diff --git a/libgomp/ChangeLog b/libgomp/ChangeLog
index 1d93273b919042e3797e634353363064e7d3afa6..36c716c518eb1e34f1f0c04cc8a0ebe783a2f038 100644
--- a/libgomp/ChangeLog
+++ b/libgomp/ChangeLog
@@ -1,3 +1,25 @@
+2011-07-15 Jakub Jelinek <jakub@redhat.com>
+
+ * config/linux/wait.h (do_spin): New inline, largely copied
+ from do_wait, just don't do futex_wait here, instead return true if
+ it should be done.
+ (do_wait): Implement using do_spin.
+ * config/linux/mutex.h (gomp_mutex_lock_slow): Add an int argument
+ to prototype.
+ (gomp_mutex_lock): Use __sync_val_compare_and_swap instead of
+ __sync_bool_compare_and_swap, pass the oldval to
+ gomp_mutex_lock_slow.
+ * config/linux/mutex.c (gomp_mutex_lock_slow): Add oldval argument.
+ If all mutex contenders are just spinning and not sleeping, don't
+ change state to 2 unnecessarily. Optimize the loop when state has
+ already become 2 to use just one atomic operation per loop instead
+ of two.
+ * config/linux/ia64/mutex.h (gomp_mutex_lock_slow): Add an int argument
+ to prototype.
+ (gomp_mutex_lock): Use __sync_val_compare_and_swap instead of
+ __sync_bool_compare_and_swap, pass the oldval to
+ gomp_mutex_lock_slow.
+
2011-06-22 Jakub Jelinek <jakub@redhat.com>
PR libgomp/49490
diff --git a/libgomp/config/linux/ia64/mutex.h b/libgomp/config/linux/ia64/mutex.h
index 6e294059b23a7d03c4bc5a9c9dace41bf0df7987..8a67673df40238dac0df8b13caa8d29a1845238b 100644
--- a/libgomp/config/linux/ia64/mutex.h
+++ b/libgomp/config/linux/ia64/mutex.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc.
+/* Copyright (C) 2005, 2008, 2009, 2011 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU OpenMP Library (libgomp).
@@ -38,11 +38,12 @@ static inline void gomp_mutex_init (gomp_mutex_t *mutex)
*mutex = 0;
}
-extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex);
+extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex, int);
static inline void gomp_mutex_lock (gomp_mutex_t *mutex)
{
- if (!__sync_bool_compare_and_swap (mutex, 0, 1))
- gomp_mutex_lock_slow (mutex);
+ int oldval = __sync_val_compare_and_swap (mutex, 0, 1);
+ if (__builtin_expect (oldval, 0))
+ gomp_mutex_lock_slow (mutex, oldval);
}
extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex);
diff --git a/libgomp/config/linux/mutex.c b/libgomp/config/linux/mutex.c
index 3ca37c19f5f633607eb5d93907d38ef73f13e050..2574f7be42a0d35aa81f03a30b2be014e2a1326b 100644
--- a/libgomp/config/linux/mutex.c
+++ b/libgomp/config/linux/mutex.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc.
+/* Copyright (C) 2005, 2008, 2009, 2011 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU OpenMP Library (libgomp).
@@ -32,15 +32,27 @@ long int gomp_futex_wake = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
long int gomp_futex_wait = FUTEX_WAIT | FUTEX_PRIVATE_FLAG;
void
-gomp_mutex_lock_slow (gomp_mutex_t *mutex)
+gomp_mutex_lock_slow (gomp_mutex_t *mutex, int oldval)
{
- do
+ while (oldval == 1)
{
- int oldval = __sync_val_compare_and_swap (mutex, 1, 2);
- if (oldval != 0)
- do_wait (mutex, 2);
+ if (do_spin (mutex, 1))
+ {
+ oldval = __sync_lock_test_and_set (mutex, 2);
+ if (oldval == 0)
+ return;
+ futex_wait (mutex, 2);
+ break;
+ }
+ else
+ {
+ oldval = __sync_val_compare_and_swap (mutex, 0, 1);
+ if (oldval == 0)
+ return;
+ }
}
- while (!__sync_bool_compare_and_swap (mutex, 0, 2));
+ while ((oldval = __sync_lock_test_and_set (mutex, 2)))
+ do_wait (mutex, 2);
}
void
diff --git a/libgomp/config/linux/mutex.h b/libgomp/config/linux/mutex.h
index 1905ce012fde79b8b57ccc5a338be8f2b895eebf..eafb0e7673401332329b7d5eaedd59eac9232228 100644
--- a/libgomp/config/linux/mutex.h
+++ b/libgomp/config/linux/mutex.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005, 2009 Free Software Foundation, Inc.
+/* Copyright (C) 2005, 2009, 2011 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU OpenMP Library (libgomp).
@@ -38,11 +38,12 @@ static inline void gomp_mutex_init (gomp_mutex_t *mutex)
*mutex = 0;
}
-extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex);
+extern void gomp_mutex_lock_slow (gomp_mutex_t *mutex, int);
static inline void gomp_mutex_lock (gomp_mutex_t *mutex)
{
- if (!__sync_bool_compare_and_swap (mutex, 0, 1))
- gomp_mutex_lock_slow (mutex);
+ int oldval = __sync_val_compare_and_swap (mutex, 0, 1);
+ if (__builtin_expect (oldval, 0))
+ gomp_mutex_lock_slow (mutex, oldval);
}
extern void gomp_mutex_unlock_slow (gomp_mutex_t *mutex);
diff --git a/libgomp/config/linux/wait.h b/libgomp/config/linux/wait.h
index 0e8abf111b09362f41173893747f268629a1ba58..4f65985581128eae3707e82794790cd86f6ede09 100644
--- a/libgomp/config/linux/wait.h
+++ b/libgomp/config/linux/wait.h
@@ -44,7 +44,7 @@ extern long int gomp_futex_wait, gomp_futex_wake;
#include <futex.h>
-static inline void do_wait (int *addr, int val)
+static inline int do_spin (int *addr, int val)
{
unsigned long long i, count = gomp_spin_count_var;
@@ -52,10 +52,16 @@ static inline void do_wait (int *addr, int val)
count = gomp_throttled_spin_count_var;
for (i = 0; i < count; i++)
if (__builtin_expect (*addr != val, 0))
- return;
+ return 0;
else
cpu_relax ();
- futex_wait (addr, val);
+ return 1;
+}
+
+static inline void do_wait (int *addr, int val)
+{
+ if (do_spin (addr, val))
+ futex_wait (addr, val);
}
#ifdef HAVE_ATTRIBUTE_VISIBILITY