diff options
| -rw-r--r-- | include/asm-i386/atomic.h | 8 | ||||
| -rw-r--r-- | include/asm-ia64/atomic.h | 8 | ||||
| -rw-r--r-- | include/asm-m68k/atomic.h | 8 | ||||
| -rw-r--r-- | include/asm-s390/atomic.h | 18 | ||||
| -rw-r--r-- | include/asm-sparc64/atomic.h | 10 | ||||
| -rw-r--r-- | include/asm-x86_64/atomic.h | 8 |
6 files changed, 50 insertions, 10 deletions
diff --git a/include/asm-i386/atomic.h b/include/asm-i386/atomic.h index 78b0032d1f29..22d80ece95cb 100644 --- a/include/asm-i386/atomic.h +++ b/include/asm-i386/atomic.h @@ -225,8 +225,14 @@ static __inline__ int atomic_sub_return(int i, atomic_t *v) ({ \ int c, old; \ c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ c = old; \ + } \ c != (u); \ }) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) diff --git a/include/asm-ia64/atomic.h b/include/asm-ia64/atomic.h index d3e0dfa99e1f..569ec7574baf 100644 --- a/include/asm-ia64/atomic.h +++ b/include/asm-ia64/atomic.h @@ -95,8 +95,14 @@ ia64_atomic64_sub (__s64 i, atomic64_t *v) ({ \ int c, old; \ c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ c = old; \ + } \ c != (u); \ }) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) diff --git a/include/asm-m68k/atomic.h b/include/asm-m68k/atomic.h index 862e497c2645..732d696d31a6 100644 --- a/include/asm-m68k/atomic.h +++ b/include/asm-m68k/atomic.h @@ -175,8 +175,14 @@ static inline void atomic_set_mask(unsigned long mask, unsigned long *v) ({ \ int c, old; \ c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ c = old; \ + } \ c != (u); \ }) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) diff --git a/include/asm-s390/atomic.h b/include/asm-s390/atomic.h index be6fefe223d6..de1d9926aa60 100644 --- a/include/asm-s390/atomic.h +++ b/include/asm-s390/atomic.h @@ -89,10 +89,15 @@ static __inline__ int atomic_cmpxchg(atomic_t *v, int old, int new) static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) { int c, old; - c = atomic_read(v); - while (c != u && (old = atomic_cmpxchg(v, c, c + a)) != c) + for (;;) { + if (unlikely(c == u)) + break; + old = atomic_cmpxchg(v, c, c + a); + if (likely(old == c)) + break; c = old; + } return c != u; } @@ -167,10 +172,15 @@ static __inline__ int atomic64_add_unless(atomic64_t *v, long long a, long long u) { long long c, old; - c = atomic64_read(v); - while (c != u && (old = atomic64_cmpxchg(v, c, c + a)) != c) + for (;;) { + if (unlikely(c == u)) + break; + old = atomic64_cmpxchg(v, c, c + a); + if (likely(old == c)) + break; c = old; + } return c != u; } diff --git a/include/asm-sparc64/atomic.h b/include/asm-sparc64/atomic.h index 25256bdc8aae..468eb48d8142 100644 --- a/include/asm-sparc64/atomic.h +++ b/include/asm-sparc64/atomic.h @@ -78,9 +78,15 @@ extern int atomic64_sub_ret(int, atomic64_t *); ({ \ int c, old; \ c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ c = old; \ - c != (u); \ + } \ + likely(c != (u)); \ }) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) diff --git a/include/asm-x86_64/atomic.h b/include/asm-x86_64/atomic.h index 4b5cd553e772..cecbf7baa6aa 100644 --- a/include/asm-x86_64/atomic.h +++ b/include/asm-x86_64/atomic.h @@ -405,8 +405,14 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v) ({ \ int c, old; \ c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ + for (;;) { \ + if (unlikely(c == (u))) \ + break; \ + old = atomic_cmpxchg((v), c, c + (a)); \ + if (likely(old == c)) \ + break; \ c = old; \ + } \ c != (u); \ }) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) |