diff options
Diffstat (limited to 'include/linux/local_lock_internal.h')
| -rw-r--r-- | include/linux/local_lock_internal.h | 72 |
1 files changed, 58 insertions, 14 deletions
diff --git a/include/linux/local_lock_internal.h b/include/linux/local_lock_internal.h index 8f82b4eb542f..eff711bf973f 100644 --- a/include/linux/local_lock_internal.h +++ b/include/linux/local_lock_internal.h @@ -4,25 +4,30 @@ #endif #include <linux/percpu-defs.h> +#include <linux/irqflags.h> #include <linux/lockdep.h> +#include <linux/debug_locks.h> +#include <asm/current.h> #ifndef CONFIG_PREEMPT_RT -typedef struct { +context_lock_struct(local_lock) { #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; struct task_struct *owner; #endif -} local_lock_t; +}; +typedef struct local_lock local_lock_t; /* local_trylock() and local_trylock_irqsave() only work with local_trylock_t */ -typedef struct { +context_lock_struct(local_trylock) { #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; struct task_struct *owner; #endif u8 acquired; -} local_trylock_t; +}; +typedef struct local_trylock local_trylock_t; #ifdef CONFIG_DEBUG_LOCK_ALLOC # define LOCAL_LOCK_DEBUG_INIT(lockname) \ @@ -84,7 +89,10 @@ do { \ local_lock_debug_init(lock); \ } while (0) -#define __local_trylock_init(lock) __local_lock_init((local_lock_t *)lock) +#define __local_trylock_init(lock) \ +do { \ + __local_lock_init((local_lock_t *)lock); \ +} while (0) #define __spinlock_nested_bh_init(lock) \ do { \ @@ -117,22 +125,25 @@ do { \ do { \ preempt_disable(); \ __local_lock_acquire(lock); \ + __acquire(lock); \ } while (0) #define __local_lock_irq(lock) \ do { \ local_irq_disable(); \ __local_lock_acquire(lock); \ + __acquire(lock); \ } while (0) #define __local_lock_irqsave(lock, flags) \ do { \ local_irq_save(flags); \ __local_lock_acquire(lock); \ + __acquire(lock); \ } while (0) #define __local_trylock(lock) \ - ({ \ + __try_acquire_ctx_lock(lock, ({ \ local_trylock_t *__tl; \ \ preempt_disable(); \ @@ -146,10 +157,10 @@ do { \ (local_lock_t *)__tl); \ } \ !!__tl; \ - }) + })) #define __local_trylock_irqsave(lock, flags) \ - ({ \ + __try_acquire_ctx_lock(lock, ({ \ local_trylock_t *__tl; \ \ local_irq_save(flags); \ @@ -163,7 +174,7 @@ do { \ (local_lock_t *)__tl); \ } \ !!__tl; \ - }) + })) /* preemption or migration must be disabled before calling __local_lock_is_locked */ #define __local_lock_is_locked(lock) READ_ONCE(this_cpu_ptr(lock)->acquired) @@ -186,18 +197,21 @@ do { \ #define __local_unlock(lock) \ do { \ + __release(lock); \ __local_lock_release(lock); \ preempt_enable(); \ } while (0) #define __local_unlock_irq(lock) \ do { \ + __release(lock); \ __local_lock_release(lock); \ local_irq_enable(); \ } while (0) #define __local_unlock_irqrestore(lock, flags) \ do { \ + __release(lock); \ __local_lock_release(lock); \ local_irq_restore(flags); \ } while (0) @@ -206,13 +220,20 @@ do { \ do { \ lockdep_assert_in_softirq(); \ local_lock_acquire((lock)); \ + __acquire(lock); \ } while (0) #define __local_unlock_nested_bh(lock) \ - local_lock_release((lock)) + do { \ + __release(lock); \ + local_lock_release((lock)); \ + } while (0) #else /* !CONFIG_PREEMPT_RT */ +#include <linux/sched.h> +#include <linux/spinlock.h> + /* * On PREEMPT_RT local_lock maps to a per CPU spinlock, which protects the * critical section while staying preemptible. @@ -267,7 +288,7 @@ do { \ } while (0) #define __local_trylock(lock) \ - ({ \ + __try_acquire_ctx_lock(lock, context_unsafe(({ \ int __locked; \ \ if (in_nmi() | in_hardirq()) { \ @@ -279,17 +300,40 @@ do { \ migrate_enable(); \ } \ __locked; \ - }) + }))) #define __local_trylock_irqsave(lock, flags) \ - ({ \ + __try_acquire_ctx_lock(lock, ({ \ typecheck(unsigned long, flags); \ flags = 0; \ __local_trylock(lock); \ - }) + })) /* migration must be disabled before calling __local_lock_is_locked */ #define __local_lock_is_locked(__lock) \ (rt_mutex_owner(&this_cpu_ptr(__lock)->lock) == current) #endif /* CONFIG_PREEMPT_RT */ + +#if defined(WARN_CONTEXT_ANALYSIS) +/* + * Because the compiler only knows about the base per-CPU variable, use this + * helper function to make the compiler think we lock/unlock the @base variable, + * and hide the fact we actually pass the per-CPU instance to lock/unlock + * functions. + */ +static __always_inline local_lock_t *__this_cpu_local_lock(local_lock_t __percpu *base) + __returns_ctx_lock(base) __attribute__((overloadable)) +{ + return this_cpu_ptr(base); +} +#ifndef CONFIG_PREEMPT_RT +static __always_inline local_trylock_t *__this_cpu_local_lock(local_trylock_t __percpu *base) + __returns_ctx_lock(base) __attribute__((overloadable)) +{ + return this_cpu_ptr(base); +} +#endif /* CONFIG_PREEMPT_RT */ +#else /* WARN_CONTEXT_ANALYSIS */ +#define __this_cpu_local_lock(base) this_cpu_ptr(base) +#endif /* WARN_CONTEXT_ANALYSIS */ |