summaryrefslogtreecommitdiff
path: root/include/linux/local_lock_internal.h
diff options
context:
space:
mode:
Diffstat (limited to 'include/linux/local_lock_internal.h')
-rw-r--r--include/linux/local_lock_internal.h134
1 files changed, 89 insertions, 45 deletions
diff --git a/include/linux/local_lock_internal.h b/include/linux/local_lock_internal.h
index a4dc479157b5..eff711bf973f 100644
--- a/include/linux/local_lock_internal.h
+++ b/include/linux/local_lock_internal.h
@@ -4,25 +4,30 @@
#endif
#include <linux/percpu-defs.h>
+#include <linux/irqflags.h>
#include <linux/lockdep.h>
+#include <linux/debug_locks.h>
+#include <asm/current.h>
#ifndef CONFIG_PREEMPT_RT
-typedef struct {
+context_lock_struct(local_lock) {
#ifdef CONFIG_DEBUG_LOCK_ALLOC
struct lockdep_map dep_map;
struct task_struct *owner;
#endif
-} local_lock_t;
+};
+typedef struct local_lock local_lock_t;
/* local_trylock() and local_trylock_irqsave() only work with local_trylock_t */
-typedef struct {
+context_lock_struct(local_trylock) {
#ifdef CONFIG_DEBUG_LOCK_ALLOC
struct lockdep_map dep_map;
struct task_struct *owner;
#endif
u8 acquired;
-} local_trylock_t;
+};
+typedef struct local_trylock local_trylock_t;
#ifdef CONFIG_DEBUG_LOCK_ALLOC
# define LOCAL_LOCK_DEBUG_INIT(lockname) \
@@ -84,7 +89,10 @@ do { \
local_lock_debug_init(lock); \
} while (0)
-#define __local_trylock_init(lock) __local_lock_init((local_lock_t *)lock)
+#define __local_trylock_init(lock) \
+do { \
+ __local_lock_init((local_lock_t *)lock); \
+} while (0)
#define __spinlock_nested_bh_init(lock) \
do { \
@@ -99,105 +107,111 @@ do { \
#define __local_lock_acquire(lock) \
do { \
- local_trylock_t *tl; \
- local_lock_t *l; \
+ local_trylock_t *__tl; \
+ local_lock_t *__l; \
\
- l = (local_lock_t *)(lock); \
- tl = (local_trylock_t *)l; \
+ __l = (local_lock_t *)(lock); \
+ __tl = (local_trylock_t *)__l; \
_Generic((lock), \
local_trylock_t *: ({ \
- lockdep_assert(tl->acquired == 0); \
- WRITE_ONCE(tl->acquired, 1); \
+ lockdep_assert(__tl->acquired == 0); \
+ WRITE_ONCE(__tl->acquired, 1); \
}), \
local_lock_t *: (void)0); \
- local_lock_acquire(l); \
+ local_lock_acquire(__l); \
} while (0)
#define __local_lock(lock) \
do { \
preempt_disable(); \
__local_lock_acquire(lock); \
+ __acquire(lock); \
} while (0)
#define __local_lock_irq(lock) \
do { \
local_irq_disable(); \
__local_lock_acquire(lock); \
+ __acquire(lock); \
} while (0)
#define __local_lock_irqsave(lock, flags) \
do { \
local_irq_save(flags); \
__local_lock_acquire(lock); \
+ __acquire(lock); \
} while (0)
#define __local_trylock(lock) \
- ({ \
- local_trylock_t *tl; \
+ __try_acquire_ctx_lock(lock, ({ \
+ local_trylock_t *__tl; \
\
preempt_disable(); \
- tl = (lock); \
- if (READ_ONCE(tl->acquired)) { \
+ __tl = (lock); \
+ if (READ_ONCE(__tl->acquired)) { \
preempt_enable(); \
- tl = NULL; \
+ __tl = NULL; \
} else { \
- WRITE_ONCE(tl->acquired, 1); \
+ WRITE_ONCE(__tl->acquired, 1); \
local_trylock_acquire( \
- (local_lock_t *)tl); \
+ (local_lock_t *)__tl); \
} \
- !!tl; \
- })
+ !!__tl; \
+ }))
#define __local_trylock_irqsave(lock, flags) \
- ({ \
- local_trylock_t *tl; \
+ __try_acquire_ctx_lock(lock, ({ \
+ local_trylock_t *__tl; \
\
local_irq_save(flags); \
- tl = (lock); \
- if (READ_ONCE(tl->acquired)) { \
+ __tl = (lock); \
+ if (READ_ONCE(__tl->acquired)) { \
local_irq_restore(flags); \
- tl = NULL; \
+ __tl = NULL; \
} else { \
- WRITE_ONCE(tl->acquired, 1); \
+ WRITE_ONCE(__tl->acquired, 1); \
local_trylock_acquire( \
- (local_lock_t *)tl); \
+ (local_lock_t *)__tl); \
} \
- !!tl; \
- })
+ !!__tl; \
+ }))
/* preemption or migration must be disabled before calling __local_lock_is_locked */
#define __local_lock_is_locked(lock) READ_ONCE(this_cpu_ptr(lock)->acquired)
#define __local_lock_release(lock) \
do { \
- local_trylock_t *tl; \
- local_lock_t *l; \
+ local_trylock_t *__tl; \
+ local_lock_t *__l; \
\
- l = (local_lock_t *)(lock); \
- tl = (local_trylock_t *)l; \
- local_lock_release(l); \
+ __l = (local_lock_t *)(lock); \
+ __tl = (local_trylock_t *)__l; \
+ local_lock_release(__l); \
_Generic((lock), \
local_trylock_t *: ({ \
- lockdep_assert(tl->acquired == 1); \
- WRITE_ONCE(tl->acquired, 0); \
+ lockdep_assert(__tl->acquired == 1); \
+ WRITE_ONCE(__tl->acquired, 0); \
}), \
local_lock_t *: (void)0); \
} while (0)
#define __local_unlock(lock) \
do { \
+ __release(lock); \
__local_lock_release(lock); \
preempt_enable(); \
} while (0)
#define __local_unlock_irq(lock) \
do { \
+ __release(lock); \
__local_lock_release(lock); \
local_irq_enable(); \
} while (0)
#define __local_unlock_irqrestore(lock, flags) \
do { \
+ __release(lock); \
__local_lock_release(lock); \
local_irq_restore(flags); \
} while (0)
@@ -206,13 +220,20 @@ do { \
do { \
lockdep_assert_in_softirq(); \
local_lock_acquire((lock)); \
+ __acquire(lock); \
} while (0)
#define __local_unlock_nested_bh(lock) \
- local_lock_release((lock))
+ do { \
+ __release(lock); \
+ local_lock_release((lock)); \
+ } while (0)
#else /* !CONFIG_PREEMPT_RT */
+#include <linux/sched.h>
+#include <linux/spinlock.h>
+
/*
* On PREEMPT_RT local_lock maps to a per CPU spinlock, which protects the
* critical section while staying preemptible.
@@ -223,12 +244,12 @@ typedef spinlock_t local_trylock_t;
#define INIT_LOCAL_LOCK(lockname) __LOCAL_SPIN_LOCK_UNLOCKED((lockname))
#define INIT_LOCAL_TRYLOCK(lockname) __LOCAL_SPIN_LOCK_UNLOCKED((lockname))
-#define __local_lock_init(l) \
+#define __local_lock_init(__l) \
do { \
- local_spin_lock_init((l)); \
+ local_spin_lock_init((__l)); \
} while (0)
-#define __local_trylock_init(l) __local_lock_init(l)
+#define __local_trylock_init(__l) __local_lock_init(__l)
#define __local_lock(__lock) \
do { \
@@ -267,7 +288,7 @@ do { \
} while (0)
#define __local_trylock(lock) \
- ({ \
+ __try_acquire_ctx_lock(lock, context_unsafe(({ \
int __locked; \
\
if (in_nmi() | in_hardirq()) { \
@@ -279,17 +300,40 @@ do { \
migrate_enable(); \
} \
__locked; \
- })
+ })))
#define __local_trylock_irqsave(lock, flags) \
- ({ \
+ __try_acquire_ctx_lock(lock, ({ \
typecheck(unsigned long, flags); \
flags = 0; \
__local_trylock(lock); \
- })
+ }))
/* migration must be disabled before calling __local_lock_is_locked */
#define __local_lock_is_locked(__lock) \
(rt_mutex_owner(&this_cpu_ptr(__lock)->lock) == current)
#endif /* CONFIG_PREEMPT_RT */
+
+#if defined(WARN_CONTEXT_ANALYSIS)
+/*
+ * Because the compiler only knows about the base per-CPU variable, use this
+ * helper function to make the compiler think we lock/unlock the @base variable,
+ * and hide the fact we actually pass the per-CPU instance to lock/unlock
+ * functions.
+ */
+static __always_inline local_lock_t *__this_cpu_local_lock(local_lock_t __percpu *base)
+ __returns_ctx_lock(base) __attribute__((overloadable))
+{
+ return this_cpu_ptr(base);
+}
+#ifndef CONFIG_PREEMPT_RT
+static __always_inline local_trylock_t *__this_cpu_local_lock(local_trylock_t __percpu *base)
+ __returns_ctx_lock(base) __attribute__((overloadable))
+{
+ return this_cpu_ptr(base);
+}
+#endif /* CONFIG_PREEMPT_RT */
+#else /* WARN_CONTEXT_ANALYSIS */
+#define __this_cpu_local_lock(base) this_cpu_ptr(base)
+#endif /* WARN_CONTEXT_ANALYSIS */
generated by cgit v1.2.3 (git 2.0.4) at 2026-10-26 01:29:45 +0000