summaryrefslogtreecommitdiff
path: root/include
diff options
context:
space:
mode:
Diffstat (limited to 'include')
-rw-r--r--include/linux/irq-entry-common.h2
-rw-r--r--include/linux/perf_event.h2
-rw-r--r--include/linux/unwind_deferred.h52
-rw-r--r--include/linux/unwind_deferred_types.h18
-rw-r--r--include/linux/unwind_user_types.h2
-rw-r--r--include/uapi/linux/perf_event.h21
6 files changed, 66 insertions, 31 deletions
diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h
index d643c7c87822..ba1ed42f8a1c 100644
--- a/include/linux/irq-entry-common.h
+++ b/include/linux/irq-entry-common.h
@@ -253,11 +253,11 @@ static __always_inline void exit_to_user_mode_prepare(struct pt_regs *regs)
static __always_inline void exit_to_user_mode(void)
{
instrumentation_begin();
+ unwind_reset_info();
trace_hardirqs_on_prepare();
lockdep_hardirqs_on_prepare();
instrumentation_end();
- unwind_reset_info();
user_enter_irqoff();
arch_exit_to_user_mode();
lockdep_hardirqs_on(CALLER_ADDR0);
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index fd1d91017b99..9870d768db4c 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -1720,7 +1720,7 @@ extern void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct p
extern void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
extern struct perf_callchain_entry *
get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
- u32 max_stack, bool crosstask, bool add_mark);
+ u32 max_stack, bool crosstask, bool add_mark, u64 defer_cookie);
extern int get_callchain_buffers(int max_stack);
extern void put_callchain_buffers(void);
extern struct perf_callchain_entry *get_callchain_entry(int *rctx);
diff --git a/include/linux/unwind_deferred.h b/include/linux/unwind_deferred.h
index 26122d00708a..bc7ae7d21900 100644
--- a/include/linux/unwind_deferred.h
+++ b/include/linux/unwind_deferred.h
@@ -6,16 +6,6 @@
#include <linux/unwind_user.h>
#include <linux/unwind_deferred_types.h>
-struct unwind_work;
-
-typedef void (*unwind_callback_t)(struct unwind_work *work, struct unwind_stacktrace *trace, u64 cookie);
-
-struct unwind_work {
- struct list_head list;
- unwind_callback_t func;
- int bit;
-};
-
#ifdef CONFIG_UNWIND_USER
enum {
@@ -44,22 +34,22 @@ void unwind_deferred_task_exit(struct task_struct *task);
static __always_inline void unwind_reset_info(void)
{
struct unwind_task_info *info = &current->unwind_info;
- unsigned long bits;
+ unsigned long bits = atomic_long_read(&info->unwind_mask);
/* Was there any unwinding? */
- if (unlikely(info->unwind_mask)) {
- bits = info->unwind_mask;
- do {
- /* Is a task_work going to run again before going back */
- if (bits & UNWIND_PENDING)
- return;
- } while (!try_cmpxchg(&info->unwind_mask, &bits, 0UL));
- current->unwind_info.id.id = 0;
-
- if (unlikely(info->cache)) {
- info->cache->nr_entries = 0;
- info->cache->unwind_completed = 0;
- }
+ if (likely(!bits))
+ return;
+
+ do {
+ /* Is a task_work going to run again before going back */
+ if (bits & UNWIND_PENDING)
+ return;
+ } while (!atomic_long_try_cmpxchg(&info->unwind_mask, &bits, 0UL));
+ current->unwind_info.id.id = 0;
+
+ if (unlikely(info->cache)) {
+ info->cache->nr_entries = 0;
+ info->cache->unwind_completed = 0;
}
}
@@ -68,9 +58,17 @@ static __always_inline void unwind_reset_info(void)
static inline void unwind_task_init(struct task_struct *task) {}
static inline void unwind_task_free(struct task_struct *task) {}
-static inline int unwind_user_faultable(struct unwind_stacktrace *trace) { return -ENOSYS; }
-static inline int unwind_deferred_init(struct unwind_work *work, unwind_callback_t func) { return -ENOSYS; }
-static inline int unwind_deferred_request(struct unwind_work *work, u64 *timestamp) { return -ENOSYS; }
+static inline int unwind_user_faultable(struct unwind_stacktrace *trace)
+{ return -ENOSYS; }
+
+static inline int
+unwind_deferred_init(struct unwind_work *work, unwind_callback_t func)
+{ return -ENOSYS; }
+
+static inline int
+unwind_deferred_request(struct unwind_work *work, u64 *timestamp)
+{ return -ENOSYS; }
+
static inline void unwind_deferred_cancel(struct unwind_work *work) {}
static inline void unwind_deferred_task_exit(struct task_struct *task) {}
diff --git a/include/linux/unwind_deferred_types.h b/include/linux/unwind_deferred_types.h
index 33b62ac25c86..18fa3932f61c 100644
--- a/include/linux/unwind_deferred_types.h
+++ b/include/linux/unwind_deferred_types.h
@@ -2,6 +2,9 @@
#ifndef _LINUX_UNWIND_USER_DEFERRED_TYPES_H
#define _LINUX_UNWIND_USER_DEFERRED_TYPES_H
+#include <linux/types.h>
+#include <linux/atomic.h>
+
struct unwind_cache {
unsigned long unwind_completed;
unsigned int nr_entries;
@@ -30,10 +33,23 @@ union unwind_task_id {
};
struct unwind_task_info {
- unsigned long unwind_mask;
+ atomic_long_t unwind_mask;
struct unwind_cache *cache;
struct callback_head work;
union unwind_task_id id;
};
+struct unwind_work;
+struct unwind_stacktrace;
+
+typedef void (*unwind_callback_t)(struct unwind_work *work,
+ struct unwind_stacktrace *trace,
+ u64 cookie);
+
+struct unwind_work {
+ struct list_head list;
+ unwind_callback_t func;
+ int bit;
+};
+
#endif /* _LINUX_UNWIND_USER_DEFERRED_TYPES_H */
diff --git a/include/linux/unwind_user_types.h b/include/linux/unwind_user_types.h
index a449f15be890..412729a269bc 100644
--- a/include/linux/unwind_user_types.h
+++ b/include/linux/unwind_user_types.h
@@ -36,8 +36,10 @@ struct unwind_user_state {
unsigned long ip;
unsigned long sp;
unsigned long fp;
+ unsigned int ws;
enum unwind_user_type current_type;
unsigned int available_types;
+ bool topmost;
bool done;
};
diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
index 78a362b80027..d292f96bc06f 100644
--- a/include/uapi/linux/perf_event.h
+++ b/include/uapi/linux/perf_event.h
@@ -463,7 +463,9 @@ struct perf_event_attr {
inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */
remove_on_exec : 1, /* event is removed from task on exec */
sigtrap : 1, /* send synchronous SIGTRAP on event */
- __reserved_1 : 26;
+ defer_callchain: 1, /* request PERF_RECORD_CALLCHAIN_DEFERRED records */
+ defer_output : 1, /* output PERF_RECORD_CALLCHAIN_DEFERRED records */
+ __reserved_1 : 24;
union {
__u32 wakeup_events; /* wake up every n events */
@@ -1239,6 +1241,22 @@ enum perf_event_type {
*/
PERF_RECORD_AUX_OUTPUT_HW_ID = 21,
+ /*
+ * This user callchain capture was deferred until shortly before
+ * returning to user space. Previous samples would have kernel
+ * callchains only and they need to be stitched with this to make full
+ * callchains.
+ *
+ * struct {
+ * struct perf_event_header header;
+ * u64 cookie;
+ * u64 nr;
+ * u64 ips[nr];
+ * struct sample_id sample_id;
+ * };
+ */
+ PERF_RECORD_CALLCHAIN_DEFERRED = 22,
+
PERF_RECORD_MAX, /* non-ABI */
};
@@ -1269,6 +1287,7 @@ enum perf_callchain_context {
PERF_CONTEXT_HV = (__u64)-32,
PERF_CONTEXT_KERNEL = (__u64)-128,
PERF_CONTEXT_USER = (__u64)-512,
+ PERF_CONTEXT_USER_DEFERRED = (__u64)-640,
PERF_CONTEXT_GUEST = (__u64)-2048,
PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176,
generated by cgit v1.2.3 (git 2.0.4) at 2025-12-11 14:50:17 +0000