From 4cf7a136115e96241f9f1089d2b53c47accf3823 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 17 Jan 2023 22:05:52 -0800 Subject: perf/core: Save the dynamic parts of sample data size The perf sample data can be divided into parts. The event->header_size and event->id_header_size keep the static part of the sample data which is determined by the sample_type flags. But other parts like CALLCHAIN and BRANCH_STACK are changing dynamically so it needs to see the actual data. In preparation of handling repeated calls for perf_prepare_sample(), it can save the dynamic size to the perf sample data to avoid the duplicate work. Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Tested-by: Jiri Olsa Acked-by: Jiri Olsa Acked-by: Song Liu Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20230118060559.615653-2-namhyung@kernel.org --- kernel/events/core.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index d56328e5080e..827082d1e7f9 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7586,7 +7586,7 @@ void perf_prepare_sample(struct perf_event_header *header, size += data->callchain->nr; - header->size += size * sizeof(u64); + data->dyn_size += size * sizeof(u64); } if (sample_type & PERF_SAMPLE_RAW) { @@ -7612,7 +7612,7 @@ void perf_prepare_sample(struct perf_event_header *header, data->raw = NULL; } - header->size += size; + data->dyn_size += size; } if (sample_type & PERF_SAMPLE_BRANCH_STACK) { @@ -7624,7 +7624,7 @@ void perf_prepare_sample(struct perf_event_header *header, size += data->br_stack->nr * sizeof(struct perf_branch_entry); } - header->size += size; + data->dyn_size += size; } if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER)) @@ -7639,7 +7639,7 @@ void perf_prepare_sample(struct perf_event_header *header, size += hweight64(mask) * sizeof(u64); } - header->size += size; + data->dyn_size += size; } if (sample_type & PERF_SAMPLE_STACK_USER) { @@ -7664,7 +7664,7 @@ void perf_prepare_sample(struct perf_event_header *header, size += sizeof(u64) + stack_size; data->stack_user_size = stack_size; - header->size += size; + data->dyn_size += size; } if (filtered_sample_type & PERF_SAMPLE_WEIGHT_TYPE) @@ -7693,7 +7693,7 @@ void perf_prepare_sample(struct perf_event_header *header, size += hweight64(mask) * sizeof(u64); } - header->size += size; + data->dyn_size += size; } if (sample_type & PERF_SAMPLE_PHYS_ADDR && @@ -7738,8 +7738,11 @@ void perf_prepare_sample(struct perf_event_header *header, size = perf_prepare_sample_aux(event, data, size); WARN_ON_ONCE(size + header->size > U16_MAX); - header->size += size; + data->dyn_size += size + sizeof(u64); /* size above */ } + + header->size += data->dyn_size; + /* * If you're adding more sample types here, you likely need to do * something about the overflowing header::size, like repurpose the -- cgit v1.2.3 From 31046500c1864b8ab25d1b9846ad10aa3f7b1821 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 17 Jan 2023 22:05:53 -0800 Subject: perf/core: Add perf_sample_save_callchain() helper When we save the callchain to the perf sample data, we need to update the sample flags and the dynamic size. To ensure this is done consistently, add the perf_sample_save_callchain() helper and convert all call sites. Suggested-by: Peter Zijlstra Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Tested-by: Jiri Olsa Acked-by: Jiri Olsa Acked-by: Song Liu Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20230118060559.615653-3-namhyung@kernel.org --- kernel/events/core.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index 827082d1e7f9..12b7d5114b06 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7578,16 +7578,8 @@ void perf_prepare_sample(struct perf_event_header *header, if (sample_type & (PERF_SAMPLE_IP | PERF_SAMPLE_CODE_PAGE_SIZE)) data->ip = perf_instruction_pointer(regs); - if (sample_type & PERF_SAMPLE_CALLCHAIN) { - int size = 1; - - if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN) - data->callchain = perf_callchain(event, regs); - - size += data->callchain->nr; - - data->dyn_size += size * sizeof(u64); - } + if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN) + perf_sample_save_callchain(data, event, regs); if (sample_type & PERF_SAMPLE_RAW) { struct perf_raw_record *raw = data->raw; -- cgit v1.2.3 From 0a9081cf0a11770f6b0affd377db8caa3ec4c793 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 17 Jan 2023 22:05:54 -0800 Subject: perf/core: Add perf_sample_save_raw_data() helper When we save the raw_data to the perf sample data, we need to update the sample flags and the dynamic size. To make sure this is done consistently, add the perf_sample_save_raw_data() helper and convert all call sites. Suggested-by: Peter Zijlstra Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Tested-by: Jiri Olsa Acked-by: Jiri Olsa Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20230118060559.615653-4-namhyung@kernel.org --- kernel/events/core.c | 31 +++++-------------------------- kernel/trace/bpf_trace.c | 6 ++---- 2 files changed, 7 insertions(+), 30 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index 12b7d5114b06..17108a23b3dd 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7581,30 +7581,10 @@ void perf_prepare_sample(struct perf_event_header *header, if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN) perf_sample_save_callchain(data, event, regs); - if (sample_type & PERF_SAMPLE_RAW) { - struct perf_raw_record *raw = data->raw; - int size; - - if (raw && (data->sample_flags & PERF_SAMPLE_RAW)) { - struct perf_raw_frag *frag = &raw->frag; - u32 sum = 0; - - do { - sum += frag->size; - if (perf_raw_frag_last(frag)) - break; - frag = frag->next; - } while (1); - - size = round_up(sum + sizeof(u32), sizeof(u64)); - raw->size = size - sizeof(u32); - frag->pad = raw->size - sum; - } else { - size = sizeof(u64); - data->raw = NULL; - } - - data->dyn_size += size; + if (filtered_sample_type & PERF_SAMPLE_RAW) { + data->raw = NULL; + data->dyn_size += sizeof(u64); + data->sample_flags |= PERF_SAMPLE_RAW; } if (sample_type & PERF_SAMPLE_BRANCH_STACK) { @@ -10120,8 +10100,7 @@ void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size, }; perf_sample_data_init(&data, 0, 0); - data.raw = &raw; - data.sample_flags |= PERF_SAMPLE_RAW; + perf_sample_save_raw_data(&data, &raw); perf_trace_buf_update(record, event_type); diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 3bbd3f0c810c..ad37608afc35 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -687,8 +687,7 @@ BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map, } perf_sample_data_init(sd, 0, 0); - sd->raw = &raw; - sd->sample_flags |= PERF_SAMPLE_RAW; + perf_sample_save_raw_data(sd, &raw); err = __bpf_perf_event_output(regs, map, flags, sd); @@ -746,8 +745,7 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, perf_fetch_caller_regs(regs); perf_sample_data_init(sd, 0, 0); - sd->raw = &raw; - sd->sample_flags |= PERF_SAMPLE_RAW; + perf_sample_save_raw_data(sd, &raw); ret = __bpf_perf_event_output(regs, map, flags, sd); out: -- cgit v1.2.3 From eb55b455ef9c7123bdfa7e8a7f1ebeaa8034eb83 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 17 Jan 2023 22:05:55 -0800 Subject: perf/core: Add perf_sample_save_brstack() helper When we saves the branch stack to the perf sample data, we needs to update the sample flags and the dynamic size. To make sure this is done consistently, add the perf_sample_save_brstack() helper and convert all call sites. Suggested-by: Peter Zijlstra Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Tested-by: Jiri Olsa Acked-by: Jiri Olsa Acked-by: Athira Rajeev Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20230118060559.615653-5-namhyung@kernel.org --- kernel/events/core.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index 17108a23b3dd..bd20705a23bd 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7310,7 +7310,7 @@ void perf_output_sample(struct perf_output_handle *handle, } if (sample_type & PERF_SAMPLE_BRANCH_STACK) { - if (data->sample_flags & PERF_SAMPLE_BRANCH_STACK) { + if (data->br_stack) { size_t size; size = data->br_stack->nr @@ -7587,16 +7587,10 @@ void perf_prepare_sample(struct perf_event_header *header, data->sample_flags |= PERF_SAMPLE_RAW; } - if (sample_type & PERF_SAMPLE_BRANCH_STACK) { - int size = sizeof(u64); /* nr */ - if (data->sample_flags & PERF_SAMPLE_BRANCH_STACK) { - if (branch_sample_hw_index(event)) - size += sizeof(u64); - - size += data->br_stack->nr - * sizeof(struct perf_branch_entry); - } - data->dyn_size += size; + if (filtered_sample_type & PERF_SAMPLE_BRANCH_STACK) { + data->br_stack = NULL; + data->dyn_size += sizeof(u64); + data->sample_flags |= PERF_SAMPLE_BRANCH_STACK; } if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER)) -- cgit v1.2.3 From bb447c27a4674628ea50341cfa4b31618f2010af Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 17 Jan 2023 22:05:56 -0800 Subject: perf/core: Set data->sample_flags in perf_prepare_sample() The perf_prepare_sample() function sets the perf_sample_data according to the attr->sample_type before copying it to the ring buffer. But BPF also wants to access the sample data so it needs to prepare the sample even before the regular path. That means perf_prepare_sample() can be called more than once. Set the data->sample_flags consistently so that it can indicate which fields are set already and skip them if sets. Also update the filtered_sample_type to have the dependent flags to reduce the number of branches. Suggested-by: Peter Zijlstra Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Tested-by: Jiri Olsa Acked-by: Jiri Olsa Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20230118060559.615653-6-namhyung@kernel.org --- kernel/events/core.c | 85 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 20 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index bd20705a23bd..7135cb99a7fe 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7046,12 +7046,21 @@ out_put: ring_buffer_put(rb); } +/* + * A set of common sample data types saved even for non-sample records + * when event->attr.sample_id_all is set. + */ +#define PERF_SAMPLE_ID_ALL (PERF_SAMPLE_TID | PERF_SAMPLE_TIME | \ + PERF_SAMPLE_ID | PERF_SAMPLE_STREAM_ID | \ + PERF_SAMPLE_CPU | PERF_SAMPLE_IDENTIFIER) + static void __perf_event_header__init_id(struct perf_event_header *header, struct perf_sample_data *data, struct perf_event *event, u64 sample_type) { data->type = event->attr.sample_type; + data->sample_flags |= data->type & PERF_SAMPLE_ID_ALL; header->size += event->id_header_size; if (sample_type & PERF_SAMPLE_TID) { @@ -7554,6 +7563,11 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs) return callchain ?: &__empty_callchain; } +static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d) +{ + return d * !!(flags & s); +} + void perf_prepare_sample(struct perf_event_header *header, struct perf_sample_data *data, struct perf_event *event, @@ -7569,14 +7583,24 @@ void perf_prepare_sample(struct perf_event_header *header, header->misc |= perf_misc_flags(regs); /* - * Clear the sample flags that have already been done by the - * PMU driver. + * Add the sample flags that are dependent to others. And clear the + * sample flags that have already been done by the PMU driver. */ - filtered_sample_type = sample_type & ~data->sample_flags; + filtered_sample_type = sample_type; + filtered_sample_type |= __cond_set(sample_type, PERF_SAMPLE_CODE_PAGE_SIZE, + PERF_SAMPLE_IP); + filtered_sample_type |= __cond_set(sample_type, PERF_SAMPLE_DATA_PAGE_SIZE | + PERF_SAMPLE_PHYS_ADDR, PERF_SAMPLE_ADDR); + filtered_sample_type |= __cond_set(sample_type, PERF_SAMPLE_STACK_USER, + PERF_SAMPLE_REGS_USER); + filtered_sample_type &= ~data->sample_flags; + __perf_event_header__init_id(header, data, event, filtered_sample_type); - if (sample_type & (PERF_SAMPLE_IP | PERF_SAMPLE_CODE_PAGE_SIZE)) + if (filtered_sample_type & PERF_SAMPLE_IP) { data->ip = perf_instruction_pointer(regs); + data->sample_flags |= PERF_SAMPLE_IP; + } if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN) perf_sample_save_callchain(data, event, regs); @@ -7593,10 +7617,15 @@ void perf_prepare_sample(struct perf_event_header *header, data->sample_flags |= PERF_SAMPLE_BRANCH_STACK; } - if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER)) + if (filtered_sample_type & PERF_SAMPLE_REGS_USER) perf_sample_regs_user(&data->regs_user, regs); - if (sample_type & PERF_SAMPLE_REGS_USER) { + /* + * It cannot use the filtered_sample_type here as REGS_USER can be set + * by STACK_USER (using __cond_set() above) and we don't want to update + * the dyn_size if it's not requested by users. + */ + if ((sample_type & ~data->sample_flags) & PERF_SAMPLE_REGS_USER) { /* regs dump ABI info */ int size = sizeof(u64); @@ -7606,9 +7635,10 @@ void perf_prepare_sample(struct perf_event_header *header, } data->dyn_size += size; + data->sample_flags |= PERF_SAMPLE_REGS_USER; } - if (sample_type & PERF_SAMPLE_STACK_USER) { + if (filtered_sample_type & PERF_SAMPLE_STACK_USER) { /* * Either we need PERF_SAMPLE_STACK_USER bit to be always * processed as the last one or have additional check added @@ -7631,23 +7661,30 @@ void perf_prepare_sample(struct perf_event_header *header, data->stack_user_size = stack_size; data->dyn_size += size; + data->sample_flags |= PERF_SAMPLE_STACK_USER; } - if (filtered_sample_type & PERF_SAMPLE_WEIGHT_TYPE) + if (filtered_sample_type & PERF_SAMPLE_WEIGHT_TYPE) { data->weight.full = 0; + data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE; + } - if (filtered_sample_type & PERF_SAMPLE_DATA_SRC) + if (filtered_sample_type & PERF_SAMPLE_DATA_SRC) { data->data_src.val = PERF_MEM_NA; + data->sample_flags |= PERF_SAMPLE_DATA_SRC; + } - if (filtered_sample_type & PERF_SAMPLE_TRANSACTION) + if (filtered_sample_type & PERF_SAMPLE_TRANSACTION) { data->txn = 0; + data->sample_flags |= PERF_SAMPLE_TRANSACTION; + } - if (sample_type & (PERF_SAMPLE_ADDR | PERF_SAMPLE_PHYS_ADDR | PERF_SAMPLE_DATA_PAGE_SIZE)) { - if (filtered_sample_type & PERF_SAMPLE_ADDR) - data->addr = 0; + if (filtered_sample_type & PERF_SAMPLE_ADDR) { + data->addr = 0; + data->sample_flags |= PERF_SAMPLE_ADDR; } - if (sample_type & PERF_SAMPLE_REGS_INTR) { + if (filtered_sample_type & PERF_SAMPLE_REGS_INTR) { /* regs dump ABI info */ int size = sizeof(u64); @@ -7660,19 +7697,22 @@ void perf_prepare_sample(struct perf_event_header *header, } data->dyn_size += size; + data->sample_flags |= PERF_SAMPLE_REGS_INTR; } - if (sample_type & PERF_SAMPLE_PHYS_ADDR && - filtered_sample_type & PERF_SAMPLE_PHYS_ADDR) + if (filtered_sample_type & PERF_SAMPLE_PHYS_ADDR) { data->phys_addr = perf_virt_to_phys(data->addr); + data->sample_flags |= PERF_SAMPLE_PHYS_ADDR; + } #ifdef CONFIG_CGROUP_PERF - if (sample_type & PERF_SAMPLE_CGROUP) { + if (filtered_sample_type & PERF_SAMPLE_CGROUP) { struct cgroup *cgrp; /* protected by RCU */ cgrp = task_css_check(current, perf_event_cgrp_id, 1)->cgroup; data->cgroup = cgroup_id(cgrp); + data->sample_flags |= PERF_SAMPLE_CGROUP; } #endif @@ -7681,13 +7721,17 @@ void perf_prepare_sample(struct perf_event_header *header, * require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr, * but the value will not dump to the userspace. */ - if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE) + if (filtered_sample_type & PERF_SAMPLE_DATA_PAGE_SIZE) { data->data_page_size = perf_get_page_size(data->addr); + data->sample_flags |= PERF_SAMPLE_DATA_PAGE_SIZE; + } - if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) + if (filtered_sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) { data->code_page_size = perf_get_page_size(data->ip); + data->sample_flags |= PERF_SAMPLE_CODE_PAGE_SIZE; + } - if (sample_type & PERF_SAMPLE_AUX) { + if (filtered_sample_type & PERF_SAMPLE_AUX) { u64 size; header->size += sizeof(u64); /* size */ @@ -7705,6 +7749,7 @@ void perf_prepare_sample(struct perf_event_header *header, WARN_ON_ONCE(size + header->size > U16_MAX); data->dyn_size += size + sizeof(u64); /* size above */ + data->sample_flags |= PERF_SAMPLE_AUX; } header->size += data->dyn_size; -- cgit v1.2.3 From a7c8d0daa87581cab8435c83cc6ecbfbcb8b60cf Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 17 Jan 2023 22:05:57 -0800 Subject: perf/core: Do not pass header for sample ID init The only thing it does for header in __perf_event_header__init_id() is to update the header size with event->id_header_size. We can do this outside and get rid of the argument for the later change. Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Tested-by: Jiri Olsa Acked-by: Jiri Olsa Acked-by: Song Liu Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20230118060559.615653-7-namhyung@kernel.org --- kernel/events/core.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index 7135cb99a7fe..47bfd9985813 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7054,14 +7054,12 @@ out_put: PERF_SAMPLE_ID | PERF_SAMPLE_STREAM_ID | \ PERF_SAMPLE_CPU | PERF_SAMPLE_IDENTIFIER) -static void __perf_event_header__init_id(struct perf_event_header *header, - struct perf_sample_data *data, +static void __perf_event_header__init_id(struct perf_sample_data *data, struct perf_event *event, u64 sample_type) { data->type = event->attr.sample_type; data->sample_flags |= data->type & PERF_SAMPLE_ID_ALL; - header->size += event->id_header_size; if (sample_type & PERF_SAMPLE_TID) { /* namespace issues */ @@ -7088,8 +7086,10 @@ void perf_event_header__init_id(struct perf_event_header *header, struct perf_sample_data *data, struct perf_event *event) { - if (event->attr.sample_id_all) - __perf_event_header__init_id(header, data, event, event->attr.sample_type); + if (event->attr.sample_id_all) { + header->size += event->id_header_size; + __perf_event_header__init_id(data, event, event->attr.sample_type); + } } static void __perf_event__output_id_sample(struct perf_output_handle *handle, @@ -7577,7 +7577,7 @@ void perf_prepare_sample(struct perf_event_header *header, u64 filtered_sample_type; header->type = PERF_RECORD_SAMPLE; - header->size = sizeof(*header) + event->header_size; + header->size = sizeof(*header) + event->header_size + event->id_header_size; header->misc = 0; header->misc |= perf_misc_flags(regs); @@ -7595,7 +7595,7 @@ void perf_prepare_sample(struct perf_event_header *header, PERF_SAMPLE_REGS_USER); filtered_sample_type &= ~data->sample_flags; - __perf_event_header__init_id(header, data, event, filtered_sample_type); + __perf_event_header__init_id(data, event, filtered_sample_type); if (filtered_sample_type & PERF_SAMPLE_IP) { data->ip = perf_instruction_pointer(regs); -- cgit v1.2.3 From f6e707156e1d5d150f288823987bee1ba0104c4c Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 17 Jan 2023 22:05:58 -0800 Subject: perf/core: Introduce perf_prepare_header() Factor out perf_prepare_header() so that it can call perf_prepare_sample() without a header if not needed. Also it checks the filtered_sample_type to avoid duplicate work when perf_prepare_sample() is called twice (or more). Suggested-by: Peter Zijlstr Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Tested-by: Jiri Olsa Acked-by: Jiri Olsa Acked-by: Song Liu Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20230118060559.615653-8-namhyung@kernel.org --- kernel/events/core.c | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index 47bfd9985813..4aa73edbaf4c 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7568,20 +7568,13 @@ static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d) return d * !!(flags & s); } -void perf_prepare_sample(struct perf_event_header *header, - struct perf_sample_data *data, +void perf_prepare_sample(struct perf_sample_data *data, struct perf_event *event, struct pt_regs *regs) { u64 sample_type = event->attr.sample_type; u64 filtered_sample_type; - header->type = PERF_RECORD_SAMPLE; - header->size = sizeof(*header) + event->header_size + event->id_header_size; - - header->misc = 0; - header->misc |= perf_misc_flags(regs); - /* * Add the sample flags that are dependent to others. And clear the * sample flags that have already been done by the PMU driver. @@ -7595,6 +7588,12 @@ void perf_prepare_sample(struct perf_event_header *header, PERF_SAMPLE_REGS_USER); filtered_sample_type &= ~data->sample_flags; + if (filtered_sample_type == 0) { + /* Make sure it has the correct data->type for output */ + data->type = event->attr.sample_type; + return; + } + __perf_event_header__init_id(data, event, filtered_sample_type); if (filtered_sample_type & PERF_SAMPLE_IP) { @@ -7646,9 +7645,10 @@ void perf_prepare_sample(struct perf_event_header *header, * up the rest of the sample size. */ u16 stack_size = event->attr.sample_stack_user; + u16 header_size = perf_sample_data_size(data, event); u16 size = sizeof(u64); - stack_size = perf_sample_ustack_size(stack_size, header->size, + stack_size = perf_sample_ustack_size(stack_size, header_size, data->regs_user.regs); /* @@ -7733,8 +7733,9 @@ void perf_prepare_sample(struct perf_event_header *header, if (filtered_sample_type & PERF_SAMPLE_AUX) { u64 size; + u16 header_size = perf_sample_data_size(data, event); - header->size += sizeof(u64); /* size */ + header_size += sizeof(u64); /* size */ /* * Given the 16bit nature of header::size, an AUX sample can @@ -7742,17 +7743,25 @@ void perf_prepare_sample(struct perf_event_header *header, * Make sure this doesn't happen by using up to U16_MAX bytes * per sample in total (rounded down to 8 byte boundary). */ - size = min_t(size_t, U16_MAX - header->size, + size = min_t(size_t, U16_MAX - header_size, event->attr.aux_sample_size); size = rounddown(size, 8); size = perf_prepare_sample_aux(event, data, size); - WARN_ON_ONCE(size + header->size > U16_MAX); + WARN_ON_ONCE(size + header_size > U16_MAX); data->dyn_size += size + sizeof(u64); /* size above */ data->sample_flags |= PERF_SAMPLE_AUX; } +} - header->size += data->dyn_size; +void perf_prepare_header(struct perf_event_header *header, + struct perf_sample_data *data, + struct perf_event *event, + struct pt_regs *regs) +{ + header->type = PERF_RECORD_SAMPLE; + header->size = perf_sample_data_size(data, event); + header->misc = perf_misc_flags(regs); /* * If you're adding more sample types here, you likely need to do @@ -7781,7 +7790,8 @@ __perf_event_output(struct perf_event *event, /* protect the callchain buffers */ rcu_read_lock(); - perf_prepare_sample(&header, data, event, regs); + perf_prepare_sample(data, event, regs); + perf_prepare_header(&header, data, event, regs); err = output_begin(&handle, data, event, header.size); if (err) -- cgit v1.2.3 From 0eed28220598cd990d094b7b9f8c832c425080c0 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 17 Jan 2023 22:05:59 -0800 Subject: perf/core: Call perf_prepare_sample() before running BPF As BPF can access sample data, it needs to populate the data. Also remove the logic to get the callchain specifically as it's covered by the perf_prepare_sample() now. Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Tested-by: Jiri Olsa Acked-by: Jiri Olsa Acked-by: Song Liu Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20230118060559.615653-9-namhyung@kernel.org --- kernel/events/core.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index 4aa73edbaf4c..380476a934e8 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -10356,13 +10356,7 @@ static void bpf_overflow_handler(struct perf_event *event, rcu_read_lock(); prog = READ_ONCE(event->prog); if (prog) { - if (prog->call_get_stack && - (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) && - !(data->sample_flags & PERF_SAMPLE_CALLCHAIN)) { - data->callchain = perf_callchain(event, regs); - data->sample_flags |= PERF_SAMPLE_CALLCHAIN; - } - + perf_prepare_sample(data, event, regs); ret = bpf_prog_run(prog, &ctx); } rcu_read_unlock(); -- cgit v1.2.3