From ad139384ecddb6f3e7e3c3c12765aafae0e39670 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 18:23:53 +0000 Subject: bpf: Fix overflow of jump offset in constant blinding bpf_jit_blind_insn() rewrites 'if rX op imm goto off' into: AX = imm ^ rnd AX ^= rnd if rX op AX goto off and subtracts 2 from off when the jump is backward. off is s16, so -32767 and -32768 wrap and the jump goes 32767 insns forward, past the end of the prog. The jump is inside the patch, hence bpf_adj_branches() doesn't check it. The kernel crashes when such prog runs with bpf_jit_harden=2. Return -ERANGE in such case. bpf_jit_blind_constants() fails the same way when blinding of other insns moves a jump out of range. Fixes: 4f3446bb809f ("bpf: add generic constant blinding for use in jits") Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20260924182353.277516-1-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/core.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 2e3bf8113ae9..7f11555a5070 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1362,7 +1362,7 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, { struct bpf_insn *to = to_buff; u32 imm_rnd = get_random_u32(); - s16 off; + int off; BUILD_BUG_ON(BPF_REG_PARAMS + 2 != MAX_BPF_JIT_REG); BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG); @@ -1438,6 +1438,8 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, off = from->off; if (off < 0) off -= 2; + if (off < S16_MIN) + return -ERANGE; *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); *to++ = BPF_JMP_REG(from->code, from->dst_reg, BPF_REG_AX, off); @@ -1458,6 +1460,8 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, off = from->off; if (off < 0) off -= 2; + if (off < S16_MIN) + return -ERANGE; *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); *to++ = BPF_JMP32_REG(from->code, from->dst_reg, BPF_REG_AX, @@ -1606,7 +1610,9 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp if (!rewritten) continue; - if (env) + if (rewritten < 0) + tmp = ERR_PTR(rewritten); + else if (env) tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten); else tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten); -- cgit v1.2.3 From ab39974240a0cff765f3bb9cce81d8001ffdb144 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 25 Sep 2026 08:46:51 +0200 Subject: bpf: Hold map BTF for the memory allocator destructor record bpf_ma_set_dtor() duplicates map->record so that the bpf_mem_alloc destructor can release the special fields of hash and rhash map elements once the allocator frees them for good. btf_record_dup() only acquires references on kernel and module BTF. Fields whose types live in the map BTF keep pointing into it: a kptr to a local type refers to map->btf, and a list_head or rb_root field carries a value_rec owned by the struct meta table of map->btf. That borrowed state can outlive the map. When RCU callbacks are still in flight, bpf_mem_alloc_destroy() copies the allocator and defers the final drain, together with the destructor context, to a workqueue. bpf_map_free() then frees map->record and drops the map's BTF reference as soon as map_free() returns. Once the program that loaded the BTF is gone as well, the BTF is freed while the worker still uses the duplicate. The worker dereferences it through btf_is_kernel() and btf_find_struct_meta() in bpf_obj_free_fields() when destroying elements that were freed under RCU, and through btf_is_kernel() in btf_record_free() when releasing the context itself: BUG: KASAN: slab-use-after-free in btf_is_kernel+0x19/0x30 Read of size 1 at addr ffff88811505dce8 by task kworker/u16:2/50 Workqueue: events_unbound free_mem_alloc_deferred Call Trace: btf_is_kernel+0x19/0x30 btf_record_free+0xc7/0xf0 htab_dtor_ctx_free+0x1a/0x30 free_mem_alloc_no_barrier+0x39/0x230 free_mem_alloc_deferred+0x2a/0x40 process_scheduled_works+0x5ce/0x9d0 worker_thread+0x42a/0x5e0 kthread+0x1f5/0x230 ret_from_fork+0x1dd/0x390 ret_from_fork_asm+0x1a/0x30 Allocated by task 358: __kmalloc_cache_noprof+0x287/0x510 btf_new_fd+0xf7/0x3d0 __sys_bpf+0x487/0x840 Freed by task 0: kfree+0x186/0x560 rcu_core+0x6f6/0xd40 Freeing a map that still holds more elements than the allocator's high watermark is enough to get there, because the bulk free during teardown queues the RCU callback that makes bpf_mem_alloc_destroy() defer. Non-preallocated hash maps and resizable hash maps both register the destructor and are affected alike. Keep one reference on the map BTF in the destructor context. Every non-kernel pointer in the duplicated record is either map->btf or memory owned by it, so a single reference covers kptrs, list heads, and rb roots, and mirrors what bpf_map_meta_alloc() does for the duplicated record of an inner map. Release it only after the duplicated record has been freed: the worker is preemptible and not in an RCU read-side critical section, so dropping the last reference first would let the BTF be freed while btf_record_free() still reads its fields. Fixes: 1df97a7453ee ("bpf: Register dtor for freeing special fields") Reported-by: Yuan Chen Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20260901062845.1379760-3-chenyuan_fl@163.com Link: https://patch.msgid.link/20260925064652.2261885-1-memxor@gmail.com --- kernel/bpf/hashtab.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index f9464e566f10..548eb1cc7c45 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -128,6 +128,7 @@ struct htab_elem { struct htab_btf_record { struct btf_record *record; + struct btf *btf; u32 key_size; }; @@ -497,8 +498,13 @@ static void htab_dtor_ctx_free(void *ctx) { struct htab_btf_record *hrec = ctx; + /* + * The duplicated record still points into the map BTF, so free it + * before dropping the reference that keeps that BTF alive. + */ btf_record_free(hrec->record); - kfree(ctx); + btf_put(hrec->btf); + kfree(hrec); } static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma, @@ -521,6 +527,15 @@ static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma, kfree(hrec); return err; } + /* + * btf_record_dup() only acquires kernel and module BTF. Fields whose + * types live in the map BTF keep pointing into it: kptrs to local + * types refer to map->btf, and graph roots carry a value record owned + * by its struct meta table. The context can outlive the map when the + * allocator defers its teardown, so hold a reference of our own. + */ + hrec->btf = map->btf; + btf_get(hrec->btf); bpf_mem_alloc_set_dtor(ma, dtor, htab_dtor_ctx_free, hrec); return 0; } -- cgit v1.2.3 From aa650d87a498844060eecb1d336f4d46f4d1b0b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Jean?= Date: Tue, 22 Sep 2026 20:27:35 +0000 Subject: audit: fix exe mark UAF in kill_rules() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kill_rules() removes mixed AUDIT_DIR and AUDIT_EXE rules when an audit tree is pruned. It drops entry->rule.exe before removing the rule from the RCU-visible filter lists. After a rule has been installed with AUDIT_ADD_RULE, which requires CAP_AUDIT_CONTROL, removing the watched directory can race with another task that is still evaluating the rule. In that case, fsnotify can free the executable mark before the reader reaches audit_mark_compare(), causing a use-after-free. KASAN reports: BUG: KASAN: slab-use-after-free in audit_mark_compare+0x8d/0xa0 Unlink the published rules from the RCU-visible lists and retain them on tree->rules for cleanup. If any rule has an executable mark, wait for a single RCU grace period before removing the marks and scheduling the entries for freeing. Otherwise, call_rcu() already provides the required deferred freeing without a synchronous wait. Cc: stable@vger.kernel.org Fixes: 34d99af52ad4 ("audit: implement audit by executable") Assisted-by: Codex:gpt-5 Signed-off-by: Jérémy Jean Reviewed-by: Ricardo Robaina Tested-by: Ricardo Robaina Reviewed-by: Bradley Morgan Signed-off-by: Paul Moore --- kernel/audit_tree.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c index 1ed19b775912..f2e81be8265e 100644 --- a/kernel/audit_tree.c +++ b/kernel/audit_tree.c @@ -545,22 +545,38 @@ static void kill_rules(struct audit_context *context, struct audit_tree *tree) { struct audit_krule *rule, *next; struct audit_entry *entry; + bool need_sync = false; list_for_each_entry_safe(rule, next, &tree->rules, rlist) { entry = container_of(rule, struct audit_entry, rule); - list_del_init(&rule->rlist); if (rule->tree) { /* not a half-baked one */ audit_tree_log_remove_rule(context, rule); - if (entry->rule.exe) - audit_remove_mark(entry->rule.exe); rule->tree = NULL; list_del_rcu(&entry->list); list_del(&entry->rule.list); - call_rcu(&entry->rcu, audit_free_rule_rcu); + if (entry->rule.exe) + need_sync = true; + } else { + list_del_init(&rule->rlist); } } + + if (list_empty(&tree->rules)) + return; + + if (need_sync) + synchronize_rcu(); + + list_for_each_entry_safe(rule, next, &tree->rules, rlist) { + entry = container_of(rule, struct audit_entry, rule); + + list_del_init(&rule->rlist); + if (entry->rule.exe) + audit_remove_mark(entry->rule.exe); + call_rcu(&entry->rcu, audit_free_rule_rcu); + } } /* -- cgit v1.2.3 From c79cf15eb35a4c703bf18f8c33c47e80dd660f8e Mon Sep 17 00:00:00 2001 From: Hao Ge Date: Thu, 27 Aug 2026 11:05:03 +0800 Subject: module: fix lost error code from codetag_load_module() If codetag_load_module() fails, err is not set to reflect the failure and load_module() returns 0 after the module has been torn down. Also, if the module is a livepatch, mod->klp_info allocated by copy_module_elf() leaks on this error path. Free it via a new livepatch_cleanup label. Link: https://lore.kernel.org/20260827030503.49171-1-hao.ge@linux.dev Fixes: 044d2aee6c57 ("alloc_tag: handle module codetag load errors as module load failures") Signed-off-by: Hao Ge Signed-off-by: Andrew Morton Reported-by: Sashiko Suggested-by: Petr Pavlu Reviewed-by: Bradley Morgan Cc: Aaron Tomlin Cc: Luis Chamberalin Cc: Sami Tolvanen Cc: Suren Baghdasaryan Cc: --- kernel/module/main.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/module/main.c b/kernel/module/main.c index d0e1e0bd2ad0..c1b34dc1e89a 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -3581,8 +3581,9 @@ static int load_module(struct load_info *info, const char __user *uargs, goto sysfs_cleanup; } - if (codetag_load_module(mod)) - goto sysfs_cleanup; + err = codetag_load_module(mod); + if (err) + goto livepatch_cleanup; /* Get rid of temporary copy. */ free_copy(info, flags); @@ -3592,6 +3593,9 @@ static int load_module(struct load_info *info, const char __user *uargs, return do_init_module(mod); + livepatch_cleanup: + if (is_livepatch_module(mod)) + free_module_elf(mod); sysfs_cleanup: mod_sysfs_teardown(mod); coming_cleanup: -- cgit v1.2.3 From 0fb5751a6e411b1b466c06e50021818d9391df98 Mon Sep 17 00:00:00 2001 From: "Florent Revest (Anthropic)" Date: Sat, 26 Sep 2026 13:55:58 +0000 Subject: bpf: Wait for an RCU tasks grace period before freeing trampoline progs When a prog is detached from a trampoline, it is freed after an RCU grace period, or an RCU tasks trace one if it is sleepable. This covers the tasks that are running the prog, since the prog's enter helper takes the matching read lock before the prog is called. On a preemptible kernel, it doesn't cover a task that was preempted in the trampoline just before the enter helper. That task holds no lock yet, and it calls the prog after it was freed: BUG: KASAN: vmalloc-out-of-bounds in __bpf_prog_enter_recur+0x3a5/0x3f0 Read of size 8 at addr ffffc90000055040 by task candidate/110 CPU: 1 UID: 0 PID: 110 Comm: candidate Not tainted 7.3.0-rc2-00014-g15071f2a1263-dirty #2 PREEMPT(full) Call Trace: __bpf_prog_enter_recur+0x3a5/0x3f0 bpf_trampoline_6442509193+0x37/0xf1 __x64_sys_futex+0x9/0x410 do_syscall_64+0xb0/0x530 ... Wait for an RCU tasks grace period before the existing one when freeing a prog that was linked to a trampoline. An RCU tasks grace period only ends once the tasks that were preempted have run again, and bpf_tramp_image_put() already relies on it to free the image. It doesn't wait for tasks that sleep in the trampoline, the next commit takes care of those. Only progs that were linked to a trampoline can be called this way, so bpf_trampoline_add_prog() marks them and other progs are still freed as before. Fixes: e21aa341785c ("bpf: Fix fexit trampoline.") Reported-by: Junseo Lim Signed-off-by: Florent Revest (Anthropic) Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260926135605.1217928-2-florent.revest@linux.dev Closes: https://lore.kernel.org/bpf/aqdrwVpanH3WGurX@omen-arch/ --- kernel/bpf/syscall.c | 19 ++++++++++++++++++- kernel/bpf/trampoline.c | 1 + 2 files changed, 19 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 244a939b9d2d..96217b99399d 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2448,6 +2448,21 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) bpf_prog_free(aux->prog); } +/* + * Progs called from a trampoline can also be reached by a task that was + * preempted in the trampoline before the prog's enter helper took its RCU + * read lock, wait for those first. + */ +static void __bpf_prog_put_rcu_tasks(struct rcu_head *rcu) +{ + struct bpf_prog *prog = container_of(rcu, struct bpf_prog_aux, rcu)->prog; + + if (prog->sleepable) + call_rcu_tasks_trace(rcu, __bpf_prog_put_rcu); + else + call_rcu(rcu, __bpf_prog_put_rcu); +} + static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) { bpf_prog_kallsyms_del_all(prog); @@ -2461,7 +2476,9 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) btf_put(prog->aux->attach_btf); if (deferred) { - if (prog->sleepable) + if (IS_ENABLED(CONFIG_TASKS_RCU) && prog->aux->tramp_linked) + call_rcu_tasks(&prog->aux->rcu, __bpf_prog_put_rcu_tasks); + else if (prog->sleepable) call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); else call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 90b70ea0d370..9d69c066a817 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -907,6 +907,7 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, } hlist_add_head(&node->tramp_hlist, prog_list); + node->link->prog->aux->tramp_linked = true; if (kind == BPF_TRAMP_FSESSION) { tr->progs_cnt[BPF_TRAMP_FENTRY]++; fexit = fsession_exit(node); -- cgit v1.2.3 From 17637e1a581a22466ac3620a91e099683ff9cc6f Mon Sep 17 00:00:00 2001 From: "Florent Revest (Anthropic)" Date: Sat, 26 Sep 2026 13:55:59 +0000 Subject: bpf: Skip detached progs in trampoline images that are still in use bpf_tramp_image_put() makes sure a trampoline image is not freed while a task may still be running in it, but nothing similar is done for the progs called by that image. Detach drops the last prog reference right away and the prog is freed after grace periods, on the basis that a task still in the traced function skips the fexit progs once the nop at ip_after_call is patched to a jump. That leaves out a task sleeping in a sleepable prog that runs before the detached one, which no grace period waits for: CPU 0 CPU 1 in image I, sleeping in prog S detach P from I's trampoline -> new image, bpf_tramp_image_put(I) bpf_prog_put(P), last ref grace periods, P freed back from S __bpf_prog_enter(P) call P->bpf_func If S and P are fexit progs the task is already past the patched jump, and fentry only images don't have one. On x86 this is an int3 in poisoned bpf_prog_pack memory: Oops: int3: 0000 [#1] SMP NOPTI CPU: 18 UID: 0 PID: 94573 Comm: x169 Not tainted 6.18.44 #1 PREEMPT(lazy) RIP: 0010:0xffffffffc0601d8d Call Trace: ? bpf_trampoline_6442515411+0x1a4/0x21b bpf_lsm_bprm_committed_creds+0x5/0x10 security_bprm_committed_creds+0x5f/0x70 begin_new_exec+0x2d6/0x410 ... We hit this in production when progs attached through trampolines got detached while their hooks were busy, and it was independently found with a fuzzer and KASAN. Have the JITs emit a patchable nop in front of each prog call sequence and record it in the image. When a prog is detached, patch its nop to a jump over the call sequence. Progs that stay attached keep running for the tasks that are in the image, and ip_after_call isn't needed anymore. The task can be in any image that isn't freed yet, not only in the current one. It sleeps in image I1 that calls S, P and Q, then P is detached and the trampoline moves to image I2, then Q is detached and its call is still in I1. So the trampoline keeps a list of its images until they are freed, and detaching a prog patches its nop in all of them. Images hold a reference on the trampoline for that long. On riscv and loongarch a jump of any range takes several instructions, and a task preempted in the middle of them could resume into half of the new sequence. The nop is a single instruction there, patched to a near branch through arch_bpf_trampoline_skip(). With the extra nops, BPF_MAX_TRAMP_LINKS progs no longer fit in a page on x86 and arm64 (and already didn't on powerpc), so lower the limit there like s390 does. Fixes: e21aa341785c ("bpf: Fix fexit trampoline.") Reported-by: Sechang Lim Suggested-by: Alexei Starovoitov Signed-off-by: Florent Revest (Anthropic) Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260926135605.1217928-3-florent.revest@linux.dev Closes: https://lore.kernel.org/bpf/20260815071927.147049-1-zirajs7@gmail.com/ --- kernel/bpf/trampoline.c | 84 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 67 insertions(+), 17 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 9d69c066a817..bf4cb3dd444d 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -401,6 +401,7 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip) head = &trampoline_ip_table[hash_64(tr->ip, TRAMPOLINE_HASH_BITS)]; hlist_add_head(&tr->hlist_ip, head); refcount_set(&tr->refcnt, 1); + INIT_LIST_HEAD(&tr->images); for (i = 0; i < BPF_TRAMP_MAX; i++) INIT_HLIST_HEAD(&tr->progs_hlist[i]); out: @@ -565,15 +566,22 @@ static void bpf_tramp_image_free(struct bpf_tramp_image *im) arch_free_bpf_trampoline(im->image, im->size); bpf_jit_uncharge_modmem(im->size); percpu_ref_exit(&im->pcref); + kfree(im->skips); kfree_rcu(im, rcu); } static void __bpf_tramp_image_put_deferred(struct work_struct *work) { struct bpf_tramp_image *im; + struct bpf_trampoline *tr; im = container_of(work, struct bpf_tramp_image, work); + tr = im->tr; + trampoline_lock(tr); + list_del(&im->list); + trampoline_unlock(tr); bpf_tramp_image_free(im); + bpf_trampoline_put(tr); } /* callback, fexit step 3 or fentry step 2 */ @@ -601,7 +609,7 @@ static void __bpf_tramp_image_put_rcu_tasks(struct rcu_head *rcu) struct bpf_tramp_image *im; im = container_of(rcu, struct bpf_tramp_image, rcu); - if (im->ip_after_call) + if (im->call_orig) /* the case of fmod_ret/fexit trampoline and CONFIG_PREEMPTION=y */ percpu_ref_kill(&im->pcref); else @@ -621,9 +629,9 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) * * The trampoline is unreachable before bpf_tramp_image_put(). * - * First, patch the trampoline to avoid calling into fexit progs. - * The progs will be freed even if the original function is still - * executing or sleeping. + * Progs are patched out of the image when they are detached, see + * bpf_trampoline_skip_prog(), so they can be freed even if a task is + * still in the image. * In case of CONFIG_PREEMPT=y use call_rcu_tasks() to wait on * first few asm instructions to execute and call into * __bpf_tramp_enter->percpu_ref_get. @@ -637,11 +645,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) * percpu_ref_kill will be waiting for. Hence the first * call_rcu_tasks() is not necessary. */ - if (im->ip_after_call) { - int err = bpf_arch_text_poke(im->ip_after_call, BPF_MOD_NOP, - BPF_MOD_JUMP, NULL, - im->ip_epilogue); - WARN_ON(err); + if (im->call_orig) { if (IS_ENABLED(CONFIG_TASKS_RCU)) call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu_tasks); else @@ -658,7 +662,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) call_rcu_tasks_trace(&im->rcu, __bpf_tramp_image_put_rcu_tasks); } -static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) +static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size, int nr_progs) { struct bpf_tramp_image *im; struct bpf_ksym *ksym; @@ -669,6 +673,10 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) if (!im) goto out; + im->skips = kzalloc_objs(*im->skips, nr_progs); + if (!im->skips) + goto out_free_im; + err = bpf_jit_charge_modmem(size); if (err) goto out_free_im; @@ -695,6 +703,7 @@ out_free_image: out_uncharge: bpf_jit_uncharge_modmem(size); out_free_im: + kfree(im->skips); kfree(im); out: return ERR_PTR(err); @@ -771,11 +780,12 @@ again: goto out; } - im = bpf_tramp_image_alloc(tr->key, size); + im = bpf_tramp_image_alloc(tr->key, size, total); if (IS_ERR(im)) { err = PTR_ERR(im); goto out; } + im->call_orig = tr->flags & BPF_TRAMP_F_CALL_ORIG; err = arch_prepare_bpf_trampoline(im, im->image, im->image + size, &tr->func.model, tr->flags, tnodes, @@ -806,8 +816,14 @@ again: #endif out_free: - if (err) + if (err) { bpf_tramp_image_free(im); + } else { + /* track the image until it is freed, for bpf_trampoline_skip_prog() */ + refcount_inc(&tr->refcnt); + im->tr = tr; + list_add(&im->list, &tr->images); + } out: /* If any error happens, restore previous flags */ if (err) @@ -921,6 +937,41 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, return 0; } +/* + * Patch the nop in front of a prog call to a jump over it. A task can be + * preempted anywhere in the image, so archs that need several instructions for + * a jump of any range patch a single near branch here instead. + */ +int __weak arch_bpf_trampoline_skip(void *nop, void *target) +{ + return bpf_arch_text_poke(nop, BPF_MOD_NOP, BPF_MOD_JUMP, NULL, target); +} + +/* + * prog was detached and can be freed, but tasks may still be running in images + * that call it, sleeping in an earlier prog for example. They can be in any + * image that is not freed yet, not only in cur_image, so patch all of them to + * jump over prog. + */ +static void bpf_trampoline_skip_prog(struct bpf_trampoline *tr, struct bpf_prog *prog) +{ + struct bpf_tramp_image *im; + int i, err; + + list_for_each_entry(im, &tr->images, list) { + for (i = 0; i < im->nr_skips; i++) { + struct bpf_tramp_skip *skip = &im->skips[i]; + + if (skip->prog != prog) + continue; + err = arch_bpf_trampoline_skip(skip->nop, skip->target); + WARN_ON_ONCE(err); + /* not a nop anymore, and prog's address can be reused */ + skip->prog = NULL; + } + } +} + static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr, struct bpf_tramp_node *node) { @@ -938,6 +989,7 @@ static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr, } hlist_del_init(&node->tramp_hlist); tr->progs_cnt[kind]--; + bpf_trampoline_skip_prog(tr, node->link->prog); } static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node, @@ -1246,11 +1298,9 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[i]))) goto out; - /* This code will be executed even when the last bpf_tramp_image - * is alive. All progs are detached from the trampoline and the - * trampoline image is patched with jmp into epilogue to skip - * fexit progs. The fentry-only trampoline will be freed via - * multiple rcu callbacks. + /* + * All progs are detached and the last image has been freed, images + * hold a reference on the trampoline until then. */ hlist_del(&tr->hlist_key); hlist_del(&tr->hlist_ip); -- cgit v1.2.3 From e955f14fa8467d70aaeea5830b4d1773d5b25500 Mon Sep 17 00:00:00 2001 From: Zhan Xusheng Date: Mon, 28 Sep 2026 10:02:04 +0800 Subject: sysctl: Negate before converting in the int read path proc_int_k2u_conv_kop() reports the sign through *negp and the magnitude through *u_ptr, but for a negative value it hands the sign-extended int to the converter and negates the result: *u_ptr = k_ptr_op ? -k_ptr_op((ulong)val) : -(ulong)val; With div_hz() and CONFIG_HZ=1000 a stored -1000 becomes (ulong)-1000 / 1000 == 18446744073709550, and negating that wraps: # echo -1 > /proc/sys/net/ipv4/tcp_fin_timeout # cat /proc/sys/net/ipv4/tcp_fin_timeout -18428297329635842066 The magnitude is HZ dependent but the wrap is not: the negation happens after the division at every CONFIG_HZ. Take the magnitude first and convert that, which is what the open-coded version did before commit 2dc164a48e6f ("sysctl: Create converter functions with two new macros") folded it into a macro. The k_ptr_op == NULL branch was already correct. All three int converters that pass a k_ptr_op are affected: proc_dointvec_jiffies(), proc_dointvec_userhz_jiffies() and proc_dointvec_ms_jiffies(). Fixes: 2dc164a48e6f ("sysctl: Create converter functions with two new macros") Cc: stable@vger.kernel.org Reviewed-by: Bradley Morgan Signed-off-by: Zhan Xusheng Signed-off-by: Joel Granados --- kernel/sysctl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sysctl.c b/kernel/sysctl.c index f7b75985d542..38597f26b34c 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -483,7 +483,7 @@ int proc_int_k2u_conv_kop(ulong *u_ptr, const int *k_ptr, bool *negp, if (val < 0) { *negp = true; - *u_ptr = k_ptr_op ? -k_ptr_op((ulong)val) : -(ulong)val; + *u_ptr = k_ptr_op ? k_ptr_op(-(ulong)val) : -(ulong)val; } else { *negp = false; *u_ptr = k_ptr_op ? k_ptr_op((ulong)val) : (ulong) val; -- cgit v1.2.3 From fdd6553e1e53b0421d89c7469c8a36d2eadb1115 Mon Sep 17 00:00:00 2001 From: Zhan Xusheng Date: Mon, 28 Sep 2026 10:02:05 +0800 Subject: time/jiffies: Saturate in mult_hz() instead of wrapping Return ULONG_MAX for values that don't fit an unsigned long. proc_int_u2k_conv_uop() now correctly rejects a result above INT_MAX. This is the erroneous behaviour that is being fixed. The input has to exceed ULONG_MAX / HZ for the product to wrap, so the value below is specific to CONFIG_HZ=1000: # echo 18446744073709552 > /proc/sys/net/ipv4/tcp_keepalive_time # cat /proc/sys/net/ipv4/tcp_keepalive_time 0 That value is now rejected with an error. The original bound ("*u_ptr > INT_MAX / HZ") was removed in commit 2dc164a48e6f ("sysctl: Create converter functions with two new macros"). Fixes: 2dc164a48e6f ("sysctl: Create converter functions with two new macros") Cc: stable@vger.kernel.org Signed-off-by: Zhan Xusheng Signed-off-by: Joel Granados --- kernel/time/jiffies.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index 80c354811538..9b3487d40cd6 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c @@ -101,6 +101,8 @@ void __init register_refined_jiffies(long cycles_per_second) #ifdef CONFIG_SYSCTL static ulong mult_hz(const ulong val) { + if (val >= ULONG_MAX / HZ) + return ULONG_MAX; return val * HZ; } -- cgit v1.2.3 From e0a6249190402a28f1e2925a81acf573157d55ef Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Tue, 29 Sep 2026 09:19:12 +0900 Subject: fprobe: Use guard(rcu_sched_notrace) and check rcu_is_watching() unregister_fprobe() and unregister_fprobe_async() (used by BPF kprobe-multi) rely on standard RCU grace periods (synchronize_rcu() and call_rcu()) to wait until in-flight fprobe handlers complete before freeing the fprobe. However, if an fprobe handler executes while RCU is not watching (such as in the idle loop or nohz_full extended quiescent states), standard RCU does not track preemption-disabled sections. Consequently, synchronize_rcu() does not wait for those executions, which can lead to a use-after-free if the fprobe is freed immediately after unregistration. Ensure handlers exit early when !rcu_is_watching(). Furthermore, fprobe_fgraph_entry() and fprobe_ftrace_entry() previously used guard(rcu)() and rcu_read_lock(), which invoke lockdep on every hit under CONFIG_PROVE_LOCKING. This adds overhead and can cause lockdep recursion if probed functions interact with lockdep. Since rhltable_lookup() and rhl_for_each_entry_rcu() use rcu_dereference_all_check() (which checks rcu_read_lock_any_held()), holding preemption disabled via rcu_read_lock_sched_notrace() is fully valid and sufficient so long as rcu_is_watching() is true. Define and use guard(rcu_sched_notrace)() across fprobe_ftrace_entry(), fprobe_fgraph_entry(), and fprobe_return(). This eliminates fast-path rcu_read_lock() and lockdep overhead while guaranteeing safe grace period synchronization. Link: https://lore.kernel.org/all/179064115227.394389.16910234241400391996.stgit@devnote2/ Reported-by: Sashiko Closes: https://sashiko.dev/#/bug/linux-e46bcd68-4a56-4f19-a255-e3772980e5e3 Fixes: 657b594b2084 ("fprobe: Fix unregister_fprobe() to wait for RCU grace period") Cc: stable@vger.kernel.org Assisted-by: LLM Signed-off-by: Masami Hiramatsu (Google) Reviewed-by: Paul E. McKenney --- kernel/trace/fprobe.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index 9f2d98181779..da286619c5d8 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -47,6 +47,10 @@ static struct rhltable fprobe_ip_table; static DEFINE_MUTEX(fprobe_mutex); static struct fgraph_ops fprobe_graph_ops; +DEFINE_LOCK_GUARD_0(rcu_sched_notrace, + rcu_read_lock_sched_notrace(), + rcu_read_unlock_sched_notrace()) + static u32 fprobe_node_hashfn(const void *data, u32 len, u32 seed) { return hash_ptr(*(unsigned long **)data, 32); @@ -329,16 +333,14 @@ static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip, struct fprobe *fp; int bit; + if (!rcu_is_watching()) + return; + bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return; - /* - * ftrace_test_recursion_trylock() disables preemption, but - * rhltable_lookup() checks whether rcu_read_lcok is held. - * So we take rcu_read_lock() here. - */ - rcu_read_lock(); + guard(rcu_sched_notrace)(); head = rhltable_lookup(&fprobe_ip_table, &ip, fprobe_rht_params); rhl_for_each_entry_rcu(node, pos, head, hlist) { @@ -353,7 +355,6 @@ static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip, else __fprobe_handler(ip, parent_ip, fp, fregs, NULL); } - rcu_read_unlock(); ftrace_test_recursion_unlock(bit); } NOKPROBE_SYMBOL(fprobe_ftrace_entry); @@ -567,10 +568,13 @@ static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops struct fprobe *fp; int used, ret; + if (!rcu_is_watching()) + return 0; + if (WARN_ON_ONCE(!fregs)) return 0; - guard(rcu)(); + guard(rcu_sched_notrace)(); head = rhltable_lookup(&fprobe_ip_table, &func, fprobe_rht_params); reserved_words = 0; rhl_for_each_entry_rcu(node, pos, head, hlist) { @@ -665,13 +669,16 @@ static void fprobe_return(struct ftrace_graph_ret *trace, int size, curr; int size_words; + if (!rcu_is_watching()) + return; + fgraph_data = (unsigned long *)fgraph_retrieve_data(gops->idx, &size); if (WARN_ON_ONCE(!fgraph_data)) return; size_words = SIZE_IN_LONG(size); ret_ip = ftrace_regs_get_instruction_pointer(fregs); - preempt_disable_notrace(); + guard(rcu_sched_notrace)(); curr = 0; while (size_words > curr) { @@ -687,7 +694,6 @@ static void fprobe_return(struct ftrace_graph_ret *trace, } curr += size; } - preempt_enable_notrace(); } NOKPROBE_SYMBOL(fprobe_return); -- cgit v1.2.3 From ffb684f2aa141eeb0caa6308422e7e05d1ded7c4 Mon Sep 17 00:00:00 2001 From: Luo Gengkun Date: Sun, 20 Sep 2026 07:50:26 +0000 Subject: perf: Fix race between perf_event_exit_task() and perf_pending_task() A race condition exists between perf_event_exit_task() and perf_pending_task() during begin_new_exec(). During begin_new_exec(), perf_event_exit_task() may be called, and the PF_EXITING flag is not set on task. So perf_sigtrap() continues to execute and triggers WARN_ON_ONCE(event->ctx->task != current). Since both task exit and exec paths can call perf_event_exit_task() which sets ctx->task to TASK_TOMBSTONE, fix this by explicitly checking if event->ctx->task equals TASK_TOMBSTONE and dropping the redundant PF_EXITING check. Fixes: 97ba62b27867 ("perf: Add support for SIGTRAP on perf events") Signed-off-by: Luo Gengkun Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260920075026.990582-1-luogengkun2@huawei.com --- kernel/events/core.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index 634d2ccbab82..c2882b5be7c3 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7626,9 +7626,11 @@ static void perf_sigtrap(struct perf_event *event) { /* * Both perf_pending_task() and perf_pending_irq() can race with the - * task exiting. + * task exiting or exec-ing. We can determine if such a race has + * occurred by checking if perf_event_exit_task(), which will set + * ctx->task to TASK_TOMBSTONE, has already been called. */ - if (current->flags & PF_EXITING) + if (event->ctx->task == TASK_TOMBSTONE) return; /* -- cgit v1.2.3 From b9d1fdc6f4ac1b6e49f9deafaf137da1407d9af1 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 29 Sep 2026 15:23:32 -0700 Subject: perf: Replace perf_event_header__init_id with full header init perf_iterate_sb() invokes its callback for each matching perf_event on the CPU and task context, passing a shared caller-allocated event structure. perf_event_header__init_id() mutated header->size in place by adding event->id_header_size, requiring sideband output callbacks to save and restore header fields across iterations. Three sideband callbacks failed to save and restore header.size around perf_event_header__init_id(): - perf_event_ksymbol_output() - perf_event_bpf_output() - perf_event_text_poke_output() When multiple events with attr.ksymbol, attr.bpf_event, or attr.text_poke and sample_id_all are active on the same CPU, each subsequent event receives a record whose header.size is inflated by all preceding events' id_header_size values while only a single id_sample is written, leaving uninitialized ring-buffer bytes at the end of the record and causing userspace perf to fail with -EFAULT ("Bad address") when parsing the sample_id trailer. Similarly, perf_event_mmap_output() set PERF_RECORD_MISC_MMAP_BUILD_ID in mmap_event->event_id.header.misc when event->attr.build_id was enabled, but only saved and restored header.size and header.type. If an event with attr.build_id was followed by an event with attr.mmap2 and !attr.build_id, the second event received PERF_RECORD_MISC_MMAP_BUILD_ID in header.misc while its payload contained maj/min/ino/ino_generation instead of a build ID. Rather than splitting header initialization between callers and output callbacks and saving/restoring mutated header fields, replace perf_event_header__init_id() with perf_event_header__init(), which initializes header->type, header->misc, and header->size alongside the sample_id fields on each invocation. Fixes: 76193a94522f ("perf, bpf: Introduce PERF_RECORD_KSYMBOL") Fixes: 6ee52e2a3fe4 ("perf, bpf: Introduce PERF_RECORD_BPF_EVENT") Fixes: e17d43b93e54 ("perf: Add perf text poke event") Fixes: 88a16a130933 ("perf: Add build id data in mmap2 event") Assisted-by: Antigravity:gemini-3.1-pro Signed-off-by: Ian Rogers Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260929222332.973435-1-irogers@google.com Cc: stable@vger.kernel.org --- kernel/events/core.c | 307 +++++++++++++++++++------------------------- kernel/events/ring_buffer.c | 7 +- 2 files changed, 132 insertions(+), 182 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index c2882b5be7c3..3aa223595e7d 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -8131,10 +8131,15 @@ static void __perf_event_header__init_id(struct perf_sample_data *data, } } -void perf_event_header__init_id(struct perf_event_header *header, - struct perf_sample_data *data, - struct perf_event *event) +void perf_event_header__init(struct perf_event_header *header, + struct perf_sample_data *data, + u32 type, u16 misc, u16 size, + struct perf_event *event) { + header->type = type; + header->misc = misc; + header->size = size; + if (event->attr.sample_id_all) { header->size += event->id_header_size; __perf_event_header__init_id(data, event, event->attr.sample_type); @@ -8971,17 +8976,16 @@ perf_event_read_event(struct perf_event *event, struct perf_output_handle handle; struct perf_sample_data sample; struct perf_read_event read_event = { - .header = { - .type = PERF_RECORD_READ, - .misc = 0, - .size = sizeof(read_event) + event->read_size, - }, .pid = perf_event_pid(event, task), .tid = perf_event_tid(event, task), }; int ret; - perf_event_header__init_id(&read_event.header, &sample, event); + perf_event_header__init(&read_event.header, &sample, + PERF_RECORD_READ, + /* misc= */ 0, + sizeof(read_event) + event->read_size, + event); ret = perf_output_begin(&handle, &sample, event, read_event.header.size); if (ret) return; @@ -9222,6 +9226,7 @@ struct perf_task_event { u32 ptid; u64 time; } event_id; + int new; }; static int perf_event_task_match(struct perf_event *event) @@ -9238,17 +9243,21 @@ static void perf_event_task_output(struct perf_event *event, struct perf_output_handle handle; struct perf_sample_data sample; struct task_struct *task = task_event->task; - int ret, size = task_event->event_id.header.size; + int ret; if (!perf_event_task_match(event)) return; - perf_event_header__init_id(&task_event->event_id.header, &sample, event); + perf_event_header__init(&task_event->event_id.header, &sample, + task_event->new ? PERF_RECORD_FORK : PERF_RECORD_EXIT, + /* misc= */ 0, + sizeof(task_event->event_id), + event); ret = perf_output_begin(&handle, &sample, event, task_event->event_id.header.size); if (ret) - goto out; + return; task_event->event_id.pid = perf_event_pid(event, task); task_event->event_id.tid = perf_event_tid(event, task); @@ -9270,8 +9279,6 @@ static void perf_event_task_output(struct perf_event *event, perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - task_event->event_id.header.size = size; } static void perf_event_task(struct task_struct *task, @@ -9288,18 +9295,7 @@ static void perf_event_task(struct task_struct *task, task_event = (struct perf_task_event){ .task = task, .task_ctx = task_ctx, - .event_id = { - .header = { - .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT, - .misc = 0, - .size = sizeof(task_event.event_id), - }, - /* .pid */ - /* .ppid */ - /* .tid */ - /* .ptid */ - /* .time */ - }, + .new = new, }; perf_iterate_sb(perf_event_task_output, @@ -9376,6 +9372,7 @@ struct perf_comm_event { u32 pid; u32 tid; } event_id; + bool exec; }; static int perf_event_comm_match(struct perf_event *event) @@ -9389,18 +9386,21 @@ static void perf_event_comm_output(struct perf_event *event, struct perf_comm_event *comm_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - int size = comm_event->event_id.header.size; int ret; if (!perf_event_comm_match(event)) return; - perf_event_header__init_id(&comm_event->event_id.header, &sample, event); + perf_event_header__init(&comm_event->event_id.header, &sample, + PERF_RECORD_COMM, + comm_event->exec ? PERF_RECORD_MISC_COMM_EXEC : 0, + sizeof(comm_event->event_id) + comm_event->comm_size, + event); ret = perf_output_begin(&handle, &sample, event, comm_event->event_id.header.size); if (ret) - goto out; + return; comm_event->event_id.pid = perf_event_pid(event, comm_event->task); comm_event->event_id.tid = perf_event_tid(event, comm_event->task); @@ -9412,8 +9412,6 @@ static void perf_event_comm_output(struct perf_event *event, perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - comm_event->event_id.header.size = size; } static void perf_event_comm_event(struct perf_comm_event *comm_event) @@ -9428,8 +9426,6 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event) comm_event->comm = comm; comm_event->comm_size = size; - comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; - perf_iterate_sb(perf_event_comm_output, comm_event, NULL); @@ -9446,15 +9442,8 @@ void perf_event_comm(struct task_struct *task, bool exec) .task = task, /* .comm */ /* .comm_size */ - .event_id = { - .header = { - .type = PERF_RECORD_COMM, - .misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0, - /* .size */ - }, - /* .pid */ - /* .tid */ - }, + /* .event_id */ + .exec = exec, }; perf_event_comm_event(&comm_event); @@ -9488,18 +9477,20 @@ static void perf_event_namespaces_output(struct perf_event *event, struct perf_namespaces_event *namespaces_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - u16 header_size = namespaces_event->event_id.header.size; int ret; if (!perf_event_namespaces_match(event)) return; - perf_event_header__init_id(&namespaces_event->event_id.header, - &sample, event); + perf_event_header__init(&namespaces_event->event_id.header, &sample, + PERF_RECORD_NAMESPACES, + /* misc= */ 0, + sizeof(namespaces_event->event_id), + event); ret = perf_output_begin(&handle, &sample, event, namespaces_event->event_id.header.size); if (ret) - goto out; + return; namespaces_event->event_id.pid = perf_event_pid(event, namespaces_event->task); @@ -9511,8 +9502,6 @@ static void perf_event_namespaces_output(struct perf_event *event, perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - namespaces_event->event_id.header.size = header_size; } static void perf_fill_ns_link_info(struct perf_ns_link_info *ns_link_info, @@ -9543,11 +9532,7 @@ void perf_event_namespaces(struct task_struct *task) namespaces_event = (struct perf_namespaces_event){ .task = task, .event_id = { - .header = { - .type = PERF_RECORD_NAMESPACES, - .misc = 0, - .size = sizeof(namespaces_event.event_id), - }, + /* .header */ /* .pid */ /* .tid */ .nr_namespaces = NR_NAMESPACES, @@ -9615,18 +9600,19 @@ static void perf_event_cgroup_output(struct perf_event *event, void *data) struct perf_cgroup_event *cgroup_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - u16 header_size = cgroup_event->event_id.header.size; int ret; + u16 size = sizeof(cgroup_event->event_id) + cgroup_event->path_size; if (!perf_event_cgroup_match(event)) return; - perf_event_header__init_id(&cgroup_event->event_id.header, - &sample, event); + perf_event_header__init(&cgroup_event->event_id.header, &sample, + PERF_RECORD_CGROUP, /* misc= */ 0, size, + event); ret = perf_output_begin(&handle, &sample, event, cgroup_event->event_id.header.size); if (ret) - goto out; + return; perf_output_put(&handle, cgroup_event->event_id); __output_copy(&handle, cgroup_event->path, cgroup_event->path_size); @@ -9634,8 +9620,6 @@ static void perf_event_cgroup_output(struct perf_event *event, void *data) perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - cgroup_event->event_id.header.size = header_size; } static void perf_event_cgroup(struct cgroup *cgrp) @@ -9650,11 +9634,6 @@ static void perf_event_cgroup(struct cgroup *cgrp) cgroup_event = (struct perf_cgroup_event){ .event_id = { - .header = { - .type = PERF_RECORD_CGROUP, - .misc = 0, - .size = sizeof(cgroup_event.event_id), - }, .id = cgroup_id(cgrp), }, }; @@ -9677,7 +9656,6 @@ static void perf_event_cgroup(struct cgroup *cgrp) while (!IS_ALIGNED(size, sizeof(u64))) cgroup_event.path[size++] = '\0'; - cgroup_event.event_id.header.size += size; cgroup_event.path_size = size; perf_iterate_sb(perf_event_cgroup_output, @@ -9733,38 +9711,40 @@ static void perf_event_mmap_output(struct perf_event *event, struct perf_mmap_event *mmap_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - int size = mmap_event->event_id.header.size; - u32 type = mmap_event->event_id.header.type; - bool use_build_id; + int size = sizeof(mmap_event->event_id) + mmap_event->file_size; + u32 type = PERF_RECORD_MMAP; + u16 misc = PERF_RECORD_MISC_USER; + bool use_build_id = false; int ret; if (!perf_event_mmap_match(event, data)) return; if (event->attr.mmap2) { - mmap_event->event_id.header.type = PERF_RECORD_MMAP2; - mmap_event->event_id.header.size += sizeof(mmap_event->maj); - mmap_event->event_id.header.size += sizeof(mmap_event->min); - mmap_event->event_id.header.size += sizeof(mmap_event->ino); - mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation); - mmap_event->event_id.header.size += sizeof(mmap_event->prot); - mmap_event->event_id.header.size += sizeof(mmap_event->flags); - } - - perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); + type = PERF_RECORD_MMAP2; + size += sizeof(mmap_event->maj); + size += sizeof(mmap_event->min); + size += sizeof(mmap_event->ino); + size += sizeof(mmap_event->ino_generation); + size += sizeof(mmap_event->prot); + size += sizeof(mmap_event->flags); + use_build_id = event->attr.build_id && mmap_event->build_id_size; + if (use_build_id) + misc |= PERF_RECORD_MISC_MMAP_BUILD_ID; + } + if (!(mmap_event->vma->vm_flags & VM_EXEC)) + misc |= PERF_RECORD_MISC_MMAP_DATA; + + perf_event_header__init(&mmap_event->event_id.header, &sample, + type, misc, size, event); ret = perf_output_begin(&handle, &sample, event, mmap_event->event_id.header.size); if (ret) - goto out; + return; mmap_event->event_id.pid = perf_event_pid(event, current); mmap_event->event_id.tid = perf_event_tid(event, current); - use_build_id = event->attr.build_id && mmap_event->build_id_size; - - if (event->attr.mmap2 && use_build_id) - mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID; - perf_output_put(&handle, mmap_event->event_id); if (event->attr.mmap2) { @@ -9789,9 +9769,6 @@ static void perf_event_mmap_output(struct perf_event *event, perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - mmap_event->event_id.header.size = size; - mmap_event->event_id.header.type = type; } static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) @@ -9887,11 +9864,6 @@ got_name: mmap_event->prot = prot; mmap_event->flags = flags; - if (!(vma->vm_flags & VM_EXEC)) - mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA; - - mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; - if (atomic_read(&nr_build_id_events)) build_id_parse_nofault(vma, mmap_event->build_id, &mmap_event->build_id_size); @@ -10011,11 +9983,7 @@ void perf_event_mmap(struct vm_area_struct *vma) /* .file_name */ /* .file_size */ .event_id = { - .header = { - .type = PERF_RECORD_MMAP, - .misc = PERF_RECORD_MISC_USER, - /* .size */ - }, + /* .header */ /* .pid */ /* .tid */ .start = vma->vm_start, @@ -10045,18 +10013,15 @@ void perf_event_aux_event(struct perf_event *event, unsigned long head, u64 size; u64 flags; } rec = { - .header = { - .type = PERF_RECORD_AUX, - .misc = 0, - .size = sizeof(rec), - }, .offset = head, .size = size, .flags = flags, }; int ret; - perf_event_header__init_id(&rec.header, &sample, event); + perf_event_header__init(&rec.header, &sample, + PERF_RECORD_AUX, /* misc= */ 0, sizeof(rec), + event); ret = perf_output_begin(&handle, &sample, event, rec.header.size); if (ret) @@ -10081,15 +10046,14 @@ void perf_log_lost_samples(struct perf_event *event, u64 lost) struct perf_event_header header; u64 lost; } lost_samples_event = { - .header = { - .type = PERF_RECORD_LOST_SAMPLES, - .misc = 0, - .size = sizeof(lost_samples_event), - }, .lost = lost, }; - perf_event_header__init_id(&lost_samples_event.header, &sample, event); + perf_event_header__init(&lost_samples_event.header, &sample, + PERF_RECORD_LOST_SAMPLES, + /* misc= */ 0, + sizeof(lost_samples_event), + event); ret = perf_output_begin(&handle, &sample, event, lost_samples_event.header.size); @@ -10114,6 +10078,8 @@ struct perf_switch_event { u32 next_prev_pid; u32 next_prev_tid; } event_id; + bool sched_in; + bool preempt; }; static int perf_event_switch_match(struct perf_event *event) @@ -10126,6 +10092,9 @@ static void perf_event_switch_output(struct perf_event *event, void *data) struct perf_switch_event *se = data; struct perf_output_handle handle; struct perf_sample_data sample; + u32 type; + u16 misc; + u16 size; int ret; if (!perf_event_switch_match(event)) @@ -10133,18 +10102,22 @@ static void perf_event_switch_output(struct perf_event *event, void *data) /* Only CPU-wide events are allowed to see next/prev pid/tid */ if (event->ctx->task) { - se->event_id.header.type = PERF_RECORD_SWITCH; - se->event_id.header.size = sizeof(se->event_id.header); + type = PERF_RECORD_SWITCH; + size = sizeof(se->event_id.header); } else { - se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE; - se->event_id.header.size = sizeof(se->event_id); + type = PERF_RECORD_SWITCH_CPU_WIDE; + size = sizeof(se->event_id); se->event_id.next_prev_pid = perf_event_pid(event, se->next_prev); se->event_id.next_prev_tid = perf_event_tid(event, se->next_prev); } + misc = se->sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT; + if (se->preempt) + misc |= PERF_RECORD_MISC_SWITCH_OUT_PREEMPT; - perf_event_header__init_id(&se->event_id.header, &sample, event); + perf_event_header__init(&se->event_id.header, &sample, + type, misc, size, event); ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size); if (ret) @@ -10170,22 +10143,11 @@ static void perf_event_switch(struct task_struct *task, switch_event = (struct perf_switch_event){ .task = task, .next_prev = next_prev, - .event_id = { - .header = { - /* .type */ - .misc = sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT, - /* .size */ - }, - /* .next_prev_pid */ - /* .next_prev_tid */ - }, + /* .event_id */ + .sched_in = sched_in, + .preempt = !sched_in && task_is_runnable(task), }; - if (!sched_in && task_is_runnable(task)) { - switch_event.event_id.header.misc |= - PERF_RECORD_MISC_SWITCH_OUT_PREEMPT; - } - perf_iterate_sb(perf_event_switch_output, &switch_event, NULL); } @@ -10205,20 +10167,17 @@ static void perf_log_throttle(struct perf_event *event, int enable) u64 id; u64 stream_id; } throttle_event = { - .header = { - .type = PERF_RECORD_THROTTLE, - .misc = 0, - .size = sizeof(throttle_event), - }, .time = perf_event_clock(event), .id = primary_event_id(event), .stream_id = event->id, }; - if (enable) - throttle_event.header.type = PERF_RECORD_UNTHROTTLE; - - perf_event_header__init_id(&throttle_event.header, &sample, event); + perf_event_header__init(&throttle_event.header, &sample, + enable ? PERF_RECORD_UNTHROTTLE + : PERF_RECORD_THROTTLE, + /* misc= */ 0, + sizeof(throttle_event), + event); ret = perf_output_begin(&handle, &sample, event, throttle_event.header.size); @@ -10257,12 +10216,14 @@ static void perf_event_ksymbol_output(struct perf_event *event, void *data) struct perf_output_handle handle; struct perf_sample_data sample; int ret; + u16 size = sizeof(ksymbol_event->event_id) + ksymbol_event->name_len; if (!perf_event_ksymbol_match(event)) return; - perf_event_header__init_id(&ksymbol_event->event_id.header, - &sample, event); + perf_event_header__init(&ksymbol_event->event_id.header, &sample, + PERF_RECORD_KSYMBOL, /* misc= */ 0, size, + event); ret = perf_output_begin(&handle, &sample, event, ksymbol_event->event_id.header.size); if (ret) @@ -10303,11 +10264,6 @@ void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len, bool unregister, .name = name, .name_len = name_len, .event_id = { - .header = { - .type = PERF_RECORD_KSYMBOL, - .size = sizeof(ksymbol_event.event_id) + - name_len, - }, .addr = addr, .len = len, .ksym_type = ksym_type, @@ -10351,8 +10307,11 @@ static void perf_event_bpf_output(struct perf_event *event, void *data) if (!perf_event_bpf_match(event)) return; - perf_event_header__init_id(&bpf_event->event_id.header, - &sample, event); + perf_event_header__init(&bpf_event->event_id.header, &sample, + PERF_RECORD_BPF_EVENT, + /* misc= */ 0, + sizeof(bpf_event->event_id), + event); ret = perf_output_begin(&handle, &sample, event, bpf_event->event_id.header.size); if (ret) @@ -10408,10 +10367,6 @@ void perf_event_bpf_event(struct bpf_prog *prog, bpf_event = (struct perf_bpf_event){ .prog = prog, .event_id = { - .header = { - .type = PERF_RECORD_BPF_EVENT, - .size = sizeof(bpf_event.event_id), - }, .type = type, .flags = flags, .id = prog->aux->id, @@ -10439,18 +10394,23 @@ static void perf_callchain_deferred_output(struct perf_event *event, void *data) struct perf_callchain_deferred_event *deferred_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - int ret, size = deferred_event->event.header.size; + int ret; + u16 size = sizeof(deferred_event->event) + (deferred_event->trace->nr * sizeof(u64)); if (!event->attr.defer_output) return; /* XXX do we really need sample_id_all for this ??? */ - perf_event_header__init_id(&deferred_event->event.header, &sample, event); + perf_event_header__init(&deferred_event->event.header, &sample, + PERF_RECORD_CALLCHAIN_DEFERRED, + PERF_RECORD_MISC_USER, + size, + event); ret = perf_output_begin(&handle, &sample, event, deferred_event->event.header.size); if (ret) - goto out; + return; perf_output_put(&handle, deferred_event->event); for (int i = 0; i < deferred_event->trace->nr; i++) { @@ -10460,8 +10420,6 @@ static void perf_callchain_deferred_output(struct perf_event *event, void *data) perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - deferred_event->event.header.size = size; } static void perf_unwind_deferred_callback(struct unwind_work *work, @@ -10470,12 +10428,6 @@ static void perf_unwind_deferred_callback(struct unwind_work *work, struct perf_callchain_deferred_event deferred_event = { .trace = trace, .event = { - .header = { - .type = PERF_RECORD_CALLCHAIN_DEFERRED, - .misc = PERF_RECORD_MISC_USER, - .size = sizeof(deferred_event.event) + - (trace->nr * sizeof(u64)), - }, .cookie = cookie, .nr = trace->nr, }, @@ -10487,7 +10439,8 @@ static void perf_unwind_deferred_callback(struct unwind_work *work, struct perf_text_poke_event { const void *old_bytes; const void *new_bytes; - size_t pad; + u16 tot; + u16 pad; u16 old_len; u16 new_len; @@ -10508,13 +10461,18 @@ static void perf_event_text_poke_output(struct perf_event *event, void *data) struct perf_text_poke_event *text_poke_event = data; struct perf_output_handle handle; struct perf_sample_data sample; + u16 size = sizeof(text_poke_event->event_id) + text_poke_event->tot + text_poke_event->pad; u64 padding = 0; int ret; if (!perf_event_text_poke_match(event)) return; - perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event); + perf_event_header__init(&text_poke_event->event_id.header, &sample, + PERF_RECORD_TEXT_POKE, + PERF_RECORD_MISC_KERNEL, + size, + event); ret = perf_output_begin(&handle, &sample, event, text_poke_event->event_id.header.size); @@ -10552,15 +10510,11 @@ void perf_event_text_poke(const void *addr, const void *old_bytes, text_poke_event = (struct perf_text_poke_event){ .old_bytes = old_bytes, .new_bytes = new_bytes, + .tot = tot, .pad = pad, .old_len = old_len, .new_len = new_len, .event_id = { - .header = { - .type = PERF_RECORD_TEXT_POKE, - .misc = PERF_RECORD_MISC_KERNEL, - .size = sizeof(text_poke_event.event_id) + tot + pad, - }, .addr = (unsigned long)addr, }, }; @@ -10591,13 +10545,12 @@ static void perf_log_itrace_start(struct perf_event *event) event->attach_state & PERF_ATTACH_ITRACE) return; - rec.header.type = PERF_RECORD_ITRACE_START; - rec.header.misc = 0; - rec.header.size = sizeof(rec); rec.pid = perf_event_pid(event, current); rec.tid = perf_event_tid(event, current); - perf_event_header__init_id(&rec.header, &sample, event); + perf_event_header__init(&rec.header, &sample, + PERF_RECORD_ITRACE_START, /* misc= */ 0, sizeof(rec), + event); ret = perf_output_begin(&handle, &sample, event, rec.header.size); if (ret) @@ -10622,12 +10575,10 @@ void perf_report_aux_output_id(struct perf_event *event, u64 hw_id) if (event->parent) event = event->parent; - rec.header.type = PERF_RECORD_AUX_OUTPUT_HW_ID; - rec.header.misc = 0; - rec.header.size = sizeof(rec); - rec.hw_id = hw_id; - - perf_event_header__init_id(&rec.header, &sample, event); + rec.hw_id = hw_id; + perf_event_header__init(&rec.header, &sample, + PERF_RECORD_AUX_OUTPUT_HW_ID, /* misc= */ 0, + sizeof(rec), event); ret = perf_output_begin(&handle, &sample, event, rec.header.size); if (ret) diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 1b1ffe0533e5..13bd42e5493d 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -246,14 +246,13 @@ __perf_output_begin(struct perf_output_handle *handle, handle->size = (1UL << page_shift) - offset; if (unlikely(have_lost)) { - lost_event.header.size = sizeof(lost_event); - lost_event.header.type = PERF_RECORD_LOST; - lost_event.header.misc = 0; lost_event.id = event->id; lost_event.lost = local_xchg(&rb->lost, 0); /* XXX mostly redundant; @data is already fully initializes */ - perf_event_header__init_id(&lost_event.header, data, event); + perf_event_header__init(&lost_event.header, data, + PERF_RECORD_LOST, /* misc= */ 0, + sizeof(lost_event), event); perf_output_put(handle, lost_event); perf_event__output_id_sample(event, handle, data); } -- cgit v1.2.3 From 357e8a77a501d96c9517f01f4a211eed5e9c9184 Mon Sep 17 00:00:00 2001 From: Zhengchuan Liang Date: Mon, 28 Sep 2026 10:59:35 -0700 Subject: perf: Require kernel access for text poke events Perf events with exclude_kernel=1 can be opened without kernel perf access. However, exclude_kernel does not suppress text-poke sideband records. Every PERF_RECORD_TEXT_POKE is marked PERF_RECORD_MISC_KERNEL and contains a raw kernel instruction address. An unprivileged task can therefore open and mmap a task-local software event with text_poke=1. Both opening a count-only tracepoint event and configuring UDP GRO for ESP-in-UDP cause updates to inline static calls; the observer receives the relocated addresses of the modified instructions. For a known kernel image, any such address reveals the runtime kernel text base despite KASLR. Call perf_allow_kernel() whenever attr.text_poke is set, regardless of exclude_kernel. Events that neither monitor kernel execution nor request text-poke records retain their existing permissions. Fixes: e17d43b93e54 ("perf: Add perf text poke event") Assisted-by: LLM Signed-off-by: Zhengchuan Liang Signed-off-by: Peter Zijlstra (Intel) Cc: stable@vger.kernel.org Link: https://patch.msgid.link/99131354c41e23188f778b92f90363775b482395.1790573390.git.zcliangcn@gmail.com --- kernel/events/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index 3aa223595e7d..7846d70be57f 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -13906,7 +13906,7 @@ SYSCALL_DEFINE5(perf_event_open, if (err) return err; - if (!attr.exclude_kernel || + if (!attr.exclude_kernel || attr.text_poke || ((attr.sample_type & PERF_SAMPLE_CALLCHAIN) && !attr.exclude_callchain_kernel)) { err = perf_allow_kernel(); -- cgit v1.2.3 From b88822d2584f78cbdad1d8256510dcac108b5630 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 1 Oct 2026 14:52:54 +0000 Subject: bpf: Fix packet range of pointers sharing an id Since commit 022ac0750883 ("bpf: use reg->var_off instead of reg->off for pointers"), find_good_pkt_pointers() sets the range of all packet pointers sharing an id from the umax of the compared pointer, and check_packet_access() requires umax + off + size <= range. That assumes the umax of two such pointers differ by exactly their constant distance. reg_bounds_sync() breaks it when var_off tightens one umax and not the other: r4 &= 0x38 if r4 > 50 goto exit ; umax 50, var_off (0x0; 0x38) r5 = pkt + r4 ; umax 50 r6 = r5 r6 += 8 ; umax 56, not 58 Comparing r6 with pkt_end sets the range to 56, and the valid 8-byte load at r5 is rejected (50 + 8 > 56). Comparing r5 sets it to 50, and the out-of-bounds 1-byte load at r6 - 7, i.e. r5 + 1, is accepted (56 - 7 + 1 <= 50). Don't call reg_bounds_sync() on a packet pointer that keeps its id (a constant was added or subtracted) or its range (an unknown non-negative value was subtracted), so that var_off cannot tighten its umax. Only update the 32-bit bounds from var_off: reg_bounds_sanity_check() wants them constant when the lower half of var_off is, e.g. for pkt + 8. This relies on nothing else changing the 64-bit bounds of a packet pointer, which holds today. var_off of such a pointer is no longer narrowed by its bounds. Adjust three verifier_align expectations; the low bits, which the alignment checks use, don't change. veristat on the selftests shows no verdict changes and +0.8% insns in test_cls_redirect_subprogs. Fixes: 022ac0750883 ("bpf: use reg->var_off instead of reg->off for pointers") Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261001145255.855630-1-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 41b49c56e123..5f874979b8d7 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -14826,7 +14826,15 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn "Tighten the scalar bounds before the arithmetic so the resulting pointer remains within the allowed range."); return -EINVAL; } - reg_bounds_sync(dst_reg); + /* + * A packet pointer that keeps its id or range is checked against a + * range set from the checked pointer's umax, so var_off must not tighten + * its umax. r32 must still match var_off for reg_bounds_sanity_check(). + */ + if (reg_is_pkt_pointer(dst_reg) && (known || dst_reg->range > 0)) + __update_reg32_bounds(dst_reg); + else + reg_bounds_sync(dst_reg); bounds_ret = sanitize_check_bounds(env, insn, dst_reg); if (bounds_ret == -EACCES) return bounds_ret; -- cgit v1.2.3 From 47f4f695cec1eb82d961fdb466cd7c99d2865b7c Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 30 Sep 2026 09:59:18 +0000 Subject: bpf: Factor out __do_call_rcu_ttrace() Move the part of do_call_rcu_ttrace() that runs after call_rcu_ttrace_in_progress is set into __do_call_rcu_ttrace(). The next patch will call it from __free_rcu(). No functional change. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20260930095920.601738-2-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/memalloc.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c index 8a8f088e83e6..08e4dde66cd5 100644 --- a/kernel/bpf/memalloc.c +++ b/kernel/bpf/memalloc.c @@ -298,19 +298,10 @@ static void enque_to_free(struct bpf_mem_cache *c, void *obj) llist_add(llnode, &c->free_by_rcu_ttrace); } -static void do_call_rcu_ttrace(struct bpf_mem_cache *c) +static void __do_call_rcu_ttrace(struct bpf_mem_cache *c) { struct llist_node *llnode, *t; - if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) { - if (unlikely(READ_ONCE(c->draining))) { - scoped_guard(raw_spinlock_irqsave, &c->lock) - llnode = llist_del_all(&c->free_by_rcu_ttrace); - free_all(c, llnode, !!c->percpu_size); - } - return; - } - WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace)); llist_for_each_safe(llnode, t, llist_del_all(&c->free_by_rcu_ttrace)) llist_add(llnode, &c->waiting_for_gp_ttrace); @@ -328,6 +319,22 @@ static void do_call_rcu_ttrace(struct bpf_mem_cache *c) call_rcu_tasks_trace(&c->rcu_ttrace, __free_rcu); } +static void do_call_rcu_ttrace(struct bpf_mem_cache *c) +{ + struct llist_node *llnode; + + if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) { + if (unlikely(READ_ONCE(c->draining))) { + scoped_guard(raw_spinlock_irqsave, &c->lock) + llnode = llist_del_all(&c->free_by_rcu_ttrace); + free_all(c, llnode, !!c->percpu_size); + } + return; + } + + __do_call_rcu_ttrace(c); +} + static void free_bulk(struct bpf_mem_cache *c) { struct bpf_mem_cache *tgt = c->tgt; -- cgit v1.2.3 From fbd97dd1d3ce32d32fb7be86acc3fdb7c04fa043 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 30 Sep 2026 09:59:19 +0000 Subject: bpf: Fix objects stuck in free_by_rcu_ttrace do_call_rcu_ttrace() returns early when call_rcu_ttrace_in_progress is set and leaves the objects in free_by_rcu_ttrace. __free_rcu() frees waiting_for_gp_ttrace only and clears the flag. Hence the objects that free_bulk() or __free_by_rcu() added while RCU tasks trace GP was in flight stay in free_by_rcu_ttrace until free_bulk() or alloc_bulk() is called for the same bpf_mem_cache again, which may never happen. The number of such objects is not bounded. Turn call_rcu_ttrace_in_progress into three states: 0 - idle 1 - __free_rcu() is queued 2 - __free_rcu() is queued and free_by_rcu_ttrace got more objects since do_call_rcu_ttrace() sets 2. __free_rcu() does cmpxchg(1 -> 0) and starts the next GP when it fails. It cannot clear the flag first and check free_by_rcu_ttrace later, since bpf_mem_alloc_destroy() frees bpf_mem_cache without waiting for RCU callbacks when the flag is zero. Now __free_rcu() queues itself, so the one that didn't see 'draining' may do call_rcu_tasks_trace() after rcu_barrier_tasks_trace() in free_mem_alloc(). Queue it under rcu_read_lock() and do synchronize_rcu() before the barriers. Calling rcu_barrier_tasks_trace() twice works too, but creating and destroying hash maps in a loop on many cpus slows down to one free_mem_alloc() per GP and kworkers pile up. Fixes: 8d5a8011b35d ("bpf: Batch call_rcu callbacks instead of SLAB_TYPESAFE_BY_RCU.") Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20260930095920.601738-3-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/memalloc.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c index 08e4dde66cd5..15684d0fc883 100644 --- a/kernel/bpf/memalloc.c +++ b/kernel/bpf/memalloc.c @@ -118,6 +118,11 @@ struct bpf_mem_cache { struct llist_head free_by_rcu_ttrace; struct llist_head waiting_for_gp_ttrace; struct rcu_head rcu_ttrace; + /* + * 0 - idle + * 1 - __free_rcu() is queued + * 2 - __free_rcu() is queued and free_by_rcu_ttrace got more objects since + */ atomic_t call_rcu_ttrace_in_progress; raw_spinlock_t lock; }; @@ -276,6 +281,8 @@ static int free_all(struct bpf_mem_cache *c, struct llist_node *llnode, bool per return cnt; } +static void __do_call_rcu_ttrace(struct bpf_mem_cache *c); + static void __free_rcu(struct rcu_head *head) { struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu_ttrace); @@ -285,7 +292,19 @@ static void __free_rcu(struct rcu_head *head) llnode = llist_del_all(&c->waiting_for_gp_ttrace); free_all(c, llnode, !!c->percpu_size); - atomic_set(&c->call_rcu_ttrace_in_progress, 0); + + /* + * do_call_rcu_ttrace() that ran while GP was in flight left its objects + * in free_by_rcu_ttrace. This cache may never free or alloc in bulk + * again, so start the next GP from here. + * 'c' can be freed as soon as call_rcu_ttrace_in_progress is zero. + */ + if (atomic_cmpxchg(&c->call_rcu_ttrace_in_progress, 1, 0) == 1) + return; + + /* Pairs with synchronize_rcu() in free_mem_alloc() */ + guard(rcu)(); + __do_call_rcu_ttrace(c); } static void enque_to_free(struct bpf_mem_cache *c, void *obj) @@ -302,6 +321,12 @@ static void __do_call_rcu_ttrace(struct bpf_mem_cache *c) { struct llist_node *llnode, *t; + /* + * Must be done before llist_del_all(). Objects that it misses were + * added by do_call_rcu_ttrace() that will set 2 after this store. + */ + atomic_set(&c->call_rcu_ttrace_in_progress, 1); + WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace)); llist_for_each_safe(llnode, t, llist_del_all(&c->free_by_rcu_ttrace)) llist_add(llnode, &c->waiting_for_gp_ttrace); @@ -323,7 +348,7 @@ static void do_call_rcu_ttrace(struct bpf_mem_cache *c) { struct llist_node *llnode; - if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) { + if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 2)) { if (unlikely(READ_ONCE(c->draining))) { scoped_guard(raw_spinlock_irqsave, &c->lock) llnode = llist_del_all(&c->free_by_rcu_ttrace); @@ -707,7 +732,12 @@ static void free_mem_alloc(struct bpf_mem_alloc *ma) * to wait for the pending __free_by_rcu(), and __free_rcu(). RCU Tasks * Trace grace period implies RCU grace period, so all __free_rcu don't * need extra call_rcu() (and thus extra rcu_barrier() here). + * + * __free_rcu() queues itself again unless it sees 'draining'. After + * synchronize_rcu() it either did that already or will not do it, so + * rcu_barrier_tasks_trace() cannot miss it. */ + synchronize_rcu(); rcu_barrier(); /* wait for __free_by_rcu */ rcu_barrier_tasks_trace(); /* wait for __free_rcu */ free_mem_alloc_no_barrier(ma); -- cgit v1.2.3 From 7a39c732a22ec4a369af9e19fbe3d666ec83eec4 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Wed, 30 Sep 2026 13:36:18 +0800 Subject: kprobes: Skip disarmed probes when checking optkprobe overlap On x86, an optkprobe at A replaces five bytes with a jump. If a disabled probe B is at A+2, get_optimized_kprobe() stops at B when arming a new probe C at A+4. It leaves A optimized: A A+1 A+2 A+3 A+4 A's jump | e9 | d0 | d1 | d2 | d3 | after C | e9 | d0 | d1 | d2 | cc | The INT3 for C overwrites the last byte of A's jump displacement, so execution can jump to the wrong address. B can have prepared optinsns while disarmed, but has no jump to unoptimize. Continue past disarmed and unprepared probes to find the active optimized probe before arming a probe in its jump. Link: https://lore.kernel.org/all/20260930053618.104498-1-leon.hwang@linux.dev/ Fixes: afd66255b9a4 ("kprobes: Introduce kprobes jump optimization") Cc: stable@vger.kernel.org Signed-off-by: Leon Hwang Signed-off-by: Masami Hiramatsu (Google) --- kernel/kprobes.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 4edd8ca5c657..e787e4948c8b 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -496,14 +496,16 @@ static bool kprobe_queued(struct kprobe *p) static struct kprobe *get_optimized_kprobe(kprobe_opcode_t *addr) { int i; - struct kprobe *p = NULL; + struct kprobe *p; struct optimized_kprobe *op; /* Don't check i == 0, since that is a breakpoint case. */ - for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++) + for (i = 1; i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++) { p = get_kprobe(addr - i); + /* A disabled probe can have prepared, but inactive, optinsns. */ + if (!p || !kprobe_optready(p) || kprobe_disarmed(p)) + continue; - if (p && kprobe_optready(p)) { op = container_of(p, struct optimized_kprobe, kp); if (arch_within_optimized_kprobe(op, addr)) return p; -- cgit v1.2.3 From 26f6b6357b1b06e6aaa9b8d796989cca785d8e1d Mon Sep 17 00:00:00 2001 From: Kuan-Wei Chiu Date: Thu, 1 Oct 2026 10:25:15 +0200 Subject: irq: Make refcount_interrupt kunit test selectable Currently, refcount_interrupt_test is built unconditionally when CONFIG_KUNIT is enabled, causing it to run unexpectedly during boot. Fix this by introducing CONFIG_REFCOUNT_INTERRUPT_KUNIT_TEST so the test can be configured independently, following standard kunit practices. Fixes: 07a88e2bcd5b ("irq: Add KUnit test for refcounted interrupt enable/disable") Signed-off-by: Kuan-Wei Chiu Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Lyude Paul Reviewed-by: Radu Rendec Reviewed-by: Boqun Feng Link: https://patch.msgid.link/20261001082519.16195-2-boqun@kernel.org --- kernel/irq/Kconfig | 12 ++++++++++++ kernel/irq/Makefile | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/irq/Kconfig b/kernel/irq/Kconfig index 05cba4e16dad..6923f37eaab4 100644 --- a/kernel/irq/Kconfig +++ b/kernel/irq/Kconfig @@ -150,6 +150,18 @@ config IRQ_KUNIT_TEST If unsure, say N. +config REFCOUNT_INTERRUPT_KUNIT_TEST + tristate "Test refcounted interrupt enable/disable" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + This builds the kunit tests for the refcounted interrupt + infrastructure. It verifies the correctness of single, nested, + and multiple interrupt enable/disable state changes and ensures + that the underlying reference counting mechanisms work as expected. + + If unsure, say N. + endmenu config GENERIC_IRQ_MULTI_HANDLER diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile index 44c4d6fc502a..0e5df962a149 100644 --- a/kernel/irq/Makefile +++ b/kernel/irq/Makefile @@ -16,4 +16,4 @@ obj-$(CONFIG_SMP) += affinity.o obj-$(CONFIG_GENERIC_IRQ_DEBUGFS) += debugfs.o obj-$(CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR) += matrix.o obj-$(CONFIG_IRQ_KUNIT_TEST) += irq_test.o -obj-$(CONFIG_KUNIT) += refcount_interrupt_test.o +obj-$(CONFIG_REFCOUNT_INTERRUPT_KUNIT_TEST) += refcount_interrupt_test.o -- cgit v1.2.3 From f35e3b5784221654f9cdbd6222275a7fa203f6c3 Mon Sep 17 00:00:00 2001 From: Chris Mason Date: Thu, 1 Oct 2026 13:50:22 +0000 Subject: futex: Fix private hash use-after-free on resize poll_state_synchronize_rcu(mm->futex.phash.batches) is used by futex_ref_drop() to check that a grace period has passed since the current hash was published. This relies on batches referencing a grace period which started after the hash pointer was assigned. __futex_pivot_hash() sets mmph->batches before it replaces mmph->hash: scoped_guard(rcu) { mmph->batches = get_state_synchronize_rcu(); rcu_assign_pointer(mmph->hash, new); } The scoped_guard(rcu) doesn't stop new grace periods from starting, and if one starts between those two assignments, futex_ref_drop() can move forward while a reader still holds a pointer to the old hash. Fix things by setting mmph->batches after assigning mmph->hash. The scoped_guard(rcu) isn't needed, so let's drop that as well. Fixes: 56180dd20c19 ("futex: Use RCU-based per-CPU reference counting instead of rcuref_t") Assisted-by: kres Signed-off-by: Chris Mason Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Paul E. McKenney Link: https://patch.msgid.link/20261001135022.2220288-1-mason@kernel.org --- kernel/futex/core.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/futex/core.c b/kernel/futex/core.c index a061f54b606d..095f9fe440e3 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -213,10 +213,12 @@ static bool __futex_pivot_hash(struct mm_struct *mm, struct futex_private_hash * futex_rehash_private(fph, new); } new->state = FR_PERCPU; - scoped_guard(rcu) { - mmph->batches = get_state_synchronize_rcu(); - rcu_assign_pointer(mmph->hash, new); - } + rcu_assign_pointer(mmph->hash, new); + /* + * mmph->batches must reference a grace period which started after + * mmph->hash was assigned. See futex_ref_drop(). + */ + mmph->batches = get_state_synchronize_rcu(); kvfree_rcu(fph, rcu); return true; } -- cgit v1.2.3 From de020dc8049bfb2b22e3b6d99c031feb2e22d112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Mete=20Kaya?= Date: Tue, 29 Sep 2026 11:10:34 +0300 Subject: bpf: Fix missing migration protection in __rhtab_map_lookup_and_delete_batch() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bpf_mem_cache_free_rcu() uses this_cpu_ptr() which requires migration to be disabled. All callers of rhtab_delete_elem() disable migration except __rhtab_map_lookup_and_delete_batch(), which calls it under rcu_read_lock() only. On CONFIG_PREEMPT_RCU, rcu_read_lock() does not disable preemption or migration, so the task can migrate between CPUs during the delete loop, causing this_cpu_ptr() to trigger: BUG: using smp_processor_id() in preemptible [00000000] code Fix by wrapping the delete loop in migrate_disable()/migrate_enable() in __rhtab_map_lookup_and_delete_batch(), matching the migration protection that the other callers already provide. Fixes: 818e00848227 ("bpf: Implement iteration ops for resizable hashtab") Reported-by: syzbot+fd7e415d891073b83e1f@syzkaller.appspotmail.com Signed-off-by: Ömer Mete Kaya Signed-off-by: Alexei Starovoitov Acked-by: Mykyta Yatsenko Link: https://patch.msgid.link/20260929081609.557899-1-omermetekaya0@gmail.com Closes: https://syzkaller.appspot.com/bug?extid=fd7e415d891073b83e1f --- kernel/bpf/hashtab.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 548eb1cc7c45..13a2356c84cf 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -3374,8 +3374,10 @@ static int __rhtab_map_lookup_and_delete_batch(struct bpf_map *map, } if (do_delete) { + migrate_disable(); for (i = 0; i < total; i++) rhtab_delete_elem(rhtab, del_elems[i], NULL, 0); + migrate_enable(); } rcu_read_unlock(); -- cgit v1.2.3