From 69fdbe63e16919a885a8f9441e248ce0ddf15b25 Mon Sep 17 00:00:00 2001 From: Woojin Ji Date: Thu, 25 Jun 2026 19:25:37 +0900 Subject: bpf: Preserve scalar zero spills for stack reads Stack reads can read back bytes that belong to a previously spilled scalar constant zero. Today mark_reg_stack_read() only treats STACK_ZERO bytes as known zero bytes, so the destination register can become unknown even though every byte in the read range is known to be zero. This can lead to rejecting otherwise valid programs once the loaded byte is used as a pointer offset. The original reproducer uses a variable-offset stack byte read emitted by clang 22.1.6 at -O2/-O3 from a small helper-based BPF C program. Fixed offset reads have a related mixed case as well: pure scalar-zero spill reads are already handled, but a fixed read spanning both STACK_ZERO and scalar const-zero STACK_SPILL bytes still falls back to unknown. Teach mark_reg_stack_read() to also consider STACK_SPILL bytes backed by a spilled scalar constant zero as zero bytes, and use that path for both variable-offset stack reads and fixed-offset mixed reads. Keep the existing pure register-fill behavior unchanged. When a zero result depends on such a spill, mark the contributing stack slots precise before accepting the const-zero result so pruning cannot reuse a zero-spill state for a later non-zero spill state. No deployed-program regression is currently known, so target bpf-next. Assisted-by: opencode:gpt-5.5 Signed-off-by: Woojin Ji Acked-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260625-bpf-stack-var-off-zero-v1-v3-1-a068210a761b@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 53 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 21a365d436a5..25aea4271cd0 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3702,14 +3702,21 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, * SCALAR. This function does not deal with register filling; the caller must * ensure that all spilled registers in the stack range have been marked as * read. + * + * STACK_SPILL bytes backed by spilled scalar const zeroes are also considered + * zero bytes. In that case, mark the contributing stack slots precise so + * pruning cannot reuse a zero-spill state for a later non-zero spill state. + * + * Returns an error if precision backtracking fails. */ -static void mark_reg_stack_read(struct bpf_verifier_env *env, - /* func where src register points to */ - struct bpf_func_state *ptr_state, - int min_off, int max_off, int dst_regno) +static int mark_reg_stack_read(struct bpf_verifier_env *env, + /* func where src register points to */ + struct bpf_func_state *ptr_state, + int min_off, int max_off, int dst_regno) { struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; + u64 zero_spill_mask = 0; int i, slot, spi; u8 *stype; int zeros = 0; @@ -3719,19 +3726,33 @@ static void mark_reg_stack_read(struct bpf_verifier_env *env, spi = slot / BPF_REG_SIZE; mark_stack_slot_scratched(env, spi); stype = ptr_state->stack[spi].slot_type; - if (stype[slot % BPF_REG_SIZE] != STACK_ZERO) - break; - zeros++; + if (stype[slot % BPF_REG_SIZE] == STACK_ZERO) { + zeros++; + continue; + } + if (stype[slot % BPF_REG_SIZE] == STACK_SPILL && + bpf_register_is_null(&ptr_state->stack[spi].spilled_ptr)) { + zero_spill_mask |= 1ull << spi; + zeros++; + continue; + } + break; } if (zeros == max_off - min_off) { /* Any access_size read into register is zero extended, * so the whole register == const_zero. */ __mark_reg_const_zero(env, &state->regs[dst_regno]); + if (zero_spill_mask) { + bpf_bt_set_frame_slot_mask(&env->bt, ptr_state->frameno, zero_spill_mask); + return mark_chain_precision_batch(env, env->cur_state); + } } else { /* have read misc data from the stack */ mark_reg_unknown(env, state->regs, dst_regno); } + + return 0; } /* Read the stack at 'off' and put the results into the register indicated by @@ -3753,6 +3774,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; struct bpf_reg_state *reg; u8 *stype, type; + int err; int insn_flags = INSN_F_STACK_ACCESS; int hist_spi = spi, hist_frame = reg_state->frameno; @@ -3835,7 +3857,10 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, __mark_reg_const_zero(env, &state->regs[dst_regno]); insn_flags = 0; /* not restoring original register state */ } else { - mark_reg_unknown(env, state->regs, dst_regno); + err = mark_reg_stack_read(env, reg_state, off, off + size, + dst_regno); + if (err) + return err; insn_flags = 0; /* not restoring original register state */ } } @@ -3880,8 +3905,11 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } return -EACCES; } - if (dst_regno >= 0) - mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); + if (dst_regno >= 0) { + err = mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); + if (err) + return err; + } insn_flags = 0; /* we are not restoring spilled register */ } if (insn_flags) @@ -3935,7 +3963,10 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg min_off = reg_smin(reg) + off; max_off = reg_smax(reg) + off; - mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); + err = mark_reg_stack_read(env, ptr_state, min_off, max_off + size, + dst_regno); + if (err) + return err; check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off); return 0; } -- cgit v1.2.3 From 859055e07697c46f6964109981aa1cd23d6bde47 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 29 Jun 2026 23:22:06 +0200 Subject: bpf: Add tracing_multi link info support Adding BPF_OBJ_GET_INFO_BY_FD support for tracing_multi links. We expose following tracing_multi link data: - attach_type of the program - number of ids - array of BTF ids - array of its related kernel addresses - array of cookies The change follows the kprobe_multi and uprobe_multi link-info convention of optional output arrays with an in/out count, On top of standard tracing link data we also expose addresses, because they are useful info for user (especially when the attachment was done via pattern). This data is hidden when kallsyms does not allow exposing kernel pointer values. Assisted-by: Codex:GPT-5 Signed-off-by: Jiri Olsa Signed-off-by: Andrii Nakryiko Acked-by: Leon Hwang Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260629212208.895962-2-jolsa@kernel.org --- kernel/trace/bpf_trace.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) (limited to 'kernel') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 75495a5c3507..76ab51deaa6b 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3700,6 +3700,60 @@ static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) kvfree(tr_link); } +static int bpf_tracing_multi_link_fill_link_info(const struct bpf_link *link, + struct bpf_link_info *info) +{ + u64 __user *ucookies = u64_to_user_ptr(info->tracing_multi.cookies); + u64 __user *uaddrs = u64_to_user_ptr(info->tracing_multi.addrs); + u32 __user *uids = u64_to_user_ptr(info->tracing_multi.ids); + struct bpf_tracing_multi_link *tr_link; + u32 ucount = info->tracing_multi.count; + bool has_cookies, show_addrs; + int err = 0; + + if ((uids || ucookies || uaddrs) && !ucount) + return -EINVAL; + + tr_link = container_of(link, struct bpf_tracing_multi_link, link); + + info->tracing_multi.attach_type = tr_link->link.attach_type; + info->tracing_multi.count = tr_link->nodes_cnt; + info->tracing_multi.btf_obj_id = btf_obj_id(tr_link->link.prog->aux->attach_btf); + + if (!uids && !ucookies && !uaddrs) + return 0; + + if (ucount < tr_link->nodes_cnt) + err = -ENOSPC; + else + ucount = tr_link->nodes_cnt; + + has_cookies = !!tr_link->cookies; + show_addrs = kallsyms_show_value(current_cred()); + + for (int i = 0; i < ucount; i++) { + struct bpf_tracing_multi_node *mnode = &tr_link->nodes[i]; + u64 addr, cookie; + u32 id; + + bpf_trampoline_unpack_key(mnode->trampoline->key, NULL, &id); + + addr = show_addrs ? mnode->trampoline->ip : 0; + cookie = has_cookies ? tr_link->cookies[i] : 0; + + if (uids && put_user(id, uids + i)) + return -EFAULT; + if (uaddrs && put_user(addr, uaddrs + i)) + return -EFAULT; + if (ucookies && put_user(cookie, ucookies + i)) + return -EFAULT; + + cond_resched(); + } + + return err; +} + #ifdef CONFIG_PROC_FS static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link, struct seq_file *seq) @@ -3730,6 +3784,7 @@ static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link, static const struct bpf_link_ops bpf_tracing_multi_link_lops = { .release = bpf_tracing_multi_link_release, .dealloc_deferred = bpf_tracing_multi_link_dealloc, + .fill_link_info = bpf_tracing_multi_link_fill_link_info, #ifdef CONFIG_PROC_FS .show_fdinfo = bpf_tracing_multi_show_fdinfo, #endif -- cgit v1.2.3 From 475b59db3bd5c7717b5441481ac06f226815cb0a Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Wed, 1 Jul 2026 11:51:07 +0800 Subject: bpf: Use BPF_CALL_IMM macro consistently in bpf_do_misc_fixups In bpf_do_misc_fixups(), the conversion from a function address to a BPF immediate value is handled using the BPF_CALL_IMM macro inside the 'patch_map_ops_generic' label block. However, immediately following it in the 'patch_call_imm' label block, the immediate value is calculated manually by subtracting __bpf_call_base from fn->func. Inspired by KaFai Wan's review comments on fixing helper call offsets, use the BPF_CALL_IMM macro in 'patch_call_imm' as well to clean this up. This removes the redundant manual pointer arithmetic and ensures coding style consistency across adjacent label blocks within the same function. Signed-off-by: Tiezhu Yang Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260701035107.8069-1-yangtiezhu@loongson.cn --- kernel/bpf/fixups.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 3cf2cc6e3ab6..12a8a4eb757f 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -2338,7 +2338,7 @@ patch_call_imm: func_id_name(insn->imm), insn->imm); return -EFAULT; } - insn->imm = fn->func - __bpf_call_base; + insn->imm = BPF_CALL_IMM(fn->func); next_insn: if (subprogs[cur_subprog + 1].start == i + delta + 1) { subprogs[cur_subprog].stack_depth += stack_depth_extra; -- cgit v1.2.3 From 2ce3f548cfc6a1fe4c53479cf8a21931cdfd51d8 Mon Sep 17 00:00:00 2001 From: Sechang Lim Date: Wed, 1 Jul 2026 08:07:51 +0000 Subject: bpf,lsm: Drop bpf_prog_free from sleepable_lsm_hooks __bpf_prog_put_rcu() is the call_rcu() callback for non-sleepable programs. security_bpf_prog_free() called from there fires bpf_prog_free in softirq; if a sleepable LSM prog is attached to that hook, might_fault() BUGs: BUG: sleeping function called from invalid context in_atomic(): 1, irqs_disabled(): 0, non_block: 0, pid: 5038 preempt_count: 101, expected: 0 Call Trace: __bpf_prog_enter_sleepable+0x1cd/0x320 kernel/bpf/trampoline.c:1255 bpf_trampoline_6442549705+0x53/0xd7 security_bpf_prog_free+0xde/0x130 security/security.c:5465 __bpf_prog_put_rcu+0xab/0xd0 kernel/bpf/syscall.c:2365 rcu_do_batch kernel/rcu/tree.c:2617 [inline] handle_softirqs+0x236/0x800 kernel/softirq.c:622 The call_rcu/call_rcu_tasks_trace split reflects the freed program's sleepability, not that of any attached observer. security_bpf_prog_free() also frees prog->aux->security, which has to stay after the grace period, so drop bpf_prog_free from sleepable_lsm_hooks rather than move the call. Non-sleepable observers still run there. Fixes: 1b67772e4e3f ("bpf,lsm: Refactor bpf_prog_alloc/bpf_prog_free LSM hooks") Signed-off-by: Sechang Lim Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260701080757.1394144-1-rhkrqnwk98@gmail.com --- kernel/bpf/bpf_lsm.c | 1 - 1 file changed, 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 1433809bb166..3983b4ce73c8 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -295,7 +295,6 @@ BTF_ID(func, bpf_lsm_bpf_map_create) BTF_ID(func, bpf_lsm_bpf_map_free) BTF_ID(func, bpf_lsm_bpf_prog) BTF_ID(func, bpf_lsm_bpf_prog_load) -BTF_ID(func, bpf_lsm_bpf_prog_free) BTF_ID(func, bpf_lsm_bpf_token_create) BTF_ID(func, bpf_lsm_bpf_token_free) BTF_ID(func, bpf_lsm_bpf_token_cmd) -- cgit v1.2.3 From 9c9ee0324c774490ae953162aaaf4561d222bd93 Mon Sep 17 00:00:00 2001 From: Yiyang Chen Date: Tue, 30 Jun 2026 08:41:26 +0000 Subject: bpf: Reject MEM_ALLOC BTF accesses past object bounds BTF struct walks relax the struct-size check for accesses through a trailing flexible array. That is valid for ordinary BTF type walking, but PTR_TO_BTF_ID | MEM_ALLOC values point to objects allocated with the static BTF type size. When walking a MEM_ALLOC object, reject the access before applying the flexible-array relaxation if the access range extends past the struct size. Apply the same policy to struct ID matching so kfunc and kptr type checks do not walk past the allocated object bounds either. Fixes: 958cf2e273f0 ("bpf: Introduce bpf_obj_new") Fixes: 36d8bdf75a93 ("bpf: Add alloc/xchg/direct_access support for local percpu kptr") Signed-off-by: Yiyang Chen Reviewed-by: Amery Hung Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/4b8c8a81102ba4b595011434c881194f264ddc59.1782807039.git.chenyy23@mails.tsinghua.edu.cn Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/btf.c | 17 +++++++++++------ kernel/bpf/verifier.c | 11 +++++++---- 2 files changed, 18 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 64572f85edc8..dff5c0d91641 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7108,7 +7108,7 @@ enum bpf_struct_walk_result { static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, const struct btf_type *t, int off, int size, u32 *next_btf_id, enum bpf_type_flag *flag, - const char **field_name) + const char **field_name, bool walk_flex_arrays) { u32 i, moff, mtrue_end, msize = 0, total_nelems = 0; const struct btf_type *mtype, *elem_type = NULL; @@ -7135,11 +7135,14 @@ again: *flag |= PTR_UNTRUSTED; if (off + size > t->size) { + struct btf_array *array_elem; + + if (!walk_flex_arrays) + goto error; + /* If the last element is a variable size array, we may * need to relax the rule. */ - struct btf_array *array_elem; - if (vlen == 0) goto error; @@ -7404,7 +7407,8 @@ int btf_struct_access(struct bpf_verifier_log *log, t = btf_type_by_id(btf, id); do { - err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, field_name); + err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, + field_name, !type_is_alloc(reg->type)); switch (err) { case WALK_PTR: @@ -7463,7 +7467,7 @@ bool btf_types_are_same(const struct btf *btf1, u32 id1, bool btf_struct_ids_match(struct bpf_verifier_log *log, const struct btf *btf, u32 id, int off, const struct btf *need_btf, u32 need_type_id, - bool strict) + bool strict, bool walk_flex_arrays) { const struct btf_type *type; enum bpf_type_flag flag = 0; @@ -7482,7 +7486,8 @@ again: type = btf_type_by_id(btf, id); if (!type) return false; - err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL); + err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL, + walk_flex_arrays); if (err != WALK_STRUCT) return false; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d46f7db20d8f..a0f292635c59 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4379,7 +4379,8 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, */ if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value, kptr_field->kptr.btf, kptr_field->kptr.btf_id, - kptr_field->type != BPF_KPTR_UNREF)) + kptr_field->type != BPF_KPTR_UNREF, + !type_is_alloc(reg->type))) goto bad_type; return 0; bad_type: @@ -7970,7 +7971,7 @@ found: if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value, btf_vmlinux, *arg_btf_id, - strict_type_match)) { + strict_type_match, !type_is_alloc(reg->type))) { verbose(env, "%s is of type %s but %s is expected\n", reg_arg_name(env, argno), btf_type_name(reg->btf, reg->btf_id), @@ -11436,7 +11437,8 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, ®_ref_id); reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->var_off.value, - meta->btf, ref_id, strict_type_match); + meta->btf, ref_id, strict_type_match, + !type_is_alloc(reg->type)); /* If kfunc is accepting a projection type (ie. __sk_buff), it cannot * actually use it -- it must cast to the underlying type. So we allow * caller to pass in the underlying type. @@ -11883,7 +11885,8 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id); t = btf_type_by_id(reg->btf, reg->btf_id); if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf, - field->graph_root.value_btf_id, true)) { + field->graph_root.value_btf_id, true, + !type_is_alloc(reg->type))) { verbose(env, "operation on %s expects arg#1 %s at offset=%d " "in struct %s, but arg is at offset=%d in struct %s\n", btf_field_type_name(head_field_type), -- cgit v1.2.3 From be39165224d03d92a05f62b9ea10eec089365480 Mon Sep 17 00:00:00 2001 From: Sechang Lim Date: Tue, 30 Jun 2026 14:54:05 +0000 Subject: bpf, sockmap: Disallow update and delete from tc, xdp, socket_filter and flow_dissector sock_map_update_common() and __sock_map_delete() hold stab->lock and call sock_map_unref() -> sock_map_del_link(), which takes sk_callback_lock for write. That gives the order stab->lock -> sk_callback_lock. The reverse order comes from the SK_SKB stream parser. sk_psock_strp_data_ready() holds sk_callback_lock for read, and after the verdict tcp_bpf_strp_read_sock() acks the consumed data inline via __tcp_cleanup_rbuf(). The ACK goes out egress, where a sched_cls program deletes from the sockmap and takes stab->lock: WARNING: possible circular locking dependency detected ------------------------------------------------------ syz.9.8824 is trying to acquire lock: (&stab->lock){+.-.}-{3:3}, at: __sock_map_delete net/core/sock_map.c:421 but task is already holding lock: (clock-AF_INET){++.-}-{3:3}, at: sk_psock_strp_data_ready net/core/skmsg.c:1173 -> #1 (clock-AF_INET){++.-}-{3:3}: _raw_write_lock_bh sock_map_del_link net/core/sock_map.c:167 sock_map_unref net/core/sock_map.c:184 sock_map_update_common net/core/sock_map.c:509 sock_map_update_elem_sys net/core/sock_map.c:588 map_update_elem kernel/bpf/syscall.c:1805 -> #0 (&stab->lock){+.-.}-{3:3}: _raw_spin_lock_bh __sock_map_delete net/core/sock_map.c:421 sock_map_delete_elem net/core/sock_map.c:452 bpf_prog_06044d24140080b6 tcx_run net/core/dev.c:4451 sch_handle_egress net/core/dev.c:4541 __dev_queue_xmit net/core/dev.c:4808 ... tcp_bpf_strp_read_sock net/ipv4/tcp_bpf.c:701 strp_data_ready net/strparser/strparser.c:402 sk_psock_strp_data_ready net/core/skmsg.c:1174 tcp_data_queue net/ipv4/tcp_input.c:5661 Possible unsafe locking scenario: CPU0 CPU1 ---- ---- rlock(clock-AF_INET); lock(&stab->lock); lock(clock-AF_INET); lock(&stab->lock); *** DEADLOCK *** A tc, xdp, socket_filter or flow_dissector program has no reason to update or delete a sockmap, and redirect does not go through here. Drop them from may_update_sockmap() so the verifier rejects it. It also closes the matching sockhash inversion. Suggested-by: John Fastabend Signed-off-by: Sechang Lim Signed-off-by: Daniel Borkmann Reviewed-by: John Fastabend Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260630145410.3648099-2-rhkrqnwk98@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a0f292635c59..3193b473762b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8496,12 +8496,7 @@ static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) if (func_id == BPF_FUNC_map_delete_elem) return true; break; - case BPF_PROG_TYPE_SOCKET_FILTER: - case BPF_PROG_TYPE_SCHED_CLS: - case BPF_PROG_TYPE_SCHED_ACT: - case BPF_PROG_TYPE_XDP: case BPF_PROG_TYPE_SK_REUSEPORT: - case BPF_PROG_TYPE_FLOW_DISSECTOR: case BPF_PROG_TYPE_SK_LOOKUP: return true; default: -- cgit v1.2.3 From 1f737e46ca6a845e6e96982ad57a31026f3521fa Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Tue, 7 Jul 2026 17:44:28 -0700 Subject: bpf: Remove artificial limitations on pointer types eligible for spilling The verifier loses precision when simulating stack spills for the following register types: - PTR_TO_TP_BUFFER - PTR_TO_INSN - CONST_PTR_TO_DYNPTR These types are not allow-listed in the is_spillable_regtype(), because of that check_stack_write_fixed_off() takes the branch that marks the slots STACK_MISC. There are no technical reasons for this limitation. This commit replaces an explicit list of pointer types in is_spillable_regtype() with explicit list of non-pointer types. The function is renamed to is_pointer_regtype() for clarity. Reported-by: Andrii Nakryiko Suggested-by: Kumar Kartikeya Dwivedi Signed-off-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260707-missing-spillable-types-v1-1-44a92121dc41@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 39 ++++++++------------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 3193b473762b..51f7965d42e3 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3304,34 +3304,6 @@ static int mark_chain_precision_batch(struct bpf_verifier_env *env, return bpf_mark_chain_precision(env, starting_state, -1, NULL); } -static bool is_spillable_regtype(enum bpf_reg_type type) -{ - switch (base_type(type)) { - case PTR_TO_MAP_VALUE: - case PTR_TO_STACK: - case PTR_TO_CTX: - case PTR_TO_PACKET: - case PTR_TO_PACKET_META: - case PTR_TO_PACKET_END: - case PTR_TO_FLOW_KEYS: - case CONST_PTR_TO_MAP: - case PTR_TO_SOCKET: - case PTR_TO_SOCK_COMMON: - case PTR_TO_TCP_SOCK: - case PTR_TO_XDP_SOCK: - case PTR_TO_BTF_ID: - case PTR_TO_BUF: - case PTR_TO_MEM: - case PTR_TO_FUNC: - case PTR_TO_MAP_KEY: - case PTR_TO_ARENA: - return true; - default: - return false; - } -} - - /* check if register is a constant scalar value */ static bool is_reg_const(struct bpf_reg_state *reg, bool subreg32) { @@ -3345,13 +3317,18 @@ static u64 reg_const_value(struct bpf_reg_state *reg, bool subreg32) return subreg32 ? tnum_subreg(reg->var_off).value : reg->var_off.value; } +static bool is_pointer_regtype(enum bpf_reg_type type) +{ + return type != SCALAR_VALUE && type != NOT_INIT; +} + static bool __is_pointer_value(bool allow_ptr_leaks, const struct bpf_reg_state *reg) { if (allow_ptr_leaks) return false; - return reg->type != SCALAR_VALUE; + return is_pointer_regtype(reg->type); } static void clear_scalar_id(struct bpf_reg_state *reg) @@ -3476,7 +3453,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, if (value_regno >= 0) reg = &cur->regs[value_regno]; if (!env->bypass_spec_v4) { - bool sanitize = reg && is_spillable_regtype(reg->type); + bool sanitize = reg && is_pointer_regtype(reg->type); for (i = 0; i < size; i++) { u8 type = state->stack[spi].slot_type[(slot - i) % @@ -3517,7 +3494,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, __mark_reg_known(tmp_reg, insn->imm); tmp_reg->type = SCALAR_VALUE; save_register_state(env, state, spi, tmp_reg, size); - } else if (reg && is_spillable_regtype(reg->type)) { + } else if (reg && is_pointer_regtype(reg->type)) { /* register containing pointer is being spilled into stack */ if (size != BPF_REG_SIZE) { verbose_linfo(env, insn_idx, "; "); -- cgit v1.2.3 From ac65c710cc643cbc52b899627577357867249530 Mon Sep 17 00:00:00 2001 From: Nicholas Dudar Date: Wed, 8 Jul 2026 05:07:50 +0200 Subject: bpf: Reject writes through untrusted BTF pointers check_ptr_to_btf_access() lets program-type btf_struct_access callbacks validate writes before the default BTF access path rejects non-read accesses. That bypasses the read-only policy for untrusted BTF pointers created by helpers such as bpf_rdonly_cast(). Reject non-read accesses through PTR_UNTRUSTED BTF pointers at the common entry point, before the callback branch to handle all cases. Fixes: 282de143ead9 ("bpf: Introduce allocated objects support") Signed-off-by: Nicholas Dudar Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Eduard Zingerman Reviewed-by: Amery Hung Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 51f7965d42e3..4f42b4e929ad 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5790,6 +5790,11 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, return -EACCES; } + if (atype != BPF_READ && (type_flag(reg->type) & PTR_UNTRUSTED)) { + verbose(env, "only read is supported\n"); + return -EACCES; + } + if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) { if (!btf_is_kernel(reg->btf)) { verifier_bug(env, "reg->btf must be kernel btf"); @@ -5802,8 +5807,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, reg_arg_name(env, argno), tname, off, size); } else { /* Writes are permitted with default btf_struct_access for - * program allocated objects (which always have id > 0), - * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC. + * program allocated objects (which always have id > 0). */ if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) { verbose(env, "only read is supported\n"); -- cgit v1.2.3 From d5a85392392c77b61a74e975a74da0e9c146f6d3 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 8 Jul 2026 09:53:36 +0200 Subject: bpf: Resolve and cache fd_array objects at load time The fd_array passed to BPF_PROG_LOAD carries the map and module BTF file descriptors a program binds. The verifier reads it more than once during a load: process_fd_array() walks it to bind the maps and BTFs, and check_and_resolve_insns() and the kfunc BTF resolver later read it again to resolve the program's BPF_PSEUDO_MAP_IDX* and module kfunc refs. For signed BPF, we need these upfront in memory, thus resolve each fd to its object once and cache it by fd_array index, then bind that cached object for the rest of the load. env->fd_array becomes a small per-slot {map, btf} cache rather than a bpfptr_t; every later reference is then an in-bounds lookup of an already-resolved object, and an index outside the cache is rejected instead of read from user memory: - continuous (fd_array_cnt given): the caller declares the length and every entry is resolved and bound up front (used also by the BPF signed loader) - sparse (no fd_array_cnt): left as the legacy path with no fd_array cache; each reference reads its fd from the caller's fd_array and resolves it on the spot. Deduplication in used_maps and the kfunc BTF table keeps this correct, and only unsigned programs use this shape. Split these into separate helpers to make it easier to follow. Signed-off-by: Daniel Borkmann Acked-by: Anton Protopopov Link: https://lore.kernel.org/bpf/20260708075343.358712-2-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 223 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 172 insertions(+), 51 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 4f42b4e929ad..e8e21d1a919a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2490,6 +2490,79 @@ int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, return 0; } +#define BPF_FD_SLOT_BTF 1UL + +static void fd_slot_set_map(struct bpf_fd_array *slot, struct bpf_map *map) +{ + slot->val = (unsigned long)map; +} + +static void fd_slot_set_btf(struct bpf_fd_array *slot, struct btf *btf) +{ + slot->val = (unsigned long)btf | BPF_FD_SLOT_BTF; +} + +static struct bpf_map *fd_slot_map(struct bpf_fd_array slot) +{ + if (slot.val & BPF_FD_SLOT_BTF) + return NULL; + return (struct bpf_map *)slot.val; +} + +static struct btf *fd_slot_btf(struct bpf_fd_array slot) +{ + if (!(slot.val & BPF_FD_SLOT_BTF)) + return NULL; + return (struct btf *)(slot.val & ~BPF_FD_SLOT_BTF); +} + +static struct btf * +fd_array_get_btf_continuous(struct bpf_verifier_env *env, u32 idx) +{ + struct btf *btf; + + if (idx >= env->fd_array_cnt) { + verbose(env, "kfunc fd_idx %u out of bounds, fd_array_cnt %u\n", + idx, env->fd_array_cnt); + return ERR_PTR(-EINVAL); + } + btf = fd_slot_btf(env->fd_array[idx]); + if (!btf) { + verbose(env, "kfunc fd_idx %u is not a module BTF\n", idx); + return ERR_PTR(-EINVAL); + } + btf_get(btf); + return btf; +} + +static struct btf * +fd_array_get_btf_sparse(struct bpf_verifier_env *env, u32 idx) +{ + struct btf *btf; + int btf_fd; + + if (copy_from_bpfptr_offset(&btf_fd, env->fd_array_raw, + (size_t)idx * sizeof(btf_fd), sizeof(btf_fd))) + return ERR_PTR(-EFAULT); + btf = btf_get_by_fd(btf_fd); + if (IS_ERR(btf)) { + verbose(env, "invalid module BTF fd specified\n"); + return btf; + } + return btf; +} + +static struct btf *fd_array_get_btf(struct bpf_verifier_env *env, u32 idx) +{ + if (env->fd_array) + return fd_array_get_btf_continuous(env, idx); + if (!bpfptr_is_null(env->fd_array_raw)) + return fd_array_get_btf_sparse(env, idx); + + verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); + return ERR_PTR(-EPROTO); +} + static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) { @@ -2498,7 +2571,6 @@ static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, struct bpf_kfunc_btf *b; struct module *mod; struct btf *btf; - int btf_fd; tab = env->prog->aux->kfunc_btf_tab; b = bsearch(&kf_btf, tab->descs, tab->nr_descs, @@ -2509,22 +2581,9 @@ static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, return ERR_PTR(-E2BIG); } - if (bpfptr_is_null(env->fd_array)) { - verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); - return ERR_PTR(-EPROTO); - } - - if (copy_from_bpfptr_offset(&btf_fd, env->fd_array, - offset * sizeof(btf_fd), - sizeof(btf_fd))) - return ERR_PTR(-EFAULT); - - btf = btf_get_by_fd(btf_fd); - if (IS_ERR(btf)) { - verbose(env, "invalid module BTF fd specified\n"); + btf = fd_array_get_btf(env, offset); + if (IS_ERR(btf)) return btf; - } - if (!btf_is_module(btf)) { verbose(env, "BTF fd for kfunc is not a module BTF\n"); btf_put(btf); @@ -17902,6 +17961,44 @@ static int add_used_map(struct bpf_verifier_env *env, int fd) return __add_used_map(env, map); } +static int fd_array_get_map_idx_continuous(struct bpf_verifier_env *env, u32 idx) +{ + struct bpf_map *map; + + if (idx >= env->fd_array_cnt) { + verbose(env, "fd_idx %u out of bounds, fd_array_cnt %u\n", + idx, env->fd_array_cnt); + return -EINVAL; + } + map = fd_slot_map(env->fd_array[idx]); + if (!map) { + verbose(env, "fd_idx %u is not a map\n", idx); + return -EINVAL; + } + return __add_used_map(env, map); +} + +static int fd_array_get_map_idx_sparse(struct bpf_verifier_env *env, u32 idx) +{ + int fd; + + if (copy_from_bpfptr_offset(&fd, env->fd_array_raw, + (size_t)idx * sizeof(fd), sizeof(fd))) + return -EFAULT; + return add_used_map(env, fd); +} + +static int fd_array_get_map_idx(struct bpf_verifier_env *env, u32 idx) +{ + if (env->fd_array) + return fd_array_get_map_idx_continuous(env, idx); + if (!bpfptr_is_null(env->fd_array_raw)) + return fd_array_get_map_idx_sparse(env, idx); + + verbose(env, "fd_idx without fd_array is invalid\n"); + return -EPROTO; +} + static int check_alu_fields(struct bpf_verifier_env *env, struct bpf_insn *insn) { u8 class = BPF_CLASS(insn->code); @@ -18119,7 +18216,6 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env) struct bpf_map *map; int map_idx; u64 addr; - u32 fd; if (i == insn_cnt - 1 || insn[1].code != 0 || insn[1].dst_reg != 0 || insn[1].src_reg != 0 || @@ -18171,21 +18267,13 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env) switch (insn[0].src_reg) { case BPF_PSEUDO_MAP_IDX_VALUE: case BPF_PSEUDO_MAP_IDX: - if (bpfptr_is_null(env->fd_array)) { - verbose(env, "fd_idx without fd_array is invalid\n"); - return -EPROTO; - } - if (copy_from_bpfptr_offset(&fd, env->fd_array, - insn[0].imm * sizeof(fd), - sizeof(fd))) - return -EFAULT; + map_idx = fd_array_get_map_idx(env, insn[0].imm); break; default: - fd = insn[0].imm; + map_idx = add_used_map(env, insn[0].imm); break; } - map_idx = add_used_map(env, fd); if (map_idx < 0) return map_idx; map = env->used_maps[map_idx]; @@ -19460,7 +19548,7 @@ struct btf *bpf_get_btf_vmlinux(void) * this case expect that every file descriptor in the array is either a map or * a BTF. Everything else is considered to be trash. */ -static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd) +static int add_fd_from_fd_array(struct bpf_verifier_env *env, u32 idx, int fd) { struct bpf_map *map; struct btf *btf; @@ -19472,51 +19560,83 @@ static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd) err = __add_used_map(env, map); if (err < 0) return err; + fd_slot_set_map(&env->fd_array[idx], map); return 0; } btf = __btf_get_by_fd(f); if (!IS_ERR(btf)) { btf_get(btf); - return __add_used_btf(env, btf); + err = __add_used_btf(env, btf); + if (err < 0) + return err; + fd_slot_set_btf(&env->fd_array[idx], btf); + return 0; } verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd); return PTR_ERR(map); } -static int process_fd_array(struct bpf_verifier_env *env, union bpf_attr *attr, bpfptr_t uattr) +/* + * A continuous fd_array is resolved into an in-memory cache with one slot + * per entry. The bound here is deliberately generous and not derived from + * the per-program object limits: Duplicate entries /are/ permitted, and + * the number of distinct maps and BTFs a program can bind is enforced when + * each entry is resolved by __add_used_map() and __add_used_btf(). + */ +#define MAX_FD_ARRAY_CNT 4096 + +static int process_fd_array_continuous(struct bpf_verifier_env *env, + bpfptr_t fd_array, u32 cnt) { - size_t size = sizeof(int); - int ret; - int fd; + int fd, ret; u32 i; - env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); - - /* - * The only difference between old (no fd_array_cnt is given) and new - * APIs is that in the latter case the fd_array is expected to be - * continuous and is scanned for map fds right away - */ - if (!attr->fd_array_cnt) - return 0; - - /* Check for integer overflow */ - if (attr->fd_array_cnt >= (U32_MAX / size)) { - verbose(env, "fd_array_cnt is too big (%u)\n", attr->fd_array_cnt); - return -EINVAL; + if (cnt > MAX_FD_ARRAY_CNT) { + verbose(env, "fd_array has too many entries (%u, max %u)\n", + cnt, MAX_FD_ARRAY_CNT); + return -E2BIG; } - for (i = 0; i < attr->fd_array_cnt; i++) { - if (copy_from_bpfptr_offset(&fd, env->fd_array, i * size, size)) + env->fd_array = kvcalloc(cnt, sizeof(*env->fd_array), + GFP_KERNEL_ACCOUNT); + if (!env->fd_array) + return -ENOMEM; + env->fd_array_cnt = cnt; + for (i = 0; i < cnt; i++) { + if (copy_from_bpfptr_offset(&fd, fd_array, + (size_t)i * sizeof(fd), sizeof(fd))) return -EFAULT; - - ret = add_fd_from_fd_array(env, fd); + ret = add_fd_from_fd_array(env, i, fd); if (ret) return ret; } + return 0; +} +static int process_fd_array(struct bpf_verifier_env *env, + union bpf_attr *attr, bpfptr_t uattr) +{ + bpfptr_t fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); + + if (bpfptr_is_null(fd_array)) { + if (attr->fd_array_cnt) { + verbose(env, "fd_array_cnt %u without fd_array is invalid\n", + attr->fd_array_cnt); + return -EINVAL; + } + return 0; + } + /* + * New API: the caller passes fd_array_cnt and a continuous array that + * is resolved and bound up front. Legacy API (no fd_array_cnt): keep + * the caller's array and resolve entries on the spot at each reference. + */ + if (attr->fd_array_cnt) + return process_fd_array_continuous(env, fd_array, + attr->fd_array_cnt); + env->fd_array_raw = fd_array; return 0; } @@ -20017,6 +20137,7 @@ err_unlock: mutex_unlock(&bpf_verifier_lock); bpf_clear_insn_aux_data(env, 0, env->prog->len); err_free_env: + kvfree(env->fd_array); bpf_stack_liveness_free(env); kvfree(env->cfg.insn_postorder); kvfree(env->scc_info); -- cgit v1.2.3 From b707068e0ed92b64bb66bae4f6f3a521f7017220 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 8 Jul 2026 09:53:37 +0200 Subject: bpf: Verify signed loader metadata at load time A signed gen_loader program carries the programs, maps and relocations it installs in a metadata array map. The loader instructions are covered by the PKCS#7 signature, but the metadata map is not: Today the loader compares the map contents from within BPF against a hash baked into its (signed) instructions, using the kernel-cached map hash. The kernel itself never actually attests that the metadata the loader installs is the metadata that was signed. This split is the core of the long-standing objection to the BPF signing scheme from the LSM / integrity side: the integrity check of a light skeleton only completes once the loader program runs, that is, after the security_bpf_prog_load() hook, so at admission time an LSM observes a program whose payload has not yet been verified. Auditing the chain link is also not a purely cryptographic operation: whoever signs or reviews an lskel has to disassemble the loader's preamble to convince themselves that the embedded hash check is present and correct [0][1]. Two acceptable fixes were identified in those threads: Complete the integrity check before the admission hook fires, or add a second hook that collects the verification result after the loader ran [2]. Covering both the loader and its maps directly with the PKCS#7 signature is what Blaise Boscaccy's patchsets proposed in several forms. Let's implement the former, without growing the UAPI, and in particular as a single unified scheme where the signature spans the raw bytes rather than derived hashes. A signed loader binds its metadata map(s) through the existing fd_array, and an exclusive map is already bound to a program digest (excl_prog_hash). So when a signature is present, collect the exclusive maps from fd_array and append their frozen contents to the instructions before verification: The signature now covers insns || metadata_0 || metadata_1 || [...] in the fd_array order, and verification completes in bpf_check(), once the fd_array maps are resolved into used_maps, before the LSM admission hook and the rest of verification. A program is either BPF_SIG_UNSIGNED or BPF_SIG_VERIFIED, with nothing in between. While folding the fd_array maps, a non-exclusive map bound to a signed program is rejected, so every map folded into the signature is exclusive. A signed loader that fails to cover its metadata thus does not load, and BPF_SIG_VERIFIED always means the instructions and every exclusive map are authentic. The maps must be frozen so the hashed bytes cannot change before the loader runs; the map <-> program digest binding is enforced by the verifier for every used map. Binding maps through fd_array_cnt makes the verifier resolve and excl-check them (excl_prog_sha vs prog->digest) before it would otherwise compute the digest, so compute prog->digest up front in bpf_check(), over the unmodified instructions the signature covers, for a load that folds metadata. Unsigned programs are not affected by the signature path; for them the LSM admission hook merely moves below fd_array resolution, with minimal bounded work in between. Note, signed loaders generated by older libbpf/ bpftool versions need to be regenerated; some of the recent fixes we've had on the signed loader side require the latter already to close gaps. Finally, some remarks around the security_bpf_prog_load() placement given there was discussion on whether a new hook is needed or the existing security_bpf_prog() hook should be reused [3]: For a new hook it would mean that just for loading a single BPF program it has to pass through four layers of LSM hooks: 1) security_bpf (cmd=PROG_LOAD): for gating various bpf subcmds 2) security_bpf_prog_load: historical admission hook (CAP/token, prog_type, attach point), pre-verification 3) security_bpf_prog_verify_signature: newly asked admission hook, same role as 2), plus the BPF signature verdict 4) security_bpf_prog: gate handing the prog fd back to userspace, verification done & signature verified The use-cases of 2) and 3) conflate, thus BPF community prefers to just keep a total of 3 LSM hooks (as-is today): 3) makes 2) incoherent given they are the /same class/ of hook, that is, access-control admission on the load and split only by _what_ they can see. Worse, with the split, for a signed BPF program security_bpf_prog_load 2) admits a program whose signature has not been checked, so a policy gating at 2) is structurally unable to express "admit only verified" and every such policy is forced onto 3) *anyway*. In other words, one doesn't get two complementary hooks, but rather, one real admission hook aka 3) plus a now-degraded /legacy/ hook 2) that can't answer the question operators actually want to ask. Reusing security_bpf_prog() 4) for admission is no alternative either: it fires only after the entire verifier (and JIT) pipeline ran, so denying a not-yet-verified program at that point burns exactly the work a denial is supposed to avoid, and by then the program has an id assigned and the kallsyms/perf/audit load events fired. Policies are free to also consume the signature verdict at 4), but admission control belongs into security_bpf_prog_load(). Hence the latter remains the only admission hook, merely moved past signature verification; with moving large allocations further down into the BPF verifier, there is now only minimal work between the old and new location: The preparation work in bpf_check() is reordered such that only the minimally necessary setup happens up front: Allocating the env, initializing the verifier log and resolving the fd_array that a signed BPF metadata map needs. The worst case allocation up until security_bpf_prog_load() is ~90K which is the env itself (~54K) plus the continuous fd_array cache (at most 32K). The insn_aux_data array is moved into a later stage in the verification. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/2f71d6c03698eb17d51f7247efde777627ee578a.camel@HansenPartnership.com [0] Link: https://lore.kernel.org/lkml/ecf0521ed302db672672ebfbc670ecfba36a6e00.camel@HansenPartnership.com [1] Link: https://lore.kernel.org/bpf/88703f00d5b7a779728451008626efa45e42db3d.camel@HansenPartnership.com [2] Link: https://lore.kernel.org/bpf/DJOFY21DYUI4.19WKQ3NPZ4H5R@gmail.com [3] Link: https://lore.kernel.org/bpf/20260708075343.358712-3-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/syscall.c | 76 +--------------- kernel/bpf/verifier.c | 234 +++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 214 insertions(+), 96 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 6db306d23b47..e898fad01aaf 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -40,7 +40,6 @@ #include #include #include -#include #include #include @@ -2886,64 +2885,6 @@ static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) } } -static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) -{ - switch (keyring_id) { - case 0: - return BPF_SIG_KEYRING_BUILTIN; - case (s32)(unsigned long)VERIFY_USE_SECONDARY_KEYRING: - return BPF_SIG_KEYRING_SECONDARY; - case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING: - return BPF_SIG_KEYRING_PLATFORM; - default: - return BPF_SIG_KEYRING_USER; - } -} - -static int bpf_prog_verify_signature(struct bpf_prog *prog, union bpf_attr *attr, - bool is_kernel, s32 *keyring_serial) -{ - bpfptr_t usig = make_bpfptr(attr->signature, is_kernel); - struct bpf_dynptr_kern sig_ptr, insns_ptr; - struct bpf_key *key = NULL; - void *sig; - int err = 0; - - /* - * Don't attempt to use kmalloc_large or vmalloc for signatures. - * Practical signature for BPF program should be below this limit. - */ - if (attr->signature_size > KMALLOC_MAX_CACHE_SIZE) - return -EINVAL; - - if (system_keyring_id_check(attr->keyring_id) == 0) - key = bpf_lookup_system_key(attr->keyring_id); - else - key = bpf_lookup_user_key(attr->keyring_id, 0); - - if (!key) - return -EINVAL; - - sig = kvmemdup_bpfptr(usig, attr->signature_size); - if (IS_ERR(sig)) { - bpf_key_put(key); - return PTR_ERR(sig); - } - - bpf_dynptr_init(&sig_ptr, sig, BPF_DYNPTR_TYPE_LOCAL, 0, - attr->signature_size); - bpf_dynptr_init(&insns_ptr, prog->insnsi, BPF_DYNPTR_TYPE_LOCAL, 0, - prog->len * sizeof(struct bpf_insn)); - - err = bpf_verify_pkcs7_signature((struct bpf_dynptr *)&insns_ptr, - (struct bpf_dynptr *)&sig_ptr, key); - if (!err) - *keyring_serial = bpf_key_serial(key); - bpf_key_put(key); - kvfree(sig); - return err; -} - static int bpf_prog_mark_insn_arrays_ready(struct bpf_prog *prog) { int err; @@ -3133,17 +3074,8 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at /* eBPF programs must be GPL compatible to use GPL-ed functions */ prog->gpl_compatible = license_is_gpl_compatible(license) ? 1 : 0; - if (attr->signature) { - err = bpf_prog_verify_signature(prog, attr, uattr.is_kernel, - &prog->aux->sig.keyring_serial); - if (err) - goto free_prog; - prog->aux->sig.keyring_type = bpf_classify_keyring(attr->keyring_id); - prog->aux->sig.verdict = BPF_SIG_VERIFIED; - } else { - prog->aux->sig.keyring_type = BPF_SIG_KEYRING_NONE; - prog->aux->sig.verdict = BPF_SIG_UNSIGNED; - } + prog->aux->sig.keyring_type = BPF_SIG_KEYRING_NONE; + prog->aux->sig.verdict = BPF_SIG_UNSIGNED; prog->orig_prog = NULL; prog->jited = 0; @@ -3189,10 +3121,6 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at if (err < 0) goto free_prog; - err = security_bpf_prog_load(prog, attr, token, uattr.is_kernel); - if (err) - goto free_prog; - /* run eBPF verifier */ err = bpf_check(&prog, attr, uattr, attr_log); if (err < 0) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e8e21d1a919a..001ac53825da 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include #include @@ -2554,6 +2556,10 @@ fd_array_get_btf_sparse(struct bpf_verifier_env *env, u32 idx) static struct btf *fd_array_get_btf(struct bpf_verifier_env *env, u32 idx) { + if (env->signature) { + verbose(env, "signed program cannot bind any BTF\n"); + return ERR_PTR(-EACCES); + } if (env->fd_array) return fd_array_get_btf_continuous(env, idx); if (!bpfptr_is_null(env->fd_array_raw)) @@ -17627,6 +17633,11 @@ static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf) if (env->used_btfs[i].btf == btf) goto ret_put; + if (env->signature) { + verbose(env, "signed program cannot bind any BTF\n"); + ret = -EACCES; + goto ret_put; + } if (env->used_btf_cnt >= MAX_USED_BTFS) { verbose(env, "The total number of btfs per program has reached the limit of %u\n", MAX_USED_BTFS); @@ -17909,6 +17920,12 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) if (env->used_maps[i] == map) return i; + if (env->signature && + env->prog->aux->sig.verdict == BPF_SIG_VERIFIED) { + verbose(env, "signed program cannot bind map '%s' not covered by the signature\n", + map->name); + return -EACCES; + } if (env->used_map_cnt >= MAX_USED_MAPS) { verbose(env, "The total number of maps per program has reached the limit of %u\n", MAX_USED_MAPS); @@ -17992,6 +18009,10 @@ static int fd_array_get_map_idx(struct bpf_verifier_env *env, u32 idx) { if (env->fd_array) return fd_array_get_map_idx_continuous(env, idx); + if (env->signature) { + verbose(env, "signed program must bind maps via a continuous fd_array (fd_array_cnt)\n"); + return -EACCES; + } if (!bpfptr_is_null(env->fd_array_raw)) return fd_array_get_map_idx_sparse(env, idx); @@ -18270,6 +18291,10 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env) map_idx = fd_array_get_map_idx(env, insn[0].imm); break; default: + if (env->signature) { + verbose(env, "signed program cannot reference a map by fd, only via fd_array index\n"); + return -EINVAL; + } map_idx = add_used_map(env, insn[0].imm); break; } @@ -19851,6 +19876,146 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } +static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) +{ + switch (keyring_id) { + case 0: + return BPF_SIG_KEYRING_BUILTIN; + case (s32)(unsigned long)VERIFY_USE_SECONDARY_KEYRING: + return BPF_SIG_KEYRING_SECONDARY; + case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING: + return BPF_SIG_KEYRING_PLATFORM; + default: + return BPF_SIG_KEYRING_USER; + } +} + +/* + * Verify the PKCS#7 signature of a loaded program. Called from bpf_check() + * once the program's metadata maps have been resolved into used_maps, so + * the exact maps folded into the signature are the ones the program binds. + * + * The signature covers the instructions followed by the frozen contents of + * each map, in @maps order: insns || map_0 || map_1 || [...]. On success the + * verdict and keyring info are recorded on prog->aux. + */ +static int bpf_prog_verify_signature(struct bpf_verifier_env *env, + union bpf_attr *attr, bool is_kernel) +{ + bpfptr_t usig = make_bpfptr(attr->signature, is_kernel); + struct bpf_dynptr_kern sig_ptr, data_ptr; + struct bpf_prog *prog = env->prog; + struct bpf_map **maps = env->used_maps; + struct bpf_key *key = NULL; + void *sig, *data = NULL; + u32 map_cnt = env->used_map_cnt; + u32 i, off, insns_sz; + u64 data_sz; + int err = 0; + + /* + * Don't attempt to use kmalloc_large or vmalloc for signatures. + * Practical signature for BPF program should be below this limit. + */ + if (!attr->signature_size || + attr->signature_size > KMALLOC_MAX_CACHE_SIZE) + return -EINVAL; + if (system_keyring_id_check(attr->keyring_id) == 0) + key = bpf_lookup_system_key(attr->keyring_id); + else + key = bpf_lookup_user_key(attr->keyring_id, 0); + if (!key) { + verbose(env, "cannot resolve signing keyring with keyring_id %d\n", + attr->keyring_id); + return -EINVAL; + } + + sig = kvmemdup_bpfptr(usig, attr->signature_size); + if (IS_ERR(sig)) { + bpf_key_put(key); + return PTR_ERR(sig); + } + + insns_sz = prog->len * sizeof(struct bpf_insn); + data_sz = insns_sz; + for (i = 0; i < map_cnt; i++) { + struct bpf_map *map = maps[i]; + + if (map->map_type != BPF_MAP_TYPE_ARRAY || + !map->ops->map_direct_value_addr) { + verbose(env, "signed program metadata map '%s' must be an array\n", + map->name); + err = -EINVAL; + goto out; + } + if (!READ_ONCE(map->frozen)) { + verbose(env, "signed program metadata map '%s' must be frozen\n", + map->name); + err = -EPERM; + goto out; + } + if (bpf_map_write_active(map)) { + verbose(env, "signed program metadata map '%s' has active writers\n", + map->name); + err = -EBUSY; + goto out; + } + if (!map->excl_prog_sha) { + verbose(env, "signed program metadata map '%s' must be exclusive\n", + map->name); + err = -EPERM; + goto out; + } + data_sz += map->value_size; + } + if (bpf_dynptr_check_size(data_sz)) { + verbose(env, "signed payload too large: %llu bytes\n", data_sz); + err = -E2BIG; + goto out; + } + data = kvmalloc(data_sz, GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!data) { + err = -ENOMEM; + goto out; + } + memcpy(data, prog->insnsi, insns_sz); + off = insns_sz; + for (i = 0; i < map_cnt; i++) { + struct bpf_map *map = maps[i]; + u64 addr; + + err = map->ops->map_direct_value_addr(map, &addr, 0); + if (err) { + verbose(env, "failed to read signed metadata map '%s': %d\n", + map->name, err); + goto out; + } + memcpy(data + off, (void *)(unsigned long)addr, + map->value_size); + off += map->value_size; + } + + bpf_dynptr_init(&data_ptr, data, BPF_DYNPTR_TYPE_LOCAL, 0, data_sz); + bpf_dynptr_init(&sig_ptr, sig, BPF_DYNPTR_TYPE_LOCAL, 0, + attr->signature_size); + + err = bpf_verify_pkcs7_signature((struct bpf_dynptr *)&data_ptr, + (struct bpf_dynptr *)&sig_ptr, key); + if (err) { + verbose(env, "signature verification failed: %d\n", err); + } else { + verbose(env, "signature verification passed\n"); + prog->aux->sig.keyring_serial = bpf_key_serial(key); + prog->aux->sig.keyring_type = bpf_classify_keyring(attr->keyring_id); + prog->aux->sig.verdict = BPF_SIG_VERIFIED; + } +out: + kvfree(data); + bpf_key_put(key); + kvfree(sig); + return err; +} + int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log) { @@ -19873,18 +20038,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, return -ENOMEM; env->bt.env = env; - - len = (*prog)->len; - env->insn_aux_data = - vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); - ret = -ENOMEM; - if (!env->insn_aux_data) - goto err_free_env; - for (i = 0; i < len; i++) - env->insn_aux_data[i].orig_idx = i; - env->succ = bpf_iarray_realloc(NULL, 2); - if (!env->succ) - goto err_free_env; env->prog = *prog; env->ops = bpf_verifier_ops[env->prog->type]; @@ -19893,22 +20046,51 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token); env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token); env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF); - - bpf_get_btf_vmlinux(); - - /* grab the mutex to protect few globals used by verifier */ - if (!is_priv) - mutex_lock(&bpf_verifier_lock); + env->signature = attr->signature; /* user could have requested verbose verifier output * and supplied buffer to store the verification trace */ ret = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size); if (ret) - goto err_unlock; + goto err_free_env; + if (env->signature) { + ret = bpf_prog_calc_tag(env->prog); + if (ret < 0) + goto err_prep; + } ret = process_fd_array(env, attr, uattr); if (ret) + goto err_prep; + + if (env->signature) { + ret = bpf_prog_verify_signature(env, attr, uattr.is_kernel); + if (ret) + goto err_prep; + } + + ret = security_bpf_prog_load(env->prog, attr, env->prog->aux->token, + uattr.is_kernel); + if (ret) + goto err_prep; + + bpf_get_btf_vmlinux(); + + /* grab the mutex to protect few globals used by verifier */ + if (!is_priv) + mutex_lock(&bpf_verifier_lock); + + len = env->prog->len; + env->insn_aux_data = + vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); + ret = -ENOMEM; + if (!env->insn_aux_data) + goto skip_full_check; + for (i = 0; i < len; i++) + env->insn_aux_data[i].orig_idx = i; + env->succ = bpf_iarray_realloc(NULL, 2); + if (!env->succ) goto skip_full_check; mark_verifier_state_clean(env); @@ -20132,18 +20314,26 @@ err_release_maps: *prog = env->prog; module_put(env->attach_btf_mod); -err_unlock: if (!is_priv) mutex_unlock(&bpf_verifier_lock); - bpf_clear_insn_aux_data(env, 0, env->prog->len); + goto err_free_env; +err_prep: + err = bpf_log_attr_finalize(attr_log, &env->log); + if (err) + ret = err; + release_insn_arrays(env); + release_maps(env); + release_btfs(env); err_free_env: + if (env->insn_aux_data) + bpf_clear_insn_aux_data(env, 0, env->prog->len); + vfree(env->insn_aux_data); kvfree(env->fd_array); bpf_stack_liveness_free(env); kvfree(env->cfg.insn_postorder); kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); - vfree(env->insn_aux_data); kvfree(env); return ret; } -- cgit v1.2.3 From a2d784869a0f252e1a277db7dc2c16d55694da72 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 8 Jul 2026 09:53:38 +0200 Subject: libbpf: Drop in-loader metadata check for load-time verification The signed gen_loader used to police its own metadata map from within BPF: emit_signature_match() read the kernel-cached map->sha[] back through hardcoded struct bpf_map offsets and compared it against a hash that compute_sha_update_offsets() baked into the signed instructions, after a BPF_OBJ_GET_INFO_BY_FD round-trip to populate map->sha[]. The kernel now verifies the metadata at BPF_PROG_LOAD time by folding the frozen contents of the loader's exclusive fd_array maps into the signature, so the loader no longer checks anything itself. Generated loaders thus carry no verification logic of their own anymore: Nothing in the signing chain depends on emitted loader bytecode doing the right thing. On the loading side, skel_internal.h now sets fd_array_cnt for a signed load so the kernel scans fd_array for the exclusive metadata map - still frozen, as the kernel requires - and the BPF_OBJ_GET_INFO_BY_FD round-trip to populate map->sha[] is gone. The struct bpf_map layout BUILD_BUG_ON()s on the kernel side are removed as well: they only pinned the ABI for the in-BPF read of map->sha[] that is no longer needed. Same for the map->excl member. Note: gen_hash is retained; it still marks a loader as signed so an untrusted host cannot re-dimension maps or override initial values now covered by the signature. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260708075343.358712-4-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/syscall.c | 7 ------- 1 file changed, 7 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index e898fad01aaf..358f2b0ce2bd 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1598,13 +1598,6 @@ static int map_create_alloc(union bpf_attr *attr, bpfptr_t uattr, struct bpf_ver err = -EFAULT; goto free_map; } - - /* See libbpf: emit_signature_match() */ - BUILD_BUG_ON(offsetof(struct bpf_map, excl) != SHA256_DIGEST_SIZE); - BUILD_BUG_ON(!__same_type(map->excl, u32)); - BUILD_BUG_ON(offsetof(struct bpf_map, sha) != 0); - BUILD_BUG_ON(!__same_type(map->sha, u8[SHA256_DIGEST_SIZE])); - map->excl = 1; } else if (attr->excl_prog_hash_size) { bpf_log(log, "Invalid excl_prog_hash_size.\n"); err = -EINVAL; -- cgit v1.2.3 From 92863e678070f57c17c868e4bfa2441a5c61ad2b Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 8 Jul 2026 23:15:34 +0200 Subject: bpf: Fix vmlinux BTF prep race in bpf_get_btf_vmlinux bpf_get_btf_vmlinux() lazily parses the vmlinux BTF under the bpf_verifier_lock, but publishes the result through a plain store and re-checks it through a plain lockless load. Nothing orders the stores initializing the struct btf inside btf_parse_vmlinux() against the store publishing the pointer: On a weakly ordered arch, a concurrent first-time caller taking the lockless fast path could in principle observe the pointer before the parsed contents are visible. The mutex_unlock() does not help such a reader given it only synchronizes with a later acquisition of the same lock. Thus, publish the pointer with smp_store_release() and read it on the fast path with smp_load_acquire(). Acquire semantics are needed rather than a dependency-ordered READ_ONCE(): btf_parse_vmlinux() also populates globals outside the returned object (e.g. bpf_ctx_convert.t). An address dependency would only order accesses performed through the pointer and not cover other globals. Fixes: 8580ac9404f6 ("bpf: Process in-kernel BTF") Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260708211537.371874-2-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 001ac53825da..9217e0f87cb5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -19559,13 +19559,25 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt struct btf *bpf_get_btf_vmlinux(void) { - if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { + /* Pairs with the smp_store_release() on the parse path below. */ + struct btf *btf = smp_load_acquire(&btf_vmlinux); + + if (!btf && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { mutex_lock(&bpf_verifier_lock); - if (!btf_vmlinux) - btf_vmlinux = btf_parse_vmlinux(); + btf = btf_vmlinux; + if (!btf) { + btf = btf_parse_vmlinux(); + /* + * Order the parsed BTF contents and the globals the + * parse populated (e.g. bpf_ctx_convert.t) before + * the pointer publication. Pairs with the acquire + * on the lockless fast path above. + */ + smp_store_release(&btf_vmlinux, btf); + } mutex_unlock(&bpf_verifier_lock); } - return btf_vmlinux; + return btf; } /* -- cgit v1.2.3 From 5e5e94d87dea92cc2e2fadaf3be84771509a86ca Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 8 Jul 2026 23:15:35 +0200 Subject: bpf: Give vmlinux BTF init its own mutex bpf_get_btf_vmlinux() serializes the lazy vmlinux BTF parse with bpf_verifier_lock, the same mutex bpf_check() holds across the whole verification of an unprivileged program (if enabled; it's disabled by default). The latter can potentially stall the mutex holder for a long time (e.g. via userfaultfd), and therefore block first-time bpf_get_btf_vmlinux() caller from any context, including privileged program loads. Give the vmlinux BTF initialization a dedicated btf_vmlinux_lock so it is independent of the unprivileged verification mutex. The parse only needs mutual exclusion against itself. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260708211537.371874-3-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/btf.c | 2 +- kernel/bpf/verifier.c | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index dff5c0d91641..8c04c340f499 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6451,7 +6451,7 @@ struct btf *btf_parse_vmlinux(void) if (IS_ERR(btf)) goto err_out; - /* btf_parse_vmlinux() runs under bpf_verifier_lock */ + /* btf_parse_vmlinux() runs under btf_vmlinux_lock */ bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]); err = btf_alloc_id(btf); if (err) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 9217e0f87cb5..40e20dfa3212 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -324,6 +324,7 @@ static const char *btf_type_name(const struct btf *btf, u32 id) } static DEFINE_MUTEX(bpf_verifier_lock); +static DEFINE_MUTEX(btf_vmlinux_lock); static DEFINE_MUTEX(bpf_percpu_ma_lock); __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) @@ -19563,7 +19564,7 @@ struct btf *bpf_get_btf_vmlinux(void) struct btf *btf = smp_load_acquire(&btf_vmlinux); if (!btf && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { - mutex_lock(&bpf_verifier_lock); + mutex_lock(&btf_vmlinux_lock); btf = btf_vmlinux; if (!btf) { btf = btf_parse_vmlinux(); @@ -19575,7 +19576,7 @@ struct btf *bpf_get_btf_vmlinux(void) */ smp_store_release(&btf_vmlinux, btf); } - mutex_unlock(&bpf_verifier_lock); + mutex_unlock(&btf_vmlinux_lock); } return btf; } @@ -20089,7 +20090,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, bpf_get_btf_vmlinux(); - /* grab the mutex to protect few globals used by verifier */ + /* Serialize verification of unprivileged programs. */ if (!is_priv) mutex_lock(&bpf_verifier_lock); -- cgit v1.2.3 From 42560699a83db261d1a671a5eadade460d0f9eee Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 8 Jul 2026 23:15:36 +0200 Subject: bpf: Account insn_aux_data allocation in bpf_check The insn_aux_data array is allocated with a plain vzalloc(), while every other allocation scoped to the verification - verifier states, explored states, the cfg/scc arrays, liveness masks, jump history - is charged to the loader's memcg via GFP_KERNEL_ACCOUNT. At 136 bytes per instruction it is one of the largest verification-time buffers, in the range of ~130MB for a program at the 1M instruction limit (worst case), and it lives across the whole verification. The buffer is also inconsistent with itself: when instruction patching grows it, the vrealloc() in bpf_patch_insn_data() already passes GFP_KERNEL_ACCOUNT. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260708211537.371874-4-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 40e20dfa3212..ad8ff228c963 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20096,7 +20096,8 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, len = env->prog->len; env->insn_aux_data = - vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); + __vmalloc(array_size(sizeof(struct bpf_insn_aux_data), len), + GFP_KERNEL_ACCOUNT | __GFP_ZERO); ret = -ENOMEM; if (!env->insn_aux_data) goto skip_full_check; -- cgit v1.2.3 From ff755b6007908730946c155bb0d90ebc55926da7 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 8 Jul 2026 23:15:37 +0200 Subject: bpf: Account scratch buffer in bpf_prog_calc_tag bpf_prog_calc_tag() copies the instructions into a plain vmalloc() scratch buffer to blind the map fds before hashing. The buffer scales with the program, up to ~8MB at the 1M instruction limit, and is allocated on every program load, but unlike the rest of the load-time scratch memory it is not charged to the loader's memcg. Use GFP_KERNEL_ACCOUNT to account it like the other allocations scoped to the verification/load. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260708211537.371874-5-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 6e19a030da6f..f2b6e4c888af 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -305,7 +305,7 @@ int bpf_prog_calc_tag(struct bpf_prog *fp) bool was_ld_map; u32 i; - dst = vmalloc(size); + dst = __vmalloc(size, GFP_KERNEL_ACCOUNT); if (!dst) return -ENOMEM; -- cgit v1.2.3 From 47b079e2117a2ee52e21f8b72935900c702fc0b5 Mon Sep 17 00:00:00 2001 From: Sanghyun Park Date: Wed, 8 Jul 2026 16:21:04 +0900 Subject: bpf: Fix use-after-free on mm_struct in bpf_find_vma() bpf_find_vma() reads task->mm and calls mmap_read_trylock(mm) without holding a reference on the mm. On a foreign task, a concurrent exit_mm() can free the mm_struct between the lockless read and the trylock, resulting in a use-after-free. mm_struct is not SLAB_TYPESAFE_BY_RCU. For the current task, task->mm is stable. For a foreign task, pin the mm under task->alloc_lock and release it with mmput_async(), mirroring commit d8e27d2d22b6 ("bpf: fix mm lifecycle in open-coded task_vma iterator"). Use spin_trylock() instead of get_task_mm() so BPF context does not block on alloc_lock. Reject irqs-disabled contexts and !CONFIG_MMU on the foreign-task path because dropping the mm reference is not safe there. Race: CPU0 (BPF program) CPU1 (exiting task) ============================ ========================== bpf_find_vma(foreign_task): mm = task->mm exit_mm(): task->mm = NULL mmput(mm) -> frees mm_struct mmap_read_trylock(mm) // UAF on mm Fixes: 7c7e3d31e785 ("bpf: Introduce helper bpf_find_vma") Signed-off-by: Sanghyun Park Reviewed-by: Puranjay Mohan Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20260708072106.199637-2-sanghyun.park.cnu@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/task_iter.c | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index e791ae065c39..b256fb9c1214 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -756,6 +756,7 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, struct mmap_unlock_irq_work *work = NULL; struct vm_area_struct *vma; bool irq_work_busy = false; + bool __maybe_unused mmput_needed = false; struct mm_struct *mm; int ret = -ENOENT; @@ -765,14 +766,38 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, if (!task) return -ENOENT; - mm = task->mm; + if (task == current) { + mm = task->mm; + } else { + /* + * Foreign task: pin task->mm against a concurrent exit_mm(). + * Use trylock on alloc_lock instead of get_task_mm()'s + * blocking task_lock() to avoid deadlocking the target task. + */ + if (!IS_ENABLED(CONFIG_MMU)) + return -EOPNOTSUPP; + if (irqs_disabled()) + return -EBUSY; + if (!spin_trylock(&task->alloc_lock)) + return -EBUSY; + mm = task->mm; + if (mm && !(task->flags & PF_KTHREAD)) { + mmget(mm); + mmput_needed = true; + } else { + mm = NULL; + } + spin_unlock(&task->alloc_lock); + } if (!mm) return -ENOENT; irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); - if (irq_work_busy || !mmap_read_trylock(mm)) - return -EBUSY; + if (irq_work_busy || !mmap_read_trylock(mm)) { + ret = -EBUSY; + goto out; + } vma = find_vma(mm, start); @@ -782,6 +807,11 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, ret = 0; } bpf_mmap_unlock_mm(work, mm); +out: +#ifdef CONFIG_MMU + if (mmput_needed) + mmput_async(mm); +#endif return ret; } -- cgit v1.2.3 From 9a6df65d5c6a9947ddab4e563e329720f44b8747 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Wed, 8 Jul 2026 18:18:05 +0800 Subject: bpf: Introduce jit_required flag and remove bpf_prog_has_kfunc_call() Introduce a 'jit_required' bitfield flag in struct bpf_prog to track whether a BPF program strictly requires the JIT compiler to run. This prevents a dangerous runtime fallback to the interpreter for features that are only implemented in the JIT compiler. Currently, bpf_prog_has_kfunc_call() is used only for kernel function calls, replace the kfunc-specific helper with the new 'jit_required' flag. This makes it easy to support other JIT-only BPF features, such as inlined helpers. Suggested-by: Alexei Starovoitov Suggested-by: KaFai Wan Suggested-by: Leon Hwang Acked-by: Leon Hwang Signed-off-by: Tiezhu Yang Signed-off-by: Eduard Zingerman --- kernel/bpf/core.c | 7 ++----- kernel/bpf/fixups.c | 5 ++--- kernel/bpf/verifier.c | 7 ++----- 3 files changed, 6 insertions(+), 13 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index f2b6e4c888af..47fe047ad30b 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -126,6 +126,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag fp->aux->main_prog_aux = aux; fp->aux->prog = fp; fp->jit_requested = ebpf_jit_enabled(); + fp->jit_required = IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON); fp->blinding_requested = bpf_jit_blinding_enabled(fp); #ifdef CONFIG_CGROUP_BPF aux->cgroup_atype = CGROUP_BPF_ATTACH_TYPE_INVALID; @@ -2670,15 +2671,11 @@ struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct /* In case of BPF to BPF calls, verifier did all the prep * work with regards to JITing, etc. */ - bool jit_needed = false; + bool jit_needed = fp->jit_required; if (fp->bpf_func) goto finalize; - if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) || - bpf_prog_has_kfunc_call(fp)) - jit_needed = true; - if (!bpf_prog_select_interpreter(fp)) jit_needed = true; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 12a8a4eb757f..02246df2f6c3 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1378,7 +1378,6 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) #ifndef CONFIG_BPF_JIT_ALWAYS_ON struct bpf_prog *prog = env->prog; struct bpf_insn *insn = prog->insnsi; - bool has_kfunc_call = bpf_prog_has_kfunc_call(prog); int depth; #endif int i, err = 0; @@ -1404,8 +1403,8 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) return err; } #ifndef CONFIG_BPF_JIT_ALWAYS_ON - if (has_kfunc_call) { - verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); + if (prog->jit_required) { + verbose(env, "program requires BPF JIT compiler but it is not available\n"); return -EINVAL; } for (i = 0; i < env->subprog_cnt; i++) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ad8ff228c963..233472a871be 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2780,6 +2780,8 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) prog_aux->kfunc_tab = tab; } + env->prog->jit_required = 1; + /* func_id == 0 is always invalid, but instead of returning an error, be * conservative and wait until the code elimination pass before returning * error, so that invalid calls that get pruned out can be in BPF programs @@ -2834,11 +2836,6 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) return 0; } -bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) -{ - return !!prog->aux->kfunc_tab; -} - static int add_subprog_and_kfunc(struct bpf_verifier_env *env) { struct bpf_subprog_info *subprog = env->subprog_info; -- cgit v1.2.3 From f1c27922576edccb99d0257827d09bd05c0304a6 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Wed, 8 Jul 2026 18:18:06 +0800 Subject: bpf: Reject programs with inlined helpers if JIT is not available When an architecture (such as LoongArch, ARM64, and RISC-V) implements bpf_jit_inlines_helper_call(), the verifier skips rewriting the helper call offset (insn->imm) in bpf_do_misc_fixups(). This is because the helper is expected to be inlined by the JIT compiler later. Therefore, insn->imm remains as the raw helper enum ID. However, if JIT is disabled at runtime (net.core.bpf_jit_enable=0) or if JIT compilation fails dynamically (e.g., due to OOM), the program falls back to the BPF interpreter. When the interpreter executes (__bpf_call_base + insn->imm) with the unpatched raw ID, it jumps into an invalid address space, triggering an instruction alignment fault or a kernel panic. Although these helpers have valid C implementations in the kernel, the omission of offset rewriting makes runtime interpreter fallback fatal. Fix this by setting 'prog->jit_required = 1' when helper call rewriting is skipped for JIT inlining. This ensures that such programs are safely rejected if JIT is not available, preventing the runtime kernel panic. Fixes: 2ddec2c80b44 ("riscv, bpf: inline bpf_get_smp_processor_id()") Suggested-by: Alexei Starovoitov Suggested-by: KaFai Wan Acked-by: Leon Hwang Signed-off-by: Tiezhu Yang Signed-off-by: Eduard Zingerman --- kernel/bpf/fixups.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 02246df2f6c3..d3be972714b2 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1840,8 +1840,10 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) } /* Skip inlining the helper call if the JIT does it. */ - if (bpf_jit_inlines_helper_call(insn->imm)) + if (bpf_jit_inlines_helper_call(insn->imm)) { + prog->jit_required = 1; goto next_insn; + } if (insn->imm == BPF_FUNC_get_route_realm) prog->dst_needed = 1; -- cgit v1.2.3 From 36ffa86c42f91c8a57071e024afc4ffb51a8958f Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Thu, 9 Jul 2026 09:34:22 +0200 Subject: bpf: Fix security_bpf_map_create error handling Commit 5816bf4273ed ("lsm,selinux: Add LSM blob support for BPF objects") made the LSM hook wrappers for BPF object creation clean up the LSM state internally upon denial, e.g. security_bpf_map_create() internally calls security_bpf_map_free() when the bpf_map_create hook returns an error. map_create() however still routes a denial to its free_map_sec label, which invokes security_bpf_map_free() a second time, so the bpf_map_free hook fires twice for a single denied map. In-tree LSMs are unaffected in practice since the blob kfree() inside security_bpf_map_free() is NULL-safe and idempotent and none of them implement bpf_map_free, but a BPF LSM program attached to that hook observes double invocations. Route the denial to free_map instead. Fixes: 5816bf4273ed ("lsm,selinux: Add LSM blob support for BPF objects") Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260709073422.379247-1-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/syscall.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 358f2b0ce2bd..0ff9e3aa293d 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1649,7 +1649,7 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_common_at err = security_bpf_map_create(map, attr, token, uattr.is_kernel); if (err) - goto free_map_sec; + goto free_map; err = bpf_map_alloc_id(map); if (err) -- cgit v1.2.3 From 2cb5f4ca695ebe552647e5ba4aad6934d6a43bae Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Thu, 9 Jul 2026 17:31:30 +0200 Subject: bpf: Drop scalar id on sign-extending narrowing stack fills When a spilled scalar is filled back with a sign-extending narrowing load (BPF_MEMSX), check_stack_read_fixed_off() copies the spilled register including its scalar id, but coerce_reg_to_size_sx() then sign-extends the filled register's value. If the same slot is also filled with a plain zero-extending load (BPF_MEM), both destination registers share the id yet hold different values. A later 'if == const' then refines the sign-extended register through sync_linked_regs() to a value it does not have at runtime (e.g. the verifier believes 0x80000000 while the register is 0xffffffff80000000), which can be turned into an out-of-bounds access. Drop the shared scalar id at the sign-extension site in check_mem_access() when sign extension actually changes the value, mirroring the BPF_MOVSX handling in check_alu_op() (no_sext = reg_umax < 2^(size*8-1)). Fixes: 3cd5c890652b ("bpf: Let the verifier assign ids on stack fills") Reported-by: STAR Labs SG Signed-off-by: Daniel Borkmann Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 233472a871be..a0830ad6bebb 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6394,11 +6394,23 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && regs[value_regno].type == SCALAR_VALUE) { - if (!is_ldsx) + if (!is_ldsx) { /* b/h/w load zero-extends, mark upper bits as known 0 */ coerce_reg_to_size(®s[value_regno], size); - else + } else { + /* + * Sign-extension can change the register value relative + * to a scalar it is linked with by id (e.g. a zero- + * extending fill of the same spilled stack slot), thus + * drop the shared id in that case. + */ + bool no_sext = reg_umax(®s[value_regno]) < + (1ULL << (size * BITS_PER_BYTE - 1)); + coerce_reg_to_size_sx(®s[value_regno], size); + if (!no_sext) + clear_scalar_id(®s[value_regno]); + } } return err; } -- cgit v1.2.3 From 2aaf67f0516fde29620d0edfc29c01b9ea7ad430 Mon Sep 17 00:00:00 2001 From: Nicholas Dudar Date: Thu, 9 Jul 2026 11:58:36 -0400 Subject: bpf: Reject rdonly/rdwr_buf_size kfunc arguments that exceed u32 max check_kfunc_args() detects a kfunc argument named rdonly_buf_size or rdwr_buf_size and stores reg->var_off.value into meta->r0_size, a u64, and does not bound it. check_kfunc_call() later copies that value into the returned register's mem_size field: meta->r0_size = reg->var_off.value; ... regs[BPF_REG_0].mem_size = meta.r0_size; regs[BPF_REG_0].mem_size is u32. A constant whose upper 32 bits are set gets truncated instead of causing a load-time rejection, so the verifier records a PTR_TO_MEM register with an approximately 4 GiB mem_size for whatever allocation the kfunc returned. A later access check against that register uses the truncated, wrong bound. Reject rdonly_buf_size/rdwr_buf_size values that exceed U32_MAX at the point meta->r0_size is set. Fixes: eb1f7f71c126 ("bpf/verifier: allow kfunc to return an allocated mem") Signed-off-by: Nicholas Dudar Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260709155837.1879230-2-main.kalliope@gmail.com Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a0830ad6bebb..03e2202cca13 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -12109,6 +12109,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } meta->r0_size = reg->var_off.value; + if (meta->r0_size > U32_MAX) { + verbose(env, "%s rdonly/rdwr_buf_size exceeds u32 max\n", + reg_arg_name(env, argno)); + return -EINVAL; + } if (regno >= 0) ret = mark_chain_precision(env, regno); else -- cgit v1.2.3 From 8740156ad33be5071b588b594c55f279457f667c Mon Sep 17 00:00:00 2001 From: Nicholas Dudar Date: Thu, 9 Jul 2026 14:27:59 -0400 Subject: bpf: Require a BPF cpumask for bpf_cpumask_populate() bpf_cpumask_populate() writes to its destination with bitmap_copy(), but the destination is typed as struct cpumask *. That allows the verifier to accept borrowed cpumask pointers returned by read-only kfuncs, such as scx_bpf_get_online_cpumask(), as a writable destination. Make the destination a struct bpf_cpumask * so populate follows the same ownership rule as the other mutating cpumask kfuncs. Query kfuncs continue to accept const struct cpumask * inputs. Fixes: 950ad93df2fc ("bpf: add kfunc for populating cpumask bits") Signed-off-by: Nicholas Dudar Acked-by: Tejun Heo Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260709182800.2037938-2-main.kalliope@gmail.com Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/cpumask.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/cpumask.c b/kernel/bpf/cpumask.c index b8c805b4b06a..1336a4efa755 100644 --- a/kernel/bpf/cpumask.c +++ b/kernel/bpf/cpumask.c @@ -449,12 +449,12 @@ __bpf_kfunc u32 bpf_cpumask_weight(const struct cpumask *cpumask) * @src__sz: Length of the BPF memory region in bytes. * * Return: - * * 0 if the struct cpumask * instance was populated successfully. + * * 0 if the struct bpf_cpumask * instance was populated successfully. * * -EACCES if the memory region is too small to populate the cpumask. * * -EINVAL if the memory region is not aligned to the size of a long * and the architecture does not support efficient unaligned accesses. */ -__bpf_kfunc int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t src__sz) +__bpf_kfunc int bpf_cpumask_populate(struct bpf_cpumask *cpumask, void *src, size_t src__sz) { unsigned long source = (unsigned long)src; @@ -467,7 +467,7 @@ __bpf_kfunc int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t !IS_ALIGNED(source, sizeof(long))) return -EINVAL; - bitmap_copy(cpumask_bits(cpumask), src, nr_cpu_ids); + bitmap_copy(cpumask_bits(&cpumask->cpumask), src, nr_cpu_ids); return 0; } -- cgit v1.2.3 From 30bdd6d1384d894931f113eb595636092d8e650c Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Sat, 11 Jul 2026 20:48:21 +0800 Subject: bpf: Mark tracing_multi trampolines as ftrace managed Since tracing_multi link does not set ftrace_managed, it would fail to release the tracing_multi link when attaching tracing_multi link and then attaching fentry link. [ 3.714215] WARNING: kernel/bpf/trampoline.c:1727 at bpf_trampoline_multi_detach+0x20b/0x240, CPU#1: test_progs/97 ... [ 3.733170] bpf_tracing_multi_link_release+0x14/0x30 [ 3.733890] bpf_link_free+0x58/0x130 [ 3.734414] bpf_link_release+0x23/0x30 Fix it by setting 'ftrace_managed = true' in register_fentry_multi(). Fixes: aef4dfa790b2 ("bpf: Add bpf_trampoline_multi_attach/detach functions") Signed-off-by: Leon Hwang Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/20260711124822.29406-2-leon.hwang@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/trampoline.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 1a721fc4bef5..6eadf64f7ec9 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -1536,6 +1536,7 @@ static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_ima if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); + tr->func.ftrace_managed = true; ftrace_hash_add(data->reg, data->entry, ip, addr); tr->cur_image = im; return 0; -- cgit v1.2.3 From c28cbef2f8986c392faf6ca94bb2088548ea6964 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Tue, 14 Jul 2026 23:40:41 -0700 Subject: bpf: Remove dynptr check in check_stack_range_initialized() For a MEM_UNINIT ("raw mode") helper argument, check_stack_range_initialized() open-coded a scan that rejected any STACK_DYNPTR slot in the range with "potential write to dynptr". This duplicated, and was stricter than, the handling that runs when the buffer is actually marked initialized. check_helper_call() later replays the write byte by byte via check_mem_access(), which goes through destroy_if_dynptr_stack_slot(), which rejects overwritting a referenced dynptr. Therefore drop the redundant scan and rely on check_mem_access(). Signed-off-by: Amery Hung Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260715064047.1793790-2-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 25 ------------------------- 1 file changed, 25 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 03e2202cca13..7dd961ede88d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6733,31 +6733,6 @@ static int check_stack_range_initialized( } if (meta && meta->raw_mode) { - /* Ensure we won't be overwriting dynptrs when simulating byte - * by byte access in check_helper_call using meta.access_size. - * This would be a problem if we have a helper in the future - * which takes: - * - * helper(uninit_mem, len, dynptr) - * - * Now, uninint_mem may overlap with dynptr pointer. Hence, it - * may end up writing to dynptr itself when touching memory from - * arg 1. This can be relaxed on a case by case basis for known - * safe cases, but reject due to the possibilitiy of aliasing by - * default. - */ - for (i = min_off; i < max_off + access_size; i++) { - int stack_off = -i - 1; - - spi = bpf_get_spi(i); - /* raw_mode may write past allocated_stack */ - if (state->allocated_stack <= stack_off) - continue; - if (state->stack[spi].slot_type[stack_off % BPF_REG_SIZE] == STACK_DYNPTR) { - verbose(env, "potential write to dynptr at off=%d disallowed\n", i); - return -EACCES; - } - } meta->access_size = access_size; meta->regno = reg_from_argno(argno); return 0; -- cgit v1.2.3 From 92ec8b1b6b24381611600162536b8b5b6a9e7323 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Tue, 14 Jul 2026 23:40:42 -0700 Subject: bpf: Factor out raw_mode-related fields in bpf_call_arg_meta To prepare for unifying the helper and kfunc call_arg_meta, group the scattered MEM_UNINIT ("raw") memory argument fields (raw_mode, regno and access_size) into a new struct arg_raw_mem_desc. The intention is to make it clear about when these are set and used instead of fields with overly generic names. Identify the raw argument once, up front, in check_raw_mode_ok() (like check_proto_release_reg() does for release_regno), recording its regno. check_stack_range_initialized() now recognizes the raw buffer by matching that regno, so the separate raw_mode flag is no longer needed, and the per-argument "meta->raw_mode = arg_type & MEM_UNINIT" assignments in check_func_arg() go away with it. A raw memory argument can be tagged either ARG_PTR_TO_MEM | MEM_UNINIT or ARG_PTR_TO_MAP_VALUE | MEM_UNINIT (the output buffer of bpf_map_pop_elem() and bpf_map_peek_elem()). Either may be passed as a PTR_TO_STACK, which reaches check_stack_range_initialized() through check_helper_mem_access(), so both must be treated as raw. Extend arg_type_is_raw_mem() to match the map value case as well; otherwise check_raw_mode_ok() would not record the regno for it and an uninitialized stack buffer passed to those helpers would be wrongly rejected for programs without CAP_PERFMON. No functional change intended. This patch does not enable raw_mode memory access for kfunc (i.e., uninit stack will not be allowed to be passed to kfunc for unprivileged programs). Existing kfuncs with arguments tagged with __uninit are either priviledged or dynptr kfuncs, which take another path to make sure the access is checked by check_mem_access(). Signed-off-by: Amery Hung Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260715064047.1793790-3-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 58 ++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 31 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7dd961ede88d..1a41f99a9133 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -256,11 +256,9 @@ struct bpf_call_arg_meta { struct bpf_map_desc map; struct bpf_dynptr_desc dynptr; struct ref_obj_desc ref_obj; - bool raw_mode; + struct arg_raw_mem_desc arg_raw_mem; bool pkt_access; u8 release_regno; - int regno; - int access_size; int mem_size; u64 msize_max_value; int func_id; @@ -6690,6 +6688,8 @@ static int check_stack_range_initialized( * but BTF based global subprog validation isn't accurate enough. */ bool allow_poison = access_size < 0 || clobber; + /* The call will initialize the memory; uninitialized stack allowed */ + bool raw_mode = meta && meta->arg_raw_mem.regno == reg_from_argno(argno); access_size = abs(access_size); @@ -6725,16 +6725,14 @@ static int check_stack_range_initialized( * helper return since specific bounds are unknown what may * cause uninitialized stack leaking. */ - if (meta && meta->raw_mode) - meta = NULL; + raw_mode = false; min_off = reg_smin(reg) + off; max_off = reg_smax(reg) + off; } - if (meta && meta->raw_mode) { - meta->access_size = access_size; - meta->regno = reg_from_argno(argno); + if (raw_mode) { + meta->arg_raw_mem.size = access_size; return 0; } @@ -7727,7 +7725,13 @@ static bool arg_type_is_mem_size(enum bpf_arg_type type) static bool arg_type_is_raw_mem(enum bpf_arg_type type) { - return base_type(type) == ARG_PTR_TO_MEM && + /* + * A map value output buffer (e.g. bpf_map_pop_elem) is also a raw + * (uninitialized) memory argument, and like ARG_PTR_TO_MEM it may be + * passed as a PTR_TO_STACK that reaches check_stack_range_initialized(). + */ + return (base_type(type) == ARG_PTR_TO_MEM || + base_type(type) == ARG_PTR_TO_MAP_VALUE) && type & MEM_UNINIT; } @@ -8403,7 +8407,6 @@ skip_type_check: verifier_bug(env, "invalid map_ptr to access map->value"); return -EFAULT; } - meta->raw_mode = arg_type & MEM_UNINIT; err = check_helper_mem_access(env, reg, argno_from_reg(regno), meta->map.ptr->value_size, arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); @@ -8446,7 +8449,6 @@ skip_type_check: /* The access to this pointer is only checked when we hit the * next is_mem_size argument below. */ - meta->raw_mode = arg_type & MEM_UNINIT; if (arg_type & MEM_FIXED_SIZE) { err = check_helper_mem_access(env, reg, argno_from_reg(regno), fn->arg_size[arg], arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, @@ -8797,26 +8799,19 @@ error: return -EINVAL; } -static bool check_raw_mode_ok(const struct bpf_func_proto *fn) +static bool check_raw_mode_ok(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) { - int count = 0; + int i; - if (arg_type_is_raw_mem(fn->arg1_type)) - count++; - if (arg_type_is_raw_mem(fn->arg2_type)) - count++; - if (arg_type_is_raw_mem(fn->arg3_type)) - count++; - if (arg_type_is_raw_mem(fn->arg4_type)) - count++; - if (arg_type_is_raw_mem(fn->arg5_type)) - count++; + for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { + if (!arg_type_is_raw_mem(fn->arg_type[i])) + continue; + if (meta->arg_raw_mem.regno) + return false; + meta->arg_raw_mem.regno = i + 1; + } - /* We only support one arg being in raw mode at the moment, - * which is sufficient for the helper functions we have - * right now. - */ - return count <= 1; + return true; } static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg) @@ -8906,7 +8901,7 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_ static int check_func_proto(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) { - return check_raw_mode_ok(fn) && + return check_raw_mode_ok(fn, meta) && check_arg_pair_ok(fn) && check_mem_arg_rw_flag_ok(fn) && check_proto_release_reg(fn, meta) && @@ -10303,8 +10298,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn /* Mark slots with STACK_MISC in case of raw mode, stack offset * is inferred from register state. */ - for (i = 0; i < meta.access_size; i++) { - err = check_mem_access(env, insn_idx, regs + meta.regno, argno_from_reg(meta.regno), i, BPF_B, + for (i = 0; i < meta.arg_raw_mem.size; i++) { + err = check_mem_access(env, insn_idx, regs + meta.arg_raw_mem.regno, + argno_from_reg(meta.arg_raw_mem.regno), i, BPF_B, BPF_WRITE, -1, false, false); if (err) return err; -- cgit v1.2.3 From 77a4974c17493b0e0bcd5010dc2ec9ad749d1a07 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Tue, 14 Jul 2026 23:40:43 -0700 Subject: bpf: Pass argno to callees in check_func_arg() instead of argno_from_reg(regno) check_func_arg() only ever handles register arguments (the caller loops over the first MAX_BPF_FUNC_REG_ARGS arguments), so a single argno_t built from the register number identifies the argument for every callee. Remove the duplicated argno_from_reg() calls to simplify check_func_arg(). 'regno' is still kept for the few places that need the raw register number directly (register reads, verbose R%d messages) and for referring to the neighbouring size/memory argument in the ARG_CONST_SIZE{,_OR_ZERO} cases. No functional change intended. Suggested-by: Eduard Zingerman Signed-off-by: Amery Hung Link: https://lore.kernel.org/bpf/20260715064047.1793790-4-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 1a41f99a9133..529ad0b4fcbd 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8275,7 +8275,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 regno = BPF_REG_1 + arg; struct bpf_reg_state *reg = reg_state(env, regno); enum bpf_arg_type arg_type = fn->arg_type[arg]; - argno_t argno = argno_from_arg(arg + 1); + argno_t argno = argno_from_reg(regno); enum bpf_reg_type type = reg->type; u32 *arg_btf_id = NULL; u32 key_size; @@ -8320,11 +8320,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) arg_btf_id = fn->arg_btf_id[arg]; - err = check_reg_type(env, reg, argno_from_reg(regno), arg_type, arg_btf_id, meta); + err = check_reg_type(env, reg, argno, arg_type, arg_btf_id, meta); if (err) return err; - err = check_func_arg_reg_off(env, reg, argno_from_reg(regno), arg_type); + err = check_func_arg_reg_off(env, reg, argno, arg_type); if (err) return err; @@ -8381,7 +8381,7 @@ skip_type_check: return -EFAULT; } key_size = meta->map.ptr->key_size; - err = check_helper_mem_access(env, reg, argno_from_reg(regno), key_size, BPF_READ, false, NULL); + err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, NULL); if (err) return err; if (can_elide_value_nullness(meta->map.ptr)) { @@ -8407,7 +8407,7 @@ skip_type_check: verifier_bug(env, "invalid map_ptr to access map->value"); return -EFAULT; } - err = check_helper_mem_access(env, reg, argno_from_reg(regno), meta->map.ptr->value_size, + err = check_helper_mem_access(env, reg, argno, meta->map.ptr->value_size, arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); break; @@ -8425,11 +8425,11 @@ skip_type_check: return -EACCES; } if (meta->func_id == BPF_FUNC_spin_lock) { - err = process_spin_lock(env, reg, argno_from_reg(regno), PROCESS_SPIN_LOCK); + err = process_spin_lock(env, reg, argno, PROCESS_SPIN_LOCK); if (err) return err; } else if (meta->func_id == BPF_FUNC_spin_unlock) { - err = process_spin_lock(env, reg, argno_from_reg(regno), 0); + err = process_spin_lock(env, reg, argno, 0); if (err) return err; } else { @@ -8438,7 +8438,7 @@ skip_type_check: } break; case ARG_PTR_TO_TIMER: - err = process_timer_helper(env, reg, argno_from_reg(regno), meta); + err = process_timer_helper(env, reg, argno, meta); if (err) return err; break; @@ -8450,7 +8450,7 @@ skip_type_check: * next is_mem_size argument below. */ if (arg_type & MEM_FIXED_SIZE) { - err = check_helper_mem_access(env, reg, argno_from_reg(regno), fn->arg_size[arg], + err = check_helper_mem_access(env, reg, argno, fn->arg_size[arg], arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); if (err) @@ -8460,21 +8460,19 @@ skip_type_check: } break; case ARG_CONST_SIZE: - err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), - argno_from_reg(regno), - fn->arg_type[arg - 1] & MEM_WRITE ? - BPF_WRITE : BPF_READ, + err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, + argno_from_reg(regno - 1), argno, + fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); break; case ARG_CONST_SIZE_OR_ZERO: - err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), - argno_from_reg(regno), - fn->arg_type[arg - 1] & MEM_WRITE ? - BPF_WRITE : BPF_READ, + err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, + argno_from_reg(regno - 1), argno, + fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, true, meta); break; case ARG_PTR_TO_DYNPTR: - err = process_dynptr_func(env, reg, argno_from_reg(regno), insn_idx, arg_type, &meta->ref_obj, + err = process_dynptr_func(env, reg, argno, insn_idx, arg_type, &meta->ref_obj, &meta->dynptr); if (err) return err; @@ -8492,7 +8490,7 @@ skip_type_check: break; case ARG_PTR_TO_CONST_STR: { - err = check_arg_const_str(env, reg, argno_from_reg(regno)); + err = check_arg_const_str(env, reg, argno); if (err) return err; break; -- cgit v1.2.3 From 8faaa93b9f6a279472cd5490030151f1635292d2 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Tue, 14 Jul 2026 23:40:44 -0700 Subject: bpf: Unify helper and kfunc allocation-size argument handling The constant "size of the PTR_TO_MEM returned in R0" argument is handled by both helpers (ARG_CONST_ALLOC_SIZE_OR_ZERO) and kfuncs (__rdonly_buf_size / __rdwr_buf_size), each with its own meta field (meta->mem_size, meta->r0_size) and duplicated validation. Add struct arg_alloc_mem_desc and a shared process_const_alloc_mem_size(), and replace both fields with meta->arg_alloc_mem. The desc records presence with a 'found' flag instead of using a non-zero size as the sentinel. This also fixes a pre-existing bug on the kfunc return path: "no size argument" was tested as r0_size == 0, so an explicit __rdonly_buf_size/__rdwr_buf_size of 0 was treated as absent and fell through to btf_resolve_size(), giving R0 the size of the pointed-to return type instead of 0. With 'found', an explicit zero size is honored and btf_resolve_size() is used only when no size argument was passed. The size is stored in a u32, matching regs[R0].mem_size. The U32_MAX check now apply to both helper and kfunc through process_const_alloc_mem_size(). Fold bpf_session_cookie return size assignment into current kfunc return size resolution path. Note that verifier saves kfunc return size through r0_size instead of mem_size. The later has no active readers so remove it. Signed-off-by: Amery Hung Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260715064047.1793790-5-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 87 ++++++++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 40 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 529ad0b4fcbd..463b49df4ff9 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -257,9 +257,9 @@ struct bpf_call_arg_meta { struct bpf_dynptr_desc dynptr; struct ref_obj_desc ref_obj; struct arg_raw_mem_desc arg_raw_mem; + struct ret_mem_desc ret_mem; bool pkt_access; u8 release_regno; - int mem_size; u64 msize_max_value; int func_id; struct btf *btf; @@ -6974,6 +6974,40 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg return err; } +static int process_const_alloc_mem_size(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, struct ret_mem_desc *ret_mem) +{ + int regno = reg_from_argno(argno); + int err; + + if (ret_mem->found) { + verifier_bug(env, "only one allocation size argument permitted"); + return -EFAULT; + } + + if (!tnum_is_const(reg->var_off)) { + verbose(env, "%s is not a const\n", reg_arg_name(env, argno)); + return -EINVAL; + } + + if (reg->var_off.value > U32_MAX) { + verbose(env, "%s allocation size exceeds u32 max\n", reg_arg_name(env, argno)); + return -EINVAL; + } + + if (regno >= 0) + err = mark_chain_precision(env, regno); + else + err = mark_stack_arg_precision(env, arg_idx_from_argno(argno)); + if (err) + return err; + + ret_mem->size = reg->var_off.value; + ret_mem->found = true; + + return 0; +} + static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, struct bpf_reg_state *size_reg, argno_t mem_argno, argno_t size_argno) { @@ -8478,13 +8512,7 @@ skip_type_check: return err; break; case ARG_CONST_ALLOC_SIZE_OR_ZERO: - if (!tnum_is_const(reg->var_off)) { - verbose(env, "R%d is not a known constant'\n", - regno); - return -EACCES; - } - meta->mem_size = reg->var_off.value; - err = mark_chain_precision(env, regno); + err = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); if (err) return err; break; @@ -10516,7 +10544,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn case RET_PTR_TO_MEM: mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; - regs[BPF_REG_0].mem_size = meta.mem_size; + regs[BPF_REG_0].mem_size = meta.ret_mem.size; break; case RET_PTR_TO_MEM_OR_BTF_ID: { @@ -12066,28 +12094,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } if (is_ret_buf_sz) { - if (meta->r0_size) { - verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc"); - return -EINVAL; - } - - if (!tnum_is_const(reg->var_off)) { - verbose(env, "%s is not a const\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - - meta->r0_size = reg->var_off.value; - if (meta->r0_size > U32_MAX) { - verbose(env, "%s rdonly/rdwr_buf_size exceeds u32 max\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (regno >= 0) - ret = mark_chain_precision(env, regno); - else - ret = mark_stack_arg_precision(env, i); - if (ret) + ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); + if (ret < 0) return ret; } continue; @@ -13066,11 +13074,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } - if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) { - meta.r0_size = sizeof(u64); - meta.r0_rdonly = false; - } - if (is_bpf_wq_set_callback_kfunc(meta.func_id)) { err = push_callback_call(env, insn, insn_idx, meta.subprogno, set_timer_callback_state); @@ -13203,15 +13206,19 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, /* kfunc returning 'void *' is equivalent to returning scalar */ mark_reg_unknown(env, regs, BPF_REG_0); } else if (!__btf_type_is_struct(ptr_type)) { - if (!meta.r0_size) { + if (!meta.ret_mem.found) { __u32 sz; if (!IS_ERR(btf_resolve_size(desc_btf, ptr_type, &sz))) { - meta.r0_size = sz; + meta.ret_mem.found = true; + meta.ret_mem.size = sz; meta.r0_rdonly = true; } + + if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) + meta.r0_rdonly = false; } - if (!meta.r0_size) { + if (!meta.ret_mem.found) { ptr_type_name = btf_name_by_offset(desc_btf, ptr_type->name_off); verbose(env, @@ -13224,7 +13231,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_MEM; - regs[BPF_REG_0].mem_size = meta.r0_size; + regs[BPF_REG_0].mem_size = meta.ret_mem.size; if (meta.r0_rdonly) regs[BPF_REG_0].type |= MEM_RDONLY; -- cgit v1.2.3 From d55149ff8c856c88cedc36557dadc6b5363b428d Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Tue, 14 Jul 2026 23:40:46 -0700 Subject: bpf: Drop redundant pkt_access from bpf_call_arg_meta meta->pkt_access is only ever a copy of fn->pkt_access, assigned once in check_helper_call() and read back in may_access_direct_pkt_data(). Have may_access_direct_pkt_data() take the bpf_func_proto and read fn->pkt_access directly, and drop the meta field along with its assignment. The only non-NULL caller, check_func_arg(), already has fn in scope; the remaining callers pass NULL and are unaffected. No functional change intended. Suggested-by: Eduard Zingerman Signed-off-by: Amery Hung Link: https://lore.kernel.org/bpf/20260715064047.1793790-7-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 463b49df4ff9..37127ae1ff28 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -258,7 +258,6 @@ struct bpf_call_arg_meta { struct ref_obj_desc ref_obj; struct arg_raw_mem_desc arg_raw_mem; struct ret_mem_desc ret_mem; - bool pkt_access; u8 release_regno; u64 msize_max_value; int func_id; @@ -4673,7 +4672,7 @@ static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state * } static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, - const struct bpf_call_arg_meta *meta, + const struct bpf_func_proto *fn, enum bpf_access_type t) { enum bpf_prog_type prog_type = resolve_prog_type(env->prog); @@ -4697,8 +4696,8 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, case BPF_PROG_TYPE_LWT_XMIT: case BPF_PROG_TYPE_SK_SKB: case BPF_PROG_TYPE_SK_MSG: - if (meta) - return meta->pkt_access; + if (fn) + return fn->pkt_access; env->seen_direct_write = true; return true; @@ -8332,7 +8331,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, } if (type_is_pkt_pointer(type) && - !may_access_direct_pkt_data(env, meta, BPF_READ)) { + !may_access_direct_pkt_data(env, fn, BPF_READ)) { verbose(env, "helper access to the packet is not allowed\n"); return -EACCES; } @@ -10285,7 +10284,6 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } memset(&meta, 0, sizeof(meta)); - meta.pkt_access = fn->pkt_access; err = check_func_proto(fn, &meta); if (err) { -- cgit v1.2.3 From bf9c1b911f4db6fa5fe088c32f1de7ee1650eee9 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Tue, 14 Jul 2026 23:40:47 -0700 Subject: bpf: Unify helper and kfunc call argument meta Helper and kfunc argument checking carried two separate meta structs: the verifier-local struct bpf_call_arg_meta and bpf_kfunc_call_arg_meta. Merge them into a single struct bpf_call_arg_meta. This is groundwork for sharing argument checking between helpers and kfuncs. While merging, drop the btf_id field from the helper meta since it is never used. No functional change. Signed-off-by: Amery Hung Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260715064047.1793790-8-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/cfg.c | 2 +- kernel/bpf/verifier.c | 94 +++++++++++++++++++++------------------------------ 2 files changed, 39 insertions(+), 57 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 26d37066465f..db3416a7c904 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -491,7 +491,7 @@ static int visit_insn(int t, struct bpf_verifier_env *env) return ret; } } else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { - struct bpf_kfunc_call_arg_meta meta; + struct bpf_call_arg_meta meta; ret = bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta); if (ret == 0 && bpf_is_iter_next_kfunc(&meta)) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 37127ae1ff28..de816063ae63 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -252,24 +252,6 @@ static int validate_ref_obj(struct bpf_verifier_env *env, struct ref_obj_desc *r return 0; } -struct bpf_call_arg_meta { - struct bpf_map_desc map; - struct bpf_dynptr_desc dynptr; - struct ref_obj_desc ref_obj; - struct arg_raw_mem_desc arg_raw_mem; - struct ret_mem_desc ret_mem; - u8 release_regno; - u64 msize_max_value; - int func_id; - struct btf *btf; - u32 btf_id; - struct btf *ret_btf; - u32 ret_btf_id; - u32 subprogno; - struct btf_field *kptr_field; - s64 const_map_key; -}; - struct bpf_kfunc_meta { struct btf *btf; const struct btf_type *proto; @@ -927,10 +909,10 @@ static void __mark_reg_known_zero(struct bpf_reg_state *reg); static bool in_rcu_cs(struct bpf_verifier_env *env); -static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta); +static bool is_kfunc_rcu_protected(struct bpf_call_arg_meta *meta); static int mark_stack_slots_iter(struct bpf_verifier_env *env, - struct bpf_kfunc_call_arg_meta *meta, + struct bpf_call_arg_meta *meta, struct bpf_reg_state *reg, int insn_idx, struct btf *btf, u32 btf_id, int nr_slots) { @@ -1063,7 +1045,7 @@ static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx); static int release_irq_state(struct bpf_verifier_state *state, int id); static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env, - struct bpf_kfunc_call_arg_meta *meta, + struct bpf_call_arg_meta *meta, struct bpf_reg_state *reg, int insn_idx, int kfunc_class) { @@ -7245,7 +7227,7 @@ static int process_timer_helper(struct bpf_verifier_env *env, struct bpf_reg_sta } static int process_timer_kfunc(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { return process_timer_func(env, reg, argno, &meta->map); } @@ -7412,23 +7394,23 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat return err; } -static bool is_iter_kfunc(struct bpf_kfunc_call_arg_meta *meta) +static bool is_iter_kfunc(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY); } -static bool is_iter_new_kfunc(struct bpf_kfunc_call_arg_meta *meta) +static bool is_iter_new_kfunc(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ITER_NEW; } -static bool is_iter_destroy_kfunc(struct bpf_kfunc_call_arg_meta *meta) +static bool is_iter_destroy_kfunc(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ITER_DESTROY; } -static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg_idx, +static bool is_kfunc_arg_iter(struct bpf_call_arg_meta *meta, int arg_idx, const struct btf_param *arg) { /* btf_check_iter_kfuncs() guarantees that first argument of any iter @@ -7442,7 +7424,7 @@ static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg_idx, } static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { struct bpf_func_state *state = bpf_func(env, reg); const struct btf_type *t; @@ -7609,7 +7591,7 @@ static int widen_imprecise_scalars(struct bpf_verifier_env *env, } static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_st, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { int iter_frameno = meta->iter.frameno; int iter_spi = meta->iter.spi; @@ -7696,7 +7678,7 @@ static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_ * bpf_iter_num_destroy(&it); */ static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; struct bpf_func_state *cur_fr = cur_st->frame[cur_st->curframe], *queued_fr; @@ -10750,27 +10732,27 @@ static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, return __mark_btf_func_reg_size(env, cur_regs(env), regno, reg_size); } -static bool is_kfunc_acquire(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ACQUIRE; } -static bool is_kfunc_release(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_release(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_RELEASE; } -static bool is_kfunc_destructive(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_destructive(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_DESTRUCTIVE; } -static bool is_kfunc_rcu(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_rcu(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_RCU; } -static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_rcu_protected(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_RCU_PROTECTED; } @@ -10991,7 +10973,7 @@ static bool is_kfunc_arg_prog_aux(const struct btf *btf, const struct btf_param * To determine whether an argument is implicit, we compare its position * against the number of arguments in the prototype w/o implicit args. */ -static bool is_kfunc_arg_implicit(const struct bpf_kfunc_call_arg_meta *meta, u32 arg_idx) +static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_idx) { const struct btf_type *func, *func_proto; u32 argn; @@ -11290,7 +11272,7 @@ static bool is_task_work_add_kfunc(u32 func_id) func_id == special_kfunc_list[KF_bpf_task_work_schedule_resume]; } -static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_ret_null(struct bpf_call_arg_meta *meta) { if (is_bpf_refcount_acquire_kfunc(meta->func_id) && meta->arg_owning_ref) return false; @@ -11298,34 +11280,34 @@ static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta) return meta->kfunc_flags & KF_RET_NULL; } -static bool is_kfunc_bpf_rcu_read_lock(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_bpf_rcu_read_lock(struct bpf_call_arg_meta *meta) { return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock]; } -static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_call_arg_meta *meta) { return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock]; } -static bool is_kfunc_bpf_preempt_disable(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_bpf_preempt_disable(struct bpf_call_arg_meta *meta) { return meta->func_id == special_kfunc_list[KF_bpf_preempt_disable]; } -static bool is_kfunc_bpf_preempt_enable(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_bpf_preempt_enable(struct bpf_call_arg_meta *meta) { return meta->func_id == special_kfunc_list[KF_bpf_preempt_enable]; } -bool bpf_is_kfunc_pkt_changing(struct bpf_kfunc_call_arg_meta *meta) +bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) { return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data]; } static enum kfunc_ptr_arg_type get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_func_state *caller, - struct bpf_reg_state *regs, struct bpf_kfunc_call_arg_meta *meta, + struct bpf_reg_state *regs, struct bpf_call_arg_meta *meta, const struct btf_type *t, const struct btf_type *ref_t, const char *ref_tname, const struct btf_param *args, int arg, int nargs, argno_t argno, struct bpf_reg_state *reg) @@ -11431,7 +11413,7 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, const struct btf_type *ref_t, const char *ref_tname, u32 ref_id, - struct bpf_kfunc_call_arg_meta *meta, + struct bpf_call_arg_meta *meta, int arg, argno_t argno) { const struct btf_type *reg_ref_t; @@ -11501,7 +11483,7 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, } static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { int err, spi, kfunc_class = IRQ_NATIVE_KFUNC; bool irq_save; @@ -11826,7 +11808,7 @@ static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env, static int __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta, + struct bpf_call_arg_meta *meta, enum btf_field_type head_field_type, struct btf_field **head_field) { @@ -11876,7 +11858,7 @@ __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_LIST_HEAD, &meta->arg_list_head.field); @@ -11884,7 +11866,7 @@ static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_RB_ROOT, &meta->arg_rbtree_root.field); @@ -11893,7 +11875,7 @@ static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, static int __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta, + struct bpf_call_arg_meta *meta, enum btf_field_type head_field_type, enum btf_field_type node_field_type, struct btf_field **node_field) @@ -11958,7 +11940,7 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta, BPF_LIST_HEAD, BPF_LIST_NODE, @@ -11967,7 +11949,7 @@ static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta, BPF_RB_ROOT, BPF_RB_NODE, @@ -11996,7 +11978,7 @@ static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env) } } -static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, +static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, int insn_idx) { const char *func_name = meta->func_name, *ref_tname; @@ -12575,7 +12557,7 @@ check_ok: int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, s32 func_id, s16 offset, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { struct bpf_kfunc_meta kfunc; int err; @@ -12734,7 +12716,7 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * int arg, int insn_idx) { struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; - struct bpf_kfunc_call_arg_meta meta; + struct bpf_call_arg_meta meta; const struct btf_param *args; const struct btf_type *t, *ref_t; const struct btf *btf; @@ -12795,7 +12777,7 @@ out: * 0 - fall-through to 'else' branch * < 0 - not fall-through to 'else' branch, return error */ -static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, +static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, struct bpf_reg_state *regs, struct bpf_insn_aux_data *insn_aux, const struct btf_type *ptr_type, struct btf *desc_btf) { @@ -12974,7 +12956,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_reg_state *regs = cur_regs(env); const char *func_name, *ptr_type_name; const struct btf_type *t, *ptr_type; - struct bpf_kfunc_call_arg_meta meta; + struct bpf_call_arg_meta meta; struct bpf_insn_aux_data *insn_aux; int err, insn_idx = *insn_idx_p; const struct btf_param *args; @@ -16728,7 +16710,7 @@ bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, struct bpf_call_summary *cs) { - struct bpf_kfunc_call_arg_meta meta; + struct bpf_call_arg_meta meta; const struct bpf_func_proto *fn; int i; -- cgit v1.2.3 From 3513ea9dab6c1a3d2dc8e6160c41f690206948b6 Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Thu, 16 Jul 2026 12:01:55 +0000 Subject: bpf: Sync tail_call_reachable with callee state on entry Currently in check_max_stack_depth_subprog, when the verifier enters a new callee branch, the local tail_call_reachable is not properly synchronized with the callee's state. Consider a main prog branching into multiple subprogs: subprog0 -> tailcall main < subprog1 -> subprog2 When the verifier finishes checking subprog0 and backtracks to main prog, the local tail_call_reachable state is left as true. As it proceeds to subprog1, this uncleared state leaks into the new branch, falsely marking subprog1 and subprog2 as tailcall reachable. Fix this by explicitly syncing tail_call_reachable with the callee's has_tail_call state on entry. The caller's state is safely preserved and restored via the existing backtracking logic. Fixes: ebf7d1f508a7 ("bpf, x64: rework pro/epilogue and tailcall handling in JIT") Reported-by: Sashiko Signed-off-by: Pu Lehui Link: https://patch.msgid.link/20260716120157.835937-2-pulehui@huaweicloud.com Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index de816063ae63..782d939c38cd 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5263,8 +5263,8 @@ continue_func: if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK; - if (subprog[idx].has_tail_call) - tail_call_reachable = true; + /* sync tail_call_reachable with callee state on entry */ + tail_call_reachable = subprog[idx].has_tail_call; frame = bpf_subprog_is_global(env, idx) ? 0 : frame + 1; if (frame >= MAX_CALL_FRAMES) { -- cgit v1.2.3 From a41d0c30d764e086c62b049e806fd12df4f4acfc Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Thu, 16 Jul 2026 12:01:56 +0000 Subject: bpf: Reject callback subprogs invoke tailcall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some JIT compilers, such as x86_64, rely on a register to pass the TCC. When subprograms of synchronous callback invoke tailcall, C helpers invoking bpf callback clobber this register, and the corrupted TCC may bypass the TCC limit, leading to infinite tailcall. Fix this by rejecting tailcall inside all subprogs of sync callback. This also cleanly consolidates the existing async and exception callback checks into a single unified `is_cb` check. Reported-by: Sashiko Reported-by: Björn Töpel Signed-off-by: Pu Lehui Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260716120157.835937-3-pulehui@huaweicloud.com Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 782d939c38cd..62d46b4c9962 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5237,10 +5237,6 @@ continue_func: if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn)) return -EFAULT; if (subprog[sidx].is_async_cb) { - if (subprog[sidx].has_tail_call) { - verifier_bug(env, "subprog has tail_call and async cb"); - return -EFAULT; - } /* async callbacks don't increase bpf prog stack size unless called directly */ if (!bpf_pseudo_call(insn + i)) continue; @@ -5281,8 +5277,8 @@ continue_func: */ if (tail_call_reachable) { for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { - if (subprog[tmp].is_exception_cb) { - verbose(env, "cannot tail call within exception cb\n"); + if (subprog[tmp].is_cb) { + verbose(env, "cannot tail call within callback\n"); return -EINVAL; } if (subprog[tmp].stack_arg_cnt) { -- cgit v1.2.3 From 918787e8f569d225c968af2c783962ae069b8ac8 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Wed, 15 Jul 2026 10:21:26 -0700 Subject: bpf: Disable raw mode for bloom filter map_peek For a bloom filter, the value argument of bpf_map_peek_elem() is always an input. Therefore, the verifier should not allow passing uninitialized stack memory to it to avoid information leak. bpf_map_peek_elem() tags its value argument ARG_PTR_TO_MAP_VALUE | MEM_UNINIT, telling the verifier the callee fills the buffer. This holds for queue/stack maps, but not for a bloom filter, which reads the buffer as an input to test set membership and never writes it. As a result, a program can pass an uninitialized stack buffer to bpf_map_peek_elem() on a bloom filter. The verifier accepts it and marks the buffer initialized on return, letting the program read back leftover kernel stack memory. Bloom maps require CAP_BPF to create, so this is a CAP_BPF-gated stack infoleak that bypasses the boundary CAP_BPF is meant to enforce (arbitrary kernel reads are gated behind CAP_PERFMON). Signed-off-by: Amery Hung Acked-by: Kumar Kartikeya Dwivedi Link: https://patch.msgid.link/20260715172127.2416388-2-ameryhung@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 62d46b4c9962..828a220647d6 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8418,6 +8418,15 @@ skip_type_check: verifier_bug(env, "invalid map_ptr to access map->value"); return -EFAULT; } + + /* + * Disable raw mode for bpf_map_peek_elem() on a bloom filter. The helper reads + * the value buffer as an input rather than filling it. + */ + if (meta->func_id == BPF_FUNC_map_peek_elem && + meta->map.ptr->map_type == BPF_MAP_TYPE_BLOOM_FILTER) + meta->arg_raw_mem.regno = 0; + err = check_helper_mem_access(env, reg, argno, meta->map.ptr->value_size, arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); -- cgit v1.2.3 From 79c9dc93fcae5e52bd1b4f96e138604d844ad759 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Wed, 15 Jul 2026 10:21:27 -0700 Subject: bpf: Zero kfunc arg meta before error paths can read it check_kfunc_call() reads meta.func_name when bpf_fetch_kfunc_arg_meta() returns -EACCES, but that error can come from fetch_kfunc_meta() (e.g. fd_array_get_btf() rejecting BTF binding for a signed program) before meta is memset(), leaving it uninitialized and risking a garbage deref in verbose(). Move the memset() to the start of bpf_fetch_kfunc_arg_meta() so meta is zeroed on every error return. The intended "not allowed" -EACCES path still sets func_name first, so its message is unchanged. Signed-off-by: Amery Hung Acked-by: Kumar Kartikeya Dwivedi Link: https://patch.msgid.link/20260715172127.2416388-3-ameryhung@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 828a220647d6..a78cdabf8560 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -12567,11 +12567,12 @@ int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, struct bpf_kfunc_meta kfunc; int err; + memset(meta, 0, sizeof(*meta)); + err = fetch_kfunc_meta(env, func_id, offset, &kfunc); if (err) return err; - memset(meta, 0, sizeof(*meta)); meta->btf = kfunc.btf; meta->func_id = kfunc.id; meta->func_proto = kfunc.proto; -- cgit v1.2.3 From f8248ac8f044ad3e79279cf3355412456bf416b0 Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Fri, 17 Jul 2026 19:37:01 +0800 Subject: bpf: Pass arena instead of scratch_page to the pte callbacks Replace the scratch_page field in the pte-callback data with the arena pointer; later patches use other arena fields from these callbacks. No functional change. Signed-off-by: Jiayuan Chen Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260717114117.350851-2-jiayuan.chen@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/arena.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 80b7b8a69446..8dbc24460890 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -143,14 +143,14 @@ static long compute_pgoff(struct bpf_arena *arena, long uaddr) } struct apply_range_data { + struct bpf_arena *arena; struct page **pages; - struct page *scratch_page; int i; }; struct clear_range_data { + struct bpf_arena *arena; struct llist_head *free_pages; - struct page *scratch_page; }; static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) @@ -180,7 +180,7 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) if (pte_none(old)) continue; - if (WARN_ON_ONCE(pte_page(old) != d->scratch_page)) + if (WARN_ON_ONCE(pte_page(old) != d->arena->scratch_page)) return -EBUSY; ptep_get_and_clear(&init_mm, addr, pte); flush_tlb_before_set(addr); @@ -227,7 +227,7 @@ static int apply_range_clear_cb(pte_t *pte, unsigned long addr, void *data) * scratches its PTE. A later bpf_arena_free_pages() over that range walks * here. Without the skip, scratch_page would be freed. */ - if (page == d->scratch_page) + if (page == d->arena->scratch_page) return 0; __llist_add(&page->pcp_llist, d->free_pages); @@ -506,8 +506,7 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) if (ret) goto out_sigsegv_memcg; - struct apply_range_data data = { .pages = &page, .i = 0, - .scratch_page = arena->scratch_page }; + struct apply_range_data data = { .arena = arena, .pages = &page, .i = 0 }; /* Account into memcg of the process that created bpf_arena */ ret = bpf_map_alloc_pages(map, NUMA_NO_NODE, 1, &page); if (ret) { @@ -696,8 +695,8 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt bpf_map_memcg_exit(old_memcg, new_memcg); return 0; } + data.arena = arena; data.pages = pages; - data.scratch_page = arena->scratch_page; if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) goto out_free_pages; @@ -873,8 +872,8 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, range_tree_set(&arena->rt, pgoff, page_cnt); init_llist_head(&free_pages); + cdata.arena = arena; cdata.free_pages = &free_pages; - cdata.scratch_page = arena->scratch_page; /* clear ptes and collect struct pages */ apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT, apply_range_clear_cb, &cdata); @@ -981,8 +980,8 @@ static void arena_free_worker(struct work_struct *work) bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); init_llist_head(&free_pages); + cdata.arena = arena; cdata.free_pages = &free_pages; - cdata.scratch_page = arena->scratch_page; arena_vm_start = bpf_arena_get_kern_vm_start(arena); user_vm_start = bpf_arena_get_user_vm_start(arena); -- cgit v1.2.3 From 89318afb141437817a88fe3e5d8f6638c6ab3ed3 Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Fri, 17 Jul 2026 19:37:02 +0800 Subject: bpf: Add memory usage for arena arena is the only map type whose map_mem_usage() still returns 0, so "bpftool map show" and fdinfo always showed 0 memlock for an arena no matter how many pages it had. Count the pages that are actually mapped into the arena: bump a counter in apply_range_set_cb() when a page goes in and drop it in apply_range_clear_cb() when a page goes out, both under the arena spinlock. map_mem_usage() then just returns nr_pages << PAGE_SHIFT. Only real data pages are counted, not the scratch page. Signed-off-by: Jiayuan Chen Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260717114117.350851-3-jiayuan.chen@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/arena.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 8dbc24460890..f046e878f7ae 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -55,8 +55,10 @@ struct bpf_arena { struct vm_struct *kern_vm; struct page *scratch_page; struct range_tree rt; - /* protects rt */ + /* protects rt and nr_pages */ rqspinlock_t spinlock; + /* number of pages currently populated in the arena */ + u64 nr_pages; struct list_head vma_list; /* protects vma_list */ struct mutex lock; @@ -196,6 +198,7 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) set_pte_at(&init_mm, addr, pte, pteval); #endif d->i++; + WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages + 1); return 0; } @@ -231,6 +234,7 @@ static int apply_range_clear_cb(pte_t *pte, unsigned long addr, void *data) return 0; __llist_add(&page->pcp_llist, d->free_pages); + WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages - 1); return 0; } @@ -413,7 +417,9 @@ static int arena_map_check_btf(struct bpf_map *map, const struct btf *btf, static u64 arena_map_mem_usage(const struct bpf_map *map) { - return 0; + struct bpf_arena *arena = container_of(map, struct bpf_arena, map); + + return (u64)READ_ONCE(arena->nr_pages) << PAGE_SHIFT; } struct vma_list { -- cgit v1.2.3 From b5a71cb2db6d84ac0042549dcec266b18429d41e Mon Sep 17 00:00:00 2001 From: Yiyang Chen Date: Fri, 17 Jul 2026 12:53:47 +0000 Subject: bpf: Reject arena frees below the arena base bpf_arena_free_pages() accepts scalar arena addresses. The runtime masks the address to the low 32 bits and reconstructs a full user address from the arena base before returning the range to the arena free tree. When the scalar value is below the low 32 bits of the arena base, full_uaddr falls below user_vm_start. The existing upper-end clipping then turns this into an out-of-range free-tree offset. A later allocation can reuse that offset and return an address below the arena mapping. Reject such frees before computing the clipped range. Fixes: 317460317a02a ("bpf: Introduce bpf_arena.") Signed-off-by: Yiyang Chen Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260717-c10-031-public-bpf-next-v2-b4-v2-1-54b555443a7c@mails.tsinghua.edu.cn Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/arena.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index f046e878f7ae..34f023a537fe 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -858,6 +858,8 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, uaddr &= PAGE_MASK; kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr; full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + if (full_uaddr < arena->user_vm_start) + return; uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); if (full_uaddr >= uaddr_end) return; -- cgit v1.2.3 From 34746b5a84ec37c0ea2bf6808c65c5ed8790eb51 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Wed, 15 Jul 2026 22:11:20 +0800 Subject: bpf: Disallow interpreter fallback for arena-related insns Since the interpreter does not support the arena-related insns, interpreter fallback should not be allowed for these insns in core.c::__bpf_prog_select_runtime(). Currently, when the interpreter executes the arena ST/LDX/STX insns, it would hit the BUG_ON() in ___bpf_prog_run() at run time. [ 2.579196] BPF interpreter: unknown opcode a2 (imm: 0x0) [ 2.579998] ------------[ cut here ]------------ [ 2.580652] kernel BUG at kernel/bpf/core.c:2349! [ 2.581314] Oops: invalid opcode: 0000 [#1] SMP PTI Set jit_required as true when arena map is used in the prog to disallow interpreter fallback for arena-related insns. Fixes: 6082b6c328b5 ("bpf: Recognize addr_space_cast instruction in the verifier.") Signed-off-by: Leon Hwang Link: https://lore.kernel.org/bpf/20260715141122.15783-2-leon.hwang@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 4446f0bde88b..2d323f13da19 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17887,6 +17887,7 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, return -EOPNOTSUPP; } env->prog->aux->arena = (void *)map; + env->prog->jit_required = true; if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) { verbose(env, "arena's user address must be set via map_extra or mmap()\n"); return -EINVAL; -- cgit v1.2.3 From 905f716362e1186c1a23447ca279e6d21f795cdb Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Wed, 15 Jul 2026 22:11:21 +0800 Subject: bpf: Disallow interpreter fallback for gotox insn The interpreter does not recognize the BPF_JMP|BPF_JA|BPF_X insn, which is used for insn_array map. Thereafter, it would hit the BUG_ON() in ___bpf_prog_run() at run time. [ 2.563726] BPF interpreter: unknown opcode 0d (imm: 0x0) [ 2.564557] ------------[ cut here ]------------ [ 2.565206] kernel BUG at kernel/bpf/core.c:2349! [ 2.565882] Oops: invalid opcode: 0000 [#1] SMP PTI Set jit_required as true when insn_array map is used in the prog in order to disallow interpreter fallback for gotox insn in core.c::__bpf_prog_select_runtime(). Fixes: 493d9e0d6083 ("bpf, x86: add support for indirect jumps") Signed-off-by: Leon Hwang Link: https://lore.kernel.org/bpf/20260715141122.15783-3-leon.hwang@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2d323f13da19..52be0a118cce 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17941,6 +17941,7 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) return err; } env->insn_array_maps[env->insn_array_map_cnt++] = map; + env->prog->jit_required = true; } return env->used_map_cnt - 1; -- cgit v1.2.3 From 7a0855e73757ee9cf25ba635a1c735018ecba742 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Wed, 15 Jul 2026 22:11:22 +0800 Subject: bpf: Disallow interpreter fallback for BPF_ADDR_PERCPU insn The BPF_MOV64_PERCPU_REG insn requires JIT to emit native code to for 'dst_reg = src_reg + '. However, the interpreter ignores the 'off' at its ALU64_MOV_X label. The 'off' indicates the insn is BPF_MOV64_PERCPU_REG insn. Then, when the interpreter loads memory from the register, it will hit a page fault. [ 2.545572] BUG: unable to handle page fault for address: ffffffffacaaf034 [ 2.546485] #PF: supervisor read access in kernel mode [ 2.547167] #PF: error_code(0x0000) - not-present page [ 2.547850] PGD 134e63067 P4D 134e63067 PUD 134e64063 PMD 10021c063 PTE 800ffffeca550062 [ 2.548912] Oops: Oops: 0000 [#1] SMP PTI Set jit_required as true in order to disallow interpreter fallback in core.c::__bpf_prog_select_runtime(), if any BPF_ADDR_PERCPU insn is patched to the prog. BTW, rename the helper bpf_map_supports_cpu_flags() to bpf_map_is_percpu_map(). Fixes: 7bdbf7446305 ("bpf: add special internal-only MOV instruction to resolve per-CPU addrs") Signed-off-by: Leon Hwang Link: https://lore.kernel.org/bpf/20260715141122.15783-4-leon.hwang@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/fixups.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index d3be972714b2..a0bddada7964 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -2008,6 +2008,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) return -EFAULT; } + if (bpf_map_is_percpu_map(map_ptr->map_type)) + prog->jit_required = true; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -2112,6 +2115,7 @@ patch_map_ops_generic: * way, it's fine to back out this inlining logic */ #ifdef CONFIG_SMP + prog->jit_required = true; insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&cpu_number); insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0); @@ -2133,6 +2137,7 @@ patch_map_ops_generic: /* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */ if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) && bpf_verifier_inlines_helper_call(env, insn->imm)) { + prog->jit_required = true; insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)¤t_task); insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); -- cgit v1.2.3 From 7ac6e1ae41a09f1dd4baeeff1d028ae49ee01232 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 19 Jul 2026 14:54:18 +0200 Subject: bpf: Zero queue and stack outputs on lock failure Queue and stack pop/peek helpers accept an uninitialized output buffer because the verifier expects the helper to initialize it. The empty-map error path clears the buffer, but a failed lock acquisition returns -EBUSY without writing it. Clear the output before returning -EBUSY so BPF programs cannot observe uninitialized stack contents after a failed helper call. Fixes: a34a9f1a19af ("bpf: Avoid deadlock when using queue and stack maps from NMI") Signed-off-by: Kumar Kartikeya Dwivedi Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260719125419.1782196-1-memxor@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/queue_stack_maps.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c index 9a5f94371e50..c1c9dee4dcdd 100644 --- a/kernel/bpf/queue_stack_maps.c +++ b/kernel/bpf/queue_stack_maps.c @@ -99,8 +99,10 @@ static long __queue_map_get(struct bpf_map *map, void *value, bool delete) int err = 0; void *ptr; - if (raw_res_spin_lock_irqsave(&qs->lock, flags)) + if (raw_res_spin_lock_irqsave(&qs->lock, flags)) { + memset(value, 0, qs->map.value_size); return -EBUSY; + } if (queue_stack_map_is_empty(qs)) { memset(value, 0, qs->map.value_size); @@ -130,8 +132,10 @@ static long __stack_map_get(struct bpf_map *map, void *value, bool delete) void *ptr; u32 index; - if (raw_res_spin_lock_irqsave(&qs->lock, flags)) + if (raw_res_spin_lock_irqsave(&qs->lock, flags)) { + memset(value, 0, qs->map.value_size); return -EBUSY; + } if (queue_stack_map_is_empty(qs)) { memset(value, 0, qs->map.value_size); -- cgit v1.2.3 From 04e19012efaec2bfd8c3b37fd8a6c3f1fe731ffc Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 19 Jul 2026 17:36:28 +0200 Subject: bpf: Fix offset warn check for bpf_res_spin_lock Sashiko pointed out correctly that the case statement for BPF_RES_SPIN_LOCK incorrectly checks offset for BPF_SPIN_LOCK. Fix it by checking res_spin_lock_off instead. Fixes: 0de2046137f9 ("bpf: Implement verifier support for rqspinlock") Signed-off-by: Kumar Kartikeya Dwivedi Link: https://patch.msgid.link/20260719153634.2908692-2-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/btf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index cbb1e49b9bcb..e7d4e9ba24e2 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -4168,7 +4168,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type rec->spin_lock_off = rec->fields[i].offset; break; case BPF_RES_SPIN_LOCK: - WARN_ON_ONCE(rec->spin_lock_off >= 0); + WARN_ON_ONCE(rec->res_spin_lock_off >= 0); /* Cache offset for faster lookup at runtime */ rec->res_spin_lock_off = rec->fields[i].offset; break; -- cgit v1.2.3 From f08619f060468076e4acbdc10e0713af20d60e65 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 19 Jul 2026 17:36:29 +0200 Subject: bpf: Preserve unique-field state across nested structs btf_find_struct_field() initializes a fresh seen mask for every recursive descent. Unique special fields in different levels of the same aggregate therefore do not see one another. The duplicate fields can reach btf_parse_fields(), where they trigger an invariant WARN_ON_ONCE(). A crafted user BTF can consequently trigger the warning before map creation checks capabilities. Initialize the seen mask once in btf_find_field() and pass the same pointer through struct, datasec, and nested-struct walks. This gives the entire field traversal one shared uniqueness state. Fixes: 64e8ee814819 ("bpf: look into the types of the fields of a struct type recursively.") Signed-off-by: Kumar Kartikeya Dwivedi Link: https://patch.msgid.link/20260719153634.2908692-3-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/btf.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index e7d4e9ba24e2..c577f00e9d88 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3751,7 +3751,7 @@ static int btf_repeat_fields(struct btf_field_info *info, int info_cnt, static int btf_find_struct_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt, - u32 level); + u32 level, u32 *seen_mask); /* Find special fields in the struct type of a field. * @@ -3762,7 +3762,7 @@ static int btf_find_struct_field(const struct btf *btf, static int btf_find_nested_struct(const struct btf *btf, const struct btf_type *t, u32 off, u32 nelems, u32 field_mask, struct btf_field_info *info, - int info_cnt, u32 level) + int info_cnt, u32 level, u32 *seen_mask) { int ret, err, i; @@ -3770,7 +3770,7 @@ static int btf_find_nested_struct(const struct btf *btf, const struct btf_type * if (level >= MAX_RESOLVE_DEPTH) return -E2BIG; - ret = btf_find_struct_field(btf, t, field_mask, info, info_cnt, level); + ret = btf_find_struct_field(btf, t, field_mask, info, info_cnt, level, seen_mask); if (ret <= 0) return ret; @@ -3827,7 +3827,7 @@ static int btf_find_field_one(const struct btf *btf, if (expected_size && expected_size != sz * nelems) return 0; ret = btf_find_nested_struct(btf, var_type, off, nelems, field_mask, - &info[0], info_cnt, level); + &info[0], info_cnt, level, seen_mask); return ret; } @@ -3892,11 +3892,11 @@ static int btf_find_field_one(const struct btf *btf, static int btf_find_struct_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt, - u32 level) + u32 level, u32 *seen_mask) { int ret, idx = 0; const struct btf_member *member; - u32 i, off, seen_mask = 0; + u32 i, off; for_each_member(i, t, member) { const struct btf_type *member_type = btf_type_by_id(btf, @@ -3910,7 +3910,7 @@ static int btf_find_struct_field(const struct btf *btf, ret = btf_find_field_one(btf, t, member_type, i, off, 0, - field_mask, &seen_mask, + field_mask, seen_mask, &info[idx], info_cnt - idx, level); if (ret < 0) return ret; @@ -3921,11 +3921,11 @@ static int btf_find_struct_field(const struct btf *btf, static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, - int info_cnt, u32 level) + int info_cnt, u32 level, u32 *seen_mask) { int ret, idx = 0; const struct btf_var_secinfo *vsi; - u32 i, off, seen_mask = 0; + u32 i, off; for_each_vsi(i, t, vsi) { const struct btf_type *var = btf_type_by_id(btf, vsi->type); @@ -3933,7 +3933,7 @@ static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, off = vsi->offset; ret = btf_find_field_one(btf, var, var_type, -1, off, vsi->size, - field_mask, &seen_mask, + field_mask, seen_mask, &info[idx], info_cnt - idx, level); if (ret < 0) @@ -3947,10 +3947,12 @@ static int btf_find_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt) { + u32 seen_mask = 0; + if (__btf_type_is_struct(t)) - return btf_find_struct_field(btf, t, field_mask, info, info_cnt, 0); + return btf_find_struct_field(btf, t, field_mask, info, info_cnt, 0, &seen_mask); else if (btf_type_is_datasec(t)) - return btf_find_datasec_var(btf, t, field_mask, info, info_cnt, 0); + return btf_find_datasec_var(btf, t, field_mask, info, info_cnt, 0, &seen_mask); return -EINVAL; } -- cgit v1.2.3 From 61e655391cb19c31f94ecd4354f624c81ce4cf75 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 19 Jul 2026 17:36:30 +0200 Subject: bpf: Mark bpf_refcount field as unique BPF_REFCOUNT is not marked as a unique field, while it should be. Fix this oversight. Fixes: d54730b50bae ("bpf: Introduce opaque bpf_refcount struct and add btf_record plumbing") Signed-off-by: Kumar Kartikeya Dwivedi Link: https://patch.msgid.link/20260719153634.2908692-4-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/btf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index c577f00e9d88..4eeeaeb69790 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3669,7 +3669,7 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_ { BPF_LIST_NODE, "bpf_list_node", false }, { BPF_RB_ROOT, "bpf_rb_root", false }, { BPF_RB_NODE, "bpf_rb_node", false }, - { BPF_REFCOUNT, "bpf_refcount", false }, + { BPF_REFCOUNT, "bpf_refcount", true }, }; int type = 0, i; const char *name = __btf_name_by_offset(btf, var_type->name_off); -- cgit v1.2.3 From 810a273919df09b345cdee74869c030aadbaa891 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 16 Jul 2026 10:51:35 +0300 Subject: bpf: dispatcher: Allocate bpf_dispatcher->rw_image with vzalloc() bpf_dispatcher->rw_image is a temporary writable buffer that arch_prepare_bpf_dispatcher() fills and then copies into bpf_dispatcher->image using bpf_arch_text_copy(). The rel32 offsets emitted by emit_bpf_dispatcher() are calculated against ->image, so ->rw_image does not need to live in the module address range. Allocate ->rw_image with vzalloc() to avoid permissions dance when EXECMEM_BPF will be backed by ROX caches. Using vzalloc() rather than vmalloc() ensures that the memory that bpf_dispatcher_update() unconditionally copies into the executable buffer is zeroed, which is not ideal but still better than random memory returned by the existing bpf_jit_alloc_exec() or plain vmalloc(). Switching from bpf_jit_alloc_exec() to vzalloc() also saves a bit of space in the more scarce module address space. Signed-off-by: Mike Rapoport (Microsoft) Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20260716-execmem-x86-rox-bpf-v0-v3-1-4e76158c01c5@kernel.org Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/dispatcher.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index ea2d60dc1fee..79f0c222c583 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -148,7 +148,10 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero, false); if (!d->image) goto out; - d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE); + /* d->rw_image doesn't need to be in module memory range, so we + * can use vzalloc. + */ + d->rw_image = vzalloc(PAGE_SIZE); if (!d->rw_image) { bpf_prog_pack_free(d->image, PAGE_SIZE); d->image = NULL; -- cgit v1.2.3 From 4946eb5d37cb6a260b9e0ec4b812c2b104fd6ea1 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 16 Jul 2026 10:51:36 +0300 Subject: bpf: Drop __weak from bpf_jit_alloc_exec() and bpf_jit_free_exec() bpf_jit_alloc_exec() and bpf_jit_free_exec() are wrappers for the corresponding execmem APIs. Architectures define the properties of the memory range needed by BPF in their initialization of execmem and don't need to override neither of them. Drop the __weak qualifier from bpf_jit_alloc_exec() and bpf_jit_free_exec(). Signed-off-by: Mike Rapoport (Microsoft) Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20260716-execmem-x86-rox-bpf-v0-v3-2-4e76158c01c5@kernel.org Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 47fe047ad30b..fc75625dc951 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1116,12 +1116,12 @@ void bpf_jit_uncharge_modmem(u32 size) atomic_long_sub(size, &bpf_jit_current); } -void *__weak bpf_jit_alloc_exec(unsigned long size) +void *bpf_jit_alloc_exec(unsigned long size) { return execmem_alloc(EXECMEM_BPF, size); } -void __weak bpf_jit_free_exec(void *addr) +void bpf_jit_free_exec(void *addr) { execmem_free(addr); } -- cgit v1.2.3 From 7516714947da93eed4d7bbb88b3781c38233f2c6 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 16 Jul 2026 10:51:37 +0300 Subject: bpf: alloc_prog_pack(): Skip ROX management for already ROX memory execmem_alloc() can return ROX memory that is already filled with architecture defined trapping instructions. In preparation for enabling this mode for BPF on x86, make sure that there is no redundant management of the ROX memory. There is no need to fill allocated memory with trapping instructions, to request permissions reset on free and to set ROX permissions as this all is handled by execmem_alloc(). Add bpf_jit_mem_is_rox() wrapper for execmem_is_rox(), use it to check if execmem_alloc() returns ROX memory and skip the redundant steps in that case. Signed-off-by: Mike Rapoport (Microsoft) Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20260716-execmem-x86-rox-bpf-v0-v3-3-4e76158c01c5@kernel.org Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/core.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index fc75625dc951..1b89c18cf246 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -916,6 +916,11 @@ static LIST_HEAD(pack_list); #define BPF_PROG_CHUNK_COUNT (BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE) +static bool bpf_jit_mem_is_rox(void) +{ + return execmem_is_rox(EXECMEM_BPF); +} + static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns) { struct bpf_prog_pack *pack; @@ -927,16 +932,18 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins pack->ptr = bpf_jit_alloc_exec(BPF_PROG_PACK_SIZE); if (!pack->ptr) goto out; - bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE); bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE); if (static_branch_unlikely(&bpf_pred_flush_enabled)) pack->arch_flush_needed = true; - set_vm_flush_reset_perms(pack->ptr); - err = set_memory_rox((unsigned long)pack->ptr, - BPF_PROG_PACK_SIZE / PAGE_SIZE); - if (err) - goto out; + if (!bpf_jit_mem_is_rox()) { + bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE); + set_vm_flush_reset_perms(pack->ptr); + err = set_memory_rox((unsigned long)pack->ptr, + BPF_PROG_PACK_SIZE / PAGE_SIZE); + if (err) + goto out; + } list_add_tail(&pack->list, &pack_list); return pack; @@ -965,7 +972,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n"); size = round_up(size, PAGE_SIZE); ptr = bpf_jit_alloc_exec(size); - if (ptr) { + if (ptr && !bpf_jit_mem_is_rox()) { int err; bpf_fill_ill_insns(ptr, size); -- cgit v1.2.3 From 5bf02dbf39fa0ec64661827fa4a1b4d5c1d942c0 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 16 Jul 2026 10:51:38 +0300 Subject: bpf, x86: Make sure allocation in arch_bpf_trampoline_size() is writable arch_bpf_trampoline_size() allocates a buffer to get actual size required for a trampoline. This buffer must be in the module address space because __arch_prepare_bpf_trampoline() calculates rel32 offsets relatively to that buffer. In preparation for enabling ROX mode for EXECMEM_BPF make sure that the allocated memory is writable. Add bpf_jit_alloc_exec_rw() wrapper for execmem_alloc_rw() and use it for buffer allocation in arch_bpf_trampoline_size(). Signed-off-by: Mike Rapoport (Microsoft) Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20260716-execmem-x86-rox-bpf-v0-v3-4-4e76158c01c5@kernel.org Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/core.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 1b89c18cf246..e2076667b245 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1128,6 +1128,11 @@ void *bpf_jit_alloc_exec(unsigned long size) return execmem_alloc(EXECMEM_BPF, size); } +void *bpf_jit_alloc_exec_rw(unsigned long size) +{ + return execmem_alloc_rw(EXECMEM_BPF, size); +} + void bpf_jit_free_exec(void *addr) { execmem_free(addr); -- cgit v1.2.3 From 2805abd089576799b15092949420e3f8ba97fabd Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Fri, 24 Jul 2026 08:52:06 -0700 Subject: bpf: Fix CFI mismatch in task work callback BPF subprograms use the bpf_callback_t ABI, but task work invokes the callback through a three-argument function pointer. This trips kCFI. Store and invoke the callback as bpf_callback_t. Fixes: 38aa7003e369 ("bpf: task work scheduling kfuncs") Signed-off-by: Mykyta Yatsenko Link: https://lore.kernel.org/bpf/20260724-task_work_cfi-v1-1-2616691781ed@meta.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/helpers.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index c18f1e16edee..88b38db47de9 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -4388,7 +4388,7 @@ struct bpf_task_work_ctx { struct bpf_map *map; void *map_val; enum task_work_notify_mode mode; - bpf_task_work_callback_t callback_fn; + bpf_callback_t callback_fn; struct rcu_head rcu; } __aligned(8); @@ -4471,7 +4471,8 @@ static void bpf_task_work_callback(struct callback_head *cb) key = (void *)map_key_from_value(ctx->map, ctx->map_val, &idx); migrate_disable(); - ctx->callback_fn(ctx->map, key, ctx->map_val); + ctx->callback_fn((u64)(long)ctx->map, (u64)(long)key, + (u64)(long)ctx->map_val, 0, 0); migrate_enable(); bpf_task_work_ctx_reset(ctx); @@ -4594,7 +4595,7 @@ static struct bpf_task_work_ctx *bpf_task_work_acquire_ctx(struct bpf_task_work } static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work *tw, - struct bpf_map *map, bpf_task_work_callback_t callback_fn, + struct bpf_map *map, void *callback_fn, struct bpf_prog_aux *aux, enum task_work_notify_mode mode) { struct bpf_prog *prog; @@ -4619,7 +4620,7 @@ static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work } ctx->task = task; - ctx->callback_fn = callback_fn; + ctx->callback_fn = (bpf_callback_t)callback_fn; ctx->prog = prog; ctx->mode = mode; ctx->map = map; -- cgit v1.2.3 From 61aaa8782bec59ecffd22e030f54ef9351bcabf9 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Wed, 22 Jul 2026 23:19:08 +0800 Subject: bpf: Fix WARNING in bpf_tracing_link_release The trampoline could be corrupted by the blindly 'tr->flags = BPF_TRAMP_F_TAIL_CALL_CTX' in verifier. 1. A fexit attached to a tail_call_reachable prog. 'tr->flags' became 'BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_TAIL_CALL_CTX'. And, the trampoline would poke the target prog's nop insn using jmp insn instead of call insn. 2. Another fexit loaded with the same tail_call_reachable prog target. 'tr->flags' became 'BPF_TRAMP_F_TAIL_CALL_CTX'. 3. Close the first fexit link. Due to no BPF_TRAMP_F_CALL_ORIG in 'tr->flags', the trampoline will fail to restore the prog's nop insn using call insn. [ 3.410719] WARNING: kernel/bpf/syscall.c:3551 at bpf_tracing_link_release+0x53/0x60, CPU#1: test_progs/98 ... [ 3.428793] bpf_link_free+0x58/0x130 [ 3.429293] bpf_link_release+0x23/0x30 Fix the warning by updating 'tr->flags' with '|=' and lock. Fixes: 2b5dcb31a19a ("bpf, x64: Fix tailcall infinite loop") Signed-off-by: Leon Hwang Reviewed-by: Pu Lehui Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/20260722151909.69142-2-leon.hwang@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/trampoline.c | 7 +++++++ kernel/bpf/verifier.c | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 6eadf64f7ec9..129d07db117e 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -670,6 +670,13 @@ out: return ERR_PTR(err); } +void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) +{ + trampoline_lock(tr); + tr->flags |= flags; + trampoline_unlock(tr); +} + static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex, const struct bpf_trampoline_ops *ops, void *data) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 52be0a118cce..66d8d9eaec05 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -19523,7 +19523,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) return -ENOMEM; if (tgt_prog && tgt_prog->aux->tail_call_reachable) - tr->flags = BPF_TRAMP_F_TAIL_CALL_CTX; + bpf_trampoline_set_flags(tr, BPF_TRAMP_F_TAIL_CALL_CTX); prog->aux->dst_trampoline = tr; return 0; -- cgit v1.2.3 From 59b9731addc7633a39d3a042ed968864f7b3aaf5 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 19 Jul 2026 13:35:49 +0200 Subject: bpf: Allow bpf_res_spin_lock() in all contexts There is no particular reason to keep bpf_res_spin_lock() disabled in tracing programs, since it is safe against reentrancy and deadlocks. Remove the restriction for tracing programs covered by the predicate is_tracing_prog_type(). This is a prerequisite before the definition of is_tracing_prog_type() is updated to include raw_tp, fentry, fexit, and fmod_ret. Existing tracing programs will be updated to use bpf_res_spin_lock() instead when it is available. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Andrii Nakryiko Reviewed-by: Emil Tsalapatis Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260719113551.1294284-2-memxor@gmail.com --- kernel/bpf/verifier.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 641c3c62c1ec..e6f35f4e715b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17816,7 +17816,9 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); return -EINVAL; } + } + if (btf_record_has_field(map->record, BPF_SPIN_LOCK)) { if (is_tracing_prog_type(prog_type)) { verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); return -EINVAL; -- cgit v1.2.3 From 5c5997836381010fc5907b36bc17d3b19407e933 Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Tue, 28 Jul 2026 02:32:59 +0000 Subject: bpf: Fix potential UAF in bpf_netns_link_update_prog In bpf_netns_link_update_prog, the checks for old_prog and prog type are currently performed locklessly before acquiring netns_bpf_mutex. This creates a race condition that can lead to a UAF issue. If two threads concurrently execute BPF_LINK_UPDATE on the same netns link, the following execution path can trigger a UAF: CPU0 CPU1 bpf_netns_link_update_prog if (old_prog && old_prog != link->prog) return -EPERM; bpf_netns_link_update_prog if (old_prog && old_prog != link->prog) ... old_prog = xchg(&link->prog, new_prog); bpf_prog_put(old_prog); if (new_prog->type != link->prog->type) <-- trigger UAF Fix this by moving the old_prog and prog->type checks inside the netns_bpf_mutex critical section. Meanwhile, use guard() to simplify lock management and avoid all the goto jumping. Fixes: 7f045a49fee0 ("bpf: Add link-based BPF program attachment to network namespace") Reported-by: Sashiko Signed-off-by: Pu Lehui Signed-off-by: Andrii Nakryiko Reviewed-by: Amery Hung Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/f87b53c0-8f00-45a6-82db-8242fa9b143f@huaweicloud.com [0] Link: https://lore.kernel.org/bpf/20260728023259.2813482-1-pulehui@huaweicloud.com --- kernel/bpf/net_namespace.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/net_namespace.c b/kernel/bpf/net_namespace.c index 25f30f9edaef..81006a242618 100644 --- a/kernel/bpf/net_namespace.c +++ b/kernel/bpf/net_namespace.c @@ -171,33 +171,28 @@ static int bpf_netns_link_update_prog(struct bpf_link *link, struct net *net; int idx, ret; + guard(mutex)(&netns_bpf_mutex); + if (old_prog && old_prog != link->prog) return -EPERM; if (new_prog->type != link->prog->type) return -EINVAL; - mutex_lock(&netns_bpf_mutex); - net = net_link->net; - if (!net || !check_net(net)) { + if (!net || !check_net(net)) /* Link auto-detached or netns dying */ - ret = -ENOLINK; - goto out_unlock; - } + return -ENOLINK; run_array = rcu_dereference_protected(net->bpf.run_array[type], lockdep_is_held(&netns_bpf_mutex)); idx = link_index(net, type, net_link); ret = bpf_prog_array_update_at(run_array, idx, new_prog); if (ret) - goto out_unlock; + return ret; old_prog = xchg(&link->prog, new_prog); bpf_prog_put(old_prog); - -out_unlock: - mutex_unlock(&netns_bpf_mutex); - return ret; + return 0; } static int bpf_netns_link_fill_info(const struct bpf_link *link, -- cgit v1.2.3 From 863f3ddd0b8ac65abfb50d3be0869268ac0e277b Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Tue, 28 Jul 2026 02:54:57 +0000 Subject: bpf: Fix potential UAF when reading bpf link info In bpf_link_show_fdinfo and bpf_link_get_info_by_fd, link->prog is accessed without holding any locks. If the prog is concurrently replaced via bpf_link_update, the old prog can be freed, leading to a potential UAF issue. Fix this by accessing link->prog under RCU protection to safely fetch the pointer and guarantee its lifetime while reading its fields. Fixes: 0c991ebc8c69 ("bpf: Implement bpf_prog replacement for an active bpf_cgroup_link") Reported-by: Sashiko Signed-off-by: Pu Lehui Signed-off-by: Andrii Nakryiko Reviewed-by: Emil Tsalapatis Reviewed-by: Amery Hung Acked-by: Leon Hwang Link: https://lore.kernel.org/bpf/f87b53c0-8f00-45a6-82db-8242fa9b143f@huaweicloud.com [0] Link: https://lore.kernel.org/bpf/20260728025457.2814876-1-pulehui@huaweicloud.com --- kernel/bpf/syscall.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 0ff9e3aa293d..0eb43ba76a8a 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3392,9 +3392,10 @@ static const char *bpf_link_type_strs[] = { static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) { const struct bpf_link *link = filp->private_data; - const struct bpf_prog *prog = link->prog; + const struct bpf_prog *prog; enum bpf_link_type type = link->type; char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; + u32 prog_id = 0; if (type < ARRAY_SIZE(bpf_link_type_strs) && bpf_link_type_strs[type]) { if (link->type == BPF_LINK_TYPE_KPROBE_MULTI) @@ -3411,13 +3412,20 @@ static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) } seq_printf(m, "link_id:\t%u\n", link->id); + rcu_read_lock(); + prog = READ_ONCE(link->prog); if (prog) { bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); + prog_id = prog->aux->id; + } + rcu_read_unlock(); + + if (prog) { seq_printf(m, "prog_tag:\t%s\n" "prog_id:\t%u\n", prog_tag, - prog->aux->id); + prog_id); } if (link->ops->show_fdinfo) link->ops->show_fdinfo(link, m); @@ -5456,6 +5464,7 @@ static int bpf_link_get_info_by_fd(struct file *file, { struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info); struct bpf_link_info info; + const struct bpf_prog *prog; u32 info_len = attr->info.info_len; int err; @@ -5470,8 +5479,12 @@ static int bpf_link_get_info_by_fd(struct file *file, info.type = link->type; info.id = link->id; - if (link->prog) - info.prog_id = link->prog->aux->id; + + rcu_read_lock(); + prog = READ_ONCE(link->prog); + if (prog) + info.prog_id = prog->aux->id; + rcu_read_unlock(); if (link->ops->fill_link_info) { err = link->ops->fill_link_info(link, &info); -- cgit v1.2.3 From f0e80dee4e32fd11e6ee1b714b75f681c7cafd3e Mon Sep 17 00:00:00 2001 From: Xu Xin Date: Wed, 29 Jul 2026 14:11:59 +0800 Subject: bpf: Log error code on trampoline unlink failure Replace silent WARN_ON_ONCE with WARN_ONCE that prints the actual error code from bpf_trampoline_unlink_prog(). This aids debugging of race conditions during link teardown, while keeping the warning rate limited to avoid log flooding. This will be very helpful for speeding up trouble-shooting of some crash UAF due to bpf_trampoline_unlink_prog failures. No change to unlink behavior. Signed-off-by: Xu Xin Signed-off-by: Andrii Nakryiko Reviewed-by: Emil Tsalapatis Acked-by: Leon Hwang Link: https://lore.kernel.org/bpf/20260729141159128mEJmS_aujBKr-cBu1p_UI@zte.com.cn --- kernel/bpf/syscall.c | 8 +++++--- kernel/bpf/trampoline.c | 12 ++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 0eb43ba76a8a..94091130bcc5 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3555,10 +3555,12 @@ static void bpf_tracing_link_release(struct bpf_link *link) { struct bpf_tracing_link *tr_link = container_of(link, struct bpf_tracing_link, link.link); + int err; - WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link.node, - tr_link->trampoline, - tr_link->tgt_prog)); + err = bpf_trampoline_unlink_prog(&tr_link->link.node, + tr_link->trampoline, + tr_link->tgt_prog); + WARN_ONCE(err, "bpf_trampoline_unlink_prog failed: %d\n", err); bpf_trampoline_put(tr_link->trampoline); diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 129d07db117e..ed7999ad6c66 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -1004,12 +1004,15 @@ static void bpf_shim_tramp_link_release(struct bpf_link *link) { struct bpf_shim_tramp_link *shim_link = container_of(link, struct bpf_shim_tramp_link, link.link); + int err; /* paired with 'shim_link->trampoline = tr' in bpf_trampoline_link_cgroup_shim */ if (!shim_link->trampoline) return; - WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link.node, shim_link->trampoline, NULL)); + err = bpf_trampoline_unlink_prog(&shim_link->link.node, shim_link->trampoline, NULL); + WARN_ONCE(err, "bpf_trampoline_unlink_prog failed: %d\n", err); + bpf_trampoline_put(shim_link->trampoline); } @@ -1720,15 +1723,16 @@ int bpf_trampoline_multi_detach(struct bpf_prog *prog, struct bpf_tracing_multi_ { struct bpf_tracing_multi_data *data = &link->data; struct bpf_tracing_multi_node *mnode; - int i; + int i, err; trampoline_lock_all(); for_each_mnode(mnode, link) { data->entry = &mnode->entry; bpf_trampoline_multi_attach_init(mnode->trampoline); - WARN_ON_ONCE(__bpf_trampoline_unlink_prog(&mnode->node, mnode->trampoline, - NULL, &trampoline_multi_ops, data)); + err = __bpf_trampoline_unlink_prog(&mnode->node, mnode->trampoline, NULL, + &trampoline_multi_ops, data); + WARN_ONCE(err, "__bpf_trampoline_unlink_prog failed: %d\n", err); } if (ftrace_hash_count(data->unreg)) -- cgit v1.2.3 From c48796aa6c392cde93946e5d5a9a1f1b1cf72feb Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 28 Jul 2026 22:01:59 -0700 Subject: bpf: Reject >8 byte return values on return-reading trampoline paths btf_distill_func_proto() builds the function model used for the fentry/fexit/fmod_ret/fsession trampolines and struct_ops. It has accepted a 16-byte __int128 return value since the trampoline was introduced: __get_type_size() returns the integer's type size, and the return-type check only rejected ret < 0. But the BPF trampoline preserves only 8 bytes of the return value (RAX on x86, i.e. R0). For an attach type that reads the target's return value the second half (RDX / R3) is neither saved nor restored, so a program attached to a function returning a 16-byte value corrupts the value seen by the real caller and itself observes only half of it. struct_ops trampolines have the same limitation. This affects the attach types that read the target's return value: fexit, fmod_ret and fsession (plus the _multi variants of fexit and fsession), and struct_ops. fentry/fentry_multi run before the target returns and are unaffected. Reject a >8 byte return value for these attach types in bpf_check_attach_target() and bpf_check_attach_btf_id_multi(), and for struct_ops in bpf_struct_ops_desc_init(). Fixes: fec56f5890d9 ("bpf: Introduce BPF trampoline") Signed-off-by: Yonghong Song Reviewed-by: Eduard Zingerman Acked-by: Leon Hwang Link: https://lore.kernel.org/bpf/20260729050159.2585809-1-yonghong.song@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/bpf_struct_ops.c | 12 ++++++++++++ kernel/bpf/verifier.c | 25 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 51b16e5f5534..4e7a48c02be5 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -445,6 +445,18 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, goto errout; } + /* + * A >8 byte return value is passed back in a register pair, + * which the struct_ops trampoline does not preserve (only + * 8 bytes of the return value are saved and restored). + */ + if (st_ops->func_models[i].ret_size > 8) { + pr_warn("func ptr %s in struct %s has a >8 byte return value, which is not supported\n", + mname, st_ops->name); + err = -EOPNOTSUPP; + goto errout; + } + stub_func_addr = *(void **)(st_ops->cfi_stubs + moff); err = prepare_arg_info(btf, st_ops->name, mname, func_proto, stub_func_addr, diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e6f35f4e715b..8d0635ee48c7 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -19027,6 +19027,20 @@ btf_attach_func_proto(struct bpf_verifier_log *log, struct btf *btf, u32 func_id return btf_type_by_id(btf, func->type); } +static bool attach_uses_trampoline_retval(enum bpf_attach_type type) +{ + switch (type) { + case BPF_MODIFY_RETURN: + case BPF_TRACE_FEXIT: + case BPF_TRACE_FEXIT_MULTI: + case BPF_TRACE_FSESSION: + case BPF_TRACE_FSESSION_MULTI: + return true; + default: + return false; + } +} + int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, @@ -19291,6 +19305,14 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, if (ret < 0) return ret; + if (tgt_info->fmodel.ret_size > 8 && + attach_uses_trampoline_retval(prog->expected_attach_type)) { + bpf_log(log, + "Attach to function %s with a >8 byte return value is not supported for this attach type\n", + tname); + return -EOPNOTSUPP; + } + /* * *.multi programs don't need an address during program * verification, we just take the module ref if needed. @@ -19565,6 +19587,9 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt err = btf_distill_func_proto(NULL, btf, t, tname, &tgt_info->fmodel); if (err < 0) return err; + if (tgt_info->fmodel.ret_size > 8 && + attach_uses_trampoline_retval(prog->expected_attach_type)) + return -EOPNOTSUPP; if (btf_is_module(btf)) { /* The bpf program already holds reference to module. */ if (WARN_ON_ONCE(!prog->aux->mod)) -- cgit v1.2.3 From 70a841617aa8f228fc9de3b82cd1bdc4cb249497 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:16 -0700 Subject: bpf: Drop process_timer_func wrappers Drop process_timer_{helper,kfunc}() since bpf_call_arg_meta is now shared by helper and kfunc. Call process_timer_func() directly. Signed-off-by: Amery Hung Reviewed-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260801074633.1595644-2-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8d0635ee48c7..616194f25dfb 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7224,18 +7224,6 @@ static int process_timer_func(struct bpf_verifier_env *env, struct bpf_reg_state return check_map_field_pointer(env, reg, argno, BPF_TIMER, map); } -static int process_timer_helper(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_call_arg_meta *meta) -{ - return process_timer_func(env, reg, argno, &meta->map); -} - -static int process_timer_kfunc(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_call_arg_meta *meta) -{ - return process_timer_func(env, reg, argno, &meta->map); -} - static int process_kptr_func(struct bpf_verifier_env *env, int regno, struct bpf_call_arg_meta *meta) { @@ -8466,7 +8454,7 @@ skip_type_check: } break; case ARG_PTR_TO_TIMER: - err = process_timer_helper(env, reg, argno, meta); + err = process_timer_func(env, reg, argno, &meta->map); if (err) return err; break; @@ -12515,7 +12503,7 @@ check_ok: reg_arg_name(env, argno)); return -EINVAL; } - ret = process_timer_kfunc(env, reg, argno, meta); + ret = process_timer_func(env, reg, argno, &meta->map); if (ret < 0) return ret; break; -- cgit v1.2.3 From b33d09b4d9141586706b6a99ce5b2f189adc7cdd Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:17 -0700 Subject: bpf: Unify const map ptr argument checking for helpers and kfuncs Both the helper ARG_CONST_MAP_PTR and the kfunc KF_ARG_PTR_TO_MAP recorded the map pointer in meta->map and, when a map was already bound by a preceding timer/workqueue/task_work argument, rejected a mismatching map. Factor the logic into a single process_map_ptr_arg() used by both paths. The bound-object name (timer, workqueue, or bpf_task_work) is derived from the bound map's btf_record, and the register numbers in the message are computed from the map argument position instead of being hard-coded. Signed-off-by: Amery Hung Reviewed-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260801074633.1595644-3-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 97 +++++++++++++++++++++++---------------------------- 1 file changed, 44 insertions(+), 53 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 616194f25dfb..4a5b54219210 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8274,6 +8274,44 @@ static int get_constant_map_key(struct bpf_verifier_env *env, static bool can_elide_value_nullness(const struct bpf_map *map); +static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, struct bpf_call_arg_meta *meta) +{ + /* Use map_uid (which is unique id of inner map) to reject: + * inner_map1 = bpf_map_lookup_elem(outer_map, key1) + * inner_map2 = bpf_map_lookup_elem(outer_map, key2) + * if (inner_map1 && inner_map2) { + * timer = bpf_map_lookup_elem(inner_map1); + * if (timer) + * // mismatch would have been allowed + * bpf_timer_init(timer, inner_map2); + * } + * + * Comparing map_ptr is enough to distinguish normal and outer maps. + */ + if (meta->map.ptr && + (meta->map.ptr != reg->map_ptr || meta->map.uid != reg->map_uid)) { + argno_t obj_argno = argno_from_reg(reg_from_argno(argno) - 1); + struct btf_record *rec = meta->map.ptr->record; + const char *obj_name = "workqueue"; + + if (rec->timer_off >= 0) + obj_name = "timer"; + else if (rec->task_work_off >= 0) + obj_name = "bpf_task_work"; + + verbose(env, "%s pointer in %s map_uid=%d ", + obj_name, reg_arg_name(env, obj_argno), meta->map.uid); + verbose(env, "doesn't match map pointer in %s map_uid=%d\n", + reg_arg_name(env, argno), reg->map_uid); + return -EINVAL; + } + + meta->map.ptr = reg->map_ptr; + meta->map.uid = reg->map_uid; + return 0; +} + static int check_func_arg(struct bpf_verifier_env *env, u32 arg, struct bpf_call_arg_meta *meta, const struct bpf_func_proto *fn, @@ -8349,29 +8387,9 @@ skip_type_check: switch (base_type(arg_type)) { case ARG_CONST_MAP_PTR: /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ - if (meta->map.ptr) { - /* Use map_uid (which is unique id of inner map) to reject: - * inner_map1 = bpf_map_lookup_elem(outer_map, key1) - * inner_map2 = bpf_map_lookup_elem(outer_map, key2) - * if (inner_map1 && inner_map2) { - * timer = bpf_map_lookup_elem(inner_map1); - * if (timer) - * // mismatch would have been allowed - * bpf_timer_init(timer, inner_map2); - * } - * - * Comparing map_ptr is enough to distinguish normal and outer maps. - */ - if (meta->map.ptr != reg->map_ptr || - meta->map.uid != reg->map_uid) { - verbose(env, - "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", - meta->map.uid, reg->map_uid); - return -EINVAL; - } - } - meta->map.ptr = reg->map_ptr; - meta->map.uid = reg->map_uid; + err = process_map_ptr_arg(env, reg, argno, meta); + if (err) + return err; break; case ARG_PTR_TO_MAP_KEY: /* bpf_map_xxx(..., map_ptr, ..., key) call: @@ -12123,36 +12141,9 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me reg_arg_name(env, argno)); return -EINVAL; } - if (meta->map.ptr && (reg->map_ptr->record->wq_off >= 0 || - reg->map_ptr->record->task_work_off >= 0)) { - /* Use map_uid (which is unique id of inner map) to reject: - * inner_map1 = bpf_map_lookup_elem(outer_map, key1) - * inner_map2 = bpf_map_lookup_elem(outer_map, key2) - * if (inner_map1 && inner_map2) { - * wq = bpf_map_lookup_elem(inner_map1); - * if (wq) - * // mismatch would have been allowed - * bpf_wq_init(wq, inner_map2); - * } - * - * Comparing map_ptr is enough to distinguish normal and outer maps. - */ - if (meta->map.ptr != reg->map_ptr || - meta->map.uid != reg->map_uid) { - if (reg->map_ptr->record->task_work_off >= 0) { - verbose(env, - "bpf_task_work pointer in R2 map_uid=%d doesn't match map pointer in R3 map_uid=%d\n", - meta->map.uid, reg->map_uid); - return -EINVAL; - } - verbose(env, - "workqueue pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", - meta->map.uid, reg->map_uid); - return -EINVAL; - } - } - meta->map.ptr = reg->map_ptr; - meta->map.uid = reg->map_uid; + ret = process_map_ptr_arg(env, reg, argno, meta); + if (ret < 0) + return ret; fallthrough; case KF_ARG_PTR_TO_ALLOC_BTF_ID: case KF_ARG_PTR_TO_BTF_ID: -- cgit v1.2.3 From c82b998777b7102f3617a46201805b7e7b899524 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:18 -0700 Subject: bpf: Split kfunc map argument into __const_map and __map Kfuncs used a single '__map' suffix (KF_ARG_PTR_TO_MAP) for two different things: a verifier-known map matched by map_uid against a bound timer/wq/task_work object (bpf_wq_init, bpf_task_work_schedule*), and an opaque 'struct bpf_map *' used only at runtime (bpf_arena_*), which may be a map fd or a PTR_TO_BTF_ID struct bpf_map (e.g. a bpf_map iterator's ctx->map). That combined path only accepted the btf map form due to type confusion. The 'if (!reg->map_ptr)' check reads reg->map_ptr, which aliases reg->btf in the bpf_reg_state union. A PTR_TO_BTF_ID register always has a non-NULL reg->btf, so the guard silently passed and validation fell through to process_kf_arg_ptr_to_btf_id(). It also recorded PTR_TO_BTF_ID info in meta->map, which would be meaningless. Split the annotation to avoid such type confusion and to align with helper: - '__const_map' -> KF_ARG_CONST_MAP_PTR: verifier-known map, handled by process_map_ptr_arg() like helper ARG_CONST_MAP_PTR. - '__map' -> KF_ARG_PTR_TO_BTF_ID: opaque struct bpf_map, validated by process_kf_arg_ptr_to_btf_id(). A map fd still matches via reg2btf_ids[CONST_PTR_TO_MAP], so bpf_arena_alloc_pages(&map) keeps working. Signed-off-by: Amery Hung Reviewed-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260801074633.1595644-4-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/helpers.c | 16 ++++++++-------- kernel/bpf/verifier.c | 45 +++++++++++++++++++++++++++------------------ 2 files changed, 35 insertions(+), 26 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 88b38db47de9..e472535bce85 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -3404,10 +3404,10 @@ __bpf_kfunc void bpf_throw(u64 cookie) WARN(1, "A call to BPF exception callback should never return\n"); } -__bpf_kfunc int bpf_wq_init(struct bpf_wq *wq, void *p__map, unsigned int flags) +__bpf_kfunc int bpf_wq_init(struct bpf_wq *wq, void *p__const_map, unsigned int flags) { struct bpf_async_kern *async = (struct bpf_async_kern *)wq; - struct bpf_map *map = p__map; + struct bpf_map *map = p__const_map; BUILD_BUG_ON(sizeof(struct bpf_async_kern) > sizeof(struct bpf_wq)); BUILD_BUG_ON(__alignof__(struct bpf_async_kern) != __alignof__(struct bpf_wq)); @@ -4643,17 +4643,17 @@ release_prog: * mode * @task: Task struct for which callback should be scheduled * @tw: Pointer to struct bpf_task_work in BPF map value for internal bookkeeping - * @map__map: bpf_map that embeds struct bpf_task_work in the values + * @map__const_map: bpf_map that embeds struct bpf_task_work in the values * @callback: pointer to BPF subprogram to call * @aux: pointer to bpf_prog_aux of the caller BPF program, implicitly set by the verifier * * Return: 0 if task work has been scheduled successfully, negative error code otherwise */ __bpf_kfunc int bpf_task_work_schedule_signal(struct task_struct *task, struct bpf_task_work *tw, - void *map__map, bpf_task_work_callback_t callback, + void *map__const_map, bpf_task_work_callback_t callback, struct bpf_prog_aux *aux) { - return bpf_task_work_schedule(task, tw, map__map, callback, aux, TWA_SIGNAL); + return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_SIGNAL); } /** @@ -4661,17 +4661,17 @@ __bpf_kfunc int bpf_task_work_schedule_signal(struct task_struct *task, struct b * mode * @task: Task struct for which callback should be scheduled * @tw: Pointer to struct bpf_task_work in BPF map value for internal bookkeeping - * @map__map: bpf_map that embeds struct bpf_task_work in the values + * @map__const_map: bpf_map that embeds struct bpf_task_work in the values * @callback: pointer to BPF subprogram to call * @aux: pointer to bpf_prog_aux of the caller BPF program, implicitly set by the verifier * * Return: 0 if task work has been scheduled successfully, negative error code otherwise */ __bpf_kfunc int bpf_task_work_schedule_resume(struct task_struct *task, struct bpf_task_work *tw, - void *map__map, bpf_task_work_callback_t callback, + void *map__const_map, bpf_task_work_callback_t callback, struct bpf_prog_aux *aux) { - return bpf_task_work_schedule(task, tw, map__map, callback, aux, TWA_RESUME); + return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_RESUME); } static int make_file_dynptr(struct file *file, u32 flags, bool may_sleep, diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 4a5b54219210..4f39e439973c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10818,6 +10818,11 @@ static bool is_kfunc_arg_map(const struct btf *btf, const struct btf_param *arg) return btf_param_match_suffix(btf, arg, "__map"); } +static bool is_kfunc_arg_const_map(const struct btf *btf, const struct btf_param *arg) +{ + return btf_param_match_suffix(btf, arg, "__const_map"); +} + static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg) { return btf_param_match_suffix(btf, arg, "__alloc"); @@ -11064,7 +11069,7 @@ enum kfunc_ptr_arg_type { KF_ARG_PTR_TO_RB_NODE, KF_ARG_PTR_TO_NULL, KF_ARG_PTR_TO_CONST_STR, - KF_ARG_PTR_TO_MAP, + KF_ARG_CONST_MAP_PTR, KF_ARG_PTR_TO_TIMER, KF_ARG_PTR_TO_WORKQUEUE, KF_ARG_PTR_TO_IRQ_FLAG, @@ -11383,8 +11388,11 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_func_state *call if (is_kfunc_arg_const_str(meta->btf, &args[arg])) return KF_ARG_PTR_TO_CONST_STR; + if (is_kfunc_arg_const_map(meta->btf, &args[arg])) + return KF_ARG_CONST_MAP_PTR; + if (is_kfunc_arg_map(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_MAP; + return KF_ARG_PTR_TO_BTF_ID; if (is_kfunc_arg_wq(meta->btf, &args[arg])) return KF_ARG_PTR_TO_WORKQUEUE; @@ -12132,19 +12140,15 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (kf_arg_type < 0) return kf_arg_type; + if (is_kfunc_arg_map(btf, &args[i])) { + ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; + ref_t = btf_type_by_id(btf_vmlinux, ref_id); + ref_tname = btf_name_by_offset(btf, ref_t->name_off); + } + switch (kf_arg_type) { case KF_ARG_PTR_TO_NULL: continue; - case KF_ARG_PTR_TO_MAP: - if (!reg->map_ptr) { - verbose(env, "pointer in %s isn't map pointer\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - ret = process_map_ptr_arg(env, reg, argno, meta); - if (ret < 0) - return ret; - fallthrough; case KF_ARG_PTR_TO_ALLOC_BTF_ID: case KF_ARG_PTR_TO_BTF_ID: if (!is_trusted_reg(env, reg)) { @@ -12160,6 +12164,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me } } fallthrough; + case KF_ARG_CONST_MAP_PTR: case KF_ARG_PTR_TO_ITER: case KF_ARG_PTR_TO_LIST_HEAD: case KF_ARG_PTR_TO_LIST_NODE: @@ -12366,12 +12371,16 @@ check_ok: if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_MAP: - /* If argument has '__map' suffix expect 'struct bpf_map *' */ - ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; - ref_t = btf_type_by_id(btf_vmlinux, ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); - fallthrough; + case KF_ARG_CONST_MAP_PTR: + if (base_type(reg->type) != CONST_PTR_TO_MAP) { + verbose(env, "pointer in %s isn't map pointer\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + ret = process_map_ptr_arg(env, reg, argno, meta); + if (ret < 0) + return ret; + break; case KF_ARG_PTR_TO_BTF_ID: /* Only base_type is checked, further checks are done here */ if ((base_type(reg->type) != PTR_TO_BTF_ID || -- cgit v1.2.3 From 9f52714dd89b489f7de5de2b60736edde07ddf5f Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:19 -0700 Subject: bpf: Pass kfunc meta to mem and mem_size check kfunc now shares the same bpf_call_arg_meta with helpers. Pass kfunc's own meta to check_mem_reg() and check_kfunc_mem_size() instead of NULL or a temporary meta on the stack. Signed-off-by: Amery Hung Reviewed-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260801074633.1595644-5-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 4f39e439973c..3fec0afacb3f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6924,7 +6924,7 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, } static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - argno_t argno, u32 mem_size) + argno_t argno, u32 mem_size, struct bpf_call_arg_meta *meta) { bool may_be_null = type_may_be_null(reg->type); struct bpf_reg_state saved_reg; @@ -6950,8 +6950,8 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg int size = base_type(reg->type) == PTR_TO_STACK ? -(int)mem_size : mem_size; - err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, NULL); - err = err ?: check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, NULL); + err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta); + err = err ?: check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, meta); if (may_be_null) *reg = saved_reg; @@ -6994,22 +6994,20 @@ static int process_const_alloc_mem_size(struct bpf_verifier_env *env, struct bpf } static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, - struct bpf_reg_state *size_reg, argno_t mem_argno, argno_t size_argno) + struct bpf_reg_state *size_reg, argno_t mem_argno, + argno_t size_argno, struct bpf_call_arg_meta *meta) { bool may_be_null = type_may_be_null(mem_reg->type); struct bpf_reg_state saved_reg; - struct bpf_call_arg_meta meta; int err; - memset(&meta, 0, sizeof(meta)); - if (may_be_null) { saved_reg = *mem_reg; mark_ptr_not_null_reg(mem_reg); } - err = check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_READ, true, &meta); - err = err ?: check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_WRITE, true, &meta); + err = check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_READ, true, meta); + err = err ?: check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_WRITE, true, meta); if (may_be_null) *mem_reg = saved_reg; @@ -9258,7 +9256,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE); if (ret < 0) return ret; - if (check_mem_reg(env, reg, argno, arg->mem_size)) + if (check_mem_reg(env, reg, argno, arg->mem_size, NULL)) return -EINVAL; if (!(arg->arg_type & PTR_MAYBE_NULL) && (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { @@ -12405,7 +12403,7 @@ check_ok: ref_tname, PTR_ERR(resolve_ret)); return -EINVAL; } - ret = check_mem_reg(env, reg, argno, type_size); + ret = check_mem_reg(env, reg, argno, type_size, meta); if (ret < 0) return ret; break; @@ -12419,7 +12417,7 @@ check_ok: if (!bpf_register_is_null(buff_reg) || !is_kfunc_arg_nullable(meta->btf, buff_arg)) { ret = check_kfunc_mem_size_reg(env, buff_reg, size_reg, - argno, next_argno); + argno, next_argno, meta); if (ret < 0) { verbose(env, "%s and ", reg_arg_name(env, argno)); verbose(env, "%s memory, len pair leads to invalid memory access\n", -- cgit v1.2.3 From 341d227fa5db567ff2e3114b5be9b1051f336e34 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Sat, 1 Aug 2026 00:46:20 -0700 Subject: bpf: Resolve map lookup result type at lookup time bpf_map_lookup_elem() is typed to return PTR_TO_MAP_VALUE for every map, but for some map kinds the looked up value is actually a different object: an inner map, a socket or an xsk socket. Until now this reinterpretation happened once the pointer was converted from its NULL-able form to a concrete value. Such reinterpretation logic placement led to mark_ptr_not_null_reg() being called for a temporary register copy in check_mem_reg() and check_kfunc_mem_size_reg() (check_mem_size_reg() was buggy because of not calling it). The temporary copy was necessary to pass reinterpreted parameters as nullable helper and kfunc arguments. Avoid this complication by refining map lookup result type right away. The test case verifier_map_in_map/on_the_inner_map_pointer needs an update because the verifier now prints a concrete NULL-able type for the lookup. Signed-off-by: Eduard Zingerman Signed-off-by: Amery Hung Link: https://lore.kernel.org/bpf/20260801074633.1595644-6-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 79 +++++++++++++++++++-------------------------------- 1 file changed, 30 insertions(+), 49 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 3fec0afacb3f..1b6ae6e5b995 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1854,32 +1854,34 @@ static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type ty reg->dynptr.first_slot = first_slot; } -static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) +/* + * Refine the return type of the bpf_map_lookup_elem() for special map types: + * map-in-map, xskmap, sockmap and sockhash. + */ +static void refine_map_lookup_value(struct bpf_reg_state *reg) { - if (base_type(reg->type) == PTR_TO_MAP_VALUE) { - const struct bpf_map *map = reg->map_ptr; + enum bpf_type_flag maybe_null = reg->type & PTR_MAYBE_NULL; + const struct bpf_map *map = reg->map_ptr; - if (map->inner_map_meta) { - reg->type = CONST_PTR_TO_MAP; - reg->map_ptr = map->inner_map_meta; - /* transfer reg's id which is unique for every map_lookup_elem - * as UID of the inner map. - */ - if (btf_record_has_field(map->inner_map_meta->record, - BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) { - reg->map_uid = reg->id; - } - } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { - reg->type = PTR_TO_XDP_SOCK; - } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || - map->map_type == BPF_MAP_TYPE_SOCKHASH) { - reg->type = PTR_TO_SOCKET; - } else { - reg->type = PTR_TO_MAP_VALUE; - } - return; + if (map->inner_map_meta) { + reg->type = CONST_PTR_TO_MAP | maybe_null; + reg->map_ptr = map->inner_map_meta; + /* transfer reg's id which is unique for every map_lookup_elem + * as UID of the inner map. + */ + if (btf_record_has_field(map->inner_map_meta->record, + BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) + reg->map_uid = reg->id; + } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { + reg->type = PTR_TO_XDP_SOCK | maybe_null; + } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || + map->map_type == BPF_MAP_TYPE_SOCKHASH) { + reg->type = PTR_TO_SOCKET | maybe_null; } +} +static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) +{ reg->type &= ~PTR_MAYBE_NULL; } @@ -6926,8 +6928,6 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, u32 mem_size, struct bpf_call_arg_meta *meta) { - bool may_be_null = type_may_be_null(reg->type); - struct bpf_reg_state saved_reg; int err; if (bpf_register_is_null(reg)) @@ -6939,23 +6939,11 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg return -EACCES; } - /* Assuming that the register contains a value check if the memory - * access is safe. Temporarily save and restore the register's state as - * the conversion shouldn't be visible to a caller. - */ - if (may_be_null) { - saved_reg = *reg; - mark_ptr_not_null_reg(reg); - } - int size = base_type(reg->type) == PTR_TO_STACK ? -(int)mem_size : mem_size; err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta); err = err ?: check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, meta); - if (may_be_null) - *reg = saved_reg; - return err; } @@ -6997,21 +6985,11 @@ static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg struct bpf_reg_state *size_reg, argno_t mem_argno, argno_t size_argno, struct bpf_call_arg_meta *meta) { - bool may_be_null = type_may_be_null(mem_reg->type); - struct bpf_reg_state saved_reg; int err; - if (may_be_null) { - saved_reg = *mem_reg; - mark_ptr_not_null_reg(mem_reg); - } - err = check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_READ, true, meta); err = err ?: check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_WRITE, true, meta); - if (may_be_null) - *mem_reg = saved_reg; - return err; } @@ -10522,10 +10500,12 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn regs[BPF_REG_0].map_ptr = meta.map.ptr; regs[BPF_REG_0].map_uid = meta.map.uid; regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag; - if (!type_may_be_null(ret_flag) && + if (type_may_be_null(ret_flag) || btf_record_has_field(meta.map.ptr->record, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK)) { regs[BPF_REG_0].id = ++env->id_gen; } + /* requires regs[BPF_REG_0].id to be set because of the map-in-map case */ + refine_map_lookup_value(®s[BPF_REG_0]); break; case RET_PTR_TO_SOCKET: mark_reg_known_zero(env, regs, BPF_REG_0); @@ -10623,7 +10603,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return -EINVAL; } - if (type_may_be_null(regs[BPF_REG_0].type)) + if (type_may_be_null(regs[BPF_REG_0].type) && !regs[BPF_REG_0].id) regs[BPF_REG_0].id = ++env->id_gen; if (is_ptr_cast_function(func_id) && @@ -12370,7 +12350,8 @@ check_ok: return ret; break; case KF_ARG_CONST_MAP_PTR: - if (base_type(reg->type) != CONST_PTR_TO_MAP) { + if (base_type(reg->type) != CONST_PTR_TO_MAP || + type_may_be_null(reg->type)) { verbose(env, "pointer in %s isn't map pointer\n", reg_arg_name(env, argno)); return -EINVAL; -- cgit v1.2.3 From d4e7fb59c0d4c746cf95054190b7b30d91995ddc Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:21 -0700 Subject: bpf: Check helper and kfunc mem+size arguments identically Helper ARG_CONST_SIZE and kfunc KF_ARG_PTR_TO_MEM_SIZE memory arguments already share check_mem_size_reg(), but the kfunc path reached it through a thin wrapper, check_kfunc_mem_size_reg(). The wrapper existed only to invoke check_mem_size_reg() twice. Once for BPF_READ and once for BPF_WRITE because a kfunc mem argument may be both read and written, whereas a helper argument carries a single access direction. Let check_mem_size_reg() take a bitmask of access directions (widening access_type to u32) and perform each requested access, then pass BPF_READ | BPF_WRITE from the kfunc call site. This removes the check_kfunc_mem_size_reg() wrapper so helper and kfunc mem+size arguments run through exactly the same code. Signed-off-by: Amery Hung Reviewed-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260801074633.1595644-7-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 1b6ae6e5b995..f61771e2a8b2 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6871,11 +6871,11 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ static int check_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, struct bpf_reg_state *size_reg, argno_t mem_argno, - argno_t size_argno, enum bpf_access_type access_type, + argno_t size_argno, u32 access_type, bool zero_size_allowed, struct bpf_call_arg_meta *meta) { - int err; + int err = 0; /* This is used to refine r0 return value bounds for helpers * that enforce this value as an upper bound on return values. @@ -6912,8 +6912,14 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, reg_arg_name(env, size_argno)); return -EACCES; } - err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), - access_type, zero_size_allowed, meta); + + if (access_type & BPF_READ) + err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), + BPF_READ, zero_size_allowed, meta); + if (!err && access_type & BPF_WRITE) + err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), + BPF_WRITE, zero_size_allowed, meta); + if (!err) { int regno = reg_from_argno(size_argno); @@ -6922,6 +6928,7 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, else err = mark_stack_arg_precision(env, arg_idx_from_argno(size_argno)); } + return err; } @@ -6981,18 +6988,6 @@ static int process_const_alloc_mem_size(struct bpf_verifier_env *env, struct bpf return 0; } -static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, - struct bpf_reg_state *size_reg, argno_t mem_argno, - argno_t size_argno, struct bpf_call_arg_meta *meta) -{ - int err; - - err = check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_READ, true, meta); - err = err ?: check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_WRITE, true, meta); - - return err; -} - enum { PROCESS_SPIN_LOCK = (1 << 0), PROCESS_RES_LOCK = (1 << 1), @@ -12397,8 +12392,8 @@ check_ok: argno_t next_argno = argno_from_arg(i + 2); if (!bpf_register_is_null(buff_reg) || !is_kfunc_arg_nullable(meta->btf, buff_arg)) { - ret = check_kfunc_mem_size_reg(env, buff_reg, size_reg, - argno, next_argno, meta); + ret = check_mem_size_reg(env, buff_reg, size_reg, argno, next_argno, + BPF_READ | BPF_WRITE, true, meta); if (ret < 0) { verbose(env, "%s and ", reg_arg_name(env, argno)); verbose(env, "%s memory, len pair leads to invalid memory access\n", -- cgit v1.2.3 From e566701b9b0c1ec398152cb972d592992984e1a8 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:23 -0700 Subject: bpf: Check fixed-size mem args of helpers and kfuncs the same way Fixed-size memory arguments went through two paths: helpers called check_helper_mem_access() directly, while kfuncs and global subprogs used check_mem_reg(). Route the helper MEM_FIXED_SIZE case through check_mem_reg() too so all three share the same check. This also fixes a bug in the helper path. When passing a NULL to PTR_MAYBE_NULL | ARG_PTR_TO_FIXED_SIZE_MEM argument, the program would be falsely rejected by check_helper_mem_access(). This is not triggerable since there is no such kind of helper. Also, note that check_reg_type() still make sure NULL cannot be passed to an argument not marked with PTR_MAYBE_NULL. It also tightens the poisoned-stack-slot check. check_mem_reg() encoded "a STACK_POISON slot may be read" as a negative access size for any PTR_TO_STACK argument, but that is only sound for global subprogs, where static stack liveness proved the callee body does not read those slots (2cb27158adb3 ("bpf: poison dead stack slots")). Since check_mem_reg() is also used for kfuncs, kfuncs accidentally inherited it and could read a poisoned (dead, possibly uninitialized) stack slot. Restrict the negative size to global subprogs (meta == NULL) so kfuncs, like helpers, require the whole argument initialized. Signed-off-by: Amery Hung Link: https://lore.kernel.org/bpf/20260801074633.1595644-9-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index f61771e2a8b2..9e0c7f0a15c1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6669,7 +6669,7 @@ static int check_stack_range_initialized( */ bool clobber = type == BPF_WRITE; /* - * Negative access_size signals global subprog/kfunc arg check where + * Negative access_size signals global subprog arg check where * STACK_POISON slots are acceptable. static stack liveness * might have determined that subprog doesn't read them, * but BTF based global subprog validation isn't accurate enough. @@ -6933,9 +6933,10 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, } static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - argno_t argno, u32 mem_size, struct bpf_call_arg_meta *meta) + argno_t argno, u32 mem_size, enum bpf_access_type access_type, + struct bpf_call_arg_meta *meta) { - int err; + int size, err = 0; if (bpf_register_is_null(reg)) return 0; @@ -6946,10 +6947,16 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg return -EACCES; } - int size = base_type(reg->type) == PTR_TO_STACK ? -(int)mem_size : mem_size; + /* + * Only a global subprog (meta == NULL) may read poisoned stack slots: + * its static stack liveness proved the callee body skips them. + */ + size = (!meta && base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size; - err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta); - err = err ?: check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, meta); + if (access_type & BPF_READ) + err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta); + if (!err && (access_type & BPF_WRITE)) + err = check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, meta); return err; } @@ -8455,9 +8462,8 @@ skip_type_check: * next is_mem_size argument below. */ if (arg_type & MEM_FIXED_SIZE) { - err = check_helper_mem_access(env, reg, argno, fn->arg_size[arg], - arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, - false, meta); + err = check_mem_reg(env, reg, argno_from_reg(regno), fn->arg_size[arg], + arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta); if (err) return err; if (arg_type & MEM_ALIGNED) @@ -9229,7 +9235,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE); if (ret < 0) return ret; - if (check_mem_reg(env, reg, argno, arg->mem_size, NULL)) + if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL)) return -EINVAL; if (!(arg->arg_type & PTR_MAYBE_NULL) && (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { @@ -12379,7 +12385,7 @@ check_ok: ref_tname, PTR_ERR(resolve_ret)); return -EINVAL; } - ret = check_mem_reg(env, reg, argno, type_size, meta); + ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, meta); if (ret < 0) return ret; break; -- cgit v1.2.3 From c0e091f30f0dd80d817c3a15d0a97962957e5786 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:24 -0700 Subject: bpf: Rename ARG_CONST_SIZE{,_OR_ZERO} to ARG_MEM_SIZE{,_OR_ZERO} ARG_CONST_SIZE does not require a constant: check_mem_size_reg() accepts any bounded scalar and verifies the memory access against its maximum (reg_umax). Rename ARG_CONST_SIZE and ARG_CONST_SIZE_OR_ZERO to ARG_MEM_SIZE and ARG_MEM_SIZE_OR_ZERO to reflect that. ARG_CONST_ALLOC_ SIZE_OR_ZERO, which does require a constant, is left unchanged. Pure rename, no functional change. Signed-off-by: Amery Hung Link: https://lore.kernel.org/bpf/20260801074633.1595644-10-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/backtrack.c | 2 +- kernel/bpf/bpf_lsm.c | 4 ++-- kernel/bpf/btf.c | 2 +- kernel/bpf/cgroup.c | 8 +++---- kernel/bpf/helpers.c | 26 ++++++++++---------- kernel/bpf/ringbuf.c | 2 +- kernel/bpf/stackmap.c | 10 ++++---- kernel/bpf/syscall.c | 4 ++-- kernel/bpf/verifier.c | 11 ++++----- kernel/trace/bpf_trace.c | 62 ++++++++++++++++++++++++------------------------ 10 files changed, 65 insertions(+), 66 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 2e4ae0ef0860..2f473ad4fd7c 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -636,7 +636,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, * r5 += 1 * ... * call bpf_perf_event_output#25 - * where .arg5_type = ARG_CONST_SIZE_OR_ZERO + * where .arg5_type = ARG_MEM_SIZE_OR_ZERO * * and this case: * r6 = 1 diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 3983b4ce73c8..82c5988417a0 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -186,7 +186,7 @@ static const struct bpf_func_proto bpf_ima_inode_hash_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_ima_inode_hash_btf_ids[0], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .allowed = bpf_ima_inode_hash_allowed, }; @@ -205,7 +205,7 @@ static const struct bpf_func_proto bpf_ima_file_hash_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_ima_file_hash_btf_ids[0], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .allowed = bpf_ima_inode_hash_allowed, }; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 4eeeaeb69790..5e8ac45ce56a 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8709,7 +8709,7 @@ const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 4355ccb78a9c..fb9357b64cad 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -2305,7 +2305,7 @@ static const struct bpf_func_proto bpf_sysctl_get_name_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -2347,7 +2347,7 @@ static const struct bpf_func_proto bpf_sysctl_get_current_value_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; BPF_CALL_3(bpf_sysctl_get_new_value, struct bpf_sysctl_kern *, ctx, char *, buf, @@ -2367,7 +2367,7 @@ static const struct bpf_func_proto bpf_sysctl_get_new_value_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; BPF_CALL_3(bpf_sysctl_set_new_value, struct bpf_sysctl_kern *, ctx, @@ -2393,7 +2393,7 @@ static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; static const struct bpf_func_proto * diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index e472535bce85..4709a5ad0474 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -278,7 +278,7 @@ const struct bpf_func_proto bpf_get_current_comm_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, }; #if defined(CONFIG_QUEUED_SPINLOCKS) || defined(CONFIG_BPF_ARCH_SPINLOCK) @@ -539,7 +539,7 @@ const struct bpf_func_proto bpf_strtol_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, .arg4_size = sizeof(s64), @@ -567,7 +567,7 @@ const struct bpf_func_proto bpf_strtoul_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, .arg4_size = sizeof(u64), @@ -583,7 +583,7 @@ static const struct bpf_func_proto bpf_strncmp_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_CONST_STR, }; @@ -627,7 +627,7 @@ const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto = { .arg1_type = ARG_ANYTHING, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; static const struct bpf_func_proto bpf_get_raw_smp_processor_id_proto = { @@ -653,7 +653,7 @@ const struct bpf_func_proto bpf_event_output_data_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(bpf_copy_from_user, void *, dst, u32, size, @@ -675,7 +675,7 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -706,7 +706,7 @@ const struct bpf_func_proto bpf_copy_from_user_task_proto = { .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_BTF_ID, .arg4_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], @@ -1093,10 +1093,10 @@ const struct bpf_func_proto bpf_snprintf_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_CONST_STR, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; static void *map_key_from_value(struct bpf_map *map, void *value, u32 *arr_idx) @@ -1888,7 +1888,7 @@ static const struct bpf_func_proto bpf_dynptr_from_mem_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL | MEM_UNINIT | MEM_WRITE, }; @@ -1943,7 +1943,7 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_DYNPTR, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, @@ -2004,7 +2004,7 @@ static const struct bpf_func_proto bpf_dynptr_write_proto = { .arg1_type = ARG_PTR_TO_DYNPTR, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE_OR_ZERO, + .arg4_type = ARG_MEM_SIZE_OR_ZERO, .arg5_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index 35ae64ade36b..c1bf7a197a96 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -634,7 +634,7 @@ const struct bpf_func_proto bpf_ringbuf_output_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_CONST_MAP_PTR, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 41fe87d7302f..463f94ba1cc4 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -781,7 +781,7 @@ const struct bpf_func_proto bpf_get_stack_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -797,7 +797,7 @@ const struct bpf_func_proto bpf_get_stack_sleepable_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -831,7 +831,7 @@ const struct bpf_func_proto bpf_get_task_stack_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -848,7 +848,7 @@ const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -911,7 +911,7 @@ const struct bpf_func_proto bpf_get_stack_proto_pe = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 94091130bcc5..d6be7f49433c 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -6559,7 +6559,7 @@ static const struct bpf_func_proto bpf_sys_bpf_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; const struct bpf_func_proto * __weak @@ -6606,7 +6606,7 @@ static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, .arg4_size = sizeof(u64), diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 9e0c7f0a15c1..89e72d0a2c46 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7704,8 +7704,7 @@ static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, static bool arg_type_is_mem_size(enum bpf_arg_type type) { - return type == ARG_CONST_SIZE || - type == ARG_CONST_SIZE_OR_ZERO; + return type == ARG_MEM_SIZE || type == ARG_MEM_SIZE_OR_ZERO; } static bool arg_type_is_raw_mem(enum bpf_arg_type type) @@ -7851,8 +7850,8 @@ static const struct bpf_reg_types dynptr_types = { static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_MAP_KEY] = &mem_types, [ARG_PTR_TO_MAP_VALUE] = &mem_types, - [ARG_CONST_SIZE] = &scalar_types, - [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, + [ARG_MEM_SIZE] = &scalar_types, + [ARG_MEM_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_MAP_PTR] = &const_map_ptr_types, [ARG_PTR_TO_CTX] = &context_types, @@ -8470,13 +8469,13 @@ skip_type_check: err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true); } break; - case ARG_CONST_SIZE: + case ARG_MEM_SIZE: err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), argno, fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); break; - case ARG_CONST_SIZE_OR_ZERO: + case ARG_MEM_SIZE_OR_ZERO: err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), argno, fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 76ab51deaa6b..891897f8a1b3 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -221,7 +221,7 @@ const struct bpf_func_proto bpf_probe_read_user_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -258,7 +258,7 @@ const struct bpf_func_proto bpf_probe_read_user_str_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -273,7 +273,7 @@ const struct bpf_func_proto bpf_probe_read_kernel_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -308,7 +308,7 @@ const struct bpf_func_proto bpf_probe_read_kernel_str_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -328,7 +328,7 @@ static const struct bpf_func_proto bpf_probe_read_compat_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -347,7 +347,7 @@ static const struct bpf_func_proto bpf_probe_read_compat_str_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; #endif /* CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE */ @@ -383,7 +383,7 @@ static const struct bpf_func_proto bpf_probe_write_user_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; #define MAX_TRACE_PRINTK_VARARGS 3 @@ -418,7 +418,7 @@ static const struct bpf_func_proto bpf_trace_printk_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, }; static void __set_printk_clr_event(struct work_struct *work) @@ -474,9 +474,9 @@ static const struct bpf_func_proto bpf_trace_vprintk_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE_OR_ZERO, + .arg4_type = ARG_MEM_SIZE_OR_ZERO, }; const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void) @@ -518,9 +518,9 @@ static const struct bpf_func_proto bpf_seq_printf_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_seq_file_ids[0], .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(bpf_seq_write, struct seq_file *, m, const void *, data, u32, len) @@ -535,7 +535,7 @@ static const struct bpf_func_proto bpf_seq_write_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_seq_file_ids[0], .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_4(bpf_seq_printf_btf, struct seq_file *, m, struct btf_ptr *, ptr, @@ -559,7 +559,7 @@ static const struct bpf_func_proto bpf_seq_printf_btf_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_seq_file_ids[0], .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -633,7 +633,7 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = { .arg1_type = ARG_CONST_MAP_PTR, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void) @@ -730,7 +730,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; static DEFINE_PER_CPU(int, bpf_event_output_nest_level); @@ -996,7 +996,7 @@ static const struct bpf_func_proto bpf_d_path_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_d_path_btf_ids[0], .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .allowed = bpf_d_path_allowed, }; @@ -1053,9 +1053,9 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, .arg5_type = ARG_ANYTHING, }; @@ -1218,7 +1218,7 @@ const struct bpf_func_proto bpf_get_branch_snapshot_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(get_func_arg, void *, ctx, u32, n, u64 *, value) @@ -1421,7 +1421,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_tp = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(bpf_get_stackid_tp, void *, tp_buff, struct bpf_map *, map, @@ -1462,7 +1462,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_tp = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -1524,12 +1524,12 @@ clear: } static const struct bpf_func_proto bpf_perf_prog_read_value_proto = { - .func = bpf_perf_prog_read_value, - .gpl_only = true, - .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .func = bpf_perf_prog_read_value, + .gpl_only = true, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_PTR_TO_UNINIT_MEM, + .arg3_type = ARG_MEM_SIZE, }; BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx, @@ -1566,7 +1566,7 @@ static const struct bpf_func_proto bpf_read_branch_records_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -1646,7 +1646,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; extern const struct bpf_func_proto bpf_skb_output_proto; @@ -1701,7 +1701,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; -- cgit v1.2.3 From f16e80c2c45175664ec6b0aaad6914fadbda395d Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:25 -0700 Subject: bpf: Fold __szk const size handling into the scalar arg path To align helper and kfunc pointer to memory argument handling, move kfunc constant memorry size argument handling to the kfunc scalar section. In addition, factor out constant scalar argument handling. The constant size argument (__szk) of a kfunc memory/size pair was recorded into meta->arg_constant by a dedicated block in the KF_ARG_PTR_TO_MEM_SIZE case, duplicating the "only one constant argument" and "must be a known constant" checks already in the generic scalar argument handling. That block also did an explicit i++ to skip the size argument. This also fixes a precision gap: the old dedicated block did not mark the size register precise, relying on check_mem_size_reg() for that. But check_mem_size_reg() is skipped when the buffer is a nullable arg passed as NULL (e.g. bpf_dynptr_slice(_rdwr) with a NULL buffer), so in that case the __szk value was recorded and used for regs[R0].mem_size without marking it precise. Routing the size through the scalar path marks it precise in all cases. Signed-off-by: Amery Hung Reviewed-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260801074633.1595644-11-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 66 +++++++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 34 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 89e72d0a2c46..6fc22a4e38b2 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6995,6 +6995,35 @@ static int process_const_alloc_mem_size(struct bpf_verifier_env *env, struct bpf return 0; } +static int process_const_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, struct bpf_call_arg_meta *meta) +{ + int regno = reg_from_argno(argno); + int err; + + if (meta->arg_constant.found) { + verifier_bug(env, "only one constant argument permitted"); + return -EFAULT; + } + + if (!tnum_is_const(reg->var_off)) { + verbose(env, "%s must be a known constant\n", reg_arg_name(env, argno)); + return -EINVAL; + } + + if (regno >= 0) + err = mark_chain_precision(env, regno); + else + err = mark_stack_arg_precision(env, arg_idx_from_argno(argno)); + if (err < 0) + return err; + + meta->arg_constant.found = true; + meta->arg_constant.value = reg->var_off.value; + + return 0; +} + enum { PROCESS_SPIN_LOCK = (1 << 0), PROCESS_RES_LOCK = (1 << 1), @@ -12054,24 +12083,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me return -EINVAL; } - if (is_kfunc_arg_constant(meta->btf, &args[i])) { - if (meta->arg_constant.found) { - verifier_bug(env, "only one constant argument permitted"); - return -EFAULT; - } - if (!tnum_is_const(reg->var_off)) { - verbose(env, "%s must be a known constant\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (regno >= 0) - ret = mark_chain_precision(env, regno); - else - ret = mark_stack_arg_precision(env, i); + if (is_kfunc_arg_constant(meta->btf, &args[i]) || + is_kfunc_arg_const_mem_size(meta->btf, &args[i], reg)) { + ret = process_const_arg(env, reg, argno, meta); if (ret < 0) return ret; - meta->arg_constant.found = true; - meta->arg_constant.value = reg->var_off.value; } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) { meta->r0_rdonly = true; is_ret_buf_sz = true; @@ -12393,7 +12409,6 @@ check_ok: struct bpf_reg_state *buff_reg = reg; const struct btf_param *buff_arg = &args[i]; struct bpf_reg_state *size_reg = get_func_arg_reg(caller, regs, i + 1); - const struct btf_param *size_arg = &args[i + 1]; argno_t next_argno = argno_from_arg(i + 2); if (!bpf_register_is_null(buff_reg) || !is_kfunc_arg_nullable(meta->btf, buff_arg)) { @@ -12406,23 +12421,6 @@ check_ok: return ret; } } - - if (is_kfunc_arg_const_mem_size(meta->btf, size_arg, size_reg)) { - if (meta->arg_constant.found) { - verifier_bug(env, "only one constant argument permitted"); - return -EFAULT; - } - if (!tnum_is_const(size_reg->var_off)) { - verbose(env, "%s must be a known constant\n", - reg_arg_name(env, next_argno)); - return -EINVAL; - } - meta->arg_constant.found = true; - meta->arg_constant.value = size_reg->var_off.value; - } - - /* Skip next '__sz' or '__szk' argument */ - i++; break; } case KF_ARG_PTR_TO_CALLBACK: -- cgit v1.2.3 From ea5ab2389801ca82018024b7cfe156b050f69f1c Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:27 -0700 Subject: bpf: Classify kfunc mem_size args from BTF without register state check_kfunc_args() already makes sure a scalar value is passed to a scalar kfunc argument. Drop the check in is_kfunc_arg_mem_size() and is_kfunc_arg_const_mem_size() to further decouple get_kfunc_ptr_arg_type() from register state (a prerequisite for generating a helper-like prototype from kfunc's BTF). Signed-off-by: Amery Hung Reviewed-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260801074633.1595644-13-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6fc22a4e38b2..d65da54f9c0f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10785,26 +10785,24 @@ static bool is_kfunc_rcu_protected(struct bpf_call_arg_meta *meta) } static bool is_kfunc_arg_mem_size(const struct btf *btf, - const struct btf_param *arg, - const struct bpf_reg_state *reg) + const struct btf_param *arg) { const struct btf_type *t; t = btf_type_skip_modifiers(btf, arg->type, NULL); - if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) + if (!btf_type_is_scalar(t)) return false; return btf_param_match_suffix(btf, arg, "__sz"); } static bool is_kfunc_arg_const_mem_size(const struct btf *btf, - const struct btf_param *arg, - const struct bpf_reg_state *reg) + const struct btf_param *arg) { const struct btf_type *t; t = btf_type_skip_modifiers(btf, arg->type, NULL); - if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) + if (!btf_type_is_scalar(t)) return false; return btf_param_match_suffix(btf, arg, "__szk"); @@ -11338,7 +11336,7 @@ bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) } static enum kfunc_ptr_arg_type -get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_func_state *caller, +get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_reg_state *regs, struct bpf_call_arg_meta *meta, const struct btf_type *t, const struct btf_type *ref_t, const char *ref_tname, const struct btf_param *args, @@ -11352,8 +11350,8 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_func_state *call return KF_ARG_PTR_TO_CTX; if (arg + 1 < nargs && - (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1)) || - is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1)))) + (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || + is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) arg_mem_size = true; /* In this function, we verify the kfunc's BTF as per the argument type, @@ -12084,7 +12082,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me } if (is_kfunc_arg_constant(meta->btf, &args[i]) || - is_kfunc_arg_const_mem_size(meta->btf, &args[i], reg)) { + is_kfunc_arg_const_mem_size(meta->btf, &args[i])) { ret = process_const_arg(env, reg, argno, meta); if (ret < 0) return ret; @@ -12129,7 +12127,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); ref_tname = btf_name_by_offset(btf, ref_t->name_off); - kf_arg_type = get_kfunc_ptr_arg_type(env, caller, regs, meta, t, ref_t, ref_tname, + kf_arg_type = get_kfunc_ptr_arg_type(env, regs, meta, t, ref_t, ref_tname, args, i, nargs, argno, reg); if (kf_arg_type < 0) return kf_arg_type; -- cgit v1.2.3 From 76fb08750481049796357f7636777bdf3c2be0a8 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:28 -0700 Subject: bpf: Handle NULL kfunc pointer args without a KF_ARG_PTR_TO_NULL type get_kfunc_ptr_arg_type() returned KF_ARG_PTR_TO_NULL when a nullable pointer argument was passed a NULL register. This folded a register-state decision (bpf_register_is_null()) into what is otherwise BTF-based argument classification, and it short-circuited before the BTF_ID/MEM resolution. Drop KF_ARG_PTR_TO_NULL and handle the NULL case in check_kfunc_args() instead: a nullable argument that is actually NULL is skipped. Note that it is okay to skip even when it is a mem+size pair because the size argument check has been moved to the scalar section. The skip is done before get_kfunc_ptr_arg_type() so that a NULL passed to a nullable non-scalar-struct argument is not newly rejected by the BTF_ID/MEM resolution. No functional change. Signed-off-by: Amery Hung Reviewed-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260801074633.1595644-14-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d65da54f9c0f..bf03ba2fc04f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11072,7 +11072,6 @@ enum kfunc_ptr_arg_type { KF_ARG_PTR_TO_CALLBACK, KF_ARG_PTR_TO_RB_ROOT, KF_ARG_PTR_TO_RB_NODE, - KF_ARG_PTR_TO_NULL, KF_ARG_PTR_TO_CONST_STR, KF_ARG_CONST_MAP_PTR, KF_ARG_PTR_TO_TIMER, @@ -11349,11 +11348,6 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) return KF_ARG_PTR_TO_CTX; - if (arg + 1 < nargs && - (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || - is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) - arg_mem_size = true; - /* In this function, we verify the kfunc's BTF as per the argument type, * leaving the rest of the verification with respect to the register * type to our caller. When a set of conditions hold in the BTF type of @@ -11362,10 +11356,6 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) return KF_ARG_PTR_TO_CTX; - if (is_kfunc_arg_nullable(meta->btf, &args[arg]) && bpf_register_is_null(reg) && - !arg_mem_size) - return KF_ARG_PTR_TO_NULL; - if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) return KF_ARG_PTR_TO_ALLOC_BTF_ID; @@ -11427,6 +11417,11 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) return KF_ARG_PTR_TO_CALLBACK; + if (arg + 1 < nargs && + (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || + is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) + arg_mem_size = true; + /* This is the catch all argument type of register types supported by * check_helper_mem_access. However, we only allow when argument type is * pointer to scalar, or struct composed (recursively) of scalars. When @@ -12127,6 +12122,9 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); ref_tname = btf_name_by_offset(btf, ref_t->name_off); + if (is_kfunc_arg_nullable(meta->btf, &args[i]) && bpf_register_is_null(reg)) + continue; + kf_arg_type = get_kfunc_ptr_arg_type(env, regs, meta, t, ref_t, ref_tname, args, i, nargs, argno, reg); if (kf_arg_type < 0) @@ -12139,8 +12137,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me } switch (kf_arg_type) { - case KF_ARG_PTR_TO_NULL: - continue; case KF_ARG_PTR_TO_ALLOC_BTF_ID: case KF_ARG_PTR_TO_BTF_ID: if (!is_trusted_reg(env, reg)) { @@ -12405,19 +12401,16 @@ check_ok: case KF_ARG_PTR_TO_MEM_SIZE: { struct bpf_reg_state *buff_reg = reg; - const struct btf_param *buff_arg = &args[i]; struct bpf_reg_state *size_reg = get_func_arg_reg(caller, regs, i + 1); argno_t next_argno = argno_from_arg(i + 2); - if (!bpf_register_is_null(buff_reg) || !is_kfunc_arg_nullable(meta->btf, buff_arg)) { - ret = check_mem_size_reg(env, buff_reg, size_reg, argno, next_argno, - BPF_READ | BPF_WRITE, true, meta); - if (ret < 0) { - verbose(env, "%s and ", reg_arg_name(env, argno)); - verbose(env, "%s memory, len pair leads to invalid memory access\n", - reg_arg_name(env, next_argno)); - return ret; - } + ret = check_mem_size_reg(env, buff_reg, size_reg, argno, next_argno, + BPF_READ | BPF_WRITE, true, meta); + if (ret < 0) { + verbose(env, "%s and ", reg_arg_name(env, argno)); + verbose(env, "%s memory, len pair leads to invalid memory access\n", + reg_arg_name(env, next_argno)); + return ret; } break; } -- cgit v1.2.3 From 90990ee10be8952d55063990088b9ad4af344e34 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:29 -0700 Subject: bpf: Distinguish fixed- and variable-size kfunc mem args with MEM_FIXED_SIZE A kfunc memory-pointer argument comes in two flavors: a fixed-size buffer whose access size is derived from the pointed-to BTF type, and a variable-size buffer paired with a following __sz/__szk size argument. Both were represented by separate kfunc_ptr_arg_type values (KF_ARG_PTR_TO_MEM vs KF_ARG_PTR_TO_MEM_SIZE) with the pointer classified as the latter when a size argument followed. Mirror how helpers describe the same distinction: classify both as KF_ARG_PTR_TO_MEM and OR in MEM_FIXED_SIZE for the fixed-size case, just as helpers use ARG_PTR_TO_MEM | MEM_FIXED_SIZE. The switches now key on base_type(kf_arg_type) so the flag rides along, and the KF_ARG_PTR_TO_MEM handler either resolves the size from BTF (MEM_FIXED_SIZE) or falls through to the mem/size-pair check, which validates the buffer against the following size register and skips it. No functional change. Currently, KF_ARG_MEM_SIZE and KF_ARG_CONST_MEM_SIZE are only reachable from ARG_PTR_TO_MEM fallthrough. A patch later will merge scalar checking into the same switch and remove the fallthrough. Signed-off-by: Amery Hung Reviewed-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260801074633.1595644-15-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index bf03ba2fc04f..1ccf3b764c51 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11059,6 +11059,8 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, } enum kfunc_ptr_arg_type { + KF_ARG_CONST_MEM_SIZE, + KF_ARG_MEM_SIZE, KF_ARG_PTR_TO_CTX, KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */ @@ -11068,7 +11070,6 @@ enum kfunc_ptr_arg_type { KF_ARG_PTR_TO_LIST_NODE, KF_ARG_PTR_TO_BTF_ID, /* Also covers reg2btf_ids conversions */ KF_ARG_PTR_TO_MEM, - KF_ARG_PTR_TO_MEM_SIZE, /* Size derived from next argument, skip it */ KF_ARG_PTR_TO_CALLBACK, KF_ARG_PTR_TO_RB_ROOT, KF_ARG_PTR_TO_RB_NODE, @@ -11334,7 +11335,7 @@ bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data]; } -static enum kfunc_ptr_arg_type +static int get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_reg_state *regs, struct bpf_call_arg_meta *meta, const struct btf_type *t, const struct btf_type *ref_t, @@ -11434,7 +11435,7 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : ""); return -EINVAL; } - return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM; + return arg_mem_size ? KF_ARG_PTR_TO_MEM : KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; } static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, @@ -12136,7 +12137,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me ref_tname = btf_name_by_offset(btf, ref_t->name_off); } - switch (kf_arg_type) { + switch (base_type(kf_arg_type)) { case KF_ARG_PTR_TO_ALLOC_BTF_ID: case KF_ARG_PTR_TO_BTF_ID: if (!is_trusted_reg(env, reg)) { @@ -12159,7 +12160,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me case KF_ARG_PTR_TO_RB_ROOT: case KF_ARG_PTR_TO_RB_NODE: case KF_ARG_PTR_TO_MEM: - case KF_ARG_PTR_TO_MEM_SIZE: case KF_ARG_PTR_TO_CALLBACK: case KF_ARG_PTR_TO_CONST_STR: case KF_ARG_PTR_TO_WORKQUEUE: @@ -12190,7 +12190,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (ret < 0) return ret; - switch (kf_arg_type) { + switch (base_type(kf_arg_type)) { case KF_ARG_PTR_TO_CTX: if (reg->type != PTR_TO_CTX) { verbose(env, "%s expected pointer to ctx, but got %s\n", @@ -12387,18 +12387,22 @@ check_ok: return ret; break; case KF_ARG_PTR_TO_MEM: - resolve_ret = btf_resolve_size(btf, ref_t, &type_size); - if (IS_ERR(resolve_ret)) { - verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", - reg_arg_name(env, argno), btf_type_str(ref_t), - ref_tname, PTR_ERR(resolve_ret)); - return -EINVAL; + if (kf_arg_type & MEM_FIXED_SIZE) { + resolve_ret = btf_resolve_size(btf, ref_t, &type_size); + if (IS_ERR(resolve_ret)) { + verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), + ref_tname, PTR_ERR(resolve_ret)); + return -EINVAL; + } + ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, meta); + if (ret < 0) + return ret; + break; } - ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_MEM_SIZE: + fallthrough; + case KF_ARG_CONST_MEM_SIZE: + case KF_ARG_MEM_SIZE: { struct bpf_reg_state *buff_reg = reg; struct bpf_reg_state *size_reg = get_func_arg_reg(caller, regs, i + 1); -- cgit v1.2.3 From c9e995ba0d117119e7955f0bd3eacfb173fe636f Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:30 -0700 Subject: bpf: Classify kfunc pointer arguments from BTF, resolve type against the register get_kfunc_ptr_arg_type() decided part of a kfunc pointer argument's type from the caller's register: a PTR_TO_BTF_ID (or reg2btf_ids) register made the argument KF_ARG_PTR_TO_BTF_ID, otherwise it fell through to a memory buffer. Folding register state into argument classification prevents describing a kfunc's arguments from its BTF alone, which is a prerequisite for generating a helper-like prototype and eventually sharing the argument checking (check_func_arg()) between helpers and kfuncs. Classify pointer arguments from BTF only, and resolve them against the register in check_kfunc_args(): - A pointer to a struct that is not paired with a __sz/__szk size argument is classified KF_ARG_PTR_TO_BTF_ID and then checked against the register. A register carrying a BTF ID (PTR_TO_BTF_ID or a reg2btf_ids type) must be referenced or trusted and is matched against the expected type. The only relaxation is when the struct is composed of scalars, the register may be verified as a fixed-size memory buffer sized from the BTF type; anything else is rejected. - A pointer paired with a size argument is always a memory buffer and is never classified as BTF_ID, so the __sz/__szk case no longer detours through BTF_ID. The new design now accepts one previously rejected case: passing PTR_TO_BTF_ID to a pointer to scalar w/o a following __sz/__szk. The argument will be classified as KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE. The PTR_TO_BTF_ID register will go through check_mem_reg() -> check_helper_mem_access() -> check_ptr_to_btf_access(). For a pointer to scalar arg, a kernel btf id will be rejected unless explicitly granted by btf_struct_access(); a program allocated btf id will be allowed. The referenced-or-trusted check thus moves into the KF_ARG_PTR_TO_BTF_ID resolution, alongside the type match. get_kfunc_ptr_arg_type() no longer needs the register, so drop its regs and reg parameters; it is now a pure function of the kfunc's BTF. When a register cannot satisfy a BTF_ID argument, report the register type passed and, when the expected struct has a reg2btf_ids mapping, the register type that would be accepted, instead of a confusing "socket". Update the affected selftest messages accordingly. Signed-off-by: Amery Hung Link: https://lore.kernel.org/bpf/20260801074633.1595644-16-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 130 +++++++++++++++++++++++++++++--------------------- 1 file changed, 76 insertions(+), 54 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 1ccf3b764c51..9045369ba569 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4926,6 +4926,18 @@ static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { [CONST_PTR_TO_MAP] = btf_bpf_map_id, }; +static enum bpf_reg_type lookup_reg2btf_ids(u32 ref_id) +{ + enum bpf_reg_type type; + + for (type = 0; type < __BPF_REG_TYPE_MAX; type++) { + if (reg2btf_ids[type] && *reg2btf_ids[type] == ref_id) + return type; + } + + return NOT_INIT; +} + static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) { /* A referenced register is always trusted. */ @@ -11336,14 +11348,11 @@ bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) } static int -get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, - struct bpf_reg_state *regs, struct bpf_call_arg_meta *meta, +get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, const struct btf_type *t, const struct btf_type *ref_t, const char *ref_tname, const struct btf_param *args, - int arg, int nargs, argno_t argno, struct bpf_reg_state *reg) + int arg, int nargs, argno_t argno) { - bool arg_mem_size = false; - if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || meta->func_id == special_kfunc_list[KF_bpf_session_is_return] || meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) @@ -11405,37 +11414,37 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) return KF_ARG_PTR_TO_RES_SPIN_LOCK; - if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) { - if (!btf_type_is_struct(ref_t)) { - verbose(env, "kernel function %s %s pointer type %s %s is not supported\n", - meta->func_name, reg_arg_name(env, argno), - btf_type_str(ref_t), ref_tname); - return -EINVAL; - } - return KF_ARG_PTR_TO_BTF_ID; - } - if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) return KF_ARG_PTR_TO_CALLBACK; if (arg + 1 < nargs && (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || - is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) - arg_mem_size = true; + is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { + if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && + !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", + reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); + return -EINVAL; + } + return KF_ARG_PTR_TO_MEM; + } - /* This is the catch all argument type of register types supported by - * check_helper_mem_access. However, we only allow when argument type is - * pointer to scalar, or struct composed (recursively) of scalars. When - * arg_mem_size is true, the pointer can be void *. + /* A pointer to a struct without a size argument is classified as KF_ARG_PTR_TO_BTF_ID */ + if (btf_type_is_struct(ref_t)) + return KF_ARG_PTR_TO_BTF_ID; + + /* + * Otherwise this is a fixed-size memory buffer supported by + * check_helper_mem_access(): a pointer to a scalar or a struct of + * scalars. The access size is derived from the pointed-to BTF type. */ - if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) && - (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) { - verbose(env, "%s pointer type %s %s must point to %sscalar, or struct with scalar\n", - reg_arg_name(env, argno), - btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : ""); + if (!btf_type_is_scalar(ref_t) && + !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", + reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; } - return arg_mem_size ? KF_ARG_PTR_TO_MEM : KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; + return KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; } static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, @@ -12126,8 +12135,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (is_kfunc_arg_nullable(meta->btf, &args[i]) && bpf_register_is_null(reg)) continue; - kf_arg_type = get_kfunc_ptr_arg_type(env, regs, meta, t, ref_t, ref_tname, - args, i, nargs, argno, reg); + kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, + args, i, nargs, argno); if (kf_arg_type < 0) return kf_arg_type; @@ -12140,19 +12149,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me switch (base_type(kf_arg_type)) { case KF_ARG_PTR_TO_ALLOC_BTF_ID: case KF_ARG_PTR_TO_BTF_ID: - if (!is_trusted_reg(env, reg)) { - if (!is_kfunc_rcu(meta)) { - verbose(env, "%s must be referenced or trusted\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (!is_rcu_reg(reg)) { - verbose(env, "%s must be a rcu pointer\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } - fallthrough; case KF_ARG_CONST_MAP_PTR: case KF_ARG_PTR_TO_ITER: case KF_ARG_PTR_TO_LIST_HEAD: @@ -12372,20 +12368,46 @@ check_ok: break; case KF_ARG_PTR_TO_BTF_ID: /* Only base_type is checked, further checks are done here */ - if ((base_type(reg->type) != PTR_TO_BTF_ID || - (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) && - !reg2btf_ids[base_type(reg->type)]) { - verbose(env, "%s is %s ", reg_arg_name(env, argno), - reg_type_str(env, reg->type)); - verbose(env, "expected %s or socket\n", - reg_type_str(env, base_type(reg->type) | - (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS))); + if (base_type(reg->type) == PTR_TO_BTF_ID || + reg2btf_ids[base_type(reg->type)]) { + if (!is_trusted_reg(env, reg) || + bpf_type_has_unsafe_modifiers(reg->type)) { + if (!is_kfunc_rcu(meta)) { + verbose(env, "%s must be referenced or trusted\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (!is_rcu_reg(reg)) { + verbose(env, "%s must be a rcu pointer\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + } + + ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno); + if (ret < 0) + return ret; + break; + } + + if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); + + verbose(env, "%s is %s expected %s %s", + reg_arg_name(env, argno), reg_type_str(env, reg->type), + btf_type_str(ref_t), ref_tname); + if (reg2btf_type != NOT_INIT) + verbose(env, " or %s", reg_type_str(env, reg2btf_type)); + verbose(env, "\n"); return -EINVAL; } - ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno); - if (ret < 0) - return ret; - break; + + /* + * If the register does not contain btf id but the argument type is a pointer to + * scalar-only struct, allow verifying it as a fixed size memory. + */ + kf_arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; + fallthrough; case KF_ARG_PTR_TO_MEM: if (kf_arg_type & MEM_FIXED_SIZE) { resolve_ret = btf_resolve_size(btf, ref_t, &type_size); -- cgit v1.2.3 From ba5b99470c4019ce936226e92d4255dd8ff4dd12 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:31 -0700 Subject: bpf: Tag nullable kfunc pointer args with PTR_MAYBE_NULL Now that get_kfunc_ptr_arg_type() classifies a kfunc pointer argument from its BTF alone, express a nullable argument by OR-ing PTR_MAYBE_NULL into the classified type, and resolve a NULL register after classification instead of before it. Previously check_kfunc_args() short-circuited a nullable argument passed a NULL register with a continue placed before get_kfunc_ptr_arg_type(), so the NULL never reached classification. That kept a register-state decision (bpf_register_is_null()) ahead of the BTF-based classification. This mirrors how helper arguments carry PTR_MAYBE_NULL in their bpf_arg_type and is a step toward describing kfuncs with a bpf_func_proto: the nullability now travels with the per-argument classification, so it is captured when the prototype is generated at add-call time. Signed-off-by: Amery Hung Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260801074633.1595644-17-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 147 +++++++++++++++++++++++--------------------------- 1 file changed, 67 insertions(+), 80 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 9045369ba569..32459f25f90b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11353,98 +11353,85 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *m const char *ref_tname, const struct btf_param *args, int arg, int nargs, argno_t argno) { - if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || - meta->func_id == special_kfunc_list[KF_bpf_session_is_return] || - meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) - return KF_ARG_PTR_TO_CTX; + int arg_type; /* In this function, we verify the kfunc's BTF as per the argument type, * leaving the rest of the verification with respect to the register * type to our caller. When a set of conditions hold in the BTF type of * arguments, we resolve it to a known kfunc_ptr_arg_type. */ - if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) - return KF_ARG_PTR_TO_CTX; - - if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_ALLOC_BTF_ID; - - if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_REFCOUNTED_KPTR; - - if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_DYNPTR; - - if (is_kfunc_arg_iter(meta, arg, &args[arg])) - return KF_ARG_PTR_TO_ITER; - - if (is_kfunc_arg_list_head(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_LIST_HEAD; - - if (is_kfunc_arg_list_node(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_LIST_NODE; - - if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_RB_ROOT; - - if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_RB_NODE; - - if (is_kfunc_arg_const_str(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_CONST_STR; - - if (is_kfunc_arg_const_map(meta->btf, &args[arg])) - return KF_ARG_CONST_MAP_PTR; - - if (is_kfunc_arg_map(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_BTF_ID; - - if (is_kfunc_arg_wq(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_WORKQUEUE; - - if (is_kfunc_arg_timer(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_TIMER; - - if (is_kfunc_arg_task_work(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_TASK_WORK; - - if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_IRQ_FLAG; - - if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_RES_SPIN_LOCK; - - if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) - return KF_ARG_PTR_TO_CALLBACK; - - if (arg + 1 < nargs && - (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || - is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { + if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || + meta->func_id == special_kfunc_list[KF_bpf_session_is_return] || + meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) + arg_type = KF_ARG_PTR_TO_CTX; + else if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) + arg_type = KF_ARG_PTR_TO_CTX; + else if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_ALLOC_BTF_ID; + else if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_REFCOUNTED_KPTR; + else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_DYNPTR; + else if (is_kfunc_arg_iter(meta, arg, &args[arg])) + arg_type = KF_ARG_PTR_TO_ITER; + else if (is_kfunc_arg_list_head(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_LIST_HEAD; + else if (is_kfunc_arg_list_node(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_LIST_NODE; + else if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_RB_ROOT; + else if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_RB_NODE; + else if (is_kfunc_arg_const_str(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_CONST_STR; + else if (is_kfunc_arg_const_map(meta->btf, &args[arg])) + arg_type = KF_ARG_CONST_MAP_PTR; + else if (is_kfunc_arg_map(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_BTF_ID; + else if (is_kfunc_arg_wq(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_WORKQUEUE; + else if (is_kfunc_arg_timer(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_TIMER; + else if (is_kfunc_arg_task_work(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_TASK_WORK; + else if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_IRQ_FLAG; + else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_RES_SPIN_LOCK; + else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_CALLBACK; + else if (arg + 1 < nargs && + (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || + is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; } - return KF_ARG_PTR_TO_MEM; + arg_type = KF_ARG_PTR_TO_MEM; + } else if (btf_type_is_struct(ref_t)) + /* A pointer to a struct without a size argument is classified as KF_ARG_PTR_TO_BTF_ID */ + arg_type = KF_ARG_PTR_TO_BTF_ID; + else { + /* + * Otherwise this is a fixed-size memory buffer supported by + * check_helper_mem_access(): a pointer to a scalar or a struct of + * scalars. The access size is derived from the pointed-to BTF type. + */ + if (!btf_type_is_scalar(ref_t) && + !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", + reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); + return -EINVAL; + } + arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; } - /* A pointer to a struct without a size argument is classified as KF_ARG_PTR_TO_BTF_ID */ - if (btf_type_is_struct(ref_t)) - return KF_ARG_PTR_TO_BTF_ID; + if (is_kfunc_arg_nullable(meta->btf, &args[arg])) + arg_type |= PTR_MAYBE_NULL; - /* - * Otherwise this is a fixed-size memory buffer supported by - * check_helper_mem_access(): a pointer to a scalar or a struct of - * scalars. The access size is derived from the pointed-to BTF type. - */ - if (!btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { - verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", - reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); - return -EINVAL; - } - return KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; + return arg_type; } static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, @@ -12132,14 +12119,14 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); ref_tname = btf_name_by_offset(btf, ref_t->name_off); - if (is_kfunc_arg_nullable(meta->btf, &args[i]) && bpf_register_is_null(reg)) - continue; - kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs, argno); if (kf_arg_type < 0) return kf_arg_type; + if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) + continue; + if (is_kfunc_arg_map(btf, &args[i])) { ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; ref_t = btf_type_by_id(btf_vmlinux, ref_id); -- cgit v1.2.3 From 1690dcf27c7368d275cdcf73793aafe4d81007c9 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:32 -0700 Subject: bpf: Classify scalar kfunc arguments from BTF Add kfunc scalar argument types, classify them in get_kfunc_arg_type() along side with pointer arguments and move scalar type verification into the main switch in check_kfunc_args(). This keeps BTF-based classification separate from register validation for every argument, paving the way for generating the kfunc argument prototype at add-call time. No functional change intended. KF_ARG_MEM_SIZE and KF_ARG_CONST_MEM_SIZE now are reachable. Therefore, remove the fallthrough from KF_ARG_PTR_TO_MEM case and adjust the register indexing. Signed-off-by: Amery Hung Reviewed-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260801074633.1595644-18-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 141 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 91 insertions(+), 50 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 32459f25f90b..23a6355a38d3 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11073,6 +11073,9 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, enum kfunc_ptr_arg_type { KF_ARG_CONST_MEM_SIZE, KF_ARG_MEM_SIZE, + KF_ARG_CONST, + KF_ARG_CONST_ALLOC_SIZE_OR_ZERO, + KF_ARG_ANYTHING, KF_ARG_PTR_TO_CTX, KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */ @@ -11348,13 +11351,39 @@ bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) } static int -get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - const struct btf_type *t, const struct btf_type *ref_t, - const char *ref_tname, const struct btf_param *args, - int arg, int nargs, argno_t argno) +get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + const struct btf_param *args, int arg, int nargs) { + const struct btf_type *t, *ref_t = NULL; + argno_t argno = argno_from_arg(arg + 1); + const char *ref_tname = NULL; int arg_type; + t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); + + /* Scalar arguments are classified from their BTF suffix/name alone. */ + if (btf_type_is_scalar(t)) { + if (is_kfunc_arg_constant(meta->btf, &args[arg])) + return KF_ARG_CONST; + if (is_kfunc_arg_const_mem_size(meta->btf, &args[arg])) + return KF_ARG_CONST_MEM_SIZE; + if (is_kfunc_arg_mem_size(meta->btf, &args[arg])) + return KF_ARG_MEM_SIZE; + if (is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdonly_buf_size") || + is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdwr_buf_size")) + return KF_ARG_CONST_ALLOC_SIZE_OR_ZERO; + return KF_ARG_ANYTHING; + } + + if (!btf_type_is_ptr(t)) { + verbose(env, "Unrecognized %s type %s\n", + reg_arg_name(env, argno), btf_type_str(t)); + return -EINVAL; + } + + ref_t = btf_type_skip_modifiers(meta->btf, t->type, NULL); + ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off); + /* In this function, we verify the kfunc's BTF as per the argument type, * leaving the rest of the verification with respect to the register * type to our caller. When a set of conditions hold in the BTF type of @@ -12043,7 +12072,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me int regno = reg_from_argno(argno); bool btf_id_fixed_off_ok = true; u32 ref_id, type_size; - bool is_ret_buf_sz = false; int kf_arg_type; if (is_kfunc_arg_prog_aux(btf, &args[i])) { @@ -12067,39 +12095,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me t = btf_type_skip_modifiers(btf, args[i].type, NULL); - if (btf_type_is_scalar(t)) { - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - return -EINVAL; - } - - if (is_kfunc_arg_constant(meta->btf, &args[i]) || - is_kfunc_arg_const_mem_size(meta->btf, &args[i])) { - ret = process_const_arg(env, reg, argno, meta); - if (ret < 0) - return ret; - } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) { - meta->r0_rdonly = true; - is_ret_buf_sz = true; - } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdwr_buf_size")) { - is_ret_buf_sz = true; - } - - if (is_ret_buf_sz) { - ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); - if (ret < 0) - return ret; - } - continue; - } - - if (!btf_type_is_ptr(t)) { - verbose(env, "Unrecognized %s type %s\n", - reg_arg_name(env, argno), btf_type_str(t)); - return -EINVAL; - } - - if ((bpf_register_is_null(reg) || type_may_be_null(reg->type)) && + if (btf_type_is_ptr(t) && (bpf_register_is_null(reg) || type_may_be_null(reg->type)) && !is_kfunc_arg_nullable(meta->btf, &args[i])) { verbose(env, "Possibly NULL pointer passed to trusted %s\n", reg_arg_name(env, argno)); @@ -12116,11 +12112,12 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (reg_is_referenced(env, reg)) update_ref_obj(&meta->ref_obj, reg); - ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); + if (btf_type_is_ptr(t)) { + ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); + ref_tname = btf_name_by_offset(btf, ref_t->name_off); + } - kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, - args, i, nargs, argno); + kf_arg_type = get_kfunc_arg_type(env, meta, args, i, nargs); if (kf_arg_type < 0) return kf_arg_type; @@ -12134,6 +12131,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me } switch (base_type(kf_arg_type)) { + case KF_ARG_CONST: + case KF_ARG_CONST_MEM_SIZE: + case KF_ARG_MEM_SIZE: + case KF_ARG_ANYTHING: + case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: case KF_ARG_PTR_TO_ALLOC_BTF_ID: case KF_ARG_PTR_TO_BTF_ID: case KF_ARG_CONST_MAP_PTR: @@ -12174,6 +12176,34 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me return ret; switch (base_type(kf_arg_type)) { + case KF_ARG_CONST: + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + return -EINVAL; + } + + ret = process_const_arg(env, reg, argno, meta); + if (ret < 0) + return ret; + break; + case KF_ARG_ANYTHING: + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + return -EINVAL; + } + break; + case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + return -EINVAL; + } + + if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) + meta->r0_rdonly = true; + ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); + if (ret < 0) + return ret; + break; case KF_ARG_PTR_TO_CTX: if (reg->type != PTR_TO_CTX) { verbose(env, "%s expected pointer to ctx, but got %s\n", @@ -12407,22 +12437,33 @@ check_ok: ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, meta); if (ret < 0) return ret; - break; } - fallthrough; + break; case KF_ARG_CONST_MEM_SIZE: + ret = process_const_arg(env, reg, argno, meta); + if (ret < 0) + return ret; + fallthrough; case KF_ARG_MEM_SIZE: { - struct bpf_reg_state *buff_reg = reg; - struct bpf_reg_state *size_reg = get_func_arg_reg(caller, regs, i + 1); - argno_t next_argno = argno_from_arg(i + 2); + struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, i - 1); + struct bpf_reg_state *size_reg = reg; + argno_t buff_argno = argno_from_arg(i); - ret = check_mem_size_reg(env, buff_reg, size_reg, argno, next_argno, + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + return -EINVAL; + } + + if (bpf_register_is_null(buff_reg)) + break; + + ret = check_mem_size_reg(env, buff_reg, size_reg, buff_argno, argno, BPF_READ | BPF_WRITE, true, meta); if (ret < 0) { - verbose(env, "%s and ", reg_arg_name(env, argno)); + verbose(env, "%s and ", reg_arg_name(env, buff_argno)); verbose(env, "%s memory, len pair leads to invalid memory access\n", - reg_arg_name(env, next_argno)); + reg_arg_name(env, argno)); return ret; } break; -- cgit v1.2.3 From a49b70400b9de06234eb99f87cf60217ed98cc0c Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:33 -0700 Subject: bpf: Generate kfunc argument prototype at add-call time Kfunc argument checking re-derives each argument's kfunc_ptr_arg_type from BTF on every verification of a call in check_kfunc_args(). Now that get_kfunc_arg_type() is a function of the kfunc's BTF alone, it no longer inspects register state. The classification can be computed once when the call is added and cached. This is a step toward describing kfuncs with a bpf_func_proto and sharing the helper argument-checking path. Generate the classification at bpf_add_kfunc_call() time: - Extend struct bpf_func_proto to be able to describe a kfunc: widen arg_type[] and the arg_btf_id[]/arg_size[] union from 5 to MAX_BPF_FUNC_ARGS, since a kfunc may take up to 12 arguments (5 in registers, 7 on the stack). - Embed a bpf_func_proto in struct bpf_kfunc_desc, populated by gen_kfunc_arg_proto() which runs get_kfunc_arg_type() for each argument and stores the result in proto.arg_type[]. Grow the descriptor table's descs[] as a flexible array to not waste memory. - check_kfunc_args() reads the cached classification from meta->fn The KF_ARG_PTR_TO_CTX classification depends on the resolved program type, and for BPF_PROG_TYPE_EXT that is the target program's type, which resolve_prog_type() reads from prog->aux->saved_dst_prog_type. That field is normally recorded later during verification in check_attach_btf_id(), after bpf_add_kfunc_call() has run. Record saved_dst_prog_type and saved_dst_attach_type from dst_prog at program load time in bpf_prog_load() so the resolved type is available at add-call time without reordering check_attach_btf_id(). This keeps e.g. an freplace of an XDP program calling bpf_xdp_metadata_rx_hash() classifying its struct xdp_md * argument as context. The classification result is unchanged; it is only computed earlier and cached. Signed-off-by: Amery Hung Link: https://lore.kernel.org/bpf/20260801074633.1595644-19-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/syscall.c | 4 +++ kernel/bpf/verifier.c | 98 ++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 85 insertions(+), 17 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index d6be7f49433c..8d111da88655 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3043,6 +3043,10 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at prog->aux->attach_btf = attach_btf; prog->aux->attach_btf_id = multi_func ? bpf_multi_func_btf_id[0] : attr->attach_btf_id; prog->aux->dst_prog = dst_prog; + if (dst_prog) { + prog->aux->saved_dst_prog_type = dst_prog->type; + prog->aux->saved_dst_attach_type = dst_prog->expected_attach_type; + } prog->aux->dev_bound = !!attr->prog_ifindex; prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 23a6355a38d3..b274004fccfd 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2721,8 +2721,12 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env, return 0; } +static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + struct bpf_func_proto *proto); + int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) { + struct bpf_call_arg_meta meta; struct bpf_kfunc_btf_tab *btf_tab; struct btf_func_model func_model; struct bpf_kfunc_desc_tab *tab; @@ -2808,11 +2812,30 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) if (err) return err; - desc = &tab->descs[tab->nr_descs++]; + memset(&meta, 0, sizeof(meta)); + meta.btf = kfunc.btf; + meta.func_id = kfunc.id; + meta.func_proto = kfunc.proto; + meta.func_name = kfunc.name; + meta.kfunc_flags = kfunc.flags ? *kfunc.flags : 0; + + tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1), GFP_KERNEL_ACCOUNT); + if (!tab) + return -ENOMEM; + prog_aux->kfunc_tab = tab; + + desc = &tab->descs[tab->nr_descs]; + memset(desc, 0, sizeof(*desc)); + + err = gen_kfunc_arg_proto(env, &meta, &desc->proto); + if (err) + return err; + desc->func_id = func_id; desc->offset = offset; desc->addr = addr; desc->func_model = func_model; + tab->nr_descs++; sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off, NULL); return 0; @@ -8332,9 +8355,9 @@ static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_stat static int check_func_arg(struct bpf_verifier_env *env, u32 arg, struct bpf_call_arg_meta *meta, - const struct bpf_func_proto *fn, int insn_idx) { + const struct bpf_func_proto *fn = meta->fn; u32 regno = BPF_REG_1 + arg; struct bpf_reg_state *reg = reg_state(env, regno); enum bpf_arg_type arg_type = fn->arg_type[arg]; @@ -8847,6 +8870,8 @@ static bool check_raw_mode_ok(const struct bpf_func_proto *fn, struct bpf_call_a int i; for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { + if (fn->arg_type[i] == ARG_DONTCARE) + break; if (!arg_type_is_raw_mem(fn->arg_type[i])) continue; if (meta->arg_raw_mem.regno) @@ -8895,6 +8920,8 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn) int i; for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { + if (fn->arg_type[i] == ARG_DONTCARE) + break; if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID) return !!fn->arg_btf_id[i]; if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK) @@ -8916,6 +8943,8 @@ static bool check_mem_arg_rw_flag_ok(const struct bpf_func_proto *fn) for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; + if (arg_type == ARG_DONTCARE) + break; if (base_type(arg_type) != ARG_PTR_TO_MEM) continue; if (!(arg_type & (MEM_WRITE | MEM_RDONLY))) @@ -8932,6 +8961,8 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_ for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; + if (arg_type == ARG_DONTCARE) + break; if (arg_type_is_release(arg_type)) { if (meta->release_regno) return false; @@ -10321,9 +10352,10 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn env->insn_aux_data[insn_idx].non_sleepable = true; meta.func_id = func_id; + meta.fn = fn; /* check args */ for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { - err = check_func_arg(env, i, &meta, fn, insn_idx); + err = check_func_arg(env, i, &meta, insn_idx); if (err) return err; } @@ -11463,6 +11495,43 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, return arg_type; } +static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + struct bpf_func_proto *proto) +{ + const struct btf *btf = meta->btf; + const struct btf_param *args; + u32 i, nargs; + int arg_type; + + args = (const struct btf_param *)(meta->func_proto + 1); + nargs = btf_type_vlen(meta->func_proto); + if (nargs > MAX_BPF_FUNC_ARGS) { + verbose(env, "Function %s has %d > %d args\n", meta->func_name, + nargs, MAX_BPF_FUNC_ARGS); + return -EINVAL; + } + if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { + verbose(env, "JIT does not support kfunc %s() with %d args\n", + meta->func_name, nargs); + return -ENOTSUPP; + } + + for (i = 0; i < nargs; i++) { + if (is_kfunc_arg_prog_aux(btf, &args[i]) || + is_kfunc_arg_ignore(btf, &args[i]) || + is_kfunc_arg_implicit(meta, i)) + continue; + + arg_type = get_kfunc_arg_type(env, meta, args, i, nargs); + if (arg_type < 0) + return arg_type; + + proto->arg_type[i] = arg_type; + } + + return 0; +} + static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, const struct btf_type *ref_t, @@ -12046,16 +12115,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me args = (const struct btf_param *)(meta->func_proto + 1); nargs = btf_type_vlen(meta->func_proto); - if (nargs > MAX_BPF_FUNC_ARGS) { - verbose(env, "Function %s has %d > %d args\n", func_name, nargs, - MAX_BPF_FUNC_ARGS); - return -EINVAL; - } - if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { - verbose(env, "JIT does not support kfunc %s() with %d args\n", - func_name, nargs); - return -ENOTSUPP; - } ret = check_outgoing_stack_args(env, caller, nargs); if (ret) @@ -12072,7 +12131,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me int regno = reg_from_argno(argno); bool btf_id_fixed_off_ok = true; u32 ref_id, type_size; - int kf_arg_type; + int kf_arg_type = meta->fn->arg_type[i]; if (is_kfunc_arg_prog_aux(btf, &args[i])) { /* Reject repeated use bpf_prog_aux */ @@ -12117,9 +12176,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me ref_tname = btf_name_by_offset(btf, ref_t->name_off); } - kf_arg_type = get_kfunc_arg_type(env, meta, args, i, nargs); - if (kf_arg_type < 0) - return kf_arg_type; if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) continue; @@ -12986,6 +13042,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int err, insn_idx = *insn_idx_p; const struct btf_param *args; u32 i, nargs, ptr_type_id; + struct bpf_kfunc_desc *desc; struct btf *desc_btf; int id; @@ -13002,6 +13059,13 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, func_name = meta.func_name; insn_aux = &env->insn_aux_data[insn_idx]; + desc = find_kfunc_desc(env->prog, insn->imm, insn->off); + if (!desc) { + verifier_bug(env, "kfunc descriptor not found for func_id %u", insn->imm); + return -EFAULT; + } + meta.fn = &desc->proto; + insn_aux->is_iter_next = bpf_is_iter_next_kfunc(&meta); if (!insn->off && -- cgit v1.2.3 From 0b10b945479c954393d62ee3229d2f224a4ca91c Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Tue, 28 Jul 2026 14:05:16 +0800 Subject: bpf: Fix mmap_lock deadlock on arena lock failure Reported by the Sashiko AI review. arena_vm_fault() returns VM_FAULT_RETRY when it can't take arena->spinlock, but it never took mmap_lock. The fault path assumes a VM_FAULT_RETRY handler already dropped mmap_lock and re-takes it on the retry, so mmap_lock gets taken twice and can deadlock: do_user_addr_fault() { fault = handle_mm_fault(...); // calls arena_vm_fault() if (fault & VM_FAULT_RETRY) goto retry; // re-locks mmap_lock mmap_read_unlock(mm); } Return VM_FAULT_SIGBUS instead, for two reasons: 1. We could keep VM_FAULT_RETRY, but then we'd have to drop the fault lock first and cap the retry ourselves, the way __folio_lock_or_retry() does. 2. A failed raw_res_spin_lock_irqsave() already means a possible deadlock was detected, so retrying just hits the same lock again. So returning VM_FAULT_RETRY here is overkill. Fixes: b8467290edab ("bpf: arena: make arena kfuncs any context safe") Signed-off-by: Jiayuan Chen Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260728060517.95183-1-jiayuan.chen@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/arena.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 34f023a537fe..555ee2531ef9 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -490,8 +490,12 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) kaddr = kbase + (u32)(vmf->address); if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) - /* Make a reasonable effort to address impossible case */ - return VM_FAULT_RETRY; + /* + * A failed lock means a possible deadlock was detected. Don't + * return VM_FAULT_RETRY: this handler never took mmap_lock, but + * the fault path would re-take it on retry and deadlock. Fail. + */ + return VM_FAULT_SIGBUS; page = vmalloc_to_page((void *)kaddr); if (page) { -- cgit v1.2.3 From 457d4ecb47aaf7a2cb46aaadd76e8c812e4f3c9e Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sun, 2 Aug 2026 22:27:26 -0700 Subject: bpf: Remove unused BTF_FMODEL_STRUCT_ARG Commit 814cba835ef6 ("bpf, x86: Fix trampoline stack size for 128-bit arguments") changed the x86 trampoline to compute the number of registers from arg_size for every argument, which removed the last user of BTF_FMODEL_STRUCT_ARG. No other architecture or verifier code looks at the flag, so remove the macro and the code in __get_type_fmodel_flags() which sets it. Keep BTF_FMODEL_SIGNED_ARG at BIT(1) rather than renumbering it to BIT(0), so BIT(0) is available for a future flag. No functional change. Signed-off-by: Yonghong Song Signed-off-by: Daniel Borkmann Acked-by: Leon Hwang Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/20260803052726.2821447-1-yonghong.song@linux.dev --- kernel/bpf/btf.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 5e8ac45ce56a..42414633cf26 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7533,8 +7533,6 @@ static u8 __get_type_fmodel_flags(const struct btf_type *t) { u8 flags = 0; - if (btf_type_is_struct(t)) - flags |= BTF_FMODEL_STRUCT_ARG; if (btf_type_is_signed_int(t)) flags |= BTF_FMODEL_SIGNED_ARG; -- cgit v1.2.3 From 180c7000712db77063b3a26f4c97e7dd9038f449 Mon Sep 17 00:00:00 2001 From: Ning Ding Date: Mon, 3 Aug 2026 04:26:08 -0700 Subject: bpf: Invalidate RCU pointers after final spin unlock In a sleepable BPF program, a spin lock can provide the only RCU protection for a kptr. The final bpf_spin_unlock() ends that protection, but the verifier leaves the pointer valid. Another CPU can then free the object before the pointer is used. A capability-limited runtime PoC triggered a task_struct use-after-free in __bpf_get_task_stack(). Record whether the program is in an RCU-protected context before releasing the lock. Invalidate RCU-protected pointers only when the unlock leaves the final such context. This preserves valid pointers in non-sleepable programs and inside an explicit RCU read-side section. Fixes: 5861d1e8dbc4 ("bpf: Allow bpf_spin_{lock,unlock} in sleepable progs") Assisted-by: Codex:gpt-5.6-sol Assisted-by: ChatGPT:GPT-5.6-Pro Signed-off-by: Ning Ding Link: https://lore.kernel.org/bpf/20260803112615.3362122-2-dingning04@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index b274004fccfd..7439afdc851a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -206,6 +206,7 @@ static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int par static int release_reference_nomark(struct bpf_verifier_state *state, int id); static int release_reference(struct bpf_verifier_env *env, int id); static void invalidate_non_owning_refs(struct bpf_verifier_env *env); +static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env); static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); static bool is_tracing_prog_type(enum bpf_prog_type type); static int ref_set_non_owning(struct bpf_verifier_env *env, @@ -7165,6 +7166,7 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state return err; } } else { + bool was_in_rcu_cs; void *ptr; int type; @@ -7192,10 +7194,13 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state verbose(env, "%s_unlock cannot be out of order\n", lock_str); return -EINVAL; } + was_in_rcu_cs = in_rcu_cs(env); if (release_lock_state(cur, type, reg->id, ptr)) { verbose(env, "%s_unlock of different lock\n", lock_str); return -EINVAL; } + if (was_in_rcu_cs && !in_rcu_cs(env)) + invalidate_rcu_protected_refs(env); invalidate_non_owning_refs(env); } -- cgit v1.2.3 From 6655c409707ec8ce9ce0850ffe4fe02331fd4d9c Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Mon, 3 Aug 2026 01:39:34 +0000 Subject: bpf, cgroup: Fix invalid storage access after __cgroup_bpf_attach failed A potential invalid storage access issue can occur after replacing a cgroup bpf prog. This occurs in the following scenario: 1. prog1 with storage is attached to a cgroup in multi-attach mode. 2. prog1 is replaced with prog2 using BPF_F_REPLACE in multi-attach mode, but fails midway (e.g. in bpf_trampoline_link_cgroup_shim or update_effective_progs). 3. A new prog3 is attached to the cgroup in multi-attach mode. The reason is that __cgroup_bpf_attach overwrites pl->storage with the new storage prior to attachment completion. When attachment fails midway, the cleanup path calls bpf_cgroup_storages_free(new_storage) to free the newly allocated storage, but fails to restore pl->storage back to old_storage. Consequently, the still-active prog1 holds invalid or dangling storage pointers, leading to an invalid memory access when prog1 executes and calls bpf_get_local_storage. Additionally, original pl->flags and cgrp->bpf.flags[atype] are left unrestored. Fix this by saving old_pl_flags, old_storage, and old_flags prior to the update, and properly restoring all of them in the cleanup path on error. Fixes: 7d9c3427894f ("bpf: Make cgroup storages shared between programs on the same cgroup") Reported-by: Sashiko Signed-off-by: Pu Lehui Signed-off-by: Andrii Nakryiko Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260803013934.4036646-1-pulehui@huaweicloud.com --- kernel/bpf/cgroup.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index fb9357b64cad..d2da5063d8f8 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -813,8 +813,10 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *old_prog = NULL; struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; struct bpf_cgroup_storage *new_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; + struct bpf_cgroup_storage *old_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; struct bpf_prog *new_prog = prog ? : link->link.prog; enum cgroup_bpf_attach_type atype; + u32 old_flags, old_pl_flags; struct bpf_prog_list *pl; struct hlist_head *progs; int err; @@ -865,6 +867,8 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, if (pl) { old_prog = pl->prog; + old_pl_flags = pl->flags; + bpf_cgroup_storages_assign(old_storage, pl->storage); } else { pl = kmalloc_obj(*pl); if (!pl) { @@ -884,6 +888,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, pl->link = link; pl->flags = flags; bpf_cgroup_storages_assign(pl->storage, storage); + old_flags = cgrp->bpf.flags[atype]; cgrp->bpf.flags[atype] = saved_flags; if (type == BPF_LSM_CGROUP) { @@ -915,12 +920,15 @@ cleanup: if (old_prog) { pl->prog = old_prog; pl->link = NULL; + pl->flags = old_pl_flags; + bpf_cgroup_storages_assign(pl->storage, old_storage); } bpf_cgroup_storages_free(new_storage); if (!old_prog) { hlist_del(&pl->node); kfree(pl); } + cgrp->bpf.flags[atype] = old_flags; return err; } -- cgit v1.2.3 From b87803391baa7e0bef60549d8841f12e549ad057 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 4 Aug 2026 22:19:16 +0200 Subject: bpf: Check load-acquire src ptr type before the load check_atomic_load() calls check_load_mem() before atomic_ptr_type_ok(). For a load-acquire that fetches into its own source register (dst_reg == src_reg), check_load_mem() overwrites src_reg's type with the type of the loaded value, so the subsequent atomic_ptr_type_ok() no longer sees the source pointer and fails to reject the disallowed types (ctx, pkt, flow_keys, sock). Since bpf_convert_ctx_accesses() does not rewrite atomic loads, the raw access to the underlying kernel object is left in place. The destination type is taken from the ctx access itself, so a load-acquire of the sk field of struct __sk_buff for example leaves the register typed as PTR_TO_SOCK_COMMON_OR_NULL, which type_is_sk_pointer() does not match either, while it actually holds unconverted struct sk_buff bytes. Once the NULL check has passed this is a type confusion, not just a leak of kernel data. Validate src_reg with check_reg_arg() and check the source pointer type with atomic_ptr_type_ok() before the load again, mirroring check_atomic_rmw(). Out-of-range register numbers are already rejected earlier by check_and_resolve_insns() (commit 503d21ef8eac ("bpf: Do register range validation early")), and the only exemption there, is_stack_arg_ldx(), requires BPF_LDX | BPF_MEM | BPF_DW and thus never matches a BPF_ATOMIC insn. atomic_ptr_type_ok() can therefore not dereference register state out of bounds, that is, the out-of-bounds read addressed by the Fixes commit below does not reappear (as proven also via selftest). Fixes: c03bb2fa327e ("bpf: Fix out-of-bounds read in check_atomic_load/store()") Reported-by: STAR Labs SG Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260804201917.253491-1-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7439afdc851a..09588b7b08b0 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6617,7 +6617,7 @@ static int check_atomic_load(struct bpf_verifier_env *env, { int err; - err = check_load_mem(env, insn, true, false, false, "atomic_load"); + err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) return err; @@ -6628,7 +6628,7 @@ static int check_atomic_load(struct bpf_verifier_env *env, return -EACCES; } - return 0; + return check_load_mem(env, insn, true, false, false, "atomic_load"); } static int check_atomic_store(struct bpf_verifier_env *env, -- cgit v1.2.3 From 8efd87051c3a2a054519955ca401229f4e84b310 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Tue, 4 Aug 2026 06:45:53 -0700 Subject: bpf: Correct the overflow check comment in bpf_iter_num_next() The comment on the s->cur + 1 >= s->end check claims the (s64) cast is needed to avoid overflow when s->cur == s->end == INT_MAX. It isn't: s->cur + 1 is computed in int and wraps before the cast, so the cast changes nothing (INT_MAX + 1 compares the same either way). The wraparound is the point. bpf_iter_num_new() sets s->cur = start - 1, which wraps to INT_MAX for start == INT_MIN, and the wrapping s->cur + 1 brings it back to start. (s64)s->cur + 1 would instead break iterators starting at INT_MIN. Drop the cast and reword the comment. No functional change; the wrap is well-defined under -fno-strict-overflow. Signed-off-by: Puranjay Mohan Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260804134601.2305303-2-puranjay@kernel.org --- kernel/bpf/bpf_iter.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index f5eaeb2493d4..b235e117e206 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -802,12 +802,11 @@ __bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it) { struct bpf_iter_num_kern *s = (void *)it; - /* check failed initialization or if we are done (same behavior); - * need to be careful about overflow, so convert to s64 for checks, - * e.g., if s->cur == s->end == INT_MAX, we can't just do - * s->cur + 1 >= s->end + /* + * s->cur < s->end while iterating, else s->cur == s->end == 0; the signed + * s->cur + 1 >= s->end holds even when s->cur + 1 wraps (start == INT_MIN). */ - if ((s64)(s->cur + 1) >= s->end) { + if (s->cur + 1 >= s->end) { s->cur = s->end = 0; return NULL; } -- cgit v1.2.3 From f8f2b567d56035ddd62ec82f2c35dcee8c516624 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Tue, 4 Aug 2026 06:45:54 -0700 Subject: bpf: Inline bpf_iter_num_new() kfunc bpf_for() expands to the bpf_iter_num_{new,next,destroy}() kfuncs, which the verifier emits as regular calls. They are tiny and only touch the 8-byte on-stack iterator state, so open-code them in bpf_fixup_kfunc_call() like the other special kfuncs there. Start with bpf_iter_num_new(): R1 points to the iterator, R2/R3 hold start/end. The inlined sequence mirrors the kfunc and returns the same -EINVAL / -E2BIG / 0. start > end is rejected first, so end - start fits in a u32; range-check it as u32 on both sides ((u32)(end - start) in the kfunc). A movsx-based check would emit a cpuv4 instruction that some JITs (x86-32, mips32, sparc64) decode as a plain move and get wrong. The emitted instructions are plain BPF, so the interpreter path stays correct and no jit_required marking is needed. Signed-off-by: Puranjay Mohan Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260804134601.2305303-3-puranjay@kernel.org --- kernel/bpf/bpf_iter.c | 4 ++-- kernel/bpf/verifier.c | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index b235e117e206..d19f1b2861d2 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -782,8 +782,8 @@ __bpf_kfunc int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) return -EINVAL; } - /* avoid overflows, e.g., if start == INT_MIN and end == INT_MAX */ - if ((s64)end - (s64)start > BPF_MAX_LOOPS) { + /* start <= end here, so end - start fits in a u32 without overflow */ + if ((u32)(end - start) > BPF_MAX_LOOPS) { s->cur = s->end = 0; return -E2BIG; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 09588b7b08b0..8401077ed8fc 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20006,6 +20006,30 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn_buf[4] = BPF_ALU64_REG(BPF_SUB, BPF_REG_0, BPF_REG_1); insn_buf[5] = BPF_ALU64_IMM(BPF_NEG, BPF_REG_0, 0); *cnt = 6; + } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_new]) { + /* inline bpf_iter_num_new(&it, start, end); R1=&it, R2=start, R3=end */ + int i = 0; + + /* if (start > end) goto einval; */ + insn_buf[i++] = BPF_JMP32_REG(BPF_JSGT, BPF_REG_2, BPF_REG_3, 8); + /* r0 = (u32)end - (u32)start; if (r0 > BPF_MAX_LOOPS) goto e2big; */ + insn_buf[i++] = BPF_MOV32_REG(BPF_REG_0, BPF_REG_3); + insn_buf[i++] = BPF_ALU32_REG(BPF_SUB, BPF_REG_0, BPF_REG_2); + insn_buf[i++] = BPF_JMP_IMM(BPF_JGT, BPF_REG_0, BPF_MAX_LOOPS, 8); + /* s->cur = start - 1; s->end = end; return 0; */ + insn_buf[i++] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_2, -1); + insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_2, 0); + insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3, 4); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0); + insn_buf[i++] = BPF_JMP_A(5); + /* einval: s->cur = s->end = 0; return -EINVAL; */ + insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); + insn_buf[i++] = BPF_JMP_A(2); + /* e2big: s->cur = s->end = 0; return -E2BIG; */ + insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, -E2BIG); + *cnt = i; } if (env->insn_aux_data[insn_idx].arg_prog) { -- cgit v1.2.3 From e93347704878aa8a54b3154114d13e57e8923e27 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Tue, 4 Aug 2026 06:45:55 -0700 Subject: bpf: Inline bpf_iter_num_next() kfunc bpf_iter_num_next() runs on every bpf_for() iteration, so inlining it drops a call from the loop body. R1 points to the iterator; the returned pointer to s->cur is R1 itself, since s->cur is first. s->cur and s->end are int, so the kfunc's s->cur + 1 >= s->end is a signed 32-bit compare and the inlined code needs no sign extension. Signed-off-by: Puranjay Mohan Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260804134601.2305303-4-puranjay@kernel.org --- kernel/bpf/verifier.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8401077ed8fc..80c3cb654b89 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20030,6 +20030,23 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, -E2BIG); *cnt = i; + } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_next]) { + /* inline bpf_iter_num_next(&it); R1=&it, returns &s->cur or NULL */ + int i = 0; + + /* r0 = s->cur + 1; if ((s32)r0 >= s->end) goto done; */ + insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0); + insn_buf[i++] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_0, 1); + insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 4); + insn_buf[i++] = BPF_JMP32_REG(BPF_JSGE, BPF_REG_0, BPF_REG_2, 3); + /* s->cur = r0; return &s->cur; */ + insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0); + insn_buf[i++] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); + insn_buf[i++] = BPF_JMP_A(2); + /* done: s->cur = s->end = 0; return NULL; */ + insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0); + *cnt = i; } if (env->insn_aux_data[insn_idx].arg_prog) { -- cgit v1.2.3 From 39f047682fe3ccc272dbebb6d1ecba8fc1e0d8b4 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Tue, 4 Aug 2026 06:45:56 -0700 Subject: bpf: Inline bpf_iter_num_destroy() as a no-op Once destroy() returns the stack slot is no longer tracked as iterator state, so zeroing it is dead work. Make the kfunc a no-op and inline the call to a single BPF_JA 0 (the fixup can't drop the instruction outright, so emit a nop; the JITs elide it). Suggested-by: Andrii Nakryiko Signed-off-by: Puranjay Mohan Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260804134601.2305303-5-puranjay@kernel.org --- kernel/bpf/bpf_iter.c | 4 +--- kernel/bpf/verifier.c | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index d19f1b2861d2..14a5fdfa0421 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -818,9 +818,7 @@ __bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it) __bpf_kfunc void bpf_iter_num_destroy(struct bpf_iter_num *it) { - struct bpf_iter_num_kern *s = (void *)it; - - s->cur = s->end = 0; + /* no-op */ } __bpf_kfunc_end_defs(); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 80c3cb654b89..4db151e24355 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20047,6 +20047,10 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0); *cnt = i; + } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_destroy]) { + /* bpf_iter_num_destroy() is a no-op; emit a nop to drop the call */ + insn_buf[0] = BPF_JMP_A(0); + *cnt = 1; } if (env->insn_aux_data[insn_idx].arg_prog) { -- cgit v1.2.3 From 15b837759a97237d647962f9943afe0d55af615a Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 3 Aug 2026 23:01:38 +0200 Subject: bpf: Factor stackid_init function from __bpf_get_stackid The new stackid_init function stores all the necessary bits for stackid trace and it will be used by other functions in following changes. Signed-off-by: Jiri Olsa Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260803210149.296496-2-jolsa@kernel.org --- kernel/bpf/stackmap.c | 95 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 36 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 463f94ba1cc4..19f9ea605a3e 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -504,33 +504,54 @@ get_callchain_entry_for_task(struct task_struct *task, u32 max_depth) #endif } -static long __bpf_get_stackid(struct bpf_map *map, - struct perf_callchain_entry *trace, u64 flags) +struct stackid { + struct stack_map_bucket *bucket; + u64 *ips; + u32 nr; + u32 len; + u32 hash; + u32 id; +}; + +static int stackid_init(struct stackid *stackid, struct bpf_map *map, + struct perf_callchain_entry *trace, u64 flags) { struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); - struct stack_map_bucket *bucket, *new_bucket, *old_bucket; - u32 hash, id, trace_nr, trace_len, i, max_depth; u32 skip = flags & BPF_F_SKIP_FIELD_MASK; - bool user = flags & BPF_F_USER_STACK; - u64 *ips; - bool hash_matches; + u32 max_depth; if (trace->nr <= skip) /* skipping more than usable stack trace */ return -EFAULT; max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags); - trace_nr = min_t(u32, trace->nr - skip, max_depth - skip); - trace_len = trace_nr * sizeof(u64); - ips = trace->ip + skip; - hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0); - id = hash & (smap->n_buckets - 1); - bucket = READ_ONCE(smap->buckets[id]); + stackid->nr = min_t(u32, trace->nr - skip, max_depth - skip); + stackid->len = stackid->nr * sizeof(u64); + stackid->ips = trace->ip + skip; + stackid->hash = jhash2((u32 *)stackid->ips, stackid->len / sizeof(u32), 0); + stackid->id = stackid->hash & (smap->n_buckets - 1); + stackid->bucket = READ_ONCE(smap->buckets[stackid->id]); + return 0; +} - hash_matches = bucket && bucket->hash == hash; +static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, + struct perf_callchain_entry *trace, u64 flags) +{ + struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); + struct stack_map_bucket *new_bucket, *old_bucket; + bool user = flags & BPF_F_USER_STACK; + bool hash_matches; + u32 trace_len, i; + int err; + + err = stackid_init(stackid, map, trace, flags); + if (err) + return err; + + hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash; /* fast cmp */ if (hash_matches && flags & BPF_F_FAST_STACK_CMP) - return id; + return stackid->id; if (stack_map_use_build_id(map)) { struct bpf_stack_build_id *id_offs; @@ -540,42 +561,42 @@ static long __bpf_get_stackid(struct bpf_map *map, pcpu_freelist_pop(&smap->freelist); if (unlikely(!new_bucket)) return -ENOMEM; - new_bucket->nr = trace_nr; + new_bucket->nr = stackid->nr; id_offs = (struct bpf_stack_build_id *)new_bucket->data; - for (i = 0; i < trace_nr; i++) - id_offs[i].ip = ips[i]; - stack_map_get_build_id_offset(id_offs, trace_nr, user, false /* !may_fault */); - trace_len = trace_nr * sizeof(struct bpf_stack_build_id); - if (hash_matches && bucket->nr == trace_nr && - memcmp(bucket->data, new_bucket->data, trace_len) == 0) { + for (i = 0; i < stackid->nr; i++) + id_offs[i].ip = stackid->ips[i]; + stack_map_get_build_id_offset(id_offs, stackid->nr, user, false /* !may_fault */); + trace_len = stackid->nr * sizeof(struct bpf_stack_build_id); + if (hash_matches && stackid->bucket->nr == stackid->nr && + memcmp(stackid->bucket->data, new_bucket->data, trace_len) == 0) { pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); - return id; + return stackid->id; } - if (bucket && !(flags & BPF_F_REUSE_STACKID)) { + if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) { pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); return -EEXIST; } } else { - if (hash_matches && bucket->nr == trace_nr && - memcmp(bucket->data, ips, trace_len) == 0) - return id; - if (bucket && !(flags & BPF_F_REUSE_STACKID)) + if (hash_matches && stackid->bucket->nr == stackid->nr && + memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0) + return stackid->id; + if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) return -EEXIST; new_bucket = (struct stack_map_bucket *) pcpu_freelist_pop(&smap->freelist); if (unlikely(!new_bucket)) return -ENOMEM; - memcpy(new_bucket->data, ips, trace_len); + memcpy(new_bucket->data, stackid->ips, stackid->len); } - new_bucket->hash = hash; - new_bucket->nr = trace_nr; + new_bucket->hash = stackid->hash; + new_bucket->nr = stackid->nr; - old_bucket = xchg(&smap->buckets[id], new_bucket); + old_bucket = xchg(&smap->buckets[stackid->id], new_bucket); if (old_bucket) pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); - return id; + return stackid->id; } BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, @@ -584,6 +605,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, u32 elem_size = stack_map_data_size(map); bool user = flags & BPF_F_USER_STACK; struct perf_callchain_entry *trace; + struct stackid stackid; bool kernel = !user; u32 max_depth; @@ -599,7 +621,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, /* couldn't fetch the stack trace */ return -EFAULT; - return __bpf_get_stackid(map, trace, flags); + return __bpf_get_stackid(&stackid, map, trace, flags); } const struct bpf_func_proto bpf_get_stackid_proto = { @@ -628,6 +650,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, { struct perf_event *event = ctx->event; struct perf_callchain_entry *trace; + struct stackid stackid; bool kernel, user; __u64 nr_kernel; int ret; @@ -653,7 +676,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, if (kernel) { trace->nr = nr_kernel; - ret = __bpf_get_stackid(map, trace, flags); + ret = __bpf_get_stackid(&stackid, map, trace, flags); } else { /* user */ u64 skip = flags & BPF_F_SKIP_FIELD_MASK; @@ -662,7 +685,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, return -EFAULT; flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; - ret = __bpf_get_stackid(map, trace, flags); + ret = __bpf_get_stackid(&stackid, map, trace, flags); } /* restore nr */ -- cgit v1.2.3 From 0ca56befcffec3a6c9d1842eae06c74e1cf41f11 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 3 Aug 2026 23:01:39 +0200 Subject: bpf: Factor stackid_fastpath function from __bpf_get_stackid The new stackid_fastpath does the fast stack hash and trace check, that does not need new bucket allocation. It covers both just-ip and buildid code paths. Signed-off-by: Jiri Olsa Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260803210149.296496-3-jolsa@kernel.org --- kernel/bpf/stackmap.c | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 19f9ea605a3e..27210b5d16fc 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -511,6 +511,7 @@ struct stackid { u32 len; u32 hash; u32 id; + bool hash_matches; }; static int stackid_init(struct stackid *stackid, struct bpf_map *map, @@ -531,28 +532,46 @@ static int stackid_init(struct stackid *stackid, struct bpf_map *map, stackid->hash = jhash2((u32 *)stackid->ips, stackid->len / sizeof(u32), 0); stackid->id = stackid->hash & (smap->n_buckets - 1); stackid->bucket = READ_ONCE(smap->buckets[stackid->id]); + stackid->hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash; return 0; } +static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map, + struct perf_callchain_entry *trace, u64 flags) +{ + int err; + + err = stackid_init(stackid, map, trace, flags); + if (err) + return err; + + /* fast cmp */ + if (stackid->hash_matches && flags & BPF_F_FAST_STACK_CMP) + return stackid->id; + + if (stack_map_use_build_id(map)) + return -ENOENT; + if (stackid->hash_matches && stackid->bucket->nr == stackid->nr && + memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0) + return stackid->id; + if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) + return -EEXIST; + return -ENOENT; +} + static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, struct perf_callchain_entry *trace, u64 flags) { struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); struct stack_map_bucket *new_bucket, *old_bucket; bool user = flags & BPF_F_USER_STACK; - bool hash_matches; u32 trace_len, i; int err; - err = stackid_init(stackid, map, trace, flags); - if (err) + err = stackid_fastpath(stackid, map, trace, flags); + if (err != -ENOENT) return err; - hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash; - /* fast cmp */ - if (hash_matches && flags & BPF_F_FAST_STACK_CMP) - return stackid->id; - if (stack_map_use_build_id(map)) { struct bpf_stack_build_id *id_offs; @@ -567,7 +586,7 @@ static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, id_offs[i].ip = stackid->ips[i]; stack_map_get_build_id_offset(id_offs, stackid->nr, user, false /* !may_fault */); trace_len = stackid->nr * sizeof(struct bpf_stack_build_id); - if (hash_matches && stackid->bucket->nr == stackid->nr && + if (stackid->hash_matches && stackid->bucket->nr == stackid->nr && memcmp(stackid->bucket->data, new_bucket->data, trace_len) == 0) { pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); return stackid->id; @@ -577,12 +596,6 @@ static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, return -EEXIST; } } else { - if (hash_matches && stackid->bucket->nr == stackid->nr && - memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0) - return stackid->id; - if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) - return -EEXIST; - new_bucket = (struct stack_map_bucket *) pcpu_freelist_pop(&smap->freelist); if (unlikely(!new_bucket)) -- cgit v1.2.3 From bb4e6f4e1b68fe60c04ca04c564c6624e837dbf4 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 3 Aug 2026 23:01:40 +0200 Subject: bpf: Factor stackid_new_bucket from __bpf_get_stackid The new stackid_new_bucket allocates the new bucket and initializes it with the trace data. Signed-off-by: Jiri Olsa Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260803210149.296496-4-jolsa@kernel.org --- kernel/bpf/stackmap.c | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 27210b5d16fc..d930d9754d7d 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -559,31 +559,52 @@ static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map, return -ENOENT; } +static struct stack_map_bucket * +stackid_new_bucket(struct stackid *stackid, struct bpf_map *map) +{ + struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); + struct bpf_stack_build_id *id_offs; + struct stack_map_bucket *bucket; + u32 i; + + bucket = (struct stack_map_bucket *) pcpu_freelist_pop(&smap->freelist); + if (unlikely(!bucket)) + return NULL; + + if (stack_map_use_build_id(map)) { + id_offs = (struct bpf_stack_build_id *)bucket->data; + for (i = 0; i < stackid->nr; i++) + id_offs[i].ip = stackid->ips[i]; + } else { + memcpy(bucket->data, stackid->ips, stackid->len); + } + + bucket->hash = stackid->hash; + bucket->nr = stackid->nr; + return bucket; +} + static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, struct perf_callchain_entry *trace, u64 flags) { struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); struct stack_map_bucket *new_bucket, *old_bucket; bool user = flags & BPF_F_USER_STACK; - u32 trace_len, i; + u32 trace_len; int err; err = stackid_fastpath(stackid, map, trace, flags); if (err != -ENOENT) return err; + new_bucket = stackid_new_bucket(stackid, map); + if (!new_bucket) + return -ENOMEM; + if (stack_map_use_build_id(map)) { struct bpf_stack_build_id *id_offs; - /* for build_id+offset, pop a bucket before slow cmp */ - new_bucket = (struct stack_map_bucket *) - pcpu_freelist_pop(&smap->freelist); - if (unlikely(!new_bucket)) - return -ENOMEM; - new_bucket->nr = stackid->nr; id_offs = (struct bpf_stack_build_id *)new_bucket->data; - for (i = 0; i < stackid->nr; i++) - id_offs[i].ip = stackid->ips[i]; stack_map_get_build_id_offset(id_offs, stackid->nr, user, false /* !may_fault */); trace_len = stackid->nr * sizeof(struct bpf_stack_build_id); if (stackid->hash_matches && stackid->bucket->nr == stackid->nr && @@ -595,17 +616,8 @@ static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); return -EEXIST; } - } else { - new_bucket = (struct stack_map_bucket *) - pcpu_freelist_pop(&smap->freelist); - if (unlikely(!new_bucket)) - return -ENOMEM; - memcpy(new_bucket->data, stackid->ips, stackid->len); } - new_bucket->hash = stackid->hash; - new_bucket->nr = stackid->nr; - old_bucket = xchg(&smap->buckets[stackid->id], new_bucket); if (old_bucket) pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); -- cgit v1.2.3 From 09b3fd6caa0b57f8a39254ee5db3af30bdd53c18 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 3 Aug 2026 23:01:41 +0200 Subject: bpf: Use stack id functions instead of __bpf_get_stackid Replacing __bpf_get_stackid calls with sequence of following functions: stackid_fastpath stackid_new_bucket stackid_install This makes code more structured and allows us to easily disable preemption only in bpf_get_stackid in following changes. Signed-off-by: Jiri Olsa Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260803210149.296496-5-jolsa@kernel.org --- kernel/bpf/stackmap.c | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index d930d9754d7d..3ee0034daf52 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -584,22 +584,13 @@ stackid_new_bucket(struct stackid *stackid, struct bpf_map *map) return bucket; } -static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, - struct perf_callchain_entry *trace, u64 flags) +static long stackid_install(struct stackid *stackid, struct bpf_map *map, + struct stack_map_bucket *new_bucket, u64 flags) { struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); - struct stack_map_bucket *new_bucket, *old_bucket; bool user = flags & BPF_F_USER_STACK; + struct stack_map_bucket *old_bucket; u32 trace_len; - int err; - - err = stackid_fastpath(stackid, map, trace, flags); - if (err != -ENOENT) - return err; - - new_bucket = stackid_new_bucket(stackid, map); - if (!new_bucket) - return -ENOMEM; if (stack_map_use_build_id(map)) { struct bpf_stack_build_id *id_offs; @@ -629,10 +620,12 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, { u32 elem_size = stack_map_data_size(map); bool user = flags & BPF_F_USER_STACK; + struct stack_map_bucket *new_bucket; struct perf_callchain_entry *trace; struct stackid stackid; bool kernel = !user; u32 max_depth; + int err; if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID))) @@ -646,7 +639,15 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, /* couldn't fetch the stack trace */ return -EFAULT; - return __bpf_get_stackid(&stackid, map, trace, flags); + err = stackid_fastpath(&stackid, map, trace, flags); + if (err != -ENOENT) + return err; + + new_bucket = stackid_new_bucket(&stackid, map); + if (!new_bucket) + return -ENOMEM; + + return stackid_install(&stackid, map, new_bucket, flags); } const struct bpf_func_proto bpf_get_stackid_proto = { @@ -674,6 +675,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, struct bpf_map *, map, u64, flags) { struct perf_event *event = ctx->event; + struct stack_map_bucket *new_bucket; struct perf_callchain_entry *trace; struct stackid stackid; bool kernel, user; @@ -701,7 +703,6 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, if (kernel) { trace->nr = nr_kernel; - ret = __bpf_get_stackid(&stackid, map, trace, flags); } else { /* user */ u64 skip = flags & BPF_F_SKIP_FIELD_MASK; @@ -710,12 +711,22 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, return -EFAULT; flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; - ret = __bpf_get_stackid(&stackid, map, trace, flags); } + ret = stackid_fastpath(&stackid, map, trace, flags); + if (ret != -ENOENT) + goto out; + + new_bucket = stackid_new_bucket(&stackid, map); + if (new_bucket) { + trace->nr = nr; + return stackid_install(&stackid, map, new_bucket, flags); + } + ret = -ENOMEM; + +out: /* restore nr */ trace->nr = nr; - return ret; } -- cgit v1.2.3 From 15f1bd8574662f1b7b26aaa2e23ebf4066f0117d Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 3 Aug 2026 23:01:42 +0200 Subject: bpf: Disable preemption in bpf_get_stackid The get_perf_callchain call needs disabled preemption plus we need it disabled as long as we access its returned trace entries buffer. Note the bpf_get_stackid_pe function is executed already with preemption disabled. Fixes: d5a3b1f69186 ("bpf: introduce BPF_MAP_TYPE_STACK_TRACE") Reported-by: Tao Chen Signed-off-by: Jiri Olsa Signed-off-by: Andrii Nakryiko Cc: stable@vger.kernel.org Link: https://lore.kernel.org/bpf/20260803210149.296496-6-jolsa@kernel.org Closes: https://lore.kernel.org/bpf/20260206090653.1336687-2-chen.dylane@linux.dev/ --- kernel/bpf/stackmap.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 3ee0034daf52..5b18d728f4b8 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -632,20 +632,22 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, return -EINVAL; max_depth = stack_map_calculate_max_depth(map->value_size, elem_size, flags); - trace = get_perf_callchain(regs, kernel, user, max_depth, - false, false, 0); - if (unlikely(!trace)) - /* couldn't fetch the stack trace */ - return -EFAULT; + scoped_guard(preempt) { + trace = get_perf_callchain(regs, kernel, user, max_depth, + false, false, 0); + if (unlikely(!trace)) + /* couldn't fetch the stack trace */ + return -EFAULT; - err = stackid_fastpath(&stackid, map, trace, flags); - if (err != -ENOENT) - return err; + err = stackid_fastpath(&stackid, map, trace, flags); + if (err != -ENOENT) + return err; - new_bucket = stackid_new_bucket(&stackid, map); - if (!new_bucket) - return -ENOMEM; + new_bucket = stackid_new_bucket(&stackid, map); + if (!new_bucket) + return -ENOMEM; + } return stackid_install(&stackid, map, new_bucket, flags); } -- cgit v1.2.3 From cbb99938e7935c9f62c891573a4b09690e5e22de Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 3 Aug 2026 23:01:43 +0200 Subject: bpf: Factor callchain_store function from __bpf_get_stack The new callchain_store function stores trace entries buffer into user supplied buffer. It covers both just-ip and buildid data. Signed-off-by: Jiri Olsa Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260803210149.296496-7-jolsa@kernel.org --- kernel/bpf/stackmap.c | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 5b18d728f4b8..ee9905d67b8e 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -741,6 +741,29 @@ const struct bpf_func_proto bpf_get_stackid_proto_pe = { .arg3_type = ARG_ANYTHING, }; +static u32 callchain_store(struct perf_callchain_entry *trace, void *buf, + u32 elem_size, u64 flags) +{ + bool user_build_id = flags & BPF_F_USER_BUILD_ID; + u32 skip = flags & BPF_F_SKIP_FIELD_MASK; + u32 trace_nr, copy_len; + u64 *ips; + + trace_nr = trace->nr - skip; + copy_len = trace_nr * elem_size; + + ips = trace->ip + skip; + if (user_build_id) { + struct bpf_stack_build_id *id_offs = buf; + + for (u32 i = 0; i < trace_nr; i++) + id_offs[i].ip = ips[i]; + } else { + memcpy(buf, ips, copy_len); + } + return trace_nr; +} + static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, struct perf_callchain_entry *trace_in, void *buf, u32 size, u64 flags, bool may_fault) @@ -753,7 +776,6 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, struct perf_callchain_entry *trace; bool kernel = !user; int err = -EINVAL; - u64 *ips; if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_USER_BUILD_ID))) @@ -798,21 +820,10 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, goto err_fault; } - trace_nr = trace->nr - skip; + trace_nr = callchain_store(trace, buf, elem_size, flags); copy_len = trace_nr * elem_size; - ips = trace->ip + skip; - if (user_build_id) { - struct bpf_stack_build_id *id_offs = buf; - u32 i; - - for (i = 0; i < trace_nr; i++) - id_offs[i].ip = ips[i]; - } else { - memcpy(buf, ips, copy_len); - } - - /* trace/ips should not be dereferenced after this point */ + /* trace should not be dereferenced after this point */ if (may_fault) rcu_read_unlock(); -- cgit v1.2.3 From 014fbe5902dccdaef69114abdcdb15e3dbe55e34 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 3 Aug 2026 23:01:44 +0200 Subject: bpf: Factor callchain_finalize function from __bpf_get_stack The new callchain_finalize function calls the build-id retrieval (if needed) and zeroes the buffer. This makes things easier for preemption fix in following change. Signed-off-by: Jiri Olsa Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260803210149.296496-8-jolsa@kernel.org --- kernel/bpf/stackmap.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index ee9905d67b8e..cdeb6c2e50da 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -764,16 +764,31 @@ static u32 callchain_store(struct perf_callchain_entry *trace, void *buf, return trace_nr; } +static long callchain_finalize(void *buf, u32 size, u32 trace_nr, u32 elem_size, + u64 flags, bool may_fault) +{ + bool user_build_id = flags & BPF_F_USER_BUILD_ID; + bool user = flags & BPF_F_USER_STACK; + u32 copy_len = trace_nr * elem_size; + + if (user_build_id) + stack_map_get_build_id_offset(buf, trace_nr, user, may_fault); + + if (size > copy_len) + memset(buf + copy_len, 0, size - copy_len); + return copy_len; +} + static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, struct perf_callchain_entry *trace_in, void *buf, u32 size, u64 flags, bool may_fault) { - u32 trace_nr, copy_len, elem_size, max_depth; bool user_build_id = flags & BPF_F_USER_BUILD_ID; bool crosstask = task && task != current; u32 skip = flags & BPF_F_SKIP_FIELD_MASK; bool user = flags & BPF_F_USER_STACK; struct perf_callchain_entry *trace; + u32 trace_nr, elem_size, max_depth; bool kernel = !user; int err = -EINVAL; @@ -821,18 +836,12 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, } trace_nr = callchain_store(trace, buf, elem_size, flags); - copy_len = trace_nr * elem_size; /* trace should not be dereferenced after this point */ if (may_fault) rcu_read_unlock(); - if (user_build_id) - stack_map_get_build_id_offset(buf, trace_nr, user, may_fault); - - if (size > copy_len) - memset(buf + copy_len, 0, size - copy_len); - return copy_len; + return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault); err_fault: err = -EFAULT; -- cgit v1.2.3 From 58cfc2201d964163fe9c4a703136eb64db799f08 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 3 Aug 2026 23:01:45 +0200 Subject: bpf: Remove trace_in argument from __bpf_get_stack Now with the new callchain_* helper functions we can process trace_in case directly in bpf_get_stack_pe function and remove it from __bpf_get_stack which makes things easier for preemption fix in following change. Signed-off-by: Jiri Olsa Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260803210149.296496-9-jolsa@kernel.org --- kernel/bpf/stackmap.c | 49 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 14 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index cdeb6c2e50da..976c4e4c1af6 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -780,7 +780,6 @@ static long callchain_finalize(void *buf, u32 size, u32 trace_nr, u32 elem_size, } static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, - struct perf_callchain_entry *trace_in, void *buf, u32 size, u64 flags, bool may_fault) { bool user_build_id = flags & BPF_F_USER_BUILD_ID; @@ -819,10 +818,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, if (may_fault) rcu_read_lock(); /* need RCU for perf's callchain below */ - if (trace_in) { - trace = trace_in; - trace->nr = min_t(u32, trace->nr, max_depth); - } else if (kernel && task) { + if (kernel && task) { trace = get_callchain_entry_for_task(task, max_depth); } else { trace = get_perf_callchain(regs, kernel, user, max_depth, @@ -853,7 +849,7 @@ clear: BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size, u64, flags) { - return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */); + return __bpf_get_stack(regs, NULL, buf, size, flags, false /* !may_fault */); } const struct bpf_func_proto bpf_get_stack_proto = { @@ -869,7 +865,7 @@ const struct bpf_func_proto bpf_get_stack_proto = { BPF_CALL_4(bpf_get_stack_sleepable, struct pt_regs *, regs, void *, buf, u32, size, u64, flags) { - return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, true /* may_fault */); + return __bpf_get_stack(regs, NULL, buf, size, flags, true /* may_fault */); } const struct bpf_func_proto bpf_get_stack_sleepable_proto = { @@ -893,7 +889,7 @@ static long __bpf_get_task_stack(struct task_struct *task, void *buf, u32 size, regs = task_pt_regs(task); if (regs) - res = __bpf_get_stack(regs, task, NULL, buf, size, flags, may_fault); + res = __bpf_get_stack(regs, task, buf, size, flags, may_fault); put_task_stack(task); return res; @@ -933,6 +929,32 @@ const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = { .arg4_type = ARG_ANYTHING, }; +static int __bpf_get_stack_pe(struct perf_callchain_entry *trace, void *buf, u32 size, + u64 flags) +{ + bool user_build_id = flags & BPF_F_USER_BUILD_ID; + u64 skip = flags & BPF_F_SKIP_FIELD_MASK; + bool user = flags & BPF_F_USER_STACK; + u32 elem_size, max_depth, nr_trace; + bool kernel = !user; + + if (kernel && user_build_id) + return -EINVAL; + + elem_size = user_build_id ? sizeof(struct bpf_stack_build_id) : sizeof(u64); + if (unlikely(size % elem_size)) + return -EINVAL; + + max_depth = stack_map_calculate_max_depth(size, elem_size, flags); + trace->nr = min_t(u32, trace->nr, max_depth); + + if (trace->nr < skip) + return -EFAULT; + + nr_trace = callchain_store(trace, buf, elem_size, flags); + return callchain_finalize(buf, size, nr_trace, elem_size, flags, false /* !may_fault */); +} + BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, void *, buf, u32, size, u64, flags) { @@ -944,7 +966,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, __u64 nr_kernel; if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)) - return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */); + return __bpf_get_stack(regs, NULL, buf, size, flags, false /* !may_fault */); if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_USER_BUILD_ID))) @@ -964,7 +986,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, __u64 nr = trace->nr; trace->nr = nr_kernel; - err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */); + err = __bpf_get_stack_pe(trace, buf, size, flags); /* restore nr */ trace->nr = nr; @@ -974,14 +996,13 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, skip += nr_kernel; if (skip > BPF_F_SKIP_FIELD_MASK) goto clear; - flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; - err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */); + err = __bpf_get_stack_pe(trace, buf, size, flags); } - return err; clear: - memset(buf, 0, size); + if (err < 0) + memset(buf, 0, size); return err; } -- cgit v1.2.3 From f5d242825ca417bb6afe35fde6e8880f97ca43fb Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 3 Aug 2026 23:01:46 +0200 Subject: bpf: Clear buf on error in __bpf_get_task_stack Both bpf_get_task_stack and bpf_get_task_stack_sleepable helpers that use __bpf_get_task_stack have buf defined as ARG_PTR_TO_UNINIT_MEM argument and we should initialize the buf on every return path. Adding missing buf memset for __bpf_get_task_stack fail paths. This provides deterministic buffer contents, which is useful when the buffer is used directly as a map key. Fixes: 06ab134ce8ec ("bpf: Refcount task stack in bpf_get_task_stack") Fixes: b992f01e6615 ("bpf: Guard against accessing NULL pt_regs in bpf_get_task_stack()") Reported-by: Sashiko Signed-off-by: Jiri Olsa Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260803210149.296496-10-jolsa@kernel.org --- kernel/bpf/stackmap.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 976c4e4c1af6..7f728d319a65 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -884,14 +884,17 @@ static long __bpf_get_task_stack(struct task_struct *task, void *buf, u32 size, struct pt_regs *regs; long res = -EINVAL; - if (!try_get_task_stack(task)) + if (!try_get_task_stack(task)) { + memset(buf, 0, size); return -EFAULT; + } regs = task_pt_regs(task); if (regs) res = __bpf_get_stack(regs, task, buf, size, flags, may_fault); + else + memset(buf, 0, size); put_task_stack(task); - return res; } -- cgit v1.2.3 From b1a47b2708d4e95dbd23aee2ec83752190897b3f Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Mon, 3 Aug 2026 23:01:47 +0200 Subject: bpf: Disable preemption in __bpf_get_stack get_perf_callchain() returns a per-CPU perf_callchain_entry buffer and releases its recursion slot via put_callchain_entry() before returning, so nothing keeps the entry reserved while __bpf_get_stack() consumes it below. A preemptible BPF program (e.g. a non-sleepable raw tracepoint program on a PREEMPT kernel, which runs under migrate_disable() but not preempt_disable()) can be scheduled out between obtaining the entry and the copy. Another task scheduled on the same CPU then reuses the same per-CPU buffer and overwrites trace->nr with a larger value. copy_len is then computed from the inflated trace->nr and can exceed the caller's buffer, causing an out-of-bounds write in the memcpy() and in the build_id path. The rcu_read_lock() taken here alone does not prevent this. It is only taken on the may_fault path, and under CONFIG_PREEMPT_RCU it does not disable preemption; it merely keeps perf's callchain buffer array alive (freed via call_rcu()) and does nothing to stop another task from reusing the entry. Disable preemption around obtaining the callchain entry and copying it into the caller's buffer, so the entry cannot be reused underneath us and trace->nr stays bounded by max_depth. Build ID resolution may fault and is therefore deferred until after preemption is re-enabled; by then the instruction pointers have already been copied into buf, so it operates only on that private copy. Note, preempt_disable() also subsumes the buffer-lifetime guarantee the rcu_read_lock() provided, since a preempt-disabled section is an RCU read-side critical section for the callchain buffers' call_rcu() reclaim. Fixes: c195651e565a ("bpf: add bpf_get_stack helper") Reported-by: Tao Chen Reported-by: STAR Labs SG Signed-off-by: Daniel Borkmann Signed-off-by: Jiri Olsa Signed-off-by: Andrii Nakryiko Cc: stable@vger.kernel.org Link: https://lore.kernel.org/bpf/20260803210149.296496-11-jolsa@kernel.org Closes: https://lore.kernel.org/bpf/20260206090653.1336687-1-chen.dylane@linux.dev/ [ changed Fixes: commit ] --- kernel/bpf/stackmap.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 7f728d319a65..121caa90707b 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -815,6 +815,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, max_depth = stack_map_calculate_max_depth(size, elem_size, flags); + preempt_disable(); if (may_fault) rcu_read_lock(); /* need RCU for perf's callchain below */ @@ -828,6 +829,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, if (unlikely(!trace) || trace->nr < skip) { if (may_fault) rcu_read_unlock(); + preempt_enable(); goto err_fault; } @@ -836,6 +838,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, /* trace should not be dereferenced after this point */ if (may_fault) rcu_read_unlock(); + preempt_enable(); return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault); -- cgit v1.2.3 From 347c1d722e3ea4ffa2850585f5c76c6e551eac83 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 3 Aug 2026 23:01:48 +0200 Subject: bpf: Avoid changing callchain in bpf_get_stack_pe There's no need to modify the trace object bpf_get_stack_pe, we just need to pass the needed callchain length in separate argument. This way we can have callchain pointers const and remove the trace->nr modification and restoration. Assisted-by: Codex:GPT-5.5 Signed-off-by: Jiri Olsa Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260803210149.296496-12-jolsa@kernel.org --- kernel/bpf/stackmap.c | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 121caa90707b..57a25b56d0ac 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -661,7 +661,7 @@ const struct bpf_func_proto bpf_get_stackid_proto = { .arg3_type = ARG_ANYTHING, }; -static __u64 count_kernel_ip(struct perf_callchain_entry *trace) +static __u64 count_kernel_ip(const struct perf_callchain_entry *trace) { __u64 nr_kernel = 0; @@ -741,15 +741,15 @@ const struct bpf_func_proto bpf_get_stackid_proto_pe = { .arg3_type = ARG_ANYTHING, }; -static u32 callchain_store(struct perf_callchain_entry *trace, void *buf, - u32 elem_size, u64 flags) +static u32 callchain_store(const struct perf_callchain_entry *trace, u32 trace_nr, + void *buf, u32 elem_size, u64 flags) { bool user_build_id = flags & BPF_F_USER_BUILD_ID; u32 skip = flags & BPF_F_SKIP_FIELD_MASK; - u32 trace_nr, copy_len; - u64 *ips; + const u64 *ips; + u32 copy_len; - trace_nr = trace->nr - skip; + trace_nr = trace_nr - skip; copy_len = trace_nr * elem_size; ips = trace->ip + skip; @@ -833,7 +833,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, goto err_fault; } - trace_nr = callchain_store(trace, buf, elem_size, flags); + trace_nr = callchain_store(trace, trace->nr, buf, elem_size, flags); /* trace should not be dereferenced after this point */ if (may_fault) @@ -935,8 +935,8 @@ const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = { .arg4_type = ARG_ANYTHING, }; -static int __bpf_get_stack_pe(struct perf_callchain_entry *trace, void *buf, u32 size, - u64 flags) +static int __bpf_get_stack_pe(const struct perf_callchain_entry *trace, u32 trace_nr, + void *buf, u32 size, u64 flags) { bool user_build_id = flags & BPF_F_USER_BUILD_ID; u64 skip = flags & BPF_F_SKIP_FIELD_MASK; @@ -952,12 +952,12 @@ static int __bpf_get_stack_pe(struct perf_callchain_entry *trace, void *buf, u32 return -EINVAL; max_depth = stack_map_calculate_max_depth(size, elem_size, flags); - trace->nr = min_t(u32, trace->nr, max_depth); + trace_nr = min_t(u32, trace_nr, max_depth); - if (trace->nr < skip) + if (trace_nr < skip) return -EFAULT; - nr_trace = callchain_store(trace, buf, elem_size, flags); + nr_trace = callchain_store(trace, trace_nr, buf, elem_size, flags); return callchain_finalize(buf, size, nr_trace, elem_size, flags, false /* !may_fault */); } @@ -965,8 +965,8 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, void *, buf, u32, size, u64, flags) { struct pt_regs *regs = (struct pt_regs *)(ctx->regs); + const struct perf_callchain_entry *trace; struct perf_event *event = ctx->event; - struct perf_callchain_entry *trace; bool kernel, user; int err = -EINVAL; __u64 nr_kernel; @@ -989,13 +989,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, nr_kernel = count_kernel_ip(trace); if (kernel) { - __u64 nr = trace->nr; - - trace->nr = nr_kernel; - err = __bpf_get_stack_pe(trace, buf, size, flags); - - /* restore nr */ - trace->nr = nr; + err = __bpf_get_stack_pe(trace, nr_kernel, buf, size, flags); } else { /* user */ u64 skip = flags & BPF_F_SKIP_FIELD_MASK; @@ -1003,7 +997,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, if (skip > BPF_F_SKIP_FIELD_MASK) goto clear; flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; - err = __bpf_get_stack_pe(trace, buf, size, flags); + err = __bpf_get_stack_pe(trace, trace->nr, buf, size, flags); } clear: -- cgit v1.2.3 From a74594607a0b310c1533e713b9097cc1cbbb85bf Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Mon, 3 Aug 2026 23:01:49 +0200 Subject: bpf: Avoid changing callchain in bpf_get_stackid_pe There's no need to modify the trace object bpf_get_stackid_pe, we just need to pass the needed callchain length in separate argument. This way we can have callchain pointers const and remove the trace->nr modification and restoration. Assisted-by: Codex:GPT-5.5 Signed-off-by: Jiri Olsa Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260803210149.296496-13-jolsa@kernel.org --- kernel/bpf/stackmap.c | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 57a25b56d0ac..8f0f3ff1a869 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -506,7 +506,7 @@ get_callchain_entry_for_task(struct task_struct *task, u32 max_depth) struct stackid { struct stack_map_bucket *bucket; - u64 *ips; + const u64 *ips; u32 nr; u32 len; u32 hash; @@ -515,21 +515,21 @@ struct stackid { }; static int stackid_init(struct stackid *stackid, struct bpf_map *map, - struct perf_callchain_entry *trace, u64 flags) + const struct perf_callchain_entry *trace, u32 trace_nr, u64 flags) { struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); u32 skip = flags & BPF_F_SKIP_FIELD_MASK; u32 max_depth; - if (trace->nr <= skip) + if (trace_nr <= skip) /* skipping more than usable stack trace */ return -EFAULT; max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags); - stackid->nr = min_t(u32, trace->nr - skip, max_depth - skip); + stackid->nr = min_t(u32, trace_nr - skip, max_depth - skip); stackid->len = stackid->nr * sizeof(u64); stackid->ips = trace->ip + skip; - stackid->hash = jhash2((u32 *)stackid->ips, stackid->len / sizeof(u32), 0); + stackid->hash = jhash2((const u32 *)stackid->ips, stackid->len / sizeof(u32), 0); stackid->id = stackid->hash & (smap->n_buckets - 1); stackid->bucket = READ_ONCE(smap->buckets[stackid->id]); stackid->hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash; @@ -537,11 +537,12 @@ static int stackid_init(struct stackid *stackid, struct bpf_map *map, } static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map, - struct perf_callchain_entry *trace, u64 flags) + const struct perf_callchain_entry *trace, u32 trace_nr, + u64 flags) { int err; - err = stackid_init(stackid, map, trace, flags); + err = stackid_init(stackid, map, trace, trace_nr, flags); if (err) return err; @@ -640,7 +641,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, /* couldn't fetch the stack trace */ return -EFAULT; - err = stackid_fastpath(&stackid, map, trace, flags); + err = stackid_fastpath(&stackid, map, trace, trace->nr, flags); if (err != -ENOENT) return err; @@ -676,12 +677,13 @@ static __u64 count_kernel_ip(const struct perf_callchain_entry *trace) BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, struct bpf_map *, map, u64, flags) { + const struct perf_callchain_entry *trace; struct perf_event *event = ctx->event; struct stack_map_bucket *new_bucket; - struct perf_callchain_entry *trace; struct stackid stackid; bool kernel, user; __u64 nr_kernel; + u32 trace_nr; int ret; /* perf_sample_data doesn't have callchain, use bpf_get_stackid */ @@ -701,13 +703,13 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, return -EFAULT; nr_kernel = count_kernel_ip(trace); - __u64 nr = trace->nr; /* save original */ if (kernel) { - trace->nr = nr_kernel; + trace_nr = nr_kernel; } else { /* user */ u64 skip = flags & BPF_F_SKIP_FIELD_MASK; + trace_nr = trace->nr; skip += nr_kernel; if (skip > BPF_F_SKIP_FIELD_MASK) return -EFAULT; @@ -715,21 +717,14 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; } - ret = stackid_fastpath(&stackid, map, trace, flags); + ret = stackid_fastpath(&stackid, map, trace, trace_nr, flags); if (ret != -ENOENT) - goto out; + return ret; new_bucket = stackid_new_bucket(&stackid, map); - if (new_bucket) { - trace->nr = nr; + if (new_bucket) return stackid_install(&stackid, map, new_bucket, flags); - } - ret = -ENOMEM; - -out: - /* restore nr */ - trace->nr = nr; - return ret; + return -ENOMEM; } const struct bpf_func_proto bpf_get_stackid_proto_pe = { -- cgit v1.2.3 From 00244bdaa423d93f4571f3f6854378ce3365e524 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Wed, 5 Aug 2026 23:08:09 +0800 Subject: bpf: Fix sleepable check for tracing/lsm prog When CONFIG_FUNCTION_ERROR_INJECTION is disabled, a sleepable tracing prog is allowed to attach to '__x64_'-alike prefix symbols. It is because the verifier does not verify whether the symbol is a kernel function or a bpf prog. That said, a sleepable tracing prog is allowed to attach to a bpf prog target whose name has '__x64_'-alike prefix. For example, a sleepable fentry prog attaches to a '__x64_sys_nop' XDP prog, and copies buffer from a user pointer with bpf_copy_from_user() helper. After attaching the XDP prog to lo interface, the kernel BUG could be triggered by 'ping -c 1 -W 1 127.0.0.1': [ 3.460756] BUG: sleeping function called from invalid context at kernel/bpf/trampoline.c:1324 Fix it by disallowing sleepable prog always when its target btf is not a kernel's btf. Fixes: 16d9c5660692 ("bpf: Always allow sleepable programs on syscalls") Signed-off-by: Leon Hwang Signed-off-by: Andrii Nakryiko Acked-by: Viktor Malik Link: https://lore.kernel.org/bpf/20260805150810.34907-2-leon.hwang@linux.dev --- kernel/bpf/verifier.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 4db151e24355..d925197c2e5f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -19022,6 +19022,9 @@ static int btf_id_allow_sleepable(u32 btf_id, unsigned long addr, const struct b const struct btf_type *t; const char *tname; + if (!btf_is_kernel(btf)) + return -EINVAL; + switch (prog->type) { case BPF_PROG_TYPE_TRACING: t = btf_type_by_id(btf, btf_id); -- cgit v1.2.3 From 11c1e836710dcba03e50454a4eedfdbaf8d3050e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Jean?= Date: Wed, 5 Aug 2026 06:02:28 +0000 Subject: bpf: Harden bloom filter sizing and indexing on 32-bit kernels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bloom_map_alloc() has two 32-bit-specific problems when the computed bitmap reaches the U32_MAX fallback case. First, BITS_TO_BYTES(U32_MAX) is evaluated with 32-bit arithmetic. The addition performed by DIV_ROUND_UP wraps, so the map allocates only the fixed-size bloom filter object while keeping bitset_mask == U32_MAX. Subsequent updates can then write past the allocated object. Second, fixing only the allocation size is not sufficient. The bloom hash is a u32, but set_bit() takes a signed long bit number and x86 test_bit() eventually feeds the index to variable_test_bit(long, ...). On 32-bit kernels, hashes in [0x80000000, U32_MAX] therefore become negative bit offsets. x86 bt/bts with a memory operand interpret those offsets relative to the supplied base, so a map with bitset_mask == U32_MAX can read or write before bloom->bitset even after allocating the full 512 MiB bitmap. Keep the U32_MAX fallback, but split each hash into a word pointer and an in-word bit number before calling test_bit() or set_bit(). The bitops argument is then always in [0, BITS_PER_LONG - 1], while BIT_WORD(h) still selects the intended word in the full bitmap. Compute the bitset size from (u64)bitset_mask + 1 before passing the final size to bpf_map_area_alloc(). This fixes the original under-allocation and keeps the allocated storage consistent with the addressable bitset. Exploitation note: local privilege escalation is possible on a 32-bit x86 kernel using the under-allocation bug from a binary with CAP_BPF. Fixes: 9330986c0300 ("bpf: Add bloom filter map implementation") Signed-off-by: Jérémy Jean Signed-off-by: Andrii Nakryiko Cc: stable@vger.kernel.org Link: https://lore.kernel.org/bpf/20260805060228.2703051-1-Jeremy.Jean@oss.cyber.gouv.fr Assisted-by: Codex:gpt-5 --- kernel/bpf/bloom_filter.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/bloom_filter.c b/kernel/bpf/bloom_filter.c index b73336c976b7..c6e7559b07de 100644 --- a/kernel/bpf/bloom_filter.c +++ b/kernel/bpf/bloom_filter.c @@ -41,7 +41,7 @@ static long bloom_map_peek_elem(struct bpf_map *map, void *value) for (i = 0; i < bloom->nr_hash_funcs; i++) { h = hash(bloom, value, map->value_size, i); - if (!test_bit(h, bloom->bitset)) + if (!test_bit(h % BITS_PER_LONG, bloom->bitset + BIT_WORD(h))) return -ENOENT; } @@ -57,9 +57,13 @@ static long bloom_map_push_elem(struct bpf_map *map, void *value, u64 flags) if (flags != BPF_ANY) return -EINVAL; + /* + * On 32-bit architectures, hashes larger than INT_MAX would be + * treated as negative by set_bit(). + */ for (i = 0; i < bloom->nr_hash_funcs; i++) { h = hash(bloom, value, map->value_size, i); - set_bit(h, bloom->bitset); + set_bit(h % BITS_PER_LONG, bloom->bitset + BIT_WORD(h)); } return 0; @@ -94,9 +98,10 @@ static int bloom_map_alloc_check(union bpf_attr *attr) static struct bpf_map *bloom_map_alloc(union bpf_attr *attr) { - u32 bitset_bytes, bitset_mask, nr_hash_funcs, nr_bits; + u32 bitset_mask, nr_hash_funcs, nr_bits; int numa_node = bpf_map_attr_numa_node(attr); struct bpf_bloom_filter *bloom; + u64 bitset_bytes; if (attr->key_size != 0 || attr->value_size == 0 || attr->max_entries == 0 || @@ -127,22 +132,16 @@ static struct bpf_map *bloom_map_alloc(union bpf_attr *attr) if (check_mul_overflow(attr->max_entries, nr_hash_funcs, &nr_bits) || check_mul_overflow(nr_bits / 5, (u32)7, &nr_bits) || nr_bits > (1UL << 31)) { - /* The bit array size is 2^32 bits but to avoid overflowing the - * u32, we use U32_MAX, which will round up to the equivalent - * number of bytes - */ - bitset_bytes = BITS_TO_BYTES(U32_MAX); bitset_mask = U32_MAX; } else { if (nr_bits <= BITS_PER_LONG) nr_bits = BITS_PER_LONG; else nr_bits = roundup_pow_of_two(nr_bits); - bitset_bytes = BITS_TO_BYTES(nr_bits); bitset_mask = nr_bits - 1; } - bitset_bytes = roundup(bitset_bytes, sizeof(unsigned long)); + bitset_bytes = BITS_TO_LONGS((u64)bitset_mask + 1) * sizeof(unsigned long); bloom = bpf_map_area_alloc(sizeof(*bloom) + bitset_bytes, numa_node); if (!bloom) -- cgit v1.2.3 From ed3b3093b6242bdb2c4acfb932d4b15db4e33948 Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Wed, 5 Aug 2026 23:33:38 +0800 Subject: bpf: Add KF_SPINLOCK_SAFE flag for kfuncs under bpf_spin_lock Introduce the KF_SPINLOCK_SAFE kfunc metadata flag in BTF so kfuncs may be explicitly marked as safe to call while holding bpf_spin_lock. Allow kfuncs defined in kernel modules to be marked with KF_SPINLOCK_SAFE. Example: BTF_ID_FLAGS(func, $kfunc_name, KF_SPINLOCK_SAFE) Signed-off-by: Kaitao Cheng Acked-by: Leon Hwang Link: https://lore.kernel.org/bpf/20260805153340.34776-2-kaitao.cheng@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d925197c2e5f..8daba32306be 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11837,11 +11837,21 @@ static bool is_bpf_stream_kfunc(u32 btf_id) btf_id == special_kfunc_list[KF_bpf_stream_print_stack]; } -static bool kfunc_spin_allowed(u32 btf_id) +static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 offset) { - return is_bpf_graph_api_kfunc(btf_id) || is_bpf_iter_num_api_kfunc(btf_id) || - is_bpf_res_spin_lock_kfunc(btf_id) || is_bpf_arena_kfunc(btf_id) || - is_bpf_stream_kfunc(btf_id); + struct bpf_kfunc_meta kfunc; + int err; + + if (is_bpf_graph_api_kfunc(func_id) || is_bpf_iter_num_api_kfunc(func_id) || + is_bpf_res_spin_lock_kfunc(func_id) || is_bpf_arena_kfunc(func_id) || + is_bpf_stream_kfunc(func_id)) + return true; + + err = fetch_kfunc_meta(env, func_id, offset, &kfunc); + if (err || !kfunc.flags) + return false; + + return *kfunc.flags & KF_SPINLOCK_SAFE; } static bool is_sync_callback_calling_kfunc(u32 btf_id) @@ -17420,7 +17430,7 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) insn->imm != BPF_FUNC_spin_unlock && insn->imm != BPF_FUNC_kptr_xchg) || (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && - (insn->off != 0 || !kfunc_spin_allowed(insn->imm)))) { + !kfunc_spin_allowed(env, insn->imm, insn->off))) { verbose(env, "function calls are not allowed while holding a lock\n"); return -EINVAL; -- cgit v1.2.3 From 7619a0ee9340b3cef114b1c7aae42c0835cf2bff Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Wed, 5 Aug 2026 23:33:39 +0800 Subject: bpf: Mark existing lock-safe kfuncs with KF_SPINLOCK_SAFE The verifier currently keeps a hard-coded list of kfuncs that may be called while holding a bpf_spin_lock. With KF_SPINLOCK_SAFE available, retaining this list creates two sources of truth and requires verifier changes whenever another lock-safe kfunc is added. Mark every kfunc currently accepted by kfunc_spin_allowed() with KF_SPINLOCK_SAFE. This covers the graph, numeric iterator, resource spin lock, arena, and stream kfuncs. Remove the obsolete category checks and make kfunc_spin_allowed() rely solely on the kfunc registration metadata. This preserves the behavior of existing kfuncs while using the same mechanism for built-in and module kfuncs. Signed-off-by: Kaitao Cheng Acked-by: Leon Hwang Link: https://lore.kernel.org/bpf/20260805153340.34776-3-kaitao.cheng@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/arena.c | 6 +++--- kernel/bpf/helpers.c | 56 +++++++++++++++++++++++++------------------------ kernel/bpf/rqspinlock.c | 8 +++---- kernel/bpf/verifier.c | 32 ---------------------------- 4 files changed, 36 insertions(+), 66 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 555ee2531ef9..7b6847200b43 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -1118,9 +1118,9 @@ __bpf_kfunc int bpf_arena_reserve_pages(void *p__map, void *ptr__ign, u32 page_c __bpf_kfunc_end_defs(); BTF_KFUNCS_START(arena_kfuncs) -BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_ARENA_RET | KF_ARENA_ARG2) -BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_ARENA_ARG2) -BTF_ID_FLAGS(func, bpf_arena_reserve_pages, KF_ARENA_ARG2) +BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_ARENA_RET | KF_ARENA_ARG2 | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_ARENA_ARG2 | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_arena_reserve_pages, KF_ARENA_ARG2 | KF_SPINLOCK_SAFE) BTF_KFUNCS_END(arena_kfuncs) static const struct btf_kfunc_id_set common_kfunc_set = { diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 4709a5ad0474..6388b6b23e49 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -4812,30 +4812,32 @@ BTF_ID_FLAGS(func, bpf_obj_drop, KF_RELEASE | KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_obj_drop_impl, KF_RELEASE) BTF_ID_FLAGS(func, bpf_percpu_obj_drop, KF_RELEASE | KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_percpu_obj_drop_impl, KF_RELEASE) -BTF_ID_FLAGS(func, bpf_refcount_acquire, KF_ACQUIRE | KF_RET_NULL | KF_RCU | KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_refcount_acquire_impl, KF_ACQUIRE | KF_RET_NULL | KF_RCU) -BTF_ID_FLAGS(func, bpf_list_push_front, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_list_push_front_impl) -BTF_ID_FLAGS(func, bpf_list_push_back, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_list_push_back_impl) -BTF_ID_FLAGS(func, bpf_list_add, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_list_pop_front, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_pop_back, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_del, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_front, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_back, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_is_first) -BTF_ID_FLAGS(func, bpf_list_is_last) -BTF_ID_FLAGS(func, bpf_list_empty) +BTF_ID_FLAGS(func, bpf_refcount_acquire, + KF_ACQUIRE | KF_RET_NULL | KF_RCU | KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_refcount_acquire_impl, + KF_ACQUIRE | KF_RET_NULL | KF_RCU | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_push_front, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_push_front_impl, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_push_back, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_push_back_impl, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_add, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_pop_front, KF_ACQUIRE | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_pop_back, KF_ACQUIRE | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_del, KF_ACQUIRE | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_front, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_back, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_is_first, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_is_last, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_empty, KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_task_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_task_release, KF_RELEASE) -BTF_ID_FLAGS(func, bpf_rbtree_remove, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_rbtree_add, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_rbtree_add_impl) -BTF_ID_FLAGS(func, bpf_rbtree_first, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_rbtree_root, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_rbtree_left, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_rbtree_right, KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_rbtree_remove, KF_ACQUIRE | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_add, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_add_impl, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_first, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_root, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_left, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_right, KF_RET_NULL | KF_SPINLOCK_SAFE) #ifdef CONFIG_CGROUPS BTF_ID_FLAGS(func, bpf_cgroup_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL) @@ -4885,9 +4887,9 @@ BTF_ID_FLAGS(func, bpf_rcu_read_lock) BTF_ID_FLAGS(func, bpf_rcu_read_unlock) BTF_ID_FLAGS(func, bpf_dynptr_slice, KF_RET_NULL) BTF_ID_FLAGS(func, bpf_dynptr_slice_rdwr, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_iter_num_new, KF_ITER_NEW) -BTF_ID_FLAGS(func, bpf_iter_num_next, KF_ITER_NEXT | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_iter_num_destroy, KF_ITER_DESTROY) +BTF_ID_FLAGS(func, bpf_iter_num_new, KF_ITER_NEW | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_iter_num_next, KF_ITER_NEXT | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_iter_num_destroy, KF_ITER_DESTROY | KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_iter_task_vma_new, KF_ITER_NEW | KF_RCU) BTF_ID_FLAGS(func, bpf_iter_task_vma_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_task_vma_destroy, KF_ITER_DESTROY) @@ -4962,8 +4964,8 @@ BTF_ID_FLAGS(func, bpf_strncasestr); #if defined(CONFIG_BPF_LSM) && defined(CONFIG_CGROUPS) BTF_ID_FLAGS(func, bpf_cgroup_read_xattr, KF_RCU) #endif -BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_task_work_schedule_signal, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_task_work_schedule_resume, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_dynptr_from_file) diff --git a/kernel/bpf/rqspinlock.c b/kernel/bpf/rqspinlock.c index e4e338cdb437..e527cb425cf4 100644 --- a/kernel/bpf/rqspinlock.c +++ b/kernel/bpf/rqspinlock.c @@ -744,10 +744,10 @@ __bpf_kfunc void bpf_res_spin_unlock_irqrestore(struct bpf_res_spin_lock *lock, __bpf_kfunc_end_defs(); BTF_KFUNCS_START(rqspinlock_kfunc_ids) -BTF_ID_FLAGS(func, bpf_res_spin_lock, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_res_spin_unlock) -BTF_ID_FLAGS(func, bpf_res_spin_lock_irqsave, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_res_spin_unlock_irqrestore) +BTF_ID_FLAGS(func, bpf_res_spin_lock, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_res_spin_unlock, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_res_spin_lock_irqsave, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_res_spin_unlock_irqrestore, KF_SPINLOCK_SAFE) BTF_KFUNCS_END(rqspinlock_kfunc_ids) static const struct btf_kfunc_id_set rqspinlock_kfunc_set = { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8daba32306be..d952bd95cbb7 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11802,20 +11802,6 @@ static bool is_bpf_rbtree_api_kfunc(u32 btf_id) btf_id == special_kfunc_list[KF_bpf_rbtree_right]; } -static bool is_bpf_iter_num_api_kfunc(u32 btf_id) -{ - return btf_id == special_kfunc_list[KF_bpf_iter_num_new] || - btf_id == special_kfunc_list[KF_bpf_iter_num_next] || - btf_id == special_kfunc_list[KF_bpf_iter_num_destroy]; -} - -static bool is_bpf_graph_api_kfunc(u32 btf_id) -{ - return is_bpf_list_api_kfunc(btf_id) || - is_bpf_rbtree_api_kfunc(btf_id) || - is_bpf_refcount_acquire_kfunc(btf_id); -} - static bool is_bpf_res_spin_lock_kfunc(u32 btf_id) { return btf_id == special_kfunc_list[KF_bpf_res_spin_lock] || @@ -11824,29 +11810,11 @@ static bool is_bpf_res_spin_lock_kfunc(u32 btf_id) btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]; } -static bool is_bpf_arena_kfunc(u32 btf_id) -{ - return btf_id == special_kfunc_list[KF_bpf_arena_alloc_pages] || - btf_id == special_kfunc_list[KF_bpf_arena_free_pages] || - btf_id == special_kfunc_list[KF_bpf_arena_reserve_pages]; -} - -static bool is_bpf_stream_kfunc(u32 btf_id) -{ - return btf_id == special_kfunc_list[KF_bpf_stream_vprintk] || - btf_id == special_kfunc_list[KF_bpf_stream_print_stack]; -} - static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 offset) { struct bpf_kfunc_meta kfunc; int err; - if (is_bpf_graph_api_kfunc(func_id) || is_bpf_iter_num_api_kfunc(func_id) || - is_bpf_res_spin_lock_kfunc(func_id) || is_bpf_arena_kfunc(func_id) || - is_bpf_stream_kfunc(func_id)) - return true; - err = fetch_kfunc_meta(env, func_id, offset, &kfunc); if (err || !kfunc.flags) return false; -- cgit v1.2.3 From d65739bf93be5160c1e0af00064874bbe262d5b6 Mon Sep 17 00:00:00 2001 From: Ning Ding Date: Wed, 5 Aug 2026 16:39:33 -0700 Subject: bpf: Account for preempt and IRQ state in RCU protection Disabling preemption or local IRQs keeps the current CPU in an RCU read-side critical section, but in_rcu_cs() does not account for either state. The verifier therefore rejects safe kptr accesses and invalidates pointers when another RCU source ends. Include preemption-disabled and IRQ-disabled state in in_rcu_cs(). Invalidate RCU-protected pointers on RCU unlock, preempt enable, or IRQ restore only after the final protection ends. Signed-off-by: Ning Ding Link: https://lore.kernel.org/bpf/20260805233940.3966981-2-dingning04@gmail.com [ kkd: Simplify was_in_rcu_cs on spin unlock and adjust the selftest. ] Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d952bd95cbb7..e6233c0081d1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4452,7 +4452,9 @@ static bool in_sleepable(struct bpf_verifier_env *env) static bool in_rcu_cs(struct bpf_verifier_env *env) { return env->cur_state->active_rcu_locks || + env->cur_state->active_preempt_locks || env->cur_state->active_locks || + env->cur_state->active_irq_id || !in_sleepable(env); } @@ -7166,7 +7168,6 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state return err; } } else { - bool was_in_rcu_cs; void *ptr; int type; @@ -7194,12 +7195,11 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state verbose(env, "%s_unlock cannot be out of order\n", lock_str); return -EINVAL; } - was_in_rcu_cs = in_rcu_cs(env); if (release_lock_state(cur, type, reg->id, ptr)) { verbose(env, "%s_unlock of different lock\n", lock_str); return -EINVAL; } - if (was_in_rcu_cs && !in_rcu_cs(env)) + if (!in_rcu_cs(env)) invalidate_rcu_protected_refs(env); invalidate_non_owning_refs(env); @@ -11663,6 +11663,9 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * err = unmark_stack_slot_irq_flag(env, reg, kfunc_class); if (err) return err; + + if (!in_rcu_cs(env)) + invalidate_rcu_protected_refs(env); } return 0; } @@ -13159,7 +13162,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); return -EINVAL; } - if (--env->cur_state->active_rcu_locks == 0) + env->cur_state->active_rcu_locks--; + if (!in_rcu_cs(env)) invalidate_rcu_protected_refs(env); } else if (preempt_disable) { env->cur_state->active_preempt_locks++; @@ -13169,6 +13173,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return -EINVAL; } env->cur_state->active_preempt_locks--; + if (!in_rcu_cs(env)) + invalidate_rcu_protected_refs(env); } if (sleepable && !in_sleepable_context(env)) { -- cgit v1.2.3 From 7db0a00445f1a40bacfe9b747405c11cb5f10fc9 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Thu, 6 Aug 2026 22:10:42 +0200 Subject: bpf: Reject load-acquire from pointers requiring fault protection A BPF_LOAD_ACQ is not rewritten to a BPF_PROBE_MEM load by the verifier, unlike a regular BPF_LDX, so the JIT emits a plain load with no exception table entry and a fault panics the kernel instead of being handled. Reject the source pointer types that a BPF_LDX would have had that fault protection applied to, i.e. the ones bpf_convert_ctx_accesses() turns into BPF_PROBE_MEM: a bare PTR_TO_BTF_ID, PTR_TO_BTF_ID | PTR_UNTRUSTED, PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED and PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED. This is reachable e.g. by loading ->mm out of a trusted task_struct yields an untrusted pointer to mm_struct, and it is NULL for a kernel thread: [...] SEC("tp_btf/sched_switch") int BPF_PROG(demo, bool preempt, struct task_struct *prev, struct task_struct *next) { struct mm_struct *mm = next->mm; /* untrusted */ out_ldx = (__u64)mm->pgd; /* BPF_LDX */ out_acq = load_acquire(&mm->pgd); /* BPF_LOAD_ACQ */ return 0; } [...] Both dereference the same pointer, but only the BPF_LDX is protected (x86-64 JIT, jump targets shown prog-relative): [...] ; out_ldx = (__u64)mm->pgd; 17: movq $-10485760, %r10 1e: movq %rsi, %r11 21: addq $184, %r11 28: subq %r10, %r11 2b: movabsq $140737498841088, %r10 35: cmpq %r10, %r11 38: ja 0x3e <-- kernel addr? 3a: xorl %edi, %edi <-- no: dst = 0, skip the load 3c: jmp 0x45 3e: movq 184(%rsi), %rdi <-- yes: load + extable entry [...] ; load_acquire(&mm->pgd) 53: movq %rsi, %rdi 56: movq 184(%rdi), %rax <-- no check, no extable entry [...] Note that BPF_PROBE_MEM is not visible in a bpftool xlated dump, as bpf_insn_prepare_dump() rewrites it back to BPF_MEM. A PTR_TRUSTED pointer is deliberately not on the list. Such a load is not converted either, but it does not need to be, since the pointer is guaranteed live, so load-acquire from it stays allowed. The check is gated on BPF_LOAD_ACQ so that atomic RMW and store-release error messages are unchanged; writes (RMW / store-release) to such pointers are already rejected elsewhere, so only load-acquire needs this. Fixes: 880442305a39 ("bpf: Introduce load-acquire and store-release instructions") Reported-by: STAR Labs SG Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260806201047.333389-1-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e6233c0081d1..648c5784178e 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4923,6 +4923,30 @@ static bool is_arena_reg(struct bpf_verifier_env *env, int regno) return reg->type == PTR_TO_ARENA; } +static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno, + struct bpf_insn *insn) +{ + const struct bpf_reg_state *reg = reg_state(env, regno); + + /* + * A BPF_LOAD_ACQ is not rewritten to a BPF_PROBE_MEM load by the + * verifier, unlike a regular BPF_LDX. The JIT would emit a plain load + * with no exception table entry, so a fault (e.g. NULL deref) crashes + * the kernel instead of being handled. + * + * Reject the source pointer types that a BPF_LDX would have had that + * fault protection applied to, i.e. the ones bpf_convert_ctx_accesses() + * turns into BPF_PROBE_MEM: a bare PTR_TO_BTF_ID and any PTR_UNTRUSTED + * pointer (untrusted btf ids, untrusted MEM_ALLOC, rdonly untrusted + * memory). A PTR_TRUSTED pointer is not among them, is not converted, + * and stays allowed. Same for the other flagged PTR_TO_BTF_ID variants + * (MEM_ALLOC, MEM_RCU, ...), hence the exact match on the base type. + */ + return insn->imm == BPF_LOAD_ACQ && + (reg->type == PTR_TO_BTF_ID || + (type_flag(reg->type) & PTR_UNTRUSTED)); +} + /* Return false if @regno contains a pointer whose type isn't supported for * atomic instruction @insn. */ @@ -4939,7 +4963,8 @@ static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, return false; if (is_arena_reg(env, regno)) return bpf_jit_supports_insn(insn, true); - + if (is_load_acq_unsafe(env, regno, insn)) + return false; return true; } -- cgit v1.2.3 From 3f562c537e9ecf4bc5e206cfffc2cc047f1b7e94 Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Fri, 7 Aug 2026 10:44:03 +0000 Subject: bpf, cgroup: Fix storage null-ptr-deref after replacing prog Syzkaller reported a storage null-ptr-deref issue after replacing prog. This occurs in the following scenario: 1. prog A, an empty prog, is attached to a cgrp. 2. prog B uses BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE and calls the bpf_get_local_storage helper. 3. link_update is called to replace prog A with prog B. The reason is that __cgroup_bpf_replace fails to alloc and assign the required cgrp storage for the incoming replacement prog. Consequently, the new prog inherits an uninit storage, leading to null-ptr-deref panic when kick the new prog. Fix this by rejecting a link update if new_prog's cgroup storage is incompatible with link->prog. Fixes: 0c991ebc8c69 ("bpf: Implement bpf_prog replacement for an active bpf_cgroup_link") Signed-off-by: Pu Lehui Signed-off-by: Andrii Nakryiko Reviewed-by: Amery Hung Acked-by: Leon Hwang Link: https://lore.kernel.org/bpf/20260728132336.2857800-1-pulehui@huaweicloud.com [0] Link: https://lore.kernel.org/bpf/f87b53c0-8f00-45a6-82db-8242fa9b143f@huaweicloud.com [1] Link: https://lore.kernel.org/bpf/20260807104403.1013064-1-pulehui@huaweicloud.com --- kernel/bpf/cgroup.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index d2da5063d8f8..8fbc942a1cc3 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1026,6 +1026,20 @@ static void replace_effective_prog(struct cgroup *cgrp, } } +static bool cgroup_bpf_storages_compatible(struct bpf_prog *old_prog, + struct bpf_prog *new_prog) +{ + enum bpf_cgroup_storage_type stype; + + for_each_cgroup_storage_type(stype) { + if (old_prog->aux->cgroup_storage[stype] != + new_prog->aux->cgroup_storage[stype]) + return false; + } + + return true; +} + /** * __cgroup_bpf_replace() - Replace link's program and propagate the change * to descendants @@ -1064,6 +1078,9 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp, if (!found) return -ENOENT; + if (!cgroup_bpf_storages_compatible(link->link.prog, new_prog)) + return -EINVAL; + cgrp->bpf.revisions[atype] += 1; old_prog = xchg(&link->link.prog, new_prog); replace_effective_prog(cgrp, atype, pl); -- cgit v1.2.3 From fa9dcacdcdf487f0ffef64bf67622f1caed509f1 Mon Sep 17 00:00:00 2001 From: Sanghyun Park Date: Wed, 5 Aug 2026 12:14:25 +0900 Subject: bpf: Fix mmap_lock leak in irq_work path stack_map_get_build_id_offset() introduced a per-CPU irq_work to defer mmap_read_unlock() from NMI context, and bpf_find_vma() later reused the same mmap_unlock_work. Both callers only check whether the work is busy before taking mmap_lock, so a nested caller can reuse the slot before the first caller queues it. Two read locks may then be acquired while only one deferred unlock runs, leaking a read lock and blocking exit_mmap(). Reserve the per-CPU slot before mmap_read_trylock(). Use the same wrapper in stackmap and bpf_find_vma() so both callers release the reservation on trylock failure. Keep rejecting the slot while the irq_work remains busy. Release it after the irq_work callback unlocks the mm. Fixes: eac9153f2b58 ("bpf/stackmap: Fix deadlock with rq_lock in bpf_get_stack()") Reported-by: syzbot+cdd6c0925e12b0af60cc@syzkaller.appspotmail.com Reported-by: sashiko-bot@kernel.org Signed-off-by: Sanghyun Park Signed-off-by: Andrii Nakryiko Signed-off-by: Daniel Borkmann Closes: https://syzkaller.appspot.com/bug?extid=cdd6c0925e12b0af60cc Closes: https://lore.kernel.org/r/20260630033745.B80201F000E9@smtp.kernel.org Link: https://lore.kernel.org/bpf/20260805031425.2157475-2-sanghyun.park.cnu@gmail.com --- kernel/bpf/mmap_unlock_work.h | 51 ++++++++++++++++++++++++------------------- kernel/bpf/stackmap.c | 28 ++++++++++++++---------- kernel/bpf/task_iter.c | 14 ++++++++---- 3 files changed, 56 insertions(+), 37 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/mmap_unlock_work.h b/kernel/bpf/mmap_unlock_work.h index 5d18d7d85bef..1834db20b861 100644 --- a/kernel/bpf/mmap_unlock_work.h +++ b/kernel/bpf/mmap_unlock_work.h @@ -4,12 +4,15 @@ #ifndef __MMAP_UNLOCK_WORK_H__ #define __MMAP_UNLOCK_WORK_H__ +#include +#include #include /* irq_work to run mmap_read_unlock() in irq_work */ struct mmap_unlock_irq_work { struct irq_work irq_work; struct mm_struct *mm; + atomic_t active; }; DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); @@ -18,32 +21,36 @@ DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); * We cannot do mmap_read_unlock() when the irq is disabled, because of * risk to deadlock with rq_lock. To look up vma when the irqs are * disabled, we need to run mmap_read_unlock() in irq_work. We use a - * percpu variable to do the irq_work. If the irq_work is already used - * by another lookup, we fall over. + * percpu variable to do the irq_work. The active flag reserves the slot + * before mmap_read_trylock() and until the irq_work callback consumes mm. */ -static inline bool bpf_mmap_unlock_get_irq_work(struct mmap_unlock_irq_work **work_ptr) +static inline struct mmap_unlock_irq_work *bpf_mmap_unlock_guard_get(void) { - struct mmap_unlock_irq_work *work = NULL; - bool irq_work_busy = false; + struct mmap_unlock_irq_work *work; - if (irqs_disabled()) { - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { - work = this_cpu_ptr(&mmap_unlock_work); - if (irq_work_is_busy(&work->irq_work)) { - /* cannot queue more up_read, fallback */ - irq_work_busy = true; - } - } else { - /* - * PREEMPT_RT does not allow to trylock mmap sem in - * interrupt disabled context. Force the fallback code. - */ - irq_work_busy = true; - } - } + if (!irqs_disabled()) + return NULL; + + /* + * PREEMPT_RT does not allow to trylock mmap sem in interrupt + * disabled context. Force the fallback code. + */ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + return ERR_PTR(-EBUSY); + + work = this_cpu_ptr(&mmap_unlock_work); + if (irq_work_is_busy(&work->irq_work) || + atomic_cmpxchg_acquire(&work->active, 0, 1)) + return ERR_PTR(-EBUSY); - *work_ptr = work; - return irq_work_busy; + return work; +} + +static inline void +bpf_mmap_unlock_guard_put(struct mmap_unlock_irq_work *work) +{ + if (work) + atomic_set_release(&work->active, 0); } static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm) diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 8f0f3ff1a869..a839041e0d00 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -414,8 +414,7 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, u32 trace_nr, bool user, bool may_fault) { - struct mmap_unlock_irq_work *work = NULL; - bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); + struct mmap_unlock_irq_work *work; bool has_user_ctx = user && current && current->mm; struct stack_map_build_id_cache cache = {}; struct vm_area_struct *vma; @@ -426,15 +425,16 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, return; } - /* If the irq_work is in use, fall back to report ips. Same - * fallback is used for kernel stack (!user) on a stackmap with - * build_id. - */ - if (!has_user_ctx || irq_work_busy || !mmap_read_trylock(current->mm)) { - /* cannot access current->mm, fall back to ips */ - for (i = 0; i < trace_nr; i++) - stack_map_build_id_set_ip(&id_offs[i]); - return; + if (!has_user_ctx) + goto fallback; + + work = bpf_mmap_unlock_guard_get(); + if (IS_ERR(work)) + goto fallback; + + if (!mmap_read_trylock(current->mm)) { + bpf_mmap_unlock_guard_put(work); + goto fallback; } for (i = 0; i < trace_nr; i++) { @@ -465,6 +465,12 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, vma->vm_pgoff); } bpf_mmap_unlock_mm(work, current->mm); + return; + +fallback: + /* cannot access current->mm, fall back to ips */ + for (i = 0; i < trace_nr; i++) + stack_map_build_id_set_ip(&id_offs[i]); } static struct perf_callchain_entry * diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index b256fb9c1214..13e1aabe6f88 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -753,9 +753,8 @@ static struct bpf_iter_reg task_vma_reg_info = { BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, bpf_callback_t, callback_fn, void *, callback_ctx, u64, flags) { - struct mmap_unlock_irq_work *work = NULL; + struct mmap_unlock_irq_work *work; struct vm_area_struct *vma; - bool irq_work_busy = false; bool __maybe_unused mmput_needed = false; struct mm_struct *mm; int ret = -ENOENT; @@ -792,9 +791,14 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, if (!mm) return -ENOENT; - irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); + work = bpf_mmap_unlock_guard_get(); + if (IS_ERR(work)) { + ret = PTR_ERR(work); + goto out; + } - if (irq_work_busy || !mmap_read_trylock(mm)) { + if (!mmap_read_trylock(mm)) { + bpf_mmap_unlock_guard_put(work); ret = -EBUSY; goto out; } @@ -1191,6 +1195,8 @@ static void do_mmap_read_unlock(struct irq_work *entry) work = container_of(entry, struct mmap_unlock_irq_work, irq_work); mmap_read_unlock_non_owner(work->mm); + work->mm = NULL; + bpf_mmap_unlock_guard_put(work); } static int __init task_iter_init(void) -- cgit v1.2.3 From 483a1bb0b6cf816fabaf99702a6e4a7938c98b07 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 7 Aug 2026 13:59:30 -0700 Subject: bpf: Do not print a newline after disassembly in bpf_verbose_insn() At the moment there are more callsites that want bpf_verbose_insn() to not print a newline after the instruction, than callsites that want a newline. Drop '\n' from disasm.c. Non-functional change. The changes in bpftool are verified by writing a bpf program using a variety of instructions and comparing `prog dump xlated` output in the following modes: plain, opcodes, visual, visual opcodes. The output before and after the changes is identical. Signed-off-by: Eduard Zingerman Signed-off-by: Daniel Borkmann Reviewed-by: Quentin Monnet Acked-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260807-static-zext-v4-1-b6c270013c77@gmail.com --- kernel/bpf/backtrack.c | 1 + kernel/bpf/disasm.c | 68 +++++++++++++++++++++++++------------------------- kernel/bpf/liveness.c | 6 ++--- kernel/bpf/verifier.c | 1 + 4 files changed, 38 insertions(+), 38 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 2f473ad4fd7c..40bd04421a99 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -285,6 +285,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, verbose(env, "stack=%s before ", env->tmp_str_buf); verbose(env, "%d: ", idx); bpf_verbose_insn(env, insn); + verbose(env, "\n"); } /* If there is a history record that some registers gained range at this insn, diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index 0391b3bc0073..50b3ca5149a0 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -139,7 +139,7 @@ static void print_bpf_end_insn(bpf_insn_print_t verbose, void *private_data, const struct bpf_insn *insn) { - verbose(private_data, "(%02x) r%d = %s%d r%d\n", + verbose(private_data, "(%02x) r%d = %s%d r%d", insn->code, insn->dst_reg, BPF_SRC(insn->code) == BPF_TO_BE ? "be" : "le", insn->imm, insn->dst_reg); @@ -149,7 +149,7 @@ static void print_bpf_bswap_insn(bpf_insn_print_t verbose, void *private_data, const struct bpf_insn *insn) { - verbose(private_data, "(%02x) r%d = bswap%d r%d\n", + verbose(private_data, "(%02x) r%d = bswap%d r%d", insn->code, insn->dst_reg, insn->imm, insn->dst_reg); } @@ -197,19 +197,19 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, else print_bpf_end_insn(verbose, cbs->private_data, insn); } else if (BPF_OP(insn->code) == BPF_NEG) { - verbose(cbs->private_data, "(%02x) %c%d = -%c%d\n", + verbose(cbs->private_data, "(%02x) %c%d = -%c%d", insn->code, class == BPF_ALU ? 'w' : 'r', insn->dst_reg, class == BPF_ALU ? 'w' : 'r', insn->dst_reg); } else if (is_addr_space_cast(insn)) { - verbose(cbs->private_data, "(%02x) r%d = addr_space_cast(r%d, %u, %u)\n", + verbose(cbs->private_data, "(%02x) r%d = addr_space_cast(r%d, %u, %u)", insn->code, insn->dst_reg, insn->src_reg, ((u32)insn->imm) >> 16, (u16)insn->imm); } else if (is_mov_percpu_addr(insn)) { - verbose(cbs->private_data, "(%02x) r%d = &(void __percpu *)(r%d)\n", + verbose(cbs->private_data, "(%02x) r%d = &(void __percpu *)(r%d)", insn->code, insn->dst_reg, insn->src_reg); } else if (BPF_SRC(insn->code) == BPF_X) { - verbose(cbs->private_data, "(%02x) %c%d %s %s%c%d\n", + verbose(cbs->private_data, "(%02x) %c%d %s %s%c%d", insn->code, class == BPF_ALU ? 'w' : 'r', insn->dst_reg, is_sdiv_smod(insn) ? bpf_alu_sign_string[BPF_OP(insn->code) >> 4] @@ -218,7 +218,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, class == BPF_ALU ? 'w' : 'r', insn->src_reg); } else { - verbose(cbs->private_data, "(%02x) %c%d %s %d\n", + verbose(cbs->private_data, "(%02x) %c%d %s %d", insn->code, class == BPF_ALU ? 'w' : 'r', insn->dst_reg, is_sdiv_smod(insn) ? bpf_alu_sign_string[BPF_OP(insn->code) >> 4] @@ -227,7 +227,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, } } else if (class == BPF_STX) { if (BPF_MODE(insn->code) == BPF_MEM) - verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = r%d\n", + verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = r%d", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, @@ -235,7 +235,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, else if (BPF_MODE(insn->code) == BPF_ATOMIC && (insn->imm == BPF_ADD || insn->imm == BPF_AND || insn->imm == BPF_OR || insn->imm == BPF_XOR)) { - verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) %s r%d\n", + verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) %s r%d", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, @@ -246,7 +246,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->imm == (BPF_AND | BPF_FETCH) || insn->imm == (BPF_OR | BPF_FETCH) || insn->imm == (BPF_XOR | BPF_FETCH))) { - verbose(cbs->private_data, "(%02x) r%d = atomic%s_fetch_%s((%s *)(r%d %+d), r%d)\n", + verbose(cbs->private_data, "(%02x) r%d = atomic%s_fetch_%s((%s *)(r%d %+d), r%d)", insn->code, insn->src_reg, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", bpf_atomic_alu_string[BPF_OP(insn->imm) >> 4], @@ -254,7 +254,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->dst_reg, insn->off, insn->src_reg); } else if (BPF_MODE(insn->code) == BPF_ATOMIC && insn->imm == BPF_CMPXCHG) { - verbose(cbs->private_data, "(%02x) r0 = atomic%s_cmpxchg((%s *)(r%d %+d), r0, r%d)\n", + verbose(cbs->private_data, "(%02x) r0 = atomic%s_cmpxchg((%s *)(r%d %+d), r0, r%d)", insn->code, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", bpf_ldst_string[BPF_SIZE(insn->code) >> 3], @@ -262,44 +262,44 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->src_reg); } else if (BPF_MODE(insn->code) == BPF_ATOMIC && insn->imm == BPF_XCHG) { - verbose(cbs->private_data, "(%02x) r%d = atomic%s_xchg((%s *)(r%d %+d), r%d)\n", + verbose(cbs->private_data, "(%02x) r%d = atomic%s_xchg((%s *)(r%d %+d), r%d)", insn->code, insn->src_reg, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); } else if (BPF_MODE(insn->code) == BPF_ATOMIC && insn->imm == BPF_LOAD_ACQ) { - verbose(cbs->private_data, "(%02x) r%d = load_acquire((%s *)(r%d %+d))\n", + verbose(cbs->private_data, "(%02x) r%d = load_acquire((%s *)(r%d %+d))", insn->code, insn->dst_reg, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->src_reg, insn->off); } else if (BPF_MODE(insn->code) == BPF_ATOMIC && insn->imm == BPF_STORE_REL) { - verbose(cbs->private_data, "(%02x) store_release((%s *)(r%d %+d), r%d)\n", + verbose(cbs->private_data, "(%02x) store_release((%s *)(r%d %+d), r%d)", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); } else { - verbose(cbs->private_data, "BUG_%02x\n", insn->code); + verbose(cbs->private_data, "BUG_%02x", insn->code); } } else if (class == BPF_ST) { if (BPF_MODE(insn->code) == BPF_MEM) { - verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = %d\n", + verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = %d", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->imm); } else if (BPF_MODE(insn->code) == 0xc0 /* BPF_NOSPEC, no UAPI */) { - verbose(cbs->private_data, "(%02x) nospec\n", insn->code); + verbose(cbs->private_data, "(%02x) nospec", insn->code); } else { - verbose(cbs->private_data, "BUG_st_%02x\n", insn->code); + verbose(cbs->private_data, "BUG_st_%02x", insn->code); } } else if (class == BPF_LDX) { if (BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) { - verbose(cbs->private_data, "BUG_ldx_%02x\n", insn->code); + verbose(cbs->private_data, "BUG_ldx_%02x", insn->code); return; } - verbose(cbs->private_data, "(%02x) r%d = *(%s *)(r%d %+d)\n", + verbose(cbs->private_data, "(%02x) r%d = *(%s *)(r%d %+d)", insn->code, insn->dst_reg, BPF_MODE(insn->code) == BPF_MEM ? bpf_ldst_string[BPF_SIZE(insn->code) >> 3] : @@ -307,12 +307,12 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->src_reg, insn->off); } else if (class == BPF_LD) { if (BPF_MODE(insn->code) == BPF_ABS) { - verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[%d]\n", + verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[%d]", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->imm); } else if (BPF_MODE(insn->code) == BPF_IND) { - verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[r%d + %d]\n", + verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[r%d + %d]", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->src_reg, insn->imm); @@ -332,12 +332,12 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, if (is_ptr && !allow_ptr_leaks) imm = 0; - verbose(cbs->private_data, "(%02x) r%d = %s\n", + verbose(cbs->private_data, "(%02x) r%d = %s", insn->code, insn->dst_reg, __func_imm_name(cbs, insn, imm, tmp, sizeof(tmp))); } else { - verbose(cbs->private_data, "BUG_ld_%02x\n", insn->code); + verbose(cbs->private_data, "BUG_ld_%02x", insn->code); return; } } else if (class == BPF_JMP32 || class == BPF_JMP) { @@ -347,35 +347,35 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, char tmp[64]; if (insn->src_reg == BPF_PSEUDO_CALL) { - verbose(cbs->private_data, "(%02x) call pc%s\n", + verbose(cbs->private_data, "(%02x) call pc%s", insn->code, __func_get_name(cbs, insn, tmp, sizeof(tmp))); } else { strcpy(tmp, "unknown"); - verbose(cbs->private_data, "(%02x) call %s#%d\n", insn->code, + verbose(cbs->private_data, "(%02x) call %s#%d", insn->code, __func_get_name(cbs, insn, tmp, sizeof(tmp)), insn->imm); } } else if (insn->code == (BPF_JMP | BPF_JA)) { - verbose(cbs->private_data, "(%02x) goto pc%+d\n", + verbose(cbs->private_data, "(%02x) goto pc%+d", insn->code, insn->off); } else if (insn->code == (BPF_JMP | BPF_JA | BPF_X)) { - verbose(cbs->private_data, "(%02x) gotox r%d\n", + verbose(cbs->private_data, "(%02x) gotox r%d", insn->code, insn->dst_reg); } else if (insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO) { - verbose(cbs->private_data, "(%02x) may_goto pc%+d\n", + verbose(cbs->private_data, "(%02x) may_goto pc%+d", insn->code, insn->off); } else if (insn->code == (BPF_JMP32 | BPF_JA)) { - verbose(cbs->private_data, "(%02x) gotol pc%+d\n", + verbose(cbs->private_data, "(%02x) gotol pc%+d", insn->code, insn->imm); } else if (insn->code == (BPF_JMP | BPF_EXIT)) { - verbose(cbs->private_data, "(%02x) exit\n", insn->code); + verbose(cbs->private_data, "(%02x) exit", insn->code); } else if (BPF_SRC(insn->code) == BPF_X) { verbose(cbs->private_data, - "(%02x) if %c%d %s %c%d goto pc%+d\n", + "(%02x) if %c%d %s %c%d goto pc%+d", insn->code, class == BPF_JMP32 ? 'w' : 'r', insn->dst_reg, bpf_jmp_string[BPF_OP(insn->code) >> 4], @@ -383,14 +383,14 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->src_reg, insn->off); } else { verbose(cbs->private_data, - "(%02x) if %c%d %s 0x%x goto pc%+d\n", + "(%02x) if %c%d %s 0x%x goto pc%+d", insn->code, class == BPF_JMP32 ? 'w' : 'r', insn->dst_reg, bpf_jmp_string[BPF_OP(insn->code) >> 4], (u32)insn->imm, insn->off); } } else { - verbose(cbs->private_data, "(%02x) %s\n", + verbose(cbs->private_data, "(%02x) %s", insn->code, bpf_class_string[class]); } } diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 0aadfbae0acc..ff1e68cc4bd1 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -497,7 +497,6 @@ static void print_instance(struct bpf_verifier_env *env, struct func_instance *i pos = env->log.end_pos; verbose(env, "%3d: ", insn_idx); bpf_verbose_insn(env, &insns[insn_idx]); - bpf_vlog_reset(&env->log, env->log.end_pos - 1); /* remove \n */ insn_pos = env->log.end_pos; verbose(env, "%*c;", bpf_vlog_alignment(insn_pos - pos), ' '); pos = env->log.end_pos; @@ -1043,7 +1042,6 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i if (!printed) { verbose(env, "%3d: ", idx); bpf_verbose_insn(env, insn); - bpf_vlog_reset(&env->log, env->log.end_pos - 1); printed = true; } verbose(env, "\tr%d: ", i); verbose_arg_track(env, &at_in[i]); @@ -1058,7 +1056,6 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i if (!printed) { verbose(env, "%3d: ", idx); bpf_verbose_insn(env, insn); - bpf_vlog_reset(&env->log, env->log.end_pos - 1); printed = true; } verbose(env, "\tsa%d: ", i); verbose_arg_track(env, &at_in[ai]); @@ -1070,7 +1067,6 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i if (!printed) { verbose(env, "%3d: ", idx); bpf_verbose_insn(env, insn); - bpf_vlog_reset(&env->log, env->log.end_pos - 1); printed = true; } verbose(env, "\tfp%+d: ", -(i + 1) * 8); verbose_arg_track(env, &at_stack_in[i]); @@ -1545,6 +1541,7 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, verbose(env, "%3d: ", idx); bpf_verbose_insn(env, &insns[idx]); + verbose(env, "\n"); /* Collect what needs printing */ if (is_ldx_stx_call && @@ -2285,6 +2282,7 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) verbose(env, "."); verbose(env, " "); bpf_verbose_insn(env, &insns[i]); + verbose(env, "\n"); if (bpf_is_ldimm64(&insns[i])) i++; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c9533ea700ba..be8818b9e640 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17562,6 +17562,7 @@ static int do_check(struct bpf_verifier_env *env) env->prev_log_pos = env->log.end_pos; verbose(env, "%d: ", env->insn_idx); bpf_verbose_insn(env, insn); + verbose(env, "\n"); env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos; env->prev_log_pos = env->log.end_pos; } -- cgit v1.2.3 From d977dca7d0736dc7d68d9ad1434961da3b42de35 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 7 Aug 2026 13:59:31 -0700 Subject: bpf: Extract is_addr_space_cast32() utility function bpf_do_misc_fixups() converts the following address space cast instructions to 32-bit moves: - cast from address space 1 (user) to address space 0 (kernel) - cast from address space 0 (kernel) to address space 1 (user) iff associated arena map has a BPF_F_NO_USER_CONV flag. Extract a predicate detecting such instructions for use in the following patches. Signed-off-by: Eduard Zingerman Signed-off-by: Daniel Borkmann Acked-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260807-static-zext-v4-2-b6c270013c77@gmail.com --- kernel/bpf/fixups.c | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index a0bddada7964..5f7843648189 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -20,6 +20,26 @@ static bool is_cmpxchg_insn(const struct bpf_insn *insn) insn->imm == BPF_CMPXCHG; } +/* Returns true if 'insn' is an address space cast instruction translated as BPF_ALU op */ +static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *insn) +{ + struct bpf_map *arena = (struct bpf_map *)prog->aux->arena; + + if (insn->code != (BPF_ALU64 | BPF_MOV | BPF_X) || insn->off != BPF_ADDR_SPACE_CAST) + return false; + + /* cast from as(1) to as(0) */ + if (insn->imm == 1) + return true; + + /* cast from as(0) to as(1) */ + if (insn->imm == 1 << 16) + return arena && arena->map_flags & BPF_F_NO_USER_CONV; + + /* non-BPF_F_NO_USER_CONV cast from as(0) to as(1) should be handled by JIT */ + return false; +} + /* Return the regno defined by the insn, or -1. */ static int insn_def_regno(const struct bpf_insn *insn) { @@ -1513,15 +1533,12 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) } for (i = 0; i < insn_cnt;) { - if (insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) && insn->imm) { - if ((insn->off == BPF_ADDR_SPACE_CAST && insn->imm == 1) || - (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) { - /* convert to 32-bit mov that clears upper 32-bit */ - insn->code = BPF_ALU | BPF_MOV | BPF_X; - /* clear off and imm, so it's a normal 'wX = wY' from JIT pov */ - insn->off = 0; - insn->imm = 0; - } /* cast from as(0) to as(1) should be handled by JIT */ + if (is_addr_space_cast32(env->prog, insn)) { + /* convert to 32-bit mov that clears upper 32-bit */ + insn->code = BPF_ALU | BPF_MOV | BPF_X; + /* clear off and imm, so it's a normal 'wX = wY' from JIT pov */ + insn->off = 0; + insn->imm = 0; goto next_insn; } -- cgit v1.2.3 From 05b71078f30555bab7a40bf2cf12e6da1ddfcaf6 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 7 Aug 2026 13:59:32 -0700 Subject: bpf: Move bpf_is_reg64() to fixups.c The following patches are going to remove bpf_is_reg64() users from everywhere except fixups.c, and also make it dependent on functions local to fixups.c. Move the function before hand to simplify the review. Non functional change. Signed-off-by: Eduard Zingerman Signed-off-by: Daniel Borkmann Acked-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260807-static-zext-v4-3-b6c270013c77@gmail.com --- kernel/bpf/fixups.c | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++ kernel/bpf/verifier.c | 90 --------------------------------------------------- 2 files changed, 90 insertions(+), 90 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 5f7843648189..d2ff416d7ad6 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -64,6 +64,96 @@ static int insn_def_regno(const struct bpf_insn *insn) } } +/* This function is supposed to be used by the zero extension optimization + * code only. It returns TRUE if the source or destination register operates + * on 64-bit, otherwise return FALSE. + */ +bool bpf_is_reg64(struct bpf_insn *insn, + u32 regno, struct bpf_reg_state *reg, enum bpf_reg_arg_type t) +{ + u8 code, class, op; + + code = insn->code; + class = BPF_CLASS(code); + op = BPF_OP(code); + if (class == BPF_JMP) { + /* BPF_EXIT for "main" will reach here. Return TRUE + * conservatively. + */ + if (op == BPF_EXIT) + return true; + if (op == BPF_CALL) { + /* BPF to BPF call will reach here because of marking + * caller saved clobber with DST_OP_NO_MARK for which we + * don't care the register def because they are anyway + * marked as NOT_INIT already. + */ + if (insn->src_reg == BPF_PSEUDO_CALL) + return false; + /* Helper call will reach here because of arg type + * check, conservatively return TRUE. + */ + if (t == SRC_OP) + return true; + + return false; + } + } + + if (class == BPF_ALU64 && op == BPF_END && (insn->imm == 16 || insn->imm == 32)) + return false; + + if (class == BPF_ALU64 || class == BPF_JMP || + (class == BPF_ALU && op == BPF_END && insn->imm == 64)) + return true; + + if (class == BPF_ALU || class == BPF_JMP32) + return false; + + if (class == BPF_LDX) { + if (t != SRC_OP) + return BPF_SIZE(code) == BPF_DW || BPF_MODE(code) == BPF_MEMSX; + /* LDX source must be ptr. */ + return true; + } + + if (class == BPF_STX) { + /* BPF_STX (including atomic variants) has one or more source + * operands, one of which is a ptr. Check whether the caller is + * asking about it. + */ + if (t == SRC_OP && reg->type != SCALAR_VALUE) + return true; + return BPF_SIZE(code) == BPF_DW; + } + + if (class == BPF_LD) { + u8 mode = BPF_MODE(code); + + /* LD_IMM64 */ + if (mode == BPF_IMM) + return true; + + /* Both LD_IND and LD_ABS return 32-bit data. */ + if (t != SRC_OP) + return false; + + /* Implicit ctx ptr. */ + if (regno == BPF_REG_6) + return true; + + /* Explicit source could be any width. */ + return true; + } + + if (class == BPF_ST) + /* The only source register for BPF_ST is a ptr. */ + return true; + + /* Conservatively return true at default. */ + return true; +} + /* Return TRUE if INSN has defined any 32-bit value explicitly. */ static bool insn_has_def32(struct bpf_insn *insn) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index be8818b9e640..abb325194168 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3058,96 +3058,6 @@ static void mark_stack_slots_scratched(struct bpf_verifier_env *env, mark_stack_slot_scratched(env, spi - i); } -/* This function is supposed to be used by the following 32-bit optimization - * code only. It returns TRUE if the source or destination register operates - * on 64-bit, otherwise return FALSE. - */ -bool bpf_is_reg64(struct bpf_insn *insn, - u32 regno, struct bpf_reg_state *reg, enum bpf_reg_arg_type t) -{ - u8 code, class, op; - - code = insn->code; - class = BPF_CLASS(code); - op = BPF_OP(code); - if (class == BPF_JMP) { - /* BPF_EXIT for "main" will reach here. Return TRUE - * conservatively. - */ - if (op == BPF_EXIT) - return true; - if (op == BPF_CALL) { - /* BPF to BPF call will reach here because of marking - * caller saved clobber with DST_OP_NO_MARK for which we - * don't care the register def because they are anyway - * marked as NOT_INIT already. - */ - if (insn->src_reg == BPF_PSEUDO_CALL) - return false; - /* Helper call will reach here because of arg type - * check, conservatively return TRUE. - */ - if (t == SRC_OP) - return true; - - return false; - } - } - - if (class == BPF_ALU64 && op == BPF_END && (insn->imm == 16 || insn->imm == 32)) - return false; - - if (class == BPF_ALU64 || class == BPF_JMP || - (class == BPF_ALU && op == BPF_END && insn->imm == 64)) - return true; - - if (class == BPF_ALU || class == BPF_JMP32) - return false; - - if (class == BPF_LDX) { - if (t != SRC_OP) - return BPF_SIZE(code) == BPF_DW || BPF_MODE(code) == BPF_MEMSX; - /* LDX source must be ptr. */ - return true; - } - - if (class == BPF_STX) { - /* BPF_STX (including atomic variants) has one or more source - * operands, one of which is a ptr. Check whether the caller is - * asking about it. - */ - if (t == SRC_OP && reg->type != SCALAR_VALUE) - return true; - return BPF_SIZE(code) == BPF_DW; - } - - if (class == BPF_LD) { - u8 mode = BPF_MODE(code); - - /* LD_IMM64 */ - if (mode == BPF_IMM) - return true; - - /* Both LD_IND and LD_ABS return 32-bit data. */ - if (t != SRC_OP) - return false; - - /* Implicit ctx ptr. */ - if (regno == BPF_REG_6) - return true; - - /* Explicit source could be any width. */ - return true; - } - - if (class == BPF_ST) - /* The only source register for BPF_ST is a ptr. */ - return true; - - /* Conservatively return true at default. */ - return true; -} - static void mark_insn_zext(struct bpf_verifier_env *env, struct bpf_reg_state *reg) { -- cgit v1.2.3 From ef1ddbfcfaef3194453c66255781e38eecd1f7de Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 7 Aug 2026 13:59:33 -0700 Subject: bpf: Track upper 32-bit register halves' liveness in compute_live_registers() Extend compute_live_registers() to track upper and lower register halves' liveness separately. This is mostly straightforward: - use/def masks are extended to track 2 bits per register; - compute_insn_live_regs() is updated to properly track these 2 bits according to the instruction semantics. Signed-off-by: Eduard Zingerman Signed-off-by: Daniel Borkmann Acked-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260807-static-zext-v4-4-b6c270013c77@gmail.com --- kernel/bpf/liveness.c | 89 ++++++++++++++++++++++++++++++++++----------------- kernel/bpf/verifier.c | 1 - 2 files changed, 59 insertions(+), 31 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index ff1e68cc4bd1..edfc2480b0f0 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -2047,29 +2047,38 @@ out: /* Each field is a register bitmask */ struct insn_live_regs { - u16 use; /* registers read by instruction */ - u16 def; /* registers written by instruction */ - u16 in; /* registers that may be alive before instruction */ - u16 out; /* registers that may be alive after instruction */ + u32 use; /* registers read by instruction */ + u32 def; /* registers written by instruction */ + u32 in; /* registers that may be alive before instruction */ + u32 out; /* registers that may be alive after instruction */ }; /* Bitmask with 1s for all caller saved registers */ #define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1) +static inline u32 reg32_mask(u32 n) { return BIT(n); } +static inline u32 reg64_mask(u32 n) { return BIT(n) | BIT(n + 16); } +static inline u32 mask_widen(u32 m) { return m | (m << 16); } +static inline u16 mask_lo(u32 m) { return (u16)m; } +static inline u16 mask_hi(u32 m) { return (u16)(m >> 16); } + /* Compute info->{use,def} fields for the instruction */ static void compute_insn_live_regs(struct bpf_verifier_env *env, struct bpf_insn *insn, struct insn_live_regs *info) { struct bpf_call_summary cs; - u8 class = BPF_CLASS(insn->code); - u8 code = BPF_OP(insn->code); - u8 mode = BPF_MODE(insn->code); - u16 src = BIT(insn->src_reg); - u16 dst = BIT(insn->dst_reg); - u16 r0 = BIT(0); - u16 def = 0; - u16 use = 0xffff; + const u8 class = BPF_CLASS(insn->code); + const u8 code = BPF_OP(insn->code); + const u8 mode = BPF_MODE(insn->code); + const u8 size = BPF_SIZE(insn->code); + const u32 src = reg64_mask(insn->src_reg); + const u32 dst = reg64_mask(insn->dst_reg); + const u32 src32 = mask_lo(src); + const u32 dst32 = mask_lo(dst); + const u32 r0 = reg64_mask(0); + u32 def = 0; + u32 use = U32_MAX; switch (class) { case BPF_LD: @@ -2080,8 +2089,8 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, use = 0; } break; - case BPF_LD | BPF_ABS: - case BPF_LD | BPF_IND: + case BPF_ABS: + case BPF_IND: /* stick with defaults */ break; } @@ -2089,7 +2098,15 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, case BPF_LDX: switch (mode) { case BPF_MEM: + /* a narrow load still redefines the whole register */ + def = dst; + use = src; + break; case BPF_MEMSX: + /* + * sign extension defines the whole register; + * src holds a pointer, hence is used as 64-bit. + */ def = dst; use = src; break; @@ -2107,12 +2124,19 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, switch (mode) { case BPF_MEM: def = 0; - use = dst | src; + use = dst | (size == BPF_DW ? src : src32); break; - case BPF_ATOMIC: + case BPF_ATOMIC: { + /* + * dst holds a pointer and is always used as 64-bit; + * the value operand and r0 are read as 32-bit for BPF_W atomics. + */ + u32 srcv = size == BPF_DW ? src : src32; + u32 r0v = size == BPF_DW ? r0 : mask_lo(r0); + switch (insn->imm) { case BPF_CMPXCHG: - use = r0 | dst | src; + use = r0v | dst | srcv; def = r0; break; case BPF_LOAD_ACQ: @@ -2121,10 +2145,10 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, break; case BPF_STORE_REL: def = 0; - use = dst | src; + use = dst | srcv; break; default: - use = dst | src; + use = dst | srcv; if (insn->imm & BPF_FETCH) def = src; else @@ -2132,6 +2156,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, } break; } + } break; case BPF_ALU: case BPF_ALU64: @@ -2145,14 +2170,14 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, if (BPF_SRC(insn->code) == BPF_K) use = 0; else - use = src; + use = class == BPF_ALU64 ? src : src32; break; default: def = dst; if (BPF_SRC(insn->code) == BPF_K) - use = dst; + use = class == BPF_ALU64 ? dst : dst32; else - use = dst | src; + use = class == BPF_ALU64 ? (dst | src) : (dst32 | src32); } break; case BPF_JMP: @@ -2178,13 +2203,14 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, use = def & ~BIT(BPF_REG_0); if (bpf_get_call_summary(env, insn, &cs)) use = GENMASK(min_t(u8, cs.num_params, MAX_BPF_FUNC_REG_ARGS), 1); + def = mask_widen(def); + use = mask_widen(use); break; default: def = 0; - if (BPF_SRC(insn->code) == BPF_K) - use = dst; - else - use = dst | src; + use = class == BPF_JMP ? dst : dst32; + if (BPF_SRC(insn->code) == BPF_X) + use |= class == BPF_JMP ? src : src32; } break; } @@ -2249,8 +2275,8 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) int insn_idx = env->cfg.insn_postorder[i]; struct insn_live_regs *live = &state[insn_idx]; struct bpf_iarray *succ; - u16 new_out = 0; - u16 new_in = 0; + u32 new_out = 0; + u32 new_in = 0; succ = bpf_insn_successors(env, insn_idx); for (int s = 0; s < succ->cnt; ++s) @@ -2264,8 +2290,11 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) } } - for (i = 0; i < insn_cnt; ++i) - insn_aux[i].live_regs_before = state[i].in; + for (i = 0; i < insn_cnt; ++i) { + u32 in = state[i].in; + + insn_aux[i].live_regs_before = mask_lo(in) | mask_hi(in); + } if (env->log.level & BPF_LOG_LEVEL2) { verbose(env, "Live regs before insn:\n"); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index abb325194168..dc1acbe0172e 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -16727,7 +16727,6 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, int i; if (bpf_helper_call(call)) { - if (bpf_get_helper_proto(env, call->imm, &fn) < 0) /* error would be reported later */ return false; -- cgit v1.2.3 From 7ce090afbf725e25fff29caacce1eaf8459b1117 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 7 Aug 2026 13:59:34 -0700 Subject: bpf: Infer zext_dst based on static register liveness analysis As reported in the thread [1], the verifier's 32-bit operations zero extension logic is broken. This logic is responsible for correct semantics of 32-bit operations on s390 architecture. According to BPF semantics, operation `w1 += 1` is supposed to zero extend the upper half of the register `r1`. On s390 the JIT relies on the verifier emitting explicit zero extension before such operations. The verifier attempts to minimize the amount of zero extensions inserted by tracking whether upper halves of the 64-bit registers are ever used. Previously such tracking worked as follows: - bpf_reg_state->subreg_def field was set by do_check_insn() for each operation defining lower but not the upper halves of the register. - Whenever an operation reading the whole register was verified, the verifier checked register's subreg_def and set bpf_insn_aux_data->zext_dst flag as true via a call to mark_insn_zext() function. - After the verification was complete, a special pass bpf_opt_subreg_zext_lo32_rnd_hi32() extended 32-bit operations with bpf_insn_aux_data->zext_dst set as true by adding explicit zero extension. Note that the logic above relies on bpf_reg_state->subreg_def, which is a property of a current verifier state. Before the commit [2] two additional steps happened: - The verifier tracked upper and lower register halves' liveness as flags REG_LIVE_READ{32,64} in bpf_reg_state->live. - The function propagate_liveness() called mark_insn_zext() in order to transfer the knowledge about which registers have their upper halves alive (and thus might require zero extension). The commit [2] removed the two steps described above, hence making possible a situation like below: - The register's upper half is set and is used on some verification path P1 and the register happens not to be marked as precise. - The checkpoint C is created while processing some instruction between register initialization and usage. - On some other verification path P2 the register's upper half is not initialized and that path ends hitting the checkpoint C. - In such a case the register's initialization on path P2 would lack zext_dst mark, making it possible for the program to inject an arbitrary value in the register's upper half. This commit replaces subreg_def based logic with computing zext_dst statically, as a part of the bpf_compute_live_registers() analysis: - The analysis now tracks usage of upper and lower halves of the registers separately. - If some instruction defines a 32-bit subregister, but not the whole register, *and* the upper half of the register is alive after that instruction, the instruction is marked as zext_dst. There is one notable drop in precision: whenever a BPF subprogram is called, all 64 bits of parameter registers are presumed to be used. The assumption is that such a drop in precision would not inflict a noticeable performance penalty. [1] https://lore.kernel.org/bpf/CAGKGUv=sOuqQtA1Ub-5JXfA4FPosJFYKAQE4B79cK+P1erxqtg@mail.gmail.com/ [2] commit 107e16979905 ("bpf: disable and remove registers chain based liveness") Fixes: 107e16979905 ("bpf: disable and remove registers chain based liveness") Reported-by: Min-gyu Kim Reported-by: STAR Labs SG Signed-off-by: Eduard Zingerman Signed-off-by: Daniel Borkmann Acked-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/CAGKGUv=sOuqQtA1Ub-5JXfA4FPosJFYKAQE4B79cK+P1erxqtg@mail.gmail.com/ Link: https://lore.kernel.org/bpf/20260807-static-zext-v4-5-b6c270013c77@gmail.com --- kernel/bpf/fixups.c | 27 +++++++----- kernel/bpf/liveness.c | 14 +++++++ kernel/bpf/verifier.c | 112 +++----------------------------------------------- 3 files changed, 37 insertions(+), 116 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index d2ff416d7ad6..447c54828cb9 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -68,8 +68,8 @@ static int insn_def_regno(const struct bpf_insn *insn) * code only. It returns TRUE if the source or destination register operates * on 64-bit, otherwise return FALSE. */ -bool bpf_is_reg64(struct bpf_insn *insn, - u32 regno, struct bpf_reg_state *reg, enum bpf_reg_arg_type t) +static bool bpf_is_reg64(struct bpf_prog *prog, struct bpf_insn *insn, + u32 regno, struct bpf_reg_state *reg, enum bpf_reg_arg_type t) { u8 code, class, op; @@ -103,6 +103,10 @@ bool bpf_is_reg64(struct bpf_insn *insn, if (class == BPF_ALU64 && op == BPF_END && (insn->imm == 16 || insn->imm == 32)) return false; + /* address space casts converted to BPF_ALU, see bpf_do_misc_fixups() */ + if (is_addr_space_cast32(prog, insn)) + return false; + if (class == BPF_ALU64 || class == BPF_JMP || (class == BPF_ALU && op == BPF_END && insn->imm == 64)) return true; @@ -154,15 +158,18 @@ bool bpf_is_reg64(struct bpf_insn *insn, return true; } -/* Return TRUE if INSN has defined any 32-bit value explicitly. */ -static bool insn_has_def32(struct bpf_insn *insn) +/* + * Return the 32-bit subregister defined by INSN, or -1 if INSN does not + * explicitly define a 32-bit value. + */ +int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn) { int dst_reg = insn_def_regno(insn); - if (dst_reg == -1) - return false; + if (dst_reg < 0 || bpf_is_reg64(prog, insn, dst_reg, NULL, DST_OP)) + return -1; - return !bpf_is_reg64(insn, dst_reg, NULL, DST_OP); + return dst_reg; } static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b) @@ -279,7 +286,7 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the * original insn at old prog. */ - data[off].zext_dst = insn_has_def32(insn + off + cnt - 1); + data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; if (cnt == 1) return; @@ -291,7 +298,7 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, for (i = off; i < off + cnt - 1; i++) { /* Expand insni[off]'s seen count to the patched range. */ data[i].seen = old_seen; - data[i].zext_dst = insn_has_def32(insn + i); + data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; } /* @@ -730,7 +737,7 @@ int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, * BPF_STX + SRC_OP, so it is safe to pass NULL * here. */ - if (bpf_is_reg64(&insn, load_reg, NULL, DST_OP)) { + if (bpf_is_reg64(env->prog, &insn, load_reg, NULL, DST_OP)) { if (class == BPF_LD && BPF_MODE(code) == BPF_IMM) i++; diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index edfc2480b0f0..ef9a5a922887 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -2232,6 +2232,7 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) struct bpf_insn *insns = env->prog->insnsi; struct insn_live_regs *state; int insn_cnt = env->prog->len; + u64 pos, insn_pos; int err = 0, i, j; bool changed; @@ -2291,9 +2292,18 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) } for (i = 0; i < insn_cnt; ++i) { + int def32 = bpf_insn_def32(env->prog, &insns[i]); + u32 out = state[i].out; u32 in = state[i].in; insn_aux[i].live_regs_before = mask_lo(in) | mask_hi(in); + /* + * On architectures where 32-bit operations do not reset upper halves + * of the registers, the verifier needs to zero extend a destination + * register if an instruction defines a 32-bit subregister and the + * upper half of that register is alive after the instruction. + */ + insn_aux[i].zext_dst = def32 >= 0 && (mask_hi(out) & BIT(def32)); } if (env->log.level & BPF_LOG_LEVEL2) { @@ -2310,7 +2320,11 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) else verbose(env, "."); verbose(env, " "); + pos = env->log.end_pos; bpf_verbose_insn(env, &insns[i]); + insn_pos = env->log.end_pos; + if (insn_aux[i].zext_dst) + verbose(env, "%*c; zext", bpf_vlog_alignment(insn_pos - pos), ' '); verbose(env, "\n"); if (bpf_is_ldimm64(&insns[i])) i++; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index dc1acbe0172e..9eabc5123e5a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2132,12 +2132,9 @@ out: /* Mark a register as having a completely unknown (scalar) value. */ void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg) { - s32 subreg_def = reg->subreg_def; - memset(reg, 0, sizeof(*reg)); reg->type = SCALAR_VALUE; reg->var_off = tnum_unknown; - reg->subreg_def = subreg_def; __mark_reg_unbounded(reg); } @@ -2213,7 +2210,6 @@ static int mark_btf_ld_reg(struct bpf_verifier_env *env, } } -#define DEF_NOT_SUBREG (0) static void init_reg_state(struct bpf_verifier_env *env, struct bpf_func_state *state) { @@ -2222,7 +2218,6 @@ static void init_reg_state(struct bpf_verifier_env *env, for (i = 0; i < MAX_BPF_REG; i++) { bpf_mark_reg_not_init(env, ®s[i]); - regs[i].subreg_def = DEF_NOT_SUBREG; } /* frame pointer */ @@ -3058,30 +3053,14 @@ static void mark_stack_slots_scratched(struct bpf_verifier_env *env, mark_stack_slot_scratched(env, spi - i); } -static void mark_insn_zext(struct bpf_verifier_env *env, - struct bpf_reg_state *reg) -{ - s32 def_idx = reg->subreg_def; - - if (def_idx == DEF_NOT_SUBREG) - return; - - env->insn_aux_data[def_idx - 1].zext_dst = true; - /* The dst will be zero extended, so won't be sub-register anymore. */ - reg->subreg_def = DEF_NOT_SUBREG; -} - static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno, enum bpf_reg_arg_type t) { - struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; struct bpf_reg_state *reg; - bool rw64; mark_reg_scratched(env, regno); reg = ®s[regno]; - rw64 = bpf_is_reg64(insn, regno, reg, t); if (t == SRC_OP) { /* check whether register used as source operand can be read */ if (reg->type == NOT_INIT) { @@ -3092,9 +3071,6 @@ static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *r if (regno == BPF_REG_FP) return 0; - if (rw64) - mark_insn_zext(env, reg); - return 0; } else { /* check whether register used as dest operand can be written to */ @@ -3102,7 +3078,6 @@ static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *r verbose(env, "frame pointer is read only\n"); return -EACCES; } - reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; if (t == DST_OP) mark_reg_unknown(env, regs, regno); } @@ -3758,11 +3733,6 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, if (size <= spill_size && bpf_stack_narrow_access_ok(off, size, spill_size)) { - /* The earlier check_reg_arg() has decided the - * subreg_def for this insn. Save it first. - */ - s32 subreg_def = state->regs[dst_regno].subreg_def; - if (env->bpf_capable && size == 4 && spill_size == 4 && get_reg_width(reg) <= 32) /* Ensure stack slot has an ID to build a relation @@ -3770,7 +3740,6 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, */ assign_scalar_id_before_mov(env, reg); state->regs[dst_regno] = *reg; - state->regs[dst_regno].subreg_def = subreg_def; /* Break the relation on a narrowing fill. * coerce_reg_to_size will adjust the boundaries. @@ -6246,12 +6215,6 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b } else { mark_reg_known_zero(env, regs, value_regno); - /* A load of ctx field could have different - * actual load size with the one encoded in the - * insn. When the dst is PTR, it is for sure not - * a sub-register. - */ - regs[value_regno].subreg_def = DEF_NOT_SUBREG; if (base_type(info.reg_type) == PTR_TO_BTF_ID) { regs[value_regno].btf = info.btf; regs[value_regno].btf_id = info.btf_id; @@ -7350,10 +7313,6 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat if (spi < 0) return spi; - /* - * For CONST_PTR_TO_DYNPTR, reg is already scratched by check_reg_arg - * in check_helper_call and mark_btf_func_reg_size in check_kfunc_call. - */ mark_stack_slots_scratched(env, spi, BPF_DYNPTR_NR_SLOTS); reg = &state->stack[spi].spilled_ptr; @@ -9457,7 +9416,6 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, /* All non-void global functions return a 64-bit SCALAR_VALUE. */ if (!subprog_returns_void(env, subprog)) { mark_reg_unknown(env, caller->regs, BPF_REG_0); - caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; } if (env->subprog_info[subprog].might_throw) { @@ -10477,9 +10435,6 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } invalidate_outgoing_stack_args(env, cur_func(env)); - /* helper call returns 64-bit value. */ - regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; - /* update return register (already marked as written above) */ ret_type = fn->ret_type; ret_flag = type_flag(ret_type); @@ -10719,30 +10674,6 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return 0; } -/* mark_btf_func_reg_size() is used when the reg size is determined by - * the BTF func_proto's return value size and argument. - */ -static void __mark_btf_func_reg_size(struct bpf_verifier_env *env, struct bpf_reg_state *regs, - u32 regno, size_t reg_size) -{ - struct bpf_reg_state *reg = ®s[regno]; - - if (regno == BPF_REG_0) { - /* Function return value */ - reg->subreg_def = reg_size == sizeof(u64) ? - DEF_NOT_SUBREG : env->insn_idx + 1; - } else if (reg_size == sizeof(u64)) { - /* Function argument */ - mark_insn_zext(env, reg); - } -} - -static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, - size_t reg_size) -{ - return __mark_btf_func_reg_size(env, cur_regs(env), regno, reg_size); -} - static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ACQUIRE; @@ -12961,7 +12892,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_call_arg_meta meta; struct bpf_insn_aux_data *insn_aux; int err, insn_idx = *insn_idx_p; - const struct btf_param *args; u32 i, nargs, ptr_type_id; struct bpf_kfunc_desc *desc; struct btf *desc_btf; @@ -13013,7 +12943,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, verbose(env, "failed to mark s32 range for retval in forked state for lock\n"); return err; } - __mark_btf_func_reg_size(env, regs, BPF_REG_0, sizeof(u32)); } else if (!insn->off && insn->imm == special_kfunc_list[KF___bpf_trap]) { verbose(env, "unexpected __bpf_trap() due to uninitialized variable?\n"); return -EFAULT; @@ -13166,7 +13095,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, u32 regno = caller_saved[i]; bpf_mark_reg_not_init(env, ®s[regno]); - regs[regno].subreg_def = DEF_NOT_SUBREG; } invalidate_outgoing_stack_args(env, cur_func(env)); @@ -13188,7 +13116,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] || meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); - mark_btf_func_reg_size(env, BPF_REG_0, t->size); } else if (btf_type_is_ptr(t)) { ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf); @@ -13279,7 +13206,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, /* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */ regs[BPF_REG_0].id = ++env->id_gen; } - mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); if (is_kfunc_acquire(&meta)) { id = acquire_reference(env, insn_idx, 0); if (id < 0) @@ -13316,18 +13242,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, caller_info->stack_arg_cnt = stack_arg_cnt; } - args = (const struct btf_param *)(meta.func_proto + 1); - for (i = 0; i < min_t(int, nargs, MAX_BPF_FUNC_REG_ARGS); i++) { - u32 regno = i + 1; - - t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL); - if (btf_type_is_ptr(t)) - mark_btf_func_reg_size(env, regno, sizeof(void *)); - else - /* scalar. ensured by check_kfunc_args() */ - mark_btf_func_reg_size(env, regno, t->size); - } - if (bpf_is_iter_next_kfunc(&meta)) { err = process_iter_next_call(env, insn_idx, &meta); if (err) @@ -14820,14 +14734,14 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, if (dst_reg->type != PTR_TO_ARENA) *dst_reg = *src_reg; - dst_reg->subreg_def = env->insn_idx + 1; - - if (BPF_CLASS(insn->code) == BPF_ALU64) + if (BPF_CLASS(insn->code) == BPF_ALU64) { /* * 32-bit operations zero upper bits automatically. * 64-bit operations need to be converted to 32. */ aux->needs_zext = true; + aux->zext_dst = true; + } /* Any arithmetic operations are allowed on arena pointers */ return 0; @@ -15023,18 +14937,14 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) if (insn->imm) { /* off == BPF_ADDR_SPACE_CAST */ mark_reg_unknown(env, regs, insn->dst_reg); - if (insn->imm == 1) { /* cast from as(1) to as(0) */ + if (insn->imm == 1) /* cast from as(1) to as(0) */ dst_reg->type = PTR_TO_ARENA; - /* PTR_TO_ARENA is 32-bit */ - dst_reg->subreg_def = env->insn_idx + 1; - } } else if (insn->off == 0) { /* case: R1 = R2 * copy register state to dest reg */ assign_scalar_id_before_mov(env, src_reg); *dst_reg = *src_reg; - dst_reg->subreg_def = DEF_NOT_SUBREG; } else { /* case: R1 = (s8, s16 s32)R2 */ if (is_pointer_value(env, insn->src_reg)) { @@ -15052,7 +14962,6 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) if (!no_sext) clear_scalar_id(dst_reg); coerce_reg_to_size_sx(dst_reg, insn->off >> 3); - dst_reg->subreg_def = DEF_NOT_SUBREG; } else { mark_reg_unknown(env, regs, insn->dst_reg); } @@ -15077,7 +14986,6 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) */ if (!is_src_reg_u32) clear_scalar_id(dst_reg); - dst_reg->subreg_def = env->insn_idx + 1; } else { /* case: W1 = (s8, s16)W2 */ bool no_sext = reg_umax(src_reg) < (1ULL << (insn->off - 1)); @@ -15087,7 +14995,6 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) *dst_reg = *src_reg; if (!no_sext) clear_scalar_id(dst_reg); - dst_reg->subreg_def = env->insn_idx + 1; coerce_subreg_to_size_sx(dst_reg, insn->off >> 3); } } else { @@ -15956,12 +15863,8 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s continue; if ((!(reg->id & BPF_ADD_CONST) && !(known_reg->id & BPF_ADD_CONST)) || reg->delta == known_reg->delta) { - s32 saved_subreg_def = reg->subreg_def; - *reg = *known_reg; - reg->subreg_def = saved_subreg_def; } else { - s32 saved_subreg_def = reg->subreg_def; s32 saved_off = reg->delta; u32 saved_id = reg->id; @@ -15971,12 +15874,11 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s /* reg = known_reg; reg += delta */ *reg = *known_reg; /* - * Must preserve off, id and subreg_def flag, - * otherwise another sync_linked_regs() will be incorrect. + * Must preserve off and id, otherwise another sync_linked_regs() + * will be incorrect. */ reg->delta = saved_off; reg->id = saved_id; - reg->subreg_def = saved_subreg_def; scalar32_min_max_add(reg, &fake_reg); scalar_min_max_add(reg, &fake_reg); @@ -16411,8 +16313,6 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) * Already marked as written above. */ mark_reg_unknown(env, regs, BPF_REG_0); - /* ld_abs load up to 32-bit skb data. */ - regs[BPF_REG_0].subreg_def = env->insn_idx + 1; /* * See bpf_gen_ld_abs() which emits a hidden BPF_EXIT with r0=0 * which must be explored by the verifier when in a subprog. -- cgit v1.2.3 From be4f8d6f2ff7afe31bd481e004b216bcb3fa6707 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 7 Aug 2026 13:59:35 -0700 Subject: bpf: Simplify the bpf_is_reg64() After the previous commit bpf_is_reg64() is only used in a context where destination register's property is queried, and only for instructions for which insn_def_regno() >= 0. Hence, simplify the function by: - removing unused parameters; - removing code paths considering BPF_JMP{,32} instructions; - streamlining the condition expressions. Signed-off-by: Eduard Zingerman Signed-off-by: Daniel Borkmann Acked-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260807-static-zext-v4-6-b6c270013c77@gmail.com --- kernel/bpf/fixups.c | 110 +++++++++++++--------------------------------------- 1 file changed, 27 insertions(+), 83 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 447c54828cb9..661e2d13a604 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -64,95 +64,43 @@ static int insn_def_regno(const struct bpf_insn *insn) } } -/* This function is supposed to be used by the zero extension optimization - * code only. It returns TRUE if the source or destination register operates - * on 64-bit, otherwise return FALSE. +/* + * For use only in combination with insn_def_regno() >= 0. + * Returns TRUE if the destination register operates on 64-bit, + * otherwise return FALSE. */ -static bool bpf_is_reg64(struct bpf_prog *prog, struct bpf_insn *insn, - u32 regno, struct bpf_reg_state *reg, enum bpf_reg_arg_type t) +static bool bpf_is_reg64(struct bpf_prog *prog, struct bpf_insn *insn) { - u8 code, class, op; - - code = insn->code; - class = BPF_CLASS(code); - op = BPF_OP(code); - if (class == BPF_JMP) { - /* BPF_EXIT for "main" will reach here. Return TRUE - * conservatively. - */ - if (op == BPF_EXIT) - return true; - if (op == BPF_CALL) { - /* BPF to BPF call will reach here because of marking - * caller saved clobber with DST_OP_NO_MARK for which we - * don't care the register def because they are anyway - * marked as NOT_INIT already. - */ - if (insn->src_reg == BPF_PSEUDO_CALL) - return false; - /* Helper call will reach here because of arg type - * check, conservatively return TRUE. - */ - if (t == SRC_OP) - return true; - - return false; - } - } + u8 class = BPF_CLASS(insn->code); + u8 mode = BPF_MODE(insn->code); + u8 size = BPF_SIZE(insn->code); + u8 op = BPF_OP(insn->code); + bool mode_mem; + + /* subregister endiness swap */ + if ((class == BPF_ALU || class == BPF_ALU64) && op == BPF_END && insn->imm != 64) + return false; - if (class == BPF_ALU64 && op == BPF_END && (insn->imm == 16 || insn->imm == 32)) + /* w0 += 1 */ + if (class == BPF_ALU && op != BPF_END) return false; /* address space casts converted to BPF_ALU, see bpf_do_misc_fixups() */ if (is_addr_space_cast32(prog, insn)) return false; - if (class == BPF_ALU64 || class == BPF_JMP || - (class == BPF_ALU && op == BPF_END && insn->imm == 64)) - return true; - - if (class == BPF_ALU || class == BPF_JMP32) + /* non 64-bit, non signed extended loads */ + mode_mem = mode == BPF_MEM || mode == BPF_PROBE_MEM || mode == BPF_PROBE_MEM32; + if (class == BPF_LDX && mode_mem && size != BPF_DW) return false; - if (class == BPF_LDX) { - if (t != SRC_OP) - return BPF_SIZE(code) == BPF_DW || BPF_MODE(code) == BPF_MEMSX; - /* LDX source must be ptr. */ - return true; - } - - if (class == BPF_STX) { - /* BPF_STX (including atomic variants) has one or more source - * operands, one of which is a ptr. Check whether the caller is - * asking about it. - */ - if (t == SRC_OP && reg->type != SCALAR_VALUE) - return true; - return BPF_SIZE(code) == BPF_DW; - } - - if (class == BPF_LD) { - u8 mode = BPF_MODE(code); - - /* LD_IMM64 */ - if (mode == BPF_IMM) - return true; - - /* Both LD_IND and LD_ABS return 32-bit data. */ - if (t != SRC_OP) - return false; - - /* Implicit ctx ptr. */ - if (regno == BPF_REG_6) - return true; - - /* Explicit source could be any width. */ - return true; - } + /* atomics, see insn_def_regno() */ + if (class == BPF_STX && size != BPF_DW) + return false; - if (class == BPF_ST) - /* The only source register for BPF_ST is a ptr. */ - return true; + /* both LD_IND and LD_ABS return 32-bit data. */ + if (class == BPF_LD && (mode == BPF_IND || mode == BPF_ABS)) + return false; /* Conservatively return true at default. */ return true; @@ -166,7 +114,7 @@ int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn) { int dst_reg = insn_def_regno(insn); - if (dst_reg < 0 || bpf_is_reg64(prog, insn, dst_reg, NULL, DST_OP)) + if (dst_reg < 0 || bpf_is_reg64(prog, insn)) return -1; return dst_reg; @@ -733,11 +681,7 @@ int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, if (load_reg == -1) continue; - /* NOTE: arg "reg" (the fourth one) is only used for - * BPF_STX + SRC_OP, so it is safe to pass NULL - * here. - */ - if (bpf_is_reg64(env->prog, &insn, load_reg, NULL, DST_OP)) { + if (bpf_is_reg64(env->prog, &insn)) { if (class == BPF_LD && BPF_MODE(code) == BPF_IMM) i++; -- cgit v1.2.3 From 04962afb3cd6a3cbf1a3679c0c95049833db36c1 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 8 Aug 2026 02:39:21 +0200 Subject: bpf: Rename 'early' BTF checking as a preparation phase BTF processing is split around subprogram discovery. The first phase gets program BTF and imports func_info because a BTF-tagged exception callback may not be referenced by any instruction. Subprogram discovery needs this metadata to find it. The later phase validates func_info and line_info against the complete subprogram table and applies CO-RE relocations. This split breaks a real dependency cycle rather than merely running the same checks early. Rename bpf_check_btf_info_early() and check_btf_func_early() to preparation names that reflect this role. Add short call-site comments to make the two phases and their responsibilities clear. No functional change is intended. Signed-off-by: Kumar Kartikeya Dwivedi Reviewed-by: Amery Hung Link: https://patch.msgid.link/20260808003938.3486067-2-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/check_btf.c | 14 +++++++------- kernel/bpf/verifier.c | 4 +++- 2 files changed, 10 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/check_btf.c b/kernel/bpf/check_btf.c index 93bebe6fe12e..0e8b3ccc7a5b 100644 --- a/kernel/bpf/check_btf.c +++ b/kernel/bpf/check_btf.c @@ -28,9 +28,9 @@ static int check_abnormal_return(struct bpf_verifier_env *env) #define MIN_BPF_FUNCINFO_SIZE 8 #define MAX_FUNCINFO_REC_SIZE 252 -static int check_btf_func_early(struct bpf_verifier_env *env, - const union bpf_attr *attr, - bpfptr_t uattr) +static int prepare_btf_func(struct bpf_verifier_env *env, + const union bpf_attr *attr, + bpfptr_t uattr) { u32 krec_size = sizeof(struct bpf_func_info); const struct btf_type *type, *func_proto; @@ -407,9 +407,9 @@ static int check_core_relo(struct bpf_verifier_env *env, return err; } -int bpf_check_btf_info_early(struct bpf_verifier_env *env, - const union bpf_attr *attr, - bpfptr_t uattr) +int bpf_prepare_btf_info(struct bpf_verifier_env *env, + const union bpf_attr *attr, + bpfptr_t uattr) { struct btf *btf; int err; @@ -429,7 +429,7 @@ int bpf_check_btf_info_early(struct bpf_verifier_env *env, } env->prog->aux->btf = btf; - err = check_btf_func_early(env, attr, uattr); + err = prepare_btf_func(env, attr, uattr); if (err) return err; return 0; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 9eabc5123e5a..ce755c8b0ee2 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20135,7 +20135,8 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, INIT_LIST_HEAD(&env->explored_states[i]); INIT_LIST_HEAD(&env->free_list); - ret = bpf_check_btf_info_early(env, attr, uattr); + /* Prepare BTF and func_info needed to discover all subprograms. */ + ret = bpf_prepare_btf_info(env, attr, uattr); if (ret < 0) goto skip_full_check; @@ -20147,6 +20148,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + /* Validate BTF against the complete subprogram layout and apply CO-RE. */ ret = bpf_check_btf_info(env, attr, uattr); if (ret < 0) goto skip_full_check; -- cgit v1.2.3 From 41f36ffa3a87b354a248be4c24f02f06cd52844d Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 8 Aug 2026 02:39:22 +0200 Subject: bpf: Split subprogram and kfunc collection add_subprog_and_kfunc() combines two operations with different ordering requirements. Subprogram discovery must precede validation of func_info and line_info, while kfunc descriptors are only needed by the verifier after its initial program setup is complete. Split the helper into add_subprogs() and add_kfuncs() so each operation can be placed according to its actual dependencies. Keep both calls adjacent and in their existing phase for now, and add short comments describing their roles. No functional change is intended for valid programs. Signed-off-by: Kumar Kartikeya Dwivedi Reviewed-by: Amery Hung Link: https://patch.msgid.link/20260808003938.3486067-3-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ce755c8b0ee2..a54f7b63eaba 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2837,7 +2837,7 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) return 0; } -static int add_subprog_and_kfunc(struct bpf_verifier_env *env) +static int add_subprogs(struct bpf_verifier_env *env) { struct bpf_subprog_info *subprog = env->subprog_info; int i, ret, insn_cnt = env->prog->len, ex_cb_insn; @@ -2849,8 +2849,7 @@ static int add_subprog_and_kfunc(struct bpf_verifier_env *env) return ret; for (i = 0; i < insn_cnt; i++, insn++) { - if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) && - !bpf_pseudo_kfunc_call(insn)) + if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) continue; if (!env->bpf_capable) { @@ -2858,11 +2857,7 @@ static int add_subprog_and_kfunc(struct bpf_verifier_env *env) return -EPERM; } - if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) - ret = add_subprog(env, i + insn->imm + 1); - else - ret = bpf_add_kfunc_call(env, insn->imm, insn->off); - + ret = add_subprog(env, i + insn->imm + 1); if (ret < 0) return ret; } @@ -2900,6 +2895,28 @@ static int add_subprog_and_kfunc(struct bpf_verifier_env *env) return 0; } +static int add_kfuncs(struct bpf_verifier_env *env) +{ + struct bpf_insn *insn = env->prog->insnsi; + int i, ret, insn_cnt = env->prog->len; + + for (i = 0; i < insn_cnt; i++, insn++) { + if (!bpf_pseudo_kfunc_call(insn)) + continue; + + if (!env->bpf_capable) { + verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); + return -EPERM; + } + + ret = bpf_add_kfunc_call(env, insn->imm, insn->off); + if (ret < 0) + return ret; + } + + return 0; +} + static int check_subprogs(struct bpf_verifier_env *env) { int i, subprog_start, subprog_end, off, cur_subprog = 0; @@ -20140,7 +20157,13 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; - ret = add_subprog_and_kfunc(env); + /* Discover all subprograms before validating their layout and BTF. */ + ret = add_subprogs(env); + if (ret < 0) + goto skip_full_check; + + /* Collect the kfunc descriptors used during verification. */ + ret = add_kfuncs(env); if (ret < 0) goto skip_full_check; -- cgit v1.2.3 From d98b2d445fc530aa34bfc7abce7e06d2e761dc01 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 8 Aug 2026 02:39:23 +0200 Subject: bpf: Collect kfuncs after resolving program resources The kfunc descriptors include argument prototypes generated while calls are collected. Some argument classifications need program auxiliary state derived from referenced maps, such as the arena associated with the program. This avoids a footgun in get_kfunc_arg_type() checks where we do validation on whether program has prog->aux->arena and it hasn't been resolved yet. check_and_resolve_insns() records used maps and populates that state. It must remain after bpf_check_btf_info(), which applies kernel-side CO-RE relocations, so that instruction validation and the program tag observe the relocated instruction stream. Move only add_kfuncs() after instruction and resource resolution. Subprogram discovery and validation remain before the full BTF phase because that phase needs the complete subprogram layout. Add a short comment describing the resource resolution phase at the call site. Signed-off-by: Kumar Kartikeya Dwivedi Reviewed-by: Amery Hung Link: https://patch.msgid.link/20260808003938.3486067-4-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a54f7b63eaba..e17084666041 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20162,11 +20162,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; - /* Collect the kfunc descriptors used during verification. */ - ret = add_kfuncs(env); - if (ret < 0) - goto skip_full_check; - ret = check_subprogs(env); if (ret < 0) goto skip_full_check; @@ -20176,10 +20171,16 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + /* Validate instructions and resolve the program's referenced resources. */ ret = check_and_resolve_insns(env); if (ret < 0) goto skip_full_check; + /* Build kfunc prototypes after resolving program resources. */ + ret = add_kfuncs(env); + if (ret < 0) + goto skip_full_check; + if (bpf_prog_is_offloaded(env->prog->aux)) { ret = bpf_prog_offload_verifier_prep(env->prog); if (ret) -- cgit v1.2.3 From 252d367163268cb8d3fe321c1fc2b15a60faf9ea Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 8 Aug 2026 02:39:24 +0200 Subject: bpf: Support __arena and __arena__nullable kfunc argument suffixes Passing an arena pointer to a kfunc takes two steps today. There is no arena pointer argument type, so the pointer crosses the boundary as a bare scalar, and the kfunc then offsets it by the arena base and casts it before it can touch the memory. Every such kfunc open-codes the same translation. Add the __arena and __arena__nullable argument suffixes to make this more convenient. The kfunc declares the parameter by its real pointer type and dereferences it directly, with the JIT rebasing the value at the call site, rN = kern_vm_start + (u32)rN. No bounds check is needed: the u32 offset stays within the guard-padded arena kernel mapping, and a fault on an unpopulated page recovers through the per-arena scratch page. A suffixed argument accepts a PTR_TO_ARENA or scalar register, matching global subprog arena arguments. __arena rebases unconditionally, so the kfunc never sees NULL and a value with zero in the low 32 bits arrives as the arena base. __arena__nullable preserves NULL for optional arguments by skipping the rebase when the truncated value, arena offset 0, is zero. Keeping the plain form NULL-free saves the NULL test on every call. The double separator makes the annotations composable: __arena__nullable also ends in __nullable and naturally follows the common nullable argument path. Plain __arena follows that path too for verifier type checking because both forms accept a constant zero; the function-model flag still determines whether the JIT preserves NULL or rebases it to the arena base. This patch adds the verifier side: the suffixes are recognized in check_kfunc_args() and distilled into argument flags in the function model stored in the kfunc descriptor. JITs retrieve the model while emitting the call, avoiding per-call state in insn_aux_data. JITs declare support with bpf_jit_supports_arena_args() and verification fails with -ENOTSUPP elsewhere. Co-developed-by: Kumar Kartikeya Dwivedi Signed-off-by: Tejun Heo Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260808003938.3486067-5-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/btf.c | 18 +++++++++++++++++- kernel/bpf/core.c | 5 +++++ kernel/bpf/verifier.c | 46 ++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 64 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 42414633cf26..4ff6148ae8e8 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7539,6 +7539,22 @@ static u8 __get_type_fmodel_flags(const struct btf_type *t) return flags; } +static u8 __get_arg_fmodel_flags(const struct btf *btf, + const struct btf_param *arg, + const struct btf_type *t) +{ + u8 flags = __get_type_fmodel_flags(t); + + if (btf_param_match_suffix(btf, arg, "__arena__nullable")) + flags |= BTF_FMODEL_ARENA_ARG | BTF_FMODEL_NULLABLE_ARG; + else if (btf_param_match_suffix(btf, arg, "__arena")) + flags |= BTF_FMODEL_ARENA_ARG; + else if (btf_param_match_suffix(btf, arg, "__nullable")) + flags |= BTF_FMODEL_NULLABLE_ARG; + + return flags; +} + int btf_distill_func_proto(struct bpf_verifier_log *log, struct btf *btf, const struct btf_type *func, @@ -7604,7 +7620,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, return -EINVAL; } m->arg_size[i] = ret; - m->arg_flags[i] = __get_type_fmodel_flags(t); + m->arg_flags[i] = __get_arg_fmodel_flags(btf, &args[i], t); } m->nr_args = nargs; return 0; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index e2076667b245..a3e1fae32eac 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3308,6 +3308,11 @@ bool __weak bpf_jit_supports_stack_args(void) return false; } +bool __weak bpf_jit_supports_arena_args(void) +{ + return false; +} + bool __weak bpf_jit_supports_far_kfunc_call(void) { return false; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e17084666041..a1b71ac39b3b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10777,7 +10777,8 @@ static bool is_kfunc_arg_refcounted_kptr(const struct btf *btf, const struct btf static bool is_kfunc_arg_nullable(const struct btf *btf, const struct btf_param *arg) { - return btf_param_match_suffix(btf, arg, "__nullable"); + return btf_param_match_suffix(btf, arg, "__nullable") || + btf_param_match_suffix(btf, arg, "__arena"); } static bool is_kfunc_arg_nonown_allowed(const struct btf *btf, const struct btf_param *arg) @@ -10795,6 +10796,12 @@ static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param return btf_param_match_suffix(btf, arg, "__irq_flag"); } +static bool is_kfunc_arg_arena(const struct btf *btf, const struct btf_param *arg) +{ + return btf_param_match_suffix(btf, arg, "__arena__nullable") || + btf_param_match_suffix(btf, arg, "__arena"); +} + static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, const struct btf_param *arg, const char *name) @@ -11015,6 +11022,7 @@ enum kfunc_ptr_arg_type { KF_ARG_PTR_TO_IRQ_FLAG, KF_ARG_PTR_TO_RES_SPIN_LOCK, KF_ARG_PTR_TO_TASK_WORK, + KF_ARG_PTR_TO_ARENA, }; enum special_kfunc_type { @@ -11300,7 +11308,6 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, reg_arg_name(env, argno), btf_type_str(t)); return -EINVAL; } - ref_t = btf_type_skip_modifiers(meta->btf, t->type, NULL); ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off); @@ -11349,7 +11356,30 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, arg_type = KF_ARG_PTR_TO_RES_SPIN_LOCK; else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) arg_type = KF_ARG_PTR_TO_CALLBACK; - else if (arg + 1 < nargs && + else if (is_kfunc_arg_arena(meta->btf, &args[arg])) { + if (!bpf_jit_supports_arena_args()) { + verbose(env, "JIT does not support kfunc %s() with arena pointer arguments\n", + meta->func_name); + return -ENOTSUPP; + } + if (!env->prog->aux->arena) { + verbose(env, + "%s arena pointer requires a program with an associated arena\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (reg_from_argno(argno) < 0) { + verbose(env, "%s arena pointer cannot be a stack argument\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + /* + * Both suffixes accept a constant zero. The function model determines + * whether the JIT rebases it to the arena base or preserves NULL. + * The common nullable path below records that verifier property. + */ + arg_type = KF_ARG_PTR_TO_ARENA; + } else if (arg + 1 < nargs && (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && @@ -12024,7 +12054,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me t = btf_type_skip_modifiers(btf, args[i].type, NULL); if (btf_type_is_ptr(t) && (bpf_register_is_null(reg) || type_may_be_null(reg->type)) && - !is_kfunc_arg_nullable(meta->btf, &args[i])) { + !type_may_be_null(kf_arg_type)) { verbose(env, "Possibly NULL pointer passed to trusted %s\n", reg_arg_name(env, argno)); return -EACCES; @@ -12077,6 +12107,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me case KF_ARG_PTR_TO_TASK_WORK: case KF_ARG_PTR_TO_IRQ_FLAG: case KF_ARG_PTR_TO_RES_SPIN_LOCK: + case KF_ARG_PTR_TO_ARENA: break; case KF_ARG_PTR_TO_DYNPTR: arg_type = ARG_PTR_TO_DYNPTR; @@ -12143,6 +12174,13 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me meta->ret_btf_id = ret; } break; + case KF_ARG_PTR_TO_ARENA: + if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a pointer to arena or scalar\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + break; case KF_ARG_PTR_TO_ALLOC_BTF_ID: if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { if (!is_bpf_obj_drop_kfunc(meta->func_id)) { -- cgit v1.2.3 From f6c33c4479a7e4d6bfc0e0e533a86376866f7360 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 8 Aug 2026 02:39:25 +0200 Subject: bpf: Support __arena and __arena__nullable on struct_ops arguments A struct_ops callback cannot receive an arena pointer directly, so passing one takes two steps. The pointer arrives as a bare u64 that the callback casts, and because the two sides address the arena through different bases it also has to be rebased by hand on the way in. Add the __arena and __arena__nullable stub argument suffixes to make this convenient. The callback declares the parameter as an arena pointer, receives it as a PTR_TO_ARENA register, and dereferences it directly, while the kernel caller just passes the natural kernel arena address (kaddr). The trampoline converts the value while saving the arguments into the BPF ctx, ctx[slot] = (u32)(kaddr - kern_vm_start), so the program never sees a kernel address and nothing rewrites the ctx after the fact. The converted value keeps the upper 32 bits clear as the JITs require of arena pointer registers and behaves like any cast_kern'ed arena pointer, so cast_user recovers the full user-visible address. __arena converts unconditionally and the kernel caller must not pass NULL. __arena__nullable preserves NULL, tested on the full 64-bit kernel pointer, and surfaces to the verifier as PTR_TO_ARENA (but not as a PTR_TO_ARENA | PTR_MAYBE_NULL). The reason is that PTR_TO_ARENA in the program's type state already encompasses NULL-ness, so it is not meaningful to force a NULL check for the program. The composite suffix intentionally ends in __nullable. Classify __arena__nullable before the generic suffix so scalar arena pointees do not take the generic nullable BTF pointer path. This patch adds the generic side. prepare_arg_info() records arena and nullable argument flags in the struct_ops function model, and bpf_tramp_arena_base() returns the arena base for a single-program struct_ops indirect trampoline. Only that trampoline converts: its program's arena is fixed at generation time. Generic trampolines can mix programs with different arenas and reject arena context arguments defensively, which is unreachable today as only struct_ops programs carry them. Architectures that do not implement the conversion are gated out at verification time with bpf_jit_supports_arena_args(). Co-developed-by: Kumar Kartikeya Dwivedi Signed-off-by: Tejun Heo Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260808003938.3486067-6-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/bpf_struct_ops.c | 56 +++++++++++++++++++++++++++++++++------------ kernel/bpf/btf.c | 10 +++++--- kernel/bpf/trampoline.c | 37 ++++++++++++++++++++++++++++++ kernel/bpf/verifier.c | 23 +++++++++++++++---- 4 files changed, 104 insertions(+), 22 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 4e7a48c02be5..d7c3030bc63b 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -147,6 +147,8 @@ void bpf_struct_ops_image_free(void *image) #define MAYBE_NULL_SUFFIX "__nullable" #define REFCOUNTED_SUFFIX "__ref" +#define ARENA_SUFFIX "__arena" +#define ARENA_MAYBE_NULL_SUFFIX "__arena__nullable" /* Prepare argument info for every nullable argument of a member of a * struct_ops type. @@ -159,7 +161,7 @@ void bpf_struct_ops_image_free(void *image) * to provide an array of struct bpf_ctx_arg_aux, which in turn provides * the information that used by the verifier to check the arguments of the * BPF struct_ops program assigned to the member. Here, we only care about - * the arguments that are marked as __nullable. + * the arguments that are marked as __nullable, __ref or __arena. * * The array of struct bpf_ctx_arg_aux is eventually assigned to * prog->aux->ctx_arg_info of BPF struct_ops programs and passed to the @@ -172,10 +174,12 @@ static int prepare_arg_info(struct btf *btf, const char *st_ops_name, const char *member_name, const struct btf_type *func_proto, void *stub_func_addr, + struct btf_func_model *model, struct bpf_struct_ops_arg_info *arg_info) { const struct btf_type *stub_func_proto, *pointed_type; - bool is_nullable = false, is_refcounted = false; + bool is_nullable = false, is_refcounted = false, is_arena = false; + bool is_arena_nullable = false; const struct btf_param *stub_args, *args; struct bpf_ctx_arg_aux *info, *info_buf; u32 nargs, arg_no, info_cnt = 0; @@ -225,27 +229,39 @@ static int prepare_arg_info(struct btf *btf, /* Prepare info for every nullable argument */ info = info_buf; for (arg_no = 0; arg_no < nargs; arg_no++) { - /* Skip arguments that is not suffixed with - * "__nullable or __ref". + bool ptr_to_arena, ptr_to_struct; + + /* + * Skip arguments that are not suffixed with "__arena__nullable", + * "__arena", "__nullable", or "__ref". */ - is_nullable = btf_param_match_suffix(btf, &stub_args[arg_no], - MAYBE_NULL_SUFFIX); + is_arena_nullable = btf_param_match_suffix(btf, &stub_args[arg_no], + ARENA_MAYBE_NULL_SUFFIX); + is_arena = btf_param_match_suffix(btf, &stub_args[arg_no], ARENA_SUFFIX); + is_nullable = !is_arena_nullable && + btf_param_match_suffix(btf, &stub_args[arg_no], MAYBE_NULL_SUFFIX); is_refcounted = btf_param_match_suffix(btf, &stub_args[arg_no], REFCOUNTED_SUFFIX); - if (is_nullable) + if (is_arena_nullable) + suffix = ARENA_MAYBE_NULL_SUFFIX; + else if (is_arena) + suffix = ARENA_SUFFIX; + else if (is_nullable) suffix = MAYBE_NULL_SUFFIX; else if (is_refcounted) suffix = REFCOUNTED_SUFFIX; else continue; - /* Should be a pointer to struct */ - pointed_type = btf_type_resolve_ptr(btf, - args[arg_no].type, - &arg_btf_id); - if (!pointed_type || - !btf_type_is_struct(pointed_type)) { + /* + * Should be a pointer to struct, or any pointer for __arena or + * __arena__nullable. + */ + pointed_type = btf_type_resolve_ptr(btf, args[arg_no].type, &arg_btf_id); + ptr_to_arena = pointed_type && (is_arena || is_arena_nullable); + ptr_to_struct = pointed_type && btf_type_is_struct(pointed_type); + if (!ptr_to_arena && !ptr_to_struct) { pr_warn("stub function %s has %s tagging to an unsupported type\n", stub_fname, suffix); goto err_out; @@ -268,7 +284,18 @@ static int prepare_arg_info(struct btf *btf, info->btf_id = arg_btf_id; info->btf = btf; info->offset = offset; - if (is_nullable) { + if (is_arena || is_arena_nullable) { + /* + * Both types get PTR_TO_ARENA. In verifier state, + * PTR_TO_ARENA encompasses potential NULL values, but + * we do not force the program to check it, or maintain + * precision around it, since it has no safety implication. + */ + info->reg_type = PTR_TO_ARENA; + model->arg_flags[arg_no] |= BTF_FMODEL_ARENA_ARG; + if (is_arena_nullable) + model->arg_flags[arg_no] |= BTF_FMODEL_NULLABLE_ARG; + } else if (is_nullable) { info->reg_type = PTR_TRUSTED | PTR_TO_BTF_ID | PTR_MAYBE_NULL; } else if (is_refcounted) { info->reg_type = PTR_TRUSTED | PTR_TO_BTF_ID; @@ -460,6 +487,7 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, stub_func_addr = *(void **)(st_ops->cfi_stubs + moff); err = prepare_arg_info(btf, st_ops->name, mname, func_proto, stub_func_addr, + &st_ops->func_models[i], arg_info + i); if (err) goto errout; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 4ff6148ae8e8..6606187ed4f4 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6963,15 +6963,19 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, return false; } - /* check for PTR_TO_RDONLY_BUF_OR_NULL or PTR_TO_RDWR_BUF_OR_NULL */ + /* + * Check for PTR_TO_RDONLY_BUF_OR_NULL, PTR_TO_RDWR_BUF_OR_NULL or + * PTR_TO_ARENA (both nullable and non-nullable cases). + */ for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; u32 type, flag; type = base_type(ctx_arg_info->reg_type); flag = type_flag(ctx_arg_info->reg_type); - if (ctx_arg_info->offset == off && type == PTR_TO_BUF && - (flag & PTR_MAYBE_NULL)) { + if (ctx_arg_info->offset == off && + (type == PTR_TO_ARENA || + (type == PTR_TO_BUF && (flag & PTR_MAYBE_NULL)))) { info->reg_type = ctx_arg_info->reg_type; return true; } diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index ed7999ad6c66..e07af35ed040 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -529,6 +529,36 @@ bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_a return tnodes; } +/* + * The arena base against which save_args() converts the arguments marked + * with BTF_FMODEL_ARENA_ARG. Only the struct_ops indirect trampoline + * converts: it dispatches to a single prog whose arena is known at + * generation time. Return 0 when there is nothing to convert. + */ +u64 bpf_tramp_arena_base(const struct btf_func_model *m, + struct bpf_tramp_nodes *tnodes, u32 flags) +{ + const struct bpf_prog *prog; + int i; + + if (!(flags & BPF_TRAMP_F_INDIRECT) || + tnodes[BPF_TRAMP_FENTRY].nr_nodes != 1) + return 0; + + for (i = 0; i < m->nr_args; i++) + if (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG) + break; + if (i == m->nr_args) + return 0; + + /* Verification rejects an arena argument without an arena. */ + prog = tnodes[BPF_TRAMP_FENTRY].nodes[0]->link->prog; + if (WARN_ON_ONCE(!prog->aux->arena)) + return 0; + + return bpf_arena_get_kern_vm_start(prog->aux->arena); +} + static void bpf_tramp_image_free(struct bpf_tramp_image *im) { bpf_image_ksym_del(&im->ksym); @@ -920,6 +950,13 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node, int cnt = 0, i; kind = bpf_attach_type_to_tramp(node->link->prog); + /* + * Arena ctx args are converted only by struct_ops indirect + * trampolines. They must never be attached to a generic trampoline. + */ + if (WARN_ON_ONCE(bpf_prog_has_arena_ctx_arg(node->link->prog))) + return -ENOTSUPP; + if (tr->extension_prog) /* cannot attach fentry/fexit if extension prog is attached. * cannot overwrite extension prog either. diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a1b71ac39b3b..7d527f7c899e 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -18685,6 +18685,7 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) { const struct btf_type *t, *func_proto; const struct bpf_struct_ops_desc *st_ops_desc; + const struct bpf_struct_ops_arg_info *arg_info; const struct bpf_struct_ops *st_ops; const struct btf_member *member; struct bpf_prog *prog = env->prog; @@ -18763,10 +18764,23 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) return -EACCES; } - for (i = 0; i < st_ops_desc->arg_info[member_idx].cnt; i++) { - if (st_ops_desc->arg_info[member_idx].info[i].refcounted) { + arg_info = &st_ops_desc->arg_info[member_idx]; + for (i = 0; i < arg_info->cnt; i++) { + const struct bpf_ctx_arg_aux *info = &arg_info->info[i]; + + if (info->refcounted) has_refcounted_arg = true; - break; + if (base_type(info->reg_type) == PTR_TO_ARENA) { + if (!bpf_jit_supports_arena_args()) { + verbose(env, "JIT does not support arena arguments\n"); + return -ENOTSUPP; + } + if (!prog->aux->arena) { + verbose(env, + "arena argument of %s requires a program with an associated arena\n", + mname); + return -EINVAL; + } } } @@ -18787,8 +18801,7 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) prog->aux->attach_func_name = mname; env->ops = st_ops->verifier_ops; - return bpf_prog_ctx_arg_info_init(prog, st_ops_desc->arg_info[member_idx].info, - st_ops_desc->arg_info[member_idx].cnt); + return bpf_prog_ctx_arg_info_init(prog, arg_info->info, arg_info->cnt); } #define SECURITY_PREFIX "security_" -- cgit v1.2.3 From fd6094ac877cf5efe6702c0cfc86802dd9a4f322 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 8 Aug 2026 02:39:33 +0200 Subject: bpf: Reject tracing/freplace progs for struct_ops with arena args Reject tracing and freplace attachments to a target program with arena context arguments. The struct_ops indirect trampoline converts those arguments before entering the target, so a generic tracing trampoline would otherwise expose arena offsets using the target BTF pointer type. Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260808003938.3486067-14-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7d527f7c899e..add3affc5703 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -19068,6 +19068,16 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, bpf_log(log, "Subprog %s doesn't exist\n", tname); return -EINVAL; } + /* + * A struct_ops indirect trampoline converts arena arguments + * before invoking its program. A tracing or extension program + * attached to the main program would see the converted offset as a + * regular BTF pointer. + */ + if (subprog == 0 && bpf_prog_has_arena_ctx_arg(tgt_prog)) { + bpf_log(log, "Cannot attach to a target with arena context arguments\n"); + return -EOPNOTSUPP; + } if (aux->func && aux->func[subprog]->aux->exception_cb) { bpf_log(log, "%s programs cannot attach to exception callback\n", -- cgit v1.2.3 From 83608e303b95d07afba1c15da0b5d9e513c2f15a Mon Sep 17 00:00:00 2001 From: Ning Ding Date: Mon, 10 Aug 2026 20:59:54 -0700 Subject: bpf: Compare iterator types during state pruning An iterator stack slot can be MEM_RCU or PTR_UNTRUSTED. These states must not be equal, or the verifier can prune an unsafe path. Compare the pointer type for STACK_ITER slots. Fixes: dfab99df147b ("bpf: teach the verifier to enforce css_iter and task_iter in RCU CS") Signed-off-by: Ning Ding Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260811035955.132989-2-dingning04@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/states.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index ea2153cf28d0..4e6aafad33bd 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -812,7 +812,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * infinite loop check triggering, see * iter_active_depths_differ() */ - if (old_reg->iter.btf != cur_reg->iter.btf || + if (old_reg->type != cur_reg->type || + old_reg->iter.btf != cur_reg->iter.btf || old_reg->iter.btf_id != cur_reg->iter.btf_id || old_reg->iter.state != cur_reg->iter.state || /* ignore {old_reg,cur_reg}->iter.depth, see above */ -- cgit v1.2.3 From 41c5dbb4be3c1ef4a5e2ce4c28de60b2be3cdccf Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 11 Aug 2026 15:15:55 +0200 Subject: bpf: Derive the atomic load register in one place check_atomic_rmw() open codes the mapping from a BPF_ATOMIC to the register it reads the old value into, the BPF_STX case of insn_def_regno() open codes the very same mapping a second time, the const folding and the liveness transfer functions a third and a fourth time, and BPF JITs need it as well to know which register a faulting BPF_PROBE_ATOMIC has to clear. Add a small helper so that all of them can share it. No functional change. The BPF_LOAD_ACQ case is there for the JITs, which do walk all instruction classes. const_reg_xfer() loses its explicit BPF_ATOMIC mode test since the helper checks class and mode itself; the BPF_PROBE_ATOMIC it additionally accepts cannot be seen there as it is only set from bpf_do_misc_fixups(), that is, after const folding has run. arg_track_xfer() keeps its mode test since that also guards the stack clearing next to it. Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260811131600.506721-1-daniel@iogearbox.net Signed-off-by: Eduard Zingerman --- kernel/bpf/const_fold.c | 11 +++-------- kernel/bpf/fixups.c | 11 +---------- kernel/bpf/liveness.c | 9 +++------ kernel/bpf/verifier.c | 13 ++----------- 4 files changed, 9 insertions(+), 35 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c index b2a19acadb91..4cf120c7b2cb 100644 --- a/kernel/bpf/const_fold.c +++ b/kernel/bpf/const_fold.c @@ -199,14 +199,9 @@ process_call: ci_out[r] = unknown; break; case BPF_STX: - if (mode != BPF_ATOMIC) - break; - if (insn->imm == BPF_CMPXCHG) - ci_out[BPF_REG_0] = unknown; - else if (insn->imm == BPF_LOAD_ACQ) - *dst = unknown; - else if (insn->imm & BPF_FETCH) - *src = unknown; + r = bpf_atomic_load_reg(insn); + if (r >= 0) + ci_out[r] = unknown; break; } } diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 661e2d13a604..c4bd70befbb5 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -49,16 +49,7 @@ static int insn_def_regno(const struct bpf_insn *insn) case BPF_ST: return -1; case BPF_STX: - if (BPF_MODE(insn->code) == BPF_ATOMIC || - BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { - if (insn->imm == BPF_CMPXCHG) - return BPF_REG_0; - else if (insn->imm == BPF_LOAD_ACQ) - return insn->dst_reg; - else if (insn->imm & BPF_FETCH) - return insn->src_reg; - } - return -1; + return bpf_atomic_load_reg(insn); default: return insn->dst_reg; } diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index ef9a5a922887..1c997aeba6fa 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -1209,12 +1209,9 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, clear_stack_for_all_offs(insn, at_out, insn->dst_reg, at_stack_out, sz); - if (insn->imm == BPF_CMPXCHG) - at_out[BPF_REG_0] = none; - else if (insn->imm == BPF_LOAD_ACQ) - *dst = none; - else if (insn->imm & BPF_FETCH) - *src = none; + r = bpf_atomic_load_reg(insn); + if (r >= 0) + at_out[r] = none; } } else if (class == BPF_ST && BPF_MODE(insn->code) == BPF_MEM) { u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index add3affc5703..61ef43325c6f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6485,21 +6485,12 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return -EACCES; } - if (insn->imm & BPF_FETCH) { - if (insn->imm == BPF_CMPXCHG) - load_reg = BPF_REG_0; - else - load_reg = insn->src_reg; - + load_reg = bpf_atomic_load_reg(insn); + if (load_reg >= 0) { /* check and record load of old value */ err = check_reg_arg(env, load_reg, DST_OP); if (err) return err; - } else { - /* This instruction accesses a memory location but doesn't - * actually load it into a register. - */ - load_reg = -1; } dst_reg = cur_regs(env) + insn->dst_reg; -- cgit v1.2.3 From 14c950ac2be8cadb63e1bfe22111ab0fdc829eb8 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 13 Aug 2026 00:19:18 +0200 Subject: bpf: Track verifier instruction stats for each subprogram The verifier currently records one instruction count for the main program and each global subprogram checked independently. Static subprograms are explored within callers, so their verification cost cannot be reported separately. Track both self and inclusive instruction counts for every subprogram. Charge each processed instruction as self work to the current subprogram and to a path-local subtotal in its function frame. When a function returns, add the callee subtotal to its inclusive count and to its parent subtotal. Fold any remaining frames when a path terminates or is pruned. Instruction subtotals are accounting state, not semantic verifier state. Clear them when a verifier state is copied so work before a path fork is charged once, rather than again when a saved branch is explored. If copying a saved state fails before all frames are allocated, skip missing frames while folding the current path. This generic frame accounting also records self and inclusive totals when an asynchronous callback starts as a fresh frame-zero state. It does not yet charge that independently explored callback path back to the main or global exploration root which scheduled it. That will be done in subsequent changes. This does not change the verification statistics output format. It only prepares the counters for per-subprogram reporting. Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260812221925.3358041-2-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 55 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 61ef43325c6f..51d754bdef5d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1593,6 +1593,8 @@ static int copy_func_state(struct bpf_func_state *dst, const struct bpf_func_state *src) { memcpy(dst, src, offsetof(struct bpf_func_state, stack)); + /* Instruction accounting is path-local, not part of verifier state. */ + dst->insns_subtotal = 0; return copy_stack_state(dst, src); } @@ -9708,6 +9710,42 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, static bool is_rbtree_lock_required_kfunc(u32 btf_id); +static void account_processed_insn(struct bpf_verifier_env *env) +{ + struct bpf_func_state *frame = cur_func(env); + + env->insn_processed++; + frame->insns_subtotal++; + env->subprog_info[frame->subprogno].insns_self++; +} + +static void account_processed_insns(struct bpf_verifier_env *env, + struct bpf_func_state *callee, + struct bpf_func_state *caller) +{ + u32 insns; + + if (!callee) + return; + + insns = callee->insns_subtotal; + + env->subprog_info[callee->subprogno].insns_total += insns; + if (caller) + caller->insns_subtotal += insns; + callee->insns_subtotal = 0; +} + +static void account_current_path(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *state = env->cur_state; + int frame; + + for (frame = state->curframe; frame >= 0; frame--) + account_processed_insns(env, state->frame[frame], + frame ? state->frame[frame - 1] : NULL); +} + /* Are we currently verifying the callback for a rbtree helper that must * be called with lock held? If so, no need to complain about unreleased * lock @@ -9804,6 +9842,7 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) verbose(env, "to caller at %d:\n", *insn_idx); print_verifier_state(env, state, caller->frameno, true); } + account_processed_insns(env, callee, caller); /* clear everything in the callee. In case of exceptional exits using * bpf_throw, this will be done by copy_verifier_state for extra frames. */ free_func_state(callee); @@ -17359,7 +17398,9 @@ static int do_check(struct bpf_verifier_env *env) insn = &insns[env->insn_idx]; insn_aux = &env->insn_aux_data[env->insn_idx]; - if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { + account_processed_insn(env); + + if (env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { verbose(env, "BPF program is too large. Processed %d insn\n", env->insn_processed); @@ -17500,6 +17541,7 @@ static int do_check(struct bpf_verifier_env *env) "speculation barrier after jump instruction may not have the desired effect")) return -EFAULT; process_bpf_exit: + account_current_path(env); mark_verifier_state_scratched(env); err = bpf_update_branch_counts(env, env->cur_state); if (err) @@ -18544,6 +18586,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) ret = do_check(env); out: + account_current_path(env); if (!ret && pop_log) bpf_vlog_reset(&env->log, 0); free_states(env); @@ -18575,7 +18618,6 @@ static int do_check_subprogs(struct bpf_verifier_env *env) struct bpf_prog_aux *aux = env->prog->aux; struct bpf_func_info_aux *sub_aux; int i, ret, new_cnt; - u32 insn_processed; if (!aux->func_info) return 0; @@ -18590,8 +18632,6 @@ again: if (!bpf_subprog_is_global(env, i)) continue; - insn_processed = env->insn_processed; - sub_aux = subprog_aux(env, i); if (!sub_aux->called || sub_aux->verified) continue; @@ -18599,7 +18639,6 @@ again: env->insn_idx = env->subprog_info[i].start; WARN_ON_ONCE(env->insn_idx == 0); ret = do_check_common(env, i); - env->subprog_info[i].insn_processed = env->insn_processed - insn_processed; if (ret) { return ret; } else if (env->log.level & BPF_LOG_LEVEL) { @@ -18626,12 +18665,10 @@ again: static int do_check_main(struct bpf_verifier_env *env) { - u32 insn_processed = env->insn_processed; int ret; env->insn_idx = 0; ret = do_check_common(env, 0); - env->subprog_info[0].insn_processed = env->insn_processed - insn_processed; if (!ret) env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; return ret; @@ -18650,10 +18687,10 @@ static void print_verification_stats(struct bpf_verifier_env *env) for (i = 1; i < subprog_cnt; i++) verbose(env, "+%d", env->subprog_info[i].stack_depth); verbose(env, " max %d\n", env->max_stack_depth); - verbose(env, "insns processed %d", env->subprog_info[0].insn_processed); + verbose(env, "insns processed %d", env->subprog_info[0].insns_total); for (i = 1; i < subprog_cnt; i++) if (bpf_subprog_is_global(env, i)) - verbose(env, "+%d", env->subprog_info[i].insn_processed); + verbose(env, "+%d", env->subprog_info[i].insns_total); verbose(env, "\n"); } verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " -- cgit v1.2.3 From 6137fb7c5f830d07909cbc789f52b1e0fffd6cd2 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 13 Aug 2026 00:19:19 +0200 Subject: bpf: Attribute async callback instructions to verification roots Asynchronous callbacks are explored as fresh frame-zero verifier states, so normal callee-to-caller accounting cannot propagate their instruction budget to the main or global subprogram whose verification scheduled them. The callback exploration still happens within the same do_check_common() invocation as that independent verification root. Record env->insn_processed at do_check_common() entry and override the root's inclusive count with the delta before returning. This includes all directly and transitively scheduled asynchronous callbacks in the root's total without maintaining a separate accounting call stack. Static subprogram and callback totals remain local to their synchronous call paths. Their self counts continue to account for each processed instruction exactly once. Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260812221925.3358041-3-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 51d754bdef5d..fced21ec2304 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -18448,6 +18448,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) struct bpf_prog_aux *aux = env->prog->aux; struct bpf_verifier_state *state; struct bpf_reg_state *regs; + u32 insn_processed = env->insn_processed; int ret, i; env->prev_linfo = NULL; @@ -18590,6 +18591,15 @@ out: if (!ret && pop_log) bpf_vlog_reset(&env->log, 0); free_states(env); + + /* + * The override is needed to account for async subprograms, which + * are verified with their own set of stack frames and thus are + * not accounted as callees by account_current_path(). + * Accumulate their total counts as total counts of the main or + * global subprog hosting the async call. + */ + env->subprog_info[subprog].insns_total = env->insn_processed - insn_processed; return ret; } -- cgit v1.2.3 From c2e6c7de883034a16132b974d6236f8189d5babe Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 13 Aug 2026 00:19:20 +0200 Subject: bpf: Show more useful info in stack depth stats Stack depth statistics list captured depths in subprogram-number order, while per-verification instruction counts are reported separately. Since libbpf determines subprogram numbers, it is hard to associate either statistic with its subprogram name or see where verifier work is spent. Now that self and inclusive instruction counts are available for every subprogram, keep the combined maximum stack depth on its own line and print one uniform record for each subprogram. Represent the main program as subprog 0, then classify each record as main, global, or static before reporting insns_self, insns_total, and stack depth. The aggregate processed count is the sum of all self counts, while each total shows verifier work rooted at that subprogram. When no subprogram name is available, print . Keep the existing aggregate "processed ... insns" record unchanged for compatibility. Suggested-by: Andrii Nakryiko Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260812221925.3358041-4-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index fced21ec2304..73d6cd563cdf 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -18693,15 +18693,20 @@ static void print_verification_stats(struct bpf_verifier_env *env) if (env->log.level & BPF_LOG_STATS) { verbose(env, "verification time %lld usec\n", div_u64(env->verification_time, 1000)); - verbose(env, "stack depth %d", env->subprog_info[0].stack_depth); - for (i = 1; i < subprog_cnt; i++) - verbose(env, "+%d", env->subprog_info[i].stack_depth); - verbose(env, " max %d\n", env->max_stack_depth); - verbose(env, "insns processed %d", env->subprog_info[0].insns_total); - for (i = 1; i < subprog_cnt; i++) - if (bpf_subprog_is_global(env, i)) - verbose(env, "+%d", env->subprog_info[i].insns_total); - verbose(env, "\n"); + verbose(env, "stack depth max %d\n", env->max_stack_depth); + for (i = 0; i < subprog_cnt; i++) { + const char *name = env->subprog_info[i].name; + const char *kind; + + if (!name || !name[0]) + name = ""; + kind = i == 0 ? "main" : + bpf_subprog_is_global(env, i) ? "global" : "static"; + verbose(env, "subprog %d (%s) %s insns_self %d insns_total %d stack %d\n", + i, name, kind, env->subprog_info[i].insns_self, + env->subprog_info[i].insns_total, + env->subprog_info[i].stack_depth); + } } verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " "total_states %d peak_states %d mark_read %d\n", -- cgit v1.2.3 From 0253073fb7d79a2dd2eae9581ea16db2aef395a6 Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Tue, 11 Aug 2026 10:46:19 +0800 Subject: bpf: Fix UAF in bpf_trampoline_multi_attach_free on update failure When bpf_trampoline_update() fails before modify_fentry_multi()/ unregister_fentry_multi() is called, cur_image is unchanged (cur_image == old_image) and ftrace still calls into it. Freeing old_image in that case causes a UAF. Only free old_image when it differs from cur_image. Fixes: aef4dfa790b2 ("bpf: Add bpf_trampoline_multi_attach/detach functions") Signed-off-by: Hui Zhu Acked-by: Leon Hwang Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/aaa3829e11e2e26bcd3bda9ee6df7a0101a718ac.1786412280.git.zhuhui@kylinos.cn Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/trampoline.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index e07af35ed040..008448bb4a1f 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -1632,7 +1632,17 @@ static void bpf_trampoline_multi_attach_init(struct bpf_trampoline *tr) static void bpf_trampoline_multi_attach_free(struct bpf_trampoline *tr) { - if (tr->multi_attach.old_image) + /* + * Only free old_image if it is no longer the active image. + * When bpf_trampoline_update() fails before modify_fentry_multi()/ + * unregister_fentry_multi() is called, cur_image is unchanged + * (cur_image == old_image) and ftrace still points to it. Freeing + * it would cause a UAF when ftrace calls into the freed memory. + * On success, cur_image is either a new image or NULL, so + * old_image != cur_image means the image is stale. + */ + if (tr->multi_attach.old_image && + tr->multi_attach.old_image != tr->cur_image) bpf_tramp_image_put(tr->multi_attach.old_image); tr->multi_attach.old_image = NULL; -- cgit v1.2.3 From 7c6beeb8c88f92866daab4516220667d1234d3c9 Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Tue, 11 Aug 2026 10:46:20 +0800 Subject: bpf: Make bpf_trampoline_multi_detach return void bpf_trampoline_multi_detach() always returns 0 and the sole caller ignores the return value. Change it to return void and drop the WARN_ON_ONCE at the call site. Signed-off-by: Hui Zhu Acked-by: Leon Hwang Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/12beba657f5c9e86a016a097750209287a2f262a.1786412280.git.zhuhui@kylinos.cn Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/trampoline.c | 4 ++-- kernel/trace/bpf_trace.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 008448bb4a1f..90b70ea0d370 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -1766,7 +1766,8 @@ rollback_put: return err; } -int bpf_trampoline_multi_detach(struct bpf_prog *prog, struct bpf_tracing_multi_link *link) +void bpf_trampoline_multi_detach(struct bpf_prog *prog, + struct bpf_tracing_multi_link *link) { struct bpf_tracing_multi_data *data = &link->data; struct bpf_tracing_multi_node *mnode; @@ -1796,7 +1797,6 @@ int bpf_trampoline_multi_detach(struct bpf_prog *prog, struct bpf_tracing_multi_ bpf_trampoline_put(mnode->trampoline); clear_tracing_multi_data(data); - return 0; } #undef for_each_mnode_cnt diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 891897f8a1b3..29260951aa87 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3687,7 +3687,7 @@ static void bpf_tracing_multi_link_release(struct bpf_link *link) struct bpf_tracing_multi_link *tr_link = container_of(link, struct bpf_tracing_multi_link, link); - WARN_ON_ONCE(bpf_trampoline_multi_detach(link->prog, tr_link)); + bpf_trampoline_multi_detach(link->prog, tr_link); } static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) -- cgit v1.2.3 From 7c3e54cb82fda75a374e2b29b25d2a399911a008 Mon Sep 17 00:00:00 2001 From: Xu Kuohai Date: Tue, 28 Jul 2026 20:25:49 +0000 Subject: bpf: Eliminate dup/restore of insn_aux_data The dup/restore of insn_aux_data was introduced to resolve the inconsistency between insnsi and insn_aux_data arrays, which occurs on the failure path where insnsi was rolled back to the original state before constants blinding, while insn_aux_data was not. After JIT failure, there is only one user, bpf_clear_insn_aux_data(), that requires insnsi and insn_aux_data to be synchronized. It accesses both insnsi and insn_aux_data using the same array size and index. However, the access to insnsi in bpf_clear_insn_aux_data() is not necessary. It is checked to skip the second slot of an ldimm64 instruction, whose jt is never set and can be absorbed into the jt check itself. So remove the access to insnsi from bpf_clear_insn_aux_data(), and add a specific length field for insn_aux_data to allow it to have a different length from the insnsi array. Then remove dup/restore of insn_aux_data. Signed-off-by: Xu Kuohai Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/5a4528f019c8d2638c019a2f37475cccc16a9503.1785240296.git.xukuohai@huawei.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/core.c | 16 ---------------- kernel/bpf/fixups.c | 36 ++---------------------------------- kernel/bpf/verifier.c | 4 ++-- 3 files changed, 4 insertions(+), 52 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index a3e1fae32eac..6a94370a2448 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2634,22 +2634,10 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc { #ifdef CONFIG_BPF_JIT struct bpf_prog *orig_prog; - struct bpf_insn_aux_data *orig_insn_aux; if (!bpf_prog_need_blind(prog)) return bpf_int_jit_compile(env, prog); - if (env) { - /* - * If env is not NULL, we are called from the end of bpf_check(), at this - * point, only insn_aux_data is used after failure, so it should be restored - * on failure. - */ - orig_insn_aux = bpf_dup_insn_aux_data(env); - if (!orig_insn_aux) - return prog; - } - orig_prog = prog; prog = bpf_jit_blind_constants(env, prog); /* @@ -2662,8 +2650,6 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc prog = bpf_int_jit_compile(env, prog); if (prog->jited) { bpf_jit_prog_release_other(prog, orig_prog); - if (env) - vfree(orig_insn_aux); return prog; } @@ -2671,8 +2657,6 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc out_restore: prog = orig_prog; - if (env) - bpf_restore_insn_aux_data(env, orig_insn_aux); #endif return prog; } diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index c4bd70befbb5..2417a3461652 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -230,6 +230,7 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, if (cnt == 1) return; prog_len = new_prog->len; + env->insn_aux_data_len = prog_len; memmove(data + off + cnt - 1, data + off, sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); @@ -496,7 +497,6 @@ static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; - struct bpf_insn *insns = env->prog->insnsi; int end = start + len; int i; @@ -505,9 +505,6 @@ void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) kvfree(aux_data[i].jt); aux_data[i].jt = NULL; } - - if (bpf_is_ldimm64(&insns[i])) - i++; } } @@ -520,7 +517,6 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) if (bpf_prog_is_offloaded(env->prog->aux)) bpf_prog_offload_remove_insns(env, off, cnt); - /* Should be called before bpf_remove_insns, as it uses prog->insnsi */ bpf_clear_insn_aux_data(env, off, cnt); err = bpf_remove_insns(env->prog, off, cnt); @@ -539,6 +535,7 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) memmove(aux_data + off, aux_data + off + cnt, sizeof(*aux_data) * (orig_prog_len - off - cnt)); + env->insn_aux_data_len -= cnt; return 0; } @@ -1057,26 +1054,6 @@ static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_s env->subprog_info[env->subprog_cnt].start = env->prog->len; } -struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env) -{ - size_t size; - void *new_aux; - - size = array_size(sizeof(struct bpf_insn_aux_data), env->prog->len); - new_aux = __vmalloc(size, GFP_KERNEL_ACCOUNT); - if (new_aux) - memcpy(new_aux, env->insn_aux_data, size); - return new_aux; -} - -void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_insn_aux_data *orig_insn_aux) -{ - /* the expanded elements are zero-filled, so no special handling is required */ - vfree(env->insn_aux_data); - env->insn_aux_data = orig_insn_aux; -} - static int jit_subprogs(struct bpf_verifier_env *env) { struct bpf_prog *prog = env->prog, **func, *tmp; @@ -1351,7 +1328,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) bool blinded = false; struct bpf_insn *insn; struct bpf_prog *prog, *orig_prog; - struct bpf_insn_aux_data *orig_insn_aux; u32 *orig_subprog_starts; if (env->subprog_cnt <= 1) @@ -1359,14 +1335,8 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) prog = orig_prog = env->prog; if (bpf_prog_need_blind(prog)) { - orig_insn_aux = bpf_dup_insn_aux_data(env); - if (!orig_insn_aux) { - err = -ENOMEM; - goto out_cleanup; - } orig_subprog_starts = bpf_dup_subprog_starts(env); if (!orig_subprog_starts) { - vfree(orig_insn_aux); err = -ENOMEM; goto out_cleanup; } @@ -1386,7 +1356,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) if (blinded) { bpf_jit_prog_release_other(prog, orig_prog); kvfree(orig_subprog_starts); - vfree(orig_insn_aux); } return 0; @@ -1416,7 +1385,6 @@ out_jit_err: out_restore: bpf_restore_subprog_starts(env, orig_subprog_starts); - bpf_restore_insn_aux_data(env, orig_insn_aux); kvfree(orig_subprog_starts); out_cleanup: /* cleanup main prog to be interpreted */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 73d6cd563cdf..7a4e1b88d73a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20213,7 +20213,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (!is_priv) mutex_lock(&bpf_verifier_lock); - len = env->prog->len; + len = env->insn_aux_data_len = env->prog->len; env->insn_aux_data = __vmalloc(array_size(sizeof(struct bpf_insn_aux_data), len), GFP_KERNEL_ACCOUNT | __GFP_ZERO); @@ -20468,7 +20468,7 @@ err_prep: release_btfs(env); err_free_env: if (env->insn_aux_data) - bpf_clear_insn_aux_data(env, 0, env->prog->len); + bpf_clear_insn_aux_data(env, 0, env->insn_aux_data_len); vfree(env->insn_aux_data); kvfree(env->fd_array); bpf_stack_liveness_free(env); -- cgit v1.2.3 From 6f033615ef8fb2374daa7e50a8ff68616bc850d2 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Thu, 13 Aug 2026 00:48:43 +0800 Subject: bpf: Trim special_kfunc_list in verifier The commit 7619a0ee9340 ("bpf: Mark existing lock-safe kfuncs with KF_SPINLOCK_SAFE") dropped some helpers in verifier, which also eliminated the use of the following kfuncs from the special_kfunc_list: * bpf_arena_reserve_pages * bpf_stream_vprintk * bpf_stream_print_stack So, drop them from the special_kfunc_list. Signed-off-by: Leon Hwang Link: https://lore.kernel.org/bpf/20260812164843.55601-1-leon.hwang@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7a4e1b88d73a..164d16c243ca 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11122,10 +11122,7 @@ enum special_kfunc_type { KF_bpf_task_work_schedule_resume, KF_bpf_arena_alloc_pages, KF_bpf_arena_free_pages, - KF_bpf_arena_reserve_pages, KF_bpf_session_is_return, - KF_bpf_stream_vprintk, - KF_bpf_stream_print_stack, }; BTF_ID_LIST(special_kfunc_list) @@ -11215,14 +11212,11 @@ BTF_ID(func, bpf_task_work_schedule_signal) BTF_ID(func, bpf_task_work_schedule_resume) BTF_ID(func, bpf_arena_alloc_pages) BTF_ID(func, bpf_arena_free_pages) -BTF_ID(func, bpf_arena_reserve_pages) #ifdef CONFIG_BPF_EVENTS BTF_ID(func, bpf_session_is_return) #else BTF_ID_UNUSED #endif -BTF_ID(func, bpf_stream_vprintk) -BTF_ID(func, bpf_stream_print_stack) static bool is_bpf_obj_new_kfunc(u32 func_id) { -- cgit v1.2.3 From 0a07e75b16b6788948198ca8576a3a72324c7ca3 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Thu, 13 Aug 2026 23:23:14 +0800 Subject: bpf: Drop duplicate blank lines in kernel/bpf/ There are many adjacent blank lines in kernel/bpf/ that have accumulated over time. Drop them for cleanup. No functional changes intended. Signed-off-by: Leon Hwang Signed-off-by: Andrii Nakryiko Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260813152324.97937-2-leon.hwang@linux.dev --- kernel/bpf/backtrack.c | 2 -- kernel/bpf/btf.c | 1 - kernel/bpf/cfg.c | 1 - kernel/bpf/fixups.c | 1 - kernel/bpf/hashtab.c | 2 -- kernel/bpf/helpers.c | 1 - kernel/bpf/liveness.c | 2 -- kernel/bpf/queue_stack_maps.c | 1 - kernel/bpf/syscall.c | 5 ----- kernel/bpf/verifier.c | 14 -------------- 10 files changed, 30 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 40bd04421a99..a2b18a9f1694 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -214,7 +214,6 @@ static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg) return bt->reg_masks[bt->frame] & (1 << reg); } - /* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */ static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask) { @@ -254,7 +253,6 @@ void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask) } } - /* For given verifier state backtrack_insn() is called from the last insn to * the first insn. Its purpose is to compute a bitmask of registers and * stack slots that needs precision in the parent verifier state. diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 6606187ed4f4..87ffde865a50 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -2534,7 +2534,6 @@ static void btf_bitfield_show(void *data, u8 bits_offset, btf_int128_print(show, print_num); } - static void btf_int_bits_show(const struct btf *btf, const struct btf_type *t, void *data, u8 bits_offset, diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index db3416a7c904..818f7afac83a 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -47,7 +47,6 @@ enum { BRANCH = 2, }; - static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off) { struct bpf_subprog_info *subprog; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 2417a3461652..0caf1bbd9494 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1466,7 +1466,6 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) return err; } - /* The function requires that first instruction in 'patch' is insnsi[prog->len - 1] */ static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len) { diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 9f394e1aa2e8..d40cb5dd446c 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -998,7 +998,6 @@ static void dec_elem_count(struct bpf_htab *htab) atomic_dec(&htab->count); } - static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) { htab_put_fd_value(htab, l); @@ -2970,7 +2969,6 @@ static int rhtab_delete_elem(struct bpf_rhtab *rhtab, struct rhtab_elem *elem, v return 0; } - static long rhtab_map_delete_elem(struct bpf_map *map, void *key) { struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 6388b6b23e49..45e2f19387b2 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -4871,7 +4871,6 @@ static const struct btf_kfunc_id_set generic_kfunc_set = { .set = &generic_btf_ids, }; - BTF_ID_LIST(generic_dtor_ids) BTF_ID(struct, task_struct) BTF_ID(func, bpf_task_release_dtor) diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 1c997aeba6fa..74fc4b3f80d6 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -269,7 +269,6 @@ bpf_insn_successors(struct bpf_verifier_env *env, u32 idx) __diag_pop(); - static inline bool update_insn(struct bpf_verifier_env *env, struct func_instance *instance, u32 frame, u32 insn_idx) { @@ -1862,7 +1861,6 @@ static int analyze_subprog(struct bpf_verifier_env *env, if (need_resched()) cond_resched(); - /* * When an instance is reused (must_write_initialized == true), * record into a fresh instance and merge afterward. This avoids diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c index c1c9dee4dcdd..6e8b18c32a10 100644 --- a/kernel/bpf/queue_stack_maps.c +++ b/kernel/bpf/queue_stack_maps.c @@ -123,7 +123,6 @@ out: return err; } - static long __stack_map_get(struct bpf_map *map, void *value, bool delete) { struct bpf_queue_stack *qs = bpf_queue_stack(map); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 8d111da88655..7d8c3e8e6d62 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -636,7 +636,6 @@ int bpf_map_alloc_pages(const struct bpf_map *map, int nid, return ret; } - static int btf_field_cmp(const void *a, const void *b) { const struct btf_field *f1 = a, *f2 = b; @@ -1830,7 +1829,6 @@ free_key: return err; } - #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) @@ -3497,7 +3495,6 @@ int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer) if (fd < 0) return fd; - id = bpf_link_alloc_id(link); if (id < 0) { put_unused_fd(fd); @@ -5505,7 +5502,6 @@ static int bpf_link_get_info_by_fd(struct file *file, return 0; } - static int token_get_info_by_fd(struct file *file, struct bpf_token *token, const union bpf_attr *attr, @@ -6507,7 +6503,6 @@ BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size) return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size, KERNEL_BPFPTR(NULL), 0); } - /* To shut up -Wmissing-prototypes. * This function is used by the kernel light skeleton * to load bpf programs when modules are loaded or during kernel boot. diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 164d16c243ca..cdb79a66b156 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -635,7 +635,6 @@ static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type, bool first_slot, int id, int parent_id); - static void mark_dynptr_stack_regs(struct bpf_verifier_env *env, struct bpf_reg_state *sreg1, struct bpf_reg_state *sreg2, @@ -1674,7 +1673,6 @@ static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_sta return true; } - void bpf_free_backedges(struct bpf_scc_visit *visit) { struct bpf_scc_backedge *backedge, *next; @@ -2291,7 +2289,6 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, return &elem->st; } - static int cmp_subprogs(const void *a, const void *b) { return ((struct bpf_subprog_info *)a)->start - @@ -3969,7 +3966,6 @@ static int check_stack_read(struct bpf_verifier_env *env, return err; } - /* check_stack_write dispatches to check_stack_write_fixed_off or * check_stack_write_var_off. * @@ -4767,7 +4763,6 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, valid = false; } - if (valid) { env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; @@ -6635,7 +6630,6 @@ static int check_stack_range_initialized( if (err) return err; - if (tnum_is_const(reg->var_off)) { min_off = max_off = reg->var_off.value + off; } else { @@ -7347,7 +7341,6 @@ static bool is_iter_new_kfunc(struct bpf_call_arg_meta *meta) return meta->kfunc_flags & KF_ITER_NEW; } - static bool is_iter_destroy_kfunc(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ITER_DESTROY; @@ -11607,7 +11600,6 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * return 0; } - static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg) { struct btf_record *rec = reg_btf_record(reg); @@ -16412,7 +16404,6 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) return 0; } - static bool return_retval_range(struct bpf_verifier_env *env, struct bpf_retval_range *range) { enum bpf_prog_type prog_type = resolve_prog_type(env->prog); @@ -18361,8 +18352,6 @@ static void release_insn_arrays(struct bpf_verifier_env *env) bpf_insn_array_release(env->insn_array_maps[i]); } - - /* The verifier does more data flow analysis than llvm and will not * explore branches that are dead at run time. Malicious programs can * have dead code too. Therefore replace all dead at-run-time code @@ -18390,8 +18379,6 @@ static void sanitize_dead_code(struct bpf_verifier_env *env) } } - - static void free_states(struct bpf_verifier_env *env) { struct bpf_verifier_state_list *sl; @@ -18678,7 +18665,6 @@ static int do_check_main(struct bpf_verifier_env *env) return ret; } - static void print_verification_stats(struct bpf_verifier_env *env) { /* Skip over hidden subprogs which are not verified. */ -- cgit v1.2.3 From bed7d65ff499e58c8504a2a3387ee95b551f6951 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Thu, 13 Aug 2026 23:23:15 +0800 Subject: bpf: Factor out check_map_mem_read helper in verifier In the next commit, percpu_array map will add map_direct_value_addr support. IOW, it will add a map_type check in the iff condition of the bpf_map_direct_read() code block, which will reduce the code block readability. Hence, factor out check_map_mem_read helper to improve the readability, and the maintainability for the percpu_array map case. Signed-off-by: Leon Hwang Signed-off-by: Andrii Nakryiko Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260813152324.97937-3-leon.hwang@linux.dev --- kernel/bpf/verifier.c | 75 +++++++++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 32 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index cdb79a66b156..4fac230122d9 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6078,6 +6078,48 @@ static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val) reg_bounds_sync(dst_reg); } +static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, + int bpf_size, int value_regno, bool is_ldsx) +{ + struct bpf_reg_state *regs = cur_regs(env); + int size = bpf_size_to_bytes(bpf_size); + struct bpf_map *map = reg->map_ptr; + + switch (map->map_type) { + case BPF_MAP_TYPE_INSN_ARRAY: + if (bpf_size != BPF_DW) { + verbose(env, "Invalid read of %d bytes from insn_array\n", size); + return -EACCES; + } + regs[value_regno] = *reg; + add_scalar_to_reg(®s[value_regno], off); + regs[value_regno].type = PTR_TO_INSN; + return 0; + default: + break; + } + + /* If map is read-only, track its contents as scalars. */ + if (tnum_is_const(reg->var_off) && + bpf_map_is_rdonly(map) && + map->ops->map_direct_value_addr) { + int map_off = off + reg->var_off.value; + u64 val = 0; + int err; + + err = bpf_map_direct_read(map, map_off, size, &val, is_ldsx); + if (err) + return err; + + regs[value_regno].type = SCALAR_VALUE; + __mark_reg_known(®s[value_regno], val); + return 0; + } + + mark_reg_unknown(env, regs, value_regno); + return 0; +} + /* check whether memory at (regno + off) is accessible for t = (read | write) * if t==write, value_regno is a register which value is stored into memory * if t==read, value_regno is a register which will receive the value from memory @@ -6132,38 +6174,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (kptr_field) { err = check_map_kptr_access(env, value_regno, insn_idx, kptr_field); } else if (t == BPF_READ && value_regno >= 0) { - struct bpf_map *map = reg->map_ptr; - - /* - * If map is read-only, track its contents as scalars, - * unless it is an insn array (see the special case below) - */ - if (tnum_is_const(reg->var_off) && - bpf_map_is_rdonly(map) && - map->ops->map_direct_value_addr && - map->map_type != BPF_MAP_TYPE_INSN_ARRAY) { - int map_off = off + reg->var_off.value; - u64 val = 0; - - err = bpf_map_direct_read(map, map_off, size, - &val, is_ldsx); - if (err) - return err; - - regs[value_regno].type = SCALAR_VALUE; - __mark_reg_known(®s[value_regno], val); - } else if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) { - if (bpf_size != BPF_DW) { - verbose(env, "Invalid read of %d bytes from insn_array\n", - size); - return -EACCES; - } - regs[value_regno] = *reg; - add_scalar_to_reg(®s[value_regno], off); - regs[value_regno].type = PTR_TO_INSN; - } else { - mark_reg_unknown(env, regs, value_regno); - } + err = check_map_mem_read(env, reg, off, bpf_size, value_regno, is_ldsx); } } else if (base_type(reg->type) == PTR_TO_MEM) { bool rdonly_mem = type_is_rdonly_mem(reg->type); -- cgit v1.2.3 From 6e61f4f8b04362d03136a4ad8f68adf72127a3af Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Thu, 13 Aug 2026 23:23:16 +0800 Subject: bpf: Introduce global percpu data Introduce global percpu data, inspired by the commit 6316f78306c1 ("Merge branch 'support-global-data'"). It enables the definition of global percpu variables in BPF, similar to the include/linux/percpu-defs.h::DEFINE_PER_CPU() macro. For example, in BPF, it is able to define a global percpu variable like: int data SEC(".percpu"); With this patch, tools like retsnoop [1] and bpfsnoop [2] can simplify their BPF code for handling LBRs. The code can be updated from static struct perf_branch_entry lbrs[1][MAX_LBR_ENTRIES] SEC(".data.lbrs"); to static struct perf_branch_entry lbrs[MAX_LBR_ENTRIES] SEC(".percpu.lbrs"); This eliminates the need to retrieve the CPU ID using the bpf_get_smp_processor_id() helper. Additionally, by reusing global percpu data map, sharing information between tail callers and callees or freplace callers and callees becomes simpler compared to reusing percpu_array maps. Links: [1] https://github.com/anakryiko/retsnoop [2] https://github.com/bpfsnoop/bpfsnoop Signed-off-by: Leon Hwang Signed-off-by: Andrii Nakryiko Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260813152324.97937-4-leon.hwang@linux.dev --- kernel/bpf/arraymap.c | 38 ++++++++++++++++++++++++++++++++++++-- kernel/bpf/const_fold.c | 1 - kernel/bpf/fixups.c | 37 +++++++++++++++++++++++++++++++++++++ kernel/bpf/verifier.c | 11 +++++++++++ 4 files changed, 84 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 248b4818178c..34865701f7f7 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -259,6 +259,37 @@ static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key) return this_cpu_ptr(array->pptrs[index & array->index_mask]); } +static int percpu_array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, u32 off) +{ + struct bpf_array *array = container_of(map, struct bpf_array, map); + + if (!bpf_jit_supports_percpu_insn()) + return -EOPNOTSUPP; + if (map->max_entries != 1) + return -EOPNOTSUPP; + if (off >= map->value_size) + return -EINVAL; + + *imm = (u64)(__force unsigned long) array->pptrs[0]; + return 0; +} + +static int percpu_array_map_direct_value_meta(const struct bpf_map *map, u64 imm, u32 *off) +{ + struct bpf_array *array = container_of(map, struct bpf_array, map); + u64 base = (u64)(__force unsigned long) array->pptrs[0]; + + if (!bpf_jit_supports_percpu_insn()) + return -EOPNOTSUPP; + if (map->max_entries != 1) + return -EOPNOTSUPP; + if (imm < base || imm >= base + array->elem_size) + return -ENOENT; + + *off = imm - base; + return 0; +} + /* emit BPF instructions equivalent to C code of percpu_array_map_lookup_elem() */ static int percpu_array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) { @@ -551,9 +582,10 @@ static int array_map_check_btf(struct bpf_map *map, const struct btf_type *key_type, const struct btf_type *value_type) { - /* One exception for keyless BTF: .bss/.data/.rodata map */ + /* One exception for keyless BTF: .bss/.data/.rodata/.percpu map */ if (btf_type_is_void(key_type)) { - if (map->map_type != BPF_MAP_TYPE_ARRAY || + if ((map->map_type != BPF_MAP_TYPE_ARRAY && + map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) || map->max_entries != 1) return -EINVAL; @@ -832,6 +864,8 @@ const struct bpf_map_ops percpu_array_map_ops = { .map_get_next_key = bpf_array_get_next_key, .map_lookup_elem = percpu_array_map_lookup_elem, .map_gen_lookup = percpu_array_map_gen_lookup, + .map_direct_value_addr = percpu_array_map_direct_value_addr, + .map_direct_value_meta = percpu_array_map_direct_value_meta, .map_update_elem = array_map_update_elem, .map_delete_elem = array_map_delete_elem, .map_lookup_percpu_elem = percpu_array_map_lookup_percpu_elem, diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c index 4cf120c7b2cb..7f1b30059cc8 100644 --- a/kernel/bpf/const_fold.c +++ b/kernel/bpf/const_fold.c @@ -182,7 +182,6 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info * u64 val = 0; if (!bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr || - map->map_type == BPF_MAP_TYPE_INSN_ARRAY || off < 0 || off + size > map->value_size || bpf_map_direct_read(map, off, size, &val, is_ldsx)) { *dst = unknown; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 0caf1bbd9494..177a3fcbb63a 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1834,6 +1834,43 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) goto next_insn; } + if (bpf_jit_supports_percpu_insn() && + insn->code == (BPF_LD | BPF_IMM | BPF_DW) && + (insn->src_reg == BPF_PSEUDO_MAP_VALUE || + insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)) { + struct bpf_map *map; + + aux = &env->insn_aux_data[i + delta]; + map = env->used_maps[aux->map_index]; + if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) + goto next_insn; + + prog->jit_required = true; + + /* + * We are *skipping* first half of ld_imm64 insn + * with 'i++;', patching over second half of it + * with that same half + mov64_percpu_reg insn. + * All because bpf_patch_insn_data() can only + * replace one 8-byte insn, which does not work + * well for ld_imm64 insn. + */ + + insn_buf[0] = insn[1]; + insn_buf[1] = BPF_MOV64_PERCPU_REG(insn->dst_reg, insn->dst_reg); + cnt = 2; + + i++; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = prog = new_prog; + insn = new_prog->insnsi + i + delta; + goto next_insn; + } + if (insn->code != (BPF_JMP | BPF_CALL)) goto next_insn; if (insn->src_reg == BPF_PSEUDO_CALL) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 4fac230122d9..6ac1afced20b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5582,6 +5582,8 @@ int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, u64 addr; int err; + if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY || map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) + return -EINVAL; err = map->ops->map_direct_value_addr(map, &addr, off); if (err) return err; @@ -6095,6 +6097,8 @@ static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state add_scalar_to_reg(®s[value_regno], off); regs[value_regno].type = PTR_TO_INSN; return 0; + case BPF_MAP_TYPE_PERCPU_ARRAY: + goto reg_unknown; default: break; } @@ -6116,6 +6120,7 @@ static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state return 0; } +reg_unknown: mark_reg_unknown(env, regs, value_regno); return 0; } @@ -8129,6 +8134,12 @@ static int check_arg_const_str(struct bpf_verifier_env *env, return -EACCES; } + if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { + verbose(env, "%s points to percpu_array map which cannot be used as const string\n", + reg_arg_name(env, argno)); + return -EACCES; + } + if (!bpf_map_is_rdonly(map)) { verbose(env, "%s does not point to a readonly map'\n", reg_arg_name(env, argno)); return -EACCES; -- cgit v1.2.3 From aacd13e1eb68f2c9049fc0cf7aed89694c3e0713 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 13 Aug 2026 01:15:05 +0200 Subject: bpf: Fix func_info_aux desync after dead code elimination The verifier keeps per-subprogram metadata in three parallel arrays: subprog_info, func_info, and func_info_aux. Dead code elimination can remove whole subprograms, and adjust_subprog_starts_after_remove() shifts subprog_info and func_info to close the gap, but leaves func_info_aux in place. From that point on, func_info_aux[i] no longer describes subprogram i. Shift func_info_aux together with func_info so the three arrays stay aligned after subprogram removal. Reported-by: Sashiko Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260808064523.DE3E71F000E9@smtp.kernel.org Link: https://lore.kernel.org/bpf/20260812231506.3558128-1-memxor@gmail.com --- kernel/bpf/fixups.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 177a3fcbb63a..70f22eb63ed5 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -402,13 +402,17 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, sizeof(*env->subprog_info) * move); env->subprog_cnt -= j - i; - /* remove func_info */ + /* remove func_info and its aux */ if (aux->func_info) { move = aux->func_info_cnt - j; memmove(aux->func_info + i, aux->func_info + j, sizeof(*aux->func_info) * move); + if (aux->func_info_aux) + memmove(aux->func_info_aux + i, + aux->func_info_aux + j, + sizeof(*aux->func_info_aux) * move); aux->func_info_cnt -= j - i; /* func_info->insn_off is set after all code rewrites, * in adjust_btf_func() - no need to adjust -- cgit v1.2.3 From b0e872a31e157d48479507c2821ba8bf6323b7ad Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Wed, 12 Aug 2026 07:47:22 -0700 Subject: bpf: Fix arm64 KASAN false positive after bpf_throw arm64 passes zero as the stack pointer while walking BPF frames, so bpf_throw() leaves stale KASAN stack poison after jumping to the exception callback. Use the frame pointer as the fallback stack watermark. Fixes: e74cb1b42213 ("arm64: stacktrace: Implement arch_bpf_stack_walk() for the BPF JIT") Signed-off-by: Mykyta Yatsenko Signed-off-by: Daniel Borkmann Tested-by: Ihor Solodrai Link: https://lore.kernel.org/bpf/20260812-hello_world-v1-1-c3c2ddcb362d@meta.com --- kernel/bpf/helpers.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 45e2f19387b2..b3cc5c8fc875 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -3395,11 +3395,13 @@ __bpf_kfunc void bpf_throw(u64 cookie) WARN_ON_ONCE(!ctx.aux->exception_boundary); WARN_ON_ONCE(!ctx.bp); WARN_ON_ONCE(!ctx.cnt); - /* Prevent KASAN false positives for CONFIG_KASAN_STACK by unpoisoning + /* + * Prevent KASAN false positives for CONFIG_KASAN_STACK by unpoisoning * deeper stack depths than ctx.sp as we do not return from bpf_throw, - * which skips compiler generated instrumentation to do the same. + * which skips compiler generated instrumentation to do the same. Some + * architectures cannot recover sp while unwinding, so fall back to bp. */ - kasan_unpoison_task_stack_below((void *)(long)ctx.sp); + kasan_unpoison_task_stack_below((void *)(long)(ctx.sp ?: ctx.bp)); ctx.aux->bpf_exception_cb(cookie, ctx.sp + ctx.aux->stack_arg_sp_adjust, ctx.bp, 0, 0); WARN(1, "A call to BPF exception callback should never return\n"); } -- cgit v1.2.3 From 6ff5b56a50c5351aeeb180e34327736576c038fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Israel=20T=C3=A9llez=20Garc=C3=ADa?= Date: Fri, 14 Aug 2026 14:48:40 +0200 Subject: bpf: Fix pending_pos walk on 32-bit ring position wrap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reservation path caches the position of the oldest not-yet-committed record in rb->pending_pos and advances it past already committed records on every reservation: while (pend_pos < prod_pos) { consumer_pos, producer_pos and pending_pos are unsigned long, i.e. 32-bit on 32-bit architectures, and Documentation/bpf/ringbuf.rst states that these counters may wrap around there. Every other comparison in the file is written as a difference, so modular arithmetic keeps them correct across the wrap. This one is an ordering comparison, and it is not wrap-safe. Once producer_pos wraps past 2^32, prod_pos is small while pend_pos still holds its pre-wrap value, so the loop condition is false and pending_pos is never advanced again. Reservations keep succeeding for a while, because bpf_ringbuf_has_space() uses differences, but new_prod_pos - pend_pos grows as the producer advances, and once it exceeds rb->mask every subsequent __bpf_ringbuf_reserve() call fails: the kernel believes a pending record spans the whole buffer. The ring never recovers, bpf_ringbuf_output() drops every event from then on, and nothing is logged. Observed on four armv7 devices (i.MX7 Dual, 6.6.52) running a tracepoint-based collector with a 512 KiB ring and 160-byte records. Every one of them stopped delivering after exactly 26846821 records and 4295491360 bytes had passed through the ring, at event rates between 441 and 862 records/s, that is after 8 h to 17 h of uptime: the trigger is the byte count, not time or load. That figure is 2^32 plus 524064 bytes, and the excess is one ring's worth of grace period, as expected while new_prod_pos - pend_pos is still below rb->mask. The last reservation that fits is the largest record boundary X with X + 160 <= 524287, and since 2^32 mod 160 = 96 the boundaries after the wrap sit at X = 64 (mod 160), giving X = 524064. Userspace kept consuming normally until the producer stopped, then read zero records for good. With this patch applied, one of the four devices took 10 GiB through the same ring with no stall, while the three unpatched ones kept wedging at the same byte count. 64-bit hosts are unaffected in practice: their counters would need 16 EiB to wrap. Compare the two positions as a difference instead. pending_pos never runs ahead of producer_pos, so the unsigned difference is the real distance between them and stays correct across the wrap. Fixes: cfa1a2329a69 ("bpf: Fix overrunning reservations in ringbuf") Signed-off-by: Israel Téllez García Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260814124843.22041-2-i.tellez@btesa.com --- kernel/bpf/ringbuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index c1bf7a197a96..99487019a8a8 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -482,7 +482,7 @@ static void *__bpf_ringbuf_reserve(struct bpf_ringbuf *rb, u64 size) prod_pos = rb->producer_pos; new_prod_pos = prod_pos + len; - while (pend_pos < prod_pos) { + while (prod_pos - pend_pos > 0) { hdr = (void *)rb->data + (pend_pos & rb->mask); hdr_len = READ_ONCE(hdr->len); if (hdr_len & BPF_RINGBUF_BUSY_BIT) -- cgit v1.2.3 From 3f611e9b820ee0d01af89bb0643ccfac76cc569d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Israel=20T=C3=A9llez=20Garc=C3=ADa?= Date: Fri, 14 Aug 2026 14:48:41 +0200 Subject: bpf: Fix available-data accounting on 32-bit wrap in overwrite mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In overwrite mode ringbuf_avail_data_sz() picks the newer of the consumer and overwrite positions before measuring how much data is available: return prod_pos - max(cons_pos, over_pos); max() is an ordering comparison, and consumer_pos, producer_pos and overwrite_pos are unsigned long, i.e. 32-bit on 32-bit architectures, where Documentation/bpf/ringbuf.rst allows them to wrap. Once one of the two positions has wrapped and the other has not, max() returns the older one: the result is then a modular difference close to 2^32, so the function reports far more available data than the ring can hold. Pollers using BPF_RB_AVAIL_DATA get a bogus figure, and epoll consumers can be woken with nothing to read. Compare distances rather than positions. prod_pos - X is the amount of data produced since X for either position, wrap or no wrap, so the newer position is simply the one with the smaller distance, which is also the value the function wants to return. 64-bit hosts are unaffected in practice: their counters would need 16 EiB to wrap. Found by review of the same class of bug fixed in "bpf: Fix pending_pos walk on 32-bit ring position wrap". Signed-off-by: Israel Téllez García Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260814124843.22041-3-i.tellez@btesa.com --- kernel/bpf/ringbuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index 99487019a8a8..3f1013d80544 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -321,7 +321,7 @@ static unsigned long ringbuf_avail_data_sz(struct bpf_ringbuf *rb) if (unlikely(rb->overwrite_mode)) { over_pos = smp_load_acquire(&rb->overwrite_pos); prod_pos = smp_load_acquire(&rb->producer_pos); - return prod_pos - max(cons_pos, over_pos); + return min(prod_pos - cons_pos, prod_pos - over_pos); } else { prod_pos = smp_load_acquire(&rb->producer_pos); return prod_pos - cons_pos; -- cgit v1.2.3 From f5b57e9e9cfd9736246eb9a5f385da451d199039 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Fri, 14 Aug 2026 08:56:23 -0700 Subject: bpf: Populate mmap-able array map memory lazily An mmap-able BPF array map (BPF_F_MMAPABLE) has its backing memory vmalloc'ed up front at map creation time. array_map_mmap() then wired up the whole mapping eagerly via remap_vmalloc_range(), which calls vm_insert_page() for every page of the map. For large maps this makes every mmap() O(number of pages): an 8MiB map inserts 2048 PTEs per mmap() and tears them all down again on munmap(), even when user space only touches a few pages (or none at all). Populate the mapping lazily instead, the same way the arena map already does. array_map_mmap() now only performs the bounds check and returns, leaving the PTEs unpopulated; pages are inserted on demand by a new array_map_mmap_fault() handler. Because the memory is already resident, the fault handler simply resolves the vmalloc page and hands it to the fault path. This makes mmap() O(1), and munmap() proportional to the number of pages that were actually faulted in rather than to the size of the map. The handler is reached through a new optional ->map_mmap_fault callback. Maps that provide it get a vm_operations_struct with a .fault handler; maps that populate their mapping eagerly keep the one they had. Both share the same open/close callbacks, so the existing VMA accounting (VM_MAYWRITE write-active tracking, freeze handling) stays centralized rather than each map installing its own vm_operations_struct. Callers that want the pages populated up front can still request that explicitly with MAP_POPULATE. Kernel-side access to the map (via the vmalloc address) is unaffected. Time for one mmap()+munmap() of an 8MiB mmap-able array map: before after no MAP_POPULATE, no access 226us 1.1us no MAP_POPULATE, access all pages 236us 1341us MAP_POPULATE, no access 312us 493us MAP_POPULATE, access all pages 318us 519us Mapping without touching the data, which is what this change targets, gets ~160x cheaper. Faulting in the whole mapping one page at a time is more expensive than the eager remap_vmalloc_range() loop, so users that do touch every page should ask for MAP_POPULATE. Note that MAP_POPULATE is not free before this change either: it adds ~85us (226us => 312us) for no benefit, as the mapping is already fully populated. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Song Liu Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260814155623.111565-1-song@kernel.org --- kernel/bpf/arraymap.c | 34 ++++++++++++++++++++++++++++++---- kernel/bpf/syscall.c | 15 ++++++++++++++- 2 files changed, 44 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 34865701f7f7..ef315b168b29 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -608,17 +608,42 @@ static int array_map_check_btf(struct bpf_map *map, static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma) { struct bpf_array *array = container_of(map, struct bpf_array, map); - pgoff_t pgoff = PAGE_ALIGN(sizeof(*array)) >> PAGE_SHIFT; if (!(map->map_flags & BPF_F_MMAPABLE)) return -EINVAL; - if (vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) > + /* use u64 math so the offset cannot overflow on 32-bit archs */ + if ((u64)vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) > PAGE_ALIGN((u64)array->map.max_entries * array->elem_size)) return -EINVAL; - return remap_vmalloc_range(vma, array_map_vmalloc_addr(array), - vma->vm_pgoff + pgoff); + /* + * Pages are faulted in on demand by array_map_mmap_fault(). Set the + * same flags that the eager remap_vmalloc_range() path used to set + * via vm_insert_page(), so that e.g. NUMA balancing keeps skipping + * these VMAs. + */ + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP | VM_MIXEDMAP); + + return 0; +} + +static vm_fault_t array_map_mmap_fault(struct bpf_map *map, + struct vm_fault *vmf) +{ + struct bpf_array *array = container_of(map, struct bpf_array, map); + struct page *page; + + page = vmalloc_to_page(array->value + ((u64)vmf->pgoff << PAGE_SHIFT)); + if (!page) + return VM_FAULT_SIGBUS; + + /* the eager remap_vmalloc_range() flushed via vm_insert_page() */ + flush_dcache_folio(page_folio(page)); + get_page(page); + vmf->page = page; + + return 0; } static bool array_map_meta_equal(const struct bpf_map *meta0, @@ -844,6 +869,7 @@ const struct bpf_map_ops array_map_ops = { .map_direct_value_addr = array_map_direct_value_addr, .map_direct_value_meta = array_map_direct_value_meta, .map_mmap = array_map_mmap, + .map_mmap_fault = array_map_mmap_fault, .map_seq_show_elem = array_map_seq_show_elem, .map_check_btf = array_map_check_btf, .map_lookup_batch = generic_map_lookup_batch, diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 7d8c3e8e6d62..6874ba1424af 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1076,11 +1076,24 @@ static void bpf_map_mmap_close(struct vm_area_struct *vma) bpf_map_write_active_dec(map); } +static vm_fault_t bpf_map_mmap_fault(struct vm_fault *vmf) +{ + struct bpf_map *map = vmf->vma->vm_private_data; + + return map->ops->map_mmap_fault(map, vmf); +} + static const struct vm_operations_struct bpf_map_default_vmops = { .open = bpf_map_mmap_open, .close = bpf_map_mmap_close, }; +static const struct vm_operations_struct bpf_map_lazy_vmops = { + .open = bpf_map_mmap_open, + .close = bpf_map_mmap_close, + .fault = bpf_map_mmap_fault, +}; + static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma) { struct bpf_map *map = filp->private_data; @@ -1116,7 +1129,7 @@ out: return err; /* set default open/close callbacks */ - vma->vm_ops = &bpf_map_default_vmops; + vma->vm_ops = map->ops->map_mmap_fault ? &bpf_map_lazy_vmops : &bpf_map_default_vmops; vma->vm_private_data = map; vm_flags_clear(vma, VM_MAYEXEC); /* If mapping is read-only, then disallow potentially re-mapping with -- cgit v1.2.3 From 5ad746166341e3c07250ee09518d7e4ab5cfb966 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:45:56 +0200 Subject: bpf: Add verifier diagnostics report helpers Add the initial diagnostics renderer for verifier reports and wire it into the BPF build. The helper emits the common failure header through the verifier log. Later patches add prose wrapping, reusable report sections, and source and instruction context for category-specific diagnostics. Gate the helpers on normal verifier log output from the start, so BPF_LOG_STATS-only loads do not collect or render diagnostics. Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260815064612.378577-2-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/Makefile | 2 +- kernel/bpf/diagnostics.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ kernel/bpf/diagnostics.h | 14 ++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 kernel/bpf/diagnostics.c create mode 100644 kernel/bpf/diagnostics.h (limited to 'kernel') diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 4dc41bf5780c..90255d80e5be 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse endif CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy) -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o diagnostics.o obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c new file mode 100644 index 000000000000..e75753552a4d --- /dev/null +++ b/kernel/bpf/diagnostics.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + +#include +#include +#include + +#include "diagnostics.h" + +bool bpf_diag_enabled(const struct bpf_verifier_env *env) +{ + return env->log.level & BPF_LOG_LEVEL; +} + +static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); + +static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) +{ + va_list args; + + if (!bpf_diag_enabled(env)) + return; + + va_start(args, fmt); + bpf_verifier_vlog(&env->log, fmt, args); + va_end(args); +} + +static void bpf_diag_header(struct bpf_verifier_env *env, const char *category, + const char *problem) +{ + char first; + + if (!bpf_diag_enabled(env)) + return; + + category = category ?: "Verifier Error"; + problem = problem ?: ""; + + if (!problem[0]) { + diag_write(env, "\nVerification failed: %s\n", category); + return; + } + + first = toupper(problem[0]); + diag_write(env, "\nVerification failed: %s: %c%s\n", category, first, problem + 1); +} diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h new file mode 100644 index 000000000000..f51aa39f0909 --- /dev/null +++ b/kernel/bpf/diagnostics.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#ifndef __BPF_DIAGNOSTICS_H +#define __BPF_DIAGNOSTICS_H + +#include +#include + +struct bpf_verifier_env; + +bool bpf_diag_enabled(const struct bpf_verifier_env *env); + +#endif /* __BPF_DIAGNOSTICS_H */ -- cgit v1.2.3 From b9c5d822f677e065971481c4bafe5d84b5451082 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:45:57 +0200 Subject: bpf: Add source and instruction diagnostic context Teach verifier diagnostics to annotate an instruction with BTF source line information and nearby BPF instructions. The renderer keeps source text in a fixed-width lane and prints instructions in a stable right-hand gutter. Wrap annotation text under the source line so long error labels remain readable while the source and instruction lanes keep their fixed layout. Keeping source and instruction context in one commit preserves the visual layout contract that later diagnostic reports rely on. Signed-off-by: Kumar Kartikeya Dwivedi Link: https://patch.msgid.link/20260815064612.378577-3-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/btf.c | 10 + kernel/bpf/core.c | 35 ++-- kernel/bpf/diagnostics.c | 490 +++++++++++++++++++++++++++++++++++++++++++++++ kernel/bpf/diagnostics.h | 7 + kernel/bpf/verifier.c | 47 ++++- 5 files changed, 561 insertions(+), 28 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 87ffde865a50..5b9d767895c9 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8316,6 +8316,16 @@ int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, return ssnprintf.len; } +int btf_type_name_to_buf(const struct btf *btf, u32 type_id, char *buf, int len) +{ + struct btf_show show = { + .btf = btf, + .state.type_id = type_id, + }; + + return snprintf(buf, len, "%s", btf_show_name(&show)); +} + #ifdef CONFIG_PROC_FS static void bpf_btf_show_fdinfo(struct seq_file *m, struct file *filp) { diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 6a94370a2448..d55e737ed75a 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3461,24 +3461,14 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_bulk_tx); #ifdef CONFIG_BPF_SYSCALL -void bpf_get_linfo_file_line(struct btf *btf, const struct bpf_line_info *linfo, - const char **filep, const char **linep, int *nump) +void bpf_get_linfo_source(struct btf *btf, const struct bpf_line_info *linfo, + struct bpf_linfo_source *src) { - /* Get base component of the file path. */ - if (filep) { - *filep = btf_name_by_offset(btf, linfo->file_name_off); - *filep = kbasename(*filep); - } - - /* Obtain the source line, and strip whitespace in prefix. */ - if (linep) { - *linep = btf_name_by_offset(btf, linfo->line_off); - while (isspace(**linep)) - *linep += 1; - } - - if (nump) - *nump = BPF_LINE_INFO_LINE_NUM(linfo->line_col); + src->file = kbasename(btf_name_by_offset(btf, linfo->file_name_off)); + src->line = btf_name_by_offset(btf, linfo->line_off); + src->file_name_off = linfo->file_name_off; + src->line_num = BPF_LINE_INFO_LINE_NUM(linfo->line_col); + src->line_col = BPF_LINE_INFO_LINE_COL(linfo->line_col); } const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn_off) @@ -3521,6 +3511,7 @@ const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep, const char **linep, int *nump) { + struct bpf_linfo_source src; int idx = -1, insn_start, insn_end, len; struct bpf_line_info *linfo; void **jited_linfo; @@ -3552,7 +3543,15 @@ int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char * if (idx == -1) return -ENOENT; - bpf_get_linfo_file_line(btf, &linfo[idx], filep, linep, nump); + bpf_get_linfo_source(btf, &linfo[idx], &src); + while (isspace(*src.line)) + src.line++; + if (filep) + *filep = src.file; + if (linep) + *linep = src.line; + if (nump) + *nump = src.line_num; return 0; } diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index e75753552a4d..815aa7938b50 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -1,12 +1,61 @@ // SPDX-License-Identifier: GPL-2.0-only // Copyright (c) 2026 Meta Platforms, Inc. and affiliates. +#include #include +#include #include +#include +#include +#include +#include #include +#include +#include "disasm.h" #include "diagnostics.h" +#define BPF_DIAG_TEXT_WIDTH 100 +#define BPF_DIAG_CONTEXT 2 +#define BPF_DIAG_CONTEXT_CNT (1 + BPF_DIAG_CONTEXT * 2) +#define BPF_DIAG_SOURCE_LANE_WIDTH 88 +#define BPF_DIAG_TAB_WIDTH 8 +#define BPF_DIAG_FMT_CHUNK_SIZE (PAGE_SIZE - sizeof(struct diag_fmt_chunk)) +#define BPF_DIAG_FMT_BUF_SIZE 256 +#define DISASM_LINE_LEN 160 + +struct disasm_line { + char text[DISASM_LINE_LEN]; + int idx; + bool valid; +}; + +struct disasm_ctx { + struct bpf_verifier_env *env; + struct seq_buf seq; +}; + +struct diag_fmt_chunk { + struct list_head node; + struct seq_buf seq; + char data[]; +}; + +struct diag_fmt_mark { + struct diag_fmt_chunk *chunk; + size_t len; +}; + +struct bpf_diag_scratch { + struct bpf_linfo_source source_lines[BPF_DIAG_CONTEXT_CNT]; + struct disasm_line disasm_lines[BPF_DIAG_CONTEXT_CNT]; +}; + +struct bpf_diag { + struct bpf_diag_scratch scratch; + struct list_head fmt_chunks; +}; + bool bpf_diag_enabled(const struct bpf_verifier_env *env) { return env->log.level & BPF_LOG_LEVEL; @@ -14,6 +63,138 @@ bool bpf_diag_enabled(const struct bpf_verifier_env *env) static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); +int bpf_diag_init(struct bpf_verifier_env *env) +{ + if (!bpf_diag_enabled(env)) + return 0; + + env->diag = kzalloc_obj(struct bpf_diag, GFP_KERNEL_ACCOUNT); + if (!env->diag) + return -ENOMEM; + + INIT_LIST_HEAD(&env->diag->fmt_chunks); + return 0; +} + +static char *diag_fmt_alloc(struct bpf_verifier_env *env, size_t size) +{ + struct bpf_diag *diag = env->diag; + struct diag_fmt_chunk *chunk; + size_t capacity, available; + char *buf; + + if (!diag || !size || size > INT_MAX) + return NULL; + + if (!list_empty(&diag->fmt_chunks)) { + chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); + available = seq_buf_get_buf(&chunk->seq, &buf); + if (available >= size) + goto commit; + } + + capacity = max_t(size_t, BPF_DIAG_FMT_CHUNK_SIZE, size); + chunk = kmalloc(struct_size(chunk, data, capacity), GFP_KERNEL_ACCOUNT); + if (!chunk) + return NULL; + + seq_buf_init(&chunk->seq, chunk->data, capacity); + list_add_tail(&chunk->node, &diag->fmt_chunks); + available = seq_buf_get_buf(&chunk->seq, &buf); + if (WARN_ON_ONCE(available < size)) + return NULL; + +commit: + seq_buf_commit(&chunk->seq, size); + return buf; +} + +char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size) +{ + char *buf; + + buf = diag_fmt_alloc(env, size); + if (buf) + buf[0] = '\0'; + return buf; +} + +const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args) +{ + va_list copy; + char *buf; + int len; + + va_copy(copy, args); + len = vsnprintf(NULL, 0, fmt, copy); + va_end(copy); + if (len < 0 || len == INT_MAX) + return ""; + + buf = diag_fmt_alloc(env, len + 1); + if (buf) + vsnprintf(buf, len + 1, fmt, args); + return buf ?: ""; +} + +const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) +{ + const char *buf; + va_list args; + + va_start(args, fmt); + buf = bpf_diag_vfmt(env, fmt, args); + va_end(args); + return buf; +} + +static struct diag_fmt_mark diag_fmt_save(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + struct diag_fmt_mark mark = {}; + + if (!diag || list_empty(&diag->fmt_chunks)) + return mark; + + mark.chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); + mark.len = mark.chunk->seq.len; + return mark; +} + +static void diag_fmt_restore(struct bpf_verifier_env *env, struct diag_fmt_mark mark) +{ + struct bpf_diag *diag = env->diag; + struct diag_fmt_chunk *chunk; + + if (!diag) + return; + + while (!list_empty(&diag->fmt_chunks)) { + chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); + if (chunk == mark.chunk) + break; + list_del(&chunk->node); + kfree(chunk); + } + + if (mark.chunk) { + mark.chunk->seq.len = mark.len; + seq_buf_str(&mark.chunk->seq); + } +} + +void bpf_diag_free(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + + if (!diag) + return; + + diag_fmt_restore(env, (struct diag_fmt_mark){}); + kfree(diag); + env->diag = NULL; +} + static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) { va_list args; @@ -26,6 +207,179 @@ static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) va_end(args); } +static void diag_print_wrapped_prefixed(struct bpf_verifier_env *env, const char *first_prefix, + const char *next_prefix, const char *text) +{ + const char *prefix = first_prefix; + + while (*text) { + const char *line = text; + int prefix_len = strlen(prefix); + int text_width = BPF_DIAG_TEXT_WIDTH - prefix_len; + int len = 0, last_space = -1; + + if (text_width < 1) + text_width = 1; + + while (line[len] && line[len] != '\n' && len < text_width) { + if (line[len] == ' ') + last_space = len; + len++; + } + + if (line[len] && line[len] != '\n' && line[len] != ' ' && last_space > 0) + len = last_space; + + diag_write(env, "%s%.*s\n", prefix, len, line); + + text = line + len; + while (*text == ' ') + text++; + if (*text == '\n') + text++; + + prefix = next_prefix; + } +} + +static int diag_line_width(unsigned int line) +{ + int width = 1; + + while (line >= 10) { + line /= 10; + width++; + } + + return width; +} + +static int diag_line_indent(const char *line) +{ + int indent = 0; + + while (*line == ' ' || *line == '\t') { + if (*line == '\t') + indent = round_up(indent + 1, BPF_DIAG_TAB_WIDTH); + else + indent++; + line++; + } + + return indent; +} + +static void disasm_print(void *private_data, const char *fmt, ...) __printf(2, 3); + +static void disasm_print(void *private_data, const char *fmt, ...) +{ + struct disasm_ctx *ctx = private_data; + va_list args; + + va_start(args, fmt); + seq_buf_vprintf(&ctx->seq, fmt, args); + va_end(args); +} + +static const char *disasm_kfunc_name(void *private_data, const struct bpf_insn *insn) +{ + struct disasm_ctx *ctx = private_data; + + return bpf_disasm_kfunc_name(ctx->env, insn); +} + +static void format_disasm_line(struct bpf_verifier_env *env, int insn_idx, + struct disasm_line *line) +{ + struct disasm_ctx ctx = { .env = env }; + struct bpf_insn *insn; + const struct bpf_insn_cbs cbs = { + .cb_call = disasm_kfunc_name, + .cb_print = disasm_print, + .private_data = &ctx, + }; + + line->idx = insn_idx; + line->valid = false; + seq_buf_init(&ctx.seq, line->text, sizeof(line->text)); + + if (insn_idx < 0 || insn_idx >= env->prog->len) + return; + + if (insn_idx > 0 && bpf_is_ldimm64(&env->prog->insnsi[insn_idx - 1])) + return; + + insn = &env->prog->insnsi[insn_idx]; + if (bpf_is_ldimm64(insn) && insn_idx + 1 >= env->prog->len) + return; + + print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); + seq_buf_str(&ctx.seq); + ctx.seq.len = strnlen(line->text, sizeof(line->text)); + while (ctx.seq.len && line->text[ctx.seq.len - 1] == '\n') + seq_buf_pop(&ctx.seq); + seq_buf_str(&ctx.seq); + + line->valid = true; +} + +static void diag_format_source_text(char *buf, size_t size, const char *line, int width) +{ + int col = 0, len = 0; + + if (!size) + return; + if (width <= 0) { + buf[0] = '\0'; + return; + } + + line = line ?: "..."; + while (*line && col < width && len + 1 < size) { + if (*line == '\t') { + int next = round_up(col + 1, BPF_DIAG_TAB_WIDTH); + + while (col < next && col < width && len + 1 < size) { + buf[len++] = ' '; + col++; + } + line++; + continue; + } + + buf[len++] = *line++; + col++; + } + + if (*line) { + int ellipsis_len = min(3, width); + + while (len > 0 && col > width - ellipsis_len) { + len--; + col--; + } + while (ellipsis_len-- && len + 1 < size) + buf[len++] = '.'; + } + + buf[len] = '\0'; +} + +static void diag_format_source_lane(char *buf, size_t size, const char *source_prefix, + int source_line_width, int line_num, const char *line) +{ + int len, text_width; + + if (line_num <= 0) { + buf[0] = '\0'; + return; + } + + len = scnprintf(buf, size, "%s%*d | ", source_prefix, source_line_width, line_num); + text_width = BPF_DIAG_SOURCE_LANE_WIDTH - len; + diag_format_source_text(buf + len, size - len, line, text_width); +} + static void bpf_diag_header(struct bpf_verifier_env *env, const char *category, const char *problem) { @@ -45,3 +399,139 @@ static void bpf_diag_header(struct bpf_verifier_env *env, const char *category, first = toupper(problem[0]); diag_write(env, "\nVerification failed: %s: %c%s\n", category, first, problem + 1); } + +static void diag_print_source_annotation(struct bpf_verifier_env *env, int line_width, int indent, + const char *label, const char *msg) +{ + const char *first_prefix, *next_prefix, *text; + + indent = min_t(int, indent, max_t(int, 0, BPF_DIAG_SOURCE_LANE_WIDTH - line_width - 8)); + text = bpf_diag_fmt(env, "%s: %s", label, msg); + first_prefix = bpf_diag_fmt(env, " %*s | %*s^-- ", line_width + 4, "", indent, ""); + next_prefix = bpf_diag_fmt(env, " %*s | %*s ", line_width + 4, "", indent, ""); + + diag_print_wrapped_prefixed(env, first_prefix, next_prefix, text); +} + +static void diag_print_insn_context(struct bpf_verifier_env *env, u32 insn_idx, + struct disasm_line *disasm_lines) +{ + int insn_width = diag_line_width(env->prog->len ? env->prog->len - 1 : 0); + int i; + + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { + int row = i - BPF_DIAG_CONTEXT; + + format_disasm_line(env, insn_idx + row, &disasm_lines[i]); + } + + diag_write(env, " Instruction context:\n"); + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { + struct disasm_line *line = &disasm_lines[i]; + + if (line->valid) + diag_write(env, " %s%*d | %s\n", + line->idx == insn_idx ? ">>> " : " ", + insn_width, line->idx, line->text); + } +} + +static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const char *label, + const char *fmt, ...) +{ + struct bpf_diag_scratch *scratch; + struct bpf_linfo_source *source_lines; + struct disasm_line *disasm_lines; + struct bpf_linfo_source src = {}; + struct diag_fmt_mark mark; + const struct bpf_line_info *linfo; + const struct bpf_subprog_info *subprog; + struct btf *btf = env->prog->aux->btf; + char *source_lane; + const char *msg; + const char *func; + int start_line, end_line, width, indent, subprogno, linfo_start, linfo_end, i; + va_list args; + + if (!bpf_diag_enabled(env)) + return; + if (!env->diag) + return; + + mark = diag_fmt_save(env); + label = label ?: "note"; + scratch = &env->diag->scratch; + source_lines = scratch->source_lines; + disasm_lines = scratch->disasm_lines; + memset(source_lines, 0, sizeof(scratch->source_lines)); + memset(disasm_lines, 0, sizeof(scratch->disasm_lines)); + + va_start(args, fmt); + msg = bpf_diag_vfmt(env, fmt, args); + va_end(args); + if (!*msg) + msg = ""; + + linfo = bpf_find_linfo(env->prog, insn_idx); + if (btf && linfo) + bpf_get_linfo_source(btf, linfo, &src); + if (!src.file || !*src.file || !src.line || !*src.line) { + diag_write(env, " insn %u\n", insn_idx); + diag_print_source_annotation(env, 0, 0, label, msg); + diag_print_insn_context(env, insn_idx, disasm_lines); + goto out_restore; + } + + subprog = bpf_find_containing_subprog(env, insn_idx); + subprogno = subprog ? subprog - env->subprog_info : -ENOENT; + func = subprogno >= 0 ? bpf_subprog_name(env, subprogno) : NULL; + if (func && *func) + diag_write(env, " %s @ %s:%d:%d\n", func, src.file, src.line_num, src.line_col); + else + diag_write(env, " %s:%d:%d\n", src.file, src.line_num, src.line_col); + + start_line = src.line_num - BPF_DIAG_CONTEXT; + end_line = src.line_num + BPF_DIAG_CONTEXT; + width = diag_line_width(end_line); + indent = diag_line_indent(src.line); + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) + source_lines[i].line_num = start_line + i; + + linfo = env->prog->aux->linfo; + linfo_start = subprog ? subprog->linfo_idx : 0; + linfo_end = subprogno >= 0 && subprogno + 1 < env->subprog_cnt ? + env->subprog_info[subprogno + 1].linfo_idx : env->prog->aux->nr_linfo; + for (i = linfo_start; i < linfo_end; i++) { + struct bpf_linfo_source line_src; + int idx; + + bpf_get_linfo_source(btf, &linfo[i], &line_src); + if (line_src.file_name_off != src.file_name_off || + line_src.line_num < start_line || line_src.line_num > end_line || + !line_src.line || !*line_src.line) + continue; + + idx = line_src.line_num - start_line; + if (!source_lines[idx].line) + source_lines[idx] = line_src; + } + + diag_write(env, " Source context:\n"); + source_lane = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); + if (!source_lane) + goto out_restore; + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { + const char *source_prefix; + + source_prefix = source_lines[i].line_num == src.line_num ? ">>> " : " "; + diag_format_source_lane(source_lane, BPF_DIAG_FMT_BUF_SIZE, source_prefix, width, + source_lines[i].line_num, source_lines[i].line); + diag_write(env, " %s\n", source_lane); + if (source_lines[i].line_num == src.line_num) + diag_print_source_annotation(env, width, indent, label, msg); + } + diag_print_insn_context(env, insn_idx, disasm_lines); + +out_restore: + diag_fmt_restore(env, mark); +} diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index f51aa39f0909..ba268b589ac9 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -5,10 +5,17 @@ #define __BPF_DIAGNOSTICS_H #include +#include #include struct bpf_verifier_env; bool bpf_diag_enabled(const struct bpf_verifier_env *env); +int bpf_diag_init(struct bpf_verifier_env *env); +char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size); +const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args) + __printf(2, 0); +const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); +void bpf_diag_free(struct bpf_verifier_env *env); #endif /* __BPF_DIAGNOSTICS_H */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6ac1afced20b..2f330230f8d5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -34,6 +34,7 @@ #include #include +#include "diagnostics.h" #include "disasm.h" static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { @@ -405,7 +406,7 @@ static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) return btf_type_is_void(type); } -static const char *subprog_name(const struct bpf_verifier_env *env, int subprog) +const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog) { struct bpf_func_info *info; @@ -2624,6 +2625,26 @@ static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) return btf_vmlinux ?: ERR_PTR(-ENOENT); } +static struct btf *find_kfunc_desc_btf_cached(struct bpf_verifier_env *env, s16 offset) +{ + struct bpf_kfunc_btf kf_btf = { .offset = offset }; + struct bpf_kfunc_btf_tab *tab; + struct bpf_kfunc_btf *b; + + if (!offset) + return btf_vmlinux ?: ERR_PTR(-ENOENT); + if (offset < 0) + return ERR_PTR(-EINVAL); + + tab = env->prog->aux->kfunc_btf_tab; + if (!tab) + return ERR_PTR(-ENOENT); + + b = bsearch(&kf_btf, tab->descs, tab->nr_descs, + sizeof(tab->descs[0]), kfunc_btf_cmp_by_off); + return b ? b->btf : ERR_PTR(-ENOENT); +} + #define KF_IMPL_SUFFIX "_impl" static const struct btf_type *find_kfunc_impl_proto(struct bpf_verifier_log *log, @@ -3031,8 +3052,8 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env) if (bpf_pseudo_func(&insn[idx])) continue; verbose(env, "recursive call from %s() to %s()\n", - subprog_name(env, cur), - subprog_name(env, callee)); + bpf_subprog_name(env, cur), + bpf_subprog_name(env, callee)); ret = -EINVAL; goto out; } @@ -3053,7 +3074,7 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env) if (env->log.level & BPF_LOG_LEVEL2) for (i = 0; i < cnt; i++) verbose(env, "topo_order[%d] = %s\n", - i, subprog_name(env, env->subprog_topo_order[i])); + i, bpf_subprog_name(env, env->subprog_topo_order[i])); out: kvfree(dfs_stack); kvfree(color); @@ -3197,7 +3218,7 @@ static void linked_regs_unpack(u64 val, struct linked_regs *s) } } -static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) +const char *bpf_disasm_kfunc_name(void *data, const struct bpf_insn *insn) { const struct btf_type *func; struct btf *desc_btf; @@ -3205,18 +3226,20 @@ static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL) return NULL; - desc_btf = find_kfunc_desc_btf(data, insn->off); + desc_btf = find_kfunc_desc_btf_cached(data, insn->off); if (IS_ERR(desc_btf)) return ""; func = btf_type_by_id(desc_btf, insn->imm); + if (!func || !btf_type_is_func(func)) + return ""; return btf_name_by_offset(desc_btf, func->name_off); } void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn) { const struct bpf_insn_cbs cbs = { - .cb_call = disasm_kfunc_name, + .cb_call = bpf_disasm_kfunc_name, .cb_print = verbose, .private_data = env, }; @@ -9408,7 +9431,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err == -EFAULT) return err; if (bpf_subprog_is_global(env, subprog)) { - const char *sub_name = subprog_name(env, subprog); + const char *sub_name = bpf_subprog_name(env, subprog); if (env->cur_state->active_locks) { verbose(env, "global function calls are not allowed while holding a lock,\n" @@ -18479,7 +18502,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) regs = state->frame[state->curframe]->regs; if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { - const char *sub_name = subprog_name(env, subprog); + const char *sub_name = bpf_subprog_name(env, subprog); struct bpf_subprog_arg_info *arg; struct bpf_reg_state *reg; @@ -18656,7 +18679,7 @@ again: return ret; } else if (env->log.level & BPF_LOG_LEVEL) { verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n", - i, subprog_name(env, i)); + i, bpf_subprog_name(env, i)); } /* We verified new global subprog, it might have called some @@ -20188,6 +20211,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, ret = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size); if (ret) goto err_free_env; + ret = bpf_diag_init(env); + if (ret) + goto err_prep; if (env->signature) { ret = bpf_prog_calc_tag(env->prog); if (ret < 0) @@ -20478,6 +20504,7 @@ err_free_env: kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); + bpf_diag_free(env); kvfree(env); return ret; } -- cgit v1.2.3 From daf8248701b621d8df7ea134793dbb750d4887c0 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:45:58 +0200 Subject: bpf: Add verifier diagnostic event log Add an environment-owned diagnostic history for verifier reports. Event payloads keep the user-facing branch history shape, while storage lives in bpf_verifier_env and follows the active verifier path. Grow the event array geometrically up to a 64 MiB limit. Once storage reaches the limit, or an allocation fails, overwrite the oldest event so diagnostics retain the newest useful suffix without adding per-event metadata. Represent saved positions as absolute logical sequence numbers. A restore truncates to a retained position. If its prefix has already been evicted, clear the abandoned suffix and preserve the missing-history position. This keeps marks stable across rotation without increasing their size. Add the branch event renderer and branch recording. Signed-off-by: Kumar Kartikeya Dwivedi Link: https://patch.msgid.link/20260815064612.378577-4-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/diagnostics.c | 130 +++++++++++++++++++++++++++++++++++++++++++++++ kernel/bpf/diagnostics.h | 3 ++ kernel/bpf/verifier.c | 21 ++++++++ 3 files changed, 154 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index 815aa7938b50..8f21b46adeca 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -22,8 +22,24 @@ #define BPF_DIAG_TAB_WIDTH 8 #define BPF_DIAG_FMT_CHUNK_SIZE (PAGE_SIZE - sizeof(struct diag_fmt_chunk)) #define BPF_DIAG_FMT_BUF_SIZE 256 +#define BPF_DIAG_EVENT_LOG_MAX_SIZE (64U << 20) #define DISASM_LINE_LEN 160 +enum bpf_diag_history_kind { + BPF_DIAG_HISTORY_BRANCH, +}; + +struct bpf_diag_history_event { + u32 insn_idx : 24; + u32 kind : 8; + u8 in_lineage : 1; + union { + struct { + bool cond_true; + } branch; + }; +}; + struct disasm_line { char text[DISASM_LINE_LEN]; int idx; @@ -46,12 +62,23 @@ struct diag_fmt_mark { size_t len; }; +struct bpf_diag_log { + struct bpf_diag_history_event *events; + /* Sequence number of the oldest retained event on the active path. */ + u64 first_seq; + u32 cnt; + u32 cap; + u32 head; + bool growth_failed; +}; + struct bpf_diag_scratch { struct bpf_linfo_source source_lines[BPF_DIAG_CONTEXT_CNT]; struct disasm_line disasm_lines[BPF_DIAG_CONTEXT_CNT]; }; struct bpf_diag { + struct bpf_diag_log log; struct bpf_diag_scratch scratch; struct list_head fmt_chunks; }; @@ -191,6 +218,7 @@ void bpf_diag_free(struct bpf_verifier_env *env) return; diag_fmt_restore(env, (struct diag_fmt_mark){}); + kvfree(diag->log.events); kfree(diag); env->diag = NULL; } @@ -207,6 +235,95 @@ static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) va_end(args); } +static u64 log_end(const struct bpf_diag_log *log) +{ + return log->first_seq + log->cnt; +} + +static u32 log_pos(const struct bpf_diag_log *log, u32 idx) +{ + u32 pos = log->head + idx; + + return pos < log->cap ? pos : pos - log->cap; +} + +u64 bpf_diag_event_log_save(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + + return diag ? log_end(&diag->log) : 0; +} + +void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos) +{ + struct bpf_diag *diag = env->diag; + struct bpf_diag_log *log; + u64 end_seq; + + if (!diag) + return; + + log = &diag->log; + end_seq = log_end(log); + if (WARN_ON_ONCE(log_pos > end_seq)) + log_pos = end_seq; + + /* + * A deep abandoned path may have rotated away the shared prefix. In + * that case, restart with an empty retained suffix and remember that + * every event before the restored mark is unavailable. + */ + if (log_pos <= log->first_seq) { + log->first_seq = log_pos; + log->head = 0; + log->cnt = 0; + return; + } + + log->cnt = log_pos - log->first_seq; +} + +static void diag_append_history(struct bpf_verifier_env *env, + const struct bpf_diag_history_event *event) +{ + struct bpf_diag_history_event *events; + struct bpf_diag *diag = env->diag; + struct bpf_diag_log *log; + u32 cap, max_events; + + if (!diag) + return; + log = &diag->log; + + if (log->cnt < log->cap) { + log->events[log_pos(log, log->cnt++)] = *event; + return; + } + + max_events = BPF_DIAG_EVENT_LOG_MAX_SIZE / sizeof(*events); + if (log->growth_failed || log->cap == max_events) + goto rotate; + + cap = min(log->cap ? log->cap * 2 : 64, max_events); + events = kvrealloc(log->events, array_size(cap, sizeof(*events)), GFP_KERNEL_ACCOUNT); + if (!events) { + log->growth_failed = true; + goto rotate; + } + log->events = events; + log->cap = cap; + log->events[log->cnt++] = *event; + return; + +rotate: + if (log->cap) { + log->events[log->head++] = *event; + if (log->head == log->cap) + log->head = 0; + } + log->first_seq++; +} + static void diag_print_wrapped_prefixed(struct bpf_verifier_env *env, const char *first_prefix, const char *next_prefix, const char *text) { @@ -535,3 +652,16 @@ static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const ch out_restore: diag_fmt_restore(env, mark); } + +void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true) +{ + struct bpf_diag_history_event event = { + .insn_idx = insn_idx, + .kind = BPF_DIAG_HISTORY_BRANCH, + .branch = { + .cond_true = cond_true, + }, + }; + + diag_append_history(env, &event); +} diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index ba268b589ac9..6eda2fd65ee1 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -16,6 +16,9 @@ char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size); const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args) __printf(2, 0); const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); +u64 bpf_diag_event_log_save(struct bpf_verifier_env *env); +void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos); void bpf_diag_free(struct bpf_verifier_env *env); +void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true); #endif /* __BPF_DIAGNOSTICS_H */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2f330230f8d5..60dcb87a2417 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17439,6 +17439,27 @@ static int do_check(struct bpf_verifier_env *env) state->last_insn_idx = env->prev_insn_idx; state->insn_idx = env->insn_idx; + /* + * Record the incoming edge so active and queued paths use the same + * branch-recording path. A zero-offset conditional has identical + * successors, so its outcome cannot be reconstructed from the edge. + */ + if (!state->speculative && prev_insn_idx >= 0 && prev_insn_idx < insn_cnt) { + struct bpf_insn *prev_insn = &insns[prev_insn_idx]; + int fallthrough_idx = prev_insn_idx + 1; + int branch_idx = prev_insn_idx + bpf_jmp_offset(prev_insn) + 1; + u8 class = BPF_CLASS(prev_insn->code); + u8 opcode = BPF_OP(prev_insn->code); + + if ((class == BPF_JMP || class == BPF_JMP32) && + opcode != BPF_JA && opcode != BPF_CALL && opcode != BPF_EXIT && + opcode <= BPF_JCOND && branch_idx != fallthrough_idx) { + if (env->insn_idx == branch_idx) + bpf_diag_record_branch(env, prev_insn_idx, true); + else if (env->insn_idx == fallthrough_idx) + bpf_diag_record_branch(env, prev_insn_idx, false); + } + } if (bpf_is_prune_point(env, env->insn_idx)) { err = bpf_is_state_visited(env, env->insn_idx); -- cgit v1.2.3 From a6debd5f25c9c79f534074e9cb460cf6495d6daf Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:45:59 +0200 Subject: bpf: Prune verifier diagnostics when switching paths Save the diagnostic event-log position with each verifier stack entry and reset the environment-owned stream together with the normal verifier log when a queued state is popped. Also reset the diagnostic stream after successful subprogram verification even when level-2 logging preserves the normal verifier log. Signed-off-by: Kumar Kartikeya Dwivedi Link: https://patch.msgid.link/20260815064612.378577-5-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 60dcb87a2417..db644690ac4b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -194,6 +194,7 @@ struct bpf_verifier_stack_elem { struct bpf_verifier_stack_elem *next; /* length of verifier log at the time this state was pushed on stack */ u32 log_pos; + u64 diag_log_pos; }; #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 @@ -1700,6 +1701,7 @@ static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, err = bpf_copy_verifier_state(cur, &head->st); if (err) return err; + bpf_diag_event_log_restore(env, head->diag_log_pos); } if (pop_log) bpf_vlog_reset(&env->log, head->log_pos); @@ -1743,6 +1745,7 @@ static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, elem->prev_insn_idx = prev_insn_idx; elem->next = env->head; elem->log_pos = env->log.end_pos; + elem->diag_log_pos = bpf_diag_event_log_save(env); env->head = elem; env->stack_size++; err = bpf_copy_verifier_state(&elem->st, cur); @@ -2264,6 +2267,7 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, elem->prev_insn_idx = prev_insn_idx; elem->next = env->head; elem->log_pos = env->log.end_pos; + elem->diag_log_pos = bpf_diag_event_log_save(env); env->head = elem; env->stack_size++; if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { @@ -18635,8 +18639,11 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) ret = do_check(env); out: account_current_path(env); - if (!ret && pop_log) - bpf_vlog_reset(&env->log, 0); + if (!ret) { + if (pop_log) + bpf_vlog_reset(&env->log, 0); + bpf_diag_event_log_restore(env, 0); + } free_states(env); /* -- cgit v1.2.3 From af4ea6e20fff383cdc2f01b9a372b4c7a0abf5ff Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:46:00 +0200 Subject: bpf: Track verifier register diagnostic events Record material register and outgoing stack argument changes so diagnostics can explain how a value reached its current type, bounds, or unreadable state. Store old and new register types, scalar ranges, tnum value and mask, map and BTF type identity, and basic operand metadata in the environment-owned diagnostic event stream. Record invalidations when packet data moves, references are released, or borrowed references leave their protected region. Register-scoped history starts at the latest matching modification and then shows later branch outcomes. Also record fixed stack spills and overwrites, and tag register fills from stack so register-scoped history can follow value flow through spilled stack slots. The type_is_map_ptr() helper previously lived as a static function in kernel/bpf/log.c since commit 0c95c9fdb696 ("bpf: emit map name in register state if applicable and available"). Move it verbatim to include/linux/bpf_verifier.h as a static inline, next to the other type classifiers, so diagnostics.c can reuse it without duplicating the case list. Signed-off-by: Kumar Kartikeya Dwivedi Link: https://patch.msgid.link/20260815064612.378577-6-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/diagnostics.c | 356 +++++++++++++++++++++++++++++++++++++++++++++++ kernel/bpf/diagnostics.h | 23 +++ kernel/bpf/log.c | 11 -- kernel/bpf/verifier.c | 131 +++++++++++++++-- 4 files changed, 498 insertions(+), 23 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index 8f21b46adeca..2e8e75815581 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -25,8 +25,83 @@ #define BPF_DIAG_EVENT_LOG_MAX_SIZE (64U << 20) #define DISASM_LINE_LEN 160 +enum bpf_diag_mod_target_kind { + BPF_DIAG_MOD_TARGET_NONE, + BPF_DIAG_MOD_TARGET_REG, + BPF_DIAG_MOD_TARGET_STACK_ARG, + BPF_DIAG_MOD_TARGET_STACK_SLOT, + BPF_DIAG_MOD_TARGET_STACK_RANGE, +}; + +struct bpf_diag_mod_target { + u32 frame_id; + union { + struct { + s16 min_off; + s16 max_off; + } range; + u16 spi; + u8 regno; + u8 stack_arg; + }; + u8 frameno; + u8 kind; +}; + +static struct bpf_diag_mod_target diag_reg_target(u32 frame_id, u8 frameno, u8 regno) +{ + return (struct bpf_diag_mod_target){ + .frame_id = frame_id, + .frameno = frameno, + .kind = BPF_DIAG_MOD_TARGET_REG, + .regno = regno, + }; +} + +static struct bpf_diag_mod_target diag_stack_arg_target(u32 frame_id, u8 frameno, u8 slot) +{ + return (struct bpf_diag_mod_target){ + .frame_id = frame_id, + .frameno = frameno, + .kind = BPF_DIAG_MOD_TARGET_STACK_ARG, + .stack_arg = slot, + }; +} + +static struct bpf_diag_mod_target diag_stack_slot_target(u32 frame_id, u8 frameno, u16 spi) +{ + return (struct bpf_diag_mod_target){ + .frame_id = frame_id, + .frameno = frameno, + .kind = BPF_DIAG_MOD_TARGET_STACK_SLOT, + .spi = spi, + }; +} + +static struct bpf_diag_mod_target diag_stack_range_target(u32 frame_id, u8 frameno, + s16 min_off, s16 max_off) +{ + return (struct bpf_diag_mod_target){ + .frame_id = frame_id, + .frameno = frameno, + .kind = BPF_DIAG_MOD_TARGET_STACK_RANGE, + .range.min_off = min_off, + .range.max_off = max_off, + }; +} + +struct bpf_diag_reg_snapshot { + u32 type; + u32 btf_id; + const struct bpf_map *map_ptr; + const struct btf *btf; + struct tnum var_off; + struct cnum64 r64; +}; + enum bpf_diag_history_kind { BPF_DIAG_HISTORY_BRANCH, + BPF_DIAG_HISTORY_MOD, }; struct bpf_diag_history_event { @@ -37,6 +112,13 @@ struct bpf_diag_history_event { struct { bool cond_true; } branch; + struct { + struct bpf_diag_mod_target target; + struct bpf_diag_mod_target origin; + struct bpf_diag_reg_snapshot old, new; + u8 reason; + bool origin_valid; + } mod; }; }; @@ -77,10 +159,22 @@ struct bpf_diag_scratch { struct disasm_line disasm_lines[BPF_DIAG_CONTEXT_CNT]; }; +struct bpf_diag_mod_scope { + struct bpf_reg_state target_reg_snapshot; + struct bpf_diag_mod_target target; + struct bpf_diag_mod_target origin; + enum bpf_diag_mod_reason reason; + u32 insn_idx; + bool active; + bool origin_valid; +}; + struct bpf_diag { struct bpf_diag_log log; struct bpf_diag_scratch scratch; struct list_head fmt_chunks; + struct bpf_diag_mod_scope mod; + u32 frame_id_gen; }; bool bpf_diag_enabled(const struct bpf_verifier_env *env) @@ -103,6 +197,12 @@ int bpf_diag_init(struct bpf_verifier_env *env) return 0; } +void bpf_diag_init_frame(struct bpf_verifier_env *env, struct bpf_func_state *state) +{ + if (env->diag) + state->diag_frame_id = ++env->diag->frame_id_gen; +} + static char *diag_fmt_alloc(struct bpf_verifier_env *env, size_t size) { struct bpf_diag *diag = env->diag; @@ -359,6 +459,28 @@ static void diag_print_wrapped_prefixed(struct bpf_verifier_env *env, const char } } +const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id) +{ + char *buf = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); + size_t len; + int ret; + + if (!buf) + return ""; + + buf[0] = '\0'; + ret = btf_type_name_to_buf(btf, type_id, buf, BPF_DIAG_FMT_BUF_SIZE); + if (ret < 0 || !buf[0]) { + scnprintf(buf, BPF_DIAG_FMT_BUF_SIZE, "BTF type ID %u", type_id); + return buf; + } + + len = strlen(buf); + if (len && buf[len - 1] == '{') + buf[len - 1] = '\0'; + return buf; +} + static int diag_line_width(unsigned int line) { int width = 1; @@ -665,3 +787,237 @@ void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool con diag_append_history(env, &event); } + +static void diag_snapshot_reg(struct bpf_diag_reg_snapshot *snapshot, + const struct bpf_reg_state *reg) +{ + snapshot->type = reg->type; + if (type_is_map_ptr(reg->type)) + snapshot->map_ptr = reg->map_ptr; + if (base_type(reg->type) == PTR_TO_BTF_ID && reg->btf && reg->btf_id) { + snapshot->btf_id = reg->btf_id; + snapshot->btf = reg->btf; + } + snapshot->var_off = reg->var_off; + snapshot->r64 = reg->r64; +} + +static bool diag_mod_insn_origin(struct bpf_verifier_env *env, u32 insn_idx, + const struct bpf_diag_mod_target *target, + struct bpf_diag_mod_target *origin) +{ + const struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; + u8 class = BPF_CLASS(insn->code); + const struct bpf_func_state *state; + + if (target->kind == BPF_DIAG_MOD_TARGET_REG && (class == BPF_ALU || class == BPF_ALU64) && + BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X) { + *origin = diag_reg_target(target->frame_id, target->frameno, insn->src_reg); + return true; + } + + if ((target->kind != BPF_DIAG_MOD_TARGET_STACK_ARG && + target->kind != BPF_DIAG_MOD_TARGET_STACK_SLOT) || + class != BPF_STX) + return false; + + state = env->cur_state->frame[env->cur_state->curframe]; + *origin = diag_reg_target(state->diag_frame_id, state->frameno, insn->src_reg); + return true; +} + +static bool diag_mod_keeps_lineage(struct bpf_verifier_env *env, + const struct bpf_diag_history_event *event) +{ + const struct bpf_insn *insn; + u8 class; + + if (event->mod.reason != BPF_DIAG_MOD_WRITE || + event->mod.target.kind != BPF_DIAG_MOD_TARGET_REG) + return false; + + insn = &env->prog->insnsi[event->insn_idx]; + class = BPF_CLASS(insn->code); + if (class != BPF_ALU && class != BPF_ALU64) + return false; + + switch (BPF_OP(insn->code)) { + case BPF_ADD: + case BPF_SUB: + case BPF_MUL: + case BPF_OR: + case BPF_AND: + case BPF_LSH: + case BPF_RSH: + case BPF_ARSH: + case BPF_XOR: + case BPF_NEG: + case BPF_END: + return true; + default: + return false; + } +} + +static void diag_record_mod(struct bpf_verifier_env *env, u32 insn_idx, + struct bpf_diag_mod_target target, + enum bpf_diag_mod_reason reason, + const struct bpf_reg_state *old_reg, + const struct bpf_reg_state *new_reg, + const struct bpf_diag_mod_target *origin) +{ + struct bpf_diag_history_event event = { + .insn_idx = insn_idx, + .kind = BPF_DIAG_HISTORY_MOD, + .mod = { + .target = target, + .reason = reason, + }, + }; + + if (old_reg) + diag_snapshot_reg(&event.mod.old, old_reg); + if (new_reg) + diag_snapshot_reg(&event.mod.new, new_reg); + if (origin) { + event.mod.origin = *origin; + event.mod.origin_valid = true; + } else if (diag_mod_insn_origin(env, insn_idx, &target, &event.mod.origin)) { + event.mod.origin_valid = true; + } + if (old_reg && new_reg && + (reason == BPF_DIAG_MOD_WRITE || reason == BPF_DIAG_MOD_SPILL) && + !memcmp(&event.mod.old, &event.mod.new, sizeof(event.mod.old)) && + !event.mod.origin_valid && + diag_mod_keeps_lineage(env, &event)) + return; + + diag_append_history(env, &event); +} + +static struct bpf_reg_state *target_to_reg(struct bpf_verifier_env *env, + const struct bpf_diag_mod_target *target) +{ + struct bpf_verifier_state *vstate = env->cur_state; + struct bpf_func_state *state; + + state = target->frameno <= vstate->curframe ? vstate->frame[target->frameno] : NULL; + + if (!state) + return NULL; + if (state->diag_frame_id != target->frame_id) + return NULL; + + switch (target->kind) { + case BPF_DIAG_MOD_TARGET_REG: + if (target->regno >= MAX_BPF_REG) + return NULL; + return &state->regs[target->regno]; + case BPF_DIAG_MOD_TARGET_STACK_ARG: + if (target->stack_arg >= state->out_stack_arg_cnt) + return NULL; + return &state->stack_arg_regs[target->stack_arg]; + case BPF_DIAG_MOD_TARGET_STACK_SLOT: + if (target->spi >= state->allocated_stack / BPF_REG_SIZE) + return NULL; + return &state->stack[target->spi].spilled_ptr; + default: + return NULL; + } +} + +static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + struct bpf_diag_mod_target *target) +{ + struct bpf_verifier_state *vstate = env->cur_state; + unsigned long addr = (unsigned long)reg; + int frame; + + for (frame = 0; frame <= vstate->curframe; frame++) { + struct bpf_func_state *state = vstate->frame[frame]; + unsigned long start, end; + u32 nslots = state->allocated_stack / BPF_REG_SIZE; + int spi; + + start = (unsigned long)state->regs; + end = (unsigned long)(state->regs + MAX_BPF_REG); + if (addr >= start && addr < end) { + *target = diag_reg_target(state->diag_frame_id, state->frameno, + reg - state->regs); + return true; + } + + start = (unsigned long)state->stack_arg_regs; + end = (unsigned long)(state->stack_arg_regs + state->out_stack_arg_cnt); + if (state->out_stack_arg_cnt && addr >= start && addr < end) { + *target = diag_stack_arg_target(state->diag_frame_id, state->frameno, + reg - state->stack_arg_regs); + return true; + } + + start = (unsigned long)state->stack; + end = (unsigned long)(state->stack + nslots); + if (nslots && addr >= start && addr < end) { + spi = ((const char *)reg - (const char *)state->stack) / + sizeof(*state->stack); + *target = diag_stack_slot_target(state->diag_frame_id, state->frameno, spi); + return true; + } + } + return false; +} + +void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason) +{ + struct bpf_diag *diag = env->diag; + + if (!diag) + return; + diag->mod.active = reg_to_target(env, reg, &diag->mod.target); + if (!diag->mod.active) + return; + diag->mod.target_reg_snapshot = *reg; + diag->mod.insn_idx = env->insn_idx; + diag->mod.reason = reason; + diag->mod.origin_valid = origin && reg_to_target(env, origin, &diag->mod.origin); +} + +void bpf_diag_mod_end(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + const struct bpf_reg_state *new_reg; + + if (!diag || !diag->mod.active) + return; + diag->mod.active = false; + /* + * Resolve the target again because the enclosing function state's stack + * may have been reallocated while the modification was in progress. + */ + new_reg = target_to_reg(env, &diag->mod.target); + if (!new_reg) + return; + diag_record_mod(env, diag->mod.insn_idx, diag->mod.target, diag->mod.reason, + &diag->mod.target_reg_snapshot, new_reg, + diag->mod.origin_valid ? &diag->mod.origin : NULL); +} + +void bpf_diag_record_scrub(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + enum bpf_diag_mod_reason reason) +{ + struct bpf_diag_mod_target target; + + if (!env->diag || reg->type == NOT_INIT || !reg_to_target(env, reg, &target)) + return; + diag_record_mod(env, env->insn_idx, target, reason, reg, NULL, NULL); +} + +void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env, + const struct bpf_func_state *state, s16 min_off, s16 max_off, + enum bpf_diag_mod_reason reason) +{ + diag_record_mod(env, env->insn_idx, + diag_stack_range_target(state->diag_frame_id, state->frameno, min_off, max_off), + reason, NULL, NULL, NULL); +} diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index 6eda2fd65ee1..c4e44b86e89d 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -8,17 +8,40 @@ #include #include +struct bpf_func_state; +struct bpf_reg_state; struct bpf_verifier_env; +struct btf; + +enum bpf_diag_mod_reason { + BPF_DIAG_MOD_WRITE, + BPF_DIAG_MOD_SPILL, + BPF_DIAG_MOD_VAR_WRITE, + BPF_DIAG_MOD_REF_RELEASE, + BPF_DIAG_MOD_PKT_DATA_CHANGE, + BPF_DIAG_MOD_NON_OWN_REF, + BPF_DIAG_MOD_CALLER_SAVED, +}; bool bpf_diag_enabled(const struct bpf_verifier_env *env); int bpf_diag_init(struct bpf_verifier_env *env); +void bpf_diag_init_frame(struct bpf_verifier_env *env, struct bpf_func_state *state); char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size); const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args) __printf(2, 0); const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); +const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id); u64 bpf_diag_event_log_save(struct bpf_verifier_env *env); void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos); void bpf_diag_free(struct bpf_verifier_env *env); void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true); +void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason); +void bpf_diag_mod_end(struct bpf_verifier_env *env); +void bpf_diag_record_scrub(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + enum bpf_diag_mod_reason reason); +void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env, + const struct bpf_func_state *state, s16 min_off, s16 max_off, + enum bpf_diag_mod_reason reason); #endif /* __BPF_DIAGNOSTICS_H */ diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index b740fa73ee26..589770ca3d3a 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -615,17 +615,6 @@ static void print_scalar_ranges(struct bpf_verifier_env *env, } } -static bool type_is_map_ptr(enum bpf_reg_type t) { - switch (base_type(t)) { - case CONST_PTR_TO_MAP: - case PTR_TO_MAP_KEY: - case PTR_TO_MAP_VALUE: - return true; - default: - return false; - } -} - /* * _a stands for append, was shortened to avoid multiline statements below. * This macro is used to output a comma separated list of attributes. diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index db644690ac4b..a5929e40f18d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1792,6 +1792,17 @@ static const int caller_saved[CALLER_SAVED_REGS] = { BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 }; +static void bpf_diag_record_caller_saved(struct bpf_verifier_env *env, + struct bpf_reg_state *regs) +{ + int i; + + for (i = 1; i < CALLER_SAVED_REGS; i++) { + bpf_diag_record_scrub(env, ®s[caller_saved[i]], + BPF_DIAG_MOD_CALLER_SAVED); + } +} + /* This helper doesn't clear reg->id */ static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) { @@ -2245,6 +2256,7 @@ static void init_func_state(struct bpf_verifier_env *env, { state->callsite = callsite; state->frameno = frameno; + bpf_diag_init_frame(env, state); state->subprogno = subprogno; state->callback_ret_range = retval_range(0, 0); init_reg_state(env, state); @@ -3362,6 +3374,7 @@ static void save_register_state(struct bpf_verifier_env *env, { int i; + bpf_diag_mod_begin(env, &state->stack[spi].spilled_ptr, reg, BPF_DIAG_MOD_SPILL); state->stack[spi].spilled_ptr = *reg; for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) @@ -3370,6 +3383,8 @@ static void save_register_state(struct bpf_verifier_env *env, /* size < 8 bytes spill */ for (; i; i--) mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]); + + bpf_diag_mod_end(env); } static bool is_bpf_st_mem(struct bpf_insn *insn) @@ -3506,6 +3521,9 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, } else { u8 type = STACK_MISC; + if (bpf_is_spilled_reg(&state->stack[spi])) + bpf_diag_record_scrub(env, &state->stack[spi].spilled_ptr, + BPF_DIAG_MOD_WRITE); scrub_special_slot(state, spi); /* when we zero initialize stack slots mark them as such */ @@ -3666,6 +3684,8 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, if (err) return err; } + bpf_diag_record_scrub_stack(env, state, min_off, max_off, + BPF_DIAG_MOD_VAR_WRITE); return 0; } @@ -3758,6 +3778,12 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, mark_stack_slot_scratched(env, spi); check_fastcall_stack_contract(env, state, env->insn_idx, off); + /* + * Refine the in-progress load record's origin to the source stack slot. + */ + if (dst_regno >= 0) + bpf_diag_mod_begin(env, &state->regs[dst_regno], reg, BPF_DIAG_MOD_WRITE); + if (bpf_is_spilled_reg(®_state->stack[spi])) { u8 spill_size = 1; @@ -4051,14 +4077,17 @@ static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_s if (spi + 1 > subprog->max_out_stack_arg_cnt) subprog->max_out_stack_arg_cnt = spi + 1; + arg = &state->stack_arg_regs[spi]; + bpf_diag_mod_begin(env, arg, value_reg, BPF_DIAG_MOD_WRITE); + if (value_reg) { state->stack_arg_regs[spi] = *value_reg; } else { /* BPF_ST: store immediate, treat as scalar */ - arg = &state->stack_arg_regs[spi]; arg->type = SCALAR_VALUE; __mark_reg_known(arg, env->prog->insnsi[env->insn_idx].imm); } + bpf_diag_mod_end(env); state->no_stack_arg_load = true; return bpf_push_jmp_history(env, env->cur_state, INSN_F_STACK_ARG_ACCESS, spi, 0, 0); @@ -4091,7 +4120,9 @@ static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_st caller = vstate->frame[vstate->curframe - 1]; arg = &caller->stack_arg_regs[spi]; cur = vstate->frame[vstate->curframe]; + bpf_diag_mod_begin(env, &cur->regs[dst_regno], arg, BPF_DIAG_MOD_WRITE); cur->regs[dst_regno] = *arg; + bpf_diag_mod_end(env); return bpf_push_jmp_history(env, env->cur_state, INSN_F_STACK_ARG_ACCESS, spi, 0, 0); } @@ -6426,15 +6457,19 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, src_reg_type = regs[insn->src_reg].type; - /* Check if (src_reg + off) is readable. The state of dst_reg will be - * updated by this call. + /* + * check_stack_read_fixed_off() may refine the modification's origin to + * the source stack slot. */ + bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, argno_from_reg(insn->src_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); err = err ?: save_aux_ptr_type(env, src_reg_type, allow_trust_mismatch); err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], ctx); + if (!err) + bpf_diag_mod_end(env); return err; } @@ -6540,10 +6575,14 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, */ err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, -1, true, false); - if (!err && load_reg >= 0) + if (!err && load_reg >= 0) { + bpf_diag_mod_begin(env, cur_regs(env) + load_reg, NULL, BPF_DIAG_MOD_WRITE); err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, load_reg, true, false); + if (!err) + bpf_diag_mod_end(env); + } if (err) return err; @@ -8945,8 +8984,10 @@ static void clear_all_pkt_pointers(struct bpf_verifier_env *env) struct bpf_reg_state *reg; bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ - if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) + if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) { + bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_PKT_DATA_CHANGE); mark_reg_invalid(env, reg); + } })); } @@ -9062,10 +9103,25 @@ static int release_reference(struct bpf_verifier_env *env, int id) return err; } + /* + * A dynptr occupies two stack slots that invalidate_dynptr() + * clears together. Record both scrubs before invalidating it. + */ + if (stack && stack->slot_type[BPF_REG_SIZE - 1] == STACK_DYNPTR) { + struct bpf_stack_state *dyn_stack = stack; + + if (reg->dynptr.first_slot) + dyn_stack--; + bpf_diag_record_scrub(env, &dyn_stack[0].spilled_ptr, + BPF_DIAG_MOD_REF_RELEASE); + bpf_diag_record_scrub(env, &dyn_stack[1].spilled_ptr, + BPF_DIAG_MOD_REF_RELEASE); + invalidate_dynptr(env, dyn_stack); + continue; + } + bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_REF_RELEASE); if (!stack || stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL) mark_reg_invalid(env, reg); - else if (stack->slot_type[BPF_REG_SIZE - 1] == STACK_DYNPTR) - invalidate_dynptr(env, stack); })); } @@ -9078,8 +9134,10 @@ static void invalidate_non_owning_refs(struct bpf_verifier_env *env) struct bpf_reg_state *reg; bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ - if (type_is_non_owning_ref(reg->type)) + if (type_is_non_owning_ref(reg->type)) { + bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_NON_OWN_REF); mark_reg_invalid(env, reg); + } })); } @@ -9092,8 +9150,10 @@ static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env) bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({ if (reg->type & MEM_RCU) { + bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE); reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL); reg->type |= PTR_UNTRUSTED; + bpf_diag_mod_end(env); } })); } @@ -9110,9 +9170,11 @@ static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id) if (reg->id != id) continue; if ((reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) { + bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE); reg->id = 0; reg->type &= ~MEM_ALLOC; reg->type |= MEM_RCU; + bpf_diag_mod_end(env); } })); @@ -9124,6 +9186,8 @@ static void clear_caller_saved_regs(struct bpf_verifier_env *env, { int i; + bpf_diag_record_caller_saved(env, regs); + /* after the call registers r0 - r5 were scratched */ for (i = 0; i < CALLER_SAVED_REGS; i++) { bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); @@ -9131,13 +9195,15 @@ static void clear_caller_saved_regs(struct bpf_verifier_env *env, } } -static void invalidate_outgoing_stack_args(const struct bpf_verifier_env *env, +static void invalidate_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *state) { int i, nslots = state->out_stack_arg_cnt; - for (i = 0; i < nslots; i++) + for (i = 0; i < nslots; i++) { + bpf_diag_record_scrub(env, &state->stack_arg_regs[i], BPF_DIAG_MOD_CALLER_SAVED); bpf_mark_reg_not_init(env, &state->stack_arg_regs[i]); + } } typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, @@ -9436,6 +9502,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return err; if (bpf_subprog_is_global(env, subprog)) { const char *sub_name = bpf_subprog_name(env, subprog); + bool returns_void; if (env->cur_state->active_locks) { verbose(env, "global function calls are not allowed while holding a lock,\n" @@ -9458,16 +9525,22 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (env->log.level & BPF_LOG_LEVEL) verbose(env, "Func#%d ('%s') is global and assumed valid.\n", subprog, sub_name); + returns_void = subprog_returns_void(env, subprog); if (env->subprog_info[subprog].changes_pkt_data) clear_all_pkt_pointers(env); /* mark global subprog for verifying after main prog */ subprog_aux(env, subprog)->called = true; + if (returns_void) + bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); + else + bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); clear_caller_saved_regs(env, caller->regs); invalidate_outgoing_stack_args(env, cur_func(env)); /* All non-void global functions return a 64-bit SCALAR_VALUE. */ - if (!subprog_returns_void(env, subprog)) { + if (!returns_void) { mark_reg_unknown(env, caller->regs, BPF_REG_0); + bpf_diag_mod_end(env); } if (env->subprog_info[subprog].might_throw) { @@ -9502,6 +9575,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; + bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); clear_caller_saved_regs(env, caller->regs); /* and go analyze first insn of the callee */ @@ -9865,7 +9939,9 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) } } else { /* return to the caller whatever r0 had in the callee */ + bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE); caller->regs[BPF_REG_0] = *r0; + bpf_diag_mod_end(env); } /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, @@ -10518,12 +10594,14 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return err; /* reset caller saved regs */ + bpf_diag_record_caller_saved(env, regs); for (i = 0; i < CALLER_SAVED_REGS; i++) { bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); } invalidate_outgoing_stack_args(env, cur_func(env)); + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); /* update return register (already marked as written above) */ ret_type = fn->ret_type; ret_flag = type_flag(ret_type); @@ -10672,6 +10750,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (err) return err; + bpf_diag_mod_end(env); + /* * In order for a release of any of the original or cast pointers * to invalidate all other pointers, reuse the same reference id for @@ -10688,6 +10768,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn __mark_reg_known_zero(r0); r0->type = SCALAR_VALUE; + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); regs[BPF_REG_0].type &= ~PTR_MAYBE_NULL; regs[BPF_REG_0].id = meta.ref_obj.id; } else if (is_acquire_function(func_id, meta.map.ptr)) { @@ -10706,6 +10787,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (err) return err; + bpf_diag_mod_end(env); + err = check_map_func_compatibility(env, meta.map.ptr, func_id); if (err) return err; @@ -13211,6 +13294,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } + bpf_diag_record_caller_saved(env, regs); + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); for (i = 0; i < CALLER_SAVED_REGS; i++) { u32 regno = caller_saved[i]; @@ -13362,6 +13447,12 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, caller_info->stack_arg_cnt = stack_arg_cnt; } + /* + * Record R0 before process_iter_next_call() snapshots the alternate + * iterator path's diagnostic position. + */ + bpf_diag_mod_end(env); + if (bpf_is_iter_next_kfunc(&meta)) { err = process_iter_next_call(env, insn_idx, &meta); if (err) @@ -15004,6 +15095,8 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) u8 opcode = BPF_OP(insn->code); int err; + bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); + if (opcode == BPF_END || opcode == BPF_NEG) { /* check src operand */ err = check_reg_arg(env, insn->dst_reg, SRC_OP); @@ -15177,7 +15270,12 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) return err; } - return reg_bounds_sanity_check(env, ®s[insn->dst_reg], "alu"); + err = reg_bounds_sanity_check(env, ®s[insn->dst_reg], "alu"); + if (err) + return err; + + bpf_diag_mod_end(env); + return 0; } static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, @@ -16271,11 +16369,13 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) return err; dst_reg = ®s[insn->dst_reg]; + bpf_diag_mod_begin(env, dst_reg, NULL, BPF_DIAG_MOD_WRITE); if (insn->src_reg == 0) { u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; dst_reg->type = SCALAR_VALUE; __mark_reg_known(®s[insn->dst_reg], imm); + bpf_diag_mod_end(env); return 0; } @@ -16299,6 +16399,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) verifier_bug(env, "pseudo btf id: unexpected dst reg type"); return -EFAULT; } + bpf_diag_mod_end(env); return 0; } @@ -16318,6 +16419,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) dst_reg->type = PTR_TO_FUNC; dst_reg->subprogno = subprogno; + bpf_diag_mod_end(env); return 0; } @@ -16328,6 +16430,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) if (map->map_type == BPF_MAP_TYPE_ARENA) { __mark_reg_unknown(env, dst_reg); dst_reg->map_ptr = map; + bpf_diag_mod_end(env); return 0; } __mark_reg_known(dst_reg, aux->map_off); @@ -16345,6 +16448,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) return -EFAULT; } + bpf_diag_mod_end(env); return 0; } @@ -16423,6 +16527,8 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) return err; /* reset caller saved regs to unreadable */ + bpf_diag_record_caller_saved(env, regs); + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); for (i = 0; i < CALLER_SAVED_REGS; i++) { bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); @@ -16433,6 +16539,7 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) * Already marked as written above. */ mark_reg_unknown(env, regs, BPF_REG_0); + bpf_diag_mod_end(env); /* * See bpf_gen_ld_abs() which emits a hidden BPF_EXIT with r0=0 * which must be explored by the verifier when in a subprog. -- cgit v1.2.3 From 9ecd70304e28985af297726cd961a33a6fec5f67 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:46:01 +0200 Subject: bpf: Track verifier reference diagnostic events Add reference acquire and release events to diagnostic history so Resource Lifetime Safety reports can show the lifetime of a specific reference id along the path. Record acquisitions after the verifier assigns the reference id. Record releases only after release_reference_nomark() succeeds, including the kptr_xchg RCU conversion path and owning-to-non-owning conversion path that consume an owning reference. Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260815064612.378577-7-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/diagnostics.c | 28 ++++++++++++++++++++++++++++ kernel/bpf/diagnostics.h | 2 ++ kernel/bpf/verifier.c | 32 +++++++++++++++++++++++++------- 3 files changed, 55 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index 2e8e75815581..ddeaff1e90b7 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -102,6 +102,8 @@ struct bpf_diag_reg_snapshot { enum bpf_diag_history_kind { BPF_DIAG_HISTORY_BRANCH, BPF_DIAG_HISTORY_MOD, + BPF_DIAG_HISTORY_REF_ACQUIRE, + BPF_DIAG_HISTORY_REF_RELEASE, }; struct bpf_diag_history_event { @@ -119,6 +121,9 @@ struct bpf_diag_history_event { u8 reason; bool origin_valid; } mod; + struct { + u32 ref_id; + } ref; }; }; @@ -1021,3 +1026,26 @@ void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env, diag_stack_range_target(state->diag_frame_id, state->frameno, min_off, max_off), reason, NULL, NULL, NULL); } + +static void diag_record_ref(struct bpf_verifier_env *env, u32 insn_idx, u8 kind, u32 ref_id) +{ + struct bpf_diag_history_event event = { + .insn_idx = insn_idx, + .kind = kind, + .ref = { + .ref_id = ref_id, + }, + }; + + diag_append_history(env, &event); +} + +void bpf_diag_record_ref_acquire(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id) +{ + diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_ACQUIRE, ref_id); +} + +void bpf_diag_record_ref_release(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id) +{ + diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_RELEASE, ref_id); +} diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index c4e44b86e89d..d17b498a3f66 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -43,5 +43,7 @@ void bpf_diag_record_scrub(struct bpf_verifier_env *env, const struct bpf_reg_st void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env, const struct bpf_func_state *state, s16 min_off, s16 max_off, enum bpf_diag_mod_reason reason); +void bpf_diag_record_ref_acquire(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id); +void bpf_diag_record_ref_release(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id); #endif /* __BPF_DIAGNOSTICS_H */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a5929e40f18d..8e32fa5fa30a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -205,7 +205,8 @@ struct bpf_verifier_stack_elem { #define BPF_PRIV_STACK_MIN_SIZE 64 static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id); -static int release_reference_nomark(struct bpf_verifier_state *state, int id); +static int __release_reference_nomark(struct bpf_verifier_state *state, int id); +static int release_reference_nomark(struct bpf_verifier_env *env, int id); static int release_reference(struct bpf_verifier_env *env, int id); static void invalidate_non_owning_refs(struct bpf_verifier_env *env); static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env); @@ -1418,6 +1419,7 @@ static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int par s->type = REF_TYPE_PTR; s->id = ++env->id_gen; s->parent_id = parent_id; + bpf_diag_record_ref_acquire(env, insn_idx, s->id); return s->id; } @@ -9017,7 +9019,7 @@ static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range reg->range = AT_PKT_END; } -static int release_reference_nomark(struct bpf_verifier_state *state, int id) +static int __release_reference_nomark(struct bpf_verifier_state *state, int id) { int i; @@ -9032,6 +9034,16 @@ static int release_reference_nomark(struct bpf_verifier_state *state, int id) return -EINVAL; } +static int release_reference_nomark(struct bpf_verifier_env *env, int id) +{ + int err; + + err = __release_reference_nomark(env->cur_state, id); + if (!err) + bpf_diag_record_ref_release(env, env->insn_idx, id); + return err; +} + static int idstack_push(struct bpf_idmap *idmap, u32 id) { int i; @@ -9074,8 +9086,10 @@ static int release_reference(struct bpf_verifier_env *env, int id) if (err) return err; - if (find_reference_state(vstate, id)) - WARN_ON_ONCE(release_reference_nomark(vstate, id)); + if (find_reference_state(vstate, id)) { + err = release_reference_nomark(env, id); + WARN_ON_ONCE(err); + } while ((id = idstack_pop(idstack))) { /* @@ -9164,7 +9178,9 @@ static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id) struct bpf_reg_state *reg; int err; - err = release_reference_nomark(env->cur_state, id); + err = release_reference_nomark(env, id); + if (err) + return err; bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ if (reg->id != id) @@ -11757,8 +11773,10 @@ static void ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 id) { struct bpf_func_state *unused; struct bpf_reg_state *reg; + int err; - WARN_ON_ONCE(release_reference_nomark(env->cur_state, id)); + err = release_reference_nomark(env, id); + WARN_ON_ONCE(err); bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ if (reg->id == id) { @@ -15890,7 +15908,7 @@ static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, * No one could have freed the reference state before * doing the NULL check. */ - WARN_ON_ONCE(release_reference_nomark(vstate, id)); + WARN_ON_ONCE(__release_reference_nomark(vstate, id)); bpf_for_each_reg_in_vstate(vstate, state, reg, ({ mark_ptr_or_null_reg(state, reg, id, is_null); -- cgit v1.2.3 From 956a66e5c33fb53003ca2bc043a90ff0b671b3a5 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:46:02 +0200 Subject: bpf: Track verifier context diagnostic events Record verifier context transitions in the diagnostic history so later reports can anchor causal paths to the critical section that made an operation invalid. This covers lock, IRQ, RCU, and preempt regions without adding any new verifier error reports. Category-specific commits decide where those recorded events should be rendered. Use context depth when selecting scoped history so nested regions anchor at the outer active region, and fall back to the earliest retained event when the matching entry was pruned. Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260815064612.378577-8-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/diagnostics.c | 39 +++++++++++++++++++++++++++++++++++++++ kernel/bpf/diagnostics.h | 12 ++++++++++++ kernel/bpf/verifier.c | 28 +++++++++++++++++++++++----- 3 files changed, 74 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index ddeaff1e90b7..15bca8a02a48 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -104,6 +104,7 @@ enum bpf_diag_history_kind { BPF_DIAG_HISTORY_MOD, BPF_DIAG_HISTORY_REF_ACQUIRE, BPF_DIAG_HISTORY_REF_RELEASE, + BPF_DIAG_HISTORY_CONTEXT, }; struct bpf_diag_history_event { @@ -124,6 +125,11 @@ struct bpf_diag_history_event { struct { u32 ref_id; } ref; + struct { + u32 depth; + u8 kind; + bool enter; + } ctx; }; }; @@ -388,6 +394,19 @@ void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos) log->cnt = log_pos - log->first_seq; } +u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state) +{ + u32 depth = 0; + int i; + + for (i = 0; i < state->acquired_refs; i++) { + if (state->refs[i].type == REF_TYPE_IRQ) + depth++; + } + + return depth; +} + static void diag_append_history(struct bpf_verifier_env *env, const struct bpf_diag_history_event *event) { @@ -1049,3 +1068,23 @@ void bpf_diag_record_ref_release(struct bpf_verifier_env *env, u32 insn_idx, u32 { diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_RELEASE, ref_id); } + +void bpf_diag_record_context(struct bpf_verifier_env *env, u32 insn_idx, + enum bpf_diag_context_kind ctx_kind, bool enter, u32 depth) +{ + /* + * Keep leave events so context rendering can stop at a depth-zero exit + * and show nested-region depth accurately for the active path. + */ + struct bpf_diag_history_event event = { + .insn_idx = insn_idx, + .kind = BPF_DIAG_HISTORY_CONTEXT, + .ctx = { + .kind = ctx_kind, + .enter = enter, + .depth = depth, + }, + }; + + diag_append_history(env, &event); +} diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index d17b498a3f66..ed64776736c6 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -11,6 +11,7 @@ struct bpf_func_state; struct bpf_reg_state; struct bpf_verifier_env; +struct bpf_verifier_state; struct btf; enum bpf_diag_mod_reason { @@ -23,6 +24,14 @@ enum bpf_diag_mod_reason { BPF_DIAG_MOD_CALLER_SAVED, }; +enum bpf_diag_context_kind { + BPF_DIAG_CONTEXT_NONE, + BPF_DIAG_CONTEXT_RCU, + BPF_DIAG_CONTEXT_PREEMPT, + BPF_DIAG_CONTEXT_IRQ, + BPF_DIAG_CONTEXT_LOCK, +}; + bool bpf_diag_enabled(const struct bpf_verifier_env *env); int bpf_diag_init(struct bpf_verifier_env *env); void bpf_diag_init_frame(struct bpf_verifier_env *env, struct bpf_func_state *state); @@ -33,6 +42,7 @@ const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __p const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id); u64 bpf_diag_event_log_save(struct bpf_verifier_env *env); void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos); +u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state); void bpf_diag_free(struct bpf_verifier_env *env); void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true); void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, @@ -45,5 +55,7 @@ void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env, enum bpf_diag_mod_reason reason); void bpf_diag_record_ref_acquire(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id); void bpf_diag_record_ref_release(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id); +void bpf_diag_record_context(struct bpf_verifier_env *env, u32 insn_idx, + enum bpf_diag_context_kind ctx_kind, bool enter, u32 depth); #endif /* __BPF_DIAGNOSTICS_H */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8e32fa5fa30a..1f2a7f480ce3 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1045,7 +1045,7 @@ static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_s } static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx); -static int release_irq_state(struct bpf_verifier_state *state, int id); +static int release_irq_state(struct bpf_verifier_env *env, int id); static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, @@ -1104,7 +1104,7 @@ static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_r return -EINVAL; } - err = release_irq_state(env->cur_state, st->id); + err = release_irq_state(env, st->id); WARN_ON_ONCE(err && err != -EACCES); if (err) { int insn_idx = 0; @@ -1439,6 +1439,8 @@ static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum r state->active_locks++; state->active_lock_id = id; state->active_lock_ptr = ptr; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_LOCK, true, + state->active_locks); return 0; } @@ -1454,6 +1456,8 @@ static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx) s->id = ++env->id_gen; state->active_irq_id = s->id; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_IRQ, true, + bpf_diag_irq_depth(state)); return s->id; } @@ -1495,8 +1499,9 @@ static bool reg_is_referenced(struct bpf_verifier_env *env, const struct bpf_reg return find_reference_state(env->cur_state, reg->id); } -static int release_lock_state(struct bpf_verifier_state *state, int type, int id, void *ptr) +static int release_lock_state(struct bpf_verifier_env *env, int type, int id, void *ptr) { + struct bpf_verifier_state *state = env->cur_state; void *prev_ptr = NULL; u32 prev_id = 0; int i; @@ -1509,6 +1514,8 @@ static int release_lock_state(struct bpf_verifier_state *state, int type, int id /* Reassign active lock (id, ptr). */ state->active_lock_id = prev_id; state->active_lock_ptr = prev_ptr; + bpf_diag_record_context(env, env->insn_idx, BPF_DIAG_CONTEXT_LOCK, + false, state->active_locks); return 0; } if (state->refs[i].type & REF_TYPE_LOCK_MASK) { @@ -1519,8 +1526,9 @@ static int release_lock_state(struct bpf_verifier_state *state, int type, int id return -EINVAL; } -static int release_irq_state(struct bpf_verifier_state *state, int id) +static int release_irq_state(struct bpf_verifier_env *env, int id) { + struct bpf_verifier_state *state = env->cur_state; u32 prev_id = 0; int i; @@ -1533,6 +1541,8 @@ static int release_irq_state(struct bpf_verifier_state *state, int id) if (state->refs[i].id == id) { release_reference_state(state, i); state->active_irq_id = prev_id; + bpf_diag_record_context(env, env->insn_idx, BPF_DIAG_CONTEXT_IRQ, + false, bpf_diag_irq_depth(state)); return 0; } else { prev_id = state->refs[i].id; @@ -7181,7 +7191,7 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state verbose(env, "%s_unlock cannot be out of order\n", lock_str); return -EINVAL; } - if (release_lock_state(cur, type, reg->id, ptr)) { + if (release_lock_state(env, type, reg->id, ptr)) { verbose(env, "%s_unlock of different lock\n", lock_str); return -EINVAL; } @@ -13242,22 +13252,30 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (rcu_lock) { env->cur_state->active_rcu_locks++; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_RCU, true, + env->cur_state->active_rcu_locks); } else if (rcu_unlock) { if (env->cur_state->active_rcu_locks == 0) { verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); return -EINVAL; } env->cur_state->active_rcu_locks--; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_RCU, false, + env->cur_state->active_rcu_locks); if (!in_rcu_cs(env)) invalidate_rcu_protected_refs(env); } else if (preempt_disable) { env->cur_state->active_preempt_locks++; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_PREEMPT, true, + env->cur_state->active_preempt_locks); } else if (preempt_enable) { if (env->cur_state->active_preempt_locks == 0) { verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name); return -EINVAL; } env->cur_state->active_preempt_locks--; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_PREEMPT, false, + env->cur_state->active_preempt_locks); if (!in_rcu_cs(env)) invalidate_rcu_protected_refs(env); } -- cgit v1.2.3 From d63284e62b3185cafe40d5fab17245ee60b6cffe Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:46:03 +0200 Subject: bpf: Report Register Type Safety errors Augment selected register-state verifier failures with Register Type Safety reports. The existing verbose verifier messages remain in place; the new reports add reason, source context, causal path, and suggestions. Cover invalid pointer dereferences, unreadable registers, missing outgoing stack arguments for bpf2bpf and kfunc calls, and rejected pointer arithmetic. Use scoped diagnostic history so reports start from the latest relevant value change and then show later branch outcomes. Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260815064612.378577-9-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/diagnostics.c | 854 +++++++++++++++++++++++++++++++++++++++++++++++ kernel/bpf/diagnostics.h | 18 + kernel/bpf/verifier.c | 128 ++++++- 3 files changed, 985 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index 15bca8a02a48..02399cad2fb0 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -15,9 +15,13 @@ #include "disasm.h" #include "diagnostics.h" +#define REGISTER_TYPE_SAFETY "Register Type Safety" + #define BPF_DIAG_TEXT_WIDTH 100 +#define BPF_DIAG_TEXT_INDENT " " #define BPF_DIAG_CONTEXT 2 #define BPF_DIAG_CONTEXT_CNT (1 + BPF_DIAG_CONTEXT * 2) +#define BPF_DIAG_HISTORY_RENDER_MAX 64 #define BPF_DIAG_SOURCE_LANE_WIDTH 88 #define BPF_DIAG_TAB_WIDTH 8 #define BPF_DIAG_FMT_CHUNK_SIZE (PAGE_SIZE - sizeof(struct diag_fmt_chunk)) @@ -133,6 +137,28 @@ struct bpf_diag_history_event { }; }; +enum bpf_diag_history_scope { + BPF_DIAG_HISTORY_SCOPE_REG, + BPF_DIAG_HISTORY_SCOPE_STACK_ARG, + BPF_DIAG_HISTORY_SCOPE_REF, + BPF_DIAG_HISTORY_SCOPE_CONTEXT, +}; + +struct bpf_diag_history_opts { + enum bpf_diag_history_scope scope; + u32 frame_id; + u32 frameno; + int regno; + int stack_arg_slot; + u32 ref_id; + enum bpf_diag_context_kind ctx_kind; + u32 ctx_depth; +}; + +static void diag_print_history(struct bpf_verifier_env *env, + const struct bpf_diag_history_opts *opts); +static bool diag_target_matches(const struct bpf_diag_mod_target *event_target, + const struct bpf_diag_mod_target *target); struct disasm_line { char text[DISASM_LINE_LEN]; int idx; @@ -505,6 +531,26 @@ const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf return buf; } +static void diag_vprint_indented(struct bpf_verifier_env *env, const char *fmt, va_list args) + __printf(2, 0); + +static void diag_vprint_indented(struct bpf_verifier_env *env, const char *fmt, va_list args) +{ + char *buf; + + if (!bpf_diag_enabled(env)) + return; + + buf = kvasprintf(GFP_KERNEL_ACCOUNT, fmt, args); + if (!buf) { + diag_write(env, "%s\n", BPF_DIAG_TEXT_INDENT); + return; + } + + diag_print_wrapped_prefixed(env, BPF_DIAG_TEXT_INDENT, BPF_DIAG_TEXT_INDENT, buf); + kfree(buf); +} + static int diag_line_width(unsigned int line) { int width = 1; @@ -663,6 +709,47 @@ static void bpf_diag_header(struct bpf_verifier_env *env, const char *category, diag_write(env, "\nVerification failed: %s: %c%s\n", category, first, problem + 1); } +static void diag_reason(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); +static void diag_suggestion(struct bpf_verifier_env *env, const char *fmt, ...) + __printf(2, 3); + +static void diag_section(struct bpf_verifier_env *env, const char *title) +{ + if (!bpf_diag_enabled(env)) + return; + + diag_write(env, "\n%s:\n", title); +} + +static void diag_reason(struct bpf_verifier_env *env, const char *fmt, ...) +{ + va_list args; + + if (!bpf_diag_enabled(env)) + return; + + diag_section(env, "Reason"); + + va_start(args, fmt); + diag_vprint_indented(env, fmt, args); + va_end(args); +} + +static void diag_suggestion(struct bpf_verifier_env *env, const char *fmt, ...) +{ + va_list args; + + if (!bpf_diag_enabled(env)) + return; + + diag_section(env, "Suggestion"); + + va_start(args, fmt); + diag_vprint_indented(env, fmt, args); + va_end(args); + diag_write(env, "\n"); +} + static void diag_print_source_annotation(struct bpf_verifier_env *env, int line_width, int indent, const char *label, const char *msg) { @@ -799,6 +886,284 @@ out_restore: diag_fmt_restore(env, mark); } +static const struct bpf_func_state *diag_current_frame(const struct bpf_verifier_env *env) +{ + return env->cur_state->frame[env->cur_state->curframe]; +} + +void bpf_diag_register_type(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *problem, const char *reason, const char *suggestion) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .regno = regno, + }; + + bpf_diag_header(env, REGISTER_TYPE_SAFETY, problem); + diag_reason(env, "%s", reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); + + if (regno >= 0) + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_type type) +{ + switch (base_type(type)) { + case NOT_INIT: + return "an uninitialized value"; + case SCALAR_VALUE: + return "an integer scalar"; + case PTR_TO_CTX: + return "a context pointer"; + case PTR_TO_STACK: + return "a stack pointer"; + case PTR_TO_MAP_VALUE: + if (type_may_be_null(type)) + return "a nullable map value pointer"; + return "a map value pointer"; + case PTR_TO_MEM: + if (type_may_be_null(type)) + return "a nullable memory pointer"; + return "a memory pointer"; + case PTR_TO_BTF_ID: + if (type_may_be_null(type)) + return "a nullable kernel object pointer"; + if (type_is_non_owning_ref(type)) + return "a borrowed allocated object pointer"; + if (type_is_ptr_alloc_obj(type)) + return "an owned allocated object pointer"; + if (type_flag(type) & PTR_UNTRUSTED) + return "an untrusted kernel object pointer"; + return "a kernel object pointer"; + default: + return reg_type_str(env, type); + } +} + +static const char *diag_arg_ordinal(int argno) +{ + switch (argno) { + case 1: + return "first"; + case 2: + return "second"; + case 3: + return "third"; + case 4: + return "fourth"; + case 5: + return "fifth"; + case 6: + return "sixth"; + case 7: + return "seventh"; + case 8: + return "eighth"; + case 9: + return "ninth"; + case 10: + return "tenth"; + case 11: + return "eleventh"; + case 12: + return "twelfth"; + default: + return NULL; + } +} + +void bpf_diag_invalid_deref(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *reg_name, const struct bpf_reg_state *reg, + enum bpf_diag_invalid_deref_kind kind, s64 offset) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .regno = regno, + }; + const char *type_name = bpf_diag_reg_type_plain(env, reg->type); + + bpf_diag_header(env, REGISTER_TYPE_SAFETY, "invalid dereference"); + + switch (kind) { + case BPF_DIAG_DEREF_SCALAR: + diag_reason(env, "%s is an integer scalar here, not a pointer to memory.", + reg_name); + break; + case BPF_DIAG_DEREF_NULLABLE_PTR: + diag_reason( + env, "%s may be NULL here (%s). The program could dereference NULL on this path, so the verifier cannot prove this access is safe.", + reg_name, type_name); + break; + case BPF_DIAG_DEREF_MODIFIED_PTR: + diag_reason( + env, "%s has offset %lld here, but this pointer type must be dereferenced in its original form.", + reg_name, offset); + break; + case BPF_DIAG_DEREF_INVALID_PTR: + default: + diag_reason( + env, "%s has type %s here, which is not valid for this memory access.", + reg_name, type_name); + break; + } + + diag_section(env, "At"); + if (kind == BPF_DIAG_DEREF_MODIFIED_PTR) + bpf_diag_source(env, insn_idx, "error", + "dereference requires the original %s pointer", type_name); + else + bpf_diag_source(env, insn_idx, "error", "invalid dereference of %s (%s)", + reg_name, type_name); + + if (regno >= 0) + diag_print_history(env, &opts); + + switch (kind) { + case BPF_DIAG_DEREF_NULLABLE_PTR: + diag_suggestion( + env, "Add a NULL check before the access and dereference the pointer only on the non-NULL path."); + break; + case BPF_DIAG_DEREF_MODIFIED_PTR: + diag_suggestion( + env, "Preserve the original pointer in another register, or use only offsets this pointer type permits before dereferencing it."); + break; + case BPF_DIAG_DEREF_SCALAR: + case BPF_DIAG_DEREF_INVALID_PTR: + default: + diag_suggestion( + env, "Preserve a pointer-valued register where needed, or reload and revalidate the pointer after scalar arithmetic, helper calls, or other operations that can invalidate it."); + break; + } +} + +void bpf_diag_unreadable_reg(struct bpf_verifier_env *env, u32 insn_idx, int regno) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .regno = regno, + }; + const struct bpf_diag_log *log = env->diag ? &env->diag->log : NULL; + struct bpf_diag_mod_target target; + bool invalidated = false; + int i; + + target = diag_reg_target(opts.frame_id, opts.frameno, regno); + for (i = log ? log->cnt : 0; i > 0; i--) { + const struct bpf_diag_history_event *event; + + event = &log->events[log_pos(log, i - 1)]; + + if (event->kind != BPF_DIAG_HISTORY_MOD || + !diag_target_matches(&event->mod.target, &target)) + continue; + invalidated = event->mod.new.type == NOT_INIT; + break; + } + + bpf_diag_header(env, REGISTER_TYPE_SAFETY, "unreadable register"); + if (invalidated) + diag_reason( + env, "R%d is not readable here. A previous operation invalidated this register, so the verifier cannot use it as an input.", + regno); + else if (log && !log->first_seq) + diag_reason(env, + "R%d has never been initialized on this path, so the verifier cannot use it as an input.", + regno); + else + diag_reason( + env, "R%d is not readable here. It may never have been initialized, or an earlier operation may have invalidated it.", + regno); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "R%d is not readable", regno); + + if (regno >= 0) + diag_print_history(env, &opts); + + if (invalidated) + diag_suggestion( + env, "Avoid using the register after it is invalidated, or initialize it again before this instruction."); + else if (log && !log->first_seq) + diag_suggestion(env, "Initialize R%d on every path before this instruction.", regno); + else + diag_suggestion( + env, "Initialize the register on every path, or initialize it again after any operation that invalidates it."); +} + +static int diag_stack_argno(u8 slot) +{ + return MAX_BPF_FUNC_REG_ARGS + slot + 1; +} + +static void diag_format_stack_arg(char *buf, size_t size, u8 slot, const char *arg_name) +{ + int argno = diag_stack_argno(slot); + const char *ordinal = diag_arg_ordinal(argno); + + if (ordinal && arg_name) + scnprintf(buf, size, "outgoing stack argument %u (%s argument, %s)", slot + 1, + ordinal, arg_name); + else if (ordinal) + scnprintf(buf, size, "outgoing stack argument %u (%s argument)", slot + 1, ordinal); + else if (arg_name) + scnprintf(buf, size, "outgoing stack argument %u (%s)", slot + 1, arg_name); + else + scnprintf(buf, size, "outgoing stack argument %u", slot + 1); +} + +void bpf_diag_stack_arg_uninit(struct bpf_verifier_env *env, u32 insn_idx, int nargs, + int stack_arg_slot, const char *callee_name, + const char *arg_name) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_STACK_ARG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .stack_arg_slot = stack_arg_slot, + }; + const char *arg_buf; + + arg_buf = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); + if (arg_buf) + diag_format_stack_arg((char *)arg_buf, BPF_DIAG_FMT_BUF_SIZE, stack_arg_slot, + arg_name); + else + arg_buf = ""; + bpf_diag_header(env, REGISTER_TYPE_SAFETY, "missing stack argument"); + if (callee_name && *callee_name) + diag_reason( + env, "Function %s expects %d arguments, but %s is not initialized at this call.", + callee_name, nargs, arg_buf); + else + diag_reason( + env, "The callee expects %d arguments, but %s is not initialized at this call.", + nargs, arg_buf); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s is not initialized", arg_buf); + + if (stack_arg_slot >= 0) + diag_print_history(env, &opts); + + diag_suggestion( + env, "Write the outgoing stack argument after any operation that may invalidate stored pointer values, and before making this call."); +} + void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true) { struct bpf_diag_history_event event = { @@ -1088,3 +1453,492 @@ void bpf_diag_record_context(struct bpf_verifier_env *env, u32 insn_idx, diag_append_history(env, &event); } + +static int diag_history_context_start_idx(const struct bpf_diag_log *log, + const struct bpf_diag_history_opts *opts) +{ + int i; + + if (!opts->ctx_depth) + return 0; + + /* Find the most recent outermost entry, or a depth-zero exit. */ + for (i = log->cnt; i > 0; i--) { + const struct bpf_diag_history_event *event; + + event = &log->events[log_pos(log, i - 1)]; + + if (event->kind != BPF_DIAG_HISTORY_CONTEXT || event->ctx.kind != opts->ctx_kind) + continue; + + if (event->ctx.enter && event->ctx.depth == 1) + return i - 1; + if (!event->ctx.enter && event->ctx.depth == 0) + return 0; + } + + return 0; +} + +struct bpf_diag_history_filter { + const struct bpf_diag_history_opts *opts; + u32 lineage_start; + bool lineage_valid; +}; + +static bool diag_target_matches(const struct bpf_diag_mod_target *event_target, + const struct bpf_diag_mod_target *target) +{ + int slot_off; + + if (event_target->frame_id != target->frame_id || event_target->frameno != target->frameno) + return false; + + if (event_target->kind == BPF_DIAG_MOD_TARGET_STACK_RANGE && + target->kind == BPF_DIAG_MOD_TARGET_STACK_SLOT) { + slot_off = -(target->spi + 1) * BPF_REG_SIZE; + return event_target->range.min_off < slot_off + BPF_REG_SIZE && + event_target->range.max_off > slot_off; + } + + if (event_target->kind != target->kind) + return false; + + switch (target->kind) { + case BPF_DIAG_MOD_TARGET_REG: + return event_target->regno == target->regno; + case BPF_DIAG_MOD_TARGET_STACK_ARG: + return event_target->stack_arg == target->stack_arg; + case BPF_DIAG_MOD_TARGET_STACK_SLOT: + return event_target->spi == target->spi; + default: + return false; + } +} + +static void diag_build_lineage(struct bpf_verifier_env *env, struct bpf_diag_log *log, + struct bpf_diag_history_filter *filter) +{ + const struct bpf_diag_history_opts *opts = filter->opts; + struct bpf_diag_mod_target target; + int i; + + for (i = 0; i < log->cnt; i++) + log->events[log_pos(log, i)].in_lineage = false; + + if (opts->scope == BPF_DIAG_HISTORY_SCOPE_REG) + target = diag_reg_target(opts->frame_id, opts->frameno, opts->regno); + else if (opts->scope == BPF_DIAG_HISTORY_SCOPE_STACK_ARG) + target = diag_stack_arg_target(opts->frame_id, opts->frameno, + opts->stack_arg_slot); + else + return; + + /* + * Find the nearest mutation of the active target. A fill or spill changes + * the target to its origin, so the same walk follows register/stack + * lineage recursively until it reaches the write that created the value. + */ + for (i = log->cnt; i > 0; i--) { + struct bpf_diag_history_event *event; + + event = &log->events[log_pos(log, i - 1)]; + if (event->kind != BPF_DIAG_HISTORY_MOD || + !diag_target_matches(&event->mod.target, &target)) + continue; + + event->in_lineage = true; + filter->lineage_start = i - 1; + filter->lineage_valid = true; + + if (event->mod.origin_valid) { + target = event->mod.origin; + continue; + } + if (event->mod.reason != BPF_DIAG_MOD_WRITE && + event->mod.reason != BPF_DIAG_MOD_SPILL) + continue; + if (diag_mod_keeps_lineage(env, event)) + continue; + break; + } +} + +static int diag_history_start_idx(const struct bpf_diag_log *log, + const struct bpf_diag_history_filter *filter) +{ + const struct bpf_diag_history_opts *opts = filter->opts; + int i; + + if (opts->scope == BPF_DIAG_HISTORY_SCOPE_CONTEXT) + return diag_history_context_start_idx(log, opts); + if (filter->lineage_valid) + return filter->lineage_start; + if (opts->scope != BPF_DIAG_HISTORY_SCOPE_REF) + return 0; + + for (i = log->cnt; i > 0; i--) { + const struct bpf_diag_history_event *event; + + event = &log->events[log_pos(log, i - 1)]; + if (event->kind == BPF_DIAG_HISTORY_REF_ACQUIRE && + event->ref.ref_id == opts->ref_id) + return i - 1; + } + + return 0; +} + +static bool diag_history_event_visible(const struct bpf_diag_history_event *event, + const struct bpf_diag_history_filter *filter) +{ + const struct bpf_diag_history_opts *opts = filter->opts; + + switch (event->kind) { + case BPF_DIAG_HISTORY_BRANCH: + return true; + case BPF_DIAG_HISTORY_MOD: + return filter->lineage_valid && event->in_lineage; + case BPF_DIAG_HISTORY_REF_ACQUIRE: + case BPF_DIAG_HISTORY_REF_RELEASE: + return opts->scope == BPF_DIAG_HISTORY_SCOPE_REF && + event->ref.ref_id == opts->ref_id; + case BPF_DIAG_HISTORY_CONTEXT: + return opts->scope == BPF_DIAG_HISTORY_SCOPE_CONTEXT && + event->ctx.kind == opts->ctx_kind; + default: + return false; + } +} + +static const char *diag_s64_bound_name(s64 value) +{ + if (value == S64_MIN) + return "S64_MIN"; + if (value == S64_MAX) + return "S64_MAX"; + return NULL; +} + +static const char *diag_u64_bound_name(u64 value) +{ + if (value == U64_MAX) + return "U64_MAX"; + return NULL; +} + +static const char *diag_s64_str(struct bpf_verifier_env *env, s64 value) +{ + return diag_s64_bound_name(value) ?: bpf_diag_fmt(env, "%lld", value); +} + +static const char *diag_u64_str(struct bpf_verifier_env *env, u64 value) +{ + return diag_u64_bound_name(value) ?: bpf_diag_fmt(env, "%llu", value); +} + +static bool diag_cnum64_unknown(struct cnum64 range) +{ + return cnum64_smin(range) == S64_MIN && cnum64_smax(range) == S64_MAX && + cnum64_umin(range) == 0 && cnum64_umax(range) == U64_MAX; +} + +static bool diag_snapshot_unknown(const struct bpf_diag_reg_snapshot *snapshot) +{ + return tnum_is_unknown(snapshot->var_off) && diag_cnum64_unknown(snapshot->r64); +} + +static const char *diag_scalar_range(struct bpf_verifier_env *env, struct cnum64 range) +{ + return bpf_diag_fmt(env, "signed range [%s, %s], unsigned range [%s, %s]", + diag_s64_str(env, cnum64_smin(range)), + diag_s64_str(env, cnum64_smax(range)), + diag_u64_str(env, cnum64_umin(range)), + diag_u64_str(env, cnum64_umax(range))); +} + +static const char *diag_var_offset(struct bpf_verifier_env *env, + const struct bpf_diag_reg_snapshot *snapshot) +{ + if (tnum_is_const(snapshot->var_off)) + return bpf_diag_fmt(env, "at offset %lld", (s64)snapshot->var_off.value); + + if (diag_snapshot_unknown(snapshot)) + return bpf_diag_fmt(env, "with unknown offset"); + + return bpf_diag_fmt(env, + "with variable offset: known bits %#llx, unknown mask %#llx, %s", + snapshot->var_off.value, snapshot->var_off.mask, + diag_scalar_range(env, snapshot->r64)); +} + +static const char *diag_reg_map_name(const struct bpf_map *map) +{ + if (!map || !map->name[0]) + return NULL; + + return map->name; +} + +static const char *diag_reg_snapshot(struct bpf_verifier_env *env, + const struct bpf_diag_reg_snapshot *snapshot) +{ + const char *type_name = reg_type_str(env, snapshot->type); + const char *offset = diag_var_offset(env, snapshot); + const char *btf = snapshot->btf && snapshot->btf_id ? + bpf_diag_fmt_btf_type(env, snapshot->btf, snapshot->btf_id) : NULL; + const char *map_name; + + if (snapshot->type == SCALAR_VALUE) { + if (tnum_is_const(snapshot->var_off)) + return bpf_diag_fmt(env, "integer scalar value %lld", + (s64)snapshot->var_off.value); + if (diag_snapshot_unknown(snapshot)) + return bpf_diag_fmt(env, "integer scalar with unknown value"); + if (cnum64_is_const(snapshot->r64)) + return bpf_diag_fmt(env, "integer scalar value %lld", + cnum64_smin(snapshot->r64)); + return bpf_diag_fmt(env, "integer scalar with %s", + diag_scalar_range(env, snapshot->r64)); + } + + if (snapshot->type == NOT_INIT) + return bpf_diag_fmt(env, "uninitialized value"); + + if (base_type(snapshot->type) == PTR_TO_CTX) + return bpf_diag_fmt(env, "context pointer %s", offset); + + if (base_type(snapshot->type) == PTR_TO_STACK) + return bpf_diag_fmt(env, "stack pointer %s", offset); + + if (base_type(snapshot->type) == PTR_TO_MAP_VALUE) { + const char *kind = type_may_be_null(snapshot->type) ? "nullable map value" : + "map value"; + + map_name = diag_reg_map_name(snapshot->map_ptr); + if (map_name) + return bpf_diag_fmt(env, "%s from %s %s", kind, map_name, offset); + return bpf_diag_fmt(env, "%s %s", kind, offset); + } + + if (base_type(snapshot->type) == CONST_PTR_TO_MAP) { + map_name = diag_reg_map_name(snapshot->map_ptr); + if (map_name) + return bpf_diag_fmt(env, "map pointer for map %s", map_name); + return bpf_diag_fmt(env, "map pointer"); + } + + if (type_is_non_owning_ref(snapshot->type)) { + if (btf) + return bpf_diag_fmt(env, "borrowed allocated object pointer type=%s", btf); + return bpf_diag_fmt(env, "borrowed allocated object pointer"); + } + + if (type_is_ptr_alloc_obj(snapshot->type)) { + if (btf) + return bpf_diag_fmt(env, "owned allocated object pointer type=%s", btf); + return bpf_diag_fmt(env, "owned allocated object pointer"); + } + + if (base_type(snapshot->type) == PTR_TO_BTF_ID && btf) + return bpf_diag_fmt(env, "%s type=%s %s", type_name, btf, offset); + + return bpf_diag_fmt(env, "%s %s", type_name, offset); +} + +static const char *diag_mod_target_desc(struct bpf_verifier_env *env, + const struct bpf_diag_mod_target *target) +{ + switch (target->kind) { + case BPF_DIAG_MOD_TARGET_REG: + return bpf_diag_fmt(env, "R%u", target->regno); + case BPF_DIAG_MOD_TARGET_STACK_ARG: + return bpf_diag_fmt(env, "stack arg%d", diag_stack_argno(target->stack_arg)); + case BPF_DIAG_MOD_TARGET_STACK_SLOT: + return bpf_diag_fmt(env, "stack slot fp%d", -(target->spi + 1) * BPF_REG_SIZE); + default: + return "value"; + } +} + +static void diag_print_mod(struct bpf_verifier_env *env, const struct bpf_diag_history_event *event) +{ + const struct bpf_diag_mod_target *target = &event->mod.target; + const char *target_desc, *reason = NULL, *old, *new; + const char *label = "update"; + + if (target->kind == BPF_DIAG_MOD_TARGET_STACK_RANGE) { + bpf_diag_source( + env, event->insn_idx, "invalidated", + "variable-offset stack write may affect bytes fp%d through fp%d", + target->range.min_off, target->range.max_off - 1); + return; + } + + old = diag_reg_snapshot(env, &event->mod.old); + new = diag_reg_snapshot(env, &event->mod.new); + target_desc = diag_mod_target_desc(env, target); + + switch (event->mod.reason) { + case BPF_DIAG_MOD_REF_RELEASE: + reason = target->kind == BPF_DIAG_MOD_TARGET_REG ? "resource release invalidated " + "this pointer" : + "resource release invalidated " + "this value"; + break; + case BPF_DIAG_MOD_PKT_DATA_CHANGE: + reason = "packet data may have moved"; + break; + case BPF_DIAG_MOD_NON_OWN_REF: + reason = "leaving the protected region invalidated this borrowed pointer"; + break; + case BPF_DIAG_MOD_CALLER_SAVED: + reason = target->kind == BPF_DIAG_MOD_TARGET_STACK_ARG ? + "call invalidated this outgoing stack argument" : + "call invalidated this caller-saved register"; + break; + case BPF_DIAG_MOD_WRITE: + if (target->kind == BPF_DIAG_MOD_TARGET_STACK_SLOT) + reason = "a later stack write overwrote this spilled value"; + break; + case BPF_DIAG_MOD_SPILL: + label = "spilled"; + break; + case BPF_DIAG_MOD_VAR_WRITE: + default: + break; + } + + if (reason) { + bpf_diag_source(env, event->insn_idx, "invalidated", + "%s: %s; previous value was %s", target_desc, reason, old); + return; + } + + bpf_diag_source(env, event->insn_idx, label, "%s changed from %s to %s", target_desc, + old, new); +} + +static void diag_print_ref_event(struct bpf_verifier_env *env, + const struct bpf_diag_history_event *event) +{ + const char *label; + + label = event->kind == BPF_DIAG_HISTORY_REF_ACQUIRE ? "acquired" : "released"; + bpf_diag_source(env, event->insn_idx, label, "owned resource (id=%u)", + event->ref.ref_id); +} + +static const char *diag_context_name(enum bpf_diag_context_kind kind) +{ + switch (kind) { + case BPF_DIAG_CONTEXT_RCU: + return "RCU read lock region"; + case BPF_DIAG_CONTEXT_PREEMPT: + return "non-preemptible region"; + case BPF_DIAG_CONTEXT_IRQ: + return "IRQ-disabled region"; + case BPF_DIAG_CONTEXT_LOCK: + return "lock region"; + case BPF_DIAG_CONTEXT_NONE: + default: + return "context"; + } +} + +static void diag_print_context_event(struct bpf_verifier_env *env, + const struct bpf_diag_history_event *event) +{ + bpf_diag_source(env, event->insn_idx, "context", "%s %s; depth is now %u", + event->ctx.enter ? "entered" : "left", + diag_context_name(event->ctx.kind), event->ctx.depth); +} + +static void diag_print_history(struct bpf_verifier_env *env, + const struct bpf_diag_history_opts *opts) +{ + const struct bpf_diag_history_event *event; + struct bpf_diag_history_filter filter = { + .opts = opts, + }; + struct bpf_diag_log *log; + struct diag_fmt_mark mark; + bool first = true; + int start_idx; + u32 i, visible_cnt = 0, visible_idx = 0; + + if (!bpf_diag_enabled(env)) + return; + + if (!env->diag) + return; + log = &env->diag->log; + + diag_build_lineage(env, log, &filter); + + start_idx = diag_history_start_idx(log, &filter); + for (i = start_idx; i < log->cnt; i++) { + event = &log->events[log_pos(log, i)]; + if (diag_history_event_visible(event, &filter)) + visible_cnt++; + } + + if (!visible_cnt && !log->first_seq && opts->scope == BPF_DIAG_HISTORY_SCOPE_STACK_ARG) + return; + + diag_section(env, "Causal path"); + mark = diag_fmt_save(env); + for (i = start_idx; i < log->cnt; i++) { + event = &log->events[log_pos(log, i)]; + if (!diag_history_event_visible(event, &filter)) + continue; + + diag_fmt_restore(env, mark); + if (visible_cnt > BPF_DIAG_HISTORY_RENDER_MAX && + visible_idx >= BPF_DIAG_HISTORY_RENDER_MAX / 2 && + visible_idx < visible_cnt - BPF_DIAG_HISTORY_RENDER_MAX / 2) { + if (visible_idx++ != BPF_DIAG_HISTORY_RENDER_MAX / 2) + continue; + if (!first) + diag_write(env, "\n"); + first = false; + diag_write(env, " %u intermediate causal-history events omitted\n", + visible_cnt - BPF_DIAG_HISTORY_RENDER_MAX); + continue; + } + visible_idx++; + + if (!first) + diag_write(env, "\n"); + first = false; + + switch (event->kind) { + case BPF_DIAG_HISTORY_BRANCH: + bpf_diag_source(env, event->insn_idx, "branch", + "took the %s branch of this conditional, goto %s", + event->branch.cond_true ? "true" : "false", + event->branch.cond_true ? "followed" : "not followed"); + break; + case BPF_DIAG_HISTORY_MOD: + diag_print_mod(env, event); + break; + case BPF_DIAG_HISTORY_REF_ACQUIRE: + case BPF_DIAG_HISTORY_REF_RELEASE: + diag_print_ref_event(env, event); + break; + case BPF_DIAG_HISTORY_CONTEXT: + diag_print_context_event(env, event); + break; + default: + break; + } + } + + if (!visible_cnt) + diag_write(env, " no retained diagnostic events on this path\n"); + if (log->first_seq) + diag_write(env, " %llu older causal-history event%s not retained because diagnostic " + "event storage reached capacity\n", + log->first_seq, log->first_seq == 1 ? "" : "s"); + diag_fmt_restore(env, mark); +} diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index ed64776736c6..d2355c46dad1 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -4,6 +4,7 @@ #ifndef __BPF_DIAGNOSTICS_H #define __BPF_DIAGNOSTICS_H +#include #include #include #include @@ -32,6 +33,13 @@ enum bpf_diag_context_kind { BPF_DIAG_CONTEXT_LOCK, }; +enum bpf_diag_invalid_deref_kind { + BPF_DIAG_DEREF_SCALAR, + BPF_DIAG_DEREF_NULLABLE_PTR, + BPF_DIAG_DEREF_MODIFIED_PTR, + BPF_DIAG_DEREF_INVALID_PTR, +}; + bool bpf_diag_enabled(const struct bpf_verifier_env *env); int bpf_diag_init(struct bpf_verifier_env *env); void bpf_diag_init_frame(struct bpf_verifier_env *env, struct bpf_func_state *state); @@ -40,10 +48,20 @@ const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list __printf(2, 0); const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id); +const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_type type); u64 bpf_diag_event_log_save(struct bpf_verifier_env *env); void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos); u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state); void bpf_diag_free(struct bpf_verifier_env *env); +void bpf_diag_register_type(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *problem, const char *reason, const char *suggestion); +void bpf_diag_invalid_deref(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *reg_name, const struct bpf_reg_state *reg, + enum bpf_diag_invalid_deref_kind kind, s64 offset); +void bpf_diag_unreadable_reg(struct bpf_verifier_env *env, u32 insn_idx, int regno); +void bpf_diag_stack_arg_uninit(struct bpf_verifier_env *env, u32 insn_idx, int nargs, + int stack_arg_slot, const char *callee_name, + const char *arg_name); void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true); void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 1f2a7f480ce3..962eb7b37e6b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3130,6 +3130,7 @@ static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *r /* check whether register used as source operand can be read */ if (reg->type == NOT_INIT) { verbose(env, "R%d !read_ok\n", regno); + bpf_diag_unreadable_reg(env, env->insn_idx, regno); return -EACCES; } /* We don't need to worry about FP liveness because it's read-only */ @@ -4149,7 +4150,8 @@ static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx) } static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, - int nargs) + int nargs, const char *callee_name, const struct btf *btf, + const struct btf_param *args) { int i, spi; @@ -4157,8 +4159,14 @@ static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_fu spi = i - MAX_BPF_FUNC_REG_ARGS; if (spi >= caller->out_stack_arg_cnt || caller->stack_arg_regs[spi].type == NOT_INIT) { + const char *arg_name = NULL; + + if (args && args[i].name_off) + arg_name = btf_name_by_offset(btf, args[i].name_off); verbose(env, "callee expects %d args, stack arg%d is not initialized\n", nargs, spi + 1); + bpf_diag_stack_arg_uninit(env, env->insn_idx, nargs, spi, + callee_name, arg_name); return -EFAULT; } } @@ -4313,6 +4321,9 @@ static int __check_ptr_off_reg(struct bpf_verifier_env *env, if (!fixed_off_ok && reg->var_off.value != 0) { verbose(env, "dereference of modified %s ptr %s off=%lld disallowed\n", reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value); + bpf_diag_invalid_deref(env, env->insn_idx, reg_from_argno(argno), + reg_arg_name(env, argno), reg, + BPF_DIAG_DEREF_MODIFIED_PTR, reg->var_off.value); return -EACCES; } @@ -6258,6 +6269,9 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (type_may_be_null(reg->type)) { verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); + bpf_diag_invalid_deref(env, insn_idx, reg_from_argno(argno), + reg_arg_name(env, argno), reg, + BPF_DIAG_DEREF_NULLABLE_PTR, 0); return -EACCES; } @@ -6408,8 +6422,16 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (t == BPF_READ && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else { + enum bpf_diag_invalid_deref_kind kind = BPF_DIAG_DEREF_INVALID_PTR; + verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); + if (reg->type == SCALAR_VALUE) + kind = BPF_DIAG_DEREF_SCALAR; + else if (type_may_be_null(reg->type)) + kind = BPF_DIAG_DEREF_NULLABLE_PTR; + bpf_diag_invalid_deref(env, insn_idx, reg_from_argno(argno), + reg_arg_name(env, argno), reg, kind, 0); return -EACCES; } @@ -9297,20 +9319,28 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_func_state *caller = cur_func(env); struct bpf_verifier_log *log = &env->log; struct ref_obj_desc ref_obj = {}; + const struct btf_param *args; + const struct btf_type *func, *func_proto; u32 i; int ret, err; ret = btf_prepare_func_args(env, subprog); if (ret) { if (bpf_in_stack_arg_cnt(sub) > 0) { - err = check_outgoing_stack_args(env, caller, sub->arg_cnt); + err = check_outgoing_stack_args(env, caller, sub->arg_cnt, + bpf_subprog_name(env, subprog), + NULL, NULL); if (err) return err; } return ret; } - ret = check_outgoing_stack_args(env, caller, sub->arg_cnt); + func = btf_type_by_id(btf, env->prog->aux->func_info[subprog].type_id); + func_proto = btf_type_by_id(btf, func->type); + args = btf_params(func_proto); + ret = check_outgoing_stack_args(env, caller, sub->arg_cnt, + bpf_subprog_name(env, subprog), btf, args); if (ret) return ret; @@ -12191,7 +12221,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me args = (const struct btf_param *)(meta->func_proto + 1); nargs = btf_type_vlen(meta->func_proto); - ret = check_outgoing_stack_args(env, caller, nargs); + ret = check_outgoing_stack_args(env, caller, nargs, func_name, btf, args); if (ret) return ret; @@ -13872,9 +13902,8 @@ static int sanitize_check_bounds(struct bpf_verifier_env *env, * If we return -EACCES, caller may want to try again treating pointer as a * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. */ -static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, - struct bpf_insn *insn, - const struct bpf_reg_state *ptr_reg, +static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn, + u32 ptr_regno, const struct bpf_reg_state *ptr_reg, const struct bpf_reg_state *off_reg) { struct bpf_verifier_state *vstate = env->cur_state; @@ -13886,6 +13915,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_sanitize_info info = {}; u8 opcode = BPF_OP(insn->code); u32 dst = insn->dst_reg; + const char *reason; int ret, bounds_ret; dst_reg = ®s[dst]; @@ -13909,12 +13939,24 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, verbose(env, "R%d 32-bit pointer arithmetic prohibited\n", dst); + reason = bpf_diag_fmt( + env, "R%d holds %s. 32-bit ALU operations on pointers discard pointer tracking, so the verifier cannot keep the result as a safe pointer.", + ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "32-bit pointer arithmetic", reason, + "Use a 64-bit ALU instruction with an allowed, bounded scalar offset."); return -EACCES; } if (ptr_reg->type & PTR_MAYBE_NULL) { verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", dst, reg_type_str(env, ptr_reg->type)); + reason = bpf_diag_fmt( + env, "R%d may be NULL (%s). Pointer arithmetic is allowed only after the program proves the pointer is non-NULL on this path.", + ptr_regno, reg_type_str(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer arithmetic before NULL check", reason, + "Make sure that a NULL check precedes any arithmetic performed on the pointer."); return -EACCES; } @@ -13944,6 +13986,12 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, default: verbose(env, "R%d pointer arithmetic on %s prohibited\n", dst, reg_type_str(env, ptr_reg->type)); + reason = bpf_diag_fmt( + env, "R%d holds %s. This pointer kind does not allow offset arithmetic.", + ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer arithmetic is not allowed", reason, + "Do not change this pointer's offset; use it only in operations accepted for its kind."); return -EACCES; } @@ -13961,9 +14009,25 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, if (base_type(ptr_reg->type) == PTR_TO_MEM && (ptr_reg->type & PTR_UNTRUSTED)) return 0; - if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type) || - !check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type)) + if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type)) { + reason = bpf_diag_fmt( + env, "The scalar offset used with R%d is unbounded or outside the verifier's safe pointer-offset range [-%u, %u].", + ptr_regno, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason, + "Clamp or bounds-check the scalar offset before applying it to the pointer."); return -EINVAL; + } + if (!check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type)) { + reason = bpf_diag_fmt( + env, "R%d already has an offset outside the verifier's safe range [-%u, %u] for %s.", + ptr_regno, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF, + bpf_diag_reg_type_plain(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason, + "Keep the base pointer within the verifier's allowed offset range before applying more arithmetic."); + return -EINVAL; + } /* pointer types do not carry 32-bit bounds at the moment. */ __mark_reg32_unbounded(dst_reg); @@ -14006,6 +14070,13 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, /* scalar -= pointer. Creates an unknown scalar */ verbose(env, "R%d tried to subtract pointer from scalar\n", dst); + reason = bpf_diag_fmt( + env, "This operation subtracts pointer register R%d from scalar register R%d. " + "The verifier only tracks pointer-minus-scalar arithmetic for allowed pointer types.", + ptr_regno, dst); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer subtracted from scalar", reason, + "Keep the pointer as the base; only add or subtract bounded scalars when permitted."); return -EACCES; } /* We don't allow subtraction from FP, because (according to @@ -14015,6 +14086,12 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, if (ptr_reg->type == PTR_TO_STACK) { verbose(env, "R%d subtraction from stack pointer prohibited\n", dst); + reason = bpf_diag_fmt( + env, "R%d is a stack pointer. The verifier does not allow BPF_SUB to move stack pointers.", + ptr_regno); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "subtraction from stack pointer", reason, + "Use addition from R10 to form stack addresses within the tracked stack frame."); return -EACCES; } dst_reg->r64 = cnum64_add(ptr_reg->r64, cnum64_negate(off_reg->r64)); @@ -14040,16 +14117,38 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, /* bitwise ops on pointers are troublesome, prohibit. */ verbose(env, "R%d bitwise operator %s on pointer prohibited\n", dst, bpf_alu_string[opcode >> 4]); + reason = bpf_diag_fmt( + env, "R%d holds %s. Bitwise operator %s would destroy the pointer value the verifier is tracking.", + ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type), + bpf_alu_string[opcode >> 4]); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "bitwise operation on pointer", reason, + "Do bitwise operations on scalar values, not on pointer-valued registers."); return -EACCES; default: /* other operators (e.g. MUL,LSH) produce non-pointer results */ verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", dst, bpf_alu_string[opcode >> 4]); + reason = bpf_diag_fmt( + env, "R%d holds %s. Operator %s is not one of the limited pointer arithmetic operations the verifier can track.", + ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type), + bpf_alu_string[opcode >> 4]); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "invalid pointer arithmetic operator", reason, + "Use only verifier-supported addition or subtraction with a bounded scalar offset, or perform this operation on a scalar value."); return -EACCES; } - if (!check_reg_sane_offset_ptr(env, dst_reg, ptr_reg->type)) + if (!check_reg_sane_offset_ptr(env, dst_reg, ptr_reg->type)) { + reason = bpf_diag_fmt( + env, "After this arithmetic, R%d would be outside the verifier's safe offset range [-%u, %u] for %s.", + dst, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF, + bpf_diag_reg_type_plain(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason, + "Tighten the scalar bounds before the arithmetic so the resulting pointer remains within the allowed range."); return -EINVAL; + } reg_bounds_sync(dst_reg); bounds_ret = sanitize_check_bounds(env, insn, dst_reg); if (bounds_ret == -EACCES) @@ -15021,15 +15120,15 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, if (err) return err; off_reg = *dst_reg; - return adjust_ptr_min_max_vals(env, insn, src_reg, &off_reg); + return adjust_ptr_min_max_vals(env, insn, insn->src_reg, src_reg, + &off_reg); } } else if (ptr_reg) { /* pointer += scalar */ err = mark_chain_precision(env, insn->src_reg); if (err) return err; - return adjust_ptr_min_max_vals(env, insn, - dst_reg, src_reg); + return adjust_ptr_min_max_vals(env, insn, insn->dst_reg, dst_reg, src_reg); } else if (dst_reg->precise) { /* if dst_reg is precise, src_reg should be precise as well */ err = mark_chain_precision(env, insn->src_reg); @@ -15044,8 +15143,7 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, __mark_reg_known(&off_reg, insn->imm); src_reg = &off_reg; if (ptr_reg) /* pointer += K */ - return adjust_ptr_min_max_vals(env, insn, - ptr_reg, src_reg); + return adjust_ptr_min_max_vals(env, insn, insn->dst_reg, ptr_reg, src_reg); } /* Got here implies adding two SCALAR_VALUEs */ -- cgit v1.2.3 From 2bdc90f5319451d825466375b4c0d7fc1e52c501 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:46:04 +0200 Subject: bpf: Report Memory Safety bounds errors Augment selected memory-range verifier failures with Memory Safety reports while preserving the existing terse verifier messages for compatibility. Cover stack spill corruption, uninitialized stack reads, variable stack helper accesses, and check_mem_region_access() range-proof failures. The bounds report spells out the required offset + access_size <= object_size proof with concrete values and uses scoped diagnostic history for causal context. Signed-off-by: Kumar Kartikeya Dwivedi Link: https://patch.msgid.link/20260815064612.378577-10-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/diagnostics.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++ kernel/bpf/diagnostics.h | 6 ++++ kernel/bpf/verifier.c | 78 +++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 158 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index 02399cad2fb0..058574a1411e 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -16,6 +17,7 @@ #include "diagnostics.h" #define REGISTER_TYPE_SAFETY "Register Type Safety" +#define MEMORY_SAFETY "Memory Safety" #define BPF_DIAG_TEXT_WIDTH 100 #define BPF_DIAG_TEXT_INDENT " " @@ -1164,6 +1166,18 @@ void bpf_diag_stack_arg_uninit(struct bpf_verifier_env *env, u32 insn_idx, int n env, "Write the outgoing stack argument after any operation that may invalidate stored pointer values, and before making this call."); } +void bpf_diag_memory(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion) +{ + bpf_diag_header(env, MEMORY_SAFETY, problem); + diag_reason(env, "%s", reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); + + diag_suggestion(env, "%s", suggestion); +} + void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true) { struct bpf_diag_history_event event = { @@ -1657,6 +1671,70 @@ static const char *diag_scalar_range(struct bpf_verifier_env *env, struct cnum64 diag_u64_str(env, cnum64_umax(range))); } +const char *bpf_diag_fmt_s64_sum(struct bpf_verifier_env *env, s64 value, int addend) +{ + s64 sum; + + if (check_add_overflow(value, (s64)addend, &sum)) + return bpf_diag_fmt(env, "%lld plus %d (%s)", value, addend, + addend < 0 ? "below S64_MIN" : "above S64_MAX"); + + return bpf_diag_fmt(env, "%lld", sum); +} + +static const char *diag_access_offset(struct bpf_verifier_env *env, int off, + const struct bpf_reg_state *reg) +{ + if (tnum_is_const(reg->var_off)) + return bpf_diag_fmt(env, "constant %s", + bpf_diag_fmt_s64_sum(env, (s64)reg->var_off.value, off)); + + if (tnum_is_unknown(reg->var_off) && diag_cnum64_unknown(reg->r64)) + return bpf_diag_fmt(env, "unbounded"); + + if (off) + return bpf_diag_fmt(env, + "variable: known bits %#llx, unknown mask %#llx, plus fixed offset %d; %s", + (u64)reg->var_off.value, reg->var_off.mask, off, + diag_scalar_range(env, reg->r64)); + return bpf_diag_fmt(env, "variable: known bits %#llx, unknown mask %#llx; %s", + (u64)reg->var_off.value, reg->var_off.mask, + diag_scalar_range(env, reg->r64)); +} + +void bpf_diag_mem_bounds(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *reg_name, const char *type_name, const char *proof, + int off, int size, u32 mem_size, const struct bpf_reg_state *reg) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .regno = regno, + }; + const char *offset_desc; + + if (!bpf_diag_enabled(env)) + return; + + offset_desc = diag_access_offset(env, off, reg); + + bpf_diag_header(env, MEMORY_SAFETY, "access outside bounds"); + diag_reason( + env, "The verifier cannot prove offset + access_size <= object_size. Here, %s. %s is %s; offset is %s; access_size is %d; object_size is %u.", + proof, reg_name, type_name, offset_desc, size, mem_size); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "access may be outside object bounds"); + + if (regno >= 0) + diag_print_history(env, &opts); + + diag_suggestion( + env, "Add or adjust a bounds check that proves offset + access_size stays within the object."); +} + static const char *diag_var_offset(struct bpf_verifier_env *env, const struct bpf_diag_reg_snapshot *snapshot) { diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index d2355c46dad1..b5feda71de3e 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -15,6 +15,7 @@ struct bpf_verifier_env; struct bpf_verifier_state; struct btf; +const char *bpf_diag_fmt_s64_sum(struct bpf_verifier_env *env, s64 value, int addend); enum bpf_diag_mod_reason { BPF_DIAG_MOD_WRITE, BPF_DIAG_MOD_SPILL, @@ -62,6 +63,11 @@ void bpf_diag_unreadable_reg(struct bpf_verifier_env *env, u32 insn_idx, int reg void bpf_diag_stack_arg_uninit(struct bpf_verifier_env *env, u32 insn_idx, int nargs, int stack_arg_slot, const char *callee_name, const char *arg_name); +void bpf_diag_memory(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion); +void bpf_diag_mem_bounds(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *reg_name, const char *type_name, const char *proof, + int off, int size, u32 mem_size, const struct bpf_reg_state *reg); void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true); void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 962eb7b37e6b..86c4212de5aa 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3470,7 +3470,16 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, bpf_is_spilled_reg(&state->stack[spi]) && !bpf_is_spilled_scalar_reg(&state->stack[spi]) && size != BPF_REG_SIZE) { + const char *reason; + verbose(env, "attempt to corrupt spilled pointer on stack\n"); + reason = bpf_diag_fmt(env, + "This store writes %d bytes at stack offset %d into a stack slot that currently holds a spilled pointer. " + "Partial writes to spilled pointers are rejected because they can corrupt pointer metadata and leak kernel pointers.", + size, off); + bpf_diag_memory( + env, insn_idx, "stack spill corruption", reason, + "Write the full 8-byte spilled pointer slot, or use a separate stack slot for scalar data before overwriting only part of it."); return -EACCES; } @@ -3762,6 +3771,21 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env, return 0; } +static void bpf_diag_stack_read_uninit(struct bpf_verifier_env *env, int off, int i, + int size) +{ + const char *reason; + + reason = bpf_diag_fmt(env, + "This rejected read uses %d bytes at stack offset %d, but byte %d in that range is uninitialized on this path. " + "Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance.", + size, off, i); + bpf_diag_memory( + env, env->insn_idx, "uninitialized stack read", reason, + "Initialize every byte in the stack range before reading it, adjust the offset and size so the read covers only initialized bytes, " + "or load with CAP_PERFMON if uninitialized stack reads are intended."); +} + /* Read the stack at 'off' and put the results into the register indicated by * 'dst_regno'. It handles reg filling if the addressed stack slot is a * spilled reg. @@ -3851,6 +3875,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); + bpf_diag_stack_read_uninit(env, off, i, size); } return -EACCES; } @@ -3909,6 +3934,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); + bpf_diag_stack_read_uninit(env, off, i, size); } return -EACCES; } @@ -4001,11 +4027,19 @@ static int check_stack_read(struct bpf_verifier_env *env, * check_stack_read_fixed_off). */ if (dst_regno < 0 && var_off) { + const char *reason; char tn_buf[48]; tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", tn_buf, off, size); + reason = bpf_diag_fmt(env, + "The helper would access the stack through variable offset %s plus fixed offset %d and size %d. " + "Helper stack memory arguments require a constant stack offset and a precise initialized range.", + tn_buf, off, size); + bpf_diag_memory( + env, env->insn_idx, "variable stack access", reason, + "Use a fixed stack offset for helper memory arguments, or copy the needed bytes into a fixed stack slot first."); return -EACCES; } /* Variable offset is prohibited for unprivileged mode for simplicity @@ -4247,6 +4281,9 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ int off, int size, u32 mem_size, bool zero_size_allowed) { + const char *proof = ""; + const char *start; + s64 max_start, max_end; int err; /* We may have adjusted the register pointing to memory region, so we @@ -4265,14 +4302,28 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ reg_smin(reg) + off < 0)) { verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n", reg_arg_name(env, argno)); - return -EACCES; + err = -EACCES; + if (bpf_diag_enabled(env)) { + start = bpf_diag_fmt_s64_sum(env, reg_smin(reg), off); + proof = bpf_diag_fmt( + env, "the minimal bound for a memory access is a negative value: %s", + start); + } + goto report_error; } + err = __check_mem_access(env, reg, argno, reg_smin(reg) + off, size, mem_size, zero_size_allowed); if (err) { verbose(env, "%s min value is outside of the allowed memory range\n", reg_arg_name(env, argno)); - return err; + if (bpf_diag_enabled(env)) { + start = bpf_diag_fmt_s64_sum(env, reg_smin(reg), off); + proof = bpf_diag_fmt( + env, "the minimal bound for a memory access is %s and is outside of the object of size %u", + start, mem_size); + } + goto report_error; } /* If we haven't set a max value then we need to bail since we can't be @@ -4282,17 +4333,36 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ if (reg_umax(reg) >= BPF_MAX_VAR_OFF) { verbose(env, "%s unbounded memory access, make sure to bounds check any such access\n", reg_arg_name(env, argno)); - return -EACCES; + err = -EACCES; + if (bpf_diag_enabled(env)) + proof = bpf_diag_fmt( + env, "the maximal bound for a memory access is %llu and exceeds maximum allowed offset of %u", + reg_umax(reg), BPF_MAX_VAR_OFF); + goto report_error; } + err = __check_mem_access(env, reg, argno, reg_umax(reg) + off, size, mem_size, zero_size_allowed); if (err) { verbose(env, "%s max value is outside of the allowed memory range\n", reg_arg_name(env, argno)); - return err; + if (bpf_diag_enabled(env)) { + max_start = (s64)reg_umax(reg) + off; + max_end = max_start + size; + proof = bpf_diag_fmt( + env, "the maximal bound for a memory access is %lld: start %lld + access_size %d, beyond object_size %u", + max_end, max_start, size, mem_size); + } + goto report_error; } return 0; + +report_error: + bpf_diag_mem_bounds(env, env->insn_idx, reg_from_argno(argno), + reg_arg_name(env, argno), reg_type_str(env, reg->type), proof, + off, size, mem_size, reg); + return err; } static int __check_ptr_off_reg(struct bpf_verifier_env *env, -- cgit v1.2.3 From 5d5764627555f6beda39bc03af56428dec0c4582 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:46:05 +0200 Subject: bpf: Report Resource Lifetime reference leaks Augment selected Resource Lifetime Safety failures with structured diagnostics while preserving the existing verifier messages. Report unreleased references from check_reference_leak() using reference-scoped diagnostic history, and add state reports for dynptr, iterator, lock, and IRQ-flag lifetime misuse. IRQ restore mismatch and out-of-order diagnostics use IRQ context-scoped history when an IRQ-disabled region is active, so retained save/restore context is still visible after per-state history removal. Signed-off-by: Kumar Kartikeya Dwivedi Link: https://patch.msgid.link/20260815064612.378577-11-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/diagnostics.c | 91 +++++++++++++++++++++++++++++++++++++++++ kernel/bpf/diagnostics.h | 9 +++++ kernel/bpf/verifier.c | 103 +++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 195 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index 058574a1411e..5d20ea9e470e 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -18,6 +18,7 @@ #define REGISTER_TYPE_SAFETY "Register Type Safety" #define MEMORY_SAFETY "Memory Safety" +#define RESOURCE_LIFETIME_SAFETY "Resource Lifetime Safety" #define BPF_DIAG_TEXT_WIDTH 100 #define BPF_DIAG_TEXT_INDENT " " @@ -1735,6 +1736,96 @@ void bpf_diag_mem_bounds(struct bpf_verifier_env *env, u32 insn_idx, int regno, env, "Add or adjust a bounds check that proves offset + access_size stays within the object."); } +static const char *diag_lock_name(const struct bpf_reference_state *lock) +{ + switch (lock->type) { + case REF_TYPE_LOCK: + return "bpf_spin_lock"; + case REF_TYPE_RES_LOCK: + return "resource spin lock"; + case REF_TYPE_RES_LOCK_IRQ: + return "IRQ-saving resource spin lock"; + default: + return "lock"; + } +} + +static void diag_res_report(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason) +{ + bpf_diag_header(env, RESOURCE_LIFETIME_SAFETY, problem); + diag_reason(env, "%s", reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); +} + +void bpf_diag_res(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion) +{ + diag_res_report(env, insn_idx, problem, reason); + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_lock(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion, + const struct bpf_reference_state *active_lock) +{ + diag_res_report(env, insn_idx, problem, reason); + + if (active_lock) { + diag_section(env, "Active lock"); + bpf_diag_source(env, active_lock->insn_idx, "acquired", + "active %s has verifier identity %d", + diag_lock_name(active_lock), active_lock->id); + } + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_irq(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion, u32 depth) +{ + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT, + .ctx_kind = BPF_DIAG_CONTEXT_IRQ, + .ctx_depth = depth, + }; + + bpf_diag_header(env, RESOURCE_LIFETIME_SAFETY, problem); + diag_reason(env, "%s", reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); + + if (depth) + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_leak(struct bpf_verifier_env *env, u32 ref_id, u32 alloc_insn, u32 fail_insn) +{ + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REF, + .ref_id = ref_id, + }; + + bpf_diag_header(env, RESOURCE_LIFETIME_SAFETY, "unreleased resource"); + diag_reason( + env, "Owned resource (id=%u) was acquired at instruction %u and still needs to be released before this exit path.", + ref_id, alloc_insn); + + diag_section(env, "At"); + bpf_diag_source(env, fail_insn, "error", + "owned resource (id=%u) still needs release", ref_id); + + diag_print_history(env, &opts); + + diag_suggestion( + env, "Release or transfer ownership of the acquired resource on every path before the program exits."); +} + static const char *diag_var_offset(struct bpf_verifier_env *env, const struct bpf_diag_reg_snapshot *snapshot) { diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index b5feda71de3e..66dd2bb655b7 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -9,6 +9,7 @@ #include #include +struct bpf_reference_state; struct bpf_func_state; struct bpf_reg_state; struct bpf_verifier_env; @@ -68,6 +69,14 @@ void bpf_diag_memory(struct bpf_verifier_env *env, u32 insn_idx, const char *pro void bpf_diag_mem_bounds(struct bpf_verifier_env *env, u32 insn_idx, int regno, const char *reg_name, const char *type_name, const char *proof, int off, int size, u32 mem_size, const struct bpf_reg_state *reg); +void bpf_diag_res(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion); +void bpf_diag_lock(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion, + const struct bpf_reference_state *active_lock); +void bpf_diag_irq(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion, u32 depth); +void bpf_diag_leak(struct bpf_verifier_env *env, u32 ref_id, u32 alloc_insn, u32 fail_insn); void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true); void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 86c4212de5aa..186176973c3c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -816,6 +816,10 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, if (dynptr_type_referenced(state->stack[spi].spilled_ptr.dynptr.type) && dynptr_ref_cnt(env, state->stack[spi].spilled_ptr.parent_id) <= 1) { verbose(env, "cannot overwrite referenced dynptr\n"); + bpf_diag_res( + env, env->insn_idx, "referenced dynptr overwrite", + "This stack slot contains a dynptr that owns or protects a referenced resource. Overwriting the last dynptr for that resource would lose the verifier-tracked release path.", + "Release or clone the dynptr so another live dynptr still tracks the referenced resource before overwriting this stack slot."); return -EINVAL; } @@ -1098,9 +1102,19 @@ static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_r if (st->irq.kfunc_class != kfunc_class) { const char *flag_kfunc = st->irq.kfunc_class == IRQ_NATIVE_KFUNC ? "native" : "lock"; const char *used_kfunc = kfunc_class == IRQ_NATIVE_KFUNC ? "native" : "lock"; + const char *reason; verbose(env, "irq flag acquired by %s kfuncs cannot be restored with %s kfuncs\n", flag_kfunc, used_kfunc); + reason = bpf_diag_fmt(env, + "This IRQ flag was saved by %s IRQ kfuncs, but the restore call " + "belongs to the %s IRQ kfunc family. Save and restore operations " + "must use the same family.", + flag_kfunc, used_kfunc); + bpf_diag_irq(env, env->insn_idx, "IRQ flag restore mismatch", reason, + "Restore the flag with the matching IRQ restore kfunc for the save " + "operation that created it.", + bpf_diag_irq_depth(env->cur_state)); return -EINVAL; } @@ -1118,6 +1132,11 @@ static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_r verbose(env, "cannot restore irq state out of order, expected id=%d acquired at insn_idx=%d\n", env->cur_state->active_irq_id, insn_idx); + bpf_diag_irq(env, env->insn_idx, "IRQ flag restore out of order", + "IRQ-disabled regions must be restored in last-in, first-out order, " + "but this restore does not match the currently active IRQ flag.", + "Restore nested IRQ flags in the reverse order they were saved.", + bpf_diag_irq_depth(env->cur_state)); return err; } @@ -7183,6 +7202,7 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state bool is_lock = flags & PROCESS_SPIN_LOCK, is_res_lock = flags & PROCESS_RES_LOCK; const char *lock_str = is_res_lock ? "bpf_res_spin" : "bpf_spin"; struct bpf_verifier_state *cur = env->cur_state; + struct bpf_reference_state *lock; bool is_const = tnum_is_const(reg->var_off); bool is_irq = flags & PROCESS_LOCK_IRQ; u64 val = reg->var_off.value; @@ -7232,14 +7252,25 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state ptr = btf; if (!is_res_lock && cur->active_locks) { - if (find_lock_state(env->cur_state, REF_TYPE_LOCK, 0, NULL)) { + lock = find_lock_state(cur, REF_TYPE_LOCK, 0, NULL); + if (lock) { verbose(env, "Locking two bpf_spin_locks are not allowed\n"); + bpf_diag_lock( + env, env->insn_idx, "nested spin lock", + "This path already holds a bpf_spin_lock. The verifier allows only one regular BPF spin lock at a time.", + "Unlock the current bpf_spin_lock before taking another one.", lock); return -EINVAL; } } else if (is_res_lock && cur->active_locks) { - if (find_lock_state(env->cur_state, REF_TYPE_RES_LOCK | REF_TYPE_RES_LOCK_IRQ, reg->id, ptr)) { + lock = find_lock_state(cur, REF_TYPE_RES_LOCK | REF_TYPE_RES_LOCK_IRQ, + reg->id, ptr); + if (lock) { verbose(env, "Acquiring the same lock again, AA deadlock detected\n"); + bpf_diag_lock( + env, env->insn_idx, "recursive resource spin lock", + "This path already holds the same resource spin lock. Taking it again would deadlock.", + "Avoid reacquiring the same resource spin lock before it is unlocked.", lock); return -EINVAL; } } @@ -7266,6 +7297,10 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state if (!cur->active_locks) { verbose(env, "%s_unlock without taking a lock\n", lock_str); + bpf_diag_res( + env, env->insn_idx, "unlock without lock", + "This unlock operation has no matching active lock on the current path.", + "Take the matching lock before this unlock, or remove the unmatched unlock path."); return -EINVAL; } @@ -7275,16 +7310,35 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state type = REF_TYPE_RES_LOCK; else type = REF_TYPE_LOCK; - if (!find_lock_state(cur, type, reg->id, ptr)) { + + lock = find_lock_state(cur, type, reg->id, ptr); + if (!lock) { verbose(env, "%s_unlock of different lock\n", lock_str); + lock = find_lock_state(cur, REF_TYPE_LOCK_MASK, cur->active_lock_id, + cur->active_lock_ptr); + bpf_diag_lock( + env, env->insn_idx, "unlock of a different lock", + "This unlock does not match any active lock with the same tracked identity on the current path.", + "Unlock the same lock object that was most recently acquired.", lock); return -EINVAL; } if (reg->id != cur->active_lock_id || ptr != cur->active_lock_ptr) { verbose(env, "%s_unlock cannot be out of order\n", lock_str); + lock = find_lock_state(cur, REF_TYPE_LOCK_MASK, cur->active_lock_id, + cur->active_lock_ptr); + bpf_diag_lock( + env, env->insn_idx, "unlock out of order", + "Locks must be released in last-in, first-out order, but this unlock does not match the currently active lock.", + "Release nested locks in the reverse order they were acquired.", lock); return -EINVAL; } if (release_lock_state(env, type, reg->id, ptr)) { verbose(env, "%s_unlock of different lock\n", lock_str); + bpf_diag_lock( + env, env->insn_idx, "unlock of a different lock", + "The verifier could not release a lock state matching this unlock operation.", + "Pass the same lock object and lock kind that were used for the matching lock operation.", + lock); return -EINVAL; } if (!in_rcu_cs(env)) @@ -7462,6 +7516,10 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat if (!is_dynptr_reg_valid_uninit(env, reg)) { verbose(env, "Dynptr has to be an uninitialized dynptr\n"); + bpf_diag_res( + env, insn_idx, "dynptr is already initialized", + "This kfunc constructs a dynptr and requires an uninitialized dynptr stack slot, but the selected slot already holds dynptr state.", + "Use a fresh stack dynptr slot, or release/destroy the existing dynptr before reusing the slot."); return -EINVAL; } @@ -7478,21 +7536,29 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */ if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) { verbose(env, "CONST_PTR_TO_DYNPTR cannot be released\n"); + bpf_diag_res( + env, insn_idx, "const dynptr release", + "This release operation was given a const dynptr. Const dynptr values are verifier-provided views and cannot be released by the program.", + "Release only mutable dynptrs that the program initialized or reserved."); return -EINVAL; } if (!is_dynptr_reg_valid_init(env, reg)) { verbose(env, "Expected an initialized dynptr as %s\n", reg_arg_name(env, argno)); + bpf_diag_res( + env, insn_idx, "uninitialized dynptr use", + "This operation requires an initialized dynptr, but the stack slot does not currently hold a valid dynptr on this path.", + "Initialize the dynptr on every path before this call, and avoid overwriting or releasing it before this use."); return -EINVAL; } /* Fold modifiers (in this case, OBJ_RELEASE) when checking expected type */ if (!is_dynptr_type_expected(env, reg, arg_type & ~OBJ_RELEASE)) { - verbose(env, - "Expected a dynptr of type %s as %s\n", - dynptr_type_str(arg_to_dynptr_type(arg_type)), - reg_arg_name(env, argno)); + enum bpf_dynptr_type expected_type = arg_to_dynptr_type(arg_type); + + verbose(env, "Expected a dynptr of type %s as %s\n", + dynptr_type_str(expected_type), reg_arg_name(env, argno)); return -EINVAL; } @@ -7579,6 +7645,10 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) { verbose(env, "expected uninitialized iter_%s as %s\n", iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno)); + bpf_diag_res( + env, insn_idx, "iterator is already initialized", + "Iterator creation requires an uninitialized iterator stack object, but this stack range already contains iterator state.", + "Use a fresh iterator stack slot, or destroy the existing iterator before reusing the slot."); return -EINVAL; } @@ -7603,6 +7673,10 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * case -EINVAL: verbose(env, "expected an initialized iter_%s as %s\n", iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno)); + bpf_diag_res( + env, insn_idx, "uninitialized iterator use", + "This iterator operation requires an initialized iterator state object, but the stack range does not contain a live iterator on this path.", + "Call the matching iterator new kfunc on every path before calling next or destroy, and do not destroy the iterator before this use."); return err; case -EPROTO: verbose(env, "expected an RCU CS when using %s\n", meta->func_name); @@ -9475,7 +9549,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, if (ret) return ret; - ret = process_dynptr_func(env, reg, argno, -1, arg->arg_type, &ref_obj, NULL); + ret = process_dynptr_func(env, reg, argno, env->insn_idx, arg->arg_type, + &ref_obj, NULL); if (ret) return ret; } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { @@ -10273,6 +10348,7 @@ static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exi continue; verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", state->refs[i].id, state->refs[i].insn_idx); + bpf_diag_leak(env, state->refs[i].id, state->refs[i].insn_idx, env->insn_idx); refs_lingering = true; } return refs_lingering ? -EINVAL : 0; @@ -11823,6 +11899,12 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * if (!is_irq_flag_reg_valid_uninit(env, reg)) { verbose(env, "expected uninitialized irq flag as %s\n", reg_arg_name(env, argno)); + bpf_diag_res(env, env->insn_idx, "IRQ flag is already initialized", + "Saving IRQ state requires an uninitialized stack slot for " + "the IRQ flag, but this slot already contains tracked IRQ " + "flag state.", + "Use a fresh stack slot for this save operation, or restore " + "the existing IRQ flag before reusing the slot."); return -EINVAL; } @@ -11839,6 +11921,11 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * if (err) { verbose(env, "expected an initialized irq flag as %s\n", reg_arg_name(env, argno)); + bpf_diag_res(env, env->insn_idx, "uninitialized IRQ flag restore", + "Restoring IRQ state requires a stack slot that was " + "initialized by a matching IRQ save operation on this path.", + "Pass the same stack slot that was previously initialized by " + "the matching IRQ save kfunc."); return err; } -- cgit v1.2.3 From 66e2727395dd994e26ace31d331013acc9ce8f2e Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:46:06 +0200 Subject: bpf: Report Call Type Safety argument errors Augment selected helper and kfunc argument-contract failures with Call Type Safety reports. Keep the existing terse verifier messages and add reason, source context, causal register or stack-argument history, and targeted suggestions. Cover helper register-type mismatch, helper and kfunc non-NULL pointer requirements, release-helper ownership requirements, scalar and constant kfunc arguments, trusted and RCU pointer contracts, kfunc memory arguments, memory/length pairs, refcounted kptrs, constant strings, and IRQ flag stack arguments. Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260815064612.378577-12-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/diagnostics.c | 46 ++++++ kernel/bpf/diagnostics.h | 3 + kernel/bpf/verifier.c | 394 ++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 387 insertions(+), 56 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index 5d20ea9e470e..99784d465881 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -19,6 +19,7 @@ #define REGISTER_TYPE_SAFETY "Register Type Safety" #define MEMORY_SAFETY "Memory Safety" #define RESOURCE_LIFETIME_SAFETY "Resource Lifetime Safety" +#define CALL_TYPE_SAFETY "Call Type Safety" #define BPF_DIAG_TEXT_WIDTH 100 #define BPF_DIAG_TEXT_INDENT " " @@ -983,6 +984,51 @@ static const char *diag_arg_ordinal(int argno) } } +void bpf_diag_call_type(struct bpf_verifier_env *env, u32 insn_idx, int argno, int regno, + int stack_arg_slot, const char *call_name, const char *arg_name, + const char *reason, const char *suggestion) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + }; + const char *ordinal = diag_arg_ordinal(argno); + const char *arg_desc; + bool print_history = true; + + if (regno >= 0) { + opts.scope = BPF_DIAG_HISTORY_SCOPE_REG; + opts.regno = regno; + } else if (stack_arg_slot >= 0) { + opts.scope = BPF_DIAG_HISTORY_SCOPE_STACK_ARG; + opts.stack_arg_slot = stack_arg_slot; + } else { + print_history = false; + } + + if (ordinal && arg_name) + arg_desc = bpf_diag_fmt(env, "%s argument (%s)", ordinal, arg_name); + else if (ordinal) + arg_desc = bpf_diag_fmt(env, "%s argument", ordinal); + else if (arg_name) + arg_desc = bpf_diag_fmt(env, "argument %s", arg_name); + else + arg_desc = "argument"; + + bpf_diag_header(env, CALL_TYPE_SAFETY, "invalid call argument"); + diag_reason(env, "The %s to %s does not satisfy the verifier contract: %s.", + arg_desc, call_name, reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "invalid %s for %s", arg_desc, call_name); + + if (print_history) + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + void bpf_diag_invalid_deref(struct bpf_verifier_env *env, u32 insn_idx, int regno, const char *reg_name, const struct bpf_reg_state *reg, enum bpf_diag_invalid_deref_kind kind, s64 offset) diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index 66dd2bb655b7..4b85a7ad2019 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -77,6 +77,9 @@ void bpf_diag_lock(struct bpf_verifier_env *env, u32 insn_idx, const char *probl void bpf_diag_irq(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, const char *reason, const char *suggestion, u32 depth); void bpf_diag_leak(struct bpf_verifier_env *env, u32 ref_id, u32 alloc_insn, u32 fail_insn); +void bpf_diag_call_type(struct bpf_verifier_env *env, u32 insn_idx, int argno, int regno, + int stack_arg_slot, const char *call_name, const char *arg_name, + const char *reason, const char *suggestion); void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true); void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 186176973c3c..30e48d4b02f9 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -890,26 +890,29 @@ static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_re return true; } -static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - enum bpf_arg_type arg_type) +static enum bpf_dynptr_type dynptr_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg) { - struct bpf_func_state *state = bpf_func(env, reg); - enum bpf_dynptr_type dynptr_type; + struct bpf_func_state *state; int spi; + if (reg->type == CONST_PTR_TO_DYNPTR) + return reg->dynptr.type; + + spi = dynptr_get_spi(env, reg); + if (spi < 0) + return BPF_DYNPTR_TYPE_INVALID; + state = bpf_func(env, reg); + return state->stack[spi].spilled_ptr.dynptr.type; +} + +static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + enum bpf_arg_type arg_type) +{ /* ARG_PTR_TO_DYNPTR takes any type of dynptr */ if (arg_type == ARG_PTR_TO_DYNPTR) return true; - dynptr_type = arg_to_dynptr_type(arg_type); - if (reg->type == CONST_PTR_TO_DYNPTR) { - return reg->dynptr.type == dynptr_type; - } else { - spi = dynptr_get_spi(env, reg); - if (spi < 0) - return false; - return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type; - } + return dynptr_reg_type(env, reg) == arg_to_dynptr_type(arg_type); } static void __mark_reg_known_zero(struct bpf_reg_state *reg); @@ -6923,14 +6926,17 @@ mark: return 0; } -static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - int access_size, enum bpf_access_type access_type, - bool zero_size_allowed, - struct bpf_call_arg_meta *meta) +static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, int access_size, + enum bpf_access_type access_type, bool zero_size_allowed, + struct bpf_call_arg_meta *meta, bool *known_memory) { struct bpf_reg_state *regs = cur_regs(env); u32 *max_access; + if (known_memory) + *known_memory = true; + switch (base_type(reg->type)) { case PTR_TO_PACKET: case PTR_TO_PACKET_META: @@ -7000,6 +7006,8 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ if (zero_size_allowed && access_size == 0 && bpf_register_is_null(reg)) return 0; + if (known_memory && base_type(reg->type) != PTR_TO_CTX) + *known_memory = false; verbose(env, "%s type=%s ", reg_arg_name(env, argno), reg_type_str(env, reg->type)); @@ -7008,6 +7016,12 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ } } +enum bpf_mem_size_failure { + BPF_MEM_SIZE_FAIL_NONE, + BPF_MEM_SIZE_FAIL_MEMORY, + BPF_MEM_SIZE_FAIL_SIZE, +}; + /* verify arguments to helpers or kfuncs consisting of a pointer and an access * size. * @@ -7018,10 +7032,14 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *size_reg, argno_t mem_argno, argno_t size_argno, u32 access_type, bool zero_size_allowed, - struct bpf_call_arg_meta *meta) + struct bpf_call_arg_meta *meta, + enum bpf_mem_size_failure *failure) { int err = 0; + if (failure) + *failure = BPF_MEM_SIZE_FAIL_NONE; + /* This is used to refine r0 return value bounds for helpers * that enforce this value as an upper bound on return values. * See do_refine_retval_range() for helpers that can refine @@ -7043,27 +7061,32 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, if (reg_smin(size_reg) < 0) { verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", reg_arg_name(env, size_argno)); - return -EACCES; + err = -EACCES; + goto size_error; } if (reg_umin(size_reg) == 0 && !zero_size_allowed) { verbose(env, "%s invalid zero-sized read: u64=[%lld,%lld]\n", reg_arg_name(env, size_argno), reg_umin(size_reg), reg_umax(size_reg)); - return -EACCES; + err = -EACCES; + goto size_error; } if (reg_umax(size_reg) >= BPF_MAX_VAR_SIZ) { verbose(env, "%s unbounded memory access, use 'var &= const' or 'if (var < const)'\n", reg_arg_name(env, size_argno)); - return -EACCES; + err = -EACCES; + goto size_error; } if (access_type & BPF_READ) err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), - BPF_READ, zero_size_allowed, meta); + BPF_READ, zero_size_allowed, meta, NULL); if (!err && access_type & BPF_WRITE) err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), - BPF_WRITE, zero_size_allowed, meta); + BPF_WRITE, zero_size_allowed, meta, NULL); + if (err && failure) + *failure = BPF_MEM_SIZE_FAIL_MEMORY; if (!err) { int regno = reg_from_argno(size_argno); @@ -7075,16 +7098,23 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, } return err; + +size_error: + if (failure) + *failure = BPF_MEM_SIZE_FAIL_SIZE; + return err; } static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, u32 mem_size, enum bpf_access_type access_type, - struct bpf_call_arg_meta *meta) + struct bpf_call_arg_meta *meta, bool *known_memory) { int size, err = 0; if (bpf_register_is_null(reg)) return 0; + if (known_memory) + *known_memory = true; if (mem_size > S32_MAX) { verbose(env, "%s memory size %u is too large\n", @@ -7099,9 +7129,11 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg size = (!meta && base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size; if (access_type & BPF_READ) - err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta); + err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta, + known_memory); if (!err && (access_type & BPF_WRITE)) - err = check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, meta); + err = check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, meta, + known_memory); return err; } @@ -7461,6 +7493,12 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno, return 0; } +static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno, + const char *call_name, const char *reason, const char *suggestion); +__printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u32 insn_idx, + argno_t argno, const char *call_name, + const char *suggestion, const char *fmt, ...); + /* * Validate dynptr arguments for helper, kfunc and subprog. * @@ -7485,7 +7523,8 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno, * and checked dynamically during runtime. */ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - argno_t argno, int insn_idx, enum bpf_arg_type arg_type, + argno_t argno, int insn_idx, const char *call_name, + enum bpf_arg_type arg_type, struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr) { int spi, err = 0; @@ -7494,6 +7533,11 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat verbose(env, "%s expected pointer to stack or const struct bpf_dynptr\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt( + env, insn_idx, argno, call_name, + "Pass the address of a stack dynptr object, or use a const dynptr pointer returned by the verifier-supported path.", + "a dynptr argument must be a pointer to a dynptr stack slot or a verifier-provided const struct bpf_dynptr, but %s is %s", + reg_arg_name(env, argno), bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } @@ -7556,9 +7600,15 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat /* Fold modifiers (in this case, OBJ_RELEASE) when checking expected type */ if (!is_dynptr_type_expected(env, reg, arg_type & ~OBJ_RELEASE)) { enum bpf_dynptr_type expected_type = arg_to_dynptr_type(arg_type); + enum bpf_dynptr_type actual_type = dynptr_reg_type(env, reg); verbose(env, "Expected a dynptr of type %s as %s\n", dynptr_type_str(expected_type), reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt( + env, insn_idx, argno, call_name, + "Use a dynptr constructor that matches this operation, or call an operation that accepts the dynptr's current type.", + "the dynptr is initialized with backing object type %s, but this operation expects dynptr type %s", + dynptr_type_str(actual_type), dynptr_type_str(expected_type)); return -EINVAL; } @@ -7622,6 +7672,11 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * if (reg->type != PTR_TO_STACK) { verbose(env, "%s expected pointer to an iterator on stack\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt( + env, insn_idx, argno, meta->func_name, + "Pass the address of a stack iterator object for iterator new, next, and destroy calls.", + "iterator state must live in verifier-tracked stack memory, but %s is %s", + reg_arg_name(env, argno), bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } @@ -7635,6 +7690,10 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * if (btf_id < 0) { verbose(env, "expected valid iter pointer as %s\n", reg_arg_name(env, argno)); + bpf_diag_call_arg( + env, insn_idx, argno, meta->func_name, + "the kfunc expects a recognized iterator state pointer, but this argument does not match a valid iterator type", + "Pass the exact iterator state type expected by this kfunc."); return -EINVAL; } t = btf_type_by_id(meta->btf, btf_id); @@ -8099,13 +8158,70 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_DYNPTR] = &dynptr_types, }; +static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno, + const char *call_name, const char *reason, + const char *suggestion) +{ + int arg = arg_from_argno(argno); + int regno = reg_from_argno(argno); + int stack_slot = -1; + + if (arg < 0 && regno >= BPF_REG_1 && regno <= BPF_REG_5) + arg = regno; + if (arg > MAX_BPF_FUNC_REG_ARGS) + stack_slot = arg - MAX_BPF_FUNC_REG_ARGS - 1; + + bpf_diag_call_type(env, insn_idx, arg, regno, stack_slot, + call_name && *call_name ? call_name : "call", + reg_arg_name(env, argno), reason, suggestion); +} + +static const char *bpf_diag_arg_name(struct bpf_verifier_env *env, argno_t argno) +{ + return bpf_diag_fmt(env, "%s", reg_arg_name(env, argno)); +} + +__printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u32 insn_idx, + argno_t argno, const char *call_name, + const char *suggestion, const char *fmt, ...) +{ + const char *reason; + va_list args; + + va_start(args, fmt); + reason = bpf_diag_vfmt(env, fmt, args); + va_end(args); + + bpf_diag_call_arg(env, insn_idx, argno, call_name, reason, suggestion); +} + +static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env, + const enum bpf_reg_type *types, int count) +{ + size_t len = 0, size = 1; + char *buf; + int i; + + for (i = 0; i < count; i++) + size += strlen(reg_type_str(env, types[i])) + (i ? 2 : 0); + + buf = bpf_diag_fmt_buf(env, size); + if (!buf) + return ""; + + for (i = 0; i < count; i++) + len += scnprintf(buf + len, size - len, "%s%s", i ? ", " : "", + reg_type_str(env, types[i])); + return buf; +} + static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type, - const u32 *arg_btf_id, - struct bpf_call_arg_meta *meta) + enum bpf_arg_type arg_type, const u32 *arg_btf_id, + struct bpf_call_arg_meta *meta, const char *call_name) { enum bpf_reg_type expected, type = reg->type; const struct bpf_reg_types *compatible; + const char *actual, *accepted; int i, j, err; compatible = compatible_reg_types[base_type(arg_type)]; @@ -8152,6 +8268,12 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re for (j = 0; j + 1 < i; j++) verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); + actual = bpf_diag_fmt(env, "%s", reg_type_str(env, reg->type)); + accepted = bpf_diag_expected_reg_types(env, compatible->types, i); + bpf_diag_call_arg_fmt(env, env->insn_idx, argno, call_name, + "Pass a value with one of the accepted pointer or scalar types for this call.", + "it has type %s, but this argument accepts %s", + actual, accepted); return -EACCES; found: @@ -8188,6 +8310,10 @@ found: (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) { verbose(env, "Possibly NULL pointer passed to helper %s\n", reg_arg_name(env, argno)); + bpf_diag_call_arg( + env, env->insn_idx, argno, call_name, + "the pointer may be NULL, but this call requires a non-NULL pointer", + "Add a NULL check and make the call only on the non-NULL path."); return -EACCES; } @@ -8574,7 +8700,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) arg_btf_id = fn->arg_btf_id[arg]; - err = check_reg_type(env, reg, argno, arg_type, arg_btf_id, meta); + err = check_reg_type(env, reg, argno, arg_type, arg_btf_id, meta, + func_id_name(meta->func_id)); if (err) return err; @@ -8587,6 +8714,10 @@ skip_type_check: !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { verbose(env, "release helper %s expects referenced PTR_TO_BTF_ID passed to %s\n", func_id_name(meta->func_id), reg_arg_name(env, argno)); + bpf_diag_call_arg( + env, insn_idx, argno, func_id_name(meta->func_id), + "release helpers require a value that owns a live resource returned by a matching acquire helper", + "Pass the resource-owning pointer returned by the matching acquire helper, and avoid calling the release helper after ownership has already been transferred or released."); return -EINVAL; } @@ -8615,7 +8746,8 @@ skip_type_check: return -EFAULT; } key_size = meta->map.ptr->key_size; - err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, NULL); + err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, NULL, + NULL); if (err) return err; if (can_elide_value_nullness(meta->map.ptr)) { @@ -8652,7 +8784,7 @@ skip_type_check: err = check_helper_mem_access(env, reg, argno, meta->map.ptr->value_size, arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, - false, meta); + false, meta, NULL); break; case ARG_PTR_TO_PERCPU_BTF_ID: if (!reg->btf_id) { @@ -8694,7 +8826,7 @@ skip_type_check: */ if (arg_type & MEM_FIXED_SIZE) { err = check_mem_reg(env, reg, argno_from_reg(regno), fn->arg_size[arg], - arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta); + arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta, NULL); if (err) return err; if (arg_type & MEM_ALIGNED) @@ -8705,17 +8837,17 @@ skip_type_check: err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), argno, fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, - false, meta); + false, meta, NULL); break; case ARG_MEM_SIZE_OR_ZERO: err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), argno, fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, - true, meta); + true, meta, NULL); break; case ARG_PTR_TO_DYNPTR: - err = process_dynptr_func(env, reg, argno, insn_idx, arg_type, &meta->ref_obj, - &meta->dynptr); + err = process_dynptr_func(env, reg, argno, insn_idx, func_id_name(meta->func_id), + arg_type, &meta->ref_obj, &meta->dynptr); if (err) return err; break; @@ -9523,7 +9655,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE); if (ret < 0) return ret; - if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL)) + if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL, + NULL)) return -EINVAL; if (!(arg->arg_type & PTR_MAYBE_NULL) && (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { @@ -9549,7 +9682,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, if (ret) return ret; - ret = process_dynptr_func(env, reg, argno, env->insn_idx, arg->arg_type, + ret = process_dynptr_func(env, reg, argno, env->insn_idx, + bpf_subprog_name(env, subprog), arg->arg_type, &ref_obj, NULL); if (ret) return ret; @@ -9561,7 +9695,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, continue; memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */ - err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta); + err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta, + bpf_subprog_name(env, subprog)); err = err ?: check_func_arg_reg_off(env, reg, argno, arg->arg_type); if (err) return err; @@ -12392,7 +12527,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me argno_t argno = argno_from_arg(i + 1); int regno = reg_from_argno(argno); bool btf_id_fixed_off_ok = true; - u32 ref_id, type_size; + u32 ref_id = args[i].type, type_size; int kf_arg_type = meta->fn->arg_type[i]; if (is_kfunc_arg_prog_aux(btf, &args[i])) { @@ -12416,29 +12551,43 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me t = btf_type_skip_modifiers(btf, args[i].type, NULL); - if (btf_type_is_ptr(t) && (bpf_register_is_null(reg) || type_may_be_null(reg->type)) && + if (btf_type_is_ptr(t)) { + ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); + ref_tname = btf_name_by_offset(btf, ref_t->name_off); + } + + if (btf_type_is_ptr(t) && + (bpf_register_is_null(reg) || type_may_be_null(reg->type)) && !type_may_be_null(kf_arg_type)) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); verbose(env, "Possibly NULL pointer passed to trusted %s\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Add a NULL check and call the kfunc only on the non-NULL path.", + "the pointer may be NULL, but this kfunc requires a non-NULL pointer to %s", + expected_type); return -EACCES; } if (regno == meta->release_regno && !is_kfunc_arg_dynptr(meta->btf, &args[i]) && !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); verbose(env, "release kfunc %s expects referenced PTR_TO_BTF_ID passed to %s\n", func_name, reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass the resource-owning pointer returned by the matching acquire kfunc, and avoid calling the release kfunc after ownership has already been transferred or released.", + "release kfuncs require a resource-owning value of type %s returned by a matching acquire kfunc", + expected_type); return -EINVAL; } if (reg_is_referenced(env, reg)) update_ref_obj(&meta->ref_obj, reg); - if (btf_type_is_ptr(t)) { - ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); - } - - if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) continue; @@ -12498,35 +12647,67 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me case KF_ARG_CONST: if (reg->type != SCALAR_VALUE) { verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an integer scalar value for this argument, not a pointer or resource object.", + "the kfunc expects an integer scalar, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } ret = process_const_arg(env, reg, argno, meta); - if (ret < 0) + if (ret < 0) { + if (ret == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a compile-time constant or a value the verifier can prove is constant at this call.", + "the kfunc requires this scalar argument to be a verifier-known constant, but %s is variable on this path", + reg_arg_name(env, argno)); return ret; + } break; case KF_ARG_ANYTHING: if (reg->type != SCALAR_VALUE) { verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an integer scalar value for this argument, not a pointer or resource object.", + "the kfunc expects an integer scalar, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } break; case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: if (reg->type != SCALAR_VALUE) { verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an integer scalar value for this argument, not a pointer or resource object.", + "the kfunc expects an integer scalar, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) meta->r0_rdonly = true; ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); - if (ret < 0) + if (ret < 0) { + if (ret == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a verifier-known constant size for this kfunc buffer argument.", + "the kfunc uses this argument as a return-buffer size, but %s is invalid or variable on this path", + reg_arg_name(env, argno)); return ret; + } break; case KF_ARG_PTR_TO_CTX: if (reg->type != PTR_TO_CTX) { verbose(env, "%s expected pointer to ctx, but got %s\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass the original program context pointer or preserve it before modifying registers.", + "the kfunc expects a context pointer, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } @@ -12560,10 +12741,19 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me } else { verbose(env, "%s expected pointer to allocated object\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a pointer returned by the matching BPF object allocation path.", + "the kfunc expects an allocated object pointer, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } if (!reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass the owned object pointer before it is released or transferred.", + "the allocated object pointer in %s must still carry verifier-tracked ownership, but this pointer no longer owns a live resource", + reg_arg_name(env, argno)); return -EINVAL; } if (meta->btf == btf_vmlinux) { @@ -12600,8 +12790,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); } - ret = process_dynptr_func(env, reg, argno, insn_idx, dynptr_arg_type, - &meta->ref_obj, &meta->dynptr); + ret = process_dynptr_func(env, reg, argno, insn_idx, func_name, + dynptr_arg_type, &meta->ref_obj, &meta->dynptr); if (ret < 0) return ret; break; @@ -12716,13 +12906,31 @@ check_ok: if (!is_trusted_reg(env, reg) || bpf_type_has_unsafe_modifiers(reg->type)) { if (!is_kfunc_rcu(meta)) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); verbose(env, "%s must be referenced or trusted\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a pointer acquired from a verifier-tracked source, or call this kfunc only inside the required protection if it accepts RCU pointers.", + "the kfunc requires a trusted or resource-owning pointer to %s, but %s is %s", + expected_type, + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } if (!is_rcu_reg(reg)) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); verbose(env, "%s must be a rcu pointer\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Use this kfunc with a pointer that is valid in an RCU read lock region.", + "the kfunc requires an RCU-protected pointer to %s, but %s is %s", + expected_type, + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } } @@ -12735,6 +12943,7 @@ check_ok: if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); + const char *expected_type; verbose(env, "%s is %s expected %s %s", reg_arg_name(env, argno), reg_type_str(env, reg->type), @@ -12742,6 +12951,12 @@ check_ok: if (reg2btf_type != NOT_INIT) verbose(env, " or %s", reg_type_str(env, reg2btf_type)); verbose(env, "\n"); + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a verifier-tracked pointer to the expected kernel object type, not a pointer to stack storage or another memory buffer.", + "the kfunc expects a pointer to %s, but this argument is %s and cannot be used as that kernel object pointer", + expected_type, + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } @@ -12753,6 +12968,8 @@ check_ok: fallthrough; case KF_ARG_PTR_TO_MEM: if (kf_arg_type & MEM_FIXED_SIZE) { + bool known_memory; + resolve_ret = btf_resolve_size(btf, ref_t, &type_size); if (IS_ERR(resolve_ret)) { verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", @@ -12760,9 +12977,28 @@ check_ok: ref_tname, PTR_ERR(resolve_ret)); return -EINVAL; } - ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, meta); - if (ret < 0) + ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, + meta, &known_memory); + if (ret < 0) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + if (known_memory) + bpf_diag_call_arg_fmt( + env, insn_idx, argno, func_name, + "Pass memory with at least the required number of accessible bytes and suitable read and write access.", + "the kfunc expects %u bytes of memory for %s, but the verifier cannot prove that %s provides a readable and writable range of that size", + type_size, expected_type, + bpf_diag_reg_type_plain(env, reg->type)); + else + bpf_diag_call_arg_fmt( + env, insn_idx, argno, func_name, + "Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.", + "the kfunc expects %u bytes of memory for %s, but it is %s and not verifier-known memory", + type_size, expected_type, + bpf_diag_reg_type_plain(env, reg->type)); return ret; + } } break; case KF_ARG_CONST_MEM_SIZE: @@ -12775,9 +13011,15 @@ check_ok: struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, i - 1); struct bpf_reg_state *size_reg = reg; argno_t buff_argno = argno_from_arg(i); + enum bpf_mem_size_failure failure; if (reg->type != SCALAR_VALUE) { verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an integer scalar length for this memory argument.", + "the kfunc expects a scalar memory size, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } @@ -12785,11 +13027,34 @@ check_ok: break; ret = check_mem_size_reg(env, buff_reg, size_reg, buff_argno, argno, - BPF_READ | BPF_WRITE, true, meta); + BPF_READ | BPF_WRITE, true, meta, &failure); if (ret < 0) { + const char *buff_arg, *size_arg; + + buff_arg = bpf_diag_arg_name(env, buff_argno); + size_arg = bpf_diag_arg_name(env, argno); verbose(env, "%s and ", reg_arg_name(env, buff_argno)); verbose(env, "%s memory, len pair leads to invalid memory access\n", reg_arg_name(env, argno)); + if (failure == BPF_MEM_SIZE_FAIL_MEMORY) { + bpf_diag_call_arg_fmt(env, insn_idx, buff_argno, func_name, + "Pass a stack, map, context, or other verifier-known memory pointer, and keep the paired length within that object.", + "it is the memory pointer in a memory/length pair with %s, but %s does not describe verifier-readable memory for the requested length", + size_arg, buff_arg); + } else if (failure == BPF_MEM_SIZE_FAIL_SIZE) { + if (reg_smin(size_reg) < 0) + bpf_diag_call_arg_fmt( + env, insn_idx, argno, func_name, + "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", + "the memory size in %s may be negative because its signed minimum is %lld", + size_arg, reg_smin(size_reg)); + else + bpf_diag_call_arg_fmt( + env, insn_idx, argno, func_name, + "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", + "the memory size in %s may reach %llu bytes, but variable memory accesses must stay below %u bytes", + size_arg, reg_umax(size_reg), BPF_MAX_VAR_SIZ); + } return ret; } break; @@ -12803,8 +13068,15 @@ check_ok: break; case KF_ARG_PTR_TO_REFCOUNTED_KPTR: if (!type_is_ptr_alloc_obj(reg->type)) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); verbose(env, "%s is neither owning or non-owning ref\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a pointer returned by the matching BPF object allocation or lookup operation for this kfunc.", + "the kfunc expects a pointer to BPF-managed refcounted object type %s, but this argument is not such an object pointer", + expected_type); return -EINVAL; } if (!type_is_non_owning_ref(reg->type)) @@ -12829,6 +13101,11 @@ check_ok: if (reg->type != PTR_TO_MAP_VALUE) { verbose(env, "%s doesn't point to a const string\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a constant string pointer that the verifier recognizes, such as a string stored in a read-only map value.", + "the kfunc expects a pointer to a constant string stored in verifier-known memory, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } ret = check_arg_const_str(env, reg, argno); @@ -12869,6 +13146,11 @@ check_ok: if (reg->type != PTR_TO_STACK) { verbose(env, "%s doesn't point to an irq flag on stack\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass the same stack slot used by bpf_local_irq_save() or bpf_res_spin_lock_irqsave().", + "the kfunc expects a stack pointer to an IRQ flag slot, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } ret = process_irq_flag(env, reg, argno, meta); -- cgit v1.2.3 From 99a6a288a82bf00ba0b01e72bf85164e848d1d18 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:46:07 +0200 Subject: bpf: Report Execution Context Safety errors Augment selected sleepability and critical-section failures with Execution Context Safety reports. Keep the existing verifier messages and add source context, path history, and suggestions tied to the active context. Use the context history recorded earlier to anchor causal paths to lock, IRQ, RCU, and preempt regions instead of unrelated register updates. Cover global calls while holding a lock, sleepable global function calls, sleepable helpers, sleepable kfunc calls from disallowed contexts, operations that exit while a context is still active, and unmatched context exits. Signed-off-by: Kumar Kartikeya Dwivedi Link: https://patch.msgid.link/20260815064612.378577-13-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/diagnostics.c | 165 ++++++++++++++++++++++++++++++++++++++++++++++- kernel/bpf/diagnostics.h | 9 +++ kernel/bpf/verifier.c | 44 +++++++++++++ 3 files changed, 217 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index 99784d465881..c69160f656e9 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -20,6 +20,7 @@ #define MEMORY_SAFETY "Memory Safety" #define RESOURCE_LIFETIME_SAFETY "Resource Lifetime Safety" #define CALL_TYPE_SAFETY "Call Type Safety" +#define EXECUTION_CONTEXT_SAFETY "Execution Context Safety" #define BPF_DIAG_TEXT_WIDTH 100 #define BPF_DIAG_TEXT_INDENT " " @@ -163,6 +164,7 @@ static void diag_print_history(struct bpf_verifier_env *env, const struct bpf_diag_history_opts *opts); static bool diag_target_matches(const struct bpf_diag_mod_target *event_target, const struct bpf_diag_mod_target *target); +static const char *diag_context_name(enum bpf_diag_context_kind kind); struct disasm_line { char text[DISASM_LINE_LEN]; int idx; @@ -1029,6 +1031,167 @@ void bpf_diag_call_type(struct bpf_verifier_env *env, u32 insn_idx, int argno, i diag_suggestion(env, "%s", suggestion); } +static const char *diag_context_constraint(enum bpf_diag_context_kind kind) +{ + switch (kind) { + case BPF_DIAG_CONTEXT_RCU: + return "RCU read-side critical sections cannot call operations that may sleep"; + case BPF_DIAG_CONTEXT_PREEMPT: + return "preemption-disabled code cannot call operations that may sleep"; + case BPF_DIAG_CONTEXT_IRQ: + return "IRQ-disabled code cannot call operations that may sleep"; + case BPF_DIAG_CONTEXT_LOCK: + return "code holding a BPF spin lock cannot call operations that may sleep"; + case BPF_DIAG_CONTEXT_NONE: + default: + return NULL; + } +} + +static const char *diag_active_context(struct bpf_verifier_env *env, u32 depth, + const char *context) +{ + if (depth == 1) + return bpf_diag_fmt(env, "an active %s (depth 1)", context); + return bpf_diag_fmt(env, "%u active %ss (depth %u)", depth, context, depth); +} + +static u32 diag_context_depth(struct bpf_verifier_env *env, enum bpf_diag_context_kind kind) +{ + switch (kind) { + case BPF_DIAG_CONTEXT_RCU: + return env->cur_state->active_rcu_locks; + case BPF_DIAG_CONTEXT_PREEMPT: + return env->cur_state->active_preempt_locks; + case BPF_DIAG_CONTEXT_IRQ: + return bpf_diag_irq_depth(env->cur_state); + case BPF_DIAG_CONTEXT_LOCK: + return env->cur_state->active_locks; + case BPF_DIAG_CONTEXT_NONE: + default: + return 0; + } +} + +void bpf_diag_ctx_forbidden(struct bpf_verifier_env *env, u32 insn_idx, + const char *operation, const char *suggestion) +{ + struct bpf_diag_history_opts opts; + enum bpf_diag_context_kind ctx_kind; + const char *constraint, *context; + u32 depth; + + if (env->cur_state->active_rcu_locks) + ctx_kind = BPF_DIAG_CONTEXT_RCU; + else if (env->cur_state->active_preempt_locks) + ctx_kind = BPF_DIAG_CONTEXT_PREEMPT; + else if (env->cur_state->active_irq_id) + ctx_kind = BPF_DIAG_CONTEXT_IRQ; + else if (env->cur_state->active_locks) + ctx_kind = BPF_DIAG_CONTEXT_LOCK; + else + ctx_kind = BPF_DIAG_CONTEXT_NONE; + + depth = diag_context_depth(env, ctx_kind); + opts = (struct bpf_diag_history_opts) { + .scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT, + .ctx_kind = ctx_kind, + .ctx_depth = depth, + }; + constraint = diag_context_constraint(ctx_kind); + context = diag_context_name(ctx_kind); + + bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, + "operation is not allowed in this context"); + if (constraint) { + if (depth) { + diag_reason( + env, "The operation %s cannot be used in %s because %s. This path is still inside %s.", + operation, context, constraint, diag_active_context(env, depth, context)); + } else { + diag_reason(env, "The operation %s cannot be used in %s because %s.", + operation, context, constraint); + } + } else { + diag_reason(env, "The operation %s cannot be used in %s.", operation, + context); + } + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s is not allowed in %s", operation, + context); + + if (ctx_kind != BPF_DIAG_CONTEXT_NONE) + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_ctx_active(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + enum bpf_diag_context_kind ctx_kind, const char *suggestion) +{ + u32 depth = diag_context_depth(env, ctx_kind); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT, + .ctx_kind = ctx_kind, + .ctx_depth = depth, + }; + const char *context = diag_context_name(ctx_kind); + + bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, + "operation is not allowed in this context"); + diag_reason( + env, "The operation %s cannot be used while this path is still inside %s. Leave the region before this operation.", + operation, diag_active_context(env, depth, context)); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s is not allowed before leaving %s", + operation, context); + + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_ctx_required(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + enum bpf_diag_context_kind ctx_kind, const char *suggestion) +{ + const char *context = diag_context_name(ctx_kind); + + bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, "required context is not active"); + diag_reason(env, "The operation %s requires an active %s, but this path is outside one.", + operation, context); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s requires %s", operation, context); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_ctx_underflow(struct bpf_verifier_env *env, u32 insn_idx, + const char *operation, enum bpf_diag_context_kind ctx_kind, + const char *suggestion) +{ + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT, + .ctx_kind = ctx_kind, + }; + const char *context = diag_context_name(ctx_kind); + + bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, "unmatched context exit"); + diag_reason( + env, "The operation %s tries to leave %s, but this path has no active %s to leave. The current depth is 0.", + operation, context, context); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s has no matching enter on this path", + operation); + + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + void bpf_diag_invalid_deref(struct bpf_verifier_env *env, u32 insn_idx, int regno, const char *reg_name, const struct bpf_reg_state *reg, enum bpf_diag_invalid_deref_kind kind, s64 offset) @@ -2057,7 +2220,7 @@ static const char *diag_context_name(enum bpf_diag_context_kind kind) return "lock region"; case BPF_DIAG_CONTEXT_NONE: default: - return "context"; + return "non-sleepable program"; } } diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index 4b85a7ad2019..95bc654e5b3e 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -80,6 +80,15 @@ void bpf_diag_leak(struct bpf_verifier_env *env, u32 ref_id, u32 alloc_insn, u32 void bpf_diag_call_type(struct bpf_verifier_env *env, u32 insn_idx, int argno, int regno, int stack_arg_slot, const char *call_name, const char *arg_name, const char *reason, const char *suggestion); +void bpf_diag_ctx_forbidden(struct bpf_verifier_env *env, u32 insn_idx, + const char *operation, const char *suggestion); +void bpf_diag_ctx_active(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + enum bpf_diag_context_kind ctx_kind, const char *suggestion); +void bpf_diag_ctx_required(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + enum bpf_diag_context_kind ctx_kind, const char *suggestion); +void bpf_diag_ctx_underflow(struct bpf_verifier_env *env, u32 insn_idx, + const char *operation, enum bpf_diag_context_kind ctx_kind, + const char *suggestion); void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true); void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 30e48d4b02f9..0f126f090a47 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7739,6 +7739,9 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * return err; case -EPROTO: verbose(env, "expected an RCU CS when using %s\n", meta->func_name); + bpf_diag_ctx_required( + env, insn_idx, meta->func_name, BPF_DIAG_CONTEXT_RCU, + "Wrap iterator use in bpf_rcu_read_lock() and bpf_rcu_read_unlock(), keeping all exit paths balanced."); return err; default: return err; @@ -9838,17 +9841,24 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return err; if (bpf_subprog_is_global(env, subprog)) { const char *sub_name = bpf_subprog_name(env, subprog); + const char *operation; bool returns_void; if (env->cur_state->active_locks) { verbose(env, "global function calls are not allowed while holding a lock,\n" "use static function instead\n"); + operation = bpf_diag_fmt(env, "global function %s()", sub_name); + bpf_diag_ctx_active(env, *insn_idx, operation, BPF_DIAG_CONTEXT_LOCK, + "Release the lock before calling the global function, or use a static function instead."); return -EINVAL; } if (env->subprog_info[subprog].might_sleep && !in_sleepable_context(env)) { verbose(env, "sleepable global function %s() called in %s\n", sub_name, non_sleepable_context_description(env)); + operation = bpf_diag_fmt(env, "sleepable global function %s()", sub_name); + bpf_diag_ctx_forbidden(env, *insn_idx, operation, + "Move the call outside the critical section, or use a non-sleepable function."); return -EINVAL; } @@ -10495,6 +10505,8 @@ static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit if (check_lock && env->cur_state->active_locks) { verbose(env, "%s cannot be used inside bpf_spin_lock-ed region\n", prefix); + bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_LOCK, + "Release the BPF spin lock before this operation on every path."); return -EINVAL; } @@ -10506,16 +10518,23 @@ static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit if (check_lock && env->cur_state->active_irq_id) { verbose(env, "%s cannot be used inside bpf_local_irq_save-ed region\n", prefix); + bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_IRQ, + "Restore the saved IRQ state before this operation on every path."); return -EINVAL; } if (check_lock && env->cur_state->active_rcu_locks) { verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix); + bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_RCU, + "Call bpf_rcu_read_unlock() before this operation on every path."); return -EINVAL; } if (check_lock && env->cur_state->active_preempt_locks) { verbose(env, "%s cannot be used inside bpf_preempt_disable-ed region\n", prefix); + bpf_diag_ctx_active( + env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_PREEMPT, + "Call bpf_preempt_enable() before this operation on every path."); return -EINVAL; } @@ -10697,6 +10716,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn enum bpf_type_flag ret_flag; struct bpf_reg_state *regs; struct bpf_call_arg_meta meta; + const char *operation; int insn_idx = *insn_idx_p; bool changes_data; int i, err, func_id; @@ -10744,6 +10764,10 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (fn->might_sleep && !in_sleepable_context(env)) { verbose(env, "sleepable helper %s#%d in %s\n", func_id_name(func_id), func_id, non_sleepable_context_description(env)); + operation = bpf_diag_fmt(env, "sleepable helper %s#%d", + func_id_name(func_id), func_id); + bpf_diag_ctx_forbidden(env, insn_idx, operation, + "Move the helper call outside the critical section, or use a non-sleepable helper."); return -EINVAL; } @@ -13591,6 +13615,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, const struct btf_type *t, *ptr_type; struct bpf_call_arg_meta meta; struct bpf_insn_aux_data *insn_aux; + const char *operation; int err, insn_idx = *insn_idx_p; u32 i, nargs, ptr_type_id; struct bpf_kfunc_desc *desc; @@ -13656,6 +13681,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, sleepable = bpf_is_kfunc_sleepable(&meta); if (sleepable && !in_sleepable(env)) { verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); + operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); + bpf_diag_ctx_forbidden(env, insn_idx, operation, + "Mark the program sleepable if the program type allows it, or use a non-sleepable kfunc."); return -EACCES; } @@ -13726,6 +13754,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } else if (rcu_unlock) { if (env->cur_state->active_rcu_locks == 0) { verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); + bpf_diag_ctx_underflow( + env, insn_idx, func_name, BPF_DIAG_CONTEXT_RCU, + "Remove the extra bpf_rcu_read_unlock() call, or ensure this path first enters an RCU read lock region."); return -EINVAL; } env->cur_state->active_rcu_locks--; @@ -13740,6 +13771,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } else if (preempt_enable) { if (env->cur_state->active_preempt_locks == 0) { verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name); + bpf_diag_ctx_underflow( + env, insn_idx, func_name, BPF_DIAG_CONTEXT_PREEMPT, + "Remove the extra bpf_preempt_enable() call, or ensure this path first disables preemption."); return -EINVAL; } env->cur_state->active_preempt_locks--; @@ -13752,6 +13786,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (sleepable && !in_sleepable_context(env)) { verbose(env, "kernel func %s is sleepable within %s\n", func_name, non_sleepable_context_description(env)); + operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); + bpf_diag_ctx_forbidden(env, insn_idx, operation, + "Move the kfunc call outside the critical section, or use a non-sleepable kfunc."); return -EACCES; } @@ -13762,6 +13799,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (is_kfunc_rcu_protected(&meta) && !in_rcu_cs(env)) { verbose(env, "kernel func %s requires RCU critical section protection\n", func_name); + bpf_diag_ctx_required( + env, insn_idx, func_name, BPF_DIAG_CONTEXT_RCU, + "Call this kfunc between bpf_rcu_read_lock() and bpf_rcu_read_unlock(), keeping all exit paths balanced."); return -EACCES; } @@ -18039,6 +18079,10 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) !kfunc_spin_allowed(env, insn->imm, insn->off))) { verbose(env, "function calls are not allowed while holding a lock\n"); + bpf_diag_ctx_active( + env, env->insn_idx, + "function call", BPF_DIAG_CONTEXT_LOCK, + "Release the BPF spin lock before making this call, or move the call outside the locked region."); return -EINVAL; } } -- cgit v1.2.3 From a8f4278353947d019c990a77d574dfd4d1dc9b46 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:46:08 +0200 Subject: bpf: Report Program Structure CFG errors Augment selected whole-program and subprogram CFG validation failures with Program Structure reports. These errors are structural rather than path-dependent, so the reports focus on source and instruction context instead of causal history. Cover direct and indirect jumps outside the program or current subprogram, unprivileged backedges, missing and out-of-range jump tables, targets in the second half of an ldimm64, unreachable instructions, subprogram fallthrough, and recursive bpf2bpf call graph edges. Format long jump-range reasons directly in diagnostics.c, and keep the fallthrough suggestion aligned with the verifier check by suggesting exit or explicit jumps. Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260815064612.378577-14-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/cfg.c | 35 +++++++++++++++++++++++++++++++++++ kernel/bpf/diagnostics.c | 19 +++++++++++++++++++ kernel/bpf/diagnostics.h | 3 +++ kernel/bpf/verifier.c | 16 ++++++++++++++++ 4 files changed, 73 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 818f7afac83a..0f13c13f4133 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -5,6 +5,8 @@ #include #include +#include "diagnostics.h" + #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) /* non-recursive DFS pseudo code @@ -112,6 +114,10 @@ static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) if (w < 0 || w >= env->prog->len) { verbose_linfo(env, t, "%d: ", t); verbose(env, "jump out of range from insn %d to %d\n", t, w); + bpf_diag_program_structure( + env, t, "jump out of range", "Keep branch targets inside the program.", + "Instruction %d jumps to instruction %d, but the program only contains instructions 0 through %d.", + t, w, env->prog->len - 1); return -EINVAL; } @@ -135,6 +141,11 @@ static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) verbose_linfo(env, t, "%d: ", t); verbose_linfo(env, w, "%d: ", w); verbose(env, "back-edge from insn %d to %d\n", t, w); + bpf_diag_program_structure( + env, t, "back-edge is not allowed", + "Load with privileges that allow this back-edge, or rewrite the control flow so it does not branch backward.", + "Instruction %d branches back to instruction %d. This program is being rejected without the privilege needed for this back-edge.", + t, w); return -EINVAL; } else if (insn_state[w] == EXPLORED) { /* forward- or cross-edge */ @@ -315,6 +326,11 @@ static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env, if (!jt) { verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start); + bpf_diag_program_structure( + env, subprog_start, "missing jump table", + "Make sure subprograms containing gotox instructions are accompanied by jump tables referencing these subprograms.", + "No jump table was found for the subprogram that starts at instruction %u.", + subprog_start); return ERR_PTR(-EINVAL); } @@ -342,6 +358,11 @@ create_jt(int t, struct bpf_verifier_env *env) if (jt->items[i] < subprog_start || jt->items[i] >= subprog_end) { verbose(env, "jump table for insn %d points outside of the subprog [%u,%u]\n", t, subprog_start, subprog_end); + bpf_diag_program_structure( + env, t, "jump table target out of range", + "Keep every jump-table target inside the same subprogram.", + "The jump table for instruction %d points outside subprogram range [%u,%u).", + t, subprog_start, subprog_end); kvfree(jt); return ERR_PTR(-EINVAL); } @@ -373,6 +394,11 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) w = jt->items[i]; if (w < 0 || w >= env->prog->len) { verbose(env, "indirect jump out of range from insn %d to %d\n", t, w); + bpf_diag_program_structure( + env, t, "indirect jump out of range", + "Keep indirect jump targets inside the program.", + "Instruction %d can jump indirectly to instruction %d, but the program only contains instructions 0 through %d.", + t, w, env->prog->len - 1); return -EINVAL; } @@ -623,12 +649,21 @@ walk_cfg: if (insn_state[i] != EXPLORED) { verbose(env, "unreachable insn %d\n", i); + bpf_diag_program_structure( + env, i, "unreachable instruction", + "Remove the unreachable instruction or add valid control flow that reaches it.", + "Instruction %d is not reachable from the program entry point.", i); ret = -EINVAL; goto err_free; } if (bpf_is_ldimm64(insn)) { if (insn_state[i + 1] != 0) { verbose(env, "jump into the middle of ldimm64 insn %d\n", i); + bpf_diag_program_structure( + env, i, "jump into ldimm64 immediate", + "Target the first instruction of the ldimm64 pair, or restructure the jump target.", + "Control flow reaches the second half of the ldimm64 instruction pair that starts at instruction %d.", + i); ret = -EINVAL; goto err_free; } diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index c69160f656e9..9fc1f8cf7312 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -21,6 +21,7 @@ #define RESOURCE_LIFETIME_SAFETY "Resource Lifetime Safety" #define CALL_TYPE_SAFETY "Call Type Safety" #define EXECUTION_CONTEXT_SAFETY "Execution Context Safety" +#define PROGRAM_STRUCTURE "Program Structure" #define BPF_DIAG_TEXT_WIDTH 100 #define BPF_DIAG_TEXT_INDENT " " @@ -1192,6 +1193,24 @@ void bpf_diag_ctx_underflow(struct bpf_verifier_env *env, u32 insn_idx, diag_suggestion(env, "%s", suggestion); } +void bpf_diag_program_structure(struct bpf_verifier_env *env, u32 insn_idx, + const char *problem, const char *suggestion, + const char *reason_fmt, ...) +{ + va_list args; + + bpf_diag_header(env, PROGRAM_STRUCTURE, problem); + diag_section(env, "Reason"); + + va_start(args, reason_fmt); + diag_vprint_indented(env, reason_fmt, args); + va_end(args); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); + + diag_suggestion(env, "%s", suggestion); +} void bpf_diag_invalid_deref(struct bpf_verifier_env *env, u32 insn_idx, int regno, const char *reg_name, const struct bpf_reg_state *reg, enum bpf_diag_invalid_deref_kind kind, s64 offset) diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index 95bc654e5b3e..ab082d2d6e37 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -89,6 +89,9 @@ void bpf_diag_ctx_required(struct bpf_verifier_env *env, u32 insn_idx, const cha void bpf_diag_ctx_underflow(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, enum bpf_diag_context_kind ctx_kind, const char *suggestion); +void bpf_diag_program_structure(struct bpf_verifier_env *env, u32 insn_idx, + const char *problem, const char *suggestion, + const char *reason_fmt, ...) __printf(5, 6); void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true); void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0f126f090a47..6a886650cc40 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3020,6 +3020,12 @@ static int check_subprogs(struct bpf_verifier_env *env) off = i + bpf_jmp_offset(&insn[i]) + 1; if (off < subprog_start || off >= subprog_end) { verbose(env, "jump out of range from insn %d to %d\n", i, off); + bpf_diag_program_structure( + env, i, "jump out of range", + "Keep branch targets within the same subprogram, or use an explicit subprogram call.", + "Instruction %d jumps to instruction %d, but subprogram %d only contains instructions %d through %d. " + "A branch target must stay inside the same subprogram.", + i, off, cur_subprog, subprog_start, subprog_end - 1); return -EINVAL; } next: @@ -3032,6 +3038,11 @@ next: code != (BPF_JMP32 | BPF_JA) && code != (BPF_JMP | BPF_JA)) { verbose(env, "last insn is not an exit or jmp\n"); + bpf_diag_program_structure( + env, i, "subprogram can fall through", + "End each subprogram with an exit or an explicit jump that keeps control flow inside the subprogram.", + "Subprogram %d reaches its last instruction %d without an exit or jump, so control could continue into the next subprogram.", + cur_subprog, i); return -EINVAL; } subprog_start = subprog_end; @@ -3104,6 +3115,11 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env) verbose(env, "recursive call from %s() to %s()\n", bpf_subprog_name(env, cur), bpf_subprog_name(env, callee)); + bpf_diag_program_structure( + env, idx, "recursive subprogram call", + "Rewrite the recursion as an explicit bounded loop, or split the logic so subprogram calls do not form a cycle.", + "This bpf2bpf call would make the subprogram call graph recursive. " + "The verifier requires a finite, acyclic call graph so it can bound stack depth and analysis."); ret = -EINVAL; goto out; } -- cgit v1.2.3 From ac545b00ca56368b8acb498107ad4f1d91a5245d Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:46:09 +0200 Subject: bpf: Report Policy helper and kfunc errors Augment selected helper and kfunc allowability failures with Policy reports. These reports explain which requested operation is forbidden and why, without adding path history for non-path-dependent policy checks. Cover unprivileged bpf2bpf and kfunc use, helper program-type restrictions, GPL-only helpers, helper-specific allow callbacks, kfunc allowability, and destructive kfunc capability checks. Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260815064612.378577-15-memxor@gmail.com Signed-off-by: Eduard Zingerman --- kernel/bpf/diagnostics.c | 14 ++++++++++++++ kernel/bpf/diagnostics.h | 2 ++ kernel/bpf/verifier.c | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index 9fc1f8cf7312..33b7d9e8e2c3 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -22,6 +22,7 @@ #define CALL_TYPE_SAFETY "Call Type Safety" #define EXECUTION_CONTEXT_SAFETY "Execution Context Safety" #define PROGRAM_STRUCTURE "Program Structure" +#define POLICY "Policy" #define BPF_DIAG_TEXT_WIDTH 100 #define BPF_DIAG_TEXT_INDENT " " @@ -1211,6 +1212,19 @@ void bpf_diag_program_structure(struct bpf_verifier_env *env, u32 insn_idx, diag_suggestion(env, "%s", suggestion); } + +void bpf_diag_policy(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + const char *reason, const char *suggestion) +{ + bpf_diag_header(env, POLICY, "operation is not allowed"); + diag_reason(env, "The %s is not allowed: %s.", operation, reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "policy check failed for %s", operation); + + diag_suggestion(env, "%s", suggestion); +} + void bpf_diag_invalid_deref(struct bpf_verifier_env *env, u32 insn_idx, int regno, const char *reg_name, const struct bpf_reg_state *reg, enum bpf_diag_invalid_deref_kind kind, s64 offset) diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index ab082d2d6e37..d1b79945008a 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -92,6 +92,8 @@ void bpf_diag_ctx_underflow(struct bpf_verifier_env *env, u32 insn_idx, void bpf_diag_program_structure(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, const char *suggestion, const char *reason_fmt, ...) __printf(5, 6); +void bpf_diag_policy(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + const char *reason, const char *suggestion); void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true); void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6a886650cc40..0bb7ee95c8bd 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2924,6 +2924,10 @@ static int add_subprogs(struct bpf_verifier_env *env) if (!env->bpf_capable) { verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); + bpf_diag_policy( + env, i, "BPF-to-BPF function call", + "loading or calling other BPF functions requires CAP_BPF or CAP_SYS_ADMIN", + "Load this program with the required capability, or avoid BPF-to-BPF function calls in unprivileged programs."); return -EPERM; } @@ -2976,6 +2980,10 @@ static int add_kfuncs(struct bpf_verifier_env *env) if (!env->bpf_capable) { verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); + bpf_diag_policy( + env, i, "kernel function call", + "calling kernel functions requires CAP_BPF or CAP_SYS_ADMIN", + "Load this program with the required capability, or avoid kernel function calls in unprivileged programs."); return -EPERM; } @@ -10748,17 +10756,31 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (err) { verbose(env, "program of this type cannot use helper %s#%d\n", func_id_name(func_id), func_id); + operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id); + bpf_diag_policy( + env, insn_idx, operation, "this program type does not allow the helper", + "Use a helper allowed for this program type, or move the logic to a compatible program type."); return err; } /* eBPF programs must be GPL compatible to use GPL-ed functions */ if (!env->prog->gpl_compatible && fn->gpl_only) { verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); + operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id); + bpf_diag_policy( + env, insn_idx, operation, + "this helper is restricted to GPL-compatible programs", + "Use a GPL-compatible license, or replace the helper with one that is available to non-GPL programs."); return -EINVAL; } if (fn->allowed && !fn->allowed(env->prog)) { verbose(env, "helper call is not allowed in probe\n"); + operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id); + bpf_diag_policy( + env, insn_idx, operation, + "the helper-specific policy callback rejected this program", + "Use the helper only from an allowed attach point or program configuration."); return -EINVAL; } @@ -13643,8 +13665,13 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; err = bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta); - if (err == -EACCES && meta.func_name) + if (err == -EACCES && meta.func_name) { verbose(env, "calling kernel function %s is not allowed\n", meta.func_name); + operation = bpf_diag_fmt(env, "kfunc %s", meta.func_name); + bpf_diag_policy( + env, insn_idx, operation, "this program cannot call the kfunc", + "Use a kfunc allowed for this program type and attach point, or change the program context."); + } if (err) return err; desc_btf = meta.btf; @@ -13691,6 +13718,10 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) { verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n"); + operation = bpf_diag_fmt(env, "destructive kfunc %s", meta.func_name); + bpf_diag_policy( + env, insn_idx, operation, "destructive kfuncs require CAP_SYS_BOOT", + "Load the program with CAP_SYS_BOOT, or avoid destructive kfuncs."); return -EACCES; } -- cgit v1.2.3 From 7ae4eb14c5f9d9bf0e0feabeab206151b1280512 Mon Sep 17 00:00:00 2001 From: Mahe Tardy Date: Thu, 13 Aug 2026 11:05:37 +0000 Subject: bpf: Add ksock kfuncs Add BPF kfuncs that allow BPF LSM programs to create and use sockets for sending data. This provides a mechanism for BPF programs to emit telemetry. For this first patch set, it's restricted to SOCK_DGRAM socket types with IPPROTO_UDP protocol but could be easily extended to SOCK_STREAM and IPPROTO_TCP in the future. The API consists of five kfuncs: bpf_ksock_create() - Create a socket (sleepable) bpf_ksock_connect() - Connect socket to remote address (sleepable) bpf_ksock_send() - Send data through the socket (sleepable) bpf_ksock_acquire() - Acquire a reference to a socket context bpf_ksock_release() - Release a reference (cleanup via queue_rcu_work since sock_release sleeps) The setup kfuncs bpf_ksock_create, bpf_ksock_connect, can be called from SYSCALL programs only. While bpf_ksock_acquire, bpf_ksock_release and bpf_ksock_send can be called from SYSCALL and LSM programs. The implementation follows the established kfunc lifecycle pattern (create/acquire/release with refcounting, kptr map storage, dtor registration). The kernel socket is wrapped in a refcounted bpf_ksock struct. Cleanup is deferred via queue_rcu_work() because sock_release() may sleep. The kfuncs are only compiled when CONFIG_INET is enabled, as they specifically support AF_INET and AF_INET6 sockets. The socket operations go through the expected LSM hooks instead of by-passing them like many kernel sockets since those are created by BPF programs and thus system users. Thus, the bpf_ksock_send() kfunc, which is exposed to LSM progs has a verifier filter protection to avoid recursion so that the whole bpf_kfunc_set kfunc set cannot be called in a program attached to security_socket_sendmsg(). Also, because of the LSM checks, we prevent the use of the kfuncs from asynchronous workqueue as the current value would then be invalid. In bpf_ksock_create(), we copy the arg values to avoid TOCTOU races since the kfunc can sleep and the arg values could be stored in a map that could be re-written by BPF progs or even userspace programs if the map is mmaped. Signed-off-by: Mahe Tardy Signed-off-by: Daniel Borkmann Reviewed-by: Jiayuan Chen Acked-by: Stanislav Fomichev Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20260813110540.103550-3-mahe.tardy@gmail.com --- kernel/bpf/verifier.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0bb7ee95c8bd..d17f14b35b79 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4564,6 +4564,9 @@ BTF_ID(struct, task_struct) #ifdef CONFIG_CRYPTO BTF_ID(struct, bpf_crypto_ctx) #endif +#ifdef CONFIG_INET +BTF_ID(struct, bpf_ksock) +#endif BTF_SET_END(rcu_protected_types) static bool rcu_protected_object(const struct btf *btf, u32 btf_id) -- cgit v1.2.3 From 4bc49ae344d65cfcef738f281ac575cf73ca2fc5 Mon Sep 17 00:00:00 2001 From: Yiyang Chen Date: Sun, 16 Aug 2026 10:56:33 +0000 Subject: bpf: Check pointer type for all atomic RMW paths Atomic RMW verification records an instruction pointer type only when the current destination is PTR_TO_ARENA. A second path can therefore reach the same instruction with an ordinary pointer without comparing it against the saved arena type. The post-verification fixup uses the saved type to rewrite the instruction to BPF_PROBE_ATOMIC for every path. Record the actual destination type for all atomic RMW paths so the existing mismatch check rejects incompatible uses of one instruction. Fixes: d503a04f8bc0 ("bpf: Add support for certain atomics in bpf_arena to x86 JIT") Signed-off-by: Yiyang Chen Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260816-bpf-next-038-mixed-atomic-v1-v2-1-4644c1886dbc@mails.tsinghua.edu.cn Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d17f14b35b79..93463caf5c9a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6739,11 +6739,9 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, if (err) return err; - if (is_arena_reg(env, insn->dst_reg)) { - err = save_aux_ptr_type(env, PTR_TO_ARENA, false); - if (err) - return err; - } + err = save_aux_ptr_type(env, dst_reg->type, false); + if (err) + return err; /* Check whether we can write into the same memory. */ err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, true, false); -- cgit v1.2.3 From 09c447564fcac5c531a19e3003d14c9c0a68fd19 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 14 Aug 2026 23:52:55 +0200 Subject: bpf: Keep fault protection when merging pointer types When the same BPF_LDX instruction is reached through paths that yield different pointer types, save_aux_ptr_type() merges them into a single type which is later used by bpf_convert_ctx_accesses() to decide whether the load has to be rewritten into a BPF_PROBE_MEM one. Before f2362a57aeff ("bpf: allow void* cast using bpf_rdonly_cast()") the merge only accepted two PTR_TO_BTF_ID pointers and unconditionally fell back to PTR_TO_BTF_ID | PTR_UNTRUSTED, so the merged type was always one that gets the BPF_PROBE_MEM rewrite. However, the mentioned commit widened the merge to also cover a PTR_TO_MEM base and replaced the fallback by a union of the PTR_UNTRUSTED and MEM_RDONLY flags. A union of flags though cannot express the property the later rewrite is built upon, some examples: - PTR_TO_MEM merged with PTR_TO_BTF_ID | PTR_UNTRUSTED gets PTR_TO_MEM | PTR_UNTRUSTED but only the MEM_RDONLY variant is valid - PTR_TO_MEM merged with a plain PTR_TO_BTF_ID gets PTR_TO_MEM dropping the rewrite the latter type would have gotten - PTR_TO_MEM | MEM_RDONLY merged with a plain PTR_TO_BTF_ID gets PTR_TO_MEM | MEM_RDONLY which is not rewritten either since only its PTR_UNTRUSTED variant is In all three cases a program can take the unsafe path at runtime with a NULL or otherwise bad pointer and panic the kernel on the faulting load: BUG: kernel NULL pointer dereference, address: 0000000000000038 RIP: 0010:bpf_prog_77531a87032eeaf1_mixed_mem_btf_id_type+0x4b/0x65 Call Trace: bpf_test_run+0x20b/0x460 bpf_prog_test_run_skb+0x650/0xbe0 __sys_bpf+0xb96/0x3140 __x64_sys_bpf+0x2c/0x40 do_syscall_64+0xba/0x590 Kernel panic - not syncing: Fatal exception in interrupt Note that the last two shapes have to be fixed right here, otherwise the merged type retains nothing which marks the load as fault prone, thus no rule in bpf_convert_ctx_accesses() can recover it. Fix it by normalizing the merged type instead. Reuse it in is_load_acq_unsafe() to avoid open coding, and trim the overly verbose comment which is more of an implementation detail of bpf_convert_ctx_accesses() anyway. Fixes: f2362a57aeff ("bpf: allow void* cast using bpf_rdonly_cast()") Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260814215301.709827-1-daniel@iogearbox.net --- kernel/bpf/verifier.c | 49 +++++++++++++++++++++++-------------------------- 1 file changed, 23 insertions(+), 26 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 93463caf5c9a..9ec23a9c6592 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5025,19 +5025,11 @@ static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno, * A BPF_LOAD_ACQ is not rewritten to a BPF_PROBE_MEM load by the * verifier, unlike a regular BPF_LDX. The JIT would emit a plain load * with no exception table entry, so a fault (e.g. NULL deref) crashes - * the kernel instead of being handled. - * - * Reject the source pointer types that a BPF_LDX would have had that - * fault protection applied to, i.e. the ones bpf_convert_ctx_accesses() - * turns into BPF_PROBE_MEM: a bare PTR_TO_BTF_ID and any PTR_UNTRUSTED - * pointer (untrusted btf ids, untrusted MEM_ALLOC, rdonly untrusted - * memory). A PTR_TRUSTED pointer is not among them, is not converted, - * and stays allowed. Same for the other flagged PTR_TO_BTF_ID variants - * (MEM_ALLOC, MEM_RCU, ...), hence the exact match on the base type. + * the kernel instead of being handled. Reject the source pointer types + * that would have needed that protection, the remaining ones stay + * allowed. */ - return insn->imm == BPF_LOAD_ACQ && - (reg->type == PTR_TO_BTF_ID || - (type_flag(reg->type) & PTR_UNTRUSTED)); + return insn->imm == BPF_LOAD_ACQ && bpf_may_fault_on_deref(reg->type); } /* Return false if @regno contains a pointer whose type isn't supported for @@ -17862,11 +17854,24 @@ static bool is_ptr_to_mem(enum bpf_reg_type type) return base_type(type) == PTR_TO_MEM; } +static enum bpf_reg_type merge_ptr_types(enum bpf_reg_type type_a, + enum bpf_reg_type type_b) +{ + bool to_mem = is_ptr_to_mem(type_a) || is_ptr_to_mem(type_b); + enum bpf_reg_type type_merged = to_mem ? PTR_TO_MEM : PTR_TO_BTF_ID; + + if (bpf_may_fault_on_deref(type_a) || bpf_may_fault_on_deref(type_b)) + type_merged |= to_mem ? MEM_RDONLY | PTR_UNTRUSTED : + PTR_UNTRUSTED; + else + type_merged |= ((type_a | type_b) & MEM_RDONLY); + return type_merged; +} + static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, bool allow_trust_mismatch) { enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type; - enum bpf_reg_type merged_type; if (*prev_type == NOT_INIT) { /* Saw a valid insn @@ -17887,20 +17892,12 @@ static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type typ is_ptr_to_mem_or_btf_id(*prev_type)) { /* * Have to support a use case when one path through - * the program yields TRUSTED pointer while another - * is UNTRUSTED. Fallback to UNTRUSTED to generate - * BPF_PROBE_MEM/BPF_PROBE_MEMSX. - * Same behavior of MEM_RDONLY flag. + * the program yields a TRUSTED pointer while another + * is UNTRUSTED. Merge them into a type which keeps + * the BPF_PROBE_MEM/BPF_PROBE_MEMSX rewrite when + * either side needs it. */ - if (is_ptr_to_mem(type) || is_ptr_to_mem(*prev_type)) - merged_type = PTR_TO_MEM; - else - merged_type = PTR_TO_BTF_ID; - if ((type & PTR_UNTRUSTED) || (*prev_type & PTR_UNTRUSTED)) - merged_type |= PTR_UNTRUSTED; - if ((type & MEM_RDONLY) || (*prev_type & MEM_RDONLY)) - merged_type |= MEM_RDONLY; - *prev_type = merged_type; + *prev_type = merge_ptr_types(type, *prev_type); } else { verbose(env, "same insn cannot be used with different pointers\n"); return -EINVAL; -- cgit v1.2.3 From f438ba7a4c3efa627dc91a132c4653725723a6bc Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 14 Aug 2026 23:52:56 +0200 Subject: bpf: Treat a fault prone PTR_TO_MEM as a pointer type mismatch reg_type_mismatch_ok() enumerates the pointer types which must not silently share a BPF_LDX with a different one, since the type recorded for the insn drives a rewrite in bpf_convert_ctx_accesses(). f2362a57aeff ("bpf: allow void* cast using bpf_rdonly_cast()") added PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED as another type in need of one, namely the BPF_PROBE_MEM rewrite, but did not add it there. Fix it by adding the missing case to reg_type_mismatch_ok(), so that a PTR_TO_MEM which may fault on deref is not mismatch ok anymore. The triage in save_aux_ptr_type() then merges them. Fixes: f2362a57aeff ("bpf: allow void* cast using bpf_rdonly_cast()") Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260814215301.709827-2-daniel@iogearbox.net --- kernel/bpf/verifier.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 9ec23a9c6592..ad3310b55b02 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17815,6 +17815,8 @@ static bool reg_type_mismatch_ok(enum bpf_reg_type type) case PTR_TO_BTF_ID: case PTR_TO_ARENA: return false; + case PTR_TO_MEM: + return !bpf_may_fault_on_deref(type); default: return true; } -- cgit v1.2.3 From ee9ad135b2087f9335eed97d062f5853e70f89fe Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 14 Aug 2026 23:52:57 +0200 Subject: bpf: Reject a store through a fault prone pointer check_ptr_to_btf_access() allows the program to store before the default BTF access path gets to reject a non read access. ac65c710cc64 ("bpf: Reject writes through untrusted BTF pointers") closed that for a PTR_UNTRUSTED pointer, but a bare PTR_TO_BTF_ID may fault on a dereference just the same and is let through. A BPF_LDX gets the BPF_PROBE_MEM rewrite in bpf_convert_ctx_accesses() and a bad address is handled, but a BPF_STX does not and cannot, there is no probed store to rewrite. The store is emitted as a plain one without an exception table entry and a bad address panics the kernel. A bpf_qdisc program can reach this, bpf_qdisc_btf_struct_access() permits a write to Qdisc::limit and Qdisc::next_sched is a plain struct Qdisc pointer which the walk turns into the compat type: struct Qdisc *next = sch->next_sched; next->limit = 1000; BUG: kernel NULL pointer dereference, address: 0000000000000014 RIP: 0010:bpf_prog_c6e14e7f32c8e325_bpf_fifo_enqueue+0x3a/0x12b Code: [...] bf e8 03 00 00 <89> 7e 14 41 8b 7f 14 [...] Kernel panic - not syncing: Fatal exception in interrupt Fix by widen the check to bpf_may_fault_on_deref() so that it covers both. Fixes: 27ae7997a661 ("bpf: Introduce BPF_PROG_TYPE_STRUCT_OPS") Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260814215301.709827-3-daniel@iogearbox.net --- kernel/bpf/verifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ad3310b55b02..58a128a8d8d0 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5988,7 +5988,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, return -EACCES; } - if (atype != BPF_READ && (type_flag(reg->type) & PTR_UNTRUSTED)) { + if (atype != BPF_READ && bpf_may_fault_on_deref(reg->type)) { verbose(env, "only read is supported\n"); return -EACCES; } -- cgit v1.2.3 From d99bda7f017b47aff45accbb321facba9f7dd799 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 14 Aug 2026 23:52:58 +0200 Subject: bpf: Rewrite any fault prone load out of a mem or btf_id pointer bpf_convert_ctx_accesses() turns a BPF_LDX into a BPF_PROBE_MEM one by matching the type recorded for the insn against a list of exact pointer types. The list cannot keep up with the flag combinations the verifier produces, and a type which is missing from it ends up as a plain load without an exception table entry, so a bad address panics the kernel instead of being handled. Two such types exist today and are reachable: - PTR_TO_BTF_ID | PTR_UNTRUSTED | MEM_ALLOC | NON_OWN_REF - PTR_TO_BTF_ID | PTR_UNTRUSTED | MEM_RCU Rather than adding the two, just drop the list and state the property itself in the default case of the switch. This is a superset of what the list matched, the untrusted PTR_TO_MEM does not have to carry MEM_RDONLY for it anymore, and it stays in sync with the verifier side which uses the same match in save_aux_ptr_type() and reg_type_mismatch_ok(). Assert that a fault prone type which does not get the rewrite for whatever reason is rejected at load time rather than left to fault at runtime to catch any future cases. Fixes: 1b12171533a9 ("bpf: Mark direct ld of stashed bpf_{rb,list}_node as non-owning ref") Fixes: 6fcd486b3a0a ("bpf: Refactor RCU enforcement in the verifier.") Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260814215301.709827-4-daniel@iogearbox.net --- kernel/bpf/fixups.c | 47 ++++++++++++++++++++++++++--------------------- kernel/bpf/verifier.c | 15 ++------------- 2 files changed, 28 insertions(+), 34 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 70f22eb63ed5..65b441e4a351 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -812,6 +812,7 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) for (i = 0; i < insn_cnt; i++, insn++) { bpf_convert_ctx_access_t convert_ctx_access; + enum bpf_reg_type ptr_type; u8 mode; if (env->insn_aux_data[i + delta].nospec) { @@ -904,7 +905,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) continue; } - switch ((int)env->insn_aux_data[i + delta].ptr_type) { + ptr_type = env->insn_aux_data[i + delta].ptr_type; + switch ((int)ptr_type) { case PTR_TO_CTX: if (!ops->convert_ctx_access) continue; @@ -920,26 +922,6 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) case PTR_TO_XDP_SOCK: convert_ctx_access = bpf_xdp_sock_convert_ctx_access; break; - case PTR_TO_BTF_ID: - case PTR_TO_BTF_ID | PTR_UNTRUSTED: - /* PTR_TO_BTF_ID | MEM_ALLOC always has a valid lifetime, unlike - * PTR_TO_BTF_ID, and an active referenced id, but the same cannot - * be said once it is marked PTR_UNTRUSTED, hence we must handle - * any faults for loads into such types. BPF_WRITE is disallowed - * for this case. - */ - case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED: - case PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED: - if (type == BPF_READ) { - if (BPF_MODE(insn->code) == BPF_MEM) - insn->code = BPF_LDX | BPF_PROBE_MEM | - BPF_SIZE((insn)->code); - else - insn->code = BPF_LDX | BPF_PROBE_MEMSX | - BPF_SIZE((insn)->code); - env->prog->aux->num_exentries++; - } - continue; case PTR_TO_ARENA: if (BPF_MODE(insn->code) == BPF_MEMSX) { if (!bpf_jit_supports_insn(insn, true)) { @@ -953,6 +935,29 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) env->prog->aux->num_exentries++; continue; default: + /* + * A pointer which may fault on a dereference must not + * be loaded from without fault protection, hence turn + * the BPF_LDX into a BPF_PROBE_MEM one so that a bad + * address is handled rather than panicking the kernel. + * A store through one is rejected earlier, there is no + * probed counterpart to rewrite it into. + */ + if (bpf_is_ptr_to_mem_or_btf_id(ptr_type) && + bpf_may_fault_on_deref(ptr_type) && + type == BPF_READ) { + if (BPF_MODE(insn->code) == BPF_MEM) + insn->code = BPF_LDX | BPF_PROBE_MEM | + BPF_SIZE(insn->code); + else + insn->code = BPF_LDX | BPF_PROBE_MEMSX | + BPF_SIZE(insn->code); + env->prog->aux->num_exentries++; + continue; + } + if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, + "access to a fault prone pointer is not rewritten as a probed one")) + return -EFAULT; continue; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 58a128a8d8d0..9f833e913e43 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17840,17 +17840,6 @@ static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) !reg_type_mismatch_ok(prev)); } -static bool is_ptr_to_mem_or_btf_id(enum bpf_reg_type type) -{ - switch (base_type(type)) { - case PTR_TO_MEM: - case PTR_TO_BTF_ID: - return true; - default: - return false; - } -} - static bool is_ptr_to_mem(enum bpf_reg_type type) { return base_type(type) == PTR_TO_MEM; @@ -17890,8 +17879,8 @@ static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type typ * Reject it. */ if (allow_trust_mismatch && - is_ptr_to_mem_or_btf_id(type) && - is_ptr_to_mem_or_btf_id(*prev_type)) { + bpf_is_ptr_to_mem_or_btf_id(type) && + bpf_is_ptr_to_mem_or_btf_id(*prev_type)) { /* * Have to support a use case when one path through * the program yields a TRUSTED pointer while another -- cgit v1.2.3 From 1b5aacd5b2419b0790e955e466d389a61c79b4b1 Mon Sep 17 00:00:00 2001 From: Junseo Lim Date: Tue, 11 Aug 2026 23:19:07 +0900 Subject: bpf: Reject negative optlen in cgroup getsockopt hook A cgroup getsockopt BPF program can shrink ctx->optlen after the kernel getsockopt handler has run. The kernel-buffer variant, used by TCP_ZEROCOPY_RECEIVE, only rejects values larger than the original length. If BPF writes a negative optlen, that value is accepted and propagated back to the TCP getsockopt code. It can then be passed to copy_to_sockptr() as a size_t and trigger the hardened usercopy bytes > INT_MAX warning. Reject negative ctx.optlen in __cgroup_bpf_run_filter_getsockopt_kern(), matching the lower-bound validation already present in the sockptr-based getsockopt hook. Fixes: 9cacf81f8161 ("bpf: Remove extra lock_sock for TCP_ZEROCOPY_RECEIVE") Reported-by: Sechang Lim Signed-off-by: Junseo Lim Signed-off-by: Daniel Borkmann Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/187a4d756275aaaee5d65eecb63c1477b3b66554.1786448307.git.zirajs7@gmail.com --- kernel/bpf/cgroup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 8fbc942a1cc3..149672c76c49 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -2260,7 +2260,7 @@ int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level, if (ret < 0) return ret; - if (ctx.optlen > *optlen) + if (ctx.optlen > *optlen || ctx.optlen < 0) return -EFAULT; /* BPF programs can shrink the buffer, export the modifications. -- cgit v1.2.3 From b26c0b2dd5195f3397e7f64fefc9c190ebba7204 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 16 Aug 2026 03:57:31 +0200 Subject: bpf: Preserve R0 lineage across helper calls check_helper_call() clears all caller-saved registers before taking the diagnostic snapshot of R0. This records NOT_INIT as the old state for every helper return and loses the lineage of the value held in R0 before the call. bpf_diag_record_caller_saved() deliberately skips R0 because the paired modification scope is responsible for it. Open the R0 modification scope before clearing caller-saved registers, matching the kfunc, ld_abs, and subprogram call paths. Reported-by: Sashiko Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260815073833.A93A91F000E9@smtp.kernel.org Link: https://lore.kernel.org/bpf/48e6f021b89562f68850fe21ef8c78719819b04cf9c4e4f50bc791937d37ace8@mail.kernel.org Link: https://lore.kernel.org/bpf/20260816015746.2632990-4-memxor@gmail.com --- kernel/bpf/verifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 9f833e913e43..4adc13584818 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10987,13 +10987,13 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn /* reset caller saved regs */ bpf_diag_record_caller_saved(env, regs); + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); for (i = 0; i < CALLER_SAVED_REGS; i++) { bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); } invalidate_outgoing_stack_args(env, cur_func(env)); - bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); /* update return register (already marked as written above) */ ret_type = fn->ret_type; ret_flag = type_flag(ret_type); -- cgit v1.2.3 From 09a0c2d678643aa8362ed83ea21b3a21566b318a Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 16 Aug 2026 03:57:33 +0200 Subject: bpf: Use canonical stack argument names in diagnostics The main diagnostic identifies the first outgoing stack slot as stack argument 1 and the sixth function argument. The causal history instead labels the same value as stack arg6, making it look like a different slot. Render causal-history targets in the verifier's canonical stack-argument location form. The first outgoing slot is now shown as *(R11-8), matching reg_arg_name(), while the main diagnostic retains its fuller slot and ordinal description. Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/eb1be5327d136b7e5bd6d68e76fef6de20c40790.camel@gmail.com Link: https://lore.kernel.org/bpf/20260816015746.2632990-6-memxor@gmail.com --- kernel/bpf/diagnostics.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index 33b7d9e8e2c3..b24be9df5fab 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -2164,7 +2164,7 @@ static const char *diag_mod_target_desc(struct bpf_verifier_env *env, case BPF_DIAG_MOD_TARGET_REG: return bpf_diag_fmt(env, "R%u", target->regno); case BPF_DIAG_MOD_TARGET_STACK_ARG: - return bpf_diag_fmt(env, "stack arg%d", diag_stack_argno(target->stack_arg)); + return bpf_diag_fmt(env, "*(R11-%u)", (target->stack_arg + 1) * BPF_REG_SIZE); case BPF_DIAG_MOD_TARGET_STACK_SLOT: return bpf_diag_fmt(env, "stack slot fp%d", -(target->spi + 1) * BPF_REG_SIZE); default: -- cgit v1.2.3 From cc782c7ad0f7416f4fcf90bf15064313cbb8a7c9 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 16 Aug 2026 03:57:34 +0200 Subject: bpf: Correct kfunc argument diagnostics The Call Type Safety diagnostics mishandle three kfunc argument classes. BTF type ID 0 represents void, but btf_show_name() also uses zero to end type traversal. A pointer that resolves to void therefore loses its pointee name and is rendered as "()". End traversal directly for concrete terminal types, but resolve referenced types before testing for ID zero, and name the void terminal type explicitly. Format the complete parameter pointer type for nullable kfunc arguments, so void pointers are reported as (void *). Also add the missing structured report when an __szk memory-size argument is not a verifier-known constant. Describe the generic bpf_refcount_acquire() contract without deriving an object type from its void pointer prototype. Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/668871823f90f69896d3db27b56db2f53e481162.camel@gmail.com Link: https://lore.kernel.org/bpf/20260816015746.2632990-7-memxor@gmail.com --- kernel/bpf/btf.c | 8 ++++---- kernel/bpf/verifier.c | 22 +++++++++++++--------- 2 files changed, 17 insertions(+), 13 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 5b9d767895c9..6967d48bba49 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -1169,19 +1169,19 @@ static const char *btf_show_name(struct btf_show *show) id = t->type; break; default: - id = 0; - break; + goto resolved; } + t = btf_type_skip_qualifiers(show->btf, id); if (!id) break; - t = btf_type_skip_qualifiers(show->btf, id); } /* We may not be able to represent this type; bail to be safe */ if (i == BTF_SHOW_MAX_ITER) return ""; +resolved: if (!name) - name = btf_name_by_offset(show->btf, t->name_off); + name = btf_type_is_void(t) ? "void" : btf_name_by_offset(show->btf, t->name_off); switch (BTF_INFO_KIND(t->info)) { case BTF_KIND_STRUCT: diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 4adc13584818..a25f3c94976a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -12616,12 +12616,12 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me !type_may_be_null(kf_arg_type)) { const char *expected_type; - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + expected_type = bpf_diag_fmt_btf_type(env, btf, args[i].type); verbose(env, "Possibly NULL pointer passed to trusted %s\n", reg_arg_name(env, argno)); bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, "Add a NULL check and call the kfunc only on the non-NULL path.", - "the pointer may be NULL, but this kfunc requires a non-NULL pointer to %s", + "the pointer may be NULL, but this kfunc requires a non-NULL value of type %s", expected_type); return -EACCES; } @@ -13058,8 +13058,14 @@ check_ok: break; case KF_ARG_CONST_MEM_SIZE: ret = process_const_arg(env, reg, argno, meta); - if (ret < 0) + if (ret < 0) { + if (ret == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a compile-time constant or a value the verifier can prove is constant at this call.", + "the kfunc requires this memory size to be a verifier-known constant, but %s is variable on this path", + reg_arg_name(env, argno)); return ret; + } fallthrough; case KF_ARG_MEM_SIZE: { @@ -13123,15 +13129,13 @@ check_ok: break; case KF_ARG_PTR_TO_REFCOUNTED_KPTR: if (!type_is_ptr_alloc_obj(reg->type)) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); verbose(env, "%s is neither owning or non-owning ref\n", reg_arg_name(env, argno)); bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a pointer returned by the matching BPF object allocation or lookup operation for this kfunc.", - "the kfunc expects a pointer to BPF-managed refcounted object type %s, but this argument is not such an object pointer", - expected_type); + "Pass an owning or non-owning pointer to a BPF-managed object containing a bpf_refcount field.", + "the kfunc expects a pointer to a BPF-managed refcounted object, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } if (!type_is_non_owning_ref(reg->type)) -- cgit v1.2.3 From 6bd520a6e3b66010e6e87ef77a1756c6b6ce30b1 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 16 Aug 2026 03:57:39 +0200 Subject: bpf: Preserve source attribution without source text GCC emits BTF line records with a file name and line number, but leaves the source line string empty. bpf_diag_source() currently treats that empty string as if the complete line record were unavailable, so diagnostics fall back to an instruction number and discard the function, file, and line attribution. Print the available source location before deciding whether source context can be rendered. When source text is absent, omit only the source context and retain the diagnostic annotation and instruction context. Fixes: b9c5d822f677 ("bpf: Add source and instruction diagnostic context") Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260816015746.2632990-12-memxor@gmail.com --- kernel/bpf/diagnostics.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index b24be9df5fab..b682fd2be443 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -833,11 +833,9 @@ static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const ch linfo = bpf_find_linfo(env->prog, insn_idx); if (btf && linfo) bpf_get_linfo_source(btf, linfo, &src); - if (!src.file || !*src.file || !src.line || !*src.line) { + if (!src.file || !*src.file) { diag_write(env, " insn %u\n", insn_idx); - diag_print_source_annotation(env, 0, 0, label, msg); - diag_print_insn_context(env, insn_idx, disasm_lines); - goto out_restore; + goto out_annotation; } subprog = bpf_find_containing_subprog(env, insn_idx); @@ -847,6 +845,8 @@ static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const ch diag_write(env, " %s @ %s:%d:%d\n", func, src.file, src.line_num, src.line_col); else diag_write(env, " %s:%d:%d\n", src.file, src.line_num, src.line_col); + if (!src.line || !*src.line) + goto out_annotation; start_line = src.line_num - BPF_DIAG_CONTEXT; end_line = src.line_num + BPF_DIAG_CONTEXT; @@ -889,7 +889,11 @@ static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const ch diag_print_source_annotation(env, width, indent, label, msg); } diag_print_insn_context(env, insn_idx, disasm_lines); + goto out_restore; +out_annotation: + diag_print_source_annotation(env, 0, 0, label, msg); + diag_print_insn_context(env, insn_idx, disasm_lines); out_restore: diag_fmt_restore(env, mark); } -- cgit v1.2.3 From fc009f4658224734e6859b8f67a681ecac5a2b22 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 16 Aug 2026 03:57:41 +0200 Subject: bpf: Distinguish function references in policy diagnostics add_subprogs() rejects both BPF-to-BPF calls and BPF_PSEUDO_FUNC loads for unprivileged programs. The latter loads a subprogram address for use as a callback, but its Policy report currently describes it as a function call and suggests avoiding calls that the program does not contain. Select the operation and suggestion from the instruction kind. Preserve the existing call wording for BPF_PSEUDO_CALL, and describe BPF_PSEUDO_FUNC as a BPF function reference. Signed-off-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/bpf/d02e6a6d3b2dc43a207b8ba836ce62497b250dede9252e7409c5212201c794b7@mail.kernel.org Link: https://lore.kernel.org/bpf/20260816015746.2632990-14-memxor@gmail.com --- kernel/bpf/verifier.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a25f3c94976a..e421ea2b80c3 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2912,6 +2912,7 @@ static int add_subprogs(struct bpf_verifier_env *env) struct bpf_subprog_info *subprog = env->subprog_info; int i, ret, insn_cnt = env->prog->len, ex_cb_insn; struct bpf_insn *insn = env->prog->insnsi; + const char *operation, *suggestion; /* Add entry function. */ ret = add_subprog(env, 0); @@ -2923,11 +2924,18 @@ static int add_subprogs(struct bpf_verifier_env *env) continue; if (!env->bpf_capable) { + if (bpf_pseudo_func(insn)) { + operation = "BPF function reference"; + suggestion = "Load this program with the required capability, or avoid BPF function references in unprivileged programs."; + } else { + operation = "BPF-to-BPF function call"; + suggestion = "Load this program with the required capability, or avoid BPF-to-BPF function calls in unprivileged programs."; + } verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); bpf_diag_policy( - env, i, "BPF-to-BPF function call", + env, i, operation, "loading or calling other BPF functions requires CAP_BPF or CAP_SYS_ADMIN", - "Load this program with the required capability, or avoid BPF-to-BPF function calls in unprivileged programs."); + suggestion); return -EPERM; } -- cgit v1.2.3