From 232475c32994411bd816d6358ca48e94010fc84a Mon Sep 17 00:00:00 2001 From: Qiliang Yuan Date: Thu, 22 Oct 2026 16:48:36 +0800 Subject: bpf: Protect the struct_ops association with a spinlock bpf_prog_assoc_struct_ops() and bpf_prog_disassoc_struct_ops() update prog->aux->st_ops_assoc under st_ops_assoc_mutex, and the latter puts the map of a non-struct_ops prog while holding it. A struct_ops map is about to dissociate its progs from map_free_pre_rcu(), which bpf_map_put() calls in whatever context drops the last reference, including softirq, e.g. bpf_struct_ops_put() from tcp_cleanup_congestion_control(). A mutex can't be taken there, and the put of the last reference under the lock of one prog would take the lock of other progs. Replace the mutex with a spinlock taken with interrupts disabled, and put the map after dropping it. Signed-off-by: Qiliang Yuan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261022-selftests-bpf-struct-ops-assoc-timer-v6-1-2c0f18e82582@gmail.com --- kernel/bpf/bpf_struct_ops.c | 19 ++++++++++--------- kernel/bpf/core.c | 3 +-- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 1178acd72296..a1ae30b145a0 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -1541,10 +1541,10 @@ int bpf_prog_assoc_struct_ops(struct bpf_prog *prog, struct bpf_map *map) { struct bpf_map *st_ops_assoc; - guard(mutex)(&prog->aux->st_ops_assoc_mutex); + guard(spinlock_irqsave)(&prog->aux->st_ops_assoc_lock); st_ops_assoc = rcu_dereference_protected(prog->aux->st_ops_assoc, - lockdep_is_held(&prog->aux->st_ops_assoc_mutex)); + lockdep_is_held(&prog->aux->st_ops_assoc_lock)); if (st_ops_assoc && st_ops_assoc == map) return 0; @@ -1569,19 +1569,20 @@ int bpf_prog_assoc_struct_ops(struct bpf_prog *prog, struct bpf_map *map) void bpf_prog_disassoc_struct_ops(struct bpf_prog *prog) { + struct bpf_prog_aux *aux = prog->aux; struct bpf_map *st_ops_assoc; - guard(mutex)(&prog->aux->st_ops_assoc_mutex); + scoped_guard(spinlock_irqsave, &aux->st_ops_assoc_lock) { + st_ops_assoc = rcu_dereference_protected(aux->st_ops_assoc, + lockdep_is_held(&aux->st_ops_assoc_lock)); + if (!st_ops_assoc || st_ops_assoc == BPF_PTR_POISON) + return; - st_ops_assoc = rcu_dereference_protected(prog->aux->st_ops_assoc, - lockdep_is_held(&prog->aux->st_ops_assoc_mutex)); - if (!st_ops_assoc || st_ops_assoc == BPF_PTR_POISON) - return; + RCU_INIT_POINTER(aux->st_ops_assoc, NULL); + } if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) bpf_map_put(st_ops_assoc); - - RCU_INIT_POINTER(prog->aux->st_ops_assoc, NULL); } /* diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 3030beb531d9..0a8ba603f44b 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -141,7 +141,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag mutex_init(&fp->aux->used_maps_mutex); mutex_init(&fp->aux->ext_mutex); mutex_init(&fp->aux->dst_mutex); - mutex_init(&fp->aux->st_ops_assoc_mutex); + spin_lock_init(&fp->aux->st_ops_assoc_lock); return fp; } @@ -291,7 +291,6 @@ void __bpf_prog_free(struct bpf_prog *fp) #endif mutex_destroy(&fp->aux->used_maps_mutex); mutex_destroy(&fp->aux->dst_mutex); - mutex_destroy(&fp->aux->st_ops_assoc_mutex); kfree(fp->aux->poke_tab); kfree(fp->aux); } -- cgit v1.2.3 From 377ab5f2ef793b034d7d710212604fd62f0f00b9 Mon Sep 17 00:00:00 2001 From: Qiliang Yuan Date: Thu, 22 Oct 2026 16:48:37 +0800 Subject: bpf: Dissociate struct_ops progs before the grace period of the map A struct_ops prog is associated with its map without a reference on the map, and finds the struct_ops through bpf_prog_get_assoc_struct_ops() under RCU. For a struct_ops map, bpf_map_put() waits for an RCU grace period before it calls bpf_struct_ops_map_free(), which clears the association and then frees the trampoline and the kdata right away. Nothing waits for a prog that reads the association after that grace period started and before the association is cleared. A timer callback of a struct_ops prog that calls test_1 of bpf_testmod's multi_st_ops through the association while the map is freed returns into the freed trampoline, whose image is filled with int3. RAX holds MAP_MAGIC (1234), which test_1 has just returned: Oops: int3: 0000 [#1] SMP KASAN NOPTI CPU: 2 UID: 0 PID: 31 Comm: ksoftirqd/2 ... RIP: 0010:0xffffffffc043114d Code: cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc RAX: 00000000000004d2 ... Call Trace: bpf_prog_ec591f4f9d13f364_timer_cb+0x96/0x1e2 bpf_timer_cb+0x15d/0x240 __hrtimer_run_queues+0x264/0x5a0 hrtimer_run_softirq+0x1a7/0x3b0 handle_softirqs+0x18c/0x5b0 Dissociate the progs in map_free_pre_rcu(), so that the grace period of bpf_map_put() follows it, and make that an RCU tasks trace grace period for every struct_ops map. It then also waits for the progs that read the association under RCU tasks trace, e.g. from a bpf_wq callback or a sleepable prog, as a tasks trace grace period implies an RCU one. Stop setting free_after_rcu_gp, which bpf_map_put() no longer reads for a struct_ops map, and update the comment of free_after_mult_rcu_gp in struct bpf_struct_ops, which now only picks the grace period of a cgroup prog array. The st_ops_assoc_in_timer_free subtest of test_progs reproduces it. Its timer callback keeps calling test_1 through the association while the map is freed, and test_1 spins in bpf_loop() when called from there: $ cd tools/testing/selftests/bpf $ make test_progs $ ./test_progs -t struct_ops_assoc/st_ops_assoc_in_timer_free On kernels of the same bpf-next commit with KASAN enabled, x86_64 VM with 32 vCPUs, before and after this patch: before after outcome Oops (int3) on run 1 100 of 100 runs passed, no Oops Fixes: 5db69b0fbdc8 ("bpf: Make struct_ops tasks_rcu grace period optional") Signed-off-by: Qiliang Yuan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261022-selftests-bpf-struct-ops-assoc-timer-v6-2-2c0f18e82582@gmail.com --- kernel/bpf/bpf_struct_ops.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index a1ae30b145a0..7c5cd6086e27 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -1030,6 +1030,7 @@ static void bpf_struct_ops_map_free_pre_rcu(struct bpf_map *map) struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; bpf_struct_ops_map_del_ksyms(st_map); + bpf_struct_ops_map_dissoc_progs(st_map); } static void bpf_struct_ops_map_free(struct bpf_map *map) @@ -1045,8 +1046,6 @@ static void bpf_struct_ops_map_free(struct bpf_map *map) if (btf_is_module(st_map->btf)) module_put(st_map->st_ops_desc->st_ops->owner); - bpf_struct_ops_map_dissoc_progs(st_map); - if (tasks_rcu && IS_ENABLED(CONFIG_TASKS_RCU)) synchronize_rcu_tasks(); @@ -1162,8 +1161,11 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) mutex_init(&st_map->lock); bpf_map_init_from_attr(map, attr); - map->free_after_mult_rcu_gp = st_ops_desc->st_ops->free_after_mult_rcu_gp; - map->free_after_rcu_gp = true; + /* + * A prog dissociated in map_free_pre_rcu() may still run the struct_ops + * from a bpf_wq callback or a sleepable prog, under RCU tasks trace. + */ + map->free_after_mult_rcu_gp = true; return map; -- cgit v1.2.3 From fb2b27d214c3b88d926a303267dd83cfdb428b13 Mon Sep 17 00:00:00 2001 From: Feng Yang Date: Thu, 22 Oct 2026 14:47:27 +0800 Subject: bpf: Propagate btf_add_type() errors in btf_check_all_metas() btf_check_all_metas() ignores the return value of btf_add_type(), so a type that could not be registered is silently dropped while the loop keeps walking over the type section. Both failure modes are reachable: - -E2BIG. Once the types array reaches its capacity (BTF_MAX_TYPE), every following btf_add_type() fails the same way, so btf_check_all_metas() still returns 0 and the BTF object is created with btf->nr_types capped while hdr->type_len claims more records. - -ENOMEM. btf->nr_types is not incremented and the array is not grown, so the next type is recorded at the slot of the dropped one and btf->types[] stays off by one with respect to the type ids for all remaining types. btf_type_by_id() then returns the wrong type for those ids for the whole lifetime of the object, and the last type is silently left unresolved. Propagate the error so that the object is rejected instead. Fixes: 69b693f0aefa ("bpf: btf: Introduce BPF Type Format (BTF)") Signed-off-by: Feng Yang Signed-off-by: Alexei Starovoitov Reviewed-by: Mahe Tardy Acked-by: Alan Maguire Link: https://patch.msgid.link/20261022064727.747304-1-yangfeng59949@163.com --- kernel/bpf/btf.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 538129ed2418..94d43612d211 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -5871,6 +5871,7 @@ static int btf_check_all_metas(struct btf_verifier_env *env) struct btf *btf = env->btf; struct btf_header *hdr; void *cur, *end; + int err; hdr = &btf->hdr; cur = btf->nohdr_data + hdr->type_off; @@ -5885,7 +5886,10 @@ static int btf_check_all_metas(struct btf_verifier_env *env) if (meta_size < 0) return meta_size; - btf_add_type(env, t); + err = btf_add_type(env, t); + if (err) + return err; + cur += meta_size; env->log_type_id++; } -- cgit v1.2.3 From 93bd472336ac2424f5f3cf05cbb24e02242343b7 Mon Sep 17 00:00:00 2001 From: Aditya Sharma Date: Thu, 1 Oct 2026 12:54:48 +0530 Subject: bpf: Remove nmi_uaccess_okay() check in bpf_send_signal_common() nmi_uaccess_okay() takes no task argument and is a statement about current. When commit 6280cf718db0 ("bpf: Implement bpf_send_signal_task() kfunc") made the PF_KTHREAD/PF_EXITING and is_global_init() test the supplied task, nmi_uaccess_okay() was left testing current. As a result bpf_send_signal_task() returns -EPERM whenever the calling context happens to be a kernel thread, regardless of which task the signal is aimed at. The same call with the same target succeeds or fails depending only on what the CPU was running: bpf_send_signal_task() from tp_btf/workqueue_execute_start : -EPERM bpf_send_signal_task() from tp_btf/sys_enter : 0 The check was carried over from bpf_probe_write_user() when bpf_send_signal() was added in commit 8b401f9ed244 ("bpf: implement bpf_send_signal() helper"). There it guards a user memory write, but bpf_send_signal_common() does no user memory access, so the check has nothing to guard here. It is also already a no-op outside x86. Remove it, and drop the part of the comment above it that described the memory access checks, as none remain. Fixes: 6280cf718db0 ("bpf: Implement bpf_send_signal_task() kfunc") Suggested-by: Andrii Nakryiko Signed-off-by: Aditya Sharma Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20260819124324.43162-1-adi.sharma@zohomail.in/T/#u Link: https://patch.msgid.link/20261001072449.1377267-2-adi.sharma@zohomail.in --- kernel/trace/bpf_trace.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index dfac5e1b8f40..14adfb85fdb2 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -867,15 +867,12 @@ static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struc siginfo = &info; } - /* Similar to bpf_probe_write_user, task needs to be - * in a sound condition and kernel memory access be - * permitted in order to send signal to the current - * task. + /* + * Similar to bpf_probe_write_user, task needs to be + * in a sound condition. */ if (unlikely(task->flags & (PF_KTHREAD | PF_EXITING))) return -EPERM; - if (unlikely(!nmi_uaccess_okay())) - return -EPERM; /* Task should not be pid=1 to avoid kernel panic. */ if (unlikely(is_global_init(task))) return -EPERM; -- cgit v1.2.3 From d09c2aa3109f5cb1d0be7d382231ab5d58583cad Mon Sep 17 00:00:00 2001 From: Xu Yunxiang Date: Sat, 24 Oct 2026 18:06:46 +0800 Subject: bpf: Track borrowed iterator results by default An open-coded iterator can return a struct pointer borrowed from its current element. The verifier has not tied such results to iterator state since the task_vma iterator was introduced, so a saved result can still be used after destroy. Commit bee9ef4a40a27 ("bpf: switch task_vma iterator from mmap_lock to per-VMA locks") also made next release the protection of the prior element. A later snapshot conversion made vm_file explicitly borrowed from that element. Associate non-RCU, non-acquired struct results with the iterator reference by default. Invalidate dependents before next forks its NULL and non-NULL paths, while retaining the iterator itself. Let destroy propagate child-reference errors. Add KF_ITER_RET_INDEPENDENT for results that outlive the current element, preserving existing RCU and acquired-reference rules. Trusted pointer field loads also need to preserve the source lifetime. Loading a trusted field does not acquire a reference to the pointed-to object. Set the field pointer's parent_id to the source's reference ID if the source is referenced, or inherit its parent_id if it is borrowed. This is a generic rule, including accesses unrelated to iterators. For example, the following access is now rejected: file = bpf_get_task_exe_file(task); if (!file) return 0; inode = file->f_inode; bpf_put_file(file); ino = inode->i_ino; The inode pointer borrows the file's lifetime. Releasing the acquired file reference removes that lifetime guarantee, so the verifier must invalidate the derived inode pointer as well. This does not imply that bpf_put_file() immediately frees the file or inode. Reading the inode while the file reference is held remains valid. Fields with independent RCU protection retain their existing lifetime rules. For an iterator result, the same propagation keeps trusted fields such as vm_file and f_inode tied to the current element. Propagate the backing object's lifetime through dynptr construction too, so its descendants remain in the invalidation graph. Share the invalidation traversal between next and release, so advancing the iterator preserves its own reference. Fixes: 4ac454682158 ("bpf: Introduce task_vma open-coded iterator kfuncs") Suggested-by: Amery Hung Signed-off-by: Xu Yunxiang Signed-off-by: Alexei Starovoitov Reviewed-by: Amery Hung Link: https://patch.msgid.link/20261024100647.2579244-2-xyx2021@mail.ustc.edu.cn --- kernel/bpf/btf.c | 13 +++++++++- kernel/bpf/verifier.c | 68 ++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 68 insertions(+), 13 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 94d43612d211..7ed1ff71a089 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -9720,6 +9720,10 @@ static int btf_check_iter_kfuncs(struct btf *btf, const char *func_name, if (!flags || (flags & (flags - 1))) return -EINVAL; + if ((func_flags & KF_ITER_RET_INDEPENDENT) && + (!(flags & KF_ITER_NEXT) || (func_flags & KF_ACQUIRE))) + return -EINVAL; + /* any BPF iter kfunc should have `struct bpf_iter_ *` first arg */ nr_args = btf_type_vlen(func); if (nr_args < 1) @@ -9760,6 +9764,12 @@ static int btf_check_iter_kfuncs(struct btf *btf, const char *func_name, t = btf_type_skip_modifiers(btf, func->type, NULL); if (!t || !btf_type_is_ptr(t)) return -EINVAL; + /* The opt-out applies only to non-acquired struct results. */ + if (func_flags & KF_ITER_RET_INDEPENDENT) { + t = btf_type_skip_modifiers(btf, t->type, NULL); + if (!t || !__btf_type_is_struct(t)) + return -EINVAL; + } } if (flags & KF_ITER_DESTROY) { @@ -9826,7 +9836,8 @@ static int btf_check_kfunc_protos(struct btf *btf, u32 func_id, u32 func_flags) if (!func || !btf_type_is_func_proto(func)) return -EINVAL; - if (func_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY)) { + if (func_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY | + KF_ITER_RET_INDEPENDENT)) { err = btf_check_iter_kfuncs(btf, func_name, func, func_flags); if (err) return err; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5601d2056218..c223bf3bc9cd 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -211,6 +211,9 @@ static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int par static int __release_reference_nomark(struct bpf_verifier_state *state, int id); static int release_reference_nomark(struct bpf_verifier_env *env, int id); static int release_reference(struct bpf_verifier_env *env, int id); +static int invalidate_reference(struct bpf_verifier_env *env, int id); +static u32 reg_lifetime_id(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg); static void invalidate_non_owning_refs(struct bpf_verifier_env *env); static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env); static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); @@ -793,8 +796,8 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ if (err) return err; - /* Track parent's id if the parent is a referenced object */ - parent_id = ref_obj->id; + /* All non-clone constructors take their backing object in R1. */ + parent_id = reg_lifetime_id(env, &cur_regs(env)[BPF_REG_1]); if (dynptr_type_referenced(type)) { int id; @@ -1074,7 +1077,7 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots) { struct bpf_func_state *state = bpf_func(env, reg); - int spi, i, j; + int spi, i, j, err; spi = iter_get_spi(env, reg, nr_slots); if (spi < 0) @@ -1084,8 +1087,11 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; - if (i == 0) - WARN_ON_ONCE(release_reference(env, st->id)); + if (i == 0) { + err = release_reference(env, st->id); + if (err) + return err; + } bpf_mark_reg_not_init(env, st); @@ -1627,6 +1633,12 @@ static bool reg_is_referenced(struct bpf_verifier_env *env, const struct bpf_reg return find_reference_state(env->cur_state, reg->id); } +static u32 reg_lifetime_id(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg) +{ + return reg_is_referenced(env, reg) ? reg->id : reg->parent_id; +} + static int release_lock_state(struct bpf_verifier_env *env, int type, int id, void *ptr) { struct bpf_verifier_state *state = env->cur_state; @@ -6848,9 +6860,14 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, flag &= ~PTR_MAYBE_NULL; if (atype == BPF_READ && value_regno >= 0) { + u32 parent_id = reg_lifetime_id(env, reg); + ret = mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); if (ret < 0) return ret; + if ((regs[value_regno].type & PTR_TRUSTED) && + !(regs[value_regno].type & (MEM_RCU | MEM_PERCPU | MEM_USER))) + regs[value_regno].parent_id = parent_id; } return 0; @@ -8636,6 +8653,21 @@ static bool is_iter_destroy_kfunc(struct bpf_call_arg_meta *meta) return meta->kfunc_flags & KF_ITER_DESTROY; } +static bool iter_next_borrows_struct(struct bpf_call_arg_meta *meta) +{ + const struct btf_type *t; + + if (!(meta->kfunc_flags & KF_ITER_NEXT) || + (meta->kfunc_flags & (KF_ITER_RET_INDEPENDENT | KF_ACQUIRE))) + return false; + + t = btf_type_skip_modifiers(meta->btf, meta->func_proto->type, NULL); + if (!t || !btf_type_is_ptr(t)) + return false; + t = btf_type_skip_modifiers(meta->btf, t->type, NULL); + return t && __btf_type_is_struct(t); +} + static bool is_kfunc_arg_iter(struct bpf_call_arg_meta *meta, int arg_idx, const struct btf_param *arg) { @@ -8746,6 +8778,13 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * meta->iter.frameno = reg->frameno; update_ref_obj(&meta->ref_obj, &bpf_stack_slot(state, spi)->spilled_ptr); + /* The iterator stays. What was borrowed from the previous element goes. */ + if (iter_next_borrows_struct(meta)) { + err = invalidate_reference(env, meta->ref_obj.id); + if (err) + return err; + } + if (is_iter_destroy_kfunc(meta)) { err = unmark_stack_slots_iter(env, reg, nr_slots); if (err) @@ -11073,8 +11112,8 @@ static int idstack_pop(struct bpf_idmap *idmap) return idmap->map[--idmap->cnt].old; } -/* Release id and objects derived from it iteratively in a DFS manner */ -static int release_reference(struct bpf_verifier_env *env, int id) +/* Invalidate id and objects derived from it iteratively in a DFS manner */ +static int invalidate_reference(struct bpf_verifier_env *env, int id) { u32 mask = (1 << STACK_SPILL) | (1 << STACK_DYNPTR); struct bpf_verifier_state *vstate = env->cur_state; @@ -11089,11 +11128,6 @@ static int release_reference(struct bpf_verifier_env *env, int id) if (err) return err; - if (find_reference_state(vstate, id)) { - err = release_reference_nomark(env, id); - WARN_ON_ONCE(err); - } - while ((id = idstack_pop(idstack))) { /* * Child references are inaccessible after parent is released, @@ -11145,6 +11179,14 @@ static int release_reference(struct bpf_verifier_env *env, int id) return 0; } +static int release_reference(struct bpf_verifier_env *env, int id) +{ + if (find_reference_state(env->cur_state, id)) + WARN_ON_ONCE(release_reference_nomark(env, id)); + + return invalidate_reference(env, id); +} + static void invalidate_non_owning_refs(struct bpf_verifier_env *env) { struct bpf_func_state *unused; @@ -15822,6 +15864,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, regs[BPF_REG_0].btf = desc_btf; regs[BPF_REG_0].type = type; regs[BPF_REG_0].btf_id = ptr_type_id; + if (iter_next_borrows_struct(&meta) && !(type & MEM_RCU)) + regs[BPF_REG_0].parent_id = meta.ref_obj.id; } if (is_kfunc_ret_null(&meta)) { -- cgit v1.2.3 From 90aedf8854c05713c02b381c41c59081b5cedbcc Mon Sep 17 00:00:00 2001 From: Weiming Shi Date: Sun, 25 Oct 2026 18:52:49 -0700 Subject: bpf: Validate stack arg reads against caller state check_stack_arg_read() bounds a stack argument index with the callee's BTF-derived argument count, then uses the index in the caller's stack_arg_regs[] array. The array's actual logical length is recorded in caller->out_stack_arg_cnt, so the bound and indexed object can diverge. A crafted BPF_PROG_LOAD can use an argument mismatch to mark a static subprogram's BTF information unreliable. Later calls return from btf_check_subprog_call() before check_outgoing_stack_args(), while check_func_call() continues the static call on -EINVAL. The callee can then read a slot that the caller never allocated or initialized. The earlier commit 4286f5deee14 ("bpf: Validate outgoing stack args when btf_prepare_func_args fails") closed this hole for the first call, but not for later calls after the subprogram has been marked unreliable. Callbacks have a related problem. The verifier can make incoming stack arguments readable from outgoing slots at the helper call site. However, helpers and kfuncs invoke callbacks through an ABI that passes at most five arguments, so the runtime callback receives no stack arguments. Async callback verification can also reach check_stack_arg_read() without a caller frame. On x86-64, KASAN reports the direct-call out-of-bounds copy as: BUG: KASAN: slab-out-of-bounds in check_load_mem+0xb51/0xc60 Read of size 80 at addr ffff8881009af550 by task poc/131 Call Trace: check_stack_arg_read at kernel/bpf/verifier.c:4234 (inlined by) check_load_mem at kernel/bpf/verifier.c:6645 do_check_common at kernel/bpf/verifier.c:19568 bpf_check at kernel/bpf/verifier.c:21318 bpf_prog_load at kernel/bpf/syscall.c:3134 Reject callback subprograms with incoming stack arguments before their state is pushed. For direct calls, retrieve the slot with bpf_get_spilled_stack_arg(), which validates the index against the caller's out_stack_arg_cnt and rejects uninitialized slots before copying the register state. Fixes: 0f6bd5e7a804 ("bpf: Support stack arguments for bpf functions") Reported-by: Signed-off-by: Weiming Shi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026015250.1674676-1-bestswngs@gmail.com --- kernel/bpf/verifier.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c223bf3bc9cd..a0a5f075d201 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4666,7 +4666,12 @@ static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_st } caller = vstate->frame[vstate->curframe - 1]; - arg = &caller->stack_arg_regs[spi]; + arg = bpf_get_spilled_stack_arg(spi, caller); + if (!arg) { + verbose(env, "invalid read from uninitialized stack arg off %d\n", off); + return -EACCES; + } + cur = vstate->frame[vstate->curframe]; bpf_diag_mod_begin(env, &cur->regs[dst_regno], arg, BPF_DIAG_MOD_WRITE); cur->regs[dst_regno] = *arg; @@ -11451,6 +11456,10 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins err = btf_check_subprog_call(env, subprog); if (err == -EFAULT) return err; + if (bpf_in_stack_arg_cnt(&env->subprog_info[subprog])) { + verbose(env, "callback subprog cannot have stack args\n"); + return -EINVAL; + } /* set_callee_state is used for direct subprog calls, but we are * interested in validating only BPF helpers that can call subprogs as -- cgit v1.2.3 From c463b378eae7fa4d8013d3ba7ca224f3a5098ec4 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:21 -0700 Subject: bpf: Reject pointer operands for 32-bit recurrences/latches Avoid inferring loop bounds for the following program shapes: 32-bit latch (A) | 32-bit recurrence (B) -------------------------+-------------------------- r1 = ... pointer ... | r1 = ... pointer ... r2 = ... pointer ... | r2 = ... pointer ... l: r1 += 8 | l: w1 += 8 ... | ... if w1 < w2 goto l | if r1 < r2 goto l (A) Scalar bounds for pointers represent only offsets from their common origin values. The 32-bit operations also depend on the runtime base address: the addresses can wrap at 32 bits without the tracked offsets wrapping. Thus, offsets alone do not determine the iteration count. (B) 32-bit operations on pointers either prohibited (wX += 1) or convert them to scalars (wX = wX). No need to deal with this at SCEV level. Fixes: 32e962577d2d ("bpf: Support 32-bit linear scalar evolution") Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-1-f373aec9825f@gmail.com --- kernel/bpf/scev.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index fea77772ca96..f9e2ca6a5a12 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -2024,7 +2024,8 @@ static void log_incompatible_scev_values(struct bpf_verifier_env *env, const cha } static bool scev_value_compatible(struct bpf_verifier_env *env, struct bpf_func_state *st, - const struct scev_value *a, const struct scev_value *b) + const struct scev_value *a, const struct scev_value *b, + const struct linear_latch *latch) { const struct bpf_reg_state *ra, *rb; @@ -2035,7 +2036,7 @@ static bool scev_value_compatible(struct bpf_verifier_env *env, struct bpf_func_ } ra = scev_reg_for_read(env, st, a->ptr_reg); rb = scev_reg_for_read(env, st, b->ptr_reg); - if (ra && rb && bpf_same_memory_origin(ra, rb)) + if (ra && rb && bpf_same_memory_origin(ra, rb) && !latch->zext32 && !latch->jmp32) return true; incompatible: @@ -2267,7 +2268,7 @@ static bool compute_max_iters(struct bpf_verifier_env *env, } return false; } - if (!scev_value_compatible(env, st, &initial_val, &bound_val)) + if (!scev_value_compatible(env, st, &initial_val, &bound_val, latch)) return false; initial = initial_val.value; @@ -2700,6 +2701,9 @@ int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_sta reg = scev_reg_for_linear(cur_func, r); /* If SCEV for r is (linear ) */ if (reg && is_simple_linear(scev, r_scev, &coefs) && is_widenable_reg_type(reg)) { + /* 32-bit ALU operations on pointers convert them to scalars */ + if (coefs.zext32 && reg->type != SCALAR_VALUE) + goto cant_widen; /* Spills at varying offsets lose precision */ if (test_bit(r, loop->store_base_regs)) { spill_base = true; -- cgit v1.2.3 From 0c33d612bdd50aaeef263410739060827377d9c0 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:22 -0700 Subject: bpf: Do not intersect with current register range on clamp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scev.c:bpf_clamp_scev_regs() is called each time verifier returns to a loop header via a backedge. It's purpose is to re-establish computed variable ranges for registers with linear SCEVs. It can be defined as: - For each register R with linear SCEV compute: - loop_range(R) := entry_range(R) * iterations_count - range(R) := range(R) ∩ loop_range(R) Such formulation is necessary to avoid ranges over-approximation in examples like below: r1 = 0; SCEVs: r1 = (linear r1 1) r3 = 0; r3 = (linear r3 1) loop: r3 += 1; r1 += 1; if r1 < 5 goto loop; r0 = 0; exit; - At loop entry r3 is widened to be in a range r3 ∈ {0,1,2,3,4}. - After increment it becomes r3 ∈ {1,2,3,4,5}. - On a second iteration the bpf_clamp_scev_regs() forces r3 to become r3 ∈ {1,2,3,4}. W/o clamping logic r3 range wouldn't shrink preventing loop from converging in states.c:states_equal(). However, intersection logic might lead to assigning an empty range to a register, as in example below: r1 = 0; verification r3 = 0; path loop: r3 += 1; | <---------. if r3 != 5 goto +0; | r3 ∈ {5} | r1 += 1; | | if r1 != 5 goto loop; '-----------' r0 = 0; exit; - At header r3 ∈ {0,1,2,3,4}. - While exploring the branch `if r3 != 5 goto ...` with r3 ∈ {5}. - On a second iteration: r3 := {0,1,2,3,4} ∩ {5} = ∅ Which stops verification with the following error: verifier bug: scev clamp produced empty range for r3 Technically this means that the current verification path is impossible at runtime and should not be traced further. This commit mitigates the above by removing the intersection logic, the function is now defined as: - For each register R with linear SCEV compute: - range(R) := entry_range(R) * iterations_count This is still sound, provided that SCEV expressions loop execution counts are correct. Fixes: 6ccf49f79327 ("bpf: Use SCEV to widen bounded loops") Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-2-f373aec9825f@gmail.com --- kernel/bpf/scev.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index f9e2ca6a5a12..d67e793d7a18 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -2896,12 +2896,6 @@ int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur continue; linear_bounds(entry_reg, max_header_count, &coefs, &bounds); - bounds.r64 = cnum64_intersect(reg->r64, bounds.r64); - bounds.r32 = cnum32_intersect(reg->r32, bounds.r32); - if (cnum64_is_empty(bounds.r64)) { - verifier_bug(env, "scev clamp produced empty range for r%d", r); - return -EFAULT; - } if (log->level & BPF_LOG_LEVEL2) { bpf_log(log, "loop header at %d, clamping ", insn_idx); log_reg(env, r); -- cgit v1.2.3 From 5dd32c8c108226912054e293ee306c98c283c512 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:23 -0700 Subject: bpf: Avoid false infinite loop reports after changes in SCEV clamping Clamping now reproduces the widened range at every back edge, so induction variable ranges no longer vary between iterations. This can produce identical checkpoint states in terminating programs, for example: int main(...) { for (int i = 0; i < 10; i++) { foo(...); for (int j = 0; j < 3; j++) ...; } } void foo(...) { ...; } Handle two cases in states.c:bpf_is_state_visited(): - For checkpoints inside foo(), look for active loops in upper frames as well. Otherwise current_loop() returns NULL and an identical callee state can trigger an infinite loop report. - For checkpoints inside the inner loop, skip the infinite loop check when the current state is inside a terminating loop in any frame. That loop bounds how many times its body is re-executed, so an identical state on a later iteration does not imply nontermination. This covers the inner loop above, re-entered with a fresh entry state on each outer iteration, and inner loops SCEV cannot analyze. The nested_terminating_prune test still rejects such an outer loop, but now reaches the jump complexity limit instead of reporting an infinite loop. Update its expected message accordingly. Fixes: 6ccf49f79327 ("bpf: Use SCEV to widen bounded loops") Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-3-f373aec9825f@gmail.com --- kernel/bpf/states.c | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index d7df6632a5c4..a6ae86f973e3 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -1358,13 +1358,33 @@ static bool need_loop_checkpoint(struct bpf_verifier_env *env, int insn_idx) return top->loop_id == insn_idx && top->terminates; } +/* Searches for an innermost loop entry starting from an innermost frame. */ static struct loop_stack_entry *current_loop(struct bpf_verifier_state *st) { - struct bpf_func_state *frame = st->frame[st->curframe]; + struct bpf_func_state *frame; + int fr; - if (frame->loop_stack_cnt == 0) - return NULL; - return &frame->loop_stack[frame->loop_stack_cnt - 1]; + for (fr = st->curframe; fr >= 0; fr--) { + frame = st->frame[fr]; + if (frame->loop_stack_cnt) + return &frame->loop_stack[frame->loop_stack_cnt - 1]; + } + return NULL; +} + +/* True if any of the loops call chain is currently in terminates. */ +static bool in_a_terminating_loop(struct bpf_verifier_state *st) +{ + struct bpf_func_state *frame; + int l, fr; + + for (fr = st->curframe; fr >= 0; fr--) { + frame = st->frame[fr]; + for (l = frame->loop_stack_cnt - 1; l >= 0; l--) + if (frame->loop_stack[l].terminates) + return true; + } + return false; } int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) @@ -1528,10 +1548,10 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) */ old_loop = current_loop(&sl->state); cur_loop = current_loop(cur); - if (old_loop && cur_loop && - old_loop->terminates && cur_loop->terminates && - old_loop->entry_state == cur_loop->entry_state) { - if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { + if (old_loop && cur_loop && in_a_terminating_loop(cur)) { + if (old_loop->terminates && cur_loop->terminates && + old_loop->entry_state == cur_loop->entry_state && + states_equal(env, &sl->state, cur, RANGE_WITHIN)) { loop = true; goto hit; } -- cgit v1.2.3 From 86d654957d40111ec158aae38d7f7b0bc119533c Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:24 -0700 Subject: bpf: Check the iteration count before the predicted exit value The comment before `total_change` computation: /* * compute_backedge_taken_count() guarantees that 'total_change' * does not overflow in 64-bit domain. */ total_change = initial + step * backedge_taken_count; Is not true before the check: if (backedge_taken_count >= U32_MAX - 1) return false; Move the check before `total_change` computation in order to reconcile the code with comments. Old position is not a bug the check was performed either way. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-4-f373aec9825f@gmail.com --- kernel/bpf/scev.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index d67e793d7a18..e3c2220f89c9 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -2307,11 +2307,15 @@ static bool compute_max_iters(struct bpf_verifier_env *env, if (!compute_backedge_taken_count(op, initial, step, bound, &backedge_taken_count)) return false; + /* + * max_header_count is u32 and U32_MAX means "infinite", + * check to avoid overflow below. + */ + if (backedge_taken_count >= U32_MAX - 1) + return false; + if (latch->zext32 || latch->jmp32) { - /* - * compute_backedge_taken_count() guarantees that 'total_change' - * does not overflow in 64-bit domain. - */ + /* total_change is guaranteed not to overflow at this point */ total_change = initial + step * backedge_taken_count; if (latch->jmp32 && is_signed_op(op)) { if ((s64)total_change < S32_MIN || (s64)total_change > S32_MAX) @@ -2322,13 +2326,6 @@ static bool compute_max_iters(struct bpf_verifier_env *env, } } - /* - * max_header_count is u32 and U32_MAX means "infinite", - * check to avoid overflow below. - */ - if (backedge_taken_count >= U32_MAX - 1) - return false; - /* The header executes once on entry and once for each taken backedge, hence +1. */ iters->max_header_count = backedge_taken_count + 1; /* -- cgit v1.2.3 From abc9a76a888f813b698f5e2a0166ace6a0d976c6 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:25 -0700 Subject: bpf: (zext32 (zext32 X)) -> (zext32 X) SCEV simplification rule The compiler can preserve an induction variable across a call by copying it to a callee-saved register. For example: w1 = 0; loop: w7 = w1; call foo; w1 = w7; w1 += 2; if w7 < 10 goto loop; When building expressions for the loop body, each 32-bit move adds a zero extension: w7 = w1; r7 = (zext32 r1) w1 = w7; r1 = (zext32 (zext32 r1)) w1 += 2; r1 = (zext32 (+ (zext32 (zext32 r1)) 2)) The existing ALU32 simplification rules allow recognizing the header SCEV for r1 as: (zext32 (linear r1 2)) Substituting this SCEV into r7's expression at the latch produces: (zext32 (zext32 (linear r1 2))) Add the simplification rule: (zext32 (zext32 X)) -> (zext32 X) This allows recognizing the saved induction variable in r7 as a 32-bit linear SCEV and computing the loop iteration bound. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-5-f373aec9825f@gmail.com --- kernel/bpf/scev.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index e3c2220f89c9..12b147faa32a 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -1502,6 +1502,10 @@ static int simplify(struct scev *scev, u32 id, void *priv) return t; } + /* (zext32 (zext32 X)) -> (zext32 X) */ + if (is_zext32(scev, id, &a) && is_zext32(scev, a, &b)) + return a; + return id; } -- cgit v1.2.3 From 899cbaae6e4c6f0a1dee01426b53e71c58d396c4 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:26 -0700 Subject: bpf: Print 32-bit SCEV bounds as 32-bit numbers - Print large 32-bit bounds without sign extension. - Omit 32-bit bounds when they coincide with 64-bit bounds. Before: loop header at ..., widening r0 to 0x80000000..0xffffffff (s32 0xffffffff80000000..-1) step 1 loop header at ..., widening r0 to 0..3 (s32 0..3) step 1 After: loop header at ..., widening r0 to 0x80000000..0xffffffff (s32 0x80000000..-1) step 1 loop header at ..., widening r0 to 0..3 step 1 Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-6-f373aec9825f@gmail.com --- kernel/bpf/log.c | 8 ++++++++ kernel/bpf/scev.c | 8 +++++--- 2 files changed, 13 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index 5432c0fc442b..fa29d9b994e4 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -535,6 +535,14 @@ void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num) verbose(env, "%#llx", num); } +void bpf_verbose_snum32(struct bpf_verifier_env *env, s32 num) +{ + if (is_snum_decimal(num)) + verbose(env, "%d", num); + else + verbose(env, "%#x", (u32)num); +} + int tnum_strn(char *str, size_t size, struct tnum a) { /* print as a constant, if tnum is fully known */ diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 12b147faa32a..04d4962448e2 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -2456,11 +2456,13 @@ static void log_bounds(struct bpf_verifier_env *env, const struct bounds *bounds bpf_verbose_snum(env, cnum64_smin(bounds->r64)); bpf_log(log, ".."); bpf_verbose_snum(env, cnum64_smax(bounds->r64)); - if (bounds->r32.size < U32_MAX) { + if (bounds->r32.size < U32_MAX && + (cnum32_smin(bounds->r32) != cnum64_smin(bounds->r64) || + cnum32_smax(bounds->r32) != cnum64_smax(bounds->r64))) { bpf_log(log, " (s32 "); - bpf_verbose_snum(env, cnum32_smin(bounds->r32)); + bpf_verbose_snum32(env, cnum32_smin(bounds->r32)); bpf_log(log, ".."); - bpf_verbose_snum(env, cnum32_smax(bounds->r32)); + bpf_verbose_snum32(env, cnum32_smax(bounds->r32)); bpf_log(log, ")"); } bpf_log(log, " step %u", bounds->step); -- cgit v1.2.3 From 815a0a2a40d6c5985a153041231731651bc4024b Mon Sep 17 00:00:00 2001 From: Mani Tofigh Date: Tue, 27 Oct 2026 13:03:42 -0400 Subject: bpf: Fix bpf_timer callback running on a freed map A bpf_timer in an inner map can fire after the map has been freed, and bpf_timer_cb() then reads the freed map: [ 50.709520] BUG: KASAN: slab-use-after-free in bpf_timer_cb+0x249/0x290 [ 50.709540] Read of size 4 at addr ff11000003bc1030 by task bpftimer/72 ... [ 50.710783] Freed by task 12: ... [ 50.710826] bpf_map_free+0x1a2/0x380 When user space closes the map, its timers are cancelled and their callbacks are cleared. Since commit 1bfbc267ec91 ("bpf: Enable bpf_timer and bpf_wq in any context"), bpf_timer_set_callback() and bpf_timer_start() run without a lock, so a program that found the timer just before the close can still use it afterwards: Task A Task B Task C ------ ------ ------ bpf_timer_start() finds the timer bpf_timer_set_callback() finds the timer close(map_fd) clears the callback cancels the timer sets the callback arms the timer deletes the map from the outer map map is freed the timer fires bpf_timer_cb() reads the freed map The timer is cancelled once more when it is finally freed, but that happens well after the inner map is freed, so a timer started this way can still fire on the freed map. Clear the timer's map pointer in bpf_async_cancel_and_free() before the map can be freed, and make the bpf_timer and bpf_wq callbacks return early when it is NULL. The map is not freed until callbacks that are already running have finished, so a callback either sees the map while it is still valid or sees NULL. The reproducer below calls the two helpers in a loop on two CPUs while a third CPU keeps creating an inner map with a timer, closing it and removing it from the outer map. This bug was identified and confirmed with the help of a deterministic hypervisor currently under development, but the non-deterministic C reproducer can typically trigger it in the 1-50s time range. With the fix, nothing is triggered after 600 seconds. Fixes: 1bfbc267ec91 ("bpf: Enable bpf_timer and bpf_wq in any context") Signed-off-by: Mani Tofigh Signed-off-by: Alexei Starovoitov Tested-by: Mahe Tardy Link: https://patch.msgid.link/20261027-bpf-timer-freed-map-v1-1-30402b0ee99b@columbia.edu --- kernel/bpf/helpers.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 648abe1ba63a..58e357302850 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1221,7 +1221,7 @@ static void bpf_async_refcount_put(struct bpf_async_cb *cb); static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer) { struct bpf_hrtimer *t = container_of(hrtimer, struct bpf_hrtimer, timer); - struct bpf_map *map = t->cb.map; + struct bpf_map *map = READ_ONCE(t->cb.map); void *value = t->cb.value; bpf_callback_t callback_fn; void *key; @@ -1229,7 +1229,7 @@ static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer) BTF_TYPE_EMIT(struct bpf_timer); callback_fn = rcu_dereference_check(t->cb.callback_fn, rcu_read_lock_bh_held()); - if (!callback_fn) + if (!callback_fn || !map) goto out; /* bpf_timer_cb() runs in hrtimer_run_softirq. It doesn't migrate and @@ -1254,7 +1254,7 @@ static void bpf_wq_work(struct work_struct *work) { struct bpf_work *w = container_of(work, struct bpf_work, work); struct bpf_async_cb *cb = &w->cb; - struct bpf_map *map = cb->map; + struct bpf_map *map; bpf_callback_t callback_fn; void *value = cb->value; void *key; @@ -1265,8 +1265,9 @@ static void bpf_wq_work(struct work_struct *work) /* keeps the program that callback_fn belongs to */ rcu_read_lock_trace(); + map = READ_ONCE(cb->map); callback_fn = READ_ONCE(cb->callback_fn); - if (!callback_fn) + if (!callback_fn || !map) goto out; key = map_key_from_value(map, value, &idx); @@ -1733,6 +1734,8 @@ static void bpf_async_cancel_and_free(struct bpf_async_kern *async) if (!cb) return; + /* Callbacks that run after this see NULL and return. */ + WRITE_ONCE(cb->map, NULL); bpf_async_update_prog_callback(cb, NULL, NULL); /* * No refcount_inc_not_zero(&cb->refcnt) here. Dropping the last -- cgit v1.2.3 From b62dee3a8bb5d6c53eef8c727aceae369fbdb3a8 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:00 +0000 Subject: bpf: Remove BPF_F_ARENA_SCALAR Rust has no address spaces. Thus a prog in Rust accesses arena through scalars. With BPF_F_ARENA_SCALAR the verifier accepts each load and each store through a scalar as an access to arena. The verifier rewrites one insn at a time. Thus it must add one insn to each access. The compiler knows the source of each pointer. It can emit one addr_space_cast for each pointer, as it does for the __arena pointers of C. flag: one move for each access cast: one cast for each pointer ax = (u32)r1 r1 = addr_space_cast(r1, 0, 1) r2 = *(u64 *)(ax + 0) r2 = *(u64 *)(r1 + 0) ax = (u32)r1 r3 = *(u64 *)(r1 + 8) r3 = *(u64 *)(ax + 8) 20 progs of rust-bpf, with the flag and with the casts that a prototype of an LLVM pass emits: flag casts moves or casts 1568 571 insns 14864 13938 jited bytes 72351 69111 No released kernel has the flag. LLVM does not emit the casts for Rust at this time. Until it does, Rust progs that access arena do not load. scx_mavd and scx_nitosis set the flag too. They are experimental schedulers in C in sched-ext/scx, and they use the flag to omit __arena. They must use __arena again. This reverts commit 4c651a91bdfc ("bpf: Treat load and store through a number as arena access"). One part stays: BPF_REG_AX in reg2pt_regs[] of the x86 JIT. The fetching AND/OR/XOR atomics in arena use it now. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-6-alexei.starovoitov@gmail.com --- kernel/bpf/core.c | 9 +------- kernel/bpf/fixups.c | 39 ------------------------------- kernel/bpf/syscall.c | 4 ---- kernel/bpf/verifier.c | 64 ++++++++++++--------------------------------------- 4 files changed, 16 insertions(+), 100 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 0a8ba603f44b..7cf37690d41f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2639,8 +2639,7 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp) /* * Whether a function has an insn that works with the arena base, which a JIT * sets up in the prologue when bpf_prog_uses_arena_insns() says so. Decided on - * the final insns of the function: an access through an arena pointer, or - * through a number in a program loaded with BPF_F_ARENA_SCALAR, has been + * the final insns of the function: an access through an arena pointer has been * rewritten to a BPF_PROBE_MEM32 form by then, a cast to the user address * space keeps BPF_ADDR_SPACE_CAST, and a kfunc call with an __arena argument, * which the JIT rebases, has a function model that says so. A load of an @@ -3518,12 +3517,6 @@ bool __weak bpf_jit_supports_arena(void) return false; } -/* Whether JIT takes BPF_REG_AX as the address of arena access */ -bool __weak bpf_jit_supports_arena_scalar(void) -{ - return false; -} - bool __weak bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { return false; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index f062ef7dd943..41bc37e205e0 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -44,31 +44,6 @@ static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *i return false; } -/* - * The insn accesses arena through a number. JITs add the base of arena - * to the register as it is, so the access goes through the low 32 bits of - * the number in BPF_REG_AX. Registers of the program are not changed: the - * number may be compared or stored later, and on another path the register - * may be PTR_TO_ARENA. - * - * Constant blinding leaves insns that use BPF_REG_AX alone, so the immediate - * of st through a number is not blinded. - */ -static int arena_scalar_access(const struct bpf_insn *insn, struct bpf_insn *buf) -{ - bool load = BPF_CLASS(insn->code) == BPF_LDX || bpf_atomic_is_load_acq(insn); - struct bpf_insn *patch = buf; - - *patch++ = BPF_MOV32_REG(BPF_REG_AX, load ? insn->src_reg : insn->dst_reg); - *patch = *insn; - if (load) - patch->src_reg = BPF_REG_AX; - else - patch->dst_reg = BPF_REG_AX; - patch++; - return patch - buf; -} - /* Return the regno defined by the insn, or -1. */ static int insn_def_regno(const struct bpf_insn *insn) { @@ -1880,20 +1855,6 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); - if (env->insn_aux_data[i + delta].arena_scalar) { - cnt = arena_scalar_access(insn, insn_buf); - - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - prog = new_prog; - env->prog = prog; - insn = prog->insnsi + i + delta; - goto next_insn; - } - /* Make sdiv/smod divide-by-minus-one exceptions impossible. */ if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) || insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) || diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 90bc7230afa8..60c4d84b9aba 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2971,13 +2971,9 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at BPF_F_XDP_HAS_FRAGS | BPF_F_XDP_DEV_BOUND_ONLY | BPF_F_TEST_REG_INVARIANTS | - BPF_F_ARENA_SCALAR | BPF_F_TOKEN_FD)) return -EINVAL; - if ((attr->prog_flags & BPF_F_ARENA_SCALAR) && !bpf_jit_supports_arena_scalar()) - return -EOPNOTSUPP; - bpf_prog_load_fixup_attach_type(attr); if (attr->prog_flags & BPF_F_TOKEN_FD) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a0a5f075d201..c1070aba621b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5500,19 +5500,6 @@ static bool is_arena_reg(struct bpf_verifier_env *env, int regno) return reg->type == PTR_TO_ARENA; } -/* - * There are no address spaces in Rust, addresses of arena are plain numbers. - * When the program is loaded with BPF_F_ARENA_SCALAR and has an arena a load or - * a store through a number is an access to arena at the low 32 bits of it, - * like the access through PTR_TO_ARENA is. The register stays a number. - */ -static bool is_arena_scalar(struct bpf_verifier_env *env, int regno) -{ - const struct bpf_reg_state *reg = reg_state(env, regno); - - return reg->type == SCALAR_VALUE && env->arena_scalar && env->prog->aux->arena; -} - static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno, struct bpf_insn *insn) { @@ -5543,7 +5530,7 @@ static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, return false; if (is_sk_reg(env, regno)) return false; - if (is_arena_reg(env, regno) || is_arena_scalar(env, regno)) + if (is_arena_reg(env, regno)) return bpf_jit_supports_insn(insn, true); if (is_load_acq_unsafe(env, regno, insn)) return false; @@ -7524,22 +7511,6 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, bool allow_trust_mismatch); -/* - * Returns the register to check the access of the current insn with. - * For a number in a program with an arena that is 'arena'. - */ -static struct bpf_reg_state *mem_access_reg(struct bpf_verifier_env *env, int regno, - struct bpf_reg_state *arena) -{ - if (!is_arena_scalar(env, regno)) - return cur_regs(env) + regno; - - memset(arena, 0, sizeof(*arena)); - arena->type = PTR_TO_ARENA; - env->insn_aux_data[env->insn_idx].arena_scalar = true; - return arena; -} - static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, bool strict_alignment_once, bool is_ldsx, bool allow_trust_mismatch, const char *ctx) @@ -7547,7 +7518,6 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); - struct bpf_reg_state arena, *src_reg; enum bpf_reg_type src_reg_type; int err; @@ -7569,16 +7539,16 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - src_reg = mem_access_reg(env, insn->src_reg, &arena); - src_reg_type = src_reg->type; + src_reg_type = regs[insn->src_reg].type; /* * check_stack_read_fixed_off() may refine the modification's origin to * the source stack slot. */ bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); - err = check_mem_access(env, env->insn_idx, src_reg, argno_from_reg(insn->src_reg), - insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, + err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, + argno_from_reg(insn->src_reg), insn->off, + BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); /* * Paths through a kernel struct pointer and through other pointers @@ -7606,7 +7576,6 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); - struct bpf_reg_state arena, *dst_reg; enum bpf_reg_type dst_reg_type; int err; @@ -7628,12 +7597,12 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - dst_reg = mem_access_reg(env, insn->dst_reg, &arena); - dst_reg_type = dst_reg->type; + dst_reg_type = regs[insn->dst_reg].type; /* Check if (dst_reg + off) is writeable. */ - err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), - insn->off, BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, + err = check_mem_access(env, env->insn_idx, regs + insn->dst_reg, + argno_from_reg(insn->dst_reg), insn->off, + BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, strict_alignment_once, false); err = err ?: save_aux_ptr_type(env, dst_reg_type, false); @@ -7643,7 +7612,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, static int check_atomic_rmw(struct bpf_verifier_env *env, struct bpf_insn *insn) { - struct bpf_reg_state arena, *dst_reg; + struct bpf_reg_state *dst_reg; int load_reg; int err; @@ -7688,9 +7657,6 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return -EACCES; } - /* load_reg may be dst_reg. Look at dst_reg before it's marked as unknown. */ - dst_reg = mem_access_reg(env, insn->dst_reg, &arena); - load_reg = bpf_atomic_load_reg(insn); if (load_reg >= 0) { /* check and record load of old value */ @@ -7699,6 +7665,8 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return err; } + dst_reg = cur_regs(env) + insn->dst_reg; + /* Check whether we can read the memory, with second call for fetch * case to simulate the register fill. */ @@ -20289,17 +20257,16 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) return check_stack_arg_write(env, state, insn->off, NULL); } - struct bpf_reg_state arena, *dst_reg; enum bpf_reg_type dst_reg_type; err = check_reg_arg(env, insn->dst_reg, SRC_OP); if (err) return err; - dst_reg = mem_access_reg(env, insn->dst_reg, &arena); - dst_reg_type = dst_reg->type; + dst_reg_type = cur_regs(env)[insn->dst_reg].type; - err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), + err = check_mem_access(env, env->insn_idx, cur_regs(env) + insn->dst_reg, + argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, false, false); if (err) @@ -23935,7 +23902,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (is_priv) env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; - env->arena_scalar = attr->prog_flags & BPF_F_ARENA_SCALAR; env->explored_states = kvzalloc_objs(struct list_head, state_htab_size(env), -- cgit v1.2.3 From 3f9924747c4422176346c34b5dff44ba8bca5180 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:02 +0000 Subject: bpf: Clear delta of a register that scalar evolution widens After 'r3 = r8; r3 += 4' the verifier knows that r3 == r8 + 4. The two registers have the same id, and r3 keeps the difference in reg->delta. When the prog compares one of them with a constant, sync_linked_regs() calculates the range of the other register from the difference of the two deltas. At a loop head scalar evolution widens a register: it replaces the register with its range over all iterations. Then the register is not linked to other registers. scratch_widened_reg_id() clears reg->id, but it does not clear reg->delta. A subsequent copy of the register gets the stale delta. Then the range of the copy is incorrect by this delta. r3 = r8 r3 += 4 ; R3=scalar(id=1+4) 1: ; r3 is widened: no id, delta is still 4 r4 = r3 ; id=2 and delta 4 in both r4 += 1 ; R4=scalar(id=2+1) if r3 > 10 goto 3f ; R3 <= 10, R4 = R3 + 1 - 4 <= 7 if r4 <= 7 goto 2f ... ; not verified, r4 is up to 11 here The verifier does not verify code that can run. Use clear_scalar_id(). It clears id and delta together. Move it to bpf_verifier.h, so that scev.c can call it. Fixes: 6ccf49f79327 ("bpf: Use SCEV to widen bounded loops") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-8-alexei.starovoitov@gmail.com --- kernel/bpf/scev.c | 2 +- kernel/bpf/verifier.c | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 04d4962448e2..3f7534cda47a 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -2391,7 +2391,7 @@ static void scratch_widened_reg_id(struct bpf_verifier_env *env, struct bpf_reg_ { switch (base_type(reg->type)) { case SCALAR_VALUE: - reg->id = 0; + clear_scalar_id(reg); break; case PTR_TO_PACKET: case PTR_TO_PACKET_META: diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c1070aba621b..8139e41ace00 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3824,12 +3824,6 @@ static bool __is_pointer_value(bool allow_ptr_leaks, return is_pointer_regtype(reg->type); } -static void clear_scalar_id(struct bpf_reg_state *reg) -{ - reg->id = 0; - reg->delta = 0; -} - static void assign_scalar_id_before_mov(struct bpf_verifier_env *env, struct bpf_reg_state *src_reg) { -- cgit v1.2.3 From 41750e5d682ab0703a9d2af3ec31a508b253b38f Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:06 +0000 Subject: bpf: Don't widen a pointer to a buffer in scalar evolution Scalar evolution widens counted loops. Since that change the verifier rejects a loop over the per-cpu values of a map element: SEC("iter/bpf_map_elem") ... for (i = 0; i < n; i++) { out[i] = *(__u32 *)pptr; pptr += 8; } loop header at 12, widening r2 to 0..56 step 8 R2 invalid variable buffer offset: off=0, var_off=(0x0; 0x38) pptr is a pointer to a buffer (PTR_TO_BUF). Scalar evolution replaces its offset with the range over all iterations. But __check_buffer_access() accepts only a constant offset. Thus a prog cannot use a widened PTR_TO_BUF. Do not widen it. Then the verifier verifies the loop one iteration at a time, as it did before. loop before SCEV with SCEV this patch -------------------------------- ----------- ---------- ---------- counts in i, reads through pptr loads rejected loads counts in p ('p < end'), no read 1M insns loads 1M insns counts in p, reads through p 1M insns rejected 1M insns The verifier cannot decide a comparison of two pointers to a buffer. Thus it follows 'p < end' until the limit of 1M insns. bpf_iter_bpf_percpu_array_map.c and bpf_iter_bpf_percpu_hash_map.c have the first loop. They load today because num_cpus is volatile. The prog reads the bound again in each iteration. Scalar evolution cannot count such a loop, and it widens nothing. Fixes: 6ccf49f79327 ("bpf: Use SCEV to widen bounded loops") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-12-alexei.starovoitov@gmail.com --- kernel/bpf/scev.c | 1 - 1 file changed, 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 3f7534cda47a..6231d16aa311 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -2379,7 +2379,6 @@ static bool is_widenable_reg_type(const struct bpf_reg_state *reg) case PTR_TO_PACKET: case PTR_TO_PACKET_META: case PTR_TO_MEM: - case PTR_TO_BUF: case PTR_TO_BTF_ID: return true; default: -- cgit v1.2.3 From 1536e9b4375f235596ab5b7d6aa0fc4573965c4e Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:08 +0000 Subject: bpf: Don't clamp a register whose type changed in the loop When a state returns to the head of a counted loop, bpf_clamp_scev_regs() replaces the range of each register that changes by a constant step. The new range is the range over all iterations. The function calculates it from the register in the state that entered the loop. For a pointer this range is the range of its offsets. A call in the loop can invalidate the pointer. bpf_skb_pull_data() does it to packet pointers, bpf_ringbuf_discard() does it to ringbuf memory. Then the register is an unknown scalar. Scalar evolution does not see that, and the clamp gives the offsets of the pointer to the scalar. r6 = bpf_ringbuf_reserve() ; R6=ringbuf_mem r7 = r6 1: if first iteration goto 2f r1 = map_value r1 += r6 ; accepted: R6 is 0 or 1 r0 = *(u8 *)(r1 + 0) goto 3f 2: bpf_ringbuf_discard(r7) ; R6=scalar() 3: r6 += 1 r9 += 1 if r9 < 2 goto 1b ; clamp: R6=scalar(umax=1) At run time r6 is the address of the ringbuf memory plus 1. The prog adds it to a pointer to a map value, and the kernel crashes in the access. Do not clamp a register that has a different type than at the entry of the loop. Such a state is not within the state at the head. Thus the verifier continues with it, and r6 stays an unknown scalar. Fixes: 6ccf49f79327 ("bpf: Use SCEV to widen bounded loops") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-14-alexei.starovoitov@gmail.com --- kernel/bpf/scev.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 6231d16aa311..89f96682af3f 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -2897,6 +2897,14 @@ int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur is_unbound_scev_reg(env, entry_st, r)) continue; + /* + * A call in the loop can change the type: it invalidates packet pointers or + * releases a reference. The bounds are the offsets of the pointer at the entry. + * They say nothing about the value that is in the register now. + */ + if (reg->type != entry_reg->type) + continue; + linear_bounds(entry_reg, max_header_count, &coefs, &bounds); if (log->level & BPF_LOG_LEVEL2) { bpf_log(log, "loop header at %d, clamping ", insn_idx); -- cgit v1.2.3 From 41b06cfd1ceaf10e382b8f2d2d720e3e0abae10b Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:09 +0000 Subject: bpf: Reset the range of a packet pointer that scalar evolution clamps A compare with pkt_end sets 'range' in all packet pointers that have the same id. Then 'range - offset' bytes from the pointer are in the packet. When a state returns to the head of a counted loop, bpf_clamp_scev_regs() gives a new id to a packet pointer, and the offsets of all iterations. It does not change the range. But the range is valid only together with the old id and the old offsets. 1: ; R6=pkt(id=1,r=0,off=0..100) r5 = r6 r5 += 100 if r5 > pkt_end goto 2f ; R6=pkt(id=1,r=200,off=0..100) r6 += 100 ; R6=pkt(id=1,r=200,off=100..200) r9 += 1 if r9 < 2 goto 1b ; clamp: R6=pkt(id=2,r=200,off=0..100) after bytes from r6 that the verifier accepts ---------------------- --------------------------------------- the compare 100, correct r6 += 100 0, correct the clamp 100, incorrect. Nothing compared this r6 Usually the verifier prunes the state at the head, and the range has no effect. The verifier continues with the state when a different part of it is not within the state at the head. A flag in a 32-bit spill is sufficient. Then the prog reads and writes after pkt_end. A write to skb_shared_info crashes the kernel. Take the range that the pointer has at the entry of the loop. No compare in the loop found this range, so it is valid in all iterations. The state at the head has the same range, so the verifier prunes as before. Fixes: 6ccf49f79327 ("bpf: Use SCEV to widen bounded loops") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-15-alexei.starovoitov@gmail.com --- kernel/bpf/scev.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 89f96682af3f..7b82fd79f9cc 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -2914,6 +2914,13 @@ int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur bpf_log(log, "\n"); } scratch_widened_reg_id(env, reg); + /* + * The range of a packet pointer goes together with its id and with the offsets + * that it had in the compare with pkt_end. Both are replaced here. What holds for + * the new offsets is the range at the entry. + */ + if (type_is_pkt_pointer(reg->type)) + reg->range = max(entry_reg->range, 0); err = bpf_set_reg_range(env, reg, bounds.r64, bounds.r32, bounds.base, bounds.step); if (err) -- cgit v1.2.3 From 19caa38daf9def4a7c995ebf3bd0e89bce707652 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:11 +0000 Subject: bpf: Finish the insn in the state that maybe_fork_scalars() splits off When a register is -1 or 0, maybe_fork_scalars() splits the state in two before an 'and' or an 'or' with a constant. The result is more accurate for each value alone. The verifier continues with -1. It pushes a copy of the state with 0 and verifies the copy later, from the same insn. today this patch N: r1 &= 8 r1 is -1 or 0 N: r1 &= 8 r1 is -1 or 0 / \ / \ r1 = -1 r1 = 0 r1 = -1 r1 = 0 r1 &= 8 push to N r1 &= 8 r1 &= 8 | | push to N+1 N+1: ... N+1: ... At insn N the copy is a part of the state that it came from: 0 is in [-1, 0]. If pruning compares the two states, it drops the copy, and no one verifies the case r1 = 0. The first state has branches that are not explored. Usually pruning does not use such states. In a loop that scalar evolution counted it does, and the head of such a loop always keeps its state. Thus a fork at the head of a counted loop loses the case r1 = 0: r9 = a map value of 8 bytes r6 = -1 1: r6 &= 8 r6 is -1 or 0, only -1 is verified: 8 r6 ^= 8 0. It is 8 when r6 was 0. r1 = r9 r1 += r6 r0 = *(u8 *)(r1 + 0) r6 = -1 or 0 r7 += 1 if r7 < 100 goto 1b from 11 to 11: safe The verifier accepts this prog. The prog reads the byte that follows the map value. With a constant of 64 Tbyte in a register it reads an address that is not mapped, and the kernel crashes. Do the insn in the copy too, and push the copy to the next insn. All other code that pushes a state does the same. No verdict changes in selftests, Cilium and scx. Each fork processes one insn less, and the sequence in which the verifier explores states changes slightly. 62 of 161 progs of Cilium process a different number of insns, from -0.7% to +3.3%. The total is 986970 -> 987013. 2 of 563 progs of scx process one insn less. Fixes: 6ccf49f79327 ("bpf: Use SCEV to widen bounded loops") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-17-alexei.starovoitov@gmail.com --- kernel/bpf/verifier.c | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8139e41ace00..02b2915f0014 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17289,11 +17289,14 @@ static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, } } +static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn, + struct bpf_reg_state *dst_reg, struct bpf_reg_state src_reg); + static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *insn, - struct bpf_reg_state *dst_reg) + struct bpf_reg_state *dst_reg, const struct bpf_reg_state *src_reg) { struct bpf_verifier_state *branch; - struct bpf_reg_state *regs; + struct bpf_reg_state *reg; bool alu32; if (reg_smin(dst_reg) == -1 && reg_smax(dst_reg) == 0) @@ -17303,19 +17306,25 @@ static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *ins else return 0; - branch = push_stack(env, env->insn_idx, env->insn_idx, false); + branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); if (IS_ERR(branch)) return PTR_ERR(branch); - regs = branch->frame[branch->curframe]->regs; + reg = &branch->frame[branch->curframe]->regs[insn->dst_reg]; if (alu32) { - __mark_reg32_known(®s[insn->dst_reg], 0); + __mark_reg32_known(reg, 0); __mark_reg32_known(dst_reg, -1ull); } else { - __mark_reg_known(®s[insn->dst_reg], 0); + __mark_reg_known(reg, 0); __mark_reg_known(dst_reg, -1ull); } - return 0; + /* + * The insn is done with in the branch too, and what gets to the next insn is a state + * like any other. At this insn the branch is a part of the state that is looked at: + * it must not be compared with it. + */ + clear_scalar_id(reg); + return adjust_scalar_min_max_vals(env, insn, reg, *src_reg); } /* WARNING: This function does calculations on 64-bit values, but the actual @@ -17420,7 +17429,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, break; case BPF_AND: if (tnum_is_const(src_reg.var_off)) { - ret = maybe_fork_scalars(env, insn, dst_reg); + ret = maybe_fork_scalars(env, insn, dst_reg, &src_reg); if (ret) return ret; } @@ -17431,7 +17440,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, break; case BPF_OR: if (tnum_is_const(src_reg.var_off)) { - ret = maybe_fork_scalars(env, insn, dst_reg); + ret = maybe_fork_scalars(env, insn, dst_reg, &src_reg); if (ret) return ret; } -- cgit v1.2.3 From 6c476d2b73056ed151c3f8b884c627c75999ed7c Mon Sep 17 00:00:00 2001 From: Feng Yang Date: Wed, 28 Oct 2026 11:19:33 +0800 Subject: bpf: Fix out-of-bounds btf->types write for split BTF btf_add_type() stores the types of a BTF in btf->types[], which can hold at most BTF_MAX_TYPE entries. The types of a split BTF get IDs start_id .. start_id + nr_types - 1, so only BTF_MAX_TYPE - start_id of them fit, but neither the limit check nor the size used to grow the array accounts for start_id: the check is an equality test against BTF_MAX_TYPE ('start_id + btf->types_size == BTF_MAX_TYPE') and the new size is clamped to BTF_MAX_TYPE. The check therefore only fires if the array happens to grow to exactly BTF_MAX_TYPE - start_id entries. Otherwise, if the module has more than BTF_MAX_TYPE types, the array is filled up to BTF_MAX_TYPE entries. The next type that needs to be added reallocates the array to the same size, after which btf->types_size no longer equals btf->nr_types and the growth path is not taken anymore, so every following type is written past the end of the array: BUG: unable to handle page fault for address: ffffc90006200000 RIP: 0010:btf_add_type+0x74/0x270 Call Trace: btf_check_all_metas btf_parse_module Bound both the array size and the limit check by BTF_MAX_TYPE - start_id. Base BTF (start_id == 0) is unaffected. Fixes: 951bb64621b8 ("bpf: Add in-kernel split BTF support") Signed-off-by: Feng Yang Signed-off-by: Alexei Starovoitov Reviewed-by: Mahe Tardy Acked-by: Ihor Solodrai Link: https://patch.msgid.link/20261028031933.277550-1-yangfeng59949@163.com --- kernel/bpf/btf.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 7ed1ff71a089..1be30487a2fa 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -1752,15 +1752,16 @@ static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t) /* Expand 'types' array */ struct btf_type **new_types; - u32 expand_by, new_size; + u32 expand_by, new_size, max_size; - if (btf->start_id + btf->types_size == BTF_MAX_TYPE) { + if (btf->start_id + btf->types_size >= BTF_MAX_TYPE) { btf_verifier_log(env, "Exceeded max num of types"); return -E2BIG; } + max_size = BTF_MAX_TYPE - btf->start_id; expand_by = max_t(u32, btf->types_size >> 2, 16); - new_size = min_t(u32, BTF_MAX_TYPE, + new_size = min_t(u32, max_size, btf->types_size + expand_by); new_types = kvzalloc_objs(*new_types, new_size, -- cgit v1.2.3 From c8a8a68d3ab4a06c8f5a13a8c00275351c8239b4 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 23 Oct 2026 15:03:02 +0000 Subject: bpf: Don't re-sort subprog_info[] for every new subprogram add_subprog() appends new subprog to env->subprog_info[] and calls sort() on the whole array, though only the last entry can be out of order. Move subprogs that start after the new one up by one entry instead. 'start' is unique, so the order of the array doesn't change. The new entry can land in a used slot, so initialize all of it. add_subprog() returns the index of the new subprog instead of the last one. The callers check for error only. It's a trivial prep for rust-bpf [1] that bumps BPF_MAX_SUBPROGS to 1024. Rust crates have more than 256 functions that are not inlined. Time in add_subprogs() when veristat loads sched_ext schedulers from scx repo, 8 cpu VM, usec: subprogs before after arena_selftest 101 386 27 lavd_enqueue 52 96 16 lavd_dispatch 48 78 11 [1] https://github.com/4ast/rust-bpf Signed-off-by: Alexei Starovoitov Reviewed-by: Mahe Tardy Link: https://patch.msgid.link/20261023150302.4097865-1-alexei.starovoitov@gmail.com --- kernel/bpf/verifier.c | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 02b2915f0014..26a5ee0708a3 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2617,12 +2617,6 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, return &elem->st; } -static int cmp_subprogs(const void *a, const void *b) -{ - return ((struct bpf_subprog_info *)a)->start - - ((struct bpf_subprog_info *)b)->start; -} - /* Find subprogram that contains instruction at 'off' */ struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off) { @@ -2657,26 +2651,29 @@ int bpf_find_subprog(struct bpf_verifier_env *env, int off) static int add_subprog(struct bpf_verifier_env *env, int off) { + struct bpf_subprog_info *subprog = env->subprog_info; int insn_cnt = env->prog->len; - int ret; + int i; if (off >= insn_cnt || off < 0) { verbose(env, "call to invalid destination\n"); return -EINVAL; } - ret = bpf_find_subprog(env, off); - if (ret >= 0) - return ret; + i = bpf_find_subprog(env, off); + if (i >= 0) + return i; if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { verbose(env, "too many subprograms\n"); return -E2BIG; } - /* determine subprog starts. The end is one before the next starts */ - env->subprog_info[env->subprog_cnt].exit_idx = U32_MAX; - env->subprog_info[env->subprog_cnt++].start = off; - sort(env->subprog_info, env->subprog_cnt, - sizeof(env->subprog_info[0]), cmp_subprogs, NULL); - return env->subprog_cnt - 1; + /* + * Determine subprog starts. The end is one before the next starts. + * The array stays sorted: subprogs that start after 'off' move up. + */ + for (i = env->subprog_cnt++; i > 0 && subprog[i - 1].start > off; i--) + subprog[i] = subprog[i - 1]; + subprog[i] = (struct bpf_subprog_info){ .start = off, .exit_idx = U32_MAX }; + return i; } static int bpf_find_exception_callback_insn_off(struct bpf_verifier_env *env) -- cgit v1.2.3