From 69fdbe63e16919a885a8f9441e248ce0ddf15b25 Mon Sep 17 00:00:00 2001 From: Woojin Ji Date: Thu, 25 Jun 2026 19:25:37 +0900 Subject: bpf: Preserve scalar zero spills for stack reads Stack reads can read back bytes that belong to a previously spilled scalar constant zero. Today mark_reg_stack_read() only treats STACK_ZERO bytes as known zero bytes, so the destination register can become unknown even though every byte in the read range is known to be zero. This can lead to rejecting otherwise valid programs once the loaded byte is used as a pointer offset. The original reproducer uses a variable-offset stack byte read emitted by clang 22.1.6 at -O2/-O3 from a small helper-based BPF C program. Fixed offset reads have a related mixed case as well: pure scalar-zero spill reads are already handled, but a fixed read spanning both STACK_ZERO and scalar const-zero STACK_SPILL bytes still falls back to unknown. Teach mark_reg_stack_read() to also consider STACK_SPILL bytes backed by a spilled scalar constant zero as zero bytes, and use that path for both variable-offset stack reads and fixed-offset mixed reads. Keep the existing pure register-fill behavior unchanged. When a zero result depends on such a spill, mark the contributing stack slots precise before accepting the const-zero result so pruning cannot reuse a zero-spill state for a later non-zero spill state. No deployed-program regression is currently known, so target bpf-next. Assisted-by: opencode:gpt-5.5 Signed-off-by: Woojin Ji Acked-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260625-bpf-stack-var-off-zero-v1-v3-1-a068210a761b@gmail.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 39a851e690ec..76b8b7627a10 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1243,6 +1243,11 @@ static inline void bpf_bt_set_frame_slot(struct backtrack_state *bt, u32 frame, bt->stack_masks[frame] |= 1ull << slot; } +static inline void bpf_bt_set_frame_slot_mask(struct backtrack_state *bt, u32 frame, u64 mask) +{ + bt->stack_masks[frame] |= mask; +} + static inline void bt_set_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot) { bt->stack_arg_masks[frame] |= 1 << slot; -- cgit v1.2.3 From 7cf9cd98cf6f0df3befc167ca6b54c07014d71de Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Wed, 24 Jun 2026 23:51:14 +0800 Subject: bpf: Copy per-CPU map value padding in copy_map_value_long() In kernel, per-CPU map elements are stored with round_up(map->value_size, 8) bytes. On UAPI lookup paths, it copies the rounded size for each CPU into a temporary buffer. However, copy_map_value_long() passes 'map->value_size' to bpf_obj_memcpy(). When the map has special fields, bpf_obj_memcpy() copies around those fields with memcpy(), and does not copy the tail padding between 'map->value_size' and round_up(map->value_size, 8). The temporary UAPI lookup buffers are allocated without __GFP_ZERO. As a result, when the per-CPU map's value size is not equal to round_up(map->value_size, 8), UAPI LOOKUP_ELEM and its variants can return stale heap contents from that padding to user space. The same issue applies to bpf_iter for per-CPU maps. Pass round_up(map->value_size, 8) to bpf_obj_memcpy() from copy_map_value_long(), so per-CPU maps both with and without special fields copy the entire per-CPU slot. Remove the now redundant round_up() from bpf_obj_memcpy()'s long_memcpy path. Fixes: 448325199f57 ("bpf: Add copy_map_value_long to copy to remote percpu memory") Signed-off-by: Leon Hwang Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260624155115.85196-2-leon.hwang@linux.dev --- include/linux/bpf.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 7719f6528445..ba09795e0bfd 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -570,7 +570,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, if (IS_ERR_OR_NULL(rec)) { if (long_memcpy) - bpf_long_memcpy(dst, src, round_up(size, 8)); + bpf_long_memcpy(dst, src, size); else memcpy(dst, src, size); return; @@ -593,7 +593,7 @@ static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src) { - bpf_obj_memcpy(map->record, dst, src, map->value_size, true); + bpf_obj_memcpy(map->record, dst, src, round_up(map->value_size, 8), true); } static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src) -- cgit v1.2.3 From 9c9ee0324c774490ae953162aaaf4561d222bd93 Mon Sep 17 00:00:00 2001 From: Yiyang Chen Date: Tue, 30 Jun 2026 08:41:26 +0000 Subject: bpf: Reject MEM_ALLOC BTF accesses past object bounds BTF struct walks relax the struct-size check for accesses through a trailing flexible array. That is valid for ordinary BTF type walking, but PTR_TO_BTF_ID | MEM_ALLOC values point to objects allocated with the static BTF type size. When walking a MEM_ALLOC object, reject the access before applying the flexible-array relaxation if the access range extends past the struct size. Apply the same policy to struct ID matching so kfunc and kptr type checks do not walk past the allocated object bounds either. Fixes: 958cf2e273f0 ("bpf: Introduce bpf_obj_new") Fixes: 36d8bdf75a93 ("bpf: Add alloc/xchg/direct_access support for local percpu kptr") Signed-off-by: Yiyang Chen Reviewed-by: Amery Hung Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/4b8c8a81102ba4b595011434c881194f264ddc59.1782807039.git.chenyy23@mails.tsinghua.edu.cn Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index ba09795e0bfd..adf53f7edf28 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -3146,7 +3146,7 @@ int btf_struct_access(struct bpf_verifier_log *log, bool btf_struct_ids_match(struct bpf_verifier_log *log, const struct btf *btf, u32 id, int off, const struct btf *need_btf, u32 need_type_id, - bool strict); + bool strict, bool walk_flex_arrays); int btf_distill_func_proto(struct bpf_verifier_log *log, struct btf *btf, -- cgit v1.2.3 From d5a85392392c77b61a74e975a74da0e9c146f6d3 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 8 Jul 2026 09:53:36 +0200 Subject: bpf: Resolve and cache fd_array objects at load time The fd_array passed to BPF_PROG_LOAD carries the map and module BTF file descriptors a program binds. The verifier reads it more than once during a load: process_fd_array() walks it to bind the maps and BTFs, and check_and_resolve_insns() and the kfunc BTF resolver later read it again to resolve the program's BPF_PSEUDO_MAP_IDX* and module kfunc refs. For signed BPF, we need these upfront in memory, thus resolve each fd to its object once and cache it by fd_array index, then bind that cached object for the rest of the load. env->fd_array becomes a small per-slot {map, btf} cache rather than a bpfptr_t; every later reference is then an in-bounds lookup of an already-resolved object, and an index outside the cache is rejected instead of read from user memory: - continuous (fd_array_cnt given): the caller declares the length and every entry is resolved and bound up front (used also by the BPF signed loader) - sparse (no fd_array_cnt): left as the legacy path with no fd_array cache; each reference reads its fd from the caller's fd_array and resolves it on the spot. Deduplication in used_maps and the kfunc BTF table keeps this correct, and only unsigned programs use this shape. Split these into separate helpers to make it easier to follow. Signed-off-by: Daniel Borkmann Acked-by: Anton Protopopov Link: https://lore.kernel.org/bpf/20260708075343.358712-2-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf_verifier.h | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 76b8b7627a10..bb57773cde37 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -898,6 +898,14 @@ struct bpf_scc_info { struct bpf_liveness; +struct bpf_fd_array { + union { + struct bpf_map *map; + struct btf *btf; + unsigned long val; + }; +}; + /* single container for all structs * one verifier_env per bpf_check() call */ @@ -989,7 +997,19 @@ struct bpf_verifier_env { u32 free_list_size; u32 explored_states_size; u32 num_backedges; - bpfptr_t fd_array; + /* + * The program's fd_array comes in two shapes, told apart by whether + * the caller passed fd_array_cnt. They are mutually exclusive: + * - continuous (fd_array_cnt given): ->fd_array holds every entry + * resolved to its object up front, indexed by fd_array position, + * with ->fd_array_cnt slots; ->fd_array_raw is unused. + * - sparse (no fd_array_cnt): ->fd_array is NULL, and entries are + * read from ->fd_array_raw (the caller's fd_array) and resolved + * on the spot at each reference. + */ + struct bpf_fd_array *fd_array; + u32 fd_array_cnt; + bpfptr_t fd_array_raw; /* bit mask to keep track of whether a register has been accessed * since the last time the function state was printed -- cgit v1.2.3 From b707068e0ed92b64bb66bae4f6f3a521f7017220 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 8 Jul 2026 09:53:37 +0200 Subject: bpf: Verify signed loader metadata at load time A signed gen_loader program carries the programs, maps and relocations it installs in a metadata array map. The loader instructions are covered by the PKCS#7 signature, but the metadata map is not: Today the loader compares the map contents from within BPF against a hash baked into its (signed) instructions, using the kernel-cached map hash. The kernel itself never actually attests that the metadata the loader installs is the metadata that was signed. This split is the core of the long-standing objection to the BPF signing scheme from the LSM / integrity side: the integrity check of a light skeleton only completes once the loader program runs, that is, after the security_bpf_prog_load() hook, so at admission time an LSM observes a program whose payload has not yet been verified. Auditing the chain link is also not a purely cryptographic operation: whoever signs or reviews an lskel has to disassemble the loader's preamble to convince themselves that the embedded hash check is present and correct [0][1]. Two acceptable fixes were identified in those threads: Complete the integrity check before the admission hook fires, or add a second hook that collects the verification result after the loader ran [2]. Covering both the loader and its maps directly with the PKCS#7 signature is what Blaise Boscaccy's patchsets proposed in several forms. Let's implement the former, without growing the UAPI, and in particular as a single unified scheme where the signature spans the raw bytes rather than derived hashes. A signed loader binds its metadata map(s) through the existing fd_array, and an exclusive map is already bound to a program digest (excl_prog_hash). So when a signature is present, collect the exclusive maps from fd_array and append their frozen contents to the instructions before verification: The signature now covers insns || metadata_0 || metadata_1 || [...] in the fd_array order, and verification completes in bpf_check(), once the fd_array maps are resolved into used_maps, before the LSM admission hook and the rest of verification. A program is either BPF_SIG_UNSIGNED or BPF_SIG_VERIFIED, with nothing in between. While folding the fd_array maps, a non-exclusive map bound to a signed program is rejected, so every map folded into the signature is exclusive. A signed loader that fails to cover its metadata thus does not load, and BPF_SIG_VERIFIED always means the instructions and every exclusive map are authentic. The maps must be frozen so the hashed bytes cannot change before the loader runs; the map <-> program digest binding is enforced by the verifier for every used map. Binding maps through fd_array_cnt makes the verifier resolve and excl-check them (excl_prog_sha vs prog->digest) before it would otherwise compute the digest, so compute prog->digest up front in bpf_check(), over the unmodified instructions the signature covers, for a load that folds metadata. Unsigned programs are not affected by the signature path; for them the LSM admission hook merely moves below fd_array resolution, with minimal bounded work in between. Note, signed loaders generated by older libbpf/ bpftool versions need to be regenerated; some of the recent fixes we've had on the signed loader side require the latter already to close gaps. Finally, some remarks around the security_bpf_prog_load() placement given there was discussion on whether a new hook is needed or the existing security_bpf_prog() hook should be reused [3]: For a new hook it would mean that just for loading a single BPF program it has to pass through four layers of LSM hooks: 1) security_bpf (cmd=PROG_LOAD): for gating various bpf subcmds 2) security_bpf_prog_load: historical admission hook (CAP/token, prog_type, attach point), pre-verification 3) security_bpf_prog_verify_signature: newly asked admission hook, same role as 2), plus the BPF signature verdict 4) security_bpf_prog: gate handing the prog fd back to userspace, verification done & signature verified The use-cases of 2) and 3) conflate, thus BPF community prefers to just keep a total of 3 LSM hooks (as-is today): 3) makes 2) incoherent given they are the /same class/ of hook, that is, access-control admission on the load and split only by _what_ they can see. Worse, with the split, for a signed BPF program security_bpf_prog_load 2) admits a program whose signature has not been checked, so a policy gating at 2) is structurally unable to express "admit only verified" and every such policy is forced onto 3) *anyway*. In other words, one doesn't get two complementary hooks, but rather, one real admission hook aka 3) plus a now-degraded /legacy/ hook 2) that can't answer the question operators actually want to ask. Reusing security_bpf_prog() 4) for admission is no alternative either: it fires only after the entire verifier (and JIT) pipeline ran, so denying a not-yet-verified program at that point burns exactly the work a denial is supposed to avoid, and by then the program has an id assigned and the kallsyms/perf/audit load events fired. Policies are free to also consume the signature verdict at 4), but admission control belongs into security_bpf_prog_load(). Hence the latter remains the only admission hook, merely moved past signature verification; with moving large allocations further down into the BPF verifier, there is now only minimal work between the old and new location: The preparation work in bpf_check() is reordered such that only the minimally necessary setup happens up front: Allocating the env, initializing the verifier log and resolving the fd_array that a signed BPF metadata map needs. The worst case allocation up until security_bpf_prog_load() is ~90K which is the env itself (~54K) plus the continuous fd_array cache (at most 32K). The insn_aux_data array is moved into a later stage in the verification. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/2f71d6c03698eb17d51f7247efde777627ee578a.camel@HansenPartnership.com [0] Link: https://lore.kernel.org/lkml/ecf0521ed302db672672ebfbc670ecfba36a6e00.camel@HansenPartnership.com [1] Link: https://lore.kernel.org/bpf/88703f00d5b7a779728451008626efa45e42db3d.camel@HansenPartnership.com [2] Link: https://lore.kernel.org/bpf/DJOFY21DYUI4.19WKQ3NPZ4H5R@gmail.com [3] Link: https://lore.kernel.org/bpf/20260708075343.358712-3-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf_verifier.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index bb57773cde37..317e99b9acc0 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -947,6 +947,7 @@ struct bpf_verifier_env { bool bypass_spec_v4; bool seen_direct_write; bool seen_exception; + bool signature; struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */ const struct bpf_line_info *prev_linfo; struct bpf_verifier_log log; -- cgit v1.2.3 From a2d784869a0f252e1a277db7dc2c16d55694da72 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 8 Jul 2026 09:53:38 +0200 Subject: libbpf: Drop in-loader metadata check for load-time verification The signed gen_loader used to police its own metadata map from within BPF: emit_signature_match() read the kernel-cached map->sha[] back through hardcoded struct bpf_map offsets and compared it against a hash that compute_sha_update_offsets() baked into the signed instructions, after a BPF_OBJ_GET_INFO_BY_FD round-trip to populate map->sha[]. The kernel now verifies the metadata at BPF_PROG_LOAD time by folding the frozen contents of the loader's exclusive fd_array maps into the signature, so the loader no longer checks anything itself. Generated loaders thus carry no verification logic of their own anymore: Nothing in the signing chain depends on emitted loader bytecode doing the right thing. On the loading side, skel_internal.h now sets fd_array_cnt for a signed load so the kernel scans fd_array for the exclusive metadata map - still frozen, as the kernel requires - and the BPF_OBJ_GET_INFO_BY_FD round-trip to populate map->sha[] is gone. The struct bpf_map layout BUILD_BUG_ON()s on the kernel side are removed as well: they only pinned the ABI for the in-BPF read of map->sha[] that is no longer needed. Same for the map->excl member. Note: gen_hash is retained; it still marks a loader as signed so an untrusted host cannot re-dimension maps or override initial values now covered by the signature. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260708075343.358712-4-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index adf53f7edf28..c1a98fa36738 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -299,7 +299,6 @@ struct bpf_map_owner { struct bpf_map { u8 sha[SHA256_DIGEST_SIZE]; - u32 excl; const struct bpf_map_ops *ops; struct bpf_map *inner_map_meta; #ifdef CONFIG_SECURITY -- cgit v1.2.3 From 9a6df65d5c6a9947ddab4e563e329720f44b8747 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Wed, 8 Jul 2026 18:18:05 +0800 Subject: bpf: Introduce jit_required flag and remove bpf_prog_has_kfunc_call() Introduce a 'jit_required' bitfield flag in struct bpf_prog to track whether a BPF program strictly requires the JIT compiler to run. This prevents a dangerous runtime fallback to the interpreter for features that are only implemented in the JIT compiler. Currently, bpf_prog_has_kfunc_call() is used only for kernel function calls, replace the kfunc-specific helper with the new 'jit_required' flag. This makes it easy to support other JIT-only BPF features, such as inlined helpers. Suggested-by: Alexei Starovoitov Suggested-by: KaFai Wan Suggested-by: Leon Hwang Acked-by: Leon Hwang Signed-off-by: Tiezhu Yang Signed-off-by: Eduard Zingerman --- include/linux/bpf.h | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index c1a98fa36738..31181e0c2b80 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1864,8 +1864,9 @@ struct bpf_prog_aux { struct bpf_prog { u16 pages; /* Number of allocated pages */ - u16 jited:1, /* Is our filter JIT'ed? */ + u32 jited:1, /* Is our filter JIT'ed? */ jit_requested:1,/* archs need to JIT the prog */ + jit_required:1, /* program strictly requires JIT compiler */ gpl_compatible:1, /* Is filter GPL compatible? */ cb_access:1, /* Is control block accessed? */ dst_needed:1, /* Do we need dst entry? */ @@ -3169,7 +3170,6 @@ const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); void bpf_task_storage_free(struct task_struct *task); void bpf_cgrp_storage_free(struct cgroup *cgroup); -bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn); @@ -3508,11 +3508,6 @@ static inline void bpf_task_storage_free(struct task_struct *task) { } -static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) -{ - return false; -} - static inline const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn) -- cgit v1.2.3 From 92ec8b1b6b24381611600162536b8b5b6a9e7323 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Tue, 14 Jul 2026 23:40:42 -0700 Subject: bpf: Factor out raw_mode-related fields in bpf_call_arg_meta To prepare for unifying the helper and kfunc call_arg_meta, group the scattered MEM_UNINIT ("raw") memory argument fields (raw_mode, regno and access_size) into a new struct arg_raw_mem_desc. The intention is to make it clear about when these are set and used instead of fields with overly generic names. Identify the raw argument once, up front, in check_raw_mode_ok() (like check_proto_release_reg() does for release_regno), recording its regno. check_stack_range_initialized() now recognizes the raw buffer by matching that regno, so the separate raw_mode flag is no longer needed, and the per-argument "meta->raw_mode = arg_type & MEM_UNINIT" assignments in check_func_arg() go away with it. A raw memory argument can be tagged either ARG_PTR_TO_MEM | MEM_UNINIT or ARG_PTR_TO_MAP_VALUE | MEM_UNINIT (the output buffer of bpf_map_pop_elem() and bpf_map_peek_elem()). Either may be passed as a PTR_TO_STACK, which reaches check_stack_range_initialized() through check_helper_mem_access(), so both must be treated as raw. Extend arg_type_is_raw_mem() to match the map value case as well; otherwise check_raw_mode_ok() would not record the regno for it and an uninitialized stack buffer passed to those helpers would be wrongly rejected for programs without CAP_PERFMON. No functional change intended. This patch does not enable raw_mode memory access for kfunc (i.e., uninit stack will not be allowed to be passed to kfunc for unprivileged programs). Existing kfuncs with arguments tagged with __uninit are either priviledged or dynptr kfuncs, which take another path to make sure the access is checked by check_mem_access(). Signed-off-by: Amery Hung Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260715064047.1793790-3-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf_verifier.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 317e99b9acc0..d3592f5b8621 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1464,6 +1464,15 @@ struct ref_obj_desc { u8 cnt; }; +/* + * A memory argument a call fills in. The verifier allows the stack to be uninitialized if + * the range is a known constant. Stack slots are marked as STACK_MISC by check_mem_access(). + */ +struct arg_raw_mem_desc { + u8 regno; + int size; +}; + struct bpf_kfunc_call_arg_meta { /* In parameters */ struct btf *btf; -- cgit v1.2.3 From 8faaa93b9f6a279472cd5490030151f1635292d2 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Tue, 14 Jul 2026 23:40:44 -0700 Subject: bpf: Unify helper and kfunc allocation-size argument handling The constant "size of the PTR_TO_MEM returned in R0" argument is handled by both helpers (ARG_CONST_ALLOC_SIZE_OR_ZERO) and kfuncs (__rdonly_buf_size / __rdwr_buf_size), each with its own meta field (meta->mem_size, meta->r0_size) and duplicated validation. Add struct arg_alloc_mem_desc and a shared process_const_alloc_mem_size(), and replace both fields with meta->arg_alloc_mem. The desc records presence with a 'found' flag instead of using a non-zero size as the sentinel. This also fixes a pre-existing bug on the kfunc return path: "no size argument" was tested as r0_size == 0, so an explicit __rdonly_buf_size/__rdwr_buf_size of 0 was treated as absent and fell through to btf_resolve_size(), giving R0 the size of the pointed-to return type instead of 0. With 'found', an explicit zero size is honored and btf_resolve_size() is used only when no size argument was passed. The size is stored in a u32, matching regs[R0].mem_size. The U32_MAX check now apply to both helper and kfunc through process_const_alloc_mem_size(). Fold bpf_session_cookie return size assignment into current kfunc return size resolution path. Note that verifier saves kfunc return size through r0_size instead of mem_size. The later has no active readers so remove it. Signed-off-by: Amery Hung Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260715064047.1793790-5-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf_verifier.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index d3592f5b8621..e3eda72cbf67 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1473,6 +1473,12 @@ struct arg_raw_mem_desc { int size; }; +/* Size of PTR_TO_MEM returned, taken from a constant allocation-size argument */ +struct ret_mem_desc { + u32 size; + bool found; +}; + struct bpf_kfunc_call_arg_meta { /* In parameters */ struct btf *btf; @@ -1484,7 +1490,6 @@ struct bpf_kfunc_call_arg_meta { u8 release_regno; bool r0_rdonly; u32 ret_btf_id; - u64 r0_size; u32 subprogno; struct { u64 value; @@ -1519,7 +1524,7 @@ struct bpf_kfunc_call_arg_meta { struct bpf_map_desc map; struct bpf_dynptr_desc dynptr; struct ref_obj_desc ref_obj; - u64 mem_size; + struct ret_mem_desc ret_mem; }; int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, -- cgit v1.2.3 From bf9c1b911f4db6fa5fe088c32f1de7ee1650eee9 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Tue, 14 Jul 2026 23:40:47 -0700 Subject: bpf: Unify helper and kfunc call argument meta Helper and kfunc argument checking carried two separate meta structs: the verifier-local struct bpf_call_arg_meta and bpf_kfunc_call_arg_meta. Merge them into a single struct bpf_call_arg_meta. This is groundwork for sharing argument checking between helpers and kfuncs. While merging, drop the btf_id field from the helper meta since it is never used. No functional change. Signed-off-by: Amery Hung Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260715064047.1793790-8-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf_verifier.h | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index e3eda72cbf67..682c2cd3b844 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1479,18 +1479,23 @@ struct ret_mem_desc { bool found; }; -struct bpf_kfunc_call_arg_meta { - /* In parameters */ +struct bpf_call_arg_meta { + /* Common */ struct btf *btf; u32 func_id; - u32 kfunc_flags; - const struct btf_type *func_proto; - const char *func_name; - /* Out parameters */ u8 release_regno; - bool r0_rdonly; u32 ret_btf_id; u32 subprogno; + struct bpf_map_desc map; + struct bpf_dynptr_desc dynptr; + struct ref_obj_desc ref_obj; + struct ret_mem_desc ret_mem; + + /* Only set by kfunc */ + bool r0_rdonly; + u32 kfunc_flags; + const struct btf_type *func_proto; + const char *func_name; struct { u64 value; bool found; @@ -1521,28 +1526,31 @@ struct bpf_kfunc_call_arg_meta { u8 spi; u8 frameno; } iter; - struct bpf_map_desc map; - struct bpf_dynptr_desc dynptr; - struct ref_obj_desc ref_obj; - struct ret_mem_desc ret_mem; + + /* Only set by helper */ + u64 msize_max_value; + s64 const_map_key; + struct btf *ret_btf; + struct btf_field *kptr_field; + struct arg_raw_mem_desc arg_raw_mem; }; int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, const struct bpf_func_proto **ptr); int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, s32 func_id, - s16 offset, struct bpf_kfunc_call_arg_meta *meta); + s16 offset, struct bpf_call_arg_meta *meta); bool bpf_is_async_callback_calling_insn(struct bpf_insn *insn); bool bpf_is_sync_callback_calling_insn(struct bpf_insn *insn); -static inline bool bpf_is_iter_next_kfunc(struct bpf_kfunc_call_arg_meta *meta) +static inline bool bpf_is_iter_next_kfunc(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ITER_NEXT; } -static inline bool bpf_is_kfunc_sleepable(struct bpf_kfunc_call_arg_meta *meta) +static inline bool bpf_is_kfunc_sleepable(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_SLEEPABLE; } -bool bpf_is_kfunc_pkt_changing(struct bpf_kfunc_call_arg_meta *meta); +bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta); struct bpf_iarray *bpf_iarray_realloc(struct bpf_iarray *old, size_t n_elem); int bpf_copy_insn_array_uniq(struct bpf_map *map, u32 start, u32 end, u32 *off); bool bpf_insn_is_cond_jump(u8 code); -- cgit v1.2.3 From 7a0855e73757ee9cf25ba635a1c735018ecba742 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Wed, 15 Jul 2026 22:11:22 +0800 Subject: bpf: Disallow interpreter fallback for BPF_ADDR_PERCPU insn The BPF_MOV64_PERCPU_REG insn requires JIT to emit native code to for 'dst_reg = src_reg + '. However, the interpreter ignores the 'off' at its ALU64_MOV_X label. The 'off' indicates the insn is BPF_MOV64_PERCPU_REG insn. Then, when the interpreter loads memory from the register, it will hit a page fault. [ 2.545572] BUG: unable to handle page fault for address: ffffffffacaaf034 [ 2.546485] #PF: supervisor read access in kernel mode [ 2.547167] #PF: error_code(0x0000) - not-present page [ 2.547850] PGD 134e63067 P4D 134e63067 PUD 134e64063 PMD 10021c063 PTE 800ffffeca550062 [ 2.548912] Oops: Oops: 0000 [#1] SMP PTI Set jit_required as true in order to disallow interpreter fallback in core.c::__bpf_prog_select_runtime(), if any BPF_ADDR_PERCPU insn is patched to the prog. BTW, rename the helper bpf_map_supports_cpu_flags() to bpf_map_is_percpu_map(). Fixes: 7bdbf7446305 ("bpf: add special internal-only MOV instruction to resolve per-CPU addrs") Signed-off-by: Leon Hwang Link: https://lore.kernel.org/bpf/20260715141122.15783-4-leon.hwang@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 31181e0c2b80..d9542127dfdf 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -4163,7 +4163,7 @@ bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image) } #endif -static inline bool bpf_map_supports_cpu_flags(enum bpf_map_type map_type) +static inline bool bpf_map_is_percpu_map(enum bpf_map_type map_type) { switch (map_type) { case BPF_MAP_TYPE_PERCPU_ARRAY: @@ -4190,7 +4190,7 @@ static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 all return -EINVAL; if (flags & (BPF_F_CPU | BPF_F_ALL_CPUS)) { - if (!bpf_map_supports_cpu_flags(map->map_type)) + if (!bpf_map_is_percpu_map(map->map_type)) return -EINVAL; if ((flags & BPF_F_CPU) && (flags & BPF_F_ALL_CPUS)) return -EINVAL; -- cgit v1.2.3 From 6bcecae4b65d80f4474c42a026a5ba229ded347e Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 8 Jul 2026 06:44:30 +0000 Subject: bpf: Extract the is_struct_ops_tramp helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the is_struct_ops_tramp helper, and use it in riscv as the current checks are somewhat hacky. Signed-off-by: Pu Lehui Reviewed-by: Björn Töpel Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/20260708064436.2971933-2-pulehui@huaweicloud.com Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d9542127dfdf..e066f44a9c05 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2196,6 +2196,12 @@ static inline bool is_tracing_multi(enum bpf_attach_type type) type == BPF_TRACE_FSESSION_MULTI; } +static inline bool is_struct_ops_tramp(const struct bpf_tramp_nodes *fentry_nodes) +{ + return fentry_nodes->nr_nodes == 1 && + fentry_nodes->nodes[0]->link->type == BPF_LINK_TYPE_STRUCT_OPS; +} + #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) /* This macro helps developer to register a struct_ops type and generate * type information correctly. Developers should use this macro to register -- cgit v1.2.3 From 5bf02dbf39fa0ec64661827fa4a1b4d5c1d942c0 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 16 Jul 2026 10:51:38 +0300 Subject: bpf, x86: Make sure allocation in arch_bpf_trampoline_size() is writable arch_bpf_trampoline_size() allocates a buffer to get actual size required for a trampoline. This buffer must be in the module address space because __arch_prepare_bpf_trampoline() calculates rel32 offsets relatively to that buffer. In preparation for enabling ROX mode for EXECMEM_BPF make sure that the allocated memory is writable. Add bpf_jit_alloc_exec_rw() wrapper for execmem_alloc_rw() and use it for buffer allocation in arch_bpf_trampoline_size(). Signed-off-by: Mike Rapoport (Microsoft) Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20260716-execmem-x86-rox-bpf-v0-v3-4-4e76158c01c5@kernel.org Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/filter.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/filter.h b/include/linux/filter.h index 14acb2455746..32d5297c557e 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1333,6 +1333,7 @@ bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, void bpf_jit_binary_free(struct bpf_binary_header *hdr); u64 bpf_jit_alloc_exec_limit(void); void *bpf_jit_alloc_exec(unsigned long size); +void *bpf_jit_alloc_exec_rw(unsigned long size); void bpf_jit_free_exec(void *addr); void bpf_jit_free(struct bpf_prog *fp); struct bpf_binary_header * -- cgit v1.2.3 From 61aaa8782bec59ecffd22e030f54ef9351bcabf9 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Wed, 22 Jul 2026 23:19:08 +0800 Subject: bpf: Fix WARNING in bpf_tracing_link_release The trampoline could be corrupted by the blindly 'tr->flags = BPF_TRAMP_F_TAIL_CALL_CTX' in verifier. 1. A fexit attached to a tail_call_reachable prog. 'tr->flags' became 'BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_TAIL_CALL_CTX'. And, the trampoline would poke the target prog's nop insn using jmp insn instead of call insn. 2. Another fexit loaded with the same tail_call_reachable prog target. 'tr->flags' became 'BPF_TRAMP_F_TAIL_CALL_CTX'. 3. Close the first fexit link. Due to no BPF_TRAMP_F_CALL_ORIG in 'tr->flags', the trampoline will fail to restore the prog's nop insn using call insn. [ 3.410719] WARNING: kernel/bpf/syscall.c:3551 at bpf_tracing_link_release+0x53/0x60, CPU#1: test_progs/98 ... [ 3.428793] bpf_link_free+0x58/0x130 [ 3.429293] bpf_link_release+0x23/0x30 Fix the warning by updating 'tr->flags' with '|=' and lock. Fixes: 2b5dcb31a19a ("bpf, x64: Fix tailcall infinite loop") Signed-off-by: Leon Hwang Reviewed-by: Pu Lehui Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/20260722151909.69142-2-leon.hwang@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e066f44a9c05..7bfc28673124 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1523,6 +1523,7 @@ int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids, struct bpf_tracing_multi_link *link); int bpf_trampoline_multi_detach(struct bpf_prog *prog, struct bpf_tracing_multi_link *link); +void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags); /* * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn @@ -1646,6 +1647,7 @@ static inline int bpf_trampoline_multi_detach(struct bpf_prog *prog, { return -ENOTSUPP; } +static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) {} #endif struct bpf_func_info_aux { -- cgit v1.2.3 From c0e091f30f0dd80d817c3a15d0a97962957e5786 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:24 -0700 Subject: bpf: Rename ARG_CONST_SIZE{,_OR_ZERO} to ARG_MEM_SIZE{,_OR_ZERO} ARG_CONST_SIZE does not require a constant: check_mem_size_reg() accepts any bounded scalar and verifies the memory access against its maximum (reg_umax). Rename ARG_CONST_SIZE and ARG_CONST_SIZE_OR_ZERO to ARG_MEM_SIZE and ARG_MEM_SIZE_OR_ZERO to reflect that. ARG_CONST_ALLOC_ SIZE_OR_ZERO, which does require a constant, is left unchanged. Pure rename, no functional change. Signed-off-by: Amery Hung Link: https://lore.kernel.org/bpf/20260801074633.1595644-10-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 4 ++-- include/linux/bpf_verifier.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 7bfc28673124..be53655d1362 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -888,8 +888,8 @@ enum bpf_arg_type { ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ ARG_PTR_TO_ARENA, - ARG_CONST_SIZE, /* number of bytes accessed from memory */ - ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ + ARG_MEM_SIZE, /* number of bytes accessed from memory */ + ARG_MEM_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ ARG_PTR_TO_CTX, /* pointer to context */ ARG_ANYTHING, /* any (initialized) argument is ok */ diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 682c2cd3b844..bb0d43814e90 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -15,7 +15,7 @@ * ensures that umax_value + (int)off + (int)size cannot overflow a u64. */ #define BPF_MAX_VAR_OFF (1 << 29) -/* Maximum variable size permitted for ARG_CONST_SIZE[_OR_ZERO]. This ensures +/* Maximum variable size permitted for ARG_MEM_SIZE[_OR_ZERO]. This ensures * that converting umax_value to int cannot overflow. */ #define BPF_MAX_VAR_SIZ (1 << 29) -- cgit v1.2.3 From f16e80c2c45175664ec6b0aaad6914fadbda395d Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:25 -0700 Subject: bpf: Fold __szk const size handling into the scalar arg path To align helper and kfunc pointer to memory argument handling, move kfunc constant memorry size argument handling to the kfunc scalar section. In addition, factor out constant scalar argument handling. The constant size argument (__szk) of a kfunc memory/size pair was recorded into meta->arg_constant by a dedicated block in the KF_ARG_PTR_TO_MEM_SIZE case, duplicating the "only one constant argument" and "must be a known constant" checks already in the generic scalar argument handling. That block also did an explicit i++ to skip the size argument. This also fixes a precision gap: the old dedicated block did not mark the size register precise, relying on check_mem_size_reg() for that. But check_mem_size_reg() is skipped when the buffer is a nullable arg passed as NULL (e.g. bpf_dynptr_slice(_rdwr) with a NULL buffer), so in that case the __szk value was recorded and used for regs[R0].mem_size without marking it precise. Routing the size through the scalar path marks it precise in all cases. Signed-off-by: Amery Hung Reviewed-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260801074633.1595644-11-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf_verifier.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index bb0d43814e90..b54c1a5c9b11 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1479,6 +1479,12 @@ struct ret_mem_desc { bool found; }; +/* A constant scalar argument; Populated by process_const_arg() */ +struct arg_constant_desc { + u64 value; + bool found; +}; + struct bpf_call_arg_meta { /* Common */ struct btf *btf; @@ -1496,10 +1502,7 @@ struct bpf_call_arg_meta { u32 kfunc_flags; const struct btf_type *func_proto; const char *func_name; - struct { - u64 value; - bool found; - } arg_constant; + struct arg_constant_desc arg_constant; /* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling, * generally to pass info about user-defined local kptr types to later -- cgit v1.2.3 From a49b70400b9de06234eb99f87cf60217ed98cc0c Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:33 -0700 Subject: bpf: Generate kfunc argument prototype at add-call time Kfunc argument checking re-derives each argument's kfunc_ptr_arg_type from BTF on every verification of a call in check_kfunc_args(). Now that get_kfunc_arg_type() is a function of the kfunc's BTF alone, it no longer inspects register state. The classification can be computed once when the call is added and cached. This is a step toward describing kfuncs with a bpf_func_proto and sharing the helper argument-checking path. Generate the classification at bpf_add_kfunc_call() time: - Extend struct bpf_func_proto to be able to describe a kfunc: widen arg_type[] and the arg_btf_id[]/arg_size[] union from 5 to MAX_BPF_FUNC_ARGS, since a kfunc may take up to 12 arguments (5 in registers, 7 on the stack). - Embed a bpf_func_proto in struct bpf_kfunc_desc, populated by gen_kfunc_arg_proto() which runs get_kfunc_arg_type() for each argument and stores the result in proto.arg_type[]. Grow the descriptor table's descs[] as a flexible array to not waste memory. - check_kfunc_args() reads the cached classification from meta->fn The KF_ARG_PTR_TO_CTX classification depends on the resolved program type, and for BPF_PROG_TYPE_EXT that is the target program's type, which resolve_prog_type() reads from prog->aux->saved_dst_prog_type. That field is normally recorded later during verification in check_attach_btf_id(), after bpf_add_kfunc_call() has run. Record saved_dst_prog_type and saved_dst_attach_type from dst_prog at program load time in bpf_prog_load() so the resolved type is available at add-call time without reordering check_attach_btf_id(). This keeps e.g. an freplace of an XDP program calling bpf_xdp_metadata_rx_hash() classifying its struct xdp_md * argument as context. The classification result is unchanged; it is only computed earlier and cached. Signed-off-by: Amery Hung Link: https://lore.kernel.org/bpf/20260801074633.1595644-19-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 36 ++++++++++++++++++------------------ include/linux/bpf_verifier.h | 9 ++++++--- 2 files changed, 24 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index be53655d1362..356884587ae1 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -960,6 +960,21 @@ enum bpf_return_type { }; static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); +/* The longest tracepoint has 12 args. + * See include/trace/bpf_probe.h + * + * Also reuse this macro for maximum number of arguments a BPF function + * or a kfunc can have. Args 1-5 are passed in registers, args 6-12 via + * stack arg slots. The JIT may map some stack arg slots to registers based + * on the native calling convention (e.g., arg 6 to R9 on x86-64). + */ +#define MAX_BPF_FUNC_ARGS 12 + +/* The maximum number of arguments passed through registers + * a single function may have. + */ +#define MAX_BPF_FUNC_REG_ARGS 5 + /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL * instructions after verifying @@ -984,7 +999,7 @@ struct bpf_func_proto { enum bpf_arg_type arg4_type; enum bpf_arg_type arg5_type; }; - enum bpf_arg_type arg_type[5]; + enum bpf_arg_type arg_type[MAX_BPF_FUNC_ARGS]; }; union { struct { @@ -994,7 +1009,7 @@ struct bpf_func_proto { u32 *arg4_btf_id; u32 *arg5_btf_id; }; - u32 *arg_btf_id[5]; + u32 *arg_btf_id[MAX_BPF_FUNC_ARGS]; struct { size_t arg1_size; size_t arg2_size; @@ -1002,7 +1017,7 @@ struct bpf_func_proto { size_t arg4_size; size_t arg5_size; }; - size_t arg_size[5]; + size_t arg_size[MAX_BPF_FUNC_ARGS]; }; int *ret_btf_id; /* return value btf_id */ bool (*allowed)(const struct bpf_prog *prog); @@ -1192,21 +1207,6 @@ struct bpf_prog_offload { u32 jited_len; }; -/* The longest tracepoint has 12 args. - * See include/trace/bpf_probe.h - * - * Also reuse this macro for maximum number of arguments a BPF function - * or a kfunc can have. Args 1-5 are passed in registers, args 6-12 via - * stack arg slots. The JIT may map some stack arg slots to registers based - * on the native calling convention (e.g., arg 6 to R9 on x86-64). - */ -#define MAX_BPF_FUNC_ARGS 12 - -/* The maximum number of arguments passed through registers - * a single function may have. - */ -#define MAX_BPF_FUNC_REG_ARGS 5 - /* The argument is a structure or a union. */ #define BTF_FMODEL_STRUCT_ARG BIT(0) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index b54c1a5c9b11..a2a40caca0a0 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1302,7 +1302,6 @@ static inline u32 type_flag(u32 type) return type & ~BPF_BASE_TYPE_MASK; } -/* only use after check_attach_btf_id() */ static inline enum bpf_prog_type resolve_prog_type(const struct bpf_prog *prog) { return (prog->type == BPF_PROG_TYPE_EXT && prog->aux->saved_dst_prog_type) ? @@ -1489,6 +1488,7 @@ struct bpf_call_arg_meta { /* Common */ struct btf *btf; u32 func_id; + const struct bpf_func_proto *fn; u8 release_regno; u32 ret_btf_id; u32 subprogno; @@ -1617,6 +1617,7 @@ enum bpf_reg_arg_type { struct bpf_kfunc_desc { struct btf_func_model func_model; + struct bpf_func_proto proto; u32 func_id; s32 imm; u16 offset; @@ -1624,13 +1625,15 @@ struct bpf_kfunc_desc { }; struct bpf_kfunc_desc_tab { + u32 nr_descs; /* Sorted by func_id (BTF ID) and offset (fd_array offset) during * verification. JITs do lookups by bpf_insn, where func_id may not be * available, therefore at the end of verification do_misc_fixups() * sorts this by imm and offset. + * + * Grown one entry at a time by bpf_add_kfunc_call(). */ - struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS]; - u32 nr_descs; + struct bpf_kfunc_desc descs[]; }; /* Functions exported from verifier.c, used by fixups.c */ -- cgit v1.2.3 From 457d4ecb47aaf7a2cb46aaadd76e8c812e4f3c9e Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sun, 2 Aug 2026 22:27:26 -0700 Subject: bpf: Remove unused BTF_FMODEL_STRUCT_ARG Commit 814cba835ef6 ("bpf, x86: Fix trampoline stack size for 128-bit arguments") changed the x86 trampoline to compute the number of registers from arg_size for every argument, which removed the last user of BTF_FMODEL_STRUCT_ARG. No other architecture or verifier code looks at the flag, so remove the macro and the code in __get_type_fmodel_flags() which sets it. Keep BTF_FMODEL_SIGNED_ARG at BIT(1) rather than renumbering it to BIT(0), so BIT(0) is available for a future flag. No functional change. Signed-off-by: Yonghong Song Signed-off-by: Daniel Borkmann Acked-by: Leon Hwang Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/20260803052726.2821447-1-yonghong.song@linux.dev --- include/linux/bpf.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 356884587ae1..73bacfc6444d 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1207,9 +1207,6 @@ struct bpf_prog_offload { u32 jited_len; }; -/* The argument is a structure or a union. */ -#define BTF_FMODEL_STRUCT_ARG BIT(0) - /* The argument is signed. */ #define BTF_FMODEL_SIGNED_ARG BIT(1) -- cgit v1.2.3 From ed3b3093b6242bdb2c4acfb932d4b15db4e33948 Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Wed, 5 Aug 2026 23:33:38 +0800 Subject: bpf: Add KF_SPINLOCK_SAFE flag for kfuncs under bpf_spin_lock Introduce the KF_SPINLOCK_SAFE kfunc metadata flag in BTF so kfuncs may be explicitly marked as safe to call while holding bpf_spin_lock. Allow kfuncs defined in kernel modules to be marked with KF_SPINLOCK_SAFE. Example: BTF_ID_FLAGS(func, $kfunc_name, KF_SPINLOCK_SAFE) Signed-off-by: Kaitao Cheng Acked-by: Leon Hwang Link: https://lore.kernel.org/bpf/20260805153340.34776-2-kaitao.cheng@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/btf.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/btf.h b/include/linux/btf.h index c09b7994de4e..3f5255d095a2 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -79,6 +79,7 @@ #define KF_ARENA_ARG1 (1 << 14) /* kfunc takes an arena pointer as its first argument */ #define KF_ARENA_ARG2 (1 << 15) /* kfunc takes an arena pointer as its second argument */ #define KF_IMPLICIT_ARGS (1 << 16) /* kfunc has implicit arguments supplied by the verifier */ +#define KF_SPINLOCK_SAFE (1 << 17) /* kfunc is allowed inside bpf_spin_lock-ed region */ /* * Tag marking a kernel function as a kfunc. This is meant to minimize the -- cgit v1.2.3 From e2577cd62060be91a3d7d11a56e5a61faae4b7f7 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Thu, 6 Aug 2026 22:10:43 +0200 Subject: bpf, riscv: Add and use bpf_atomic_is_load_acq() helper A load-acquire is the only BPF_STX class instruction that reads from src_reg into dst_reg, that is, it has the operand roles of a BPF_LDX. JIT code which tells loads from stores apart by instruction class alone has to special case it, for example when deciding which register holds the faulting address and which one to clear from an exception handler. riscv64 already does so, open coded as a bare insn->imm test. Add a bpf_atomic_is_load_acq() helper and convert riscv64 over to it, so that the x86-64 and arm64 JITs can use the same helper in subsequent patches. Unlike bpf_atomic_is_load_store(), which presumes that its argument is already known to be a BPF_ATOMIC instruction, the new helper is called from code which still sees all instruction classes, so it checks class and mode itself. Also, move bpf_atomic_is_load_store() to filter.h next to BPF_ATOMIC_OP, so that both helpers stay together. No functional change intended. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260806201047.333389-2-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 15 --------------- include/linux/filter.h | 31 +++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 73bacfc6444d..d79bf7557ef6 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1132,21 +1132,6 @@ static inline bool bpf_pseudo_func(const struct bpf_insn *insn) return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC; } -/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an - * atomic load or store, and false if it is a read-modify-write instruction. - */ -static inline bool -bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn) -{ - switch (atomic_insn->imm) { - case BPF_LOAD_ACQ: - case BPF_STORE_REL: - return true; - default: - return false; - } -} - struct bpf_prog_ops { int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); diff --git a/include/linux/filter.h b/include/linux/filter.h index 32d5297c557e..41b02d53e222 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -383,6 +383,37 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn) /* Legacy alias */ #define BPF_STX_XADD(SIZE, DST, SRC, OFF) BPF_ATOMIC_OP(SIZE, BPF_ADD, DST, SRC, OFF) +/* + * Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an + * atomic load or store, and false if it is a read-modify-write instruction. + */ +static inline bool +bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn) +{ + switch (atomic_insn->imm) { + case BPF_LOAD_ACQ: + case BPF_STORE_REL: + return true; + default: + return false; + } +} + +/* + * A load-acquire is the only BPF_STX class instruction that reads into + * dst_reg from src_reg + off16, i.e. it has the operand roles of a BPF_LDX. + * Unlike bpf_atomic_is_load_store(), @insn is not assumed to be a BPF_ATOMIC + * instruction here, so that callers which walk all instruction classes can + * use this directly. + */ +static inline bool bpf_atomic_is_load_acq(const struct bpf_insn *insn) +{ + return BPF_CLASS(insn->code) == BPF_STX && + (BPF_MODE(insn->code) == BPF_ATOMIC || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && + insn->imm == BPF_LOAD_ACQ; +} + /* Memory store, *(uint *) (dst_reg + off16) = imm32 */ #define BPF_ST_MEM(SIZE, DST, OFF, IMM) \ -- cgit v1.2.3 From 7ce090afbf725e25fff29caacce1eaf8459b1117 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 7 Aug 2026 13:59:34 -0700 Subject: bpf: Infer zext_dst based on static register liveness analysis As reported in the thread [1], the verifier's 32-bit operations zero extension logic is broken. This logic is responsible for correct semantics of 32-bit operations on s390 architecture. According to BPF semantics, operation `w1 += 1` is supposed to zero extend the upper half of the register `r1`. On s390 the JIT relies on the verifier emitting explicit zero extension before such operations. The verifier attempts to minimize the amount of zero extensions inserted by tracking whether upper halves of the 64-bit registers are ever used. Previously such tracking worked as follows: - bpf_reg_state->subreg_def field was set by do_check_insn() for each operation defining lower but not the upper halves of the register. - Whenever an operation reading the whole register was verified, the verifier checked register's subreg_def and set bpf_insn_aux_data->zext_dst flag as true via a call to mark_insn_zext() function. - After the verification was complete, a special pass bpf_opt_subreg_zext_lo32_rnd_hi32() extended 32-bit operations with bpf_insn_aux_data->zext_dst set as true by adding explicit zero extension. Note that the logic above relies on bpf_reg_state->subreg_def, which is a property of a current verifier state. Before the commit [2] two additional steps happened: - The verifier tracked upper and lower register halves' liveness as flags REG_LIVE_READ{32,64} in bpf_reg_state->live. - The function propagate_liveness() called mark_insn_zext() in order to transfer the knowledge about which registers have their upper halves alive (and thus might require zero extension). The commit [2] removed the two steps described above, hence making possible a situation like below: - The register's upper half is set and is used on some verification path P1 and the register happens not to be marked as precise. - The checkpoint C is created while processing some instruction between register initialization and usage. - On some other verification path P2 the register's upper half is not initialized and that path ends hitting the checkpoint C. - In such a case the register's initialization on path P2 would lack zext_dst mark, making it possible for the program to inject an arbitrary value in the register's upper half. This commit replaces subreg_def based logic with computing zext_dst statically, as a part of the bpf_compute_live_registers() analysis: - The analysis now tracks usage of upper and lower halves of the registers separately. - If some instruction defines a 32-bit subregister, but not the whole register, *and* the upper half of the register is alive after that instruction, the instruction is marked as zext_dst. There is one notable drop in precision: whenever a BPF subprogram is called, all 64 bits of parameter registers are presumed to be used. The assumption is that such a drop in precision would not inflict a noticeable performance penalty. [1] https://lore.kernel.org/bpf/CAGKGUv=sOuqQtA1Ub-5JXfA4FPosJFYKAQE4B79cK+P1erxqtg@mail.gmail.com/ [2] commit 107e16979905 ("bpf: disable and remove registers chain based liveness") Fixes: 107e16979905 ("bpf: disable and remove registers chain based liveness") Reported-by: Min-gyu Kim Reported-by: STAR Labs SG Signed-off-by: Eduard Zingerman Signed-off-by: Daniel Borkmann Acked-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/CAGKGUv=sOuqQtA1Ub-5JXfA4FPosJFYKAQE4B79cK+P1erxqtg@mail.gmail.com/ Link: https://lore.kernel.org/bpf/20260807-static-zext-v4-5-b6c270013c77@gmail.com --- include/linux/bpf_verifier.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index a2a40caca0a0..2c74d676ede9 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -162,11 +162,6 @@ struct bpf_reg_state { * pointing to bpf_func_state. */ u32 frameno; - /* Tracks subreg definition. The stored value is the insn_idx of the - * writing insn. This is safe because subreg_def is used before any insn - * patching which only happens after main verification finished. - */ - s32 subreg_def; /* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */ bool precise; }; @@ -1637,7 +1632,6 @@ struct bpf_kfunc_desc_tab { }; /* Functions exported from verifier.c, used by fixups.c */ -bool bpf_is_reg64(struct bpf_insn *insn, u32 regno, struct bpf_reg_state *reg, enum bpf_reg_arg_type t); void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len); void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog); bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env); @@ -1661,5 +1655,6 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env); int bpf_jit_subprogs(struct bpf_verifier_env *env); int bpf_fixup_call_args(struct bpf_verifier_env *env); int bpf_do_misc_fixups(struct bpf_verifier_env *env); +int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn); #endif /* _LINUX_BPF_VERIFIER_H */ -- cgit v1.2.3 From 04962afb3cd6a3cbf1a3679c0c95049833db36c1 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 8 Aug 2026 02:39:21 +0200 Subject: bpf: Rename 'early' BTF checking as a preparation phase BTF processing is split around subprogram discovery. The first phase gets program BTF and imports func_info because a BTF-tagged exception callback may not be referenced by any instruction. Subprogram discovery needs this metadata to find it. The later phase validates func_info and line_info against the complete subprogram table and applies CO-RE relocations. This split breaks a real dependency cycle rather than merely running the same checks early. Rename bpf_check_btf_info_early() and check_btf_func_early() to preparation names that reflect this role. Add short call-site comments to make the two phases and their responsibilities clear. No functional change is intended. Signed-off-by: Kumar Kartikeya Dwivedi Reviewed-by: Amery Hung Link: https://patch.msgid.link/20260808003938.3486067-2-memxor@gmail.com Signed-off-by: Eduard Zingerman --- include/linux/bpf_verifier.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 2c74d676ede9..1ccf82d7ff8d 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1172,8 +1172,8 @@ static inline void bpf_trampoline_unpack_key(u64 key, u32 *obj_id, u32 *btf_id) *btf_id = key & 0x7FFFFFFF; } -int bpf_check_btf_info_early(struct bpf_verifier_env *env, - const union bpf_attr *attr, bpfptr_t uattr); +int bpf_prepare_btf_info(struct bpf_verifier_env *env, + const union bpf_attr *attr, bpfptr_t uattr); int bpf_check_btf_info(struct bpf_verifier_env *env, const union bpf_attr *attr, bpfptr_t uattr); -- cgit v1.2.3 From 252d367163268cb8d3fe321c1fc2b15a60faf9ea Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 8 Aug 2026 02:39:24 +0200 Subject: bpf: Support __arena and __arena__nullable kfunc argument suffixes Passing an arena pointer to a kfunc takes two steps today. There is no arena pointer argument type, so the pointer crosses the boundary as a bare scalar, and the kfunc then offsets it by the arena base and casts it before it can touch the memory. Every such kfunc open-codes the same translation. Add the __arena and __arena__nullable argument suffixes to make this more convenient. The kfunc declares the parameter by its real pointer type and dereferences it directly, with the JIT rebasing the value at the call site, rN = kern_vm_start + (u32)rN. No bounds check is needed: the u32 offset stays within the guard-padded arena kernel mapping, and a fault on an unpopulated page recovers through the per-arena scratch page. A suffixed argument accepts a PTR_TO_ARENA or scalar register, matching global subprog arena arguments. __arena rebases unconditionally, so the kfunc never sees NULL and a value with zero in the low 32 bits arrives as the arena base. __arena__nullable preserves NULL for optional arguments by skipping the rebase when the truncated value, arena offset 0, is zero. Keeping the plain form NULL-free saves the NULL test on every call. The double separator makes the annotations composable: __arena__nullable also ends in __nullable and naturally follows the common nullable argument path. Plain __arena follows that path too for verifier type checking because both forms accept a constant zero; the function-model flag still determines whether the JIT preserves NULL or rebases it to the arena base. This patch adds the verifier side: the suffixes are recognized in check_kfunc_args() and distilled into argument flags in the function model stored in the kfunc descriptor. JITs retrieve the model while emitting the call, avoiding per-call state in insn_aux_data. JITs declare support with bpf_jit_supports_arena_args() and verification fails with -ENOTSUPP elsewhere. Co-developed-by: Kumar Kartikeya Dwivedi Signed-off-by: Tejun Heo Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260808003938.3486067-5-memxor@gmail.com Signed-off-by: Eduard Zingerman --- include/linux/bpf.h | 6 ++++++ include/linux/filter.h | 1 + 2 files changed, 7 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d79bf7557ef6..ba1b9d8ac348 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1195,6 +1195,12 @@ struct bpf_prog_offload { /* The argument is signed. */ #define BTF_FMODEL_SIGNED_ARG BIT(1) +/* The argument is an arena pointer. */ +#define BTF_FMODEL_ARENA_ARG BIT(2) + +/* The argument is nullable. */ +#define BTF_FMODEL_NULLABLE_ARG BIT(3) + struct btf_func_model { u8 ret_size; u8 ret_flags; diff --git a/include/linux/filter.h b/include/linux/filter.h index 41b02d53e222..4edba8182db1 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1214,6 +1214,7 @@ bool bpf_jit_supports_subprog_tailcalls(void); bool bpf_jit_supports_percpu_insn(void); bool bpf_jit_supports_kfunc_call(void); bool bpf_jit_supports_stack_args(void); +bool bpf_jit_supports_arena_args(void); bool bpf_jit_supports_far_kfunc_call(void); bool bpf_jit_supports_exceptions(void); bool bpf_jit_supports_ptr_xchg(void); -- cgit v1.2.3 From f6c33c4479a7e4d6bfc0e0e533a86376866f7360 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 8 Aug 2026 02:39:25 +0200 Subject: bpf: Support __arena and __arena__nullable on struct_ops arguments A struct_ops callback cannot receive an arena pointer directly, so passing one takes two steps. The pointer arrives as a bare u64 that the callback casts, and because the two sides address the arena through different bases it also has to be rebased by hand on the way in. Add the __arena and __arena__nullable stub argument suffixes to make this convenient. The callback declares the parameter as an arena pointer, receives it as a PTR_TO_ARENA register, and dereferences it directly, while the kernel caller just passes the natural kernel arena address (kaddr). The trampoline converts the value while saving the arguments into the BPF ctx, ctx[slot] = (u32)(kaddr - kern_vm_start), so the program never sees a kernel address and nothing rewrites the ctx after the fact. The converted value keeps the upper 32 bits clear as the JITs require of arena pointer registers and behaves like any cast_kern'ed arena pointer, so cast_user recovers the full user-visible address. __arena converts unconditionally and the kernel caller must not pass NULL. __arena__nullable preserves NULL, tested on the full 64-bit kernel pointer, and surfaces to the verifier as PTR_TO_ARENA (but not as a PTR_TO_ARENA | PTR_MAYBE_NULL). The reason is that PTR_TO_ARENA in the program's type state already encompasses NULL-ness, so it is not meaningful to force a NULL check for the program. The composite suffix intentionally ends in __nullable. Classify __arena__nullable before the generic suffix so scalar arena pointees do not take the generic nullable BTF pointer path. This patch adds the generic side. prepare_arg_info() records arena and nullable argument flags in the struct_ops function model, and bpf_tramp_arena_base() returns the arena base for a single-program struct_ops indirect trampoline. Only that trampoline converts: its program's arena is fixed at generation time. Generic trampolines can mix programs with different arenas and reject arena context arguments defensively, which is unreachable today as only struct_ops programs carry them. Architectures that do not implement the conversion are gated out at verification time with bpf_jit_supports_arena_args(). Co-developed-by: Kumar Kartikeya Dwivedi Signed-off-by: Tejun Heo Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260808003938.3486067-6-memxor@gmail.com Signed-off-by: Eduard Zingerman --- include/linux/bpf.h | 9 +++++++++ include/linux/bpf_verifier.h | 10 ++++++++++ 2 files changed, 19 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index ba1b9d8ac348..b4a10c9878cf 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1274,6 +1274,15 @@ struct bpf_tramp_nodes { int nr_nodes; }; +/* + * The arena base against which a struct_ops trampoline converts the + * arguments marked with BTF_FMODEL_ARENA_ARG while saving them into the BPF + * ctx, ctx[arg] = (u32)(kaddr - kern_vm_start). Zero when the trampoline + * converts nothing. + */ +u64 bpf_tramp_arena_base(const struct btf_func_model *m, + struct bpf_tramp_nodes *tnodes, u32 flags); + struct bpf_tramp_run_ctx; /* Different use cases for BPF trampoline: diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 1ccf82d7ff8d..93f7c2075eea 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1297,6 +1297,16 @@ static inline u32 type_flag(u32 type) return type & ~BPF_BASE_TYPE_MASK; } +static inline bool bpf_prog_has_arena_ctx_arg(const struct bpf_prog *prog) +{ + int i; + + for (i = 0; i < prog->aux->ctx_arg_info_size; i++) + if (base_type(prog->aux->ctx_arg_info[i].reg_type) == PTR_TO_ARENA) + return true; + return false; +} + static inline enum bpf_prog_type resolve_prog_type(const struct bpf_prog *prog) { return (prog->type == BPF_PROG_TYPE_EXT && prog->aux->saved_dst_prog_type) ? -- cgit v1.2.3 From 41c5dbb4be3c1ef4a5e2ce4c28de60b2be3cdccf Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 11 Aug 2026 15:15:55 +0200 Subject: bpf: Derive the atomic load register in one place check_atomic_rmw() open codes the mapping from a BPF_ATOMIC to the register it reads the old value into, the BPF_STX case of insn_def_regno() open codes the very same mapping a second time, the const folding and the liveness transfer functions a third and a fourth time, and BPF JITs need it as well to know which register a faulting BPF_PROBE_ATOMIC has to clear. Add a small helper so that all of them can share it. No functional change. The BPF_LOAD_ACQ case is there for the JITs, which do walk all instruction classes. const_reg_xfer() loses its explicit BPF_ATOMIC mode test since the helper checks class and mode itself; the BPF_PROBE_ATOMIC it additionally accepts cannot be seen there as it is only set from bpf_do_misc_fixups(), that is, after const folding has run. arg_track_xfer() keeps its mode test since that also guards the stack clearing next to it. Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260811131600.506721-1-daniel@iogearbox.net Signed-off-by: Eduard Zingerman --- include/linux/filter.h | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'include/linux') diff --git a/include/linux/filter.h b/include/linux/filter.h index 4edba8182db1..15d83684c6e9 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -414,6 +414,30 @@ static inline bool bpf_atomic_is_load_acq(const struct bpf_insn *insn) insn->imm == BPF_LOAD_ACQ; } +/* + * Given an instruction @insn, return the number of the BPF register that a + * BPF_ATOMIC reads the value at its memory operand into, or -1 if there is + * no such register. That is the register a BPF_PROBE_ATOMIC has to clear when + * the access faults. Like bpf_atomic_is_load_acq(), @insn is not assumed to + * be a BPF_ATOMIC here. + */ +static inline int bpf_atomic_load_reg(const struct bpf_insn *insn) +{ + if (BPF_CLASS(insn->code) != BPF_STX || + (BPF_MODE(insn->code) != BPF_ATOMIC && + BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)) + return -1; + + switch (insn->imm) { + case BPF_LOAD_ACQ: + return insn->dst_reg; + case BPF_CMPXCHG: + return BPF_REG_0; + default: + return (insn->imm & BPF_FETCH) ? insn->src_reg : -1; + } +} + /* Memory store, *(uint *) (dst_reg + off16) = imm32 */ #define BPF_ST_MEM(SIZE, DST, OFF, IMM) \ -- cgit v1.2.3 From 14c950ac2be8cadb63e1bfe22111ab0fdc829eb8 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Thu, 13 Aug 2026 00:19:18 +0200 Subject: bpf: Track verifier instruction stats for each subprogram The verifier currently records one instruction count for the main program and each global subprogram checked independently. Static subprograms are explored within callers, so their verification cost cannot be reported separately. Track both self and inclusive instruction counts for every subprogram. Charge each processed instruction as self work to the current subprogram and to a path-local subtotal in its function frame. When a function returns, add the callee subtotal to its inclusive count and to its parent subtotal. Fold any remaining frames when a path terminates or is pruned. Instruction subtotals are accounting state, not semantic verifier state. Clear them when a verifier state is copied so work before a path fork is charged once, rather than again when a saved branch is explored. If copying a saved state fails before all frames are allocated, skip missing frames while folding the current path. This generic frame accounting also records self and inclusive totals when an asynchronous callback starts as a fresh frame-zero state. It does not yet charge that independently explored callback path back to the main or global exploration root which scheduled it. That will be done in subsequent changes. This does not change the verification statistics output format. It only prepares the counters for per-subprogram reporting. Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260812221925.3358041-2-memxor@gmail.com Signed-off-by: Eduard Zingerman --- include/linux/bpf_verifier.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 93f7c2075eea..7c376451db82 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -380,6 +380,8 @@ struct bpf_func_state { * | number of simulations is tracked in frame N */ u32 callback_depth; + /* Instructions processed in this frame and callees on the current path. */ + u32 insns_subtotal; /* The following fields should be last. See copy_func_state() */ /* The state of the stack. Each element of the array describes BPF_REG_SIZE @@ -798,7 +800,8 @@ struct bpf_subprog_info { u32 exit_idx; /* Index of one of the BPF_EXIT instructions in this subprogram */ u16 stack_depth; /* max. stack depth used by this function */ u16 stack_extra; - u32 insn_processed; + u32 insns_total; + u32 insns_self; /* offsets in range [stack_depth .. fastcall_stack_off) * are used for bpf_fastcall spills and fills. */ -- cgit v1.2.3 From 7c6beeb8c88f92866daab4516220667d1234d3c9 Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Tue, 11 Aug 2026 10:46:20 +0800 Subject: bpf: Make bpf_trampoline_multi_detach return void bpf_trampoline_multi_detach() always returns 0 and the sole caller ignores the return value. Change it to return void and drop the WARN_ON_ONCE at the call site. Signed-off-by: Hui Zhu Acked-by: Leon Hwang Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/12beba657f5c9e86a016a097750209287a2f262a.1786412280.git.zhuhui@kylinos.cn Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index b4a10c9878cf..f4e8d372253a 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1518,8 +1518,8 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids, struct bpf_tracing_multi_link *link); -int bpf_trampoline_multi_detach(struct bpf_prog *prog, - struct bpf_tracing_multi_link *link); +void bpf_trampoline_multi_detach(struct bpf_prog *prog, + struct bpf_tracing_multi_link *link); void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags); /* @@ -1639,10 +1639,9 @@ static inline int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids, { return -ENOTSUPP; } -static inline int bpf_trampoline_multi_detach(struct bpf_prog *prog, - struct bpf_tracing_multi_link *link) +static inline void bpf_trampoline_multi_detach(struct bpf_prog *prog, + struct bpf_tracing_multi_link *link) { - return -ENOTSUPP; } static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) {} #endif -- cgit v1.2.3 From 7c3e54cb82fda75a374e2b29b25d2a399911a008 Mon Sep 17 00:00:00 2001 From: Xu Kuohai Date: Tue, 28 Jul 2026 20:25:49 +0000 Subject: bpf: Eliminate dup/restore of insn_aux_data The dup/restore of insn_aux_data was introduced to resolve the inconsistency between insnsi and insn_aux_data arrays, which occurs on the failure path where insnsi was rolled back to the original state before constants blinding, while insn_aux_data was not. After JIT failure, there is only one user, bpf_clear_insn_aux_data(), that requires insnsi and insn_aux_data to be synchronized. It accesses both insnsi and insn_aux_data using the same array size and index. However, the access to insnsi in bpf_clear_insn_aux_data() is not necessary. It is checked to skip the second slot of an ldimm64 instruction, whose jt is never set and can be absorbed into the jt check itself. So remove the access to insnsi from bpf_clear_insn_aux_data(), and add a specific length field for insn_aux_data to allow it to have a different length from the insnsi array. Then remove dup/restore of insn_aux_data. Signed-off-by: Xu Kuohai Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/5a4528f019c8d2638c019a2f37475cccc16a9503.1785240296.git.xukuohai@huawei.com Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf_verifier.h | 1 + include/linux/filter.h | 13 ------------- 2 files changed, 1 insertion(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 7c376451db82..27b43fda9b17 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -946,6 +946,7 @@ struct bpf_verifier_env { bool seen_direct_write; bool seen_exception; bool signature; + u32 insn_aux_data_len; struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */ const struct bpf_line_info *prev_linfo; struct bpf_verifier_log log; diff --git a/include/linux/filter.h b/include/linux/filter.h index 15d83684c6e9..4a9bc6a848f2 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1267,25 +1267,12 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, #ifdef CONFIG_BPF_SYSCALL struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, const struct bpf_insn *patch, u32 len); -struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env); -void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_insn_aux_data *orig_insn_aux); #else static inline struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, const struct bpf_insn *patch, u32 len) { return ERR_PTR(-ENOTSUPP); } - -static inline struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env) -{ - return NULL; -} - -static inline void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_insn_aux_data *orig_insn_aux) -{ -} #endif /* CONFIG_BPF_SYSCALL */ int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt); -- cgit v1.2.3 From f5b57e9e9cfd9736246eb9a5f385da451d199039 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Fri, 14 Aug 2026 08:56:23 -0700 Subject: bpf: Populate mmap-able array map memory lazily An mmap-able BPF array map (BPF_F_MMAPABLE) has its backing memory vmalloc'ed up front at map creation time. array_map_mmap() then wired up the whole mapping eagerly via remap_vmalloc_range(), which calls vm_insert_page() for every page of the map. For large maps this makes every mmap() O(number of pages): an 8MiB map inserts 2048 PTEs per mmap() and tears them all down again on munmap(), even when user space only touches a few pages (or none at all). Populate the mapping lazily instead, the same way the arena map already does. array_map_mmap() now only performs the bounds check and returns, leaving the PTEs unpopulated; pages are inserted on demand by a new array_map_mmap_fault() handler. Because the memory is already resident, the fault handler simply resolves the vmalloc page and hands it to the fault path. This makes mmap() O(1), and munmap() proportional to the number of pages that were actually faulted in rather than to the size of the map. The handler is reached through a new optional ->map_mmap_fault callback. Maps that provide it get a vm_operations_struct with a .fault handler; maps that populate their mapping eagerly keep the one they had. Both share the same open/close callbacks, so the existing VMA accounting (VM_MAYWRITE write-active tracking, freeze handling) stays centralized rather than each map installing its own vm_operations_struct. Callers that want the pages populated up front can still request that explicitly with MAP_POPULATE. Kernel-side access to the map (via the vmalloc address) is unaffected. Time for one mmap()+munmap() of an 8MiB mmap-able array map: before after no MAP_POPULATE, no access 226us 1.1us no MAP_POPULATE, access all pages 236us 1341us MAP_POPULATE, no access 312us 493us MAP_POPULATE, access all pages 318us 519us Mapping without touching the data, which is what this change targets, gets ~160x cheaper. Faulting in the whole mapping one page at a time is more expensive than the eager remap_vmalloc_range() loop, so users that do touch every page should ask for MAP_POPULATE. Note that MAP_POPULATE is not free before this change either: it adds ~85us (226us => 312us) for no benefit, as the mapping is already fully populated. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Song Liu Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260814155623.111565-1-song@kernel.org --- include/linux/bpf.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index f4e8d372253a..04cadd987169 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -145,6 +145,7 @@ struct bpf_map_ops { int (*map_direct_value_meta)(const struct bpf_map *map, u64 imm, u32 *off); int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma); + vm_fault_t (*map_mmap_fault)(struct bpf_map *map, struct vm_fault *vmf); __poll_t (*map_poll)(struct bpf_map *map, struct file *filp, struct poll_table_struct *pts); unsigned long (*map_get_unmapped_area)(struct file *filep, unsigned long addr, -- cgit v1.2.3 From b9c5d822f677e065971481c4bafe5d84b5451082 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:45:57 +0200 Subject: bpf: Add source and instruction diagnostic context Teach verifier diagnostics to annotate an instruction with BTF source line information and nearby BPF instructions. The renderer keeps source text in a fixed-width lane and prints instructions in a stable right-hand gutter. Wrap annotation text under the source line so long error labels remain readable while the source and instruction lanes keep their fixed layout. Keeping source and instruction context in one commit preserves the visual layout contract that later diagnostic reports rely on. Signed-off-by: Kumar Kartikeya Dwivedi Link: https://patch.msgid.link/20260815064612.378577-3-memxor@gmail.com Signed-off-by: Eduard Zingerman --- include/linux/bpf.h | 12 ++++++++++-- include/linux/bpf_verifier.h | 4 ++++ include/linux/btf.h | 1 + 3 files changed, 15 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 04cadd987169..ffa5626411ac 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -4147,8 +4147,16 @@ static inline bool bpf_is_subprog(const struct bpf_prog *prog) } const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn_off); -void bpf_get_linfo_file_line(struct btf *btf, const struct bpf_line_info *linfo, - const char **filep, const char **linep, int *nump); +struct bpf_linfo_source { + const char *file; + const char *line; + u32 file_name_off; + int line_num; + int line_col; +}; + +void bpf_get_linfo_source(struct btf *btf, const struct bpf_line_info *linfo, + struct bpf_linfo_source *src); int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep, const char **linep, int *nump); struct bpf_prog *bpf_prog_find_from_stack(void); diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 27b43fda9b17..579a288bc8de 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -833,6 +833,7 @@ static inline u16 bpf_in_stack_arg_cnt(const struct bpf_subprog_info *sub) return 0; } +struct bpf_diag; struct bpf_verifier_env; struct backtrack_state { @@ -950,6 +951,7 @@ struct bpf_verifier_env { struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */ const struct bpf_line_info *prev_linfo; struct bpf_verifier_log log; + struct bpf_diag *diag; struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */ /* subprog indices sorted in topological order: leaves first, callers last */ int subprog_topo_order[BPF_MAX_SUBPROGS + 2]; @@ -1433,8 +1435,10 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate, u32 frameno); u32 bpf_vlog_alignment(u32 pos); +const char *bpf_disasm_kfunc_name(void *data, const struct bpf_insn *insn); struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off); +const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog); int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); diff --git a/include/linux/btf.h b/include/linux/btf.h index 3f5255d095a2..7ea13768c979 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -214,6 +214,7 @@ int btf_type_seq_show_flags(const struct btf *btf, u32 type_id, void *obj, */ int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, char *buf, int len, u64 flags); +int btf_type_name_to_buf(const struct btf *btf, u32 type_id, char *buf, int len); int btf_get_fd_by_id(u32 id); u32 btf_obj_id(const struct btf *btf); -- cgit v1.2.3 From af4ea6e20fff383cdc2f01b9a372b4c7a0abf5ff Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:46:00 +0200 Subject: bpf: Track verifier register diagnostic events Record material register and outgoing stack argument changes so diagnostics can explain how a value reached its current type, bounds, or unreadable state. Store old and new register types, scalar ranges, tnum value and mask, map and BTF type identity, and basic operand metadata in the environment-owned diagnostic event stream. Record invalidations when packet data moves, references are released, or borrowed references leave their protected region. Register-scoped history starts at the latest matching modification and then shows later branch outcomes. Also record fixed stack spills and overwrites, and tag register fills from stack so register-scoped history can follow value flow through spilled stack slots. The type_is_map_ptr() helper previously lived as a static function in kernel/bpf/log.c since commit 0c95c9fdb696 ("bpf: emit map name in register state if applicable and available"). Move it verbatim to include/linux/bpf_verifier.h as a static inline, next to the other type classifiers, so diagnostics.c can reuse it without duplicating the case list. Signed-off-by: Kumar Kartikeya Dwivedi Link: https://patch.msgid.link/20260815064612.378577-6-memxor@gmail.com Signed-off-by: Eduard Zingerman --- include/linux/bpf_verifier.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 579a288bc8de..bc2af02547fe 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -354,6 +354,11 @@ struct bpf_func_state { * 0 = main function, 1 = first callee. */ u32 frameno; + /* + * Unique diagnostic identity for this function invocation. Frame depth is + * reused after returns, while this ID is preserved across state clones. + */ + u32 diag_frame_id; /* subprog number == index within subprog_info * zero == main subprog */ @@ -1351,6 +1356,18 @@ static inline bool type_is_non_owning_ref(u32 type) return type_is_ptr_alloc_obj(type) && type_flag(type) & NON_OWN_REF; } +static inline bool type_is_map_ptr(enum bpf_reg_type type) +{ + switch (base_type(type)) { + case CONST_PTR_TO_MAP: + case PTR_TO_MAP_KEY: + case PTR_TO_MAP_VALUE: + return true; + default: + return false; + } +} + static inline bool type_is_pkt_pointer(enum bpf_reg_type type) { type = base_type(type); -- cgit v1.2.3 From 5bd369c05576ec12f62f02c65b576bf0bf074131 Mon Sep 17 00:00:00 2001 From: Mahe Tardy Date: Thu, 13 Aug 2026 11:05:36 +0000 Subject: net: Add connect_socket() helper Add a helper that connects an existing socket while invoking the LSM hook. Reuse it in __sys_connect_file() to avoid duplicating the connect logic. Other socket operations have equivalent helpers that trigger the appropriate LSM hooks that can be reused, this one was the only one missing. This will be used in the next commit for a new BPF kfunc that needs to connect a socket and trigger the LSM hook. Signed-off-by: Mahe Tardy Signed-off-by: Daniel Borkmann Reviewed-by: Jiayuan Chen Reviewed-by: Kuniyuki Iwashima Acked-by: Song Liu Acked-by: Stanislav Fomichev Link: https://lore.kernel.org/bpf/20260813110540.103550-2-mahe.tardy@gmail.com --- include/linux/socket.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/socket.h b/include/linux/socket.h index 2a8d7b14f1d1..5a5eb1250103 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -461,6 +461,8 @@ extern struct file *__sys_socket_file(int family, int type, int protocol); extern int __sys_bind(int fd, struct sockaddr __user *umyaddr, int addrlen); extern int __sys_bind_socket(struct socket *sock, struct sockaddr_storage *address, int addrlen); +int connect_socket(struct socket *sock, struct sockaddr_storage *addr, + int addrlen, int flags); extern int __sys_connect_file(struct file *file, struct sockaddr_storage *addr, int addrlen, int file_flags); extern int __sys_connect(int fd, struct sockaddr __user *uservaddr, -- cgit v1.2.3 From 7ae4eb14c5f9d9bf0e0feabeab206151b1280512 Mon Sep 17 00:00:00 2001 From: Mahe Tardy Date: Thu, 13 Aug 2026 11:05:37 +0000 Subject: bpf: Add ksock kfuncs Add BPF kfuncs that allow BPF LSM programs to create and use sockets for sending data. This provides a mechanism for BPF programs to emit telemetry. For this first patch set, it's restricted to SOCK_DGRAM socket types with IPPROTO_UDP protocol but could be easily extended to SOCK_STREAM and IPPROTO_TCP in the future. The API consists of five kfuncs: bpf_ksock_create() - Create a socket (sleepable) bpf_ksock_connect() - Connect socket to remote address (sleepable) bpf_ksock_send() - Send data through the socket (sleepable) bpf_ksock_acquire() - Acquire a reference to a socket context bpf_ksock_release() - Release a reference (cleanup via queue_rcu_work since sock_release sleeps) The setup kfuncs bpf_ksock_create, bpf_ksock_connect, can be called from SYSCALL programs only. While bpf_ksock_acquire, bpf_ksock_release and bpf_ksock_send can be called from SYSCALL and LSM programs. The implementation follows the established kfunc lifecycle pattern (create/acquire/release with refcounting, kptr map storage, dtor registration). The kernel socket is wrapped in a refcounted bpf_ksock struct. Cleanup is deferred via queue_rcu_work() because sock_release() may sleep. The kfuncs are only compiled when CONFIG_INET is enabled, as they specifically support AF_INET and AF_INET6 sockets. The socket operations go through the expected LSM hooks instead of by-passing them like many kernel sockets since those are created by BPF programs and thus system users. Thus, the bpf_ksock_send() kfunc, which is exposed to LSM progs has a verifier filter protection to avoid recursion so that the whole bpf_kfunc_set kfunc set cannot be called in a program attached to security_socket_sendmsg(). Also, because of the LSM checks, we prevent the use of the kfuncs from asynchronous workqueue as the current value would then be invalid. In bpf_ksock_create(), we copy the arg values to avoid TOCTOU races since the kfunc can sleep and the arg values could be stored in a map that could be re-written by BPF progs or even userspace programs if the map is mmaped. Signed-off-by: Mahe Tardy Signed-off-by: Daniel Borkmann Reviewed-by: Jiayuan Chen Acked-by: Stanislav Fomichev Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20260813110540.103550-3-mahe.tardy@gmail.com --- include/linux/bpf_ksock.h | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 include/linux/bpf_ksock.h (limited to 'include/linux') diff --git a/include/linux/bpf_ksock.h b/include/linux/bpf_ksock.h new file mode 100644 index 000000000000..cb387fb75e43 --- /dev/null +++ b/include/linux/bpf_ksock.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2026 Isovalent */ + +#ifndef _BPF_KSOCK_H +#define _BPF_KSOCK_H + +#include +#include +#include + +/** + * struct bpf_ksock_create_opts - BPF kernel socket creation parameters + * @family: Address family: AF_INET or AF_INET6. + * @type: Socket type: only SOCK_DGRAM supported for now. + * @protocol: Protocol number (e.g. IPPROTO_UDP), or 0 for the default protocol + * of the given type. + * @reserved: Must be zero. Reserved for future use. + */ +struct bpf_ksock_create_opts { + __u8 family; + __u8 type; + __u8 protocol; + __u8 reserved; +}; + +/** + * union bpf_ksock_addr - IPv4 or IPv6 socket address + * @sin: IPv4 socket address. + * @sin6: IPv6 socket address. + */ +union bpf_ksock_addr { + struct sockaddr_in sin; + struct sockaddr_in6 sin6; +}; + +#endif /* _BPF_KSOCK_H */ -- cgit v1.2.3 From 09c447564fcac5c531a19e3003d14c9c0a68fd19 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 14 Aug 2026 23:52:55 +0200 Subject: bpf: Keep fault protection when merging pointer types When the same BPF_LDX instruction is reached through paths that yield different pointer types, save_aux_ptr_type() merges them into a single type which is later used by bpf_convert_ctx_accesses() to decide whether the load has to be rewritten into a BPF_PROBE_MEM one. Before f2362a57aeff ("bpf: allow void* cast using bpf_rdonly_cast()") the merge only accepted two PTR_TO_BTF_ID pointers and unconditionally fell back to PTR_TO_BTF_ID | PTR_UNTRUSTED, so the merged type was always one that gets the BPF_PROBE_MEM rewrite. However, the mentioned commit widened the merge to also cover a PTR_TO_MEM base and replaced the fallback by a union of the PTR_UNTRUSTED and MEM_RDONLY flags. A union of flags though cannot express the property the later rewrite is built upon, some examples: - PTR_TO_MEM merged with PTR_TO_BTF_ID | PTR_UNTRUSTED gets PTR_TO_MEM | PTR_UNTRUSTED but only the MEM_RDONLY variant is valid - PTR_TO_MEM merged with a plain PTR_TO_BTF_ID gets PTR_TO_MEM dropping the rewrite the latter type would have gotten - PTR_TO_MEM | MEM_RDONLY merged with a plain PTR_TO_BTF_ID gets PTR_TO_MEM | MEM_RDONLY which is not rewritten either since only its PTR_UNTRUSTED variant is In all three cases a program can take the unsafe path at runtime with a NULL or otherwise bad pointer and panic the kernel on the faulting load: BUG: kernel NULL pointer dereference, address: 0000000000000038 RIP: 0010:bpf_prog_77531a87032eeaf1_mixed_mem_btf_id_type+0x4b/0x65 Call Trace: bpf_test_run+0x20b/0x460 bpf_prog_test_run_skb+0x650/0xbe0 __sys_bpf+0xb96/0x3140 __x64_sys_bpf+0x2c/0x40 do_syscall_64+0xba/0x590 Kernel panic - not syncing: Fatal exception in interrupt Note that the last two shapes have to be fixed right here, otherwise the merged type retains nothing which marks the load as fault prone, thus no rule in bpf_convert_ctx_accesses() can recover it. Fix it by normalizing the merged type instead. Reuse it in is_load_acq_unsafe() to avoid open coding, and trim the overly verbose comment which is more of an implementation detail of bpf_convert_ctx_accesses() anyway. Fixes: f2362a57aeff ("bpf: allow void* cast using bpf_rdonly_cast()") Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260814215301.709827-1-daniel@iogearbox.net --- include/linux/bpf_verifier.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index bc2af02547fe..ae0274a4bf35 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1308,6 +1308,16 @@ static inline u32 type_flag(u32 type) return type & ~BPF_BASE_TYPE_MASK; } +static inline bool bpf_may_fault_on_deref(enum bpf_reg_type type) +{ + /* + * The pointer types which must not be dereferenced without fault + * protection, that is, the ones bpf_convert_ctx_accesses() has to + * turn a BPF_LDX into a BPF_PROBE_MEM one for. + */ + return type == PTR_TO_BTF_ID || (type_flag(type) & PTR_UNTRUSTED); +} + static inline bool bpf_prog_has_arena_ctx_arg(const struct bpf_prog *prog) { int i; -- cgit v1.2.3 From d99bda7f017b47aff45accbb321facba9f7dd799 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 14 Aug 2026 23:52:58 +0200 Subject: bpf: Rewrite any fault prone load out of a mem or btf_id pointer bpf_convert_ctx_accesses() turns a BPF_LDX into a BPF_PROBE_MEM one by matching the type recorded for the insn against a list of exact pointer types. The list cannot keep up with the flag combinations the verifier produces, and a type which is missing from it ends up as a plain load without an exception table entry, so a bad address panics the kernel instead of being handled. Two such types exist today and are reachable: - PTR_TO_BTF_ID | PTR_UNTRUSTED | MEM_ALLOC | NON_OWN_REF - PTR_TO_BTF_ID | PTR_UNTRUSTED | MEM_RCU Rather than adding the two, just drop the list and state the property itself in the default case of the switch. This is a superset of what the list matched, the untrusted PTR_TO_MEM does not have to carry MEM_RDONLY for it anymore, and it stays in sync with the verifier side which uses the same match in save_aux_ptr_type() and reg_type_mismatch_ok(). Assert that a fault prone type which does not get the rewrite for whatever reason is rejected at load time rather than left to fault at runtime to catch any future cases. Fixes: 1b12171533a9 ("bpf: Mark direct ld of stashed bpf_{rb,list}_node as non-owning ref") Fixes: 6fcd486b3a0a ("bpf: Refactor RCU enforcement in the verifier.") Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260814215301.709827-4-daniel@iogearbox.net --- include/linux/bpf_verifier.h | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index ae0274a4bf35..5fad59fdab0d 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1308,6 +1308,17 @@ static inline u32 type_flag(u32 type) return type & ~BPF_BASE_TYPE_MASK; } +static inline bool bpf_is_ptr_to_mem_or_btf_id(enum bpf_reg_type type) +{ + switch (base_type(type)) { + case PTR_TO_MEM: + case PTR_TO_BTF_ID: + return true; + default: + return false; + } +} + static inline bool bpf_may_fault_on_deref(enum bpf_reg_type type) { /* -- cgit v1.2.3