From 7cf9cd98cf6f0df3befc167ca6b54c07014d71de Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Wed, 24 Jun 2026 23:51:14 +0800 Subject: bpf: Copy per-CPU map value padding in copy_map_value_long() In kernel, per-CPU map elements are stored with round_up(map->value_size, 8) bytes. On UAPI lookup paths, it copies the rounded size for each CPU into a temporary buffer. However, copy_map_value_long() passes 'map->value_size' to bpf_obj_memcpy(). When the map has special fields, bpf_obj_memcpy() copies around those fields with memcpy(), and does not copy the tail padding between 'map->value_size' and round_up(map->value_size, 8). The temporary UAPI lookup buffers are allocated without __GFP_ZERO. As a result, when the per-CPU map's value size is not equal to round_up(map->value_size, 8), UAPI LOOKUP_ELEM and its variants can return stale heap contents from that padding to user space. The same issue applies to bpf_iter for per-CPU maps. Pass round_up(map->value_size, 8) to bpf_obj_memcpy() from copy_map_value_long(), so per-CPU maps both with and without special fields copy the entire per-CPU slot. Remove the now redundant round_up() from bpf_obj_memcpy()'s long_memcpy path. Fixes: 448325199f57 ("bpf: Add copy_map_value_long to copy to remote percpu memory") Signed-off-by: Leon Hwang Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260624155115.85196-2-leon.hwang@linux.dev --- include/linux/bpf.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux/bpf.h') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 7719f6528445..ba09795e0bfd 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -570,7 +570,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, if (IS_ERR_OR_NULL(rec)) { if (long_memcpy) - bpf_long_memcpy(dst, src, round_up(size, 8)); + bpf_long_memcpy(dst, src, size); else memcpy(dst, src, size); return; @@ -593,7 +593,7 @@ static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src) { - bpf_obj_memcpy(map->record, dst, src, map->value_size, true); + bpf_obj_memcpy(map->record, dst, src, round_up(map->value_size, 8), true); } static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src) -- cgit v1.2.3 From 9c9ee0324c774490ae953162aaaf4561d222bd93 Mon Sep 17 00:00:00 2001 From: Yiyang Chen Date: Tue, 30 Jun 2026 08:41:26 +0000 Subject: bpf: Reject MEM_ALLOC BTF accesses past object bounds BTF struct walks relax the struct-size check for accesses through a trailing flexible array. That is valid for ordinary BTF type walking, but PTR_TO_BTF_ID | MEM_ALLOC values point to objects allocated with the static BTF type size. When walking a MEM_ALLOC object, reject the access before applying the flexible-array relaxation if the access range extends past the struct size. Apply the same policy to struct ID matching so kfunc and kptr type checks do not walk past the allocated object bounds either. Fixes: 958cf2e273f0 ("bpf: Introduce bpf_obj_new") Fixes: 36d8bdf75a93 ("bpf: Add alloc/xchg/direct_access support for local percpu kptr") Signed-off-by: Yiyang Chen Reviewed-by: Amery Hung Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/4b8c8a81102ba4b595011434c881194f264ddc59.1782807039.git.chenyy23@mails.tsinghua.edu.cn Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux/bpf.h') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index ba09795e0bfd..adf53f7edf28 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -3146,7 +3146,7 @@ int btf_struct_access(struct bpf_verifier_log *log, bool btf_struct_ids_match(struct bpf_verifier_log *log, const struct btf *btf, u32 id, int off, const struct btf *need_btf, u32 need_type_id, - bool strict); + bool strict, bool walk_flex_arrays); int btf_distill_func_proto(struct bpf_verifier_log *log, struct btf *btf, -- cgit v1.2.3 From a2d784869a0f252e1a277db7dc2c16d55694da72 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 8 Jul 2026 09:53:38 +0200 Subject: libbpf: Drop in-loader metadata check for load-time verification The signed gen_loader used to police its own metadata map from within BPF: emit_signature_match() read the kernel-cached map->sha[] back through hardcoded struct bpf_map offsets and compared it against a hash that compute_sha_update_offsets() baked into the signed instructions, after a BPF_OBJ_GET_INFO_BY_FD round-trip to populate map->sha[]. The kernel now verifies the metadata at BPF_PROG_LOAD time by folding the frozen contents of the loader's exclusive fd_array maps into the signature, so the loader no longer checks anything itself. Generated loaders thus carry no verification logic of their own anymore: Nothing in the signing chain depends on emitted loader bytecode doing the right thing. On the loading side, skel_internal.h now sets fd_array_cnt for a signed load so the kernel scans fd_array for the exclusive metadata map - still frozen, as the kernel requires - and the BPF_OBJ_GET_INFO_BY_FD round-trip to populate map->sha[] is gone. The struct bpf_map layout BUILD_BUG_ON()s on the kernel side are removed as well: they only pinned the ABI for the in-BPF read of map->sha[] that is no longer needed. Same for the map->excl member. Note: gen_hash is retained; it still marks a loader as signed so an untrusted host cannot re-dimension maps or override initial values now covered by the signature. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260708075343.358712-4-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux/bpf.h') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index adf53f7edf28..c1a98fa36738 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -299,7 +299,6 @@ struct bpf_map_owner { struct bpf_map { u8 sha[SHA256_DIGEST_SIZE]; - u32 excl; const struct bpf_map_ops *ops; struct bpf_map *inner_map_meta; #ifdef CONFIG_SECURITY -- cgit v1.2.3 From 9a6df65d5c6a9947ddab4e563e329720f44b8747 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Wed, 8 Jul 2026 18:18:05 +0800 Subject: bpf: Introduce jit_required flag and remove bpf_prog_has_kfunc_call() Introduce a 'jit_required' bitfield flag in struct bpf_prog to track whether a BPF program strictly requires the JIT compiler to run. This prevents a dangerous runtime fallback to the interpreter for features that are only implemented in the JIT compiler. Currently, bpf_prog_has_kfunc_call() is used only for kernel function calls, replace the kfunc-specific helper with the new 'jit_required' flag. This makes it easy to support other JIT-only BPF features, such as inlined helpers. Suggested-by: Alexei Starovoitov Suggested-by: KaFai Wan Suggested-by: Leon Hwang Acked-by: Leon Hwang Signed-off-by: Tiezhu Yang Signed-off-by: Eduard Zingerman --- include/linux/bpf.h | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'include/linux/bpf.h') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index c1a98fa36738..31181e0c2b80 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1864,8 +1864,9 @@ struct bpf_prog_aux { struct bpf_prog { u16 pages; /* Number of allocated pages */ - u16 jited:1, /* Is our filter JIT'ed? */ + u32 jited:1, /* Is our filter JIT'ed? */ jit_requested:1,/* archs need to JIT the prog */ + jit_required:1, /* program strictly requires JIT compiler */ gpl_compatible:1, /* Is filter GPL compatible? */ cb_access:1, /* Is control block accessed? */ dst_needed:1, /* Do we need dst entry? */ @@ -3169,7 +3170,6 @@ const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); void bpf_task_storage_free(struct task_struct *task); void bpf_cgrp_storage_free(struct cgroup *cgroup); -bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn); @@ -3508,11 +3508,6 @@ static inline void bpf_task_storage_free(struct task_struct *task) { } -static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) -{ - return false; -} - static inline const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn) -- cgit v1.2.3 From 7a0855e73757ee9cf25ba635a1c735018ecba742 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Wed, 15 Jul 2026 22:11:22 +0800 Subject: bpf: Disallow interpreter fallback for BPF_ADDR_PERCPU insn The BPF_MOV64_PERCPU_REG insn requires JIT to emit native code to for 'dst_reg = src_reg + '. However, the interpreter ignores the 'off' at its ALU64_MOV_X label. The 'off' indicates the insn is BPF_MOV64_PERCPU_REG insn. Then, when the interpreter loads memory from the register, it will hit a page fault. [ 2.545572] BUG: unable to handle page fault for address: ffffffffacaaf034 [ 2.546485] #PF: supervisor read access in kernel mode [ 2.547167] #PF: error_code(0x0000) - not-present page [ 2.547850] PGD 134e63067 P4D 134e63067 PUD 134e64063 PMD 10021c063 PTE 800ffffeca550062 [ 2.548912] Oops: Oops: 0000 [#1] SMP PTI Set jit_required as true in order to disallow interpreter fallback in core.c::__bpf_prog_select_runtime(), if any BPF_ADDR_PERCPU insn is patched to the prog. BTW, rename the helper bpf_map_supports_cpu_flags() to bpf_map_is_percpu_map(). Fixes: 7bdbf7446305 ("bpf: add special internal-only MOV instruction to resolve per-CPU addrs") Signed-off-by: Leon Hwang Link: https://lore.kernel.org/bpf/20260715141122.15783-4-leon.hwang@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux/bpf.h') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 31181e0c2b80..d9542127dfdf 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -4163,7 +4163,7 @@ bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image) } #endif -static inline bool bpf_map_supports_cpu_flags(enum bpf_map_type map_type) +static inline bool bpf_map_is_percpu_map(enum bpf_map_type map_type) { switch (map_type) { case BPF_MAP_TYPE_PERCPU_ARRAY: @@ -4190,7 +4190,7 @@ static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 all return -EINVAL; if (flags & (BPF_F_CPU | BPF_F_ALL_CPUS)) { - if (!bpf_map_supports_cpu_flags(map->map_type)) + if (!bpf_map_is_percpu_map(map->map_type)) return -EINVAL; if ((flags & BPF_F_CPU) && (flags & BPF_F_ALL_CPUS)) return -EINVAL; -- cgit v1.2.3 From 6bcecae4b65d80f4474c42a026a5ba229ded347e Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 8 Jul 2026 06:44:30 +0000 Subject: bpf: Extract the is_struct_ops_tramp helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the is_struct_ops_tramp helper, and use it in riscv as the current checks are somewhat hacky. Signed-off-by: Pu Lehui Reviewed-by: Björn Töpel Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/20260708064436.2971933-2-pulehui@huaweicloud.com Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux/bpf.h') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d9542127dfdf..e066f44a9c05 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2196,6 +2196,12 @@ static inline bool is_tracing_multi(enum bpf_attach_type type) type == BPF_TRACE_FSESSION_MULTI; } +static inline bool is_struct_ops_tramp(const struct bpf_tramp_nodes *fentry_nodes) +{ + return fentry_nodes->nr_nodes == 1 && + fentry_nodes->nodes[0]->link->type == BPF_LINK_TYPE_STRUCT_OPS; +} + #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) /* This macro helps developer to register a struct_ops type and generate * type information correctly. Developers should use this macro to register -- cgit v1.2.3 From 61aaa8782bec59ecffd22e030f54ef9351bcabf9 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Wed, 22 Jul 2026 23:19:08 +0800 Subject: bpf: Fix WARNING in bpf_tracing_link_release The trampoline could be corrupted by the blindly 'tr->flags = BPF_TRAMP_F_TAIL_CALL_CTX' in verifier. 1. A fexit attached to a tail_call_reachable prog. 'tr->flags' became 'BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_TAIL_CALL_CTX'. And, the trampoline would poke the target prog's nop insn using jmp insn instead of call insn. 2. Another fexit loaded with the same tail_call_reachable prog target. 'tr->flags' became 'BPF_TRAMP_F_TAIL_CALL_CTX'. 3. Close the first fexit link. Due to no BPF_TRAMP_F_CALL_ORIG in 'tr->flags', the trampoline will fail to restore the prog's nop insn using call insn. [ 3.410719] WARNING: kernel/bpf/syscall.c:3551 at bpf_tracing_link_release+0x53/0x60, CPU#1: test_progs/98 ... [ 3.428793] bpf_link_free+0x58/0x130 [ 3.429293] bpf_link_release+0x23/0x30 Fix the warning by updating 'tr->flags' with '|=' and lock. Fixes: 2b5dcb31a19a ("bpf, x64: Fix tailcall infinite loop") Signed-off-by: Leon Hwang Reviewed-by: Pu Lehui Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/20260722151909.69142-2-leon.hwang@linux.dev Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux/bpf.h') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e066f44a9c05..7bfc28673124 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1523,6 +1523,7 @@ int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids, struct bpf_tracing_multi_link *link); int bpf_trampoline_multi_detach(struct bpf_prog *prog, struct bpf_tracing_multi_link *link); +void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags); /* * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn @@ -1646,6 +1647,7 @@ static inline int bpf_trampoline_multi_detach(struct bpf_prog *prog, { return -ENOTSUPP; } +static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) {} #endif struct bpf_func_info_aux { -- cgit v1.2.3 From c0e091f30f0dd80d817c3a15d0a97962957e5786 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:24 -0700 Subject: bpf: Rename ARG_CONST_SIZE{,_OR_ZERO} to ARG_MEM_SIZE{,_OR_ZERO} ARG_CONST_SIZE does not require a constant: check_mem_size_reg() accepts any bounded scalar and verifies the memory access against its maximum (reg_umax). Rename ARG_CONST_SIZE and ARG_CONST_SIZE_OR_ZERO to ARG_MEM_SIZE and ARG_MEM_SIZE_OR_ZERO to reflect that. ARG_CONST_ALLOC_ SIZE_OR_ZERO, which does require a constant, is left unchanged. Pure rename, no functional change. Signed-off-by: Amery Hung Link: https://lore.kernel.org/bpf/20260801074633.1595644-10-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux/bpf.h') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 7bfc28673124..be53655d1362 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -888,8 +888,8 @@ enum bpf_arg_type { ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ ARG_PTR_TO_ARENA, - ARG_CONST_SIZE, /* number of bytes accessed from memory */ - ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ + ARG_MEM_SIZE, /* number of bytes accessed from memory */ + ARG_MEM_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ ARG_PTR_TO_CTX, /* pointer to context */ ARG_ANYTHING, /* any (initialized) argument is ok */ -- cgit v1.2.3 From a49b70400b9de06234eb99f87cf60217ed98cc0c Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Sat, 1 Aug 2026 00:46:33 -0700 Subject: bpf: Generate kfunc argument prototype at add-call time Kfunc argument checking re-derives each argument's kfunc_ptr_arg_type from BTF on every verification of a call in check_kfunc_args(). Now that get_kfunc_arg_type() is a function of the kfunc's BTF alone, it no longer inspects register state. The classification can be computed once when the call is added and cached. This is a step toward describing kfuncs with a bpf_func_proto and sharing the helper argument-checking path. Generate the classification at bpf_add_kfunc_call() time: - Extend struct bpf_func_proto to be able to describe a kfunc: widen arg_type[] and the arg_btf_id[]/arg_size[] union from 5 to MAX_BPF_FUNC_ARGS, since a kfunc may take up to 12 arguments (5 in registers, 7 on the stack). - Embed a bpf_func_proto in struct bpf_kfunc_desc, populated by gen_kfunc_arg_proto() which runs get_kfunc_arg_type() for each argument and stores the result in proto.arg_type[]. Grow the descriptor table's descs[] as a flexible array to not waste memory. - check_kfunc_args() reads the cached classification from meta->fn The KF_ARG_PTR_TO_CTX classification depends on the resolved program type, and for BPF_PROG_TYPE_EXT that is the target program's type, which resolve_prog_type() reads from prog->aux->saved_dst_prog_type. That field is normally recorded later during verification in check_attach_btf_id(), after bpf_add_kfunc_call() has run. Record saved_dst_prog_type and saved_dst_attach_type from dst_prog at program load time in bpf_prog_load() so the resolved type is available at add-call time without reordering check_attach_btf_id(). This keeps e.g. an freplace of an XDP program calling bpf_xdp_metadata_rx_hash() classifying its struct xdp_md * argument as context. The classification result is unchanged; it is only computed earlier and cached. Signed-off-by: Amery Hung Link: https://lore.kernel.org/bpf/20260801074633.1595644-19-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'include/linux/bpf.h') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index be53655d1362..356884587ae1 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -960,6 +960,21 @@ enum bpf_return_type { }; static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); +/* The longest tracepoint has 12 args. + * See include/trace/bpf_probe.h + * + * Also reuse this macro for maximum number of arguments a BPF function + * or a kfunc can have. Args 1-5 are passed in registers, args 6-12 via + * stack arg slots. The JIT may map some stack arg slots to registers based + * on the native calling convention (e.g., arg 6 to R9 on x86-64). + */ +#define MAX_BPF_FUNC_ARGS 12 + +/* The maximum number of arguments passed through registers + * a single function may have. + */ +#define MAX_BPF_FUNC_REG_ARGS 5 + /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL * instructions after verifying @@ -984,7 +999,7 @@ struct bpf_func_proto { enum bpf_arg_type arg4_type; enum bpf_arg_type arg5_type; }; - enum bpf_arg_type arg_type[5]; + enum bpf_arg_type arg_type[MAX_BPF_FUNC_ARGS]; }; union { struct { @@ -994,7 +1009,7 @@ struct bpf_func_proto { u32 *arg4_btf_id; u32 *arg5_btf_id; }; - u32 *arg_btf_id[5]; + u32 *arg_btf_id[MAX_BPF_FUNC_ARGS]; struct { size_t arg1_size; size_t arg2_size; @@ -1002,7 +1017,7 @@ struct bpf_func_proto { size_t arg4_size; size_t arg5_size; }; - size_t arg_size[5]; + size_t arg_size[MAX_BPF_FUNC_ARGS]; }; int *ret_btf_id; /* return value btf_id */ bool (*allowed)(const struct bpf_prog *prog); @@ -1192,21 +1207,6 @@ struct bpf_prog_offload { u32 jited_len; }; -/* The longest tracepoint has 12 args. - * See include/trace/bpf_probe.h - * - * Also reuse this macro for maximum number of arguments a BPF function - * or a kfunc can have. Args 1-5 are passed in registers, args 6-12 via - * stack arg slots. The JIT may map some stack arg slots to registers based - * on the native calling convention (e.g., arg 6 to R9 on x86-64). - */ -#define MAX_BPF_FUNC_ARGS 12 - -/* The maximum number of arguments passed through registers - * a single function may have. - */ -#define MAX_BPF_FUNC_REG_ARGS 5 - /* The argument is a structure or a union. */ #define BTF_FMODEL_STRUCT_ARG BIT(0) -- cgit v1.2.3 From 457d4ecb47aaf7a2cb46aaadd76e8c812e4f3c9e Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sun, 2 Aug 2026 22:27:26 -0700 Subject: bpf: Remove unused BTF_FMODEL_STRUCT_ARG Commit 814cba835ef6 ("bpf, x86: Fix trampoline stack size for 128-bit arguments") changed the x86 trampoline to compute the number of registers from arg_size for every argument, which removed the last user of BTF_FMODEL_STRUCT_ARG. No other architecture or verifier code looks at the flag, so remove the macro and the code in __get_type_fmodel_flags() which sets it. Keep BTF_FMODEL_SIGNED_ARG at BIT(1) rather than renumbering it to BIT(0), so BIT(0) is available for a future flag. No functional change. Signed-off-by: Yonghong Song Signed-off-by: Daniel Borkmann Acked-by: Leon Hwang Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/20260803052726.2821447-1-yonghong.song@linux.dev --- include/linux/bpf.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'include/linux/bpf.h') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 356884587ae1..73bacfc6444d 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1207,9 +1207,6 @@ struct bpf_prog_offload { u32 jited_len; }; -/* The argument is a structure or a union. */ -#define BTF_FMODEL_STRUCT_ARG BIT(0) - /* The argument is signed. */ #define BTF_FMODEL_SIGNED_ARG BIT(1) -- cgit v1.2.3 From e2577cd62060be91a3d7d11a56e5a61faae4b7f7 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Thu, 6 Aug 2026 22:10:43 +0200 Subject: bpf, riscv: Add and use bpf_atomic_is_load_acq() helper A load-acquire is the only BPF_STX class instruction that reads from src_reg into dst_reg, that is, it has the operand roles of a BPF_LDX. JIT code which tells loads from stores apart by instruction class alone has to special case it, for example when deciding which register holds the faulting address and which one to clear from an exception handler. riscv64 already does so, open coded as a bare insn->imm test. Add a bpf_atomic_is_load_acq() helper and convert riscv64 over to it, so that the x86-64 and arm64 JITs can use the same helper in subsequent patches. Unlike bpf_atomic_is_load_store(), which presumes that its argument is already known to be a BPF_ATOMIC instruction, the new helper is called from code which still sees all instruction classes, so it checks class and mode itself. Also, move bpf_atomic_is_load_store() to filter.h next to BPF_ATOMIC_OP, so that both helpers stay together. No functional change intended. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260806201047.333389-2-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 15 --------------- 1 file changed, 15 deletions(-) (limited to 'include/linux/bpf.h') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 73bacfc6444d..d79bf7557ef6 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1132,21 +1132,6 @@ static inline bool bpf_pseudo_func(const struct bpf_insn *insn) return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC; } -/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an - * atomic load or store, and false if it is a read-modify-write instruction. - */ -static inline bool -bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn) -{ - switch (atomic_insn->imm) { - case BPF_LOAD_ACQ: - case BPF_STORE_REL: - return true; - default: - return false; - } -} - struct bpf_prog_ops { int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); -- cgit v1.2.3 From 252d367163268cb8d3fe321c1fc2b15a60faf9ea Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 8 Aug 2026 02:39:24 +0200 Subject: bpf: Support __arena and __arena__nullable kfunc argument suffixes Passing an arena pointer to a kfunc takes two steps today. There is no arena pointer argument type, so the pointer crosses the boundary as a bare scalar, and the kfunc then offsets it by the arena base and casts it before it can touch the memory. Every such kfunc open-codes the same translation. Add the __arena and __arena__nullable argument suffixes to make this more convenient. The kfunc declares the parameter by its real pointer type and dereferences it directly, with the JIT rebasing the value at the call site, rN = kern_vm_start + (u32)rN. No bounds check is needed: the u32 offset stays within the guard-padded arena kernel mapping, and a fault on an unpopulated page recovers through the per-arena scratch page. A suffixed argument accepts a PTR_TO_ARENA or scalar register, matching global subprog arena arguments. __arena rebases unconditionally, so the kfunc never sees NULL and a value with zero in the low 32 bits arrives as the arena base. __arena__nullable preserves NULL for optional arguments by skipping the rebase when the truncated value, arena offset 0, is zero. Keeping the plain form NULL-free saves the NULL test on every call. The double separator makes the annotations composable: __arena__nullable also ends in __nullable and naturally follows the common nullable argument path. Plain __arena follows that path too for verifier type checking because both forms accept a constant zero; the function-model flag still determines whether the JIT preserves NULL or rebases it to the arena base. This patch adds the verifier side: the suffixes are recognized in check_kfunc_args() and distilled into argument flags in the function model stored in the kfunc descriptor. JITs retrieve the model while emitting the call, avoiding per-call state in insn_aux_data. JITs declare support with bpf_jit_supports_arena_args() and verification fails with -ENOTSUPP elsewhere. Co-developed-by: Kumar Kartikeya Dwivedi Signed-off-by: Tejun Heo Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260808003938.3486067-5-memxor@gmail.com Signed-off-by: Eduard Zingerman --- include/linux/bpf.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux/bpf.h') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d79bf7557ef6..ba1b9d8ac348 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1195,6 +1195,12 @@ struct bpf_prog_offload { /* The argument is signed. */ #define BTF_FMODEL_SIGNED_ARG BIT(1) +/* The argument is an arena pointer. */ +#define BTF_FMODEL_ARENA_ARG BIT(2) + +/* The argument is nullable. */ +#define BTF_FMODEL_NULLABLE_ARG BIT(3) + struct btf_func_model { u8 ret_size; u8 ret_flags; -- cgit v1.2.3 From f6c33c4479a7e4d6bfc0e0e533a86376866f7360 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 8 Aug 2026 02:39:25 +0200 Subject: bpf: Support __arena and __arena__nullable on struct_ops arguments A struct_ops callback cannot receive an arena pointer directly, so passing one takes two steps. The pointer arrives as a bare u64 that the callback casts, and because the two sides address the arena through different bases it also has to be rebased by hand on the way in. Add the __arena and __arena__nullable stub argument suffixes to make this convenient. The callback declares the parameter as an arena pointer, receives it as a PTR_TO_ARENA register, and dereferences it directly, while the kernel caller just passes the natural kernel arena address (kaddr). The trampoline converts the value while saving the arguments into the BPF ctx, ctx[slot] = (u32)(kaddr - kern_vm_start), so the program never sees a kernel address and nothing rewrites the ctx after the fact. The converted value keeps the upper 32 bits clear as the JITs require of arena pointer registers and behaves like any cast_kern'ed arena pointer, so cast_user recovers the full user-visible address. __arena converts unconditionally and the kernel caller must not pass NULL. __arena__nullable preserves NULL, tested on the full 64-bit kernel pointer, and surfaces to the verifier as PTR_TO_ARENA (but not as a PTR_TO_ARENA | PTR_MAYBE_NULL). The reason is that PTR_TO_ARENA in the program's type state already encompasses NULL-ness, so it is not meaningful to force a NULL check for the program. The composite suffix intentionally ends in __nullable. Classify __arena__nullable before the generic suffix so scalar arena pointees do not take the generic nullable BTF pointer path. This patch adds the generic side. prepare_arg_info() records arena and nullable argument flags in the struct_ops function model, and bpf_tramp_arena_base() returns the arena base for a single-program struct_ops indirect trampoline. Only that trampoline converts: its program's arena is fixed at generation time. Generic trampolines can mix programs with different arenas and reject arena context arguments defensively, which is unreachable today as only struct_ops programs carry them. Architectures that do not implement the conversion are gated out at verification time with bpf_jit_supports_arena_args(). Co-developed-by: Kumar Kartikeya Dwivedi Signed-off-by: Tejun Heo Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260808003938.3486067-6-memxor@gmail.com Signed-off-by: Eduard Zingerman --- include/linux/bpf.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'include/linux/bpf.h') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index ba1b9d8ac348..b4a10c9878cf 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1274,6 +1274,15 @@ struct bpf_tramp_nodes { int nr_nodes; }; +/* + * The arena base against which a struct_ops trampoline converts the + * arguments marked with BTF_FMODEL_ARENA_ARG while saving them into the BPF + * ctx, ctx[arg] = (u32)(kaddr - kern_vm_start). Zero when the trampoline + * converts nothing. + */ +u64 bpf_tramp_arena_base(const struct btf_func_model *m, + struct bpf_tramp_nodes *tnodes, u32 flags); + struct bpf_tramp_run_ctx; /* Different use cases for BPF trampoline: -- cgit v1.2.3 From 7c6beeb8c88f92866daab4516220667d1234d3c9 Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Tue, 11 Aug 2026 10:46:20 +0800 Subject: bpf: Make bpf_trampoline_multi_detach return void bpf_trampoline_multi_detach() always returns 0 and the sole caller ignores the return value. Change it to return void and drop the WARN_ON_ONCE at the call site. Signed-off-by: Hui Zhu Acked-by: Leon Hwang Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/12beba657f5c9e86a016a097750209287a2f262a.1786412280.git.zhuhui@kylinos.cn Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'include/linux/bpf.h') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index b4a10c9878cf..f4e8d372253a 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1518,8 +1518,8 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids, struct bpf_tracing_multi_link *link); -int bpf_trampoline_multi_detach(struct bpf_prog *prog, - struct bpf_tracing_multi_link *link); +void bpf_trampoline_multi_detach(struct bpf_prog *prog, + struct bpf_tracing_multi_link *link); void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags); /* @@ -1639,10 +1639,9 @@ static inline int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids, { return -ENOTSUPP; } -static inline int bpf_trampoline_multi_detach(struct bpf_prog *prog, - struct bpf_tracing_multi_link *link) +static inline void bpf_trampoline_multi_detach(struct bpf_prog *prog, + struct bpf_tracing_multi_link *link) { - return -ENOTSUPP; } static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) {} #endif -- cgit v1.2.3 From f5b57e9e9cfd9736246eb9a5f385da451d199039 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Fri, 14 Aug 2026 08:56:23 -0700 Subject: bpf: Populate mmap-able array map memory lazily An mmap-able BPF array map (BPF_F_MMAPABLE) has its backing memory vmalloc'ed up front at map creation time. array_map_mmap() then wired up the whole mapping eagerly via remap_vmalloc_range(), which calls vm_insert_page() for every page of the map. For large maps this makes every mmap() O(number of pages): an 8MiB map inserts 2048 PTEs per mmap() and tears them all down again on munmap(), even when user space only touches a few pages (or none at all). Populate the mapping lazily instead, the same way the arena map already does. array_map_mmap() now only performs the bounds check and returns, leaving the PTEs unpopulated; pages are inserted on demand by a new array_map_mmap_fault() handler. Because the memory is already resident, the fault handler simply resolves the vmalloc page and hands it to the fault path. This makes mmap() O(1), and munmap() proportional to the number of pages that were actually faulted in rather than to the size of the map. The handler is reached through a new optional ->map_mmap_fault callback. Maps that provide it get a vm_operations_struct with a .fault handler; maps that populate their mapping eagerly keep the one they had. Both share the same open/close callbacks, so the existing VMA accounting (VM_MAYWRITE write-active tracking, freeze handling) stays centralized rather than each map installing its own vm_operations_struct. Callers that want the pages populated up front can still request that explicitly with MAP_POPULATE. Kernel-side access to the map (via the vmalloc address) is unaffected. Time for one mmap()+munmap() of an 8MiB mmap-able array map: before after no MAP_POPULATE, no access 226us 1.1us no MAP_POPULATE, access all pages 236us 1341us MAP_POPULATE, no access 312us 493us MAP_POPULATE, access all pages 318us 519us Mapping without touching the data, which is what this change targets, gets ~160x cheaper. Faulting in the whole mapping one page at a time is more expensive than the eager remap_vmalloc_range() loop, so users that do touch every page should ask for MAP_POPULATE. Note that MAP_POPULATE is not free before this change either: it adds ~85us (226us => 312us) for no benefit, as the mapping is already fully populated. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Song Liu Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260814155623.111565-1-song@kernel.org --- include/linux/bpf.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux/bpf.h') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index f4e8d372253a..04cadd987169 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -145,6 +145,7 @@ struct bpf_map_ops { int (*map_direct_value_meta)(const struct bpf_map *map, u64 imm, u32 *off); int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma); + vm_fault_t (*map_mmap_fault)(struct bpf_map *map, struct vm_fault *vmf); __poll_t (*map_poll)(struct bpf_map *map, struct file *filp, struct poll_table_struct *pts); unsigned long (*map_get_unmapped_area)(struct file *filep, unsigned long addr, -- cgit v1.2.3 From b9c5d822f677e065971481c4bafe5d84b5451082 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 15 Aug 2026 08:45:57 +0200 Subject: bpf: Add source and instruction diagnostic context Teach verifier diagnostics to annotate an instruction with BTF source line information and nearby BPF instructions. The renderer keeps source text in a fixed-width lane and prints instructions in a stable right-hand gutter. Wrap annotation text under the source line so long error labels remain readable while the source and instruction lanes keep their fixed layout. Keeping source and instruction context in one commit preserves the visual layout contract that later diagnostic reports rely on. Signed-off-by: Kumar Kartikeya Dwivedi Link: https://patch.msgid.link/20260815064612.378577-3-memxor@gmail.com Signed-off-by: Eduard Zingerman --- include/linux/bpf.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'include/linux/bpf.h') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 04cadd987169..ffa5626411ac 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -4147,8 +4147,16 @@ static inline bool bpf_is_subprog(const struct bpf_prog *prog) } const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn_off); -void bpf_get_linfo_file_line(struct btf *btf, const struct bpf_line_info *linfo, - const char **filep, const char **linep, int *nump); +struct bpf_linfo_source { + const char *file; + const char *line; + u32 file_name_off; + int line_num; + int line_col; +}; + +void bpf_get_linfo_source(struct btf *btf, const struct bpf_line_info *linfo, + struct bpf_linfo_source *src); int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep, const char **linep, int *nump); struct bpf_prog *bpf_prog_find_from_stack(void); -- cgit v1.2.3