diff options
| author | Kumar Kartikeya Dwivedi <memxor@gmail.com> | 2026-08-03 00:29:19 +0200 |
|---|---|---|
| committer | Kumar Kartikeya Dwivedi <memxor@gmail.com> | 2026-08-03 00:31:27 +0200 |
| commit | 8f876c773b79ed66dadc9507f50dbff39c6a9ec7 (patch) | |
| tree | 793ca0c879f0df3dcaf40136dfab4277400f220b /kernel | |
| parent | 28e911d61d66b92a3bded8b54622ed3cd2795bf6 (diff) | |
| parent | a49b70400b9de06234eb99f87cf60217ed98cc0c (diff) | |
Merge branch 'generate-bpf_func_proto-for-kfunc'
Amery Hung says:
====================
Generate bpf_func_proto for kfunc
Hi,
This is the second of three patch sets to unify kfunc and helper
argument verification. It:
1) further aligns the kfunc and helper argument checks,
2) makes kfunc argument type classification depend solely on BTF, and
3) generates a bpf_func_proto for each kfunc.
With classification now a pure function of the kfunc's BTF, it is computed
once at add-call time and cached in the generated bpf_func_proto, rather
than re-derived on every verification of the call. Along the way it also
fixes a few issues.
The next patch set will align the argument register compatibility checks
and route helper and kfunc argument verification through a single shared
function.
[1/3] https://lore.kernel.org/bpf/20260715064047.1793790-1-ameryhung@gmail.com/
Changelog
v2 -> v3:
- Drop a patch that introduces SCALAR_MAYBE_ZERO (Eduard)
- Drop patch make helper handle mem+size at mem arg, and instead make
kfunc also handle mem+size at size
- patch 5: New patch replacing temporary mark_ptr_not_null_reg hack
with refine_ptr_not_null_reg (Eduard)
- patch 8: Only allow global subprog to read poisoned stack slots
(Eduard)
- patch 11: Test a precision gap when passing NULL to nullable-mem +
size arg (Eduard)
- patch 15: Reorganize BTF_ID, MEM, MEM+SIZE classification for
clarity (Eduard)
- patch 18: Emded bpf_func_proto in bpf_kfunc_desc and dynamically
resize bpf_kfunc_desc_tab; Record saved_dst_prog_type early in
bpf_prog_load to avoid introducing a fallback logic in
resolve_prog_type (Eduard)
Link: https://lore.kernel.org/bpf/20260724190813.1458271-1-ameryhung@gmail.com/
v1 -> v2:
- patch 2: use reg_arg_name() for the map-mismatch message; derive the
object register correctly on both helper and kfunc paths
- patch 3: reject non-CONST_PTR_TO_MAP regs (base_type check) to fix
map-value type confusion
- patch 15: also reject referenced regs with unsafe modifiers (e.g.
MEM_PERCPU)
- patch 17: reject non-SCALAR_VALUE for KF_ARG_MEM_SIZE
====================
Link: https://patch.msgid.link/20260801074633.1595644-1-ameryhung@gmail.com
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/bpf/backtrack.c | 2 | ||||
| -rw-r--r-- | kernel/bpf/bpf_lsm.c | 4 | ||||
| -rw-r--r-- | kernel/bpf/btf.c | 2 | ||||
| -rw-r--r-- | kernel/bpf/cgroup.c | 8 | ||||
| -rw-r--r-- | kernel/bpf/helpers.c | 42 | ||||
| -rw-r--r-- | kernel/bpf/ringbuf.c | 2 | ||||
| -rw-r--r-- | kernel/bpf/stackmap.c | 10 | ||||
| -rw-r--r-- | kernel/bpf/syscall.c | 8 | ||||
| -rw-r--r-- | kernel/bpf/verifier.c | 830 | ||||
| -rw-r--r-- | kernel/trace/bpf_trace.c | 62 |
10 files changed, 524 insertions, 446 deletions
diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 2e4ae0ef0860..2f473ad4fd7c 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -636,7 +636,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, * r5 += 1 * ... * call bpf_perf_event_output#25 - * where .arg5_type = ARG_CONST_SIZE_OR_ZERO + * where .arg5_type = ARG_MEM_SIZE_OR_ZERO * * and this case: * r6 = 1 diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 3983b4ce73c8..82c5988417a0 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -186,7 +186,7 @@ static const struct bpf_func_proto bpf_ima_inode_hash_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_ima_inode_hash_btf_ids[0], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .allowed = bpf_ima_inode_hash_allowed, }; @@ -205,7 +205,7 @@ static const struct bpf_func_proto bpf_ima_file_hash_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_ima_file_hash_btf_ids[0], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .allowed = bpf_ima_inode_hash_allowed, }; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 4eeeaeb69790..5e8ac45ce56a 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8709,7 +8709,7 @@ const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 4355ccb78a9c..fb9357b64cad 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -2305,7 +2305,7 @@ static const struct bpf_func_proto bpf_sysctl_get_name_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -2347,7 +2347,7 @@ static const struct bpf_func_proto bpf_sysctl_get_current_value_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; BPF_CALL_3(bpf_sysctl_get_new_value, struct bpf_sysctl_kern *, ctx, char *, buf, @@ -2367,7 +2367,7 @@ static const struct bpf_func_proto bpf_sysctl_get_new_value_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; BPF_CALL_3(bpf_sysctl_set_new_value, struct bpf_sysctl_kern *, ctx, @@ -2393,7 +2393,7 @@ static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; static const struct bpf_func_proto * diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 88b38db47de9..4709a5ad0474 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -278,7 +278,7 @@ const struct bpf_func_proto bpf_get_current_comm_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, }; #if defined(CONFIG_QUEUED_SPINLOCKS) || defined(CONFIG_BPF_ARCH_SPINLOCK) @@ -539,7 +539,7 @@ const struct bpf_func_proto bpf_strtol_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, .arg4_size = sizeof(s64), @@ -567,7 +567,7 @@ const struct bpf_func_proto bpf_strtoul_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, .arg4_size = sizeof(u64), @@ -583,7 +583,7 @@ static const struct bpf_func_proto bpf_strncmp_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_CONST_STR, }; @@ -627,7 +627,7 @@ const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto = { .arg1_type = ARG_ANYTHING, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; static const struct bpf_func_proto bpf_get_raw_smp_processor_id_proto = { @@ -653,7 +653,7 @@ const struct bpf_func_proto bpf_event_output_data_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(bpf_copy_from_user, void *, dst, u32, size, @@ -675,7 +675,7 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -706,7 +706,7 @@ const struct bpf_func_proto bpf_copy_from_user_task_proto = { .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_BTF_ID, .arg4_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], @@ -1093,10 +1093,10 @@ const struct bpf_func_proto bpf_snprintf_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_CONST_STR, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; static void *map_key_from_value(struct bpf_map *map, void *value, u32 *arr_idx) @@ -1888,7 +1888,7 @@ static const struct bpf_func_proto bpf_dynptr_from_mem_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL | MEM_UNINIT | MEM_WRITE, }; @@ -1943,7 +1943,7 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_DYNPTR, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, @@ -2004,7 +2004,7 @@ static const struct bpf_func_proto bpf_dynptr_write_proto = { .arg1_type = ARG_PTR_TO_DYNPTR, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE_OR_ZERO, + .arg4_type = ARG_MEM_SIZE_OR_ZERO, .arg5_type = ARG_ANYTHING, }; @@ -3404,10 +3404,10 @@ __bpf_kfunc void bpf_throw(u64 cookie) WARN(1, "A call to BPF exception callback should never return\n"); } -__bpf_kfunc int bpf_wq_init(struct bpf_wq *wq, void *p__map, unsigned int flags) +__bpf_kfunc int bpf_wq_init(struct bpf_wq *wq, void *p__const_map, unsigned int flags) { struct bpf_async_kern *async = (struct bpf_async_kern *)wq; - struct bpf_map *map = p__map; + struct bpf_map *map = p__const_map; BUILD_BUG_ON(sizeof(struct bpf_async_kern) > sizeof(struct bpf_wq)); BUILD_BUG_ON(__alignof__(struct bpf_async_kern) != __alignof__(struct bpf_wq)); @@ -4643,17 +4643,17 @@ release_prog: * mode * @task: Task struct for which callback should be scheduled * @tw: Pointer to struct bpf_task_work in BPF map value for internal bookkeeping - * @map__map: bpf_map that embeds struct bpf_task_work in the values + * @map__const_map: bpf_map that embeds struct bpf_task_work in the values * @callback: pointer to BPF subprogram to call * @aux: pointer to bpf_prog_aux of the caller BPF program, implicitly set by the verifier * * Return: 0 if task work has been scheduled successfully, negative error code otherwise */ __bpf_kfunc int bpf_task_work_schedule_signal(struct task_struct *task, struct bpf_task_work *tw, - void *map__map, bpf_task_work_callback_t callback, + void *map__const_map, bpf_task_work_callback_t callback, struct bpf_prog_aux *aux) { - return bpf_task_work_schedule(task, tw, map__map, callback, aux, TWA_SIGNAL); + return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_SIGNAL); } /** @@ -4661,17 +4661,17 @@ __bpf_kfunc int bpf_task_work_schedule_signal(struct task_struct *task, struct b * mode * @task: Task struct for which callback should be scheduled * @tw: Pointer to struct bpf_task_work in BPF map value for internal bookkeeping - * @map__map: bpf_map that embeds struct bpf_task_work in the values + * @map__const_map: bpf_map that embeds struct bpf_task_work in the values * @callback: pointer to BPF subprogram to call * @aux: pointer to bpf_prog_aux of the caller BPF program, implicitly set by the verifier * * Return: 0 if task work has been scheduled successfully, negative error code otherwise */ __bpf_kfunc int bpf_task_work_schedule_resume(struct task_struct *task, struct bpf_task_work *tw, - void *map__map, bpf_task_work_callback_t callback, + void *map__const_map, bpf_task_work_callback_t callback, struct bpf_prog_aux *aux) { - return bpf_task_work_schedule(task, tw, map__map, callback, aux, TWA_RESUME); + return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_RESUME); } static int make_file_dynptr(struct file *file, u32 flags, bool may_sleep, diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index 35ae64ade36b..c1bf7a197a96 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -634,7 +634,7 @@ const struct bpf_func_proto bpf_ringbuf_output_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_CONST_MAP_PTR, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 41fe87d7302f..463f94ba1cc4 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -781,7 +781,7 @@ const struct bpf_func_proto bpf_get_stack_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -797,7 +797,7 @@ const struct bpf_func_proto bpf_get_stack_sleepable_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -831,7 +831,7 @@ const struct bpf_func_proto bpf_get_task_stack_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -848,7 +848,7 @@ const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -911,7 +911,7 @@ const struct bpf_func_proto bpf_get_stack_proto_pe = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 94091130bcc5..8d111da88655 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3043,6 +3043,10 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at prog->aux->attach_btf = attach_btf; prog->aux->attach_btf_id = multi_func ? bpf_multi_func_btf_id[0] : attr->attach_btf_id; prog->aux->dst_prog = dst_prog; + if (dst_prog) { + prog->aux->saved_dst_prog_type = dst_prog->type; + prog->aux->saved_dst_attach_type = dst_prog->expected_attach_type; + } prog->aux->dev_bound = !!attr->prog_ifindex; prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; @@ -6559,7 +6563,7 @@ static const struct bpf_func_proto bpf_sys_bpf_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; const struct bpf_func_proto * __weak @@ -6606,7 +6610,7 @@ static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, .arg4_size = sizeof(u64), diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8d0635ee48c7..b274004fccfd 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1854,32 +1854,34 @@ static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type ty reg->dynptr.first_slot = first_slot; } -static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) +/* + * Refine the return type of the bpf_map_lookup_elem() for special map types: + * map-in-map, xskmap, sockmap and sockhash. + */ +static void refine_map_lookup_value(struct bpf_reg_state *reg) { - if (base_type(reg->type) == PTR_TO_MAP_VALUE) { - const struct bpf_map *map = reg->map_ptr; + enum bpf_type_flag maybe_null = reg->type & PTR_MAYBE_NULL; + const struct bpf_map *map = reg->map_ptr; - if (map->inner_map_meta) { - reg->type = CONST_PTR_TO_MAP; - reg->map_ptr = map->inner_map_meta; - /* transfer reg's id which is unique for every map_lookup_elem - * as UID of the inner map. - */ - if (btf_record_has_field(map->inner_map_meta->record, - BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) { - reg->map_uid = reg->id; - } - } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { - reg->type = PTR_TO_XDP_SOCK; - } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || - map->map_type == BPF_MAP_TYPE_SOCKHASH) { - reg->type = PTR_TO_SOCKET; - } else { - reg->type = PTR_TO_MAP_VALUE; - } - return; + if (map->inner_map_meta) { + reg->type = CONST_PTR_TO_MAP | maybe_null; + reg->map_ptr = map->inner_map_meta; + /* transfer reg's id which is unique for every map_lookup_elem + * as UID of the inner map. + */ + if (btf_record_has_field(map->inner_map_meta->record, + BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) + reg->map_uid = reg->id; + } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { + reg->type = PTR_TO_XDP_SOCK | maybe_null; + } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || + map->map_type == BPF_MAP_TYPE_SOCKHASH) { + reg->type = PTR_TO_SOCKET | maybe_null; } +} +static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) +{ reg->type &= ~PTR_MAYBE_NULL; } @@ -2719,8 +2721,12 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env, return 0; } +static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + struct bpf_func_proto *proto); + int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) { + struct bpf_call_arg_meta meta; struct bpf_kfunc_btf_tab *btf_tab; struct btf_func_model func_model; struct bpf_kfunc_desc_tab *tab; @@ -2806,11 +2812,30 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) if (err) return err; - desc = &tab->descs[tab->nr_descs++]; + memset(&meta, 0, sizeof(meta)); + meta.btf = kfunc.btf; + meta.func_id = kfunc.id; + meta.func_proto = kfunc.proto; + meta.func_name = kfunc.name; + meta.kfunc_flags = kfunc.flags ? *kfunc.flags : 0; + + tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1), GFP_KERNEL_ACCOUNT); + if (!tab) + return -ENOMEM; + prog_aux->kfunc_tab = tab; + + desc = &tab->descs[tab->nr_descs]; + memset(desc, 0, sizeof(*desc)); + + err = gen_kfunc_arg_proto(env, &meta, &desc->proto); + if (err) + return err; + desc->func_id = func_id; desc->offset = offset; desc->addr = addr; desc->func_model = func_model; + tab->nr_descs++; sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off, NULL); return 0; @@ -4924,6 +4949,18 @@ static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { [CONST_PTR_TO_MAP] = btf_bpf_map_id, }; +static enum bpf_reg_type lookup_reg2btf_ids(u32 ref_id) +{ + enum bpf_reg_type type; + + for (type = 0; type < __BPF_REG_TYPE_MAX; type++) { + if (reg2btf_ids[type] && *reg2btf_ids[type] == ref_id) + return type; + } + + return NOT_INIT; +} + static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) { /* A referenced register is always trusted. */ @@ -6667,7 +6704,7 @@ static int check_stack_range_initialized( */ bool clobber = type == BPF_WRITE; /* - * Negative access_size signals global subprog/kfunc arg check where + * Negative access_size signals global subprog arg check where * STACK_POISON slots are acceptable. static stack liveness * might have determined that subprog doesn't read them, * but BTF based global subprog validation isn't accurate enough. @@ -6869,11 +6906,11 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ static int check_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, struct bpf_reg_state *size_reg, argno_t mem_argno, - argno_t size_argno, enum bpf_access_type access_type, + argno_t size_argno, u32 access_type, bool zero_size_allowed, struct bpf_call_arg_meta *meta) { - int err; + int err = 0; /* This is used to refine r0 return value bounds for helpers * that enforce this value as an upper bound on return values. @@ -6910,8 +6947,14 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, reg_arg_name(env, size_argno)); return -EACCES; } - err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), - access_type, zero_size_allowed, meta); + + if (access_type & BPF_READ) + err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), + BPF_READ, zero_size_allowed, meta); + if (!err && access_type & BPF_WRITE) + err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), + BPF_WRITE, zero_size_allowed, meta); + if (!err) { int regno = reg_from_argno(size_argno); @@ -6920,15 +6963,15 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, else err = mark_stack_arg_precision(env, arg_idx_from_argno(size_argno)); } + return err; } static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - argno_t argno, u32 mem_size) + argno_t argno, u32 mem_size, enum bpf_access_type access_type, + struct bpf_call_arg_meta *meta) { - bool may_be_null = type_may_be_null(reg->type); - struct bpf_reg_state saved_reg; - int err; + int size, err = 0; if (bpf_register_is_null(reg)) return 0; @@ -6939,22 +6982,16 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg return -EACCES; } - /* Assuming that the register contains a value check if the memory - * access is safe. Temporarily save and restore the register's state as - * the conversion shouldn't be visible to a caller. + /* + * Only a global subprog (meta == NULL) may read poisoned stack slots: + * its static stack liveness proved the callee body skips them. */ - if (may_be_null) { - saved_reg = *reg; - mark_ptr_not_null_reg(reg); - } - - int size = base_type(reg->type) == PTR_TO_STACK ? -(int)mem_size : mem_size; + size = (!meta && base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size; - err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, NULL); - err = err ?: check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, NULL); - - if (may_be_null) - *reg = saved_reg; + if (access_type & BPF_READ) + err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta); + if (!err && (access_type & BPF_WRITE)) + err = check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, meta); return err; } @@ -6993,28 +7030,33 @@ static int process_const_alloc_mem_size(struct bpf_verifier_env *env, struct bpf return 0; } -static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, - struct bpf_reg_state *size_reg, argno_t mem_argno, argno_t size_argno) +static int process_const_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, struct bpf_call_arg_meta *meta) { - bool may_be_null = type_may_be_null(mem_reg->type); - struct bpf_reg_state saved_reg; - struct bpf_call_arg_meta meta; + int regno = reg_from_argno(argno); int err; - memset(&meta, 0, sizeof(meta)); + if (meta->arg_constant.found) { + verifier_bug(env, "only one constant argument permitted"); + return -EFAULT; + } - if (may_be_null) { - saved_reg = *mem_reg; - mark_ptr_not_null_reg(mem_reg); + if (!tnum_is_const(reg->var_off)) { + verbose(env, "%s must be a known constant\n", reg_arg_name(env, argno)); + return -EINVAL; } - err = check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_READ, true, &meta); - err = err ?: check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_WRITE, true, &meta); + if (regno >= 0) + err = mark_chain_precision(env, regno); + else + err = mark_stack_arg_precision(env, arg_idx_from_argno(argno)); + if (err < 0) + return err; - if (may_be_null) - *mem_reg = saved_reg; + meta->arg_constant.found = true; + meta->arg_constant.value = reg->var_off.value; - return err; + return 0; } enum { @@ -7224,18 +7266,6 @@ static int process_timer_func(struct bpf_verifier_env *env, struct bpf_reg_state return check_map_field_pointer(env, reg, argno, BPF_TIMER, map); } -static int process_timer_helper(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_call_arg_meta *meta) -{ - return process_timer_func(env, reg, argno, &meta->map); -} - -static int process_timer_kfunc(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_call_arg_meta *meta) -{ - return process_timer_func(env, reg, argno, &meta->map); -} - static int process_kptr_func(struct bpf_verifier_env *env, int regno, struct bpf_call_arg_meta *meta) { @@ -7738,8 +7768,7 @@ static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, static bool arg_type_is_mem_size(enum bpf_arg_type type) { - return type == ARG_CONST_SIZE || - type == ARG_CONST_SIZE_OR_ZERO; + return type == ARG_MEM_SIZE || type == ARG_MEM_SIZE_OR_ZERO; } static bool arg_type_is_raw_mem(enum bpf_arg_type type) @@ -7885,8 +7914,8 @@ static const struct bpf_reg_types dynptr_types = { static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_MAP_KEY] = &mem_types, [ARG_PTR_TO_MAP_VALUE] = &mem_types, - [ARG_CONST_SIZE] = &scalar_types, - [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, + [ARG_MEM_SIZE] = &scalar_types, + [ARG_MEM_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_MAP_PTR] = &const_map_ptr_types, [ARG_PTR_TO_CTX] = &context_types, @@ -8286,11 +8315,49 @@ static int get_constant_map_key(struct bpf_verifier_env *env, static bool can_elide_value_nullness(const struct bpf_map *map); +static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, struct bpf_call_arg_meta *meta) +{ + /* Use map_uid (which is unique id of inner map) to reject: + * inner_map1 = bpf_map_lookup_elem(outer_map, key1) + * inner_map2 = bpf_map_lookup_elem(outer_map, key2) + * if (inner_map1 && inner_map2) { + * timer = bpf_map_lookup_elem(inner_map1); + * if (timer) + * // mismatch would have been allowed + * bpf_timer_init(timer, inner_map2); + * } + * + * Comparing map_ptr is enough to distinguish normal and outer maps. + */ + if (meta->map.ptr && + (meta->map.ptr != reg->map_ptr || meta->map.uid != reg->map_uid)) { + argno_t obj_argno = argno_from_reg(reg_from_argno(argno) - 1); + struct btf_record *rec = meta->map.ptr->record; + const char *obj_name = "workqueue"; + + if (rec->timer_off >= 0) + obj_name = "timer"; + else if (rec->task_work_off >= 0) + obj_name = "bpf_task_work"; + + verbose(env, "%s pointer in %s map_uid=%d ", + obj_name, reg_arg_name(env, obj_argno), meta->map.uid); + verbose(env, "doesn't match map pointer in %s map_uid=%d\n", + reg_arg_name(env, argno), reg->map_uid); + return -EINVAL; + } + + meta->map.ptr = reg->map_ptr; + meta->map.uid = reg->map_uid; + return 0; +} + static int check_func_arg(struct bpf_verifier_env *env, u32 arg, struct bpf_call_arg_meta *meta, - const struct bpf_func_proto *fn, int insn_idx) { + const struct bpf_func_proto *fn = meta->fn; u32 regno = BPF_REG_1 + arg; struct bpf_reg_state *reg = reg_state(env, regno); enum bpf_arg_type arg_type = fn->arg_type[arg]; @@ -8361,29 +8428,9 @@ skip_type_check: switch (base_type(arg_type)) { case ARG_CONST_MAP_PTR: /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ - if (meta->map.ptr) { - /* Use map_uid (which is unique id of inner map) to reject: - * inner_map1 = bpf_map_lookup_elem(outer_map, key1) - * inner_map2 = bpf_map_lookup_elem(outer_map, key2) - * if (inner_map1 && inner_map2) { - * timer = bpf_map_lookup_elem(inner_map1); - * if (timer) - * // mismatch would have been allowed - * bpf_timer_init(timer, inner_map2); - * } - * - * Comparing map_ptr is enough to distinguish normal and outer maps. - */ - if (meta->map.ptr != reg->map_ptr || - meta->map.uid != reg->map_uid) { - verbose(env, - "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", - meta->map.uid, reg->map_uid); - return -EINVAL; - } - } - meta->map.ptr = reg->map_ptr; - meta->map.uid = reg->map_uid; + err = process_map_ptr_arg(env, reg, argno, meta); + if (err) + return err; break; case ARG_PTR_TO_MAP_KEY: /* bpf_map_xxx(..., map_ptr, ..., key) call: @@ -8466,7 +8513,7 @@ skip_type_check: } break; case ARG_PTR_TO_TIMER: - err = process_timer_helper(env, reg, argno, meta); + err = process_timer_func(env, reg, argno, &meta->map); if (err) return err; break; @@ -8478,22 +8525,21 @@ skip_type_check: * next is_mem_size argument below. */ if (arg_type & MEM_FIXED_SIZE) { - err = check_helper_mem_access(env, reg, argno, fn->arg_size[arg], - arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, - false, meta); + err = check_mem_reg(env, reg, argno_from_reg(regno), fn->arg_size[arg], + arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta); if (err) return err; if (arg_type & MEM_ALIGNED) err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true); } break; - case ARG_CONST_SIZE: + case ARG_MEM_SIZE: err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), argno, fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); break; - case ARG_CONST_SIZE_OR_ZERO: + case ARG_MEM_SIZE_OR_ZERO: err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), argno, fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, @@ -8824,6 +8870,8 @@ static bool check_raw_mode_ok(const struct bpf_func_proto *fn, struct bpf_call_a int i; for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { + if (fn->arg_type[i] == ARG_DONTCARE) + break; if (!arg_type_is_raw_mem(fn->arg_type[i])) continue; if (meta->arg_raw_mem.regno) @@ -8872,6 +8920,8 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn) int i; for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { + if (fn->arg_type[i] == ARG_DONTCARE) + break; if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID) return !!fn->arg_btf_id[i]; if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK) @@ -8893,6 +8943,8 @@ static bool check_mem_arg_rw_flag_ok(const struct bpf_func_proto *fn) for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; + if (arg_type == ARG_DONTCARE) + break; if (base_type(arg_type) != ARG_PTR_TO_MEM) continue; if (!(arg_type & (MEM_WRITE | MEM_RDONLY))) @@ -8909,6 +8961,8 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_ for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; + if (arg_type == ARG_DONTCARE) + break; if (arg_type_is_release(arg_type)) { if (meta->release_regno) return false; @@ -9252,7 +9306,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE); if (ret < 0) return ret; - if (check_mem_reg(env, reg, argno, arg->mem_size)) + if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL)) return -EINVAL; if (!(arg->arg_type & PTR_MAYBE_NULL) && (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { @@ -10298,9 +10352,10 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn env->insn_aux_data[insn_idx].non_sleepable = true; meta.func_id = func_id; + meta.fn = fn; /* check args */ for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { - err = check_func_arg(env, i, &meta, fn, insn_idx); + err = check_func_arg(env, i, &meta, insn_idx); if (err) return err; } @@ -10518,10 +10573,12 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn regs[BPF_REG_0].map_ptr = meta.map.ptr; regs[BPF_REG_0].map_uid = meta.map.uid; regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag; - if (!type_may_be_null(ret_flag) && + if (type_may_be_null(ret_flag) || btf_record_has_field(meta.map.ptr->record, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK)) { regs[BPF_REG_0].id = ++env->id_gen; } + /* requires regs[BPF_REG_0].id to be set because of the map-in-map case */ + refine_map_lookup_value(®s[BPF_REG_0]); break; case RET_PTR_TO_SOCKET: mark_reg_known_zero(env, regs, BPF_REG_0); @@ -10619,7 +10676,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return -EINVAL; } - if (type_may_be_null(regs[BPF_REG_0].type)) + if (type_may_be_null(regs[BPF_REG_0].type) && !regs[BPF_REG_0].id) regs[BPF_REG_0].id = ++env->id_gen; if (is_ptr_cast_function(func_id) && @@ -10772,26 +10829,24 @@ static bool is_kfunc_rcu_protected(struct bpf_call_arg_meta *meta) } static bool is_kfunc_arg_mem_size(const struct btf *btf, - const struct btf_param *arg, - const struct bpf_reg_state *reg) + const struct btf_param *arg) { const struct btf_type *t; t = btf_type_skip_modifiers(btf, arg->type, NULL); - if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) + if (!btf_type_is_scalar(t)) return false; return btf_param_match_suffix(btf, arg, "__sz"); } static bool is_kfunc_arg_const_mem_size(const struct btf *btf, - const struct btf_param *arg, - const struct bpf_reg_state *reg) + const struct btf_param *arg) { const struct btf_type *t; t = btf_type_skip_modifiers(btf, arg->type, NULL); - if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) + if (!btf_type_is_scalar(t)) return false; return btf_param_match_suffix(btf, arg, "__szk"); @@ -10812,6 +10867,11 @@ static bool is_kfunc_arg_map(const struct btf *btf, const struct btf_param *arg) return btf_param_match_suffix(btf, arg, "__map"); } +static bool is_kfunc_arg_const_map(const struct btf *btf, const struct btf_param *arg) +{ + return btf_param_match_suffix(btf, arg, "__const_map"); +} + static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg) { return btf_param_match_suffix(btf, arg, "__alloc"); @@ -11043,6 +11103,11 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, } enum kfunc_ptr_arg_type { + KF_ARG_CONST_MEM_SIZE, + KF_ARG_MEM_SIZE, + KF_ARG_CONST, + KF_ARG_CONST_ALLOC_SIZE_OR_ZERO, + KF_ARG_ANYTHING, KF_ARG_PTR_TO_CTX, KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */ @@ -11052,13 +11117,11 @@ enum kfunc_ptr_arg_type { KF_ARG_PTR_TO_LIST_NODE, KF_ARG_PTR_TO_BTF_ID, /* Also covers reg2btf_ids conversions */ KF_ARG_PTR_TO_MEM, - KF_ARG_PTR_TO_MEM_SIZE, /* Size derived from next argument, skip it */ KF_ARG_PTR_TO_CALLBACK, KF_ARG_PTR_TO_RB_ROOT, KF_ARG_PTR_TO_RB_NODE, - KF_ARG_PTR_TO_NULL, KF_ARG_PTR_TO_CONST_STR, - KF_ARG_PTR_TO_MAP, + KF_ARG_CONST_MAP_PTR, KF_ARG_PTR_TO_TIMER, KF_ARG_PTR_TO_WORKQUEUE, KF_ARG_PTR_TO_IRQ_FLAG, @@ -11319,108 +11382,154 @@ bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data]; } -static enum kfunc_ptr_arg_type -get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_func_state *caller, - struct bpf_reg_state *regs, struct bpf_call_arg_meta *meta, - const struct btf_type *t, const struct btf_type *ref_t, - const char *ref_tname, const struct btf_param *args, - int arg, int nargs, argno_t argno, struct bpf_reg_state *reg) +static int +get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + const struct btf_param *args, int arg, int nargs) { - bool arg_mem_size = false; + const struct btf_type *t, *ref_t = NULL; + argno_t argno = argno_from_arg(arg + 1); + const char *ref_tname = NULL; + int arg_type; - if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || - meta->func_id == special_kfunc_list[KF_bpf_session_is_return] || - meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) - return KF_ARG_PTR_TO_CTX; + t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); - if (arg + 1 < nargs && - (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1)) || - is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1)))) - arg_mem_size = true; + /* Scalar arguments are classified from their BTF suffix/name alone. */ + if (btf_type_is_scalar(t)) { + if (is_kfunc_arg_constant(meta->btf, &args[arg])) + return KF_ARG_CONST; + if (is_kfunc_arg_const_mem_size(meta->btf, &args[arg])) + return KF_ARG_CONST_MEM_SIZE; + if (is_kfunc_arg_mem_size(meta->btf, &args[arg])) + return KF_ARG_MEM_SIZE; + if (is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdonly_buf_size") || + is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdwr_buf_size")) + return KF_ARG_CONST_ALLOC_SIZE_OR_ZERO; + return KF_ARG_ANYTHING; + } + + if (!btf_type_is_ptr(t)) { + verbose(env, "Unrecognized %s type %s\n", + reg_arg_name(env, argno), btf_type_str(t)); + return -EINVAL; + } + + ref_t = btf_type_skip_modifiers(meta->btf, t->type, NULL); + ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off); /* In this function, we verify the kfunc's BTF as per the argument type, * leaving the rest of the verification with respect to the register * type to our caller. When a set of conditions hold in the BTF type of * arguments, we resolve it to a known kfunc_ptr_arg_type. */ - if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) - return KF_ARG_PTR_TO_CTX; - - if (is_kfunc_arg_nullable(meta->btf, &args[arg]) && bpf_register_is_null(reg) && - !arg_mem_size) - return KF_ARG_PTR_TO_NULL; - - if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_ALLOC_BTF_ID; - - if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_REFCOUNTED_KPTR; - - if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_DYNPTR; - - if (is_kfunc_arg_iter(meta, arg, &args[arg])) - return KF_ARG_PTR_TO_ITER; - - if (is_kfunc_arg_list_head(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_LIST_HEAD; - - if (is_kfunc_arg_list_node(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_LIST_NODE; - - if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_RB_ROOT; - - if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_RB_NODE; - - if (is_kfunc_arg_const_str(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_CONST_STR; + if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || + meta->func_id == special_kfunc_list[KF_bpf_session_is_return] || + meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) + arg_type = KF_ARG_PTR_TO_CTX; + else if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) + arg_type = KF_ARG_PTR_TO_CTX; + else if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_ALLOC_BTF_ID; + else if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_REFCOUNTED_KPTR; + else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_DYNPTR; + else if (is_kfunc_arg_iter(meta, arg, &args[arg])) + arg_type = KF_ARG_PTR_TO_ITER; + else if (is_kfunc_arg_list_head(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_LIST_HEAD; + else if (is_kfunc_arg_list_node(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_LIST_NODE; + else if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_RB_ROOT; + else if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_RB_NODE; + else if (is_kfunc_arg_const_str(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_CONST_STR; + else if (is_kfunc_arg_const_map(meta->btf, &args[arg])) + arg_type = KF_ARG_CONST_MAP_PTR; + else if (is_kfunc_arg_map(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_BTF_ID; + else if (is_kfunc_arg_wq(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_WORKQUEUE; + else if (is_kfunc_arg_timer(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_TIMER; + else if (is_kfunc_arg_task_work(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_TASK_WORK; + else if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_IRQ_FLAG; + else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_RES_SPIN_LOCK; + else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_CALLBACK; + else if (arg + 1 < nargs && + (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || + is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { + if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && + !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", + reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); + return -EINVAL; + } + arg_type = KF_ARG_PTR_TO_MEM; + } else if (btf_type_is_struct(ref_t)) + /* A pointer to a struct without a size argument is classified as KF_ARG_PTR_TO_BTF_ID */ + arg_type = KF_ARG_PTR_TO_BTF_ID; + else { + /* + * Otherwise this is a fixed-size memory buffer supported by + * check_helper_mem_access(): a pointer to a scalar or a struct of + * scalars. The access size is derived from the pointed-to BTF type. + */ + if (!btf_type_is_scalar(ref_t) && + !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", + reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); + return -EINVAL; + } + arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; + } - if (is_kfunc_arg_map(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_MAP; + if (is_kfunc_arg_nullable(meta->btf, &args[arg])) + arg_type |= PTR_MAYBE_NULL; - if (is_kfunc_arg_wq(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_WORKQUEUE; + return arg_type; +} - if (is_kfunc_arg_timer(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_TIMER; +static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + struct bpf_func_proto *proto) +{ + const struct btf *btf = meta->btf; + const struct btf_param *args; + u32 i, nargs; + int arg_type; - if (is_kfunc_arg_task_work(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_TASK_WORK; + args = (const struct btf_param *)(meta->func_proto + 1); + nargs = btf_type_vlen(meta->func_proto); + if (nargs > MAX_BPF_FUNC_ARGS) { + verbose(env, "Function %s has %d > %d args\n", meta->func_name, + nargs, MAX_BPF_FUNC_ARGS); + return -EINVAL; + } + if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { + verbose(env, "JIT does not support kfunc %s() with %d args\n", + meta->func_name, nargs); + return -ENOTSUPP; + } - if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_IRQ_FLAG; + for (i = 0; i < nargs; i++) { + if (is_kfunc_arg_prog_aux(btf, &args[i]) || + is_kfunc_arg_ignore(btf, &args[i]) || + is_kfunc_arg_implicit(meta, i)) + continue; - if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_RES_SPIN_LOCK; + arg_type = get_kfunc_arg_type(env, meta, args, i, nargs); + if (arg_type < 0) + return arg_type; - if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) { - if (!btf_type_is_struct(ref_t)) { - verbose(env, "kernel function %s %s pointer type %s %s is not supported\n", - meta->func_name, reg_arg_name(env, argno), - btf_type_str(ref_t), ref_tname); - return -EINVAL; - } - return KF_ARG_PTR_TO_BTF_ID; + proto->arg_type[i] = arg_type; } - if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) - return KF_ARG_PTR_TO_CALLBACK; - - /* This is the catch all argument type of register types supported by - * check_helper_mem_access. However, we only allow when argument type is - * pointer to scalar, or struct composed (recursively) of scalars. When - * arg_mem_size is true, the pointer can be void *. - */ - if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) && - (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) { - verbose(env, "%s pointer type %s %s must point to %sscalar, or struct with scalar\n", - reg_arg_name(env, argno), - btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : ""); - return -EINVAL; - } - return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM; + return 0; } static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, @@ -12006,16 +12115,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me args = (const struct btf_param *)(meta->func_proto + 1); nargs = btf_type_vlen(meta->func_proto); - if (nargs > MAX_BPF_FUNC_ARGS) { - verbose(env, "Function %s has %d > %d args\n", func_name, nargs, - MAX_BPF_FUNC_ARGS); - return -EINVAL; - } - if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { - verbose(env, "JIT does not support kfunc %s() with %d args\n", - func_name, nargs); - return -ENOTSUPP; - } ret = check_outgoing_stack_args(env, caller, nargs); if (ret) @@ -12032,8 +12131,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me int regno = reg_from_argno(argno); bool btf_id_fixed_off_ok = true; u32 ref_id, type_size; - bool is_ret_buf_sz = false; - int kf_arg_type; + int kf_arg_type = meta->fn->arg_type[i]; if (is_kfunc_arg_prog_aux(btf, &args[i])) { /* Reject repeated use bpf_prog_aux */ @@ -12056,52 +12154,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me t = btf_type_skip_modifiers(btf, args[i].type, NULL); - if (btf_type_is_scalar(t)) { - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - return -EINVAL; - } - - if (is_kfunc_arg_constant(meta->btf, &args[i])) { - if (meta->arg_constant.found) { - verifier_bug(env, "only one constant argument permitted"); - return -EFAULT; - } - if (!tnum_is_const(reg->var_off)) { - verbose(env, "%s must be a known constant\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (regno >= 0) - ret = mark_chain_precision(env, regno); - else - ret = mark_stack_arg_precision(env, i); - if (ret < 0) - return ret; - meta->arg_constant.found = true; - meta->arg_constant.value = reg->var_off.value; - } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) { - meta->r0_rdonly = true; - is_ret_buf_sz = true; - } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdwr_buf_size")) { - is_ret_buf_sz = true; - } - - if (is_ret_buf_sz) { - ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); - if (ret < 0) - return ret; - } - continue; - } - - if (!btf_type_is_ptr(t)) { - verbose(env, "Unrecognized %s type %s\n", - reg_arg_name(env, argno), btf_type_str(t)); - return -EINVAL; - } - - if ((bpf_register_is_null(reg) || type_may_be_null(reg->type)) && + if (btf_type_is_ptr(t) && (bpf_register_is_null(reg) || type_may_be_null(reg->type)) && !is_kfunc_arg_nullable(meta->btf, &args[i])) { verbose(env, "Possibly NULL pointer passed to trusted %s\n", reg_arg_name(env, argno)); @@ -12118,76 +12171,36 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (reg_is_referenced(env, reg)) update_ref_obj(&meta->ref_obj, reg); - ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); + if (btf_type_is_ptr(t)) { + ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); + ref_tname = btf_name_by_offset(btf, ref_t->name_off); + } - kf_arg_type = get_kfunc_ptr_arg_type(env, caller, regs, meta, t, ref_t, ref_tname, - args, i, nargs, argno, reg); - if (kf_arg_type < 0) - return kf_arg_type; - switch (kf_arg_type) { - case KF_ARG_PTR_TO_NULL: + if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) continue; - case KF_ARG_PTR_TO_MAP: - if (!reg->map_ptr) { - verbose(env, "pointer in %s isn't map pointer\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (meta->map.ptr && (reg->map_ptr->record->wq_off >= 0 || - reg->map_ptr->record->task_work_off >= 0)) { - /* Use map_uid (which is unique id of inner map) to reject: - * inner_map1 = bpf_map_lookup_elem(outer_map, key1) - * inner_map2 = bpf_map_lookup_elem(outer_map, key2) - * if (inner_map1 && inner_map2) { - * wq = bpf_map_lookup_elem(inner_map1); - * if (wq) - * // mismatch would have been allowed - * bpf_wq_init(wq, inner_map2); - * } - * - * Comparing map_ptr is enough to distinguish normal and outer maps. - */ - if (meta->map.ptr != reg->map_ptr || - meta->map.uid != reg->map_uid) { - if (reg->map_ptr->record->task_work_off >= 0) { - verbose(env, - "bpf_task_work pointer in R2 map_uid=%d doesn't match map pointer in R3 map_uid=%d\n", - meta->map.uid, reg->map_uid); - return -EINVAL; - } - verbose(env, - "workqueue pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", - meta->map.uid, reg->map_uid); - return -EINVAL; - } - } - meta->map.ptr = reg->map_ptr; - meta->map.uid = reg->map_uid; - fallthrough; + + if (is_kfunc_arg_map(btf, &args[i])) { + ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; + ref_t = btf_type_by_id(btf_vmlinux, ref_id); + ref_tname = btf_name_by_offset(btf, ref_t->name_off); + } + + switch (base_type(kf_arg_type)) { + case KF_ARG_CONST: + case KF_ARG_CONST_MEM_SIZE: + case KF_ARG_MEM_SIZE: + case KF_ARG_ANYTHING: + case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: case KF_ARG_PTR_TO_ALLOC_BTF_ID: case KF_ARG_PTR_TO_BTF_ID: - if (!is_trusted_reg(env, reg)) { - if (!is_kfunc_rcu(meta)) { - verbose(env, "%s must be referenced or trusted\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (!is_rcu_reg(reg)) { - verbose(env, "%s must be a rcu pointer\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } - fallthrough; + case KF_ARG_CONST_MAP_PTR: case KF_ARG_PTR_TO_ITER: case KF_ARG_PTR_TO_LIST_HEAD: case KF_ARG_PTR_TO_LIST_NODE: case KF_ARG_PTR_TO_RB_ROOT: case KF_ARG_PTR_TO_RB_NODE: case KF_ARG_PTR_TO_MEM: - case KF_ARG_PTR_TO_MEM_SIZE: case KF_ARG_PTR_TO_CALLBACK: case KF_ARG_PTR_TO_CONST_STR: case KF_ARG_PTR_TO_WORKQUEUE: @@ -12218,7 +12231,35 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (ret < 0) return ret; - switch (kf_arg_type) { + switch (base_type(kf_arg_type)) { + case KF_ARG_CONST: + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + return -EINVAL; + } + + ret = process_const_arg(env, reg, argno, meta); + if (ret < 0) + return ret; + break; + case KF_ARG_ANYTHING: + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + return -EINVAL; + } + break; + case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + return -EINVAL; + } + + if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) + meta->r0_rdonly = true; + ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); + if (ret < 0) + return ret; + break; case KF_ARG_PTR_TO_CTX: if (reg->type != PTR_TO_CTX) { verbose(env, "%s expected pointer to ctx, but got %s\n", @@ -12387,75 +12428,100 @@ check_ok: if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_MAP: - /* If argument has '__map' suffix expect 'struct bpf_map *' */ - ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; - ref_t = btf_type_by_id(btf_vmlinux, ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); - fallthrough; - case KF_ARG_PTR_TO_BTF_ID: - /* Only base_type is checked, further checks are done here */ - if ((base_type(reg->type) != PTR_TO_BTF_ID || - (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) && - !reg2btf_ids[base_type(reg->type)]) { - verbose(env, "%s is %s ", reg_arg_name(env, argno), - reg_type_str(env, reg->type)); - verbose(env, "expected %s or socket\n", - reg_type_str(env, base_type(reg->type) | - (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS))); + case KF_ARG_CONST_MAP_PTR: + if (base_type(reg->type) != CONST_PTR_TO_MAP || + type_may_be_null(reg->type)) { + verbose(env, "pointer in %s isn't map pointer\n", + reg_arg_name(env, argno)); return -EINVAL; } - ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno); + ret = process_map_ptr_arg(env, reg, argno, meta); if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_MEM: - resolve_ret = btf_resolve_size(btf, ref_t, &type_size); - if (IS_ERR(resolve_ret)) { - verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", - reg_arg_name(env, argno), btf_type_str(ref_t), - ref_tname, PTR_ERR(resolve_ret)); + case KF_ARG_PTR_TO_BTF_ID: + /* Only base_type is checked, further checks are done here */ + if (base_type(reg->type) == PTR_TO_BTF_ID || + reg2btf_ids[base_type(reg->type)]) { + if (!is_trusted_reg(env, reg) || + bpf_type_has_unsafe_modifiers(reg->type)) { + if (!is_kfunc_rcu(meta)) { + verbose(env, "%s must be referenced or trusted\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (!is_rcu_reg(reg)) { + verbose(env, "%s must be a rcu pointer\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + } + + ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno); + if (ret < 0) + return ret; + break; + } + + if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); + + verbose(env, "%s is %s expected %s %s", + reg_arg_name(env, argno), reg_type_str(env, reg->type), + btf_type_str(ref_t), ref_tname); + if (reg2btf_type != NOT_INIT) + verbose(env, " or %s", reg_type_str(env, reg2btf_type)); + verbose(env, "\n"); return -EINVAL; } - ret = check_mem_reg(env, reg, argno, type_size); + + /* + * If the register does not contain btf id but the argument type is a pointer to + * scalar-only struct, allow verifying it as a fixed size memory. + */ + kf_arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; + fallthrough; + case KF_ARG_PTR_TO_MEM: + if (kf_arg_type & MEM_FIXED_SIZE) { + resolve_ret = btf_resolve_size(btf, ref_t, &type_size); + if (IS_ERR(resolve_ret)) { + verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), + ref_tname, PTR_ERR(resolve_ret)); + return -EINVAL; + } + ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, meta); + if (ret < 0) + return ret; + } + break; + case KF_ARG_CONST_MEM_SIZE: + ret = process_const_arg(env, reg, argno, meta); if (ret < 0) return ret; - break; - case KF_ARG_PTR_TO_MEM_SIZE: + fallthrough; + case KF_ARG_MEM_SIZE: { - struct bpf_reg_state *buff_reg = reg; - const struct btf_param *buff_arg = &args[i]; - struct bpf_reg_state *size_reg = get_func_arg_reg(caller, regs, i + 1); - const struct btf_param *size_arg = &args[i + 1]; - argno_t next_argno = argno_from_arg(i + 2); - - if (!bpf_register_is_null(buff_reg) || !is_kfunc_arg_nullable(meta->btf, buff_arg)) { - ret = check_kfunc_mem_size_reg(env, buff_reg, size_reg, - argno, next_argno); - if (ret < 0) { - verbose(env, "%s and ", reg_arg_name(env, argno)); - verbose(env, "%s memory, len pair leads to invalid memory access\n", - reg_arg_name(env, next_argno)); - return ret; - } - } + struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, i - 1); + struct bpf_reg_state *size_reg = reg; + argno_t buff_argno = argno_from_arg(i); - if (is_kfunc_arg_const_mem_size(meta->btf, size_arg, size_reg)) { - if (meta->arg_constant.found) { - verifier_bug(env, "only one constant argument permitted"); - return -EFAULT; - } - if (!tnum_is_const(size_reg->var_off)) { - verbose(env, "%s must be a known constant\n", - reg_arg_name(env, next_argno)); - return -EINVAL; - } - meta->arg_constant.found = true; - meta->arg_constant.value = size_reg->var_off.value; + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + return -EINVAL; } - /* Skip next '__sz' or '__szk' argument */ - i++; + if (bpf_register_is_null(buff_reg)) + break; + + ret = check_mem_size_reg(env, buff_reg, size_reg, buff_argno, argno, + BPF_READ | BPF_WRITE, true, meta); + if (ret < 0) { + verbose(env, "%s and ", reg_arg_name(env, buff_argno)); + verbose(env, "%s memory, len pair leads to invalid memory access\n", + reg_arg_name(env, argno)); + return ret; + } break; } case KF_ARG_PTR_TO_CALLBACK: @@ -12515,7 +12581,7 @@ check_ok: reg_arg_name(env, argno)); return -EINVAL; } - ret = process_timer_kfunc(env, reg, argno, meta); + ret = process_timer_func(env, reg, argno, &meta->map); if (ret < 0) return ret; break; @@ -12976,6 +13042,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int err, insn_idx = *insn_idx_p; const struct btf_param *args; u32 i, nargs, ptr_type_id; + struct bpf_kfunc_desc *desc; struct btf *desc_btf; int id; @@ -12992,6 +13059,13 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, func_name = meta.func_name; insn_aux = &env->insn_aux_data[insn_idx]; + desc = find_kfunc_desc(env->prog, insn->imm, insn->off); + if (!desc) { + verifier_bug(env, "kfunc descriptor not found for func_id %u", insn->imm); + return -EFAULT; + } + meta.fn = &desc->proto; + insn_aux->is_iter_next = bpf_is_iter_next_kfunc(&meta); if (!insn->off && diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 76ab51deaa6b..891897f8a1b3 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -221,7 +221,7 @@ const struct bpf_func_proto bpf_probe_read_user_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -258,7 +258,7 @@ const struct bpf_func_proto bpf_probe_read_user_str_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -273,7 +273,7 @@ const struct bpf_func_proto bpf_probe_read_kernel_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -308,7 +308,7 @@ const struct bpf_func_proto bpf_probe_read_kernel_str_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -328,7 +328,7 @@ static const struct bpf_func_proto bpf_probe_read_compat_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -347,7 +347,7 @@ static const struct bpf_func_proto bpf_probe_read_compat_str_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; #endif /* CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE */ @@ -383,7 +383,7 @@ static const struct bpf_func_proto bpf_probe_write_user_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; #define MAX_TRACE_PRINTK_VARARGS 3 @@ -418,7 +418,7 @@ static const struct bpf_func_proto bpf_trace_printk_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, }; static void __set_printk_clr_event(struct work_struct *work) @@ -474,9 +474,9 @@ static const struct bpf_func_proto bpf_trace_vprintk_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE_OR_ZERO, + .arg4_type = ARG_MEM_SIZE_OR_ZERO, }; const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void) @@ -518,9 +518,9 @@ static const struct bpf_func_proto bpf_seq_printf_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_seq_file_ids[0], .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(bpf_seq_write, struct seq_file *, m, const void *, data, u32, len) @@ -535,7 +535,7 @@ static const struct bpf_func_proto bpf_seq_write_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_seq_file_ids[0], .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_4(bpf_seq_printf_btf, struct seq_file *, m, struct btf_ptr *, ptr, @@ -559,7 +559,7 @@ static const struct bpf_func_proto bpf_seq_printf_btf_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_seq_file_ids[0], .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -633,7 +633,7 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = { .arg1_type = ARG_CONST_MAP_PTR, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void) @@ -730,7 +730,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; static DEFINE_PER_CPU(int, bpf_event_output_nest_level); @@ -996,7 +996,7 @@ static const struct bpf_func_proto bpf_d_path_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_d_path_btf_ids[0], .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .allowed = bpf_d_path_allowed, }; @@ -1053,9 +1053,9 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, .arg5_type = ARG_ANYTHING, }; @@ -1218,7 +1218,7 @@ const struct bpf_func_proto bpf_get_branch_snapshot_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(get_func_arg, void *, ctx, u32, n, u64 *, value) @@ -1421,7 +1421,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_tp = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(bpf_get_stackid_tp, void *, tp_buff, struct bpf_map *, map, @@ -1462,7 +1462,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_tp = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -1524,12 +1524,12 @@ clear: } static const struct bpf_func_proto bpf_perf_prog_read_value_proto = { - .func = bpf_perf_prog_read_value, - .gpl_only = true, - .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .func = bpf_perf_prog_read_value, + .gpl_only = true, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_PTR_TO_UNINIT_MEM, + .arg3_type = ARG_MEM_SIZE, }; BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx, @@ -1566,7 +1566,7 @@ static const struct bpf_func_proto bpf_read_branch_records_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -1646,7 +1646,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; extern const struct bpf_func_proto bpf_skb_output_proto; @@ -1701,7 +1701,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; |
