summaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
authorKumar Kartikeya Dwivedi <memxor@gmail.com>2026-08-03 00:29:19 +0200
committerKumar Kartikeya Dwivedi <memxor@gmail.com>2026-08-03 00:31:27 +0200
commit8f876c773b79ed66dadc9507f50dbff39c6a9ec7 (patch)
tree793ca0c879f0df3dcaf40136dfab4277400f220b /kernel
parent28e911d61d66b92a3bded8b54622ed3cd2795bf6 (diff)
parenta49b70400b9de06234eb99f87cf60217ed98cc0c (diff)
Merge branch 'generate-bpf_func_proto-for-kfunc'
Amery Hung says: ==================== Generate bpf_func_proto for kfunc Hi, This is the second of three patch sets to unify kfunc and helper argument verification. It: 1) further aligns the kfunc and helper argument checks, 2) makes kfunc argument type classification depend solely on BTF, and 3) generates a bpf_func_proto for each kfunc. With classification now a pure function of the kfunc's BTF, it is computed once at add-call time and cached in the generated bpf_func_proto, rather than re-derived on every verification of the call. Along the way it also fixes a few issues. The next patch set will align the argument register compatibility checks and route helper and kfunc argument verification through a single shared function. [1/3] https://lore.kernel.org/bpf/20260715064047.1793790-1-ameryhung@gmail.com/ Changelog v2 -> v3: - Drop a patch that introduces SCALAR_MAYBE_ZERO (Eduard) - Drop patch make helper handle mem+size at mem arg, and instead make kfunc also handle mem+size at size - patch 5: New patch replacing temporary mark_ptr_not_null_reg hack with refine_ptr_not_null_reg (Eduard) - patch 8: Only allow global subprog to read poisoned stack slots (Eduard) - patch 11: Test a precision gap when passing NULL to nullable-mem + size arg (Eduard) - patch 15: Reorganize BTF_ID, MEM, MEM+SIZE classification for clarity (Eduard) - patch 18: Emded bpf_func_proto in bpf_kfunc_desc and dynamically resize bpf_kfunc_desc_tab; Record saved_dst_prog_type early in bpf_prog_load to avoid introducing a fallback logic in resolve_prog_type (Eduard) Link: https://lore.kernel.org/bpf/20260724190813.1458271-1-ameryhung@gmail.com/ v1 -> v2: - patch 2: use reg_arg_name() for the map-mismatch message; derive the object register correctly on both helper and kfunc paths - patch 3: reject non-CONST_PTR_TO_MAP regs (base_type check) to fix map-value type confusion - patch 15: also reject referenced regs with unsafe modifiers (e.g. MEM_PERCPU) - patch 17: reject non-SCALAR_VALUE for KF_ARG_MEM_SIZE ==================== Link: https://patch.msgid.link/20260801074633.1595644-1-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Diffstat (limited to 'kernel')
-rw-r--r--kernel/bpf/backtrack.c2
-rw-r--r--kernel/bpf/bpf_lsm.c4
-rw-r--r--kernel/bpf/btf.c2
-rw-r--r--kernel/bpf/cgroup.c8
-rw-r--r--kernel/bpf/helpers.c42
-rw-r--r--kernel/bpf/ringbuf.c2
-rw-r--r--kernel/bpf/stackmap.c10
-rw-r--r--kernel/bpf/syscall.c8
-rw-r--r--kernel/bpf/verifier.c830
-rw-r--r--kernel/trace/bpf_trace.c62
10 files changed, 524 insertions, 446 deletions
diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c
index 2e4ae0ef0860..2f473ad4fd7c 100644
--- a/kernel/bpf/backtrack.c
+++ b/kernel/bpf/backtrack.c
@@ -636,7 +636,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
* r5 += 1
* ...
* call bpf_perf_event_output#25
- * where .arg5_type = ARG_CONST_SIZE_OR_ZERO
+ * where .arg5_type = ARG_MEM_SIZE_OR_ZERO
*
* and this case:
* r6 = 1
diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c
index 3983b4ce73c8..82c5988417a0 100644
--- a/kernel/bpf/bpf_lsm.c
+++ b/kernel/bpf/bpf_lsm.c
@@ -186,7 +186,7 @@ static const struct bpf_func_proto bpf_ima_inode_hash_proto = {
.arg1_type = ARG_PTR_TO_BTF_ID,
.arg1_btf_id = &bpf_ima_inode_hash_btf_ids[0],
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE,
+ .arg3_type = ARG_MEM_SIZE,
.allowed = bpf_ima_inode_hash_allowed,
};
@@ -205,7 +205,7 @@ static const struct bpf_func_proto bpf_ima_file_hash_proto = {
.arg1_type = ARG_PTR_TO_BTF_ID,
.arg1_btf_id = &bpf_ima_file_hash_btf_ids[0],
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE,
+ .arg3_type = ARG_MEM_SIZE,
.allowed = bpf_ima_inode_hash_allowed,
};
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 4eeeaeb69790..5e8ac45ce56a 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -8709,7 +8709,7 @@ const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = {
.gpl_only = false,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg2_type = ARG_CONST_SIZE,
+ .arg2_type = ARG_MEM_SIZE,
.arg3_type = ARG_ANYTHING,
.arg4_type = ARG_ANYTHING,
};
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index 4355ccb78a9c..fb9357b64cad 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -2305,7 +2305,7 @@ static const struct bpf_func_proto bpf_sysctl_get_name_proto = {
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_MEM | MEM_WRITE,
- .arg3_type = ARG_CONST_SIZE,
+ .arg3_type = ARG_MEM_SIZE,
.arg4_type = ARG_ANYTHING,
};
@@ -2347,7 +2347,7 @@ static const struct bpf_func_proto bpf_sysctl_get_current_value_proto = {
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE,
+ .arg3_type = ARG_MEM_SIZE,
};
BPF_CALL_3(bpf_sysctl_get_new_value, struct bpf_sysctl_kern *, ctx, char *, buf,
@@ -2367,7 +2367,7 @@ static const struct bpf_func_proto bpf_sysctl_get_new_value_proto = {
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE,
+ .arg3_type = ARG_MEM_SIZE,
};
BPF_CALL_3(bpf_sysctl_set_new_value, struct bpf_sysctl_kern *, ctx,
@@ -2393,7 +2393,7 @@ static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = {
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg3_type = ARG_CONST_SIZE,
+ .arg3_type = ARG_MEM_SIZE,
};
static const struct bpf_func_proto *
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 88b38db47de9..4709a5ad0474 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -278,7 +278,7 @@ const struct bpf_func_proto bpf_get_current_comm_proto = {
.gpl_only = false,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_UNINIT_MEM,
- .arg2_type = ARG_CONST_SIZE,
+ .arg2_type = ARG_MEM_SIZE,
};
#if defined(CONFIG_QUEUED_SPINLOCKS) || defined(CONFIG_BPF_ARCH_SPINLOCK)
@@ -539,7 +539,7 @@ const struct bpf_func_proto bpf_strtol_proto = {
.gpl_only = false,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg2_type = ARG_CONST_SIZE,
+ .arg2_type = ARG_MEM_SIZE,
.arg3_type = ARG_ANYTHING,
.arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED,
.arg4_size = sizeof(s64),
@@ -567,7 +567,7 @@ const struct bpf_func_proto bpf_strtoul_proto = {
.gpl_only = false,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg2_type = ARG_CONST_SIZE,
+ .arg2_type = ARG_MEM_SIZE,
.arg3_type = ARG_ANYTHING,
.arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED,
.arg4_size = sizeof(u64),
@@ -583,7 +583,7 @@ static const struct bpf_func_proto bpf_strncmp_proto = {
.gpl_only = false,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg2_type = ARG_CONST_SIZE,
+ .arg2_type = ARG_MEM_SIZE,
.arg3_type = ARG_PTR_TO_CONST_STR,
};
@@ -627,7 +627,7 @@ const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto = {
.arg1_type = ARG_ANYTHING,
.arg2_type = ARG_ANYTHING,
.arg3_type = ARG_PTR_TO_UNINIT_MEM,
- .arg4_type = ARG_CONST_SIZE,
+ .arg4_type = ARG_MEM_SIZE,
};
static const struct bpf_func_proto bpf_get_raw_smp_processor_id_proto = {
@@ -653,7 +653,7 @@ const struct bpf_func_proto bpf_event_output_data_proto = {
.arg2_type = ARG_CONST_MAP_PTR,
.arg3_type = ARG_ANYTHING,
.arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg5_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg5_type = ARG_MEM_SIZE_OR_ZERO,
};
BPF_CALL_3(bpf_copy_from_user, void *, dst, u32, size,
@@ -675,7 +675,7 @@ const struct bpf_func_proto bpf_copy_from_user_proto = {
.might_sleep = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_UNINIT_MEM,
- .arg2_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg2_type = ARG_MEM_SIZE_OR_ZERO,
.arg3_type = ARG_ANYTHING,
};
@@ -706,7 +706,7 @@ const struct bpf_func_proto bpf_copy_from_user_task_proto = {
.might_sleep = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_UNINIT_MEM,
- .arg2_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg2_type = ARG_MEM_SIZE_OR_ZERO,
.arg3_type = ARG_ANYTHING,
.arg4_type = ARG_PTR_TO_BTF_ID,
.arg4_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
@@ -1093,10 +1093,10 @@ const struct bpf_func_proto bpf_snprintf_proto = {
.gpl_only = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE,
- .arg2_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg2_type = ARG_MEM_SIZE_OR_ZERO,
.arg3_type = ARG_PTR_TO_CONST_STR,
.arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
- .arg5_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg5_type = ARG_MEM_SIZE_OR_ZERO,
};
static void *map_key_from_value(struct bpf_map *map, void *value, u32 *arr_idx)
@@ -1888,7 +1888,7 @@ static const struct bpf_func_proto bpf_dynptr_from_mem_proto = {
.gpl_only = false,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_UNINIT_MEM,
- .arg2_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg2_type = ARG_MEM_SIZE_OR_ZERO,
.arg3_type = ARG_ANYTHING,
.arg4_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL | MEM_UNINIT | MEM_WRITE,
};
@@ -1943,7 +1943,7 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = {
.gpl_only = false,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_UNINIT_MEM,
- .arg2_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg2_type = ARG_MEM_SIZE_OR_ZERO,
.arg3_type = ARG_PTR_TO_DYNPTR,
.arg4_type = ARG_ANYTHING,
.arg5_type = ARG_ANYTHING,
@@ -2004,7 +2004,7 @@ static const struct bpf_func_proto bpf_dynptr_write_proto = {
.arg1_type = ARG_PTR_TO_DYNPTR,
.arg2_type = ARG_ANYTHING,
.arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg4_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg4_type = ARG_MEM_SIZE_OR_ZERO,
.arg5_type = ARG_ANYTHING,
};
@@ -3404,10 +3404,10 @@ __bpf_kfunc void bpf_throw(u64 cookie)
WARN(1, "A call to BPF exception callback should never return\n");
}
-__bpf_kfunc int bpf_wq_init(struct bpf_wq *wq, void *p__map, unsigned int flags)
+__bpf_kfunc int bpf_wq_init(struct bpf_wq *wq, void *p__const_map, unsigned int flags)
{
struct bpf_async_kern *async = (struct bpf_async_kern *)wq;
- struct bpf_map *map = p__map;
+ struct bpf_map *map = p__const_map;
BUILD_BUG_ON(sizeof(struct bpf_async_kern) > sizeof(struct bpf_wq));
BUILD_BUG_ON(__alignof__(struct bpf_async_kern) != __alignof__(struct bpf_wq));
@@ -4643,17 +4643,17 @@ release_prog:
* mode
* @task: Task struct for which callback should be scheduled
* @tw: Pointer to struct bpf_task_work in BPF map value for internal bookkeeping
- * @map__map: bpf_map that embeds struct bpf_task_work in the values
+ * @map__const_map: bpf_map that embeds struct bpf_task_work in the values
* @callback: pointer to BPF subprogram to call
* @aux: pointer to bpf_prog_aux of the caller BPF program, implicitly set by the verifier
*
* Return: 0 if task work has been scheduled successfully, negative error code otherwise
*/
__bpf_kfunc int bpf_task_work_schedule_signal(struct task_struct *task, struct bpf_task_work *tw,
- void *map__map, bpf_task_work_callback_t callback,
+ void *map__const_map, bpf_task_work_callback_t callback,
struct bpf_prog_aux *aux)
{
- return bpf_task_work_schedule(task, tw, map__map, callback, aux, TWA_SIGNAL);
+ return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_SIGNAL);
}
/**
@@ -4661,17 +4661,17 @@ __bpf_kfunc int bpf_task_work_schedule_signal(struct task_struct *task, struct b
* mode
* @task: Task struct for which callback should be scheduled
* @tw: Pointer to struct bpf_task_work in BPF map value for internal bookkeeping
- * @map__map: bpf_map that embeds struct bpf_task_work in the values
+ * @map__const_map: bpf_map that embeds struct bpf_task_work in the values
* @callback: pointer to BPF subprogram to call
* @aux: pointer to bpf_prog_aux of the caller BPF program, implicitly set by the verifier
*
* Return: 0 if task work has been scheduled successfully, negative error code otherwise
*/
__bpf_kfunc int bpf_task_work_schedule_resume(struct task_struct *task, struct bpf_task_work *tw,
- void *map__map, bpf_task_work_callback_t callback,
+ void *map__const_map, bpf_task_work_callback_t callback,
struct bpf_prog_aux *aux)
{
- return bpf_task_work_schedule(task, tw, map__map, callback, aux, TWA_RESUME);
+ return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_RESUME);
}
static int make_file_dynptr(struct file *file, u32 flags, bool may_sleep,
diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c
index 35ae64ade36b..c1bf7a197a96 100644
--- a/kernel/bpf/ringbuf.c
+++ b/kernel/bpf/ringbuf.c
@@ -634,7 +634,7 @@ const struct bpf_func_proto bpf_ringbuf_output_proto = {
.ret_type = RET_INTEGER,
.arg1_type = ARG_CONST_MAP_PTR,
.arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.arg4_type = ARG_ANYTHING,
};
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index 41fe87d7302f..463f94ba1cc4 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -781,7 +781,7 @@ const struct bpf_func_proto bpf_get_stack_proto = {
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.arg4_type = ARG_ANYTHING,
};
@@ -797,7 +797,7 @@ const struct bpf_func_proto bpf_get_stack_sleepable_proto = {
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.arg4_type = ARG_ANYTHING,
};
@@ -831,7 +831,7 @@ const struct bpf_func_proto bpf_get_task_stack_proto = {
.arg1_type = ARG_PTR_TO_BTF_ID,
.arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.arg4_type = ARG_ANYTHING,
};
@@ -848,7 +848,7 @@ const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = {
.arg1_type = ARG_PTR_TO_BTF_ID,
.arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.arg4_type = ARG_ANYTHING,
};
@@ -911,7 +911,7 @@ const struct bpf_func_proto bpf_get_stack_proto_pe = {
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.arg4_type = ARG_ANYTHING,
};
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 94091130bcc5..8d111da88655 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -3043,6 +3043,10 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at
prog->aux->attach_btf = attach_btf;
prog->aux->attach_btf_id = multi_func ? bpf_multi_func_btf_id[0] : attr->attach_btf_id;
prog->aux->dst_prog = dst_prog;
+ if (dst_prog) {
+ prog->aux->saved_dst_prog_type = dst_prog->type;
+ prog->aux->saved_dst_attach_type = dst_prog->expected_attach_type;
+ }
prog->aux->dev_bound = !!attr->prog_ifindex;
prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS;
@@ -6559,7 +6563,7 @@ static const struct bpf_func_proto bpf_sys_bpf_proto = {
.ret_type = RET_INTEGER,
.arg1_type = ARG_ANYTHING,
.arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg3_type = ARG_CONST_SIZE,
+ .arg3_type = ARG_MEM_SIZE,
};
const struct bpf_func_proto * __weak
@@ -6606,7 +6610,7 @@ static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = {
.gpl_only = false,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg2_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg2_type = ARG_MEM_SIZE_OR_ZERO,
.arg3_type = ARG_ANYTHING,
.arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED,
.arg4_size = sizeof(u64),
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 8d0635ee48c7..b274004fccfd 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1854,32 +1854,34 @@ static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type ty
reg->dynptr.first_slot = first_slot;
}
-static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
+/*
+ * Refine the return type of the bpf_map_lookup_elem() for special map types:
+ * map-in-map, xskmap, sockmap and sockhash.
+ */
+static void refine_map_lookup_value(struct bpf_reg_state *reg)
{
- if (base_type(reg->type) == PTR_TO_MAP_VALUE) {
- const struct bpf_map *map = reg->map_ptr;
+ enum bpf_type_flag maybe_null = reg->type & PTR_MAYBE_NULL;
+ const struct bpf_map *map = reg->map_ptr;
- if (map->inner_map_meta) {
- reg->type = CONST_PTR_TO_MAP;
- reg->map_ptr = map->inner_map_meta;
- /* transfer reg's id which is unique for every map_lookup_elem
- * as UID of the inner map.
- */
- if (btf_record_has_field(map->inner_map_meta->record,
- BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) {
- reg->map_uid = reg->id;
- }
- } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
- reg->type = PTR_TO_XDP_SOCK;
- } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
- map->map_type == BPF_MAP_TYPE_SOCKHASH) {
- reg->type = PTR_TO_SOCKET;
- } else {
- reg->type = PTR_TO_MAP_VALUE;
- }
- return;
+ if (map->inner_map_meta) {
+ reg->type = CONST_PTR_TO_MAP | maybe_null;
+ reg->map_ptr = map->inner_map_meta;
+ /* transfer reg's id which is unique for every map_lookup_elem
+ * as UID of the inner map.
+ */
+ if (btf_record_has_field(map->inner_map_meta->record,
+ BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK))
+ reg->map_uid = reg->id;
+ } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
+ reg->type = PTR_TO_XDP_SOCK | maybe_null;
+ } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
+ map->map_type == BPF_MAP_TYPE_SOCKHASH) {
+ reg->type = PTR_TO_SOCKET | maybe_null;
}
+}
+static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
+{
reg->type &= ~PTR_MAYBE_NULL;
}
@@ -2719,8 +2721,12 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env,
return 0;
}
+static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
+ struct bpf_func_proto *proto);
+
int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset)
{
+ struct bpf_call_arg_meta meta;
struct bpf_kfunc_btf_tab *btf_tab;
struct btf_func_model func_model;
struct bpf_kfunc_desc_tab *tab;
@@ -2806,11 +2812,30 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset)
if (err)
return err;
- desc = &tab->descs[tab->nr_descs++];
+ memset(&meta, 0, sizeof(meta));
+ meta.btf = kfunc.btf;
+ meta.func_id = kfunc.id;
+ meta.func_proto = kfunc.proto;
+ meta.func_name = kfunc.name;
+ meta.kfunc_flags = kfunc.flags ? *kfunc.flags : 0;
+
+ tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1), GFP_KERNEL_ACCOUNT);
+ if (!tab)
+ return -ENOMEM;
+ prog_aux->kfunc_tab = tab;
+
+ desc = &tab->descs[tab->nr_descs];
+ memset(desc, 0, sizeof(*desc));
+
+ err = gen_kfunc_arg_proto(env, &meta, &desc->proto);
+ if (err)
+ return err;
+
desc->func_id = func_id;
desc->offset = offset;
desc->addr = addr;
desc->func_model = func_model;
+ tab->nr_descs++;
sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
kfunc_desc_cmp_by_id_off, NULL);
return 0;
@@ -4924,6 +4949,18 @@ static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
[CONST_PTR_TO_MAP] = btf_bpf_map_id,
};
+static enum bpf_reg_type lookup_reg2btf_ids(u32 ref_id)
+{
+ enum bpf_reg_type type;
+
+ for (type = 0; type < __BPF_REG_TYPE_MAX; type++) {
+ if (reg2btf_ids[type] && *reg2btf_ids[type] == ref_id)
+ return type;
+ }
+
+ return NOT_INIT;
+}
+
static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg)
{
/* A referenced register is always trusted. */
@@ -6667,7 +6704,7 @@ static int check_stack_range_initialized(
*/
bool clobber = type == BPF_WRITE;
/*
- * Negative access_size signals global subprog/kfunc arg check where
+ * Negative access_size signals global subprog arg check where
* STACK_POISON slots are acceptable. static stack liveness
* might have determined that subprog doesn't read them,
* but BTF based global subprog validation isn't accurate enough.
@@ -6869,11 +6906,11 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_
static int check_mem_size_reg(struct bpf_verifier_env *env,
struct bpf_reg_state *mem_reg,
struct bpf_reg_state *size_reg, argno_t mem_argno,
- argno_t size_argno, enum bpf_access_type access_type,
+ argno_t size_argno, u32 access_type,
bool zero_size_allowed,
struct bpf_call_arg_meta *meta)
{
- int err;
+ int err = 0;
/* This is used to refine r0 return value bounds for helpers
* that enforce this value as an upper bound on return values.
@@ -6910,8 +6947,14 @@ static int check_mem_size_reg(struct bpf_verifier_env *env,
reg_arg_name(env, size_argno));
return -EACCES;
}
- err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg),
- access_type, zero_size_allowed, meta);
+
+ if (access_type & BPF_READ)
+ err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg),
+ BPF_READ, zero_size_allowed, meta);
+ if (!err && access_type & BPF_WRITE)
+ err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg),
+ BPF_WRITE, zero_size_allowed, meta);
+
if (!err) {
int regno = reg_from_argno(size_argno);
@@ -6920,15 +6963,15 @@ static int check_mem_size_reg(struct bpf_verifier_env *env,
else
err = mark_stack_arg_precision(env, arg_idx_from_argno(size_argno));
}
+
return err;
}
static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
- argno_t argno, u32 mem_size)
+ argno_t argno, u32 mem_size, enum bpf_access_type access_type,
+ struct bpf_call_arg_meta *meta)
{
- bool may_be_null = type_may_be_null(reg->type);
- struct bpf_reg_state saved_reg;
- int err;
+ int size, err = 0;
if (bpf_register_is_null(reg))
return 0;
@@ -6939,22 +6982,16 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg
return -EACCES;
}
- /* Assuming that the register contains a value check if the memory
- * access is safe. Temporarily save and restore the register's state as
- * the conversion shouldn't be visible to a caller.
+ /*
+ * Only a global subprog (meta == NULL) may read poisoned stack slots:
+ * its static stack liveness proved the callee body skips them.
*/
- if (may_be_null) {
- saved_reg = *reg;
- mark_ptr_not_null_reg(reg);
- }
-
- int size = base_type(reg->type) == PTR_TO_STACK ? -(int)mem_size : mem_size;
+ size = (!meta && base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size;
- err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, NULL);
- err = err ?: check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, NULL);
-
- if (may_be_null)
- *reg = saved_reg;
+ if (access_type & BPF_READ)
+ err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta);
+ if (!err && (access_type & BPF_WRITE))
+ err = check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, meta);
return err;
}
@@ -6993,28 +7030,33 @@ static int process_const_alloc_mem_size(struct bpf_verifier_env *env, struct bpf
return 0;
}
-static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg,
- struct bpf_reg_state *size_reg, argno_t mem_argno, argno_t size_argno)
+static int process_const_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
+ argno_t argno, struct bpf_call_arg_meta *meta)
{
- bool may_be_null = type_may_be_null(mem_reg->type);
- struct bpf_reg_state saved_reg;
- struct bpf_call_arg_meta meta;
+ int regno = reg_from_argno(argno);
int err;
- memset(&meta, 0, sizeof(meta));
+ if (meta->arg_constant.found) {
+ verifier_bug(env, "only one constant argument permitted");
+ return -EFAULT;
+ }
- if (may_be_null) {
- saved_reg = *mem_reg;
- mark_ptr_not_null_reg(mem_reg);
+ if (!tnum_is_const(reg->var_off)) {
+ verbose(env, "%s must be a known constant\n", reg_arg_name(env, argno));
+ return -EINVAL;
}
- err = check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_READ, true, &meta);
- err = err ?: check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_WRITE, true, &meta);
+ if (regno >= 0)
+ err = mark_chain_precision(env, regno);
+ else
+ err = mark_stack_arg_precision(env, arg_idx_from_argno(argno));
+ if (err < 0)
+ return err;
- if (may_be_null)
- *mem_reg = saved_reg;
+ meta->arg_constant.found = true;
+ meta->arg_constant.value = reg->var_off.value;
- return err;
+ return 0;
}
enum {
@@ -7224,18 +7266,6 @@ static int process_timer_func(struct bpf_verifier_env *env, struct bpf_reg_state
return check_map_field_pointer(env, reg, argno, BPF_TIMER, map);
}
-static int process_timer_helper(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
- struct bpf_call_arg_meta *meta)
-{
- return process_timer_func(env, reg, argno, &meta->map);
-}
-
-static int process_timer_kfunc(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
- struct bpf_call_arg_meta *meta)
-{
- return process_timer_func(env, reg, argno, &meta->map);
-}
-
static int process_kptr_func(struct bpf_verifier_env *env, int regno,
struct bpf_call_arg_meta *meta)
{
@@ -7738,8 +7768,7 @@ static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx,
static bool arg_type_is_mem_size(enum bpf_arg_type type)
{
- return type == ARG_CONST_SIZE ||
- type == ARG_CONST_SIZE_OR_ZERO;
+ return type == ARG_MEM_SIZE || type == ARG_MEM_SIZE_OR_ZERO;
}
static bool arg_type_is_raw_mem(enum bpf_arg_type type)
@@ -7885,8 +7914,8 @@ static const struct bpf_reg_types dynptr_types = {
static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
[ARG_PTR_TO_MAP_KEY] = &mem_types,
[ARG_PTR_TO_MAP_VALUE] = &mem_types,
- [ARG_CONST_SIZE] = &scalar_types,
- [ARG_CONST_SIZE_OR_ZERO] = &scalar_types,
+ [ARG_MEM_SIZE] = &scalar_types,
+ [ARG_MEM_SIZE_OR_ZERO] = &scalar_types,
[ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types,
[ARG_CONST_MAP_PTR] = &const_map_ptr_types,
[ARG_PTR_TO_CTX] = &context_types,
@@ -8286,11 +8315,49 @@ static int get_constant_map_key(struct bpf_verifier_env *env,
static bool can_elide_value_nullness(const struct bpf_map *map);
+static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
+ argno_t argno, struct bpf_call_arg_meta *meta)
+{
+ /* Use map_uid (which is unique id of inner map) to reject:
+ * inner_map1 = bpf_map_lookup_elem(outer_map, key1)
+ * inner_map2 = bpf_map_lookup_elem(outer_map, key2)
+ * if (inner_map1 && inner_map2) {
+ * timer = bpf_map_lookup_elem(inner_map1);
+ * if (timer)
+ * // mismatch would have been allowed
+ * bpf_timer_init(timer, inner_map2);
+ * }
+ *
+ * Comparing map_ptr is enough to distinguish normal and outer maps.
+ */
+ if (meta->map.ptr &&
+ (meta->map.ptr != reg->map_ptr || meta->map.uid != reg->map_uid)) {
+ argno_t obj_argno = argno_from_reg(reg_from_argno(argno) - 1);
+ struct btf_record *rec = meta->map.ptr->record;
+ const char *obj_name = "workqueue";
+
+ if (rec->timer_off >= 0)
+ obj_name = "timer";
+ else if (rec->task_work_off >= 0)
+ obj_name = "bpf_task_work";
+
+ verbose(env, "%s pointer in %s map_uid=%d ",
+ obj_name, reg_arg_name(env, obj_argno), meta->map.uid);
+ verbose(env, "doesn't match map pointer in %s map_uid=%d\n",
+ reg_arg_name(env, argno), reg->map_uid);
+ return -EINVAL;
+ }
+
+ meta->map.ptr = reg->map_ptr;
+ meta->map.uid = reg->map_uid;
+ return 0;
+}
+
static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
struct bpf_call_arg_meta *meta,
- const struct bpf_func_proto *fn,
int insn_idx)
{
+ const struct bpf_func_proto *fn = meta->fn;
u32 regno = BPF_REG_1 + arg;
struct bpf_reg_state *reg = reg_state(env, regno);
enum bpf_arg_type arg_type = fn->arg_type[arg];
@@ -8361,29 +8428,9 @@ skip_type_check:
switch (base_type(arg_type)) {
case ARG_CONST_MAP_PTR:
/* bpf_map_xxx(map_ptr) call: remember that map_ptr */
- if (meta->map.ptr) {
- /* Use map_uid (which is unique id of inner map) to reject:
- * inner_map1 = bpf_map_lookup_elem(outer_map, key1)
- * inner_map2 = bpf_map_lookup_elem(outer_map, key2)
- * if (inner_map1 && inner_map2) {
- * timer = bpf_map_lookup_elem(inner_map1);
- * if (timer)
- * // mismatch would have been allowed
- * bpf_timer_init(timer, inner_map2);
- * }
- *
- * Comparing map_ptr is enough to distinguish normal and outer maps.
- */
- if (meta->map.ptr != reg->map_ptr ||
- meta->map.uid != reg->map_uid) {
- verbose(env,
- "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n",
- meta->map.uid, reg->map_uid);
- return -EINVAL;
- }
- }
- meta->map.ptr = reg->map_ptr;
- meta->map.uid = reg->map_uid;
+ err = process_map_ptr_arg(env, reg, argno, meta);
+ if (err)
+ return err;
break;
case ARG_PTR_TO_MAP_KEY:
/* bpf_map_xxx(..., map_ptr, ..., key) call:
@@ -8466,7 +8513,7 @@ skip_type_check:
}
break;
case ARG_PTR_TO_TIMER:
- err = process_timer_helper(env, reg, argno, meta);
+ err = process_timer_func(env, reg, argno, &meta->map);
if (err)
return err;
break;
@@ -8478,22 +8525,21 @@ skip_type_check:
* next is_mem_size argument below.
*/
if (arg_type & MEM_FIXED_SIZE) {
- err = check_helper_mem_access(env, reg, argno, fn->arg_size[arg],
- arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ,
- false, meta);
+ err = check_mem_reg(env, reg, argno_from_reg(regno), fn->arg_size[arg],
+ arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta);
if (err)
return err;
if (arg_type & MEM_ALIGNED)
err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true);
}
break;
- case ARG_CONST_SIZE:
+ case ARG_MEM_SIZE:
err = check_mem_size_reg(env, reg_state(env, regno - 1), reg,
argno_from_reg(regno - 1), argno,
fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ,
false, meta);
break;
- case ARG_CONST_SIZE_OR_ZERO:
+ case ARG_MEM_SIZE_OR_ZERO:
err = check_mem_size_reg(env, reg_state(env, regno - 1), reg,
argno_from_reg(regno - 1), argno,
fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ,
@@ -8824,6 +8870,8 @@ static bool check_raw_mode_ok(const struct bpf_func_proto *fn, struct bpf_call_a
int i;
for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
+ if (fn->arg_type[i] == ARG_DONTCARE)
+ break;
if (!arg_type_is_raw_mem(fn->arg_type[i]))
continue;
if (meta->arg_raw_mem.regno)
@@ -8872,6 +8920,8 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn)
int i;
for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
+ if (fn->arg_type[i] == ARG_DONTCARE)
+ break;
if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID)
return !!fn->arg_btf_id[i];
if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK)
@@ -8893,6 +8943,8 @@ static bool check_mem_arg_rw_flag_ok(const struct bpf_func_proto *fn)
for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
enum bpf_arg_type arg_type = fn->arg_type[i];
+ if (arg_type == ARG_DONTCARE)
+ break;
if (base_type(arg_type) != ARG_PTR_TO_MEM)
continue;
if (!(arg_type & (MEM_WRITE | MEM_RDONLY)))
@@ -8909,6 +8961,8 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_
for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
enum bpf_arg_type arg_type = fn->arg_type[i];
+ if (arg_type == ARG_DONTCARE)
+ break;
if (arg_type_is_release(arg_type)) {
if (meta->release_regno)
return false;
@@ -9252,7 +9306,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE);
if (ret < 0)
return ret;
- if (check_mem_reg(env, reg, argno, arg->mem_size))
+ if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL))
return -EINVAL;
if (!(arg->arg_type & PTR_MAYBE_NULL) &&
(type_may_be_null(reg->type) || bpf_register_is_null(reg))) {
@@ -10298,9 +10352,10 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
env->insn_aux_data[insn_idx].non_sleepable = true;
meta.func_id = func_id;
+ meta.fn = fn;
/* check args */
for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
- err = check_func_arg(env, i, &meta, fn, insn_idx);
+ err = check_func_arg(env, i, &meta, insn_idx);
if (err)
return err;
}
@@ -10518,10 +10573,12 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
regs[BPF_REG_0].map_ptr = meta.map.ptr;
regs[BPF_REG_0].map_uid = meta.map.uid;
regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag;
- if (!type_may_be_null(ret_flag) &&
+ if (type_may_be_null(ret_flag) ||
btf_record_has_field(meta.map.ptr->record, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK)) {
regs[BPF_REG_0].id = ++env->id_gen;
}
+ /* requires regs[BPF_REG_0].id to be set because of the map-in-map case */
+ refine_map_lookup_value(&regs[BPF_REG_0]);
break;
case RET_PTR_TO_SOCKET:
mark_reg_known_zero(env, regs, BPF_REG_0);
@@ -10619,7 +10676,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
return -EINVAL;
}
- if (type_may_be_null(regs[BPF_REG_0].type))
+ if (type_may_be_null(regs[BPF_REG_0].type) && !regs[BPF_REG_0].id)
regs[BPF_REG_0].id = ++env->id_gen;
if (is_ptr_cast_function(func_id) &&
@@ -10772,26 +10829,24 @@ static bool is_kfunc_rcu_protected(struct bpf_call_arg_meta *meta)
}
static bool is_kfunc_arg_mem_size(const struct btf *btf,
- const struct btf_param *arg,
- const struct bpf_reg_state *reg)
+ const struct btf_param *arg)
{
const struct btf_type *t;
t = btf_type_skip_modifiers(btf, arg->type, NULL);
- if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE)
+ if (!btf_type_is_scalar(t))
return false;
return btf_param_match_suffix(btf, arg, "__sz");
}
static bool is_kfunc_arg_const_mem_size(const struct btf *btf,
- const struct btf_param *arg,
- const struct bpf_reg_state *reg)
+ const struct btf_param *arg)
{
const struct btf_type *t;
t = btf_type_skip_modifiers(btf, arg->type, NULL);
- if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE)
+ if (!btf_type_is_scalar(t))
return false;
return btf_param_match_suffix(btf, arg, "__szk");
@@ -10812,6 +10867,11 @@ static bool is_kfunc_arg_map(const struct btf *btf, const struct btf_param *arg)
return btf_param_match_suffix(btf, arg, "__map");
}
+static bool is_kfunc_arg_const_map(const struct btf *btf, const struct btf_param *arg)
+{
+ return btf_param_match_suffix(btf, arg, "__const_map");
+}
+
static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg)
{
return btf_param_match_suffix(btf, arg, "__alloc");
@@ -11043,6 +11103,11 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env,
}
enum kfunc_ptr_arg_type {
+ KF_ARG_CONST_MEM_SIZE,
+ KF_ARG_MEM_SIZE,
+ KF_ARG_CONST,
+ KF_ARG_CONST_ALLOC_SIZE_OR_ZERO,
+ KF_ARG_ANYTHING,
KF_ARG_PTR_TO_CTX,
KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */
KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */
@@ -11052,13 +11117,11 @@ enum kfunc_ptr_arg_type {
KF_ARG_PTR_TO_LIST_NODE,
KF_ARG_PTR_TO_BTF_ID, /* Also covers reg2btf_ids conversions */
KF_ARG_PTR_TO_MEM,
- KF_ARG_PTR_TO_MEM_SIZE, /* Size derived from next argument, skip it */
KF_ARG_PTR_TO_CALLBACK,
KF_ARG_PTR_TO_RB_ROOT,
KF_ARG_PTR_TO_RB_NODE,
- KF_ARG_PTR_TO_NULL,
KF_ARG_PTR_TO_CONST_STR,
- KF_ARG_PTR_TO_MAP,
+ KF_ARG_CONST_MAP_PTR,
KF_ARG_PTR_TO_TIMER,
KF_ARG_PTR_TO_WORKQUEUE,
KF_ARG_PTR_TO_IRQ_FLAG,
@@ -11319,108 +11382,154 @@ bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta)
return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data];
}
-static enum kfunc_ptr_arg_type
-get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_func_state *caller,
- struct bpf_reg_state *regs, struct bpf_call_arg_meta *meta,
- const struct btf_type *t, const struct btf_type *ref_t,
- const char *ref_tname, const struct btf_param *args,
- int arg, int nargs, argno_t argno, struct bpf_reg_state *reg)
+static int
+get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
+ const struct btf_param *args, int arg, int nargs)
{
- bool arg_mem_size = false;
+ const struct btf_type *t, *ref_t = NULL;
+ argno_t argno = argno_from_arg(arg + 1);
+ const char *ref_tname = NULL;
+ int arg_type;
- if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||
- meta->func_id == special_kfunc_list[KF_bpf_session_is_return] ||
- meta->func_id == special_kfunc_list[KF_bpf_session_cookie])
- return KF_ARG_PTR_TO_CTX;
+ t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL);
- if (arg + 1 < nargs &&
- (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1)) ||
- is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1))))
- arg_mem_size = true;
+ /* Scalar arguments are classified from their BTF suffix/name alone. */
+ if (btf_type_is_scalar(t)) {
+ if (is_kfunc_arg_constant(meta->btf, &args[arg]))
+ return KF_ARG_CONST;
+ if (is_kfunc_arg_const_mem_size(meta->btf, &args[arg]))
+ return KF_ARG_CONST_MEM_SIZE;
+ if (is_kfunc_arg_mem_size(meta->btf, &args[arg]))
+ return KF_ARG_MEM_SIZE;
+ if (is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdonly_buf_size") ||
+ is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdwr_buf_size"))
+ return KF_ARG_CONST_ALLOC_SIZE_OR_ZERO;
+ return KF_ARG_ANYTHING;
+ }
+
+ if (!btf_type_is_ptr(t)) {
+ verbose(env, "Unrecognized %s type %s\n",
+ reg_arg_name(env, argno), btf_type_str(t));
+ return -EINVAL;
+ }
+
+ ref_t = btf_type_skip_modifiers(meta->btf, t->type, NULL);
+ ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off);
/* In this function, we verify the kfunc's BTF as per the argument type,
* leaving the rest of the verification with respect to the register
* type to our caller. When a set of conditions hold in the BTF type of
* arguments, we resolve it to a known kfunc_ptr_arg_type.
*/
- if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg))
- return KF_ARG_PTR_TO_CTX;
-
- if (is_kfunc_arg_nullable(meta->btf, &args[arg]) && bpf_register_is_null(reg) &&
- !arg_mem_size)
- return KF_ARG_PTR_TO_NULL;
-
- if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg]))
- return KF_ARG_PTR_TO_ALLOC_BTF_ID;
-
- if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg]))
- return KF_ARG_PTR_TO_REFCOUNTED_KPTR;
-
- if (is_kfunc_arg_dynptr(meta->btf, &args[arg]))
- return KF_ARG_PTR_TO_DYNPTR;
-
- if (is_kfunc_arg_iter(meta, arg, &args[arg]))
- return KF_ARG_PTR_TO_ITER;
-
- if (is_kfunc_arg_list_head(meta->btf, &args[arg]))
- return KF_ARG_PTR_TO_LIST_HEAD;
-
- if (is_kfunc_arg_list_node(meta->btf, &args[arg]))
- return KF_ARG_PTR_TO_LIST_NODE;
-
- if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg]))
- return KF_ARG_PTR_TO_RB_ROOT;
-
- if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg]))
- return KF_ARG_PTR_TO_RB_NODE;
-
- if (is_kfunc_arg_const_str(meta->btf, &args[arg]))
- return KF_ARG_PTR_TO_CONST_STR;
+ if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||
+ meta->func_id == special_kfunc_list[KF_bpf_session_is_return] ||
+ meta->func_id == special_kfunc_list[KF_bpf_session_cookie])
+ arg_type = KF_ARG_PTR_TO_CTX;
+ else if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg))
+ arg_type = KF_ARG_PTR_TO_CTX;
+ else if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg]))
+ arg_type = KF_ARG_PTR_TO_ALLOC_BTF_ID;
+ else if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg]))
+ arg_type = KF_ARG_PTR_TO_REFCOUNTED_KPTR;
+ else if (is_kfunc_arg_dynptr(meta->btf, &args[arg]))
+ arg_type = KF_ARG_PTR_TO_DYNPTR;
+ else if (is_kfunc_arg_iter(meta, arg, &args[arg]))
+ arg_type = KF_ARG_PTR_TO_ITER;
+ else if (is_kfunc_arg_list_head(meta->btf, &args[arg]))
+ arg_type = KF_ARG_PTR_TO_LIST_HEAD;
+ else if (is_kfunc_arg_list_node(meta->btf, &args[arg]))
+ arg_type = KF_ARG_PTR_TO_LIST_NODE;
+ else if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg]))
+ arg_type = KF_ARG_PTR_TO_RB_ROOT;
+ else if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg]))
+ arg_type = KF_ARG_PTR_TO_RB_NODE;
+ else if (is_kfunc_arg_const_str(meta->btf, &args[arg]))
+ arg_type = KF_ARG_PTR_TO_CONST_STR;
+ else if (is_kfunc_arg_const_map(meta->btf, &args[arg]))
+ arg_type = KF_ARG_CONST_MAP_PTR;
+ else if (is_kfunc_arg_map(meta->btf, &args[arg]))
+ arg_type = KF_ARG_PTR_TO_BTF_ID;
+ else if (is_kfunc_arg_wq(meta->btf, &args[arg]))
+ arg_type = KF_ARG_PTR_TO_WORKQUEUE;
+ else if (is_kfunc_arg_timer(meta->btf, &args[arg]))
+ arg_type = KF_ARG_PTR_TO_TIMER;
+ else if (is_kfunc_arg_task_work(meta->btf, &args[arg]))
+ arg_type = KF_ARG_PTR_TO_TASK_WORK;
+ else if (is_kfunc_arg_irq_flag(meta->btf, &args[arg]))
+ arg_type = KF_ARG_PTR_TO_IRQ_FLAG;
+ else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg]))
+ arg_type = KF_ARG_PTR_TO_RES_SPIN_LOCK;
+ else if (is_kfunc_arg_callback(env, meta->btf, &args[arg]))
+ arg_type = KF_ARG_PTR_TO_CALLBACK;
+ else if (arg + 1 < nargs &&
+ (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) ||
+ is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) {
+ if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) &&
+ !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) {
+ verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n",
+ reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname);
+ return -EINVAL;
+ }
+ arg_type = KF_ARG_PTR_TO_MEM;
+ } else if (btf_type_is_struct(ref_t))
+ /* A pointer to a struct without a size argument is classified as KF_ARG_PTR_TO_BTF_ID */
+ arg_type = KF_ARG_PTR_TO_BTF_ID;
+ else {
+ /*
+ * Otherwise this is a fixed-size memory buffer supported by
+ * check_helper_mem_access(): a pointer to a scalar or a struct of
+ * scalars. The access size is derived from the pointed-to BTF type.
+ */
+ if (!btf_type_is_scalar(ref_t) &&
+ !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) {
+ verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n",
+ reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname);
+ return -EINVAL;
+ }
+ arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE;
+ }
- if (is_kfunc_arg_map(meta->btf, &args[arg]))
- return KF_ARG_PTR_TO_MAP;
+ if (is_kfunc_arg_nullable(meta->btf, &args[arg]))
+ arg_type |= PTR_MAYBE_NULL;
- if (is_kfunc_arg_wq(meta->btf, &args[arg]))
- return KF_ARG_PTR_TO_WORKQUEUE;
+ return arg_type;
+}
- if (is_kfunc_arg_timer(meta->btf, &args[arg]))
- return KF_ARG_PTR_TO_TIMER;
+static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
+ struct bpf_func_proto *proto)
+{
+ const struct btf *btf = meta->btf;
+ const struct btf_param *args;
+ u32 i, nargs;
+ int arg_type;
- if (is_kfunc_arg_task_work(meta->btf, &args[arg]))
- return KF_ARG_PTR_TO_TASK_WORK;
+ args = (const struct btf_param *)(meta->func_proto + 1);
+ nargs = btf_type_vlen(meta->func_proto);
+ if (nargs > MAX_BPF_FUNC_ARGS) {
+ verbose(env, "Function %s has %d > %d args\n", meta->func_name,
+ nargs, MAX_BPF_FUNC_ARGS);
+ return -EINVAL;
+ }
+ if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) {
+ verbose(env, "JIT does not support kfunc %s() with %d args\n",
+ meta->func_name, nargs);
+ return -ENOTSUPP;
+ }
- if (is_kfunc_arg_irq_flag(meta->btf, &args[arg]))
- return KF_ARG_PTR_TO_IRQ_FLAG;
+ for (i = 0; i < nargs; i++) {
+ if (is_kfunc_arg_prog_aux(btf, &args[i]) ||
+ is_kfunc_arg_ignore(btf, &args[i]) ||
+ is_kfunc_arg_implicit(meta, i))
+ continue;
- if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg]))
- return KF_ARG_PTR_TO_RES_SPIN_LOCK;
+ arg_type = get_kfunc_arg_type(env, meta, args, i, nargs);
+ if (arg_type < 0)
+ return arg_type;
- if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) {
- if (!btf_type_is_struct(ref_t)) {
- verbose(env, "kernel function %s %s pointer type %s %s is not supported\n",
- meta->func_name, reg_arg_name(env, argno),
- btf_type_str(ref_t), ref_tname);
- return -EINVAL;
- }
- return KF_ARG_PTR_TO_BTF_ID;
+ proto->arg_type[i] = arg_type;
}
- if (is_kfunc_arg_callback(env, meta->btf, &args[arg]))
- return KF_ARG_PTR_TO_CALLBACK;
-
- /* This is the catch all argument type of register types supported by
- * check_helper_mem_access. However, we only allow when argument type is
- * pointer to scalar, or struct composed (recursively) of scalars. When
- * arg_mem_size is true, the pointer can be void *.
- */
- if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) &&
- (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) {
- verbose(env, "%s pointer type %s %s must point to %sscalar, or struct with scalar\n",
- reg_arg_name(env, argno),
- btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : "");
- return -EINVAL;
- }
- return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM;
+ return 0;
}
static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,
@@ -12006,16 +12115,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me
args = (const struct btf_param *)(meta->func_proto + 1);
nargs = btf_type_vlen(meta->func_proto);
- if (nargs > MAX_BPF_FUNC_ARGS) {
- verbose(env, "Function %s has %d > %d args\n", func_name, nargs,
- MAX_BPF_FUNC_ARGS);
- return -EINVAL;
- }
- if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) {
- verbose(env, "JIT does not support kfunc %s() with %d args\n",
- func_name, nargs);
- return -ENOTSUPP;
- }
ret = check_outgoing_stack_args(env, caller, nargs);
if (ret)
@@ -12032,8 +12131,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me
int regno = reg_from_argno(argno);
bool btf_id_fixed_off_ok = true;
u32 ref_id, type_size;
- bool is_ret_buf_sz = false;
- int kf_arg_type;
+ int kf_arg_type = meta->fn->arg_type[i];
if (is_kfunc_arg_prog_aux(btf, &args[i])) {
/* Reject repeated use bpf_prog_aux */
@@ -12056,52 +12154,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me
t = btf_type_skip_modifiers(btf, args[i].type, NULL);
- if (btf_type_is_scalar(t)) {
- if (reg->type != SCALAR_VALUE) {
- verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno));
- return -EINVAL;
- }
-
- if (is_kfunc_arg_constant(meta->btf, &args[i])) {
- if (meta->arg_constant.found) {
- verifier_bug(env, "only one constant argument permitted");
- return -EFAULT;
- }
- if (!tnum_is_const(reg->var_off)) {
- verbose(env, "%s must be a known constant\n",
- reg_arg_name(env, argno));
- return -EINVAL;
- }
- if (regno >= 0)
- ret = mark_chain_precision(env, regno);
- else
- ret = mark_stack_arg_precision(env, i);
- if (ret < 0)
- return ret;
- meta->arg_constant.found = true;
- meta->arg_constant.value = reg->var_off.value;
- } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) {
- meta->r0_rdonly = true;
- is_ret_buf_sz = true;
- } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdwr_buf_size")) {
- is_ret_buf_sz = true;
- }
-
- if (is_ret_buf_sz) {
- ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem);
- if (ret < 0)
- return ret;
- }
- continue;
- }
-
- if (!btf_type_is_ptr(t)) {
- verbose(env, "Unrecognized %s type %s\n",
- reg_arg_name(env, argno), btf_type_str(t));
- return -EINVAL;
- }
-
- if ((bpf_register_is_null(reg) || type_may_be_null(reg->type)) &&
+ if (btf_type_is_ptr(t) && (bpf_register_is_null(reg) || type_may_be_null(reg->type)) &&
!is_kfunc_arg_nullable(meta->btf, &args[i])) {
verbose(env, "Possibly NULL pointer passed to trusted %s\n",
reg_arg_name(env, argno));
@@ -12118,76 +12171,36 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me
if (reg_is_referenced(env, reg))
update_ref_obj(&meta->ref_obj, reg);
- ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
- ref_tname = btf_name_by_offset(btf, ref_t->name_off);
+ if (btf_type_is_ptr(t)) {
+ ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
+ ref_tname = btf_name_by_offset(btf, ref_t->name_off);
+ }
- kf_arg_type = get_kfunc_ptr_arg_type(env, caller, regs, meta, t, ref_t, ref_tname,
- args, i, nargs, argno, reg);
- if (kf_arg_type < 0)
- return kf_arg_type;
- switch (kf_arg_type) {
- case KF_ARG_PTR_TO_NULL:
+ if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type))
continue;
- case KF_ARG_PTR_TO_MAP:
- if (!reg->map_ptr) {
- verbose(env, "pointer in %s isn't map pointer\n",
- reg_arg_name(env, argno));
- return -EINVAL;
- }
- if (meta->map.ptr && (reg->map_ptr->record->wq_off >= 0 ||
- reg->map_ptr->record->task_work_off >= 0)) {
- /* Use map_uid (which is unique id of inner map) to reject:
- * inner_map1 = bpf_map_lookup_elem(outer_map, key1)
- * inner_map2 = bpf_map_lookup_elem(outer_map, key2)
- * if (inner_map1 && inner_map2) {
- * wq = bpf_map_lookup_elem(inner_map1);
- * if (wq)
- * // mismatch would have been allowed
- * bpf_wq_init(wq, inner_map2);
- * }
- *
- * Comparing map_ptr is enough to distinguish normal and outer maps.
- */
- if (meta->map.ptr != reg->map_ptr ||
- meta->map.uid != reg->map_uid) {
- if (reg->map_ptr->record->task_work_off >= 0) {
- verbose(env,
- "bpf_task_work pointer in R2 map_uid=%d doesn't match map pointer in R3 map_uid=%d\n",
- meta->map.uid, reg->map_uid);
- return -EINVAL;
- }
- verbose(env,
- "workqueue pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n",
- meta->map.uid, reg->map_uid);
- return -EINVAL;
- }
- }
- meta->map.ptr = reg->map_ptr;
- meta->map.uid = reg->map_uid;
- fallthrough;
+
+ if (is_kfunc_arg_map(btf, &args[i])) {
+ ref_id = *reg2btf_ids[CONST_PTR_TO_MAP];
+ ref_t = btf_type_by_id(btf_vmlinux, ref_id);
+ ref_tname = btf_name_by_offset(btf, ref_t->name_off);
+ }
+
+ switch (base_type(kf_arg_type)) {
+ case KF_ARG_CONST:
+ case KF_ARG_CONST_MEM_SIZE:
+ case KF_ARG_MEM_SIZE:
+ case KF_ARG_ANYTHING:
+ case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO:
case KF_ARG_PTR_TO_ALLOC_BTF_ID:
case KF_ARG_PTR_TO_BTF_ID:
- if (!is_trusted_reg(env, reg)) {
- if (!is_kfunc_rcu(meta)) {
- verbose(env, "%s must be referenced or trusted\n",
- reg_arg_name(env, argno));
- return -EINVAL;
- }
- if (!is_rcu_reg(reg)) {
- verbose(env, "%s must be a rcu pointer\n",
- reg_arg_name(env, argno));
- return -EINVAL;
- }
- }
- fallthrough;
+ case KF_ARG_CONST_MAP_PTR:
case KF_ARG_PTR_TO_ITER:
case KF_ARG_PTR_TO_LIST_HEAD:
case KF_ARG_PTR_TO_LIST_NODE:
case KF_ARG_PTR_TO_RB_ROOT:
case KF_ARG_PTR_TO_RB_NODE:
case KF_ARG_PTR_TO_MEM:
- case KF_ARG_PTR_TO_MEM_SIZE:
case KF_ARG_PTR_TO_CALLBACK:
case KF_ARG_PTR_TO_CONST_STR:
case KF_ARG_PTR_TO_WORKQUEUE:
@@ -12218,7 +12231,35 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me
if (ret < 0)
return ret;
- switch (kf_arg_type) {
+ switch (base_type(kf_arg_type)) {
+ case KF_ARG_CONST:
+ if (reg->type != SCALAR_VALUE) {
+ verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno));
+ return -EINVAL;
+ }
+
+ ret = process_const_arg(env, reg, argno, meta);
+ if (ret < 0)
+ return ret;
+ break;
+ case KF_ARG_ANYTHING:
+ if (reg->type != SCALAR_VALUE) {
+ verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno));
+ return -EINVAL;
+ }
+ break;
+ case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO:
+ if (reg->type != SCALAR_VALUE) {
+ verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno));
+ return -EINVAL;
+ }
+
+ if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size"))
+ meta->r0_rdonly = true;
+ ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem);
+ if (ret < 0)
+ return ret;
+ break;
case KF_ARG_PTR_TO_CTX:
if (reg->type != PTR_TO_CTX) {
verbose(env, "%s expected pointer to ctx, but got %s\n",
@@ -12387,75 +12428,100 @@ check_ok:
if (ret < 0)
return ret;
break;
- case KF_ARG_PTR_TO_MAP:
- /* If argument has '__map' suffix expect 'struct bpf_map *' */
- ref_id = *reg2btf_ids[CONST_PTR_TO_MAP];
- ref_t = btf_type_by_id(btf_vmlinux, ref_id);
- ref_tname = btf_name_by_offset(btf, ref_t->name_off);
- fallthrough;
- case KF_ARG_PTR_TO_BTF_ID:
- /* Only base_type is checked, further checks are done here */
- if ((base_type(reg->type) != PTR_TO_BTF_ID ||
- (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) &&
- !reg2btf_ids[base_type(reg->type)]) {
- verbose(env, "%s is %s ", reg_arg_name(env, argno),
- reg_type_str(env, reg->type));
- verbose(env, "expected %s or socket\n",
- reg_type_str(env, base_type(reg->type) |
- (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS)));
+ case KF_ARG_CONST_MAP_PTR:
+ if (base_type(reg->type) != CONST_PTR_TO_MAP ||
+ type_may_be_null(reg->type)) {
+ verbose(env, "pointer in %s isn't map pointer\n",
+ reg_arg_name(env, argno));
return -EINVAL;
}
- ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno);
+ ret = process_map_ptr_arg(env, reg, argno, meta);
if (ret < 0)
return ret;
break;
- case KF_ARG_PTR_TO_MEM:
- resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
- if (IS_ERR(resolve_ret)) {
- verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n",
- reg_arg_name(env, argno), btf_type_str(ref_t),
- ref_tname, PTR_ERR(resolve_ret));
+ case KF_ARG_PTR_TO_BTF_ID:
+ /* Only base_type is checked, further checks are done here */
+ if (base_type(reg->type) == PTR_TO_BTF_ID ||
+ reg2btf_ids[base_type(reg->type)]) {
+ if (!is_trusted_reg(env, reg) ||
+ bpf_type_has_unsafe_modifiers(reg->type)) {
+ if (!is_kfunc_rcu(meta)) {
+ verbose(env, "%s must be referenced or trusted\n",
+ reg_arg_name(env, argno));
+ return -EINVAL;
+ }
+ if (!is_rcu_reg(reg)) {
+ verbose(env, "%s must be a rcu pointer\n",
+ reg_arg_name(env, argno));
+ return -EINVAL;
+ }
+ }
+
+ ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno);
+ if (ret < 0)
+ return ret;
+ break;
+ }
+
+ if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) {
+ enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id);
+
+ verbose(env, "%s is %s expected %s %s",
+ reg_arg_name(env, argno), reg_type_str(env, reg->type),
+ btf_type_str(ref_t), ref_tname);
+ if (reg2btf_type != NOT_INIT)
+ verbose(env, " or %s", reg_type_str(env, reg2btf_type));
+ verbose(env, "\n");
return -EINVAL;
}
- ret = check_mem_reg(env, reg, argno, type_size);
+
+ /*
+ * If the register does not contain btf id but the argument type is a pointer to
+ * scalar-only struct, allow verifying it as a fixed size memory.
+ */
+ kf_arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE;
+ fallthrough;
+ case KF_ARG_PTR_TO_MEM:
+ if (kf_arg_type & MEM_FIXED_SIZE) {
+ resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
+ if (IS_ERR(resolve_ret)) {
+ verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n",
+ reg_arg_name(env, argno), btf_type_str(ref_t),
+ ref_tname, PTR_ERR(resolve_ret));
+ return -EINVAL;
+ }
+ ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, meta);
+ if (ret < 0)
+ return ret;
+ }
+ break;
+ case KF_ARG_CONST_MEM_SIZE:
+ ret = process_const_arg(env, reg, argno, meta);
if (ret < 0)
return ret;
- break;
- case KF_ARG_PTR_TO_MEM_SIZE:
+ fallthrough;
+ case KF_ARG_MEM_SIZE:
{
- struct bpf_reg_state *buff_reg = reg;
- const struct btf_param *buff_arg = &args[i];
- struct bpf_reg_state *size_reg = get_func_arg_reg(caller, regs, i + 1);
- const struct btf_param *size_arg = &args[i + 1];
- argno_t next_argno = argno_from_arg(i + 2);
-
- if (!bpf_register_is_null(buff_reg) || !is_kfunc_arg_nullable(meta->btf, buff_arg)) {
- ret = check_kfunc_mem_size_reg(env, buff_reg, size_reg,
- argno, next_argno);
- if (ret < 0) {
- verbose(env, "%s and ", reg_arg_name(env, argno));
- verbose(env, "%s memory, len pair leads to invalid memory access\n",
- reg_arg_name(env, next_argno));
- return ret;
- }
- }
+ struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, i - 1);
+ struct bpf_reg_state *size_reg = reg;
+ argno_t buff_argno = argno_from_arg(i);
- if (is_kfunc_arg_const_mem_size(meta->btf, size_arg, size_reg)) {
- if (meta->arg_constant.found) {
- verifier_bug(env, "only one constant argument permitted");
- return -EFAULT;
- }
- if (!tnum_is_const(size_reg->var_off)) {
- verbose(env, "%s must be a known constant\n",
- reg_arg_name(env, next_argno));
- return -EINVAL;
- }
- meta->arg_constant.found = true;
- meta->arg_constant.value = size_reg->var_off.value;
+ if (reg->type != SCALAR_VALUE) {
+ verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno));
+ return -EINVAL;
}
- /* Skip next '__sz' or '__szk' argument */
- i++;
+ if (bpf_register_is_null(buff_reg))
+ break;
+
+ ret = check_mem_size_reg(env, buff_reg, size_reg, buff_argno, argno,
+ BPF_READ | BPF_WRITE, true, meta);
+ if (ret < 0) {
+ verbose(env, "%s and ", reg_arg_name(env, buff_argno));
+ verbose(env, "%s memory, len pair leads to invalid memory access\n",
+ reg_arg_name(env, argno));
+ return ret;
+ }
break;
}
case KF_ARG_PTR_TO_CALLBACK:
@@ -12515,7 +12581,7 @@ check_ok:
reg_arg_name(env, argno));
return -EINVAL;
}
- ret = process_timer_kfunc(env, reg, argno, meta);
+ ret = process_timer_func(env, reg, argno, &meta->map);
if (ret < 0)
return ret;
break;
@@ -12976,6 +13042,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
int err, insn_idx = *insn_idx_p;
const struct btf_param *args;
u32 i, nargs, ptr_type_id;
+ struct bpf_kfunc_desc *desc;
struct btf *desc_btf;
int id;
@@ -12992,6 +13059,13 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
func_name = meta.func_name;
insn_aux = &env->insn_aux_data[insn_idx];
+ desc = find_kfunc_desc(env->prog, insn->imm, insn->off);
+ if (!desc) {
+ verifier_bug(env, "kfunc descriptor not found for func_id %u", insn->imm);
+ return -EFAULT;
+ }
+ meta.fn = &desc->proto;
+
insn_aux->is_iter_next = bpf_is_iter_next_kfunc(&meta);
if (!insn->off &&
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 76ab51deaa6b..891897f8a1b3 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -221,7 +221,7 @@ const struct bpf_func_proto bpf_probe_read_user_proto = {
.gpl_only = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_UNINIT_MEM,
- .arg2_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg2_type = ARG_MEM_SIZE_OR_ZERO,
.arg3_type = ARG_ANYTHING,
};
@@ -258,7 +258,7 @@ const struct bpf_func_proto bpf_probe_read_user_str_proto = {
.gpl_only = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_UNINIT_MEM,
- .arg2_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg2_type = ARG_MEM_SIZE_OR_ZERO,
.arg3_type = ARG_ANYTHING,
};
@@ -273,7 +273,7 @@ const struct bpf_func_proto bpf_probe_read_kernel_proto = {
.gpl_only = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_UNINIT_MEM,
- .arg2_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg2_type = ARG_MEM_SIZE_OR_ZERO,
.arg3_type = ARG_ANYTHING,
};
@@ -308,7 +308,7 @@ const struct bpf_func_proto bpf_probe_read_kernel_str_proto = {
.gpl_only = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_UNINIT_MEM,
- .arg2_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg2_type = ARG_MEM_SIZE_OR_ZERO,
.arg3_type = ARG_ANYTHING,
};
@@ -328,7 +328,7 @@ static const struct bpf_func_proto bpf_probe_read_compat_proto = {
.gpl_only = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_UNINIT_MEM,
- .arg2_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg2_type = ARG_MEM_SIZE_OR_ZERO,
.arg3_type = ARG_ANYTHING,
};
@@ -347,7 +347,7 @@ static const struct bpf_func_proto bpf_probe_read_compat_str_proto = {
.gpl_only = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_UNINIT_MEM,
- .arg2_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg2_type = ARG_MEM_SIZE_OR_ZERO,
.arg3_type = ARG_ANYTHING,
};
#endif /* CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE */
@@ -383,7 +383,7 @@ static const struct bpf_func_proto bpf_probe_write_user_proto = {
.ret_type = RET_INTEGER,
.arg1_type = ARG_ANYTHING,
.arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg3_type = ARG_CONST_SIZE,
+ .arg3_type = ARG_MEM_SIZE,
};
#define MAX_TRACE_PRINTK_VARARGS 3
@@ -418,7 +418,7 @@ static const struct bpf_func_proto bpf_trace_printk_proto = {
.gpl_only = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg2_type = ARG_CONST_SIZE,
+ .arg2_type = ARG_MEM_SIZE,
};
static void __set_printk_clr_event(struct work_struct *work)
@@ -474,9 +474,9 @@ static const struct bpf_func_proto bpf_trace_vprintk_proto = {
.gpl_only = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg2_type = ARG_CONST_SIZE,
+ .arg2_type = ARG_MEM_SIZE,
.arg3_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
- .arg4_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg4_type = ARG_MEM_SIZE_OR_ZERO,
};
const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void)
@@ -518,9 +518,9 @@ static const struct bpf_func_proto bpf_seq_printf_proto = {
.arg1_type = ARG_PTR_TO_BTF_ID,
.arg1_btf_id = &btf_seq_file_ids[0],
.arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg3_type = ARG_CONST_SIZE,
+ .arg3_type = ARG_MEM_SIZE,
.arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
- .arg5_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg5_type = ARG_MEM_SIZE_OR_ZERO,
};
BPF_CALL_3(bpf_seq_write, struct seq_file *, m, const void *, data, u32, len)
@@ -535,7 +535,7 @@ static const struct bpf_func_proto bpf_seq_write_proto = {
.arg1_type = ARG_PTR_TO_BTF_ID,
.arg1_btf_id = &btf_seq_file_ids[0],
.arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
};
BPF_CALL_4(bpf_seq_printf_btf, struct seq_file *, m, struct btf_ptr *, ptr,
@@ -559,7 +559,7 @@ static const struct bpf_func_proto bpf_seq_printf_btf_proto = {
.arg1_type = ARG_PTR_TO_BTF_ID,
.arg1_btf_id = &btf_seq_file_ids[0],
.arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.arg4_type = ARG_ANYTHING,
};
@@ -633,7 +633,7 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = {
.arg1_type = ARG_CONST_MAP_PTR,
.arg2_type = ARG_ANYTHING,
.arg3_type = ARG_PTR_TO_UNINIT_MEM,
- .arg4_type = ARG_CONST_SIZE,
+ .arg4_type = ARG_MEM_SIZE,
};
const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void)
@@ -730,7 +730,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = {
.arg2_type = ARG_CONST_MAP_PTR,
.arg3_type = ARG_ANYTHING,
.arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg5_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg5_type = ARG_MEM_SIZE_OR_ZERO,
};
static DEFINE_PER_CPU(int, bpf_event_output_nest_level);
@@ -996,7 +996,7 @@ static const struct bpf_func_proto bpf_d_path_proto = {
.arg1_type = ARG_PTR_TO_BTF_ID,
.arg1_btf_id = &bpf_d_path_btf_ids[0],
.arg2_type = ARG_PTR_TO_MEM | MEM_WRITE,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.allowed = bpf_d_path_allowed,
};
@@ -1053,9 +1053,9 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = {
.gpl_only = false,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_MEM | MEM_WRITE,
- .arg2_type = ARG_CONST_SIZE,
+ .arg2_type = ARG_MEM_SIZE,
.arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg4_type = ARG_CONST_SIZE,
+ .arg4_type = ARG_MEM_SIZE,
.arg5_type = ARG_ANYTHING,
};
@@ -1218,7 +1218,7 @@ const struct bpf_func_proto bpf_get_branch_snapshot_proto = {
.gpl_only = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_UNINIT_MEM,
- .arg2_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg2_type = ARG_MEM_SIZE_OR_ZERO,
};
BPF_CALL_3(get_func_arg, void *, ctx, u32, n, u64 *, value)
@@ -1421,7 +1421,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_tp = {
.arg2_type = ARG_CONST_MAP_PTR,
.arg3_type = ARG_ANYTHING,
.arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg5_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg5_type = ARG_MEM_SIZE_OR_ZERO,
};
BPF_CALL_3(bpf_get_stackid_tp, void *, tp_buff, struct bpf_map *, map,
@@ -1462,7 +1462,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_tp = {
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.arg4_type = ARG_ANYTHING,
};
@@ -1524,12 +1524,12 @@ clear:
}
static const struct bpf_func_proto bpf_perf_prog_read_value_proto = {
- .func = bpf_perf_prog_read_value,
- .gpl_only = true,
- .ret_type = RET_INTEGER,
- .arg1_type = ARG_PTR_TO_CTX,
- .arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE,
+ .func = bpf_perf_prog_read_value,
+ .gpl_only = true,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_PTR_TO_UNINIT_MEM,
+ .arg3_type = ARG_MEM_SIZE,
};
BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx,
@@ -1566,7 +1566,7 @@ static const struct bpf_func_proto bpf_read_branch_records_proto = {
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.arg4_type = ARG_ANYTHING,
};
@@ -1646,7 +1646,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = {
.arg2_type = ARG_CONST_MAP_PTR,
.arg3_type = ARG_ANYTHING,
.arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
- .arg5_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg5_type = ARG_MEM_SIZE_OR_ZERO,
};
extern const struct bpf_func_proto bpf_skb_output_proto;
@@ -1701,7 +1701,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = {
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
- .arg3_type = ARG_CONST_SIZE_OR_ZERO,
+ .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.arg4_type = ARG_ANYTHING,
};