summaryrefslogtreecommitdiff
path: root/kernel/bpf
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/bpf')
-rw-r--r--kernel/bpf/arena.c2
-rw-r--r--kernel/bpf/arraymap.c2
-rw-r--r--kernel/bpf/cfg.c13
-rw-r--r--kernel/bpf/core.c27
-rw-r--r--kernel/bpf/fixups.c13
-rw-r--r--kernel/bpf/helpers.c5
-rw-r--r--kernel/bpf/liveness.c25
-rw-r--r--kernel/bpf/syscall.c26
-rw-r--r--kernel/bpf/verifier.c27
9 files changed, 101 insertions, 39 deletions
diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c
index 802656c6fd3c..49a8f7b1beef 100644
--- a/kernel/bpf/arena.c
+++ b/kernel/bpf/arena.c
@@ -511,7 +511,7 @@ static int arena_map_direct_value_addr(const struct bpf_map *map, u64 *imm, u32
{
struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
- if ((u64)off > arena->user_vm_end - arena->user_vm_start)
+ if ((u64)off >= arena->user_vm_end - arena->user_vm_start)
return -ERANGE;
*imm = (unsigned long)arena->user_vm_start;
return 0;
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
index 5e25e0353509..dfb2110ab733 100644
--- a/kernel/bpf/arraymap.c
+++ b/kernel/bpf/arraymap.c
@@ -827,7 +827,7 @@ const struct bpf_map_ops array_map_ops = {
};
const struct bpf_map_ops percpu_array_map_ops = {
- .map_meta_equal = bpf_map_meta_equal,
+ .map_meta_equal = array_map_meta_equal,
.map_alloc_check = array_map_alloc_check,
.map_alloc = array_map_alloc,
.map_free = array_map_free,
diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c
index 998f42a8189a..26d37066465f 100644
--- a/kernel/bpf/cfg.c
+++ b/kernel/bpf/cfg.c
@@ -64,11 +64,19 @@ static void mark_subprog_might_sleep(struct bpf_verifier_env *env, int off)
subprog->might_sleep = true;
}
+static void mark_subprog_might_throw(struct bpf_verifier_env *env, int off)
+{
+ struct bpf_subprog_info *subprog;
+
+ subprog = bpf_find_containing_subprog(env, off);
+ subprog->might_throw = true;
+}
+
/* 't' is an index of a call-site.
* 'w' is a callee entry point.
* Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED.
* Rely on DFS traversal order and absence of recursive calls to guarantee that
- * callee's change_pkt_data marks would be correct at that moment.
+ * callee's effect marks would be correct at that moment.
*/
static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w)
{
@@ -78,6 +86,7 @@ static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w)
callee = bpf_find_containing_subprog(env, w);
caller->changes_pkt_data |= callee->changes_pkt_data;
caller->might_sleep |= callee->might_sleep;
+ caller->might_throw |= callee->might_throw;
}
enum {
@@ -509,6 +518,8 @@ static int visit_insn(int t, struct bpf_verifier_env *env)
mark_subprog_might_sleep(env, t);
if (ret == 0 && bpf_is_kfunc_pkt_changing(&meta))
mark_subprog_changes_pkt_data(env, t);
+ if (ret == 0 && bpf_is_throw_kfunc(insn))
+ mark_subprog_might_throw(env, t);
}
return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 8b018ff48875..6aa2a8b24030 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1771,6 +1771,9 @@ static u32 abs_s32(s32 x)
return x >= 0 ? (u32)x : -(u32)x;
}
+static u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5,
+ const struct bpf_insn *insn);
+
/**
* ___bpf_prog_run - run eBPF program on a given context
* @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers
@@ -2077,10 +2080,9 @@ select_insn:
CONT;
JMP_CALL_ARGS:
- BPF_R0 = (__bpf_call_base_args + insn->imm)(BPF_R1, BPF_R2,
- BPF_R3, BPF_R4,
- BPF_R5,
- insn + insn->off + 1);
+ BPF_R0 = interpreters_args[insn->off](BPF_R1, BPF_R2, BPF_R3,
+ BPF_R4, BPF_R5,
+ insn + insn->imm + 1);
CONT;
JMP_TAIL_CALL: {
@@ -2394,13 +2396,22 @@ EVAL4(PROG_NAME_LIST, 416, 448, 480, 512)
#undef PROG_NAME_LIST
#ifdef CONFIG_BPF_SYSCALL
-void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
+int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
{
stack_depth = max_t(u32, stack_depth, 1);
- insn->off = (s16) insn->imm;
- insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] -
- __bpf_call_base_args;
+ /* Prevent out-of-bounds read to interpreters_args */
+ if (stack_depth > MAX_BPF_STACK)
+ return -EINVAL;
+ insn->off = (round_up(stack_depth, 32) / 32) - 1;
insn->code = BPF_JMP | BPF_CALL_ARGS;
+ return 0;
+}
+
+s32 bpf_call_args_imm(s16 idx)
+{
+ if (WARN_ON_ONCE(idx < 0 || idx >= ARRAY_SIZE(interpreters_args)))
+ return 0;
+ return BPF_CALL_IMM(interpreters_args[idx]);
}
#endif
#endif
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index fba9e8c00878..3692adf62558 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -1250,9 +1250,9 @@ static int jit_subprogs(struct bpf_verifier_env *env)
}
if (!bpf_pseudo_call(insn))
continue;
- insn->off = env->insn_aux_data[i].call_imm;
- subprog = bpf_find_subprog(env, i + insn->off + 1);
- insn->imm = subprog;
+ insn->imm = env->insn_aux_data[i].call_imm;
+ subprog = bpf_find_subprog(env, i + insn->imm + 1);
+ insn->off = subprog;
}
prog->jited = 1;
@@ -1416,7 +1416,12 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env)
depth = get_callee_stack_depth(env, insn, i);
if (depth < 0)
return depth;
- bpf_patch_call_args(insn, depth);
+ err = bpf_patch_call_args(insn, depth);
+ if (err) {
+ verbose(env, "stack depth %d exceeds interpreter stack depth limit\n",
+ depth);
+ return err;
+ }
}
err = 0;
#endif
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 2bb60200c266..b5314c9fed3c 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -4241,8 +4241,13 @@ __bpf_kfunc int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p,
data_len = __bpf_dynptr_size(data_ptr);
data = __bpf_dynptr_data(data_ptr, data_len);
+ if (!data)
+ return -EINVAL;
+
sig_len = __bpf_dynptr_size(sig_ptr);
sig = __bpf_dynptr_data(sig_ptr, sig_len);
+ if (!sig)
+ return -EINVAL;
return verify_pkcs7_signature(data, data_len, sig, sig_len,
trusted_keyring->key,
diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
index 332e6e003f27..58197d73b120 100644
--- a/kernel/bpf/liveness.c
+++ b/kernel/bpf/liveness.c
@@ -1914,26 +1914,15 @@ int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env)
return -ENOMEM;
}
- instance = call_instance(env, NULL, 0, 0);
- if (IS_ERR(instance)) {
- err = PTR_ERR(instance);
- goto out;
- }
- err = analyze_subprog(env, NULL, info, instance, callsites);
- if (err)
- goto out;
-
/*
- * Subprogs and callbacks that don't receive FP-derived arguments
- * cannot access ancestor stack frames, so they were skipped during
- * the recursive walk above. Async callbacks (timer, workqueue) are
- * also not reachable from the main program's call graph. Analyze
- * all unvisited subprogs as independent roots at depth 0.
+ * Analyze every subprog in reverse topological order (callers
+ * before callees) so that each subprog is analyzed before its
+ * callees, allowing the recursive walk inside analyze_subprog()
+ * to naturally reach callees that receive FP-derived args.
*
- * Use reverse topological order (callers before callees) so that
- * each subprog is analyzed before its callees, allowing the
- * recursive walk inside analyze_subprog() to naturally
- * reach nested callees that also lack FP-derived args.
+ * Subprogs and callbacks that don't receive FP-derived arguments
+ * cannot access ancestor stack frames are analyzed independently.
+ * Async callbacks (timer, workqueue) are handled the same way.
*/
for (k = env->subprog_cnt - 1; k >= 0; k--) {
int sub = env->subprog_topo_order[k];
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index a3c0214ca934..630d530782fe 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -4919,6 +4919,29 @@ out:
return map;
}
+static void prepare_dump_pseudo_call(struct bpf_insn *insn)
+{
+ s32 call_off = insn->imm;
+
+ /*
+ * BPF_CALL_ARGS only exists for interpreter fallback.
+ * 1. For interpreter (BPF_CALL_ARGS): insn->off is the index of
+ * interpreters_args array, so here using bpf_call_args_imm()
+ * to get the real address offset.
+ * 2. For JIT (BPF_CALL): insn->off is the subprog id.
+ */
+ if (insn->code == (BPF_JMP | BPF_CALL_ARGS))
+ insn->imm = bpf_call_args_imm(insn->off);
+ else
+ insn->imm = insn->off;
+
+ /* Avoid dumping a truncated and misleading pc-relative offset. */
+ if (call_off > S16_MAX || call_off < S16_MIN)
+ insn->off = 0;
+ else
+ insn->off = call_off;
+}
+
static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
const struct cred *f_cred)
{
@@ -4944,6 +4967,9 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
}
if (code == (BPF_JMP | BPF_CALL) ||
code == (BPF_JMP | BPF_CALL_ARGS)) {
+ /* Restore the legacy xlated dump layout. */
+ if (insns[i].src_reg == BPF_PSEUDO_CALL)
+ prepare_dump_pseudo_call(&insns[i]);
if (code == (BPF_JMP | BPF_CALL_ARGS))
insns[i].code = BPF_JMP | BPF_CALL;
if (!bpf_dump_raw_ok(f_cred))
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 69d75515ed3f..7fb88e1cd7c4 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -442,7 +442,6 @@ static bool is_dynptr_ref_function(enum bpf_func_id func_id)
static bool is_sync_callback_calling_kfunc(u32 btf_id);
static bool is_async_callback_calling_kfunc(u32 btf_id);
static bool is_callback_calling_kfunc(u32 btf_id);
-static bool is_bpf_throw_kfunc(struct bpf_insn *insn);
static bool is_bpf_wq_set_callback_kfunc(u32 btf_id);
static bool is_task_work_add_kfunc(u32 func_id);
@@ -5405,7 +5404,7 @@ continue_func:
if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) {
bool err = false;
- if (!is_bpf_throw_kfunc(insn + i))
+ if (!bpf_is_throw_kfunc(insn + i))
continue;
for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) {
if (subprog[tmp].is_cb) {
@@ -9499,6 +9498,9 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
return 0;
}
+static int process_bpf_exit_full(struct bpf_verifier_env *env,
+ bool *do_print_state, bool exception_exit);
+
static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
int *insn_idx)
{
@@ -9552,6 +9554,17 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
}
+ if (env->subprog_info[subprog].might_throw) {
+ struct bpf_verifier_state *branch;
+
+ branch = push_stack(env, *insn_idx + 1, *insn_idx, false);
+ if (IS_ERR(branch)) {
+ verbose(env, "failed to push state for global subprog exception path\n");
+ return PTR_ERR(branch);
+ }
+ return process_bpf_exit_full(env, NULL, true);
+ }
+
/* continue with next insn after call */
return 0;
}
@@ -11263,7 +11276,11 @@ BTF_ID(func, bpf_task_work_schedule_resume)
BTF_ID(func, bpf_arena_alloc_pages)
BTF_ID(func, bpf_arena_free_pages)
BTF_ID(func, bpf_arena_reserve_pages)
+#ifdef CONFIG_BPF_EVENTS
BTF_ID(func, bpf_session_is_return)
+#else
+BTF_ID_UNUSED
+#endif
BTF_ID(func, bpf_stream_vprintk)
BTF_ID(func, bpf_stream_print_stack)
@@ -11778,7 +11795,7 @@ static bool is_async_callback_calling_kfunc(u32 btf_id)
is_task_work_add_kfunc(btf_id);
}
-static bool is_bpf_throw_kfunc(struct bpf_insn *insn)
+bool bpf_is_throw_kfunc(struct bpf_insn *insn)
{
return bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
insn->imm == special_kfunc_list[KF_bpf_throw];
@@ -12968,8 +12985,6 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_ca
}
static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name);
-static int process_bpf_exit_full(struct bpf_verifier_env *env,
- bool *do_print_state, bool exception_exit);
static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
int *insn_idx_p)
@@ -13350,7 +13365,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie])
env->prog->call_session_cookie = true;
- if (is_bpf_throw_kfunc(insn))
+ if (bpf_is_throw_kfunc(insn))
return process_bpf_exit_full(env, NULL, true);
return 0;
generated by cgit v1.2.3 (git 2.0.4) at 2026-05-25 07:35:15 +0000