summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/linux/bpf.h8
-rw-r--r--include/linux/filter.h3
-rw-r--r--kernel/bpf/core.c27
-rw-r--r--kernel/bpf/fixups.c13
-rw-r--r--kernel/bpf/syscall.c26
-rw-r--r--tools/testing/selftests/bpf/prog_tests/verifier.c2
-rw-r--r--tools/testing/selftests/bpf/progs/verifier_call_large_imm.c66
7 files changed, 129 insertions, 16 deletions
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 01e203964892..cd191c5fdb0a 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -2917,7 +2917,13 @@ int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size,
int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size);
#ifndef CONFIG_BPF_JIT_ALWAYS_ON
-void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
+int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
+s32 bpf_call_args_imm(s16 idx);
+#else
+static inline s32 bpf_call_args_imm(s16 idx)
+{
+ return 0;
+}
#endif
struct btf *bpf_get_btf_vmlinux(void);
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 1ec6d5ba64cc..88a241aac36a 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1151,9 +1151,6 @@ bool sk_filter_charge(struct sock *sk, struct sk_filter *fp);
void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp);
u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
-#define __bpf_call_base_args \
- ((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \
- (void *)__bpf_call_base)
struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog);
void bpf_jit_compile(struct bpf_prog *prog);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 8b018ff48875..6aa2a8b24030 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1771,6 +1771,9 @@ static u32 abs_s32(s32 x)
return x >= 0 ? (u32)x : -(u32)x;
}
+static u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5,
+ const struct bpf_insn *insn);
+
/**
* ___bpf_prog_run - run eBPF program on a given context
* @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers
@@ -2077,10 +2080,9 @@ select_insn:
CONT;
JMP_CALL_ARGS:
- BPF_R0 = (__bpf_call_base_args + insn->imm)(BPF_R1, BPF_R2,
- BPF_R3, BPF_R4,
- BPF_R5,
- insn + insn->off + 1);
+ BPF_R0 = interpreters_args[insn->off](BPF_R1, BPF_R2, BPF_R3,
+ BPF_R4, BPF_R5,
+ insn + insn->imm + 1);
CONT;
JMP_TAIL_CALL: {
@@ -2394,13 +2396,22 @@ EVAL4(PROG_NAME_LIST, 416, 448, 480, 512)
#undef PROG_NAME_LIST
#ifdef CONFIG_BPF_SYSCALL
-void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
+int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
{
stack_depth = max_t(u32, stack_depth, 1);
- insn->off = (s16) insn->imm;
- insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] -
- __bpf_call_base_args;
+ /* Prevent out-of-bounds read to interpreters_args */
+ if (stack_depth > MAX_BPF_STACK)
+ return -EINVAL;
+ insn->off = (round_up(stack_depth, 32) / 32) - 1;
insn->code = BPF_JMP | BPF_CALL_ARGS;
+ return 0;
+}
+
+s32 bpf_call_args_imm(s16 idx)
+{
+ if (WARN_ON_ONCE(idx < 0 || idx >= ARRAY_SIZE(interpreters_args)))
+ return 0;
+ return BPF_CALL_IMM(interpreters_args[idx]);
}
#endif
#endif
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index fba9e8c00878..3692adf62558 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -1250,9 +1250,9 @@ static int jit_subprogs(struct bpf_verifier_env *env)
}
if (!bpf_pseudo_call(insn))
continue;
- insn->off = env->insn_aux_data[i].call_imm;
- subprog = bpf_find_subprog(env, i + insn->off + 1);
- insn->imm = subprog;
+ insn->imm = env->insn_aux_data[i].call_imm;
+ subprog = bpf_find_subprog(env, i + insn->imm + 1);
+ insn->off = subprog;
}
prog->jited = 1;
@@ -1416,7 +1416,12 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env)
depth = get_callee_stack_depth(env, insn, i);
if (depth < 0)
return depth;
- bpf_patch_call_args(insn, depth);
+ err = bpf_patch_call_args(insn, depth);
+ if (err) {
+ verbose(env, "stack depth %d exceeds interpreter stack depth limit\n",
+ depth);
+ return err;
+ }
}
err = 0;
#endif
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index a3c0214ca934..630d530782fe 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -4919,6 +4919,29 @@ out:
return map;
}
+static void prepare_dump_pseudo_call(struct bpf_insn *insn)
+{
+ s32 call_off = insn->imm;
+
+ /*
+ * BPF_CALL_ARGS only exists for interpreter fallback.
+ * 1. For interpreter (BPF_CALL_ARGS): insn->off is the index of
+ * interpreters_args array, so here using bpf_call_args_imm()
+ * to get the real address offset.
+ * 2. For JIT (BPF_CALL): insn->off is the subprog id.
+ */
+ if (insn->code == (BPF_JMP | BPF_CALL_ARGS))
+ insn->imm = bpf_call_args_imm(insn->off);
+ else
+ insn->imm = insn->off;
+
+ /* Avoid dumping a truncated and misleading pc-relative offset. */
+ if (call_off > S16_MAX || call_off < S16_MIN)
+ insn->off = 0;
+ else
+ insn->off = call_off;
+}
+
static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
const struct cred *f_cred)
{
@@ -4944,6 +4967,9 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
}
if (code == (BPF_JMP | BPF_CALL) ||
code == (BPF_JMP | BPF_CALL_ARGS)) {
+ /* Restore the legacy xlated dump layout. */
+ if (insns[i].src_reg == BPF_PSEUDO_CALL)
+ prepare_dump_pseudo_call(&insns[i]);
if (code == (BPF_JMP | BPF_CALL_ARGS))
insns[i].code = BPF_JMP | BPF_CALL;
if (!bpf_dump_raw_ok(f_cred))
diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c
index a96b25ebff23..06cd24e37b3f 100644
--- a/tools/testing/selftests/bpf/prog_tests/verifier.c
+++ b/tools/testing/selftests/bpf/prog_tests/verifier.c
@@ -22,6 +22,7 @@
#include "verifier_bswap.skel.h"
#include "verifier_btf_ctx_access.skel.h"
#include "verifier_btf_unreliable_prog.skel.h"
+#include "verifier_call_large_imm.skel.h"
#include "verifier_cfg.skel.h"
#include "verifier_cgroup_inv_retcode.skel.h"
#include "verifier_cgroup_skb.skel.h"
@@ -170,6 +171,7 @@ void test_verifier_bpf_trap(void) { RUN(verifier_bpf_trap); }
void test_verifier_bswap(void) { RUN(verifier_bswap); }
void test_verifier_btf_ctx_access(void) { RUN(verifier_btf_ctx_access); }
void test_verifier_btf_unreliable_prog(void) { RUN(verifier_btf_unreliable_prog); }
+void test_verifier_call_large_imm(void) { RUN(verifier_call_large_imm); }
void test_verifier_cfg(void) { RUN(verifier_cfg); }
void test_verifier_cgroup_inv_retcode(void) { RUN(verifier_cgroup_inv_retcode); }
void test_verifier_cgroup_skb(void) { RUN(verifier_cgroup_skb); }
diff --git a/tools/testing/selftests/bpf/progs/verifier_call_large_imm.c b/tools/testing/selftests/bpf/progs/verifier_call_large_imm.c
new file mode 100644
index 000000000000..7998df07f6a6
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/verifier_call_large_imm.c
@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_misc.h"
+
+int call_happened = 0;
+
+/*
+ * 32765 is the exact minimum number of padding instructions needed to
+ * trigger the verifier failure, because:
+ * 1. Counting the wrapper instructions around the padding block (one
+ * "r0=0" and two "exit" instructions), the actual jump distance
+ * evaluates to N + 3.
+ * 2. To overflow the s16 max bound (32767), we need N + 3 > 32767.
+ * Thus, N = 32765 is the exact minimum padding size required.
+ */
+static __attribute__((noinline)) void padding_subprog(void)
+{
+ asm volatile (
+ "r0 = 0;"
+ ".rept 32765;"
+ "r0 += 0;"
+ ".endr;"
+ ::: __clobber_all);
+}
+
+static __attribute__((noinline)) int target_subprog(void)
+{
+ /* Use volatile variable here to prevent optimization. */
+ volatile int magic_ret = 3;
+ return magic_ret;
+}
+
+SEC("syscall")
+__success __retval(3)
+int call_large_imm_test(void *ctx)
+{
+ /*
+ * Landing pad to handle call error on kernel without the fix,
+ * preventing kernel panic.
+ */
+ asm volatile (
+ "r0 = 0;"
+ ".rept 32768;"
+ "r0 += 0;"
+ ".endr;"
+ ::: __clobber_all);
+
+ /*
+ * The call_happened variable is 1 only when the call insn wrongly
+ * go back to the landing pad above.
+ */
+ if (call_happened == 1) {
+ /* Use volatile variable here to prevent optimization. */
+ volatile int flag = -1;
+ return flag;
+ }
+
+ call_happened = 1;
+
+ padding_subprog();
+
+ return target_subprog();
+}
+
+char LICENSE[] SEC("license") = "GPL";
generated by cgit v1.2.3 (git 2.0.4) at 2026-05-31 14:00:06 +0000