summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/arm64/net/bpf_jit_comp.c6
-rw-r--r--arch/loongarch/net/bpf_jit.c6
-rw-r--r--arch/powerpc/net/bpf_jit_comp.c6
-rw-r--r--arch/riscv/net/bpf_jit_core.c6
-rw-r--r--arch/s390/net/bpf_jit_comp.c7
-rw-r--r--arch/x86/net/bpf_jit_comp.c14
-rw-r--r--include/linux/bpf.h2
-rw-r--r--include/linux/filter.h13
-rw-r--r--kernel/bpf/core.c57
-rw-r--r--kernel/bpf/fixups.c1
10 files changed, 101 insertions, 17 deletions
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 506c4201fec8..1d5f4cf88d33 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -2284,8 +2284,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
if (ctx.offset == NULL)
goto out_off;
- ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
- ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
+ if (bpf_prog_uses_arena_insns(prog)) {
+ ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
+ ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
+ }
out_cnt = bpf_out_stack_arg_cnt(env, prog);
if (out_cnt) {
diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index e78eb582c300..9393e4b0db8e 100644
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -2245,8 +2245,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
memset(&ctx, 0, sizeof(ctx));
ctx.prog = prog;
- ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
- ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
+ if (bpf_prog_uses_arena_insns(prog)) {
+ ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
+ ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
+ }
ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL);
if (ctx.offset == NULL)
diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index 7a612688e7a2..0d6ea6df0106 100644
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c
@@ -241,8 +241,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
/* Make sure that the stack is quadword aligned. */
cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
- cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena);
- cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena);
+ if (bpf_prog_uses_arena_insns(fp)) {
+ cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena);
+ cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena);
+ }
cgctx.is_subprog = bpf_is_subprog(fp);
cgctx.exception_boundary = fp->aux->exception_boundary;
cgctx.exception_cb = fp->aux->exception_cb;
diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c
index 5265dd5bd39e..f2497ec7d378 100644
--- a/arch/riscv/net/bpf_jit_core.c
+++ b/arch/riscv/net/bpf_jit_core.c
@@ -73,8 +73,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
goto skip_init_ctx;
}
- ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
- ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
+ if (bpf_prog_uses_arena_insns(prog)) {
+ ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
+ ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
+ }
stack_arg_cnt = bpf_out_stack_arg_cnt(env, prog);
/* First 3 stack args in regs, rest on stack */
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index 14c905697352..8eaaac7028a1 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -2304,10 +2304,11 @@ static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp,
else
jit->frame_off = 0;
- kern_arena = bpf_arena_get_kern_vm_start(fp->aux->arena);
- if (kern_arena)
+ if (bpf_prog_uses_arena_insns(fp)) {
+ kern_arena = bpf_arena_get_kern_vm_start(fp->aux->arena);
jit->kern_arena = _EMIT_CONST_U64(kern_arena);
- jit->user_arena = bpf_arena_get_user_vm_start(fp->aux->arena);
+ jit->user_arena = bpf_arena_get_user_vm_start(fp->aux->arena);
+ }
bpf_jit_prologue(jit, fp);
if (bpf_set_addr(jit, 0) < 0)
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index d3843045dc71..72a2a0a89e03 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -888,7 +888,7 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog,
pop_r12(&prog);
} else {
pop_callee_regs(&prog, callee_regs_used);
- if (bpf_arena_get_kern_vm_start(bpf_prog->aux->arena))
+ if (bpf_prog_uses_arena_insns(bpf_prog))
pop_r12(&prog);
}
@@ -954,7 +954,7 @@ static void emit_bpf_tail_call_direct(struct bpf_prog *bpf_prog,
pop_r12(&prog);
} else {
pop_callee_regs(&prog, callee_regs_used);
- if (bpf_arena_get_kern_vm_start(bpf_prog->aux->arena))
+ if (bpf_prog_uses_arena_insns(bpf_prog))
pop_r12(&prog);
}
@@ -2443,7 +2443,7 @@ static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog,
if (!(flags & BTF_FMODEL_ARENA_ARG))
continue;
- if (WARN_ON_ONCE(!bpf_prog->aux->arena))
+ if (WARN_ON_ONCE(!bpf_prog_uses_arena_insns(bpf_prog)))
return -EINVAL;
/* mov eN, eN: truncate and clear the upper 32 bits */
@@ -2522,7 +2522,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
void __percpu *priv_frame_ptr = NULL;
u16 out_stack_arg_cnt, outgoing_rsp;
- u64 arena_vm_start, user_vm_start;
+ u64 arena_vm_start = 0, user_vm_start = 0;
void __percpu *priv_stack_ptr;
int i, excnt = 0;
int ilen, proglen = 0;
@@ -2562,8 +2562,10 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
* [below] callee-saved regs
* [below] outgoing args 7+ (= rsp)
*/
- arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena);
- user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena);
+ if (bpf_prog_uses_arena_insns(bpf_prog)) {
+ arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena);
+ user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena);
+ }
detect_reg_usage(insn, insn_cnt, callee_regs_used);
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index dae33b1a7427..7750df973494 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1898,6 +1898,8 @@ struct bpf_prog_aux {
bool has_prologue;
bool might_sleep;
bool kprobe_write_ctx;
+ /* has insns that work with the arena base, see bpf_prog_has_arena_insns() */
+ bool uses_arena_insns;
struct {
s32 keyring_serial;
u8 keyring_type;
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 9339c6131f8f..6f4c9bd024a0 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1110,6 +1110,19 @@ static inline unsigned int bpf_prog_size(unsigned int proglen)
offsetof(struct bpf_prog, insns[proglen]));
}
+bool bpf_prog_has_arena_insns(const struct bpf_prog *prog);
+
+/*
+ * Whether a JIT has to set up the arena base for @prog. A program bound to
+ * an arena need not touch it, libbpf binds the arena of an object to every
+ * program in it. The flag caches bpf_prog_has_arena_insns() for the function
+ * about to be JITed, see bpf_prog_jit_compile() and jit_subprogs().
+ */
+static inline bool bpf_prog_uses_arena_insns(const struct bpf_prog *prog)
+{
+ return prog->aux->uses_arena_insns;
+}
+
static inline bool bpf_prog_was_classic(const struct bpf_prog *prog)
{
/* When classic BPF programs have been loaded and the arch
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 6e32db9777c2..fc22d66d194a 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -2637,11 +2637,68 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp)
return select_interpreter;
}
+/*
+ * Whether a function has an insn that works with the arena base, which a JIT
+ * sets up in the prologue when bpf_prog_uses_arena_insns() says so. Decided on
+ * the final insns of the function: an access through an arena pointer, or
+ * through a number in a program loaded with BPF_F_ARENA_SCALAR, has been
+ * rewritten to a BPF_PROBE_MEM32 form by then, a cast to the user address
+ * space keeps BPF_ADDR_SPACE_CAST, and a kfunc call with an __arena argument,
+ * which the JIT rebases, has a function model that says so. A load of an
+ * __arena struct member is rewritten with an immediate and needs none;
+ * dereferencing the result does.
+ */
+bool bpf_prog_has_arena_insns(const struct bpf_prog *prog)
+{
+ const struct bpf_insn *insn = prog->insnsi;
+ const struct btf_func_model *m;
+ int i, j;
+
+ if (!prog->aux->arena)
+ return false;
+
+ for (i = 0; i < prog->len; i++, insn++) {
+ switch (BPF_CLASS(insn->code)) {
+ case BPF_LDX:
+ if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 ||
+ BPF_MODE(insn->code) == BPF_PROBE_MEM32SX)
+ return true;
+ break;
+ case BPF_ST:
+ if (BPF_MODE(insn->code) == BPF_PROBE_MEM32)
+ return true;
+ break;
+ case BPF_STX:
+ if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 ||
+ BPF_MODE(insn->code) == BPF_PROBE_ATOMIC)
+ return true;
+ break;
+ case BPF_ALU64:
+ if (BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X &&
+ insn->off == BPF_ADDR_SPACE_CAST)
+ return true;
+ break;
+ case BPF_JMP:
+ if (!bpf_pseudo_kfunc_call(insn))
+ break;
+ m = bpf_jit_find_kfunc_model(prog, insn);
+ if (!m)
+ break;
+ for (j = 0; j < m->nr_args; j++)
+ if (m->arg_flags[j] & BTF_FMODEL_ARENA_ARG)
+ return true;
+ break;
+ }
+ }
+ return false;
+}
+
static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
#ifdef CONFIG_BPF_JIT
struct bpf_prog *orig_prog;
+ prog->aux->uses_arena_insns = bpf_prog_has_arena_insns(prog);
if (!bpf_prog_need_blind(prog))
return bpf_int_jit_compile(env, prog);
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 34b53ebb5841..409b9104c723 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -1448,6 +1448,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
func[i]->aux->jited_linfo = prog->aux->jited_linfo;
func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
func[i]->aux->arena = prog->aux->arena;
+ func[i]->aux->uses_arena_insns = bpf_prog_has_arena_insns(func[i]);
func[i]->aux->used_maps = env->used_maps;
func[i]->aux->used_map_cnt = env->used_map_cnt;
num_exentries = 0;
generated by cgit v1.2.3 (git 2.0.4) at 2026-10-31 22:01:04 +0000