diff options
| author | Puranjay Mohan <puranjay@kernel.org> | 2026-08-04 06:45:55 -0700 |
|---|---|---|
| committer | Andrii Nakryiko <andrii@kernel.org> | 2026-08-05 10:44:35 -0700 |
| commit | e93347704878aa8a54b3154114d13e57e8923e27 (patch) | |
| tree | c197e6039070f2ec6e03489dfc6571693ed4eb44 /kernel | |
| parent | f8f2b567d56035ddd62ec82f2c35dcee8c516624 (diff) | |
bpf: Inline bpf_iter_num_next() kfunc
bpf_iter_num_next() runs on every bpf_for() iteration, so inlining it
drops a call from the loop body. R1 points to the iterator; the returned
pointer to s->cur is R1 itself, since s->cur is first.
s->cur and s->end are int, so the kfunc's s->cur + 1 >= s->end is a
signed 32-bit compare and the inlined code needs no sign extension.
Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20260804134601.2305303-4-puranjay@kernel.org
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/bpf/verifier.c | 17 |
1 files changed, 17 insertions, 0 deletions
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8401077ed8fc..80c3cb654b89 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20030,6 +20030,23 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, -E2BIG); *cnt = i; + } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_next]) { + /* inline bpf_iter_num_next(&it); R1=&it, returns &s->cur or NULL */ + int i = 0; + + /* r0 = s->cur + 1; if ((s32)r0 >= s->end) goto done; */ + insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0); + insn_buf[i++] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_0, 1); + insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 4); + insn_buf[i++] = BPF_JMP32_REG(BPF_JSGE, BPF_REG_0, BPF_REG_2, 3); + /* s->cur = r0; return &s->cur; */ + insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0); + insn_buf[i++] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); + insn_buf[i++] = BPF_JMP_A(2); + /* done: s->cur = s->end = 0; return NULL; */ + insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0); + *cnt = i; } if (env->insn_aux_data[insn_idx].arg_prog) { |