diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2026-10-29 08:10:23 -1000 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2026-10-29 08:10:23 -1000 |
| commit | 46806af187f448b0d5481f284e7c900a57ee6a6d (patch) | |
| tree | 4a664c1e3f24c91ad6df8b5c5e3bf2fd10b4ee1d /tools/testing | |
| parent | d011f648a708b231f7ef28c18ff30c815b07fe56 (diff) | |
| parent | c8a8a68d3ab4a06c8f5a13a8c00275351c8239b4 (diff) | |
Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf
Pull bpf fixes from Alexei Starovoitov:
- Remove BPF_F_ARENA_SCALAR flag and .arena.data section in libbpf that
were added for rust-bpf in this merge window. We found a cleaner
approach that does it on compiler side instead of kernel/libbpf. No
released kernel has them (Alexei Starovoitov)
- Fix bpf_send_signal_task() returning -EPERM when it is called from a
kernel thread (Aditya Sharma)
- Various fixes (Alexei Starovoitov):
- Use emit_rsb_call() to call bpf_unwind_target() in x86 JIT to fix
objtool warning during the build
- Fix scalar evolution: clear delta of a widened register, don't
widen a pointer to a buffer, don't clamp a register whose type
changed in the loop, and reset the range of a clamped packet
pointer
- Finish the insn in the state that maybe_fork_scalars() splits
off, so that pruning doesn't drop it at the head of a counted
loop
- Preserve the registers that carry stack arguments across
arch_bpf_timed_may_goto() on x86-64, arm64 and riscv64
- Don't re-sort subprog_info[] for every new subprogram
- Fix flaky map_kptr and rcu_reentry selftests
- Use memmove() in bpf_skb_load_bytes(), bpf_skb_load_bytes_relative()
and bpf_xdp_load_bytes(), since the destination can be a slice of the
same packet (Ayushman Rout)
- Fixes and follow-ups for scalar evolution: reject pointer operands in
32-bit recurrences and latches, don't intersect with the current
register range on clamp to avoid a false 'verifier bug', avoid false
infinite loop reports, and recognize a 32-bit copy of the induction
variable (Eduard Zingerman)
- Fix out-of-bounds write to btf->types[] for split BTF and propagate
btf_add_type() errors in btf_check_all_metas() (Feng Yang)
- Fix use-after-free in bpf_timer callback that fires after its inner
map was freed (Mani Tofigh)
- Fix x86 JIT stopping before jumps converge: size backward jumps by
the insns they jump over (Mykyta Yatsenko)
- Fix use-after-free of struct_ops trampoline: dissociate struct_ops
progs before the grace period of the map (Qiliang Yuan)
- Validate stack argument reads against the caller state to fix
out-of-bounds read in the verifier, and reject callbacks with stack
arguments (Weiming Shi)
- Tie borrowed iterator results and pointers loaded from trusted fields
to the lifetime of their source, so that iterator next, destroy and
reference release invalidate them (Xu Yunxiang)
* tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: (48 commits)
bpf: Don't re-sort subprog_info[] for every new subprogram
bpf: Fix out-of-bounds btf->types write for split BTF
selftests/bpf: Test stack arguments across may_goto
bpf: Keep stack arguments in arch_bpf_timed_may_goto()
selftests/bpf: Test maybe_fork_scalars() at the head of a counted loop
bpf: Finish the insn in the state that maybe_fork_scalars() splits off
selftests/bpf: Test the clamp of pointers in scalar evolution
bpf: Reset the range of a packet pointer that scalar evolution clamps
bpf: Don't clamp a register whose type changed in the loop
selftests/bpf: Test a loop that moves a pointer to a buffer
bpf: Don't widen a pointer to a buffer in scalar evolution
selftests/bpf: Avoid a page fault in rcu_reentry while the handler is armed
selftests/bpf: Wait for the last maps of map_kptr to be freed
selftests/bpf: Test that a widened register is not linked
bpf: Clear delta of a register that scalar evolution widens
bpf, x86: Remove r0_on_stack from fetching atomics in arena
bpf: Remove BPF_F_ARENA_SCALAR
selftests/bpf: Remove tests for arena access through numbers
libbpf: Keep format strings of bpf_printk() in .rodata
libbpf: Remove .arena.data
...
Diffstat (limited to 'tools/testing')
35 files changed, 1520 insertions, 1779 deletions
diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 1e4af62e8312..c99e0f623e10 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -423,16 +423,6 @@ $(LIBARENA_ASAN_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BPF_DEPS) +$(MAKE) -C libarena libarena_asan.skel.h $(LIBARENA_MAKE_ARGS) endif -# #![no_std] looks for compiler_builtins too. Nothing of it is used. -ifneq ($(RUST_CORE),) -$(RUST_CORE): $(RUST_CORE_SRC) - $(call msg,RUSTC,,$@) - $(Q)mkdir -p $(@D) - +$(Q)$(RUSTC_BPF) -A warnings --edition 2024 --crate-name core --out-dir $(@D) $< - +$(Q)echo '#![feature(compiler_builtins)] #![compiler_builtins] #![no_std]' | \ - $(RUSTC_BPF) -A warnings --crate-name compiler_builtins --out-dir $(@D) - -endif - # Generated test list headers define gen_tests_hdr @@ -468,7 +458,7 @@ RUNNER_PREREQS := $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(BPFTOOL) \ $(VERIFY_SIG_HDR) $(PRIVATE_KEY) $(VERIFICATION_CERT) \ $(LIBARENA_SKEL) $(LIBARENA_ASAN_SKEL) \ prog_tests/tests.h map_tests/tests.h \ - $(RUNNER_OBJS) $(RUST_CORE) + $(RUNNER_OBJS) # Runtime fixtures for each test_progs flavor. RUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \ diff --git a/tools/testing/selftests/bpf/Makefile.buildvars b/tools/testing/selftests/bpf/Makefile.buildvars index ff3476bc40a4..d2a0c0031b87 100644 --- a/tools/testing/selftests/bpf/Makefile.buildvars +++ b/tools/testing/selftests/bpf/Makefile.buildvars @@ -109,31 +109,6 @@ HOST_INCLUDE_DIR := $(INCLUDE_DIR) endif RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids -# Programs in Rust are built by upstream rustc. It has no prebuilt core for -# the bpf target, so it has to come with the source of core: -# rustup component add rust-src -# core is built as edition 2024, which it is since rustc 1.87. -# rustc emits LLVM bitcode and clang makes the object of it, so clang has to be -# 23 or newer and not older than LLVM of rustc. -# Otherwise RUST_CORE is empty and the tests are skipped. -RUSTC ?= rustc -RUST_CORE_SRC := $(wildcard $(shell $(RUSTC) --print sysroot 2>/dev/null)$\ - /lib/rustlib/src/rust/library/core/src/lib.rs) -ifneq ($(RUST_CORE_SRC),) -ifeq ($(shell { clang=$$(echo __clang_major__ | $(CLANG) -E -P -x c -) && \ - llvm=$$($(srctree)/scripts/rustc-llvm-version.sh $(RUSTC)) && \ - [ $$($(srctree)/scripts/rustc-version.sh $(RUSTC)) -ge 108700 ] && \ - [ $$clang -ge 23 ] && [ $$clang -ge $$((llvm / 10000)) ]; } \ - 2>/dev/null && echo y),y) -RUST_CORE := $(BUILD_DIR)/rust/libcore.rlib -endif -endif -# RUSTC_BOOTSTRAP=1 is to build core with a stable rustc, like the kernel does. -# panic=abort is a stop gap until panic=unwind is supported. -RUSTC_BPF = RUSTC_BOOTSTRAP=1 $(RUSTC) -O -C panic=abort --crate-type rlib \ - --target $(if $(IS_LITTLE_ENDIAN),bpfel,bpfeb)-unknown-none \ - -L $(dir $(RUST_CORE)) - DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool ifneq ($(CROSS_COMPILE),) CROSS_BPFTOOL := $(SCRATCH_DIR)/sbin/bpftool diff --git a/tools/testing/selftests/bpf/Makefile.skel b/tools/testing/selftests/bpf/Makefile.skel index 6570c48f92b0..e8699903feef 100644 --- a/tools/testing/selftests/bpf/Makefile.skel +++ b/tools/testing/selftests/bpf/Makefile.skel @@ -20,8 +20,7 @@ ifneq ($(BPF_CC),) BPF_SRCS := $(notdir $(wildcard progs/*.c)) BPF_OBJS := $(patsubst %.c,$(RDIR)/%.bpf.o,$(BPF_SRCS)) -SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c \ - data_in_arena_extern.c data_in_arena_nomap.c +SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \ linked_vars.skel.h linked_maps.skel.h linked_arena.skel.h \ @@ -138,21 +137,6 @@ $(LINKED_SKELS_H): $(RDIR)/%.skel.h: $$(addprefix $(RDIR)/,$$($$*.skel.h-deps)) $(BPFTOOL) $(GEN_SKEL) | $(RDIR) $(Q)$(cmd_bpf_link_skel) -# Programs in Rust, see Makefile.buildvars. No skeletons: the objects may be absent. -ifneq ($(RUST_CORE),) -ifeq ($(BPF_CC),$(CLANG)) -RUST_OBJS := $(patsubst progs/%.rs,$(RDIR)/%.bpf.o,$(wildcard progs/*.rs)) -BPF_OBJS += $(RUST_OBJS) - -$(RUST_OBJS): $(RDIR)/%.bpf.o: progs/%.rs $(RUST_CORE) | $(RDIR) - $(call msg,RUSTC,$(BINARY),$@) - +$(Q)$(RUSTC_BPF) --edition 2021 -C debuginfo=2 --emit=llvm-bc \ - $(patsubst -mcpu=%,-C target-cpu=%,$(filter -mcpu=%,$(BPF_CC_FLAGS))) \ - -o $(dir $(RUST_CORE))$(BINARY)-$*.bc $< && \ - $(BPF_CC) $(BPF_CC_FLAGS) -c $(dir $(RUST_CORE))$(BINARY)-$*.bc -o $@ $(call skip_on_fail,BPF) -endif -endif - # Programs in C++. No skeletons: gcc doesn't build them. ifeq ($(BPF_CC),$(CLANG)) CXX_OBJS := $(patsubst progs/%.cpp,$(RDIR)/%.bpf.o,$(wildcard progs/*.cpp)) diff --git a/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c b/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c deleted file mode 100644 index 2ac2e9a652fe..000000000000 --- a/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include <test_progs.h> -#include "sysctl_helpers.h" -#include "verifier_arena_scalar.skel.h" - -/* The same tests with constants of the programs blinded */ -void serial_test_arena_scalar_blinded(void) -{ - const char *harden = "/proc/sys/net/core/bpf_jit_harden"; - char old[16] = {}; - - if (!is_jit_enabled()) { - test__skip(); - return; - } - if (sysctl_set_or_fail(harden, old, "2")) - return; - RUN_TESTS(verifier_arena_scalar); - sysctl_set_or_fail(harden, NULL, old); -} diff --git a/tools/testing/selftests/bpf/prog_tests/data_in_arena.c b/tools/testing/selftests/bpf/prog_tests/data_in_arena.c deleted file mode 100644 index cb1023507c01..000000000000 --- a/tools/testing/selftests/bpf/prog_tests/data_in_arena.c +++ /dev/null @@ -1,216 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include <test_progs.h> -#include "data_in_arena.skel.h" -#include "data_in_arena_decl.skel.h" -#include "data_in_arena_fail.skel.h" - -static int run_prog(struct bpf_program *prog) -{ - LIBBPF_OPTS(bpf_test_run_opts, topts); - - if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(prog), &topts), "test_run")) - return -1; - return topts.retval; -} - -static void run(struct data_in_arena *skel, int counter) -{ - int i; - - ASSERT_EQ(run_prog(skel->progs.use_data), counter + 7 + 1 + 2, "retval"); - ASSERT_EQ(skel->bss->sum, counter + 7 + 1 + 2, "sum"); - ASSERT_EQ(skel->data->counter, counter + 1, "counter"); - ASSERT_EQ(skel->data->pair[1], 5, "pair[1]"); - for (i = 0; i < 4; i++) - ASSERT_EQ(skel->bss->table[i], 10 * (i + 1) + i, "table"); -} - -static void test_in_arena(void) -{ - struct bpf_map_info info = {}; - struct data_in_arena *skel; - __u32 len = sizeof(info); - struct bpf_map *arena; - size_t sz; - - skel = data_in_arena__open(); - if (!ASSERT_OK_PTR(skel, "open")) - return; - - arena = bpf_object__find_map_by_name(skel->obj, "arena"); - if (!ASSERT_OK_PTR(arena, "arena")) - goto out; - ASSERT_EQ(bpf_map__type(arena), BPF_MAP_TYPE_ARENA, "arena type"); - ASSERT_EQ(bpf_map__max_entries(arena), 1, "arena pages"); - ASSERT_OK(bpf_map__set_max_entries(arena, 8), "arena resize"); - ASSERT_FALSE(bpf_map__autocreate(skel->maps.data), "data autocreate"); - ASSERT_FALSE(bpf_map__autocreate(skel->maps.bss), "bss autocreate"); - ASSERT_FALSE(bpf_map__autocreate(skel->maps.rodata), "rodata autocreate"); - ASSERT_EQ(bpf_map__set_autocreate(skel->maps.data, true), -EOPNOTSUPP, "set_autocreate"); - ASSERT_EQ(bpf_map__set_value_size(skel->maps.bss, 4096), -EOPNOTSUPP, "set_value_size"); - ASSERT_EQ(bpf_map__initial_value(skel->maps.data, &sz), skel->data, "initial_value"); - ASSERT_EQ(sz, sizeof(*skel->data), "initial_value size"); - - /* initial values are set the usual way */ - skel->data->counter = 100; - - if (!ASSERT_OK(data_in_arena__load(skel), "load")) - goto out; - /* there are no maps behind the sections */ - ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.data), &info, &len), "data map"); - ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.bss), &info, &len), "bss map"); - ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.rodata), &info, &len), - "rodata map"); - run(skel, 100); - - /* pointers to data next to pointers to functions */ - ASSERT_EQ(run_prog(skel->progs.use_ops), 42 + 1 + 'e', "use_ops"); - ASSERT_EQ(skel->data->counter, 102, "counter"); - - /* alignment of sections and pointers to data in data */ - ASSERT_EQ((unsigned long)&skel->bss->aligned64 % 64, 0, "alignment"); - ASSERT_EQ(run_prog(skel->progs.use_ptrs), 0, "use_ptrs"); - ASSERT_EQ(skel->data->x, 43, "x"); - ASSERT_EQ(skel->bss->aligned64.v[7], 7, "aligned64"); - ASSERT_EQ(skel->data->px, &skel->data->x, "px"); - - /* format strings of bpf_printk() and BPF_SNPRINTF() */ - ASSERT_EQ(run_prog(skel->progs.use_printk), sizeof("43-7"), "use_printk"); - ASSERT_STREQ(skel->bss->out, "43-7", "out"); -out: - data_in_arena__destroy(skel); -} - -/* The object has an arena map and __arena variables */ -static void test_declared_arena(void) -{ - struct data_in_arena_decl *skel; - struct bpf_map *map; - int arenas = 0; - - skel = data_in_arena_decl__open(); - if (!ASSERT_OK_PTR(skel, "open")) - return; - bpf_object__for_each_map(map, skel->obj) - arenas += bpf_map__type(map) == BPF_MAP_TYPE_ARENA; - ASSERT_EQ(arenas, 1, "no second arena"); - skel->data->counter = 6; - if (!ASSERT_OK(data_in_arena_decl__load(skel), "load")) - goto out; - ASSERT_EQ(run_prog(skel->progs.use_data), 6 + 7 + 11, "retval"); - ASSERT_EQ(run_prog(skel->progs.use_data), 7 + 7 + 12, "retval"); - ASSERT_EQ(skel->bss->sum, 7 + 7 + 12, "sum"); - ASSERT_EQ(skel->data->counter, 8, "counter"); -out: - data_in_arena_decl__destroy(skel); -} - -/* A pointer in data that can't be made an address of arena fails the load */ -static void test_ptr_to_map(void) -{ - struct data_in_arena_fail *skel; - - skel = data_in_arena_fail__open(); - if (!ASSERT_OK_PTR(skel, "open")) - return; - ASSERT_ERR(data_in_arena_fail__load(skel), "load"); - data_in_arena_fail__destroy(skel); -} - -/* So does a pointer to a variable of the kernel. There is no skeleton: the open fails. */ -static void test_ptr_to_extern(void) -{ - struct bpf_object *obj; - - obj = bpf_object__open_file("./data_in_arena_extern.bpf.o", NULL); - if (!ASSERT_ERR_PTR(obj, "open")) - bpf_object__close(obj); -} - -/* __arena variables and no arena map. There is no skeleton: the open fails. */ -static void test_arena_var_no_map(void) -{ - struct bpf_object *obj; - - obj = bpf_object__open_file("./data_in_arena_nomap.bpf.o", NULL); - if (!ASSERT_ERR_PTR(obj, "open")) - bpf_object__close(obj); -} - -/* The address of an arena that libbpf doesn't create is not known. No pointers in data then. */ -static void test_not_my_arena(bool pin) -{ - LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE); - struct data_in_arena *skel; - struct bpf_map *arena; - int fd = -1; - - skel = data_in_arena__open(); - if (!ASSERT_OK_PTR(skel, "open")) - return; - arena = bpf_object__find_map_by_name(skel->obj, "arena"); - if (!ASSERT_OK_PTR(arena, "arena")) - goto out; - if (pin) { - ASSERT_OK(bpf_map__set_pin_path(arena, "/sys/fs/bpf/data_in_arena"), "pin_path"); - } else { - fd = bpf_map_create(BPF_MAP_TYPE_ARENA, "arena", 0, 0, 1, &opts); - if (!ASSERT_GE(fd, 0, "map_create")) - goto out; - ASSERT_OK(bpf_map__reuse_fd(arena, fd), "reuse_fd"); - } - ASSERT_EQ(data_in_arena__load(skel), -ENOTSUP, "load"); -out: - if (fd >= 0) - close(fd); - data_in_arena__destroy(skel); -} - -/* The object is there if rustc and clang can build it, see Makefile.buildvars */ -static void test_rust(void) -{ - const char *file = "./data_in_arena_rust.bpf.o"; - struct bpf_object *obj; - - if (access(file, R_OK)) { - test__skip(); - return; - } - obj = bpf_object__open_file(file, NULL); - if (!ASSERT_OK_PTR(obj, "open")) - return; - if (!ASSERT_OK(bpf_object__load(obj), "load")) - goto out; - /* libbpf goes on without BTF when the kernel doesn't take it */ - ASSERT_GE(bpf_object__btf_fd(obj), 0, "btf_fd"); - ASSERT_EQ(run_prog(bpf_object__find_program_by_name(obj, "list_in_data")), - 100 + 20 + 3, "retval"); -out: - bpf_object__close(obj); -} - -void test_data_in_arena(void) -{ -#if !defined(__x86_64__) && !defined(__aarch64__) - /* other JITs don't take BPF_F_ARENA_SCALAR */ - test__skip(); - return; -#endif - if (test__start_subtest("arena")) - test_in_arena(); - if (test__start_subtest("declared_arena")) - test_declared_arena(); - if (test__start_subtest("ptr_to_map")) - test_ptr_to_map(); - if (test__start_subtest("ptr_to_extern")) - test_ptr_to_extern(); - if (test__start_subtest("arena_var_no_map")) - test_arena_var_no_map(); - if (test__start_subtest("pinned_arena")) - test_not_my_arena(true); - if (test__start_subtest("reused_arena")) - test_not_my_arena(false); - if (test__start_subtest("rust")) - test_rust(); -} diff --git a/tools/testing/selftests/bpf/prog_tests/jit_insns.c b/tools/testing/selftests/bpf/prog_tests/jit_insns.c index dabefdb8a353..52bffe27199b 100644 --- a/tools/testing/selftests/bpf/prog_tests/jit_insns.c +++ b/tools/testing/selftests/bpf/prog_tests/jit_insns.c @@ -2882,6 +2882,63 @@ static void test_convergence(void) } /* + * On x86-64 a backward jump sized from its own previous-pass address grew + * here while others shrank, so two passes gave the same image length with + * jumps still changing. The fillers are 7 and 2 bytes of native code, the + * conditional jumps are never taken. + */ +static void test_same_len_not_converged(void) +{ + struct bpf_insn insns[] = { + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, IN0), + BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_1, IN1), + BPF_JMP_IMM(BPF_JA, 0, 0, 1), + BPF_JMP_IMM(BPF_JA, 0, 0, 29), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 20), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, -3), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 22), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 2), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 23), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_JMP_IMM(BPF_JA, 0, 0, -22), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 3), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 2), + BPF_JMP_IMM(BPF_JA, 0, 0, -22), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 10), + BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, RES), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + int fd; + + fd = load_prog(insns, ARRAY_SIZE(insns), "same len not converged"); + if (fd < 0) + return; + ASSERT_GT(jited_len(fd), 0, "same len not converged jited"); + prog_run(fd, 1, 2, 11, "same len not converged"); + close(fd); +} + +/* * Calls into a subprogram with all five arguments, which clobbers a subset * of r6-r9 and uses the deepest stack slot the budget leaves, directly or * through another one clobbering the rest, so the JIT has to save and @@ -3087,6 +3144,8 @@ void test_jit_insns(void) test_max_jmp(); if (test__start_subtest("convergence")) test_convergence(); + if (test__start_subtest("same_len_not_converged")) + test_same_len_not_converged(); if (test__start_subtest("calls")) test_calls(); if (test__start_subtest("helper_calls")) diff --git a/tools/testing/selftests/bpf/prog_tests/map_kptr.c b/tools/testing/selftests/bpf/prog_tests/map_kptr.c index 17e707dddda8..f103f8cd458b 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_kptr.c +++ b/tools/testing/selftests/bpf/prog_tests/map_kptr.c @@ -225,6 +225,11 @@ void serial_test_map_kptr(void) /* Observe refcount dropping to 1 on map release. */ test_map_kptr_success(true); + + /* The next test counts the same references */ + ASSERT_OK(kern_sync_rcu_tasks_trace(skel), "sync rcu_tasks_trace"); + ASSERT_OK(kern_sync_rcu(), "sync rcu"); + wait_for_map_release(); } rcu_tasks_trace_gp__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/rcu_reentry.c b/tools/testing/selftests/bpf/prog_tests/rcu_reentry.c index de23a14b3d40..10b4ea913921 100644 --- a/tools/testing/selftests/bpf/prog_tests/rcu_reentry.c +++ b/tools/testing/selftests/bpf/prog_tests/rcu_reentry.c @@ -23,10 +23,17 @@ static bool have_tree_srcu(void) return kallsyms_find("srcu_expedite_current", &addr) == 0; } +/* Arm the handler for this thread, then trigger call_rcu_tasks_trace(). */ +static noinline int arm_and_delete(struct rcu_reentry *skel, int pid, int map_fd, int *pidfd) +{ + skel->bss->target_pid = pid; + return bpf_map_delete_elem(map_fd, pidfd); +} + void test_rcu_reentry(void) { struct rcu_reentry *skel; - int err, pidfd = -1, map_fd; + int err, pidfd = -1, bad_fd = -1, map_fd; cpu_set_t set, old_set; bool affinity_saved; __u64 val = 1; @@ -63,9 +70,13 @@ void test_rcu_reentry(void) if (!ASSERT_OK(err, "boot_create")) goto restore; - /* Arm the handler for this thread, then trigger call_rcu_tasks_trace(). */ - skel->bss->target_pid = syscall(__NR_gettid); - err = bpf_map_delete_elem(map_fd, &pidfd); + /* + * The handler fires at the first rcu_segcblist_enqueue() in this thread, and it deletes + * the element. A page fault can get there too, when the file system reads the page. + * Go through the same code first, without a pid, to bring its text and .bss in. + */ + arm_and_delete(skel, 0, map_fd, &bad_fd); + err = arm_and_delete(skel, syscall(__NR_gettid), map_fd, &pidfd); if (!ASSERT_OK(err, "boot_delete")) goto restore; diff --git a/tools/testing/selftests/bpf/prog_tests/send_signal.c b/tools/testing/selftests/bpf/prog_tests/send_signal.c index 7ac4d5a488aa..ecec2ee8b19a 100644 --- a/tools/testing/selftests/bpf/prog_tests/send_signal.c +++ b/tools/testing/selftests/bpf/prog_tests/send_signal.c @@ -261,6 +261,110 @@ static void test_send_signal_nmi(bool signal_thread, bool remote) test_send_signal_common(&attr, signal_thread, remote); } +static void test_send_signal_kworker(void) +{ + struct test_send_signal_kern *skel; + int pipe_c2p[2], pipe_p2c[2]; + struct sigaction sa = {}; + char buf[256]; + int err = -1; + pid_t pid; + + if (!ASSERT_OK(pipe(pipe_c2p), "pipe_c2p")) + return; + + if (!ASSERT_OK(pipe(pipe_p2c), "pipe_p2c")) { + close(pipe_c2p[0]); + close(pipe_c2p[1]); + return; + } + + pid = fork(); + if (!ASSERT_GE(pid, 0, "fork")) { + close(pipe_c2p[0]); + close(pipe_c2p[1]); + close(pipe_p2c[0]); + close(pipe_p2c[1]); + return; + } + + if (pid == 0) { + /* install signal handler and notify parent */ + sa.sa_sigaction = sigusr1_siginfo_handler; + sa.sa_flags = SA_RESTART | SA_SIGINFO; + ASSERT_NEQ(sigaction(SIGUSR1, &sa, NULL), -1, "sigaction"); + + close(pipe_c2p[0]); /* close read */ + close(pipe_p2c[1]); /* close write */ + + /* notify parent signal handler is installed */ + ASSERT_EQ(write(pipe_c2p[1], buf, 1), 1, "pipe_write"); + + /* make sure parent enabled bpf program to send_signal */ + ASSERT_EQ(read(pipe_p2c[0], buf, 1), 1, "pipe_read"); + + /* + * The signal is sent from workqueue context, so nothing has + * to be triggered from here. + */ + while (!sigusr1_received) + sleep(1); + + buf[0] = sigusr1_received; + + ASSERT_EQ(sigusr1_received, 8, "sigusr1_received"); + ASSERT_EQ(write(pipe_c2p[1], buf, 1), 1, "pipe_write"); + + close(pipe_c2p[1]); + close(pipe_p2c[0]); + exit(0); + } + + close(pipe_c2p[1]); /* close write */ + close(pipe_p2c[0]); /* close read */ + + skel = test_send_signal_kern__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + goto skel_open_load_failure; + + /* wait until child signal handler installed */ + ASSERT_EQ(read(pipe_c2p[0], buf, 1), 1, "pipe_read"); + + /* pid == 0 keeps the other programs of this skeleton inactive */ + skel->bss->kworker_test = 1; + skel->bss->pid = 0; + skel->bss->sig = SIGUSR1; + skel->bss->target_pid = pid; + + err = test_send_signal_kern__attach(skel); + if (!ASSERT_OK(err, "skel_attach")) { + err = -1; + goto destroy_skel; + } + + /* notify child that bpf program can send_signal now */ + ASSERT_EQ(write(pipe_p2c[1], buf, 1), 1, "pipe_write"); + + /* wait for result, workqueues run on their own */ + err = read_with_timeout(pipe_c2p[0], buf, 1, 10 * 1000 * 1000); + if (!ASSERT_GT(err, 0, "reading pipe")) + goto destroy_skel; + + ASSERT_EQ(buf[0], 8, "incorrect result"); + +destroy_skel: + test_send_signal_kern__destroy(skel); +skel_open_load_failure: + close(pipe_c2p[0]); + close(pipe_p2c[1]); + /* + * Child is either about to exit cleanly or stuck in case of errors. + * Nudge it to exit. + */ + kill(pid, SIGKILL); + wait(NULL); +} + void test_send_signal(void) { if (test__start_subtest("send_signal_tracepoint")) @@ -289,4 +393,6 @@ void test_send_signal(void) test_send_signal_perf(true, true); if (test__start_subtest("send_signal_nmi_thread_remote")) test_send_signal_nmi(true, true); + if (test__start_subtest("send_signal_kworker")) + test_send_signal_kworker(); } diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c index 461ded722351..9b646d134a45 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c @@ -136,7 +136,8 @@ out: struct_ops_assoc_in_timer__destroy(skel); } -static void test_st_ops_assoc_in_timer_no_uref(void) +static void run_st_ops_assoc_in_timer_no_uref(int timer_ns, int wait_map_free, + int spin_loops) { struct struct_ops_assoc_in_timer *skel = NULL; struct bpf_link *link; @@ -148,7 +149,8 @@ static void test_st_ops_assoc_in_timer_no_uref(void) err = bpf_program__assoc_struct_ops(skel->progs.syscall_prog, skel->maps.st_ops_map, NULL); - ASSERT_OK(err, "bpf_program__assoc_struct_ops"); + if (!ASSERT_OK(err, "bpf_program__assoc_struct_ops")) + goto out; link = bpf_map__attach_struct_ops(skel->maps.st_ops_map); if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) @@ -157,12 +159,18 @@ static void test_st_ops_assoc_in_timer_no_uref(void) /* * Run .test_1 by calling kfunc bpf_kfunc_multi_st_ops_test_1_prog_arg() and checks * the return value. .test_1 will also schedule timer_cb that runs .test_1 again. - * timer_cb will run 500ms after syscall_prog runs, when the user space no longer - * holds a reference to st_ops_map. + * timer_cb will run timer_ns after syscall_prog runs and, with wait_map_free, every + * timer_ns after that until the map is freed. With 500ms, it first runs when the + * user space no longer holds a reference to st_ops_map. */ - skel->bss->timer_ns = 500000000; + skel->bss->timer_ns = timer_ns; + skel->bss->wait_map_free = wait_map_free; + skel->bss->spin_loops = spin_loops; err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.syscall_prog), NULL); - ASSERT_OK(err, "bpf_prog_test_run_opts"); + if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) { + bpf_link__destroy(link); + goto out; + } /* Detach and close struct_ops map to cause it to be freed */ bpf_link__destroy(link); @@ -178,6 +186,21 @@ out: struct_ops_assoc_in_timer__destroy(skel); } +static void test_st_ops_assoc_in_timer_no_uref(void) +{ + run_st_ops_assoc_in_timer_no_uref(500000000, 1, 0); +} + +/* + * Keep calling .test_1 through the association from the timer callback, and + * stay in it for a while each time, while the map is being freed. The map + * must not be freed under a running .test_1. + */ +static void test_st_ops_assoc_in_timer_free(void) +{ + run_st_ops_assoc_in_timer_no_uref(0, 1, 1 << 20); +} + void test_struct_ops_assoc(void) { if (test__start_subtest("st_ops_assoc")) @@ -188,4 +211,6 @@ void test_struct_ops_assoc(void) test_st_ops_assoc_in_timer(); if (test__start_subtest("st_ops_assoc_in_timer_no_uref")) test_st_ops_assoc_in_timer_no_uref(); + if (test__start_subtest("st_ops_assoc_in_timer_free")) + test_st_ops_assoc_in_timer_free(); } diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 04740ba37e91..6a494c381266 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -13,7 +13,6 @@ #include "verifier_and.skel.h" #include "verifier_arena.skel.h" #include "verifier_arena_large.skel.h" -#include "verifier_arena_scalar.skel.h" #include "verifier_arena_globals1.skel.h" #include "verifier_arena_globals2.skel.h" #include "verifier_array_access.skel.h" @@ -207,7 +206,6 @@ void test_verifier_align(void) { RUN(verifier_align); } void test_verifier_and(void) { RUN(verifier_and); } void test_verifier_arena(void) { RUN(verifier_arena); } void test_verifier_arena_large(void) { RUN(verifier_arena_large); } -void test_verifier_arena_scalar(void) { RUN(verifier_arena_scalar); } void test_verifier_arena_globals1(void) { RUN(verifier_arena_globals1); } void test_verifier_arena_globals2(void) { RUN(verifier_arena_globals2); } void test_verifier_basic_stack(void) { RUN(verifier_basic_stack); } diff --git a/tools/testing/selftests/bpf/progs/arena_noarena.c b/tools/testing/selftests/bpf/progs/arena_noarena.c index edbd40cd5be8..1f46e5aa403b 100644 --- a/tools/testing/selftests/bpf/progs/arena_noarena.c +++ b/tools/testing/selftests/bpf/progs/arena_noarena.c @@ -21,19 +21,4 @@ int arena_arg_no_arena(void *ctx) return 0; } -SEC("syscall") -__flag(BPF_F_ARENA_SCALAR) -__arch_x86_64 -__arch_arm64 -__description("arena_scalar: no arena, no access through a number") -__failure __msg("R6 invalid mem access 'scalar'") -__naked void no_arena(void) -{ - asm volatile (" \ - r6 = 0x100000000000 ll; \ - r0 = *(u64 *)(r6 + 0); \ - exit; \ -" ::: __clobber_all); -} - char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c index a980a3a399d3..f4fb8f81afa7 100644 --- a/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c @@ -26,6 +26,11 @@ void subprog_bad_ptr_7args(long *a, int b, int c, int d, int e, int f, int g) { } +int callback_6args(long a, long b, long c, long d, long e, long f) +{ + return a + b + c + d + e + f; +} + #else int subprog_bad_order_6args(void) @@ -47,4 +52,9 @@ void subprog_bad_ptr_7args(void) { } +int callback_6args(void) +{ + return 0; +} + #endif diff --git a/tools/testing/selftests/bpf/progs/data_in_arena.c b/tools/testing/selftests/bpf/progs/data_in_arena.c deleted file mode 100644 index 18f38103c0fc..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena.c +++ /dev/null @@ -1,112 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include <linux/bpf.h> -#include <bpf/bpf_helpers.h> - -/* global data of the object is in arena */ -char data_in_arena SEC(".arena.data"); - -int counter = 5; -long pair[2] = { 1, 2 }; -int x = 42; -long sum; -long table[4]; -struct { - long v[8]; -} aligned64 __attribute__((aligned(64))); -const volatile int ro = 7; -const volatile long ro_table[4] = { 10, 20, 30, 40 }; - -/* const strings stay in a map for helpers and kfuncs, a copy of them is in arena */ -const char hello[] SEC(".rodata.str.hello") = "hello"; - -/* pointers to data that are stored in data */ -int *px = &x; -const char *str = hello; -int *const volatile cpx SEC(".data.rel.ro") = &x; - -SEC("syscall") -int use_data(void *ctx) -{ - int i; - - for (i = 0; i < 4; i++) - table[i] = ro_table[i] + i; - sum = counter + ro + pair[0] + pair[1]; - counter++; - __sync_fetch_and_add(&pair[1], 3); - return sum; -} - -typedef int (*op_fn)(int); - -static __noinline int add1(int v) -{ - return v + 1; -} - -/* - * Pointers to functions and to data in read-only data of a program with callx. - * Volatile, so that the compiler doesn't replace the pointers with what - * they point to. - */ -static const volatile struct { - op_fn fn; - int *data; - const char *name; -} ops SEC(".data.rel.ro") = { add1, &x, hello }; - -SEC("syscall") -int use_ops(void *ctx) -{ - /* a program has the arena when its code refers to it */ - counter++; -#ifdef __clang__ - return ops.fn(*ops.data) + ops.name[1]; -#else - /* gcc doesn't support indirect calls */ - return add1(*ops.data) + ops.name[1]; -#endif -} - -SEC("syscall") -int use_ptrs(void *ctx) -{ - unsigned long addr = (unsigned long)&aligned64; - char local[4] = "abc"; - - /* hide the address from the compiler, it knows that '& 63' is 0 */ - asm volatile ("" : "+r"(addr)); - if (addr & 63) - return 1; - aligned64.v[7] = 7; - if (*px != 42) - return 2; - *px = 43; - if (x != 43 || *cpx != 43) - return 3; - if (str[0] != 'h' || str[4] != 'o' || str[5]) - return 4; - /* the literal is in a map */ - if (bpf_strncmp(local, sizeof(local), "abc")) - return 5; - return 0; -} - -char out[16]; - -/* format strings are in a map */ -SEC("syscall") -int use_printk(void *ctx) -{ - char buf[sizeof(out)]; - int i, n; - - bpf_printk("counter %d", counter); - n = BPF_SNPRINTF(buf, sizeof(buf), "%d-%d", x, ro); - for (i = 0; i < sizeof(out); i++) - out[i] = buf[i]; - return n; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_decl.c b/tools/testing/selftests/bpf/progs/data_in_arena_decl.c deleted file mode 100644 index 01bc4fea8f0c..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena_decl.c +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#define BPF_NO_KFUNC_PROTOTYPES -#include <vmlinux.h> -#include <bpf/bpf_helpers.h> -#include "bpf_experimental.h" -#include <bpf_arena_common.h> - -struct { - __uint(type, BPF_MAP_TYPE_ARENA); - __uint(map_flags, BPF_F_MMAPABLE); - __uint(max_entries, 4); -} arena SEC(".maps"); - -/* global data of the object is in arena */ -char data_in_arena SEC(".arena.data"); - -int counter = 5; -long sum; -const volatile int ro = 7; - -#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) -int __arena avar = 11; -#else -int avar = 11; -#endif - -SEC("syscall") -int use_data(void *ctx) -{ - sum = counter + ro + avar; - counter++; - avar++; - return sum; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_extern.c b/tools/testing/selftests/bpf/progs/data_in_arena_extern.c deleted file mode 100644 index dc1ad5ca3bdd..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena_extern.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include <linux/bpf.h> -#include <bpf/bpf_helpers.h> - -/* global data of the object is in arena */ -char data_in_arena SEC(".arena.data"); - -extern const int bpf_prog_active __ksym; - -/* the variable of the kernel is not in arena */ -const void *kp = &bpf_prog_active; - -SEC("syscall") -int ptr_to_extern(void *ctx) -{ - return *(int *)kp; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_fail.c b/tools/testing/selftests/bpf/progs/data_in_arena_fail.c deleted file mode 100644 index ad8afe9afc96..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena_fail.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include <linux/bpf.h> -#include <bpf/bpf_helpers.h> - -/* global data of the object is in arena */ -char data_in_arena SEC(".arena.data"); - -int x = 42; -/* the table is read-only data with pointers. It's not in arena. */ -int *const volatile tbl[1] SEC(".data.rel.ro") = { &x }; -int *const volatile *pp = tbl; - -SEC("syscall") -int ptr_to_map(void *ctx) -{ - return **pp; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c b/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c deleted file mode 100644 index ccf66321eb36..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include <vmlinux.h> -#include <bpf/bpf_helpers.h> -#include "bpf_arena_common.h" - -/* global data of the object is in arena */ -char data_in_arena SEC(".arena.data"); - -int counter = 5; -/* needs an arena map that is declared. The one that libbpf creates won't do. */ -int __arena_global avar = 11; - -SEC("syscall") -int arena_var(void *ctx) -{ - return avar + counter; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs b/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs deleted file mode 100644 index 3dd9d4208232..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs +++ /dev/null @@ -1,73 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -// Why .data, .bss and .rodata of a program in Rust are in arena. -// -// A reference in Rust is an address. It doesn't say what it points to and it -// can be stored in data. The list below has a node in each of the sections. -// The nodes are linked by references that are in the data: -// IN_BSS.next is stored by the program, -// IN_DATA.next is a relocation in .data against .rodata. -// sum() loads the references back and reads the three nodes with the same insn. -// -// When the sections are array maps libbpf skips the relocation in .data, and -// what sum() loads from a node is a number that can't be dereferenced: -// R1 invalid mem access 'scalar' -// In arena the address of a node is a number to begin with. - -#![no_std] -#![no_main] - -// Tell libbpf to keep .data, .bss and .rodata in arena. -#[used] -#[link_section = ".arena.data"] -static DATA_IN_ARENA: u8 = 0; - -#[used] -#[link_section = "license"] -static LICENSE: [u8; 4] = *b"GPL\0"; - -// panic=abort is a stop gap until panic=unwind is supported. -// Nothing here panics, so the handler is not a part of the program. -#[panic_handler] -fn panic(_info: &core::panic::PanicInfo) -> ! { - loop {} -} - -pub struct Node { - val: u32, - next: Option<&'static Node>, -} - -// no_mangle makes them visible outside, so LLVM can't fold the list into a constant. -#[no_mangle] -static IN_RODATA: Node = Node { val: 3, next: None }; -#[no_mangle] -static mut IN_DATA: Node = Node { - val: 20, - next: Some(&IN_RODATA), -}; -#[no_mangle] -static mut IN_BSS: Node = Node { val: 0, next: None }; - -#[inline(never)] -fn sum(mut node: Option<&Node>) -> u32 { - let mut sum = 0; - // The verifier wants a bound. - for _ in 0..8 { - let Some(n) = node else { break }; - sum += n.val; - node = n.next; - } - sum -} - -#[no_mangle] -#[link_section = "syscall"] -pub extern "C" fn list_in_data(_ctx: *mut u8) -> u32 { - unsafe { - let head = &mut *&raw mut IN_BSS; - head.val = 100; - head.next = Some(&*&raw const IN_DATA); - sum(Some(head)) - } -} diff --git a/tools/testing/selftests/bpf/progs/iters_testmod.c b/tools/testing/selftests/bpf/progs/iters_testmod.c index 145750bb64ef..8896335668b4 100644 --- a/tools/testing/selftests/bpf/progs/iters_testmod.c +++ b/tools/testing/selftests/bpf/progs/iters_testmod.c @@ -8,6 +8,44 @@ char _license[] SEC("license") = "GPL"; +struct bpf_iter_testmod_current { + u64 __opaque; +}; + +SEC("raw_tp/sys_enter") +__success +int iter_independent_lifetime_after_next(const void *ctx) +{ + struct bpf_iter_testmod_current it; + struct task_struct *task; + + bpf_iter_testmod_current_new(&it); + task = bpf_iter_testmod_current_next(&it); + if (!task) + goto out; + + bpf_iter_testmod_current_next(&it); + bpf_kfunc_trusted_task_test(task); +out: + bpf_iter_testmod_current_destroy(&it); + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_independent_lifetime_after_destroy(const void *ctx) +{ + struct bpf_iter_testmod_current it; + struct task_struct *task; + + bpf_iter_testmod_current_new(&it); + task = bpf_iter_testmod_current_next(&it); + bpf_iter_testmod_current_destroy(&it); + if (task) + bpf_kfunc_trusted_task_test(task); + return 0; +} + SEC("raw_tp/sys_enter") __success int iter_next_trusted(const void *ctx) @@ -29,6 +67,298 @@ out: } SEC("raw_tp/sys_enter") +__failure __msg("invalid mem access 'scalar'") +int iter_next_trusted_after_destroy(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + bpf_iter_task_vma_destroy(&vma_it); + return vma_ptr->vm_start; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("invalid mem access 'scalar'") +int iter_next_trusted_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr, *next_vma_ptr; + u64 vm_start; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + next_vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!next_vma_ptr) + goto out; + + vm_start = vma_ptr->vm_start; + bpf_iter_task_vma_destroy(&vma_it); + return vm_start; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("invalid mem access 'scalar'") +int iter_next_trusted_after_next_null(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr, *next_vma_ptr; + u64 vm_start; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + next_vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (next_vma_ptr) + goto out; + + vm_start = vma_ptr->vm_start; + bpf_iter_task_vma_destroy(&vma_it); + return vm_start; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("invalid mem access 'scalar'") +int iter_next_trusted_field_after_destroy(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct file *file_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + file_ptr = vma_ptr->vm_file; + bpf_iter_task_vma_destroy(&vma_it); + if (file_ptr) + return file_ptr->f_mode; + return 0; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("invalid mem access 'scalar'") +int iter_next_trusted_field_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr, *next_vma_ptr; + struct file *file_ptr; + u32 mode; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + file_ptr = vma_ptr->vm_file; + if (!file_ptr) + goto out; + + next_vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!next_vma_ptr) + goto out; + + mode = file_ptr->f_mode; + bpf_iter_task_vma_destroy(&vma_it); + return mode; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("Leaking reference id=") +int iter_next_file_dynptr_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct bpf_dynptr dynptr; + struct file *file_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + file_ptr = vma_ptr->vm_file; + if (!file_ptr) + goto out; + + bpf_dynptr_from_file(file_ptr, 0, &dynptr); + bpf_iter_task_vma_next(&vma_it); + bpf_dynptr_file_discard(&dynptr); +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("Leaking reference id=") +int iter_next_file_dynptr_after_destroy(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct bpf_dynptr dynptr; + struct file *file_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + file_ptr = vma_ptr->vm_file; + if (!file_ptr) + goto out; + + bpf_dynptr_from_file(file_ptr, 0, &dynptr); + bpf_iter_task_vma_destroy(&vma_it); + bpf_dynptr_file_discard(&dynptr); + return 0; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_next_file_dynptr_discard_before_advance(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct bpf_dynptr dynptr; + struct file *file_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + file_ptr = vma_ptr->vm_file; + if (!file_ptr) + goto out; + + bpf_dynptr_from_file(file_ptr, 0, &dynptr); + bpf_dynptr_file_discard(&dynptr); + bpf_iter_task_vma_next(&vma_it); +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_next_trusted_current_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (vma_ptr) + bpf_kfunc_trusted_vma_test(vma_ptr); +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_next_trusted_rcu_field_after_destroy(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct mm_struct *mm_ptr; + struct file *file_ptr = NULL; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (vma_ptr) { + mm_ptr = vma_ptr->vm_mm; + /* exe_file has RCU protection independent of the iterator. */ + if (mm_ptr) + file_ptr = mm_ptr->exe_file; + } + + bpf_iter_task_vma_destroy(&vma_it); + if (file_ptr) + return file_ptr->f_mode; + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_next_trusted_rcu_field_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct mm_struct *mm_ptr; + struct file *file_ptr = NULL; + u32 mode = 0; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (vma_ptr) { + mm_ptr = vma_ptr->vm_mm; + if (mm_ptr) + file_ptr = mm_ptr->exe_file; + } + + bpf_iter_task_vma_next(&vma_it); + if (file_ptr) + mode = file_ptr->f_mode; + bpf_iter_task_vma_destroy(&vma_it); + return mode; +} + +SEC("raw_tp/sys_enter") __failure __msg("Possibly NULL pointer passed to trusted R1") int iter_next_trusted_or_null(const void *ctx) { @@ -78,6 +408,46 @@ out: } SEC("raw_tp/sys_enter") +__success +int iter_next_rcu_after_destroy(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task task_it; + struct task_struct *task_ptr; + + bpf_iter_task_new(&task_it, cur_task, 0); + + task_ptr = bpf_iter_task_next(&task_it); + bpf_iter_task_destroy(&task_it); + if (!task_ptr) + return 0; + + bpf_kfunc_rcu_task_test(task_ptr); + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_next_rcu_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task task_it; + struct task_struct *task_ptr; + + bpf_iter_task_new(&task_it, cur_task, 0); + + task_ptr = bpf_iter_task_next(&task_it); + if (!task_ptr) + goto out; + + bpf_iter_task_next(&task_it); + bpf_kfunc_rcu_task_test(task_ptr); +out: + bpf_iter_task_destroy(&task_it); + return 0; +} + +SEC("raw_tp/sys_enter") __failure __msg("Possibly NULL pointer passed to trusted R1") int iter_next_rcu_or_null(const void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/struct_ops_assoc_in_timer.c b/tools/testing/selftests/bpf/progs/struct_ops_assoc_in_timer.c index 0bed49e9f217..47a44c8e8b11 100644 --- a/tools/testing/selftests/bpf/progs/struct_ops_assoc_in_timer.c +++ b/tools/testing/selftests/bpf/progs/struct_ops_assoc_in_timer.c @@ -25,6 +25,13 @@ int test_err; int timer_ns; int timer_test_1_ret; int timer_cb_run; +int wait_map_free; +int spin_loops; + +static int spin(u32 i, void *ctx) +{ + return 0; +} __noinline static int timer_cb(void *map, int *key, struct bpf_timer *timer) { @@ -34,6 +41,12 @@ __noinline static int timer_cb(void *map, int *key, struct bpf_timer *timer) timer_test_1_ret = bpf_kfunc_multi_st_ops_test_1_assoc(&args); recur--; + /* Try again later until the map is freed */ + if (wait_map_free && timer_test_1_ret != -1) { + bpf_timer_start(timer, timer_ns, 0); + return 0; + } + timer_cb_run++; return 0; @@ -53,6 +66,9 @@ int BPF_PROG(test_1, struct st_ops_args *args) bpf_timer_init(timer, &array_map, 1); bpf_timer_set_callback(timer, timer_cb); bpf_timer_start(timer, timer_ns, 0); + } else if (spin_loops) { + /* Stay in the trampoline while the map may be freed */ + bpf_loop(spin_loops, spin, NULL, 0); } return MAP_MAGIC; diff --git a/tools/testing/selftests/bpf/progs/test_send_signal_kern.c b/tools/testing/selftests/bpf/progs/test_send_signal_kern.c index 176a355e3062..f9c4496d9fe1 100644 --- a/tools/testing/selftests/bpf/progs/test_send_signal_kern.c +++ b/tools/testing/selftests/bpf/progs/test_send_signal_kern.c @@ -9,6 +9,7 @@ void bpf_task_release(struct task_struct *p) __ksym; int bpf_send_signal_task(struct task_struct *task, int sig, enum pid_type type, u64 value) __ksym; __u32 sig = 0, pid = 0, status = 0, signal_thread = 0, target_pid = 0; +__u32 kworker_test = 0; static __always_inline int bpf_send_signal_test(void *ctx) { @@ -60,6 +61,34 @@ int send_signal_tp_sched(void *ctx) return bpf_send_signal_test(ctx); } +/* + * Send a signal to a task other than current, from a context where current + * is a kernel thread. No filtering on current is possible here, so this is + * driven by target_pid, and gated on kworker_test so that it stays inactive + * in the other tests. + */ +SEC("tp_btf/workqueue_execute_start") +int send_signal_kworker(void *ctx) +{ + struct task_struct *target_task; + int ret; + + if (status != 0 || !kworker_test || target_pid == 0) + return 0; + + target_task = bpf_task_from_pid(target_pid); + if (!target_task) + return 0; + + ret = bpf_send_signal_task(target_task, sig, PIDTYPE_TGID, 8); + bpf_task_release(target_task); + + if (ret == 0) + status = 1; + + return 0; +} + SEC("perf_event") int send_signal_perf(void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index 1ce9c4b78481..b17a6dfc100d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c @@ -611,6 +611,21 @@ int mixed_arena_scalar_alu64_arena_first(void *ctx) } SEC("syscall") +__description("arena: a number is not an address in arena") +__failure __msg("R6 invalid mem access 'scalar'") +__naked void access_through_number(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r6 = 0x100000000000 ll; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" : + : __imm_addr(arena) + : __clobber_all); +} + +SEC("syscall") __success __retval(0) int scalar_xor_arena_ptr(void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c deleted file mode 100644 index 6f0efd9202ca..000000000000 --- a/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c +++ /dev/null @@ -1,1155 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include <linux/bpf.h> -#include <bpf/bpf_helpers.h> -#include "../../../include/linux/filter.h" -#include "bpf_misc.h" - -void *bpf_arena_alloc_pages(void *map, void *addr, __u32 page_cnt, int node_id, - __u64 flags) __ksym; -void bpf_arena_free_pages(void *map, void *ptr, __u32 page_cnt) __ksym; - -#ifdef __TARGET_ARCH_arm64 -#define ARENA_VM_START (1ull << 32) -#else -#define ARENA_VM_START (1ull << 44) -#endif - -struct { - __uint(type, BPF_MAP_TYPE_ARENA); - __uint(map_flags, BPF_F_MMAPABLE); - __uint(max_entries, 4); - __ulong(map_extra, ARENA_VM_START); -} arena SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_HASH); - __uint(max_entries, 1); - __type(key, int); - __type(value, long long); -} hash SEC(".maps"); - -/* JITs that take BPF_F_ARENA_SCALAR */ -#define __arena_scalar __flag(BPF_F_ARENA_SCALAR) __arch_x86_64 __arch_arm64 - -/* BTF FUNC records are not generated for kfuncs referenced from inline assembly */ -void __kfunc_btf_root(void) -{ - bpf_arena_alloc_pages(0, 0, 0, 0, 0); - bpf_arena_free_pages(0, 0, 0); -} - -/* Tests start with r6 = address of a new page as the user space sees it, a number */ - -SEC("syscall") -__arena_scalar -__description("arena_scalar: load and store of every size") -__success __retval(0) -__load_if_JITed() -__naked void ld_st_sizes(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = r6; \ - r1 = 0x1122334455667788 ll; \ - *(u64 *)(r6 + 0) = r1; \ - *(u32 *)(r6 + 8) = r1; \ - *(u16 *)(r6 + 12) = r1; \ - *(u8 *)(r6 + 14) = r1; \ - *(u64 *)(r6 + 16) = 0x1234; \ - *(u32 *)(r6 + 24) = 0x5678; \ - *(u16 *)(r6 + 28) = 0x9a; \ - *(u8 *)(r6 + 30) = 0xbc; \ - r0 = 1; \ - r2 = *(u64 *)(r6 + 0); \ - if r2 != r1 goto 9f; \ - r0 = 2; \ - r2 = *(u32 *)(r6 + 8); \ - if r2 != 0x55667788 goto 9f; \ - r0 = 3; \ - r2 = *(u16 *)(r6 + 12); \ - if r2 != 0x7788 goto 9f; \ - r0 = 4; \ - r2 = *(u8 *)(r6 + 14); \ - if r2 != 0x88 goto 9f; \ - r0 = 5; \ - r2 = *(u64 *)(r6 + 16); \ - if r2 != 0x1234 goto 9f; \ - r0 = 6; \ - r2 = *(u32 *)(r6 + 24); \ - if r2 != 0x5678 goto 9f; \ - r0 = 7; \ - r2 = *(u16 *)(r6 + 28); \ - if r2 != 0x9a goto 9f; \ - r0 = 8; \ - r2 = *(u8 *)(r6 + 30); \ - if r2 != 0xbc goto 9f; \ - /* the address is what it was */ \ - r0 = 9; \ - if r6 != r7 goto 9f; \ - r0 = 10; \ - r7 >>= 32; \ - if r7 == 0 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: load into the register that holds the address") -__success __retval(0) -__load_if_JITed() -__naked void ld_into_base(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r1 = 77; \ - *(u64 *)(r6 + 0) = r1; \ - r6 = *(u64 *)(r6 + 0); \ - r0 = 1; \ - if r6 != 77 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: sign extending load") -__success __retval(0) -__load_if_JITed() -__naked void ldsx(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = r6; \ - *(u64 *)(r6 + 0) = 0x80; \ - .8byte %[ldsx_insn]; /* r2 = *(s8 *)(r6 + 0) */ \ - r0 = 1; \ - if r2 != -128 goto 9f; \ - r0 = 2; \ - if r6 != r7 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(ldsx_insn, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | BPF_B, - BPF_REG_2, BPF_REG_6, 0, 0)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: address loaded from arena") -__success __retval(0) -__load_if_JITed() -__naked void ptr_chase(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - /* page[0] = &page[64]; page[64] = 5; */ \ - r1 = r6; \ - r1 += 64; \ - *(u64 *)(r6 + 0) = r1; \ - *(u64 *)(r1 + 0) = 5; \ - r2 = *(u64 *)(r6 + 0); \ - r0 = 1; \ - if r2 != r1 goto 9f; \ - r3 = *(u64 *)(r2 + 0); \ - r0 = 2; \ - if r3 != 5 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: atomics") -__success __retval(0) -__load_if_JITed() -__naked void atomics(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = r6; \ - *(u64 *)(r6 + 8) = 1; \ - r1 = 2; \ - lock *(u64 *)(r6 + 8) += r1; \ - r1 = 4; \ - .8byte %[fetch_add_insn]; /* r1 = atomic_fetch_add((u64 *)(r6 + 8), r1) */ \ - r0 = 1; \ - if r1 != 3 goto 9f; \ - r1 = 8; \ - .8byte %[xchg_insn]; /* r1 = xchg_64(r6 + 8, r1) */ \ - r0 = 2; \ - if r1 != 7 goto 9f; \ - r0 = 8; \ - r1 = 16; \ - .8byte %[cmpxchg_insn]; /* r0 = cmpxchg_64(r6 + 8, r0, r1) */ \ - r2 = r0; \ - r0 = 3; \ - if r2 != 8 goto 9f; \ - r2 = *(u64 *)(r6 + 8); \ - r0 = 4; \ - if r2 != 16 goto 9f; \ - r0 = 5; \ - if r6 != r7 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(fetch_add_insn, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_6, BPF_REG_1, 8)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: fetching and/or/xor") -__success __retval(0) -__load_if_JITed() -__naked void fetch_and_or_xor(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = 0x11000000110 ll; \ - *(u64 *)(r6 + 8) = r7; \ - r0 = r6; \ - r0 ^= 0x55; \ - r1 = 0x1100000011 ll; \ - .8byte %[fetch_or_dw]; /* r1 = atomic_fetch_or((u64 *)(r6 + 8), r1) */ \ - r2 = r6; \ - r2 ^= 0x55; \ - if r0 == r2 goto 1f; \ - r0 = 99; \ - goto 9f; \ -1: r0 = 1; \ - if r1 != r7 goto 9f; \ - r7 = 0x11100000111 ll; \ - r1 = 0x11000000111 ll; \ - .8byte %[fetch_and_dw]; /* r1 = atomic_fetch_and((u64 *)(r6 + 8), r1) */ \ - r0 = 2; \ - if r1 != r7 goto 9f; \ - r7 = 0x11000000111 ll; \ - r1 = 0x101 ll; \ - .8byte %[fetch_xor_dw]; /* r1 = atomic_fetch_xor((u64 *)(r6 + 8), r1) */ \ - r0 = 3; \ - if r1 != r7 goto 9f; \ - r7 = 0x11000000010 ll; \ - r2 = *(u64 *)(r6 + 8); \ - r0 = 4; \ - if r2 != r7 goto 9f; \ - *(u32 *)(r6 + 16) = 0x110; \ - r1 = 0xdead000000000011 ll; \ - .8byte %[fetch_or_w]; /* r1 = atomic_fetch_or((u32 *)(r6 + 16), r1) */ \ - r0 = 5; \ - if r1 != 0x110 goto 9f; \ - r1 = 0xdead000000000101 ll; \ - .8byte %[fetch_and_w]; /* r1 = atomic_fetch_and((u32 *)(r6 + 16), r1) */ \ - r0 = 6; \ - if r1 != 0x111 goto 9f; \ - r1 = 0xdead000000000011 ll; \ - .8byte %[fetch_xor_w]; /* r1 = atomic_fetch_xor((u32 *)(r6 + 16), r1) */ \ - r0 = 7; \ - if r1 != 0x101 goto 9f; \ - r2 = *(u32 *)(r6 + 16); \ - r0 = 8; \ - if r2 != 0x110 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(fetch_and_dw, BPF_ATOMIC_OP(BPF_DW, BPF_AND | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(fetch_xor_dw, BPF_ATOMIC_OP(BPF_DW, BPF_XOR | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(fetch_or_w, BPF_ATOMIC_OP(BPF_W, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 16)), - __imm_insn(fetch_and_w, BPF_ATOMIC_OP(BPF_W, BPF_AND | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 16)), - __imm_insn(fetch_xor_w, BPF_ATOMIC_OP(BPF_W, BPF_XOR | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 16)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: fetching and/or/xor with the operand in r0") -__success __retval(0) -__load_if_JITed() -__naked void fetch_and_or_xor_r0(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = 0x11000000110 ll; \ - *(u64 *)(r6 + 8) = r7; \ - r0 = 0x1100000011 ll; \ - .8byte %[fetch_or_dw]; /* r0 = atomic_fetch_or((u64 *)(r6 + 8), r0) */ \ - r2 = r0; \ - r0 = 1; \ - if r2 != r7 goto 9f; \ - r7 = 0x11100000111 ll; \ - r0 = 0x11000000111 ll; \ - .8byte %[fetch_and_dw]; /* r0 = atomic_fetch_and((u64 *)(r6 + 8), r0) */ \ - r2 = r0; \ - r0 = 2; \ - if r2 != r7 goto 9f; \ - r7 = 0x11000000111 ll; \ - r0 = 0x101 ll; \ - .8byte %[fetch_xor_dw]; /* r0 = atomic_fetch_xor((u64 *)(r6 + 8), r0) */ \ - r2 = r0; \ - r0 = 3; \ - if r2 != r7 goto 9f; \ - r7 = 0x11000000010 ll; \ - r2 = *(u64 *)(r6 + 8); \ - r0 = 4; \ - if r2 != r7 goto 9f; \ - *(u32 *)(r6 + 16) = 0x110; \ - r0 = 0xdead000000000011 ll; \ - .8byte %[fetch_or_w]; /* r0 = atomic_fetch_or((u32 *)(r6 + 16), r0) */ \ - r2 = r0; \ - r0 = 5; \ - if r2 != 0x110 goto 9f; \ - r0 = 0xdead000000000101 ll; \ - .8byte %[fetch_and_w]; /* r0 = atomic_fetch_and((u32 *)(r6 + 16), r0) */ \ - r2 = r0; \ - r0 = 6; \ - if r2 != 0x111 goto 9f; \ - r0 = 0xdead000000000011 ll; \ - .8byte %[fetch_xor_w]; /* r0 = atomic_fetch_xor((u32 *)(r6 + 16), r0) */ \ - r2 = r0; \ - r0 = 7; \ - if r2 != 0x101 goto 9f; \ - r2 = *(u32 *)(r6 + 16); \ - r0 = 8; \ - if r2 != 0x110 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 8)), - __imm_insn(fetch_and_dw, BPF_ATOMIC_OP(BPF_DW, BPF_AND | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 8)), - __imm_insn(fetch_xor_dw, BPF_ATOMIC_OP(BPF_DW, BPF_XOR | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 8)), - __imm_insn(fetch_or_w, BPF_ATOMIC_OP(BPF_W, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 16)), - __imm_insn(fetch_and_w, BPF_ATOMIC_OP(BPF_W, BPF_AND | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 16)), - __imm_insn(fetch_xor_w, BPF_ATOMIC_OP(BPF_W, BPF_XOR | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 16)) - : __clobber_all); -} - -/* - * The page is gone, so the access faults. The fetched register has to come back - * 0 and everything else has to be left alone, r0 included. Reading the address - * again has to give 0 as well: had the fault been recovered by the arena mapping - * its zeroed scratch page instead of by the JIT's exception table entry, the - * access would have succeeded and left a 1 there. - */ -SEC("syscall") -__arena_scalar -__description("arena_scalar: fetching or through a number to a freed page") -__success __retval(0) -__load_if_JITed() -__naked void fetch_or_freed_page(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - *(u64 *)(r6 + 8) = 0x55; \ - r1 = %[arena] ll; \ - r2 = r6; \ - r3 = 1; \ - call %[bpf_arena_free_pages]; \ - r0 = r6; \ - r0 ^= 0x55; \ - r1 = 1; \ - .8byte %[fetch_or_dw]; /* faults: r1 = atomic_fetch_or((u64 *)(r6 + 8), r1) */ \ - r2 = r6; \ - r2 ^= 0x55; \ - if r0 == r2 goto 1f; \ - r0 = 1; \ - goto 9f; \ -1: r0 = 2; \ - if r1 != 0 goto 9f; \ - r3 = *(u64 *)(r6 + 8); \ - r0 = 3; \ - if r3 != 0 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm(bpf_arena_free_pages), - __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 8)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: fetching or with the operand in r0 to a freed page") -__success __retval(0) -__load_if_JITed() -__naked void fetch_or_r0_freed_page(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - *(u64 *)(r6 + 8) = 0x55; \ - r1 = %[arena] ll; \ - r2 = r6; \ - r3 = 1; \ - call %[bpf_arena_free_pages]; \ - r0 = 1; \ - .8byte %[fetch_or_dw]; /* faults: r0 = atomic_fetch_or((u64 *)(r6 + 8), r0) */ \ - r2 = r0; \ - r0 = 1; \ - if r2 != 0 goto 9f; \ - r3 = *(u64 *)(r6 + 8); \ - r0 = 2; \ - if r3 != 0 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm(bpf_arena_free_pages), - __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 8)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: cmpxchg through r0") -__success __retval(0) -__load_if_JITed() -__naked void cmpxchg_r0(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - /* The address is in r0. The value at the address is not equal to it. */ \ - *(u64 *)(r6 + 0) = 3; \ - r0 = r6; \ - r1 = 5; \ - .8byte %[cmpxchg_insn]; /* r0 = cmpxchg_64(r0 + 0, r0, r1) */ \ - r2 = r0; \ - r0 = 1; \ - if r2 != 3 goto 9f; \ - r2 = *(u64 *)(r6 + 0); \ - r0 = 2; \ - if r2 != 3 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_0, BPF_REG_1, 0)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: xchg into the register that holds the address") -__success __retval(0) -__load_if_JITed() -__naked void xchg_into_base(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - *(u64 *)(r6 + 0) = 3; \ - r1 = r6; \ - .8byte %[xchg_insn]; /* r1 = xchg_64(r1 + 0, r1) */ \ - r0 = 1; \ - if r1 != 3 goto 9f; \ - r2 = *(u64 *)(r6 + 0); \ - r0 = 2; \ - if r2 != r6 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_1, 0)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: xchg into the register that holds a pointer to stack") -__failure __msg("misaligned access off (0x0; 0xffffffffffffffff)+0 size 8") -__naked void xchg_into_stack_ptr(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r1 = 0; \ - *(u64 *)(r10 - 8) = r1; \ - r1 = r10; \ - r1 += -8; \ - .8byte %[xchg_insn]; /* r1 = xchg_64(r1 + 0, r1) */ \ - r0 = 0; \ - exit; \ -" : - : __imm_addr(arena), - __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_1, 0)) - : __clobber_all); -} - -#ifdef CAN_USE_LOAD_ACQ_STORE_REL - -SEC("syscall") -__arena_scalar -__description("arena_scalar: load-acquire and store-release") -__success __retval(0) -__load_if_JITed() -__naked void load_acq_store_rel(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = r6; \ - r1 = 0x1234; \ - .8byte %[store_release_insn]; /* store_release((u64 *)(r6 + 8), r1) */ \ - .8byte %[load_acquire_insn]; /* r2 = load_acquire((u64 *)(r6 + 8)) */ \ - r0 = 1; \ - if r2 != 0x1234 goto 9f; \ - .8byte %[load_acquire8_insn]; /* w2 = load_acquire((u8 *)(r6 + 8)) */ \ - r0 = 2; \ - if r2 != 0x34 goto 9f; \ - r0 = 3; \ - if r6 != r7 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(store_release_insn, - BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(load_acquire_insn, - BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_6, 8)), - __imm_insn(load_acquire8_insn, - BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_6, 8)) - : __clobber_all); -} - -#endif /* CAN_USE_LOAD_ACQ_STORE_REL */ - -SEC("syscall") -__arena_scalar -__description("arena_scalar: number that is not an address in arena") -__success __retval(0) -__load_if_JITed() -__naked void not_in_arena(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - /* nothing is allocated: all loads read 0, stores are dropped */ \ - r6 = 0xdeadbeef00000000 ll; \ - r0 = 1; \ - r2 = *(u64 *)(r6 + 0); \ - if r2 != 0 goto 9f; \ - r0 = 2; \ - r2 = *(u8 *)(r6 - 32768); \ - if r2 != 0 goto 9f; \ - r6 = 0x12345678ffffffff ll; \ - r0 = 3; \ - r2 = *(u64 *)(r6 + 32760); \ - if r2 != 0 goto 9f; \ - *(u64 *)(r6 + 32760) = 1; \ - *(u8 *)(r6 + 32767) = r2; \ - r1 = 1; \ - lock *(u64 *)(r6 + 32760) += r1; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: store through the address of the stack as a number") -__success __retval(0) -__load_if_JITed() -__naked void stack_addr_as_number(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - *(u64 *)(r10 - 8) = 5; \ - r6 = r10; \ - r6 |= 0; \ - /* r6 is a number now. The store goes to arena, not to the stack. */ \ - *(u64 *)(r6 - 8) = 7; \ - r1 = 9; \ - *(u64 *)(r6 - 8) = r1; \ - lock *(u64 *)(r6 - 8) += r1; \ - r2 = *(u64 *)(r10 - 8); \ - r0 = 1; \ - if r2 != 5 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: 64-bit math on the address stays 64-bit") -__success __retval(0) -__load_if_JITed() -__naked void alu64_after_access(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r2 = *(u64 *)(r6 + 0); \ - r7 = r6; \ - r7 += 8; \ - r7 -= r6; \ - r0 = 1; \ - if r7 != 8 goto 9f; \ - r7 = r6; \ - r7 >>= 32; \ - r0 = 2; \ - if r7 == 0 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: st of an immediate changes no register") -__success __retval(0) -__load_if_JITed() -__naked void st_keeps_regs(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r0 = r6; \ - r1 = 0x1111; \ - r2 = 0x2222; \ - *(u64 *)(r0 + 0) = 5; \ - r3 = r0; \ - r0 = 1; \ - if r3 != r6 goto 9f; \ - r0 = 2; \ - if r1 != 0x1111 goto 9f; \ - r0 = 3; \ - if r2 != 0x2222 goto 9f; \ - r0 = 0x3333; \ - r1 = r6; \ - *(u32 *)(r1 + 8) = -7; \ - r3 = r0; \ - r0 = 4; \ - if r3 != 0x3333 goto 9f; \ - r0 = 5; \ - if r1 != r6 goto 9f; \ - r0 = 6; \ - r3 = *(u64 *)(r6 + 0); \ - if r3 != 5 goto 9f; \ - r0 = 7; \ - r3 = *(u64 *)(r6 + 8); \ - r4 = 0xfffffff9 ll; \ - if r3 != r4 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: access through a number with garbage in the upper half") -__success __retval(0) -__load_if_JITed() -__naked void st_value_garbage(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r1 = 0xdeadbeef00001000 ll; \ - *(u64 *)(r6 + 2048) = r1; \ - r7 = *(u64 *)(r6 + 2048); \ - r8 = *(u64 *)(r6 + 2048); \ - r1 = 3; \ - *(u64 *)(r8 + 0) = 1; \ - r0 = 1; \ - if r8 != r7 goto 9f; \ - *(u8 *)(r8 + 1) = 1; \ - r0 = 2; \ - if r8 != r7 goto 9f; \ - *(u32 *)(r8 + 4) = r1; \ - r0 = 3; \ - if r8 != r7 goto 9f; \ - r2 = *(u16 *)(r8 + 2); \ - r0 = 4; \ - if r8 != r7 goto 9f; \ - r0 = 5; \ - if r1 != 3 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: access through a number without the upper half") -__success __retval(0) -__load_if_JITed() -__naked void st_value_small(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r1 = 0x1000 ll; \ - *(u64 *)(r6 + 2048) = r1; \ - r7 = *(u64 *)(r6 + 2048); \ - r8 = *(u64 *)(r6 + 2048); \ - r1 = 3; \ - *(u64 *)(r8 + 0) = 1; \ - r0 = 1; \ - if r8 != r7 goto 9f; \ - *(u8 *)(r8 + 1) = 1; \ - r0 = 2; \ - if r8 != r7 goto 9f; \ - *(u32 *)(r8 + 4) = r1; \ - r0 = 3; \ - if r8 != r7 goto 9f; \ - r2 = *(u16 *)(r8 + 2); \ - r0 = 4; \ - if r8 != r7 goto 9f; \ - r0 = 5; \ - if r1 != 3 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: atomics through a number that is not an address in arena") -__success __retval(0) -__load_if_JITed() -__naked void atomic_value(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r1 = 0xdeadbeef00001000 ll; \ - *(u64 *)(r6 + 2048) = r1; \ - r7 = *(u64 *)(r6 + 2048); \ - r8 = *(u64 *)(r6 + 2048); \ - r1 = 3; \ - lock *(u64 *)(r8 + 0) += r1; \ - r0 = 1; \ - if r8 != r7 goto 9f; \ - .8byte %[fetch_add_insn]; \ - r0 = 2; \ - if r8 != r7 goto 9f; \ - .8byte %[xchg_insn]; \ - r0 = 3; \ - if r8 != r7 goto 9f; \ - r0 = 0; \ - .8byte %[cmpxchg_insn]; \ - r0 = 4; \ - if r8 != r7 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(fetch_add_insn, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, - BPF_REG_8, BPF_REG_1, 0)), - __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_8, BPF_REG_1, 0)), - __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_8, BPF_REG_1, 0)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: number and pointer to arena at the same insn") -__success __retval(0) -__load_if_JITed() -__naked void mixed_number_arena(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - *(u64 *)(r6 + 8) = 0x1234; \ - /* a number that the verifier does not know, 0 at run time */ \ - r7 = *(u64 *)(r6 + 16); \ - r8 = r6; \ - if r7 != 0 goto 1f; \ - .8byte %[cast_kern_insn]; \ -1: *(u8 *)(r8 + 0) = 1; \ - *(u8 *)(r8 + 1) = r7; \ - r2 = *(u8 *)(r8 + 1); \ - lock *(u64 *)(r8 + 24) += r7; \ - r0 = 0; \ - if r7 != 0 goto 9f; \ - /* pointer to arena only: JIT adds all 64 bits of r8 to the base */ \ - r0 = 2; \ - r2 = *(u64 *)(r8 + 8); \ - if r2 != 0x1234 goto 9f; \ - r0 = 3; \ - r2 = *(u8 *)(r6 + 0); \ - if r2 != 1 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(cast_kern_insn, BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, - BPF_REG_8, BPF_REG_8, 1, 1)) - : __clobber_all); -} - -SEC("syscall") -__description("arena_scalar: no flag, no access through a number") -__failure __msg("R6 invalid mem access 'scalar'") -__naked void no_flag(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r6 = 0x100000000000 ll; \ - r0 = *(u64 *)(r6 + 0); \ - exit; \ -" : - : __imm_addr(arena) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: pointer that is NULL is an address in arena") -__success __retval(0) -__load_if_JITed() -__naked void null_ptr(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r1 = 0; \ - *(u32 *)(r10 - 4) = r1; \ - r2 = r10; \ - r2 += -4; \ - r1 = %[hash] ll; \ - call %[bpf_map_lookup_elem]; \ - r1 = r0; \ - r0 = 1; \ - if r1 != 0 goto 9f; \ - /* nothing is allocated: the load reads 0 */ \ - r0 = *(u64 *)(r1 + 0); \ -9: exit; \ -" : - : __imm_addr(arena), - __imm_addr(hash), - __imm(bpf_map_lookup_elem) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: pointer that is NULL with an offset is an address in arena") -__success __retval(0) -__load_if_JITed() -__naked void null_ptr_off(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r1 = 0; \ - *(u32 *)(r10 - 4) = r1; \ - r2 = r10; \ - r2 += -4; \ - r1 = %[hash] ll; \ - call %[bpf_map_lookup_elem]; \ - r1 = r0; \ - r0 = 1; \ - if r1 != 0 goto 9f; \ - r1 += 8; \ - *(u64 *)(r1 + 0) = 5; \ - r0 = *(u64 *)(r1 + 0); \ -9: exit; \ -" : - : __imm_addr(arena), - __imm_addr(hash), - __imm(bpf_map_lookup_elem) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: number that is less than a page is an address in arena") -__success __retval(0) -__load_if_JITed() -__naked void small_number(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - call %[bpf_get_prandom_u32]; \ - r1 = r0; \ - r1 &= 0xfff; \ - r0 = *(u64 *)(r1 + 0); \ - exit; \ -" : - : __imm_addr(arena), - __imm(bpf_get_prandom_u32) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: number is not a pointer for a helper") -__failure __msg("R2 type=scalar expected=") -__naked void helper_arg(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r1 = %[hash] ll; \ - r2 = r6; \ - call %[bpf_map_lookup_elem]; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm_addr(hash), - __imm(bpf_arena_alloc_pages), - __imm(bpf_map_lookup_elem) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: number and pointer to stack at the same insn") -__failure __msg("same insn cannot be used with different pointers") -__load_if_JITed() -__naked void mixed_number_stack(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - *(u64 *)(r10 - 8) = 0; \ - call %[bpf_get_prandom_u32]; \ - if w0 != 0 goto 1f; \ - r6 = r10; \ - r6 += -8; \ -1: r0 = *(u64 *)(r6 + 0); \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm(bpf_get_prandom_u32) - : __clobber_all); -} - -static int st_cb(__u64 idx, void *ctx) -{ - volatile long *p = *(volatile long **)ctx; - - p[idx] = 7; - return 0; -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: store through a number in a callback") -__success __retval(0) -__load_if_JITed() -int st_in_callback(void *unused) -{ - volatile long *p = bpf_arena_alloc_pages(&arena, NULL, 1, -1, 0); - - if (!p) - return 100; - bpf_loop(4, st_cb, &p, 0); - return p[0] + p[3] + p[4] - 14; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c index df8d5309657e..05efa9f4a522 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c @@ -2160,6 +2160,47 @@ l0_%=: r0 = 0; \ } /* + * The insn that forks is the head of a loop that scalar evolution counted. The state with + * [-1, 0] is kept there, and the state with 0 that is split off is within it. + */ +SEC("socket") +__description("maybe_fork_scalars: AND at the head of a counted loop rejects OOB") +__failure __msg("invalid access to map value") +__naked void and_scalar_fork_at_long_loop_head_rejects_oob(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64*)(r10 - 8) = r1; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map_hash_8b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r9 = r0; \ + r7 = 0; \ + r6 = -1; \ +l1_%=: r6 &= 8; \ + /* r6 is 8 (current) or 0 (pushed) */ \ + r6 ^= 8; \ + /* pushed path: r6 = 8, OOB for value_size=8 */ \ + r1 = r9; \ + r1 += r6; \ + r0 = *(u8*)(r1 + 0); \ + r1 = *(u64*)(r9 + 0); \ + r6 = -1; \ + if r1 == 0 goto l2_%=; \ + r6 = 0; \ +l2_%=: r7 += 1; \ + if r7 < 100 goto l1_%=; \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b) + : __clobber_all); +} + +/* * Last jump can be detected as always taken because the intersection of R5 and * R7 32bit tnums produces a constant that isn't within R7's s32 bounds. */ diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 661ce9ca8f93..fa01a616af9d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -1377,6 +1377,40 @@ __naked __noinline __used static int probe_read_stack(void) : __clobber_all); } +/* + * r3 = r8 + 4 is linked to r8 when it enters the loop. Widening drops the link. If + * it leaves the 4 behind then r4 = r3 + 1 is taken for r3 - 3 when r3 is compared: + * r4 <= 7 instead of r4 <= 11, and the read of r9 looks like dead code. + */ +SEC("socket") +__failure __msg("R9 !read_ok") +__naked void widened_reg_is_not_linked(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r8 = r0;" + "r8 &= 0xff;" + "r3 = r8;" + "r3 += 4;" + "r1 = 0;" +"1:" + "r4 = r3;" + "r4 += 1;" + "if r3 > 10 goto 3f;" + "if r4 <= 7 goto 2f;" + "r0 = *(u64 *)(r9 + 0);" +"2:" + "r3 += 1;" + "r1 += 1;" + "if r1 < 2 goto 1b;" +"3:" + "r0 = r8;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + /* The unknown SCEV of an unbound STACK_MISC slot must not prevent widening. */ SEC("tracepoint") __success @@ -1592,7 +1626,7 @@ __msg_next(" r0=(+ r0 -1) / (linear (+ r0 -1) -1)") __msg("loop header at 1, header_count is 3 ") __msg("loop header at 1, widening r0 to 1..3 step 1") __msg("1: R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,var_off=(0x0; 0x3)) loop_stack=1") -__msg("loop header at 1, clamping r0 to 1..2 (s32 1..2) step 1") +__msg("loop header at 1, clamping r0 to 1..3 step 1") __msg("processed 5 insns") __naked void post_cond_jne_neg_step(void) { @@ -1604,6 +1638,61 @@ __naked void post_cond_jne_neg_step(void) " ::: __clobber_all); } +/* + * r1 and r3 are widened independently, so the r3 == 5 path reaches the back + * edge with r3 past the last value it takes at the header. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, header_count is 5 ") +__msg("loop header at 2, widening r1 to 0..4 step 1") +__msg("loop header at 2, widening r3 to 0..4 step 1") +__msg("3: (55) if r3 != 0x5 goto pc+1 {{.*}}; R3=5") +__msg("loop header at 2, clamping r3 to 0..4 step 1") +__msg("from 6 to 2: safe") +__naked void clamp_dependent_overshoot(void) +{ + asm volatile (" \ + r1 = 0; \ + r3 = 0; \ +1: r3 += 1; \ + if r3 != 5 goto 2f; \ + r2 = 0; \ +2: r1 += 1; \ + if r1 != 5 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* Same with a 32-bit dependent register, SCEV (zext32 (linear r3 1)) */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, header_count is 5 ") +__msg("loop header at 2, widening r1 to 0..4 step 1") +__msg("loop header at 2, widening r3 to 0..4 step 1") +__msg("3: (56) if w3 != 0x5 goto pc+1 {{.*}}; R3=5") +__msg("loop header at 2, clamping r3 to 0..4 step 1") +__msg("from 6 to 2: safe") +__naked void clamp_dependent_overshoot32(void) +{ + asm volatile (" \ + r1 = 0; \ + w3 = 0; \ +1: w3 += 1; \ + if w3 != 5 goto 2f; \ + r2 = 0; \ +2: r1 += 1; \ + if r1 != 5 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + SEC("xdp") __success __log_level(2) @@ -1771,7 +1860,7 @@ __success __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) __msg("loop header at 1, header_count is 4") -__msg("loop header at 1, widening r0 to 0..3 (s32 0..3) step 1") +__msg("loop header at 1, widening r0 to 0..3 step 1") __msg("loop header at 1, clamping r0") __msg("4: R0=3") __naked void pre_cond_je32(void) @@ -2512,6 +2601,104 @@ __naked void no_widen_any_stack_origin(void) " ::: __clobber_all); } +__naked __noinline __used static int scrub_slot_callee(void) +{ + asm volatile (" \ + *(u32 *)(r1 + 4) = 1; \ + call %[bpf_get_prandom_u32]; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * The callee turns the caller's fp-8 into misc, so the states at the call site + * differ between the first and the second iteration, while the states inside + * the callee are identical. Verification fails with an infinite loop error if + * states.c:current_loop() does not search upper frames for active loops. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 3, widening r6 to 0..9 step 1") +__msg("loop header at 3, clamping r6 to 0..9 step 1") +__msg("12: safe") +__naked void clamp_callee_state_pruning(void) +{ + asm volatile (" \ + w1 = 1; \ + *(u32 *)(r10 - 8) = r1; \ + r6 = 0; \ +1: r1 = r10; \ + r1 += -8; \ + call scrub_slot_callee; \ + r6 += 1; \ + if r6 < 10 goto 1b; \ + r0 = *(u32 *)(r10 - 8); \ + exit; \ +" ::: __clobber_all); +} + +/* + * The inner loop turns fp-8 from a spill into misc. On the next outer + * iteration, the states inside the inner loop match its first invocation. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 3, widening r6 to 0..9 step 1") +__msg("loop header at 4, widening r7 to 0..2 step 1") +__msg("loop header at 3, clamping r6 to 0..9 step 1") +__naked void clamp_nested_loop_state_pruning(void) +{ + asm volatile (" \ + w1 = 1; \ + *(u32 *)(r10 - 8) = r1; \ + r6 = 0; \ +1: r7 = 0; \ +2: *(u32 *)(r10 - 4) = 1; \ + r7 += 1; \ + if r7 < 3 goto 2b; \ + r6 += 1; \ + if r6 < 10 goto 1b; \ + r0 = *(u32 *)(r10 - 8); \ + exit; \ +" ::: __clobber_all); +} + +/* + * Same as above, but the inner loop is not recognized by SCEV and is unrolled + * with a precise r7. The states at its header still match between the outer + * iterations and must not be reported as an infinite loop: the path between + * them crosses the back edge of the terminating outer loop. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 3, widening r6 to 0..9 step 1") +__msg("loop header at 5, can't compute iterations count") +__msg("loop header at 3, clamping r6 to 0..9 step 1") +__naked void clamp_nested_unrolled_loop_state_pruning(void) +{ + asm volatile (" \ + w1 = 1; \ + *(u32 *)(r10 - 8) = r1; \ + r6 = 0; \ +1: *(u32 *)(r10 - 4) = 1; \ + r7 = 1; \ +2: r7 *= 2; \ + if r7 < 4 goto 2b; \ + r6 += 1; \ + if r6 < 10 goto 1b; \ + r0 = *(u32 *)(r10 - 8); \ + exit; \ +" ::: __clobber_all); +} + /* * r7 starts as 1 + 6*k and invariant r8 as 1 + 9*k (0 <= k <= 3). * The (any r7 r8) union must preserve base 1 and step gcd(6, 9) = 3. @@ -2971,6 +3158,115 @@ __naked void ptr_stack_other_frame(void) " ::: __clobber_all); } +SEC("xdp") +__success +__log_level(2) +__msg("scev: incompatible latch operands r6 (map_value), r7 (map_value)") +__msg("loop header at 11, can't compute iterations count at 14") +__naked void ptr_map_value_jmp32(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r7 = r6; \ + r7 += 4; \ + r9 = 0; \ +1:" LIMIT_ITERATIONS \ +" r6 += 1; \ + if w6 < w7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + +SEC("xdp") +__failure +__log_level(2) +__msg("loop header at 11, can't compute iterations count") +__msg("R6 32-bit pointer arithmetic prohibited") +__naked void ptr_map_value_alu32_latch(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r7 = r6; \ + r7 += 4; \ + r9 = 0; \ +1:" LIMIT_ITERATIONS \ +" w6 += 1; \ + if r6 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg("loop header at 11, can't compute iterations count") +__naked void ptr_map_value_alu32_latch2(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r7 = r6; \ + r7 += 4; \ + r9 = 0; \ +1:" LIMIT_ITERATIONS \ +" r6 += 1; \ + w8 = w6; \ + if r8 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg("loop header at 8, can't widen r0, expr is (zext32 (+ r0 1))") +__naked void ptr_map_value_alu32_non_latch(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r1 = 0; \ +1: r1 += 1; \ + w0 = w0; \ + w0 += 1; \ + if r1 < 4 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + /* Metadata and packet data have different origins. */ SEC("xdp") __success @@ -3221,21 +3517,23 @@ __naked void ptr_mem_other_reservation(void) : __clobber_all); } +/* An access to a buffer at a variable offset is rejected: the pointer must not be widened. */ SEC("iter/bpf_map_elem") __success __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) __msg("header_count is 4 ") -__msg("widening r6 to 0..3 step 1") +__msg("can't widen r6, expr is (+ r6 8)") __naked void ptr_buf(void) { asm volatile (" \ r6 = *(u64 *)(r1 + %[value]); \ if r6 == 0 goto 2f; \ - r7 = r6; \ - r7 += 4; \ -1: r6 += 1; \ - if r6 < r7 goto 1b; \ + r7 = 0; \ +1: r0 = *(u32 *)(r6 + 0); \ + r6 += 8; \ + r7 += 1; \ + if r7 < 4 goto 1b; \ 2: r0 = 0; \ exit; \ " : @@ -3351,7 +3649,7 @@ SEC("xdp") __failure __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) -__msg("infinite loop detected at insn 6") +__msg("The sequence of 8193 jumps is too complex.") __naked void nested_terminating_prune(void) { asm volatile (" \ @@ -3448,4 +3746,194 @@ __naked void dominating_exit_bounds_count(void) " ::: __clobber_all); } +/* + * The counter is copied to a callee-saved register for the call and the copy is compared. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg(" scev at latch 5:") +__msg(" r7=(zext32 r1) / (zext32 (linear r1 2))") +__msg("loop header at 1, header_count is 100 ") +__msg("loop header at 1, widening r1 to 0..198 step 2") +__naked void post_cond_jlt32_copy(void) +{ + asm volatile (" \ + w1 = 0; \ +1: w7 = w1; \ + call %[bpf_get_prandom_u32]; \ + w1 = w7; \ + w1 += 2; \ + if w7 < 198 goto 1b; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * r6 is a pointer to ringbuf memory with a step of 1. bpf_ringbuf_discard() in the first iteration + * makes a number of it. The offsets 0..1 of the pointer are not the range of this number: it's an + * address. The flag in the 32-bit spill is not widened, so the second iteration is verified + * on its own. + */ +SEC("socket") +__failure __msg("math between map_value pointer and register with unbounded min value") +__naked void clamp_released_pointer(void) +{ + asm volatile ( + "r1 = 0;" + "*(u32 *)(r10 - 12) = r1;" + "r2 = r10;" + "r2 += -12;" + "r1 = %[small_map] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 9f;" + "r8 = r0;" + "r1 = %[ringbuf] ll;" + "r2 = 16;" + "r3 = 0;" + "call %[bpf_ringbuf_reserve];" + "if r0 == 0 goto 9f;" + "r6 = r0;" + "r7 = r0;" + "r9 = 0;" + "w1 = 0;" + "*(u32 *)(r10 - 8) = r1;" +"1:" + "w4 = *(u32 *)(r10 - 8);" + "if w4 == 0 goto 2f;" + "r1 = r8;" + "r1 += r6;" + "r0 = *(u8 *)(r1 + 0);" + "goto 3f;" +"2:" + "r1 = r7;" + "r2 = 0;" + "call %[bpf_ringbuf_discard];" +"3:" + "w4 = 1;" + "*(u32 *)(r10 - 8) = r4;" + "r6 += 1;" + "r9 += 1;" + "if r9 < 2 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), __imm(bpf_ringbuf_reserve), __imm(bpf_ringbuf_discard), + __imm_addr(small_map), __imm_addr(ringbuf) + : __clobber_all); +} + +/* The same with a packet pointer and bpf_skb_pull_data() */ +SEC("tc") +__failure __msg("math between map_value pointer and register with unbounded min value") +__naked void clamp_invalidated_pkt_pointer(void) +{ + asm volatile ( + "r7 = r1;" + "r1 = 0;" + "*(u32 *)(r10 - 12) = r1;" + "r2 = r10;" + "r2 += -12;" + "r1 = %[small_map] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 9f;" + "r8 = r0;" + "r6 = *(u32 *)(r7 + %[data]);" + "r9 = 0;" + "w1 = 0;" + "*(u32 *)(r10 - 8) = r1;" +"1:" + "w4 = *(u32 *)(r10 - 8);" + "if w4 == 0 goto 2f;" + "r1 = r8;" + "r1 += r6;" + "r0 = *(u8 *)(r1 + 0);" + "goto 3f;" +"2:" + "r1 = r7;" + "r2 = 0;" + "call %[bpf_skb_pull_data];" +"3:" + "w4 = 1;" + "*(u32 *)(r10 - 8) = r4;" + "r6 += 1;" + "r9 += 1;" + "if r9 < 2 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), __imm(bpf_skb_pull_data), __imm_addr(small_map), + __imm_const(data, offsetof(struct __sk_buff, data)) + : __clobber_all); +} + +/* + * The compare with data_end is for r6 of this iteration: 100 bytes from r6 are in the packet. Then + * r6 moves by 100. The range that the compare found is not for the r6 that returns to the head. + */ +SEC("tc") +__failure __msg("invalid access to packet") +__naked void clamp_pkt_pointer_keeps_no_range(void) +{ + asm volatile ( + "r7 = *(u32 *)(r1 + %[data_end]);" + "r6 = *(u32 *)(r1 + %[data]);" + "r9 = 0;" + "w1 = 0;" + "*(u32 *)(r10 - 8) = r1;" +"1:" + "w4 = *(u32 *)(r10 - 8);" + "if w4 == 0 goto 2f;" + /* second iteration: nothing compared this r6 with data_end */ + "r0 = *(u64 *)(r6 + 56);" +"2:" + "r5 = r6;" + "r5 += 100;" + "if r5 > r7 goto 9f;" + "w4 = 1;" + "*(u32 *)(r10 - 8) = r4;" + "r6 += 100;" + "r9 += 1;" + "if r9 < 2 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + : + : __imm_const(data, offsetof(struct __sk_buff, data)), + __imm_const(data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + +/* A compare in front of the loop is for all iterations: the range at the entry stays */ +SEC("tc") +__success __retval(0) +__naked void clamp_pkt_pointer_keeps_range_of_entry(void) +{ + asm volatile ( + "r7 = *(u32 *)(r1 + %[data_end]);" + "r6 = *(u32 *)(r1 + %[data]);" + "r5 = r6;" + "r5 += 40;" + "if r5 > r7 goto 9f;" + "r9 = 0;" +"1:" + "r0 = *(u64 *)(r6 + 0);" + "r6 += 8;" + "r9 += 1;" + "if r9 < 4 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + : + : __imm_const(data, offsetof(struct __sk_buff, data)), + __imm_const(data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c index b9f799f6e405..15a12497e074 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c @@ -433,6 +433,125 @@ __naked void stack_arg_sequential_calls(void) ); } +/* + * A JIT passes the first stack arguments in registers. may_goto is not a call: they must be + * the same after it, also when it looks at the clock. That is once in 65535 iterations. + */ +__naked __noinline __used +static long loops_then_reads_8args(long a, long b, long c, long d, long e, long f, long g, long h) +{ + asm volatile ( + "r1 = 0;" + "1:" + "may_goto 2f;" + "r1 += 1;" + "if r1 < 70000 goto 1b;" + "2:" + "r0 = *(u64 *)(r11 + 8);" + "r1 = *(u64 *)(r11 + 16);" + "r0 += r1;" + "r1 = *(u64 *)(r11 + 24);" + "r0 += r1;" + "exit;" + ); +} + +SEC("tc") +__description("stack_arg: may_goto looks at the clock, then the callee reads its args") +__success __retval(876) +__naked void stack_arg_may_goto_in_callee(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 24) = 800;" + "*(u64 *)(r11 - 16) = 70;" + "*(u64 *)(r11 - 8) = 6;" + "call loops_then_reads_8args;" + "exit;" + ::: __clobber_all + ); +} + +__naked __noinline __used +static long spins_then_reads_8args(long a, long b, long c, long d, long e, long f, long g, long h) +{ + asm volatile ( + "1:" + "may_goto 2f;" + "goto 1b;" + "2:" + "r0 = *(u64 *)(r11 + 8);" + "r1 = *(u64 *)(r11 + 16);" + "r0 += r1;" + "r1 = *(u64 *)(r11 + 24);" + "r0 += r1;" + "exit;" + ); +} + +SEC("tc") +__description("stack_arg: the time of may_goto is up, then the callee reads its args") +__success __retval(876) +__naked void stack_arg_may_goto_timeout_in_callee(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 24) = 800;" + "*(u64 *)(r11 - 16) = 70;" + "*(u64 *)(r11 - 8) = 6;" + "call spins_then_reads_8args;" + "exit;" + ::: __clobber_all + ); +} + +__naked __noinline __used +static long reads_8args(long a, long b, long c, long d, long e, long f, long g, long h) +{ + asm volatile ( + "r0 = *(u64 *)(r11 + 8);" + "r1 = *(u64 *)(r11 + 16);" + "r0 += r1;" + "r1 = *(u64 *)(r11 + 24);" + "r0 += r1;" + "exit;" + ); +} + +SEC("tc") +__description("stack_arg: may_goto looks at the clock between the stores of the args and the call") +__success __retval(876) +__naked void stack_arg_may_goto_in_caller(void) +{ + asm volatile ( + "*(u64 *)(r11 - 24) = 800;" + "*(u64 *)(r11 - 16) = 70;" + "*(u64 *)(r11 - 8) = 6;" + "r6 = 0;" + "1:" + "may_goto 2f;" + "r6 += 1;" + "if r6 < 70000 goto 1b;" + "2:" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "call reads_8args;" + "exit;" + ::: __clobber_all + ); +} + #else SEC("socket") diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c index ab1955852233..5bfcd300c7a0 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c @@ -5,6 +5,17 @@ #include <bpf/bpf_helpers.h> #include "bpf_misc.h" +struct timer_value { + struct bpf_timer timer; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct timer_value); +} timer_map SEC(".maps"); + #if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) @@ -130,6 +141,25 @@ static void subprog_bad_ptr_7args(long *a, int b, int c, int d, int e, int f, in ); } +__noinline __used __naked +static int callback_6args(long a, long b, long c, long d, long e, long f) +{ + asm volatile ("r0 = *(u64 *)(r11 + 8);" + "exit;" + ::: __clobber_all + ); +} + +__noinline __used __naked +static void subprog_call_bad_ptr_7args(void) +{ + asm volatile ("*(u64 *)(r11 - 8) = 0;" + "call subprog_bad_ptr_7args;" + "exit;" + ::: __clobber_all + ); +} + SEC("tc") __description("stack_arg: read without caller write") __failure @@ -152,7 +182,7 @@ __naked void stack_arg_read_without_write_1(void) SEC("tc") __description("stack_arg: read with not-initialized caller write") __failure -__msg("R0 !read_ok") +__msg("invalid read from uninitialized stack arg off 8") __btf_func_path("btf__verifier_stack_arg_order.bpf.o") __naked void stack_arg_read_without_write_2(void) { @@ -171,6 +201,95 @@ __naked void stack_arg_read_without_write_2(void) ); } +SEC("tc") +__description("stack_arg: unreliable callee with too few stack args") +__failure +__msg("invalid read from uninitialized stack arg off 16") +__btf_func_path("btf__verifier_stack_arg_order.bpf.o") +__naked void stack_arg_read_with_short_bridge_write(void) +{ + asm volatile ("r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "*(u64 *)(r11 - 8) = 0;" + "*(u64 *)(r11 - 16) = 0;" + "call subprog_bad_ptr_7args;" + "call subprog_call_bad_ptr_7args;" + "exit;" + ::: __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: synchronous callback with stack args") +__failure +__msg("callback subprog cannot have stack args") +__btf_func_path("btf__verifier_stack_arg_order.bpf.o") +__naked void stack_arg_sync_callback(void) +{ + asm volatile ("r6 = 0;" + "*(u64 *)(r11 - 8) = r6;" + "r1 = 1;" + "r2 = %[callback_6args];" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 8) = 6;" + "call callback_6args;" + "r0 = 0;" + "exit;" + : + : __imm_ptr(callback_6args), + __imm(bpf_loop) + : __clobber_common, "r6" + ); +} + +SEC("tc") +__description("stack_arg: asynchronous callback with stack args") +__failure +__msg("callback subprog cannot have stack args") +__btf_func_path("btf__verifier_stack_arg_order.bpf.o") +__naked void stack_arg_async_callback(void) +{ + asm volatile ("r1 = 0;" + "*(u32 *)(r10 - 4) = r1;" + "r1 = %[timer_map] ll;" + "r2 = r10;" + "r2 += -4;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto l0_%=;" + "r6 = r0;" + "r7 = 0;" + "*(u64 *)(r11 - 8) = r7;" + "r1 = r6;" + "r2 = %[callback_6args];" + "call %[bpf_timer_set_callback];" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 8) = 6;" + "call callback_6args;" + "l0_%=: r0 = 0;" + "exit;" + : + : __imm_addr(timer_map), + __imm_ptr(callback_6args), + __imm(bpf_map_lookup_elem), + __imm(bpf_timer_set_callback) + : __clobber_common, "r6", "r7" + ); +} + #else SEC("socket") diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c index 7fa7bc207850..e6a9021b197b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c @@ -588,35 +588,37 @@ __naked void stack_check_size_512_with_may_goto(void) /* * Verify that old PTR_TO_STACK state is considered a super-set of - * new PTR_TO_STACK state when new variable range is a sub-range - * of the old range, e.g. old [-72, -16] vs new [-64, -16]. + * new PTR_TO_STACK state when the new offset is within the old range, + * e.g. old [-72, -16] vs new {-16}. */ SEC("socket") __success __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) /* - * r7 is widened to [-72, -16] at the loop header (insn 3), - * the loop body sees [-64, -8] after 'r7 += 8' + * r7 is widened to [-72, -16] at the loop header (insn 5), + * the loop body sees [-64, -8] after 'r7 += 8'. */ -__msg("loop header at 3, widening r7 to -72..-16 step 8") +__msg("loop header at 5, widening r7 to -72..-16 step 1") __msg("R7=fp(smin=smin32=-64,smax=smax32=-8") -/* - * back-edge state is clamped to the remaining iterations and pruned - * at the header, because [-64, -16] is within the widened [-72, -16] - */ -__msg("loop header at 3, clamping r7 to -64..-16 (s32 -64..-8) step 8") -__msg("from 5 to 3: safe") -__msg("processed 9 insns") +/* 'r7 = r8' sets r7 to fp-16; its ANY SCEV is not clamped. */ +__msg("R7=fp-16 R8=fp-16 loop_stack=5") +__not_msg("clamping r7") +/* back-edge state is pruned because {-16} is within [-72, -16]. */ +__msg("from 8 to 5: safe") +__msg("processed 12 insns") __naked void stack_ptr_subrange_in_loop(void) { asm volatile (" \ r7 = r10; \ r7 += -72; \ + r8 = r10; \ + r8 += -16; \ r6 = 0; \ - r7 += 8; \ +1: r7 += 8; \ + r7 = r8; \ r6 += 1; \ - if r6 < 8 goto -3; \ + if r6 < 8 goto 1b; \ r0 = 0; \ exit; \ " ::: __clobber_all); diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_accept.c b/tools/testing/selftests/bpf/progs/verifier_vfs_accept.c index 55398c04290a..25f37a3dd7cf 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_accept.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_accept.c @@ -13,6 +13,23 @@ static char buf[64]; SEC("lsm.s/file_open") __success +int BPF_PROG(inode_field_before_file_put) +{ + struct file *file; + ino_t ino; + + file = bpf_get_task_exe_file(bpf_get_current_task_btf()); + if (!file) + return 0; + + ino = file->f_inode->i_ino; + bpf_put_file(file); + __sink(ino); + return 0; +} + +SEC("lsm.s/file_open") +__success int BPF_PROG(get_task_exe_file_and_put_kfunc_from_current_sleepable) { struct file *acquired; diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c index b5f456d57669..ed2bd2faebf3 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c @@ -13,6 +13,25 @@ static char buf[PATH_MAX]; SEC("lsm.s/file_open") +__failure __msg("invalid mem access 'scalar'") +int BPF_PROG(inode_field_after_file_put) +{ + struct inode *inode; + struct file *file; + ino_t ino; + + file = bpf_get_task_exe_file(bpf_get_current_task_btf()); + if (!file) + return 0; + + inode = file->f_inode; + bpf_put_file(file); + ino = inode->i_ino; + __sink(ino); + return 0; +} + +SEC("lsm.s/file_open") __failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(get_task_exe_file_kfunc_null) { diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 6380f52f4dd2..76fbe64755ef 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -281,6 +281,27 @@ __bpf_kfunc void bpf_iter_testmod_seq_destroy(struct bpf_iter_testmod_seq *it) it->cnt = 0; } +__bpf_kfunc int bpf_iter_testmod_current_new(struct bpf_iter_testmod_current *it) +{ + it->remaining = 2; + return 0; +} + +__bpf_kfunc struct task_struct * +bpf_iter_testmod_current_next(struct bpf_iter_testmod_current *it) +{ + if (!it->remaining) + return NULL; + it->remaining--; + /* current remains alive independently of the iterator. */ + return current; +} + +__bpf_kfunc void bpf_iter_testmod_current_destroy(struct bpf_iter_testmod_current *it) +{ + it->remaining = 0; +} + __bpf_kfunc void bpf_kfunc_common_test(void) { } @@ -1016,6 +1037,10 @@ BTF_ID_FLAGS(func, bpf_iter_testmod_seq_new, KF_ITER_NEW) BTF_ID_FLAGS(func, bpf_iter_testmod_seq_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_testmod_seq_destroy, KF_ITER_DESTROY) BTF_ID_FLAGS(func, bpf_iter_testmod_seq_value) +BTF_ID_FLAGS(func, bpf_iter_testmod_current_new, KF_ITER_NEW) +BTF_ID_FLAGS(func, bpf_iter_testmod_current_next, + KF_ITER_NEXT | KF_RET_NULL | KF_ITER_RET_INDEPENDENT) +BTF_ID_FLAGS(func, bpf_iter_testmod_current_destroy, KF_ITER_DESTROY) BTF_ID_FLAGS(func, bpf_kfunc_common_test) BTF_ID_FLAGS(func, bpf_kfunc_arena_arg_test) BTF_ID_FLAGS(func, bpf_kfunc_arena_cap_test) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h index 4ba9def74404..aef780261ef4 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h @@ -31,6 +31,10 @@ struct bpf_iter_testmod_seq { int cnt; }; +struct bpf_iter_testmod_current { + u64 remaining; +}; + struct bpf_testmod_ops { int (*test_1)(void); void (*test_2)(int a, int b); diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 3b810f1e438f..e9902e651d72 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -33,6 +33,11 @@ struct prog_test_ref_kfunc { #endif struct bpf_iter_testmod_seq; +struct bpf_iter_testmod_current; + +int bpf_iter_testmod_current_new(struct bpf_iter_testmod_current *it) __ksym; +struct task_struct *bpf_iter_testmod_current_next(struct bpf_iter_testmod_current *it) __ksym; +void bpf_iter_testmod_current_destroy(struct bpf_iter_testmod_current *it) __ksym; struct prog_test_pass1 { int x0; diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 824e5912ebcb..d24e4fedae46 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -590,8 +590,6 @@ static int parse_test_spec(struct test_loader *tester, update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear); } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) { update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear); - } else if (strcmp(val, "BPF_F_ARENA_SCALAR") == 0) { - update_flags(&spec->prog_flags, BPF_F_ARENA_SCALAR, clear); } else /* assume numeric value */ { err = parse_int(val, &flags, "test prog flags"); if (err) |