diff options
Diffstat (limited to 'tools/testing/selftests/bpf')
341 files changed, 37561 insertions, 1730 deletions
diff --git a/tools/testing/selftests/bpf/.gitignore b/tools/testing/selftests/bpf/.gitignore index 986a6389186b..b815bf0d8877 100644 --- a/tools/testing/selftests/bpf/.gitignore +++ b/tools/testing/selftests/bpf/.gitignore @@ -17,7 +17,6 @@ test_verifier_log feature urandom_read test_sockmap -test_lirc_mode2_user flow_dissector_load test_tcpnotify_user test_libbpf diff --git a/tools/testing/selftests/bpf/DENYLIST.riscv64 b/tools/testing/selftests/bpf/DENYLIST.riscv64 index ca1beae7fe8f..75a9690fb2f2 100644 --- a/tools/testing/selftests/bpf/DENYLIST.riscv64 +++ b/tools/testing/selftests/bpf/DENYLIST.riscv64 @@ -1,2 +1,3 @@ # riscv64 deny list for BPF CI and local vmtest exceptions # JIT does not support exceptions +probe_user # kprobe hit in trap diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 93c707116fad..1e4af62e8312 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -3,93 +3,18 @@ include ../../../build/Build.include include ../../../scripts/Makefile.arch include ../../../scripts/Makefile.include -OBJCOPY ?= $(CROSS_COMPILE)objcopy - -CURDIR := $(abspath .) -TOOLSDIR := $(abspath ../../..) -LIBDIR := $(TOOLSDIR)/lib -BPFDIR := $(LIBDIR)/bpf -TOOLSINCDIR := $(TOOLSDIR)/include -TOOLSARCHINCDIR := $(TOOLSDIR)/arch/$(SRCARCH)/include -BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool -APIDIR := $(TOOLSINCDIR)/uapi -ifneq ($(O),) -GENDIR := $(O)/include/generated -else -GENDIR := $(abspath ../../../../include/generated) -endif -GENHDR := $(GENDIR)/autoconf.h -PKG_CONFIG ?= $(CROSS_COMPILE)pkg-config - -ifneq ($(wildcard $(GENHDR)),) - GENFLAGS := -DHAVE_GENHDR -endif - BPF_GCC ?= $(shell command -v bpf-gcc;) -ifdef ASAN -SAN_CFLAGS ?= -fsanitize=address -fno-omit-frame-pointer -else -SAN_CFLAGS ?= -endif -SAN_LDFLAGS ?= $(SAN_CFLAGS) -RELEASE ?= -OPT_FLAGS ?= $(if $(RELEASE),-O2,-O0) - -LIBELF_CFLAGS := $(shell $(PKG_CONFIG) libelf --cflags 2>/dev/null) -LIBELF_LIBS := $(shell $(PKG_CONFIG) libelf --libs 2>/dev/null || echo -lelf) SKIP_DOCS ?= SKIP_LLVM ?= SKIP_LIBBFD ?= SKIP_CRYPTO ?= -# When BPF_STRICT_BUILD is 1, any BPF object, skeleton, test object, or -# benchmark compilation failure is fatal. Set to 0 to tolerate failures -# and continue building the remaining tests. -BPF_STRICT_BUILD ?= 1 -PERMISSIVE := $(filter 0,$(BPF_STRICT_BUILD)) - -ifeq ($(srctree),) -srctree := $(patsubst %/,%,$(dir $(CURDIR))) -srctree := $(patsubst %/,%,$(dir $(srctree))) -srctree := $(patsubst %/,%,$(dir $(srctree))) -srctree := $(patsubst %/,%,$(dir $(srctree))) -endif - -COMMON_CFLAGS = -g $(OPT_FLAGS) -rdynamic -std=gnu11 \ - -Wall -Werror -fno-omit-frame-pointer \ - -Wno-unused-but-set-variable \ - $(GENFLAGS) $(SAN_CFLAGS) $(LIBELF_CFLAGS) \ - -I$(CURDIR) -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ - -I$(TOOLSINCDIR) -I$(TOOLSARCHINCDIR) -I$(APIDIR) -I$(OUTPUT) \ - -I$(CURDIR)/libarena/include -LDFLAGS += $(SAN_LDFLAGS) -LDLIBS += $(LIBELF_LIBS) -lz -lrt -lpthread - -PCAP_CFLAGS := $(shell $(PKG_CONFIG) --cflags libpcap 2>/dev/null && echo "-DTRAFFIC_MONITOR=1") -PCAP_LIBS := $(shell $(PKG_CONFIG) --libs libpcap 2>/dev/null) -LDLIBS += $(PCAP_LIBS) -CFLAGS += $(COMMON_CFLAGS) $(PCAP_CFLAGS) - -# Some utility functions use LLVM libraries -jit_disasm_helpers.c-CFLAGS = $(LLVM_CFLAGS) - -ifneq ($(LLVM),) -# Silence some warnings when compiled with clang -CFLAGS += -Wno-unused-command-line-argument -endif - # Check whether bpf cpu=v4 is supported or not by clang ifneq ($(shell $(CLANG) --target=bpf -mcpu=help 2>&1 | grep 'v4'),) CLANG_CPUV4 := 1 endif -# Check whether clang supports BPF address sanitizer (requires LLVM 22+) -CLANG_HAS_ARENA_ASAN := $(shell echo 'int x;' | \ - $(CLANG) --target=bpf -fsanitize=kernel-address \ - -mllvm -asan-shadow-addr-space=1 \ - -x c -c - -o /dev/null 2>/dev/null && echo 1) - # Order correspond to 'make run_tests' order TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_progs \ test_sockmap \ @@ -101,16 +26,6 @@ TEST_INST_SUBDIRS := no_alu32 ifneq ($(BPF_GCC),) TEST_GEN_PROGS += test_progs-bpf_gcc TEST_INST_SUBDIRS += bpf_gcc - -# The following tests contain C code that, although technically legal, -# triggers GCC warnings that cannot be disabled: declaration of -# anonymous struct types in function parameter lists. -progs/btf_dump_test_case_bitfields.c-bpf_gcc-CFLAGS := -Wno-error -progs/btf_dump_test_case_namespacing.c-bpf_gcc-CFLAGS := -Wno-error -progs/btf_dump_test_case_packing.c-bpf_gcc-CFLAGS := -Wno-error -progs/btf_dump_test_case_padding.c-bpf_gcc-CFLAGS := -Wno-error -progs/btf_dump_test_case_syntax.c-bpf_gcc-CFLAGS := -Wno-error - endif ifneq ($(CLANG_CPUV4),) @@ -118,12 +33,12 @@ TEST_GEN_PROGS += test_progs-cpuv4 TEST_INST_SUBDIRS += cpuv4 endif -TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c) +TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c) \ + bpftool_btf_dump_sorted.expected \ + bpftool_btf_dump_unsorted.expected # Order correspond to 'make run_tests' order -TEST_PROGS := test_kmod.sh \ - test_lirc_mode2.sh \ - test_bpftool_build.sh \ +TEST_PROGS := test_bpftool_build.sh \ test_doc_build.sh \ test_xsk.sh \ test_xdp_features.sh @@ -140,7 +55,6 @@ TEST_GEN_PROGS_EXTENDED = \ bench \ flow_dissector_load \ test_cpp \ - test_lirc_mode2_user \ veristat \ xdp_features \ xdp_hw_metadata \ @@ -167,7 +81,17 @@ override define CLEAN $(Q)$(MAKE) docs-clean endef +# Append build flags before lib.mk's user flags. References to +# Makefile.buildvars definitions expand when the flags are used. +# The arena-ASAN define is appended after lib.mk. +CFLAGS += $(COMMON_CFLAGS) $(PCAP_CFLAGS) $(CLANG_WARN_CFLAGS) +LDFLAGS += $(SAN_LDFLAGS) include ../lib.mk +include Makefile.buildvars + +BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool +HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a +BPF_TARGET_ENDIAN:=$(if $(IS_LITTLE_ENDIAN),--target=bpfel,--target=bpfeb) NON_CHECK_FEAT_TARGETS := clean docs-clean emit_tests CHECK_FEAT := $(filter-out $(NON_CHECK_FEAT_TARGETS),$(or $(MAKECMDGOALS), "none")) @@ -212,21 +136,9 @@ ifeq ($(feature-llvm),1) endif endif -SCRATCH_DIR := $(OUTPUT)/tools -BUILD_DIR := $(SCRATCH_DIR)/build -INCLUDE_DIR := $(SCRATCH_DIR)/include -BPFOBJ := $(BUILD_DIR)/libbpf/libbpf.a -ifneq ($(CROSS_COMPILE),) -HOST_BUILD_DIR := $(BUILD_DIR)/host -HOST_SCRATCH_DIR := $(OUTPUT)/host-tools -HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include -else -HOST_BUILD_DIR := $(BUILD_DIR) -HOST_SCRATCH_DIR := $(SCRATCH_DIR) -HOST_INCLUDE_DIR := $(INCLUDE_DIR) -endif -HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a -RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids +# Some utility functions use LLVM libraries +$(OUTPUT)/jit_disasm_helpers.o: CFLAGS += $(LLVM_CFLAGS) + VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ ../../../../vmlinux \ @@ -245,7 +157,7 @@ $(notdir $(TEST_GEN_PROGS) $(TEST_KMODS) \ $(TEST_GEN_PROGS_EXTENDED)): %: $(OUTPUT)/% ; # sort removes libbpf duplicates when not cross-building -MAKE_DIRS := $(sort $(BUILD_DIR)/libbpf $(HOST_BUILD_DIR)/libbpf \ +MAKE_DIRS := $(sort $(OUTPUT) $(BUILD_DIR)/libbpf $(HOST_BUILD_DIR)/libbpf \ $(BUILD_DIR)/bpftool $(HOST_BUILD_DIR)/bpftool \ $(HOST_BUILD_DIR)/resolve_btfids \ $(INCLUDE_DIR)) @@ -257,9 +169,11 @@ $(OUTPUT)/%.o: %.c $(call msg,CC,,$@) $(Q)$(CC) $(CFLAGS) -c $(filter %.c,$^) $(LDLIBS) -o $@ +# Keep header prerequisites off the link command: clang treats +# bare headers as precompiled-header jobs and refuses -o. $(OUTPUT)/%:%.c $(call msg,BINARY,,$@) - $(Q)$(LINK.c) $^ $(LDLIBS) -o $@ + $(Q)$(LINK.c) $(filter-out %.h,$^) $(LDLIBS) -o $@ # LLVM's ld.lld doesn't support all the architectures, so use it only on x86 ifeq ($(SRCARCH),$(filter $(SRCARCH),x86 riscv)) @@ -312,17 +226,6 @@ $(TEST_KMOD_TARGETS): $(addprefix test_kmods/,$(TEST_KMODS)) $(call msg,MOD,,$@) $(Q)$(if $(PERMISSIVE),if [ -f test_kmods/$(@F) ]; then )cp test_kmods/$(@F) $@$(if $(PERMISSIVE),; fi) - -DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool -ifneq ($(CROSS_COMPILE),) -CROSS_BPFTOOL := $(SCRATCH_DIR)/sbin/bpftool -TRUNNER_BPFTOOL := $(CROSS_BPFTOOL) -USE_BOOTSTRAP := "" -else -TRUNNER_BPFTOOL := $(DEFAULT_BPFTOOL) -USE_BOOTSTRAP := "bootstrap/" -endif - $(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED): $(BPFOBJ) TESTING_HELPERS := $(OUTPUT)/testing_helpers.o @@ -337,13 +240,29 @@ $(OUTPUT)/test_sockmap: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(OUTPUT)/test_tcpnotify_user: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(TRACE_HELPERS) $(OUTPUT)/test_sock_fields: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(OUTPUT)/test_tag: $(TESTING_HELPERS) -$(OUTPUT)/test_lirc_mode2_user: $(TESTING_HELPERS) -$(OUTPUT)/flow_dissector_load: $(TESTING_HELPERS) -$(OUTPUT)/test_maps: $(TESTING_HELPERS) +$(OUTPUT)/flow_dissector_load: $(TESTING_HELPERS) flow_dissector_load.h $(OUTPUT)/test_verifier: $(TESTING_HELPERS) $(CAP_HELPERS) $(UNPRIV_HELPERS) -$(OUTPUT)/xsk.o: $(BPFOBJ) -BPFTOOL ?= $(DEFAULT_BPFTOOL) +# Defined ahead of the first rule that lists it as a prerequisite. +VERIFY_SIG_HDR := verification_cert.h + +# The userspace objects the runner instances link (RUNNER_OBJS, see +# Makefile.buildvars) are built here, once, so that no runner sub-make +# ever writes a file this Makefile also builds. +# +# Helper C files include headers installed to $(INCLUDE_DIR) by the +# $(TRUNNER_BPFTOOL) and $(BPFOBJ) targets. +RUNNER_OBJS := $(sort $(RUNNER_OBJS-test_progs) $(RUNNER_OBJS-test_maps)) +$(RUNNER_OBJS): $(VERIFY_SIG_HDR) $(BPFOBJ) $(TRUNNER_BPFTOOL) +$(OUTPUT)/test_progs.o: prog_tests/tests.h +$(OUTPUT)/test_maps.o: map_tests/tests.h + +# find_bit.c is the only runner source outside this directory, so it +# needs a rule of its own. +$(OUTPUT)/find_bit.o: $(TOOLSDIR)/lib/find_bit.c + $(call msg,CC,,$@) + $(Q)$(CC) $(CFLAGS) -c $< $(LDLIBS) -o $@ + $(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \ $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/bpftool $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOLDIR) \ @@ -411,7 +330,7 @@ endif # vmlinux.h is first dumped to a temporary file and then compared to # the previous version. This helps to avoid unnecessary re-builds of -# $(TRUNNER_BPF_OBJS) +# BPF objects. $(INCLUDE_DIR)/vmlinux.h: $(VMLINUX_BTF) $(BPFTOOL) | $(INCLUDE_DIR) ifeq ($(VMLINUX_H),) $(call msg,GEN,,$@) @@ -423,7 +342,9 @@ else endif $(RESOLVE_BTFIDS): $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/resolve_btfids \ - $(TOOLSDIR)/bpf/resolve_btfids/main.c \ + $(TOOLSDIR)/bpf/resolve_btfids/main.c \ + $(TOOLSDIR)/bpf/resolve_btfids/btf_colors.c \ + $(TOOLSDIR)/bpf/resolve_btfids/btf_colors.h \ $(TOOLSDIR)/lib/rbtree.c \ $(TOOLSDIR)/lib/zalloc.c \ $(TOOLSDIR)/lib/string.c \ @@ -450,360 +371,17 @@ $(shell $(1) $(2) -dM -E - </dev/null | grep '__loongarch_grlen ' | awk '{printf $(shell $(1) $(2) -dM -E - </dev/null | grep -E 'MIPS(EL|EB)|_MIPS_SZ(PTR|LONG) |_MIPS_SIM |_ABI(O32|N32|64) ' | awk '{printf("-D%s=%s ", $$2, $$3)}') endef -# Determine target endianness. -IS_LITTLE_ENDIAN := $(shell $(CC) -dM -E - </dev/null | \ - grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__') -MENDIAN:=$(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian) -BPF_TARGET_ENDIAN:=$(if $(IS_LITTLE_ENDIAN),--target=bpfel,--target=bpfeb) - ifneq ($(CROSS_COMPILE),) CLANG_TARGET_ARCH = --target=$(notdir $(CROSS_COMPILE:%-=%)) endif CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH)) -BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN) \ - -I$(INCLUDE_DIR) -I$(CURDIR) -I$(APIDIR) \ - -I$(TOOLSINCDIR) -I$(CURDIR)/libarena/include \ - -I$(abspath $(OUTPUT)/../usr/include) \ - -std=gnu11 \ - -fno-strict-aliasing \ - -Wno-microsoft-anon-tag \ - -fms-extensions \ - -Wno-compare-distinct-pointer-types \ - -Wno-initializer-overrides \ - # -# TODO: enable me -Wsign-compare - CLANG_CFLAGS = $(CLANG_SYS_INCLUDES) -$(OUTPUT)/test_l4lb_noinline.o: BPF_CFLAGS += -fno-inline -$(OUTPUT)/test_xdp_noinline.o: BPF_CFLAGS += -fno-inline - -$(OUTPUT)/flow_dissector_load.o: flow_dissector_load.h -$(OUTPUT)/cgroup_getset_retval_hooks.o: cgroup_getset_retval_hooks.h - -# Build BPF object using Clang -# $1 - input .c file -# $2 - output .o file -# $3 - CFLAGS -# $4 - binary name -define CLANG_BPF_BUILD_RULE - $(call msg,CLNG-BPF,$4,$2) - $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v3 -o $2 $(if $(PERMISSIVE),|| \ - ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) -endef -# Similar to CLANG_BPF_BUILD_RULE, but with disabled alu32 -define CLANG_NOALU32_BPF_BUILD_RULE - $(call msg,CLNG-BPF,$4,$2) - $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v2 -o $2 $(if $(PERMISSIVE),|| \ - ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) -endef -# Similar to CLANG_BPF_BUILD_RULE, but with cpu-v4 -define CLANG_CPUV4_BPF_BUILD_RULE - $(call msg,CLNG-BPF,$4,$2) - $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v4 -o $2 $(if $(PERMISSIVE),|| \ - ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) -endef -# Build BPF object using GCC -define GCC_BPF_BUILD_RULE - $(call msg,GCC-BPF,$4,$2) - $(Q)$(BPF_GCC) $3 -DBPF_NO_PRESERVE_ACCESS_INDEX -Wno-attributes -O2 -c $1 -o $2 $(if $(PERMISSIVE),|| \ - ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) -endef - -SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c - -LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \ - linked_vars.skel.h linked_maps.skel.h \ - test_subskeleton.skel.h test_subskeleton_lib.skel.h \ - test_usdt.skel.h tracing_multi.skel.h \ - tracing_multi_module.skel.h \ - tracing_multi_intersect.skel.h \ - tracing_multi_session.skel.h - -LSKELS := fexit_sleep.c trace_printk.c trace_vprintk.c map_ptr_kern.c \ - core_kern.c core_kern_overflow.c test_ringbuf.c \ - test_ringbuf_n.c test_ringbuf_map_key.c test_ringbuf_write.c \ - test_ringbuf_overwrite.c - -LSKELS_SIGNED := fentry_test.c fexit_test.c atomics.c - -# Generate both light skeleton and libbpf skeleton for these -LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c kfunc_call_test.c \ - kfunc_call_test_subprog.c test_global_percpu_data.c -SKEL_BLACKLIST += $$(LSKELS) $$(LSKELS_SIGNED) - -test_static_linked.skel.h-deps := test_static_linked1.bpf.o test_static_linked2.bpf.o -linked_funcs.skel.h-deps := linked_funcs1.bpf.o linked_funcs2.bpf.o -linked_vars.skel.h-deps := linked_vars1.bpf.o linked_vars2.bpf.o -linked_maps.skel.h-deps := linked_maps1.bpf.o linked_maps2.bpf.o -# In the subskeleton case, we want the test_subskeleton_lib.subskel.h file -# but that's created as a side-effect of the skel.h generation. -test_subskeleton.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o test_subskeleton.bpf.o -test_subskeleton_lib.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o -test_usdt.skel.h-deps := test_usdt.bpf.o test_usdt_multispec.bpf.o -xsk_xdp_progs.skel.h-deps := xsk_xdp_progs.bpf.o -xdp_hw_metadata.skel.h-deps := xdp_hw_metadata.bpf.o -xdp_features.skel.h-deps := xdp_features.bpf.o -tracing_multi.skel.h-deps := tracing_multi_attach.bpf.o tracing_multi_check.bpf.o -tracing_multi_module.skel.h-deps := tracing_multi_attach_module.bpf.o tracing_multi_check.bpf.o -tracing_multi_intersect.skel.h-deps := tracing_multi_intersect_attach.bpf.o tracing_multi_check.bpf.o -tracing_multi_session.skel.h-deps := tracing_multi_session_attach.bpf.o tracing_multi_check.bpf.o - -LINKED_BPF_OBJS := $(foreach skel,$(LINKED_SKELS),$($(skel)-deps)) -LINKED_BPF_SRCS := $(patsubst %.bpf.o,%.c,$(LINKED_BPF_OBJS)) - -HEADERS_FOR_BPF_OBJS := $(wildcard $(BPFDIR)/*.bpf.h) \ - $(wildcard $(CURDIR)/libarena/include/*.[ch]) \ - $(addprefix $(BPFDIR)/, bpf_core_read.h \ - bpf_endian.h \ - bpf_helpers.h \ - bpf_tracing.h) - -# Set up extra TRUNNER_XXX "temporary" variables in the environment (relies on -# $eval()) and pass control to DEFINE_TEST_RUNNER_RULES. -# Parameters: -# $1 - test runner base binary name (e.g., test_progs) -# $2 - test runner extra "flavor" (e.g., no_alu32, cpuv4, bpf_gcc, etc) -define DEFINE_TEST_RUNNER - -LSKEL_SIGN := -S -k $(PRIVATE_KEY) -i $(VERIFICATION_CERT) -TRUNNER_OUTPUT := $(OUTPUT)$(if $2,/)$2 -TRUNNER_BINARY := $1$(if $2,-)$2 -TRUNNER_TEST_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.test.o, \ - $$(notdir $$(wildcard $(TRUNNER_TESTS_DIR)/*.c))) -TRUNNER_EXTRA_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.o, \ - $$(filter %.c,$(TRUNNER_EXTRA_SOURCES))) -TRUNNER_LIB_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.o, \ - $$(filter %.c,$(TRUNNER_LIB_SOURCES))) -TRUNNER_EXTRA_HDRS := $$(filter %.h,$(TRUNNER_EXTRA_SOURCES)) -TRUNNER_TESTS_HDR := $(TRUNNER_TESTS_DIR)/tests.h -TRUNNER_BPF_SRCS := $$(notdir $$(wildcard $(TRUNNER_BPF_PROGS_DIR)/*.c)) -TRUNNER_BPF_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.bpf.o, $$(TRUNNER_BPF_SRCS)) -TRUNNER_BPF_SKELS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.skel.h, \ - $$(filter-out $(SKEL_BLACKLIST) $(LINKED_BPF_SRCS),\ - $$(TRUNNER_BPF_SRCS))) -TRUNNER_BPF_LSKELS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.lskel.h, $$(LSKELS) $$(LSKELS_EXTRA)) -TRUNNER_BPF_SKELS_LINKED := $$(addprefix $$(TRUNNER_OUTPUT)/,$(LINKED_SKELS)) -TRUNNER_BPF_LSKELS_SIGNED := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.lskel.h, $$(LSKELS_SIGNED)) -TEST_GEN_FILES += $$(TRUNNER_BPF_OBJS) - -# Evaluate rules now with extra TRUNNER_XXX variables above already defined -$$(eval $$(call DEFINE_TEST_RUNNER_RULES,$1,$2)) - -endef - -# Using TRUNNER_XXX variables, provided by callers of DEFINE_TEST_RUNNER and -# set up by DEFINE_TEST_RUNNER itself, create test runner build rules with: -# $1 - test runner base binary name (e.g., test_progs) -# $2 - test runner extra "flavor" (e.g., no_alu32, cpuv4, bpf_gcc, etc) -define DEFINE_TEST_RUNNER_RULES - -# Permissive build behaviour (skip-on-failure compile, partial-link) only -# applies to test_progs and its flavors; runners that use strong cross-object -# references (e.g. test_maps) keep strict semantics even when permissive. -# The check is inlined per-runner so $1 is substituted at $(call) time and -# the result is baked into each rule's recipe. - -ifeq ($($(TRUNNER_OUTPUT)-dir),) -$(TRUNNER_OUTPUT)-dir := y -$(TRUNNER_OUTPUT): - $$(call msg,MKDIR,,$$@) - $(Q)mkdir -p $$@ -endif - -# ensure we set up BPF objects generation rule just once for a given -# input/output directory combination -ifeq ($($(TRUNNER_BPF_PROGS_DIR)$(if $2,-)$2-bpfobjs),) -$(TRUNNER_BPF_PROGS_DIR)$(if $2,-)$2-bpfobjs := y -$(TRUNNER_BPF_OBJS): $(TRUNNER_OUTPUT)/%.bpf.o: \ - $(TRUNNER_BPF_PROGS_DIR)/%.c \ - $(TRUNNER_BPF_PROGS_DIR)/*.h \ - $$(INCLUDE_DIR)/vmlinux.h \ - $(HEADERS_FOR_BPF_OBJS) \ - | $(TRUNNER_OUTPUT) $$(BPFOBJ) - $$(call $(TRUNNER_BPF_BUILD_RULE),$$<,$$@, \ - $(TRUNNER_BPF_CFLAGS) \ - $$($$<-CFLAGS) \ - $$($$<-$2-CFLAGS),$(TRUNNER_BINARY)) - -$(TRUNNER_BPF_SKELS): %.skel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) - $(Q)$(if $(PERMISSIVE),if [ ! -f $$< ]; then \ - $$(RM) $$@ $$(@:.skel.h=.subskel.h); \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - exit 0; \ - fi;) \ - printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2; \ - $$(BPFTOOL) gen object $$(<:.o=.linked1.o) $$< && \ - $$(BPFTOOL) gen object $$(<:.o=.linked2.o) $$(<:.o=.linked1.o) && \ - $$(BPFTOOL) gen object $$(<:.o=.linked3.o) $$(<:.o=.linked2.o) && \ - diff $$(<:.o=.linked2.o) $$(<:.o=.linked3.o) && \ - $$(BPFTOOL) gen skeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$@ && \ - $$(BPFTOOL) gen subskeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$(@:.skel.h=.subskel.h) $(if $(PERMISSIVE),|| { \ - $$(RM) $$@ $$(@:.skel.h=.subskel.h); \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - }) && \ - rm -f $$(<:.o=.linked1.o) $$(<:.o=.linked2.o) $$(<:.o=.linked3.o) - -$(TRUNNER_BPF_LSKELS): %.lskel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) - $(Q)$(if $(PERMISSIVE),if [ ! -f $$< ]; then \ - $$(RM) $$@; \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - exit 0; \ - fi;) \ - printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2; \ - $$(BPFTOOL) gen object $$(<:.o=.llinked1.o) $$< && \ - $$(BPFTOOL) gen object $$(<:.o=.llinked2.o) $$(<:.o=.llinked1.o) && \ - $$(BPFTOOL) gen object $$(<:.o=.llinked3.o) $$(<:.o=.llinked2.o) && \ - diff $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) && \ - $$(BPFTOOL) gen skeleton -L $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@ $(if $(PERMISSIVE),|| { \ - $$(RM) $$@; \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - }) && \ - rm -f $$(<:.o=.llinked1.o) $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) - -$(TRUNNER_BPF_LSKELS_SIGNED): %.lskel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) - $(Q)$(if $(PERMISSIVE),if [ ! -f $$< ]; then \ - $$(RM) $$@; \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - exit 0; \ - fi;) \ - printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY) (signed)] $$(notdir $$@)' 1>&2; \ - $$(BPFTOOL) gen object $$(<:.o=.llinked1.o) $$< && \ - $$(BPFTOOL) gen object $$(<:.o=.llinked2.o) $$(<:.o=.llinked1.o) && \ - $$(BPFTOOL) gen object $$(<:.o=.llinked3.o) $$(<:.o=.llinked2.o) && \ - diff $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) && \ - $$(BPFTOOL) gen skeleton $(LSKEL_SIGN) $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@ $(if $(PERMISSIVE),|| { \ - $$(RM) $$@; \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - }) && \ - rm -f $$(<:.o=.llinked1.o) $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) - -$(LINKED_BPF_OBJS): %: $(TRUNNER_OUTPUT)/% - -# .SECONDEXPANSION here allows to correctly expand %-deps variables as prerequisites -.SECONDEXPANSION: -$(TRUNNER_BPF_SKELS_LINKED): $(TRUNNER_OUTPUT)/%: $$$$(%-deps) $(BPFTOOL) | $(TRUNNER_OUTPUT) - $(Q)$(if $(PERMISSIVE),for f in $$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps)); do \ - if [ ! -f $$$$f ]; then \ - $$(RM) $$@ $$(@:.skel.h=.subskel.h); \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - exit 0; \ - fi; \ - done;) \ - printf ' %-12s %s\n' 'LINK-BPF' '[$(TRUNNER_BINARY)] $$(notdir $$(@:.skel.h=.bpf.o))' 1>&2; \ - $$(BPFTOOL) gen object $$(@:.skel.h=.linked1.o) $$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps)) && \ - $$(BPFTOOL) gen object $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked1.o) && \ - $$(BPFTOOL) gen object $$(@:.skel.h=.linked3.o) $$(@:.skel.h=.linked2.o) && \ - diff $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked3.o) && \ - printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2 && \ - $$(BPFTOOL) gen skeleton $$(@:.skel.h=.linked3.o) name $$(notdir $$(@:.skel.h=)) > $$@ && \ - $$(BPFTOOL) gen subskeleton $$(@:.skel.h=.linked3.o) name $$(notdir $$(@:.skel.h=)) > $$(@:.skel.h=.subskel.h) $(if $(PERMISSIVE),|| { \ - $$(RM) $$@ $$(@:.skel.h=.subskel.h); \ - printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ - }) && \ - rm -f $$(@:.skel.h=.linked1.o) $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked3.o) - -# When the compiler generates a %.d file, only skel basenames (not -# full paths) are specified as prerequisites for corresponding %.o -# file. vpath directives below instruct make to search for skel files -# in TRUNNER_OUTPUT, if they are not present in the working directory. -vpath %.skel.h $(TRUNNER_OUTPUT) -vpath %.lskel.h $(TRUNNER_OUTPUT) -vpath %.subskel.h $(TRUNNER_OUTPUT) - -endif - -# ensure we set up tests.h header generation rule just once -ifeq ($($(TRUNNER_TESTS_DIR)-tests-hdr),) -$(TRUNNER_TESTS_DIR)-tests-hdr := y -$(TRUNNER_TESTS_HDR): $(TRUNNER_TESTS_DIR)/*.c - $$(call msg,TEST-HDR,$(TRUNNER_BINARY),$$@) - $$(shell (echo '/* Generated header, do not edit */'; \ - sed -n -E 's/^void (serial_)?test_([a-zA-Z0-9_]+)\((void)?\).*/DEFINE_TEST(\2)/p' \ - $(TRUNNER_TESTS_DIR)/*.c | sort ; \ - ) > $$@) -endif - -$(TRUNNER_OUTPUT)/resolve_btfids.test.o: $(RESOLVE_BTFIDS) $(TRUNNER_OUTPUT)/btf_data.bpf.o -$(TRUNNER_OUTPUT)/resolve_btfids.test.o: private TEST_NEEDS_BTFIDS = 1 - -# compile individual test files -# Note: we cd into output directory to ensure embedded BPF object is found -$(TRUNNER_TEST_OBJS): $(TRUNNER_OUTPUT)/%.test.o: \ - $(TRUNNER_TESTS_DIR)/%.c \ - | $(TRUNNER_OUTPUT)/%.test.d - $$(call msg,TEST-OBJ,$(TRUNNER_BINARY),$$@) - $(Q)(cd $$(@D) && $$(CC) -I. $$(CFLAGS) -MMD -MT $$@ -c $(CURDIR)/$$< $$(LDLIBS) -o $$(@F)) $(if $(filter test_progs%,$1),$(if $(PERMISSIVE),|| \ - ($(RM) $$@; printf ' %-12s %s\n' 'SKIP-TEST' '$$(notdir $$@)' 1>&2))) - $$(if $$(TEST_NEEDS_BTFIDS), \ - $(Q)if [ -f $$@ ]; then \ - $(if $(filter 1,$(V)),true,printf ' %-8s%s %s\n' "BTFIDS" " [$(TRUNNER_BINARY)]" "$$(notdir $$@)"); \ - $(RESOLVE_BTFIDS) --btf $(TRUNNER_OUTPUT)/btf_data.bpf.o $$@; \ - $(RESOLVE_BTFIDS) --patch_btfids $$@.BTF_ids $$@; \ - fi) - -$(TRUNNER_TEST_OBJS:.o=.d): $(TRUNNER_OUTPUT)/%.test.d: \ - $(TRUNNER_TESTS_DIR)/%.c \ - $(TRUNNER_EXTRA_HDRS) \ - $$(BPFOBJ) | $(TRUNNER_OUTPUT) \ - $(TRUNNER_BPF_SKELS) \ - $(TRUNNER_BPF_LSKELS) \ - $(TRUNNER_BPF_LSKELS_SIGNED) \ - $(TRUNNER_BPF_SKELS_LINKED) - -ifeq ($(filter clean docs-clean emit_tests,$(MAKECMDGOALS)),) -include $(wildcard $(TRUNNER_TEST_OBJS:.o=.d)) -endif - -# add per extra obj CFGLAGS definitions -$(foreach N,$(patsubst $(TRUNNER_OUTPUT)/%.o,%,$(TRUNNER_EXTRA_OBJS)), \ - $(eval $(TRUNNER_OUTPUT)/$(N).o: CFLAGS += $($(N).c-CFLAGS))) - -$(TRUNNER_EXTRA_OBJS): $(TRUNNER_OUTPUT)/%.o: \ - %.c \ - $(TRUNNER_EXTRA_HDRS) \ - $(VERIFY_SIG_HDR) \ - $(TRUNNER_TESTS_HDR) \ - $$(BPFOBJ) | $(TRUNNER_OUTPUT) - $$(call msg,EXT-OBJ,$(TRUNNER_BINARY),$$@) - $(Q)$$(CC) $$(CFLAGS) -c $$< $$(LDLIBS) -o $$@ - -$(TRUNNER_LIB_OBJS): $(TRUNNER_OUTPUT)/%.o:$(TOOLSDIR)/lib/%.c - $$(call msg,LIB-OBJ,$(TRUNNER_BINARY),$$@) - $(Q)$$(CC) $$(CFLAGS) -c $$< $$(LDLIBS) -o $$@ - -# non-flavored in-srctree builds receive special treatment, in particular, we -# do not need to copy extra resources (see e.g. test_btf_dump_case()) -$(TRUNNER_BINARY)-extras: $(TRUNNER_EXTRA_FILES) | $(TRUNNER_OUTPUT) -ifneq ($2:$(OUTPUT),:$(shell pwd)) - $$(call msg,EXT-COPY,$(TRUNNER_BINARY),$(TRUNNER_EXTRA_FILES)) - $(Q)rsync -aq $(if $(PERMISSIVE),--ignore-missing-args) $$^ $(TRUNNER_OUTPUT)/ -endif - -# some X.test.o files have runtime dependencies on Y.bpf.o files -$(OUTPUT)/$(TRUNNER_BINARY): | $(TRUNNER_BPF_OBJS) - -$(OUTPUT)/$(TRUNNER_BINARY): $(if $(filter test_progs%,$1),$(if $(PERMISSIVE),$$(wildcard $(TRUNNER_TEST_OBJS)),$(TRUNNER_TEST_OBJS)),$(TRUNNER_TEST_OBJS)) \ - $(TRUNNER_EXTRA_OBJS) $$(BPFOBJ) \ - $(TRUNNER_LIB_OBJS) \ - $(TRUNNER_BPFTOOL) \ - $(OUTPUT)/veristat \ - | $(TRUNNER_BINARY)-extras \ - $(if $(filter test_progs%,$1),$(if $(PERMISSIVE),$(TRUNNER_TEST_OBJS))) - $$(call msg,BINARY,,$$@) - $(Q)$$(CC) $$(CFLAGS) $(if $(filter test_progs%,$1),$(if $(PERMISSIVE),$$(filter %.a %.o,$$(wildcard $(TRUNNER_TEST_OBJS)) $$(filter-out $(TRUNNER_TEST_OBJS),$$^)),$$(filter %.a %.o,$$^)),$$(filter %.a %.o,$$^)) $$(LDLIBS) $$(LLVM_LDLIBS) $$(LDFLAGS) $$(LLVM_LDFLAGS) -o $$@ - $(Q)ln -sf $(if $2,..,.)/tools/build/bpftool/$(USE_BOOTSTRAP)bpftool \ - $(OUTPUT)/$(if $2,$2/)bpftool - -endef - -VERIFY_SIG_SETUP := $(CURDIR)/verify_sig_setup.sh -VERIFY_SIG_HDR := verification_cert.h -VERIFICATION_CERT := $(BUILD_DIR)/signing_key.der -PRIVATE_KEY := $(BUILD_DIR)/signing_key.pem - -$(VERIFICATION_CERT) $(PRIVATE_KEY): $(VERIFY_SIG_SETUP) +# genkey writes both outputs. Pattern rules group them even on make +# versions before 4.3, preventing concurrent genkey invocations. +# Restrict the pattern to *_key files because genkey uses fixed names. +$(BUILD_DIR)/%_key.pem $(BUILD_DIR)/%_key.der: $(VERIFY_SIG_SETUP) $(Q)mkdir -p $(BUILD_DIR) $(Q)$(VERIFY_SIG_SETUP) genkey $(BUILD_DIR) @@ -825,108 +403,179 @@ LIBARENA_MAKE_ARGS = \ BPF_TARGET_ENDIAN="$(BPF_TARGET_ENDIAN)" \ Q="$(Q)" -LIBARENA_BPF_DEPS := $(wildcard libarena/Makefile \ - libarena/include/* \ - libarena/include/libarena/* \ - libarena/src/* \ - libarena/selftests/* \ - libarena/*.bpf.o) +LIBARENA_COMMON_DEPS := $(wildcard libarena/Makefile \ + libarena/include/* \ + libarena/include/libarena/* \ + libarena/src/*) +LIBARENA_BPF_DEPS := $(LIBARENA_COMMON_DEPS) $(wildcard libarena/selftests/*) +LIBARENA_BENCH_BPF_DEPS := $(LIBARENA_COMMON_DEPS) $(wildcard libarena/benchs/*) -LIBARENA_SKEL := libarena/libarena.skel.h +LIBARENA_BENCH_SKEL := libarena/libarena_bench.skel.h $(LIBARENA_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BPF_DEPS) +$(MAKE) -C libarena libarena.skel.h $(LIBARENA_MAKE_ARGS) -ifneq ($(CLANG_HAS_ARENA_ASAN),) -LIBARENA_ASAN_SKEL := libarena/libarena_asan.skel.h -CFLAGS += -DHAS_BPF_ARENA_ASAN +$(LIBARENA_BENCH_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BENCH_BPF_DEPS) | $(LIBARENA_SKEL) + +$(MAKE) -C libarena benchmarks $(LIBARENA_MAKE_ARGS) +ifneq ($(CLANG_HAS_ARENA_ASAN),) $(LIBARENA_ASAN_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BPF_DEPS) +$(MAKE) -C libarena libarena_asan.skel.h $(LIBARENA_MAKE_ARGS) endif -# Define test_progs test runner. -TRUNNER_TESTS_DIR := prog_tests -TRUNNER_BPF_PROGS_DIR := progs -TRUNNER_EXTRA_SOURCES := test_progs.c \ - cgroup_helpers.c \ - trace_helpers.c \ - network_helpers.c \ - testing_helpers.c \ - btf_helpers.c \ - cap_helpers.c \ - unpriv_helpers.c \ - sysctl_helpers.c \ - netlink_helpers.c \ - jit_disasm_helpers.c \ - io_helpers.c \ - test_loader.c \ - xsk.c \ - disasm.c \ - disasm_helpers.c \ - json_writer.c \ - $(VERIFY_SIG_HDR) \ - flow_dissector_load.h \ - ip_check_defrag_frags.h \ - bpftool_helpers.c \ - usdt_1.c usdt_2.c \ - $(LIBARENA_SKEL) \ - $(LIBARENA_ASAN_SKEL) -TRUNNER_LIB_SOURCES := find_bit.c -TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \ - $(OUTPUT)/liburandom_read.so \ - $(OUTPUT)/xdp_synproxy \ - $(OUTPUT)/sign-file \ - $(OUTPUT)/uprobe_multi \ - $(TEST_KMOD_TARGETS) \ - ima_setup.sh \ - $(VERIFY_SIG_SETUP) \ - $(wildcard progs/btf_dump_test_case_*.c) \ - $(wildcard progs/*.bpf.o) -TRUNNER_BPF_BUILD_RULE := CLANG_BPF_BUILD_RULE -TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(CLANG_CFLAGS) -DENABLE_ATOMICS_TESTS -$(eval $(call DEFINE_TEST_RUNNER,test_progs)) - -# Define test_progs-no_alu32 test runner. -TRUNNER_BPF_BUILD_RULE := CLANG_NOALU32_BPF_BUILD_RULE -TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(CLANG_CFLAGS) -$(eval $(call DEFINE_TEST_RUNNER,test_progs,no_alu32)) - -# Define test_progs-cpuv4 test runner. +# #![no_std] looks for compiler_builtins too. Nothing of it is used. +ifneq ($(RUST_CORE),) +$(RUST_CORE): $(RUST_CORE_SRC) + $(call msg,RUSTC,,$@) + $(Q)mkdir -p $(@D) + +$(Q)$(RUSTC_BPF) -A warnings --edition 2024 --crate-name core --out-dir $(@D) $< + +$(Q)echo '#![feature(compiler_builtins)] #![compiler_builtins] #![no_std]' | \ + $(RUSTC_BPF) -A warnings --crate-name compiler_builtins --out-dir $(@D) - +endif + +# Generated test list headers + +define gen_tests_hdr + $(call msg,TEST-HDR,,$@) + $(Q)(echo '/* Generated header, do not edit */'; \ + sed -n -E 's/^void (serial_)?test_([a-zA-Z0-9_]+)\((void)?\).*/DEFINE_TEST(\2)/p' \ + $(@D)/*.c | sort) > $@ +endef + +prog_tests/tests.h: $(wildcard prog_tests/*.c) + $(gen_tests_hdr) + +map_tests/tests.h: $(wildcard map_tests/*.c) + $(gen_tests_hdr) + +# Test runner instances, one sub-make each (see Makefile.runner). + +# Share LLVM probe results with runners. Pass CC explicitly to +# runners so it does not affect libbpf's compiler selection. +export LLVM_LDLIBS LLVM_LDFLAGS + +# Pass assembled flags on the command line so runner assignments +# cannot append them again. +RUNNER_MAKE := $(MAKE) --no-print-directory -f Makefile.runner \ + OUTPUT=$(OUTPUT) CC='$(CC)' CLANG='$(CLANG)' \ + CFLAGS='$(CFLAGS)' LDFLAGS='$(LDFLAGS)' + +# Shared inputs prepared before runner sub-makes start. +# Also build veristat, which test_veristat executes. +RUNNER_PREREQS := $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(BPFTOOL) \ + $(TRUNNER_BPFTOOL) $(RESOLVE_BTFIDS) \ + $(OUTPUT)/veristat \ + $(VERIFY_SIG_HDR) $(PRIVATE_KEY) $(VERIFICATION_CERT) \ + $(LIBARENA_SKEL) $(LIBARENA_ASAN_SKEL) \ + prog_tests/tests.h map_tests/tests.h \ + $(RUNNER_OBJS) $(RUST_CORE) + +# Runtime fixtures for each test_progs flavor. +RUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \ + $(OUTPUT)/liburandom_read.so \ + $(OUTPUT)/xdp_synproxy \ + $(OUTPUT)/sign-file \ + $(OUTPUT)/uprobe_multi \ + $(TEST_KMOD_TARGETS) \ + ima_setup.sh \ + $(VERIFY_SIG_SETUP) \ + $(wildcard progs/btf_dump_test_case_*.c) \ + $(wildcard progs/*.bpf.o) \ + bpftool_btf_dump_sorted.expected \ + bpftool_btf_dump_unsorted.expected + +# Prepare fixtures in parallel with runner compilation. +# Copy into flavor and out-of-tree directories; the in-tree +# default runner uses the originals. +RUNNER_EXTRA_DIRS := $(addprefix $(OUTPUT)/,$(TEST_INST_SUBDIRS)) +ifneq ($(realpath $(OUTPUT)),$(CURDIR)) +RUNNER_EXTRA_DIRS += $(OUTPUT) +test_progs: $(OUTPUT)/runner-extras +else +test_progs: $(RUNNER_EXTRA_FILES) +endif +RUNNER_EXTRAS := $(addsuffix /runner-extras,$(RUNNER_EXTRA_DIRS)) + +.PHONY: $(RUNNER_EXTRAS) +$(RUNNER_EXTRAS): $(RUNNER_EXTRA_FILES) + $(Q)mkdir -p $(@D) + $(Q)rsync -aq $(if $(PERMISSIVE),--ignore-missing-args) \ + $(RUNNER_EXTRA_FILES) $(@D)/ + +all: $(RUNNER_EXTRAS) +$(addprefix test_progs-,$(TEST_INST_SUBDIRS)): test_progs-%: \ + $(OUTPUT)/%/runner-extras + +# The main Makefile cannot tell whether $(OUTPUT)/test_progs is stale - +# only the runner sub-make knows its full dependency graph. FORCE makes +# the delegating rules below always run; their dependents still rebuild +# on mtime only. +FORCE: + +# Build default BPF outputs here for standalone consumers and install. +# Standalone consumers depend on individual skeletons; unflavored +# runners depend on the complete set before their sub-makes start. +RDIR := $(OUTPUT) +FLAVOR := +BINARY := test_progs +BPF_CC := $(CLANG) +BPF_CC_MSG := CLNG-BPF +BPF_SYS_INCLUDES := $(CLANG_SYS_INCLUDES) +BPF_CC_FLAGS := -O2 $(BPF_TARGET_ENDIAN) -mcpu=v3 +BPF_DEFINES := -DENABLE_ATOMICS_TESTS +include Makefile.skel + +DEFAULT_RUNNER_ARGS := RUNNER=test_progs FLAVOR= TESTS_DIR=prog_tests \ + BPF_CC='$(BPF_CC)' BPF_CC_MSG=$(BPF_CC_MSG) \ + BPF_SYS_INCLUDES='$(BPF_SYS_INCLUDES)' \ + BPF_CC_FLAGS='$(BPF_CC_FLAGS)' BPF_DEFINES=$(BPF_DEFINES) +CLANG_RUNNER_ARGS := BPF_CC='$(CLANG)' BPF_CC_MSG=CLNG-BPF \ + BPF_SYS_INCLUDES='$(CLANG_SYS_INCLUDES)' + +$(OUTPUT)/test_progs: $(RUNNER_PREREQS) $(BPF_OBJS) $(ALL_SKELS) FORCE + +$(Q)$(RUNNER_MAKE) $(DEFAULT_RUNNER_ARGS) + +$(OUTPUT)/test_progs-no_alu32: $(RUNNER_PREREQS) FORCE + +$(Q)$(RUNNER_MAKE) RUNNER=test_progs FLAVOR=no_alu32 TESTS_DIR=prog_tests \ + $(CLANG_RUNNER_ARGS) BPF_CC_FLAGS='-O2 $(BPF_TARGET_ENDIAN) -mcpu=v2' + ifneq ($(CLANG_CPUV4),) -TRUNNER_BPF_BUILD_RULE := CLANG_CPUV4_BPF_BUILD_RULE -TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(CLANG_CFLAGS) -DENABLE_ATOMICS_TESTS -$(eval $(call DEFINE_TEST_RUNNER,test_progs,cpuv4)) +$(OUTPUT)/test_progs-cpuv4: $(RUNNER_PREREQS) FORCE + +$(Q)$(RUNNER_MAKE) RUNNER=test_progs FLAVOR=cpuv4 TESTS_DIR=prog_tests \ + $(CLANG_RUNNER_ARGS) BPF_CC_FLAGS='-O2 $(BPF_TARGET_ENDIAN) -mcpu=v4' \ + BPF_DEFINES=-DENABLE_ATOMICS_TESTS endif -# Define test_progs BPF-GCC-flavored test runner. ifneq ($(BPF_GCC),) -TRUNNER_BPF_BUILD_RULE := GCC_BPF_BUILD_RULE -TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(call get_sys_includes,gcc,) -$(eval $(call DEFINE_TEST_RUNNER,test_progs,bpf_gcc)) +# Sys includes come from the *host* gcc, not $(BPF_GCC), as the rule +# this replaces had it. +GCC_SYS_INCLUDES := $(call get_sys_includes,gcc,) + +$(OUTPUT)/test_progs-bpf_gcc: $(RUNNER_PREREQS) FORCE + +$(Q)$(RUNNER_MAKE) RUNNER=test_progs FLAVOR=bpf_gcc TESTS_DIR=prog_tests \ + BPF_CC='$(BPF_GCC)' BPF_CC_MSG=GCC-BPF \ + BPF_SYS_INCLUDES='$(GCC_SYS_INCLUDES)' \ + BPF_CC_FLAGS='-DBPF_NO_PRESERVE_ACCESS_INDEX -Wno-attributes -O2' endif -# Define test_maps test runner. -TRUNNER_TESTS_DIR := map_tests -TRUNNER_BPF_PROGS_DIR := progs -TRUNNER_EXTRA_SOURCES := test_maps.c -TRUNNER_LIB_SOURCES := -TRUNNER_EXTRA_FILES := -TRUNNER_BPF_BUILD_RULE := $$(error no BPF objects should be built) -TRUNNER_BPF_CFLAGS := -$(eval $(call DEFINE_TEST_RUNNER,test_maps)) +# test_maps compiles map_tests/*.c against the default flavor's skeletons +$(OUTPUT)/test_maps: $(RUNNER_PREREQS) $(BPF_OBJS) \ + $(ALL_SKELS) FORCE + +$(Q)$(RUNNER_MAKE) RUNNER=test_maps FLAVOR= TESTS_DIR=map_tests + +TEST_GEN_FILES += $(BPF_OBJS) # Define test_verifier test runner. # It is much simpler than test_maps/test_progs and sufficiently different from # them (e.g., test.h is using completely pattern), that it's worth just # explicitly defining all the rules explicitly. verifier/tests.h: verifier/*.c - $(shell ( cd verifier/; \ + $(Q)( cd verifier/; \ echo '/* Generated header, do not edit */'; \ echo '#ifdef FILL_ARRAY'; \ ls *.c 2> /dev/null | sed -e 's@\(.*\)@#include \"\1\"@'; \ echo '#endif' \ - ) > verifier/tests.h) + ) > verifier/tests.h $(OUTPUT)/test_verifier: test_verifier.c verifier/tests.h $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ @@ -962,11 +611,11 @@ $(OUTPUT)/test_cpp: test_cpp.cpp $(OUTPUT)/test_core_extern.skel.h $(BPFOBJ) $(call msg,CXX,,$@) $(Q)$(CXX) $(CXXFLAGS) $(filter %.a %.o %.cpp,$^) $(LDLIBS) -o $@ -# Benchmark runner +# Benchmark runner. Every benchs/bench_*.c is compiled and linked in. +BENCH_OBJS := $(sort $(patsubst benchs/%.c,$(OUTPUT)/%.o,$(wildcard benchs/bench_*.c))) $(OUTPUT)/bench_%.o: benchs/bench_%.c bench.h $(BPFOBJ) $(call msg,CC,,$@) - $(Q)$(CC) $(CFLAGS) -O2 -c $(filter %.c,$^) $(LDLIBS) -o $@ $(if $(PERMISSIVE),|| \ - ($(RM) $@; printf ' %-12s %s\n' 'SKIP-BENCH' '$(notdir $@)' 1>&2)) + $(Q)$(CC) $(CFLAGS) -O2 -c $(filter %.c,$^) $(LDLIBS) -o $@ $(call skip_on_fail,BENCH) $(OUTPUT)/bench_rename.o: $(OUTPUT)/test_overhead.skel.h $(OUTPUT)/bench_trigger.o: $(OUTPUT)/trigger_bench.skel.h $(OUTPUT)/bench_ringbufs.o: $(OUTPUT)/ringbuf_bench.skel.h \ @@ -986,6 +635,7 @@ $(OUTPUT)/bench_sockmap.o: $(OUTPUT)/bench_sockmap_prog.skel.h $(OUTPUT)/bench_lpm_trie_map.o: $(OUTPUT)/lpm_trie_bench.skel.h $(OUTPUT)/lpm_trie_map.skel.h $(OUTPUT)/bench_bpf_nop.o: $(OUTPUT)/bpf_nop_bench.skel.h bench_bpf_timing.h $(OUTPUT)/bench_xdp_lb.o: $(OUTPUT)/xdp_lb_bench.skel.h bench_bpf_timing.h +$(OUTPUT)/bench_libarena.o: $(LIBARENA_BENCH_SKEL) $(OUTPUT)/bench_bpf_timing.o: bench_bpf_timing.h $(OUTPUT)/bench.o: bench.h testing_helpers.h $(BPFOBJ) $(OUTPUT)/bench: LDLIBS += -lm @@ -993,32 +643,11 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \ $(TESTING_HELPERS) \ $(TRACE_HELPERS) \ $(CGROUP_HELPERS) \ - $(OUTPUT)/bench_count.o \ - $(OUTPUT)/bench_rename.o \ - $(OUTPUT)/bench_trigger.o \ - $(OUTPUT)/bench_ringbufs.o \ - $(OUTPUT)/bench_bloom_filter_map.o \ - $(OUTPUT)/bench_bpf_loop.o \ - $(OUTPUT)/bench_bpf_for.o \ - $(OUTPUT)/bench_strncmp.o \ - $(OUTPUT)/bench_bpf_hashmap_full_update.o \ - $(OUTPUT)/bench_local_storage.o \ - $(OUTPUT)/bench_local_storage_rcu_tasks_trace.o \ - $(OUTPUT)/bench_bpf_hashmap_lookup.o \ - $(OUTPUT)/bench_local_storage_create.o \ - $(OUTPUT)/bench_htab_mem.o \ - $(OUTPUT)/bench_bpf_crypto.o \ - $(OUTPUT)/bench_sockmap.o \ - $(OUTPUT)/bench_lpm_trie_map.o \ - $(OUTPUT)/bench_bpf_timing.o \ - $(OUTPUT)/bench_bpf_nop.o \ - $(OUTPUT)/bench_xdp_lb.o \ + $(BENCH_OBJS) \ $(OUTPUT)/usdt_1.o \ - $(OUTPUT)/usdt_2.o \ - # + $(OUTPUT)/usdt_2.o $(call msg,BINARY,,$@) - $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@ $(if $(PERMISSIVE),|| \ - ($(RM) $@; printf ' %-12s %s\n' 'SKIP-LINK' '$(notdir $@) (some benchmarks may have been skipped)' 1>&2)) + $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@ $(call skip_on_fail,LINK,,(some benchmarks may have been skipped)) # This works around GCC warning about snprintf truncating strings like: # diff --git a/tools/testing/selftests/bpf/Makefile.buildvars b/tools/testing/selftests/bpf/Makefile.buildvars new file mode 100644 index 000000000000..ff3476bc40a4 --- /dev/null +++ b/tools/testing/selftests/bpf/Makefile.buildvars @@ -0,0 +1,187 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Toolchain, path and flag definitions for the BPF selftests build, +# kept apart from the rules so that more than one makefile can include +# them. Included by the Makefile after ../lib.mk and by Makefile.runner +# in its place - the note above the Makefile's include explains how the +# flags are ordered around it. + +TOOLSDIR := $(abspath ../../..) +LIBDIR := $(TOOLSDIR)/lib +BPFDIR := $(LIBDIR)/bpf +TOOLSINCDIR := $(TOOLSDIR)/include +TOOLSARCHINCDIR := $(TOOLSDIR)/arch/$(SRCARCH)/include +APIDIR := $(TOOLSINCDIR)/uapi +ifneq ($(O),) +GENDIR := $(O)/include/generated +else +GENDIR := $(abspath ../../../../include/generated) +endif +GENHDR := $(GENDIR)/autoconf.h +PKG_CONFIG ?= $(CROSS_COMPILE)pkg-config + +ifneq ($(wildcard $(GENHDR)),) + GENFLAGS := -DHAVE_GENHDR +endif + +ifdef ASAN +SAN_CFLAGS ?= -fsanitize=address -fno-omit-frame-pointer +else +SAN_CFLAGS ?= +endif +SAN_LDFLAGS ?= $(SAN_CFLAGS) +RELEASE ?= +OPT_FLAGS ?= $(if $(RELEASE),-O2,-O0) + +LIBELF_CFLAGS := $(shell $(PKG_CONFIG) libelf --cflags 2>/dev/null) +LIBELF_LIBS := $(shell $(PKG_CONFIG) libelf --libs 2>/dev/null || echo -lelf) + +# When BPF_STRICT_BUILD is 1, any BPF object, skeleton, test object, or +# benchmark compilation failure is fatal. Set to 0 to tolerate failures +# and continue building the remaining tests. +BPF_STRICT_BUILD ?= 1 +PERMISSIVE := $(filter 0,$(BPF_STRICT_BUILD)) + +# Permissive-mode recipe suffix: on failure, remove the target, emit a +# SKIP marker and report success so the rest of the build continues. +# $(1) - SKIP tag (BPF, TEST, BENCH, SKEL, LINK, ...) +# $(2) - optional extra files to remove alongside the target +# $(3) - optional note appended to the SKIP message +skip_on_fail = $(if $(PERMISSIVE),|| { $(RM) $@ $(2); \ + printf ' %-12s %s\n' 'SKIP-$(1)' '$(notdir $@)$(if $(3), $(3))' 1>&2; }) + +# Permissive-mode recipe prefix: when a prerequisite object is missing +# (its compile failed and was skipped), remove the target and $(3), +# emit a SKIP marker and report success without running the rest of +# the recipe. +# $(1) - SKIP tag +# $(2) - input files to check +# $(3) - optional extra files to remove alongside the target +skip_if_missing = $(if $(PERMISSIVE),for f in $(2); do [ -f $$f ] || { \ + $(RM) $@ $(3); \ + printf ' %-12s %s\n' 'SKIP-$(1)' '$(notdir $@)' 1>&2; exit 0; }; done;) + +ifeq ($(srctree),) +srctree := $(patsubst %/,%,$(dir $(CURDIR))) +srctree := $(patsubst %/,%,$(dir $(srctree))) +srctree := $(patsubst %/,%,$(dir $(srctree))) +srctree := $(patsubst %/,%,$(dir $(srctree))) +endif + +COMMON_CFLAGS = -g $(OPT_FLAGS) -rdynamic -std=gnu11 \ + -Wall -Werror -fno-omit-frame-pointer \ + -Wno-unused-but-set-variable \ + $(GENFLAGS) $(SAN_CFLAGS) $(LIBELF_CFLAGS) \ + -I$(CURDIR) -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ + -I$(TOOLSINCDIR) -I$(TOOLSARCHINCDIR) -I$(APIDIR) -I$(OUTPUT) \ + -I$(CURDIR)/libarena/include +LDLIBS += $(LIBELF_LIBS) -lz -lrt -lpthread + +PCAP_CFLAGS := $(shell $(PKG_CONFIG) --cflags libpcap 2>/dev/null && echo "-DTRAFFIC_MONITOR=1") +PCAP_LIBS := $(shell $(PKG_CONFIG) --libs libpcap 2>/dev/null) +LDLIBS += $(PCAP_LIBS) + +ifneq ($(LLVM),) +# Silence some warnings when compiled with clang +CLANG_WARN_CFLAGS := -Wno-unused-command-line-argument +endif + +# Check whether clang supports BPF address sanitizer (requires LLVM 22+). +# This runs after ../lib.mk has resolved CLANG from LLVM=<suffix or path>, +# so it probes the compiler that builds the BPF objects. +CLANG_HAS_ARENA_ASAN := $(shell echo 'int x;' | \ + $(CLANG) --target=bpf -fsanitize=kernel-address \ + -mllvm -asan-shadow-addr-space=1 \ + -x c -c - -o /dev/null 2>/dev/null && echo 1) + +SCRATCH_DIR := $(OUTPUT)/tools +BUILD_DIR := $(SCRATCH_DIR)/build +INCLUDE_DIR := $(SCRATCH_DIR)/include +BPFOBJ := $(BUILD_DIR)/libbpf/libbpf.a +ifneq ($(CROSS_COMPILE),) +HOST_BUILD_DIR := $(BUILD_DIR)/host +HOST_SCRATCH_DIR := $(OUTPUT)/host-tools +HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include +else +HOST_BUILD_DIR := $(BUILD_DIR) +HOST_SCRATCH_DIR := $(SCRATCH_DIR) +HOST_INCLUDE_DIR := $(INCLUDE_DIR) +endif +RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids + +# Programs in Rust are built by upstream rustc. It has no prebuilt core for +# the bpf target, so it has to come with the source of core: +# rustup component add rust-src +# core is built as edition 2024, which it is since rustc 1.87. +# rustc emits LLVM bitcode and clang makes the object of it, so clang has to be +# 23 or newer and not older than LLVM of rustc. +# Otherwise RUST_CORE is empty and the tests are skipped. +RUSTC ?= rustc +RUST_CORE_SRC := $(wildcard $(shell $(RUSTC) --print sysroot 2>/dev/null)$\ + /lib/rustlib/src/rust/library/core/src/lib.rs) +ifneq ($(RUST_CORE_SRC),) +ifeq ($(shell { clang=$$(echo __clang_major__ | $(CLANG) -E -P -x c -) && \ + llvm=$$($(srctree)/scripts/rustc-llvm-version.sh $(RUSTC)) && \ + [ $$($(srctree)/scripts/rustc-version.sh $(RUSTC)) -ge 108700 ] && \ + [ $$clang -ge 23 ] && [ $$clang -ge $$((llvm / 10000)) ]; } \ + 2>/dev/null && echo y),y) +RUST_CORE := $(BUILD_DIR)/rust/libcore.rlib +endif +endif +# RUSTC_BOOTSTRAP=1 is to build core with a stable rustc, like the kernel does. +# panic=abort is a stop gap until panic=unwind is supported. +RUSTC_BPF = RUSTC_BOOTSTRAP=1 $(RUSTC) -O -C panic=abort --crate-type rlib \ + --target $(if $(IS_LITTLE_ENDIAN),bpfel,bpfeb)-unknown-none \ + -L $(dir $(RUST_CORE)) + +DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool +ifneq ($(CROSS_COMPILE),) +CROSS_BPFTOOL := $(SCRATCH_DIR)/sbin/bpftool +TRUNNER_BPFTOOL := $(CROSS_BPFTOOL) +USE_BOOTSTRAP := "" +else +TRUNNER_BPFTOOL := $(DEFAULT_BPFTOOL) +USE_BOOTSTRAP := "bootstrap/" +endif + +BPFTOOL ?= $(DEFAULT_BPFTOOL) + +# Determine target endianness. +IS_LITTLE_ENDIAN := $(shell $(CC) -dM -E - </dev/null | \ + grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__') +MENDIAN:=$(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian) + +BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN) \ + -I$(INCLUDE_DIR) -I$(CURDIR) -I$(APIDIR) \ + -I$(TOOLSINCDIR) -I$(CURDIR)/libarena/include \ + -I$(abspath $(OUTPUT)/../usr/include) \ + -std=gnu11 \ + -fno-strict-aliasing \ + -Wno-microsoft-anon-tag \ + -fms-extensions \ + -Wno-compare-distinct-pointer-types \ + -Wno-initializer-overrides \ + # +# TODO: enable me -Wsign-compare + +VERIFY_SIG_SETUP := $(CURDIR)/verify_sig_setup.sh +VERIFICATION_CERT := $(BUILD_DIR)/signing_key.der +PRIVATE_KEY := $(BUILD_DIR)/signing_key.pem + +LIBARENA_SKEL := libarena/libarena.skel.h + +ifneq ($(CLANG_HAS_ARENA_ASAN),) +LIBARENA_ASAN_SKEL := libarena/libarena_asan.skel.h +CFLAGS += -DHAS_BPF_ARENA_ASAN +endif + +RUNNER_OBJS-test_progs := $(addprefix $(OUTPUT)/, \ + test_progs.o cgroup_helpers.o trace_helpers.o \ + network_helpers.o testing_helpers.o btf_helpers.o \ + cap_helpers.o unpriv_helpers.o sysctl_helpers.o \ + netlink_helpers.o jit_disasm_helpers.o io_helpers.o \ + test_loader.o xsk.o disasm.o disasm_helpers.o \ + json_writer.o bpftool_helpers.o usdt_1.o usdt_2.o \ + find_bit.o) +RUNNER_OBJS-test_maps := $(addprefix $(OUTPUT)/, \ + test_maps.o testing_helpers.o) diff --git a/tools/testing/selftests/bpf/Makefile.runner b/tools/testing/selftests/bpf/Makefile.runner new file mode 100644 index 000000000000..d266c4246888 --- /dev/null +++ b/tools/testing/selftests/bpf/Makefile.runner @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Build one test_progs flavor or test_maps. +# Invoked by the main Makefile after shared build inputs are ready. +# Flavored outputs live in $(OUTPUT)/$(FLAVOR); unflavored outputs +# share $(OUTPUT) with the main Makefile. +# +# Command-line parameters: +# OUTPUT selftests output directory, resolved by lib.mk +# RUNNER test_progs | test_maps +# FLAVOR empty | no_alu32 | cpuv4 | bpf_gcc +# TESTS_DIR prog_tests | map_tests +# BPF_* see Makefile.skel; BPF_CC is empty for test_maps +# CC, CLANG compilers resolved by the main Makefile +# CFLAGS, LDFLAGS assembled flags, including lib.mk's additions +# +# Exported by the main Makefile: +# LLVM_LDLIBS, LLVM_LDFLAGS: LLVM feature-probe results + +include ../../../build/Build.include +include ../../../scripts/Makefile.arch +include ../../../scripts/Makefile.include + +# Same message helpers as ../lib.mk, which only the top level includes. +ifeq ($(V),1) +Q = +msg = +else +Q = @ +msg = @printf ' %-8s%s %s%s\n' "$(1)" "$(if $(2), [$(2)])" "$(notdir $(3))" "$(if $(4), $(4))"; +MAKEFLAGS += --no-print-directory +endif + +include Makefile.buildvars + +BINARY := $(RUNNER)$(if $(FLAVOR),-$(FLAVOR)) +RDIR := $(OUTPUT)$(if $(FLAVOR),/$(FLAVOR)) + +all: $(OUTPUT)/$(BINARY) +.PHONY: all + +# Delete partially updated (corrupted) files on error +.DELETE_ON_ERROR: + +# No built-in rules: every file built here has an explicit rule, and the +# objects that come from the main Makefile have to be an error when +# missing rather than a silent rebuild by the built-in %.o recipes. +MAKEFLAGS += -r + +# Permissive build behaviour (skip-on-failure compile, partial-link) only +# applies to test_progs and its flavors; runners that use strong cross-object +# references (e.g. test_maps) keep strict semantics even when permissive. +PERMISSIVE_TESTS := $(if $(filter test_progs,$(RUNNER)),$(PERMISSIVE)) + +$(RDIR): + $(call msg,MKDIR,,$@) + $(Q)mkdir -p $@ + +# Per-runner sources + +ifeq ($(RUNNER),test_progs) +GENERATED_HDRS := $(LIBARENA_SKEL) $(LIBARENA_ASAN_SKEL) +endif + +TEST_SRCS := $(notdir $(wildcard $(TESTS_DIR)/*.c)) +TEST_OBJS := $(patsubst %.c,$(RDIR)/%.test.o,$(TEST_SRCS)) +TEST_DEPS := $(TEST_OBJS:.o=.d) + +# BPF objects and skeletons - rules shared with the main Makefile + +include Makefile.skel + +# When the compiler generates a %.d file, only skel basenames (not +# full paths) are specified as prerequisites for corresponding %.o +# file. vpath directives below instruct make to search for skel files +# in $(RDIR), if they are not present in the working directory. +vpath %.skel.h $(RDIR) +vpath %.lskel.h $(RDIR) +vpath %.subskel.h $(RDIR) + +# Test objects and runner binary + +$(RDIR)/resolve_btfids.test.o: $(RESOLVE_BTFIDS) $(RDIR)/btf_data.bpf.o +$(RDIR)/resolve_btfids.test.o: private TEST_NEEDS_BTFIDS = 1 + +# compile individual test files +# Note: we cd into output directory to ensure embedded BPF object is found +# The %.test.d dependency files are a side effect of the -MMD below; the +# separate no-recipe rule for them orders the first compilation after +# skeleton generation and re-orders test object compilation when headers +# change. +$(TEST_OBJS): $(RDIR)/%.test.o: $(TESTS_DIR)/%.c | $(RDIR)/%.test.d + $(call msg,TEST-OBJ,$(BINARY),$@) + $(Q)(cd $(@D) && $(CC) -I. $(CFLAGS) -MMD -MT $@ -c $(CURDIR)/$< $(LDLIBS) -o $(@F)) $(if \ + $(PERMISSIVE_TESTS),$(call skip_on_fail,TEST)) + $(if $(TEST_NEEDS_BTFIDS), \ + $(Q)if [ -f $@ ]; then \ + $(if $(filter 1,$(V)),true,printf ' %-8s%s %s\n' "BTFIDS" " [$(BINARY)]" "$(notdir $@)"); \ + $(RESOLVE_BTFIDS) --btf $(RDIR)/btf_data.bpf.o $@; \ + $(RESOLVE_BTFIDS) --patch_btfids $@.BTF_ids $@; \ + fi) + +$(TEST_DEPS): $(RDIR)/%.test.d: $(TESTS_DIR)/%.c $(GENERATED_HDRS) $(BPFOBJ) \ + | $(RDIR) $(ALL_SKELS) + +include $(wildcard $(TEST_DEPS)) + +cmd_runner_link = $(CC) $(CFLAGS) \ + $(if $(PERMISSIVE_TESTS),$(wildcard $(TEST_OBJS)) \ + $(filter-out $(TEST_OBJS),$(filter %.a %.o,$^)), \ + $(filter %.a %.o,$^)) \ + $(LDLIBS) $(LLVM_LDLIBS) $(LDFLAGS) $(LLVM_LDFLAGS) -o $@ + +# BPF objects are also needed at runtime. +# In permissive mode, existing test objects are normal prerequisites +# so edits trigger relinking. Order-only prerequisites attempt all +# tests; the recipe links the objects that exist afterward. +$(OUTPUT)/$(BINARY): $(if $(PERMISSIVE_TESTS),$(wildcard $(TEST_OBJS)),$(TEST_OBJS)) \ + $(RUNNER_OBJS-$(RUNNER)) $(BPFOBJ) \ + $(TRUNNER_BPFTOOL) \ + | $(BPF_OBJS) \ + $(if $(PERMISSIVE_TESTS),$(TEST_OBJS)) + $(call msg,BINARY,,$@) + $(Q)$(cmd_runner_link) + $(Q)ln -sf $(if $(FLAVOR),..,.)/tools/build/bpftool/$(USE_BOOTSTRAP)bpftool $(RDIR)/bpftool diff --git a/tools/testing/selftests/bpf/Makefile.skel b/tools/testing/selftests/bpf/Makefile.skel new file mode 100644 index 000000000000..6570c48f92b0 --- /dev/null +++ b/tools/testing/selftests/bpf/Makefile.skel @@ -0,0 +1,167 @@ +# SPDX-License-Identifier: GPL-2.0 +# BPF object and skeleton rules for one flavor. +# Included by the main Makefile for default outputs and by runners +# for their flavor. Unflavored runners require the default outputs first. +# +# Parameters (variables defined by the includer): +# RDIR output directory of this flavor +# FLAVOR flavor name, empty for the default +# BINARY runner binary name, for build-log messages +# BPF_CC BPF compiler +# BPF_CC_MSG build-log tag (CLNG-BPF/GCC-BPF) +# BPF_CC_FLAGS per-flavor compiler flags +# BPF_SYS_INCLUDES system include flags for $(BPF_CC) +# BPF_DEFINES extra defines (-DENABLE_ATOMICS_TESTS) + +# BPF objects and skeletons + +ifneq ($(BPF_CC),) + +BPF_SRCS := $(notdir $(wildcard progs/*.c)) +BPF_OBJS := $(patsubst %.c,$(RDIR)/%.bpf.o,$(BPF_SRCS)) + +SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c \ + data_in_arena_extern.c data_in_arena_nomap.c + +LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \ + linked_vars.skel.h linked_maps.skel.h linked_arena.skel.h \ + test_subskeleton.skel.h test_subskeleton_lib.skel.h \ + test_usdt.skel.h tracing_multi.skel.h \ + tracing_multi_module.skel.h \ + tracing_multi_intersect.skel.h \ + tracing_multi_session.skel.h + +LSKELS := fexit_sleep.c trace_printk.c trace_vprintk.c map_ptr_kern.c \ + core_kern.c core_kern_overflow.c test_ringbuf.c \ + test_ringbuf_n.c test_ringbuf_map_key.c test_ringbuf_write.c \ + test_ringbuf_overwrite.c callx_rodata.c + +LSKELS_SIGNED := fentry_test.c fexit_test.c atomics.c + +# Generate both light skeleton and libbpf skeleton for these +LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c kfunc_call_test.c \ + kfunc_call_test_subprog.c test_global_percpu_data.c +SKEL_BLACKLIST += $(LSKELS) $(LSKELS_SIGNED) + +test_static_linked.skel.h-deps := test_static_linked1.bpf.o test_static_linked2.bpf.o +linked_funcs.skel.h-deps := linked_funcs1.bpf.o linked_funcs2.bpf.o +linked_vars.skel.h-deps := linked_vars1.bpf.o linked_vars2.bpf.o +linked_maps.skel.h-deps := linked_maps1.bpf.o linked_maps2.bpf.o +linked_arena.skel.h-deps := linked_arena1.bpf.o linked_arena2.bpf.o +# In the subskeleton case, we want the test_subskeleton_lib.subskel.h file +# but that's created as a side-effect of the skel.h generation. +test_subskeleton.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o test_subskeleton.bpf.o +test_subskeleton_lib.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o +test_usdt.skel.h-deps := test_usdt.bpf.o test_usdt_multispec.bpf.o +tracing_multi.skel.h-deps := tracing_multi_attach.bpf.o tracing_multi_check.bpf.o +tracing_multi_module.skel.h-deps := tracing_multi_attach_module.bpf.o tracing_multi_check.bpf.o +tracing_multi_intersect.skel.h-deps := tracing_multi_intersect_attach.bpf.o tracing_multi_check.bpf.o +tracing_multi_session.skel.h-deps := tracing_multi_session_attach.bpf.o tracing_multi_check.bpf.o + +LINKED_BPF_OBJS := $(foreach skel,$(LINKED_SKELS),$($(skel)-deps)) +LINKED_BPF_SRCS := $(patsubst %.bpf.o,%.c,$(LINKED_BPF_OBJS)) + +SKELS := $(patsubst %.c,$(RDIR)/%.skel.h, \ + $(filter-out $(SKEL_BLACKLIST) $(LINKED_BPF_SRCS),$(BPF_SRCS))) +LSKELS_H := $(patsubst %.c,$(RDIR)/%.lskel.h,$(LSKELS) $(LSKELS_EXTRA)) +LSKELS_SIGNED_H := $(patsubst %.c,$(RDIR)/%.lskel.h,$(LSKELS_SIGNED)) +LINKED_SKELS_H := $(addprefix $(RDIR)/,$(LINKED_SKELS)) +ALL_SKELS := $(SKELS) $(LSKELS_H) $(LSKELS_SIGNED_H) $(LINKED_SKELS_H) + +HEADERS_FOR_BPF_OBJS := $(wildcard $(BPFDIR)/*.bpf.h) \ + $(wildcard $(CURDIR)/libarena/include/*.[ch]) \ + $(addprefix $(BPFDIR)/, bpf_core_read.h \ + bpf_endian.h \ + bpf_helpers.h \ + bpf_tracing.h) + +# The following tests contain C code that, although technically legal, +# triggers GCC warnings that cannot be disabled: declaration of +# anonymous struct types in function parameter lists. +progs/btf_dump_test_case_bitfields.c-bpf_gcc-CFLAGS := -Wno-error +progs/btf_dump_test_case_namespacing.c-bpf_gcc-CFLAGS := -Wno-error +progs/btf_dump_test_case_packing.c-bpf_gcc-CFLAGS := -Wno-error +progs/btf_dump_test_case_padding.c-bpf_gcc-CFLAGS := -Wno-error +progs/btf_dump_test_case_syntax.c-bpf_gcc-CFLAGS := -Wno-error + +cmd_bpf_compile = $(BPF_CC) $(BPF_CFLAGS) $(BPF_SYS_INCLUDES) $(BPF_DEFINES) \ + $($<-CFLAGS) $($<-$(FLAVOR)-CFLAGS) $(BPF_CC_FLAGS) -c $< -o $@ $(call skip_on_fail,BPF) + +$(BPF_OBJS): $(RDIR)/%.bpf.o: progs/%.c progs/*.h \ + $(INCLUDE_DIR)/vmlinux.h $(HEADERS_FOR_BPF_OBJS) \ + | $(RDIR) $(BPFOBJ) + $(call msg,$(BPF_CC_MSG),$(BINARY),$@) + $(Q)$(cmd_bpf_compile) + +GEN_SKEL := ./gen_bpf_skel.sh + +cmd_bpf_gen_skel = $(call skip_if_missing,SKEL,$<,$(@:.skel.h=.subskel.h))printf \ + ' %-8s%s %s\n' 'GEN-SKEL' ' [$(BINARY)]' '$(notdir $@)'; \ + BPFTOOL=$(BPFTOOL) $(GEN_SKEL) --name $* \ + --skel $@ --subskel $(@:.skel.h=.subskel.h) $< $(call \ + skip_on_fail,SKEL,$(@:.skel.h=.subskel.h)) + +$(SKELS): $(RDIR)/%.skel.h: $(RDIR)/%.bpf.o $(BPFTOOL) $(GEN_SKEL) | $(RDIR) + $(Q)$(cmd_bpf_gen_skel) + +cmd_bpf_gen_lskel = $(call skip_if_missing,SKEL,$<)printf \ + ' %-8s%s %s\n' 'GEN-SKEL' ' [$(BINARY)]' '$(notdir $@)'; \ + BPFTOOL=$(BPFTOOL) $(GEN_SKEL) --name $*_lskel --lskel \ + --skel $@ $< $(call skip_on_fail,SKEL) + +$(LSKELS_H): $(RDIR)/%.lskel.h: $(RDIR)/%.bpf.o $(BPFTOOL) $(GEN_SKEL) | $(RDIR) + $(Q)$(cmd_bpf_gen_lskel) + +cmd_bpf_gen_signed_lskel = $(call skip_if_missing,SKEL,$<)printf \ + ' %-8s%s %s\n' 'GEN-SKEL' ' [$(BINARY) (signed)]' '$(notdir $@)'; \ + BPFTOOL=$(BPFTOOL) PRIVATE_KEY=$(PRIVATE_KEY) \ + VERIFICATION_CERT=$(VERIFICATION_CERT) \ + $(GEN_SKEL) --sign --name $*_lskel \ + --skel $@ $< $(call skip_on_fail,SKEL) + +$(LSKELS_SIGNED_H): $(RDIR)/%.lskel.h: $(RDIR)/%.bpf.o $(BPFTOOL) \ + $(GEN_SKEL) $(PRIVATE_KEY) $(VERIFICATION_CERT) | $(RDIR) + $(Q)$(cmd_bpf_gen_signed_lskel) + +cmd_bpf_link_skel = $(call skip_if_missing,SKEL,$(addprefix $(RDIR)/,\ + $($*.skel.h-deps)),$(@:.skel.h=.subskel.h))printf \ + ' %-8s%s %s\n' 'LINK-BPF' ' [$(BINARY)]' '$(notdir $(@:.skel.h=.bpf.o))'; \ + printf ' %-8s%s %s\n' 'GEN-SKEL' ' [$(BINARY)]' '$(notdir $@)'; \ + BPFTOOL=$(BPFTOOL) $(GEN_SKEL) --name $* \ + --skel $@ --subskel $(@:.skel.h=.subskel.h) \ + $(addprefix $(RDIR)/,$($*.skel.h-deps)) $(call \ + skip_on_fail,SKEL,$(@:.skel.h=.subskel.h)) + +# .SECONDEXPANSION here allows to correctly expand %-deps variables as prerequisites +.SECONDEXPANSION: +$(LINKED_SKELS_H): $(RDIR)/%.skel.h: $$(addprefix $(RDIR)/,$$($$*.skel.h-deps)) \ + $(BPFTOOL) $(GEN_SKEL) | $(RDIR) + $(Q)$(cmd_bpf_link_skel) + +# Programs in Rust, see Makefile.buildvars. No skeletons: the objects may be absent. +ifneq ($(RUST_CORE),) +ifeq ($(BPF_CC),$(CLANG)) +RUST_OBJS := $(patsubst progs/%.rs,$(RDIR)/%.bpf.o,$(wildcard progs/*.rs)) +BPF_OBJS += $(RUST_OBJS) + +$(RUST_OBJS): $(RDIR)/%.bpf.o: progs/%.rs $(RUST_CORE) | $(RDIR) + $(call msg,RUSTC,$(BINARY),$@) + +$(Q)$(RUSTC_BPF) --edition 2021 -C debuginfo=2 --emit=llvm-bc \ + $(patsubst -mcpu=%,-C target-cpu=%,$(filter -mcpu=%,$(BPF_CC_FLAGS))) \ + -o $(dir $(RUST_CORE))$(BINARY)-$*.bc $< && \ + $(BPF_CC) $(BPF_CC_FLAGS) -c $(dir $(RUST_CORE))$(BINARY)-$*.bc -o $@ $(call skip_on_fail,BPF) +endif +endif + +# Programs in C++. No skeletons: gcc doesn't build them. +ifeq ($(BPF_CC),$(CLANG)) +CXX_OBJS := $(patsubst progs/%.cpp,$(RDIR)/%.bpf.o,$(wildcard progs/*.cpp)) +BPF_OBJS += $(CXX_OBJS) + +$(CXX_OBJS): $(RDIR)/%.bpf.o: progs/%.cpp | $(RDIR) + $(call msg,$(BPF_CC_MSG),$(BINARY),$@) + $(Q)$(BPF_CC) -x c++ -g -Wall -Werror -fexceptions -funwind-tables -fno-rtti \ + $(BPF_CC_FLAGS) -c $< -o $@ $(call skip_on_fail,BPF) +endif + +endif # BPF_CC diff --git a/tools/testing/selftests/bpf/README.rst b/tools/testing/selftests/bpf/README.rst index 07c834433b38..75ff1142c185 100644 --- a/tools/testing/selftests/bpf/README.rst +++ b/tools/testing/selftests/bpf/README.rst @@ -187,7 +187,7 @@ The symptom for ``bpf_iter/netlink`` looks like ... 15: (bf) r2 = r7 16: (0f) r2 += r1 - ; BPF_SEQ_PRINTF(seq, "%pK %-3d ", s, s->sk_protocol); + ; BPF_SEQ_PRINTF(seq, "%p %-3d ", s, s->sk_protocol); 17: (7b) *(u64 *)(r7 +0) = r2 only read is supported diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c index 465233ea5f79..d5f494184f9d 100644 --- a/tools/testing/selftests/bpf/bench.c +++ b/tools/testing/selftests/bpf/bench.c @@ -288,6 +288,7 @@ extern struct argp bench_crypto_argp; extern struct argp bench_sockmap_argp; extern struct argp bench_lpm_trie_map_argp; extern struct argp bench_xdp_lb_argp; +extern struct argp bench_libarena_argp; static const struct argp_child bench_parsers[] = { { &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 }, @@ -306,6 +307,7 @@ static const struct argp_child bench_parsers[] = { { &bench_sockmap_argp, 0, "bpf sockmap benchmark", 0 }, { &bench_lpm_trie_map_argp, 0, "LPM trie map benchmark", 0 }, { &bench_xdp_lb_argp, 0, "XDP load-balancer benchmark", 0 }, + { &bench_libarena_argp, 0, "libarena allocator benchmark", 0 }, {}, }; @@ -585,6 +587,8 @@ extern const struct bench bench_lpm_trie_delete; extern const struct bench bench_lpm_trie_free; extern const struct bench bench_bpf_nop; extern const struct bench bench_xdp_lb; +extern const struct bench bench_libarena_malloc; +extern const struct bench bench_libarena_calloc; static const struct bench *benchs[] = { &bench_count_global, @@ -669,6 +673,8 @@ static const struct bench *benchs[] = { &bench_lpm_trie_free, &bench_bpf_nop, &bench_xdp_lb, + &bench_libarena_malloc, + &bench_libarena_calloc, }; static void find_benchmark(void) diff --git a/tools/testing/selftests/bpf/benchs/bench_libarena.c b/tools/testing/selftests/bpf/benchs/bench_libarena.c new file mode 100644 index 000000000000..24e432244bf5 --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/bench_libarena.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <argp.h> +#include <limits.h> +#include <string.h> + +#include "bench.h" + +#include <libarena/common.h> +#include <libarena/asan.h> +#include <libarena/buddy.h> +#include <libarena/userspace.h> + +#include "libarena/libarena_bench.skel.h" + +static struct { + __u64 alloc_size; + __u64 nallocs; +} args = { + .alloc_size = 64, + .nallocs = 10000, +}; + +static struct { + struct libarena_bench *skel; + int bench_fd; + int reset_fd; +} ctx; + +enum { + ARG_LIBARENA_ALLOC_SIZE = 12000, + ARG_LIBARENA_NALLOCS, +}; + +static const struct argp_option opts[] = { + { "alloc_size", ARG_LIBARENA_ALLOC_SIZE, "BYTES", 0, + "Size of each arena allocation" }, + { "nallocs", ARG_LIBARENA_NALLOCS, "ITERS", 0, + "Number of allocation per measurement" }, + {}, +}; + +static error_t parse_arg(int key, char *arg, struct argp_state *state) +{ + unsigned long value; + + switch (key) { + case ARG_LIBARENA_ALLOC_SIZE: + value = strtoull(arg, NULL, 10); + if (!value || value >= UINT_MAX) { + fprintf(stderr, "invalid alloc_size: %ld", value); + argp_usage(state); + } + args.alloc_size = value; + break; + case ARG_LIBARENA_NALLOCS: + args.nallocs = strtoull(arg, NULL, 10); + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +const struct argp bench_libarena_argp = { + .options = opts, + .parser = parse_arg, +}; + +static void validate(void) +{ + if (env.consumer_cnt != 0) { + fprintf(stderr, "benchmark doesn't support consumers\n"); + exit(1); + } + + if (env.producer_cnt != 1) { + fprintf(stderr, "benchmark supports exactly one producer\n"); + exit(1); + } +} + +static void setup_common(void) +{ + struct arena_alloc_reserve_args reserve_args = { + .nr_pages = ARENA_RESERVE_PAGES_DFL, + }; + int err; + + setup_libbpf(); + + ctx.skel = libarena_bench__open_and_load(); + if (!ctx.skel) { + fprintf(stderr, "failed to open and load skeleton\n"); + exit(1); + } + + err = libarena_run_prog_args( + bpf_program__fd(ctx.skel->progs.arena_alloc_reserve), + &reserve_args, sizeof(reserve_args)); + if (err) { + fprintf(stderr, "failed to reserve arena pages: %d\n", err); + exit(1); + } + + err = libarena_run_prog( + bpf_program__fd(ctx.skel->progs.arena_buddy_reset)); + if (err) { + fprintf(stderr, "failed to initialize arena allocator: %d\n", err); + exit(1); + } + + ctx.skel->bss->bench_alloc_size = args.alloc_size; + ctx.skel->bss->bench_nallocs = args.nallocs; + ctx.reset_fd = bpf_program__fd(ctx.skel->progs.arena_buddy_reset); +} + +static void malloc_setup(void) +{ + setup_common(); + ctx.bench_fd = bpf_program__fd(ctx.skel->progs.bench_malloc); +} + +static void calloc_setup(void) +{ + setup_common(); + ctx.bench_fd = bpf_program__fd(ctx.skel->progs.bench_calloc); +} + +static void *producer(void *input) +{ + int err; + + while (true) { + err = libarena_run_prog(ctx.bench_fd); + if (err) { + fprintf(stderr, "libarena benchmark failed: %d\n", err); + exit(1); + } + + err = libarena_run_prog(ctx.reset_fd); + if (err) { + fprintf(stderr, "libarena alloc reset failed: %d\n", err); + exit(1); + } + } + + return NULL; +} + +static void measure(struct bench_res *res) +{ + res->duration_ns = atomic_swap(&ctx.skel->bss->bench_duration_ns, 0); + res->hits = atomic_swap(&ctx.skel->bss->bench_hits, 0); +} + +static void report_progress(int iter, struct bench_res *res, long delta_ns) +{ + double latency_ns = 0.0; + + if (res->hits) + latency_ns = res->duration_ns / (double)res->hits; + + printf("Iter %3d (%7.3lfus): latency %8.3lf ns/op (%ld allocations)\n", + iter, (delta_ns - 1000000000) / 1000.0, latency_ns, res->hits); +} + +static void report_final(struct bench_res res[], int res_cnt) +{ + unsigned long duration_ns = 0; + long hits = 0; + int i; + + for (i = 0; i < res_cnt; i++) { + duration_ns += res[i].duration_ns; + hits += res[i].hits; + } + + if (!hits || !res_cnt) { + printf("Summary: no runs measured\n"); + return; + } + + printf("Summary: %.3lf ns/op, %.0lf invocations for %u allocations/invocation)\n", + duration_ns / (double)hits, hits / (double)res_cnt, + ctx.skel->bss->bench_nallocs); +} + +const struct bench bench_libarena_malloc = { + .name = "libarena-malloc", + .argp = &bench_libarena_argp, + .validate = validate, + .setup = malloc_setup, + .producer_thread = producer, + .measure = measure, + .report_progress = report_progress, + .report_final = report_final, +}; + +const struct bench bench_libarena_calloc = { + .name = "libarena-calloc", + .argp = &bench_libarena_argp, + .validate = validate, + .setup = calloc_setup, + .producer_thread = producer, + .measure = measure, + .report_progress = report_progress, + .report_final = report_final, +}; diff --git a/tools/testing/selftests/bpf/benchs/run_bench_libarena.sh b/tools/testing/selftests/bpf/benchs/run_bench_libarena.sh new file mode 100755 index 000000000000..10afe4d52ebf --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/run_bench_libarena.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +source ./benchs/run_common.sh + +set -eufo pipefail + +RUN_BENCH="./bench -d3 -q" + +summarize_libarena() +{ + local bench="$1" + local summary + + summary=$(printf '%s\n' "$2" | tail -n1) + summary=${summary#Summary: } + printf "%-20s %s\n" "$bench" "$summary" +} + +header "libarena sequential malloc\n" + +for size in 16 64 256 1024 4096; do +subtitle "allocation size: $size" +printf "\t-------------------\n" + for nallocs in 10 50 100 500 1000 5000 10000; do + summarize_libarena "malloc:" \ + "$($RUN_BENCH --alloc_size "$size" --nallocs "$nallocs" libarena-malloc)" + summarize_libarena "calloc:" \ + "$($RUN_BENCH --alloc_size "$size" --nallocs "$nallocs" libarena-calloc)" + done +done diff --git a/tools/testing/selftests/bpf/bpf_kfuncs.h b/tools/testing/selftests/bpf/bpf_kfuncs.h index ae71e9b69051..1b408fe9e0fb 100644 --- a/tools/testing/selftests/bpf/bpf_kfuncs.h +++ b/tools/testing/selftests/bpf/bpf_kfuncs.h @@ -78,18 +78,29 @@ extern void bpf_key_put(struct bpf_key *key) __ksym; extern int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_ptr, const struct bpf_dynptr *sig_ptr, struct bpf_key *trusted_keyring) __ksym; - -struct dentry; -/* Description +/* + * Description * Returns xattr of a dentry * Returns * Error code */ +struct dentry; extern int bpf_get_dentry_xattr(struct dentry *dentry, const char *name, struct bpf_dynptr *value_ptr) __ksym __weak; - extern int bpf_set_dentry_xattr(struct dentry *dentry, const char *name__str, const struct bpf_dynptr *value_p, int flags) __ksym __weak; extern int bpf_remove_dentry_xattr(struct dentry *dentry, const char *name__str) __ksym __weak; +/* + * Description + * Attach a xattr to an inode that is being created, from a program on the + * inode_init_security LSM hook. *xattrs* and *xattr_count* must be the + * hook's own arguments, passed through unmodified. + * Returns + * 0 on success, a negative value on error + */ +struct xattr; +extern int bpf_inode_init_xattr(struct xattr *xattrs, int *xattr_count, + const char *name__str, + const struct bpf_dynptr *value_p) __ksym __weak; #endif diff --git a/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected b/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected new file mode 100644 index 000000000000..5470b2b43229 --- /dev/null +++ b/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected @@ -0,0 +1,48 @@ +#ifndef __VMLINUX_H__ +#define __VMLINUX_H__ + +#ifndef BPF_NO_PRESERVE_ACCESS_INDEX +#pragma clang attribute push (__attribute__((preserve_access_index)), apply_to = record) +#endif + +#ifndef __ksym +#define __ksym __attribute__((section(".ksyms"))) +#endif + +#ifndef __weak +#define __weak __attribute__((weak)) +#endif + +#ifndef __bpf_fastcall +#if __has_attribute(bpf_fastcall) +#define __bpf_fastcall __attribute__((bpf_fastcall)) +#else +#define __bpf_fastcall +#endif +#endif + +enum { + E0 = 1, +}; + +struct holey { + int c; + long: 32; + long: 32; + int tail; +}; + +struct s { + int f; +}; + + +/* BPF kfuncs */ +#ifndef BPF_NO_KFUNC_PROTOTYPES +#endif + +#ifndef BPF_NO_PRESERVE_ACCESS_INDEX +#pragma clang attribute pop +#endif + +#endif /* __VMLINUX_H__ */ diff --git a/tools/testing/selftests/bpf/bpftool_btf_dump_unsorted.expected b/tools/testing/selftests/bpf/bpftool_btf_dump_unsorted.expected new file mode 100644 index 000000000000..4310adc2ed51 --- /dev/null +++ b/tools/testing/selftests/bpf/bpftool_btf_dump_unsorted.expected @@ -0,0 +1,48 @@ +#ifndef __VMLINUX_H__ +#define __VMLINUX_H__ + +#ifndef BPF_NO_PRESERVE_ACCESS_INDEX +#pragma clang attribute push (__attribute__((preserve_access_index)), apply_to = record) +#endif + +#ifndef __ksym +#define __ksym __attribute__((section(".ksyms"))) +#endif + +#ifndef __weak +#define __weak __attribute__((weak)) +#endif + +#ifndef __bpf_fastcall +#if __has_attribute(bpf_fastcall) +#define __bpf_fastcall __attribute__((bpf_fastcall)) +#else +#define __bpf_fastcall +#endif +#endif + +struct holey { + int c; + long: 32; + long: 32; + int tail; +}; + +enum { + E0 = 1, +}; + +struct s { + int f; +}; + + +/* BPF kfuncs */ +#ifndef BPF_NO_KFUNC_PROTOTYPES +#endif + +#ifndef BPF_NO_PRESERVE_ACCESS_INDEX +#pragma clang attribute pop +#endif + +#endif /* __VMLINUX_H__ */ diff --git a/tools/testing/selftests/bpf/bpftool_helpers.c b/tools/testing/selftests/bpf/bpftool_helpers.c index 0a2a4f0a2794..9380b24bbf52 100644 --- a/tools/testing/selftests/bpf/bpftool_helpers.c +++ b/tools/testing/selftests/bpf/bpftool_helpers.c @@ -11,7 +11,7 @@ #define BPFTOOL_DEFAULT_PATH "tools/sbin/bpftool" -static int detect_bpftool_path(char *buffer, size_t size) +int detect_bpftool_path(char *buffer, size_t size) { char tmp[PATH_MAX]; const char *env_path; @@ -53,6 +53,7 @@ static int run_command(char *args, char *output_buf, size_t output_max_len) static char bpftool_path[PATH_MAX] = {}; bool suppress_output = !(output_buf && output_max_len); char command[BPFTOOL_FULL_CMD_MAX_LEN]; + size_t n; FILE *f; int ret; @@ -68,8 +69,10 @@ static int run_command(char *args, char *output_buf, size_t output_max_len) if (!f) return 1; - if (!suppress_output) - fread(output_buf, 1, output_max_len, f); + if (!suppress_output) { + n = fread(output_buf, 1, output_max_len - 1, f); + output_buf[n] = '\0'; + } ret = pclose(f); return ret; diff --git a/tools/testing/selftests/bpf/bpftool_helpers.h b/tools/testing/selftests/bpf/bpftool_helpers.h index dec1ba201410..17e3be8e7073 100644 --- a/tools/testing/selftests/bpf/bpftool_helpers.h +++ b/tools/testing/selftests/bpf/bpftool_helpers.h @@ -7,5 +7,6 @@ #define MAX_BPFTOOL_CMD_LEN (256) +int detect_bpftool_path(char *buffer, size_t size); int run_bpftool_command(char *args); int get_bpftool_command_output(char *args, char *output_buf, size_t output_max_len); diff --git a/tools/testing/selftests/bpf/btf_helpers.c b/tools/testing/selftests/bpf/btf_helpers.c index 1c1c2c26690a..e04b61c08cd8 100644 --- a/tools/testing/selftests/bpf/btf_helpers.c +++ b/tools/testing/selftests/bpf/btf_helpers.c @@ -27,11 +27,14 @@ static const char * const btf_kind_str_mapping[] = { [BTF_KIND_DECL_TAG] = "DECL_TAG", [BTF_KIND_TYPE_TAG] = "TYPE_TAG", [BTF_KIND_ENUM64] = "ENUM64", + [BTF_KIND_LOC_PARAM] = "LOC_PARAM", + [BTF_KIND_LOC_PROTO] = "LOC_PROTO", + [BTF_KIND_LOCSEC] = "LOCSEC", }; static const char *btf_kind_str(__u16 kind) { - if (kind > BTF_KIND_ENUM64) + if (kind > BTF_KIND_LOCSEC) return "UNKNOWN"; return btf_kind_str_mapping[kind]; } @@ -111,9 +114,12 @@ int fprintf_btf_type_raw(FILE *out, const struct btf *btf, __u32 id) case BTF_KIND_VOLATILE: case BTF_KIND_RESTRICT: case BTF_KIND_TYPEDEF: - case BTF_KIND_TYPE_TAG: fprintf(out, " type_id=%u", t->type); break; + case BTF_KIND_TYPE_TAG: + fprintf(out, " type_id=%u%s", t->type, + btf_kflag(t) ? " kflag=1" : ""); + break; case BTF_KIND_ARRAY: { const struct btf_array *arr = btf_array(t); @@ -200,9 +206,40 @@ int fprintf_btf_type_raw(FILE *out, const struct btf *btf, __u32 id) fprintf(out, " size=%u", t->size); break; case BTF_KIND_DECL_TAG: - fprintf(out, " type_id=%u component_idx=%d", - t->type, btf_decl_tag(t)->component_idx); + fprintf(out, " type_id=%u component_idx=%d%s", + t->type, btf_decl_tag(t)->component_idx, + btf_kflag(t) ? " kflag=1" : ""); break; + case BTF_KIND_LOC_PARAM: { + struct btf_loc_param *p = btf_loc_param(t); + + fprintf(out, " size=%u flags=0x%x vlen=%u", t->size, p->flags, vlen); + for (i = 0; i < vlen; i++) { + if (p->flags & BTF_LOC_PARAM_SIGNED) + fprintf(out, "\n\tvalue=%d", (__s32)p->values[i]); + else + fprintf(out, "\n\tvalue=%u", p->values[i]); + } + break; + } + case BTF_KIND_LOC_PROTO: { + const __u32 *p = btf_loc_proto_params(t); + + fprintf(out, " vlen=%u", vlen); + for (i = 0; i < vlen; i++, p++) + fprintf(out, "\n\ttype_id=%u", *p); + break; + } + case BTF_KIND_LOCSEC: { + const struct btf_loc *l = btf_locsec_locs(t); + + fprintf(out, " vlen=%u", vlen); + for (i = 0; i < vlen; i++, l++) { + fprintf(out, "\n\tfunc_type_id=%u loc_proto_type_id=%u offset=%u", + l->func, l->loc_proto, l->offset); + } + break; + } default: break; } diff --git a/tools/testing/selftests/bpf/cgroup_helpers.c b/tools/testing/selftests/bpf/cgroup_helpers.c index 45cd0b479fe3..4183ff6150c2 100644 --- a/tools/testing/selftests/bpf/cgroup_helpers.c +++ b/tools/testing/selftests/bpf/cgroup_helpers.c @@ -188,6 +188,73 @@ int write_cgroup_file_parent(const char *relative_path, const char *file, return __write_cgroup_file(cgroup_path, file, buf); } +static int __read_cgroup_file(const char *cgroup_path, const char *file, + char *buf, size_t len) +{ + char file_path[PATH_MAX + 1]; + ssize_t got; + int fd; + + snprintf(file_path, sizeof(file_path), "%s/%s", cgroup_path, file); + fd = open(file_path, O_RDONLY); + if (fd < 0) { + log_err("Opening %s", file_path); + return 1; + } + + got = read(fd, buf, len - 1); + if (got < 0) { + log_err("Reading %s", file_path); + close(fd); + return 1; + } + buf[got] = '\0'; + close(fd); + return 0; +} + +/** + * read_cgroup_file() - Read from a cgroup file + * @relative_path: The cgroup path, relative to the workdir + * @file: The name of the file in cgroupfs to read from + * @buf: Buffer to read into, NUL-terminated on success + * @len: Size of @buf + * + * Read from a file in the given cgroup's directory. + * + * If successful, 0 is returned. + */ +int read_cgroup_file(const char *relative_path, const char *file, + char *buf, size_t len) +{ + char cgroup_path[PATH_MAX - 24]; + + format_cgroup_path(cgroup_path, relative_path); + return __read_cgroup_file(cgroup_path, file, buf, len); +} + +/** + * read_cgroup_file_parent() - Read from a cgroup file in the parent process + * workdir + * @relative_path: The cgroup path, relative to the parent process workdir + * @file: The name of the file in cgroupfs to read from + * @buf: Buffer to read into, NUL-terminated on success + * @len: Size of @buf + * + * Read from a file in the given cgroup's directory under the parent process + * workdir. + * + * If successful, 0 is returned. + */ +int read_cgroup_file_parent(const char *relative_path, const char *file, + char *buf, size_t len) +{ + char cgroup_path[PATH_MAX - 24]; + + format_parent_cgroup_path(cgroup_path, relative_path); + return __read_cgroup_file(cgroup_path, file, buf, len); +} + /** * setup_cgroup_environment() - Setup the cgroup environment * diff --git a/tools/testing/selftests/bpf/cgroup_helpers.h b/tools/testing/selftests/bpf/cgroup_helpers.h index 3857304be874..d42d2e13044e 100644 --- a/tools/testing/selftests/bpf/cgroup_helpers.h +++ b/tools/testing/selftests/bpf/cgroup_helpers.h @@ -15,6 +15,10 @@ int write_cgroup_file(const char *relative_path, const char *file, const char *buf); int write_cgroup_file_parent(const char *relative_path, const char *file, const char *buf); +int read_cgroup_file(const char *relative_path, const char *file, + char *buf, size_t len); +int read_cgroup_file_parent(const char *relative_path, const char *file, + char *buf, size_t len); int cgroup_setup_and_join(const char *relative_path); int get_root_cgroup(void); int create_and_get_cgroup(const char *relative_path); diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config index ea7044f30adc..49dfe939467c 100644 --- a/tools/testing/selftests/bpf/config +++ b/tools/testing/selftests/bpf/config @@ -12,15 +12,18 @@ CONFIG_BPF_SYSCALL=y # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set CONFIG_CGROUP_BPF=y CONFIG_CRYPTO_HMAC=y +CONFIG_CRYPTO_MLDSA=y CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_SHA512=y CONFIG_CRYPTO_USER_API=y CONFIG_CRYPTO_USER_API_HASH=y CONFIG_CRYPTO_USER_API_SKCIPHER=y -CONFIG_CRYPTO_SKCIPHER=y +CONFIG_CRYPTO_CBC=y CONFIG_CRYPTO_ECB=y CONFIG_CRYPTO_AES=y CONFIG_DEBUG_INFO=y CONFIG_DEBUG_INFO_BTF=y +CONFIG_DEBUG_INFO_BTF_INLINE=y CONFIG_DEBUG_INFO_DWARF4=y CONFIG_DMABUF_HEAPS=y CONFIG_DMABUF_HEAPS_SYSTEM=y @@ -55,6 +58,7 @@ CONFIG_LIRC=y CONFIG_LIVEPATCH=y CONFIG_LWTUNNEL=y CONFIG_LWTUNNEL_BPF=y +CONFIG_MEMCG=y CONFIG_MODULE_SIG=y CONFIG_MODULE_SRCVERSION_ALL=y CONFIG_MODULE_UNLOAD=y @@ -121,7 +125,8 @@ CONFIG_SAMPLE_LIVEPATCH=m CONFIG_SECURITY=y CONFIG_SECURITYFS=y CONFIG_SYN_COOKIES=y -CONFIG_TEST_BPF=m +CONFIG_TMPFS=y +CONFIG_TMPFS_XATTR=y CONFIG_UDMABUF=y CONFIG_USERFAULTFD=y CONFIG_VSOCKETS=y diff --git a/tools/testing/selftests/bpf/gen_bpf_skel.sh b/tools/testing/selftests/bpf/gen_bpf_skel.sh new file mode 100755 index 000000000000..60942f24b094 --- /dev/null +++ b/tools/testing/selftests/bpf/gen_bpf_skel.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Link BPF object file(s) with "bpftool gen object" and generate a +# skeleton (or light skeleton) header from the result. +# +# Usage: +# BPFTOOL=<bpftool> gen_bpf_skel.sh --name NAME --skel OUT +# [options] OBJ... +# +# --name NAME skeleton object name ("name NAME" for bpftool) +# --skel OUT output header ("foo.skel.h" or "foo.lskel.h") +# --subskel OUT also generate a subskeleton header into OUT +# --lskel generate a light skeleton (bpftool gen skeleton -L) +# --sign sign the (light) skeleton, with the key and +# certificate taken from $PRIVATE_KEY and +# $VERIFICATION_CERT +# +# The inputs are linked three times and the 2nd and 3rd results compared, +# as a regression test for the determinism of "bpftool gen object". +# +# Intermediate files are named after the output header with a "linked" +# infix - "llinked" for light or signed skeletons, so that generating a +# .skel.h and a .lskel.h from the same .bpf.o in parallel never races on +# the intermediates. +# +# The bpftool binary is taken from $BPFTOOL (default: bpftool from PATH). +# On failure all outputs and intermediates are removed and the script exits +# non-zero; permissive-mode skipping is the caller's business (see +# skip_on_fail in the Makefile). + +set -u + +bpftool=${BPFTOOL:-bpftool} +name='' skel='' subskel='' +lskel=0 sign=0 + +while [ $# -gt 0 ]; do + case "$1" in + --name) name=$2; shift 2 ;; + --skel) skel=$2; shift 2 ;; + --subskel) subskel=$2; shift 2 ;; + --lskel) lskel=1; shift ;; + --sign) sign=1; shift ;; + --) shift; break ;; + -*) echo "$0: unknown option: $1" >&2; exit 1 ;; + *) break ;; + esac +done + +if [ -z "$name" ] || [ -z "$skel" ] || [ $# -eq 0 ]; then + echo "usage: $0 --name NAME --skel OUT [options] OBJ..." >&2 + exit 1 +fi + +infix=linked +if [ "$lskel" -eq 1 ] || [ "$sign" -eq 1 ]; then + infix=llinked +fi + +if [ "$sign" -eq 1 ] && { [ -z "${PRIVATE_KEY-}" ] || [ -z "${VERIFICATION_CERT-}" ]; }; then + echo "$0: --sign requires PRIVATE_KEY and VERIFICATION_CERT in the environment" >&2 + exit 1 +fi + +base=${skel%.skel.h} +base=${base%.lskel.h} +t1=$base.${infix}1.o +t2=$base.${infix}2.o +t3=$base.${infix}3.o + +fail() { + rm -f "$skel" ${subskel:+"$subskel"} "$t1" "$t2" "$t3" + exit 1 +} + +"$bpftool" gen object "$t1" "$@" || fail +"$bpftool" gen object "$t2" "$t1" || fail +"$bpftool" gen object "$t3" "$t2" || fail +if ! cmp -s "$t2" "$t3"; then + echo "$0: bpftool gen object is not deterministic for $skel" >&2 + fail +fi + +args=() +if [ "$sign" -eq 1 ]; then + args+=(-S -k "$PRIVATE_KEY" -i "$VERIFICATION_CERT") +fi +if [ "$lskel" -eq 1 ]; then + args+=(-L) +fi +"$bpftool" gen skeleton ${args[@]+"${args[@]}"} "$t3" name "$name" > "$skel" || fail + +if [ -n "$subskel" ]; then + "$bpftool" gen subskeleton "$t3" name "$name" > "$subskel" || fail +fi + +rm -f "$t1" "$t2" "$t3" +exit 0 diff --git a/tools/testing/selftests/bpf/libarena/Makefile b/tools/testing/selftests/bpf/libarena/Makefile index 5e2ab514805e..91164d43bd61 100644 --- a/tools/testing/selftests/bpf/libarena/Makefile +++ b/tools/testing/selftests/bpf/libarena/Makefile @@ -27,10 +27,17 @@ BPFDIR=$(abspath $(LIBARENA)/..) INCLUDE_DIR ?= $(BPFDIR)/tools/include LIBBPF_INCLUDE ?= $(INCLUDE_DIR) -# Scan src/ and selftests/ to generate the final binaries -LIBARENA_SOURCES = $(wildcard $(LIBARENA)/src/*.bpf.c) $(wildcard $(LIBARENA)/selftests/*.bpf.c) -LIBARENA_OBJECTS = $(notdir $(LIBARENA_SOURCES:.bpf.c=.bpf.o)) -LIBARENA_OBJECTS_ASAN = $(notdir $(LIBARENA_SOURCES:.bpf.c=_asan.bpf.o)) +# Build selftests and benchmarks into separate BPF objects and skeletons. +LIBARENA_CORE_SOURCES = $(wildcard $(LIBARENA)/src/*.bpf.c) +LIBARENA_TEST_SOURCES = $(wildcard $(LIBARENA)/selftests/*.bpf.c) +LIBARENA_BENCH_SOURCES = $(wildcard $(LIBARENA)/benchs/*.bpf.c) + +LIBARENA_OBJECTS = $(notdir $(LIBARENA_CORE_SOURCES:.bpf.c=.bpf.o) \ + $(LIBARENA_TEST_SOURCES:.bpf.c=.bpf.o)) +LIBARENA_OBJECTS_ASAN = $(notdir $(LIBARENA_CORE_SOURCES:.bpf.c=_asan.bpf.o) \ + $(LIBARENA_TEST_SOURCES:.bpf.c=_asan.bpf.o)) +LIBARENA_BENCH_OBJECTS = $(notdir $(LIBARENA_CORE_SOURCES:.bpf.c=.bpf.o) \ + $(LIBARENA_BENCH_SOURCES:.bpf.c=.bpf.o)) INCLUDES = -I$(LIBARENA)/include -I$(BPFDIR) ifneq ($(INCLUDE_DIR),) @@ -53,17 +60,25 @@ override BPF_CFLAGS += -O2 -g override BPF_CFLAGS += -Wno-incompatible-pointer-types-discards-qualifiers # Required for suppressing harmless vmlinux.h-related warnings. override BPF_CFLAGS += -Wno-missing-declarations +override BPF_CFLAGS += -fno-strict-aliasing override BPF_CFLAGS += $(INCLUDES) CFLAGS = -O2 -no-pie CFLAGS += $(INCLUDES) -vpath %.bpf.c $(LIBARENA)/src $(LIBARENA)/selftests -vpath %.c $(LIBARENA)/src $(LIBARENA)/selftests +vpath %.bpf.c $(LIBARENA)/src $(LIBARENA)/selftests $(LIBARENA)/benchs +vpath %.c $(LIBARENA)/src $(LIBARENA)/selftests $(LIBARENA)/benchs skeletons: libarena.skel.h libarena_asan.skel.h .PHONY: skeletons +benchmarks: libarena_bench.skel.h +.PHONY: benchmarks + +libarena_bench.skel.h: libarena_bench.bpf.o + $(call msg,GEN-SKEL,libarena,$@) + $(Q)$(BPFTOOL) gen skeleton $< name "libarena_bench" > $@ + libarena_asan.skel.h: libarena_asan.bpf.o $(call msg,GEN-SKEL,libarena,$@) $(Q)$(BPFTOOL) gen skeleton $< name "libarena_asan" > $@ @@ -80,6 +95,10 @@ libarena.bpf.o: $(LIBARENA_OBJECTS) $(call msg,GEN-OBJ,libarena,$@) $(Q)$(BPFTOOL) gen object $@ $^ +libarena_bench.bpf.o: $(LIBARENA_BENCH_OBJECTS) + $(call msg,GEN-OBJ,libarena,$@) + $(Q)$(BPFTOOL) gen object $@ $^ + %_asan.bpf.o: %.bpf.c $(call msg,CLNG-BPF,libarena,$@) $(Q)$(CLANG) $(BPF_CFLAGS) $(ASAN_FLAGS) -DBPF_ARENA_ASAN $(BPF_TARGET_ENDIAN) -c $< -o $@ diff --git a/tools/testing/selftests/bpf/libarena/benchs/bench_malloc.bpf.c b/tools/testing/selftests/bpf/libarena/benchs/bench_malloc.bpf.c new file mode 100644 index 000000000000..e06397dc9a40 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/benchs/bench_malloc.bpf.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/buddy.h> + +u32 bench_alloc_size; +u32 bench_nallocs; +long bench_hits; +long bench_duration_ns; + +SEC("syscall") +int bench_malloc(void) +{ + void __arena *mem; + u64 start_ns; + u32 i; + + start_ns = bpf_ktime_get_ns(); + for (i = zero; i < bench_nallocs && can_loop; i++) { + mem = arena_malloc(bench_alloc_size); + if (!mem) + return -ENOMEM; + } + + __sync_add_and_fetch(&bench_duration_ns, + bpf_ktime_get_ns() - start_ns); + __sync_add_and_fetch(&bench_hits, i); + return 0; +} + +SEC("syscall") +int bench_calloc(void) +{ + void __arena *mem; + u64 start_ns; + u32 i; + + start_ns = bpf_ktime_get_ns(); + for (i = zero; i < bench_nallocs && can_loop; i++) { + mem = arena_calloc(1, bench_alloc_size); + if (!mem) + return -ENOMEM; + } + + __sync_add_and_fetch(&bench_duration_ns, + bpf_ktime_get_ns() - start_ns); + __sync_add_and_fetch(&bench_hits, i); + return 0; +} diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h b/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h index 71d9db610263..055b5892cb63 100644 --- a/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ #ifndef BPF_ARENA_SPIN_LOCK_H #define BPF_ARENA_SPIN_LOCK_H diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h b/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h index 43c306e17f19..65582b2010ad 100644 --- a/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ #ifndef BPF_ATOMIC_H #define BPF_ATOMIC_H diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h b/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h index 9ba90689d6ba..b32a420d4e1a 100644 --- a/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause #pragma once /* diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h index e2431ea6fdd6..163e2b83d943 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h @@ -1,5 +1,8 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause #pragma once +#include <bpf_atomic.h> + #define BITS_PER_BYTE 8 #define BYTES_TO_BITS(nb) ((nb) * BITS_PER_BYTE) @@ -15,11 +18,8 @@ struct arena_bitmap { struct arena_bitmap __arena *bmp_alloc(size_t bits); void bmp_free(struct arena_bitmap __arena *bmp); -void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp); -void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); void bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp); void bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); -bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp); bool bmp_test_and_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); bool bmp_test_and_set_bit(u32 bit, struct arena_bitmap __arena *bmp); @@ -32,3 +32,27 @@ void bmp_copy(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap bool bmp_intersects(size_t bits, struct arena_bitmap __arena *arg1, struct arena_bitmap __arena *arg2); bool bmp_subset(size_t bits, struct arena_bitmap __arena *big, struct arena_bitmap __arena *small); void bmp_print(size_t bits, struct arena_bitmap __arena *bmp); + +static __always_inline +void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + volatile u64 __arena *word = &bmp->bits[BIT_WORD(bit)]; + + *word |= BIT_MASK(bit); +} + +static __always_inline +void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + volatile u64 __arena *word = &bmp->bits[BIT_WORD(bit)]; + + *word &= ~BIT_MASK(bit); +} + +static __always_inline +bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + u64 word = READ_ONCE(bmp->bits[BIT_WORD(bit)]); + + return word & BIT_MASK(bit); +} diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/common.h b/tools/testing/selftests/bpf/libarena/include/libarena/common.h index 931ace9a49e2..a6220355a697 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/common.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/common.h @@ -49,6 +49,7 @@ extern volatile u64 asan_violated; int arena_fls(__u64 word); void __arena *arena_malloc(size_t size); +void __arena *arena_calloc(size_t ncount, size_t size); void arena_free(void __arena *ptr); /* @@ -61,6 +62,76 @@ void arena_free(void __arena *ptr); */ #define arena_subprog_init() do { asm volatile ("" :: "r"(&arena)); } while (0) +/* + * BPF does not currently support the memset intrinsics. for large + * sequential copies, or assignments of large data structures, + * the frontend will generate an intrinsic that causes the BPF + * backend to exit due to a missing implementation. Provide + * implementations for the intrinsic. + */ +static inline int arena_memset(s8 __arena *dst, s8 val, size_t size) +{ + size_t headalign; + size_t tailalign; + u8 uval = (u8)val; + size_t val64; + size_t i; + + /* + * Calculate how many bytes to the next word-aligned one. + * We get this by truncating the 2s complement of the + * pointer to the last 3 bits. Intuitively, since + * + * The N LSBs of dst and -dst add to 1 << N, which + * is why dst + (-dst) = 0x0ULL through overflow. So the + * last N = 3 bits of the negative are the number of + * bytes to align dst on the last 3 bits. + * + */ + headalign = -(u64)dst & (sizeof(u64) - 1); + if (!headalign || size < headalign) + goto ptraligned; + + for (i = zero; i < headalign && can_loop; i++) + dst[i] = uval; + + dst += headalign; + size -= headalign; + +ptraligned: + + /* + * Make a word with all bytes equal to the byte we are setting. + * Since 1 byte -> 2 hex digits. + * + * Shifting the value by a 0 bytes is equal to multiplication by 0x01 + * Shifting by 1 bytes is equal to multiplication by 0x01 << 8, + * ... + * Shifting by 7 bytes is equal to multiplication by 0x01 << 56. + * + * End operation to replicate the byte into all the bytes of a word + * is (since a | b = a + b when a & b == 0): + * + * val + val * (1UL << 8) + val * (1UL << 16) + .. + val * (1UL << 56) + * = val * (1UL << 56 + 1UL << 48 + ... + 1UL << 0) + * = val * (0x01UL << 56 | 0x01UL << 48 + ... + 1UL << 0) + * = val * 0x0101 0101 0101 0101 + */ + val64 = (u8)val * 0x0101010101010101ULL; + + /* Pointer is now aligned, use word-aligned assignments. */ + for (i = zero; i < size / sizeof(u64) && can_loop; i++) + ((u64 __arena *)dst)[i] = val64; + + /* Go back to byte-aligned for the tail. */ + tailalign = size % sizeof(u64); + dst += size - tailalign; + for (i = zero; i < tailalign && can_loop; i++) + dst[i] = uval; + + return 0; +} + #else /* ! __BPF__ */ #include <stdint.h> diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c index 76319a529f02..e66b3a26ca3d 100644 --- a/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c +++ b/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + #include <libarena/common.h> #include <libarena/asan.h> diff --git a/tools/testing/selftests/bpf/libarena/src/asan.bpf.c b/tools/testing/selftests/bpf/libarena/src/asan.bpf.c index 5135d5c72a46..de656b69d13a 100644 --- a/tools/testing/selftests/bpf/libarena/src/asan.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/asan.bpf.c @@ -103,23 +103,6 @@ volatile bool asan_inited = false; */ volatile bool asan_report_once = false; -/* - * BPF does not currently support the memset/memcpy/memcmp intrinsics. - * For large sequential copies, or assignments of large data structures, - * the frontend will generate an intrinsic that causes the BPF backend - * to exit due to a missing implementation. Provide a simple implementation - * just for memset to use it for poisoning/unpoisoning the map. - */ -__weak int asan_memset(s8 __arena *dst, s8 val, size_t size) -{ - size_t i; - - for (i = zero; i < size && can_loop; i++) - dst[i] = val; - - return 0; -} - /* Validate a 1-byte access, always within a single byte. */ static __always_inline bool memory_is_poisoned_1(s8 __arena *addr) { @@ -422,7 +405,7 @@ __hidden __noasan int asan_poison(void __arena *addr, s8 val, size_t size) shadow = mem_to_shadow(addr); len = size >> ASAN_SHADOW_SHIFT; - asan_memset(shadow, val, len); + arena_memset(shadow, val, len); return 0; } @@ -463,7 +446,7 @@ __hidden __noasan int asan_unpoison(void __arena *addr, size_t size) shadow = mem_to_shadow(addr); len = size >> ASAN_SHADOW_SHIFT; - asan_memset(shadow, 0, len); + arena_memset(shadow, 0, len); /* * If we are allocating a non-granule aligned region, we need to adjust diff --git a/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c index 5ff8e688ddc7..0390f20ce366 100644 --- a/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c @@ -35,24 +35,6 @@ void bmp_free(struct arena_bitmap __arena *bmp) } __weak -void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp) -{ - bmp->bits[BIT_WORD(bit)] |= BIT_MASK(bit); -} - -__weak -void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) -{ - bmp->bits[BIT_WORD(bit)] &= ~BIT_MASK(bit); -} - -__weak -bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp) -{ - return bmp->bits[BIT_WORD(bit)] & BIT_MASK(bit); -} - -__weak bool bmp_test_and_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) { u64 val = BIT_MASK(bit); diff --git a/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c b/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c index c674ee5cfcc1..2490ab1396de 100644 --- a/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c @@ -5,6 +5,8 @@ #include <libarena/asan.h> #include <libarena/buddy.h> +#include <bpf_arena_spin_lock.h> + /* * Buddy allocator arena-based implementation. * @@ -45,15 +47,8 @@ enum { BUDDY_CHUNK_PAGES = BUDDY_CHUNK_BYTES / __PAGE_SIZE }; -static inline int buddy_lock(struct buddy __arena *buddy) -{ - return arena_spin_lock(&buddy->lock); -} - -static inline void buddy_unlock(struct buddy __arena *buddy) -{ - arena_spin_unlock(&buddy->lock); -} +#define buddy_lock(buddy, flags) (arena_spin_lock_irqsave(&(buddy)->lock, (flags))) +#define buddy_unlock(buddy, flags) (arena_spin_unlock_irqrestore(&(buddy)->lock, (flags))) /* * Reserve part of the arena address space for the allocator. We use @@ -385,6 +380,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) { u64 order, ord, min_order, max_order; struct buddy_chunk __arena *chunk; + unsigned long flags; size_t left; int power2; u64 vaddr; @@ -416,7 +412,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) return NULL; } - if (buddy_lock(buddy)) { + if (buddy_lock(buddy, flags)) { /* * We cannot reclaim the vaddr space, but that is ok - this * operation should always succeed. The error path is to catch @@ -520,7 +516,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) arena_stderr( "chunk has size of 0x%lx bytes (left %lx bytes)\n", sizeof(*chunk), left); - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return NULL; } @@ -531,7 +527,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) order = (power2 >= BUDDY_MIN_ALLOC_SHIFT) ? power2 - BUDDY_MIN_ALLOC_SHIFT : 0; if (idx_set_allocated(chunk, idx, true)) { - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return NULL; } @@ -547,7 +543,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) */ min_order = left ? order + 1 : order; if (add_leftovers_to_freelist(chunk, idx, min_order, max_order)) { - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return NULL; } @@ -556,7 +552,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) max_order = order; } - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return chunk; } @@ -564,6 +560,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) __weak int buddy_init(struct buddy __arena *buddy) { struct buddy_chunk __arena *chunk; + unsigned long flags; int ret; if (!asan_ready()) @@ -579,7 +576,7 @@ __weak int buddy_init(struct buddy __arena *buddy) chunk = buddy_chunk_get(buddy); - if (buddy_lock(buddy)) { + if (buddy_lock(buddy, flags)) { bpf_arena_free_pages(&arena, chunk, BUDDY_CHUNK_PAGES); return -EINVAL; } @@ -591,7 +588,7 @@ __weak int buddy_init(struct buddy __arena *buddy) /* Put the chunk at the beginning of the list. */ buddy->first_chunk = chunk; - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return chunk ? 0 : -ENOMEM; } @@ -730,9 +727,10 @@ static u64 buddy_alloc_from_existing_chunks(struct buddy __arena *buddy, int ord */ static u64 buddy_alloc_from_new_chunk(struct buddy __arena *buddy, struct buddy_chunk __arena *chunk, int order) { + unsigned long flags; u64 address; - if (buddy_lock(buddy)) + if (buddy_lock(buddy, flags)) return (u64)NULL; @@ -745,7 +743,7 @@ static u64 buddy_alloc_from_new_chunk(struct buddy __arena *buddy, struct buddy_ address = buddy_chunk_alloc(buddy->first_chunk, order); - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return (u64)address; } @@ -754,6 +752,7 @@ void __arena *buddy_alloc(struct buddy __arena *buddy, size_t size) { void __arena *address = NULL; struct buddy_chunk __arena *chunk; + unsigned long flags; int order; if (!buddy) @@ -765,11 +764,11 @@ void __arena *buddy_alloc(struct buddy __arena *buddy, size_t size) return NULL; } - if (buddy_lock(buddy)) + if (buddy_lock(buddy, flags)) return NULL; address = (u8 __arena *)buddy_alloc_from_existing_chunks(buddy, order); - buddy_unlock(buddy); + buddy_unlock(buddy, flags); if (address) goto done; @@ -880,6 +879,7 @@ static __always_inline int buddy_free_unlocked(struct buddy __arena *buddy, u64 __weak int buddy_free(struct buddy __arena *buddy, void __arena *addr) { + unsigned long flags; int ret; if (!buddy) @@ -889,13 +889,13 @@ __weak int buddy_free(struct buddy __arena *buddy, void __arena *addr) if (!addr) return 0; - ret = buddy_lock(buddy); + ret = buddy_lock(buddy, flags); if (ret) return ret; ret = buddy_free_unlocked(buddy, (u64)addr); - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return ret; } diff --git a/tools/testing/selftests/bpf/libarena/src/common.bpf.c b/tools/testing/selftests/bpf/libarena/src/common.bpf.c index 41b1de3452fe..d1af60b81838 100644 --- a/tools/testing/selftests/bpf/libarena/src/common.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/common.bpf.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <limits.h> + #include <libarena/common.h> #include <libarena/asan.h> #include <libarena/buddy.h> @@ -55,10 +57,33 @@ __weak void __arena *arena_malloc(size_t size) return buddy_alloc(&buddy, size); } +__weak void __arena *arena_calloc(size_t ncount, size_t size) +{ + void __arena *mem; + size_t total; + + /* + * Ideally we'd be using __builtin_mul_overflow here, + * but the BPF compiler backend doesn't implement __multi3. + * There are ways to optimize the division away from the + * overflow check, but any costs are dwarfed by the + * buddy_alloc() call. Keep it simple for now. + */ + if (unlikely(ncount && size >= ULLONG_MAX / ncount)) + return NULL; + + total = ncount * size; + + mem = buddy_alloc(&buddy, total); + if (likely(mem)) + arena_memset(mem, 0, total); + + return mem; +} + __weak void arena_free(void __arena *ptr) { buddy_free(&buddy, ptr); } - char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/network_helpers.c b/tools/testing/selftests/bpf/network_helpers.c index cdf2d7d3ab32..d27dc79475de 100644 --- a/tools/testing/selftests/bpf/network_helpers.c +++ b/tools/testing/selftests/bpf/network_helpers.c @@ -49,6 +49,8 @@ errno = __save; \ }) +#define CONNECT_MIN_TIMEOUT_MS 5000 + struct ipv4_packet pkt_v4 = { .eth.h_proto = __bpf_constant_htons(ETH_P_IP), .iph.ihl = 5, @@ -291,6 +293,37 @@ error_close: return -1; } +static int connect_with_timeout(int fd, const struct sockaddr_storage *addr, + socklen_t addrlen, int timeout_ms) +{ + int connect_timeout_ms = MAX(timeout_ms, CONNECT_MIN_TIMEOUT_MS); + + /* + * Override timeout configuration with a larger value for the + * connection + */ + if (settimeo(fd, connect_timeout_ms)) { + log_err("Failed to set connect timeout"); + return -1; + } + + if (connect(fd, (const struct sockaddr *)addr, addrlen)) { + log_err("Failed to connect"); + return -1; + } + + /* + * If the timeout configured by the test is different from the + * connect timeout, restore it + */ + if (timeout_ms != connect_timeout_ms && settimeo(fd, timeout_ms)) { + log_err("Failed to set timeout for connected socket"); + return -1; + } + + return 0; +} + int connect_to_addr(int type, const struct sockaddr_storage *addr, socklen_t addrlen, const struct network_helper_opts *opts) { @@ -305,8 +338,7 @@ int connect_to_addr(int type, const struct sockaddr_storage *addr, socklen_t add return -1; } - if (connect(fd, (const struct sockaddr *)addr, addrlen)) { - log_err("Failed to connect to server"); + if (connect_with_timeout(fd, addr, addrlen, opts->timeout_ms)) { save_errno_close(fd); return -1; } @@ -376,20 +408,12 @@ int connect_fd_to_fd(int client_fd, int server_fd, int timeout_ms) struct sockaddr_storage addr; socklen_t len = sizeof(addr); - if (settimeo(client_fd, timeout_ms)) - return -1; - if (getsockname(server_fd, (struct sockaddr *)&addr, &len)) { log_err("Failed to get server addr"); return -1; } - if (connect(client_fd, (const struct sockaddr *)&addr, len)) { - log_err("Failed to connect to server"); - return -1; - } - - return 0; + return connect_with_timeout(client_fd, &addr, len, timeout_ms); } int make_sockaddr(int family, const char *addr_str, __u16 port, diff --git a/tools/testing/selftests/bpf/network_helpers.h b/tools/testing/selftests/bpf/network_helpers.h index 75133119c04a..878c9fc5c37c 100644 --- a/tools/testing/selftests/bpf/network_helpers.h +++ b/tools/testing/selftests/bpf/network_helpers.h @@ -129,12 +129,23 @@ static __u16 csum_fold(__u32 csum) static __wsum csum_partial(const void *buf, int len, __wsum sum) { - __u16 *p = (__u16 *)buf; + const __u8 *p = buf; int num_u16 = len >> 1; int i; for (i = 0; i < num_u16; i++) - sum += p[i]; + sum += ((const __u16 *)p)[i]; + + /* + * RFC 1071: an odd-length buffer's trailing byte is paired with + * a zero pad byte to form the final 16-bit word. + */ + if (len & 1) { + __u16 tail = 0; + + __builtin_memcpy(&tail, p + len - 1, 1); + sum += tail; + } return sum; } diff --git a/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c b/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c new file mode 100644 index 000000000000..aa7562f48737 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> +#include "aggregate_arg_func.skel.h" +#include "aggregate_arg_kfunc.skel.h" + +void test_aggregate_arg(void) +{ + RUN_TESTS(aggregate_arg_func); + RUN_TESTS(aggregate_arg_kfunc); +} diff --git a/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c new file mode 100644 index 000000000000..07d9d6e1d6b8 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> +#include <bpf/btf.h> +#include "aggregate_ret_func.skel.h" +#include "aggregate_ret_kfunc.skel.h" +#include "aggregate_ret_kfunc_arena.skel.h" + +static bool testmod_has_arena_tagged_member(void) +{ + struct btf *vmlinux_btf, *module_btf = NULL; + const struct btf_type *t; + bool tagged = false; + __s32 id; + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!vmlinux_btf) + return false; + + module_btf = btf__load_module_btf("bpf_testmod", vmlinux_btf); + if (!module_btf) + goto out; + + /* prog_test_ret_arena::a is 'void __arena_tag *': PTR -> TYPE_TAG -> void */ + id = btf__find_by_name_kind(module_btf, "prog_test_ret_arena", BTF_KIND_STRUCT); + if (id <= 0) + goto out; + + t = btf__type_by_id(module_btf, btf_members(btf__type_by_id(module_btf, id))[0].type); + if (!t || !btf_is_ptr(t)) + goto out; + + t = btf__type_by_id(module_btf, t->type); + tagged = t && btf_is_type_tag(t) && + !strcmp(btf__name_by_offset(module_btf, t->name_off), "arena"); + +out: + btf__free(module_btf); + btf__free(vmlinux_btf); + + return tagged; +} + +void test_aggregate_ret(void) +{ + RUN_TESTS(aggregate_ret_func); + RUN_TESTS(aggregate_ret_kfunc); + + if (testmod_has_arena_tagged_member()) + RUN_TESTS(aggregate_ret_kfunc_arena); + else + test__skip(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/arena_atomics.c b/tools/testing/selftests/bpf/prog_tests/arena_atomics.c index 1ad5d03d07ad..222c63b15de0 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_atomics.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_atomics.c @@ -68,6 +68,12 @@ static void test_and(struct arena_atomics *skel) ASSERT_EQ(skel->arena->and64_value, 0x010ull << 32, "and64_value"); ASSERT_EQ(skel->arena->and32_value, 0x010, "and32_value"); + + ASSERT_EQ(skel->arena->and64_result, 0x110ull << 32, "and64_result"); + ASSERT_EQ(skel->arena->and32_result, 0x110, "and32_result"); + + ASSERT_EQ(skel->arena->and64_noreturn_value, 0x010ull << 32, "and64_noreturn_value"); + ASSERT_EQ(skel->arena->and32_noreturn_value, 0x010, "and32_noreturn_value"); } static void test_or(struct arena_atomics *skel) @@ -85,6 +91,12 @@ static void test_or(struct arena_atomics *skel) ASSERT_EQ(skel->arena->or64_value, 0x111ull << 32, "or64_value"); ASSERT_EQ(skel->arena->or32_value, 0x111, "or32_value"); + + ASSERT_EQ(skel->arena->or64_result, 0x110ull << 32, "or64_result"); + ASSERT_EQ(skel->arena->or32_result, 0x110, "or32_result"); + + ASSERT_EQ(skel->arena->or64_noreturn_value, 0x111ull << 32, "or64_noreturn_value"); + ASSERT_EQ(skel->arena->or32_noreturn_value, 0x111, "or32_noreturn_value"); } static void test_xor(struct arena_atomics *skel) @@ -102,6 +114,12 @@ static void test_xor(struct arena_atomics *skel) ASSERT_EQ(skel->arena->xor64_value, 0x101ull << 32, "xor64_value"); ASSERT_EQ(skel->arena->xor32_value, 0x101, "xor32_value"); + + ASSERT_EQ(skel->arena->xor64_result, 0x110ull << 32, "xor64_result"); + ASSERT_EQ(skel->arena->xor32_result, 0x110, "xor32_result"); + + ASSERT_EQ(skel->arena->xor64_noreturn_value, 0x101ull << 32, "xor64_noreturn_value"); + ASSERT_EQ(skel->arena->xor32_noreturn_value, 0x101, "xor32_noreturn_value"); } static void test_cmpxchg(struct arena_atomics *skel) @@ -146,6 +164,27 @@ static void test_xchg(struct arena_atomics *skel) ASSERT_EQ(skel->arena->xchg32_result, 1, "xchg32_result"); } +static void test_fetch_r0(struct arena_atomics *skel) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, prog_fd; + + /* No need to attach it, just run it directly */ + prog_fd = bpf_program__fd(skel->progs.fetch_r0); + err = bpf_prog_test_run_opts(prog_fd, &topts); + if (!ASSERT_OK(err, "test_run_opts err")) + return; + if (!ASSERT_OK(topts.retval, "test_run_opts retval")) + return; + + ASSERT_EQ(skel->arena->fetch_src_r0_value, 0x11100000111ull, "fetch_src_r0_value"); + ASSERT_EQ(skel->arena->fetch_src_r0_result, 0x11000000110ull, "fetch_src_r0_result"); + + ASSERT_EQ(skel->arena->fetch_dst_r0_value, 0x11100000111ull, "fetch_dst_r0_value"); + ASSERT_EQ(skel->arena->fetch_dst_r0_result, 0x11000000110ull, "fetch_dst_r0_result"); + ASSERT_EQ(skel->arena->fetch_dst_r0_readback, 0x11100000111ull, "fetch_dst_r0_readback"); +} + static void test_uaf(struct arena_atomics *skel) { LIBBPF_OPTS(bpf_test_run_opts, topts); @@ -160,6 +199,18 @@ static void test_uaf(struct arena_atomics *skel) return; ASSERT_EQ(skel->arena->uaf_recovery_fails, 0, "uaf_recovery_fails"); + +#ifndef __powerpc64__ + /* + * A fault clears the fetched register and leaves the address alone. The + * powerpc64 JIT does not clear the register of a faulting atomic yet. + */ + ASSERT_EQ(skel->arena->uaf_r1_after_dst_r0, 0, "uaf_r1_after_dst_r0"); + ASSERT_EQ(skel->arena->uaf_r0_after_dst_r0, skel->arena->uaf_r0_before_dst_r0, + "uaf_r0_after_dst_r0"); + ASSERT_EQ(skel->arena->uaf_r0_after_src_r0, 0, "uaf_r0_after_src_r0"); + ASSERT_EQ(skel->arena->uaf_reread, 0, "uaf_reread"); +#endif } static void test_load_acquire(struct arena_atomics *skel) @@ -256,6 +307,8 @@ void serial_test_arena_atomics(void) test_cmpxchg(skel); if (test__start_subtest("xchg")) test_xchg(skel); + if (test__start_subtest("fetch_r0")) + test_fetch_r0(skel); if (test__start_subtest("uaf")) test_uaf(skel); if (test__start_subtest("load_acquire")) diff --git a/tools/testing/selftests/bpf/prog_tests/arena_memcg.c b/tools/testing/selftests/bpf/prog_tests/arena_memcg.c new file mode 100644 index 000000000000..c76a7eb2f01d --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_memcg.c @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE +#include <test_progs.h> +#include <fcntl.h> +#include <linux/magic.h> +#include <sys/vfs.h> +#include <sys/wait.h> +#include <unistd.h> +#include <sys/user.h> +#ifndef PAGE_SIZE /* on some archs it comes in sys/user.h */ +#define PAGE_SIZE getpagesize() +#endif + +#include "cgroup_helpers.h" +#include "arena_memcg.skel.h" + +#define CG_PATH "/arena_memcg" + +/* Reclaimable page cache the child builds up before it gets capped. */ +#define RECLAIMABLE (128 * 1024 * 1024) +/* Headroom left under memory.max, far less than the arena we fault in. */ +#define HEADROOM (8 * 1024 * 1024) +/* Arena to fault in; it only fits by reclaiming the page cache. */ +#define ARENA_FAULT (64 * 1024 * 1024) +/* Child exit code for "this environment cannot host the test". */ +#define CHILD_UNSUPPORTED 9 + +static void dump_memcg(void) +{ + char buf[512]; + + if (!read_cgroup_file(CG_PATH, "memory.max", buf, sizeof(buf))) + fprintf(stderr, "memory.max: %s", buf); + if (!read_cgroup_file(CG_PATH, "memory.peak", buf, sizeof(buf))) + fprintf(stderr, "memory.peak: %s", buf); + if (!read_cgroup_file(CG_PATH, "memory.events", buf, sizeof(buf))) + fprintf(stderr, "memory.events:\n%s", buf); +} + +/* + * Fill the page cache with @size bytes of clean, reclaimable pages by + * reading a sparse temp file, the way the cgroup selftests do. Returns the + * fd, which must stay open: closing it drops the cache. Returns -EOPNOTSUPP + * if the working directory cannot back such a file, -1 on error. + */ +static int alloc_pagecache(size_t size) +{ + struct statfs stfs; + char buf[4096]; + size_t off; + int fd; + + fd = open(".", O_TMPFILE | O_RDWR | O_EXCL, 0600); + if (fd < 0) + return errno == EOPNOTSUPP ? -EOPNOTSUPP : -1; + /* tmpfs hands out shmem pages, which are not reclaimable without swap */ + if (fstatfs(fd, &stfs) || stfs.f_type == TMPFS_MAGIC) { + close(fd); + return -EOPNOTSUPP; + } + if (ftruncate(fd, size)) + goto err; + for (off = 0; off < size; off += sizeof(buf)) + if (read(fd, buf, sizeof(buf)) < 0) + goto err; + return fd; +err: + close(fd); + return -1; +} + +void serial_test_arena_memcg(void) +{ + int cgroup_fd = -1, status, err; + const long ps = PAGE_SIZE; + char buf[64]; + pid_t pid; + + err = setup_cgroup_environment(); + if (!ASSERT_OK(err, "setup_cgroup_environment")) + goto out; + + cgroup_fd = create_and_get_cgroup(CG_PATH); + if (!ASSERT_OK_FD(cgroup_fd, "create_and_get_cgroup")) + goto out; + + /* No memory controller -> nothing to test. */ + if (read_cgroup_file(CG_PATH, "memory.current", buf, sizeof(buf))) { + fprintf(stderr, "%s:SKIP:no memory controller or other env error\n", + __func__); + test__skip(); + goto out; + } + + pid = fork(); + if (!ASSERT_GE(pid, 0, "fork")) + goto out; + if (pid == 0) { + struct arena_memcg *cskel; + __u32 i, npages; + char *base; + size_t sz; + long cur; + int fd; + + /* + * Everything runs in the child: the arena vma is VM_DONTCOPY so + * it does not survive fork(), and only the child should be under + * the limit. The work dir belongs to the parent, so use the + * _parent() helpers; errors come back as an exit code, ASSERT_* + * does not reach the parent from here. + */ + + /* Step 1: join the memcg, so what follows is charged to it. */ + snprintf(buf, sizeof(buf), "%d", getpid()); + if (write_cgroup_file_parent(CG_PATH, "cgroup.procs", buf)) + _exit(2); + + /* + * Step 2: load the arena. A map is charged to whoever creates + * it, hence joining first. + */ + cskel = arena_memcg__open_and_load(); + if (!cskel) + _exit(3); + base = bpf_map__initial_value(cskel->maps.arena, &sz); + if (!base) + _exit(4); + npages = ARENA_FAULT / ps; + if (npages > bpf_map__max_entries(cskel->maps.arena)) + _exit(5); + + /* Step 3: make RECLAIMABLE bytes of clean page cache. */ + fd = alloc_pagecache(RECLAIMABLE); + if (fd == -EOPNOTSUPP) + _exit(CHILD_UNSUPPORTED); + if (fd < 0) + _exit(6); + + /* + * Step 4: set memory.max to what we use now plus HEADROOM. The + * page cache is already inside the limit, so only HEADROOM is + * left. + */ + if (read_cgroup_file_parent(CG_PATH, "memory.current", buf, sizeof(buf))) + _exit(7); + cur = strtol(buf, NULL, 10); + snprintf(buf, sizeof(buf), "%ld", cur + HEADROOM); + if (write_cgroup_file_parent(CG_PATH, "memory.max", buf)) + _exit(8); + + /* + * Step 5: fault ARENA_FAULT of arena in, much more than + * HEADROOM. Once it hits memory.max every further page has to + * come from reclaiming the page cache. With the fix the + * fault-in reclaims and all of it succeeds; without it the + * allocation cannot reclaim and we die on a valid address. + */ + for (i = 0; i < npages; i++) + base[(size_t)i * ps] = 1; + _exit(0); /* fd deliberately kept open until here */ + } + + if (!ASSERT_EQ(waitpid(pid, &status, 0), pid, "waitpid")) + goto out; + + /* The working directory cannot hold a reclaimable page cache. */ + if (WIFEXITED(status) && WEXITSTATUS(status) == CHILD_UNSUPPORTED) { + fprintf(stderr, "%s:SKIP:no disk-backed O_TMPFILE in cwd\n", __func__); + test__skip(); + goto out; + } + + /* A non-zero exit means the child failed to set up; the code says where. */ + if (WIFEXITED(status) && WEXITSTATUS(status)) { + ASSERT_OK(WEXITSTATUS(status), "child setup"); + goto out; + } + + /* + * With the fix the arena fault-in reclaims the page cache and every + * fault succeeds, so the child exits 0. Without it the allocation + * cannot reclaim, fails once the headroom is used up, and the child + * dies with SIGSEGV on a valid arena address. + */ + if (!ASSERT_TRUE(WIFEXITED(status) && !WEXITSTATUS(status), + "child faulted the arena in")) { + if (WIFSIGNALED(status)) + fprintf(stderr, "child killed by signal %d\n", WTERMSIG(status)); + dump_memcg(); + } +out: + if (cgroup_fd >= 0) + close(cgroup_fd); + cleanup_cgroup_environment(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/arena_race.c b/tools/testing/selftests/bpf/prog_tests/arena_race.c new file mode 100644 index 000000000000..f9d0e9c3a006 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_race.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE +#include <sched.h> +#include <setjmp.h> +#include <signal.h> +#include <sys/syscall.h> +#include <test_progs.h> + +#include "arena_race.skel.h" + +struct free_thread_ctx { + struct arena_race *skel; + int err; + __u32 retval; +}; + +struct fault_thread_ctx { + __u64 *addr; + int stop; +}; + +static int run_prog(struct bpf_program *prog, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int err; + + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + return ASSERT_OK(err, name) && ASSERT_OK(opts.retval, name) ? 0 : -1; +} + +/* Trigger the sleepable free page path. */ +static void *run_free_thread(void *arg) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct free_thread_ctx *ctx = arg; + + ctx->skel->bss->target_tid = sys_gettid(); + ctx->err = bpf_prog_test_run_opts( + bpf_program__fd(ctx->skel->progs.free_page), &opts); + ctx->retval = opts.retval; + return NULL; +} + +static __thread sigjmp_buf reader_env; +static __thread bool reader_armed; + +/* Only the reader expects SIGBUS; anywhere else it keeps its default action. */ +static void reader_sigbus(int sig) +{ + if (!reader_armed) { + signal(sig, SIG_DFL); + raise(sig); + return; + } + siglongjmp(reader_env, 1); +} + +/* Continuously fault in the address. */ +static void *fault_reader_thread(void *arg) +{ + struct fault_thread_ctx *ctx = arg; + __u64 *addr; + + /* Touching a page while BPF frees it raises SIGBUS; skip that read. */ + sigsetjmp(reader_env, 1); + reader_armed = true; + while (!READ_ONCE(ctx->stop)) { + addr = __atomic_load_n(&ctx->addr, __ATOMIC_ACQUIRE); + (void)READ_ONCE(*addr); + } + reader_armed = false; + return NULL; +} + +static int wait_for(int *p) +{ + __u64 deadline = get_time_ns() + 5ULL * 1000 * 1000 * 1000; + + while (!READ_ONCE(*p)) { + if (get_time_ns() > deadline) + return -ETIMEDOUT; + } + return 0; +} + +/* Skip the subtest, saying why, if @func cannot be traced with fentry. */ +static bool require_fentry_target(const char *test, const char *func) +{ + if (libbpf_find_vmlinux_btf_id(func, BPF_TRACE_FENTRY) > 0) + return true; + printf("%s:SKIP: %s is not an fentry target\n", test, func); + test__skip(); + return false; +} + +static struct arena_race *setup_arena(__u64 **addr, bool trace_zap) +{ + struct arena_race *skel; + size_t arena_sz; + char *base; + int err; + + skel = arena_race__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return NULL; + err = bpf_program__set_autoload(skel->progs.trace_zap, trace_zap); + if (!ASSERT_OK(err, "trace_zap_autoload")) + goto err_out; + + err = arena_race__load(skel); + if (!ASSERT_OK(err, "load")) + goto err_out; + err = arena_race__attach(skel); + if (!ASSERT_OK(err, "attach")) + goto err_out; + + if (run_prog(skel->progs.alloc_old, "alloc_old")) + goto err_out; + if (skel->bss->skip) { + printf("%s:SKIP: compiler doesn't support arena_cast\n", __func__); + test__skip(); + goto err_out; + } + + base = bpf_map__initial_value(skel->maps.arena, &arena_sz); + if (!ASSERT_OK_PTR(base, "arena_base")) + goto err_out; + *addr = (__u64 *)(base + getpagesize()); + if (!ASSERT_EQ((unsigned long)skel->bss->ptr, (unsigned long)*addr, + "arena_ptr")) + goto err_out; + return skel; + +err_out: + arena_race__destroy(skel); + return NULL; +} + +static void test_realloc_during_free(bool deferred) +{ + struct free_thread_ctx ctx = {}; + struct arena_race *skel; + pthread_t thread; + __u64 *addr; + bool thread_created = false, zap_seen = false, completed = false; + int err; + + if (!require_fentry_target(__func__, "zap_vma_range")) + return; + + skel = setup_arena(&addr, true); + if (!skel) + return; + + /* Map the page into user space, so that the free has to zap it. */ + if (!ASSERT_EQ(READ_ONCE(*addr), skel->rodata->old_marker, "old_marker")) + goto out; + + /* Pause the free after its TLB flush, before it releases the range. */ + skel->bss->pause_on_zap = 1; + + if (deferred) { + /* + * Test the nonsleepable free path that gets + * deferred to a worker in the kernel. We do + * so by triggering the arena operation from + * a nonsleepable tracepoint context. + */ + skel->bss->trigger_pid_tgid = + ((__u64)getpid() << 32) | (__u32)sys_gettid(); + skel->bss->trigger_syscall = SYS_getpgid; + skel->bss->deferred_free = 1; + if (!ASSERT_GE(syscall(SYS_getpgid, 0), 0, "deferred_free")) + goto release; + } else { + /* + * Test the sleepable arena free path through a + * syscall test prog. + */ + ctx.skel = skel; + err = pthread_create(&thread, NULL, run_free_thread, &ctx); + if (!ASSERT_OK(err, "pthread_create")) { + skel->bss->release = 1; + goto out; + } + thread_created = true; + } + + /* Wait until the free reaches the user zap. */ + err = wait_for(&skel->bss->zap_entered); + if (!ASSERT_OK(err, "zap_entered")) + goto release; + + zap_seen = true; + + /* The range must stay allocated until the free completes. */ + run_prog(skel->progs.try_realloc, "realloc_during_free"); + ASSERT_NULL(skel->bss->realloc_ptr, "realloc_during_free"); + +release: + skel->bss->release = 1; + if (thread_created) { + ASSERT_OK(pthread_join(thread, NULL), "pthread_join"); + thread_created = false; + completed = ASSERT_OK(ctx.err, "free_run") && + ASSERT_OK(ctx.retval, "free_retval"); + } else if (skel->bss->target_tid) { + err = wait_for(&skel->bss->worker_exited); + completed = ASSERT_OK(err, "worker_exited"); + } + ASSERT_FALSE(skel->bss->timed_out, "zap_timed_out"); + + if (zap_seen && completed && + !run_prog(skel->progs.try_realloc, "realloc_after_free")) { + ASSERT_EQ((unsigned long)skel->bss->realloc_ptr, + (unsigned long)addr, "realloc_after_free"); + ASSERT_EQ(*addr, skel->rodata->new_marker, "new_marker"); + } +out: + arena_race__destroy(skel); +} + +/* + * Force a race between a faulting thread in userspace and a + * free operation on the arena. + */ +static void test_fault_free_realloc(void) +{ + struct sigaction sigbus = { .sa_handler = reader_sigbus }, old_sigbus; + struct fault_thread_ctx fault = {}; + struct arena_race *skel; + pthread_t fault_thread; + bool fault_created = false; + __u64 *addr; + __u64 expected, value; + int err, i; + + skel = setup_arena(&addr, false); + if (!skel) + return; + + skel->bss->realloc_after_free = 1; + + if (!ASSERT_OK(sigaction(SIGBUS, &sigbus, &old_sigbus), "sigaction_sigbus")) + goto out_destroy; + fault.addr = addr; + err = pthread_create(&fault_thread, NULL, fault_reader_thread, &fault); + if (!ASSERT_OK(err, "pthread_create_fault")) + goto out; + fault_created = true; + + for (i = 0; i < 1000; i++) { + LIBBPF_OPTS(bpf_test_run_opts, opts); + + /* Keep deferred frees from earlier iterations on separate slots. */ + if (i) { + skel->bss->page_idx = i; + if (run_prog(skel->progs.alloc_old, "alloc_next")) + goto out; + addr = (__u64 *)(unsigned long)skel->bss->ptr; + __atomic_store_n(&fault.addr, addr, __ATOMIC_RELEASE); + } + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.free_page), + &opts); + if (err) { + ASSERT_OK(err, "free_realloc"); + break; + } + if (opts.retval) { + ASSERT_OK(opts.retval, "free_realloc"); + break; + } + /* The reader may populate the slot first, or free may defer. */ + if (!skel->bss->realloc_ptr) + continue; + value = *addr; + expected = skel->bss->current_marker; + if (value != expected) { + ASSERT_EQ(value, expected, "marker_after_realloc"); + break; + } + } + + WRITE_ONCE(fault.stop, 1); + if (fault_created) { + ASSERT_OK(pthread_join(fault_thread, NULL), "pthread_join_fault"); + fault_created = false; + } +out: + WRITE_ONCE(fault.stop, 1); + if (fault_created) + pthread_join(fault_thread, NULL); + sigaction(SIGBUS, &old_sigbus, NULL); +out_destroy: + arena_race__destroy(skel); +} + +void serial_test_arena_race(void) +{ + cpu_set_t cpuset; + int err; + + err = sched_getaffinity(0, sizeof(cpuset), &cpuset); + if (!ASSERT_OK(err, "sched_getaffinity")) + return; + if (CPU_COUNT(&cpuset) < 2) { + printf("%s:SKIP: at least two runnable CPUs are required\n", __func__); + test__skip(); + return; + } + + if (test__start_subtest("realloc_during_free")) + test_realloc_during_free(false); + if (test__start_subtest("realloc_during_deferred_free")) + test_realloc_during_free(true); + if (test__start_subtest("fault_free_realloc")) + test_fault_free_realloc(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c b/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c new file mode 100644 index 000000000000..2ac2e9a652fe --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "sysctl_helpers.h" +#include "verifier_arena_scalar.skel.h" + +/* The same tests with constants of the programs blinded */ +void serial_test_arena_scalar_blinded(void) +{ + const char *harden = "/proc/sys/net/core/bpf_jit_harden"; + char old[16] = {}; + + if (!is_jit_enabled()) { + test__skip(); + return; + } + if (sysctl_set_or_fail(harden, old, "2")) + return; + RUN_TESTS(verifier_arena_scalar); + sysctl_set_or_fail(harden, NULL, old); +} diff --git a/tools/testing/selftests/bpf/prog_tests/attach_probe.c b/tools/testing/selftests/bpf/prog_tests/attach_probe.c index e8c1a619e330..41136e1c7752 100644 --- a/tools/testing/selftests/bpf/prog_tests/attach_probe.c +++ b/tools/testing/selftests/bpf/prog_tests/attach_probe.c @@ -543,8 +543,8 @@ static void test_kprobe_sleepable(void) return; /* sleepable kprobe test case needs flags set before loading */ - if (!ASSERT_OK(bpf_program__set_flags(skel->progs.handle_kprobe_sleepable, - BPF_F_SLEEPABLE), "kprobe_sleepable_flags")) + if (!ASSERT_OK(bpf_program__add_flags(skel->progs.handle_kprobe_sleepable, BPF_F_SLEEPABLE), + "kprobe_sleepable_flags")) goto cleanup; if (!ASSERT_OK(test_attach_kprobe_sleepable__load(skel), diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c index 0222a9a5d076..4ffe4fe6b78e 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c @@ -1,9 +1,11 @@ // SPDX-License-Identifier: GPL-2.0 #include <bpf/bpf.h> +#include <bpf/btf.h> #include <test_progs.h> -#if defined(__x86_64__) || defined(__powerpc__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__powerpc__) || defined(__aarch64__) || \ + (defined(__riscv) && __riscv_xlen == 64) static int map_create(__u32 map_type, __u32 max_entries) { const char *map_name = "insn_array"; @@ -453,43 +455,672 @@ cleanup: close(map_fd); } +#define GOTOX_LOG_SZ (256 * 1024) + +/* Load and return the fd, which the caller owns. */ +static int gotox_prog_load_funcs_fd(struct bpf_insn *insns, __u32 insn_cnt, + int *fd_array, __u32 fd_array_cnt, char *log, + int btf_fd, struct bpf_func_info *fi, __u32 fi_cnt) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts); + + log[0] = 0; + opts.fd_array = fd_array; + opts.fd_array_cnt = fd_array_cnt; + opts.log_buf = log; + opts.log_size = GOTOX_LOG_SZ; + opts.log_level = 1; + if (fi_cnt) { + opts.prog_btf_fd = btf_fd; + opts.func_info = fi; + opts.func_info_cnt = fi_cnt; + opts.func_info_rec_size = sizeof(*fi); + } + + return bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, &opts); +} + +/* Load for the outcome only: 0 or the negative errno. */ +static int gotox_prog_load_funcs(struct bpf_insn *insns, __u32 insn_cnt, + int *fd_array, __u32 fd_array_cnt, char *log, + int btf_fd, struct bpf_func_info *fi, __u32 fi_cnt) +{ + int prog_fd; + + prog_fd = gotox_prog_load_funcs_fd(insns, insn_cnt, fd_array, fd_array_cnt, log, + btf_fd, fi, fi_cnt); + if (prog_fd < 0) + return prog_fd; + close(prog_fd); + return 0; +} + +static int gotox_prog_load(struct bpf_insn *insns, __u32 insn_cnt, + int *fd_array, __u32 fd_array_cnt, char *log) +{ + return gotox_prog_load_funcs(insns, insn_cnt, fd_array, fd_array_cnt, log, + -1, NULL, 0); +} + +static int gotox_jt_create_offs(const __u32 *offs, __u32 cnt) +{ + struct bpf_insn_array_value val = {}; + int map_fd; + __u32 i; + + map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, cnt); + if (!ASSERT_GE(map_fd, 0, "map_create")) + return map_fd; + + for (i = 0; i < cnt; i++) { + val.orig_off = offs[i]; + if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &i, &val, 0), 0, + "bpf_map_update_elem")) + goto err; + } + + if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) + goto err; + + return map_fd; +err: + close(map_fd); + return -1; +} + +#define GOTOX_SUB_START 4 +#define GOTOX_MAIN_TGT 2 +#define GOTOX_SUB_TGT 8 +#define GOTOX_TWO_INSN_CNT 10 + +static void gotox_two_subprogs_fill(struct bpf_insn *insns, __u32 jt_idx, __u32 jt_off) +{ + insns[0] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[1] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + GOTOX_SUB_START - 1 - 1); + insns[GOTOX_MAIN_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[3] = BPF_EXIT_INSN(); + + /* r1 = &jt[0], by index 'jt_idx' into fd_array */ + insns[GOTOX_SUB_START] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = jt_idx, + }; + insns[GOTOX_SUB_START + 1] = (struct bpf_insn) { .imm = 0 }; + insns[6] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, jt_off * 8); + insns[7] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); + insns[GOTOX_SUB_TGT] = BPF_MOV64_IMM(BPF_REG_0, 1); + insns[9] = BPF_EXIT_INSN(); +} + +/* + * Load 'insns' with the two jump tables 'jt_a' and 'jt_b', in this order in + * fd_array, and check the outcome. 'fi' describes the subprograms when given. + */ +static void check_gotox_two_maps(struct bpf_insn *insns, __u32 insn_cnt, + const __u32 *jt_a, __u32 a_cnt, + const __u32 *jt_b, __u32 b_cnt, + int btf_fd, struct bpf_func_info *fi, __u32 fi_cnt, + int expect_err, const char *expect_msg) +{ + int map_fd[2] = { -1, -1 }; + char *log; + int err; + + log = calloc(1, GOTOX_LOG_SZ); + if (!ASSERT_OK_PTR(log, "calloc log")) + return; + + map_fd[0] = gotox_jt_create_offs(jt_a, a_cnt); + if (map_fd[0] < 0) + goto free_log; + map_fd[1] = gotox_jt_create_offs(jt_b, b_cnt); + if (map_fd[1] < 0) + goto close_maps; + + if (fi) + err = gotox_prog_load_funcs(insns, insn_cnt, map_fd, 2, log, + btf_fd, fi, fi_cnt); + else + err = gotox_prog_load(insns, insn_cnt, map_fd, 2, log); + if (expect_err) + ASSERT_EQ(err, expect_err, "program should have been rejected"); + else + ASSERT_EQ(err, 0, "bpf(BPF_PROG_LOAD)"); + if (expect_msg) + ASSERT_HAS_SUBSTR(log, expect_msg, "verifier log"); + +close_maps: + close(map_fd[0]); + if (map_fd[1] >= 0) + close(map_fd[1]); +free_log: + free(log); +} + +static void check_gotox_target_other_subprog(void) +{ + const __u32 jt_main[] = { GOTOX_MAIN_TGT }; + const __u32 jt_sub[] = { GOTOX_SUB_TGT }; + struct bpf_insn insns[GOTOX_TWO_INSN_CNT]; + + gotox_two_subprogs_fill(insns, 0, 0); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), + jt_sub, ARRAY_SIZE(jt_sub), -1, NULL, 0, + -EINVAL, "indirect jump from insn 7 to 2 leaves the subprog [4,10)"); +} + +static void check_gotox_jt_per_subprog(void) +{ + const __u32 jt_main[] = { GOTOX_MAIN_TGT }; + const __u32 jt_sub[] = { GOTOX_SUB_TGT }; + struct bpf_insn insns[GOTOX_TWO_INSN_CNT]; + + gotox_two_subprogs_fill(insns, 1, 0); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), + jt_sub, ARRAY_SIZE(jt_sub), -1, NULL, 0, + 0, NULL); +} + +#define GOTOX_DEAD_MAIN_LEN 7 +#define GOTOX_DEAD_SUB_LEN 6 +#define GOTOX_DEAD_INSN_CNT (GOTOX_DEAD_MAIN_LEN + 3 * GOTOX_DEAD_SUB_LEN) +#define GOTOX_DEAD_LIVE_CNT 10 + +/* r1 = &jt[0], by index 'jt_idx' into fd_array; r1 = *r1; gotox r1; r0 = 0; exit */ +static void gotox_sub_fill(struct bpf_insn *insns, __u32 start, __u32 jt_idx) +{ + insns[start] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = jt_idx, + }; + insns[start + 1] = (struct bpf_insn) { .imm = 0 }; + insns[start + 2] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + insns[start + 3] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); + insns[start + 4] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[start + 5] = BPF_EXIT_INSN(); +} + +/* + * Two subprograms which are only called from dead code sit in front of the + * live one, and all three own a jump table. Dead code removal drops both in + * one go, which compacts subprog_info[] along with the tables. + */ +static void check_gotox_jt_dead_subprogs(void) +{ + const __u32 sub_start[3] = { GOTOX_DEAD_MAIN_LEN, + GOTOX_DEAD_MAIN_LEN + GOTOX_DEAD_SUB_LEN, + GOTOX_DEAD_MAIN_LEN + 2 * GOTOX_DEAD_SUB_LEN }; + struct bpf_insn insns[GOTOX_DEAD_INSN_CNT]; + int map_fd[3] = { -1, -1, -1 }; + struct bpf_prog_info info = {}; + __u32 len = sizeof(info), tgt, i; + int prog_fd; + char *log; + + log = calloc(1, GOTOX_LOG_SZ); + if (!ASSERT_OK_PTR(log, "calloc log")) + return; + + insns[0] = BPF_MOV64_IMM(BPF_REG_1, 0); + insns[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2); + insns[2] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + sub_start[0] - 2 - 1); + insns[3] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + sub_start[1] - 3 - 1); + insns[4] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + sub_start[2] - 4 - 1); + insns[5] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[6] = BPF_EXIT_INSN(); + + for (i = 0; i < ARRAY_SIZE(sub_start); i++) { + gotox_sub_fill(insns, sub_start[i], i); + tgt = sub_start[i] + 4; + map_fd[i] = gotox_jt_create_offs(&tgt, 1); + if (map_fd[i] < 0) + goto close_maps; + } + + prog_fd = gotox_prog_load_funcs_fd(insns, ARRAY_SIZE(insns), map_fd, + ARRAY_SIZE(map_fd), log, -1, NULL, 0); + if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) + goto close_maps; + + /* Both dead subprograms and the calls to them must be gone. */ + if (ASSERT_OK(bpf_prog_get_info_by_fd(prog_fd, &info, &len), + "bpf_prog_get_info_by_fd")) + ASSERT_EQ(info.xlated_prog_len / sizeof(struct bpf_insn), + GOTOX_DEAD_LIVE_CNT, "xlated insn count"); + close(prog_fd); +close_maps: + for (i = 0; i < ARRAY_SIZE(map_fd); i++) + if (map_fd[i] >= 0) + close(map_fd[i]); + free(log); +} + +#define GOTOX_FWD_GOTOX 11 +#define GOTOX_FWD_OWN_TGT 12 +#define GOTOX_FWD_SUB_START 14 +#define GOTOX_FWD_INSN_CNT 16 + +static void gotox_from_main_fill(struct bpf_insn *insns) +{ + insns[0] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1); + insns[1] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + GOTOX_FWD_SUB_START - 1 - 1); + insns[2] = BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_6, + offsetof(struct xdp_md, ingress_ifindex)); + insns[3] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_2, 0, 4); + + /* r1 = &jt_leaves[0], by index 1 into fd_array */ + insns[4] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = 1, + }; + insns[5] = (struct bpf_insn) { .imm = 0 }; + insns[6] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + insns[7] = BPF_JMP_A(3); + + /* r1 = &jt_own[0], by index 0 into fd_array */ + insns[8] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = 0, + }; + insns[9] = (struct bpf_insn) { .imm = 0 }; + insns[10] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + + insns[GOTOX_FWD_GOTOX] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); + insns[GOTOX_FWD_OWN_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[13] = BPF_EXIT_INSN(); + insns[GOTOX_FWD_SUB_START] = BPF_MOV64_IMM(BPF_REG_0, 1); + insns[15] = BPF_EXIT_INSN(); +} + +static void check_gotox_target_subprog_from_main(void) +{ + const __u32 jt_own[] = { GOTOX_FWD_OWN_TGT }; + const __u32 jt_leaves[] = { GOTOX_FWD_SUB_START }; + struct bpf_insn insns[GOTOX_FWD_INSN_CNT]; + + gotox_from_main_fill(insns); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_own, ARRAY_SIZE(jt_own), + jt_leaves, ARRAY_SIZE(jt_leaves), -1, NULL, 0, + -EINVAL, "indirect jump from insn 11 to 14 leaves the subprog [0,14)"); +} + +/* + * The only map of the subprog holding the gotox reaches past that subprog, so + * the subprog is left without a jump table at all. + */ +static void check_gotox_jt_spans_subprogs(void) +{ + const __u32 jt_span[] = { GOTOX_FWD_OWN_TGT, GOTOX_FWD_SUB_START }; + const __u32 jt_leaves[] = { GOTOX_FWD_SUB_START }; + struct bpf_insn insns[GOTOX_FWD_INSN_CNT]; + + gotox_from_main_fill(insns); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_span, ARRAY_SIZE(jt_span), + jt_leaves, ARRAY_SIZE(jt_leaves), -1, NULL, 0, + -EINVAL, "jump table of subprog starting at 0 spans multiple subprogs"); +} + +/* + * The subprog holding the gotox has a well formed jump table of its own and + * also collects a map that reaches past its end. The spanning map is still + * rejected, even though the subprog is not left without a table. + */ +static void check_gotox_jt_spans_with_own_table(void) +{ + const __u32 jt_own[] = { GOTOX_FWD_OWN_TGT }; + const __u32 jt_span[] = { GOTOX_FWD_OWN_TGT, GOTOX_FWD_SUB_START }; + struct bpf_insn insns[GOTOX_FWD_INSN_CNT]; + + gotox_from_main_fill(insns); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_own, ARRAY_SIZE(jt_own), + jt_span, ARRAY_SIZE(jt_span), -1, NULL, 0, + -EINVAL, "jump table of subprog starting at 0 spans multiple subprogs"); +} + +#define GOTOX_SLICE_SUB_START 6 +#define GOTOX_SLICE_GOTOX 14 +#define GOTOX_SLICE_SUB_TGT 15 +#define GOTOX_SLICE_INSN_CNT 17 + +static void gotox_slice_fill(struct bpf_insn *insns) +{ + insns[0] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[1] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, + GOTOX_SLICE_SUB_START - 1 - 1); + insns[2] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[3] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[4] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[5] = BPF_EXIT_INSN(); + + insns[GOTOX_SLICE_SUB_START] = + BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, + offsetof(struct xdp_md, ingress_ifindex)); + insns[7] = BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 1); + insns[8] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1); + insns[9] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); + + /* r1 = &jt_main[0], by index 0 into fd_array */ + insns[10] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = 0, + }; + insns[11] = (struct bpf_insn) { .imm = 0 }; + insns[12] = BPF_ALU64_REG(BPF_ADD, BPF_REG_1, BPF_REG_2); + insns[13] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + + insns[GOTOX_SLICE_GOTOX] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); + insns[GOTOX_SLICE_SUB_TGT] = BPF_MOV64_IMM(BPF_REG_0, 1); + insns[16] = BPF_EXIT_INSN(); +} + +static void check_gotox_index_slice_other_subprog(void) +{ + const __u32 jt_main[] = { 2, 3, 4 }; + const __u32 jt_sub[] = { GOTOX_SLICE_SUB_TGT }; + struct bpf_insn insns[GOTOX_SLICE_INSN_CNT]; + + gotox_slice_fill(insns); + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), + jt_sub, ARRAY_SIZE(jt_sub), -1, NULL, 0, + -EINVAL, "indirect jump from insn 14 to 3 leaves the subprog [6,17)"); +} + +static int gotox_btf_create(const __u32 *starts, const __u8 *linkage, __u32 cnt, + struct bpf_func_info *fi, struct btf **pbtf) +{ + int int_id, proto_id, id; + struct btf *btf; + char name[24]; + __u32 i; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf__new_empty")) + return -1; + + int_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_GT(int_id, 0, "btf__add_int")) + goto err; + + proto_id = btf__add_func_proto(btf, int_id); + if (!ASSERT_GT(proto_id, 0, "btf__add_func_proto")) + goto err; + + for (i = 0; i < cnt; i++) { + snprintf(name, sizeof(name), "gotox_f%u", i); + id = btf__add_func(btf, name, linkage[i], proto_id); + if (!ASSERT_GT(id, 0, "btf__add_func")) + goto err; + fi[i].insn_off = starts[i]; + fi[i].type_id = id; + } + + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf__load_into_kernel")) + goto err; + + *pbtf = btf; + return btf__fd(btf); +err: + btf__free(btf); + return -1; +} + +static void check_gotox_target_other_global_subprog(void) +{ + const __u32 starts[] = { 0, GOTOX_SUB_START }; + const __u8 linkage[] = { BTF_FUNC_GLOBAL, BTF_FUNC_GLOBAL }; + const __u32 jt_main[] = { GOTOX_MAIN_TGT }; + const __u32 jt_sub[] = { GOTOX_SUB_TGT }; + struct bpf_insn insns[GOTOX_TWO_INSN_CNT]; + struct bpf_func_info fi[2]; + struct btf *btf = NULL; + int btf_fd; + + gotox_two_subprogs_fill(insns, 0, 0); + + btf_fd = gotox_btf_create(starts, linkage, ARRAY_SIZE(starts), fi, &btf); + if (btf_fd < 0) + return; + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), + jt_sub, ARRAY_SIZE(jt_sub), btf_fd, fi, ARRAY_SIZE(fi), + -EINVAL, "indirect jump from insn 7 to 2 leaves the subprog [4,10)"); + btf__free(btf); +} + +#define GOTOX_CB_MAIN_TGT 6 +#define GOTOX_CB_START 8 +#define GOTOX_CB_GOTOX 11 +#define GOTOX_CB_TGT 12 +#define GOTOX_CB_INSN_CNT 14 + +static void gotox_callback_fill(struct bpf_insn *insns) +{ + insns[0] = BPF_MOV64_IMM(BPF_REG_1, 1); + /* r2 = &callback */ + insns[1] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_2, + .src_reg = BPF_PSEUDO_FUNC, + .imm = GOTOX_CB_START - 1 - 1, + }; + insns[2] = (struct bpf_insn) { .imm = 0 }; + insns[3] = BPF_MOV64_IMM(BPF_REG_3, 0); + insns[4] = BPF_MOV64_IMM(BPF_REG_4, 0); + insns[5] = BPF_EMIT_CALL(BPF_FUNC_loop); + insns[GOTOX_CB_MAIN_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[7] = BPF_EXIT_INSN(); + + /* r1 = &jt_main[0], by index 0 into fd_array */ + insns[GOTOX_CB_START] = (struct bpf_insn) { + .code = BPF_LD | BPF_DW | BPF_IMM, + .dst_reg = BPF_REG_1, + .src_reg = BPF_PSEUDO_MAP_IDX_VALUE, + .imm = 0, + }; + insns[9] = (struct bpf_insn) { .imm = 0 }; + insns[10] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); + insns[GOTOX_CB_GOTOX] = BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0); + insns[GOTOX_CB_TGT] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[13] = BPF_EXIT_INSN(); +} + +static void check_gotox_callback_leaves_subprog(void) +{ + const __u32 starts[] = { 0, GOTOX_CB_START }; + const __u8 linkage[] = { BTF_FUNC_GLOBAL, BTF_FUNC_STATIC }; + const __u32 jt_main[] = { GOTOX_CB_MAIN_TGT }; + const __u32 jt_cb[] = { GOTOX_CB_TGT }; + struct bpf_insn insns[GOTOX_CB_INSN_CNT]; + struct bpf_func_info fi[2]; + struct btf *btf = NULL; + int btf_fd; + + gotox_callback_fill(insns); + + btf_fd = gotox_btf_create(starts, linkage, ARRAY_SIZE(starts), fi, &btf); + if (btf_fd < 0) + return; + + check_gotox_two_maps(insns, ARRAY_SIZE(insns), jt_main, ARRAY_SIZE(jt_main), + jt_cb, ARRAY_SIZE(jt_cb), btf_fd, fi, ARRAY_SIZE(fi), + -EINVAL, "indirect jump from insn 11 to 6 leaves the subprog [8,14)"); + btf__free(btf); +} + static void check_bpf_side(void) { check_bpf_no_lookup(); } +/* Return the xlated index of the first insn equal to 'insn', or -1. */ +static int xlated_insn_index(int prog_fd, const struct bpf_insn *insn) +{ + struct bpf_insn *xlated; + int idx = -1; + __u32 cnt, i; + + if (!ASSERT_OK(get_xlated_program(prog_fd, &xlated, &cnt), "get_xlated_program")) + return -1; + for (i = 0; i < cnt; i++) + if (!memcmp(&xlated[i], insn, sizeof(*insn))) { + idx = i; + break; + } + free(xlated); + return idx; +} + +/* + * Load 'insns' with a jump table naming insn 0, which a prologue prepend + * pushes down, and check that the slot followed it. + */ +static void check_gotox_target_prepend(enum bpf_prog_type prog_type, + struct bpf_insn *insns, __u32 insn_cnt) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts); + struct bpf_insn_array_value val = {}; + int prog_fd = -1, map_fd, idx; + __u32 key = 0; + + map_fd = map_create(BPF_MAP_TYPE_INSN_ARRAY, 1); + if (!ASSERT_GE(map_fd, 0, "map_create")) + return; + + val.orig_off = 0; + if (!ASSERT_EQ(bpf_map_update_elem(map_fd, &key, &val, 0), 0, "bpf_map_update_elem")) + goto cleanup; + if (!ASSERT_EQ(bpf_map_freeze(map_fd), 0, "bpf_map_freeze")) + goto cleanup; + + opts.fd_array = &map_fd; + opts.fd_array_cnt = 1; + prog_fd = bpf_prog_load(prog_type, NULL, "GPL", insns, insn_cnt, &opts); + if (!ASSERT_GE(prog_fd, 0, "bpf(BPF_PROG_LOAD)")) + goto cleanup; + + idx = xlated_insn_index(prog_fd, &insns[0]); + if (!ASSERT_GT(idx, 0, "insn 0 was prepended to")) + goto cleanup; + if (!ASSERT_EQ(bpf_map_lookup_elem(map_fd, &key, &val), 0, "bpf_map_lookup_elem")) + goto cleanup; + ASSERT_EQ(val.xlated_off, (__u32)idx, "gotox target followed insn 0 past the prepend"); +cleanup: + if (prog_fd >= 0) + close(prog_fd); + close(map_fd); +} + +static void check_gotox_target_prologue_shift(void) +{ + struct bpf_insn insns[] = { + /* insn 0: gotox target and subprog start */ + BPF_MOV64_IMM(BPF_REG_0, 0), + /* may_goto +4 -> exit block, bounds the loop */ + BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 4, 0), + /* r1 = &jt[0] (insns 2 and 3) */ + BPF_LD_IMM64_RAW(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0), /* r1 = ips[0] */ + /* insn 5: gotox r1 -> insn 0 */ + BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_1, 0, 0, 0), + BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), /* insn 6: exit block */ + BPF_EXIT_INSN(), + }; + + check_gotox_target_prepend(BPF_PROG_TYPE_XDP, insns, ARRAY_SIZE(insns)); +} + +static void check_gotox_target_ctx_prologue_shift(void) +{ + struct bpf_insn insns[] = { + /* insn 0: gotox target and subprog start */ + BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, -8), + BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, 0x5a5a, 12), /* second visit exits */ + BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0x5a5a), + BPF_MOV64_REG(BPF_REG_6, BPF_REG_1), /* r6 = ctx */ + BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_6, offsetof(struct __sk_buff, data)), + BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_6, offsetof(struct __sk_buff, data_end)), + BPF_MOV64_REG(BPF_REG_4, BPF_REG_2), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 1), + BPF_JMP_REG(BPF_JGT, BPF_REG_4, BPF_REG_3, 5), + /* insn 9: direct packet write -> tc unclone prologue prepended at insn 0 */ + BPF_ST_MEM(BPF_B, BPF_REG_2, 0, 0), + /* r5 = &jt[0] (insns 10 and 11) */ + BPF_LD_IMM64_RAW(BPF_REG_5, BPF_PSEUDO_MAP_IDX_VALUE, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_5, BPF_REG_5, 0), /* r5 = ips[0] */ + /* insn 13: gotox r5 -> insn 0 */ + BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_5, 0, 0, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), /* insn 14: exit block */ + BPF_EXIT_INSN(), + }; + + check_gotox_target_prepend(BPF_PROG_TYPE_SCHED_CLS, insns, ARRAY_SIZE(insns)); +} + static void __test_bpf_insn_array(void) { /* Test if offsets are adjusted properly */ - if (test__start_subtest("one2one")) check_one_to_one_mapping(); - if (test__start_subtest("simple")) check_simple(); - if (test__start_subtest("deletions")) check_deletions(); - if (test__start_subtest("deletions-with-functions")) check_deletions_with_functions(); - if (test__start_subtest("blindness")) check_blindness(); - /* Check all kinds of operations and related restrictions */ - if (test__start_subtest("incorrect-index")) check_incorrect_index(); - if (test__start_subtest("load-unfrozen-map")) check_load_unfrozen_map(); - if (test__start_subtest("no-map-reuse")) check_no_map_reuse(); - if (test__start_subtest("bpf-side-ops")) check_bpf_side(); + if (test__start_subtest("gotox-jt-spans-subprogs")) + check_gotox_jt_spans_subprogs(); + if (test__start_subtest("gotox-jt-spans-with-own-table")) + check_gotox_jt_spans_with_own_table(); + if (test__start_subtest("gotox-target-other-subprog")) + check_gotox_target_other_subprog(); + if (test__start_subtest("gotox-jt-per-subprog")) + check_gotox_jt_per_subprog(); + if (test__start_subtest("gotox-jt-dead-subprogs")) + check_gotox_jt_dead_subprogs(); + if (test__start_subtest("gotox-target-subprog-from-main")) + check_gotox_target_subprog_from_main(); + if (test__start_subtest("gotox-index-slice-other-subprog")) + check_gotox_index_slice_other_subprog(); + if (test__start_subtest("gotox-target-other-global-subprog")) + check_gotox_target_other_global_subprog(); + if (test__start_subtest("gotox-callback-leaves-subprog")) + check_gotox_callback_leaves_subprog(); + if (test__start_subtest("gotox-target-prologue-shift")) + check_gotox_target_prologue_shift(); + if (test__start_subtest("gotox-target-ctx-prologue-shift")) + check_gotox_target_ctx_prologue_shift(); } #else static void __test_bpf_insn_array(void) diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c index 14d4c1793aed..d74a9db54c9a 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c @@ -13,13 +13,13 @@ struct { const char *prog_name; const char *err_msg; } test_bpf_nf_fail_tests[] = { - { "alloc_release", "kernel function bpf_ct_release R1 expected pointer to STRUCT nf_conn but" }, - { "insert_insert", "kernel function bpf_ct_insert_entry R1 expected pointer to STRUCT nf_conn___init but" }, - { "lookup_insert", "kernel function bpf_ct_insert_entry R1 expected pointer to STRUCT nf_conn___init but" }, - { "set_timeout_after_insert", "kernel function bpf_ct_set_timeout R1 expected pointer to STRUCT nf_conn___init but" }, - { "set_status_after_insert", "kernel function bpf_ct_set_status R1 expected pointer to STRUCT nf_conn___init but" }, - { "change_timeout_after_alloc", "kernel function bpf_ct_change_timeout R1 expected pointer to STRUCT nf_conn but" }, - { "change_status_after_alloc", "kernel function bpf_ct_change_status R1 expected pointer to STRUCT nf_conn but" }, + { "alloc_release", "bpf_ct_release R1 expected pointer to STRUCT nf_conn but" }, + { "insert_insert", "bpf_ct_insert_entry R1 expected pointer to STRUCT nf_conn___init but" }, + { "lookup_insert", "bpf_ct_insert_entry R1 expected pointer to STRUCT nf_conn___init but" }, + { "set_timeout_after_insert", "bpf_ct_set_timeout R1 expected pointer to STRUCT nf_conn___init but" }, + { "set_status_after_insert", "bpf_ct_set_status R1 expected pointer to STRUCT nf_conn___init but" }, + { "change_timeout_after_alloc", "bpf_ct_change_timeout R1 expected pointer to STRUCT nf_conn but" }, + { "change_status_after_alloc", "bpf_ct_change_status R1 expected pointer to STRUCT nf_conn but" }, { "write_not_allowlisted_field", "no write support to nf_conn at off" }, { "lookup_null_bpf_tuple", "Possibly NULL pointer passed to trusted R2" }, { "lookup_null_bpf_opts", "Possibly NULL pointer passed to trusted R4" }, diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c b/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c index 6dbd1487343c..122ecb7e98e2 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c @@ -11,6 +11,7 @@ #include "bpf_qdisc_fail__invalid_dynptr.skel.h" #include "bpf_qdisc_fail__invalid_dynptr_slice.skel.h" #include "bpf_qdisc_fail__invalid_dynptr_cross_frame.skel.h" +#include "bpf_qdisc_fail__invalid_dynptr_returned_slice.skel.h" #include "bpf_qdisc_fail__untrusted_write.skel.h" #include "bpf_qdisc_dynptr_use_after_invalidate_clone.skel.h" @@ -230,6 +231,7 @@ void test_ns_bpf_qdisc(void) test_incompl_ops(); RUN_TESTS(bpf_qdisc_fail__invalid_dynptr); RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_cross_frame); + RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_returned_slice); RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_slice); RUN_TESTS(bpf_qdisc_fail__untrusted_write); RUN_TESTS(bpf_qdisc_dynptr_use_after_invalidate_clone); diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops.c b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops.c new file mode 100644 index 000000000000..6435ed1c2cf6 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops.c @@ -0,0 +1,560 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <test_progs.h> +#include <network_helpers.h> +#include <bpf/btf.h> +#include "cgroup_helpers.h" +#include "bpf_tcp_ops.skel.h" + +#define CGROUP_PATH "/bpf_tcp_ops" +#define TEST_NETNS "bpf_tcp_ops" + +static __s32 get_bpf_tcp_ops_type_id(void) +{ + struct btf *vmlinux_btf; + __s32 type_id; + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(vmlinux_btf, "load_vmlinux_btf")) + return -1; + + type_id = btf__find_by_name_kind(vmlinux_btf, "bpf_tcp_ops", BTF_KIND_STRUCT); + btf__free(vmlinux_btf); + + ASSERT_GT(type_id, 0, "find_bpf_tcp_ops"); + return type_id; +} + +static void reset_order(struct bpf_tcp_ops *skel) +{ + memset(skel->bss->listen_order, 0, sizeof(skel->bss->listen_order)); + memset(skel->bss->connect_order, 0, sizeof(skel->bss->connect_order)); + skel->bss->listen_cnt = 0; + skel->bss->connect_cnt = 0; +} + +static void do_listen_connect(int family) +{ + const char *addr = family == AF_INET ? "127.0.0.1" : "::1"; + int server_fd, client_fd; + + server_fd = start_server(family, SOCK_STREAM, addr, 0, 0); + if (!ASSERT_GE(server_fd, 0, "start_server")) + return; + + client_fd = connect_to_fd(server_fd, 0); + if (ASSERT_OK_FD(client_fd, "connect_to_fd")) + close(client_fd); + + close(server_fd); +} + +/* + * Attach ops1 and ops2 normally (in that order), then ops3 with + * BPF_F_PREORDER. Expected execution order: [3, 1, 2] — ops3 runs + * first despite being attached last, ops1 before ops2 by attach order. + */ +static void test_order(int cgroup_fd, struct bpf_tcp_ops *skel, int family) +{ + LIBBPF_OPTS(bpf_cgroup_opts, preorder_opts, .flags = BPF_F_PREORDER); + struct bpf_link *link1 = NULL, *link2 = NULL, *link3 = NULL; + + link1 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link1, "attach_ops1")) + goto done; + + link2 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops2, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link2, "attach_ops2")) + goto done; + + link3 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops3, cgroup_fd, + &preorder_opts); + if (!ASSERT_OK_PTR(link3, "attach_ops3_preorder")) + goto done; + + reset_order(skel); + do_listen_connect(family); + + ASSERT_EQ(skel->bss->listen_cnt, 3, "listen_cnt"); + ASSERT_EQ(skel->bss->listen_order[0], 3, "listen_order[0]"); + ASSERT_EQ(skel->bss->listen_order[1], 1, "listen_order[1]"); + ASSERT_EQ(skel->bss->listen_order[2], 2, "listen_order[2]"); + + ASSERT_EQ(skel->bss->connect_cnt, 3, "connect_cnt"); + ASSERT_EQ(skel->bss->connect_order[0], 3, "connect_order[0]"); + ASSERT_EQ(skel->bss->connect_order[1], 1, "connect_order[1]"); + ASSERT_EQ(skel->bss->connect_order[2], 2, "connect_order[2]"); + +done: + bpf_link__destroy(link3); + bpf_link__destroy(link2); + bpf_link__destroy(link1); +} + +static void run_order_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_order(cgroup_fd, skel, AF_INET); + test_order(cgroup_fd, skel, AF_INET6); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +/* + * Position a new attachment relative to an existing one. Attach ops1, then + * ops2 with BPF_F_BEFORE ops1, then ops3 with BPF_F_AFTER ops2. Expected + * execution order: [2, 3, 1]. For struct_ops, relative_fd refers to a link + * fd, so BPF_F_LINK must be set. + */ +static void test_before_after(int cgroup_fd, struct bpf_tcp_ops *skel) +{ + LIBBPF_OPTS(bpf_cgroup_opts, opts); + struct bpf_link *link1 = NULL, *link2 = NULL, *link3 = NULL; + + link1 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link1, "attach_ops1")) + goto done; + + opts.flags = BPF_F_BEFORE | BPF_F_LINK; + opts.relative_fd = bpf_link__fd(link1); + link2 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops2, cgroup_fd, &opts); + if (!ASSERT_OK_PTR(link2, "attach_ops2_before")) + goto done; + + opts.flags = BPF_F_AFTER | BPF_F_LINK; + opts.relative_fd = bpf_link__fd(link2); + link3 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops3, cgroup_fd, &opts); + if (!ASSERT_OK_PTR(link3, "attach_ops3_after")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + + ASSERT_EQ(skel->bss->listen_cnt, 3, "listen_cnt"); + ASSERT_EQ(skel->bss->listen_order[0], 2, "listen_order[0]"); + ASSERT_EQ(skel->bss->listen_order[1], 3, "listen_order[1]"); + ASSERT_EQ(skel->bss->listen_order[2], 1, "listen_order[2]"); + +done: + bpf_link__destroy(link3); + bpf_link__destroy(link2); + bpf_link__destroy(link1); +} + +static void run_before_after_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_before_after(cgroup_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +static void test_query(int cgroup_fd, struct bpf_tcp_ops *skel) +{ + LIBBPF_OPTS(bpf_prog_query_opts, query_opts); + struct bpf_map_info info = {}; + __u32 info_len = sizeof(info); + struct bpf_link *link1 = NULL, *link2 = NULL; + __u32 map1_id, map2_id, map_ids[2] = {}; + __s32 type_id; + + type_id = get_bpf_tcp_ops_type_id(); + if (type_id <= 0) + return; + + bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.tcp_ops1), &info, &info_len); + map1_id = info.id; + + bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.tcp_ops2), &info, &info_len); + map2_id = info.id; + + link1 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link1, "attach_ops1")) + goto done; + + link2 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops2, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link2, "attach_ops2")) + goto done; + + /* query effective: expect 2 entries in attachment order */ + query_opts.type_id = type_id; + query_opts.prog_ids = map_ids; + query_opts.count = ARRAY_SIZE(map_ids); + query_opts.query_flags = BPF_F_QUERY_EFFECTIVE; + ASSERT_OK(bpf_prog_query_opts(cgroup_fd, BPF_STRUCT_OPS, &query_opts), + "query_effective"); + ASSERT_EQ(query_opts.count, 2, "query_effective_count"); + ASSERT_EQ(map_ids[0], map1_id, "map_ids[0]"); + ASSERT_EQ(map_ids[1], map2_id, "map_ids[1]"); + + /* query attached (non-effective): expect 2 entries */ + memset(map_ids, 0, sizeof(map_ids)); + query_opts.query_flags = 0; + query_opts.count = ARRAY_SIZE(map_ids); + ASSERT_OK(bpf_prog_query_opts(cgroup_fd, BPF_STRUCT_OPS, &query_opts), + "query_attached"); + ASSERT_EQ(query_opts.count, 2, "query_attached_count"); + ASSERT_EQ(map_ids[0], map1_id, "attached_map_ids[0]"); + ASSERT_EQ(map_ids[1], map2_id, "attached_map_ids[1]"); + +done: + bpf_link__destroy(link2); + bpf_link__destroy(link1); +} + +static void run_query_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_query(cgroup_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +/* Must match progs/bpf_tcp_ops.c */ +#define OPS_RETVAL1 11 +#define OPS_RETVAL2 22 + +/* + * Attach three struct_ops implementing timeout_init to the same cgroup; they + * run in attach order [retval1, retval2, retval3]. timeout_init's return value + * is chained: the first prog reads the kernel seed via bpf_get_retval() (0, + * since no legacy sockops prog is attached) and returns OPS_RETVAL1; each + * subsequent prog must then observe the previous prog's return value. This + * proves the trampoline inherits the retval across an array of struct_ops. + */ +static void test_retval(int cgroup_fd, struct bpf_tcp_ops *skel) +{ + struct bpf_link *link1 = NULL, *link2 = NULL, *link3 = NULL; + + skel->bss->retval_saw1 = -1; + skel->bss->retval_saw2 = -1; + skel->bss->retval_saw3 = -1; + + link1 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops_retval1, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link1, "attach_retval1")) + goto done; + + link2 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops_retval2, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link2, "attach_retval2")) + goto done; + + link3 = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops_retval3, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link3, "attach_retval3")) + goto done; + + do_listen_connect(AF_INET6); + + /* First prog inherits the kernel seed (no legacy sockops -> 0). */ + ASSERT_EQ(skel->bss->retval_saw1, 0, "retval_saw1"); + /* Each subsequent prog inherits the previous prog's return value. */ + ASSERT_EQ(skel->bss->retval_saw2, OPS_RETVAL1, "retval_saw2"); + ASSERT_EQ(skel->bss->retval_saw3, OPS_RETVAL2, "retval_saw3"); + +done: + bpf_link__destroy(link3); + bpf_link__destroy(link2); + bpf_link__destroy(link1); +} + +static void run_retval_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_retval(cgroup_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +/* + * bpf_link__update_map() swaps the struct_ops map backing an attached link. + * The link keeps its position, including BPF_F_PREORDER, across the update. + * Attach ops1 (normal) and ops2 (preorder): order [2, 1]. Update the normal + * link to ops3 -> [2, 3]; update the preorder link to ops1 -> [1, 3]. + */ +static void test_update(int cgroup_fd, struct bpf_tcp_ops *skel) +{ + LIBBPF_OPTS(bpf_cgroup_opts, preorder_opts, .flags = BPF_F_PREORDER); + struct bpf_link *link = NULL, *link_pre = NULL; + + link = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link, "attach_ops1")) + goto done; + + link_pre = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops2, cgroup_fd, + &preorder_opts); + if (!ASSERT_OK_PTR(link_pre, "attach_ops2_preorder")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + ASSERT_EQ(skel->bss->listen_cnt, 2, "cnt_initial"); + ASSERT_EQ(skel->bss->listen_order[0], 2, "order0_initial"); + ASSERT_EQ(skel->bss->listen_order[1], 1, "order1_initial"); + + /* Update the normal link's map (ops1 -> ops3); position is unchanged. */ + if (!ASSERT_OK(bpf_link__update_map(link, skel->maps.tcp_ops3), "update_normal")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + ASSERT_EQ(skel->bss->listen_order[0], 2, "order0_after_normal"); + ASSERT_EQ(skel->bss->listen_order[1], 3, "order1_after_normal"); + + /* Update the preorder link's map (ops2 -> ops1); it stays first. */ + if (!ASSERT_OK(bpf_link__update_map(link_pre, skel->maps.tcp_ops1), "update_preorder")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + ASSERT_EQ(skel->bss->listen_order[0], 1, "order0_after_preorder"); + ASSERT_EQ(skel->bss->listen_order[1], 3, "order1_after_preorder"); + +done: + bpf_link__destroy(link_pre); + bpf_link__destroy(link); +} + +static void run_update_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_update(cgroup_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +/* + * Two-level hierarchy. Attach ops1 to the parent and ops2 to the child, then + * trigger from a socket in the child. Descendant progs run before ancestor + * progs, so the order is [2 (child), 1 (parent)]. + */ +static void test_hierarchy(int parent_fd, int child_fd, struct bpf_tcp_ops *skel) +{ + struct bpf_link *plink = NULL, *clink = NULL; + + plink = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, parent_fd, NULL); + if (!ASSERT_OK_PTR(plink, "attach_parent")) + goto done; + + clink = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops2, child_fd, NULL); + if (!ASSERT_OK_PTR(clink, "attach_child")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + + ASSERT_EQ(skel->bss->listen_cnt, 2, "listen_cnt"); + ASSERT_EQ(skel->bss->listen_order[0], 2, "listen_order[0]"); + ASSERT_EQ(skel->bss->listen_order[1], 1, "listen_order[1]"); + +done: + bpf_link__destroy(clink); + bpf_link__destroy(plink); +} + +static void run_hierarchy_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int parent_fd, child_fd = -1; + + parent_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(parent_fd, 0, "join_parent_cgroup")) + return; + + child_fd = create_and_get_cgroup(CGROUP_PATH "/child"); + if (!ASSERT_GE(child_fd, 0, "create_child_cgroup")) + goto done; + + if (!ASSERT_OK(join_cgroup(CGROUP_PATH "/child"), "join_child_cgroup")) + goto done; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_hierarchy(parent_fd, child_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + if (child_fd >= 0) { + close(child_fd); + join_cgroup(CGROUP_PATH); + remove_cgroup(CGROUP_PATH "/child"); + } + close(parent_fd); +} + +/* + * Attach ops1 to the parent, then create and join the child cgroup. The child + * is created after the attach, so it must inherit the parent's effective progs + * via cgroup_bpf_inherit(). A socket in the child runs the parent's prog. + */ +static void test_inherit(int parent_fd, struct bpf_tcp_ops *skel) +{ + struct bpf_link *plink = NULL; + int child_fd = -1; + + plink = bpf_map__attach_cgroup_opts(skel->maps.tcp_ops1, parent_fd, NULL); + if (!ASSERT_OK_PTR(plink, "attach_parent")) + goto done; + + child_fd = create_and_get_cgroup(CGROUP_PATH "/child"); + if (!ASSERT_GE(child_fd, 0, "create_child_cgroup")) + goto done; + + if (!ASSERT_OK(join_cgroup(CGROUP_PATH "/child"), "join_child_cgroup")) + goto done; + + reset_order(skel); + do_listen_connect(AF_INET6); + + ASSERT_EQ(skel->bss->listen_cnt, 1, "listen_cnt"); + ASSERT_EQ(skel->bss->listen_order[0], 1, "listen_order[0]"); + +done: + if (child_fd >= 0) { + close(child_fd); + join_cgroup(CGROUP_PATH); + remove_cgroup(CGROUP_PATH "/child"); + } + bpf_link__destroy(plink); +} + +static void run_inherit_subtest(void) +{ + struct bpf_tcp_ops *skel = NULL; + struct netns_obj *ns = NULL; + int parent_fd; + + parent_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(parent_fd, 0, "join_parent_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + test_inherit(parent_fd, skel); + +done: + bpf_tcp_ops__destroy(skel); + netns_free(ns); + close(parent_fd); +} + +void test_bpf_tcp_ops(void) +{ + if (test__start_subtest("order")) + run_order_subtest(); + if (test__start_subtest("before_after")) + run_before_after_subtest(); + if (test__start_subtest("query")) + run_query_subtest(); + if (test__start_subtest("retval")) + run_retval_subtest(); + if (test__start_subtest("update")) + run_update_subtest(); + if (test__start_subtest("hierarchy")) + run_hierarchy_subtest(); + if (test__start_subtest("inherit")) + run_inherit_subtest(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops_hdr.c b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops_hdr.c new file mode 100644 index 000000000000..244605e69e40 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops_hdr.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <test_progs.h> +#include <network_helpers.h> +#include "cgroup_helpers.h" +#include "bpf_tcp_ops_hdr.skel.h" + +#define CGROUP_PATH "/bpf_tcp_ops_hdr" +#define TEST_NETNS "bpf_tcp_ops_hdr" + +#define TEST_OPT_D0 0xAB +#define TEST_OPT_D1 0xCD + +static void run_hdr_opt(void) +{ + struct bpf_tcp_ops_hdr *skel = NULL; + struct bpf_link *link = NULL; + struct netns_obj *ns = NULL; + int cgroup_fd, lfd = -1, fd = -1; + + cgroup_fd = test__join_cgroup(CGROUP_PATH); + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) + return; + + ns = netns_new(TEST_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto done; + + skel = bpf_tcp_ops_hdr__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto done; + + link = bpf_map__attach_cgroup_opts(skel->maps.test_hdr_ops, cgroup_fd, NULL); + if (!ASSERT_OK_PTR(link, "attach_cgroup")) + goto done; + + /* + * One direction of data is enough to exercise all hooks: both peers + * share the cgroup struct_ops, so the sender runs hdr_opt_len/write + * and the receiver runs parse. + */ + lfd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0); + if (!ASSERT_GE(lfd, 0, "start_server")) + goto done; + + fd = connect_to_fd(lfd, 0); + if (!ASSERT_OK_FD(fd, "connect_to_fd")) + goto done; + + if (!ASSERT_OK(send_recv_data(lfd, fd, 64), "send_recv_data")) + goto done; + + /* Reserve + write hooks ran while sending. */ + ASSERT_GT(skel->bss->hdr_opt_len_cnt, 0, "hdr_opt_len_cnt"); + ASSERT_GT(skel->bss->write_cnt, 0, "write_cnt"); + /* Parse hook ran and recovered our option on the receive side. */ + ASSERT_GT(skel->bss->parse_cnt, 0, "parse_cnt"); + ASSERT_GT(skel->bss->found_cnt, 0, "found_cnt"); + ASSERT_EQ(skel->bss->found_d0, TEST_OPT_D0, "found_d0"); + ASSERT_EQ(skel->bss->found_d1, TEST_OPT_D1, "found_d1"); + +done: + if (fd >= 0) + close(fd); + if (lfd >= 0) + close(lfd); + bpf_link__destroy(link); + bpf_tcp_ops_hdr__destroy(skel); + netns_free(ns); + close(cgroup_fd); +} + +void test_bpf_tcp_ops_hdr(void) +{ + run_hdr_opt(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c index 73f669014b69..902a70d5afeb 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c @@ -35,7 +35,7 @@ static int check_load(const char *file, enum bpf_prog_type type) } bpf_program__set_type(prog, type); - bpf_program__set_flags(prog, testing_prog_flags()); + bpf_program__add_flags(prog, testing_prog_flags()); bpf_program__set_log_level(prog, 4 | extra_prog_load_log_flags); err = bpf_object__load(obj); diff --git a/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c b/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c new file mode 100644 index 000000000000..534aa79a4278 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <bpftool_helpers.h> +#include <bpf/btf.h> +#include <limits.h> +#include <unistd.h> +#include "btf_helpers.h" +#include "testing_helpers.h" + +#define DUMP_BUF_SZ (16 * 1024) + +#define EXPECTED_SORTED "bpftool_btf_dump_sorted.expected" +#define EXPECTED_UNSORTED "bpftool_btf_dump_unsorted.expected" + +/* + * struct holey { + * int c; + * <64-bit hole> + * int tail; + * }; + * enum { E0 = 1 }; + * struct s { int f; }; + */ +static struct btf *mk_btf(void) +{ + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "new_empty")) + return NULL; + + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + btf__add_int(btf, "long int", 4, BTF_INT_SIGNED); + + btf__add_struct(btf, "holey", 16); + btf__add_field(btf, "c", 1, 0, 0); + btf__add_field(btf, "tail", 1, 96, 0); + + btf__add_enum(btf, NULL, 4); + btf__add_enum_value(btf, "E0", 1); + + btf__add_struct(btf, "s", 4); + btf__add_field(btf, "f", 1, 0, 0); + + VALIDATE_RAW_BTF( + btf, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] INT 'long int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[3] STRUCT 'holey' size=16 vlen=2\n" + "\t'c' type_id=1 bits_offset=0\n" + "\t'tail' type_id=1 bits_offset=96", + "[4] ENUM '(anon)' encoding=UNSIGNED size=4 vlen=1\n" + "\t'E0' val=1", + "[5] STRUCT 's' size=4 vlen=1\n" + "\t'f' type_id=1 bits_offset=0"); + + return btf; +} + +static struct btf *mk_loc_btf(void) +{ + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "new_empty")) + return NULL; + + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + btf__add_func_proto(btf, 1); + btf__add_func_param(btf, "arg1", 1); + btf__add_func_param(btf, "arg2", 1); + btf__add_func_param(btf, "arg3", 1); + btf__add_func_param(btf, "arg4", 1); + btf__add_func_param(btf, "arg5", 1); + btf__add_func_param(btf, "arg6", 1); + btf__add_func(btf, "foo", BTF_FUNC_STATIC, 2); + + btf__add_loc_param(btf, 4, BTF_LOC_PARAM_REG); + btf__add_loc_param_value(btf, 1); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_REG | + BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET); + btf__add_loc_param_value(btf, 2); + btf__add_loc_param_value(btf, 0x10); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_REG | + BTF_LOC_PARAM_OFFSET | BTF_LOC_PARAM_SIGNED); + btf__add_loc_param_value(btf, BTF_LOC_PARAM_FBREG); + btf__add_loc_param_value(btf, -0x10); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_ADDR | BTF_LOC_PARAM_CONST); + btf__add_loc_param_value(btf, 0x9abcdef0); + btf__add_loc_param_value(btf, 0x12345678); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_CONST); + btf__add_loc_param_value(btf, 0xfeedface); + btf__add_loc_param_value(btf, 0xdeadbeef); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_REG | + BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET | + BTF_LOC_PARAM_SIGNED); + btf__add_loc_param_value(btf, BTF_LOC_PARAM_FBREG); + btf__add_loc_param_value(btf, -0x20); + btf__add_loc_proto(btf); + btf__add_loc_proto_param(btf, 4); + btf__add_loc_proto_param(btf, 5); + btf__add_loc_proto_param(btf, 6); + btf__add_loc_proto_param(btf, 7); + btf__add_loc_proto_param(btf, 8); + btf__add_loc_proto_param(btf, 9); + btf__add_locsec(btf, "inline.text"); + btf__add_locsec_loc(btf, 3, 10, 64); + + return btf; +} + +static int btf_to_tmpfile(const struct btf *btf, char *path) +{ + ssize_t written; + const void *raw; + __u32 sz; + int fd; + + raw = btf__raw_data(btf, &sz); + if (!ASSERT_OK_PTR(raw, "raw_data")) + return -1; + + snprintf(path, PATH_MAX, "/tmp/bpftool_btf_dump.XXXXXX"); + fd = mkstemp(path); + if (!ASSERT_OK_FD(fd, "mkstemp_btf")) + return -1; + + written = write(fd, raw, sz); + close(fd); + if (!ASSERT_EQ(written, sz, "write_btf")) { + unlink(path); + return -1; + } + + return 0; +} + +static char *dump_c(const char *btf_path, bool sorted) +{ + char args[MAX_BPFTOOL_CMD_LEN]; + char *buf; + int err; + + buf = malloc(DUMP_BUF_SZ); + if (!ASSERT_OK_PTR(buf, "alloc_dump")) + return NULL; + + snprintf(args, sizeof(args), "btf dump file %s format c%s", + btf_path, sorted ? "" : " unsorted"); + + err = get_bpftool_command_output(args, buf, DUMP_BUF_SZ); + if (!ASSERT_OK(err, "btf_dump_format_c")) { + free(buf); + return NULL; + } + + return buf; +} + +static char *dump_raw(const char *btf_path) +{ + char args[MAX_BPFTOOL_CMD_LEN]; + char *buf; + int err; + + buf = malloc(DUMP_BUF_SZ); + if (!ASSERT_OK_PTR(buf, "alloc_dump")) + return NULL; + + snprintf(args, sizeof(args), "btf dump file %s", btf_path); + err = get_bpftool_command_output(args, buf, DUMP_BUF_SZ); + if (!ASSERT_OK(err, "btf_dump_raw")) { + free(buf); + return NULL; + } + + return buf; +} + +static char *read_expected(const char *path) +{ + char *buf = NULL; + size_t cap = 0; + FILE *f; + int err; + + f = fopen(path, "r"); + if (!f) { + err = errno; + PRINT_FAIL("can't open expected output '%s': errno %d\n", path, err); + return NULL; + } + + /* no NUL in a generated header, so this reads to the end */ + if (getdelim(&buf, &cap, '\0', f) < 0) { + err = errno; + PRINT_FAIL("can't read expected output '%s': errno %d\n", path, err); + free(buf); + buf = NULL; + } + + fclose(f); + return buf; +} + +static void test_dump(const char *btf_path, bool sorted) +{ + const char *exp_path; + char *dump, *exp; + + exp_path = sorted ? EXPECTED_SORTED : EXPECTED_UNSORTED; + + dump = dump_c(btf_path, sorted); + if (!dump) + return; + + exp = read_expected(exp_path); + if (!exp) + goto out_dump; + + ASSERT_TEXT_EQ(dump, exp, sorted ? "cmp_sorted" : "cmp_unsorted"); + + free(exp); +out_dump: + free(dump); +} + +static void test_loc_dump(const char *btf_path) +{ + const char expected[] = + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED\n" + "[2] FUNC_PROTO '(anon)' ret_type_id=1 vlen=6\n" + "\t'arg1' type_id=1\n" + "\t'arg2' type_id=1\n" + "\t'arg3' type_id=1\n" + "\t'arg4' type_id=1\n" + "\t'arg5' type_id=1\n" + "\t'arg6' type_id=1\n" + "[3] FUNC 'foo' type_id=2 linkage=static\n" + "[4] LOC_PARAM '(anon)' size=4 flags=0x8 vlen=1 values='reg1'\n" + "[5] LOC_PARAM '(anon)' size=8 flags=0x38 vlen=2 values='*(reg2 + 0x10)'\n" + "[6] LOC_PARAM '(anon)' size=8 flags=0x29 vlen=2 values='fbreg - 0x10'\n" + "[7] LOC_PARAM '(anon)' size=8 flags=0x6 vlen=2 values='address 0x123456789abcdef0'\n" + "[8] LOC_PARAM '(anon)' size=8 flags=0x2 vlen=2 values='const 0xdeadbeeffeedface'\n" + "[9] LOC_PARAM '(anon)' size=8 flags=0x39 vlen=2 values='*(fbreg - 0x20)'\n" + "[10] LOC_PROTO '(anon)' vlen=6\n" + "\ttype_id=4 value='reg1'\n" + "\ttype_id=5 value='*(reg2 + 0x10)'\n" + "\ttype_id=6 value='fbreg - 0x10'\n" + "\ttype_id=7 value='address 0x123456789abcdef0'\n" + "\ttype_id=8 value='const 0xdeadbeeffeedface'\n" + "\ttype_id=9 value='*(fbreg - 0x20)'\n" + "[11] LOCSEC 'inline.text' vlen=1\n" + "\tfunc='foo(arg1 [reg1], arg2 [*(reg2 + 0x10)], arg3 [fbreg - 0x10], arg4 [address 0x123456789abcdef0], arg5 [const 0xdeadbeeffeedface], arg6 [*(fbreg - 0x20)])' func_type_id=3 loc_proto_type_id=10 offset=64\n"; + char *dump; + + dump = dump_raw(btf_path); + if (!dump) + return; + + ASSERT_TEXT_EQ(dump, expected, "cmp_loc_dump"); + free(dump); +} + +void test_bpftool_btf_dump(void) +{ + char path[PATH_MAX]; + struct btf *btf = NULL; + + btf = mk_btf(); + if (!btf) + return; + + if (btf_to_tmpfile(btf, path)) + goto out_btf; + + if (test__start_subtest("c_sorted")) + test_dump(path, true); + if (test__start_subtest("c_unsorted")) + test_dump(path, false); + unlink(path); + + btf__free(btf); + + btf = mk_loc_btf(); + if (!btf) + return; + + if (btf_to_tmpfile(btf, path)) + goto out_btf; + + if (test__start_subtest("loc_dump")) + test_loc_dump(path); + unlink(path); +out_btf: + btf__free(btf); +} diff --git a/tools/testing/selftests/bpf/prog_tests/bpftool_map_dump.c b/tools/testing/selftests/bpf/prog_tests/bpftool_map_dump.c new file mode 100644 index 000000000000..187b3f784307 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpftool_map_dump.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <sys/socket.h> +#include <unistd.h> +#include <bpf/btf.h> +#include <test_progs.h> +#include "bpftool_helpers.h" + +#define PIN_PATH "/sys/fs/bpf/test_bpftool_map_dump_sk_storage" +#define QUEUE_PIN_PATH "/sys/fs/bpf/test_bpftool_map_dump_queue" +#define JSON_ERROR "{\"error\":\"can't dump sk_storage map" + +/* An empty socket storage map with an int value. */ +static int create_sk_storage_map(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, + .map_flags = BPF_F_NO_PREALLOC); + struct btf *btf; + int int_id, fd = -1; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf__new_empty")) + return -1; + + int_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_GT(int_id, 0, "btf__add_int")) + goto out; + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf__load_into_kernel")) + goto out; + + opts.btf_fd = btf__fd(btf); + opts.btf_key_type_id = int_id; + opts.btf_value_type_id = int_id; + fd = bpf_map_create(BPF_MAP_TYPE_SK_STORAGE, "sk_storage", sizeof(int), + sizeof(int), 0, &opts); + ASSERT_OK_FD(fd, "bpf_map_create"); +out: + btf__free(btf); + return fd; +} + +static void test_sk_storage(void) +{ + int map_fd, sk_fd = -1, value = 42, ret; + char output[1024] = {}; + + map_fd = create_sk_storage_map(); + if (map_fd < 0) + return; + + sk_fd = socket(AF_INET, SOCK_STREAM, 0); + if (!ASSERT_OK_FD(sk_fd, "socket")) + goto out; + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &sk_fd, &value, BPF_NOEXIST), + "add socket storage")) + goto out; + if (!ASSERT_OK(bpf_obj_pin(map_fd, PIN_PATH), "pin map")) + goto out; + + /* + * Socket storage can't be iterated, so the dump must fail with an + * error and nothing else, rather than print an empty map that holds + * an element. The buffer is cleared before each run so that a run + * that fails to start can't match stale or uninitialised output. + */ + ret = get_bpftool_command_output("-j map dump pinned " PIN_PATH, + output, sizeof(output)); + ASSERT_NEQ(ret, 0, "json dump fails"); + ASSERT_STRNEQ(output, JSON_ERROR, strlen(JSON_ERROR), + "json dump is only the error"); + + memset(output, 0, sizeof(output)); + ret = get_bpftool_command_output("map dump pinned " PIN_PATH, + output, sizeof(output)); + ASSERT_NEQ(ret, 0, "plain dump fails"); + ASSERT_NULL(strchr(output, '['), "plain dump prints no elements"); + + memset(output, 0, sizeof(output)); + get_bpftool_command_output("map dump pinned " PIN_PATH " 2>&1 >/dev/null", + output, sizeof(output)); + ASSERT_HAS_SUBSTR(output, + "can't dump sk_storage map: Operation not supported", + "plain dump explains why on stderr"); + +out: + unlink(PIN_PATH); + if (sk_fd >= 0) + close(sk_fd); + close(map_fd); +} + +/* + * A queue map has no BTF, so its plain dump prints an element count + * rather than JSON. A dump that ignored the error would print "Found 0 + * elements" for a queue that holds one. + */ +static void test_queue(void) +{ + char output[1024] = {}; + int map_fd, value = 42, ret; + + map_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, "queue", 0, sizeof(int), 1, + NULL); + if (!ASSERT_OK_FD(map_fd, "bpf_map_create")) + return; + if (!ASSERT_OK(bpf_map_update_elem(map_fd, NULL, &value, BPF_ANY), + "push to queue")) + goto out; + if (!ASSERT_OK(bpf_obj_pin(map_fd, QUEUE_PIN_PATH), "pin map")) + goto out; + + ret = get_bpftool_command_output("map dump pinned " QUEUE_PIN_PATH, + output, sizeof(output)); + ASSERT_NEQ(ret, 0, "plain dump fails"); + ASSERT_NULL(strstr(output, "Found"), "plain dump prints no count"); + + memset(output, 0, sizeof(output)); + get_bpftool_command_output("map dump pinned " QUEUE_PIN_PATH " 2>&1 >/dev/null", + output, sizeof(output)); + ASSERT_HAS_SUBSTR(output, "can't dump queue map", + "plain dump explains why on stderr"); + +out: + unlink(QUEUE_PIN_PATH); + close(map_fd); +} + +void test_bpftool_map_dump(void) +{ + if (test__start_subtest("sk_storage")) + test_sk_storage(); + if (test__start_subtest("queue")) + test_queue(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/bpftool_ringbuf.c b/tools/testing/selftests/bpf/prog_tests/bpftool_ringbuf.c new file mode 100644 index 000000000000..e9724c55d1dc --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpftool_ringbuf.c @@ -0,0 +1,417 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <ctype.h> +#include <fcntl.h> +#include <poll.h> +#include <signal.h> +#include <sys/mman.h> +#include <sys/wait.h> +#include <test_progs.h> +#include "bpftool_helpers.h" +#include "bpftool_ringbuf.skel.h" + +#define WAIT_STEPS 500 +#define WAIT_US 10000 + +struct consumer { + pid_t pid; + int fd; +}; + +static void consumer_cleanup(struct consumer *child) +{ + if (child->pid > 0) { + kill(child->pid, SIGKILL); + while (waitpid(child->pid, NULL, 0) < 0 && errno == EINTR) + ; + child->pid = -1; + } + if (child->fd >= 0) { + close(child->fd); + child->fd = -1; + } +} + +static bool consumer_start(struct consumer *child, int map_fd, const char *format, + const char *pin_path, const char *option, + bool pair, bool capture_errors) +{ + struct bpf_map_info info = {}; + __u32 len = sizeof(info); + char path[PATH_MAX]; + char *argv[12], id[16]; + int out[2], ready[2], n = 0, err, err_fd; + struct pollfd pfd; + + if (!ASSERT_OK(detect_bpftool_path(path, sizeof(path)), "bpftool path") || + !ASSERT_OK(bpf_map_get_info_by_fd(map_fd, &info, &len), "map info")) + return false; + snprintf(id, sizeof(id), "%u", info.id); + argv[n++] = path; + if (format) + argv[n++] = (char *)format; + argv[n++] = "map"; + argv[n++] = "event_pipe"; + argv[n++] = pin_path ? "pinned" : "id"; + argv[n++] = pin_path ? (char *)pin_path : id; + if (option) { + argv[n++] = (char *)option; + argv[n++] = "0"; + if (pair) { + argv[n++] = "index"; + argv[n++] = "0"; + } + } + argv[n] = NULL; + if (!ASSERT_OK(pipe2(out, O_CLOEXEC), "output pipe")) + return false; + if (!ASSERT_OK(pipe2(ready, O_CLOEXEC), "exec pipe")) { + close(out[0]); + close(out[1]); + return false; + } + child->pid = fork(); + if (!child->pid) { + close(out[0]); + close(ready[0]); + err_fd = capture_errors ? out[1] : open("/dev/null", O_WRONLY); + if (dup2(out[1], STDOUT_FILENO) < 0 || + dup2(err_fd, STDERR_FILENO) < 0) + goto exec_fail; + if (!capture_errors) + close(err_fd); + close(out[1]); + execv(path, argv); +exec_fail: + err = errno; + if (write(ready[1], &err, sizeof(err)) != sizeof(err)) + _exit(126); + _exit(127); + } + close(out[1]); + close(ready[1]); + child->fd = out[0]; + pfd = (struct pollfd) { .fd = ready[0], .events = POLLIN }; + /* EOF on the close-on-exec pipe distinguishes exec from inherited handlers. */ + err = child->pid > 0 ? poll(&pfd, 1, WAIT_STEPS * WAIT_US / 1000) : -1; + if (!ASSERT_GT(child->pid, 0, "fork") || + !ASSERT_GT(err, 0, "exec timeout") || + !ASSERT_EQ(read(ready[0], &err, sizeof(err)), 0, "exec")) { + close(ready[0]); + consumer_cleanup(child); + return false; + } + close(ready[0]); + return true; +} + +static bool consumer_finish(struct consumer *child, int signo, bool success, + char *output, size_t size) +{ + int status = 0, i; + pid_t ret = 0; + ssize_t n; + size_t used = 0; + + if (!ASSERT_OK(fcntl(child->fd, F_SETFL, O_NONBLOCK), "nonblocking output")) + return false; + if (signo && !ASSERT_OK(kill(child->pid, signo), "signal consumer")) + return false; + for (i = 0; i < WAIT_STEPS; i++) { + while (used < size - 1 && + (n = read(child->fd, output + used, size - 1 - used)) > 0) + used += n; + ret = waitpid(child->pid, &status, WNOHANG); + if (ret == child->pid) + break; + if (ret < 0 && errno != EINTR) + break; + usleep(WAIT_US); + } + if (!ASSERT_EQ(ret, child->pid, "bounded consumer exit")) + return false; + child->pid = -1; + while (used < size - 1 && (n = read(child->fd, output + used, size - 1 - used)) > 0) + used += n; + output[used] = '\0'; + return ASSERT_TRUE(WIFEXITED(status), "normal exit") && + ASSERT_EQ(WEXITSTATUS(status) == 0, success, "exit status"); +} + +static bool consumer_ready(struct consumer *child) +{ + unsigned long long caught; + char path[64], line[256]; + int i; + FILE *f; + + snprintf(path, sizeof(path), "/proc/%d/status", child->pid); + for (i = 0; i < WAIT_STEPS; i++) { + f = fopen(path, "r"); + if (!f) + break; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "SigCgt: %llx", &caught) == 1 && + (caught & (1ULL << (SIGINT - 1))) && + (caught & (1ULL << (SIGTERM - 1)))) { + fclose(f); + return true; + } + } + fclose(f); + usleep(WAIT_US); + } + return ASSERT_TRUE(false, "consumer signal handlers ready"); +} + +static bool emit_record(struct bpftool_ringbuf *skel, int record) +{ + char packet[64] = {}; + + LIBBPF_OPTS(bpf_test_run_opts, opts, + .data_in = packet, + .data_size_in = sizeof(packet), + ); + + skel->bss->record = record; + return ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.produce), + &opts), "produce record") && + ASSERT_OK(skel->bss->output_err, "ringbuf output"); +} + +static bool consumed(unsigned long *position, unsigned long expected) +{ + int i; + + for (i = 0; i < WAIT_STEPS; i++) { + if (__atomic_load_n(position, __ATOMIC_ACQUIRE) == expected) + return true; + usleep(WAIT_US); + } + return ASSERT_EQ(*position, expected, "consumer position"); +} + +static void check_json(char *output, const char *expected) +{ + char *src = output, *dst = output; + bool quoted = false, escaped = false; + + /* Ignore formatting whitespace while checking the entire JSON document. */ + while (*src) { + if (quoted || !isspace((unsigned char)*src)) + *dst++ = *src; + if (!escaped && *src == '"') + quoted = !quoted; + escaped = !escaped && quoted && *src == '\\'; + src++; + } + *dst = '\0'; + ASSERT_STREQ(output, expected, "JSON records"); +} + +static void test_consumer(const char *format, bool idle, int signo, bool pinned) +{ + struct consumer child = { .pid = -1, .fd = -1 }; + struct bpftool_ringbuf *skel; + unsigned long *position = MAP_FAILED; + int page_size = getpagesize(), fd; + char pin_dir[] = "/sys/fs/bpf/bpftool_ringbuf_XXXXXX"; + char pin_path[sizeof(pin_dir) + sizeof("/map")]; + bool dir_created = false, map_pinned = false; + char output[4096]; + struct pollfd pfd; + + skel = bpftool_ringbuf__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + bpf_map__set_max_entries(skel->maps.ringbuf, page_size); + if (!ASSERT_OK(bpftool_ringbuf__load(skel), "load")) + goto out; + fd = bpf_map__fd(skel->maps.ringbuf); + if (pinned) { + if (!ASSERT_OK_PTR(mkdtemp(pin_dir), "create pin directory")) + goto out; + dir_created = true; + snprintf(pin_path, sizeof(pin_path), "%s/map", pin_dir); + if (!ASSERT_OK(bpf_obj_pin(fd, pin_path), "pin ringbuf")) + goto out; + map_pinned = true; + } + position = mmap(NULL, page_size, PROT_READ, MAP_SHARED, fd, 0); + if (!ASSERT_NEQ(position, MAP_FAILED, "consumer mmap")) + goto out; + if (!idle && (!emit_record(skel, 0) || !emit_record(skel, 1))) + goto out; + if (!consumer_start(&child, fd, format, pinned ? pin_path : NULL, + NULL, false, false) || + !consumer_ready(&child)) + goto out; + if (!idle) { + /* Both prefilled records occupy 16 bytes including their headers. */ + if (!consumed(position, 32)) + goto out; + pfd = (struct pollfd) { .fd = child.fd, .events = POLLIN }; + if (!ASSERT_GT(poll(&pfd, 1, WAIT_STEPS * WAIT_US / 1000), 0, + "records flushed before exit") || + !ASSERT_TRUE(pfd.revents & POLLIN, "record output readable") || + !emit_record(skel, 2) || !consumed(position, 64)) + goto out; + } + if (!consumer_finish(&child, signo, true, output, sizeof(output))) + goto out; + if (format && !strcmp(format, "-p")) + ASSERT_STREQ(output, idle ? "[]\n" : + "[{\n" + " \"size\": 2,\n" + " \"data\": [0,255\n" + " ]\n" + " },{\n" + " \"size\": 5,\n" + " \"data\": [1,2,3,4,5\n" + " ]\n" + " },{\n" + " \"size\": 17,\n" + " \"data\": [16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32\n" + " ]\n" + " }\n" + "]\n", "pretty JSON records"); + else if (format) + check_json(output, idle ? "[]" : + "[{\"size\":2,\"data\":[0,255]}," + "{\"size\":5,\"data\":[1,2,3,4,5]}," + "{\"size\":17,\"data\":[16,17,18,19,20,21,22,23," + "24,25,26,27,28,29,30,31,32]}]"); + else + ASSERT_STREQ(output, idle ? "" : + "== size: 2 =====\n00 ff\n" + "== size: 5 =====\n01 02 03 04 05\n" + "== size: 17 =====\n10 11 12 13 14 15 16 17 " + "18 19 1a 1b 1c 1d 1e 1f\n20\n", "plain records"); +out: + consumer_cleanup(&child); + if (position != MAP_FAILED) + munmap(position, page_size); + if (map_pinned) + ASSERT_OK(unlink(pin_path), "unpin ringbuf"); + if (dir_created) + ASSERT_OK(rmdir(pin_dir), "remove pin directory"); + bpftool_ringbuf__destroy(skel); +} + +static void test_perf_consumer(void) +{ + struct consumer child = { .pid = -1, .fd = -1 }; + unsigned long long seconds, nanoseconds; + struct bpftool_ringbuf *skel; + char output[16384], expected[16384]; + int nr_cpus, cpu, index, offset = 0, i, used = 0, fields; + struct pollfd pfd; + + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "possible cpus")) + return; + skel = bpftool_ringbuf__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + bpf_map__set_max_entries(skel->maps.ringbuf, getpagesize()); + bpf_map__set_max_entries(skel->maps.perfbuf, nr_cpus); + if (!ASSERT_OK(bpftool_ringbuf__load(skel), "load") || + !consumer_start(&child, bpf_map__fd(skel->maps.perfbuf), NULL, + NULL, NULL, false, false) || !consumer_ready(&child) || + !emit_record(skel, 3)) + goto out; + /* One large record flushes the existing buffered perf output callback. */ + pfd = (struct pollfd) { .fd = child.fd, .events = POLLIN }; + if (!ASSERT_GT(poll(&pfd, 1, WAIT_STEPS * WAIT_US / 1000), 0, + "perf record output") || + !ASSERT_TRUE(pfd.revents & POLLIN, "perf output readable") || + !consumer_finish(&child, SIGINT, true, output, sizeof(output))) + goto out; + fields = sscanf(output, "== @%llu.%llu CPU: %d index: %d =====\n%n", + &seconds, &nanoseconds, &cpu, &index, &offset); + if (!ASSERT_EQ(fields, 4, "perf header") || + !ASSERT_GT(offset, 0, "perf payload offset")) + goto out; + ASSERT_GT(seconds * 1000000000ULL + nanoseconds, 0, "perf timestamp"); + ASSERT_LT(nanoseconds, 1000000000ULL, "perf timestamp nanoseconds"); + ASSERT_GE(cpu, 0, "perf cpu"); + ASSERT_LT(cpu, nr_cpus, "perf cpu range"); + ASSERT_EQ(index, cpu, "perf index"); + for (i = 0; i < sizeof(skel->rodata->perf_payload); i++) { + const char *separator = !i ? "" : !(i % 16) ? "\n" : + !(i % 8) ? " " : " "; + + used += snprintf(expected + used, sizeof(expected) - used, + "%s%02x", separator, i == 1 ? 0xff : 0); + } + snprintf(expected + used, sizeof(expected) - used, "\n"); + ASSERT_STREQ(output + offset, expected, "perf payload"); +out: + consumer_cleanup(&child); + bpftool_ringbuf__destroy(skel); +} + +static void test_reject(enum bpf_map_type type, const char *option, bool pair, bool json) +{ + struct consumer child = { .pid = -1, .fd = -1 }; + bool ring = type == BPF_MAP_TYPE_RINGBUF || type == BPF_MAP_TYPE_USER_RINGBUF; + const char *expected = option && !strcmp(option, "foobar") ? + "{\"error\":\"what is 'foobar'?\"}" : option ? + "{\"error\":\"ring buffer maps do not support cpu or index arguments\"}" : + "{\"error\":\"map is not a perf event array or ring buffer\"}"; + char output[4096]; + int fd; + + fd = bpf_map_create(type, NULL, ring ? 0 : 4, ring ? 0 : 4, + ring ? getpagesize() : 1, NULL); + if (!ASSERT_GE(fd, 0, "create map")) + return; + if (consumer_start(&child, fd, json ? "-j" : NULL, NULL, option, pair, true) && + consumer_finish(&child, 0, false, output, sizeof(output))) { + if (json) + check_json(output, expected); + else + ASSERT_GT(strlen(output), 0, "error diagnostic"); + } + consumer_cleanup(&child); + close(fd); +} + +void test_bpftool_ringbuf(void) +{ + if (test__start_subtest("perf_event_array")) + test_perf_consumer(); + if (test__start_subtest("plain")) + test_consumer(NULL, false, SIGINT, false); + if (test__start_subtest("json")) + test_consumer("-j", false, SIGTERM, false); + if (test__start_subtest("pretty_json")) + test_consumer("-p", false, SIGINT, false); + if (test__start_subtest("pinned_plain")) + test_consumer(NULL, false, SIGTERM, true); + if (test__start_subtest("pinned_json")) + test_consumer("-j", false, SIGINT, true); + if (test__start_subtest("pinned_pretty_json")) + test_consumer("-p", false, SIGTERM, true); + if (test__start_subtest("idle_sigint")) + test_consumer(NULL, true, SIGINT, false); + if (test__start_subtest("idle_sigterm_json")) + test_consumer("-j", true, SIGTERM, false); + if (test__start_subtest("idle_sigint_pretty_json")) + test_consumer("-p", true, SIGINT, false); + if (test__start_subtest("reject_array")) + test_reject(BPF_MAP_TYPE_ARRAY, NULL, false, false); + if (test__start_subtest("reject_user_ringbuf")) + test_reject(BPF_MAP_TYPE_USER_RINGBUF, NULL, false, false); + if (test__start_subtest("reject_cpu")) + test_reject(BPF_MAP_TYPE_RINGBUF, "cpu", false, false); + if (test__start_subtest("reject_index")) + test_reject(BPF_MAP_TYPE_RINGBUF, "index", false, false); + if (test__start_subtest("reject_cpu_index")) + test_reject(BPF_MAP_TYPE_RINGBUF, "cpu", true, false); + if (test__start_subtest("reject_user_ringbuf_json")) + test_reject(BPF_MAP_TYPE_USER_RINGBUF, NULL, false, true); + if (test__start_subtest("reject_unknown_json")) + test_reject(BPF_MAP_TYPE_RINGBUF, "foobar", false, true); + if (test__start_subtest("reject_cpu_index_json")) + test_reject(BPF_MAP_TYPE_RINGBUF, "cpu", true, true); +} diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 67b9015cbd98..24ab62b2834a 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -424,7 +424,7 @@ static struct btf_raw_test raw_tests[] = { .err_str = "Invalid type", }, { - .descr = "global data test #8, invalid var size", + .descr = "global data test #8, var is smaller than its type", .raw_types = { /* int */ BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ @@ -457,8 +457,6 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 0, .value_type_id = 7, .max_entries = 1, - .btf_load_err = true, - .err_str = "Invalid size", }, { .descr = "global data test #9, invalid var size", @@ -498,7 +496,7 @@ static struct btf_raw_test raw_tests[] = { .err_str = "Invalid size", }, { - .descr = "global data test #10, invalid var size", + .descr = "global data test #10, section is smaller than map value", .raw_types = { /* int */ BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ @@ -531,8 +529,7 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 0, .value_type_id = 7, .max_entries = 1, - .btf_load_err = true, - .err_str = "Invalid size", + .map_create_err = true, }, { .descr = "global data test #11, multiple section members", @@ -1987,14 +1984,14 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "typedef (invalid name, invalid identifier)", + .descr = "typedef (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPEDEF_ENC(NAME_TBD, 1), /* [2] */ BTF_END_RAW, }, - .str_sec = "\0__!int", - .str_sec_size = sizeof("\0__!int"), + .str_sec = "\0__\7int", + .str_sec_size = sizeof("\0__\7int"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "typedef_check_btf", .key_size = sizeof(int), @@ -2112,15 +2109,15 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "fwd type (invalid name, invalid identifier)", + .descr = "fwd type (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_FWD, 0, 0), 0), /* [2] */ BTF_END_RAW, }, - .str_sec = "\0__!skb", - .str_sec_size = sizeof("\0__!skb"), + .str_sec = "\0__\7skb", + .str_sec_size = sizeof("\0__\7skb"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "fwd_type_check_btf", .key_size = sizeof(int), @@ -2175,7 +2172,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "struct type (invalid name, invalid identifier)", + .descr = "struct type (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(NAME_TBD, @@ -2183,8 +2180,8 @@ static struct btf_raw_test raw_tests[] = { BTF_MEMBER_ENC(NAME_TBD, 1, 0), BTF_END_RAW, }, - .str_sec = "\0A!\0B", - .str_sec_size = sizeof("\0A!\0B"), + .str_sec = "\0A\7\0B", + .str_sec_size = sizeof("\0A\7\0B"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "struct_type_check_btf", .key_size = sizeof(int), @@ -2217,7 +2214,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "struct member (invalid name, invalid identifier)", + .descr = "struct member (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(NAME_TBD, @@ -2225,8 +2222,8 @@ static struct btf_raw_test raw_tests[] = { BTF_MEMBER_ENC(NAME_TBD, 1, 0), BTF_END_RAW, }, - .str_sec = "\0A\0B*", - .str_sec_size = sizeof("\0A\0B*"), + .str_sec = "\0A\0B\7", + .str_sec_size = sizeof("\0A\0B\7"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "struct_type_check_btf", .key_size = sizeof(int), @@ -2260,7 +2257,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "enum type (invalid name, invalid identifier)", + .descr = "enum type (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(NAME_TBD, @@ -2269,8 +2266,8 @@ static struct btf_raw_test raw_tests[] = { BTF_ENUM_ENC(NAME_TBD, 0), BTF_END_RAW, }, - .str_sec = "\0A!\0B", - .str_sec_size = sizeof("\0A!\0B"), + .str_sec = "\0A\7\0B", + .str_sec_size = sizeof("\0A\7\0B"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "enum_type_check_btf", .key_size = sizeof(int), @@ -2306,7 +2303,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "enum member (invalid name, invalid identifier)", + .descr = "enum member (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(0, @@ -2315,8 +2312,8 @@ static struct btf_raw_test raw_tests[] = { BTF_ENUM_ENC(NAME_TBD, 0), BTF_END_RAW, }, - .str_sec = "\0A!", - .str_sec_size = sizeof("\0A!"), + .str_sec = "\0A\7", + .str_sec_size = sizeof("\0A\7"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "enum_type_check_btf", .key_size = sizeof(int), @@ -2625,14 +2622,14 @@ static struct btf_raw_test raw_tests[] = { .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ - /* void (*)(int a, unsigned int !!!) */ + /* void (*)(int a, unsigned int \7) */ BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), BTF_END_RAW, }, - .str_sec = "\0a\0!!!", - .str_sec_size = sizeof("\0a\0!!!"), + .str_sec = "\0a\0\7", + .str_sec_size = sizeof("\0a\0\7"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "func_proto_type_check_btf", .key_size = sizeof(int), @@ -2775,12 +2772,12 @@ static struct btf_raw_test raw_tests[] = { BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), - /* void !!!(int a, unsigned int b) */ + /* void \7(int a, unsigned int b) */ BTF_FUNC_ENC(NAME_TBD, 3), /* [4] */ BTF_END_RAW, }, - .str_sec = "\0a\0b\0!!!", - .str_sec_size = sizeof("\0a\0b\0!!!"), + .str_sec = "\0a\0b\0\7", + .str_sec_size = sizeof("\0a\0b\0\7"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "func_type_check_btf", .key_size = sizeof(int), @@ -2793,7 +2790,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "func (Some arg has no name)", + .descr = "func (Some arg of global func has no name)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ @@ -2802,7 +2799,8 @@ static struct btf_raw_test raw_tests[] = { BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), BTF_FUNC_PROTO_ARG_ENC(0, 2), /* void func(int a, unsigned int) */ - BTF_FUNC_ENC(NAME_TBD, 3), /* [4] */ + BTF_TYPE_ENC(NAME_TBD, /* [4] */ + BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 3), BTF_END_RAW, }, .str_sec = "\0a\0func", @@ -2819,6 +2817,57 @@ static struct btf_raw_test raw_tests[] = { }, { + .descr = "func (Some arg of static func has no name)", + .raw_types = { + /* int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* unsigned int */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void (*)(int a, unsigned int) */ + BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(0, 2), + /* static void func(int a, unsigned int) */ + BTF_FUNC_ENC(NAME_TBD, 3), /* [4] */ + BTF_END_RAW, + }, + .str_sec = "\0a\0func", + .str_sec_size = sizeof("\0a\0func"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, +}, + +{ + .descr = "func (vararg of global func has no name)", + .raw_types = { + /* int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* void (*)(int a, ...) */ + BTF_FUNC_PROTO_ENC(0, 2), /* [2] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(0, 0), + /* void func(int a, ...) */ + BTF_TYPE_ENC(NAME_TBD, /* [3] */ + BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), + BTF_END_RAW, + }, + .str_sec = "\0a\0func", + .str_sec_size = sizeof("\0a\0func"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, +}, + +{ .descr = "func (Non zero vlen)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ @@ -3585,15 +3634,28 @@ static struct btf_raw_test raw_tests[] = { .btf_load_err = true, }, { - .descr = "type name '?foo' is not ok", + .descr = "type name '?foo' is ok", .raw_types = { /* union ?foo; */ BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_FWD, 1, 0), 0), /* [1] */ BTF_END_RAW, }, BTF_STR_SEC("\0?foo"), - .err_str = "Invalid name", - .btf_load_err = true, +}, +{ + .descr = "names of Rust types and functions are ok", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_NTH(1), 0, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_NTH(2), 1, 4), /* [2] */ + BTF_MEMBER_ENC(NAME_NTH(3), 1, 0), + BTF_FWD_ENC(NAME_NTH(4), 0), /* [3] */ + BTF_TYPEDEF_ENC(NAME_NTH(5), 2), /* [4] */ + BTF_FUNC_PROTO_ENC(0, 1), /* [5] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(6), 1), + BTF_FUNC_ENC(NAME_NTH(7), 5), /* [6] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0u32\0Option<&str>\0__0\0*const str\0{impl#9}<[u8; 4]>\0self\0fmt<str>"), }, { @@ -7063,6 +7125,132 @@ static struct btf_dedup_test dedup_tests[] = { }, }, { + .descr = "dedup: array element comparison", + .input = { + .raw_types = { + /* signed int */ + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* unsigned int */ + BTF_TYPE_INT_ENC(NAME_NTH(2), 0, 0, 32, 4), /* [2] */ + /* signed int[1] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [3] */ + /* duplicate signed int[1] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [4] */ + /* unsigned int[1] */ + BTF_TYPE_ARRAY_ENC(2, 2, 1), /* [5] */ + /* struct s { signed int a[1]; signed int b[1]; } */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [6] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 3, 32), + /* struct s { signed int a[1]; unsigned int b[1]; } */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [7] */ + BTF_MEMBER_ENC(NAME_NTH(4), 4, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 5, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0unsigned int\0s\0a\0b"), + }, + .expect = { + .raw_types = { + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(NAME_NTH(2), 0, 0, 32, 4), /* [2] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [3] */ + BTF_TYPE_ARRAY_ENC(2, 2, 1), /* [4] */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [5] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 3, 32), + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [6] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 4, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0unsigned int\0s\0a\0b"), + }, +}, +{ + .descr = "dedup: array element comparison with same index type", + .input = { + .raw_types = { + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(NAME_NTH(2), 0, 0, 32, 4), /* [2] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [3] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [4] */ + BTF_TYPE_ARRAY_ENC(2, 1, 1), /* [5] */ + /* struct s { int a[1]; int b[1]; } */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [6] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 3, 32), + /* struct s { int a[1]; unsigned int b[1]; } */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [7] */ + BTF_MEMBER_ENC(NAME_NTH(4), 4, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 5, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0unsigned int\0s\0a\0b"), + }, + .expect = { + .raw_types = { + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(NAME_NTH(2), 0, 0, 32, 4), /* [2] */ + BTF_TYPE_ARRAY_ENC(1, 1, 1), /* [3] */ + BTF_TYPE_ARRAY_ENC(2, 1, 1), /* [4] */ + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [5] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 3, 32), + BTF_STRUCT_ENC(NAME_NTH(3), 2, 8), /* [6] */ + BTF_MEMBER_ENC(NAME_NTH(4), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(5), 4, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0unsigned int\0s\0a\0b"), + }, +}, +{ + .descr = "dedup: identical array element comparison", + .input = { + .raw_types = { + /* int */ + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* struct container { struct elem first[1]; struct elem second[1]; } */ + BTF_STRUCT_ENC(NAME_NTH(2), 2, 8), /* [2] */ + BTF_MEMBER_ENC(NAME_NTH(3), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(4), 3, 32), + /* struct elem[1] */ + BTF_TYPE_ARRAY_ENC(4, 1, 1), /* [3] */ + /* struct elem { int x; } */ + BTF_STRUCT_ENC(NAME_NTH(5), 1, 4), /* [4] */ + BTF_MEMBER_ENC(NAME_NTH(6), 1, 0), + /* duplicate struct container */ + BTF_STRUCT_ENC(NAME_NTH(2), 2, 8), /* [5] */ + BTF_MEMBER_ENC(NAME_NTH(3), 6, 0), + BTF_MEMBER_ENC(NAME_NTH(4), 7, 32), + /* duplicate struct elem[1] */ + BTF_TYPE_ARRAY_ENC(8, 1, 1), /* [6] */ + BTF_TYPE_ARRAY_ENC(9, 1, 1), /* [7] */ + /* duplicate struct elem */ + BTF_STRUCT_ENC(NAME_NTH(5), 1, 4), /* [8] */ + BTF_MEMBER_ENC(NAME_NTH(6), 1, 0), + BTF_STRUCT_ENC(NAME_NTH(5), 1, 4), /* [9] */ + BTF_MEMBER_ENC(NAME_NTH(6), 1, 0), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0container\0first\0second\0elem\0x"), + }, + .expect = { + .raw_types = { + BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_NTH(2), 2, 8), /* [2] */ + BTF_MEMBER_ENC(NAME_NTH(3), 3, 0), + BTF_MEMBER_ENC(NAME_NTH(4), 3, 32), + BTF_TYPE_ARRAY_ENC(4, 1, 1), /* [3] */ + BTF_STRUCT_ENC(NAME_NTH(5), 1, 4), /* [4] */ + BTF_MEMBER_ENC(NAME_NTH(6), 1, 0), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0container\0first\0second\0elem\0x"), + }, +}, +{ .descr = "dedup: struct example #1", /* * struct s { diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c b/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c index 9d6161151593..5c15a6b7f4fb 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c @@ -554,6 +554,120 @@ cleanup: btf__free(vmlinux_btf); } +static void test_split_loc(void) +{ + struct btf *btf1, *btf2; + int err; + + btf1 = btf__new_empty(); + if (!ASSERT_OK_PTR(btf1, "empty_main_btf")) + return; + + btf__set_pointer_size(btf1, 8); /* enforce 64-bit arch */ + + + btf__add_int(btf1, "long", 8, BTF_INT_SIGNED); /* [1] long */ + btf__add_ptr(btf1, 1); /* [2] ptr to long */ + btf__add_func_proto(btf1, 1); /* [3] long (*)(long, long *); */ + btf__add_func_param(btf1, "p1", 1); + btf__add_func_param(btf1, "p2", 2); + btf__add_func(btf1, "foo", BTF_FUNC_STATIC, 3); /* [4] long foo(long, long *); */ + btf__add_loc_param(btf1, 8, BTF_LOC_PARAM_CONST); + btf__add_loc_param_value(btf1, 3735928559); + btf__add_loc_param_value(btf1, 4277009102); /* [5] loc value */ + btf__add_loc_param(btf1, 8, BTF_LOC_PARAM_REG); /* [6] loc reg 1 */ + btf__add_loc_param_value(btf1, 1); + btf__add_loc_proto(btf1); /* [7] loc proto */ + btf__add_loc_proto_param(btf1, 5); /* param value */ + btf__add_loc_proto_param(btf1, 6); /* param reg 1 */ + + VALIDATE_RAW_BTF( + btf1, + "[1] INT 'long' size=8 bits_offset=0 nr_bits=64 encoding=SIGNED", + "[2] PTR '(anon)' type_id=1", + "[3] FUNC_PROTO '(anon)' ret_type_id=1 vlen=2\n" + "\t'p1' type_id=1\n" + "\t'p2' type_id=2", + "[4] FUNC 'foo' type_id=3 linkage=static", + "[5] LOC_PARAM '(anon)' size=8 flags=0x2 vlen=2\n" + "\tvalue=3735928559\n" + "\tvalue=4277009102", + "[6] LOC_PARAM '(anon)' size=8 flags=0x8 vlen=1\n" + "\tvalue=1", + "[7] LOC_PROTO '(anon)' vlen=2\n" + "\ttype_id=5\n" + "\ttype_id=6"); + + btf2 = btf__new_empty_split(btf1); + if (!ASSERT_OK_PTR(btf2, "empty_split_btf")) + goto cleanup; + btf__add_loc_param(btf2, 8, BTF_LOC_PARAM_REG); + btf__add_loc_param_value(btf2, 1); /* [8] loc reg 1 */ + btf__add_loc_proto(btf2); /* [9] loc proto */ + btf__add_loc_proto_param(btf2, 5); /* param value */ + btf__add_loc_proto_param(btf2, 8); /* param reg 1 */ + btf__add_locsec(btf2, "inline.text"); /* [10] locsec "inline.text" */ + /* add duplicate locsec */ + btf__add_locsec_loc(btf2, 4, 9, 128); + btf__add_locsec(btf2, "inline.text"); /* [11] locsec "inline.text" */ + btf__add_locsec_loc(btf2, 4, 9, 128); + + VALIDATE_RAW_BTF( + btf2, + "[1] INT 'long' size=8 bits_offset=0 nr_bits=64 encoding=SIGNED", + "[2] PTR '(anon)' type_id=1", + "[3] FUNC_PROTO '(anon)' ret_type_id=1 vlen=2\n" + "\t'p1' type_id=1\n" + "\t'p2' type_id=2", + "[4] FUNC 'foo' type_id=3 linkage=static", + "[5] LOC_PARAM '(anon)' size=8 flags=0x2 vlen=2\n" + "\tvalue=3735928559\n" + "\tvalue=4277009102", + "[6] LOC_PARAM '(anon)' size=8 flags=0x8 vlen=1\n" + "\tvalue=1", + "[7] LOC_PROTO '(anon)' vlen=2\n" + "\ttype_id=5\n" + "\ttype_id=6", + "[8] LOC_PARAM '(anon)' size=8 flags=0x8 vlen=1\n" + "\tvalue=1", + "[9] LOC_PROTO '(anon)' vlen=2\n" + "\ttype_id=5\n" + "\ttype_id=8", + "[10] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=4 loc_proto_type_id=9 offset=128", + "[11] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=4 loc_proto_type_id=9 offset=128"); + + err = btf__dedup(btf2, NULL); + if (!ASSERT_OK(err, "btf_dedup")) + goto cleanup; + + VALIDATE_RAW_BTF( + btf2, + "[1] INT 'long' size=8 bits_offset=0 nr_bits=64 encoding=SIGNED", + "[2] PTR '(anon)' type_id=1", + "[3] FUNC_PROTO '(anon)' ret_type_id=1 vlen=2\n" + "\t'p1' type_id=1\n" + "\t'p2' type_id=2", + "[4] FUNC 'foo' type_id=3 linkage=static", + "[5] LOC_PARAM '(anon)' size=8 flags=0x2 vlen=2\n" + "\tvalue=3735928559\n" + "\tvalue=4277009102", + "[6] LOC_PARAM '(anon)' size=8 flags=0x8 vlen=1\n" + "\tvalue=1", + "[7] LOC_PROTO '(anon)' vlen=2\n" + "\ttype_id=5\n" + "\ttype_id=6", + "[8] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=4 loc_proto_type_id=7 offset=128", + "[9] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=4 loc_proto_type_id=7 offset=128"); + +cleanup: + btf__free(btf2); + btf__free(btf1); +} + void test_btf_dedup_split() { if (test__start_subtest("split_simple")) @@ -566,4 +680,6 @@ void test_btf_dedup_split() test_split_dup_struct_in_cu(); if (test__start_subtest("split_module")) test_split_module(); + if (test__start_subtest("split_loc")) + test_split_loc(); } diff --git a/tools/testing/selftests/bpf/prog_tests/btf_distill.c b/tools/testing/selftests/bpf/prog_tests/btf_distill.c index fb67ae195a73..f0900b47fd48 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_distill.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_distill.c @@ -671,6 +671,110 @@ cleanup: btf__free(btf1); } +/* LOC_PARAM, LOC_PROTO should be added to split BTF. */ +static void test_distilled_loc(void) +{ + struct btf *btf1 = NULL, *btf2 = NULL, *btf3 = NULL; + struct btf *btf4 = NULL, *btf5 = NULL; + + btf1 = btf__new_empty(); + if (!ASSERT_OK_PTR(btf1, "empty_main_btf")) + return; + + btf__add_int(btf1, "int", 4, BTF_INT_SIGNED); /* [1] int */ + btf__add_func_proto(btf1, 1); /* [2] int (*)(int); */ + btf__add_func_param(btf1, "p1", 1); + btf__add_func(btf1, "foo", BTF_FUNC_STATIC, 2); /* [3] int foo(int); */ + btf__add_loc_param(btf1, 4, BTF_LOC_PARAM_SIGNED | BTF_LOC_PARAM_CONST); + btf__add_loc_param_value(btf1, -1); /* [4] loc value */ + btf__add_loc_proto(btf1); /* [5] loc proto */ + btf__add_loc_proto_param(btf1, 4); /* param value */ + btf__add_int(btf1, "unsigned int", 4, 0); /* [6] unsigned int */ + + VALIDATE_RAW_BTF( + btf1, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] FUNC_PROTO '(anon)' ret_type_id=1 vlen=1\n" + "\t'p1' type_id=1", + "[3] FUNC 'foo' type_id=2 linkage=static", + "[4] LOC_PARAM '(anon)' size=4 flags=0x3 vlen=1\n" + "\tvalue=-1", + "[5] LOC_PROTO '(anon)' vlen=1\n" + "\ttype_id=4", + "[6] INT 'unsigned int' size=4 bits_offset=0 nr_bits=32 encoding=(none)"); + + btf2 = btf__new_empty_split(btf1); + if (!ASSERT_OK_PTR(btf2, "empty_split_btf")) + goto cleanup; + + btf__add_locsec(btf2, "inline.text"); /* [6] locsec */ + btf__add_locsec_loc(btf2, 3, 5, 256); /* "foo" offset 256 */ + VALIDATE_RAW_BTF( + btf2, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] FUNC_PROTO '(anon)' ret_type_id=1 vlen=1\n" + "\t'p1' type_id=1", + "[3] FUNC 'foo' type_id=2 linkage=static", + "[4] LOC_PARAM '(anon)' size=4 flags=0x3 vlen=1\n" + "\tvalue=-1", + "[5] LOC_PROTO '(anon)' vlen=1\n" + "\ttype_id=4", + "[6] INT 'unsigned int' size=4 bits_offset=0 nr_bits=32 encoding=(none)", + "[7] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=3 loc_proto_type_id=5 offset=256"); + + if (!ASSERT_EQ(0, btf__distill_base(btf2, &btf3, &btf4), + "distilled_base") || + !ASSERT_OK_PTR(btf3, "distilled_base") || + !ASSERT_OK_PTR(btf4, "distilled_split") || + !ASSERT_EQ(2, btf__type_cnt(btf3), "distilled_base_type_cnt")) + goto cleanup; + + VALIDATE_RAW_BTF( + btf4, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + /* remainder is split BTF */ + "[2] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=4 loc_proto_type_id=6 offset=256", + "[3] FUNC_PROTO '(anon)' ret_type_id=1 vlen=1\n" + "\t'p1' type_id=1", + "[4] FUNC 'foo' type_id=3 linkage=static", + "[5] LOC_PARAM '(anon)' size=4 flags=0x3 vlen=1\n" + "\tvalue=-1", + "[6] LOC_PROTO '(anon)' vlen=1\n" + "\ttype_id=5"); + + btf5 = btf__new_empty(); + if (!ASSERT_OK_PTR(btf5, "empty_reloc_btf")) + goto cleanup; + btf__add_int(btf5, "int", 4, BTF_INT_SIGNED); /* [1] int */ + btf__add_int(btf5, "char", 1, 0); /* [2] char */ + + if (!ASSERT_EQ(btf__relocate(btf4, btf5), 0, "relocate_split")) + goto cleanup; + VALIDATE_RAW_BTF( + btf4, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] INT 'char' size=1 bits_offset=0 nr_bits=8 encoding=(none)", + /* remainder is split BTF */ + "[3] LOCSEC 'inline.text' vlen=1\n" + "\tfunc_type_id=5 loc_proto_type_id=7 offset=256", + "[4] FUNC_PROTO '(anon)' ret_type_id=1 vlen=1\n" + "\t'p1' type_id=1", + "[5] FUNC 'foo' type_id=4 linkage=static", + "[6] LOC_PARAM '(anon)' size=4 flags=0x3 vlen=1\n" + "\tvalue=-1", + "[7] LOC_PROTO '(anon)' vlen=1\n" + "\ttype_id=6"); + +cleanup: + btf__free(btf5); + btf__free(btf4); + btf__free(btf3); + btf__free(btf2); + btf__free(btf1); +} + void test_btf_distill(void) { if (test__start_subtest("distilled_base")) @@ -689,4 +793,6 @@ void test_btf_distill(void) test_distilled_base_vmlinux(); if (test__start_subtest("distilled_endianness")) test_distilled_endianness(); + if (test__start_subtest("distilled_loc")) + test_distilled_loc(); } diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index 9f1b50e07a29..35f214e254da 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -1,8 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> #include <bpf/btf.h> - -static int duration = 0; +#include "testing_helpers.h" void btf_dump_printf(void *ctx, const char *fmt, va_list args) { @@ -45,12 +44,75 @@ done: return err; } +/* + * Expected output is embedded in the test case source, between + * START-EXPECTED-OUTPUT and END-EXPECTED-OUTPUT markers. A region is either + * plain C, where a declaration doubles as its own expectation, or C wrapped in + * a block comment, where the rendered form differs from the source. All + * regions of a file concatenate into one expectation, compared against one + * whole-file dump. + * + * Returns a malloc'd buffer for the caller to free, or NULL on failure. + */ +static char *read_expected_output(const char *path) +{ + size_t out_sz = 0, line_cap = 0; + char *out = NULL, *line = NULL; + bool in_region = false; + FILE *f, *out_file; + + f = fopen(path, "r"); + if (!f) + return NULL; + + out_file = open_memstream(&out, &out_sz); + if (!out_file) { + fclose(f); + return NULL; + } + + while (getline(&line, &line_cap, f) > 0) { + const char *p; + + if (strstr(line, "START-EXPECTED-OUTPUT")) { + in_region = true; + continue; + } + if (strstr(line, "END-EXPECTED-OUTPUT")) + in_region = false; + if (!in_region) + continue; + + p = line + strspn(line, " \t"); + + /* opening or closing line of a commented out region */ + if (!strncmp(p, "/*", 2) || !strncmp(p, "*/", 2)) + continue; + + /* + * Only a '*' directly after the indentation is a comment + * prefix. Without one the line is taken as it is, leading + * whitespace included. + */ + p = *p == '*' ? p + 1 : line; + + fputs(p, out_file); + } + + free(line); + fclose(f); + fclose(out_file); + return out; +} + static int test_btf_dump_case(int n, struct btf_dump_test_case *t) { - char test_file[256], out_file[256], diff_cmd[1024]; + char *dump = NULL, *expected = NULL; struct btf *btf = NULL; - int err = 0, fd = -1; - FILE *f = NULL; + char test_file[256]; + size_t dump_sz = 0; + int err = 0; + FILE *f; snprintf(test_file, sizeof(test_file), "%s.bpf.o", t->file); @@ -69,29 +131,21 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) if (!t->known_ptr_sz) { btf__set_pointer_size(btf, 8); } else { - CHECK(btf__pointer_size(btf) != 8, "ptr_sz", "exp %d, got %zu\n", - 8, btf__pointer_size(btf)); - } + size_t ptr_sz = btf__pointer_size(btf); - snprintf(out_file, sizeof(out_file), "/tmp/%s.output.XXXXXX", t->file); - fd = mkstemp(out_file); - if (!ASSERT_GE(fd, 0, "create_tmp")) { - err = fd; - goto done; + ASSERT_EQ(ptr_sz, (size_t)8, "ptr_sz"); } - f = fdopen(fd, "w"); - if (CHECK(f == NULL, "open_tmp", "failed to open file: %s(%d)\n", - strerror(errno), errno)) { - close(fd); + + f = open_memstream(&dump, &dump_sz); + if (!ASSERT_OK_PTR(f, "open_memstream")) { + err = -errno; goto done; } err = btf_dump_all_types(btf, f); fclose(f); - close(fd); - if (CHECK(err, "btf_dump", "failure during C dumping: %d\n", err)) { + if (!ASSERT_OK(err, "btf_dump")) goto done; - } snprintf(test_file, sizeof(test_file), "progs/%s.c", t->file); if (access(test_file, R_OK) == -1) @@ -100,28 +154,18 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) * without preserving the directory structure. */ snprintf(test_file, sizeof(test_file), "%s.c", t->file); - /* - * Diff test output and expected test output, contained between - * START-EXPECTED-OUTPUT and END-EXPECTED-OUTPUT lines in test case. - * For expected output lines, everything before '*' is stripped out. - * Also lines containing comment start and comment end markers are - * ignored. - */ - snprintf(diff_cmd, sizeof(diff_cmd), - "awk '/START-EXPECTED-OUTPUT/{out=1;next} " - "/END-EXPECTED-OUTPUT/{out=0} " - "/\\/\\*|\\*\\//{next} " /* ignore comment start/end lines */ - "out {sub(/^[ \\t]*\\*/, \"\"); print}' '%s' | diff -u - '%s'", - test_file, out_file); - err = system(diff_cmd); - if (CHECK(err, "diff", - "differing test output, output=%s, err=%d, diff cmd:\n%s\n", - out_file, err, diff_cmd)) + + expected = read_expected_output(test_file); + if (!ASSERT_OK_PTR(expected, "read_expected_output")) { + err = -errno; goto done; + } - remove(out_file); + ASSERT_TEXT_EQ(dump, expected, "compare_text_to_expected"); done: + free(expected); + free(dump); btf__free(btf); return err; } @@ -175,7 +219,7 @@ static void test_ctx__dump_and_compare(struct test_ctx *t, fflush(t->dump_buf_file); t->dump_buf[t->dump_buf_sz] = 0; /* some libc implementations don't do this */ - ASSERT_STREQ(t->dump_buf, expected_output, message); + ASSERT_TEXT_EQ(t->dump_buf, expected_output, message); } static void test_btf_dump_incremental(void) @@ -266,6 +310,8 @@ static void test_btf_dump_incremental(void) ASSERT_OK(err, "field_ok"); err = btf__add_field(btf, "s", 6, 64, 0); ASSERT_OK(err, "field_ok"); + /* attribute on a type added after the first dump: the later walk indexes it */ + ASSERT_EQ(btf__add_decl_attr(btf, "round2", 7, -1), 8, "round2_attr"); test_ctx__dump_and_compare(&t, "struct s___2 {\n" @@ -274,7 +320,270 @@ static void test_btf_dump_incremental(void) " Y___2 = 1,\n" " } y;\n" " struct s s;\n" -"};\n\n" , "c_dump1"); +"} __attribute__((round2));\n\n", "c_dump1"); + + test_ctx__free(&t); +} + +static void test_btf_dump_decl_tags(void) +{ + struct test_ctx t = {}; + struct btf *btf; + int id; + + if (test_ctx__init(&t)) + return; + + btf = t.btf; + + /* + * Generate BTF corresponding to the following C code: + * + * struct s1 { int f; } __attribute__((attr1)) + * __attribute__((btf_decl_tag("plain_tag"))); + * typedef struct { int f; } __attribute__((a1)) __attribute__((a2)) t1; + * union u1 { int f; } __attribute__((uattr)); + * struct p1 { char c; int i; } __attribute__((packed)) __attribute__((pattr)); + * struct outer { struct { int g; } __attribute__((inner)) in; }; + * struct s2 { int f; } __attribute__((on_s2)); + * struct s3 { int f; } __attribute__((on_s3)); + * + * plus attributes on a var and a func, which btf_dump does not declare. + */ + id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + ASSERT_EQ(id, 1, "int_id"); + id = btf__add_int(btf, "char", 1, BTF_INT_CHAR); + ASSERT_EQ(id, 2, "char_id"); + + /* a named record with one attribute of each form */ + id = btf__add_struct(btf, "s1", 4); + ASSERT_EQ(id, 3, "s1_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "s1_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "attr1", 3, -1), 4, "s1_attr"); + /* no kflag: rendered as a btf_decl_tag(), not as a bare attribute */ + ASSERT_EQ(btf__add_decl_tag(btf, "plain_tag", 3, -1), 5, "s1_plain_tag"); + + /* + * an anonymous record behind a typedef, with two attributes. This is + * the case that cannot be expressed by a caller appending to the dump: + * the attribute has to go before the declarator. + */ + id = btf__add_struct(btf, NULL, 4); + ASSERT_EQ(id, 6, "anon_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "anon_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "a1", 6, -1), 7, "anon_attr1"); + ASSERT_EQ(btf__add_decl_attr(btf, "a2", 6, -1), 8, "anon_attr2"); + id = btf__add_typedef(btf, "t1", 6); + ASSERT_EQ(id, 9, "typedef_id"); + + /* unions are records too */ + id = btf__add_union(btf, "u1", 4); + ASSERT_EQ(id, 10, "u1_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "u1_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "uattr", 10, -1), 11, "u1_attr"); + + /* composes with the packed attribute, which is derived from layout */ + id = btf__add_struct(btf, "p1", 5); + ASSERT_EQ(id, 12, "p1_id"); + ASSERT_OK(btf__add_field(btf, "c", 2, 0, 0), "p1_field_c"); + ASSERT_OK(btf__add_field(btf, "i", 1, 8, 0), "p1_field_i"); + ASSERT_EQ(btf__add_decl_attr(btf, "pattr", 12, -1), 13, "p1_attr"); + + /* an anonymous record inlined at a member, the third emission position */ + id = btf__add_struct(btf, NULL, 4); + ASSERT_EQ(id, 14, "inner_id"); + ASSERT_OK(btf__add_field(btf, "g", 1, 0, 0), "inner_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "inner", 14, -1), 15, "inner_attr"); + id = btf__add_struct(btf, "outer", 4); + ASSERT_EQ(id, 16, "outer_id"); + ASSERT_OK(btf__add_field(btf, "in", 14, 0, 0), "outer_field"); + + /* not rendered: a var has no declaration in the C output */ + id = btf__add_var(btf, "v", BTF_VAR_GLOBAL_ALLOCATED, 3); + ASSERT_EQ(id, 17, "var_id"); + ASSERT_EQ(btf__add_decl_attr(btf, "var_attr", 17, -1), 18, "var_attr"); + /* not rendered: neither does a func */ + id = btf__add_func_proto(btf, 1); + ASSERT_EQ(id, 19, "proto_id"); + id = btf__add_func(btf, "fn", BTF_FUNC_GLOBAL, 19); + ASSERT_EQ(id, 20, "func_id"); + ASSERT_EQ(btf__add_decl_attr(btf, "func_attr", 20, -1), 21, "func_attr"); + + /* + * Tags added in reverse target order: the tag with the lower type ID + * names the record with the higher one, so the index is in emission + * order only once btf_dump_resize() has sorted it. Without the sort + * the lookups land outside their runs and the records lose their + * attributes. + */ + id = btf__add_struct(btf, "s2", 4); + ASSERT_EQ(id, 22, "s2_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "s2_field"); + id = btf__add_struct(btf, "s3", 4); + ASSERT_EQ(id, 23, "s3_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "s3_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "on_s3", 23, -1), 24, "s3_attr"); + ASSERT_EQ(btf__add_decl_attr(btf, "on_s2", 22, -1), 25, "s2_attr"); + + test_ctx__dump_and_compare(&t, +"struct s1 {\n" +" int f;\n" +"} __attribute__((attr1)) __attribute__((btf_decl_tag(\"plain_tag\")));\n" +"\n" +"typedef struct {\n" +" int f;\n" +"} __attribute__((a1)) __attribute__((a2)) t1;\n" +"\n" +"union u1 {\n" +" int f;\n" +"} __attribute__((uattr));\n" +"\n" +"struct p1 {\n" +" char c;\n" +" int i;\n" +"} __attribute__((packed)) __attribute__((pattr));\n" +"\n" +"struct outer {\n" +" struct {\n" +" int g;\n" +" } __attribute__((inner)) in;\n" +"};\n" +"\n" +"struct s2 {\n" +" int f;\n" +"} __attribute__((on_s2));\n" +"\n" +"struct s3 {\n" +" int f;\n" +"} __attribute__((on_s3));\n" +"\n", "dump_and_compare"); + + test_ctx__free(&t); +} + +/* + * An attribute on a member is emitted at the member declaration, and one on + * the record at the closing brace. Both are keyed by the record in the index, + * so this also checks that they do not leak into each other's position. + */ +static void test_btf_dump_decl_tags_members(void) +{ + struct test_ctx t = {}; + struct btf *btf; + int id; + + if (test_ctx__init(&t)) + return; + + btf = t.btf; + + /* + * Generate BTF corresponding to the following C code: + * + * struct m1 { + * int (*fp)(void) __attribute__((fp_attr)); + * int a; + * int b __attribute__((b1)) __attribute__((b2)); + * int arr[4] __attribute__((arr_attr)); + * } __attribute__((rec)); + * struct bits { int x : 3 __attribute__((x_attr)); int y : 29; }; + */ + id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + ASSERT_EQ(id, 1, "int_id"); + id = btf__add_array(btf, 1, 1, 4); + ASSERT_EQ(id, 2, "array_id"); + id = btf__add_func_proto(btf, 1); + ASSERT_EQ(id, 3, "proto_id"); + id = btf__add_ptr(btf, 3); + ASSERT_EQ(id, 4, "ptr_id"); + + id = btf__add_struct(btf, "m1", 32); + ASSERT_EQ(id, 5, "m1_id"); + ASSERT_OK(btf__add_field(btf, "fp", 4, 0, 0), "m1_fp"); + ASSERT_OK(btf__add_field(btf, "a", 1, 64, 0), "m1_a"); + ASSERT_OK(btf__add_field(btf, "b", 1, 96, 0), "m1_b"); + ASSERT_OK(btf__add_field(btf, "arr", 2, 128, 0), "m1_arr"); + ASSERT_EQ(btf__add_decl_attr(btf, "rec", 5, -1), 6, "m1_rec_attr"); + /* the attribute follows the whole declarator, pointer and array alike */ + ASSERT_EQ(btf__add_decl_attr(btf, "fp_attr", 5, 0), 7, "m1_fp_attr"); + /* two on one member, to pin the ordering */ + ASSERT_EQ(btf__add_decl_attr(btf, "b1", 5, 2), 8, "m1_b1"); + ASSERT_EQ(btf__add_decl_attr(btf, "b2", 5, 2), 9, "m1_b2"); + ASSERT_EQ(btf__add_decl_attr(btf, "arr_attr", 5, 3), 10, "m1_arr_attr"); + + /* a bit-field: the attribute has to follow the width */ + id = btf__add_struct(btf, "bits", 4); + ASSERT_EQ(id, 11, "bits_id"); + ASSERT_OK(btf__add_field(btf, "x", 1, 0, 3), "bits_x"); + ASSERT_OK(btf__add_field(btf, "y", 1, 3, 29), "bits_y"); + ASSERT_EQ(btf__add_decl_attr(btf, "x_attr", 11, 0), 12, "bits_x_attr"); + + test_ctx__dump_and_compare(&t, +"struct m1 {\n" +" int (*fp)(void) __attribute__((fp_attr));\n" +" int a;\n" +" int b __attribute__((b1)) __attribute__((b2));\n" +" int arr[4] __attribute__((arr_attr));\n" +"} __attribute__((rec));\n" +"\n" +"struct bits {\n" +" int x: 3 __attribute__((x_attr));\n" +" int y: 29;\n" +"};\n" +"\n", "dump_and_compare"); + + test_ctx__free(&t); +} + +/* + * A typedef is the third and last kind btf_dump declares. Its attributes go + * after the declarator, so a typedef of an anonymous record can carry three + * groups in one declaration, binding to three different entities. + */ +static void test_btf_dump_decl_tags_typedef(void) +{ + struct test_ctx t = {}; + struct btf *btf; + int id; + + if (test_ctx__init(&t)) + return; + + btf = t.btf; + + /* + * Generate BTF corresponding to the following C code: + * + * typedef int td __attribute__((t1)) + * __attribute__((btf_decl_tag("t2"))); + * typedef struct { + * int f __attribute__((memb_attr)); + * } __attribute__((rec_attr)) both __attribute__((both_attr)); + */ + id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + ASSERT_EQ(id, 1, "int_id"); + id = btf__add_typedef(btf, "td", 1); + ASSERT_EQ(id, 2, "td_id"); + ASSERT_EQ(btf__add_decl_attr(btf, "t1", 2, -1), 3, "td_attr1"); + /* no kflag: rendered as a btf_decl_tag(), not as a bare attribute */ + ASSERT_EQ(btf__add_decl_tag(btf, "t2", 2, -1), 4, "td_attr2"); + + id = btf__add_struct(btf, NULL, 4); + ASSERT_EQ(id, 5, "anon_id"); + ASSERT_OK(btf__add_field(btf, "f", 1, 0, 0), "anon_field"); + ASSERT_EQ(btf__add_decl_attr(btf, "rec_attr", 5, -1), 6, "anon_rec_attr"); + ASSERT_EQ(btf__add_decl_attr(btf, "memb_attr", 5, 0), 7, "anon_memb_attr"); + id = btf__add_typedef(btf, "both", 5); + ASSERT_EQ(id, 8, "both_id"); + ASSERT_EQ(btf__add_decl_attr(btf, "both_attr", 8, -1), 9, "both_attr"); + + test_ctx__dump_and_compare(&t, +"typedef int td __attribute__((t1)) __attribute__((btf_decl_tag(\"t2\")));\n" +"\n" +"typedef struct {\n" +" int f __attribute__((memb_attr));\n" +"} __attribute__((rec_attr)) both __attribute__((both_attr));\n" +"\n", "dump_and_compare"); test_ctx__free(&t); } @@ -295,8 +604,10 @@ static void test_btf_dump_type_tags(void) * struct s { * void __attribute__((btf_type_tag(\"void_tag\"))) *p1; * void __attribute__((void_attr)) *p2; + * void __arena *p3; * }; * + * The arena tag is rendered with the address space it denotes. */ id = btf__add_type_tag(btf, "void_tag", 0); @@ -309,17 +620,25 @@ static void test_btf_dump_type_tags(void) id = btf__add_ptr(btf, id); ASSERT_EQ(id, 4, "void_ptr_id2"); - id = btf__add_struct(btf, "s", 8); - ASSERT_EQ(id, 5, "struct_id"); + id = btf__add_type_tag(btf, "arena", 0); + ASSERT_EQ(id, 5, "arena_tag_id"); + id = btf__add_ptr(btf, id); + ASSERT_EQ(id, 6, "void_ptr_id3"); + + id = btf__add_struct(btf, "s", 24); + ASSERT_EQ(id, 7, "struct_id"); err = btf__add_field(btf, "p1", 2, 0, 0); ASSERT_OK(err, "field_ok1"); - err = btf__add_field(btf, "p2", 4, 0, 0); + err = btf__add_field(btf, "p2", 4, 64, 0); ASSERT_OK(err, "field_ok2"); + err = btf__add_field(btf, "p3", 6, 128, 0); + ASSERT_OK(err, "field_ok3"); test_ctx__dump_and_compare(&t, "struct s {\n" " void __attribute__((btf_type_tag(\"void_tag\"))) *p1;\n" " void __attribute__((void_attr)) *p2;\n" +" void __attribute__((address_space(1))) __attribute__((btf_type_tag(\"arena\"))) *p3;\n" "};\n\n", "dump_and_compare"); test_ctx__free(&t); @@ -1060,6 +1379,15 @@ void test_btf_dump() { if (test__start_subtest("btf_dump: incremental")) test_btf_dump_incremental(); + if (test__start_subtest("btf_dump: decl_tags")) + test_btf_dump_decl_tags(); + + if (test__start_subtest("btf_dump: decl_tags_members")) + test_btf_dump_decl_tags_members(); + + if (test__start_subtest("btf_dump: decl_tags_typedef")) + test_btf_dump_decl_tags_typedef(); + if (test__start_subtest("btf_dump: type_tags")) test_btf_dump_type_tags(); diff --git a/tools/testing/selftests/bpf/prog_tests/btf_field_iter.c b/tools/testing/selftests/bpf/prog_tests/btf_field_iter.c index 32159d3eb281..dcb5429d141d 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_field_iter.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_field_iter.c @@ -31,8 +31,11 @@ struct field_data { { .ids = { 11 }, .strs = { "decltag" } }, { .ids = { 6 }, .strs = { "typetag" } }, { .ids = {}, .strs = { "e64", "eval1", "eval2", "eval3" } }, - { .ids = { 15, 16 }, .strs = { "datasec1" } } - + { .ids = { 15, 16 }, .strs = { "datasec1" } }, + { .ids = {}, .strs = { "" } }, + { .ids = {}, .strs = { "" } }, + { .ids = { 22, 23 }, .strs = { "" } }, + { .ids = { 14, 24 }, .strs = { ".loc" } } }; /* Fabricate BTF with various types and check BTF field iteration finds types, @@ -88,6 +91,19 @@ void test_btf_field_iter(void) btf__add_datasec_var_info(btf, 15, 0, 4); btf__add_datasec_var_info(btf, 16, 4, 8); + btf__add_loc_param(btf, 4, BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_SIGNED); + /* [22] loc value -1 */ + btf__add_loc_param_value(btf, -1); + btf__add_loc_param(btf, 8, BTF_LOC_PARAM_REG); /* [23] loc reg 1 */ + btf__add_loc_param_value(btf, 1); + + btf__add_loc_proto(btf); /* [24] loc proto */ + btf__add_loc_proto_param(btf, 22); /* param value -1, */ + btf__add_loc_proto_param(btf, 23); /* param reg 1 */ + + btf__add_locsec(btf, ".loc"); /* [25] locsec ".loc" */ + btf__add_locsec_loc(btf, 14, 24, 128); /* "func" */ + VALIDATE_RAW_BTF( btf, "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", @@ -123,7 +139,16 @@ void test_btf_field_iter(void) "\t'eval3' val=3000", "[21] DATASEC 'datasec1' size=12 vlen=2\n" "\ttype_id=15 offset=0 size=4\n" - "\ttype_id=16 offset=4 size=8"); + "\ttype_id=16 offset=4 size=8", + "[22] LOC_PARAM '(anon)' size=4 flags=0x3 vlen=1\n" + "\tvalue=-1", + "[23] LOC_PARAM '(anon)' size=8 flags=0x8 vlen=1\n" + "\tvalue=1", + "[24] LOC_PROTO '(anon)' vlen=2\n" + "\ttype_id=22\n" + "\ttype_id=23", + "[25] LOCSEC '.loc' vlen=1\n" + "\tfunc_type_id=14 loc_proto_type_id=24 offset=128"); for (id = 1; id < btf__type_cnt(btf); id++) { struct btf_type *t = btf_type_by_id(btf, id); diff --git a/tools/testing/selftests/bpf/prog_tests/btf_inline.c b/tools/testing/selftests/bpf/prog_tests/btf_inline.c new file mode 100644 index 000000000000..e630e5f0f5fa --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_inline.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026, Oracle and/or its affiliates. */ + +#include <test_progs.h> +#include <bpf/btf.h> +#include <bpf/libbpf.h> + +#define BTF_SYSFS_DIR "/sys/kernel/btf" +#define BTF_INLINE_SUFFIX ".inline" + +/* + * For a specific inline site, verify we have the right function, + * loc proto and loc param representation and that the offset is + * reasonable given the caller where it was inlined. + */ +void test_btf_inline(void) +{ + struct btf *inline_btf = NULL, *btf = NULL, *vmlinux_btf = NULL; + const char *inline_caller = "bpf_testmod_test_write"; + const char *inline_func = "testmod_inline_len"; + unsigned long long caller_addr, base_addr; + bool skip = false, found_loc = false; + const struct btf_loc_param *lp; + int locsec_id, func_id, n, i; + const struct btf_type *t; + struct btf_loc *l; + const __u32 *p; + int err = 0; + + if (!env.has_testmod) { + test__skip(); + return; + } + + /* LOCSEC offsets are relative to the ELF section base. */ + base_addr = module_get_base_addr_sec("bpf_testmod", ".text"); + if (!ASSERT_NEQ(base_addr, 0, "base_addr_nonzero")) + return; + + if (!ASSERT_OK(kallsyms_find(inline_caller, &caller_addr), + "find_inline_caller")) + return; + if (!ASSERT_NEQ(caller_addr, 0, "caller_addr_nonzero")) + return; + if (!ASSERT_GE(caller_addr, base_addr, "caller_addr_ge_base_addr")) + return; + caller_addr -= base_addr; + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(vmlinux_btf, "vmlinux_btf")) + return; + + btf = btf__parse_split(BTF_SYSFS_DIR "/bpf_testmod", vmlinux_btf); + if (!ASSERT_OK_PTR(btf, "bpf_testmod_btf")) + goto out; + + inline_btf = btf__parse_split(BTF_SYSFS_DIR "/bpf_testmod" BTF_INLINE_SUFFIX, + btf); + err = libbpf_get_error(inline_btf); + /* pahole may not have inline BTF feature support. */ + if (err == -ENOENT) { + skip = true; + goto out; + } + if (!ASSERT_OK(err, "parse_inline_btf")) + goto out; + locsec_id = btf__find_by_name_kind(inline_btf, "inline.text", BTF_KIND_LOCSEC); + if (locsec_id < 0) { + skip = true; + goto out; + } + func_id = btf__find_by_name_kind(inline_btf, inline_func, BTF_KIND_FUNC); + if (!ASSERT_GT(func_id, 0, "inline_caller_func")) + goto out; + t = btf__type_by_id(inline_btf, locsec_id); + n = btf_vlen(t); + for (i = 0, l = btf_locsec_locs(t); i < n; i++, l++) { + if (l->func == func_id) { + found_loc = true; + break; + } + } + if (!ASSERT_TRUE(found_loc, "found_loc")) + goto out; + if (!ASSERT_GT(l->loc_proto, 0, "loc_proto_id")) + goto out; + if (!ASSERT_GT(l->offset, 0, "loc_offset")) + goto out; + t = btf__type_by_id(inline_btf, l->loc_proto); + if (!ASSERT_OK_PTR(t, "loc_proto_ptr")) + goto out; + if (!ASSERT_EQ(btf_vlen(t), 1, "loc_proto_one_param")) + goto out; + p = btf_loc_proto_params(t); + if (!ASSERT_GT(*p, 0, "loc_proto_has_loc_param")) + goto out; + t = btf__type_by_id(inline_btf, *p); + if (!ASSERT_OK_PTR(t, "loc_param_ptr")) + goto out; + lp = btf_loc_param(t); + if (!ASSERT_TRUE(lp->flags & BTF_LOC_PARAM_REG, "param_has_reg")) + goto out; + if (!ASSERT_GT(l->offset, caller_addr, "inline_gt_caller")) + goto out; + /* simple sanity test to roughly ensure inline site still in function */ + if (!ASSERT_LT(l->offset, caller_addr + 512, "inline_in_caller")) + goto out; +out: + btf__free(inline_btf); + btf__free(btf); + btf__free(vmlinux_btf); + if (skip) + test__skip(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/btf_module_allowlist.c b/tools/testing/selftests/bpf/prog_tests/btf_module_allowlist.c new file mode 100644 index 000000000000..3189133ee58b --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_module_allowlist.c @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <bpf/btf.h> +#include "btf_module_allowlist.skel.h" + +static void test_load_with_list(const char **list, size_t count, + int expected_ret) +{ + struct btf_module_allowlist *skel; + int ret; + LIBBPF_OPTS(bpf_object_open_opts, opts, + .btf_module_allowlist = list, + .btf_module_allowlist_cnt = count, + ); + + skel = btf_module_allowlist__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "btf_module_allowlist__open_opts")) + return; + + ret = btf_module_allowlist__load(skel); + ASSERT_EQ(ret, expected_ret, "btf_module_allowlist__load"); + + btf_module_allowlist__destroy(skel); +} + +static void btf_module_allowlist_default(void) +{ + struct btf_module_allowlist *skel; + int ret; + + skel = btf_module_allowlist__open(); + if (!ASSERT_OK_PTR(skel, "btf_module_allowlist__open")) + return; + + ret = btf_module_allowlist__load(skel); + ASSERT_OK(ret, "btf_module_allowlist__load"); + + btf_module_allowlist__destroy(skel); +} + +static void btf_module_allowlist_allow(void) +{ + const char *mod_list[] = { "bpf_testmod" }; + + test_load_with_list(mod_list, 1, 0); +} + +/* + * A non-NULL allowlist with a count of 0 explicitly disables module BTFs. + * The pointer must be non-NULL to distinguish this from an unspecified + * allowlist. + */ +static void btf_module_allowlist_emptylist(void) +{ + const char *mod_list[] = { NULL }; + + test_load_with_list(mod_list, 0, -ESRCH); +} + +/* + * bpf_testmod is not in the allowlist, so its BTF should not be loaded. + */ +static void btf_module_allowlist_skipunlisted(void) +{ + const char *mod_list[] = { "module_nonexist" }; + struct btf_module_allowlist *skel; + int ret; + LIBBPF_OPTS(bpf_object_open_opts, opts, + .btf_module_allowlist = mod_list, + .btf_module_allowlist_cnt = 1, + ); + + skel = btf_module_allowlist__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "btf_module_allowlist__open_opts")) + return; + + ret = bpf_program__set_attach_target(skel->progs.test_btf_module_allowlist, 0, + "bpf_testmod:bpf_testmod_fentry_test1"); + ASSERT_EQ(ret, -ESRCH, "bpf_program__set_attach_target"); + btf_module_allowlist__destroy(skel); +} + +static void test_invalid_input(const char **list, size_t count, + const char *test_name) +{ + struct btf_module_allowlist *skel; + char assert_name[64]; + int err; + LIBBPF_OPTS(bpf_object_open_opts, opts, + .btf_module_allowlist = list, + .btf_module_allowlist_cnt = count, + ); + + snprintf(assert_name, sizeof(assert_name), "%s: open_opts", test_name); + skel = btf_module_allowlist__open_opts(&opts); + err = errno; + if (!ASSERT_NULL(skel, assert_name)) { + btf_module_allowlist__destroy(skel); + return; + } + snprintf(assert_name, sizeof(assert_name), "%s: open_opts err", test_name); + ASSERT_EQ(err, EINVAL, assert_name); +} + +static void btf_module_allowlist_invalidinput(void) +{ + const char *names[] = { NULL }; + const char *empty_names[] = { "" }; + const char *duplicate_names[] = { "bpf_testmod", "bpf_testmod" }; + + test_invalid_input(NULL, 1, "null_list"); + test_invalid_input(names, 1, "null_name"); + test_invalid_input(empty_names, 1, "empty_name"); + test_invalid_input(duplicate_names, 2, "duplicate_name"); + test_invalid_input(names, (size_t)SSIZE_MAX + 1, "large_count"); +} + +void test_btf_module_allowlist(void) +{ + struct btf *vmlinux_btf = NULL; + struct btf *module_btf = NULL; + + if (!env.has_testmod) { + printf("%s:SKIP: bpf_testmod is not available\n", __func__); + test__skip(); + return; + } + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!vmlinux_btf) { + printf("%s:SKIP: vmlinux_btf is not available\n", __func__); + test__skip(); + return; + } + + /* Ensure bpf_testmod BTF is available. */ + module_btf = btf__load_module_btf("bpf_testmod", vmlinux_btf); + if (!module_btf) { + printf("%s:SKIP: bpf_testmod's BTF is not available\n", __func__); + btf__free(vmlinux_btf); + test__skip(); + return; + } + + btf__free(module_btf); + btf__free(vmlinux_btf); + + if (test__start_subtest("btf_module_allowlist_default")) + btf_module_allowlist_default(); + + if (test__start_subtest("btf_module_allowlist_allow")) + btf_module_allowlist_allow(); + + if (test__start_subtest("btf_module_allowlist_skipunlisted")) + btf_module_allowlist_skipunlisted(); + + if (test__start_subtest("btf_module_allowlist_emptylist")) + btf_module_allowlist_emptylist(); + + if (test__start_subtest("btf_module_allowlist_invalidinput")) + btf_module_allowlist_invalidinput(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/btf_rust.c b/tools/testing/selftests/bpf/prog_tests/btf_rust.c new file mode 100644 index 000000000000..daf777cadfda --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_rust.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include <bpf/btf.h> +#include "bpftool_helpers.h" + +#define FUNC_NAME "write_fmt<scx_simple::BpfStream>" +#define KSYM_NAME "write_fmt_scx_simple__BpfStream_" + +/* The name of a function of Rust is a part of the name of the program in kallsyms */ +static void test_func_name(void) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_prog_load_opts, opts); + int int_id, proto_id, prog_fd = -1, i; + struct bpf_func_info func_info = {}; + struct bpf_prog_info info = {}; + unsigned long long addr; + __u32 len = sizeof(info); + char sym[128], *p = sym; + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf")) + return; + int_id = btf__add_int(btf, "i32", 4, BTF_INT_SIGNED); + ASSERT_GT(int_id, 0, "int"); + proto_id = btf__add_func_proto(btf, int_id); + ASSERT_GT(proto_id, 0, "proto"); + ASSERT_OK(btf__add_func_param(btf, "ctx", int_id), "param"); + func_info.type_id = btf__add_func(btf, FUNC_NAME, BTF_FUNC_STATIC, proto_id); + ASSERT_GT(func_info.type_id, 0, "func"); + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf load")) + goto out; + + opts.prog_btf_fd = btf__fd(btf); + opts.func_info = &func_info; + opts.func_info_cnt = 1; + opts.func_info_rec_size = sizeof(func_info); + prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, + ARRAY_SIZE(insns), &opts); + if (!ASSERT_GE(prog_fd, 0, "prog load")) + goto out; + if (!ASSERT_OK(bpf_prog_get_info_by_fd(prog_fd, &info, &len), "prog info")) + goto out; + if (!info.jited_prog_len) { + test__skip(); + goto out; + } + + p += sprintf(p, "bpf_prog_"); + for (i = 0; i < BPF_TAG_SIZE; i++) + p += sprintf(p, "%02x", info.tag[i]); + sprintf(p, "_%s", KSYM_NAME); + ASSERT_OK(kallsyms_find(sym, &addr), sym); +out: + if (prog_fd >= 0) + close(prog_fd); + btf__free(btf); +} + +#define PIN_PATH "/sys/fs/bpf/btf_rust_piece" + +/* + * A piece of a static that LLVM split has the type of the whole static. + * It's the last variable in the section, so its type ends past the map value. + */ +static void test_piece(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + int int_id, struct_id, var_id, piece_id, sec_id, map_fd = -1, key = 0; + __u32 value[2] = { 0x11111111, 0x22222222 }; + char line[256] = {}, out[1024] = {}; + struct btf *btf; + FILE *f = NULL; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf")) + return; + int_id = btf__add_int(btf, "u32", 4, 0); + ASSERT_GT(int_id, 0, "int"); + struct_id = btf__add_struct(btf, "Whole", 8); + ASSERT_GT(struct_id, 0, "struct"); + ASSERT_OK(btf__add_field(btf, "a", int_id, 0, 0), "field"); + ASSERT_OK(btf__add_field(btf, "b", int_id, 32, 0), "field"); + var_id = btf__add_var(btf, "CNT", BTF_VAR_STATIC, int_id); + ASSERT_GT(var_id, 0, "var"); + piece_id = btf__add_var(btf, "WHOLE.1", BTF_VAR_STATIC, struct_id); + ASSERT_GT(piece_id, 0, "piece"); + sec_id = btf__add_datasec(btf, ".bss", sizeof(value)); + ASSERT_GT(sec_id, 0, "datasec"); + ASSERT_OK(btf__add_datasec_var_info(btf, var_id, 0, 4), "var info"); + ASSERT_OK(btf__add_datasec_var_info(btf, piece_id, 4, 4), "piece info"); + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf load")) + goto out; + + opts.btf_fd = btf__fd(btf); + opts.btf_value_type_id = sec_id; + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, ".bss", sizeof(key), sizeof(value), 1, &opts); + if (!ASSERT_GE(map_fd, 0, "map create")) + goto out; + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &key, value, 0), "map update")) + goto out; + + /* the variable is printed, the piece is not */ + unlink(PIN_PATH); + if (!ASSERT_OK(bpf_obj_pin(map_fd, PIN_PATH), "pin")) + goto out; + f = fopen(PIN_PATH, "r"); + if (!ASSERT_OK_PTR(f, "open")) + goto out; + while (fgets(line, sizeof(line), f) && line[0] == '#') + ; + ASSERT_HAS_SUBSTR(line, "286331153", "var"); + ASSERT_NULL(strstr(line, "572662306"), "piece"); + + /* the same for bpftool */ + if (!ASSERT_OK(get_bpftool_command_output("map dump pinned " PIN_PATH, out, sizeof(out)), + "bpftool")) + goto out; + ASSERT_HAS_SUBSTR(out, "CNT", "var"); + ASSERT_NULL(strstr(out, "WHOLE.1"), "piece"); +out: + if (f) + fclose(f); + unlink(PIN_PATH); + if (map_fd >= 0) + close(map_fd); + btf__free(btf); +} + +/* Serial: programs are not in kallsyms while another test sets bpf_jit_harden */ +void serial_test_btf_rust(void) +{ + if (test__start_subtest("func_name")) + test_func_name(); + if (test__start_subtest("piece")) + test_piece(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/btf_sysfs.c b/tools/testing/selftests/bpf/prog_tests/btf_sysfs.c index 3923e64c4c1d..3fde8598189f 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_sysfs.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_sysfs.c @@ -3,15 +3,21 @@ #include <test_progs.h> #include <bpf/btf.h> +#include <dirent.h> +#include <limits.h> #include <sys/stat.h> #include <sys/mman.h> #include <fcntl.h> #include <unistd.h> +#define BTF_SYSFS_DIR "/sys/kernel/btf" +#define BTF_INLINE_SUFFIX ".inline" + static void test_btf_mmap_sysfs(const char *path, struct btf *base) { struct stat st; __u64 btf_size, end; + size_t mmap_len = 0; void *raw_data = NULL; int fd = -1; long page_size; @@ -26,6 +32,7 @@ static void test_btf_mmap_sysfs(const char *path, struct btf *base) btf_size = st.st_size; end = (btf_size + page_size - 1) / page_size * page_size; + mmap_len = btf_size; fd = open(path, O_RDONLY); if (!ASSERT_GE(fd, 0, "open_btf")) @@ -39,11 +46,13 @@ static void test_btf_mmap_sysfs(const char *path, struct btf *base) if (!ASSERT_EQ(raw_data, MAP_FAILED, "mmap_btf_shared")) goto cleanup; - raw_data = mmap(NULL, end + 1, PROT_READ, MAP_PRIVATE, fd, 0); + mmap_len = end + 1; + raw_data = mmap(NULL, mmap_len, PROT_READ, MAP_PRIVATE, fd, 0); if (!ASSERT_EQ(raw_data, MAP_FAILED, "mmap_btf_invalid_size")) goto cleanup; - raw_data = mmap(NULL, end, PROT_READ, MAP_PRIVATE, fd, 0); + mmap_len = end; + raw_data = mmap(NULL, mmap_len, PROT_READ, MAP_PRIVATE, fd, 0); if (!ASSERT_OK_PTR(raw_data, "mmap_btf")) goto cleanup; @@ -70,12 +79,84 @@ static void test_btf_mmap_sysfs(const char *path, struct btf *base) cleanup: btf__free(btf); if (raw_data && raw_data != MAP_FAILED) - munmap(raw_data, btf_size); + munmap(raw_data, mmap_len); if (fd >= 0) close(fd); } +static void test_btf_inline_sysfs_all(void) +{ + struct btf *vmlinux_btf; + struct dirent *dentry; + DIR *dir; + int err = 0; + + dir = opendir(BTF_SYSFS_DIR); + if (!ASSERT_OK_PTR(dir, "open_btf_sysfs")) + return; + + vmlinux_btf = btf__parse(BTF_SYSFS_DIR "/vmlinux", NULL); + if (!ASSERT_OK_PTR(vmlinux_btf, "parse_vmlinux_btf")) { + closedir(dir); + return; + } + + while ((dentry = readdir(dir)) != NULL) { + struct btf *base_btf = NULL, *module_btf = NULL, *inline_btf = NULL; + char btf_path[PATH_MAX], inline_path[PATH_MAX]; + struct stat st; + + /* Skip ".", ".." and "foo.inline" */ + if (strstr(dentry->d_name, ".")) + continue; + + if (strcmp(dentry->d_name, "vmlinux") == 0) + base_btf = vmlinux_btf; + + if (snprintf(btf_path, sizeof(btf_path), "%s/%s", + BTF_SYSFS_DIR, dentry->d_name) >= sizeof(btf_path) || + snprintf(inline_path, sizeof(inline_path), "%s/%s%s", + BTF_SYSFS_DIR, dentry->d_name, BTF_INLINE_SUFFIX) >= + sizeof(inline_path)) { + ASSERT_FAIL("BTF sysfs path is too long\n"); + break; + } + + if (!base_btf) { + module_btf = btf__parse_split(btf_path, vmlinux_btf); + err = libbpf_get_error(module_btf); + if (err) { + /* A module can be unloaded while its sysfs entry is iterated. */ + if (err == -ENOENT) + continue; + ASSERT_OK(err, "parse_module_btf"); + continue; + } + base_btf = module_btf; + } + if (stat(inline_path, &st)) { + err = errno; + if (err != ENOENT) + ASSERT_OK(err, "stat_inline_btf"); + btf__free(module_btf); + continue; + } + if (base_btf == vmlinux_btf) + test_btf_mmap_sysfs(inline_path, base_btf); + inline_btf = btf__parse_split(inline_path, base_btf); + err = libbpf_get_error(inline_btf); + if (!err) + btf__free(inline_btf); + ASSERT_OK(err, "parse_inline_btf"); + btf__free(module_btf); + } + closedir(dir); + + btf__free(vmlinux_btf); +} + void test_btf_sysfs(void) { test_btf_mmap_sysfs("/sys/kernel/btf/vmlinux", NULL); + test_btf_inline_sysfs_all(); } diff --git a/tools/testing/selftests/bpf/prog_tests/btf_write.c b/tools/testing/selftests/bpf/prog_tests/btf_write.c index 5c84723cf254..3bed3a28c06a 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_write.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_write.c @@ -606,6 +606,38 @@ cleanup: btf__free(base); } +static void test_btf_attrs(void) +{ + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "new_empty")) + return; + + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + btf__add_struct(btf, "s", 4); + btf__add_field(btf, "f", 1, 0, 0); + + btf__add_decl_tag(btf, "tag", 2, -1); + btf__add_decl_attr(btf, "tag", 2, -1); + btf__add_decl_attr(btf, "member_attr", 2, 0); + btf__add_type_tag(btf, "tag", 1); + btf__add_type_attr(btf, "tag", 1); + + VALIDATE_RAW_BTF( + btf, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] STRUCT 's' size=4 vlen=1\n" + "\t'f' type_id=1 bits_offset=0", + "[3] DECL_TAG 'tag' type_id=2 component_idx=-1", + "[4] DECL_TAG 'tag' type_id=2 component_idx=-1 kflag=1", + "[5] DECL_TAG 'member_attr' type_id=2 component_idx=0 kflag=1", + "[6] TYPE_TAG 'tag' type_id=1", + "[7] TYPE_TAG 'tag' type_id=1 kflag=1"); + + btf__free(btf); +} + void test_btf_write() { if (test__start_subtest("btf_add")) @@ -614,4 +646,6 @@ void test_btf_write() test_btf_add_btf(); if (test__start_subtest("btf_add_btf_split")) test_btf_add_btf_split(); + if (test__start_subtest("btf_attrs")) + test_btf_attrs(); } diff --git a/tools/testing/selftests/bpf/prog_tests/call_rcu.c b/tools/testing/selftests/bpf/prog_tests/call_rcu.c new file mode 100644 index 000000000000..e4d271d43d87 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/call_rcu.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> +#include "call_rcu.skel.h" +#include "call_rcu_fail.skel.h" + +struct elem { + __u64 pad; + struct bpf_rcu_head rh; + __u64 val; +}; + +/* + * Force the grace period, then poll: the callback still has to be invoked + * afterwards, and call_rcu() is lazy on a CONFIG_RCU_LAZY kernel. + */ +static bool wait_for_callbacks(struct call_rcu *skel, int expected) +{ + int i, got = 0; + + kern_sync_rcu(); + for (i = 0; i < 3000; i++) { + got = __atomic_load_n(&skel->bss->callbacks, __ATOMIC_ACQUIRE); + if (got >= expected) + return true; + usleep(10000); + } + return ASSERT_GE(got, expected, "callbacks"); +} + +static void test_call_rcu_run(bool trace) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct bpf_program *prog; + struct call_rcu *skel; + struct elem elem; + __u32 key = 1; + int err; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + prog = trace ? skel->progs.arm_trace : skel->progs.arm; + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(opts.retval, 0, "retval")) + goto out; + + ASSERT_EQ(skel->bss->arm_err, 0, "arm_err"); + ASSERT_EQ(skel->bss->busy_err, -EBUSY, "busy_err"); + + if (!wait_for_callbacks(skel, 1)) + goto out; + + ASSERT_EQ(skel->bss->cb_key, key, "cb_key"); + ASSERT_EQ(skel->bss->cb_val, 0xdeadbeef, "cb_val"); + ASSERT_EQ(skel->bss->cb_max_entries, bpf_map__max_entries(skel->maps.arr), "cb_map"); + + err = bpf_map__lookup_elem(skel->maps.arr, &key, sizeof(key), &elem, sizeof(elem), 0); + if (ASSERT_OK(err, "lookup")) + ASSERT_EQ(elem.val, 0, "value_cleared"); + + /* The head is disarmed before the callback runs, so it can be reused. */ + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + if (!ASSERT_OK(err, "test_run_again")) + goto out; + ASSERT_EQ(skel->bss->arm_err, 0, "rearm_err"); + wait_for_callbacks(skel, 2); +out: + call_rcu__destroy(skel); +} + +static void test_call_rcu_chain(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct call_rcu *skel; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + skel->bss->chain = 1; + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm), &opts), "test_run")) + goto out; + + if (!wait_for_callbacks(skel, 1)) + goto out; + ASSERT_EQ(skel->bss->chain_err, 0, "chain_err"); + wait_for_callbacks(skel, 2); +out: + call_rcu__destroy(skel); +} + +/* + * A callback that keeps re-arming must stop once the map loses its last user + * reference, or it pins the program for good. Hold an independent fd on .bss + * so the refusal is still readable after the skeleton is gone. + */ +static void test_call_rcu_teardown(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int i, err, fd = 0, bss_fd = -1; + struct bpf_prog_info pinfo = {}; + struct bpf_map_info minfo = {}; + __u32 len, prog_id, zero = 0; + struct call_rcu *skel; + char *buf = NULL; + size_t off, vsz; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + len = sizeof(pinfo); + if (!ASSERT_OK(bpf_prog_get_info_by_fd(bpf_program__fd(skel->progs.arm), &pinfo, &len), + "prog_info")) + goto out; + prog_id = pinfo.id; + + len = sizeof(minfo); + if (!ASSERT_OK(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.bss), &minfo, &len), + "bss_info")) + goto out; + bss_fd = bpf_map_get_fd_by_id(minfo.id); + if (!ASSERT_GE(bss_fd, 0, "bss_fd")) + goto out; + + vsz = bpf_map__value_size(skel->maps.bss); + off = (char *)&skel->bss->chain_err - (char *)skel->bss; + buf = malloc(vsz); + if (!ASSERT_OK_PTR(buf, "buf")) + goto out; + + skel->bss->chain = INT_MAX; + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm), &opts); + if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(opts.retval, 0, "retval")) + goto out; + if (!ASSERT_EQ(skel->bss->arm_err, 0, "arm_err")) + goto out; + /* The chain has to be running before the map reference goes away. */ + if (!wait_for_callbacks(skel, 2)) + goto out; + + call_rcu__destroy(skel); + skel = NULL; + + for (i = 0; i < 3000; i++) { + fd = bpf_prog_get_fd_by_id(prog_id); + if (fd < 0) + break; + close(fd); + usleep(10000); + } + if (!ASSERT_EQ(fd, -ENOENT, "prog_freed")) + goto out; + + if (ASSERT_OK(bpf_map_lookup_elem(bss_fd, &zero, buf), "bss_lookup")) + ASSERT_EQ(*(int *)(buf + off), -EPERM, "chain_refused"); +out: + free(buf); + if (bss_fd >= 0) + close(bss_fd); + call_rcu__destroy(skel); +} + +static void test_call_rcu_bad_map(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + struct call_rcu *skel; + int fd; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + opts.btf_fd = bpf_object__btf_fd(skel->obj); + opts.btf_key_type_id = bpf_map__btf_key_type_id(skel->maps.arr); + opts.btf_value_type_id = bpf_map__btf_value_type_id(skel->maps.arr); + + fd = bpf_map_create(BPF_MAP_TYPE_HASH, "rcu_hash", sizeof(__u32), + bpf_map__value_size(skel->maps.arr), 1, &opts); + ASSERT_EQ(fd, -EOPNOTSUPP, "hash_rejected"); + if (fd >= 0) + close(fd); + + call_rcu__destroy(skel); +} + +/* Iterating would hand the program a writable pointer to the head. */ +static void test_call_rcu_iter(void) +{ + LIBBPF_OPTS(bpf_iter_attach_opts, opts); + union bpf_iter_link_info linfo = {}; + struct bpf_link *link; + struct call_rcu *skel; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + linfo.map.map_fd = bpf_map__fd(skel->maps.arr); + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + + link = bpf_program__attach_iter(skel->progs.dump, &opts); + if (!ASSERT_ERR_PTR(link, "iter_rejected")) + bpf_link__destroy(link); + else + ASSERT_EQ(libbpf_get_error(link), -EOPNOTSUPP, "iter_errno"); + + call_rcu__destroy(skel); +} + +static void test_call_rcu_inner_map(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + struct call_rcu *skel; + int fd; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + opts.inner_map_fd = bpf_map__fd(skel->maps.arr); + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "rcu_outer", + sizeof(__u32), sizeof(__u32), 1, &opts); + ASSERT_EQ(fd, -EOPNOTSUPP, "inner_map_rejected"); + if (fd >= 0) + close(fd); + + call_rcu__destroy(skel); +} + +/* Two heads in the same map must be independent. */ +static void test_call_rcu_two_heads(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct call_rcu *skel; + + skel = call_rcu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.arm_both), &opts), + "test_run")) + goto out; + if (!ASSERT_EQ(skel->bss->arm_err, 0, "arm_err")) + goto out; + if (!wait_for_callbacks(skel, 2)) + goto out; + ASSERT_EQ(skel->bss->cb_keys, 0x3, "both_keys"); +out: + call_rcu__destroy(skel); +} + +void test_call_rcu(void) +{ + if (test__start_subtest("run")) + test_call_rcu_run(false); + if (test__start_subtest("run_tasks_trace")) + test_call_rcu_run(true); + if (test__start_subtest("chain")) + test_call_rcu_chain(); + if (test__start_subtest("two_heads")) + test_call_rcu_two_heads(); + if (test__start_subtest("teardown")) + test_call_rcu_teardown(); + if (test__start_subtest("bad_map")) + test_call_rcu_bad_map(); + if (test__start_subtest("iter")) + test_call_rcu_iter(); + if (test__start_subtest("inner_map")) + test_call_rcu_inner_map(); + RUN_TESTS(call_rcu_fail); +} diff --git a/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c b/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c new file mode 100644 index 000000000000..cc63b84e4d94 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * The kernel replaces the offsets of functions in a frozen read-only map with + * their addresses when the program is loaded. The program has to be the only + * user of the map, and no other program can use the map after that. + */ +#include <test_progs.h> +#include <linux/filter.h> +#include <bpf/btf.h> + +#if defined(__x86_64__) || defined(__aarch64__) || \ + (defined(__riscv) && __riscv_xlen == 64) + +#define CALLEE_INSN 6 +#define DATA 0x1234 + +/* + * main: r2 = &value; r2 = *(u64 *)(r2 + 8); r1 = 10; callx r2; exit + * add1: r0 = r1; r0 += 1; exit + * + * where value is { DATA, offset of add1 in the program }. + */ +static const struct bpf_insn callx_insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_2, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_2, 8), + BPF_MOV64_IMM(BPF_REG_1, 10), + BPF_RAW_INSN(BPF_JMP | BPF_CALL | BPF_X, BPF_REG_2, 0, 0, 0), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_0, BPF_REG_1), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1), + BPF_EXIT_INSN(), +}; + +/* reads the data of the map */ +static const struct bpf_insn reader_insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_2, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0), + BPF_EXIT_INSN(), +}; + +/* refers to the map and is rejected: r0 is not set */ +static const struct bpf_insn bad_insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_2, 0, 0), + BPF_EXIT_INSN(), +}; + +static char log_buf[16 * 1024]; +static struct bpf_func_info func_info[2]; +static int btf_fd; + +static int create_map(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_RDONLY_PROG); + __u64 value[2] = { DATA, CALLEE_INSN * sizeof(struct bpf_insn) }; + int fd, zero = 0; + + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "callx_rodata", sizeof(int), sizeof(value), 1, + &opts); + if (!ASSERT_OK_FD(fd, "map_create")) + return -1; + if (!ASSERT_OK(bpf_map_update_elem(fd, &zero, value, 0), "map_update") || + !ASSERT_OK(bpf_map_freeze(fd), "map_freeze")) { + close(fd); + return -1; + } + return fd; +} + +static int load(const struct bpf_insn *prog_insns, int cnt, int map_fd) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts, + .log_buf = log_buf, + .log_size = sizeof(log_buf), + .log_level = 1, + ); + struct bpf_insn insns[ARRAY_SIZE(callx_insns)]; + + if (prog_insns == callx_insns) { + /* add1() is referred to by the data only and is found through func_info */ + opts.prog_btf_fd = btf_fd; + opts.func_info = func_info; + opts.func_info_cnt = 2; + opts.func_info_rec_size = sizeof(func_info[0]); + } + memcpy(insns, prog_insns, cnt * sizeof(insns[0])); + insns[0].imm = map_fd; + log_buf[0] = 0; + return bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "callx_map", "GPL", insns, cnt, &opts); +} + +#define LOAD(insns, map_fd) load(insns, ARRAY_SIZE(insns), map_fd) + +static void run(int prog_fd, int expected, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + char pkt[64] = {}; + + topts.data_in = pkt; + topts.data_size_in = sizeof(pkt); + if (ASSERT_OK(bpf_prog_test_run_opts(prog_fd, &topts), name)) + ASSERT_EQ(topts.retval, expected, name); +} + +static void __test_callx_func_ptr_map(void) +{ + int int_id, proto_id, map_fd = -1, prog_fd = -1, reader_fd = -1, fd, zero = 0; + int main_id, add1_id; + struct btf *btf; + __u64 value[2]; + + /* there is no support for callx in the interpreter */ + if (!is_jit_enabled()) { + test__skip(); + return; + } + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf_new")) + return; + int_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + proto_id = btf__add_func_proto(btf, int_id); + main_id = btf__add_func(btf, "main_prog", BTF_FUNC_GLOBAL, proto_id); + add1_id = btf__add_func(btf, "add1", BTF_FUNC_STATIC, proto_id); + if (!ASSERT_GT(main_id, 0, "btf_add_main") || + !ASSERT_GT(add1_id, 0, "btf_add_add1") || + !ASSERT_OK(btf__load_into_kernel(btf), "btf_load")) + goto out; + func_info[0].insn_off = 0; + func_info[0].type_id = main_id; + func_info[1].insn_off = CALLEE_INSN; + func_info[1].type_id = add1_id; + btf_fd = btf__fd(btf); + + /* + * Another program relies on what the map has: it's verified with + * the data folded into constants. Pointers to functions are not looked + * for in such map. + */ + map_fd = create_map(); + if (map_fd < 0) + goto out; + reader_fd = LOAD(reader_insns, map_fd); + if (!ASSERT_OK_FD(reader_fd, "load_reader")) + goto out; + fd = LOAD(callx_insns, map_fd); + if (!ASSERT_LT(fd, 0, "load_shared")) + close(fd); + ASSERT_HAS_SUBSTR(log_buf, "unreachable insn 6", "log_shared"); + run(reader_fd, DATA, "run_reader"); + close(reader_fd); + reader_fd = -1; + close(map_fd); + + map_fd = create_map(); + if (map_fd < 0) + goto out; + + /* a program that is rejected is not a user, libbpf loads it again to get the log */ + fd = LOAD(bad_insns, map_fd); + if (!ASSERT_LT(fd, 0, "load_bad")) + close(fd); + + prog_fd = LOAD(callx_insns, map_fd); + if (!ASSERT_OK_FD(prog_fd, "load_callx")) { + printf("%s\n", log_buf); + goto out; + } + run(prog_fd, 11, "run_callx"); + + /* the offset of the function is gone from the map, the data is intact */ + if (ASSERT_OK(bpf_map_lookup_elem(map_fd, &zero, value), "map_lookup")) { + ASSERT_EQ(value[0], DATA, "data"); + ASSERT_NEQ(value[1], CALLEE_INSN * sizeof(struct bpf_insn), "pointer"); + } + + /* no other program can use the map now, another instance of the same one too */ + fd = LOAD(reader_insns, map_fd); + if (!ASSERT_EQ(fd, -EBUSY, "load_reader_after")) + close(fd); + ASSERT_HAS_SUBSTR(log_buf, "has addresses of functions of another program", "log_reader"); + fd = LOAD(callx_insns, map_fd); + if (!ASSERT_EQ(fd, -EBUSY, "load_second_instance")) + close(fd); + + run(prog_fd, 11, "run_callx_again"); +out: + if (reader_fd >= 0) + close(reader_fd); + if (prog_fd >= 0) + close(prog_fd); + if (map_fd >= 0) + close(map_fd); + btf__free(btf); +} + +#else + +static void __test_callx_func_ptr_map(void) +{ + test__skip(); +} + +#endif + +void test_callx_func_ptr_map(void) +{ + __test_callx_func_ptr_map(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c b/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c new file mode 100644 index 000000000000..f6ef67c3141e --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> + +#include "callx_rodata.lskel.h" + +#if defined(__x86_64__) || defined(__aarch64__) || \ + (defined(__riscv) && __riscv_xlen == 64) + +static void run(int prog_fd, int expected, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + char pkt[64] = {}; + + topts.data_in = pkt; + topts.data_size_in = sizeof(pkt); + if (ASSERT_OK(bpf_prog_test_run_opts(prog_fd, &topts), name)) + ASSERT_EQ(topts.retval, expected, name); +} + +/* + * Every program that has callx gets its own copy of .rodata with the offsets + * of its functions. The copies have what user space puts into .rodata before + * the load. The program that doesn't have callx uses .rodata. + */ +static void __test_callx_rodata_lskel(void) +{ + struct callx_rodata_lskel *skel; + + /* there is no support for callx in the interpreter */ + if (!is_jit_enabled()) { + test__skip(); + return; + } + + skel = callx_rodata_lskel__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + /* gcc doesn't support indirect calls */ + if (skel->rodata->skip) { + test__skip(); + goto out; + } + + skel->rodata->bias = 7; + + if (!ASSERT_OK(callx_rodata_lskel__load(skel), "load")) + goto out; + + skel->bss->op_idx = 0; + run(skel->progs.select_op.prog_fd, 17, "add_bias"); + skel->bss->op_idx = 1; + run(skel->progs.select_op.prog_fd, 30, "mul3"); + skel->bss->op_idx = 2; + run(skel->progs.select_op.prog_fd, -1, "out_of_range"); + /* mul3(add_bias(4)) */ + run(skel->progs.both_ops.prog_fd, 33, "both_ops"); + run(skel->progs.read_bias.prog_fd, 7, "read_bias"); +out: + callx_rodata_lskel__destroy(skel); +} + +#else + +static void __test_callx_rodata_lskel(void) +{ + test__skip(); +} + +#endif + +void test_callx_rodata_lskel(void) +{ + __test_callx_rodata_lskel(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/cb_refs.c b/tools/testing/selftests/bpf/prog_tests/cb_refs.c deleted file mode 100644 index 490e15e7126d..000000000000 --- a/tools/testing/selftests/bpf/prog_tests/cb_refs.c +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include "bpf/libbpf.h" -#include <test_progs.h> -#include <network_helpers.h> - -#include "cb_refs.skel.h" - -static char log_buf[1024 * 1024]; - -struct { - const char *prog_name; - const char *err_msg; -} cb_refs_tests[] = { - { "underflow_prog", "release kfunc bpf_kfunc_call_test_release expects referenced PTR_TO_BTF_ID passed to R1" }, - { "leak_prog", "Possibly NULL pointer passed to helper R2" }, - { "nested_cb", "Unreleased reference id=5 alloc_insn=2" }, /* alloc_insn=2{4,5} */ - { "non_cb_transfer_ref", "Unreleased reference id=4 alloc_insn=1" }, /* alloc_insn=1{1,2} */ -}; - -void test_cb_refs(void) -{ - LIBBPF_OPTS(bpf_object_open_opts, opts, .kernel_log_buf = log_buf, - .kernel_log_size = sizeof(log_buf), - .kernel_log_level = 1); - struct bpf_program *prog; - struct cb_refs *skel; - int i; - - for (i = 0; i < ARRAY_SIZE(cb_refs_tests); i++) { - LIBBPF_OPTS(bpf_test_run_opts, run_opts, - .data_in = &pkt_v4, - .data_size_in = sizeof(pkt_v4), - .repeat = 1, - ); - skel = cb_refs__open_opts(&opts); - if (!ASSERT_OK_PTR(skel, "cb_refs__open_and_load")) - return; - prog = bpf_object__find_program_by_name(skel->obj, cb_refs_tests[i].prog_name); - bpf_program__set_autoload(prog, true); - if (!ASSERT_ERR(cb_refs__load(skel), "cb_refs__load")) - bpf_prog_test_run_opts(bpf_program__fd(prog), &run_opts); - if (!ASSERT_OK_PTR(strstr(log_buf, cb_refs_tests[i].err_msg), "expected error message")) { - fprintf(stderr, "Expected: %s\n", cb_refs_tests[i].err_msg); - fprintf(stderr, "Verifier: %s\n", log_buf); - } - cb_refs__destroy(skel); - } -} diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c b/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c index 2bb5773d6f99..ea342859d6ce 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c @@ -465,7 +465,7 @@ struct exposed_hook { .expected_err = EXPECTED_ERR, \ }, -#include "cgroup_getset_retval_hooks.h" +#include "progs/cgroup_getset_retval_hooks.h" #undef BPF_RETVAL_HOOK }; diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c b/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c index 3bd27d2ea668..cc6ab2c263da 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c @@ -110,7 +110,7 @@ static void cleanup_bpffs(void) ASSERT_OK(rmdir(BPFFS_ATTACH_COUNTERS), "rmdir "BPFFS_ATTACH_COUNTERS); /* Unmount bpffs, if it wasn't already mounted when we started */ - if (mounted_bpffs) + if (!mounted_bpffs) return; ASSERT_OK(umount(BPFFS_ROOT), "unmount bpffs"); @@ -151,7 +151,7 @@ static void cleanup_cgroups(void) /* Sets up cgroup hiearchary, returns 0 on success. */ static int setup_hierarchy(void) { - return setup_bpffs() || setup_cgroups(); + return setup_cgroups() || setup_bpffs(); } static void destroy_hierarchy(void) diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_mprog_opts.c b/tools/testing/selftests/bpf/prog_tests/cgroup_mprog_opts.c index bb60704a3ef9..9ec5e4f96312 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_mprog_opts.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_mprog_opts.c @@ -273,11 +273,64 @@ cleanup: close(cg); } +static int find_prog_attach_flags(const struct bpf_prog_query_opts *opts, + __u32 prog_id, __u32 *flags) +{ + __u32 i; + + for (i = 0; i < opts->count; i++) { + if (opts->prog_ids[i] == prog_id) { + *flags = opts->prog_attach_flags[i]; + return 0; + } + } + return -ENOENT; +} + +static void assert_preorder_query_flags(int cg, int atype, + __u32 id1, __u32 id2, + __u32 id3, __u32 id4) +{ + LIBBPF_OPTS(bpf_prog_query_opts, optq); + __u32 prog_attach_flags[10] = {}; + __u32 prog_ids[10] = {}; + __u32 flags; + int err; + + optq.prog_ids = prog_ids; + optq.prog_attach_flags = prog_attach_flags; + optq.count = 10; + err = bpf_prog_query_opts(cg, atype, &optq); + if (!ASSERT_OK(err, "prog_query")) + return; + + ASSERT_EQ(optq.count, 4, "count"); + + /* Match by prog_id to avoid relying on query order. */ + err = find_prog_attach_flags(&optq, id1, &flags); + if (ASSERT_OK(err, "find id1")) + ASSERT_EQ(flags, BPF_F_ALLOW_MULTI, "flags id1"); + + err = find_prog_attach_flags(&optq, id2, &flags); + if (ASSERT_OK(err, "find id2")) + ASSERT_EQ(flags, BPF_F_ALLOW_MULTI | BPF_F_PREORDER, + "flags id2"); + + err = find_prog_attach_flags(&optq, id3, &flags); + if (ASSERT_OK(err, "find id3")) + ASSERT_EQ(flags, BPF_F_ALLOW_MULTI | BPF_F_PREORDER, + "flags id3"); + + err = find_prog_attach_flags(&optq, id4, &flags); + if (ASSERT_OK(err, "find id4")) + ASSERT_EQ(flags, BPF_F_ALLOW_MULTI, "flags id4"); +} + static void test_preorder_prog_attach_detach(int atype) { LIBBPF_OPTS(bpf_prog_attach_opts, opta); LIBBPF_OPTS(bpf_prog_detach_opts, optd); - __u32 fd1, fd2, fd3, fd4; + __u32 fd1, fd2, fd3, fd4, id1, id2, id3, id4; struct cgroup_mprog *skel; int cg, err; @@ -294,6 +347,11 @@ static void test_preorder_prog_attach_detach(int atype) fd3 = bpf_program__fd(skel->progs.getsockopt_3); fd4 = bpf_program__fd(skel->progs.getsockopt_4); + id1 = id_from_prog_fd(fd1); + id2 = id_from_prog_fd(fd2); + id3 = id_from_prog_fd(fd3); + id4 = id_from_prog_fd(fd4); + assert_mprog_count(cg, atype, 0); LIBBPF_OPTS_RESET(opta, @@ -357,6 +415,8 @@ static void test_preorder_prog_attach_detach(int atype) assert_mprog_count(cg, atype, 4); + assert_preorder_query_flags(cg, atype, id1, id2, id3, id4); + err = bpf_prog_detach_opts(fd4, cg, atype, &optd); ASSERT_OK(err, "prog_detach"); assert_mprog_count(cg, atype, 3); @@ -386,7 +446,7 @@ static void test_preorder_link_attach_detach(int atype) LIBBPF_OPTS(bpf_cgroup_opts, opta); struct bpf_link *link1, *link2, *link3, *link4; struct cgroup_mprog *skel; - __u32 fd2; + __u32 fd2, id1, id2, id3, id4; int cg; cg = test__join_cgroup("/preorder_link_attach_detach"); @@ -399,6 +459,11 @@ static void test_preorder_link_attach_detach(int atype) fd2 = bpf_program__fd(skel->progs.getsockopt_2); + id1 = id_from_prog_fd(bpf_program__fd(skel->progs.getsockopt_1)); + id2 = id_from_prog_fd(fd2); + id3 = id_from_prog_fd(bpf_program__fd(skel->progs.getsockopt_3)); + id4 = id_from_prog_fd(bpf_program__fd(skel->progs.getsockopt_4)); + assert_mprog_count(cg, atype, 0); LIBBPF_OPTS_RESET(opta, @@ -460,6 +525,8 @@ static void test_preorder_link_attach_detach(int atype) assert_mprog_count(cg, atype, 4); + assert_preorder_query_flags(cg, atype, id1, id2, id3, id4); + bpf_link__destroy(link4); assert_mprog_count(cg, atype, 3); diff --git a/tools/testing/selftests/bpf/prog_tests/clone_module_kfunc.c b/tools/testing/selftests/bpf/prog_tests/clone_module_kfunc.c new file mode 100644 index 000000000000..fa6ccbaa7fd9 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/clone_module_kfunc.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> +#include "kfunc_call_test.skel.h" + +/* + * bpf_program__clone() loads a program from a prepared object with the + * fd_array of the object, where module BTF fds start at slot 1 and slot 0 is + * empty. It used to pass the fill index of that array as fd_array_cnt, which + * made the kernel resolve slot 0 and fail the load. + */ +void test_clone_module_kfunc(void) +{ + struct kfunc_call_test *skel; + int err, fd; + + skel = kfunc_call_test__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + + err = bpf_object__prepare(skel->obj); + if (!ASSERT_OK(err, "prepare")) + goto out; + + fd = bpf_program__clone(skel->progs.kfunc_call_test1, NULL); + if (ASSERT_GE(fd, 0, "clone")) + close(fd); +out: + kfunc_call_test__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/copy_from_user_bprm.c b/tools/testing/selftests/bpf/prog_tests/copy_from_user_bprm.c new file mode 100644 index 000000000000..310370689427 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/copy_from_user_bprm.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <errno.h> +#include <sys/wait.h> +#include <unistd.h> + +#include <test_progs.h> + +#include "copy_from_user_bprm.skel.h" + +void test_copy_from_user_bprm(void) +{ + char data[] = "first\0second-argument\0SOME_ENV=a\0OTHER_ENV=something"; + char arg0[] = "first"; + char arg1[] = "second-argument"; + char env0[] = "SOME_ENV=a"; + char env1[] = "OTHER_ENV=something"; + struct copy_from_user_bprm *skel; + pid_t child; + int status; + + skel = copy_from_user_bprm__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + if (!ASSERT_OK(copy_from_user_bprm__attach(skel), "attach")) + goto out; + + child = fork(); + if (!ASSERT_GE(child, 0, "fork")) + goto out; + + if (!child) { + char *const argv[] = { arg0, arg1, NULL }; + char *const envp[] = { env0, env1, NULL }; + + skel->bss->monitored_pid = getpid(); + execvpe("true", argv, envp); + _exit(errno); + } + + if (!ASSERT_EQ(waitpid(child, &status, 0), child, "waitpid")) + goto out; + + if (ASSERT_TRUE(WIFEXITED(status), "child_exited")) + ASSERT_EQ(WEXITSTATUS(status), EPERM, "exec_errno"); + + ASSERT_EQ(skel->bss->bprm_argc, 2, "bprm_argc"); + ASSERT_EQ(skel->bss->bprm_envc, 2, "bprm_envc"); + ASSERT_EQ(skel->bss->data_len, sizeof(data), "data_len"); + ASSERT_EQ(skel->bss->invalid_flags_ret, -EINVAL, "invalid_flags_ret"); + ASSERT_EQ(skel->bss->copy_ret, 0, "copy_ret"); + ASSERT_EQ(skel->bss->arg0_ret, sizeof(arg0), "arg0_ret"); + ASSERT_EQ(skel->bss->arg1_ret, sizeof(arg1), "arg1_ret"); + ASSERT_EQ(skel->bss->env0_ret, sizeof(env0), "env0_ret"); + ASSERT_EQ(skel->bss->env1_ret, sizeof(env1), "env1_ret"); + ASSERT_EQ(memcmp(skel->bss->data, data, sizeof(data)), 0, "data"); + ASSERT_EQ(memcmp(skel->bss->arg0, arg0, sizeof(arg0)), 0, "arg0"); + ASSERT_EQ(memcmp(skel->bss->arg1, arg1, sizeof(arg1)), 0, "arg1"); + ASSERT_EQ(memcmp(skel->bss->env0, env0, sizeof(env0)), 0, "env0"); + ASSERT_EQ(memcmp(skel->bss->env1, env1, sizeof(env1)), 0, "env1"); + +out: + copy_from_user_bprm__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/core_reloc.c b/tools/testing/selftests/bpf/prog_tests/core_reloc.c index 08963c82f30b..80432c4656b5 100644 --- a/tools/testing/selftests/bpf/prog_tests/core_reloc.c +++ b/tools/testing/selftests/bpf/prog_tests/core_reloc.c @@ -7,6 +7,7 @@ #include <sys/mman.h> #include <sys/syscall.h> #include <bpf/btf.h> +#include "bpftool_helpers.h" static int duration = 0; @@ -985,16 +986,15 @@ static size_t roundup_page(size_t sz) static int run_btfgen(const char *src_btf, const char *dst_btf, const char *objpath) { - char command[4096]; + char args[4096]; int n; - n = snprintf(command, sizeof(command), - "./bpftool gen min_core_btf %s %s %s", + n = snprintf(args, sizeof(args), "gen min_core_btf %s %s %s", src_btf, dst_btf, objpath); - if (n < 0 || n >= sizeof(command)) + if (n < 0 || n >= sizeof(args)) return -1; - return system(command); + return run_bpftool_command(args); } static void run_core_reloc_tests(bool use_btfgen) diff --git a/tools/testing/selftests/bpf/prog_tests/data_in_arena.c b/tools/testing/selftests/bpf/prog_tests/data_in_arena.c new file mode 100644 index 000000000000..cb1023507c01 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/data_in_arena.c @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "data_in_arena.skel.h" +#include "data_in_arena_decl.skel.h" +#include "data_in_arena_fail.skel.h" + +static int run_prog(struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(prog), &topts), "test_run")) + return -1; + return topts.retval; +} + +static void run(struct data_in_arena *skel, int counter) +{ + int i; + + ASSERT_EQ(run_prog(skel->progs.use_data), counter + 7 + 1 + 2, "retval"); + ASSERT_EQ(skel->bss->sum, counter + 7 + 1 + 2, "sum"); + ASSERT_EQ(skel->data->counter, counter + 1, "counter"); + ASSERT_EQ(skel->data->pair[1], 5, "pair[1]"); + for (i = 0; i < 4; i++) + ASSERT_EQ(skel->bss->table[i], 10 * (i + 1) + i, "table"); +} + +static void test_in_arena(void) +{ + struct bpf_map_info info = {}; + struct data_in_arena *skel; + __u32 len = sizeof(info); + struct bpf_map *arena; + size_t sz; + + skel = data_in_arena__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + arena = bpf_object__find_map_by_name(skel->obj, "arena"); + if (!ASSERT_OK_PTR(arena, "arena")) + goto out; + ASSERT_EQ(bpf_map__type(arena), BPF_MAP_TYPE_ARENA, "arena type"); + ASSERT_EQ(bpf_map__max_entries(arena), 1, "arena pages"); + ASSERT_OK(bpf_map__set_max_entries(arena, 8), "arena resize"); + ASSERT_FALSE(bpf_map__autocreate(skel->maps.data), "data autocreate"); + ASSERT_FALSE(bpf_map__autocreate(skel->maps.bss), "bss autocreate"); + ASSERT_FALSE(bpf_map__autocreate(skel->maps.rodata), "rodata autocreate"); + ASSERT_EQ(bpf_map__set_autocreate(skel->maps.data, true), -EOPNOTSUPP, "set_autocreate"); + ASSERT_EQ(bpf_map__set_value_size(skel->maps.bss, 4096), -EOPNOTSUPP, "set_value_size"); + ASSERT_EQ(bpf_map__initial_value(skel->maps.data, &sz), skel->data, "initial_value"); + ASSERT_EQ(sz, sizeof(*skel->data), "initial_value size"); + + /* initial values are set the usual way */ + skel->data->counter = 100; + + if (!ASSERT_OK(data_in_arena__load(skel), "load")) + goto out; + /* there are no maps behind the sections */ + ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.data), &info, &len), "data map"); + ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.bss), &info, &len), "bss map"); + ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.rodata), &info, &len), + "rodata map"); + run(skel, 100); + + /* pointers to data next to pointers to functions */ + ASSERT_EQ(run_prog(skel->progs.use_ops), 42 + 1 + 'e', "use_ops"); + ASSERT_EQ(skel->data->counter, 102, "counter"); + + /* alignment of sections and pointers to data in data */ + ASSERT_EQ((unsigned long)&skel->bss->aligned64 % 64, 0, "alignment"); + ASSERT_EQ(run_prog(skel->progs.use_ptrs), 0, "use_ptrs"); + ASSERT_EQ(skel->data->x, 43, "x"); + ASSERT_EQ(skel->bss->aligned64.v[7], 7, "aligned64"); + ASSERT_EQ(skel->data->px, &skel->data->x, "px"); + + /* format strings of bpf_printk() and BPF_SNPRINTF() */ + ASSERT_EQ(run_prog(skel->progs.use_printk), sizeof("43-7"), "use_printk"); + ASSERT_STREQ(skel->bss->out, "43-7", "out"); +out: + data_in_arena__destroy(skel); +} + +/* The object has an arena map and __arena variables */ +static void test_declared_arena(void) +{ + struct data_in_arena_decl *skel; + struct bpf_map *map; + int arenas = 0; + + skel = data_in_arena_decl__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + bpf_object__for_each_map(map, skel->obj) + arenas += bpf_map__type(map) == BPF_MAP_TYPE_ARENA; + ASSERT_EQ(arenas, 1, "no second arena"); + skel->data->counter = 6; + if (!ASSERT_OK(data_in_arena_decl__load(skel), "load")) + goto out; + ASSERT_EQ(run_prog(skel->progs.use_data), 6 + 7 + 11, "retval"); + ASSERT_EQ(run_prog(skel->progs.use_data), 7 + 7 + 12, "retval"); + ASSERT_EQ(skel->bss->sum, 7 + 7 + 12, "sum"); + ASSERT_EQ(skel->data->counter, 8, "counter"); +out: + data_in_arena_decl__destroy(skel); +} + +/* A pointer in data that can't be made an address of arena fails the load */ +static void test_ptr_to_map(void) +{ + struct data_in_arena_fail *skel; + + skel = data_in_arena_fail__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + ASSERT_ERR(data_in_arena_fail__load(skel), "load"); + data_in_arena_fail__destroy(skel); +} + +/* So does a pointer to a variable of the kernel. There is no skeleton: the open fails. */ +static void test_ptr_to_extern(void) +{ + struct bpf_object *obj; + + obj = bpf_object__open_file("./data_in_arena_extern.bpf.o", NULL); + if (!ASSERT_ERR_PTR(obj, "open")) + bpf_object__close(obj); +} + +/* __arena variables and no arena map. There is no skeleton: the open fails. */ +static void test_arena_var_no_map(void) +{ + struct bpf_object *obj; + + obj = bpf_object__open_file("./data_in_arena_nomap.bpf.o", NULL); + if (!ASSERT_ERR_PTR(obj, "open")) + bpf_object__close(obj); +} + +/* The address of an arena that libbpf doesn't create is not known. No pointers in data then. */ +static void test_not_my_arena(bool pin) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE); + struct data_in_arena *skel; + struct bpf_map *arena; + int fd = -1; + + skel = data_in_arena__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + arena = bpf_object__find_map_by_name(skel->obj, "arena"); + if (!ASSERT_OK_PTR(arena, "arena")) + goto out; + if (pin) { + ASSERT_OK(bpf_map__set_pin_path(arena, "/sys/fs/bpf/data_in_arena"), "pin_path"); + } else { + fd = bpf_map_create(BPF_MAP_TYPE_ARENA, "arena", 0, 0, 1, &opts); + if (!ASSERT_GE(fd, 0, "map_create")) + goto out; + ASSERT_OK(bpf_map__reuse_fd(arena, fd), "reuse_fd"); + } + ASSERT_EQ(data_in_arena__load(skel), -ENOTSUP, "load"); +out: + if (fd >= 0) + close(fd); + data_in_arena__destroy(skel); +} + +/* The object is there if rustc and clang can build it, see Makefile.buildvars */ +static void test_rust(void) +{ + const char *file = "./data_in_arena_rust.bpf.o"; + struct bpf_object *obj; + + if (access(file, R_OK)) { + test__skip(); + return; + } + obj = bpf_object__open_file(file, NULL); + if (!ASSERT_OK_PTR(obj, "open")) + return; + if (!ASSERT_OK(bpf_object__load(obj), "load")) + goto out; + /* libbpf goes on without BTF when the kernel doesn't take it */ + ASSERT_GE(bpf_object__btf_fd(obj), 0, "btf_fd"); + ASSERT_EQ(run_prog(bpf_object__find_program_by_name(obj, "list_in_data")), + 100 + 20 + 3, "retval"); +out: + bpf_object__close(obj); +} + +void test_data_in_arena(void) +{ +#if !defined(__x86_64__) && !defined(__aarch64__) + /* other JITs don't take BPF_F_ARENA_SCALAR */ + test__skip(); + return; +#endif + if (test__start_subtest("arena")) + test_in_arena(); + if (test__start_subtest("declared_arena")) + test_declared_arena(); + if (test__start_subtest("ptr_to_map")) + test_ptr_to_map(); + if (test__start_subtest("ptr_to_extern")) + test_ptr_to_extern(); + if (test__start_subtest("arena_var_no_map")) + test_arena_var_no_map(); + if (test__start_subtest("pinned_arena")) + test_not_my_arena(true); + if (test__start_subtest("reused_arena")) + test_not_my_arena(false); + if (test__start_subtest("rust")) + test_rust(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c b/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c index fb2cea710db3..1a5dc1adae73 100644 --- a/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c +++ b/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c @@ -188,8 +188,9 @@ static void subtest_dmabuf_iter_check_default_iter(struct dmabuf_iter *skel) ASSERT_EQ(sscanf(line, "%ld", &bufinfo.size), 1, "read size"); } else if (f % FIELD_COUNT == NAME) { - ASSERT_EQ(sscanf(line, "%s", bufinfo.name), 1, - "read name"); + /* Keep an empty name for a buffer without one */ + if (sscanf(line, "%s", bufinfo.name) != 1) + bufinfo.name[0] = '\0'; } else if (f % FIELD_COUNT == EXPORTER) { ASSERT_EQ(sscanf(line, "%31s", bufinfo.exporter), 1, "read exporter"); diff --git a/tools/testing/selftests/bpf/prog_tests/exceptions.c b/tools/testing/selftests/bpf/prog_tests/exceptions.c index 639866ce09a9..1bd15c4c125b 100644 --- a/tools/testing/selftests/bpf/prog_tests/exceptions.c +++ b/tools/testing/selftests/bpf/prog_tests/exceptions.c @@ -6,6 +6,7 @@ #include "exceptions_ext.skel.h" #include "exceptions_fail.skel.h" #include "exceptions_assert.skel.h" +#include "exceptions_dead_subprog.skel.h" static char log_buf[1024 * 1024]; @@ -321,7 +322,7 @@ static void test_exceptions_success(void) bpf_program__fd(skel->progs.exception_throw_subprog), "subprog"), "set_attach_target")) goto done; - }), "subprog() is not a global function", 0); + }), "Cannot replace static functions", 0); if (test__start_subtest("non-throwing extension -> throwing subprog")) /* non-throwing extension -> throwing subprog : BAD (!global) */ @@ -332,7 +333,7 @@ static void test_exceptions_success(void) bpf_program__fd(skel->progs.exception_throw_subprog), "throwing_subprog"), "set_attach_target")) goto done; - }), "throwing_subprog() is not a global function", 0); + }), "Cannot replace static functions", 0); if (test__start_subtest("non-throwing extension -> non-throwing subprog")) /* non-throwing extension -> non-throwing global subprog : OK */ @@ -410,9 +411,15 @@ static void test_exceptions_assertions(void) RUN_TESTS(exceptions_assert); } +static void test_exceptions_dead_subprog(void) +{ + RUN_TESTS(exceptions_dead_subprog); +} + void test_exceptions(void) { test_exceptions_success(); test_exceptions_failure(); test_exceptions_assertions(); + test_exceptions_dead_subprog(); } diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c index 2523c07a16c6..d2f2dcc0f2e1 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c @@ -164,8 +164,10 @@ static void test_fexit_bpf2bpf_common(const char *obj_file, goto close_prog; close_prog: - for (i = 0; i < prog_cnt; i++) - bpf_link__destroy(link[i]); + if (link) { + for (i = 0; i < prog_cnt; i++) + bpf_link__destroy(link[i]); + } bpf_object__close(obj); bpf_object__close(tgt_obj); free(link); @@ -441,6 +443,20 @@ static void test_func_replace_int_with_void(void) " doesn't match type INT of global_func2()"); } +static void test_func_replace_ret_pair(void) +{ + const char *msg = "Return type of new_agg_ret_target_func() has size 8 " + "and of agg_ret_target_func() size 16"; + + /* + * An extension cannot replace a function whose return value comes back + * in the R0:R2 pair: only R0 is checked at the extension's exit, so it + * would leave R2 stale for the target's callers. + */ + test_obj_load_failure_common("freplace_ret_pair.bpf.o", + "./aggregate_ret_target.bpf.o", msg); +} + static int find_prog_btf_id(const char *name, __u32 attach_prog_fd) { struct bpf_prog_info info = {}; @@ -660,6 +676,8 @@ void serial_test_fexit_bpf2bpf(void) test_func_replace_progmap(); if (test__start_subtest("freplace_int_with_void")) test_func_replace_int_with_void(); + if (test__start_subtest("freplace_ret_pair")) + test_func_replace_ret_pair(); if (test__start_subtest("freplace_void")) test_func_replace_void(); if (test__start_subtest("sleepable_fentry_to_xdp")) diff --git a/tools/testing/selftests/bpf/prog_tests/file_reader.c b/tools/testing/selftests/bpf/prog_tests/file_reader.c index 48aae7ea0e4b..e59c6c87e9d5 100644 --- a/tools/testing/selftests/bpf/prog_tests/file_reader.c +++ b/tools/testing/selftests/bpf/prog_tests/file_reader.c @@ -7,10 +7,12 @@ #include "file_reader_fail.skel.h" #include <dlfcn.h> #include <sys/mman.h> +#include <sys/stat.h> const char *user_ptr = "hello world"; char file_contents[256000]; void *addr; +__u64 beyond_eof_offset; void *get_executable_base_addr(void) { @@ -26,12 +28,18 @@ void *get_executable_base_addr(void) static int initialize_file_contents(void) { + struct stat st; int fd, page_sz = sysconf(_SC_PAGESIZE); ssize_t n = 0, cur; fd = open("/proc/self/exe", O_RDONLY); if (!ASSERT_OK_FD(fd, "Open /proc/self/exe\n")) return 1; + if (!ASSERT_OK(fstat(fd, &st), "fstat /proc/self/exe")) { + close(fd); + return 1; + } + beyond_eof_offset = st.st_size + (1ULL << 30); do { cur = read(fd, file_contents + n, sizeof(file_contents) - n); @@ -75,6 +83,7 @@ static void run_test(const char *prog_name) memcpy(skel->bss->user_buf, file_contents, sizeof(file_contents)); skel->bss->pid = getpid(); + skel->bss->beyond_eof_offset = beyond_eof_offset; err = file_reader__load(skel); if (!ASSERT_OK(err, "file_reader__load")) @@ -110,6 +119,12 @@ void test_file_reader(void) if (test__start_subtest("on_open_validate_file_read")) run_test("on_open_validate_file_read"); + if (test__start_subtest("on_open_non_sleepable_first")) + run_test("on_open_non_sleepable_first"); + + if (test__start_subtest("on_open_sleepable_first")) + run_test("on_open_sleepable_first"); + if (test__start_subtest("negative")) RUN_TESTS(file_reader_fail); } diff --git a/tools/testing/selftests/bpf/prog_tests/flow_dissector.c b/tools/testing/selftests/bpf/prog_tests/flow_dissector.c index fb4892681464..f6fefb14cda3 100644 --- a/tools/testing/selftests/bpf/prog_tests/flow_dissector.c +++ b/tools/testing/selftests/bpf/prog_tests/flow_dissector.c @@ -706,7 +706,7 @@ void test_flow_dissector_skb_less_direct_attach(void) struct bpf_flow *skel; struct netns_obj *ns; - ns = netns_new("flow_dissector_skb_less_indirect_attach_ns", true); + ns = netns_new("flow_dissector_skb_less_direct_attach_ns", true); if (!ASSERT_OK_PTR(ns, "create and open netns")) return; diff --git a/tools/testing/selftests/bpf/prog_tests/flow_dissector_classification.c b/tools/testing/selftests/bpf/prog_tests/flow_dissector_classification.c index 80b153d3ddec..421dfa6c4ea3 100644 --- a/tools/testing/selftests/bpf/prog_tests/flow_dissector_classification.c +++ b/tools/testing/selftests/bpf/prog_tests/flow_dissector_classification.c @@ -27,7 +27,7 @@ #define TEST_NAME_MAX_LEN (32 + SUBTEST_NAME_MAX_LEN) #define MAX_SOURCE_PORTS 3 #define TEST_PACKETS_COUNT 10 -#define TEST_PACKET_LEN 100 +#define TEST_PACKET_LEN 99 #define TEST_PACKET_PATTERN 'a' #define TEST_IPV4 "192.168.0.1/32" #define TEST_IPV6 "100::a/128" diff --git a/tools/testing/selftests/bpf/prog_tests/global_data_init.c b/tools/testing/selftests/bpf/prog_tests/global_data_init.c index 5671c31085cd..7c539cbcf3a1 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_data_init.c +++ b/tools/testing/selftests/bpf/prog_tests/global_data_init.c @@ -68,12 +68,9 @@ static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_f { struct test_global_percpu_data__percpu *data = NULL; int i, err, key = 0, num_online, run = 0; - __u64 args[2] = {0x1234ULL, 0x5678ULL}; size_t data_sz; bool *online; LIBBPF_OPTS(bpf_test_run_opts, topts, - .ctx_in = args, - .ctx_size_in = sizeof(args), .flags = BPF_F_TEST_RUN_ON_CPU, ); @@ -81,7 +78,7 @@ static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_f if (!ASSERT_OK(err, "parse_cpu_mask_file")) return; - data_sz = map ? bpf_map__value_size(map) : sizeof(*data); + data_sz = sizeof(*data); data = calloc(1, data_sz); if (!ASSERT_OK_PTR(data, "calloc percpu data")) goto out; @@ -109,7 +106,7 @@ static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_f break; ASSERT_EQ(*runp, ++run, "run"); - ASSERT_EQ(data->cpu_id[0], i, "cpu_id"); + ASSERT_EQ(data->cpu_id, i, "cpu_id"); ASSERT_EQ(data->data, 1, "data"); ASSERT_TRUE(data->set, "set"); ASSERT_EQ(data->nums[6], 0xc0de, "nums[6]"); @@ -127,7 +124,6 @@ static void test_global_percpu_data_init(void) { struct test_global_percpu_data__percpu init_value = {}; struct test_global_percpu_data__percpu *init_data; - const __u32 desired_sz = sysconf(_SC_PAGE_SIZE); struct test_global_percpu_data *skel = NULL; size_t init_data_sz; struct bpf_map *map; @@ -163,9 +159,11 @@ static void test_global_percpu_data_init(void) if (!ASSERT_EQ(bpf_map__type(map), BPF_MAP_TYPE_PERCPU_ARRAY, "bpf_map__type")) goto out; + init_value.set = 1; init_value.data = 2; init_value.nums[6] = -1; init_value.struct_data.i = 2; + init_value.struct_data.set = 1; init_value.struct_data.nums[6] = -1; err = bpf_map__set_initial_value(map, &init_value, sizeof(init_value)); if (!ASSERT_OK(err, "bpf_map__set_initial_value")) @@ -177,35 +175,24 @@ static void test_global_percpu_data_init(void) ASSERT_EQ(init_data->data, init_value.data, "init_value data"); ASSERT_EQ(init_data->set, init_value.set, "init_value set"); + ASSERT_EQ(init_data->nums[6], init_value.nums[6], "init_value nums[6]"); ASSERT_EQ(init_data->struct_data.i, init_value.struct_data.i, "init_value struct_data.i"); + ASSERT_EQ(init_data->struct_data.set, init_value.struct_data.set, + "init_value struct_data.set"); ASSERT_EQ(init_data->struct_data.nums[6], init_value.struct_data.nums[6], "init_value struct_data.nums[6]"); ASSERT_EQ(init_data_sz, sizeof(init_value), "init_value size"); ASSERT_EQ((void *) init_data, (void *) skel->percpu, "skel->percpu eq init_data"); ASSERT_EQ(skel->percpu->data, init_value.data, "skel->percpu->data"); ASSERT_EQ(skel->percpu->set, init_value.set, "skel->percpu->set"); + ASSERT_EQ(skel->percpu->nums[6], init_value.nums[6], "skel->percpu->nums[6]"); ASSERT_EQ(skel->percpu->struct_data.i, init_value.struct_data.i, "skel->percpu->struct_data.i"); + ASSERT_EQ(skel->percpu->struct_data.set, init_value.struct_data.set, + "skel->percpu->struct_data.set"); ASSERT_EQ(skel->percpu->struct_data.nums[6], init_value.struct_data.nums[6], "skel->percpu->struct_data.nums[6]"); - ASSERT_GT(desired_sz, sizeof(init_value), "desired_sz"); - err = bpf_map__set_value_size(map, desired_sz); - if (!ASSERT_OK(err, "bpf_map__set_value_size")) - goto out; - if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "percpu value size")) - goto out; - if (!ASSERT_NEQ(bpf_map__btf_value_type_id(map), 0, "percpu BTF value type")) - goto out; - - init_data = bpf_map__initial_value(map, &init_data_sz); - if (!ASSERT_OK_PTR(init_data, "resized bpf_map__initial_value")) - goto out; - if (!ASSERT_EQ(init_data_sz, desired_sz, "resized initial value size")) - goto out; - if (!ASSERT_EQ(init_data->data, init_value.data, "resized initial value data")) - goto out; - err = test_global_percpu_data__load(skel); if (!ASSERT_OK(err, "test_global_percpu_data__load")) goto out; @@ -236,6 +223,67 @@ out: test_global_percpu_data_lskel__destroy(lskel); } +#define PCPU_MIN_UNIT_SIZE (32 << 10) + +static void test_global_percpu_data_map_resize(void) +{ + const __u32 page_size = sysconf(_SC_PAGE_SIZE); + const __u32 pcpu_max_value_size = ALIGN(PCPU_MIN_UNIT_SIZE, page_size); + const __u32 desired_sz = page_size * 2; + struct test_global_percpu_data *skel; + const int arr_value = -1; + struct bpf_map *map; + int err, prog_fd; + size_t data_sz; + + skel = test_global_percpu_data__open(); + if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) + return; + + map = skel->maps.percpu_arr; + skel->percpu_arr->arr[0] = arr_value; + err = bpf_map__set_value_size(map, desired_sz); + if (!ASSERT_OK(err, "bpf_map__set_value_size")) + goto out; + if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "bpf_map__value_size")) + goto out; + if (!ASSERT_NEQ(bpf_map__btf_value_type_id(map), 0, "bpf_map__btf_value_type_id")) + goto out; + + skel->percpu_arr = bpf_map__initial_value(map, &data_sz); + if (!ASSERT_EQ(data_sz, desired_sz, "data_sz")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu_arr, "skel->percpu_arr")) + goto out; + if (!ASSERT_EQ(skel->percpu_arr->arr[0], arr_value, "arr[0]")) + goto out; + + err = test_global_percpu_data__load(skel); + if (bpf_map__value_size(map) > pcpu_max_value_size) { + if (ASSERT_EQ(err, -E2BIG, "test_global_percpu_data__load E2BIG")) + /* + * When the PAGE_SIZE is 64KiB, the new value size will + * be 128KiB. Creating a percpu_array map with the new + * value size will fail with -E2BIG due to percpu_array + * map value size limit 64KiB, which is calculated by + * PFN_ALIGN(32KiB). + * Let's skip the test for such case. + */ + test__skip(); + goto out; + } + if (!ASSERT_OK(err, "test_global_percpu_data__load")) + goto out; + + map = skel->maps.percpu; + prog_fd = bpf_program__fd(skel->progs.update_percpu_data); + test_percpu_data_on_cpus(map, bpf_map__fd(map), prog_fd, &skel->bss->run); + ASSERT_EQ(skel->bss->arr_sum, arr_value * skel->bss->run, "arr_sum"); + +out: + test_global_percpu_data__destroy(skel); +} + static int create_rdonly_percpu_array(void) { LIBBPF_OPTS(bpf_map_create_opts, map_opts, @@ -388,6 +436,8 @@ void test_global_percpu_data(void) test_global_percpu_data_init(); if (test__start_subtest("lskel")) test_global_percpu_data_lskel(); + if (test__start_subtest("map_resize")) + test_global_percpu_data_map_resize(); if (test__start_subtest("rdonly_direct_read")) test_global_percpu_data_rdonly_direct_read(); if (test__start_subtest("rdonly_direct_write")) diff --git a/tools/testing/selftests/bpf/prog_tests/jit_insns.c b/tools/testing/selftests/bpf/prog_tests/jit_insns.c new file mode 100644 index 000000000000..dabefdb8a353 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/jit_insns.c @@ -0,0 +1,3155 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include <endian.h> +#include <stdint.h> +#include <test_progs.h> +#include <bpf/btf.h> +#include <linux/filter.h> + +#include "sysctl_helpers.h" +#include "jit_insns_arena.skel.h" + +#define NR_CASES 512 +#define CTX_SPILL -16 /* stack slot holding the ctx pointer */ +#define MEM_SLOT -8 /* stack slot for the atomic operations */ + +struct sweep_ctx { + __u64 in[NR_CASES][2]; + __u64 out[NR_CASES][4]; +}; + +#define IN_OFF(k, j) (offsetof(struct sweep_ctx, in) + ((k) * 2 + (j)) * 8) +#define OUT_OFF(k, j) (offsetof(struct sweep_ctx, out) + ((k) * 4 + (j)) * 8) + +/* The cases of one program: operands in, results out, expected results */ +struct batch { + struct bpf_insn *insns; + int cnt, cap; + struct sweep_ctx ctx; + __u64 exp[NR_CASES][4]; + __u8 nr_out[NR_CASES]; + int nr; + char desc[80]; +}; + +static char log[1 << 16]; + +#define EMIT(b, insn) ((b)->insns[(b)->cnt++] = (insn)) + +static void emit_ld_imm64(struct batch *b, int reg, __u64 imm) +{ + EMIT(b, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, reg, 0, 0, (__u32)imm)); + EMIT(b, BPF_RAW_INSN(0, 0, 0, 0, imm >> 32)); +} + +static void batch_reset(struct batch *b) +{ + b->cnt = 0; + b->nr = 0; + /* The ctx pointer lives on the stack, every register is an operand */ + EMIT(b, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, CTX_SPILL)); +} + +static bool batch_init(struct batch *b) +{ + b->cap = NR_CASES * 24 + 8; + b->insns = malloc(b->cap * sizeof(*b->insns)); + if (!ASSERT_OK_PTR(b->insns, "insns")) + return false; + batch_reset(b); + return true; +} + +static void batch_fini(struct batch *b) +{ + free(b->insns); +} + +static int load_prog(struct bpf_insn *insns, int cnt, const char *desc) +{ + LIBBPF_OPTS(bpf_prog_load_opts, lopts, + .prog_flags = BPF_F_SLEEPABLE, + .log_buf = log, + .log_size = sizeof(log), + ); + int fd; + + fd = bpf_prog_load(BPF_PROG_TYPE_SYSCALL, NULL, "GPL", insns, cnt, &lopts); + if (!ASSERT_GE(fd, 0, desc)) + fprintf(stderr, "%s", log); + return fd; +} + +/* Run the batch, check the results, start a new one */ +static void batch_flush(struct batch *b) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = &b->ctx, + .ctx_size_in = sizeof(b->ctx), + ); + char buf[160]; + int fd, k, j; + + if (!b->nr) + return; + EMIT(b, BPF_MOV64_IMM(BPF_REG_0, 0)); + EMIT(b, BPF_EXIT_INSN()); + snprintf(buf, sizeof(buf), "%s: load %d cases", b->desc, b->nr); + fd = load_prog(b->insns, b->cnt, buf); + if (fd >= 0 && ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), b->desc)) { + for (k = 0; k < b->nr; k++) { + for (j = 0; j < b->nr_out[k]; j++) { + if (b->ctx.out[k][j] == b->exp[k][j]) + continue; + snprintf(buf, sizeof(buf), "%s: in0=0x%llx in1=0x%llx out%d", + b->desc, b->ctx.in[k][0], b->ctx.in[k][1], j); + ASSERT_EQ(b->ctx.out[k][j], b->exp[k][j], buf); + } + } + } + if (fd >= 0) + close(fd); + batch_reset(b); +} + +/* Add a case: record the operands and load the ctx pointer into rc */ +static int batch_case(struct batch *b, int rc, __u64 in0, __u64 in1) +{ + int k; + + if (b->nr == NR_CASES) + batch_flush(b); + k = b->nr++; + b->ctx.in[k][0] = in0; + b->ctx.in[k][1] = in1; + b->nr_out[k] = 0; + EMIT(b, BPF_LDX_MEM(BPF_DW, rc, BPF_REG_10, CTX_SPILL)); + return k; +} + +static void batch_expect(struct batch *b, int k, int j, __u64 val) +{ + b->exp[k][j] = val; + if (b->nr_out[k] <= j) + b->nr_out[k] = j + 1; +} + +#define LD_IN(b, k, j, rc, reg) EMIT(b, BPF_LDX_MEM(BPF_DW, reg, rc, IN_OFF(k, j))) +#define ST_OUT(b, k, j, rc, reg) EMIT(b, BPF_STX_MEM(BPF_DW, rc, reg, OUT_OFF(k, j))) + +/* A register that is neither of the two operand registers nor r0 */ +static int scratch_reg(int r1, int r2) +{ + int r; + + for (r = BPF_REG_1; r <= BPF_REG_9; r++) + if (r != r1 && r != r2) + return r; + return BPF_REG_9; +} + +static int size_bits(int size) +{ + switch (size) { + case BPF_B: + return 8; + case BPF_H: + return 16; + case BPF_W: + return 32; + default: + return 64; + } +} + +/* The first size bytes of v in memory, as a load of that size reads them */ +static __u64 head_bytes(__u64 v, int size) +{ + int bits = size_bits(size); + + if (bits == 64) + return v; +#if __BYTE_ORDER == __BIG_ENDIAN + return v >> (64 - bits); +#else + return v & ((1ULL << bits) - 1); +#endif +} + +/* v with its first size bytes in memory replaced by those of w */ +static __u64 head_merge(__u64 v, __u64 w, int size) +{ + int bits = size_bits(size); + __u64 mask; + + if (bits == 64) + return w; + mask = (1ULL << bits) - 1; +#if __BYTE_ORDER == __BIG_ENDIAN + return (v & ~(mask << (64 - bits))) | ((w & mask) << (64 - bits)); +#else + return (v & ~mask) | (w & mask); +#endif +} + +/* v as a sign extending load of that size reads it */ +static __s64 sign_extend(__u64 v, int size) +{ + switch (size) { + case BPF_B: + return (__s8)v; + case BPF_H: + return (__s16)v; + case BPF_W: + return (__s32)v; + default: + return v; + } +} + +static const char *alu_name(int op) +{ + switch (BPF_OP(op)) { + case BPF_MOV: return "mov"; + case BPF_NEG: return "neg"; + case BPF_AND: return "and"; + case BPF_OR: return "or"; + case BPF_XOR: return "xor"; + case BPF_LSH: return "lsh"; + case BPF_RSH: return "rsh"; + case BPF_ARSH: return "arsh"; + case BPF_ADD: return "add"; + case BPF_SUB: return "sub"; + case BPF_MUL: return "mul"; + case BPF_DIV: return "div"; + case BPF_MOD: return "mod"; + case BPF_OP(BPF_XCHG): return "xchg"; + case BPF_OP(BPF_CMPXCHG): return "cmpxchg"; + default: return "?"; + } +} + +static const char *jmp_name(int op) +{ + switch (op) { + case BPF_JSET: return "jset"; + case BPF_JEQ: return "jeq"; + case BPF_JNE: return "jne"; + case BPF_JGT: return "jgt"; + case BPF_JGE: return "jge"; + case BPF_JLT: return "jlt"; + case BPF_JLE: return "jle"; + case BPF_JSGT: return "jsgt"; + case BPF_JSGE: return "jsge"; + case BPF_JSLT: return "jslt"; + case BPF_JSLE: return "jsle"; + default: return "?"; + } +} + +/* ALU result computation, with the ISA result for every operand */ +enum { F_ALU32 = 1, F_SIGNED = 2 }; + +static void alu_result(__u64 *res, __u64 v1, __u64 v2, __u8 op, __u32 flags) +{ + bool is_signed = flags & F_SIGNED; + int mask = flags & F_ALU32 ? 31 : 63; + + /* Narrow operands for ALU32 */ + if (flags & F_ALU32) { + if (is_signed) { + v1 = (__u64)(__s32)v1; + v2 = (__u64)(__s32)v2; + } else { + v1 = (__u32)v1; + v2 = (__u32)v2; + } + } + + *res = 0; + switch (op) { + case BPF_MOV: + *res = v2; + break; + case BPF_NEG: + *res = -v1; + break; + case BPF_AND: + *res = v1 & v2; + break; + case BPF_OR: + *res = v1 | v2; + break; + case BPF_XOR: + *res = v1 ^ v2; + break; + case BPF_LSH: + *res = v1 << (v2 & mask); + break; + case BPF_RSH: + *res = v1 >> (v2 & mask); + break; + case BPF_ARSH: + if (flags & F_ALU32) + v1 = (__u64)(__s32)v1; + v2 &= mask; + *res = v1 >> v2; + if (v2 > 0 && v1 > INT64_MAX) + *res |= ~0ULL << (64 - v2); + break; + case BPF_ADD: + *res = v1 + v2; + break; + case BPF_SUB: + *res = v1 - v2; + break; + case BPF_MUL: + *res = v1 * v2; + break; + /* By zero: the quotient is zero and the dividend keeps, by -1: negation and zero */ + case BPF_DIV: + if (v2 == 0) + *res = 0; + else if (!is_signed) + *res = v1 / v2; + else if ((__s64)v2 == -1) + *res = -v1; + else + *res = (__u64)((__s64)v1 / (__s64)v2); + break; + case BPF_MOD: + if (v2 == 0) + *res = v1; + else if (!is_signed) + *res = v1 % v2; + else if ((__s64)v2 == -1) + *res = 0; + else + *res = (__u64)((__s64)v1 % (__s64)v2); + break; + } +} + +/* + * Operand pattern generator for the power-of-two magnitude tests: all + * combinations of magnitudes and signs with a block of contiguous values + * around each magnitude, with every dstep'th magnitude of the destination, + * and a larger block for the same magnitude of both operands. The first + * pattern is where the immediates take every encoding a JIT has, the + * second where the sign and overflow boundaries of both operands meet. + */ +static __s64 value(int msb, int delta, int sign) +{ + return sign * (1LL << msb) + delta; +} + +#define PATTERN_BLOCK1 1 +#define PATTERN_BLOCK2 3 + +typedef void (*emit_fn)(struct batch *b, void *arg, __s64 dst, __s64 src); + +static void fill_pattern(struct batch *b, void *arg, int dbits, int sbits, + int block1, int dstep, int block2, emit_fn emit) +{ + static const int sgn[][2] = { { 1, 1 }, { 1, -1 }, { -1, 1 }, { -1, -1 } }; + int di, si, bt, db, sb, k; + + for (di = 0; di < dbits - 1; di += dstep) + for (si = 0; si < sbits - 1; si++) + for (k = 0; k < ARRAY_SIZE(sgn); k++) + for (db = -(block1 / 2); db < (block1 + 1) / 2; db++) + for (sb = -(block1 / 2); sb < (block1 + 1) / 2; sb++) + emit(b, arg, value(di, db, sgn[k][0]), + value(si, sb, sgn[k][1])); + + for (bt = 0; bt < (dbits > sbits ? dbits : sbits) - 1; bt++) + for (k = 0; k < ARRAY_SIZE(sgn); k++) + for (db = -(block2 / 2); db < (block2 + 1) / 2; db++) + for (sb = -(block2 / 2); sb < (block2 + 1) / 2; sb++) + emit(b, arg, value(bt % dbits, db, sgn[k][0]), + value(bt % sbits, sb, sgn[k][1])); + batch_flush(b); +} + +/* Operands at the edges: zero, one and the limits of every width */ +static const __u64 edges[] = { + 0, 1, 0x7f, 0x80, 0xff, 0x7fff, 0x8000, 0xffff, 0x7fffffff, 0x80000000, + 0xffffffff, 0x100000000ULL, 0x7fffffffffffffffULL, 0x8000000000000000ULL, + 0xffffffff80000000ULL, 0xffffffff7fffffffULL, 0xffffffffffffffffULL, +}; + +static void fill_edges(struct batch *b, void *arg, bool imm, emit_fn emit) +{ + int i, j; + + for (i = 0; i < ARRAY_SIZE(edges); i++) + for (j = 0; j < ARRAY_SIZE(edges); j++) { + /* The immediate has to fit its 32 bits */ + if (imm && (__s64)edges[j] != (__s32)edges[j]) + continue; + emit(b, arg, edges[i], edges[j]); + } + batch_flush(b); +} + +/* ALU operations with an immediate and with a register for all magnitudes */ +static const int alu_ops[] = { + BPF_MOV, BPF_AND, BPF_OR, BPF_XOR, BPF_ADD, BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD, +}; + +struct alu_arg { + int op; + __u32 flags; +}; + +static void emit_alu_imm(struct batch *b, void *arg, __s64 dst, __s64 imm) +{ + struct alu_arg *a = arg; + bool alu32 = a->flags & F_ALU32; + __u64 res; + int k; + + /* The verifier rejects a zero divisor and a shift count beyond the width */ + if ((a->op == BPF_DIV || a->op == BPF_MOD) && !imm) + return; + if ((a->op == BPF_LSH || a->op == BPF_RSH || a->op == BPF_ARSH) && + (__u64)imm >= (alu32 ? 32 : 64)) + return; + if (a->op == BPF_NEG) + imm = 0; + alu_result(&res, dst, (__s32)imm, a->op, a->flags); + k = batch_case(b, BPF_REG_3, dst, (__s32)imm); + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); + EMIT(b, BPF_RAW_INSN((alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(a->op) | BPF_K, + BPF_REG_1, 0, a->flags & F_SIGNED ? 1 : 0, imm)); + ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(b, k, 0, alu32 ? (__u32)res : res); +} + +static void emit_alu_reg(struct batch *b, void *arg, __s64 dst, __s64 src) +{ + struct alu_arg *a = arg; + bool alu32 = a->flags & F_ALU32; + __u64 res; + int k; + + /* Shift counts beyond the width have their own subtest */ + if ((a->op == BPF_LSH || a->op == BPF_RSH || a->op == BPF_ARSH) && + (alu32 ? (__u32)src : (__u64)src) >= (alu32 ? 32 : 64)) + return; + alu_result(&res, dst, src, a->op, a->flags); + k = batch_case(b, BPF_REG_3, dst, src); + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); + LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2); + EMIT(b, BPF_RAW_INSN((alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(a->op) | BPF_X, + BPF_REG_1, BPF_REG_2, a->flags & F_SIGNED ? 1 : 0, 0)); + ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(b, k, 0, alu32 ? (__u32)res : res); +} + +static void test_alu_magnitudes(bool reg) +{ + struct batch b; + int o, f; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(alu_ops); o++) { + struct alu_arg a = { alu_ops[o], f ? F_ALU32 : 0 }; + + snprintf(b.desc, sizeof(b.desc), "alu%d_%s_%c magnitudes", + f ? 32 : 64, alu_name(a.op), reg ? 'x' : 'k'); + if (reg) + fill_pattern(&b, &a, 64, 64, 0, 1, PATTERN_BLOCK2, emit_alu_reg); + else + fill_pattern(&b, &a, 64, 32, PATTERN_BLOCK1, 1, 5, emit_alu_imm); + } + for (o = 0; o < 2; o++) { + struct alu_arg a = { o ? BPF_MOD : BPF_DIV, F_SIGNED | (f ? F_ALU32 : 0) }; + + snprintf(b.desc, sizeof(b.desc), "alu%d_s%s_%c magnitudes", + f ? 32 : 64, alu_name(a.op), reg ? 'x' : 'k'); + if (reg) + fill_pattern(&b, &a, 64, 64, 0, 1, PATTERN_BLOCK2, emit_alu_reg); + else + fill_pattern(&b, &a, 64, 32, PATTERN_BLOCK1, 1, 5, emit_alu_imm); + } + } + batch_fini(&b); +} + +static void test_alu_imm(void) +{ + test_alu_magnitudes(false); +} + +static void test_alu_reg(void) +{ + test_alu_magnitudes(true); +} + +static void test_alu_edges(void) +{ + static const struct alu_arg ops[] = { + { BPF_MOV }, { BPF_AND }, { BPF_OR }, { BPF_XOR }, { BPF_LSH }, { BPF_RSH }, + { BPF_ARSH }, { BPF_ADD }, { BPF_SUB }, { BPF_MUL }, { BPF_DIV }, { BPF_MOD }, + { BPF_DIV, F_SIGNED }, { BPF_MOD, F_SIGNED }, { BPF_NEG }, + }; + struct batch b; + int o, f, r, i; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(ops); o++) { + struct alu_arg a = { ops[o].op, ops[o].flags | (f ? F_ALU32 : 0) }; + + for (r = 0; r < (a.op == BPF_NEG ? 1 : 2); r++) { + snprintf(b.desc, sizeof(b.desc), "alu%d_%s%s_%c edges", + f ? 32 : 64, a.flags & F_SIGNED ? "s" : "", alu_name(a.op), + r ? 'x' : 'k'); + if (a.op == BPF_NEG) { + for (i = 0; i < ARRAY_SIZE(edges); i++) + emit_alu_imm(&b, &a, edges[i], 0); + batch_flush(&b); + } else if (r) { + fill_edges(&b, &a, false, emit_alu_reg); + } else { + fill_edges(&b, &a, true, emit_alu_imm); + } + } + } + } + batch_fini(&b); +} + +/* ALU operations with an immediate for every destination register */ +static const int alu_reg_ops[] = { + BPF_MOV, BPF_AND, BPF_OR, BPF_XOR, BPF_LSH, BPF_RSH, BPF_ARSH, + BPF_ADD, BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD, +}; + +static void alu_imm_regs(struct batch *b, int op, __u32 flags) +{ + bool alu32 = flags & F_ALU32; + __u32 imm = 0x01234567U; + __u64 dst, res; + int rd, rc, k; + + if (flags & F_SIGNED) + dst = alu32 ? (__u64)-76543210LL : (__u64)-7654321076543210LL; + else + dst = alu32 ? 0x76543210ULL : 0x7edcba9876543210ULL; + if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) + imm &= 31; + if (op == BPF_NEG) + imm = 0; + alu_result(&res, dst, imm, op, flags); + if (alu32) + res = (__u32)res; + + snprintf(b->desc, sizeof(b->desc), "alu%d_%s%s_k registers", alu32 ? 32 : 64, + flags & F_SIGNED ? "s" : "", alu_name(op)); + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) { + rc = scratch_reg(rd, rd); + k = batch_case(b, rc, dst, 0); + LD_IN(b, k, 0, rc, rd); + EMIT(b, BPF_RAW_INSN((alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(op) | BPF_K, + rd, 0, flags & F_SIGNED ? 1 : 0, imm)); + ST_OUT(b, k, 0, rc, rd); + batch_expect(b, k, 0, res); + } + batch_flush(b); +} + +static void test_alu_imm_regs(void) +{ + struct batch b; + int o, f; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(alu_reg_ops); o++) + alu_imm_regs(&b, alu_reg_ops[o], f ? F_ALU32 : 0); + alu_imm_regs(&b, BPF_NEG, f ? F_ALU32 : 0); + alu_imm_regs(&b, BPF_DIV, F_SIGNED | (f ? F_ALU32 : 0)); + alu_imm_regs(&b, BPF_MOD, F_SIGNED | (f ? F_ALU32 : 0)); + } + batch_fini(&b); +} + +/* + * Division and modulo by constants for every width, signedness and + * destination register with a set of divisors and dividends around the + * multiple boundaries and the extremes of each range. This covers a JIT + * replacing the division with a multiplication by a magic number in all + * of its forms: magic numbers needing the extra add step, zero shifts, + * both signed corrections, and the divisors like 1, -1 and the ones + * above 2^63 that are still emitted as a real division. The dividends + * of the 32-bit operations carry garbage in the upper 32 bits. + */ +static const __s32 unsigned_divisors[] = { + 2, 3, 7, 10, 25, 641, 1000, 4096, 65536, 1000000007, 0x7fffffff, + -2, -10, -0x7fffffff - 1, -1, 1, +}; + +static const __s32 signed_divisors[] = { + 2, 3, 7, 25, 1000, 0x40000000, 0x7fffffff, + -2, -3, -5, -7, -1000, -0x7fffffff, -0x7fffffff - 1, 1, -1, +}; + +static int dividends(__u32 flags, __s32 imm, __u64 *n) +{ + bool alu32 = flags & F_ALU32, is_signed = flags & F_SIGNED; + __u64 d = is_signed ? (imm < 0 ? -(__s64)imm : imm) : + alu32 ? (__u32)imm : (__u64)(__s64)imm; + int cnt = 0, i; + + n[cnt++] = 0; + n[cnt++] = d - 1; + n[cnt++] = d + 1; + n[cnt++] = 3 * d + 2; + if (!alu32) { + n[cnt++] = (d << 40) - 1; + n[cnt++] = (d << 40) + 1; + n[cnt++] = (1ULL << 63) - 1; + n[cnt++] = 1ULL << 63; + n[cnt++] = ~0ULL; + n[cnt++] = 0xdaeb8ebd244a330cULL; + } else { + n[cnt++] = (d << 16) - 1; + n[cnt++] = (d << 16) + 1; + n[cnt++] = (1ULL << 31) - 1; + n[cnt++] = 1ULL << 31; + n[cnt++] = ~0U; + n[cnt++] = 0x8c1e86f1; + } + if (is_signed) { + n[cnt++] = -1; + n[cnt++] = -d - 1; + n[cnt++] = -3 * d - 2; + n[cnt++] = alu32 ? -(1ULL << 31) + 1 : -(1ULL << 63) + 1; + } + if (alu32) + for (i = 0; i < cnt; i++) + n[i] = (__u32)n[i] | ((0xa5a50000ULL + i) << 32); + return cnt; +} + +static __u64 div_result(int op, __u32 flags, __s32 imm, __u64 n) +{ + __u64 res; + + alu_result(&res, n, (__s32)imm, op, flags); + return flags & F_ALU32 ? (__u32)res : res; +} + +static void div_const(struct batch *b, int op, __u32 flags) +{ + bool alu32 = flags & F_ALU32, is_signed = flags & F_SIGNED; + const __s32 *divs = is_signed ? signed_divisors : unsigned_divisors; + int cnt = is_signed ? ARRAY_SIZE(signed_divisors) : ARRAY_SIZE(unsigned_divisors); + int code = (alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(op) | BPF_K; + int i, j, nr, rd, rc, k; + __u64 n[16]; + + snprintf(b->desc, sizeof(b->desc), "alu%d_%s%s_k constants", alu32 ? 32 : 64, + is_signed ? "s" : "", alu_name(op)); + for (i = 0; i < cnt; i++) { + nr = dividends(flags, divs[i], n); + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) { + rc = scratch_reg(rd, rd); + for (j = 0; j < nr; j++) { + k = batch_case(b, rc, n[j], divs[i]); + LD_IN(b, k, 0, rc, rd); + EMIT(b, BPF_RAW_INSN(code, rd, 0, is_signed, divs[i])); + ST_OUT(b, k, 0, rc, rd); + batch_expect(b, k, 0, div_result(op, flags, divs[i], n[j])); + } + } + } + batch_flush(b); +} + +static void test_div_const(void) +{ + struct batch b; + int f; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + div_const(&b, BPF_DIV, f ? F_ALU32 : 0); + div_const(&b, BPF_MOD, f ? F_ALU32 : 0); + div_const(&b, BPF_DIV, F_SIGNED | (f ? F_ALU32 : 0)); + div_const(&b, BPF_MOD, F_SIGNED | (f ? F_ALU32 : 0)); + } + batch_fini(&b); +} + +/* ALU operations for all combinations of operand registers */ +static void alu_reg_pairs(struct batch *b, int op, __u32 flags) +{ + bool alu32 = flags & F_ALU32, divmod = op == BPF_DIV || op == BPF_MOD; + int code = (alu32 ? BPF_ALU : BPF_ALU64) | BPF_OP(op) | BPF_X; + int off = flags & F_SIGNED ? 1 : 0, rd, rs, rc, k, n, nr = 1; + __u64 dst, srcs[3], src, res, same; + + if (flags & F_SIGNED) + dst = alu32 ? (__u64)-76543210LL : (__u64)-7654321076543210LL; + else + dst = alu32 ? 0x76543210ULL : 0x7edcba9876543210ULL; + srcs[0] = alu32 ? 0x01234567ULL : 0x0123456789abcdefULL; + if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) + srcs[0] &= 31; + /* A zero divisor, in 32 bits also one with only the upper half set */ + if (divmod) { + srcs[nr++] = 0; + if (alu32) + srcs[nr++] = 1ULL << 32; + } + + snprintf(b->desc, sizeof(b->desc), "alu%d_%s%s_x register pairs", alu32 ? 32 : 64, + flags & F_SIGNED ? "s" : "", alu_name(op)); + for (n = 0; n < nr; n++) { + src = srcs[n]; + alu_result(&res, dst, src, op, flags); + alu_result(&same, src, src, op, flags); + if (alu32) { + res = (__u32)res; + same = (__u32)same; + } + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) { + for (rs = BPF_REG_0; rs <= BPF_REG_9; rs++) { + rc = scratch_reg(rd, rs); + k = batch_case(b, rc, rd == rs ? src : dst, src); + LD_IN(b, k, 0, rc, rd); + if (rs != rd) + LD_IN(b, k, 1, rc, rs); + EMIT(b, BPF_RAW_INSN(code, rd, rs, off, 0)); + ST_OUT(b, k, 0, rc, rd); + batch_expect(b, k, 0, rd == rs ? same : res); + } + } + } + batch_flush(b); +} + +static void test_alu_reg_pairs(void) +{ + struct batch b; + int o, f; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(alu_reg_ops); o++) + alu_reg_pairs(&b, alu_reg_ops[o], f ? F_ALU32 : 0); + alu_reg_pairs(&b, BPF_DIV, F_SIGNED | (f ? F_ALU32 : 0)); + alu_reg_pairs(&b, BPF_MOD, F_SIGNED | (f ? F_ALU32 : 0)); + } + batch_fini(&b); +} + +static const __s64 shift_vals[] = { + 0x0123456789abcdefLL, /* dword > 0, word < 0 */ + 0xfedcba9876543210LL, /* dword < 0, word > 0 */ + 0xfedcba0198765432LL, /* dword < 0, word < 0 */ + 0x0123458967abcdefLL, /* dword > 0, word > 0 */ +}; +static const int shift_ops[] = { BPF_LSH, BPF_RSH, BPF_ARSH }; + +/* Shifts for all shift counts, by immediate, by register and by the shifted register */ +static void test_shifts(void) +{ + struct batch b; + int o, f, v, imm, k; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + int bits = f ? 32 : 64; + + for (o = 0; o < ARRAY_SIZE(shift_ops); o++) { + int op = shift_ops[o]; + + snprintf(b.desc, sizeof(b.desc), "alu%d_%s all shift counts", bits, + alu_name(op)); + for (v = 0; v < ARRAY_SIZE(shift_vals); v++) { + for (imm = 0; imm < bits; imm++) { + __u64 reg = shift_vals[v], res; + + if (f) + reg = op == BPF_ARSH ? (__u64)(__s32)reg : + (__u32)reg; + alu_result(&res, reg, imm, op, 0); + if (f) + res = (__u32)res; + + k = batch_case(&b, BPF_REG_3, shift_vals[v], imm); + LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1); + if (f) + EMIT(&b, BPF_ALU32_IMM(op, BPF_REG_1, imm)); + else + EMIT(&b, BPF_ALU64_IMM(op, BPF_REG_1, imm)); + ST_OUT(&b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(&b, k, 0, res); + + k = batch_case(&b, BPF_REG_3, shift_vals[v], imm); + LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1); + LD_IN(&b, k, 1, BPF_REG_3, BPF_REG_2); + if (f) + EMIT(&b, BPF_ALU32_REG(op, BPF_REG_1, BPF_REG_2)); + else + EMIT(&b, BPF_ALU64_REG(op, BPF_REG_1, BPF_REG_2)); + ST_OUT(&b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(&b, k, 0, res); + } + } + /* The count in the shifted register */ + for (imm = 0; imm < bits; imm++) { + __u64 res; + + alu_result(&res, imm, imm, op, 0); + k = batch_case(&b, BPF_REG_3, imm, 0); + LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1); + if (f) + EMIT(&b, BPF_ALU32_REG(op, BPF_REG_1, BPF_REG_1)); + else + EMIT(&b, BPF_ALU64_REG(op, BPF_REG_1, BPF_REG_1)); + ST_OUT(&b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(&b, k, 0, f ? (__u32)res : res); + } + batch_flush(&b); + } + } + batch_fini(&b); +} + +/* + * Shifts by register with counts at and above the width and negative ones, + * which the ISA masks by the width. Apart in a subtest of its own, as not + * every JIT does so. + */ +static void test_shift_counts(void) +{ + static const __s64 counts[] = { 32, 33, 63, 64, 65, 127, -1, -3, 1LL << 32 | 3 }; + struct batch b; + int o, f, v, c, k; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + int bits = f ? 32 : 64; + + for (o = 0; o < ARRAY_SIZE(shift_ops); o++) { + int op = shift_ops[o]; + + snprintf(b.desc, sizeof(b.desc), "alu%d_%s_x counts beyond the width", + bits, alu_name(op)); + for (c = 0; c < ARRAY_SIZE(counts); c++) { + __u64 cnt = f ? (__u32)counts[c] : counts[c]; + + for (v = 0; v < ARRAY_SIZE(shift_vals); v++) { + __u64 reg = shift_vals[v], res; + + if (f) + reg = op == BPF_ARSH ? (__u64)(__s32)reg : + (__u32)reg; + alu_result(&res, reg, cnt & (bits - 1), op, 0); + k = batch_case(&b, BPF_REG_3, shift_vals[v], counts[c]); + LD_IN(&b, k, 0, BPF_REG_3, BPF_REG_1); + LD_IN(&b, k, 1, BPF_REG_3, BPF_REG_2); + if (f) + EMIT(&b, BPF_ALU32_REG(op, BPF_REG_1, BPF_REG_2)); + else + EMIT(&b, BPF_ALU64_REG(op, BPF_REG_1, BPF_REG_2)); + ST_OUT(&b, k, 0, BPF_REG_3, BPF_REG_1); + batch_expect(&b, k, 0, f ? (__u32)res : res); + } + } + batch_flush(&b); + } + } + batch_fini(&b); +} + +/* Atomic operations on the stack for all magnitudes of memory and source operand */ +static const int atomic_ops[] = { + BPF_ADD, BPF_AND, BPF_OR, BPF_XOR, + BPF_ADD | BPF_FETCH, BPF_AND | BPF_FETCH, BPF_OR | BPF_FETCH, BPF_XOR | BPF_FETCH, + BPF_XCHG, +}; + +struct atomic_arg { + int op; + int size; +}; + +static void emit_atomic(struct batch *b, void *arg, __s64 dst, __s64 src) +{ + struct atomic_arg *a = arg; + bool w = a->size == BPF_W; + __u64 keep = 0x0123456789abcdefULL, res, fetch; + int k; + + if (a->op == BPF_XCHG) + res = src; + else + alu_result(&res, w ? (__u32)dst : dst, w ? (__u32)src : src, BPF_OP(a->op), 0); + /* The source register gets the old value with fetch, keeps its own without */ + fetch = a->op & BPF_FETCH ? (w ? (__u32)dst : dst) : src; + + k = batch_case(b, BPF_REG_3, dst, src); + /* The kept value fills the slot, a 32-bit operation leaves the other half alone */ + emit_ld_imm64(b, BPF_REG_0, keep); + EMIT(b, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, MEM_SLOT)); + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); + EMIT(b, BPF_STX_MEM(a->size, BPF_REG_10, BPF_REG_1, MEM_SLOT)); + LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2); + EMIT(b, BPF_ATOMIC_OP(a->size, a->op, BPF_REG_10, BPF_REG_2, MEM_SLOT)); + EMIT(b, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, MEM_SLOT)); + ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_1); + ST_OUT(b, k, 1, BPF_REG_3, BPF_REG_2); + ST_OUT(b, k, 2, BPF_REG_3, BPF_REG_0); + batch_expect(b, k, 0, head_merge(keep, w ? (__u32)res : res, a->size)); + batch_expect(b, k, 1, fetch); + batch_expect(b, k, 2, keep); +} + +static void emit_cmpxchg(struct batch *b, void *arg, __s64 dst, __s64 src) +{ + int size = *(int *)arg; + bool w = size == BPF_W; + __u64 old = w ? (__u32)dst : dst; + int k; + + k = batch_case(b, BPF_REG_3, dst, src); + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); + /* The inverted value fills the slot, a 32-bit operation leaves the other half alone */ + EMIT(b, BPF_MOV64_REG(BPF_REG_0, BPF_REG_1)); + EMIT(b, BPF_ALU64_IMM(BPF_XOR, BPF_REG_0, -1)); + EMIT(b, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, MEM_SLOT)); + EMIT(b, BPF_STX_MEM(size, BPF_REG_10, BPF_REG_1, MEM_SLOT)); + LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2); + /* Mismatch: the memory keeps its value, r0 gets it */ + EMIT(b, BPF_ATOMIC_OP(size, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, MEM_SLOT)); + ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_0); + /* + * Match: the memory gets the source, r0 the old value. r0 holds the + * full value, so the 32-bit compare has to ignore its upper half, and + * the source register has to keep its own. + */ + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_0); + EMIT(b, BPF_ATOMIC_OP(size, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, MEM_SLOT)); + EMIT(b, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, MEM_SLOT)); + ST_OUT(b, k, 1, BPF_REG_3, BPF_REG_1); + ST_OUT(b, k, 2, BPF_REG_3, BPF_REG_0); + ST_OUT(b, k, 3, BPF_REG_3, BPF_REG_2); + batch_expect(b, k, 0, old); + batch_expect(b, k, 1, head_merge(~(__u64)dst, w ? (__u32)src : src, size)); + batch_expect(b, k, 2, old); + batch_expect(b, k, 3, src); +} + +static void test_atomics(void) +{ + static const int sizes[] = { BPF_DW, BPF_W }; + struct batch b; + int o, s; + + if (!batch_init(&b)) + return; + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + int size = sizes[s]; + + for (o = 0; o < ARRAY_SIZE(atomic_ops); o++) { + struct atomic_arg a = { atomic_ops[o], size }; + + snprintf(b.desc, sizeof(b.desc), "atomic%d_%s%s magnitudes", + size == BPF_DW ? 64 : 32, alu_name(a.op), + a.op == BPF_XCHG ? "" : a.op & BPF_FETCH ? "_fetch" : ""); + fill_pattern(&b, &a, 64, 64, 0, 1, PATTERN_BLOCK2, emit_atomic); + } + snprintf(b.desc, sizeof(b.desc), "atomic%d_cmpxchg magnitudes", + size == BPF_DW ? 64 : 32); + fill_pattern(&b, &size, 64, 64, 0, 1, PATTERN_BLOCK2, emit_cmpxchg); + } + batch_fini(&b); +} + +/* Atomic operations for all combinations of pointer and source register */ +static void atomic_reg_pairs(struct batch *b, int size, int op) +{ + bool w = size == BPF_W; + __u64 mem, upd; + int rd, rs, rc, k, n; + + if (w) { + mem = 0xfedcba9801234567ULL; + upd = 0x76543210U; + } else { + mem = 0x0123456789abcdefULL; + upd = 0xfedcba9876543210ULL; + } + + snprintf(b->desc, sizeof(b->desc), "atomic%d_%s%s register pairs", w ? 32 : 64, + alu_name(op), op == BPF_XCHG || op == BPF_CMPXCHG ? "" : + op & BPF_FETCH ? "_fetch" : ""); + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) { + for (rs = BPF_REG_0; rs <= BPF_REG_9; rs++) { + __u64 regs[10], old = w ? (__u32)mem : mem, new, src, cmp; + + /* The source would be the pointer, r0 the compare value */ + if (rs == rd || (op == BPF_CMPXCHG && rd == BPF_REG_0)) + continue; + rc = scratch_reg(rd, rs); + k = batch_case(b, rc, mem, upd); + /* r0 inverted into the slot, memory from in0, source from in1, pointer */ + LD_IN(b, k, 0, rc, BPF_REG_0); + EMIT(b, BPF_ALU64_IMM(BPF_XOR, BPF_REG_0, -1)); + EMIT(b, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, MEM_SLOT)); + LD_IN(b, k, 0, rc, rs); + EMIT(b, BPF_STX_MEM(size, BPF_REG_10, rs, MEM_SLOT)); + LD_IN(b, k, 1, rc, rs); + EMIT(b, BPF_MOV64_REG(rd, BPF_REG_10)); + EMIT(b, BPF_ATOMIC_OP(size, op, rd, rs, MEM_SLOT)); + /* r0 holds the pointer when it is the destination */ + n = 0; + if (rd != BPF_REG_0) + ST_OUT(b, k, n++, rc, BPF_REG_0); + ST_OUT(b, k, n++, rc, rs); + EMIT(b, BPF_LDX_MEM(BPF_DW, rs, BPF_REG_10, MEM_SLOT)); + ST_OUT(b, k, n, rc, rs); + + /* Model with the register aliasing of r0 */ + regs[BPF_REG_0] = ~mem; + regs[rs] = upd; + src = w ? (__u32)regs[rs] : regs[rs]; + if (op == BPF_XCHG) { + new = src; + regs[rs] = old; + } else if (op == BPF_CMPXCHG) { + cmp = w ? (__u32)regs[BPF_REG_0] : regs[BPF_REG_0]; + new = cmp == old ? src : old; + regs[BPF_REG_0] = old; + } else { + alu_result(&new, old, src, BPF_OP(op), 0); + if (w) + new = (__u32)new; + if (op & BPF_FETCH) + regs[rs] = old; + } + n = 0; + if (rd != BPF_REG_0) + batch_expect(b, k, n++, regs[BPF_REG_0]); + batch_expect(b, k, n++, regs[rs]); + batch_expect(b, k, n, head_merge(~mem, new, size)); + } + } + batch_flush(b); +} + +static void test_atomic_reg_pairs(void) +{ + static const int sizes[] = { BPF_DW, BPF_W }; + struct batch b; + int o, s; + + if (!batch_init(&b)) + return; + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + for (o = 0; o < ARRAY_SIZE(atomic_ops); o++) + atomic_reg_pairs(&b, sizes[s], atomic_ops[o]); + atomic_reg_pairs(&b, sizes[s], BPF_CMPXCHG); + } + batch_fini(&b); +} + +/* A maximal chain of atomic adds on one stack slot */ +static void test_xadd_chain(void) +{ + static const int sizes[] = { BPF_DW, BPF_W }; + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct bpf_insn *insns; + int len = BPF_MAXINSNS, s, i, fd; + + insns = malloc(len * sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, "insns")) + return; + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + insns[0] = BPF_MOV32_IMM(BPF_REG_0, 1); + insns[1] = BPF_ST_MEM(sizes[s], BPF_REG_10, -40, 42); + for (i = 2; i < len - 2; i++) + insns[i] = BPF_STX_XADD(sizes[s], BPF_REG_10, BPF_REG_0, -40); + insns[len - 2] = BPF_LDX_MEM(sizes[s], BPF_REG_0, BPF_REG_10, -40); + insns[len - 1] = BPF_EXIT_INSN(); + + fd = load_prog(insns, len, "xadd chain"); + if (fd < 0) + break; + if (ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), "xadd chain run")) + ASSERT_EQ(opts.retval, 42 + len - 4, "xadd chain result"); + close(fd); + } + free(insns); +} + +/* 64-bit immediate loads for all magnitudes, byte patterns and registers */ +static void ld_imm64_case(struct batch *b, int rd, __u64 imm) +{ + int rc = scratch_reg(rd, rd); + int k = batch_case(b, rc, imm, 0); + + emit_ld_imm64(b, rd, imm); + ST_OUT(b, k, 0, rc, rd); + batch_expect(b, k, 0, imm); +} + +static void test_ld_imm64(void) +{ + static const struct { + __u8 base1, mask1, base2, mask2; + } bytes[] = { + { 0, 0xff, 0xff, 0xff }, /* checker */ + { 1, 0x81, 0x80, 0x80 }, /* positive and negative */ + { 1, 0x81, 0, 0xff }, /* positive and zero */ + { 0x80, 0x80, 0, 0xff }, /* negative and zero */ + }; + /* Zero, a 32-bit, a sign extended 32-bit and a full immediate for every register */ + static const __u64 imms[] = { + 0, 1, 0x7fffffff, 0x80000000, 0xffffffff, 0x100000000ULL, + 0xffffffffffffffffULL, 0xffffffff80000000ULL, 0xffffffff7fffffffULL, + 0x0123456789abcdefULL, + }; + int block = 64, bit, adj, sign, p, pattern, index, rd, i; + struct batch b; + + if (!batch_init(&b)) + return; + snprintf(b.desc, sizeof(b.desc), "ld_imm64 magnitudes"); + for (bit = 0; bit <= 62; bit++) + for (adj = -block / 2; adj < block / 2; adj++) + for (sign = -1; sign <= 1; sign += 2) + ld_imm64_case(&b, BPF_REG_1, sign * ((1LL << bit) + adj)); + batch_flush(&b); + + for (p = 0; p < ARRAY_SIZE(bytes); p++) { + __u32 rand = 1; + + snprintf(b.desc, sizeof(b.desc), "ld_imm64 byte pattern %d", p); + for (pattern = 0; pattern < 256; pattern++) { + __u64 imm = 0; + + for (index = 0; index < 8; index++) { + int byte; + + if (pattern & (1 << index)) + byte = (bytes[p].base1 & bytes[p].mask1) | + (rand & ~bytes[p].mask1); + else + byte = (bytes[p].base2 & bytes[p].mask2) | + (rand & ~bytes[p].mask2); + imm = (imm << 8) | (__u8)byte; + } + rand = rand * 1664525 + 1013904223; + ld_imm64_case(&b, BPF_REG_1, imm); + } + batch_flush(&b); + } + snprintf(b.desc, sizeof(b.desc), "ld_imm64 registers"); + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) + for (i = 0; i < ARRAY_SIZE(imms); i++) + ld_imm64_case(&b, rd, imms[i]); + batch_flush(&b); + batch_fini(&b); +} + +/* Conditional jumps with an immediate and with a register for all magnitudes */ +static const int jmp_ops[] = { + BPF_JSET, BPF_JEQ, BPF_JNE, BPF_JGT, BPF_JGE, BPF_JLT, BPF_JLE, + BPF_JSGT, BPF_JSGE, BPF_JSLT, BPF_JSLE, +}; + +static bool jmp_match(__s64 v1, __s64 v2, __u8 op) +{ + switch (op) { + case BPF_JSET: + return !!(v1 & v2); + case BPF_JEQ: + return v1 == v2; + case BPF_JNE: + return v1 != v2; + case BPF_JGT: + return (__u64)v1 > (__u64)v2; + case BPF_JGE: + return (__u64)v1 >= (__u64)v2; + case BPF_JLT: + return (__u64)v1 < (__u64)v2; + case BPF_JLE: + return (__u64)v1 <= (__u64)v2; + case BPF_JSGT: + return v1 > v2; + case BPF_JSGE: + return v1 >= v2; + case BPF_JSLT: + return v1 < v2; + case BPF_JSLE: + return v1 <= v2; + } + return false; +} + +struct jmp_arg { + int op; + bool is32; + bool reg; +}; + +static void emit_jmp(struct batch *b, void *arg, __s64 dst, __s64 src) +{ + struct jmp_arg *a = arg; + bool match; + int k; + + if (a->is32) + match = jmp_match((__s32)dst, (__s32)src, a->op); + else + match = jmp_match(dst, a->reg ? src : (__s32)src, a->op); + + k = batch_case(b, BPF_REG_3, dst, src); + LD_IN(b, k, 0, BPF_REG_3, BPF_REG_1); + if (a->reg) + LD_IN(b, k, 1, BPF_REG_3, BPF_REG_2); + EMIT(b, BPF_MOV64_IMM(BPF_REG_4, 1)); + if (a->reg && a->is32) + EMIT(b, BPF_JMP32_REG(a->op, BPF_REG_1, BPF_REG_2, 1)); + else if (a->reg) + EMIT(b, BPF_JMP_REG(a->op, BPF_REG_1, BPF_REG_2, 1)); + else if (a->is32) + EMIT(b, BPF_JMP32_IMM(a->op, BPF_REG_1, src, 1)); + else + EMIT(b, BPF_JMP_IMM(a->op, BPF_REG_1, src, 1)); + EMIT(b, BPF_MOV64_IMM(BPF_REG_4, 0)); + ST_OUT(b, k, 0, BPF_REG_3, BPF_REG_4); + batch_expect(b, k, 0, match); +} + +static void test_jmp_magnitudes(bool reg) +{ + struct batch b; + int o, f; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(jmp_ops); o++) { + struct jmp_arg a = { jmp_ops[o], f, reg }; + + snprintf(b.desc, sizeof(b.desc), "jmp%s_%s_%c magnitudes", f ? "32" : "", + jmp_name(a.op), reg ? 'x' : 'k'); + if (reg) + fill_pattern(&b, &a, 64, 64, 0, 1, PATTERN_BLOCK2, emit_jmp); + else + fill_pattern(&b, &a, 64, 32, PATTERN_BLOCK1, 8, PATTERN_BLOCK2, + emit_jmp); + } + } + batch_fini(&b); +} + +static void test_jmp_imm(void) +{ + test_jmp_magnitudes(false); +} + +static void test_jmp_reg(void) +{ + test_jmp_magnitudes(true); +} + +static void test_jmp_edges(void) +{ + struct batch b; + int o, f, r; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(jmp_ops); o++) { + for (r = 0; r < 2; r++) { + struct jmp_arg a = { jmp_ops[o], f, r }; + + snprintf(b.desc, sizeof(b.desc), "jmp%s_%s_%c edges", f ? "32" : "", + jmp_name(a.op), r ? 'x' : 'k'); + fill_edges(&b, &a, !r, emit_jmp); + } + } + } + batch_fini(&b); +} + +/* A conditional jump on two registers, with the operands swapped for n == 1 */ +static void jmp_reg_pair(struct batch *b, bool is32, int op, int rd, int rs, int n) +{ + const __u64 a = 0x7edcba9876543210ULL, c = 0x0123456789abcdefULL; + __u64 v1 = n ? c : a, v2 = n ? a : c; + int rc, rt, k; + bool match; + + /* Two scratch registers besides the operands */ + rc = scratch_reg(rd, rs); + for (rt = BPF_REG_1; rt < BPF_REG_9; rt++) + if (rt != rd && rt != rs && rt != rc) + break; + if (rd == rs) + v2 = v1; + match = is32 ? jmp_match((__s32)v1, (__s32)v2, op) : jmp_match(v1, v2, op); + + k = batch_case(b, rc, v1, v2); + LD_IN(b, k, 0, rc, rd); + if (rs != rd) + LD_IN(b, k, 1, rc, rs); + EMIT(b, BPF_MOV64_IMM(rt, 1)); + if (is32) + EMIT(b, BPF_JMP32_REG(op, rd, rs, 1)); + else + EMIT(b, BPF_JMP_REG(op, rd, rs, 1)); + EMIT(b, BPF_MOV64_IMM(rt, 0)); + ST_OUT(b, k, 0, rc, rt); + batch_expect(b, k, 0, match); +} + +/* Conditional jumps for all combinations of operand registers, taken and not */ +static void test_jmp_reg_pairs(void) +{ + struct batch b; + int o, f, rd, rs, n; + + if (!batch_init(&b)) + return; + for (f = 0; f < 2; f++) { + for (o = 0; o < ARRAY_SIZE(jmp_ops); o++) { + snprintf(b.desc, sizeof(b.desc), "jmp%s_%s_x register pairs", + f ? "32" : "", jmp_name(jmp_ops[o])); + for (rd = BPF_REG_0; rd <= BPF_REG_9; rd++) + for (rs = BPF_REG_0; rs <= BPF_REG_9; rs++) + for (n = 0; n < 2; n++) + jmp_reg_pair(&b, f, jmp_ops[o], rd, rs, n); + batch_flush(&b); + } + } + batch_fini(&b); +} + +/* The offset of a jump instruction, which the 32-bit unconditional jump keeps in imm */ +static void set_jmp_off(struct bpf_insn *insn, bool ja32, int off) +{ + if (ja32) + insn->imm = off; + else + insn->off = off; +} + +static struct bpf_insn jmp_insn(bool ja32) +{ + return ja32 ? BPF_RAW_INSN(BPF_JMP32 | BPF_JA, 0, 0, 0, 0) : BPF_JMP_IMM(BPF_JA, 0, 0, 0); +} + +/* + * Taken jumps over regions of doubling length, forward or backward, up to + * the maximal offset of the jump: 16 bits, or 2^17 for the 32-bit + * unconditional jump. The operands are reloaded in every region, so that + * the verifier cannot decide the jumps from the bounds of the region + * before and fold the regions away. r0 gains the region number before the + * jump and loses it again only on the target, so the not taken path of a + * conditional jump, the guard of an unconditional one and a jump landing + * short of its target, or forward past it, exit with a nonzero r0. A + * backward jump overshooting its target comes back to itself and hangs + * the run instead. + */ +static void staggered_jumps(const struct bpf_insn *jmp, __s64 r1, __s64 r2, bool backward, + const char *desc) +{ + bool ja32 = BPF_CLASS(jmp->code) == BPF_JMP32 && BPF_OP(jmp->code) == BPF_JA; + bool ja = BPF_OP(jmp->code) == BPF_JA; + struct { __u64 r1, r2; } ctx = { r1, r2 }; + LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = &ctx, + .ctx_size_in = sizeof(ctx), + ); + int max = ja32 ? 1 << 17 : INT16_MAX; + int len = 256 + 2 * max, i = 0, n = 0, gap, fd; + struct bpf_insn *insns; + + insns = malloc(len * sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, "insns")) + return; + insns[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1); + insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0); + for (gap = backward ? 4 : 2; gap <= max; gap = gap < max / 2 ? gap * 2 : max) { + int k, a, t, x, g, j; + + insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6, 0); + insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_6, 8); + insns[i++] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, ++n); + if (backward) { + /* + * Over the region to its guard and jump, which comes back + * to the start of the region, from where it leads on to + * the next one. + */ + a = i++; + t = i; + insns[i++] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_0, n); + for (k = 0; k < gap - 3 - ja; k++) + insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0); + x = i++; + g = i; + if (ja) + insns[i++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_2, 0, 1); + j = i; + insns[i] = *jmp; + set_jmp_off(&insns[i++], ja32, t - (j + 1)); + insns[i++] = BPF_EXIT_INSN(); + insns[a] = jmp_insn(ja32); + set_jmp_off(&insns[a], ja32, g - (a + 1)); + insns[x] = jmp_insn(ja32); + set_jmp_off(&insns[x], ja32, i - (x + 1)); + } else { + /* An unconditional jump needs a way into its region */ + if (ja) + insns[i++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_2, 0, 1); + insns[i] = *jmp; + set_jmp_off(&insns[i++], ja32, gap); + for (k = 0; k < gap - 1; k++) + insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_3, 0); + insns[i++] = BPF_EXIT_INSN(); + insns[i++] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_0, n); + } + if (gap == max) + break; + } + insns[i++] = BPF_EXIT_INSN(); + if (!ASSERT_LE(i, len, "insn count")) + goto out; + + fd = load_prog(insns, i, desc); + if (fd >= 0) { + if (ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), desc)) + ASSERT_EQ(opts.retval, 0, desc); + close(fd); + } +out: + free(insns); +} + +static void test_staggered_jumps(void) +{ + /* A representative set, with operands which make the jump taken */ + static const struct { + const char *desc; + struct bpf_insn jmp; + __s64 r1, r2; + } kinds[] = { + { "staggered ja", + BPF_JMP_IMM(BPF_JA, 0, 0, 0), 0, 1 }, + { "staggered gotol", + BPF_RAW_INSN(BPF_JMP32 | BPF_JA, 0, 0, 0, 0), 0, 1 }, + { "staggered jmp_jeq_k", + BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 1234, 0), 1234, 0 }, + { "staggered jmp_jne_x", + BPF_JMP_REG(BPF_JNE, BPF_REG_1, BPF_REG_2, 0), 4321, 1234 }, + { "staggered jmp_jset_k", + BPF_JMP_IMM(BPF_JSET, BPF_REG_1, 0x82, 0), 0x86, 0 }, + { "staggered jmp_jsgt_x", + BPF_JMP_REG(BPF_JSGT, BPF_REG_1, BPF_REG_2, 0), -1, -2 }, + { "staggered jmp32_jlt_k", + BPF_JMP32_IMM(BPF_JLT, BPF_REG_1, 0x80000000, 0), 1234, 0 }, + { "staggered jmp32_jsge_x", + BPF_JMP32_REG(BPF_JSGE, BPF_REG_1, BPF_REG_2, 0), -2, -2 }, + }; + char desc[64]; + int i, d; + + for (i = 0; i < ARRAY_SIZE(kinds); i++) { + for (d = 0; d < 2; d++) { + snprintf(desc, sizeof(desc), "%s %s", kinds[i].desc, + d ? "backward" : "forward"); + staggered_jumps(&kinds[i].jmp, kinds[i].r1, kinds[i].r2, d, desc); + } + } +} + +/* + * The pattern sweeps: one program per pattern and register choice, run + * once per input with the operands in the ctx and the result handed back + * through it. + */ +struct pat_ctx { + __u64 in0; + __u64 in1; + __u64 res; +}; + +#define IN0 offsetof(struct pat_ctx, in0) +#define IN1 offsetof(struct pat_ctx, in1) +#define RES offsetof(struct pat_ctx, res) + +/* r6 holds the ctx */ +static const int pat_regs[] = { + BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, + BPF_REG_5, BPF_REG_7, BPF_REG_8, BPF_REG_9, +}; + +/* The second register of a pattern: the same one, the next and a far one */ +#define NR_PAIRS 3 +static const int pair_off[NR_PAIRS] = { 0, 1, 5 }; + +static int pair_reg(int i, int k) +{ + return pat_regs[(i + pair_off[k]) % ARRAY_SIZE(pat_regs)]; +} + +static const __u64 inputs[] = { + 0, 0x1122334455667788ULL, 0xfedcba9876543210ULL, 0x80, 0xffffffffffffff7fULL, +}; + +struct prog { + struct bpf_insn insns[16]; + int cnt; +}; + +static void prog_start(struct prog *p) +{ + p->cnt = 0; + EMIT(p, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1)); +} + +/* Hand the result in reg back and return */ +static void prog_end(struct prog *p, int reg) +{ + EMIT(p, BPF_STX_MEM(BPF_DW, BPF_REG_6, reg, RES)); + EMIT(p, BPF_MOV64_IMM(BPF_REG_0, 0)); + EMIT(p, BPF_EXIT_INSN()); +} + +static int prog_load(struct prog *p, const char *desc) +{ + LIBBPF_OPTS(bpf_prog_load_opts, lopts, + .prog_flags = BPF_F_SLEEPABLE, + .log_buf = log, + .log_size = sizeof(log), + ); + int fd; + + fd = bpf_prog_load(BPF_PROG_TYPE_SYSCALL, NULL, "GPL", p->insns, p->cnt, &lopts); + if (!ASSERT_GE(fd, 0, desc)) + fprintf(stderr, "%s", log); + return fd; +} + +static void prog_run(int fd, __u64 in0, __u64 in1, __u64 expected, const char *desc) +{ + struct pat_ctx ctx = { .in0 = in0, .in1 = in1 }; + LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = &ctx, + .ctx_size_in = sizeof(ctx), + ); + char buf[128]; + + snprintf(buf, sizeof(buf), "%s in0=0x%llx in1=0x%llx", desc, in0, in1); + if (ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), buf)) + ASSERT_EQ(ctx.res, expected, buf); +} + +static void test_ldst_zero_off(void) +{ + static const int sizes[] = { BPF_B, BPF_H, BPF_W, BPF_DW }; + static const int aops[] = { + BPF_ADD, BPF_ADD | BPF_FETCH, BPF_OR, BPF_XCHG, BPF_CMPXCHG, + }; + static const __s32 imms[] = { 0x5a, -1 }; + struct prog p; + char desc[64]; + int b, k, s, j, fd; + + for (b = 0; b < ARRAY_SIZE(pat_regs); b++) { + int rb = pat_regs[b]; + + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + int size = sizes[s]; + + for (k = 0; k < NR_PAIRS; k++) { + int rd = pair_reg(b, k); + + /* Load with a zero offset over all ones, also into the base */ + prog_start(&p); + EMIT(&p, BPF_MOV64_IMM(rd, -1)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_6)); + EMIT(&p, BPF_LDX_MEM(size, rd, rb, 0)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "ldx size=%d base=r%d dst=r%d", + size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) + prog_run(fd, inputs[j], 0, + head_bytes(inputs[j], size), desc); + close(fd); + + /* Sign extending load with a zero offset from the stack */ + if (size != BPF_DW) { + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, rd, -8)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, rb, -8)); + EMIT(&p, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | size, + rd, rb, 0, 0)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), + "ldsx size=%d base=r%d dst=r%d", size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __s64 v = head_bytes(inputs[j], size); + + v = sign_extend(v, size); + prog_run(fd, inputs[j], 0, v, desc); + } + close(fd); + } + + if (rd == rb) + continue; + /* Store with a zero offset over in0 */ + prog_start(&p); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_6)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN1)); + EMIT(&p, BPF_STX_MEM(size, rb, rd, 0)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "stx size=%d base=r%d src=r%d", + size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 in1 = inputs[(j + 1) % ARRAY_SIZE(inputs)]; + + prog_run(fd, inputs[j], in1, + head_merge(inputs[j], in1, size), desc); + } + close(fd); + } + + /* Store an immediate with a zero offset into a zeroed stack slot */ + for (k = 0; k < ARRAY_SIZE(imms); k++) { + int rd = pair_reg(b, 1); + + prog_start(&p); + EMIT(&p, BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, rb, -8)); + EMIT(&p, BPF_ST_MEM(size, rb, 0, imms[k])); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_10, -8)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "st size=%d base=r%d imm=%d", + size, rb, imms[k]); + fd = prog_load(&p, desc); + if (fd < 0) + return; + prog_run(fd, 0, 0, + head_merge(0, (__u64)(__s64)imms[k], size), desc); + close(fd); + } + + /* Atomics with a zero offset on a stack slot holding in0 */ + for (k = 0; k < ARRAY_SIZE(aops); k++) { + int rd = pair_reg(b, 1), op = aops[k]; + + if (size == BPF_B || size == BPF_H) + break; + /* cmpxchg compares with r0 */ + if (op == BPF_CMPXCHG && (rb == BPF_REG_0 || rd == BPF_REG_0)) + continue; + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, rd, -8)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN1)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, rb, -8)); + if (op == BPF_CMPXCHG) + EMIT(&p, BPF_LDX_MEM(size, BPF_REG_0, BPF_REG_10, -8)); + EMIT(&p, BPF_ATOMIC_OP(size, op, rb, rd, 0)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_10, -8)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "atomic 0x%x size=%d base=r%d src=r%d", + op, size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 in1 = inputs[(j + 1) % ARRAY_SIZE(inputs)]; + __u64 v = head_bytes(inputs[j], size); + /* The source is the low half, whichever the byte order */ + __u64 w = size == BPF_W ? (__u32)in1 : in1; + __u64 r = BPF_OP(op) == BPF_ADD ? v + w : + BPF_OP(op) == BPF_OR ? v | w : w; + + prog_run(fd, inputs[j], in1, + head_merge(inputs[j], r, size), desc); + } + close(fd); + } + } + } +} + +/* + * Loads, stores and stores of an immediate with the offsets around the + * limits of the short displacement forms, negative, large and unaligned + * ones, with every register as base. The memory is an array map value with + * the base register pointing at its middle, the loads hand their results + * back through a second value and the stores are checked against a copy + * kept here. The unaligned offsets are left out where the verifier insists + * on aligned accesses. + */ +#define MEM_SIZE 65536 +#define MEM_BASE (MEM_SIZE / 2) +#define MEM_KEY 0 +#define RES_KEY 1 +#define SLOT_KEY -4 /* stack slots: map key, results and base pointer */ +#define SLOT_RES -16 +#define SLOT_BASE -24 +#define STORE_VAL 0x8899aabbccddeeffULL + +struct off_prog { + struct bpf_insn insns[128]; + int cnt; +}; + +/* The map holding the memory and the results, a copy of the memory and a buffer */ +struct off_mem { + int map_fd; + __u8 *mem, *exp, *res; +}; + +enum { K_LDX, K_LDSX, K_STX, K_ST }; + +/* The offsets for an access of size, aligned unless unaligned ones are ok */ +static int offsets_for(int size, bool unaligned, int *off) +{ + int n = size_bits(size) / 8, cnt = 0; + + off[cnt++] = n; + off[cnt++] = 128 - n; + off[cnt++] = 128; + off[cnt++] = 256 - n; + off[cnt++] = 256; + off[cnt++] = 4096; + off[cnt++] = MEM_BASE - n; + off[cnt++] = -n; + off[cnt++] = -128; + off[cnt++] = -128 - n; + off[cnt++] = -256; + off[cnt++] = -4096; + off[cnt++] = -MEM_BASE; + if (n > 1 && unaligned) { + off[cnt++] = 13; + off[cnt++] = 127; + off[cnt++] = -13; + off[cnt++] = -129; + } + return cnt; +} + +/* The bytes a store of size writes from the register value v */ +static void store_bytes(__u8 *dst, __u64 v, int size) +{ + int n = size_bits(size) / 8; + +#if __BYTE_ORDER == __BIG_ENDIAN + memcpy(dst, (__u8 *)&v + 8 - n, n); +#else + memcpy(dst, &v, n); +#endif +} + +/* The value a load of size reads from the bytes at src */ +static __u64 load_bytes(const __u8 *src, int size) +{ + __u64 v = 0; + + memcpy(&v, src, size_bits(size) / 8); + return head_bytes(v, size); +} + +/* A register that is neither of the two given ones */ +static int other_reg(int r1, int r2) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(pat_regs); i++) + if (pat_regs[i] != r1 && pat_regs[i] != r2) + return pat_regs[i]; + return BPF_REG_9; +} + +static void off_prog_ld_imm64(struct off_prog *p, int reg, __u64 imm) +{ + EMIT(p, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, reg, 0, 0, (__u32)imm)); + EMIT(p, BPF_RAW_INSN(0, 0, 0, 0, imm >> 32)); +} + +/* Look up the value of key into r0, exit with 1 when the lookup fails */ +static void off_prog_lookup(struct off_prog *p, int map_fd, int key) +{ + EMIT(p, BPF_ST_MEM(BPF_W, BPF_REG_10, SLOT_KEY, key)); + EMIT(p, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, BPF_REG_1, BPF_PSEUDO_MAP_FD, 0, map_fd)); + EMIT(p, BPF_RAW_INSN(0, 0, 0, 0, 0)); + EMIT(p, BPF_MOV64_REG(BPF_REG_2, BPF_REG_10)); + EMIT(p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, SLOT_KEY)); + EMIT(p, BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem)); + EMIT(p, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2)); + EMIT(p, BPF_MOV64_IMM(BPF_REG_0, 1)); + EMIT(p, BPF_EXIT_INSN()); +} + +/* rb points at the middle of the memory, the results value is spilled */ +static void off_prog_start(struct off_prog *p, int map_fd, int rb) +{ + p->cnt = 0; + off_prog_lookup(p, map_fd, RES_KEY); + EMIT(p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, SLOT_RES)); + off_prog_lookup(p, map_fd, MEM_KEY); + EMIT(p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, MEM_BASE)); + EMIT(p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, SLOT_BASE)); + if (rb != BPF_REG_0) + EMIT(p, BPF_MOV64_REG(rb, BPF_REG_0)); +} + +static int off_prog_load(struct off_prog *p, const char *desc, bool quiet) +{ + LIBBPF_OPTS(bpf_prog_load_opts, lopts, + .prog_flags = BPF_F_SLEEPABLE, + .log_buf = log, + .log_size = sizeof(log), + ); + int fd; + + EMIT(p, BPF_MOV64_IMM(BPF_REG_0, 0)); + EMIT(p, BPF_EXIT_INSN()); + fd = bpf_prog_load(BPF_PROG_TYPE_SYSCALL, NULL, "GPL", p->insns, p->cnt, &lopts); + if (quiet) + return fd; + if (!ASSERT_GE(fd, 0, desc)) + fprintf(stderr, "%s", log); + return fd; +} + +static int off_prog_run(int fd, const char *desc) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int err; + + err = bpf_prog_test_run_opts(fd, &opts); + close(fd); + if (!ASSERT_OK(err, desc) || !ASSERT_EQ(opts.retval, 0, desc)) + return -1; + return 0; +} + +/* Whether the verifier accepts a halfword load from an odd offset */ +static bool unaligned_ok(int map_fd) +{ + struct off_prog p; + int fd; + + off_prog_start(&p, map_fd, BPF_REG_1); + EMIT(&p, BPF_LDX_MEM(BPF_H, BPF_REG_0, BPF_REG_1, 1)); + fd = off_prog_load(&p, "unaligned probe", true); + if (fd < 0) + return false; + close(fd); + return true; +} + +/* One program loading from all the offsets of size through rb into rd */ +static void load_offsets(struct off_mem *m, int kind, int size, int rb, int rd, + const int *off, int nr) +{ + __u32 mem_key = MEM_KEY, res_key = RES_KEY; + int rs = other_reg(rb, rd), fd, i; + __u64 *res = (__u64 *)m->res; + int map_fd = m->map_fd; + struct off_prog p; + char desc[64]; + + off_prog_start(&p, map_fd, rb); + for (i = 0; i < nr; i++) { + /* The base gets reloaded when it is the destination, all ones otherwise */ + if (rd == rb) + EMIT(&p, BPF_LDX_MEM(BPF_DW, rb, BPF_REG_10, SLOT_BASE)); + else + EMIT(&p, BPF_MOV64_IMM(rd, -1)); + if (kind == K_LDX) + EMIT(&p, BPF_LDX_MEM(size, rd, rb, off[i])); + else + EMIT(&p, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | size, rd, rb, off[i], 0)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rs, BPF_REG_10, SLOT_RES)); + EMIT(&p, BPF_STX_MEM(BPF_DW, rs, rd, i * 8)); + } + snprintf(desc, sizeof(desc), "%s size=%d base=r%d dst=r%d", + kind == K_LDX ? "ldx" : "ldsx", size, rb, rd); + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &mem_key, m->mem, 0), desc)) + return; + fd = off_prog_load(&p, desc, false); + if (fd < 0) + return; + if (off_prog_run(fd, desc) || !ASSERT_OK(bpf_map_lookup_elem(map_fd, &res_key, res), desc)) + return; + for (i = 0; i < nr; i++) { + __u64 v = load_bytes(m->mem + MEM_BASE + off[i], size); + + if (kind == K_LDSX) + v = sign_extend(v, size); + if (res[i] != v) { + snprintf(desc, sizeof(desc), "%s size=%d base=r%d dst=r%d off=%d", + kind == K_LDX ? "ldx" : "ldsx", size, rb, rd, off[i]); + ASSERT_EQ(res[i], v, desc); + } + } +} + +/* One program storing rd, or the immediate imm when rd < 0, at all the offsets */ +static void store_offsets(struct off_mem *m, int size, int rb, int rd, __s32 imm, + const int *off, int nr) +{ + __u64 val = rd >= 0 ? STORE_VAL : (__u64)(__s64)imm; + __u8 *mem = m->mem, *exp = m->exp, *res = m->res; + __u32 mem_key = MEM_KEY; + int map_fd = m->map_fd; + char desc[64], buf[96]; + struct off_prog p; + int fd, i; + + off_prog_start(&p, map_fd, rb); + if (rd >= 0) { + EMIT(&p, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, rd, 0, 0, (__u32)STORE_VAL)); + EMIT(&p, BPF_RAW_INSN(0, 0, 0, 0, STORE_VAL >> 32)); + } + memcpy(exp, mem, MEM_SIZE); + for (i = 0; i < nr; i++) { + if (rd >= 0) + EMIT(&p, BPF_STX_MEM(size, rb, rd, off[i])); + else + EMIT(&p, BPF_ST_MEM(size, rb, off[i], imm)); + store_bytes(exp + MEM_BASE + off[i], val, size); + } + if (rd >= 0) + snprintf(desc, sizeof(desc), "stx size=%d base=r%d src=r%d", size, rb, rd); + else + snprintf(desc, sizeof(desc), "st size=%d base=r%d imm=%d", size, rb, imm); + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &mem_key, mem, 0), desc)) + return; + fd = off_prog_load(&p, desc, false); + if (fd < 0) + return; + if (off_prog_run(fd, desc) || !ASSERT_OK(bpf_map_lookup_elem(map_fd, &mem_key, res), desc)) + return; + for (i = 0; i < MEM_SIZE && res[i] == exp[i]; i++) + ; + if (i < MEM_SIZE) { + snprintf(buf, sizeof(buf), "%s: byte at offset %d", desc, i - MEM_BASE); + ASSERT_EQ(res[i], exp[i], buf); + } +} + +static void test_ldst_offsets(void) +{ + static const int sizes[] = { BPF_B, BPF_H, BPF_W, BPF_DW }; + static const __s32 imms[] = { -1, 0x7fffffff, -0x7fffffff - 1 }; + int b, k, s, i, nr, off[32]; + struct off_mem m = {}; + bool unaligned; + + m.map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "ldst_mem", sizeof(__u32), MEM_SIZE, 2, NULL); + if (!ASSERT_GE(m.map_fd, 0, "ldst_mem map")) + return; + m.mem = malloc(MEM_SIZE); + m.exp = malloc(MEM_SIZE); + m.res = malloc(MEM_SIZE); + if (!ASSERT_OK_PTR(m.mem, "mem") || !ASSERT_OK_PTR(m.exp, "exp") || + !ASSERT_OK_PTR(m.res, "res")) + goto out; + for (i = 0; i < MEM_SIZE; i++) + m.mem[i] = (i * 7 + 3) ^ (i >> 8); + unaligned = unaligned_ok(m.map_fd); + + for (b = 0; b < ARRAY_SIZE(pat_regs); b++) { + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + int rb = pat_regs[b], size = sizes[s]; + + nr = offsets_for(size, unaligned, off); + /* Loads and sign extending loads, also into the base */ + for (k = 0; k < NR_PAIRS; k++) { + int rd = pair_reg(b, k); + + load_offsets(&m, K_LDX, size, rb, rd, off, nr); + if (size != BPF_DW) + load_offsets(&m, K_LDSX, size, rb, rd, off, nr); + } + /* Stores of a register and of an immediate */ + for (k = 1; k < NR_PAIRS; k++) + store_offsets(&m, size, rb, pair_reg(b, k), 0, off, nr); + for (k = 0; k < ARRAY_SIZE(imms); k++) + store_offsets(&m, size, rb, -1, imms[k], off, nr); + } + } +out: + free(m.mem); + free(m.exp); + free(m.res); + close(m.map_fd); +} + +static void test_movsx(void) +{ + static const struct { + bool is64; + int off; + } ops[] = { + { true, 8 }, { true, 16 }, { true, 32 }, { false, 8 }, { false, 16 }, + }; + struct prog p; + char desc[64]; + int s, k, o, j, fd; + + for (s = 0; s < ARRAY_SIZE(pat_regs); s++) { + int rs = pat_regs[s]; + + for (k = 0; k < NR_PAIRS; k++) { + int rd = pair_reg(s, k); + + for (o = 0; o < ARRAY_SIZE(ops); o++) { + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rs, BPF_REG_6, IN0)); + if (ops[o].is64) + EMIT(&p, BPF_MOVSX64_REG(rd, rs, ops[o].off)); + else + EMIT(&p, BPF_MOVSX32_REG(rd, rs, ops[o].off)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "movsx%d off=%d dst=r%d src=r%d", + ops[o].is64 ? 64 : 32, ops[o].off, rd, rs); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __s64 v = inputs[j]; + + if (ops[o].off == 8) + v = (__s8)v; + else if (ops[o].off == 16) + v = (__s16)v; + else + v = (__s32)v; + prog_run(fd, inputs[j], 0, + ops[o].is64 ? (__u64)v : (__u32)v, desc); + } + close(fd); + } + } + } +} + +/* Branch on dst and imm, with 1 in res when taken and 0 otherwise */ +static void jmp_prog(struct prog *p, bool is64, int op, int d, __s32 imm, int off) +{ + int rd = pat_regs[d], rt = pair_reg(d, 1); + + prog_start(p); + EMIT(p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + EMIT(p, BPF_MOV64_IMM(rt, 1)); + if (is64) + EMIT(p, BPF_JMP_IMM(op, rd, imm, off)); + else + EMIT(p, BPF_JMP32_IMM(op, rd, imm, off)); + EMIT(p, BPF_MOV64_IMM(rt, 0)); + prog_end(p, rt); +} + +static void test_jset_imm(void) +{ + static const __s32 masks[] = { + 0x1, 0x40, 0x80, 0xff, 0x100, 0x7fff, 0x7fffffff, -1, -0x7fffffff - 1, + }; + struct prog p; + char desc[64]; + int d, m, j, fd; + bool is64; + + for (d = 0; d < ARRAY_SIZE(pat_regs); d++) { + int rd = pat_regs[d]; + + for (is64 = false; ; is64 = true) { + for (m = 0; m < ARRAY_SIZE(masks); m++) { + jmp_prog(&p, is64, BPF_JSET, d, masks[m], 1); + snprintf(desc, sizeof(desc), "jset%d dst=r%d mask=0x%x", + is64 ? 64 : 32, rd, masks[m]); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 v = inputs[j]; + bool taken; + + if (is64) + taken = v & (__u64)(__s64)masks[m]; + else + taken = (__u32)v & (__u32)masks[m]; + prog_run(fd, v, 0, taken, desc); + } + close(fd); + } + if (is64) + break; + } + } +} + +static bool cmp_taken(int op, bool is64, __u64 v, __u64 c) +{ + __s64 sv = is64 ? (__s64)v : (__s32)v, sc = is64 ? (__s64)c : (__s32)c; + + switch (op) { + case BPF_JEQ: + return v == c; + case BPF_JNE: + return v != c; + case BPF_JGT: + return v > c; + case BPF_JGE: + return v >= c; + case BPF_JLT: + return v < c; + case BPF_JLE: + return v <= c; + case BPF_JSGT: + return sv > sc; + case BPF_JSGE: + return sv >= sc; + case BPF_JSLT: + return sv < sc; + default: + return sv <= sc; + } +} + +static void test_cmp_imm(void) +{ + static const __s32 imms[] = { + 1, -1, 0x7f, 0x80, -0x80, -0x81, 0x1000, 0x7fffffff, + }; + static const int ops[] = { + BPF_JEQ, BPF_JNE, BPF_JGT, BPF_JGE, BPF_JLT, BPF_JLE, + BPF_JSGT, BPF_JSGE, BPF_JSLT, BPF_JSLE, + }; + struct prog p; + char desc[64]; + int d, o, m, j, fd; + bool is64; + + for (d = 0; d < ARRAY_SIZE(pat_regs); d++) { + int rd = pat_regs[d]; + + for (is64 = false; ; is64 = true) { + for (o = 0; o < ARRAY_SIZE(ops); o++) { + for (m = 0; m < ARRAY_SIZE(imms); m++) { + jmp_prog(&p, is64, ops[o], d, imms[m], 1); + snprintf(desc, sizeof(desc), "jmp%d op=0x%x dst=r%d imm=%d", + is64 ? 64 : 32, ops[o], rd, imms[m]); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 v = is64 ? inputs[j] : (__u32)inputs[j]; + __u64 c = is64 ? (__u64)(__s64)imms[m] : + (__u32)imms[m]; + __u64 taken = cmp_taken(ops[o], is64, v, c); + + prog_run(fd, inputs[j], 0, taken, desc); + } + close(fd); + if (m) + continue; + /* A jump over nothing lands on the same path either way */ + jmp_prog(&p, is64, ops[o], d, imms[m], 0); + snprintf(desc, sizeof(desc), "jmp%d op=0x%x dst=r%d off=0", + is64 ? 64 : 32, ops[o], rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) + prog_run(fd, inputs[j], 0, 0, desc); + close(fd); + } + } + if (is64) + break; + } + } +} + +static __u64 alu_model(int op, __u64 v, __u64 c) +{ + switch (op) { + case BPF_ADD: + return v + c; + case BPF_SUB: + return v - c; + case BPF_AND: + return v & c; + case BPF_OR: + return v | c; + case BPF_XOR: + return v ^ c; + default: + return v * c; + } +} + +static void test_alu_consts(void) +{ + static const struct { + int op; + __s32 imm; + } ops[] = { + { BPF_ADD, 1 }, { BPF_ADD, -1 }, { BPF_SUB, 1 }, { BPF_SUB, -1 }, + { BPF_ADD, 128 }, { BPF_SUB, 128 }, { BPF_ADD, -128 }, + { BPF_AND, 0xff }, { BPF_AND, 0xffff }, { BPF_AND, 0x7f }, + { BPF_AND, 0x3ff }, { BPF_AND, 0x7fffffff }, + { BPF_OR, 0xff }, { BPF_XOR, -1 }, { BPF_XOR, 0x80 }, + { BPF_MUL, 2 }, { BPF_MUL, 3 }, { BPF_MUL, 4 }, { BPF_MUL, 5 }, + { BPF_MUL, 7 }, { BPF_MUL, 8 }, { BPF_MUL, 9 }, { BPF_MUL, 16 }, + { BPF_MUL, 0x40000000 }, { BPF_MUL, -3 }, { BPF_MUL, -0x7fffffff - 1 }, + { BPF_MUL, 0 }, { BPF_MUL, 1 }, { BPF_MUL, -1 }, + }; + struct prog p; + char desc[64]; + int d, o, j, fd; + bool is64; + + for (d = 0; d < ARRAY_SIZE(pat_regs); d++) { + int rd = pat_regs[d]; + + for (is64 = false; ; is64 = true) { + for (o = 0; o < ARRAY_SIZE(ops); o++) { + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + if (is64) + EMIT(&p, BPF_ALU64_IMM(ops[o].op, rd, ops[o].imm)); + else + EMIT(&p, BPF_ALU32_IMM(ops[o].op, rd, ops[o].imm)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "alu%d op=0x%x dst=r%d imm=%d", + is64 ? 64 : 32, ops[o].op, rd, ops[o].imm); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 v = is64 ? inputs[j] : (__u32)inputs[j]; + __u64 c = is64 ? (__u64)(__s64)ops[o].imm : + (__u32)ops[o].imm; + __u64 r = alu_model(ops[o].op, v, c); + + prog_run(fd, inputs[j], 0, is64 ? r : (__u32)r, desc); + } + close(fd); + } + if (is64) + break; + } + } +} + +static void test_bswap(void) +{ + static const int widths[] = { 16, 32, 64 }; + struct prog p; + char desc[64]; + int d, w, o, j, fd; + + for (d = 0; d < ARRAY_SIZE(pat_regs); d++) { + int rd = pat_regs[d]; + + for (w = 0; w < ARRAY_SIZE(widths); w++) { + int bits = widths[w]; + + /* To big endian, to little endian, and the unconditional swap */ + for (o = 0; o < 3; o++) { + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + if (o == 0) + EMIT(&p, BPF_ENDIAN(BPF_TO_BE, rd, bits)); + else if (o == 1) + EMIT(&p, BPF_ENDIAN(BPF_TO_LE, rd, bits)); + else + EMIT(&p, BPF_RAW_INSN(BPF_ALU64 | BPF_END | BPF_TO_LE, + rd, 0, 0, bits)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "%s%d dst=r%d", + o == 0 ? "be" : o == 1 ? "le" : "bswap", bits, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 v = inputs[j], r; + + if (bits == 16) + r = o == 0 ? htobe16(v) : o == 1 ? htole16(v) : + __builtin_bswap16(v); + else if (bits == 32) + r = o == 0 ? htobe32(v) : o == 1 ? htole32(v) : + __builtin_bswap32(v); + else + r = o == 0 ? htobe64(v) : o == 1 ? htole64(v) : + __builtin_bswap64(v); + prog_run(fd, v, 0, r, desc); + } + close(fd); + } + } + } +} + +static void test_probe_mem(void) +{ + static const struct { + int size; + bool sx; + } loads[] = { + { BPF_B, false }, { BPF_H, false }, { BPF_W, false }, { BPF_DW, false }, + { BPF_B, true }, { BPF_H, true }, { BPF_W, true }, + }; + int kfunc_id, type_id, b, k, l, j, fd; + struct bpf_insn call; + struct btf *btf; + struct prog p; + char desc[64]; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "vmlinux btf")) + return; + kfunc_id = btf__find_by_name_kind(btf, "bpf_rdonly_cast", BTF_KIND_FUNC); + type_id = btf__find_by_name_kind(btf, "timespec64", BTF_KIND_STRUCT); + btf__free(btf); + if (!ASSERT_GT(kfunc_id, 0, "bpf_rdonly_cast id") || + !ASSERT_GT(type_id, 0, "timespec64 id")) + return; + call = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, kfunc_id); + + /* + * The kernel copy of the ctx through an untrusted pointer in every + * register, in every size, zero and sign extending over all ones + */ + for (b = 0; b < ARRAY_SIZE(pat_regs); b++) { + int rb = pat_regs[b]; + + for (k = 0; k < NR_PAIRS; k++) { + int rd = pair_reg(b, k); + + for (l = 0; l < ARRAY_SIZE(loads); l++) { + int size = loads[l].size, mode = loads[l].sx ? BPF_MEMSX : BPF_MEM; + + prog_start(&p); + EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_6)); + EMIT(&p, BPF_MOV32_IMM(BPF_REG_2, type_id)); + EMIT(&p, call); + if (rb != BPF_REG_0) + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_0)); + if (rd != rb) + EMIT(&p, BPF_MOV64_IMM(rd, -1)); + EMIT(&p, BPF_RAW_INSN(BPF_LDX | mode | size, rd, rb, 0, 0)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), + "probe_mem%s size=%d base=r%d dst=r%d", + loads[l].sx ? "sx" : "", size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __s64 v = head_bytes(inputs[j], size); + + if (loads[l].sx) + v = sign_extend(v, size); + prog_run(fd, inputs[j], 0, v, desc); + } + close(fd); + } + } + } + + /* NULL and a user address are not touched and read as zero */ + for (j = 0; j < 2; j++) { + __u64 addr = j ? 0x00007ffffffff000ULL : 0; + + prog_start(&p); + EMIT(&p, BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, BPF_REG_1, 0, 0, (__u32)addr)); + EMIT(&p, BPF_RAW_INSN(0, 0, 0, 0, addr >> 32)); + EMIT(&p, BPF_MOV32_IMM(BPF_REG_2, type_id)); + EMIT(&p, call); + EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0)); + prog_end(&p, BPF_REG_3); + snprintf(desc, sizeof(desc), "probe_mem addr=0x%llx", addr); + fd = prog_load(&p, desc); + if (fd < 0) + return; + prog_run(fd, 0, 0, 0, desc); + close(fd); + } +} + +/* + * Load-acquire and store-release of every size with every register as + * base, on a stack slot holding in0: the loads zero extend like the plain + * loads and the stores replace the first bytes of the slot like the plain + * stores. + */ +static void test_acq_rel(void) +{ + static const int sizes[] = { BPF_B, BPF_H, BPF_W, BPF_DW }; + struct prog p; + char desc[64]; + int b, k, s, j, fd; + + for (b = 0; b < ARRAY_SIZE(pat_regs); b++) { + int rb = pat_regs[b]; + + for (s = 0; s < ARRAY_SIZE(sizes); s++) { + int size = sizes[s]; + + for (k = 0; k < NR_PAIRS; k++) { + int rd = pair_reg(b, k); + + /* Load-acquire from the slot, also into the base */ + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, rd, -8)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, rb, -8)); + EMIT(&p, BPF_ATOMIC_OP(size, BPF_LOAD_ACQ, rd, rb, 0)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "load_acq size=%d base=r%d dst=r%d", + size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) + prog_run(fd, inputs[j], 0, + head_bytes(inputs[j], size), desc); + close(fd); + + if (rd == rb) + continue; + /* Store-release of in1 over the slot holding in0 */ + prog_start(&p); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN0)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, rd, -8)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_6, IN1)); + EMIT(&p, BPF_MOV64_REG(rb, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, rb, -8)); + EMIT(&p, BPF_ATOMIC_OP(size, BPF_STORE_REL, rb, rd, 0)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, rd, BPF_REG_10, -8)); + prog_end(&p, rd); + snprintf(desc, sizeof(desc), "store_rel size=%d base=r%d src=r%d", + size, rb, rd); + fd = prog_load(&p, desc); + if (fd < 0) + return; + for (j = 0; j < ARRAY_SIZE(inputs); j++) { + __u64 in1 = inputs[(j + 1) % ARRAY_SIZE(inputs)]; + + prog_run(fd, inputs[j], in1, + head_merge(inputs[j], in1, size), desc); + } + close(fd); + } + } + } +} + +/* + * A jump with the maximal offset over instructions which expand to several + * native ones, so that the native offset outgrows the field of a short + * branch and takes the JITs through their branch conversion. The region + * must not grow in the verifier, or the jump could not reach over it any + * more: the divisions are by constants, as a division by a register gets + * a zero check in front, and the region sticks to one width, as a 32-bit + * result read by a 64-bit instruction gets a zero extension on the JITs + * which need one. The region is the not taken path and ends with a result + * different from the taken one, so a jump landing inside it fails the run. + */ +static void max_jmp(bool alu32, bool taken) +{ + static const int ops[] = { + BPF_LSH, BPF_RSH, BPF_ARSH, BPF_ADD, BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD, + }; + struct { __u64 r0, r2; } ctx = { taken, 0x0123456789abcdefULL }; + LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = &ctx, + .ctx_size_in = sizeof(ctx), + ); + int len = INT16_MAX + 5, i = 0, fd; + struct bpf_insn *insns; + char desc[64]; + + snprintf(desc, sizeof(desc), "max jump alu%d %s", alu32 ? 32 : 64, + taken ? "taken" : "not taken"); + insns = malloc(len * sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, "insns")) + return; + insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, 8); + insns[i++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0); + insns[i++] = BPF_MOV64_REG(BPF_REG_3, BPF_REG_2); + insns[i++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 1, INT16_MAX); + while (i < len - 2) { + int op = ops[(i >> 1) % ARRAY_SIZE(ops)]; + + if (op == BPF_DIV || op == BPF_MOD) + insns[i++] = alu32 ? BPF_ALU32_IMM(op, BPF_REG_3, 7) : + BPF_ALU64_IMM(op, BPF_REG_3, 7); + else + insns[i++] = alu32 ? BPF_ALU32_REG(op, BPF_REG_3, BPF_REG_2) : + BPF_ALU64_REG(op, BPF_REG_3, BPF_REG_2); + } + insns[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 2); + insns[i++] = BPF_EXIT_INSN(); + ASSERT_EQ(i, len, "insn count"); + + fd = load_prog(insns, i, desc); + if (fd >= 0) { + if (ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), desc)) + ASSERT_EQ(opts.retval, taken ? 1 : 2, desc); + close(fd); + } + free(insns); +} + +static void test_max_jmp(void) +{ + max_jmp(false, true); + max_jmp(false, false); + max_jmp(true, true); + max_jmp(true, false); +} + +/* + * Fillers of two and three bytes of native code on x86-64: a 32-bit and a + * 64-bit move between two low registers, which give a region every length + * from two bytes on and which constant blinding leaves alone. + */ +static int fill_bytes(struct bpf_insn *insns, int n, int bytes) +{ + if (bytes & 1) { + insns[n++] = BPF_MOV64_REG(BPF_REG_2, BPF_REG_3); + bytes -= 3; + } + for (; bytes > 0; bytes -= 2) + insns[n++] = BPF_MOV32_REG(BPF_REG_2, BPF_REG_3); + return n; +} + +static void prog_result(struct bpf_insn *insns, int *n) +{ + insns[(*n)++] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 10); + insns[(*n)++] = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, RES); + insns[(*n)++] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[(*n)++] = BPF_EXIT_INSN(); +} + +/* A region of x86-64 bytes through fill_bytes(), or of so many 64 and 32-bit moves */ +struct fill { + int bytes, wide, narrow; +}; + +static int fill_moves(struct bpf_insn *insns, int n, int wide, int narrow) +{ + while (wide-- > 0) + insns[n++] = BPF_MOV64_REG(BPF_REG_2, BPF_REG_3); + while (narrow-- > 0) + insns[n++] = BPF_MOV32_REG(BPF_REG_2, BPF_REG_3); + return n; +} + +static int fill_region(struct bpf_insn *insns, int n, const struct fill *f) +{ + if (f->wide || f->narrow) + return fill_moves(insns, n, f->wide, f->narrow); + return fill_bytes(insns, n, f->bytes); +} + +static int fill_insns(const struct fill *f) +{ + return f->wide + f->narrow + f->bytes / 2 + 1; +} + +/* + * Jumps whose lengths depend on each other: a conditional jump forward + * over a region, a jump back from its end to its start or to the jump in + * front of it, and an unconditional jump out of it. The region takes every + * length around the short branch limit, in front of the exit and behind + * it, so the three jumps cross the limit at different lengths and in both + * directions. A chain of nop jumps in it, gone one per pass, shifts the + * phase in which the cycle meets the padding passes, as settling jumps + * elsewhere do. This is generic to any JIT with short and long branch + * forms: the ranges of the running one are probed, see find_ranges(), and + * the cycle swept across each of them. r0 goes in with 0 to 3 and picks + * one of the four paths through the jumps. + */ +static void jump_cycle(bool back_cond, bool to_start, int inner, const struct fill *front, + const struct fill *back, const char *desc) +{ + struct bpf_insn *insns; + int n = 0, disp, head, chain, to_back[2], out, i, fd; + + insns = malloc((32 + 2 * inner + fill_insns(front) + fill_insns(back)) * sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, desc)) + return; + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, IN0); + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_1, IN1); + insns[n++] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 3); + insns[n++] = BPF_MOV64_IMM(BPF_REG_3, 0); + /* The dispatch into the chain, see nop_chain() */ + disp = n; + n += inner; + head = n++; + n = fill_region(insns, n, front); + chain = n; + for (i = 0; i < inner; i++) { + insns[disp + i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_4, 0x1000 + i, chain - disp - 1); + insns[n++] = BPF_JMP_IMM(BPF_JA, 0, 0, inner - 1 - i); + } + to_back[0] = n++; + to_back[1] = n++; + out = n++; + insns[to_back[0]] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 1, n - to_back[0] - 1); + insns[to_back[1]] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 3, n - to_back[1] - 1); + insns[n++] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1); + n = fill_region(insns, n, back); + /* Taken with 2 and not with 4, so the verifier keeps the condition */ + if (back_cond) + insns[n] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 4, head + to_start - n - 1); + else + insns[n] = BPF_JMP_IMM(BPF_JA, 0, 0, head + to_start - n - 1); + n++; + insns[head] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 2, n - head - 1); + insns[out] = BPF_JMP_IMM(BPF_JA, 0, 0, n - out - 1); + prog_result(insns, &n); + + fd = load_prog(insns, n, desc); + free(insns); + if (fd < 0) + return; + prog_run(fd, 0, 0, 10, desc); + prog_run(fd, 1, 0, 12, desc); + prog_run(fd, 2, 0, 12, desc); + prog_run(fd, 3, 0, 14, desc); + close(fd); +} + +/* + * A region which ends in a chain of jumps to its end: the last one jumps + * to the next instruction, which the JIT drops as a nop, whereupon the one + * before it becomes one, so the chain goes away one jump per pass and the + * jump over the region shrinks late, in the passes which pad the image. + * The filler in front of the chain puts that jump around the short branch + * limit, as a conditional and as an unconditional one, over a chain of ja + * and one of gotol. Each jump of the chain is the target of a conditional + * jump in front of the region on a value the verifier cannot see, as the + * chain is dead behind its first jump otherwise. + */ +static void nop_chain(bool over_cond, bool ja32, int len, int fill) +{ + struct bpf_insn insns[256]; + int n = 0, disp, over, chain, i, fd; + char desc[64]; + + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, IN0); + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_1, IN1); + insns[n++] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 3); + insns[n++] = BPF_MOV64_IMM(BPF_REG_3, 0); + /* The dispatch into the chain, filled in once the chain is placed */ + disp = n; + n += len; + if (!over_cond) + insns[n++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 2, 1); + over = n++; + n = fill_bytes(insns, n, fill); + chain = n; + for (i = 0; i < len; i++) { + insns[disp + i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_4, 0x1000 + i, chain - disp - 1); + insns[n++] = ja32 ? BPF_RAW_INSN(BPF_JMP32 | BPF_JA, 0, 0, 0, len - 1 - i) : + BPF_JMP_IMM(BPF_JA, 0, 0, len - 1 - i); + } + insns[n++] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1); + if (over_cond) + insns[over] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 2, n - over - 1); + else + insns[over] = BPF_JMP_IMM(BPF_JA, 0, 0, n - over - 1); + prog_result(insns, &n); + + snprintf(desc, sizeof(desc), "nop chain of %s under %s len=%d fill=%d", + ja32 ? "gotol" : "ja", over_cond ? "jcc" : "ja", len, fill); + fd = load_prog(insns, n, desc); + if (fd < 0) + return; + prog_run(fd, 0, 0, 10, desc); + prog_run(fd, 1, 0, 11, desc); + prog_run(fd, 2, 0, 13, desc); + prog_run(fd, 3, 0, 13, desc); + close(fd); +} + +/* The native length of a program's image, 0 without a JIT */ +static __u32 jited_len(int fd) +{ + struct bpf_prog_info info = {}; + __u32 len = sizeof(info); + + if (!ASSERT_OK(bpf_prog_get_info_by_fd(fd, &info, &len), "prog info")) + return 0; + return info.jited_prog_len; +} + +/* A conditional jump over the moves, for the length of its image */ +static int probe_len(int wide, int narrow, __u32 *len) +{ + struct bpf_insn *insns; + int n = 0, fd; + + insns = malloc((wide + narrow + 8) * sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, "probe insns")) + return -1; + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, IN0); + insns[n++] = BPF_MOV64_IMM(BPF_REG_3, 0); + insns[n++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 2, wide + narrow); + n = fill_moves(insns, n, wide, narrow); + insns[n++] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[n++] = BPF_EXIT_INSN(); + fd = load_prog(insns, n, "branch range probe"); + free(insns); + if (fd < 0) + return -1; + *len = jited_len(fd); + close(fd); + return 0; +} + +/* + * The short branch ranges of the running JIT, as counts of 64-bit moves a + * conditional jump reaches over, and the native lengths of the two moves. + * The image of the probe grows by the moves alone until the jump does not + * reach over them any more and takes a longer form, which adds the + * difference of the forms: find the first count that does so in every + * power of two of moves up to the 16-bit offset of the jump, as a JIT may + * have several forms. Without a JIT or with fixed length branches there is + * none. + */ +static int find_ranges(int *ranges, int max, int *wide_bytes, int *narrow_bytes) +{ + __u32 base, len; + int excess, plateau = 0, c, lo, hi, mid, k = 0; + + if (probe_len(8, 0, &base) || !base) + return 0; + if (probe_len(24, 0, &len)) + return 0; + *wide_bytes = (len - base) / 16; + if (probe_len(8, 16, &len)) + return 0; + *narrow_bytes = (len - base) / 16; + if (!*wide_bytes || !*narrow_bytes) + return 0; + for (c = 32; k < max && c <= 16384; c *= 2) { + if (probe_len(c, 0, &len)) + return k; + excess = len - base - (c - 8) * *wide_bytes; + if (excess <= plateau) + continue; + lo = c / 2; + hi = c; + while (hi - lo > 1) { + mid = (lo + hi) / 2; + if (probe_len(mid, 0, &len)) + return k; + if (len - base - (mid - 8) * *wide_bytes > plateau) + hi = mid; + else + lo = mid; + } + ranges[k++] = hi; + plateau = excess; + } + return k; +} + +static int gcd(int a, int b) +{ + while (b) { + int t = a % b; + + a = b; + b = t; + } + return a; +} + +/* The jump cycle with the region in front of the exit and then behind it */ +static void cycle_bytes(int kind, int inner, int bytes) +{ + struct fill front = { .bytes = bytes }, back = {}; + char desc[96]; + + snprintf(desc, sizeof(desc), "jump cycle %s back to %s inner=%d front=%d", + kind & 1 ? "jcc" : "ja", kind & 2 ? "start" : "head", inner, bytes); + jump_cycle(kind & 1, kind & 2, inner, &front, &back, desc); + front.bytes = 0; + back.bytes = bytes; + snprintf(desc, sizeof(desc), "jump cycle %s back to %s inner=%d back=%d", + kind & 1 ? "jcc" : "ja", kind & 2 ? "start" : "head", inner, bytes); + jump_cycle(kind & 1, kind & 2, inner, &front, &back, desc); +} + +/* The jump cycle with a region of so many native bytes of moves */ +static void cycle_moves(int kind, int range, int bytes, int wide, int narrow) +{ + struct fill front = {}, back = {}; + char desc[96]; + int b; + + for (b = 0; b * narrow <= bytes; b++) + if ((bytes - b * narrow) % wide == 0) + break; + if (b * narrow > bytes) + return; + front.wide = (bytes - b * narrow) / wide; + front.narrow = b; + snprintf(desc, sizeof(desc), "jump cycle %s back to %s range %d at %d bytes", + kind & 1 ? "jcc" : "ja", kind & 2 ? "start" : "head", range, bytes); + jump_cycle(kind & 1, kind & 2, 0, &front, &back, desc); +} + +static void test_convergence(void) +{ + /* The chain gone, the jump over it is at and below the limit */ + static const int fills[] = { 100, 110, 116, 120 }; + /* The last jump of a chain is a nop the verifier already removes */ + static const int inners[] = { 0, 2, 3, 4, 5 }; + int ranges[4], wide, narrow, g, nr, bytes, lo, hi, c, f, i, l, k; + + /* The x86-64 imm8 range, byte by byte */ + for (c = 0; c < 4; c++) + for (i = 0; i < ARRAY_SIZE(inners); i++) + for (f = 100; f <= 132; f++) + cycle_bytes(c, inners[i], f); + for (c = 0; c < 4; c++) + for (f = 0; f < ARRAY_SIZE(fills); f++) + for (l = 2; l <= 26; l++) + nop_chain(c & 1, c & 2, l, fills[f]); + + /* + * The short branch ranges of this JIT, which the fillers of 64 and + * 32-bit moves approach at the granularity their lengths allow + */ + nr = find_ranges(ranges, ARRAY_SIZE(ranges), &wide, &narrow); + for (k = 0; k < nr; k++) { + printf("short branch range at %d bytes, moves of %d and %d bytes\n", + ranges[k] * wide, wide, narrow); + g = gcd(wide, narrow); + lo = ranges[k] * wide - 128; + hi = ranges[k] * wide + 32; + for (c = 0; c < 4; c++) + for (bytes = lo; bytes <= hi; bytes += g) + cycle_moves(c, ranges[k] * wide, bytes, wide, narrow); + } +} + +/* + * Calls into a subprogram with all five arguments, which clobbers a subset + * of r6-r9 and uses the deepest stack slot the budget leaves, directly or + * through another one clobbering the rest, so the JIT has to save and + * restore what it uses. + */ +/* Values to keep in r6-r9 across calls */ +static const __u64 kept[] = { + 0x1111111111111111ULL, 0x2222222222222222ULL, + 0x3333333333333333ULL, 0x4444444444444444ULL, +}; + +static void test_calls(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + __u64 expected = 1 + 2 + 3 + 4 + 5; + struct bpf_insn insns[64]; + struct pat_ctx ctx; + char desc[64]; + int nested, mask, n, call, r, fd, leaf; + + /* The frames above the leaf take up to 32 bytes each of the budget */ + leaf = (is_large_stack_supported() ? 2048 : 512) - 2 * 32; + for (r = 0; r < 4; r++) + expected += kept[r]; + opts.ctx_in = &ctx; + opts.ctx_size_in = sizeof(ctx); + for (nested = 0; nested < 2; nested++) { + for (mask = 0; mask < 16; mask++) { + n = 0; + insns[n++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, -8); + for (r = 0; r < 4; r++) { + insns[n++] = BPF_RAW_INSN(BPF_LD | BPF_DW | BPF_IMM, BPF_REG_6 + r, + 0, 0, (__u32)kept[r]); + insns[n++] = BPF_RAW_INSN(0, 0, 0, 0, kept[r] >> 32); + } + for (r = 1; r <= 5; r++) + insns[n++] = BPF_MOV64_IMM(r, r); + call = n++; + for (r = 6; r <= 9; r++) + insns[n++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, r); + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -8); + insns[n++] = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, RES); + insns[n++] = BPF_MOV64_IMM(BPF_REG_0, 0); + insns[n++] = BPF_EXIT_INSN(); + insns[call] = BPF_CALL_REL(n - call - 1); + if (nested) { + /* Clobber the subset, pass the arguments on */ + for (r = 0; r < 4; r++) + if (mask & (1 << r)) + insns[n++] = BPF_MOV64_IMM(BPF_REG_6 + r, 0); + call = n++; + insns[n++] = BPF_EXIT_INSN(); + insns[call] = BPF_CALL_REL(n - call - 1); + } + /* Clobber the rest, sum the arguments through the deepest slot */ + for (r = 0; r < 4; r++) + if ((nested ? ~mask : mask) & (1 << r)) + insns[n++] = BPF_MOV64_IMM(BPF_REG_6 + r, 0); + insns[n++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, -leaf); + insns[n++] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2); + for (r = 3; r <= 5; r++) + insns[n++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, r); + insns[n++] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -leaf); + insns[n++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); + insns[n++] = BPF_EXIT_INSN(); + + snprintf(desc, sizeof(desc), "call%s clobbering 0x%x", + nested ? " nested" : "", mask); + fd = load_prog(insns, n, desc); + if (fd < 0) + return; + memset(&ctx, 0, sizeof(ctx)); + if (ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), desc)) + ASSERT_EQ(ctx.res, expected, desc); + close(fd); + } + } +} + +/* + * Helper calls with up to five arguments and a kfunc call: a dynptr over a + * map value, a read from it into the stack and its size, with r6-r9 live + * across the calls. The returns, the bytes read, the size and r6-r9 add up. + */ +static void test_helper_calls(void) +{ + static const int offs[] = { 0, 8, 1000, 4088 }; + struct off_prog p; + struct btf *btf; + __u8 mem[4096]; + __u64 expected; + __u32 key = 0; + char desc[64]; + int map_fd, kfunc_id, i, r, fd; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "vmlinux btf")) + return; + kfunc_id = btf__find_by_name_kind(btf, "bpf_dynptr_size", BTF_KIND_FUNC); + btf__free(btf); + if (!ASSERT_GT(kfunc_id, 0, "bpf_dynptr_size id")) + return; + for (i = 0; i < sizeof(mem); i++) + mem[i] = i * 7 + 3; + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "helper_mem", sizeof(key), sizeof(mem), 1, + NULL); + if (!ASSERT_GE(map_fd, 0, "helper_mem map")) + return; + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &key, mem, 0), "helper_mem update")) + goto out; + + for (i = 0; i < ARRAY_SIZE(offs); i++) { + p.cnt = 0; + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, -64)); + for (r = 0; r < 4; r++) + off_prog_ld_imm64(&p, BPF_REG_6 + r, kept[r]); + off_prog_lookup(&p, map_fd, key); + /* The dynptr at -80 over the value, the returns summed at -96 */ + EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_0)); + EMIT(&p, BPF_MOV64_IMM(BPF_REG_2, sizeof(mem))); + EMIT(&p, BPF_MOV64_IMM(BPF_REG_3, 0)); + EMIT(&p, BPF_MOV64_REG(BPF_REG_4, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, -80)); + EMIT(&p, BPF_EMIT_CALL(BPF_FUNC_dynptr_from_mem)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -96)); + /* Eight bytes from the offset into -88 */ + EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -88)); + EMIT(&p, BPF_MOV64_IMM(BPF_REG_2, 8)); + EMIT(&p, BPF_MOV64_REG(BPF_REG_3, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, -80)); + EMIT(&p, BPF_MOV64_IMM(BPF_REG_4, offs[i])); + EMIT(&p, BPF_MOV64_IMM(BPF_REG_5, 0)); + EMIT(&p, BPF_EMIT_CALL(BPF_FUNC_dynptr_read)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -96)); + EMIT(&p, BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -96)); + /* The size through the kfunc */ + EMIT(&p, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10)); + EMIT(&p, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -80)); + EMIT(&p, BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, + kfunc_id)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -88)); + EMIT(&p, BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -96)); + EMIT(&p, BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1)); + for (r = BPF_REG_6; r <= BPF_REG_9; r++) + EMIT(&p, BPF_ALU64_REG(BPF_ADD, BPF_REG_0, r)); + EMIT(&p, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_10, -64)); + EMIT(&p, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, RES)); + snprintf(desc, sizeof(desc), "helper calls off=%d", offs[i]); + fd = off_prog_load(&p, desc, false); + if (fd < 0) + goto out; + expected = sizeof(mem) + load_bytes(mem + offs[i], BPF_DW); + for (r = 0; r < 4; r++) + expected += kept[r]; + prog_run(fd, 0, 0, expected, desc); + close(fd); + } +out: + close(map_fd); +} + +void test_jit_insns(void) +{ + RUN_TESTS(jit_insns_arena); + if (test__start_subtest("alu_imm")) + test_alu_imm(); + if (test__start_subtest("alu_reg")) + test_alu_reg(); + if (test__start_subtest("alu_imm_regs")) + test_alu_imm_regs(); + if (test__start_subtest("div_const")) + test_div_const(); + if (test__start_subtest("alu_reg_pairs")) + test_alu_reg_pairs(); + if (test__start_subtest("alu_edges")) + test_alu_edges(); + if (test__start_subtest("shifts")) + test_shifts(); + if (test__start_subtest("shift_counts")) + test_shift_counts(); + if (test__start_subtest("atomics")) + test_atomics(); + if (test__start_subtest("atomic_reg_pairs")) + test_atomic_reg_pairs(); + if (test__start_subtest("xadd_chain")) + test_xadd_chain(); + if (test__start_subtest("ld_imm64")) + test_ld_imm64(); + if (test__start_subtest("jmp_imm")) + test_jmp_imm(); + if (test__start_subtest("jmp_reg")) + test_jmp_reg(); + if (test__start_subtest("jmp_reg_pairs")) + test_jmp_reg_pairs(); + if (test__start_subtest("jmp_edges")) + test_jmp_edges(); + if (test__start_subtest("staggered_jumps")) + test_staggered_jumps(); + if (test__start_subtest("max_jmp")) + test_max_jmp(); + if (test__start_subtest("convergence")) + test_convergence(); + if (test__start_subtest("calls")) + test_calls(); + if (test__start_subtest("helper_calls")) + test_helper_calls(); + if (test__start_subtest("ldst_zero_off")) + test_ldst_zero_off(); + if (test__start_subtest("ldst_offsets")) + test_ldst_offsets(); + if (test__start_subtest("acq_rel")) + test_acq_rel(); + if (test__start_subtest("movsx")) + test_movsx(); + if (test__start_subtest("jset_imm")) + test_jset_imm(); + if (test__start_subtest("cmp_imm")) + test_cmp_imm(); + if (test__start_subtest("alu_consts")) + test_alu_consts(); + if (test__start_subtest("bswap")) + test_bswap(); + if (test__start_subtest("probe_mem")) + test_probe_mem(); +} + +/* The families with an immediate once more with the constants blinded */ +void serial_test_jit_insns_blinded(void) +{ + const char *harden = "/proc/sys/net/core/bpf_jit_harden"; + char old[16] = {}; + + if (!is_jit_enabled()) { + test__skip(); + return; + } + if (sysctl_set_or_fail(harden, old, "2")) + return; + if (test__start_subtest("alu_imm")) + test_alu_imm(); + if (test__start_subtest("alu_imm_regs")) + test_alu_imm_regs(); + if (test__start_subtest("alu_edges")) + test_alu_edges(); + if (test__start_subtest("div_const")) + test_div_const(); + if (test__start_subtest("shifts")) + test_shifts(); + if (test__start_subtest("xadd_chain")) + test_xadd_chain(); + if (test__start_subtest("ld_imm64")) + test_ld_imm64(); + if (test__start_subtest("jmp_imm")) + test_jmp_imm(); + if (test__start_subtest("jmp_edges")) + test_jmp_edges(); + if (test__start_subtest("ldst_zero_off")) + test_ldst_zero_off(); + if (test__start_subtest("ldst_offsets")) + test_ldst_offsets(); + if (test__start_subtest("jset_imm")) + test_jset_imm(); + if (test__start_subtest("cmp_imm")) + test_cmp_imm(); + if (test__start_subtest("alu_consts")) + test_alu_consts(); + sysctl_set_or_fail(harden, NULL, old); +} diff --git a/tools/testing/selftests/bpf/prog_tests/kasan.c b/tools/testing/selftests/bpf/prog_tests/kasan.c new file mode 100644 index 000000000000..9911b96fec9e --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/kasan.c @@ -0,0 +1,479 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +/* + * Tests validating that KASAN reports are properly instrumented and + * generated on a wide variety of instructions. The running kernel needs + * kasan_multi_shot to run multiple kasan-generating subtests at once + */ +#include <bpf/bpf.h> +#include <errno.h> +#include <fcntl.h> +#include <linux/if_ether.h> +#include <unistd.h> +#include <test_progs.h> +#include <unpriv_helpers.h> +#include "sysctl_helpers.h" +#include "kasan.skel.h" +#include "kasan_harden.skel.h" + +#define SUBTEST_NAME_MAX_LEN 128 +#define PROG_NAME_MAX_LEN 128 + +#define MAX_LOG_SIZE (8 * 1024) +#define READ_CHUNK_SIZE 256 + +#define KASAN_PATTERN_SLAB_UAF "BUG: KASAN: slab-use-after-free " \ + "in bpf_prog_%02x%02x%02x%02x%02x%02x%02x%02x_%s" +#define KASAN_PATTERN_SLAB_OOB "BUG: KASAN: slab-out-of-bounds " \ + "in bpf_prog_%02x%02x%02x%02x%02x%02x%02x%02x_%s" +#define KASAN_PATTERN_REPORT "%s of size %d at addr" + +static char klog_buffer[MAX_LOG_SIZE]; +static char record[MAX_LOG_SIZE]; + +struct test_spec { + char *prog_type; + bool is_write; + bool only_32_or_64; + bool needs_load_acq_store_rel; + bool needs_st; + bool skip_multi_size_testing; + bool skip_on_stack_testing; + int run_size; + bool expect_no_report; + bool rnd_hi32; + bool is_oob; +}; + +struct kasan_write_val { + __u8 data_1; + __u16 data_2; + __u32 data_4; + __u64 data_8; +}; + +struct test_ctx { + __u8 prog_tag[BPF_TAG_SIZE]; + struct bpf_object *obj; + int *access_size; + bool skip_load_acq_store_rel; + bool skip_st_tests; + struct bpf_program *prog; + char prog_name[SUBTEST_NAME_MAX_LEN]; + int klog_fd; +}; + +static int open_kernel_logs(void) +{ + int fd; + + fd = open("/dev/kmsg", O_RDONLY | O_NONBLOCK); + + return fd; +} + +static void skip_kernel_logs(int fd) +{ + lseek(fd, 0, SEEK_END); +} + +static int read_kernel_logs(int fd, char *buf, size_t max_len) +{ + size_t total = 0; + ssize_t n; + + buf[0] = '\0'; + while (1) { + char *msg, *eol; + size_t len; + + n = read(fd, record, sizeof(record) - 1); + if (n == 0) + break; + + if (n < 0) { + if (errno == EAGAIN) + break; + return n; + } + record[n] = '\0'; + + /* + * Each kmsg record starts with some metadata, separated + * from the actual content by a semi-colon + */ + msg = strchr(record, ';'); + if (!msg) + continue; + msg++; + eol = strchr(msg, '\n'); + if (eol) + *eol = '\0'; + + len = strlen(msg); + if (total + len + 2 > max_len) + break; + memcpy(buf + total, msg, len); + total += len; + buf[total++] = '\n'; + buf[total] = '\0'; + } + + return total; +} + +static int check_kasan_report_in_kernel_logs(char *buf, struct test_ctx *ctx, + bool is_write, int size, + bool is_oob) +{ + char access_log[READ_CHUNK_SIZE]; + const char *pattern; + char *kasan_report_start; + int nsize; + + pattern = is_oob ? KASAN_PATTERN_SLAB_OOB : KASAN_PATTERN_SLAB_UAF; + nsize = snprintf(access_log, READ_CHUNK_SIZE, pattern, + ctx->prog_tag[0], ctx->prog_tag[1], ctx->prog_tag[2], + ctx->prog_tag[3], ctx->prog_tag[4], ctx->prog_tag[5], + ctx->prog_tag[6], ctx->prog_tag[7], ctx->prog_name); + if (!ASSERT_GE(nsize, 0, "format kasan access header line")) + return nsize; + /* + * Searched kasan report is valid if + * - it contains the expected kasan pattern + * - the description of the faulty access is found somewhere + * after the header (not necessarily on the very next line, + * because other kernel messages may interleave) + * - faulty access properties match the tested type and size + */ + kasan_report_start = strstr(buf, access_log); + + if (!kasan_report_start) + return 1; + + nsize = snprintf(access_log, READ_CHUNK_SIZE, KASAN_PATTERN_REPORT, + is_write ? "Write" : "Read", size); + if (!ASSERT_GE(nsize, 0, "format kasan access report line")) + return nsize; + + if (!strstr(kasan_report_start, access_log)) + return 1; + + return 0; +} + +static void exec_subtest(struct test_ctx *ctx, struct test_spec *test, + int access_size, bool on_stack) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct bpf_prog_info info; + uint8_t buf[ETH_HLEN] = {0}; + int ret, prog_fd; + __u32 info_len; + + ctx->prog = bpf_object__find_program_by_name(ctx->obj, + ctx->prog_name); + if (!ASSERT_OK_PTR(ctx->prog, "find test prog")) + return; + + info_len = sizeof(info); + memset(&info, 0, info_len); + prog_fd = bpf_program__fd(ctx->prog); + if (!ASSERT_OK_FD(prog_fd, "get prog fd")) + return; + ret = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len); + if (!ASSERT_OK(ret, "fetch loaded program info")) + return; + memcpy(ctx->prog_tag, info.tag, BPF_TAG_SIZE); + + skip_kernel_logs(ctx->klog_fd); + + topts.sz = sizeof(struct bpf_test_run_opts); + topts.data_size_in = ETH_HLEN; + topts.data_in = buf; + if (ctx->access_size) + *ctx->access_size = access_size; + ret = bpf_prog_test_run_opts(bpf_program__fd(ctx->prog), + &topts); + if (!ASSERT_OK(ret, "run prog")) + return; + + ret = read_kernel_logs(ctx->klog_fd, klog_buffer, MAX_LOG_SIZE); + if (!ASSERT_GE(ret, 0, "read kernel logs")) + return; + + ret = check_kasan_report_in_kernel_logs(klog_buffer, ctx, + test->is_write, access_size, + test->is_oob); + if (on_stack || test->expect_no_report) + ASSERT_NEQ(ret, 0, "no report should be generated"); + else + ASSERT_OK(ret, "report should be generated"); +} + +static void run_subtest_with_size_and_location(struct test_ctx *ctx, + struct test_spec *test, + int access_size, + bool on_stack) +{ + char subtest_name[SUBTEST_NAME_MAX_LEN]; + + if (test->skip_multi_size_testing) { + snprintf(subtest_name, SUBTEST_NAME_MAX_LEN, "%s%s", + test->prog_type, + test->skip_on_stack_testing ? "" : + on_stack ? "_on_stack" : + "_not_on_stack"); + } else { + snprintf(subtest_name, SUBTEST_NAME_MAX_LEN, "%s_%d_%s", + test->prog_type, access_size, + on_stack ? "on_stack" : "not_on_stack"); + } + + snprintf(ctx->prog_name, PROG_NAME_MAX_LEN, "%s%s", test->prog_type, + test->skip_on_stack_testing ? "" : + on_stack ? "_on_stack" : + "_not_on_stack"); + + if (!test__start_subtest(subtest_name)) + return; + + if (test->needs_load_acq_store_rel && ctx->skip_load_acq_store_rel) { + test__skip(); + return; + } + + if (test->needs_st && ctx->skip_st_tests) { + test__skip(); + return; + } + + exec_subtest(ctx, test, access_size, on_stack); +} + +static void run_subtest_with_size(struct test_ctx *ctx, struct test_spec *test, + int size) +{ + run_subtest_with_size_and_location(ctx, test, size, false); + if (!test->skip_on_stack_testing) + run_subtest_with_size_and_location(ctx, test, size, true); +} + +static void run_subtest(struct test_ctx *ctx, struct test_spec *test) +{ + if (test->skip_multi_size_testing) { + run_subtest_with_size(ctx, test, test->run_size); + return; + } + + if (!test->only_32_or_64) { + run_subtest_with_size(ctx, test, 1); + run_subtest_with_size(ctx, test, 2); + } + run_subtest_with_size(ctx, test, 4); + run_subtest_with_size(ctx, test, 8); +} + +static void run_blinding_subtest(void) +{ + struct test_spec blinding_spec = { + .prog_type = "st_blinded", + .is_write = true, + }; + char bpf_jit_harden_orig[2]; + struct kasan_harden *skel; + struct test_ctx *ctx; + + if (!test__start_subtest("st_blinded")) + return; + + ctx = calloc(1, sizeof(*ctx)); + if (!ASSERT_OK_PTR(ctx, "alloc blinding ctx")) + return; + ctx->klog_fd = -1; + + if (sysctl_set_or_fail("/proc/sys/net/core/bpf_jit_harden", + bpf_jit_harden_orig, "2")) + goto free_ctx; + + skel = kasan_harden__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open and load blinded prog")) + goto restore; + + if (skel->data->skip_st_tests) { + test__skip(); + goto destroy; + } + + ctx->klog_fd = open_kernel_logs(); + if (!ASSERT_OK_FD(ctx->klog_fd, "open kernel logs")) + goto destroy; + + ctx->obj = skel->obj; + strncpy(ctx->prog_name, "st_blinded", PROG_NAME_MAX_LEN); + + exec_subtest(ctx, &blinding_spec, 1, false); + +destroy: + close(ctx->klog_fd); + kasan_harden__destroy(skel); +restore: + sysctl_set_or_fail("/proc/sys/net/core/bpf_jit_harden", NULL, + bpf_jit_harden_orig); +free_ctx: + free(ctx); +} + +static struct test_spec tests[] = { + { + .prog_type = "st", + .is_write = true, + .needs_st = true + }, + { + .prog_type = "stx", + .is_write = true + }, + { + .prog_type = "ldx", + .is_write = false + }, + { + .prog_type = "simple_atomic", + .is_write = true, + .only_32_or_64 = true + }, + { + .prog_type = "simple_atomic_fetch", + .is_write = true, + .skip_multi_size_testing = true, + .run_size = 8, + }, + { + .prog_type = "load_acquire", + .is_write = false, + .needs_load_acq_store_rel = true + }, + { + .prog_type = "store_release", + .is_write = true, + .needs_load_acq_store_rel = true + }, + { + .prog_type = "ldx_patched", + .is_write = false, + .skip_multi_size_testing = true, + .run_size = 4, + .rnd_hi32 = true + }, + { + .prog_type = "verifier_paths_stack_and_non_stack", + .is_write = true, + .skip_multi_size_testing = true, + .skip_on_stack_testing = true, + .run_size = 1 + }, + { + .prog_type = "ldx_oob", + .is_write = false, + .skip_on_stack_testing = true, + .is_oob = true + }, + { + .prog_type = "ldx_self_alias_on_stack", + .is_write = false, + .skip_multi_size_testing = true, + .skip_on_stack_testing = true, + .run_size = 8, + .expect_no_report = true + } +}; + +void serial_test_kasan(void) +{ + struct kasan_write_val val; + struct test_spec *test; + struct test_ctx *ctx; + struct kasan *skel; + __u32 key = 0; + int i, ret; + + ctx = calloc(1, sizeof(struct test_ctx)); + if (!ASSERT_OK_PTR(ctx, "alloc test ctx")) + return; + + if (!is_jit_enabled() || !get_kasan_jit_enabled() || + !get_kasan_multi_shot_enabled()) { + test__skip(); + goto end; + } + + skel = kasan__open(); + if (!ASSERT_OK_PTR(skel, "open prog")) + goto end; + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + char prog_name[SUBTEST_NAME_MAX_LEN]; + struct bpf_program *prog; + + if (!tests[i].rnd_hi32) + continue; + + snprintf(prog_name, SUBTEST_NAME_MAX_LEN, "%s_%s", + tests[i].prog_type, "on_stack"); + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "find rnd_hi32 on_stack prog")) + goto destroy; + bpf_program__add_flags(prog, BPF_F_TEST_RND_HI32); + snprintf(prog_name, SUBTEST_NAME_MAX_LEN, "%s_%s", + tests[i].prog_type, "not_on_stack"); + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "find rnd_hi32 not_on_stack prog")) + goto destroy; + bpf_program__add_flags(prog, BPF_F_TEST_RND_HI32); + } + + if (!ASSERT_OK(kasan__load(skel), "load prog")) + goto destroy; + + ctx->obj = skel->obj; + ctx->access_size = &skel->bss->access_size; + ctx->skip_load_acq_store_rel = skel->data->skip_load_acq_store_rel_tests; + ctx->skip_st_tests = skel->data->skip_st_tests; + + ctx->klog_fd = open_kernel_logs(); + if (!ASSERT_OK_FD(ctx->klog_fd, "open kernel logs")) + goto destroy; + + /* Fill map with recognizable values */ + ret = bpf_map__lookup_elem(skel->maps.test_map, &key, sizeof(key), + &val, sizeof(val), 0); + if (!ASSERT_OK(ret, "get map")) + goto close; + val.data_1 = 0xAA; + val.data_2 = 0xBBBB; + val.data_4 = 0xCCCCCCCC; + val.data_8 = 0xDDDDDDDDDDDDDDDD; + ret = bpf_map__update_elem(skel->maps.test_map, &key, sizeof(key), + &val, sizeof(val), 0); + if (!ASSERT_OK(ret, "set map")) + goto close; + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + test = &tests[i]; + run_subtest(ctx, test); + } + + /* + * Blinding subtest is handled differently as it needs the + * corresponding program to be loaded with bpf_jit_harden raised + */ + run_blinding_subtest(); + +close: + close(ctx->klog_fd); +destroy: + kasan__destroy(skel); +end: + free(ctx); +} diff --git a/tools/testing/selftests/bpf/prog_tests/kernel_flag.c b/tools/testing/selftests/bpf/prog_tests/kernel_flag.c index 97b00c7efe94..c723c9fbfbcb 100644 --- a/tools/testing/selftests/bpf/prog_tests/kernel_flag.c +++ b/tools/testing/selftests/bpf/prog_tests/kernel_flag.c @@ -10,12 +10,32 @@ void test_kernel_flag(void) struct test_kernel_flag *lsm_skel; struct kfunc_call_test *skel = NULL; struct kfunc_call_test_lskel *lskel = NULL; + struct bpf_program *prog; + __u32 flags; int ret; - lsm_skel = test_kernel_flag__open_and_load(); + lsm_skel = test_kernel_flag__open(); if (!ASSERT_OK_PTR(lsm_skel, "lsm_skel")) return; + prog = lsm_skel->progs.bpf; + flags = bpf_program__flags(prog); + ASSERT_EQ(flags & BPF_F_SLEEPABLE, + BPF_F_SLEEPABLE, "sleepable in program flags"); + + ret = bpf_program__add_flags(prog, BPF_F_ANY_ALIGNMENT); + ASSERT_OK(ret, "bpf_program__add_flags ret"); + ASSERT_EQ(bpf_program__flags(prog), flags | BPF_F_ANY_ALIGNMENT, + "bpf_program__add_flags value"); + + ret = bpf_program__clear_flags(prog, BPF_F_ANY_ALIGNMENT); + ASSERT_OK(ret, "bpf_program__clear_flags ret"); + ASSERT_EQ(bpf_program__flags(prog), flags, "bpf_program__clear_flags value"); + + ret = test_kernel_flag__load(lsm_skel); + if (!ASSERT_OK(ret, "test_kernel_flag__load")) + goto close_prog; + lsm_skel->bss->monitored_tid = sys_gettid(); ret = test_kernel_flag__attach(lsm_skel); diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c index 2b39cc1b09f9..0063e60d6f2f 100644 --- a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c +++ b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c @@ -70,7 +70,7 @@ static struct kfunc_test_params kfunc_tests[] = { TC_FAIL(kfunc_call_test_get_mem_fail_oversized, 0, "allocation size exceeds u32 max"), TC_FAIL(kfunc_call_test_get_mem_fail_not_const, 0, "is not a const"), TC_FAIL(kfunc_call_test_mem_acquire_fail, 0, "acquire kernel function does not return PTR_TO_BTF_ID"), - TC_FAIL(kfunc_call_test_pointer_arg_type_mismatch, 0, "R1 expected pointer to ctx, but got scalar"), + TC_FAIL(kfunc_call_test_pointer_arg_type_mismatch, 0, "R1 type=scalar expected=ctx"), TC_FAIL(kfunc_call_test_spin_lock_unsafe, 0, "function calls are not allowed while holding a lock"), /* success cases */ diff --git a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c index 2e0ddef77ba5..651123cd8602 100644 --- a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c +++ b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c @@ -361,8 +361,7 @@ static void test_attach_api_fails(void) sl_skel->bss->user_ptr = sl_skel; - err = bpf_program__set_flags(sl_skel->progs.handle_kprobe_multi_sleepable, - BPF_F_SLEEPABLE); + err = bpf_program__add_flags(sl_skel->progs.handle_kprobe_multi_sleepable, BPF_F_SLEEPABLE); if (!ASSERT_OK(err, "sleep_skel_set_flags")) goto cleanup; diff --git a/tools/testing/selftests/bpf/prog_tests/ksock_sendto.c b/tools/testing/selftests/bpf/prog_tests/ksock_sendto.c new file mode 100644 index 000000000000..498f0e1be6ee --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/ksock_sendto.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include "test_progs.h" +#include "network_helpers.h" +#include "ksock_sendto.skel.h" + +struct sendto_test_case { + const char *name; + bool connected; + bool use_sendto_addr; +}; + +static bool recv_payload(int fd, const void *expected, size_t len) +{ + char buf[64]; + ssize_t n; + + n = recv(fd, buf, sizeof(buf), MSG_TRUNC); + if (!ASSERT_EQ(n, len, "recv length")) + return false; + return ASSERT_MEMEQ(buf, expected, len, "recv payload"); +} + +static void test_sendto_case(const struct sendto_test_case *test) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct ksock_sendto *skel; + int receivers[2] = { -1, -1 }; + int err, i, port, sendto_fd; + char buf[64]; + ssize_t n; + + skel = ksock_sendto__open_and_load(); + if (!ASSERT_OK_PTR(skel, "ksock_sendto open and load")) + return; + + /* Use separate receivers for the default peer and explicit destination. */ + for (i = 0; i < ARRAY_SIZE(receivers); i++) { + receivers[i] = start_server(AF_INET, SOCK_DGRAM, "127.0.0.1", 0, 1000); + if (!ASSERT_OK_FD(receivers[i], "receiver socket")) + goto out; + } + + port = get_socket_local_port(receivers[0]); + if (!ASSERT_GE(port, 0, "connected port")) + goto out; + skel->bss->connected_port = port; + port = get_socket_local_port(receivers[1]); + if (!ASSERT_GE(port, 0, "sendto port")) + goto out; + skel->bss->sendto_port = port; + skel->bss->connected = test->connected; + skel->bss->use_sendto_addr = test->use_sendto_addr; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_sendto), &opts); + if (!ASSERT_OK(err, "run test_sendto") || !ASSERT_OK(opts.retval, "test_sendto retval")) + goto out; + + /* Check sendto() outcome */ + if (!ASSERT_EQ(skel->data->sendto_ret, sizeof(skel->data->sendto_data), "sendto bytes")) + goto out; + sendto_fd = receivers[test->use_sendto_addr ? 1 : 0]; + if (!recv_payload(sendto_fd, skel->data->sendto_data, sizeof(skel->data->sendto_data))) + goto out; + + /* Check send() outcome */ + if (test->connected) { + if (!ASSERT_EQ(skel->data->send_ret, sizeof(skel->data->send_data), "send bytes")) + goto out; + if (!recv_payload(receivers[0], skel->data->send_data, + sizeof(skel->data->send_data))) + goto out; + } else if (!ASSERT_EQ(skel->data->send_ret, -EDESTADDRREQ, + "send after sendto remains unconnected")) { + goto out; + } + + /* Catch unexpected deliveries to either destination. */ + for (i = 0; i < ARRAY_SIZE(receivers); i++) { + n = recv(receivers[i], buf, sizeof(buf), MSG_DONTWAIT); + if (!ASSERT_EQ(n, -1, "no extra datagram") || + !ASSERT_EQ(errno, EAGAIN, "recv errno")) + goto out; + } + +out: + for (i = 0; i < ARRAY_SIZE(receivers); i++) { + if (receivers[i] >= 0) + close(receivers[i]); + } + ksock_sendto__destroy(skel); +} + +void test_ksock_sendto(void) +{ + static const struct sendto_test_case tests[] = { + { .name = "unconnected", .use_sendto_addr = true }, + { .name = "connected_null_addr", .connected = true }, + { .name = "connected_explicit_addr", .connected = true, .use_sendto_addr = true }, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + if (test__start_subtest(tests[i].name)) + test_sendto_case(&tests[i]); + } +} diff --git a/tools/testing/selftests/bpf/prog_tests/ksock_wq.c b/tools/testing/selftests/bpf/prog_tests/ksock_wq.c index d6dc20b8f95b..12c2275e8177 100644 --- a/tools/testing/selftests/bpf/prog_tests/ksock_wq.c +++ b/tools/testing/selftests/bpf/prog_tests/ksock_wq.c @@ -4,6 +4,7 @@ #include <unistd.h> #include "test_progs.h" +#include "network_helpers.h" #include "ksock_wq.skel.h" #define CALLBACK_WAIT_RETRIES 1000 @@ -13,13 +14,22 @@ void test_ksock_wq(void) { LIBBPF_OPTS(bpf_test_run_opts, opts); struct ksock_wq *skel; + char recv_data[sizeof(skel->data->send_data)]; u32 callback_done; - int err, i; + int err, i, port, rfd; skel = ksock_wq__open_and_load(); if (!ASSERT_OK_PTR(skel, "ksock_wq open and load")) return; + rfd = start_server(AF_INET, SOCK_DGRAM, "127.0.0.1", 0, 1000); + if (!ASSERT_OK_FD(rfd, "receiver socket")) + goto out; + port = get_socket_local_port(rfd); + if (!ASSERT_GE(port, 0, "receiver port")) + goto out; + skel->bss->remote_port = port; + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.ksock_wq_start), &opts); if (!ASSERT_OK(err, "run ksock_wq_start")) @@ -28,18 +38,27 @@ void test_ksock_wq(void) goto out; for (i = 0; i < CALLBACK_WAIT_RETRIES; i++) { - if (__atomic_load_n(&skel->bss->callback_done, __ATOMIC_ACQUIRE)) + if (__atomic_load_n(&skel->bss->callback_done, __ATOMIC_ACQUIRE) == 2) break; usleep(CALLBACK_WAIT_US); } callback_done = __atomic_load_n(&skel->bss->callback_done, __ATOMIC_ACQUIRE); - if (!ASSERT_EQ(callback_done, 1, "workqueue callback completed")) + if (!ASSERT_EQ(callback_done, 2, "workqueue callbacks completed")) goto out; - ASSERT_EQ(skel->bss->create_err, -EOPNOTSUPP, - "workqueue create rejected"); + ASSERT_EQ(skel->bss->create_ret, -EOPNOTSUPP, "workqueue create rejected"); + + if (!ASSERT_EQ(skel->bss->connect_send_ret, sizeof(recv_data), + "workqueue send bytes")) + goto out; + err = recv(rfd, recv_data, sizeof(recv_data), 0); + if (ASSERT_EQ(err, sizeof(recv_data), "recv len")) + ASSERT_MEMEQ(recv_data, skel->data->send_data, sizeof(recv_data), + "payload match"); out: + if (rfd >= 0) + close(rfd); ksock_wq__destroy(skel); } diff --git a/tools/testing/selftests/bpf/prog_tests/linked_externs.c b/tools/testing/selftests/bpf/prog_tests/linked_externs.c new file mode 100644 index 000000000000..ff568f5cfc42 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/linked_externs.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> +#include <bpf/btf.h> +#include "linked_arena.skel.h" + +static void test_skel(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct linked_arena *skel; + int err; + + skel = linked_arena__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + + if (skel->data->skip_tests) { + printf("%s:SKIP: no addr_space_cast support in the BPF compiler\n", __func__); + test__skip(); + goto cleanup; + } + + err = linked_arena__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + ASSERT_EQ(skel->arena->a_val, 1, "a_val_init"); + ASSERT_EQ(skel->arena->b_val, 2, "b_val_init"); + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.sum1), &opts); + ASSERT_OK(err, "sum1_run"); + ASSERT_EQ(opts.retval, 1 + 2, "sum1_retval"); + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.bump2), &opts); + ASSERT_OK(err, "bump2_run"); + ASSERT_EQ(opts.retval, 11 + 22, "bump2_retval"); + + ASSERT_EQ(skel->arena->a_val, 11, "a_val"); + ASSERT_EQ(skel->arena->b_val, 22, "b_val"); + +cleanup: + linked_arena__destroy(skel); +} + +static int link_objs(const char *out, const char *in1, const char *in2) +{ + struct bpf_linker *linker; + int err; + + linker = bpf_linker__new(out, NULL); + if (!ASSERT_OK_PTR(linker, "linker_new")) + return -EINVAL; + + err = bpf_linker__add_file(linker, in1, NULL); + if (!ASSERT_OK(err, in1)) + goto cleanup; + if (in2) { + err = bpf_linker__add_file(linker, in2, NULL); + if (!ASSERT_OK(err, in2)) + goto cleanup; + } + err = bpf_linker__finalize(linker); + ASSERT_OK(err, "finalize"); + +cleanup: + bpf_linker__free(linker); + return err; +} + +/* link in1 on its own first, then link the result with in2 */ +static void test_relink(const char *in1, const char *in2, const char *sec_name) +{ + char out1[] = "/tmp/linked_externs.XXXXXX", out2[] = "/tmp/linked_externs.XXXXXX"; + const struct btf_var_secinfo *vi; + struct bpf_object *obj = NULL; + const struct btf_type *t; + struct btf *btf; + int i, id, n; + + close(mkstemp(out1)); + close(mkstemp(out2)); + + if (!ASSERT_OK(link_objs(out1, in1, NULL), "link_stage1") || + !ASSERT_OK(link_objs(out2, out1, in2), "link_stage2")) + goto cleanup; + + obj = bpf_object__open_file(out2, NULL); + if (!ASSERT_OK_PTR(obj, "obj_open")) + goto cleanup; + + btf = bpf_object__btf(obj); + id = btf__find_by_name_kind(btf, sec_name, BTF_KIND_DATASEC); + if (!ASSERT_GT(id, 0, "find_datasec")) + goto cleanup; + + t = btf__type_by_id(btf, id); + vi = btf_var_secinfos(t); + n = btf_vlen(t); + for (i = 0; i < n; i++) { + t = btf__type_by_id(btf, vi[i].type); + ASSERT_NEQ(btf_var(t)->linkage, BTF_VAR_GLOBAL_EXTERN, "var_resolved"); + } + + ASSERT_OK(bpf_object__load(obj), "obj_load"); + +cleanup: + bpf_object__close(obj); + unlink(out1); + unlink(out2); +} + +void test_linked_externs(void) +{ + if (test__start_subtest("skel_arena")) + test_skel(); + if (test__start_subtest("relink_arena")) + test_relink("linked_arena1.bpf.o", "linked_arena2.bpf.o", ".addr_space.1"); + if (test__start_subtest("relink_maps")) + test_relink("linked_maps1.bpf.o", "linked_maps2.bpf.o", ".maps"); +} diff --git a/tools/testing/selftests/bpf/prog_tests/lirc_mode2.c b/tools/testing/selftests/bpf/prog_tests/lirc_mode2.c new file mode 100644 index 000000000000..7b7352dbcd82 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/lirc_mode2.c @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0 +// test ir decoder +// +// Copyright (C) 2018 Sean Young <sean@mess.org> + +// A lirc chardev is a device representing a consumer IR (cir) device which +// can receive infrared signals from remote control and/or transmit IR. +// +// IR is sent as a series of pulses and space somewhat like morse code. The +// BPF program can decode this into scancodes so that rc-core can translate +// this into input key codes using the rc keymap. +// +// This test works by sending IR over rc-loopback, so the IR is processed by +// BPF and then decoded into scancodes. The lirc chardev must be the one +// associated with rc-loopback, see the output of ir-keytable(1). +// +// The following CONFIG options must be enabled for the test to succeed: +// CONFIG_RC_CORE=y +// CONFIG_BPF_LIRC_MODE2=y +// CONFIG_RC_LOOPBACK=y +// CONFIG_LIRC=y + +#include <linux/input.h> +#include <linux/lirc.h> +#include <glob.h> +#include <limits.h> +#include <poll.h> +#include <test_progs.h> +#include "lirc_mode2.skel.h" + +/* + * Read the DEVNAME= line out of the first uevent file that matches + * pattern, and turn it into a /dev/<name> path. + */ +static bool find_devname(const char *pattern, char *path, size_t path_sz) +{ + glob_t gl = {}; + bool found = false; + FILE *f; + + if (glob(pattern, 0, NULL, &gl) || gl.gl_pathc == 0) + goto out; + + f = fopen(gl.gl_pathv[0], "r"); + if (!f) + goto out; + + char line[256]; + + while (fgets(line, sizeof(line), f)) { + char *val; + + if (strncmp(line, "DEVNAME=", 8)) + continue; + + val = line + 8; + val[strcspn(val, "\n")] = '\0'; + snprintf(path, path_sz, "/dev/%s", val); + found = true; + break; + } + + fclose(f); +out: + globfree(&gl); + return found; +} + +/* Load rc-loopback and find the lirc and input chardevs it created. */ +static bool find_loopback_devices(char *lirc_path, char *input_path, + size_t path_sz) +{ + glob_t gl = {}; + bool found = false; + + /* Ignore failure, we check for the resulting devices below. */ + SYS_NOFAIL("modprobe rc-loopback > /dev/null 2>&1"); + + if (glob("/sys/class/rc/rc*", 0, NULL, &gl)) + goto out; + + for (size_t i = 0; i < gl.gl_pathc; i++) { + const char *rcdir = gl.gl_pathv[i]; + char uevent_path[PATH_MAX]; + char uevent[4096]; + char pattern[PATH_MAX]; + FILE *f; + size_t n; + + snprintf(uevent_path, sizeof(uevent_path), "%s/uevent", rcdir); + f = fopen(uevent_path, "r"); + if (!f) + continue; + n = fread(uevent, 1, sizeof(uevent) - 1, f); + fclose(f); + uevent[n] = '\0'; + + if (!strstr(uevent, "DRV_NAME=rc-loopback")) + continue; + + snprintf(pattern, sizeof(pattern), "%s/lirc*/uevent", rcdir); + if (!find_devname(pattern, lirc_path, path_sz)) + continue; + + snprintf(pattern, sizeof(pattern), "%s/input*/event*/uevent", rcdir); + if (!find_devname(pattern, input_path, path_sz)) + continue; + + found = true; + break; + } + +out: + if (!found) { + fprintf(stderr, "No rc devices found\n"); + fprintf(stderr, "Enable CONFIG_RC_LOOPBACK and CONFIG_BPF_LIRC_MODE2\n"); + } + + globfree(&gl); + return found; +} + +void test_lirc_mode2(void) +{ + char lirc_path[PATH_MAX], input_path[PATH_MAX]; + int lircfd = -1, inputfd = -1, progfd, progfd2 = -1; + struct lirc_mode2 *skel = NULL, *skel2 = NULL; + __u32 prog_ids[10], prog_flags[10], prog_cnt; + struct bpf_prog_info info; + __u32 info_len, prog_id, prog_id2; + int testir1 = 0x8ead; /* keydown flag (0x8000) | scancode 0xead */ + int testir2 = 0x4081; /* pointer_rel flag (0x4000) | rel_x=1 | rel_y=1 */ + struct input_event event; + struct pollfd pfd = {}; + int ret; + + if (getuid() != 0) { + test__skip(); + return; + } + + if (!find_loopback_devices(lirc_path, input_path, sizeof(lirc_path))) { + test__skip(); + return; + } + + skel = lirc_mode2__open_and_load(); + if (!ASSERT_OK_PTR(skel, "lirc_mode2__open_and_load")) + return; + + progfd = bpf_program__fd(skel->progs.bpf_decoder); + + memset(&info, 0, sizeof(info)); + info_len = sizeof(info); + ret = bpf_prog_get_info_by_fd(progfd, &info, &info_len); + if (!ASSERT_OK(ret, "get first program's info")) + goto out; + prog_id = info.id; + + lircfd = open(lirc_path, O_RDWR | O_NONBLOCK); + if (!ASSERT_GE(lircfd, 0, "open lirc device")) + goto out; + + /* Try to detach it before it was ever attached, should fail. */ + ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); + if (!ASSERT_EQ(ret, -ENOENT, "detach unattached program")) + goto out; + + inputfd = open(input_path, O_RDONLY | O_NONBLOCK); + if (!ASSERT_GE(inputfd, 0, "open input device")) + goto out; + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs before attach")) + goto out; + if (!ASSERT_EQ(prog_cnt, 0, "no programs should be attached yet")) + goto out; + + /* Invalid attach flags must be rejected, and must not attach. */ + ret = bpf_prog_attach(progfd, lircfd, BPF_LIRC_MODE2, 1); + if (!ASSERT_EQ(ret, -EINVAL, "attach with invalid flags")) + goto out; + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after rejected attach")) + goto out; + if (!ASSERT_EQ(prog_cnt, 0, "rejected attach should not attach")) + goto out; + + ret = bpf_prog_attach(progfd, lircfd, BPF_LIRC_MODE2, 0); + if (!ASSERT_OK(ret, "attach program to lirc device")) + goto out; + + /* Invalid query flags must be rejected too, without upsetting state. */ + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 1, prog_flags, prog_ids, + &prog_cnt); + ASSERT_EQ(ret, -EINVAL, "query with invalid flags"); + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after attach")) + goto out_detach; + if (!ASSERT_EQ(prog_cnt, 1, "one program should be attached")) + goto out_detach; + ASSERT_EQ(prog_ids[0], prog_id, "queried id should match attached program"); + + /* Write raw IR */ + ret = write(lircfd, &testir1, sizeof(testir1)); + if (!ASSERT_EQ(ret, sizeof(testir1), "send test IR message 1")) + goto out_detach; + + pfd.fd = inputfd; + pfd.events = POLLIN; + + for (;;) { + poll(&pfd, 1, 100); + + /* Read decoded IR */ + ret = read(inputfd, &event, sizeof(event)); + if (!ASSERT_EQ(ret, sizeof(event), "read decoded IR 1")) + goto out_detach; + + if (event.type == EV_MSC && event.code == MSC_SCAN && + event.value == 0xead) + break; + } + + /* Write raw IR */ + ret = write(lircfd, &testir2, sizeof(testir2)); + if (!ASSERT_EQ(ret, sizeof(testir2), "send test IR message 2")) + goto out_detach; + + for (;;) { + poll(&pfd, 1, 100); + + /* Read decoded IR */ + ret = read(inputfd, &event, sizeof(event)); + if (!ASSERT_EQ(ret, sizeof(event), "read decoded IR 2")) + goto out_detach; + + if (event.type == EV_REL && event.code == REL_Y && + event.value == 1) + break; + } + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after IR was decoded")) + goto out_detach; + if (!ASSERT_EQ(prog_cnt, 1, "one program should still be attached")) + goto out_detach; + + /* + * The lirc chardev can hold more than one attached program at once. + * Load a second, independent instance and check it can be attached + * alongside the first, queried, and then detached on its own + * without disturbing the first program's attachment. + */ + skel2 = lirc_mode2__open_and_load(); + if (!ASSERT_OK_PTR(skel2, "lirc_mode2__open_and_load (2nd)")) + goto out_detach; + + progfd2 = bpf_program__fd(skel2->progs.bpf_decoder); + + memset(&info, 0, sizeof(info)); + info_len = sizeof(info); + ret = bpf_prog_get_info_by_fd(progfd2, &info, &info_len); + if (!ASSERT_OK(ret, "get second program's info")) + goto out_detach; + prog_id2 = info.id; + + ret = bpf_prog_attach(progfd2, lircfd, BPF_LIRC_MODE2, 0); + if (!ASSERT_OK(ret, "attach second program to lirc device")) + goto out_detach; + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after second attach")) + goto out_detach2; + if (!ASSERT_EQ(prog_cnt, 2, "two programs should be attached")) + goto out_detach2; + ASSERT_TRUE((prog_ids[0] == prog_id && prog_ids[1] == prog_id2) || + (prog_ids[0] == prog_id2 && prog_ids[1] == prog_id), + "queried ids should be the two attached programs"); + + /* Detach the second program; the first should remain attached. */ + ret = bpf_prog_detach2(progfd2, lircfd, BPF_LIRC_MODE2); + if (!ASSERT_OK(ret, "detach second program")) + goto out_detach2; + + /* Detaching an already-detached program should now fail. */ + ret = bpf_prog_detach2(progfd2, lircfd, BPF_LIRC_MODE2); + ASSERT_EQ(ret, -ENOENT, "detach second program again"); + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after second detach")) + goto out_detach; + if (!ASSERT_EQ(prog_cnt, 1, "one program should remain attached")) + goto out_detach; + ASSERT_EQ(prog_ids[0], prog_id, "remaining program should be the first one"); + +out_detach: + /* Let's try detaching it now it is actually attached. */ + ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); + ASSERT_OK(ret, "detach program from lirc device"); + + /* Detaching it again should now fail the same way. */ + ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); + ASSERT_EQ(ret, -ENOENT, "detach program from lirc device again"); + goto out; + +out_detach2: + /* Best-effort cleanup of the second program before bailing out. */ + bpf_prog_detach2(progfd2, lircfd, BPF_LIRC_MODE2); + goto out_detach; + +out: + if (inputfd >= 0) + close(inputfd); + if (lircfd >= 0) + close(lircfd); + lirc_mode2__destroy(skel2); + lirc_mode2__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/lsm_inode_init_xattr.c b/tools/testing/selftests/bpf/prog_tests/lsm_inode_init_xattr.c new file mode 100644 index 000000000000..b91c5c659542 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/lsm_inode_init_xattr.c @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE +#include <fcntl.h> +#include <errno.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <sys/mount.h> +#include <sys/stat.h> +#include <sys/xattr.h> +#include <test_progs.h> + +#include "lsm_inode_init_xattr.skel.h" +#include "lsm_inode_init_xattr_budget.skel.h" +#include "lsm_inode_init_xattr_value.skel.h" + +#define INIT_XATTRS_MAX 2 +#define VALUE_SIZE_MAX (64 * 1024) +#define TESTDIR "/tmp/test_progs_lsm_init_xattr" +#define RAMFSDIR "/tmp/test_progs_lsm_init_xattr_ramfs" + +static bool testdir_mounted; +static bool ramfsdir_mounted; + +static int testdir_setup(void) +{ + if (mkdir(TESTDIR, 0755) && errno != EEXIST) + return -errno; + if (mount("tmpfs", TESTDIR, "tmpfs", 0, NULL)) + return -errno; + testdir_mounted = true; + if (mkdir(RAMFSDIR, 0755) && errno != EEXIST) + return -errno; + if (mount("ramfs", RAMFSDIR, "ramfs", 0, NULL)) + return -errno; + ramfsdir_mounted = true; + return 0; +} + +static void testdir_cleanup(void) +{ + if (ramfsdir_mounted) + umount(RAMFSDIR); + rmdir(RAMFSDIR); + if (testdir_mounted) + umount(TESTDIR); + rmdir(TESTDIR); +} + +static bool lsm_is_active(const char *name) +{ + char buf[512], *tok; + int fd, len; + + fd = open("/sys/kernel/security/lsm", O_RDONLY); + if (fd < 0) + return true; + len = read(fd, buf, sizeof(buf) - 1); + close(fd); + if (len <= 0) + return true; + buf[len] = '\0'; + for (tok = strtok(buf, ",\n"); tok; tok = strtok(NULL, ",\n")) + if (!strcmp(tok, name)) + return true; + return false; +} + +static int read_label(const char *path, const char *name, char *buf, size_t sz) +{ + int ret = getxattr(path, name, buf, sz); + + return ret < 0 ? -errno : ret; +} + +static void assert_label(const char *path, const char *name, const char *want) +{ + char buf[64] = {}; + int ret; + + ret = read_label(path, name, buf, sizeof(buf)); + if (!ASSERT_EQ(ret, (int)strlen(want) + 1, name)) + return; + ASSERT_STREQ(buf, want, name); +} + +static struct lsm_inode_init_xattr *policy_attach(void) +{ + struct lsm_inode_init_xattr *skel; + + skel = lsm_inode_init_xattr__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return NULL; + + skel->bss->monitored_pid = getpid(); + if (!ASSERT_OK(lsm_inode_init_xattr__attach(skel), "skel_attach")) { + lsm_inode_init_xattr__destroy(skel); + return NULL; + } + return skel; +} + +static void test_init_labels(void) +{ + struct lsm_inode_init_xattr *skel; + const char *file = TESTDIR "/file"; + const char *subdir = TESTDIR "/subdir"; + int fd = -1; + + skel = policy_attach(); + if (!skel) + return; + + fd = open(file, O_CREAT | O_RDWR, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + goto out; + + if (!ASSERT_TRUE(skel->bss->hook_ran, "hook_ran")) + goto out; + + ASSERT_OK(skel->data->zone_err, "zone_err"); + ASSERT_OK(skel->data->origin_err, "origin_err"); + assert_label(file, "security.bpf.zone", "default"); + assert_label(file, "security.bpf.origin", "created"); + + if (!ASSERT_OK(mkdir(subdir, 0755), "mkdir")) + goto out; + assert_label(subdir, "security.bpf.zone", "default"); + assert_label(subdir, "security.bpf.origin", "created"); + rmdir(subdir); +out: + if (fd >= 0) + close(fd); + remove(file); + lsm_inode_init_xattr__destroy(skel); +} + +static void test_inherit_from_parent(void) +{ + const char *zonedir = TESTDIR "/zoned"; + const char *file = TESTDIR "/zoned/file"; + struct lsm_inode_init_xattr *skel; + int dirfd = -1, fd = -1, err; + struct { + char v[32]; + __u32 len; + } label = {}; + + if (!ASSERT_OK(mkdir(zonedir, 0755), "mkdir_zoned")) + return; + + skel = policy_attach(); + if (!skel) + goto out_dir; + + dirfd = open(zonedir, O_RDONLY | O_DIRECTORY); + if (!ASSERT_GE(dirfd, 0, "open_zoned")) + goto out; + + strncpy(label.v, "restricted", sizeof(label.v) - 1); + label.len = strlen("restricted") + 1; + + err = bpf_map_update_elem(bpf_map__fd(skel->maps.inode_zone), &dirfd, + &label, BPF_ANY); + if (!ASSERT_OK(err, "seed_parent_zone")) + goto out; + + fd = open(file, O_CREAT | O_RDWR, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + goto out; + + if (!ASSERT_TRUE(skel->bss->hook_ran, "hook_ran")) + goto out; + + ASSERT_EQ(skel->bss->inherited, 1, "inherited"); + ASSERT_OK(skel->data->zone_err, "zone_err"); + assert_label(file, "security.bpf.zone", "restricted"); + assert_label(file, "security.bpf.origin", "created"); +out: + if (fd >= 0) + close(fd); + if (dirfd >= 0) + close(dirfd); + remove(file); + lsm_inode_init_xattr__destroy(skel); +out_dir: + rmdir(zonedir); +} + +static void test_refused_claims(void) +{ + const char *file = TESTDIR "/refused"; + struct lsm_inode_init_xattr *skel; + char buf[64]; + int fd = -1; + + skel = policy_attach(); + if (!skel) + return; + + fd = open(file, O_CREAT | O_RDWR, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + goto out; + + if (!ASSERT_TRUE(skel->bss->hook_ran, "hook_ran")) + goto out; + + ASSERT_EQ(skel->data->overflow_err, -ENOSPC, "overflow_err"); + ASSERT_EQ(read_label(file, "security.bpf.overflow", buf, sizeof(buf)), + -ENODATA, "overflow_absent"); + + ASSERT_EQ(skel->data->toolong_err, -EINVAL, "toolong_err"); + + ASSERT_EQ(skel->data->selinux_err, -EPERM, "selinux_err"); + ASSERT_EQ(skel->data->user_err, -EPERM, "user_err"); + + if (!lsm_is_active("selinux")) + ASSERT_EQ(read_label(file, "security.selinux", buf, sizeof(buf)), + -ENODATA, "selinux_absent"); + ASSERT_EQ(read_label(file, "user.zone", buf, sizeof(buf)), + -ENODATA, "user_absent"); +out: + if (fd >= 0) + close(fd); + remove(file); + lsm_inode_init_xattr__destroy(skel); +} + +static void test_null_xattrs(void) +{ + const char *file = RAMFSDIR "/file"; + struct lsm_inode_init_xattr *skel; + int fd = -1; + + skel = policy_attach(); + if (!skel) + return; + + fd = open(file, O_CREAT | O_RDWR, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + goto out; + + if (!ASSERT_TRUE(skel->bss->hook_ran, "hook_ran")) + goto out; + ASSERT_EQ(skel->data->zone_err, -EOPNOTSUPP, "zone_err"); +out: + if (fd >= 0) + close(fd); + remove(file); + lsm_inode_init_xattr__destroy(skel); +} + +static void test_shared_budget(void) +{ + struct lsm_inode_init_xattr_budget *skel[INIT_XATTRS_MAX + 1] = {}; + struct bpf_link *link[INIT_XATTRS_MAX + 1] = {}; + const char *file = TESTDIR "/budget"; + int claimed = 0, refused = 0; + int i, fd = -1; + + for (i = 0; i <= INIT_XATTRS_MAX; i++) { + skel[i] = lsm_inode_init_xattr_budget__open(); + if (!ASSERT_OK_PTR(skel[i], "skel_open")) + goto out; + + snprintf(skel[i]->rodata->xattr_name, + sizeof(skel[i]->rodata->xattr_name), + "security.bpf.slot%d", i); + + if (!ASSERT_OK(lsm_inode_init_xattr_budget__load(skel[i]), + "skel_load")) + goto out; + + skel[i]->bss->monitored_pid = getpid(); + + link[i] = bpf_program__attach_lsm(skel[i]->progs.claim_one); + if (!ASSERT_OK_PTR(link[i], "attach")) + goto out; + } + + fd = open(file, O_CREAT | O_RDWR, 0644); + if (!ASSERT_GE(fd, 0, "create_file")) + goto out; + + for (i = 0; i <= INIT_XATTRS_MAX; i++) { + int err = skel[i]->data->claim_err; + + ASSERT_TRUE(skel[i]->bss->hook_ran, "hook_ran"); + if (!err) + claimed++; + else if (ASSERT_EQ(err, -ENOSPC, "claim_err")) + refused++; + } + + ASSERT_EQ(claimed, INIT_XATTRS_MAX, "claimed"); + ASSERT_EQ(refused, 1, "refused"); +out: + if (fd >= 0) + close(fd); + remove(file); + for (i = 0; i <= INIT_XATTRS_MAX; i++) { + bpf_link__destroy(link[i]); + lsm_inode_init_xattr_budget__destroy(skel[i]); + } +} + +static int claim_value(const char *file, __u32 len, __u64 flags, + int hook_retval, int *open_errno) +{ + struct lsm_inode_init_xattr_value *skel; + struct bpf_link *link = NULL; + int err = 1, fd; + + skel = lsm_inode_init_xattr_value__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return 1; + + skel->rodata->value_len = len; + skel->rodata->dynptr_flags = flags; + skel->rodata->hook_retval = hook_retval; + + if (!ASSERT_OK(lsm_inode_init_xattr_value__load(skel), "skel_load")) + goto out; + + skel->bss->monitored_pid = getpid(); + + link = bpf_program__attach_lsm(skel->progs.claim_value); + if (!ASSERT_OK_PTR(link, "attach")) + goto out; + + fd = open(file, O_CREAT | O_RDWR | O_EXCL, 0644); + *open_errno = fd < 0 ? errno : 0; + if (fd >= 0) + close(fd); + + if (ASSERT_TRUE(skel->bss->hook_ran, "hook_ran")) + err = skel->data->claim_err; +out: + bpf_link__destroy(link); + lsm_inode_init_xattr_value__destroy(skel); + return err; +} + +static void test_value_shapes(void) +{ + const char *file = TESTDIR "/value"; + char buf[64]; + int err, open_errno; + + remove(file); + + err = claim_value(file, 0, 0, 0, &open_errno); + ASSERT_OK(err, "empty_value_err"); + ASSERT_OK(open_errno, "empty_value_open"); + ASSERT_EQ(read_label(file, "security.bpf.value", buf, sizeof(buf)), + 0, "empty_value_len"); + remove(file); + + err = claim_value(file, sizeof(buf), 1, 0, &open_errno); + ASSERT_EQ(err, -EINVAL, "bad_dynptr_err"); + ASSERT_EQ(read_label(file, "security.bpf.value", buf, sizeof(buf)), + -ENODATA, "bad_dynptr_absent"); + remove(file); + + err = claim_value(file, VALUE_SIZE_MAX + 1, 0, 0, &open_errno); + ASSERT_EQ(err, -E2BIG, "oversized_err"); + ASSERT_EQ(read_label(file, "security.bpf.value", buf, sizeof(buf)), + -ENODATA, "oversized_absent"); + remove(file); + + err = claim_value(file, 8, 0, -EPERM, &open_errno); + ASSERT_OK(err, "denied_claim_err"); + ASSERT_EQ(open_errno, EPERM, "denied_open"); + ASSERT_EQ(read_label(file, "security.bpf.value", buf, sizeof(buf)), + -ENOENT, "denied_absent"); + remove(file); +} + +void test_lsm_inode_init_xattr(void) +{ + if (!ASSERT_OK(testdir_setup(), "testdir_setup")) + goto out; + + if (test__start_subtest("init_labels")) + test_init_labels(); + if (test__start_subtest("inherit_from_parent")) + test_inherit_from_parent(); + if (test__start_subtest("refused_claims")) + test_refused_claims(); + if (test__start_subtest("null_xattrs")) + test_null_xattrs(); + if (test__start_subtest("shared_budget")) + test_shared_budget(); + if (test__start_subtest("value_shapes")) + test_value_shapes(); +out: + testdir_cleanup(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c index 39e8a3b8b6af..5c5560d45c5b 100644 --- a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c +++ b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only +#include <net/if.h> #include <netinet/in.h> +#include <sys/ioctl.h> #include "network_helpers.h" #include "test_progs.h" @@ -141,6 +143,54 @@ fail: return 1; } +/* + * A veth whose peer sits in another netns with the same ifindex gets its + * carrier-on handled as a non-urgent linkwatch event, i.e. up to 1s late. + * Until then IPv6 considers the link not ready (no ff00::/8 route, no + * link-local address) and silently drops neighbour solicitations, so wait + * for all links to be operationally up (IFF_RUNNING) before sending traffic. + */ +static int wait_for_oper_up(const char *ns, const char *dev) +{ + struct nstoken *nstoken; + struct ifreq ifr = {}; + int i, fd, ret = -1; + + nstoken = open_netns(ns); + if (!ASSERT_OK_PTR(nstoken, "open ns")) + return -1; + + fd = socket(AF_INET, SOCK_DGRAM, 0); + if (!ASSERT_OK_FD(fd, "socket")) + goto out; + + strncpy(ifr.ifr_name, dev, IFNAMSIZ - 1); + for (i = 0; i < 50; i++) { + if (!ASSERT_OK(ioctl(fd, SIOCGIFFLAGS, &ifr), "SIOCGIFFLAGS")) + break; + if (ifr.ifr_flags & IFF_RUNNING) { + ret = 0; + break; + } + usleep(100000); + } + close(fd); +out: + ASSERT_OK(ret, dev); + close_netns(nstoken); + return ret; +} + +static int wait_for_links(const char *ns1, const char *ns2, const char *ns3) +{ + if (wait_for_oper_up(ns1, "veth1") || wait_for_oper_up(ns1, "veth5") || + wait_for_oper_up(ns2, "veth2") || wait_for_oper_up(ns2, "veth3") || + wait_for_oper_up(ns2, "veth6") || wait_for_oper_up(ns2, "veth7") || + wait_for_oper_up(ns3, "veth4") || wait_for_oper_up(ns3, "veth8")) + return -1; + return 0; +} + static int configure_vrf(const char *ns1, const char *ns2) { if (!ns1 || !ns2) @@ -304,6 +354,9 @@ static int setup_network(char *ns1, char *ns2, char *ns3, const char *vrf) if (!ASSERT_OK(configure_ns3(ns3), "configure ns3 routes")) goto fail; + if (!ASSERT_OK(wait_for_links(ns1, ns2, ns3), "wait for links")) + goto fail; + /* Link bottom route to the GRE tunnels */ SYS(fail, "ip -n %s route add %s/32 dev veth5 via %s %s", ns1, IP4_ADDR_GRE, IP4_ADDR_6, vrf); diff --git a/tools/testing/selftests/bpf/prog_tests/map_excl.c b/tools/testing/selftests/bpf/prog_tests/map_excl.c index 3f4422b9ffa6..9ff4cd3de2ec 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_excl.c +++ b/tools/testing/selftests/bpf/prog_tests/map_excl.c @@ -161,12 +161,49 @@ static void test_map_excl_create_validation(void) ASSERT_EQ(fd, -EINVAL, "reject size with NULL hash pointer"); } +/* + * The exclusive map bound through a continuous fd_array rather than by an + * insn. The kernel resolves the array before it verifies the program, so it + * has to have the digest of the program at hand by then. + */ +static void test_map_excl_fd_array(void) +{ + struct map_excl *skel = map_excl__open(); + LIBBPF_OPTS(bpf_prog_load_opts, opts); + int err, fd, map_fd; + + if (!ASSERT_OK_PTR(skel, "map_excl__open")) + return; + err = bpf_map__set_exclusive_program(skel->maps.excl_map, skel->progs.should_have_access); + if (!ASSERT_OK(err, "bpf_map__set_exclusive_program")) + goto out; + + bpf_program__set_autoload(skel->progs.should_have_access, true); + bpf_program__set_autoload(skel->progs.should_not_have_access, true); + err = bpf_object__prepare(skel->obj); + if (!ASSERT_OK(err, "map_excl prepare")) + goto out; + + map_fd = bpf_map__fd(skel->maps.excl_map); + opts.fd_array = &map_fd; + opts.fd_array_cnt = 1; + fd = bpf_program__clone(skel->progs.should_have_access, &opts); + if (ASSERT_GE(fd, 0, "exclusive program with the map in fd_array")) + close(fd); + fd = bpf_program__clone(skel->progs.should_not_have_access, &opts); + ASSERT_EQ(fd, -EACCES, "other program with the map in fd_array"); +out: + map_excl__destroy(skel); +} + void test_map_excl(void) { if (test__start_subtest("map_excl_allowed")) test_map_excl_allowed(); if (test__start_subtest("map_excl_denied")) test_map_excl_denied(); + if (test__start_subtest("map_excl_fd_array")) + test_map_excl_fd_array(); if (test__start_subtest("map_excl_no_map_in_map")) test_map_excl_no_map_in_map(); if (test__start_subtest("map_excl_no_map_iter")) diff --git a/tools/testing/selftests/bpf/prog_tests/may_goto_far.c b/tools/testing/selftests/bpf/prog_tests/may_goto_far.c new file mode 100644 index 000000000000..8d0ff30bd068 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/may_goto_far.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include <linux/filter.h> + +#define MAY_GOTO(off) BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, off, 0) +#define FILL_CNT 33000 + +static struct bpf_insn *fill(struct bpf_insn *insn, int cnt, int reg) +{ + while (cnt--) + *insn++ = BPF_MOV64_REG(BPF_REG_0, reg); + return insn; +} + +/* + * Syscall prog that loops until may_goto expires. Its target is 'off' insns + * away: + * + * r6 = r7 = 0, but not a constant for the verifier + * r8 = 2 + * r9 = 0 + * r0 = r8 x FILL_CNT or 1 + * 1: if r6 == r9 goto +1 + * exit + * r6 = r8 + * r0 = r9 x N when off is negative + * if r7 != r9 goto 3f + * 2: may_goto off (1b or 4f) + * goto 2b + * 3: r0 = r8 x N when off is positive + * 4: exit + * + * The prog returns 0 when may_goto jumps to its target and 2 when + * it lands in one of 'r0 = r8' areas. There are no constants between + * may_goto and its target to keep the distance when constants are blinded. + */ +static void test_far(int off) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts, .prog_flags = BPF_F_SLEEPABLE); + LIBBPF_OPTS(bpf_test_run_opts, topts); + int front = off < 0 ? 1 : FILL_CNT; + int mid = off < 0 ? -off - 5 : 1; + int back = off < 0 ? FILL_CNT : off - 1; + struct bpf_insn *insns, *insn; + int fd, err; + + insns = calloc(front + mid + back + 16, sizeof(*insns)); + if (!ASSERT_OK_PTR(insns, "calloc")) + return; + + insn = insns; + *insn++ = BPF_EMIT_CALL(BPF_FUNC_ktime_get_ns); + *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_0); + *insn++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_6, 63); + *insn++ = BPF_MOV64_REG(BPF_REG_7, BPF_REG_0); + *insn++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 63); + *insn++ = BPF_MOV64_IMM(BPF_REG_8, 2); + *insn++ = BPF_MOV64_IMM(BPF_REG_9, 0); + insn = fill(insn, front, BPF_REG_8); + *insn++ = BPF_JMP_REG(BPF_JEQ, BPF_REG_6, BPF_REG_9, 1); + *insn++ = BPF_EXIT_INSN(); + *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_8); + insn = fill(insn, mid, BPF_REG_9); + *insn++ = BPF_JMP_REG(BPF_JNE, BPF_REG_7, BPF_REG_9, 2); + *insn++ = MAY_GOTO(off); + *insn++ = BPF_JMP_A(-2); + insn = fill(insn, back, BPF_REG_8); + *insn++ = BPF_EXIT_INSN(); + + fd = bpf_prog_load(BPF_PROG_TYPE_SYSCALL, NULL, "GPL", insns, insn - insns, &opts); + free(insns); + if (!ASSERT_GE(fd, 0, "prog_load")) + return; + + err = bpf_prog_test_run_opts(fd, &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "retval"); + close(fd); +} + +void test_may_goto_far(void) +{ + if (test__start_subtest("32762")) + test_far(32762); + if (test__start_subtest("32767")) + test_far(32767); + if (test__start_subtest("-32767")) + test_far(-32767); + if (test__start_subtest("-32768")) + test_far(-32768); +} diff --git a/tools/testing/selftests/bpf/prog_tests/may_goto_priv_stack.c b/tools/testing/selftests/bpf/prog_tests/may_goto_priv_stack.c new file mode 100644 index 000000000000..4f8476d36250 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/may_goto_priv_stack.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "may_goto_priv_stack.skel.h" + +static void check_regs(__u64 *regs) +{ + int i; + + for (i = 0; i < 6; i++) + ASSERT_EQ(regs[i], 0, "reg"); + /* count is refreshed at least once when may_goto is timed */ + ASSERT_GT(regs[6], 0xffff, "iterations"); +} + +void test_may_goto_priv_stack(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct may_goto_priv_stack *skel; + int err; + + skel = may_goto_priv_stack__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + err = may_goto_priv_stack__attach(skel); + if (!ASSERT_OK(err, "attach")) + goto out; + + memset(skel->bss->regs_64, 0xff, sizeof(skel->bss->regs_64)); + memset(skel->bss->regs_128, 0xff, sizeof(skel->bss->regs_128)); + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.priv_stack_64), &topts); + if (!ASSERT_OK(err, "test_run")) + goto out; + + check_regs(skel->bss->regs_64); + check_regs(skel->bss->regs_128); +out: + may_goto_priv_stack__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/memcg_proactive_reclaim.c b/tools/testing/selftests/bpf/prog_tests/memcg_proactive_reclaim.c new file mode 100644 index 000000000000..c22dcfabe856 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/memcg_proactive_reclaim.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <limits.h> +#include <linux/magic.h> +#include <sys/statfs.h> +#include <unistd.h> + +#include "cgroup_helpers.h" +#include "memcg_proactive_reclaim.skel.h" + +#define CG_PATH "/memcg_proactive_reclaim" + +/* + * Large enough that a single reclaim pass cannot come close to it, so that + * "reclaimed less than was asked for" is not page-size noise: one pass is + * capped at MEMCG_CHARGE_BATCH pages, which is 256 KiB on 4K pages but 4 MiB + * on 64K pages. + */ +#define FILE_SIZE (32 * 1024 * 1024UL) +#define BUF_SIZE (64 * 1024) + +struct reclaim_args { + __u64 cgroup_id; + __u64 size; +}; + +/* + * The data file has to sit on a writable regular filesystem: tmpfs pages are + * charged as shmem, so whether they can be reclaimed at all depends on swap + * being available, and a read-only mount cannot host the file at all (EROFS + * applies to root too). /tmp is tmpfs on many systems, and test_progs is + * routinely run from a tmpfs working directory, so both candidates need the + * check. + */ +static const char *workload_dir(void) +{ + static const char * const dirs[] = { "/tmp", "." }; + struct statfs st; + int i; + + for (i = 0; i < ARRAY_SIZE(dirs); i++) + if (!statfs(dirs[i], &st) && st.f_type != TMPFS_MAGIC && + st.f_type != RAMFS_MAGIC && !access(dirs[i], W_OK)) + return dirs[i]; + + return NULL; +} + +void test_memcg_proactive_reclaim(void) +{ + struct memcg_proactive_reclaim *skel = NULL; + struct reclaim_args args = {}; + + LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = &args, + .ctx_size_in = sizeof(args)); + + char data_file[PATH_MAX]; + static char buf[BUF_SIZE]; + __u64 cgroup_id; + const char *dir; + off_t off; + int cg_fd = -1, data_fd = -1, err; + + /* + * Both candidate directories can be unusable: tmpfs (initramfs-based + * CI, for instance) or a read-only mount. That is an environment + * limitation, not a test failure. + */ + dir = workload_dir(); + if (!dir) { + test__skip(); + return; + } + + snprintf(data_file, sizeof(data_file), + "%s/memcg_proactive_reclaim_XXXXXX", dir); + data_fd = mkstemp(data_file); + if (!ASSERT_GE(data_fd, 0, "mkstemp")) + return; + + cg_fd = cgroup_setup_and_join(CG_PATH); + if (!ASSERT_OK_FD(cg_fd, "cgroup_setup_and_join")) + goto out; + + cgroup_id = get_cgroup_id(CG_PATH); + if (!ASSERT_GT(cgroup_id, 0, "get_cgroup_id")) + goto out; + + skel = memcg_proactive_reclaim__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto out; + + /* Charge FILE_SIZE of clean page cache to the cgroup. */ + for (off = 0; off < (off_t)FILE_SIZE; off += sizeof(buf)) + if (!ASSERT_EQ(write(data_fd, buf, sizeof(buf)), + (ssize_t)sizeof(buf), "write")) + goto out; + if (!ASSERT_OK(fsync(data_fd), "fsync")) + goto out; + + args.cgroup_id = cgroup_id; + args.size = FILE_SIZE; + skel->bss->reclaimed = 0; + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.memcg_proactive_reclaim), + &opts); + if (!ASSERT_OK(err, "test_run")) + goto out; + if (!ASSERT_EQ(opts.retval, 0, "retval")) + goto out; + + /* + * A single call is a single bounded pass: it reclaims something, but + * stops well short of the requested size instead of retrying until the + * goal is reached the way a write to memory.reclaim does. + */ + ASSERT_GT(skel->bss->reclaimed, 0, "reclaimed"); + ASSERT_LT(skel->bss->reclaimed, (__s64)FILE_SIZE, "single pass"); + +out: + if (skel) + memcg_proactive_reclaim__destroy(skel); + if (cg_fd >= 0) + close(cg_fd); + if (data_fd >= 0) { + close(data_fd); + unlink(data_file); + } + cleanup_cgroup_environment(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/missed.c b/tools/testing/selftests/bpf/prog_tests/missed.c index ed8857ae914a..b5fb0267e944 100644 --- a/tools/testing/selftests/bpf/prog_tests/missed.c +++ b/tools/testing/selftests/bpf/prog_tests/missed.c @@ -36,7 +36,7 @@ static void test_missed_perf_kprobe(void) ASSERT_EQ(info.type, BPF_LINK_TYPE_PERF_EVENT, "info.type"); ASSERT_EQ(info.perf_event.type, BPF_PERF_EVENT_KPROBE, "info.perf_event.type"); - ASSERT_EQ(info.perf_event.kprobe.missed, 1, "info.perf_event.kprobe.missed"); + ASSERT_GE(info.perf_event.kprobe.missed, 1, "info.perf_event.kprobe.missed"); cleanup: missed_kprobe__destroy(skel); @@ -120,9 +120,9 @@ static void test_missed_tp_recursion(void) ASSERT_EQ(topts.retval, 0, "test_run"); ASSERT_EQ(get_missed_count(bpf_program__fd(skel->progs.test1)), 0, "test1_recursion_misses"); - ASSERT_EQ(get_missed_count(bpf_program__fd(skel->progs.test2)), 1, "test2_recursion_misses"); - ASSERT_EQ(get_missed_count(bpf_program__fd(skel->progs.test3)), 1, "test3_recursion_misses"); - ASSERT_EQ(get_missed_count(bpf_program__fd(skel->progs.test4)), 1, "test4_recursion_misses"); + ASSERT_GE(get_missed_count(bpf_program__fd(skel->progs.test2)), 1, "test2_recursion_misses"); + ASSERT_GE(get_missed_count(bpf_program__fd(skel->progs.test3)), 1, "test3_recursion_misses"); + ASSERT_GE(get_missed_count(bpf_program__fd(skel->progs.test4)), 1, "test4_recursion_misses"); cleanup: missed_tp_recursion__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c b/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c index 7b4a1e24363b..67124dd4d8a7 100644 --- a/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c +++ b/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c @@ -449,6 +449,95 @@ static void test_lru_percpu_hash_cpu_flag_create(void) test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_LRU_PERCPU_HASH, 0); } +static bool verify_map_value_all_cpus(int map_fd, void *key, void *values, size_t elem_sz, + u32 expected_value, int nr_cpus) +{ + int i, err; + + err = bpf_map_lookup_elem(map_fd, key, values); + if (!ASSERT_OK(err, "bpf_map_lookup_elem all_cpus")) + return false; + + for (i = 0 ; i < nr_cpus ; i++) { + if (!ASSERT_EQ(*(u32 *)(values + elem_sz * i), expected_value, + "bpf_map_lookup_elem value all_cpus")) + return false; + } + + return true; +} + +static bool verify_map_value_first_cpu(int map_fd, void *key, void *values, size_t elem_sz, + u32 expected_value, int nr_cpus) +{ + int i, err; + + err = bpf_map_lookup_elem(map_fd, key, values); + if (!ASSERT_OK(err, "bpf_map_lookup_elem cpu")) + return false; + + if (!ASSERT_EQ(*(u32 *)values, expected_value, "bpf_map_lookup_elem value cpu")) + return false; + + for (i = 1 ; i < nr_cpus ; i++) { + if (!ASSERT_NEQ(*(u32 *)(values + elem_sz * i), expected_value, + "bpf_map_lookup_elem value cpu")) + return false; + } + + return true; +} + +static void test_percpu_cgroup_storage_flags_combination(struct bpf_map *map, int nr_cpus, + struct bpf_cgroup_storage_key *key) +{ + size_t value_sz = sizeof(u32), key_sz, elem_sz; + u32 *values, test_value = 0xAABBCCDD; + int err, map_fd; + u64 flags; + + key_sz = sizeof(*key); + elem_sz = roundup(value_sz, 8); + map_fd = bpf_map__fd(map); + values = calloc(nr_cpus, elem_sz); + if (!ASSERT_OK_PTR(values, "calloc values")) + return; + + flags = BPF_NOEXIST | BPF_EXIST; + err = bpf_map__update_elem(map, key, key_sz, values, elem_sz * nr_cpus, flags); + if (!ASSERT_EQ(err, -EINVAL, "bpf_map__update_elem noexist|exist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_NOEXIST; + err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); + if (!ASSERT_EQ(err, -EINVAL, "bpf_map__update_elem all_cpus|noexist")) + goto out; + + flags = BPF_F_CPU | BPF_NOEXIST; + err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); + if (!ASSERT_EQ(err, -EINVAL, "bpf_map__update_elem cpu|noexist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_EXIST; + values[0] = test_value; + err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); + if (!ASSERT_OK(err, "bpf_map__update_elem all_cpus|exist")) + goto out; + + if (!verify_map_value_all_cpus(map_fd, key, values, elem_sz, test_value, nr_cpus)) + goto out; + + flags = BPF_F_CPU | BPF_EXIST; + values[0] = --test_value; + err = bpf_map__update_elem(map, key, key_sz, values, value_sz, flags); + if (!ASSERT_OK(err, "bpf_map__update_elem cpu|exist")) + goto out; + + verify_map_value_first_cpu(map_fd, key, values, elem_sz, test_value, nr_cpus); +out: + free(values); +} + static void test_percpu_cgroup_storage_cpu_flag(void) { struct percpu_alloc_array *skel = NULL; @@ -489,6 +578,7 @@ static void test_percpu_cgroup_storage_cpu_flag(void) goto out; test_percpu_map_op_cpu_flag(map, &key, sizeof(key), 1, nr_cpus, false); + test_percpu_cgroup_storage_flags_combination(map, nr_cpus, &key); out: bpf_prog_detach2(-1, cgroup, BPF_CGROUP_INET_EGRESS); close(cgroup); @@ -537,6 +627,164 @@ static void test_hash_cpu_flag(void) test_map_op_cpu_flag(BPF_MAP_TYPE_HASH); } +static void test_percpu_map_flags_combination(enum bpf_map_type map_type) +{ + u32 max_entries = 3, key = 0, first_value = 0xAABBCCDD, test_value = first_value; + size_t value_sz = sizeof(u32), elem_sz; + int err, map_fd, nr_cpus; + bool is_hash_map; + u32 *values; + u64 flags; + + is_hash_map = (map_type == BPF_MAP_TYPE_PERCPU_HASH || + map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) + return; + + elem_sz = roundup(value_sz, 8); + values = calloc(nr_cpus, elem_sz); + if (!ASSERT_OK_PTR(values, "calloc values")) + return; + + map_fd = bpf_map_create(map_type, "test_flags_combination_map", + sizeof(u32), value_sz, max_entries, NULL); + if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) { + free(values); + return; + } + + flags = BPF_NOEXIST | BPF_EXIST; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (!ASSERT_EQ(err, -EINVAL, "bpf_map_update_elem noexist|exist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_NOEXIST; + values[0] = test_value; + key = 0; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (is_hash_map) { + if (!ASSERT_OK(err, "bpf_map_update_elem all_cpus|noexist")) + goto out; + + if (!verify_map_value_all_cpus(map_fd, &key, values, elem_sz, test_value, nr_cpus)) + goto out; + + err = bpf_map_update_elem(map_fd, &key, values, flags); + } + if (!ASSERT_EQ(err, -EEXIST, "bpf_map_update_elem all_cpus|noexist")) + goto out; + + flags = BPF_F_CPU | BPF_NOEXIST; + values[0] = --test_value; + key = 1; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (is_hash_map) { + if (!ASSERT_OK(err, "bpf_map_update_elem cpu|noexist")) + goto out; + + if (!verify_map_value_first_cpu(map_fd, &key, values, elem_sz, test_value, nr_cpus)) + goto out; + + err = bpf_map_update_elem(map_fd, &key, values, flags); + } + if (!ASSERT_EQ(err, -EEXIST, "bpf_map_update_elem cpu|noexist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_EXIST; + values[0] = --test_value; + key = 2; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (is_hash_map) { + if (!ASSERT_EQ(err, -ENOENT, "bpf_map_update_elem all_cpus|exist")) + goto out; + } else { + if (!ASSERT_OK(err, "bpf_map_update_elem all_cpus|exist")) + goto out; + + if (!verify_map_value_all_cpus(map_fd, &key, values, elem_sz, test_value, nr_cpus)) + goto out; + } + + flags = BPF_F_CPU | BPF_EXIST; + values[0] = --test_value; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (is_hash_map) { + if (!ASSERT_EQ(err, -ENOENT, "bpf_map_update_elem cpu|exist")) + goto out; + } else { + if (!ASSERT_OK(err, "bpf_map_update_elem cpu|exist")) + goto out; + + if (!verify_map_value_first_cpu(map_fd, &key, values, elem_sz, test_value, nr_cpus)) + goto out; + } + + if (map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH) + goto out; + + /* Percpu LRU hash map should not delete old entries unnecessarily */ + flags = BPF_F_ALL_CPUS | BPF_NOEXIST; + key = 2; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (!ASSERT_OK(err, "bpf_map_update_elem lru_percpu_hash all_cpus|noexist")) + goto out; + + flags = BPF_F_ALL_CPUS | BPF_EXIST; + err = bpf_map_update_elem(map_fd, &key, values, flags); + if (!ASSERT_OK(err, "bpf_map_update_elem lru_percpu_hash all_cpus|exist")) + goto out; + + key = 0; + verify_map_value_all_cpus(map_fd, &key, values, elem_sz, first_value, nr_cpus); + +out: + close(map_fd); + free(values); +} + +static bool pin_current_cpu(cpu_set_t *old_mask) +{ + cpu_set_t new_mask; + int err, cpu; + + err = sched_getaffinity(0, sizeof(*old_mask), old_mask); + if (!ASSERT_OK(err, "sched_getaffinity")) + return false; + + cpu = sched_getcpu(); + if (!ASSERT_GE(cpu, 0, "sched_getcpu")) + return false; + + CPU_ZERO(&new_mask); + CPU_SET(cpu, &new_mask); + + err = sched_setaffinity(0, sizeof(new_mask), &new_mask); + return ASSERT_OK(err, "sched_setaffinity"); +} + +static void test_percpu_hash_flags_combination(void) +{ + test_percpu_map_flags_combination(BPF_MAP_TYPE_PERCPU_HASH); +} + +static void test_lru_percpu_hash_flags_combination(void) +{ + cpu_set_t old_mask; + + if (!pin_current_cpu(&old_mask)) + return; + + test_percpu_map_flags_combination(BPF_MAP_TYPE_LRU_PERCPU_HASH); + + sched_setaffinity(0, sizeof(old_mask), &old_mask); +} + +static void test_percpu_array_flags_combination(void) +{ + test_percpu_map_flags_combination(BPF_MAP_TYPE_PERCPU_ARRAY); +} + void test_percpu_alloc(void) { if (test__start_subtest("array")) @@ -565,4 +813,10 @@ void test_percpu_alloc(void) test_array_cpu_flag(); if (test__start_subtest("cpu_flag_hash")) test_hash_cpu_flag(); + if (test__start_subtest("cpu_flag_combination_percpu_hash")) + test_percpu_hash_flags_combination(); + if (test__start_subtest("cpu_flag_combination_lru_percpu_hash")) + test_lru_percpu_hash_flags_combination(); + if (test__start_subtest("cpu_flag_combination_percpu_array")) + test_percpu_array_flags_combination(); } diff --git a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c index 7607cfc2408c..ddb6cdc1579f 100644 --- a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c +++ b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c @@ -13,6 +13,7 @@ static void clear_test_state(struct test_state *state) void test_prog_tests_framework(void) { struct test_state *state = env.test_state; + int subtest_cnt; /* in all the ASSERT calls below we need to return on the first * error due to the fact that we are cleaning the test state after @@ -30,25 +31,30 @@ void test_prog_tests_framework(void) return; clear_test_state(state); + subtest_cnt = 0; if (test__start_subtest("test_skip_subtest")) { test__skip(); test__end_subtest(); + subtest_cnt++; } if (test__start_subtest("test_skip_subtest")) { test__skip(); test__end_subtest(); + subtest_cnt++; } - if (!ASSERT_EQ(state->skip_cnt, 2, "skip_cnt_check")) + if (!ASSERT_EQ(state->skip_cnt, subtest_cnt, "skip_cnt_check")) return; if (!ASSERT_EQ(state->subtest_num, 3, "subtest_num_check")) return; clear_test_state(state); + subtest_cnt = 0; if (test__start_subtest("test_fail_subtest")) { test__fail(); test__end_subtest(); + subtest_cnt++; } - if (!ASSERT_EQ(state->error_cnt, 1, "error_cnt_check")) + if (!ASSERT_EQ(state->error_cnt, subtest_cnt, "error_cnt_check")) return; if (!ASSERT_EQ(state->subtest_num, 4, "subtest_num_check")) return; @@ -163,6 +169,12 @@ void test_prog_tests_framework_expected_msgs(void) for (i = 0; i < ARRAY_SIZE(cases); i++) { if (test__start_subtest(cases[i].name)) { + if (!env.subtest_state->stdout_saved) { + test__skip(); + test__end_subtest(); + continue; + } + error_cnt = env.subtest_state->error_cnt; msgs.patterns = cases[i].pats; msgs.cnt = 0; @@ -179,3 +191,30 @@ void test_prog_tests_framework_expected_msgs(void) } } } + +void test_prog_tests_framework_compare_text(void) +{ + int err; + + if (test__start_subtest("compare_text_match")) { + err = compare_text_to_expected("same\n", "same\n"); + ASSERT_EQ(err, 0, "match_rc"); + test__end_subtest(); + } + + if (test__start_subtest("compare_text_mismatch")) { + if (!env.subtest_state->stdout_saved) { + test__skip(); + } else { + err = compare_text_to_expected("line two\n", "line one\n"); + fflush(stdout); + + ASSERT_EQ(err, -1, "mismatch_rc"); + ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, "-line one", + "diff_has_expected"); + ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, "+line two", + "diff_has_actual"); + } + test__end_subtest(); + } +} diff --git a/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c b/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c index 41441325e179..a01a0bcbdee7 100644 --- a/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c +++ b/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c @@ -2,6 +2,8 @@ #include <test_progs.h> #include <network_helpers.h> +#define U32_MAX ((u32)UINT_MAX) + enum { QUEUE, STACK, @@ -101,8 +103,35 @@ out: bpf_object__close(obj); } +static void test_queue_stack_map_alloc_check(void) +{ + int fd; + + fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, 1000000, 8192, NULL); + ASSERT_EQ(fd, -E2BIG, "queue_oversize"); + if (fd >= 0) + close(fd); + + fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, 1, U32_MAX, NULL); + ASSERT_EQ(fd, -E2BIG, "queue_u32max"); + if (fd >= 0) + close(fd); + + fd = bpf_map_create(BPF_MAP_TYPE_STACK, NULL, 0, 1000000, 8192, NULL); + ASSERT_EQ(fd, -E2BIG, "stack_oversize"); + if (fd >= 0) + close(fd); + + /* A normal-sized map must still be created successfully. */ + fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, 64, 100, NULL); + ASSERT_GE(fd, 0, "queue_normal"); + if (fd >= 0) + close(fd); +} + void test_queue_stack_map(void) { test_queue_stack_map_by_type(QUEUE); test_queue_stack_map_by_type(STACK); + test_queue_stack_map_alloc_check(); } diff --git a/tools/testing/selftests/bpf/prog_tests/select_reuseport.c b/tools/testing/selftests/bpf/prog_tests/select_reuseport.c index 3dbcc091f16c..3b4b233e21d1 100644 --- a/tools/testing/selftests/bpf/prog_tests/select_reuseport.c +++ b/tools/testing/selftests/bpf/prog_tests/select_reuseport.c @@ -6,6 +6,7 @@ #include <stdbool.h> #include <string.h> #include <errno.h> +#include <error.h> #include <assert.h> #include <fcntl.h> #include <linux/bpf.h> @@ -21,6 +22,7 @@ #include "bpf_util.h" #include "test_progs.h" +#include "socket_helpers.h" #include "test_select_reuseport_common.h" #define MAX_TEST_NAME 80 @@ -740,18 +742,6 @@ static const char *maptype_str(enum bpf_map_type type) } } -static const char *family_str(sa_family_t family) -{ - switch (family) { - case AF_INET: - return "IPv4"; - case AF_INET6: - return "IPv6"; - default: - return "unknown"; - } -} - static const char *sotype_str(int sotype) { switch (sotype) { @@ -794,7 +784,7 @@ static void test_config(int sotype, sa_family_t family, bool inany) snprintf(s, sizeof(s), "%s %s/%s %s %s", maptype_str(inner_map_type), - family_str(family), sotype_str(sotype), + inet_family_str(family), sotype_str(sotype), inany ? "INANY" : "LOOPBACK", t->name); if (!test__start_subtest(s)) diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c index 77381d345435..e43dba8fed8e 100644 --- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -32,11 +32,22 @@ enum { BPF_SIG_KEYRING_SECONDARY, BPF_SIG_KEYRING_PLATFORM, BPF_SIG_KEYRING_USER, + BPF_SIG_KEYRING_BPF, }; -static int load_loader(const void *insns, __u32 insns_sz, int map_fd, - const void *sig, __u32 sig_sz, __s32 keyring_id, - __u32 fd_array_cnt) +#ifndef KEY_SPEC_BPF_KEYRING +#define KEY_SPEC_BPF_KEYRING -9 +#endif + +/* verify_sig_setup.sh exits with this when openssl cannot do ML-DSA. */ +#define SETUP_SKIP (-77) + +/* FIPS-204 ML-DSA-87 signature size, see include/crypto/mldsa.h. */ +#define MLDSA87_SIGNATURE_SIZE 4627 + +static int load_loader_log(const void *insns, __u32 insns_sz, int map_fd, + const void *sig, __u32 sig_sz, __s32 keyring_id, + __u32 fd_array_cnt, char *log_buf, __u32 log_sz) { union bpf_attr attr; int fd; @@ -48,18 +59,31 @@ static int load_loader(const void *insns, __u32 insns_sz, int map_fd, attr.license = ptr_to_u64("Dual BSD/GPL"); attr.prog_flags = BPF_F_SLEEPABLE; attr.fd_array = ptr_to_u64(&map_fd); + attr.fd_array_cnt = fd_array_cnt; if (sig) { attr.signature = ptr_to_u64(sig); attr.signature_size = sig_sz; attr.keyring_id = keyring_id; } - attr.fd_array_cnt = fd_array_cnt; + if (log_buf) { + attr.log_level = 1; + attr.log_buf = ptr_to_u64(log_buf); + attr.log_size = log_sz; + } memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, offsetofend(union bpf_attr, keyring_id)); return fd < 0 ? -errno : fd; } +static int load_loader(const void *insns, __u32 insns_sz, int map_fd, + const void *sig, __u32 sig_sz, __s32 keyring_id, + __u32 fd_array_cnt) +{ + return load_loader_log(insns, insns_sz, map_fd, sig, sig_sz, keyring_id, + fd_array_cnt, NULL, 0); +} + static int run_gen_loader(const void *insns, __u32 insns_sz, const void *data, __u32 data_sz, const void *excl, __u32 excl_sz, @@ -156,12 +180,30 @@ static int run_setup(const char *cmd, const char *dir) } if (waitpid(pid, &status, 0) < 0) return -errno; - return (WIFEXITED(status) && - WEXITSTATUS(status) == 0) ? 0 : -EINVAL; + if (!WIFEXITED(status)) + return -EINVAL; + return -WEXITSTATUS(status); } -static int sign_buf(const char *dir, const void *buf, __u32 len, - void *sig, __u32 *sig_sz) +static void genkey_dir_fini(const char *dir) +{ + static const char * const files[] = { + "signing_key.der", "signing_key.pem", "x509.genkey", + }; + char path[PATH_MAX]; + size_t i; + + if (!dir) + return; + for (i = 0; i < ARRAY_SIZE(files); i++) { + snprintf(path, sizeof(path), "%s/%s", dir, files[i]); + unlink(path); + } + rmdir(dir); +} + +static int sign_buf_digest(const char *dir, const void *buf, __u32 len, + void *sig, __u32 *sig_sz, const char *digest) { char data_tmpl[PATH_MAX], key[PATH_MAX]; char sigpath[PATH_MAX + sizeof(".p7s")]; @@ -176,6 +218,7 @@ static int sign_buf(const char *dir, const void *buf, __u32 len, fd = mkstemp(data_tmpl); if (fd < 0) return -errno; + snprintf(sigpath, sizeof(sigpath), "%s.p7s", data_tmpl); if (write(fd, buf, len) != (ssize_t)len) { close(fd); ret = -EIO; @@ -190,7 +233,7 @@ static int sign_buf(const char *dir, const void *buf, __u32 len, } if (pid == 0) { snprintf(key, sizeof(key), "%s/signing_key.pem", dir); - execlp("./sign-file", "./sign-file", "-d", "sha256", + execlp("./sign-file", "./sign-file", "-d", digest, key, key, data_tmpl, NULL); exit(1); } @@ -200,34 +243,38 @@ static int sign_buf(const char *dir, const void *buf, __u32 len, goto out; } - snprintf(sigpath, sizeof(sigpath), "%s.p7s", data_tmpl); if (stat(sigpath, &st) < 0) { ret = -errno; goto out; } if (st.st_size > (off_t)*sig_sz) { ret = -E2BIG; - goto out_sig; + goto out; } fd = open(sigpath, O_RDONLY); if (fd < 0) { ret = -errno; - goto out_sig; + goto out; } if (read(fd, sig, st.st_size) != st.st_size) { close(fd); ret = -EIO; - goto out_sig; + goto out; } close(fd); *sig_sz = st.st_size; -out_sig: - unlink(sigpath); out: + unlink(sigpath); unlink(data_tmpl); return ret; } +static int sign_buf(const char *dir, const void *buf, __u32 len, + void *sig, __u32 *sig_sz) +{ + return sign_buf_digest(dir, buf, len, sig, sig_sz, "sha256"); +} + struct gen_loader_fixture { struct test_signed_loader *skel; struct gen_loader_opts gopts; @@ -332,6 +379,51 @@ static void signature_enforced(void) gen_loader_fixture_fini(&f); } +/* + * CO-RE relocations would have the kernel rewrite the insns after the + * signature was checked over them, with values the loader picks through the + * BTF it supplies. A signed load that carries any is rejected before the + * signature is examined, so junk serves as one. + */ +static void signed_core_relo_rejected(void) +{ + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + struct bpf_core_relo relo = { + .insn_off = 0, + .type_id = 1, + .kind = BPF_CORE_TYPE_ID_LOCAL, + }; + char log_buf[1024] = {}; + union bpf_attr attr; + int fd; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.signature = ptr_to_u64(junk); + attr.signature_size = sizeof(junk); + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + attr.core_relos = ptr_to_u64(&relo); + attr.core_relo_cnt = 1; + attr.core_relo_rec_size = sizeof(relo); + attr.log_level = 1; + attr.log_buf = ptr_to_u64(log_buf); + attr.log_size = sizeof(log_buf); + fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, offsetofend(union bpf_attr, keyring_id)); + fd = fd < 0 ? -errno : fd; + ASSERT_EQ(fd, -EACCES, "signed load with CO-RE relocations rejected"); + ASSERT_HAS_SUBSTR(log_buf, "signed program cannot carry CO-RE relocations", "log"); + if (fd >= 0) + close(fd); +} + static void signed_nonexcl_fd_array_rejected(void) { static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; @@ -457,7 +549,7 @@ static void signed_btf_fd_array_rejected(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -529,7 +621,6 @@ static void signature_failure_logs(void) static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; char log_buf[1024] = {}; struct gen_loader_fixture f; - union bpf_attr attr; int fd; if (gen_loader_fixture_init(&f) == 0) { @@ -538,22 +629,9 @@ static void signature_failure_logs(void) * failure is reported through the verifier log. A present-but- * invalid signature is rejected and the log says why. */ - memset(&attr, 0, sizeof(attr)); - attr.prog_type = BPF_PROG_TYPE_SYSCALL; - attr.insns = ptr_to_u64(f.gopts.insns); - attr.insn_cnt = f.gopts.insns_sz / sizeof(struct bpf_insn); - attr.license = ptr_to_u64("Dual BSD/GPL"); - attr.prog_flags = BPF_F_SLEEPABLE; - attr.signature = ptr_to_u64(junk); - attr.signature_size = sizeof(junk); - attr.keyring_id = KEY_SPEC_SESSION_KEYRING; - attr.log_level = 1; - attr.log_buf = ptr_to_u64(log_buf); - attr.log_size = sizeof(log_buf); - memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); - - fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, - offsetofend(union bpf_attr, keyring_id)); + fd = load_loader_log(f.gopts.insns, f.gopts.insns_sz, -1, junk, + sizeof(junk), KEY_SPEC_SESSION_KEYRING, 0, + log_buf, sizeof(log_buf)); ASSERT_LT(fd, 0, "invalid signature rejected at load"); if (fd >= 0) close(fd); @@ -571,8 +649,9 @@ static void signature_too_large(void) if (gen_loader_fixture_init(&f) == 0) { /* - * signature_size beyond the kernel's bound (KMALLOC_MAX_CACHE_SIZE) - * is rejected before the buffer is read. + * signature_size beyond the kernel's bound + * (BPF_PROG_MAX_SIGNATURE_SIZE) is rejected before the buffer + * is read. */ fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, 64 << 20, KEY_SPEC_SESSION_KEYRING, 0); @@ -626,6 +705,280 @@ static void signature_bad_keyring(void) gen_loader_fixture_fini(&f); } +static bool keyring_unsealed_boot(void) +{ + char val = 0; + int fd; + + fd = open("/sys/module/bpf/parameters/keyring_unsealed", O_RDONLY); + if (fd < 0) + return false; + if (read(fd, &val, 1) != 1) + val = 0; + close(fd); + return val == 'Y' || val == '1'; +} + +static int bpf_keyring_lookup(int *nr_keys) +{ + char line[512], type[32], desc[64]; + int serial = -ENOENT; + FILE *f; + + f = fopen("/proc/keys", "r"); + if (!f) + return -errno; + + while (fgets(line, sizeof(line), f)) { + unsigned int hex; + char *sum; + + if (sscanf(line, "%x %*s %*s %*s %*s %*s %*s %31s %63s", + &hex, type, desc) != 3) + continue; + if (strcmp(type, "keyring") || strcmp(desc, ".bpf:")) + continue; + + serial = (int)hex; + if (nr_keys) { + sum = strstr(line, ".bpf: "); + *nr_keys = !sum || !strncmp(sum + 6, "empty", 5) ? + 0 : atoi(sum + 6); + } + break; + } + fclose(f); + return serial; +} + +static long keyctl_ret(int cmd, unsigned long arg2, unsigned long arg3) +{ + long ret = syscall(__NR_keyctl, cmd, arg2, arg3); + + return ret < 0 ? -errno : ret; +} + +/* + * What the bpf keyring still needs once it got provisioned: KEY_POS_SEARCH + * for the in-kernel search during verification, and the user view/read bits + * so it stays visible in /proc/keys, rest is dropped so the enrolled is + * therefore final. + */ +#define BPF_KEYRING_PERM_LOCKED 0x08030000 +/* What bpf_keyring_init() grants at boot. */ +#define BPF_KEYRING_PERM_INITIAL 0x082f0000 + +static void bpf_keyring_sealed(void) +{ + static const __u8 junk[64] = {}; + struct gen_loader_fixture f; + int serial, key, fd; + + if (keyring_unsealed_boot()) { + printf("%s:SKIP:the bpf keyring was unsealed at boot\n", __func__); + test__skip(); + return; + } + serial = bpf_keyring_lookup(NULL); + if (serial >= 0) { + ASSERT_EQ(keyctl_ret(KEYCTL_GET_KEYRING_ID, + KEY_SPEC_BPF_KEYRING, 0), serial, + "KEY_SPEC_BPF_KEYRING resolves to the bpf keyring"); + key = syscall(__NR_add_key, "user", "sealprobe", "x", 1, + KEY_SPEC_BPF_KEYRING); + if (key >= 0) + syscall(__NR_keyctl, KEYCTL_UNLINK, key, + KEY_SPEC_BPF_KEYRING); + ASSERT_EQ(key < 0 ? -errno : 0, -EPERM, + "nothing links into a sealed keyring"); + } + if (gen_loader_fixture_init(&f) == 0) { + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, + sizeof(junk), KEY_SPEC_BPF_KEYRING, 0); + ASSERT_EQ(fd, -ENOKEY, "sealed bpf keyring rejected"); + if (fd >= 0) + close(fd); + } + gen_loader_fixture_fini(&f); +} + +static int try_load(const struct gen_loader_fixture *f, const void *sig, + __u32 sig_sz, __s32 keyring_id, char *log_buf, __u32 log_sz) +{ + int map_fd, prog_fd; + + map_fd = setup_meta_map(f); + if (!ASSERT_OK_FD(map_fd, "meta_map")) + return map_fd; + prog_fd = load_loader_log(f->gopts.insns, f->gopts.insns_sz, map_fd, + sig, sig_sz, keyring_id, 1, log_buf, log_sz); + close(map_fd); + if (prog_fd >= 0) + close(prog_fd); + return prog_fd; +} + +/* + * This needs bpf.keyring_unsealed=1 on the guest kernel command line, which + * vmtest.sh can pass via KERNEL_CMDLINE_EXTRA. There is no way to unseal the + * keyring from here, so without it the test skips. It also only works once + * per boot, as restricting a keyring cannot be undone. + */ +static void bpf_keyring_provisioned(void) +{ + char dir_tmpl[] = "/tmp/bpfkeyringXXXXXX"; + char bad_tmpl[] = "/tmp/bpfkeyringbadXXXXXX"; + __u8 *sig = NULL, *bad = NULL, *buf = NULL; + int serial, err; + int nr_keys = 0, der_fd = -1; + struct gen_loader_fixture f; + __u32 sig_sz = 8192, bad_sz; + bool have_fixture = false; + char *dir, *bad_dir = NULL; + char log_buf[1024] = {}; + char path[PATH_MAX]; + __u8 der[4096]; + ssize_t der_sz; + + serial = bpf_keyring_lookup(&nr_keys); + if (serial < 0) { + printf("%s:SKIP:no bpf keyring (needs CONFIG_KEYS)\n", __func__); + test__skip(); + return; + } + if (nr_keys != 0) { + printf("%s:SKIP:the bpf keyring has already been provisioned\n", + __func__); + test__skip(); + return; + } + + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("genkey", dir), "verify_sig_setup genkey")) + goto rmdir; + + snprintf(path, sizeof(path), "%s/signing_key.der", dir); + der_fd = open(path, O_RDONLY); + if (!ASSERT_OK_FD(der_fd, "open signing_key.der")) + goto rmdir; + der_sz = read(der_fd, der, sizeof(der)); + close(der_fd); + if (!ASSERT_GT(der_sz, 0, "read signing_key.der")) + goto rmdir; + + ASSERT_EQ(keyctl_ret(KEYCTL_GET_KEYRING_ID, KEY_SPEC_BPF_KEYRING, 0), + serial, "KEY_SPEC_BPF_KEYRING resolves to the bpf keyring"); + + err = syscall(__NR_add_key, "asymmetric", "", der, (size_t)der_sz, + KEY_SPEC_BPF_KEYRING); + if (err < 0 && errno == EPERM) { + printf("%s:SKIP:the bpf keyring is sealed, need bpf.keyring_unsealed=1\n", + __func__); + test__skip(); + goto rmdir; + } + if (!ASSERT_GE(err, 0, "add the signing key to the bpf keyring")) + goto rmdir; + + sig = malloc(sig_sz); + if (!ASSERT_OK_PTR(sig, "sig buf")) + goto out; + have_fixture = true; + if (gen_loader_fixture_init(&f) != 0) + goto out; + + buf = malloc((size_t)f.gopts.insns_sz + f.data_sz); + if (!ASSERT_OK_PTR(buf, "signbuf")) + goto out; + memcpy(buf, f.gopts.insns, f.gopts.insns_sz); + memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz); + if (!ASSERT_OK(sign_buf(dir, buf, f.gopts.insns_sz + f.data_sz, sig, + &sig_sz), "sign insns||metadata")) + goto out; + + ASSERT_EQ(try_load(&f, sig, sig_sz, KEY_SPEC_BPF_KEYRING, NULL, 0), + -ENOKEY, "unrestricted keyring still not consulted"); + + ASSERT_EQ(try_load(&f, sig, sig_sz, KEY_SPEC_SESSION_KEYRING, NULL, 0), + -EPERM, "caller-supplied keyring refused before provisioning"); + + if (!ASSERT_OK(syscall(__NR_keyctl, KEYCTL_RESTRICT_KEYRING, + KEY_SPEC_BPF_KEYRING, NULL, NULL), + "restrict bpf keyring")) + goto out; + + if (!ASSERT_OK_FD(try_load(&f, sig, sig_sz, KEY_SPEC_BPF_KEYRING, + NULL, 0), + "load signed by a key in the .bpf keyring")) + goto out; + + bad_dir = mkdtemp(bad_tmpl); + if (!ASSERT_OK_PTR(bad_dir, "mkdtemp unenrolled")) + goto out; + if (!ASSERT_OK(run_setup("genkey", bad_dir), "verify_sig_setup genkey unenrolled")) + goto out; + bad_sz = 8192; + bad = malloc(bad_sz); + if (!ASSERT_OK_PTR(bad, "bad sig buf")) + goto out; + if (!ASSERT_OK(sign_buf(bad_dir, buf, f.gopts.insns_sz + f.data_sz, bad, + &bad_sz), "sign with an unenrolled key")) + goto out; + + ASSERT_EQ(try_load(&f, bad, bad_sz, KEY_SPEC_BPF_KEYRING, log_buf, + sizeof(log_buf)), -ENOKEY, + "key outside the bpf keyring refused"); + ASSERT_HAS_SUBSTR(log_buf, "signature verification failed", + "the bpf keyring was consulted"); + + f.blob[0] ^= 0xff; + err = try_load(&f, sig, sig_sz, KEY_SPEC_BPF_KEYRING, NULL, 0); + f.blob[0] ^= 0xff; + ASSERT_EQ(err, -EKEYREJECTED, "tampered metadata refused"); + + ASSERT_EQ(try_load(&f, sig, sig_sz, KEY_SPEC_SESSION_KEYRING, NULL, 0), + -EPERM, "caller-supplied keyring refused once .bpf is in use"); + + err = keyctl_ret(KEYCTL_UNLINK, KEY_SPEC_SESSION_KEYRING, serial); + ASSERT_EQ(err, -ENOENT, "keyring writable while the user bits are there"); + + err = keyctl_ret(KEYCTL_SETPERM, serial, BPF_KEYRING_PERM_LOCKED); + if (!ASSERT_OK(err, "drop the user bits on the bpf keyring")) + goto out; + + ASSERT_OK_FD(try_load(&f, sig, sig_sz, KEY_SPEC_BPF_KEYRING, NULL, 0), + "load still verified against the locked keyring"); + ASSERT_EQ(keyctl_ret(KEYCTL_GET_KEYRING_ID, KEY_SPEC_BPF_KEYRING, 0), + -EACCES, "the special id grants no rights of its own"); + + err = keyctl_ret(KEYCTL_UNLINK, KEY_SPEC_SESSION_KEYRING, serial); + ASSERT_EQ(err, -EACCES, "unlink refused"); + err = keyctl_ret(KEYCTL_CLEAR, serial, 0); + ASSERT_EQ(err, -EACCES, "clear refused"); + err = keyctl_ret(KEYCTL_REVOKE, serial, 0); + ASSERT_EQ(err, -EACCES, "revoke refused"); + err = keyctl_ret(KEYCTL_INVALIDATE, serial, 0); + ASSERT_EQ(err, -EACCES, "invalidate refused"); + err = keyctl_ret(KEYCTL_SET_TIMEOUT, serial, 1); + ASSERT_EQ(err, -EACCES, "timeout refused"); + err = keyctl_ret(KEYCTL_SETPERM, serial, BPF_KEYRING_PERM_INITIAL); + ASSERT_EQ(err, -EACCES, "the bits cannot be granted back"); + + ASSERT_EQ(bpf_keyring_lookup(&nr_keys), serial, "keyring still there"); + ASSERT_EQ(nr_keys, 1, "the enrolled key survived"); +out: + if (have_fixture) + gen_loader_fixture_fini(&f); + genkey_dir_fini(bad_dir); + free(buf); + free(bad); + free(sig); +rmdir: + genkey_dir_fini(dir); +} + /* * A signed loader must ignore ctx-supplied map dimensions: the host cannot * resize a signed program's maps via the loader ctx. Drive a one-map program @@ -831,7 +1184,7 @@ static void signature_authenticates_insns(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -931,7 +1284,7 @@ static void signature_authenticates_metadata(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -1267,7 +1620,7 @@ static void lsm_signature_verdict(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) goto out; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); dir = NULL; goto out; @@ -1450,7 +1803,7 @@ static void loadtime_verify(struct bpf_object *obj, int expect_maps) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -1525,6 +1878,89 @@ static void loadtime_with_map(void) } /* + * End-to-end signed load with a post-quantum key. ML-DSA (FIPS-204) is wired + * through the X.509 and PKCS#7 parsers, and BPF reaches them via + * verify_pkcs7_signature() without knowing the algorithm, so an ML-DSA key in + * the keyring should verify an ML-DSA signed program with no BPF-side work. + */ +static void mldsa_signed_load(void) +{ + char dir_tmpl[] = "/tmp/bpfmldsaXXXXXX"; + int map_fd = -1, prog_fd = -1, err; + __u8 *sig = NULL, *buf = NULL; + struct gen_loader_fixture f; + bool have_fixture = false; + __u32 sig_sz = 16384; + char *dir; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + + err = run_setup("setup-mldsa", dir); + if (err == SETUP_SKIP) { + printf("%s:SKIP:no working ML-DSA signing, set SELFTESTS_VERBOSE=1\n", + __func__); + test__skip(); + genkey_dir_fini(dir); + return; + } + if (!ASSERT_OK(err, "verify_sig_setup setup-mldsa")) { + genkey_dir_fini(dir); + return; + } + + sig = malloc(sig_sz); + if (!ASSERT_OK_PTR(sig, "sig buf")) + goto out; + have_fixture = true; + if (gen_loader_fixture_init(&f) != 0) + goto out; + + buf = malloc((size_t)f.gopts.insns_sz + f.data_sz); + if (!ASSERT_OK_PTR(buf, "signbuf")) + goto out; + memcpy(buf, f.gopts.insns, f.gopts.insns_sz); + memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz); + + /* + * ML-DSA hashes the message itself, but openssl before 4.0 cannot + * produce a CMS message without signedAttrs for it, and with those in + * play only SHA-512 is permitted for the messageDigest attribute. + */ + if (!ASSERT_OK(sign_buf_digest(dir, buf, f.gopts.insns_sz + f.data_sz, + sig, &sig_sz, "sha512"), + "sign insns||metadata with ML-DSA")) + goto out; + + /* + * Guard against the setup silently handing back some other key type: + * an RSA or ECDSA signature is a few hundred bytes, where an ML-DSA-87 + * one cannot be smaller than the raw signature it carries. + */ + ASSERT_GT(sig_sz, MLDSA87_SIGNATURE_SIZE, "ML-DSA-87 signature size"); + + map_fd = setup_meta_map(&f); + if (!ASSERT_OK_FD(map_fd, "meta_map")) + goto out; + prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, sig, + sig_sz, KEY_SPEC_SESSION_KEYRING, 1); + ASSERT_OK_FD(prog_fd, "ML-DSA signed loader load"); +out: + if (prog_fd >= 0) + close(prog_fd); + if (map_fd >= 0) + close(map_fd); + if (have_fixture) + gen_loader_fixture_fini(&f); + free(buf); + free(sig); + run_setup("cleanup", dir); +} + +/* * A signed program need not bind any map. A plain BPF_PROG_TYPE_SYSCALL * program with no fd_array is signed over its instructions alone: the kernel * verifies the signature, folds no metadata, and the program loads. Exercise @@ -1548,7 +1984,7 @@ static void signed_no_fd_array(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -1619,7 +2055,7 @@ static void signed_map_by_fd_rejected(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) goto out_map; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); goto out_map; } @@ -1681,7 +2117,7 @@ static void signed_sparse_fd_array_rejected(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) goto out_map; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); goto out_map; } @@ -1735,7 +2171,7 @@ static void signed_module_kfunc_rejected(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -1777,64 +2213,72 @@ cleanup: run_setup("cleanup", dir); } +enum subtest_boot { + BOOT_ANY, + BOOT_SEALED, + BOOT_UNSEALED, +}; + +static const struct { + const char *name; + void (*fn)(void); + enum subtest_boot boot; +} subtests[] = { + { "loadtime_no_map", loadtime_no_map, BOOT_SEALED }, + { "loadtime_with_map", loadtime_with_map, BOOT_SEALED }, + { "metadata_match", metadata_match, BOOT_ANY }, + { "signature_enforced", signature_enforced, BOOT_SEALED }, + { "signed_nonexcl_fd_array_rejected", signed_nonexcl_fd_array_rejected, BOOT_SEALED }, + { "signed_unfrozen_fd_array_rejected", signed_unfrozen_fd_array_rejected, BOOT_SEALED }, + { "signed_nonarray_fd_array_rejected", signed_nonarray_fd_array_rejected, BOOT_SEALED }, + { "signed_btf_fd_array_rejected", signed_btf_fd_array_rejected, BOOT_ANY }, + { "signed_core_relo_rejected", signed_core_relo_rejected, BOOT_ANY }, + { "signed_module_kfunc_rejected", signed_module_kfunc_rejected, BOOT_SEALED }, + { "signature_failure_logs", signature_failure_logs, BOOT_SEALED }, + { "signature_too_large", signature_too_large, BOOT_ANY }, + { "signature_zero_size", signature_zero_size, BOOT_ANY }, + { "signature_bad_keyring", signature_bad_keyring, BOOT_SEALED }, + { "bpf_keyring_sealed", bpf_keyring_sealed, BOOT_ANY }, + { "mldsa_signed_load", mldsa_signed_load, BOOT_SEALED }, + { "metadata_ctx_max_entries_ignored", metadata_ctx_max_entries_ignored, BOOT_ANY }, + { "metadata_ctx_initial_value_ignored", metadata_ctx_initial_value_ignored, BOOT_ANY }, + { "signature_authenticates_insns", signature_authenticates_insns, BOOT_SEALED }, + { "signature_authenticates_metadata", signature_authenticates_metadata, BOOT_SEALED }, + { "hash_requires_frozen", hash_requires_frozen, BOOT_ANY }, + { "no_update_after_freeze", no_update_after_freeze, BOOT_ANY }, + { "freeze_writable_mmap", freeze_writable_mmap, BOOT_ANY }, + { "no_writable_mmap_frozen", no_writable_mmap_frozen, BOOT_ANY }, + { "map_hash_matches_libbpf", map_hash_matches_libbpf, BOOT_ANY }, + { "map_hash_multi_element", map_hash_multi_element, BOOT_ANY }, + { "map_hash_bad_size", map_hash_bad_size, BOOT_ANY }, + { "map_hash_unsupported_type", map_hash_unsupported_type, BOOT_ANY }, + { "lsm_signature_verdict", lsm_signature_verdict, BOOT_SEALED }, + { "signed_no_fd_array", signed_no_fd_array, BOOT_SEALED }, + { "signed_map_by_fd_rejected", signed_map_by_fd_rejected, BOOT_SEALED }, + { "signed_sparse_fd_array_rejected", signed_sparse_fd_array_rejected, BOOT_SEALED }, + { "bpf_keyring_provisioned", bpf_keyring_provisioned, BOOT_UNSEALED }, +}; + void test_signed_loader(void) { - if (test__start_subtest("loadtime_no_map")) - loadtime_no_map(); - if (test__start_subtest("loadtime_with_map")) - loadtime_with_map(); - if (test__start_subtest("metadata_match")) - metadata_match(); - if (test__start_subtest("signature_enforced")) - signature_enforced(); - if (test__start_subtest("signed_nonexcl_fd_array_rejected")) - signed_nonexcl_fd_array_rejected(); - if (test__start_subtest("signed_unfrozen_fd_array_rejected")) - signed_unfrozen_fd_array_rejected(); - if (test__start_subtest("signed_nonarray_fd_array_rejected")) - signed_nonarray_fd_array_rejected(); - if (test__start_subtest("signed_btf_fd_array_rejected")) - signed_btf_fd_array_rejected(); - if (test__start_subtest("signed_module_kfunc_rejected")) - signed_module_kfunc_rejected(); - if (test__start_subtest("signature_failure_logs")) - signature_failure_logs(); - if (test__start_subtest("signature_too_large")) - signature_too_large(); - if (test__start_subtest("signature_zero_size")) - signature_zero_size(); - if (test__start_subtest("signature_bad_keyring")) - signature_bad_keyring(); - if (test__start_subtest("metadata_ctx_max_entries_ignored")) - metadata_ctx_max_entries_ignored(); - if (test__start_subtest("metadata_ctx_initial_value_ignored")) - metadata_ctx_initial_value_ignored(); - if (test__start_subtest("signature_authenticates_insns")) - signature_authenticates_insns(); - if (test__start_subtest("signature_authenticates_metadata")) - signature_authenticates_metadata(); - if (test__start_subtest("hash_requires_frozen")) - hash_requires_frozen(); - if (test__start_subtest("no_update_after_freeze")) - no_update_after_freeze(); - if (test__start_subtest("freeze_writable_mmap")) - freeze_writable_mmap(); - if (test__start_subtest("no_writable_mmap_frozen")) - no_writable_mmap_frozen(); - if (test__start_subtest("map_hash_matches_libbpf")) - map_hash_matches_libbpf(); - if (test__start_subtest("map_hash_multi_element")) - map_hash_multi_element(); - if (test__start_subtest("map_hash_bad_size")) - map_hash_bad_size(); - if (test__start_subtest("map_hash_unsupported_type")) - map_hash_unsupported_type(); - if (test__start_subtest("lsm_signature_verdict")) - lsm_signature_verdict(); - if (test__start_subtest("signed_no_fd_array")) - signed_no_fd_array(); - if (test__start_subtest("signed_map_by_fd_rejected")) - signed_map_by_fd_rejected(); - if (test__start_subtest("signed_sparse_fd_array_rejected")) - signed_sparse_fd_array_rejected(); + bool unsealed = keyring_unsealed_boot(); + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(subtests); i++) { + if (!test__start_subtest(subtests[i].name)) + continue; + if (subtests[i].boot == BOOT_SEALED && unsealed) { + printf("%s:SKIP:needs a boot without bpf.keyring_unsealed=1\n", + subtests[i].name); + test__skip(); + continue; + } + if (subtests[i].boot == BOOT_UNSEALED && !unsealed) { + printf("%s:SKIP:needs bpf.keyring_unsealed=1\n", + subtests[i].name); + test__skip(); + continue; + } + subtests[i].fn(); + } } diff --git a/tools/testing/selftests/bpf/prog_tests/skb_csum_complete.c b/tools/testing/selftests/bpf/prog_tests/skb_csum_complete.c new file mode 100644 index 000000000000..919f24dbacd3 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/skb_csum_complete.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <network_helpers.h> +#include "skb_csum_complete.skel.h" + +static void run(struct bpf_program *prog, int expected_err) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .repeat = 1, + .flags = BPF_F_TEST_SKB_CHECKSUM_COMPLETE, + ); + int err; + + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &topts); + if (!ASSERT_EQ(err, expected_err, "test_run")) + return; + if (!expected_err) + ASSERT_EQ(topts.retval, 0, "retval"); +} + +void test_skb_csum_complete(void) +{ + struct skb_csum_complete *skel; + + skel = skb_csum_complete__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + if (test__start_subtest("change_tail_trim")) + run(skel->progs.change_tail_trim, 0); + if (test__start_subtest("change_tail_grow_write")) + run(skel->progs.change_tail_grow_write, 0); + if (test__start_subtest("store_no_recompute")) + run(skel->progs.store_no_recompute, -EBADMSG); + + skb_csum_complete__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/snprintf.c b/tools/testing/selftests/bpf/prog_tests/snprintf.c index 4e4a82d54f79..ccfb02bd4e7c 100644 --- a/tools/testing/selftests/bpf/prog_tests/snprintf.c +++ b/tools/testing/selftests/bpf/prog_tests/snprintf.c @@ -15,8 +15,8 @@ #define EXP_SYM_OUT "schedule schedule+0x0/" #define MIN_SYM_RET sizeof(EXP_SYM_OUT) -/* The third specifier, %p, is a hashed pointer which changes on every reboot */ -#define EXP_ADDR_OUT "0000000000000000 ffff00000add4e55 " +/* The second specifier, %p, is a hashed pointer which changes on every reboot */ +#define EXP_ADDR_OUT "ffff00000add4e55 " #define EXP_ADDR_RET sizeof(EXP_ADDR_OUT "unknownhashedptr") #define EXP_STR_OUT "str1 a b c d e longstr" diff --git a/tools/testing/selftests/bpf/prog_tests/socket_helpers.h b/tools/testing/selftests/bpf/prog_tests/socket_helpers.h index 0d59503a0c73..4a570998612d 100644 --- a/tools/testing/selftests/bpf/prog_tests/socket_helpers.h +++ b/tools/testing/selftests/bpf/prog_tests/socket_helpers.h @@ -470,4 +470,16 @@ static inline const char *socket_kind_to_str(int sock_fd) return "???"; } +static inline const char *inet_family_str(sa_family_t family) +{ + switch (family) { + case AF_INET: + return "IPv4"; + case AF_INET6: + return "IPv6"; + default: + return "unknown"; + } +} + #endif // __SOCKET_HELPERS__ diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c index 1c96a3cf4b97..235e1d56f044 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c @@ -1232,22 +1232,6 @@ static void test_ops_cleanup(const struct bpf_map *map) } } -static const char *family_str(sa_family_t family) -{ - switch (family) { - case AF_INET: - return "IPv4"; - case AF_INET6: - return "IPv6"; - case AF_UNIX: - return "Unix"; - case AF_VSOCK: - return "VSOCK"; - default: - return "unknown"; - } -} - static const char *map_type_str(const struct bpf_map *map) { int type; @@ -1315,7 +1299,7 @@ static void test_ops(struct test_sockmap_listen *skel, struct bpf_map *map, char s[MAX_TEST_NAME]; int map_fd; - family_name = family_str(family); + family_name = inet_family_str(family); map_name = map_type_str(map); sotype_name = sotype_str(sotype); map_fd = bpf_map__fd(map); @@ -1355,7 +1339,7 @@ static void test_redir(struct test_sockmap_listen *skel, struct bpf_map *map, const struct redir_test *t; char s[MAX_TEST_NAME]; - family_name = family_str(family); + family_name = inet_family_str(family); map_name = map_type_str(map); for (t = tests; t < tests + ARRAY_SIZE(tests); t++) { @@ -1387,7 +1371,7 @@ static void test_reuseport(struct test_sockmap_listen *skel, const struct reuseport_test *t; char s[MAX_TEST_NAME]; - family_name = family_str(family); + family_name = inet_family_str(family); map_name = map_type_str(map); sotype_name = sotype_str(sotype); diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt.c b/tools/testing/selftests/bpf/prog_tests/sockopt.c index 6c96f2d9fccf..7dfdc4c12dec 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockopt.c +++ b/tools/testing/selftests/bpf/prog_tests/sockopt.c @@ -955,7 +955,7 @@ static struct sockopt_test { /* ==================== prog_type ==================== */ { - .descr = "can attach only BPF_CGROUP_SETSOCKOP", + .descr = "can attach only BPF_CGROUP_SETSOCKOPT", .insns = { /* return 1 */ BPF_MOV64_IMM(BPF_REG_0, 1), @@ -969,7 +969,7 @@ static struct sockopt_test { }, { - .descr = "can attach only BPF_CGROUP_GETSOCKOP", + .descr = "can attach only BPF_CGROUP_GETSOCKOPT", .insns = { /* return 1 */ BPF_MOV64_IMM(BPF_REG_0, 1), diff --git a/tools/testing/selftests/bpf/prog_tests/stream.c b/tools/testing/selftests/bpf/prog_tests/stream.c index e4e9374309e2..ae400dcfbf24 100644 --- a/tools/testing/selftests/bpf/prog_tests/stream.c +++ b/tools/testing/selftests/bpf/prog_tests/stream.c @@ -1,11 +1,17 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ #include <test_progs.h> +#include <linux/perf_event.h> +#include <poll.h> +#include <sys/epoll.h> #include <sys/mman.h> +#include <sys/syscall.h> #include "stream.skel.h" #include "stream_fail.skel.h" +#define NMI_TIMEOUT_NS (5ULL * 1000 * 1000 * 1000) + void test_stream_failure(void) { RUN_TESTS(stream_fail); @@ -46,6 +52,10 @@ void test_stream_syscall(void) ret = -errno; ASSERT_EQ(ret, -EFAULT, "bad stream buf"); + ASSERT_LT(bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, buf, UINT_MAX, NULL), 0, "error"); + ret = -errno; + ASSERT_EQ(ret, -EINVAL, "large stream buf"); + ret = bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, buf, 2, NULL); ASSERT_EQ(ret, 2, "bytes"); ret = bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, buf, 2, NULL); @@ -58,6 +68,549 @@ void test_stream_syscall(void) stream__destroy(skel); } +static bool stream_fd_trigger(struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int ret; + + ret = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + return ASSERT_OK(ret, "test_run") && ASSERT_OK(opts.retval, "retval"); +} + +static void test_stream_fd_open(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts); + struct stream *skel; + int fd, prog_fd; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + prog_fd = bpf_program__fd(skel->progs.stream_syscall); + fd = bpf_prog_stream_open(0, BPF_STREAM_STDOUT, NULL); + ASSERT_EQ(fd, -EINVAL, "bad_prog_fd"); + + fd = bpf_prog_stream_open(prog_fd, 0, NULL); + ASSERT_EQ(fd, -ENOENT, "bad_stream_id"); + + opts.flags = BPF_F_RDONLY; + fd = bpf_prog_stream_open(prog_fd, BPF_STREAM_STDOUT, &opts); + ASSERT_EQ(fd, -EINVAL, "access_flag"); + + opts.flags = 1U << 31; + fd = bpf_prog_stream_open(prog_fd, BPF_STREAM_STDOUT, &opts); + ASSERT_EQ(fd, -EINVAL, "unknown_flag"); + + fd = bpf_prog_stream_open(prog_fd, BPF_STREAM_STDERR, NULL); + if (ASSERT_OK_FD(fd, "stderr")) + close(fd); + + stream__destroy(skel); +} + +static void test_stream_fd_nonblock(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts, + .flags = BPF_F_STREAM_NONBLOCK, + ); + struct pollfd pfd = { .events = POLLIN | POLLHUP }; + struct stream *skel; + char buf[4] = {}; + int fd, flags, ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_syscall), + BPF_STREAM_STDOUT, &opts); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out_destroy; + pfd.fd = fd; + + flags = fcntl(fd, F_GETFD); + ASSERT_GE(flags, 0, "getfd"); + ASSERT_NEQ(flags & FD_CLOEXEC, 0, "cloexec"); + flags = fcntl(fd, F_GETFL); + ASSERT_GE(flags, 0, "getfl"); + ASSERT_EQ(flags & O_ACCMODE, O_RDONLY, "readonly"); + ASSERT_NEQ(flags & O_NONBLOCK, 0, "nonblock"); + + ret = write(fd, "x", 1); + ASSERT_EQ(ret, -1, "write"); + ASSERT_EQ(errno, EBADF, "write_errno"); + ret = lseek(fd, 0, SEEK_SET); + ASSERT_EQ(ret, -1, "lseek"); + ASSERT_EQ(errno, ESPIPE, "lseek_errno"); + + ret = poll(&pfd, 1, 0); + ASSERT_EQ(ret, 0, "poll_empty"); + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, -1, "read_empty"); + ASSERT_EQ(errno, EAGAIN, "read_empty_errno"); + + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_close; + ret = poll(&pfd, 1, 0); + ASSERT_EQ(ret, 1, "poll_data"); + ASSERT_NEQ(pfd.revents & POLLIN, 0, "pollin"); + ASSERT_EQ(pfd.revents & POLLHUP, 0, "no_pollhup"); + + ret = read(fd, buf, 2); + ASSERT_EQ(ret, 2, "read_first"); + ASSERT_OK(memcmp(buf, "fo", 2), "read_first_data"); + pfd.revents = 0; + ret = poll(&pfd, 1, 0); + ASSERT_EQ(ret, 1, "poll_partial"); + ASSERT_NEQ(pfd.revents & POLLIN, 0, "pollin_partial"); + + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, 1, "read_rest"); + ASSERT_EQ(buf[0], 'o', "read_rest_data"); + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, -1, "read_drained"); + ASSERT_EQ(errno, EAGAIN, "read_drained_errno"); + +out_close: + close(fd); +out_destroy: + stream__destroy(skel); +} + +static void test_stream_fd_empty(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts, + .flags = BPF_F_STREAM_NONBLOCK, + ); + struct pollfd pfd = { .events = POLLIN | POLLHUP }; + struct stream *skel; + char buf[4]; + int fd, ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_empty), + BPF_STREAM_STDOUT, &opts); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out_destroy; + pfd.fd = fd; + + /* Empty output produces neither data nor readiness. */ + if (!stream_fd_trigger(skel->progs.stream_empty)) + goto out_close; + ret = poll(&pfd, 1, 0); + ASSERT_EQ(ret, 0, "poll_empty_write"); + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, -1, "read_empty_write"); + ASSERT_EQ(errno, EAGAIN, "read_empty_write_errno"); + +out_close: + close(fd); +out_destroy: + stream__destroy(skel); +} + +struct stream_fd_read_ctx { + int fd; + ssize_t ret; + char buf[4]; +}; + +static void *stream_fd_read_thread(void *arg) +{ + struct stream_fd_read_ctx *ctx = arg; + + ctx->ret = read(ctx->fd, ctx->buf, sizeof(ctx->buf)); + return NULL; +} + +static int stream_fd_timed_join(pthread_t thread) +{ + struct timespec timeout; + + clock_gettime(CLOCK_REALTIME, &timeout); + timeout.tv_sec += 5; + return pthread_timedjoin_np(thread, NULL, &timeout); +} + +static void test_stream_fd_blocking(void) +{ + struct stream_fd_read_ctx ctx = {}; + struct stream *skel; + pthread_t thread; + bool thread_live = false; + int fd, flags, ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_syscall), + BPF_STREAM_STDOUT, NULL); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out_destroy; + ctx.fd = fd; + + flags = fcntl(fd, F_GETFL); + ASSERT_GE(flags, 0, "getfl"); + ASSERT_EQ(flags & O_NONBLOCK, 0, "blocking"); + + ret = pthread_create(&thread, NULL, stream_fd_read_thread, &ctx); + if (!ASSERT_OK(ret, "pthread_create")) + goto out_close; + thread_live = true; + + usleep(50000); + ret = pthread_tryjoin_np(thread, NULL); + if (!ASSERT_EQ(ret, EBUSY, "read_blocks")) { + thread_live = ret != 0; + goto out_thread; + } + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_thread; + + ret = stream_fd_timed_join(thread); + if (!ASSERT_OK(ret, "pthread_join")) + goto out_thread; + thread_live = false; + ASSERT_EQ(ctx.ret, 3, "read_len"); + ASSERT_OK(memcmp(ctx.buf, "foo", 3), "read_data"); + + memset(&ctx, 0, sizeof(ctx)); + ctx.fd = fd; + ret = pthread_create(&thread, NULL, stream_fd_read_thread, &ctx); + if (!ASSERT_OK(ret, "pthread_create_eof")) + goto out_close; + thread_live = true; + + usleep(50000); + ret = pthread_tryjoin_np(thread, NULL); + if (!ASSERT_EQ(ret, EBUSY, "read_eof_blocks")) { + thread_live = ret != 0; + goto out_thread; + } + + stream__destroy(skel); + skel = NULL; + ret = stream_fd_timed_join(thread); + if (!ASSERT_OK(ret, "pthread_join_eof")) + goto out_thread; + thread_live = false; + ASSERT_EQ(ctx.ret, 0, "read_eof"); + +out_thread: + if (thread_live) { + stream__destroy(skel); + skel = NULL; + ret = stream_fd_timed_join(thread); + if (ret) { + pthread_cancel(thread); + pthread_join(thread, NULL); + } + } +out_close: + close(fd); +out_destroy: + stream__destroy(skel); +} + +static void test_stream_fd_hup(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts, + .flags = BPF_F_STREAM_NONBLOCK, + ); + struct epoll_event event = { + .events = EPOLLIN | EPOLLET, + }; + struct pollfd pfd = { .events = POLLIN | POLLHUP }; + struct stream *skel; + char buf[4] = {}; + int epfd = -1, fd, ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_syscall), + BPF_STREAM_STDOUT, &opts); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out_destroy; + pfd.fd = fd; + epfd = epoll_create1(EPOLL_CLOEXEC); + if (!ASSERT_OK_FD(epfd, "epoll_create")) + goto out_close; + event.data.fd = fd; + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &event); + if (!ASSERT_OK(ret, "epoll_ctl")) + goto out_close; + + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_close; + ret = epoll_wait(epfd, &event, 1, 5000); + if (!ASSERT_EQ(ret, 1, "epoll_wait_data")) + goto out_close; + ASSERT_NEQ(event.events & EPOLLIN, 0, "epollin"); + ASSERT_EQ(event.events & EPOLLHUP, 0, "no_epollhup"); + + stream__destroy(skel); + skel = NULL; + event.events = 0; + ret = epoll_wait(epfd, &event, 1, 5000); + if (!ASSERT_EQ(ret, 1, "epoll_wait_hup")) + goto out_close; + ASSERT_NEQ(event.events & EPOLLIN, 0, "epollin_with_hup"); + ASSERT_NEQ(event.events & EPOLLHUP, 0, "epollhup"); + + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, 3, "read_buffered"); + ASSERT_OK(memcmp(buf, "foo", 3), "read_buffered_data"); + pfd.revents = 0; + ret = poll(&pfd, 1, 0); + ASSERT_EQ(ret, 1, "poll_drained_hup"); + ASSERT_EQ(pfd.revents & POLLIN, 0, "no_pollin_after_drain"); + ASSERT_NEQ(pfd.revents & POLLHUP, 0, "pollhup_after_drain"); + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, 0, "read_eof"); + +out_close: + if (epfd >= 0) + close(epfd); + close(fd); +out_destroy: + stream__destroy(skel); +} + +static void test_stream_fd_edge(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts, + .flags = BPF_F_STREAM_NONBLOCK, + ); + struct epoll_event event = { + .events = EPOLLIN | EPOLLET, + }; + struct stream *skel; + char buf[8] = {}; + int epfd = -1, fd, ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_syscall), + BPF_STREAM_STDOUT, &opts); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out_destroy; + epfd = epoll_create1(EPOLL_CLOEXEC); + if (!ASSERT_OK_FD(epfd, "epoll_create")) + goto out_close; + event.data.fd = fd; + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &event); + if (!ASSERT_OK(ret, "epoll_ctl")) + goto out_close; + + /* Output into an empty stream raises an edge. */ + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_close; + ret = epoll_wait(epfd, &event, 1, 5000); + if (!ASSERT_EQ(ret, 1, "epoll_wait_first")) + goto out_close; + ASSERT_NEQ(event.events & EPOLLIN, 0, "epollin_first"); + + /* + * Leave data behind, add more, then drain everything. Whether output + * into a non-empty stream raises an edge of its own is not part of the + * contract, so discard any pending event before re-arming the stream. + */ + ret = read(fd, buf, 2); + ASSERT_EQ(ret, 2, "read_partial"); + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_close; + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, 4, "read_rest"); + ASSERT_OK(memcmp(buf, "ofoo", 4), "read_rest_data"); + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, -1, "read_drained"); + ASSERT_EQ(errno, EAGAIN, "read_drained_errno"); + epoll_wait(epfd, &event, 1, 0); + + /* Output into the drained stream raises an edge again. */ + if (!stream_fd_trigger(skel->progs.stream_syscall)) + goto out_close; + ret = epoll_wait(epfd, &event, 1, 5000); + if (!ASSERT_EQ(ret, 1, "epoll_wait_after_drain")) + goto out_close; + ASSERT_NEQ(event.events & EPOLLIN, 0, "epollin_after_drain"); + +out_close: + if (epfd >= 0) + close(epfd); + close(fd); +out_destroy: + stream__destroy(skel); +} + +static void test_stream_fd_nmi_epoll(void) +{ + LIBBPF_OPTS(bpf_prog_stream_open_opts, opts, + .flags = BPF_F_STREAM_NONBLOCK, + ); + struct perf_event_attr attr = { + .size = sizeof(attr), + .type = PERF_TYPE_HARDWARE, + .config = PERF_COUNT_HW_CPU_CYCLES, + .freq = 1, + .sample_freq = 10, + }; + struct epoll_event event = { + .events = EPOLLIN, + }; + struct bpf_link *link = NULL; + struct stream *skel; + __u64 deadline; + char buf[4] = {}; + int epfd = -1, fd = -1, pmu_fd = -1; + int ret; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + fd = bpf_prog_stream_open(bpf_program__fd(skel->progs.stream_nmi), + BPF_STREAM_STDOUT, &opts); + if (!ASSERT_OK_FD(fd, "stream_open")) + goto out; + + epfd = epoll_create1(EPOLL_CLOEXEC); + if (!ASSERT_OK_FD(epfd, "epoll_create")) + goto out; + event.data.fd = fd; + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &event); + if (!ASSERT_OK(ret, "epoll_ctl")) + goto out; + + pmu_fd = syscall(__NR_perf_event_open, &attr, 0, -1, -1, + PERF_FLAG_FD_CLOEXEC); + if (pmu_fd < 0 && (errno == ENOENT || errno == EOPNOTSUPP)) { + printf("%s:SKIP:no PERF_COUNT_HW_CPU_CYCLES\n", __func__); + test__skip(); + goto out; + } + if (!ASSERT_GE(pmu_fd, 0, "perf_event_open")) + goto out; + + link = bpf_program__attach_perf_event(skel->progs.stream_nmi, pmu_fd); + if (!ASSERT_OK_PTR(link, "attach_perf_event")) { + link = NULL; + goto out; + } + pmu_fd = -1; + + deadline = get_time_ns() + NMI_TIMEOUT_NS; + do { + ret = epoll_wait(epfd, &event, 1, 0); + } while (ret == 0 && get_time_ns() < deadline); + if (!ASSERT_EQ(ret, 1, "epoll_wait")) + goto out; + ASSERT_NEQ(event.events & EPOLLIN, 0, "epollin"); + + ret = read(fd, buf, sizeof(buf)); + ASSERT_EQ(ret, 3, "read_len"); + ASSERT_OK(memcmp(buf, "nmi", 3), "read_data"); + +out: + bpf_link__destroy(link); + if (pmu_fd >= 0) + close(pmu_fd); + if (epfd >= 0) + close(epfd); + if (fd >= 0) + close(fd); + stream__destroy(skel); +} + +void test_stream_fd(void) +{ + if (test__start_subtest("open")) + test_stream_fd_open(); + if (test__start_subtest("nonblock")) + test_stream_fd_nonblock(); + if (test__start_subtest("empty")) + test_stream_fd_empty(); + if (test__start_subtest("blocking")) + test_stream_fd_blocking(); + if (test__start_subtest("hup")) + test_stream_fd_hup(); + if (test__start_subtest("edge")) + test_stream_fd_edge(); + if (test__start_subtest("nmi_epoll")) + test_stream_fd_nmi_epoll(); +} + +void test_stream_oversize(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct stream *skel; + int ret, prog_fd; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + prog_fd = bpf_program__fd(skel->progs.stream_oversize); + ret = bpf_prog_test_run_opts(prog_fd, &opts); + ASSERT_OK(ret, "oversize run"); + ASSERT_OK(opts.retval, "oversize retval"); + + stream__destroy(skel); +} + +void test_stream_partial_read(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct stream *skel; + int ret, prog_fd; + long page_size; + char *page, *buf; + char rest[8] = {}; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + prog_fd = bpf_program__fd(skel->progs.stream_syscall); + ret = bpf_prog_test_run_opts(prog_fd, &opts); + ASSERT_OK(ret, "ret"); + ASSERT_OK(opts.retval, "retval"); + + page_size = sysconf(_SC_PAGESIZE); + page = mmap(NULL, page_size * 2, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (!ASSERT_NEQ(page, MAP_FAILED, "mmap")) { + stream__destroy(skel); + return; + } + /* Leave only the first page mapped so a straddling copy faults. */ + if (!ASSERT_OK(munmap(page + page_size, page_size), "munmap second page")) { + munmap(page, page_size * 2); + stream__destroy(skel); + return; + } + + buf = page + page_size - 1; + ret = bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, buf, 3, NULL); + ASSERT_EQ(ret, 1, "partial bytes"); + ASSERT_EQ(buf[0], 'f', "first byte"); + + ret = bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, rest, sizeof(rest), NULL); + ASSERT_EQ(ret, 2, "remaining bytes"); + ASSERT_OK(memcmp(rest, "oo", 2), "remaining data"); + + munmap(page, page_size); + stream__destroy(skel); +} + static void test_address(struct bpf_program *prog, unsigned long *fault_addr_p) { LIBBPF_OPTS(bpf_test_run_opts, opts); diff --git a/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c b/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c index 98db9bafa44b..2b3ec2b79091 100644 --- a/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c +++ b/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c @@ -4,6 +4,7 @@ #include "struct_ops_private_stack.skel.h" #include "struct_ops_private_stack_fail.skel.h" #include "struct_ops_private_stack_recur.skel.h" +#include "struct_ops_private_stack_large.skel.h" #if defined(__x86_64__) || defined(__aarch64__) || defined(__powerpc64__) static void test_private_stack(void) @@ -78,6 +79,34 @@ cleanup: struct_ops_private_stack_recur__destroy(skel); } +/* Two frames of 2 KiB each on the private stack */ +static void test_private_stack_large(void) +{ + struct struct_ops_private_stack_large *skel; + struct bpf_link *link; + + if (!is_large_stack_supported()) { + test__skip(); + return; + } + + skel = struct_ops_private_stack_large__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack_large__open_and_load")) + return; + + link = bpf_map__attach_struct_ops(skel->maps.testmod_1); + if (!ASSERT_OK_PTR(link, "attach_struct_ops")) + goto cleanup; + + ASSERT_OK(trigger_module_test_read(256), "trigger_read"); + + ASSERT_EQ(skel->bss->val, 100 + 30 + 12, "val"); + + bpf_link__destroy(link); +cleanup: + struct_ops_private_stack_large__destroy(skel); +} + static void __test_struct_ops_private_stack(void) { if (test__start_subtest("private_stack")) @@ -86,6 +115,8 @@ static void __test_struct_ops_private_stack(void) test_private_stack_fail(); if (test__start_subtest("private_stack_recur")) test_private_stack_recur(); + if (test__start_subtest("private_stack_large")) + test_private_stack_large(); } #else static void __test_struct_ops_private_stack(void) diff --git a/tools/testing/selftests/bpf/prog_tests/summarization.c b/tools/testing/selftests/bpf/prog_tests/summarization.c index 6951786044ca..3e4bcf071735 100644 --- a/tools/testing/selftests/bpf/prog_tests/summarization.c +++ b/tools/testing/selftests/bpf/prog_tests/summarization.c @@ -53,17 +53,6 @@ static void test_aux(const char *main_prog_name, err = summarization_freplace__load(freplace); print_verifier_log(log); - /* The might_sleep extension doesn't work yet as sleepable calls are not - * allowed, but preserve the check in case it's supported later and then - * this particular combination can be enabled. - */ - if (!strcmp("might_sleep", replacement) && err) { - ASSERT_HAS_SUBSTR(log, "sleepable helper bpf_copy_from_user#", "error log"); - ASSERT_EQ(err, -EINVAL, "err"); - test__skip(); - goto out; - } - if (expect_load) { ASSERT_OK(err, "summarization_freplace__load"); } else { @@ -86,9 +75,8 @@ out: * Also test a case when main subprogram itself is replaced and is a single * subprogram in a program. * - * This holds for might_sleep programs. It is ok to replace might_sleep with - * might_sleep and with does_not_sleep, but does_not_sleep cannot be replaced - * with might_sleep. + * An extension is verified as a non-sleepable program, hence nothing that runs + * sleepable can be replaced. */ void test_summarization_freplace(void) { @@ -111,7 +99,7 @@ void test_summarization_freplace(void) }, }; const char *pkt_err = "Extension program changes packet data"; - const char *slp_err = "Extension program may sleep"; + const char *slp_err = "Cannot extend a function that runs without RCU read lock"; struct { const char *func; bool has_side_effect; @@ -136,7 +124,8 @@ void test_summarization_freplace(void) if (!test__start_subtest(buf)) continue; test_aux(mains[t][i].main, mains[t][i].to_be_replaced, replacements[t][j].func, - mains[t][i].has_side_effect || !replacements[t][j].has_side_effect, + t == 0 && + (mains[t][i].has_side_effect || !replacements[t][j].has_side_effect), replacements[t][j].err_msg); } } diff --git a/tools/testing/selftests/bpf/prog_tests/tailcalls.c b/tools/testing/selftests/bpf/prog_tests/tailcalls.c index c5c9d6c359bb..88d223110ecd 100644 --- a/tools/testing/selftests/bpf/prog_tests/tailcalls.c +++ b/tools/testing/selftests/bpf/prog_tests/tailcalls.c @@ -6,9 +6,11 @@ #include "tailcall_bpf2bpf_hierarchy2.skel.h" #include "tailcall_bpf2bpf_hierarchy3.skel.h" #include "tailcall_freplace.skel.h" +#include "tailcall_freplace_multi.skel.h" #include "tc_bpf2bpf.skel.h" #include "tailcall_fail.skel.h" #include "tailcall_cgrp_storage_owner.skel.h" +#include "tailcall_large_stack.skel.h" #include "tailcall_cgrp_storage_no_storage.skel.h" #include "tailcall_cgrp_storage.skel.h" #include "tailcall_sleepable.skel.h" @@ -1655,6 +1657,77 @@ out: tc_bpf2bpf__destroy(tc_skel); } +static void test_tailcall_freplace_multi(void) +{ + struct tailcall_freplace_multi *freplace_skel = NULL; + struct bpf_link *link_subprog = NULL, *link_entry = NULL; + struct tc_bpf2bpf *tc_skel = NULL; + int tc_prog_fd, map_fd, key = 0, err; + + tc_skel = tc_bpf2bpf__open_and_load(); + if (!ASSERT_OK_PTR(tc_skel, "tc_bpf2bpf__open_and_load")) + return; + + tc_prog_fd = bpf_program__fd(tc_skel->progs.entry_tc); + + freplace_skel = tailcall_freplace_multi__open(); + if (!ASSERT_OK_PTR(freplace_skel, "tailcall_freplace_multi__open")) + goto out; + + err = bpf_program__set_attach_target(freplace_skel->progs.subprog_freplace, + tc_prog_fd, "subprog_tc"); + if (!ASSERT_OK(err, "set_attach_target subprog_tc")) + goto out; + + err = bpf_program__set_attach_target(freplace_skel->progs.entry_freplace, + tc_prog_fd, "entry_tc"); + if (!ASSERT_OK(err, "set_attach_target entry_tc")) + goto out; + + err = tailcall_freplace_multi__load(freplace_skel); + if (!ASSERT_OK(err, "tailcall_freplace_multi__load")) + goto out; + + map_fd = bpf_map__fd(freplace_skel->maps.jmp_table); + + link_subprog = bpf_program__attach_freplace(freplace_skel->progs.subprog_freplace, + tc_prog_fd, "subprog_tc"); + if (!ASSERT_OK_PTR(link_subprog, "attach_freplace subprog_tc")) + goto out; + + link_entry = bpf_program__attach_freplace(freplace_skel->progs.entry_freplace, + tc_prog_fd, "entry_tc"); + if (!ASSERT_OK_PTR(link_entry, "attach_freplace entry_tc")) + goto out; + + err = bpf_map_update_elem(map_fd, &key, &tc_prog_fd, BPF_ANY); + if (!ASSERT_ERR(err, "update jmp_table with extended prog")) + goto out; + + err = bpf_link__destroy(link_entry); + link_entry = NULL; + if (!ASSERT_OK(err, "destroy entry link")) + goto out; + + err = bpf_map_update_elem(map_fd, &key, &tc_prog_fd, BPF_ANY); + if (!ASSERT_ERR(err, "update jmp_table with still extended prog")) + goto out; + + err = bpf_link__destroy(link_subprog); + link_subprog = NULL; + if (!ASSERT_OK(err, "destroy subprog link")) + goto out; + + err = bpf_map_update_elem(map_fd, &key, &tc_prog_fd, BPF_ANY); + ASSERT_OK(err, "update jmp_table"); + +out: + bpf_link__destroy(link_subprog); + bpf_link__destroy(link_entry); + tailcall_freplace_multi__destroy(freplace_skel); + tc_bpf2bpf__destroy(tc_skel); +} + static void test_tailcall_failure() { RUN_TESTS(tailcall_fail); @@ -1953,6 +2026,45 @@ out: tailcall_bpf2bpf2__destroy(skel_tc); } +/* + * test_tailcall_large_stack runs a tail call made from a subprog with a 1536 + * byte frame, under a 240-byte caller, into a program with a 2 KiB frame: + * + * entry (240) --call-> subprog_tail (1536) --tailcall-> classifier_0 (2048) + */ +static void test_tailcall_large_stack(void) +{ + struct tailcall_large_stack *skel; + int err, prog_fd, map_fd, key = 0; + char buff[128] = {}; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .data_in = buff, + .data_size_in = sizeof(buff), + .repeat = 1, + ); + + if (!is_large_stack_supported()) { + test__skip(); + return; + } + + skel = tailcall_large_stack__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tailcall_large_stack__open_and_load")) + return; + + prog_fd = bpf_program__fd(skel->progs.classifier_0); + map_fd = bpf_map__fd(skel->maps.jmp_table); + err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY); + if (!ASSERT_OK(err, "update jmp_table")) + goto out; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.entry), &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 42 + 7, "retval"); +out: + tailcall_large_stack__destroy(skel); +} + void test_tailcalls(void) { if (test__start_subtest("tailcall_1")) @@ -2005,6 +2117,8 @@ void test_tailcalls(void) test_tailcall_freplace(); if (test__start_subtest("tailcall_bpf2bpf_freplace")) test_tailcall_bpf2bpf_freplace(); + if (test__start_subtest("tailcall_freplace_multi")) + test_tailcall_freplace_multi(); if (test__start_subtest("tailcall_failure")) test_tailcall_failure(); if (test__start_subtest("tailcall_sleepable")) @@ -2022,4 +2136,6 @@ void test_tailcalls(void) test_tailcall_callback(); if (test__start_subtest("tailcall_bpf2bpf_fexit_links")) test_tailcall_bpf2bpf_fexit_links(); + if (test__start_subtest("tailcall_large_stack")) + test_tailcall_large_stack(); } diff --git a/tools/testing/selftests/bpf/prog_tests/tcp_autolowat.c b/tools/testing/selftests/bpf/prog_tests/tcp_autolowat.c new file mode 100644 index 000000000000..337f9d34a39c --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tcp_autolowat.c @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright 2026 Google LLC */ +#include <sys/epoll.h> + +#include "test_progs.h" +#include "cgroup_helpers.h" +#include "network_helpers.h" + +#include "tcp_autolowat.skel.h" + +#define SOL_BPF 0xdeadbeef +#define BPF_TCP_AUTOLOWAT 0x8badf00d + +struct rpc_descriptor { + u32 header_len; + u32 payload_len; +}; + +enum rpc_event_type { + RPC_EVENT_END, + RPC_EVENT_AUTOLOWAT, + RPC_EVENT_SEND, + RPC_EVENT_RECV, + RPC_EVENT_EPOLL, + RPC_EVENT_RCVLOWAT, +}; + +struct rpc_event { + enum rpc_event_type type; + union { + int len; + int nfds; + int val; + int rcvlowat; + }; +}; + +#define RPC_DESC_SIZE (sizeof(struct rpc_descriptor)) + +static struct rpc_test_case { + char data[4096]; + struct rpc_descriptor desc[32]; + struct rpc_event event[32]; +} rpc_test_cases[] = { + { + .desc = { + { .header_len = 100, .payload_len = 150 }, + }, + .event = { + { .type = RPC_EVENT_AUTOLOWAT, .val = 1}, + /* Single full RPC message in skb. */ + { .type = RPC_EVENT_SEND, .len = RPC_DESC_SIZE + 100 + 150}, + { .type = RPC_EVENT_EPOLL, .nfds = 1}, + { .type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE + 100 + 150}, + }, + }, + { + .desc = { + {.header_len = 100, .payload_len = 150}, + {.header_len = 100, .payload_len = 150}, + {.header_len = 100, .payload_len = 150}, + }, + .event = { + { .type = RPC_EVENT_AUTOLOWAT, .val = 1}, + /* Two full RPC messages in skb. */ + {.type = RPC_EVENT_SEND, .len = (RPC_DESC_SIZE + 100 + 150) * 2}, + {.type = RPC_EVENT_EPOLL, .nfds = 1}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = (RPC_DESC_SIZE + 100 + 150) * 2}, + /* Single full RPC message in skb. */ + { .type = RPC_EVENT_SEND, .len = RPC_DESC_SIZE + 100 + 150}, + { .type = RPC_EVENT_EPOLL, .nfds = 1}, + { .type = RPC_EVENT_RCVLOWAT, .rcvlowat = (RPC_DESC_SIZE + 100 + 150) * 3}, + }, + }, + { + .desc = { + {.header_len = 100, .payload_len = 150}, + {.header_len = 100, .payload_len = 150}, + {.header_len = 100, .payload_len = 150}, + }, + .event = { + { .type = RPC_EVENT_AUTOLOWAT, .val = 1}, + /* Two full RPC messages in skb. */ + {.type = RPC_EVENT_SEND, .len = (RPC_DESC_SIZE + 100 + 150) * 2}, + {.type = RPC_EVENT_EPOLL, .nfds = 1}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = (RPC_DESC_SIZE + 100 + 150) * 2}, + /* Only the next descriptor in skb. */ + { .type = RPC_EVENT_SEND, .len = RPC_DESC_SIZE}, + { .type = RPC_EVENT_EPOLL, .nfds = 1}, + { .type = RPC_EVENT_RCVLOWAT, .rcvlowat = (RPC_DESC_SIZE + 100 + 150) * 2}, + }, + }, + { + .desc = { + {.header_len = 100, .payload_len = 150}, + {.header_len = 200, .payload_len = 500}, + }, + .event = { + { .type = RPC_EVENT_AUTOLOWAT, .val = 1}, + /* The first descriptor is partial. */ + {.type = RPC_EVENT_SEND, .len = 1}, + {.type = RPC_EVENT_EPOLL, .nfds = 0}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE}, + /* The first descriptor is available. */ + {.type = RPC_EVENT_SEND, .len = RPC_DESC_SIZE - 1}, + {.type = RPC_EVENT_EPOLL, .nfds = 0}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE + 150 + 100}, + /* The first header is ready. */ + {.type = RPC_EVENT_SEND, .len = 100}, + {.type = RPC_EVENT_EPOLL, .nfds = 0}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE + 150 + 100}, + /* skb has the first payload and 1 byte of the next descriptor. */ + {.type = RPC_EVENT_SEND, .len = 150 + 1}, + {.type = RPC_EVENT_EPOLL, .nfds = 1}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE + 150 + 100}, + /* After reading the first RPC message, SO_RCVLOWAT should be RPC_DESC_SIZE. */ + {.type = RPC_EVENT_RECV, .len = RPC_DESC_SIZE + 150 + 100}, + {.type = RPC_EVENT_EPOLL, .nfds = 0}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE}, + /* The second descriptor is available. */ + {.type = RPC_EVENT_SEND, .len = RPC_DESC_SIZE - 1}, + {.type = RPC_EVENT_EPOLL, .nfds = 0}, + {.type = RPC_EVENT_RCVLOWAT, .rcvlowat = RPC_DESC_SIZE + 200 + 500}, + }, + }, +}; + +struct tcp_autolowat_test_cb { + int saved_netns; + union { + int fd[4]; + struct { + int server, client, child; + int epoll; + }; + }; +}; + +static void tcp_autolowat_teardown_cb(struct tcp_autolowat_test_cb *cb) +{ + int i, err; + + for (i = 0; i < ARRAY_SIZE(cb->fd); i++) { + if (cb->fd[i] != -1) + close(cb->fd[i]); + } + + if (cb->saved_netns != -1) { + err = setns(cb->saved_netns, CLONE_NEWNET); + ASSERT_OK(err, "restore netns"); + + close(cb->saved_netns); + } +} + +static int tcp_autolowat_setup_cb(struct tcp_autolowat_test_cb *cb, int family) +{ + struct epoll_event ev = {}; + int err; + int i; + + for (i = 0; i < ARRAY_SIZE(cb->fd); i++) + cb->fd[i] = -1; + + cb->saved_netns = open("/proc/self/ns/net", O_RDONLY); + if (!ASSERT_OK_FD(cb->saved_netns, "save netns")) + goto err; + + err = unshare(CLONE_NEWNET); + if (!ASSERT_OK(err, "unshare")) + goto err; + + err = system("ip link set dev lo up"); + if (!ASSERT_OK(err, "set up lo")) + goto err; + + cb->server = start_server(family, SOCK_STREAM, NULL, 0, 0); + if (!ASSERT_OK_FD(cb->server, "start_server")) + goto err; + + cb->client = connect_to_fd(cb->server, 0); + if (!ASSERT_OK_FD(cb->client, "connect_to_fd")) + goto err; + + cb->child = accept(cb->server, NULL, NULL); + if (!ASSERT_OK_FD(cb->child, "accept")) + goto err; + + cb->epoll = epoll_create1(0); + if (!ASSERT_OK_FD(cb->epoll, "epoll_create")) + goto err; + + ev.events = EPOLLIN; + ev.data.fd = cb->child; + + err = epoll_ctl(cb->epoll, EPOLL_CTL_ADD, cb->child, &ev); + if (!ASSERT_OK(err, "epoll_ctl")) + goto err; + + return 0; + +err: + tcp_autolowat_teardown_cb(cb); + return -1; +} + +static int tcp_autolowat_build_data(struct rpc_test_case *test_case) +{ + struct rpc_descriptor *desc = test_case->desc; + char *ptr = test_case->data; + int rpc_size; + + memset(ptr, 0, sizeof(test_case->data)); + + while (desc->header_len + desc->payload_len) { + rpc_size = sizeof(*desc) + desc->header_len + desc->payload_len; + + if (!ASSERT_LE(ptr + rpc_size - test_case->data, + sizeof(test_case->data), "data overflow")) + return 1; + + memcpy(ptr, desc, sizeof(*desc)); + ptr += rpc_size; + desc++; + } + + if (!ASSERT_GT(ptr - test_case->data, 0, "no data")) + return 1; + + return 0; +} + +static void tcp_autolowat_run_rpc_test(struct tcp_autolowat_test_cb *cb, + struct rpc_test_case *test_case) +{ + struct rpc_event *event = test_case->event; + char *ptr = test_case->data; + struct epoll_event ev; + socklen_t optlen; + int err, optval; + char buf[4096]; + + if (tcp_autolowat_build_data(test_case)) + return; + + while (1) { + switch (event->type) { + case RPC_EVENT_END: + return; + case RPC_EVENT_AUTOLOWAT: + err = setsockopt(cb->child, SOL_BPF, BPF_TCP_AUTOLOWAT, + &event->val, sizeof(event->val)); + if (!ASSERT_OK(err, "setsockopt")) + return; + break; + case RPC_EVENT_SEND: + err = send(cb->client, ptr, event->len, 0); + if (!ASSERT_EQ(err, event->len, "send")) + return; + + ptr += event->len; + break; + case RPC_EVENT_RECV: + err = recv(cb->child, buf, event->len, 0); + if (!ASSERT_EQ(err, event->len, "recv")) + return; + break; + case RPC_EVENT_EPOLL: + err = epoll_wait(cb->epoll, &ev, 1, 100); + if (!ASSERT_EQ(err, event->nfds, "epoll_wait")) + return; + break; + case RPC_EVENT_RCVLOWAT: + optval = 0; + optlen = sizeof(optval); + + err = getsockopt(cb->child, SOL_SOCKET, SO_RCVLOWAT, &optval, &optlen); + if (!ASSERT_OK(err, "getsockopt") || + !ASSERT_EQ(optval, event->rcvlowat, "rcvlowat")) + return; + break; + } + + event++; + } +} + +static void tcp_autolowat_run_rpc_tests(struct tcp_autolowat *skel, int family) +{ + struct tcp_autolowat_test_cb cb; + int err; + int i; + + for (i = 0; i < ARRAY_SIZE(rpc_test_cases); i++) { + memset(skel->bss->test_name, 0, sizeof(skel->bss->test_name)); + + snprintf(skel->bss->test_name, sizeof(skel->bss->test_name), + "AF_INET%c rpc_test_cases[%d]", + family == AF_INET ? ' ' : '6', i); + + if (!test__start_subtest(skel->bss->test_name)) + continue; + + err = tcp_autolowat_setup_cb(&cb, family); + if (err) + continue; + + tcp_autolowat_run_rpc_test(&cb, &rpc_test_cases[i]); + tcp_autolowat_teardown_cb(&cb); + } +} + +static void tcp_autolowat_run_tests(struct tcp_autolowat *skel) +{ + tcp_autolowat_run_rpc_tests(skel, AF_INET); + tcp_autolowat_run_rpc_tests(skel, AF_INET6); +} + +void test_tcp_autolowat(void) +{ + struct tcp_autolowat *skel; + struct bpf_link *link[2]; + int cgroup; + + skel = tcp_autolowat__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + cgroup = test__join_cgroup("/tcp_autolowat"); + if (!ASSERT_GE(cgroup, 0, "join_cgroup")) + goto destroy_skel; + + link[0] = bpf_map__attach_cgroup_opts(skel->maps.tcp_autolowat_ops, cgroup, NULL); + if (!ASSERT_OK_PTR(link[0], "attach_cgroup(tcp_autolowat_ops)")) + goto close_cgroup; + + link[1] = bpf_program__attach_cgroup(skel->progs.tcp_autolowat_setsockopt, cgroup); + if (!ASSERT_OK_PTR(link[1], "attach_cgroup(SETSOCKOPT)")) + goto destroy_sockops; + + tcp_autolowat_run_tests(skel); + + bpf_link__destroy(link[1]); +destroy_sockops: + bpf_link__destroy(link[0]); +close_cgroup: + close(cgroup); +destroy_skel: + tcp_autolowat__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c index 7f9f54ba3fbe..e7580b9f0fc2 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c @@ -5,6 +5,15 @@ #include "struct_ops_arena.skel.h" #include "struct_ops_arena_attach.skel.h" #include "struct_ops_arena_fail.skel.h" +#include "struct_ops_arena_member.skel.h" +#include "struct_ops_arena_member_render.skel.h" +#include "struct_ops_arena_member_fail__union.skel.h" +#include "struct_ops_arena_member_fail__xchg.skel.h" +#include "struct_ops_arena_member_fail__mismatch.skel.h" +#include "struct_ops_arena_member_fail__mismatch_mem.skel.h" +#include "struct_ops_arena_member_fail__write.skel.h" +#include "struct_ops_arena_member_fail__no_arena.skel.h" +#include "struct_ops_arena_autobind.skel.h" #if defined(__x86_64__) || defined(__aarch64__) /* @@ -101,6 +110,167 @@ static void arena_arg_attach(void) struct_ops_arena__destroy(skel); } + +/* Attach @st_ops and run @trigger once, it returns 0 when everything checked out. */ +static void attach_and_trigger(struct bpf_map *st_ops, struct bpf_program *trigger) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct bpf_link *link; + int err; + + link = bpf_map__attach_struct_ops(st_ops); + if (!ASSERT_OK_PTR(link, "attach_struct_ops")) + return; + + err = bpf_prog_test_run_opts(bpf_program__fd(trigger), &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "trigger_retval"); + + bpf_link__destroy(link); +} + +/* + * Attach a callback that reads arena pointers out of its argument struct. + * The header is shared with the module, so the members are arena pointers + * on the program side as well. + */ +static void arena_member(void) +{ + struct struct_ops_arena_member *skel; + + skel = struct_ops_arena_member__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_member__open_and_load")) + return; + + attach_and_trigger(skel->maps.testmod_arena_member, skel->progs.trigger); + struct_ops_arena_member__destroy(skel); +} + +/* @prog_fd lists exactly one map, the one with @map_id. */ +static void assert_prog_map(int prog_fd, __u32 map_id, const char *name) +{ + __u32 map_ids[2] = {}, len = sizeof(struct bpf_prog_info); + struct bpf_prog_info info = { + .nr_map_ids = ARRAY_SIZE(map_ids), + .map_ids = ptr_to_u64(map_ids), + }; + int err; + + err = bpf_prog_get_info_by_fd(prog_fd, &info, &len); + if (!ASSERT_OK(err, name)) + return; + if (!ASSERT_EQ(info.nr_map_ids, 1, name)) + return; + ASSERT_EQ(map_ids[0], map_id, name); +} + +/* + * libbpf binds the arena of an object to every program in it: a callback + * with an __arena argument loads without referring to the arena, and an + * unrelated program gets the arena as well. + */ +static void arena_autobind(void) +{ + struct struct_ops_arena_autobind *skel; + struct bpf_map_info info = {}; + __u32 len = sizeof(info); + int err; + + skel = struct_ops_arena_autobind__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_autobind__open_and_load")) + return; + + err = bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.arena), &info, &len); + if (!ASSERT_OK(err, "arena_info")) + goto out; + assert_prog_map(bpf_program__fd(skel->progs.test_arena_cb), info.id, "test_arena_cb"); + assert_prog_map(bpf_program__fd(skel->progs.unrelated), info.id, "unrelated"); + + attach_and_trigger(skel->maps.testmod_arena_autobind, skel->progs.trigger); +out: + struct_ops_arena_autobind__destroy(skel); +} + +/* + * The same binding through bpf_object__prepare() and bpf_program__clone(), the + * way veristat loads a program: the arena takes slot 0 of a continuous + * fd_array, next to the module BTF of the kfunc the trigger calls. + */ +static void arena_autobind_clone(void) +{ + struct struct_ops_arena_autobind *skel; + struct bpf_map_info info = {}; + __u32 len = sizeof(info); + struct bpf_program *prog; + int err, fd; + + skel = struct_ops_arena_autobind__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_autobind__open")) + return; + + err = bpf_object__prepare(skel->obj); + if (!ASSERT_OK(err, "prepare")) + goto out; + err = bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.arena), &info, &len); + if (!ASSERT_OK(err, "arena_info")) + goto out; + bpf_object__for_each_program(prog, skel->obj) { + fd = bpf_program__clone(prog, NULL); + if (!ASSERT_GE(fd, 0, bpf_program__name(prog))) + continue; + assert_prog_map(fd, info.id, bpf_program__name(prog)); + close(fd); + } +out: + struct_ops_arena_autobind__destroy(skel); +} + +/* + * An arena exclusive to a program is bound to that program all the same: the + * kernel checks an exclusive map against the digest of the final program, so + * the binding through the fd_array passes for the program the map is + * exclusive to. + */ +static void arena_autobind_excl(void) +{ + struct struct_ops_arena_autobind *skel; + struct bpf_map_info info = {}; + __u32 len = sizeof(info); + int err; + + skel = struct_ops_arena_autobind__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_autobind__open")) + return; + /* the other programs need the arena too, and cannot have it */ + bpf_map__set_autocreate(skel->maps.testmod_arena_autobind, false); + bpf_program__set_autoload(skel->progs.trigger, false); + err = bpf_map__set_exclusive_program(skel->maps.arena, skel->progs.unrelated); + if (!ASSERT_OK(err, "set_exclusive_program")) + goto out; + + err = struct_ops_arena_autobind__load(skel); + if (!ASSERT_OK(err, "load")) + goto out; + err = bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.arena), &info, &len); + if (!ASSERT_OK(err, "arena_info")) + goto out; + assert_prog_map(bpf_program__fd(skel->progs.unrelated), info.id, "unrelated"); +out: + struct_ops_arena_autobind__destroy(skel); +} + +/* The same through the vmlinux.h rendering of the members, which takes casts. */ +static void arena_member_render(void) +{ + struct struct_ops_arena_member_render *skel; + + skel = struct_ops_arena_member_render__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_member_render__open_and_load")) + return; + + attach_and_trigger(skel->maps.testmod_arena_member_render, skel->progs.trigger); + struct_ops_arena_member_render__destroy(skel); +} #endif /* @@ -116,12 +286,37 @@ void serial_test_struct_ops_arena(void) * arena arguments", so the programs cannot even load. */ #if defined(__x86_64__) || defined(__aarch64__) + int has_tag; + if (test__start_subtest("arena_arg")) arena_arg(); if (test__start_subtest("arena_arg_fail")) arena_arg_fail(); if (test__start_subtest("arena_arg_attach")) arena_arg_attach(); + /* the tags reach the module BTF only from a toolchain with btf_type_tag support */ + has_tag = testmod_member_has_type_tag("bpf_testmod_arena_member_args", "ptr", "arena"); + if (ASSERT_GE(has_tag, 0, "testmod_btf") && has_tag) { + if (test__start_subtest("arena_member")) + arena_member(); + if (test__start_subtest("arena_member_render")) + arena_member_render(); + RUN_TESTS(struct_ops_arena_member_fail__union); + RUN_TESTS(struct_ops_arena_member_fail__xchg); + RUN_TESTS(struct_ops_arena_member_fail__mismatch); + RUN_TESTS(struct_ops_arena_member_fail__mismatch_mem); + RUN_TESTS(struct_ops_arena_member_fail__write); + RUN_TESTS(struct_ops_arena_member_fail__no_arena); + } else if (test__start_subtest("arena_member")) { + /* so that the missing member tests show up as skipped */ + test__skip(); + } + if (test__start_subtest("arena_autobind")) + arena_autobind(); + if (test__start_subtest("arena_autobind_clone")) + arena_autobind_clone(); + if (test__start_subtest("arena_autobind_excl")) + arena_autobind_excl(); #else test__skip(); #endif diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c index 0f321e889862..8767e5f8bcc7 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c @@ -3,7 +3,62 @@ #include <test_progs.h> #include "struct_ops_multi_args.skel.h" +static void test_refcounted_multi(void) +{ + struct struct_ops_multi_args *skel; + char log_buf[4096] = {}; + int err; + + skel = struct_ops_multi_args__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_multi_args__open")) + return; + + bpf_program__set_log_buf(skel->progs.test_refcounted_multi, log_buf, sizeof(log_buf)); + + err = struct_ops_multi_args__load(skel); + if (!ASSERT_EQ(err, -EINVAL, "struct_ops_multi_args__load")) + goto out; + + ASSERT_HAS_SUBSTR(log_buf, "program with __ref argument cannot tail call", + "check_verifier_log_message"); + +out: + struct_ops_multi_args__destroy(skel); +} + +static void test_trampoline_stack_args(void) +{ + struct struct_ops_multi_args *skel; + struct bpf_link *link = NULL; + int err; + + skel = struct_ops_multi_args__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_multi_args__open")) + return; + + bpf_program__set_autoload(skel->progs.test_refcounted_multi, false); + skel->struct_ops.testmod_ref_acquire->test_refcounted_multi = NULL; + + err = struct_ops_multi_args__load(skel); + if (!ASSERT_OK(err, "struct_ops_multi_args__load")) + goto out; + + link = bpf_map__attach_struct_ops(skel->maps.testmod_ref_acquire); + if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) + goto out; + + ASSERT_EQ(skel->bss->trampoline_stack_arg9, 9999, "check_stack_passed_arg9"); + +out: + bpf_link__destroy(link); + struct_ops_multi_args__destroy(skel); +} + void test_struct_ops_multi_args(void) { - RUN_TESTS(struct_ops_multi_args); + if (test__start_subtest("test_refcounted_multi")) + test_refcounted_multi(); + + if (test__start_subtest("test_trampoline_stack_args")) + test_trampoline_stack_args(); } diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_nullable.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_nullable.c new file mode 100644 index 000000000000..028f070f2e5f --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_nullable.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> + +#include "struct_ops_nullable.skel.h" +#include "struct_ops_nullable_fail.skel.h" +#include "struct_ops_nullable_fail__percpu.skel.h" +#include "struct_ops_nullable_probed__plain.skel.h" +#include "struct_ops_nullable_probed__scalar.skel.h" +#include "struct_ops_nullable_probed__untrusted.skel.h" + +/* + * Attach a callback that gets a trusted and a trusted __nullable task pointer + * through its argument struct and drive it through the + * bpf_testmod_ops3_call_test_nullable() kfunc. + */ +static void nullable_member(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct struct_ops_nullable *skel; + struct bpf_link *link; + int err; + + skel = struct_ops_nullable__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_nullable__open_and_load")) + return; + + link = bpf_map__attach_struct_ops(skel->maps.testmod_nullable); + if (!ASSERT_OK_PTR(link, "attach_struct_ops")) + goto out; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.trigger), &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "trigger_retval"); + + bpf_link__destroy(link); +out: + struct_ops_nullable__destroy(skel); +} + +/* Serialized because it attaches the singleton bpf_testmod_ops3. */ +void serial_test_struct_ops_nullable(void) +{ + int has_tag; + + /* the tag reaches the module BTF only from a toolchain with btf_type_tag support */ + has_tag = testmod_member_has_type_tag("bpf_testmod_nullable_args", "task_or_null", "nullable"); + if (!ASSERT_GE(has_tag, 0, "testmod_btf")) + return; + if (!has_tag) { + printf("%s:SKIP: no nullable type tag in the bpf_testmod BTF\n", __func__); + test__skip(); + return; + } + if (test__start_subtest("nullable_member")) + nullable_member(); + RUN_TESTS(struct_ops_nullable_fail); + RUN_TESTS(struct_ops_nullable_fail__percpu); + RUN_TESTS(struct_ops_nullable_probed__plain); + RUN_TESTS(struct_ops_nullable_probed__scalar); + RUN_TESTS(struct_ops_nullable_probed__untrusted); +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_task_work.c b/tools/testing/selftests/bpf/prog_tests/test_task_work.c index 774b31a5f6ca..ca426094e9c4 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_task_work.c +++ b/tools/testing/selftests/bpf/prog_tests/test_task_work.c @@ -5,10 +5,15 @@ #include <stdio.h> #include "task_work.skel.h" #include "task_work_fail.skel.h" +#include "task_work_race.skel.h" +#include "task_work_busy.skel.h" #include <linux/bpf.h> #include <linux/perf_event.h> #include <sys/syscall.h> #include <time.h> +#include <pthread.h> +#include <sched.h> +#include <unistd.h> static int perf_event_open(__u32 type, __u64 config, int pid) { @@ -85,7 +90,7 @@ static void task_work_run(const char *prog_name, const char *map_name) skel = task_work__open(); if (!ASSERT_OK_PTR(skel, "task_work__open")) - return; + goto cleanup; bpf_object__for_each_program(prog, skel->obj) { bpf_program__set_autoload(prog, false); @@ -155,3 +160,692 @@ void test_task_work(void) RUN_TESTS(task_work_fail); } + +#define TASK_WORK_RACE_ROUNDS 1500 +#define TASK_WORK_POLL_US 100 +#define TASK_WORK_POLL_ITERS 20000 +#define TASK_WORK_START_POLL_ITERS 50000 + +/* Must match progs/task_work_race.c. */ +enum task_work_race_status { + RACE_ARM_SEQ, + RACE_READY_SEQ, + /* High 32 bits hold the sequence, low 32 bits the signed error. */ + RACE_RESULT, +}; + +struct task_work_value { + __u32 seq; + char data[60]; + struct bpf_task_work tw; +}; + +struct task_work_race_ctx { + int stop; + int setup_err; + int trigger_tid; + int target_tid; +}; + +struct task_work_race_thread_arg { + struct task_work_race_ctx *ctx; + int cpu; + int *tid; +}; + +static int task_work_race_pin_cpu(int cpu) +{ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return pthread_setaffinity_np(pthread_self(), sizeof(set), &set); +} + +static void *task_work_race_worker(void *arg) +{ + struct task_work_race_thread_arg *thread = arg; + struct task_work_race_ctx *ctx = thread->ctx; + int err; + + err = task_work_race_pin_cpu(thread->cpu); + if (err) + __atomic_store_n(&ctx->setup_err, err, __ATOMIC_RELEASE); + __atomic_store_n(thread->tid, syscall(__NR_gettid), __ATOMIC_RELEASE); + while (!__atomic_load_n(&ctx->stop, __ATOMIC_ACQUIRE)) + getppid(); + return NULL; +} + +static int task_work_status(struct bpf_map *map, int idx, __s64 *value) +{ + return bpf_map_lookup_elem(bpf_map__fd(map), &idx, value); +} + +static int task_work_set_status(struct bpf_map *map, int idx, __s64 value) +{ + return bpf_map_update_elem(bpf_map__fd(map), &idx, &value, BPF_ANY); +} + +/* Wait for seq, keeping polling errors separate from the scheduling result. */ +static int task_work_wait_seq(struct bpf_map *map, int idx, __u32 seq, + __s32 *result) +{ + int i; + + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { + __s64 value = 0; + + if (!task_work_status(map, idx, &value) && + (__u32)((__u64)value >> 32) == seq) { + *result = (__s32)value; + return 0; + } + usleep(TASK_WORK_POLL_US); + } + return -ETIMEDOUT; +} + +static int task_work_race_wait_ready(struct task_work_race *skel, __u32 seq) +{ + int i; + + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { + __s64 value = 0; + + if (!task_work_status(skel->maps.status, RACE_READY_SEQ, &value) && + value == seq) + return 0; + if (!task_work_status(skel->maps.status, RACE_RESULT, &value) && + (__u32)((__u64)value >> 32) == seq) + /* + * The BPF program finished between the two reads: + * RESULT makes waiting for READY unnecessary. Any error + * that occurred before READY is checked by the caller. + */ + return 0; + usleep(TASK_WORK_POLL_US); + } + return -ETIMEDOUT; +} + +static int task_work_race_prepare_elem(struct task_work_race *skel) +{ + struct task_work_value value = {}; + int key = 0; + + if (!bpf_map_lookup_elem(bpf_map__fd(skel->maps.hmap), &key, &value)) + return 0; + if (errno != ENOENT) + return -errno; + if (bpf_map_update_elem(bpf_map__fd(skel->maps.hmap), &key, &value, + BPF_NOEXIST)) + return -errno; + return 0; +} + +static int task_work_race_arm(struct task_work_race *skel, __u32 seq) +{ + if (task_work_set_status(skel->maps.status, RACE_ARM_SEQ, seq)) + return -errno; + return task_work_race_wait_ready(skel, seq); +} + +static int task_work_race_check_done(struct task_work_race *skel, __u32 seq) +{ + __s32 result; + int err; + + err = task_work_wait_seq(skel->maps.status, RACE_RESULT, seq, &result); + return err ?: result; +} + +static int task_work_race_wait_callback(struct task_work_race *skel, + __u32 seq) +{ + int i, key = seq; + + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { + __u64 completed = 0; + + if (!bpf_map_lookup_elem(bpf_map__fd(skel->maps.completed), + &key, &completed) && completed) + return 0; + usleep(TASK_WORK_POLL_US); + } + return -ETIMEDOUT; +} + +void serial_test_task_work_race(void) +{ + struct task_work_race_ctx ctx = {}; + struct task_work_race_thread_arg trigger_arg = { + .ctx = &ctx, + .tid = &ctx.trigger_tid, + }; + struct task_work_race_thread_arg target_arg = { + .ctx = &ctx, + .tid = &ctx.target_tid, + }; + struct task_work_race *skel; + pthread_t trigger, target; + cpu_set_t allowed; + int cpu, cpu_count = 0; + bool target_started = false, trigger_started = false; + int err, i, key = 0; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) { + ASSERT_OK(-errno, "sched_getaffinity"); + return; + } + for (cpu = 0; cpu < CPU_SETSIZE && cpu_count < 2; cpu++) { + if (!CPU_ISSET(cpu, &allowed)) + continue; + if (!cpu_count) + trigger_arg.cpu = cpu; + else + target_arg.cpu = cpu; + cpu_count++; + } + if (cpu_count < 2) { + printf("%s:SKIP:need two CPUs in the process affinity mask\n", + __func__); + test__skip(); + return; + } + + skel = task_work_race__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + if (!ASSERT_OK(task_work_race__attach(skel), "attach")) + goto cleanup; + + err = pthread_create(&target, NULL, task_work_race_worker, &target_arg); + if (!ASSERT_OK(err, "pthread_create target")) + goto cleanup; + target_started = true; + err = pthread_create(&trigger, NULL, task_work_race_worker, &trigger_arg); + if (!ASSERT_OK(err, "pthread_create trigger")) + goto stop; + trigger_started = true; + + for (i = 0; i < TASK_WORK_START_POLL_ITERS; i++) { + if (__atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE) && + __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE)) + break; + usleep(TASK_WORK_POLL_US); + } + if (!ASSERT_TRUE(__atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE) && + __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE), + "thread start")) + goto stop; + if (!ASSERT_OK(__atomic_load_n(&ctx.setup_err, __ATOMIC_ACQUIRE), + "thread affinity")) + goto stop; + + skel->bss->trigger_tid = __atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE); + skel->bss->target_tid = __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE); + + for (i = 0; i < TASK_WORK_RACE_ROUNDS; i++) { + __u32 seq = i + 1; + int variant = i % 3; + + if (!ASSERT_OK(task_work_race_prepare_elem(skel), "prepare elem") || + !ASSERT_OK(task_work_race_arm(skel, seq), "round ready")) + goto stop; + + if (variant == 2 && + !ASSERT_OK(bpf_map_delete_elem(bpf_map__fd(skel->maps.hmap), + &key), "early delete")) + goto stop; + + err = task_work_race_check_done(skel, seq); + /* + * Variant 2 deletes the element after READY. A scheduling attempt + * racing with deletion can return -EBUSY, which the BPF program + * retries without publishing RESULT. The retry can then report + * -ENOENT because the element is gone; accept that terminal result. + * A later variant 0 round recreates the element. + */ + if (variant == 2 && err == -ENOENT) + err = 0; + if (!ASSERT_OK(err, "schedule")) { + fprintf(stderr, "round %d schedule failed: %d\n", i, err); + goto stop; + } + + if (variant != 2 && + !ASSERT_OK(task_work_race_wait_callback(skel, seq), + "callback")) + goto stop; + + if (variant == 1 && + !ASSERT_OK(bpf_map_delete_elem(bpf_map__fd(skel->maps.hmap), + &key), "late delete")) + goto stop; + } + + /* A distinct completion generation prevents an old callback satisfying this. */ + if (!ASSERT_OK(task_work_race_prepare_elem(skel), "final prepare") || + !ASSERT_OK(task_work_race_arm(skel, TASK_WORK_RACE_ROUNDS + 1), + "final ready") || + !ASSERT_OK(task_work_race_check_done(skel, + TASK_WORK_RACE_ROUNDS + 1), + "final schedule") || + !ASSERT_OK(task_work_race_wait_callback(skel, + TASK_WORK_RACE_ROUNDS + 1), + "final callback")) + goto stop; + +stop: + __atomic_store_n(&ctx.stop, 1, __ATOMIC_RELEASE); + if (trigger_started) + pthread_join(trigger, NULL); + if (target_started) + pthread_join(target, NULL); +cleanup: + task_work_race__destroy(skel); +} + +#define TASK_WORK_BUSY_ROUNDS 32 +#define TASK_WORK_BUSY_MAX_ATTEMPTS 96 + +/* Must match progs/task_work_busy.c. */ +enum task_work_busy_status { + BUSY_ARM_SEQ, + /* + * High 32 bits hold the sequence. BUSY_WINDOW low bits: 1 open, + * 2 the in-window attempt was seen before the handler returned, + * 3 the wait bound was reached without it. Result slots: signed + * error in the low 32 bits. + */ + BUSY_WINDOW, + BUSY_ACK, + BUSY_GEN1_RESULT, + BUSY_GEN2_RESULT, + BUSY_CB1_SEQ, + BUSY_CB2_COUNT, + BUSY_CB2_SEQ, + BUSY_RECOVER_SEQ, + BUSY_LAST_WAIT_NS, + BUSY_STATUS_MAX, +}; + +enum task_work_busy_attempt { + TWB_VALID, + TWB_MISSED, + TWB_HARD_FAIL, +}; + +static const char *twb_attempt_name(enum task_work_busy_attempt r) +{ + switch (r) { + case TWB_VALID: + return "valid"; + case TWB_MISSED: + return "missed"; + default: + return "fail"; + } +} + +static int task_work_busy_wait_value(struct task_work_busy *skel, int idx, + __s64 want) +{ + int i; + + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { + __s64 value = 0; + + if (!task_work_status(skel->maps.status, idx, &value) && value == want) + return 0; + usleep(TASK_WORK_POLL_US); + } + return -ETIMEDOUT; +} + +/* Wait until the window slot reports a final state (2 or 3) for seq. */ +static int task_work_busy_wait_window(struct task_work_busy *skel, __u32 seq, + __s64 *window) +{ + int i; + + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { + __s64 value = 0; + + if (!task_work_status(skel->maps.status, BUSY_WINDOW, &value) && + (__u32)((__u64)value >> 32) == seq && + (__u32)value >= 2) { + *window = value; + return 0; + } + usleep(TASK_WORK_POLL_US); + } + return -ETIMEDOUT; +} + +static int task_work_busy_prepare_elem(struct task_work_busy *skel, int key) +{ + struct task_work_value value = {}; + + if (!bpf_map_update_elem(bpf_map__fd(skel->maps.hmap), &key, &value, + BPF_NOEXIST)) + return 0; + return -errno; +} + +static int task_work_busy_reset(struct task_work_busy *skel) +{ + static const int zeroed[] = { + BUSY_WINDOW, BUSY_ACK, BUSY_GEN1_RESULT, BUSY_GEN2_RESULT, + BUSY_CB1_SEQ, BUSY_CB2_COUNT, BUSY_CB2_SEQ, BUSY_RECOVER_SEQ, + BUSY_LAST_WAIT_NS, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(zeroed); i++) + if (task_work_set_status(skel->maps.status, zeroed[i], 0)) + return -errno; + return 0; +} + +/* + * join waits for clear_child_tid, which is cleared before exit_task_work(). + * PID lookup stops finding the worker only after its exit task_work has run. + * Probe with signal 0; no signal is delivered. Do not create new workers + * until both old TIDs have disappeared. + */ +static int task_work_busy_wait_exit(int tid) +{ + int i; + + for (i = 0; i < TASK_WORK_POLL_ITERS; i++) { + if (syscall(__NR_tgkill, getpid(), tid, 0)) { + if (errno == ESRCH) + return 0; + return -errno; + } + usleep(TASK_WORK_POLL_US); + } + return -ETIMEDOUT; +} + +/* + * Run one attempt on dedicated workers and report its outcome. The + * workers are stopped, joined and waited out of PID lookup before another + * attempt can reset the shared state. join alone does not wait for their + * exit task_work to finish. + */ +static enum task_work_busy_attempt +task_work_busy_run_attempt(struct task_work_busy *skel, int seq, + int trigger_cpu, int target_cpu, __s64 *out_wait_ns) +{ + struct task_work_race_ctx ctx = {}; + struct task_work_race_thread_arg trigger_arg = { + .ctx = &ctx, + .cpu = trigger_cpu, + .tid = &ctx.trigger_tid, + }; + struct task_work_race_thread_arg target_arg = { + .ctx = &ctx, + .cpu = target_cpu, + .tid = &ctx.target_tid, + }; + enum task_work_busy_attempt result = TWB_HARD_FAIL; + pthread_t trigger, target; + bool trigger_started = false, target_started = false; + bool cleanup_ok = true; + __s64 window = 0, wait_ns = 0, count = -1; + __s32 res = 0; + int err, i; + + *out_wait_ns = 0; + /* Do not let a reused TID match the previous attempt's filter. */ + skel->bss->trigger_tid = 0; + skel->bss->target_tid = 0; + + /* Clear old state before enabling workers that can read it. */ + if (!ASSERT_OK(task_work_busy_prepare_elem(skel, seq), "prepare elem")) + goto out; + if (!ASSERT_OK(task_work_busy_reset(skel), "reset slots")) + goto out; + + err = pthread_create(&target, NULL, task_work_race_worker, &target_arg); + if (err) { + ASSERT_OK(err, "pthread_create target"); + goto out; + } + target_started = true; + err = pthread_create(&trigger, NULL, task_work_race_worker, &trigger_arg); + if (err) { + ASSERT_OK(err, "pthread_create trigger"); + goto out; + } + trigger_started = true; + + for (i = 0; i < TASK_WORK_START_POLL_ITERS; i++) { + if (__atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE) && + __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE)) + break; + usleep(TASK_WORK_POLL_US); + } + if (!ASSERT_TRUE(__atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE) && + __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE), + "thread start")) + goto out; + if (!ASSERT_OK(__atomic_load_n(&ctx.setup_err, __ATOMIC_ACQUIRE), + "thread affinity")) + goto out; + + skel->bss->trigger_tid = __atomic_load_n(&ctx.trigger_tid, __ATOMIC_ACQUIRE); + skel->bss->target_tid = __atomic_load_n(&ctx.target_tid, __ATOMIC_ACQUIRE); + + /* Arm last, once this attempt's value, slots and TIDs are in place. */ + if (!ASSERT_OK(task_work_set_status(skel->maps.status, BUSY_ARM_SEQ, seq), + "arm")) + goto out; + + /* A fresh value has no expected BUSY source: check the first result. */ + err = task_work_wait_seq(skel->maps.status, BUSY_GEN1_RESULT, seq, &res); + if (!ASSERT_OK(err, "gen1 result")) + goto out; + if (!ASSERT_OK(res, "gen1 schedule")) { + fprintf(stderr, "attempt %d: gen1 failed: %d\n", seq, res); + goto out; + } + + if (!ASSERT_OK(task_work_busy_wait_window(skel, seq, &window), + "window close")) + goto out; + + if ((__u32)window == 3) { + /* + * The attempt did not arrive inside the wait bound. The + * elapsed time is a diagnostic only; count + * the miss and retry with a fresh attempt. + */ + result = TWB_MISSED; + goto out; + } + + /* + * Valid window: the handler saw this sequence's attempt before + * returning, so the attempt ran while the scheduling irq_work was + * still BUSY. The elapsed time is a diagnostic only: host vCPU + * preemption can stretch wall-clock time without affecting the + * gate behavior under test. + */ + err = task_work_wait_seq(skel->maps.status, BUSY_ACK, seq, &res); + if (!ASSERT_OK(err, "in-window attempt")) + goto out; + if (res != -EBUSY) { + fprintf(stderr, + "attempt %d: in-window reuse %s (%d), expected -EBUSY\n", + seq, res == 0 ? "accepted" : "failed", res); + ASSERT_EQ(res, -EBUSY, "in-window BUSY gate rejection"); + goto out; + } + + /* + * The window was released; retry until BUSY clears. The second + * callback must then run exactly once. + */ + if (!ASSERT_OK(task_work_set_status(skel->maps.status, BUSY_RECOVER_SEQ, seq), + "enable recovery")) + goto out; + err = task_work_wait_seq(skel->maps.status, BUSY_GEN2_RESULT, seq, &res); + if (!ASSERT_OK(err, "recovery result")) + goto out; + if (!ASSERT_OK(res, "recovery schedule")) { + fprintf(stderr, "attempt %d: recovery failed: %d\n", seq, res); + goto out; + } + if (!ASSERT_OK(task_work_busy_wait_value(skel, BUSY_CB2_SEQ, seq), + "gen2 callback")) + goto out; + result = TWB_VALID; +out: + __atomic_store_n(&ctx.stop, 1, __ATOMIC_RELEASE); + if (trigger_started && + (!ASSERT_OK(pthread_join(trigger, NULL), "trigger join") || + !ASSERT_OK(task_work_busy_wait_exit(ctx.trigger_tid), "trigger exit"))) { + cleanup_ok = false; + result = TWB_HARD_FAIL; + } + if (target_started && + (!ASSERT_OK(pthread_join(target, NULL), "target join") || + !ASSERT_OK(task_work_busy_wait_exit(ctx.target_tid), "target exit"))) { + cleanup_ok = false; + result = TWB_HARD_FAIL; + } + /* Both workers have passed exit_task_work; the count is now stable. */ + if (result == TWB_VALID) { + if (!ASSERT_OK(task_work_status(skel->maps.status, BUSY_CB2_COUNT, &count), + "cb2 count")) { + result = TWB_HARD_FAIL; + } else if (!ASSERT_EQ(count, 1, "gen2 callback exactly once")) { + fprintf(stderr, "attempt %d: gen2 callback count %lld\n", + seq, (long long)count); + result = TWB_HARD_FAIL; + } + } + /* + * Only a captured final window belongs to this attempt. The fexit + * runs in the trigger's interrupted context, so successful worker + * cleanup also completes its diagnostic write. Read once here; + * the elapsed value does not decide whether the attempt passed. + */ + if (cleanup_ok && window) { + err = task_work_status(skel->maps.status, BUSY_LAST_WAIT_NS, &wait_ns); + if (err) { + err = -errno; + if (result != TWB_HARD_FAIL) { + ASSERT_OK(err, "wait diagnostic"); + result = TWB_HARD_FAIL; + } else { + fprintf(stderr, + "attempt %d: elapsed diagnostic unavailable: %d\n", + seq, err); + } + } + } + *out_wait_ns = wait_ns; + fprintf(stderr, "attempt %d: %s (window %u wait_ns %lld)\n", + seq, twb_attempt_name(result), (__u32)window, (long long)wait_ns); + return result; +} + +void serial_test_task_work_busy(void) +{ + struct task_work_busy *skel; + struct bpf_link *link_tp = NULL, *link_fexit = NULL; + cpu_set_t allowed; + int cpu, cpu_count = 0, trigger_cpu = -1, target_cpu = -1; + int attempt, attempts_started = 0, valid = 0, missed = 0; + bool hard_fail = false; + __s64 max_wait_ns = 0; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) { + ASSERT_OK(-errno, "sched_getaffinity"); + return; + } + for (cpu = 0; cpu < CPU_SETSIZE && cpu_count < 2; cpu++) { + if (!CPU_ISSET(cpu, &allowed)) + continue; + if (!cpu_count) + trigger_cpu = cpu; + else + target_cpu = cpu; + cpu_count++; + } + if (cpu_count < 2) { + printf("%s:SKIP:need two CPUs in the process affinity mask\n", + __func__); + test__skip(); + return; + } + + skel = task_work_busy__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + if (skel->kconfig->CONFIG_PREEMPT_RT) { + printf("%s:SKIP:CONFIG_PREEMPT_RT=y runs irq_work in a kernel thread\n", + __func__); + test__skip(); + goto cleanup; + } + + link_tp = bpf_program__attach_tracepoint(skel->progs.busy_sched_work, + "syscalls", + "sys_enter_getppid"); + if (!ASSERT_OK_PTR(link_tp, "attach tracepoint")) + goto cleanup; + + link_fexit = bpf_program__attach_trace(skel->progs.busy_fexit_task_work_add); + if (!ASSERT_OK_PTR(link_fexit, "attach fexit")) + goto cleanup; + + for (attempt = 1; + attempt <= TASK_WORK_BUSY_MAX_ATTEMPTS && + valid < TASK_WORK_BUSY_ROUNDS; + attempt++) { + enum task_work_busy_attempt res; + __s64 wait_ns = 0; + + attempts_started = attempt; + res = task_work_busy_run_attempt(skel, attempt, trigger_cpu, + target_cpu, &wait_ns); + switch (res) { + case TWB_VALID: + valid++; + if (wait_ns > max_wait_ns) + max_wait_ns = wait_ns; + break; + case TWB_MISSED: + missed++; + break; + case TWB_HARD_FAIL: + hard_fail = true; + break; + } + if (hard_fail) + break; + } + + fprintf(stderr, + "task_work_busy: %d/%d valid, %d missed, %d attempts, max_wait %lld ns\n", + valid, TASK_WORK_BUSY_ROUNDS, missed, attempts_started, + (long long)max_wait_ns); + + /* + * The coverage assertion only makes sense when no hard + * failure aborted the loop; after a hard failure keep the primary + * reason instead of deriving more assertion noise from it. + */ + if (!hard_fail) + ASSERT_EQ(valid, TASK_WORK_BUSY_ROUNDS, "valid busy windows"); +cleanup: + bpf_link__destroy(link_tp); + bpf_link__destroy(link_fexit); + task_work_busy__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_tc_edt.c b/tools/testing/selftests/bpf/prog_tests/test_tc_edt.c index 462512fb191f..1391978a3e30 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_tc_edt.c +++ b/tools/testing/selftests/bpf/prog_tests/test_tc_edt.c @@ -90,7 +90,7 @@ static void cleanup(void) remove_netns(SERVER_NS); } -static void run_test(void) +static void run_test(struct test_tc_edt *skel) { int server_fd, client_fd, err; double rate_mbps, rate_error; @@ -113,12 +113,19 @@ static void run_test(void) if (!ASSERT_OK_FD(client_fd, "connect client")) return; - ts_start = get_time_ns(); err = send_recv_data(server_fd, client_fd, TX_BYTES_COUNT); ts_end = get_time_ns(); close_netns(nstoken); ASSERT_OK(err, "send_recv_data"); + /* + * Start from the first packet with data rather than before + * send_recv_data(), which also sets up the sender. + */ + ts_start = skel->bss->tx_start_ns; + if (!ASSERT_GT(ts_start, 0, "tx_start_ns")) + return; + rate_mbps = TX_BYTES_COUNT / ((ts_end - ts_start) / 1000.0); rate_error = fabs((rate_mbps - TARGET_RATE_MBPS) * 100.0 / TARGET_RATE_MBPS); @@ -138,7 +145,7 @@ void test_tc_edt(void) if (!ASSERT_OK(setup(skel), "global setup")) return; - run_test(); + run_test(skel); cleanup(); test_tc_edt__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/test_tunnel.c b/tools/testing/selftests/bpf/prog_tests/test_tunnel.c index eb9309931272..7b35bf949805 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_tunnel.c +++ b/tools/testing/selftests/bpf/prog_tests/test_tunnel.c @@ -161,9 +161,9 @@ static int add_vxlan_tunnel(void) IP4_ADDR2_VETH1, MAC_VETH1); /* root namespace */ - SYS(fail, "ip link add dev %s type vxlan external gbp dstport 4789", - VXLAN_TUNL_DEV1); - SYS(fail, "ip link set dev %s address %s up", VXLAN_TUNL_DEV1, MAC_TUNL_DEV1); + SYS(fail, "ip link add dev %s address %s type vxlan external gbp dstport 4789", + VXLAN_TUNL_DEV1, MAC_TUNL_DEV1); + SYS(fail, "ip link set dev %s up", VXLAN_TUNL_DEV1); SYS(fail, "ip addr add dev %s %s/24", VXLAN_TUNL_DEV1, IP4_ADDR_TUNL_DEV1); SYS(fail, "ip neigh add %s lladdr %s dev %s", IP4_ADDR_TUNL_DEV0, MAC_TUNL_DEV0, VXLAN_TUNL_DEV1); @@ -198,11 +198,10 @@ static int add_ip6vxlan_tunnel(void) IP6VXLAN_TUNL_DEV0, MAC_TUNL_DEV0); /* root namespace */ - SYS(fail, "ip link add dev %s type vxlan external dstport 4789", - IP6VXLAN_TUNL_DEV1); - SYS(fail, "ip addr add dev %s %s/24", IP6VXLAN_TUNL_DEV1, IP4_ADDR_TUNL_DEV1); - SYS(fail, "ip link set dev %s address %s up", + SYS(fail, "ip link add dev %s address %s type vxlan external dstport 4789", IP6VXLAN_TUNL_DEV1, MAC_TUNL_DEV1); + SYS(fail, "ip addr add dev %s %s/24", IP6VXLAN_TUNL_DEV1, IP4_ADDR_TUNL_DEV1); + SYS(fail, "ip link set dev %s up", IP6VXLAN_TUNL_DEV1); return 0; fail: diff --git a/tools/testing/selftests/bpf/prog_tests/timer.c b/tools/testing/selftests/bpf/prog_tests/timer.c index 09ff21e1ad2f..178223190b8b 100644 --- a/tools/testing/selftests/bpf/prog_tests/timer.c +++ b/tools/testing/selftests/bpf/prog_tests/timer.c @@ -262,6 +262,34 @@ static int timer_cancel_async(struct timer *timer_skel) return 0; } +/* + * A timer callback which re-arms itself and reaches a loop through a call: + * the two entries into the callback meet at the loop, in a frame of its own. + */ +static int timer_loop_rearm(struct timer *timer_skel) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, prog_fd, i; + + err = timer__attach(timer_skel); + if (!ASSERT_OK(err, "timer_attach")) + return err; + + timer_skel->bss->loop_rearm = 1; + + prog_fd = bpf_program__fd(timer_skel->progs.test_loop_rearm); + err = bpf_prog_test_run_opts(prog_fd, &topts); + if (!ASSERT_OK(err, "test_run")) + return err; + + for (i = 0; i < 100 && timer_skel->bss->loop_sum < 120 * 2; i++) + usleep(1000); + + timer__detach(timer_skel); + ASSERT_EQ(timer_skel->bss->loop_sum, 120 * 2, "loop_sum"); + return 0; +} + static void test_timer(int (*timer_test_fn)(struct timer *timer_skel)) { struct timer *timer_skel = NULL; @@ -287,6 +315,11 @@ void serial_test_timer(void) RUN_TESTS(timer_failure); } +void serial_test_timer_loop_rearm(void) +{ + test_timer(timer_loop_rearm); +} + void serial_test_timer_stress(void) { test_timer(timer_stress); diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c index 0aa9532a05cf..cc727011f3a2 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c @@ -662,6 +662,97 @@ cleanup: free(ids2); } +/* + * Find two BTF ids that resolve to the same address. The stub of a syscall + * without arguments is an alias of its __do_sys_ function. + */ +static bool get_same_addr_ids(__u32 *ids) +{ + const char *prefix = "__x64_sys_"; + __u32 nr, type_id; + bool found = false; + struct btf *btf; + char alias[128]; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "btf__load_vmlinux_btf")) + return false; + + nr = btf__type_cnt(btf); + for (type_id = 1; type_id < nr && !found; type_id++) { + const struct btf_type *type = btf__type_by_id(btf, type_id); + const char *str; + long addr; + __s32 id; + + if (!btf_is_func(type)) + continue; + + str = btf__name_by_offset(btf, type->name_off); + if (!str || strncmp(str, prefix, strlen(prefix))) + continue; + + snprintf(alias, sizeof(alias), "__do_sys_%s", str + strlen(prefix)); + id = btf__find_by_name_kind(btf, alias, BTF_KIND_FUNC); + if (id <= 0) + continue; + + addr = ksym_get_addr(str); + if (!addr || addr != ksym_get_addr(alias)) + continue; + + ids[0] = type_id; + ids[1] = id; + found = true; + } + + btf__free(btf); + return found; +} + +static void test_attach_same_addr(void) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tracing_multi_fail *skel; + struct bpf_link *link; + __u32 ids[2]; + int i; + + if (!ASSERT_OK(load_kallsyms(), "load_kallsyms")) + return; + + if (!get_same_addr_ids(ids)) { + test__skip(); + return; + } + + skel = tracing_multi_fail__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_fail__open_and_load")) + return; + + /* Each id attaches fine on its own.. */ + for (i = 0; i < 2; i++) { + opts.ids = &ids[i]; + opts.cnt = 1; + + link = bpf_program__attach_tracing_multi(skel->progs.test_fentry, NULL, &opts); + if (!ASSERT_OK_PTR(link, "attach_single")) + goto cleanup; + bpf_link__destroy(link); + } + + /* ..but there's only one call site, so not both of them. */ + opts.ids = ids; + opts.cnt = 2; + + link = bpf_program__attach_tracing_multi(skel->progs.test_fentry, NULL, &opts); + if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "attach_same_addr")) + bpf_link__destroy(link); + +cleanup: + tracing_multi_fail__destroy(skel); +} + void serial_test_tracing_multi_bench_attach(void) { LIBBPF_OPTS(bpf_tracing_multi_opts, opts); @@ -748,6 +839,12 @@ void serial_test_tracing_multi_bench_attach(void) if (!btf_type_is_traceable_func(btf, type)) continue; + /* + * BTF can have more functions of the same name and they all + * resolve to the same address, take just the first one. + */ + tdelete(&str, &root, compare); + err = libbpf_ensure_mem((void **) &ids, &cap, sizeof(*ids), cnt + 1); if (err) goto cleanup; @@ -1023,6 +1120,8 @@ void test_tracing_multi_test(void) test_session(); if (test__start_subtest("attach_api_fails")) test_attach_api_fails(); + if (test__start_subtest("attach_same_addr")) + test_attach_same_addr(); RUN_TESTS(tracing_multi_verifier); if (test__start_subtest("fentry_after_multi")) test_fentry_after_multi(); diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c index 15b95d0235b5..de358f335d98 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c @@ -5,6 +5,10 @@ #include "tracing_struct.skel.h" #include "tracing_struct_many_args.skel.h" #include "tracing_struct_int128.skel.h" +#include "tracing_struct_int128_tgt.skel.h" +#include "tracing_struct_int128_tracer.skel.h" +#include "tracing_stack_arg_tailcall_tgt.skel.h" +#include "tracing_stack_arg_tailcall_tracer.skel.h" static void test_struct_args(void) { @@ -117,9 +121,11 @@ static void test_int128_args(void) { /* * __int128 arguments are passed in a register pair on x86_64 and - * arm64, which the trampoline packs into two context slots. Other - * architectures pass a __int128 differently (e.g. s390x passes larger - * arguments by reference), so only exercise this on x86_64 and arm64. + * arm64, which the trampoline packs into two context slots. Past the + * registers, both start an argument aligned to 16 bytes at an even + * stack slot, which the trampoline packs too. Other architectures + * pass a __int128 differently (e.g. s390x passes larger arguments by + * reference), so only exercise this on x86_64 and arm64. */ #if defined(__x86_64__) || defined(__aarch64__) struct tracing_struct_int128 *skel; @@ -139,6 +145,20 @@ static void test_int128_args(void) ASSERT_EQ(skel->bss->t_c, 3, "t:c"); ASSERT_EQ(skel->bss->t_ret, 6, "t ret"); + ASSERT_EQ(skel->bss->s_g, 7, "s:g"); + ASSERT_EQ(skel->bss->s_h_lo, 8, "s:h.lo"); + ASSERT_EQ(skel->bss->s_h_hi, 9, "s:h.hi"); + ASSERT_EQ(skel->bss->s_i, 10, "s:i"); + ASSERT_EQ(skel->bss->s_ret, 385, "s ret"); + + ASSERT_EQ(skel->bss->b_f_a, 6, "b:f.a"); + ASSERT_EQ(skel->bss->b_f_b, 7, "b:f.b"); + ASSERT_EQ(skel->bss->b_g, 8, "b:g"); + ASSERT_EQ(skel->bss->b_h, 9, "b:h"); + ASSERT_EQ(skel->bss->b_i_lo, 10, "b:i.lo"); + ASSERT_EQ(skel->bss->b_i_hi, 11, "b:i.hi"); + ASSERT_EQ(skel->bss->b_ret, 506, "b ret"); + destroy_skel: tracing_struct_int128__destroy(skel); #else @@ -146,6 +166,191 @@ destroy_skel: #endif } +static void test_int128_tgt_prog(void) +{ +#if defined(__x86_64__) || defined(__aarch64__) || defined(__s390x__) + struct tracing_struct_int128_tracer *tracer = NULL; + struct tracing_struct_int128_tgt *tgt; + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, fd; + + tgt = tracing_struct_int128_tgt__open_and_load(); + if (!ASSERT_OK_PTR(tgt, "tgt__open_and_load")) + return; + if (!tgt->rodata->has_struct_arg) { + test__skip(); + goto out; + } + fd = bpf_program__fd(tgt->progs.run); + + tracer = tracing_struct_int128_tracer__open(); + if (!ASSERT_OK_PTR(tracer, "tracer__open")) + goto out; + + err = bpf_program__set_attach_target(tracer->progs.odd_entry, fd, "callee_odd"); + err = err ?: bpf_program__set_attach_target(tracer->progs.odd_exit, fd, "callee_odd"); + if (tgt->rodata->has_stack_arg) { + err = err ?: bpf_program__set_attach_target(tracer->progs.stack_entry, fd, + "callee_stack"); + err = err ?: bpf_program__set_attach_target(tracer->progs.stack_exit, fd, + "callee_stack"); + err = err ?: bpf_program__set_attach_target(tracer->progs.split_entry, fd, + "callee_split"); + err = err ?: bpf_program__set_attach_target(tracer->progs.split_exit, fd, + "callee_split"); + } else { + bpf_program__set_autoload(tracer->progs.stack_entry, false); + bpf_program__set_autoload(tracer->progs.stack_exit, false); + bpf_program__set_autoload(tracer->progs.split_entry, false); + bpf_program__set_autoload(tracer->progs.split_exit, false); + } + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + err = tracing_struct_int128_tracer__load(tracer); + if (!ASSERT_OK(err, "tracer__load")) + goto out; + err = tracing_struct_int128_tracer__attach(tracer); + if (!ASSERT_OK(err, "tracer__attach")) + goto out; + + err = bpf_prog_test_run_opts(fd, &topts); + if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(topts.retval, 0, "retval")) + goto out; + + ASSERT_EQ(tracer->bss->odd_a, 1, "odd:a"); + ASSERT_EQ(tracer->bss->odd_lo, 8, "odd:v.lo"); + ASSERT_EQ(tracer->bss->odd_hi, 9, "odd:v.hi"); + ASSERT_EQ(tracer->bss->odd_b, 2, "odd:b"); + ASSERT_EQ(tgt->bss->ret_odd, 52, "odd ret"); + ASSERT_EQ(tracer->bss->odd_ret, 52, "odd fexit ret"); + + if (tgt->rodata->has_stack_arg) { + ASSERT_EQ(tracer->bss->stack_g, 7, "stack:g"); + ASSERT_EQ(tracer->bss->stack_lo, 8, "stack:h.lo"); + ASSERT_EQ(tracer->bss->stack_hi, 9, "stack:h.hi"); + ASSERT_EQ(tracer->bss->stack_i, 10, "stack:i"); + ASSERT_EQ(tgt->bss->ret_stack, 385, "stack ret"); + ASSERT_EQ(tracer->bss->stack_ret, 385, "stack fexit ret"); + + ASSERT_EQ(tracer->bss->split_e, 5, "split:e"); + ASSERT_EQ(tracer->bss->split_lo, 8, "split:h.lo"); + ASSERT_EQ(tracer->bss->split_hi, 9, "split:h.hi"); + ASSERT_EQ(tracer->bss->split_i, 10, "split:i"); + ASSERT_EQ(tgt->bss->ret_split, 246, "split ret"); + ASSERT_EQ(tracer->bss->split_ret, 246, "split fexit ret"); + } + +out: + tracing_struct_int128_tracer__destroy(tracer); + tracing_struct_int128_tgt__destroy(tgt); +#else + test__skip(); +#endif +} + +/* + * A static function whose BTF is unreliable gets a trampoline that only knows + * of the register arguments, so one taking stack arguments must be refused. + */ +static void test_tgt_prog_unreliable(void) +{ +#if defined(__x86_64__) || defined(__aarch64__) + struct tracing_struct_int128_tracer *tracer = NULL; + struct tracing_struct_int128_tgt *tgt; + char log_buf[4096] = {}; + struct bpf_program *prog; + int err, fd; + + tgt = tracing_struct_int128_tgt__open_and_load(); + if (!ASSERT_OK_PTR(tgt, "tgt__open_and_load")) + return; + if (!tgt->rodata->has_stack_arg) { + test__skip(); + goto out; + } + fd = bpf_program__fd(tgt->progs.run); + + tracer = tracing_struct_int128_tracer__open(); + if (!ASSERT_OK_PTR(tracer, "tracer__open")) + goto out; + + bpf_object__for_each_program(prog, tracer->obj) + bpf_program__set_autoload(prog, false); + bpf_program__set_autoload(tracer->progs.unreliable_entry, true); + bpf_program__set_log_buf(tracer->progs.unreliable_entry, log_buf, sizeof(log_buf)); + err = bpf_program__set_attach_target(tracer->progs.unreliable_entry, fd, + "callee_unreliable"); + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + err = tracing_struct_int128_tracer__load(tracer); + ASSERT_ERR(err, "tracer__load"); + ASSERT_HAS_SUBSTR(log_buf, "unreliable BTF and stack arguments", "log"); + +out: + tracing_struct_int128_tracer__destroy(tracer); + tracing_struct_int128_tgt__destroy(tgt); +#else + test__skip(); +#endif +} + +/* + * A trampoline on a function of a tail_call_reachable program keeps the tail + * call counter pointer on its stack, and has to hand the function its stack + * arguments past it. + */ +static void test_stack_arg_tailcall(void) +{ +#if defined(__x86_64__) || defined(__aarch64__) + struct tracing_stack_arg_tailcall_tracer *tracer = NULL; + struct tracing_stack_arg_tailcall_tgt *tgt; + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, fd; + + tgt = tracing_stack_arg_tailcall_tgt__open_and_load(); + if (!ASSERT_OK_PTR(tgt, "tgt__open_and_load")) + return; + if (!tgt->rodata->has_stack_arg) { + test__skip(); + goto out; + } + fd = bpf_program__fd(tgt->progs.run); + + tracer = tracing_stack_arg_tailcall_tracer__open(); + if (!ASSERT_OK_PTR(tracer, "tracer__open")) + goto out; + + err = bpf_program__set_attach_target(tracer->progs.stack_entry, fd, "callee_stack"); + err = err ?: bpf_program__set_attach_target(tracer->progs.stack_exit, fd, "callee_stack"); + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + err = tracing_stack_arg_tailcall_tracer__load(tracer); + if (!ASSERT_OK(err, "tracer__load")) + goto out; + err = tracing_stack_arg_tailcall_tracer__attach(tracer); + if (!ASSERT_OK(err, "tracer__attach")) + goto out; + + err = bpf_prog_test_run_opts(fd, &topts); + if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(topts.retval, 0, "retval")) + goto out; + + ASSERT_EQ(tracer->bss->stack_g, 7, "stack:g"); + ASSERT_EQ(tracer->bss->stack_h, 8, "stack:h"); + ASSERT_EQ(tgt->bss->ret_stack, 204, "stack ret"); + ASSERT_EQ(tracer->bss->stack_ret, 204, "stack fexit ret"); + +out: + tracing_stack_arg_tailcall_tracer__destroy(tracer); + tracing_stack_arg_tailcall_tgt__destroy(tgt); +#else + test__skip(); +#endif +} + static void test_union_args(void) { struct tracing_struct *skel; @@ -181,6 +386,12 @@ void test_tracing_struct(void) test_struct_many_args(); if (test__start_subtest("int128_args")) test_int128_args(); + if (test__start_subtest("int128_tgt_prog")) + test_int128_tgt_prog(); + if (test__start_subtest("tgt_prog_unreliable")) + test_tgt_prog_unreliable(); + if (test__start_subtest("stack_arg_tailcall")) + test_stack_arg_tailcall(); if (test__start_subtest("union_args")) test_union_args(); } diff --git a/tools/testing/selftests/bpf/prog_tests/unwind_attach.c b/tools/testing/selftests/bpf/prog_tests/unwind_attach.c new file mode 100644 index 000000000000..37e03b3e0ee8 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/unwind_attach.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include "verifier_unwind.skel.h" +#include "unwind_attach.skel.h" + +static char log_buf[16 * 1024]; + +static int run(int fd) +{ + char data[64] = {}; + LIBBPF_OPTS(bpf_test_run_opts, opts, + .data_in = data, + .data_size_in = sizeof(data), + .repeat = 1, + ); + + ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), "test_run"); + return opts.retval; +} + +/* + * msg: why it cannot be loaded. Otherwise retval is what the target returns + * with the prog attached. + */ +static void attach(int tgt_fd, const char *func, const char *name, const char *msg, int retval) +{ + struct bpf_link *link = NULL; + struct unwind_attach *skel; + struct bpf_program *prog; + int err; + + skel = unwind_attach__open(); + if (!ASSERT_OK_PTR(skel, "unwind_attach__open")) + return; + + prog = bpf_object__find_program_by_name(skel->obj, name); + if (!ASSERT_OK_PTR(prog, name)) + goto out; + bpf_program__set_autoload(prog, true); + log_buf[0] = 0; + bpf_program__set_log_buf(prog, log_buf, sizeof(log_buf)); + err = bpf_program__set_attach_target(prog, tgt_fd, func); + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + err = unwind_attach__load(skel); + if (msg) { + ASSERT_ERR(err, name); + ASSERT_HAS_SUBSTR(log_buf, msg, name); + goto out; + } + if (!ASSERT_OK(err, name)) + goto out; + + link = bpf_program__attach(prog); + if (!ASSERT_OK_PTR(link, "attach")) + goto out; + ASSERT_EQ(run(tgt_fd), retval, "retval"); + if (prog == skel->progs.fentry_unwind) + ASSERT_EQ(skel->bss->fentry_cnt, 1, "fentry_cnt"); +out: + bpf_link__destroy(link); + unwind_attach__destroy(skel); +} + +void test_unwind_attach(void) +{ + struct verifier_unwind *tgt; + struct bpf_program *prog; + int tgt_fd, plain_fd, shared_fd, tail_fd, throws_fd, err, key = 0; + + if (!is_jit_enabled()) { + printf("%s:SKIP:JIT is disabled\n", __func__); + test__skip(); + return; + } + tgt = verifier_unwind__open(); + if (!ASSERT_OK_PTR(tgt, "verifier_unwind__open")) + return; + bpf_object__for_each_program(prog, tgt->obj) + bpf_program__set_autoload(prog, prog == tgt->progs.calls_global_unwinds || + prog == tgt->progs.calls_global_never_unwinds || + prog == tgt->progs.shared_pad || + prog == tgt->progs.tail_call_in_subprog || + prog == tgt->progs.exception_callback_unwinds); + err = verifier_unwind__load(tgt); + if (err == -ENOTSUPP) { + printf("%s:SKIP:JIT does not support unwinding\n", __func__); + test__skip(); + goto out; + } + if (!ASSERT_OK(err, "verifier_unwind__load")) + goto out; + /* + * It calls global_unwinds(), which unwinds with r0 = 5, and returns r0 + 10 + * from the pad. + */ + tgt_fd = bpf_program__fd(tgt->progs.calls_global_unwinds); + plain_fd = bpf_program__fd(tgt->progs.calls_global_never_unwinds); + ASSERT_EQ(run(tgt_fd), 15, "retval"); + + attach(tgt_fd, "global_unwinds", "fentry_unwind", NULL, 15); + /* unwind_exit would return to the trampoline */ + attach(tgt_fd, "global_unwinds", "fexit_unwind", + "Only fentry can attach to global_unwinds", 0); + /* An extension leaves either way, like what it replaced */ + attach(tgt_fd, "global_unwinds", "freplace_returns", NULL, 9); + attach(tgt_fd, "global_unwinds", "freplace_unwinds", NULL, 13); + /* There is no unwind_goto after the call */ + attach(plain_fd, "global_never_unwinds", "freplace_unwinds", + "Extension program can unwind, while original does not", 0); + ASSERT_EQ(run(tgt_fd), 15, "retval"); + + /* + * An exception callback returns from the frame of the main prog. Here + * that's a return to a call that unwind_goto follows. unwind_exit of + * the callback has to be exit: the pad wasn't verified with it. + */ + shared_fd = bpf_program__fd(tgt->progs.shared_pad); + attach(shared_fd, "global_never_unwinds", "freplace_throws", NULL, 3); + + tail_fd = bpf_program__fd(tgt->progs.tail_call_in_subprog); + throws_fd = bpf_program__fd(tgt->progs.exception_callback_unwinds); + err = bpf_map__update_elem(tgt->maps.prog_array, &key, sizeof(key), + &throws_fd, sizeof(throws_fd), 0); + if (ASSERT_OK(err, "prog_array")) + ASSERT_EQ(run(tail_fd), 7, "retval"); +out: + verifier_unwind__destroy(tgt); +} diff --git a/tools/testing/selftests/bpf/prog_tests/unwind_cxx.c b/tools/testing/selftests/bpf/prog_tests/unwind_cxx.c new file mode 100644 index 000000000000..9faba99133ad --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/unwind_cxx.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <bpf/btf.h> + +/* The progs are under #ifdef __BPF_FEATURE_UNWIND */ +static bool llvm_lowers_unwinding(const char *file) +{ + struct btf *btf = btf__parse(file, NULL); + bool ret; + + ret = btf && btf__find_by_name_kind(btf, "catches", BTF_KIND_FUNC) > 0; + btf__free(btf); + return ret; +} + +static void run(struct bpf_object *obj, const char *name, int len, int retval, int dtors) +{ + struct bpf_program *prog = bpf_object__find_program_by_name(obj, name); + struct bpf_map *map = bpf_object__find_map_by_name(obj, ".data.dtors"); + char data[256] = {}; + LIBBPF_OPTS(bpf_test_run_opts, opts, + .data_in = data, + .data_size_in = len, + .repeat = 1, + ); + int key = 0, val = 0; + + if (!ASSERT_OK_PTR(prog, name) || !ASSERT_OK_PTR(map, "dtors")) + return; + ASSERT_OK(bpf_map__update_elem(map, &key, sizeof(key), &val, sizeof(val), 0), "clear"); + ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(prog), &opts), "test_run"); + ASSERT_EQ(opts.retval, retval, "retval"); + ASSERT_OK(bpf_map__lookup_elem(map, &key, sizeof(key), &val, sizeof(val), 0), "lookup"); + ASSERT_EQ(val, dtors, "dtors"); +} + +void test_unwind_cxx(void) +{ + const char *file = "./unwind_cxx.bpf.o"; + struct bpf_object *obj; + struct bpf_program *prog; + int err = -EINVAL; + + if (access(file, R_OK)) { + printf("%s:SKIP:gcc doesn't build %s\n", __func__, file); + test__skip(); + return; + } + if (!is_jit_enabled()) { + printf("%s:SKIP:JIT is disabled\n", __func__); + test__skip(); + return; + } + if (!llvm_lowers_unwinding(file)) { + printf("%s:SKIP:llvm doesn't lower unwinding\n", __func__); + test__skip(); + return; + } + obj = bpf_object__open_file(file, NULL); + if (!ASSERT_OK_PTR(obj, "open")) + return; + prog = bpf_object__find_program_by_name(obj, "catches_aggregate"); + if (!ASSERT_OK_PTR(prog, "catches_aggregate")) + goto out; + bpf_program__set_autoload(prog, false); + err = bpf_object__load(obj); + if (err == -ENOTSUPP) { + printf("%s:SKIP:JIT does not support unwinding\n", __func__); + test__skip(); + goto out; + } + if (!ASSERT_OK(err, "load")) + goto out; + + /* inner's destructor, then middle's */ + run(obj, "catches", 64, 65, 12); + run(obj, "catches", 200, 42, 12); + run(obj, "does_not_catch", 64, 65, 12); + run(obj, "does_not_catch", 200, 0, 12); +out: + bpf_object__close(obj); + if (err) + return; + + if (!test__start_subtest("aggregate_return")) + return; + obj = bpf_object__open_file(file, NULL); + if (!ASSERT_OK_PTR(obj, "open_aggregate")) + return; + bpf_object__for_each_program(prog, obj) + bpf_program__set_autoload(prog, + strcmp(bpf_program__name(prog), "catches_aggregate") == 0); + if (!ASSERT_OK(bpf_object__load(obj), "load_aggregate")) + goto out_aggregate; + run(obj, "catches_aggregate", 64, 71, 0); + run(obj, "catches_aggregate", 200, 42, 0); +out_aggregate: + bpf_object__close(obj); +} diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c index f0baf5738b75..7fbcfddc3cc3 100644 --- a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +++ b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c @@ -94,14 +94,14 @@ static int child_func(void *arg) /* wait for parent's kick */ err = read(child->go[0], &c, 1); if (err != 1) - exit(err); + _exit(err); uprobe_multi_func_1(); uprobe_multi_func_2(); uprobe_multi_func_3(); usdt_trigger(); - exit(errno); + _exit(errno); } static int spawn_child_flag(struct child *child, bool clone_vm) diff --git a/tools/testing/selftests/bpf/prog_tests/usdt.c b/tools/testing/selftests/bpf/prog_tests/usdt.c index 8004c9568ffa..eff1e57ab13c 100644 --- a/tools/testing/selftests/bpf/prog_tests/usdt.c +++ b/tools/testing/selftests/bpf/prog_tests/usdt.c @@ -250,6 +250,7 @@ cleanup: #ifdef __x86_64__ extern void usdt_1(void); extern void usdt_2(void); +extern void usdt_2_cross_page(void); extern void usdt_red_zone_trigger(void); static unsigned char nop1[1] = { 0x90 }; @@ -343,6 +344,57 @@ cleanup: } /* + * Test attachment to a USDT probe whose nop10 crosses a page boundary. + * The kernel can't optimize such nop10, so libbpf keeps the uprobe on + * the preceding 1-byte nop. Do not assume any particular placement + * here, though: however the probe ends up attached, the attachment + * must succeed and the probe must fire. + */ +static void subtest_optimized_attach_cross_page(void) +{ + long page_sz = sysconf(_SC_PAGESIZE); + struct test_usdt *skel; + __u8 *addr = NULL; + long i; + + /* combo is placed up to a page of padding after the function start */ + for (i = 0; i < 2 * page_sz; i++) { + if (!memcmp((void *)usdt_2_cross_page + i, nop1_nop10_combo, 11)) { + addr = (void *)usdt_2_cross_page + i; + break; + } + } + if (!ASSERT_OK_PTR(addr, "find_nop1_nop10_combo")) + return; + + /* layout sanity check: the nop10 must cross the page boundary */ + if (!ASSERT_GT((unsigned long)(addr + 1) % page_sz + 10, page_sz, + "nop10_crosses_page")) + return; + + skel = test_usdt__open_and_load(); + if (!ASSERT_OK_PTR(skel, "test_usdt__open_and_load")) + return; + + skel->bss->my_pid = getpid(); + + skel->links.usdt0 = bpf_program__attach_usdt(skel->progs.usdt0, + 0 /*self*/, "/proc/self/exe", + "optimized_attach", + "usdt_2_cross_page", NULL); + if (!ASSERT_OK_PTR(skel->links.usdt0, "bpf_program__attach_usdt")) + goto cleanup; + + usdt_2_cross_page(); + usdt_2_cross_page(); + + ASSERT_EQ(skel->bss->usdt0_called, 2, "usdt0_called"); + +cleanup: + test_usdt__destroy(skel); +} + +/* * Test that USDT arguments survive nop10 optimization in a function where * the compiler places operands in the red zone. * @@ -660,6 +712,8 @@ void test_usdt(void) subtest_basic_usdt(true); if (test__start_subtest("optimized_attach")) subtest_optimized_attach(); + if (test__start_subtest("optimized_attach_cross_page")) + subtest_optimized_attach_cross_page(); if (test__start_subtest("optimized_red_zone")) subtest_optimized_red_zone(); #endif diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 8b439e194bcc..04740ba37e91 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -3,12 +3,17 @@ #include <test_progs.h> #include "arena_kfunc.skel.h" +#include "arena_noarena.skel.h" #include "arena_kfunc_jit.skel.h" #include "cap_helpers.h" +#include "cb_refs.skel.h" +#include "verifier_aggregate_arg.skel.h" +#include "verifier_aggregate_ret.skel.h" #include "verifier_align.skel.h" #include "verifier_and.skel.h" #include "verifier_arena.skel.h" #include "verifier_arena_large.skel.h" +#include "verifier_arena_scalar.skel.h" #include "verifier_arena_globals1.skel.h" #include "verifier_arena_globals2.skel.h" #include "verifier_array_access.skel.h" @@ -16,16 +21,20 @@ #include "verifier_basic_stack.skel.h" #include "verifier_bitfield_write.skel.h" #include "verifier_bounds.skel.h" +#include "verifier_bounds_step.skel.h" #include "verifier_bounds_deduction.skel.h" #include "verifier_bounds_deduction_non_const.skel.h" #include "verifier_bounds_mix_sign_unsign.skel.h" #include "verifier_bpf_get_stack.skel.h" #include "verifier_bpf_trap.skel.h" #include "verifier_bswap.skel.h" +#include "verifier_btf_array_access.skel.h" #include "verifier_btf_ctx_access.skel.h" #include "verifier_btf_flex_array.skel.h" #include "verifier_btf_unreliable_prog.skel.h" #include "verifier_call_large_imm.skel.h" +#include "verifier_callx.skel.h" +#include "verifier_callx_rodata.skel.h" #include "verifier_cfg.skel.h" #include "verifier_cgroup_inv_retcode.skel.h" #include "verifier_cgroup_skb.skel.h" @@ -50,18 +59,24 @@ #include "verifier_helper_packet_access.skel.h" #include "verifier_helper_restricted.skel.h" #include "verifier_helper_value_access.skel.h" +#include "verifier_idoms.skel.h" #include "verifier_int_ptr.skel.h" #include "verifier_iterating_callbacks.skel.h" #include "verifier_jeq_infer_not_null.skel.h" #include "verifier_jit_convergence.skel.h" +#include "verifier_kfunc_packet_access.skel.h" +#include "verifier_kfunc_uninit.skel.h" +#include "verifier_kfunc_uninit_multi.skel.h" #include "verifier_kfunc_perfmon.skel.h" #include "verifier_ld_ind.skel.h" +#include "verifier_large_stack.skel.h" #include "verifier_ldsx.skel.h" #include "verifier_leak_ptr.skel.h" #include "verifier_linked_scalars.skel.h" #include "verifier_live_stack.skel.h" #include "verifier_liveness_exp.skel.h" #include "verifier_load_acquire.skel.h" +#include "verifier_loop_hierarchy.skel.h" #include "verifier_loops1.skel.h" #include "verifier_lwt.skel.h" #include "verifier_map_in_map.skel.h" @@ -94,6 +109,7 @@ #include "verifier_ringbuf.skel.h" #include "verifier_runtime_jit.skel.h" #include "verifier_scalar_ids.skel.h" +#include "verifier_scev.skel.h" #include "verifier_sdiv.skel.h" #include "verifier_search_pruning.skel.h" #include "verifier_sock.skel.h" @@ -114,6 +130,7 @@ #include "verifier_typedef.skel.h" #include "verifier_uninit.skel.h" #include "verifier_unpriv.skel.h" +#include "verifier_unwind.skel.h" #include "verifier_unpriv_perf.skel.h" #include "verifier_value_adj_spill.skel.h" #include "verifier_value.skel.h" @@ -129,6 +146,7 @@ #include "verifier_bits_iter.skel.h" #include "verifier_set_retval.skel.h" #include "verifier_lsm.skel.h" +#include "verifier_lsm_init_xattr.skel.h" #include "verifier_jit_inline.skel.h" #include "irq.skel.h" #include "verifier_ctx_ptr_param.skel.h" @@ -168,29 +186,47 @@ static void run_tests_aux(const char *skel_name, #define RUN(skel) run_tests_aux(#skel, skel##__elf_bytes, NULL) +/* for tests of what the interpreter doesn't support */ +#define RUN_JITED(skel) do { \ + if (is_jit_enabled()) \ + RUN(skel); \ + else \ + test__skip(); \ +} while (0) + void test_arena_kfunc(void) { RUN_TESTS(arena_kfunc); } +void test_arena_noarena(void) { RUN_TESTS(arena_noarena); } void test_arena_kfunc_jit(void) { RUN_TESTS(arena_kfunc_jit); } +void test_cb_refs(void) { RUN_TESTS(cb_refs); } + +void test_verifier_aggregate_arg(void) { RUN_TESTS(verifier_aggregate_arg); } +void test_verifier_aggregate_ret(void) { RUN_TESTS(verifier_aggregate_ret); } void test_verifier_align(void) { RUN(verifier_align); } void test_verifier_and(void) { RUN(verifier_and); } void test_verifier_arena(void) { RUN(verifier_arena); } void test_verifier_arena_large(void) { RUN(verifier_arena_large); } +void test_verifier_arena_scalar(void) { RUN(verifier_arena_scalar); } void test_verifier_arena_globals1(void) { RUN(verifier_arena_globals1); } void test_verifier_arena_globals2(void) { RUN(verifier_arena_globals2); } void test_verifier_basic_stack(void) { RUN(verifier_basic_stack); } void test_verifier_bitfield_write(void) { RUN(verifier_bitfield_write); } void test_verifier_bounds(void) { RUN(verifier_bounds); } +void test_verifier_bounds_step(void) { RUN(verifier_bounds_step); } void test_verifier_bounds_deduction(void) { RUN(verifier_bounds_deduction); } void test_verifier_bounds_deduction_non_const(void) { RUN(verifier_bounds_deduction_non_const); } void test_verifier_bounds_mix_sign_unsign(void) { RUN(verifier_bounds_mix_sign_unsign); } void test_verifier_bpf_get_stack(void) { RUN(verifier_bpf_get_stack); } void test_verifier_bpf_trap(void) { RUN(verifier_bpf_trap); } void test_verifier_bswap(void) { RUN(verifier_bswap); } +void test_verifier_btf_array_access(void) { RUN(verifier_btf_array_access); } void test_verifier_btf_ctx_access(void) { RUN(verifier_btf_ctx_access); } void test_verifier_btf_flex_array(void) { RUN(verifier_btf_flex_array); } void test_verifier_btf_unreliable_prog(void) { RUN(verifier_btf_unreliable_prog); } void test_verifier_call_large_imm(void) { RUN(verifier_call_large_imm); } +void test_verifier_callx(void) { RUN_JITED(verifier_callx); } +void test_verifier_callx_rodata(void) { RUN_JITED(verifier_callx_rodata); } void test_verifier_cfg(void) { RUN(verifier_cfg); } void test_verifier_cgroup_inv_retcode(void) { RUN(verifier_cgroup_inv_retcode); } void test_verifier_cgroup_skb(void) { RUN(verifier_cgroup_skb); } @@ -215,18 +251,24 @@ void test_verifier_helper_access_var_len(void) { RUN(verifier_helper_access_var_ void test_verifier_helper_packet_access(void) { RUN(verifier_helper_packet_access); } void test_verifier_helper_restricted(void) { RUN(verifier_helper_restricted); } void test_verifier_helper_value_access(void) { RUN(verifier_helper_value_access); } +void test_verifier_idoms(void) { RUN(verifier_idoms); } void test_verifier_int_ptr(void) { RUN(verifier_int_ptr); } void test_verifier_iterating_callbacks(void) { RUN(verifier_iterating_callbacks); } void test_verifier_jeq_infer_not_null(void) { RUN(verifier_jeq_infer_not_null); } void test_verifier_jit_convergence(void) { RUN(verifier_jit_convergence); } +void test_verifier_kfunc_packet_access(void) { RUN_TESTS(verifier_kfunc_packet_access); } +void test_verifier_kfunc_uninit(void) { RUN_TESTS(verifier_kfunc_uninit); } +void test_verifier_kfunc_uninit_multi(void) { RUN_TESTS(verifier_kfunc_uninit_multi); } void test_verifier_kfunc_perfmon(void) { RUN(verifier_kfunc_perfmon); } void test_verifier_load_acquire(void) { RUN(verifier_load_acquire); } void test_verifier_ld_ind(void) { RUN(verifier_ld_ind); } +void test_verifier_large_stack(void) { RUN(verifier_large_stack); } void test_verifier_ldsx(void) { RUN(verifier_ldsx); } void test_verifier_leak_ptr(void) { RUN(verifier_leak_ptr); } void test_verifier_linked_scalars(void) { RUN(verifier_linked_scalars); } void test_verifier_live_stack(void) { RUN(verifier_live_stack); } void test_verifier_liveness_exp(void) { RUN(verifier_liveness_exp); } +void test_verifier_loop_hierarchy(void) { RUN(verifier_loop_hierarchy); } void test_verifier_loops1(void) { RUN(verifier_loops1); } void test_verifier_lwt(void) { RUN(verifier_lwt); } void test_verifier_map_in_map(void) { RUN(verifier_map_in_map); } @@ -258,6 +300,7 @@ void test_verifier_regalloc(void) { RUN(verifier_regalloc); } void test_verifier_ringbuf(void) { RUN(verifier_ringbuf); } void test_verifier_runtime_jit(void) { RUN(verifier_runtime_jit); } void test_verifier_scalar_ids(void) { RUN(verifier_scalar_ids); } +void test_verifier_scev(void) { RUN(verifier_scev); } void test_verifier_sdiv(void) { RUN(verifier_sdiv); } void test_verifier_search_pruning(void) { RUN(verifier_search_pruning); } void test_verifier_sock(void) { RUN(verifier_sock); } @@ -278,6 +321,7 @@ void test_verifier_tailcall_jit(void) { RUN(verifier_tailcall_jit); } void test_verifier_typedef(void) { RUN(verifier_typedef); } void test_verifier_uninit(void) { RUN(verifier_uninit); } void test_verifier_unpriv(void) { RUN(verifier_unpriv); } +void test_verifier_unwind(void) { RUN(verifier_unwind); } void test_verifier_unpriv_perf(void) { RUN(verifier_unpriv_perf); } void test_verifier_value_adj_spill(void) { RUN(verifier_value_adj_spill); } void test_verifier_value(void) { RUN(verifier_value); } @@ -292,6 +336,7 @@ void test_verifier_xdp_direct_packet_access(void) { RUN(verifier_xdp_direct_pack void test_verifier_bits_iter(void) { RUN(verifier_bits_iter); } void test_verifier_set_retval(void) { RUN(verifier_set_retval); } void test_verifier_lsm(void) { RUN(verifier_lsm); } +void test_verifier_lsm_init_xattr(void) { RUN(verifier_lsm_init_xattr); } void test_irq(void) { RUN(irq); } void test_verifier_mtu(void) { RUN(verifier_mtu); } void test_verifier_jit_inline(void) { RUN(verifier_jit_inline); } diff --git a/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c index f327feb8e38c..12b146d205d7 100644 --- a/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c +++ b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c @@ -257,7 +257,7 @@ static void test_verify_pkcs7_sig_from_map(void) if (!ASSERT_OK_PTR(tmp_dir, "mkdtemp")) return; - ret = _run_setup_process(tmp_dir, "setup"); + ret = _run_setup_process(tmp_dir, "setup-rsa"); if (!ASSERT_OK(ret, "_run_setup_process")) goto close_prog; @@ -458,7 +458,7 @@ static void test_pkcs7_sig_fsverity(void) snprintf(data_path, PATH_MAX, "%s/data-file", tmp_dir); snprintf(sig_path, PATH_MAX, "%s/sig-file", tmp_dir); - ret = _run_setup_process(tmp_dir, "setup"); + ret = _run_setup_process(tmp_dir, "setup-rsa"); if (!ASSERT_OK(ret, "_run_setup_process")) goto out; diff --git a/tools/testing/selftests/bpf/prog_tests/wq.c b/tools/testing/selftests/bpf/prog_tests/wq.c index 84831eecc935..97926e35aaf2 100644 --- a/tools/testing/selftests/bpf/prog_tests/wq.c +++ b/tools/testing/selftests/bpf/prog_tests/wq.c @@ -64,10 +64,9 @@ static void test_failure_map_no_btf(void) goto out; err = bpf_map__reuse_fd(skel->maps.array, map_fd); - if (!ASSERT_OK(err, "map reuse fd")) { - close(map_fd); + close(map_fd); + if (!ASSERT_OK(err, "map reuse fd")) goto out; - } insns = bpf_program__insns(skel->progs.test_map_no_btf); if (!ASSERT_OK_PTR(insns, "insns ptr")) diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c index fce203640f8c..a894b1ab46f4 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c @@ -18,6 +18,9 @@ static void test_xdp_update_frags(void) return; prog = bpf_object__next_program(obj, NULL); + ASSERT_EQ(bpf_program__flags(prog) & BPF_F_XDP_HAS_FRAGS, + BPF_F_XDP_HAS_FRAGS, "frags in program flags"); + if (bpf_object__load(obj)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c b/tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c index a8ab05216c38..dff6b3e7266e 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c @@ -146,6 +146,11 @@ static void test_xdp_with_devmap_frags_helpers(void) if (!ASSERT_OK_PTR(skel, "test_xdp_with_devmap_helpers__open_and_load")) return; + ASSERT_EQ(bpf_program__flags(skel->progs.xdp_dummy_dm_frags) & BPF_F_XDP_HAS_FRAGS, + BPF_F_XDP_HAS_FRAGS, "frags in program flags"); + ASSERT_EQ(bpf_program__flags(skel->progs.xdp_dummy_dm) & BPF_F_XDP_HAS_FRAGS, + 0, "frags not in program flags"); + dm_fd_frags = bpf_program__fd(skel->progs.xdp_dummy_dm_frags); map_fd = bpf_map__fd(skel->maps.dm_ports); err = bpf_prog_get_info_by_fd(dm_fd_frags, &info, &len); diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c b/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c index 5c31054ad4a4..3ab963458903 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c @@ -408,14 +408,14 @@ void test_xdp_metadata(void) prog = bpf_object__find_program_by_name(bpf_obj->obj, "rx"); bpf_program__set_ifindex(prog, rx_ifindex); - bpf_program__set_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__add_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY); /* Make sure we can load a dev-bound program that performs * XDP_REDIRECT into a devmap. */ new_prog = bpf_object__find_program_by_name(bpf_obj->obj, "redirect"); bpf_program__set_ifindex(new_prog, rx_ifindex); - bpf_program__set_flags(new_prog, BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__add_flags(new_prog, BPF_F_XDP_DEV_BOUND_ONLY); if (!ASSERT_OK(xdp_metadata__load(bpf_obj), "load skeleton")) goto out; diff --git a/tools/testing/selftests/bpf/prog_tests/xsk.c b/tools/testing/selftests/bpf/prog_tests/xsk.c index 6e2f63ee2a6c..772f3dd7ce64 100644 --- a/tools/testing/selftests/bpf/prog_tests/xsk.c +++ b/tools/testing/selftests/bpf/prog_tests/xsk.c @@ -75,7 +75,7 @@ static void test_xsk(const struct test_spec *test_to_run, enum test_mode mode) if (!ASSERT_OK_PTR(ifobj_rx, "create ifobj_rx")) goto delete_tx; - if (!ASSERT_OK(configure_ifobj(ifobj_tx, ifobj_rx), "conigure ifobj")) + if (!ASSERT_OK(configure_ifobj(ifobj_tx, ifobj_rx), "configure ifobj")) goto delete_rx; ret = get_hw_ring_size(ifobj_tx->ifname, &ifobj_tx->ring); diff --git a/tools/testing/selftests/bpf/progs/aggregate_arg_func.c b/tools/testing/selftests/bpf/progs/aggregate_arg_func.c new file mode 100644 index 000000000000..2132b926a8f0 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_arg_func.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +#ifdef __SIZEOF_INT128__ +typedef unsigned __int128 u128; +#endif + +struct pair { + __u64 lo; + __u64 hi; +}; + +struct too_big { + __u64 a; + __u64 b; + __u64 c; +}; + +#if defined(__clang__) + +__noinline __u64 global_arg_pair(int a, struct pair p, int c) +{ + return (__u64)a + p.lo + p.hi + c; +} + +SEC("tc") +__success __retval(0x33) +__naked int aggregate_arg_pair_asm(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 0x10;" /* p.lo */ + "r3 = 0x20;" /* p.hi */ + "r4 = 2;" + "call %[global_arg_pair];" + "exit;" + : + : __imm(global_arg_pair) + : __clobber_all); +} + +SEC("tc") +__failure __msg("R2 type=fp expected=scalar") +__naked int aggregate_arg_pair_ptr_fail(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = r10;" /* a stack pointer where p.lo belongs */ + "r3 = 0x20;" + "r4 = 2;" + "call %[global_arg_pair];" + "exit;" + : + : __imm(global_arg_pair) + : __clobber_all); +} + +#endif + +__noinline __u64 global_arg_too_big(struct too_big s) +{ + return s.a + s.b + s.c; +} + +SEC("tc") +__failure __msg("in global_arg_too_big() has size 24, only 1 to 16 bytes can be passed by value") +__naked int aggregate_arg_too_big_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "call %[global_arg_too_big];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_too_big) + : __clobber_all); +} + +#if defined(__BPF_FEATURE_STACK_ARGUMENT) + +/* + * One slot per parameter, so the slot count is settled before the parameters + * are walked. The cases below reach the same count only once they have been. + */ +__noinline __u64 global_arg_six_scalars(int a, int b, int c, int d, int e, int f) +{ + return (__u64)a + b + c + d + e + f; +} + +SEC("tc") +__failure __msg("global function global_arg_six_scalars() needs 6 > 5 argument slots") +__naked int aggregate_arg_six_scalars_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call %[global_arg_six_scalars];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_six_scalars) + : __clobber_all); +} + +__noinline __u64 global_arg_split(int a, int b, int c, int d, struct pair p) +{ + return (__u64)a + b + c + d + p.lo + p.hi; +} + +SEC("tc") +__failure __msg("global function global_arg_split() needs 6 > 5 argument slots") +__naked int aggregate_arg_split_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call %[global_arg_split];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_split) + : __clobber_all); +} + +__noinline __u64 global_arg_past_regs(struct pair p, struct pair q, int a, struct pair r) +{ + return p.lo + p.hi + q.lo + q.hi + a + r.lo + r.hi; +} + +SEC("tc") +__failure __msg("global function global_arg_past_regs() needs 7 > 5 argument slots") +__naked int aggregate_arg_past_regs_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call %[global_arg_past_regs];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_past_regs) + : __clobber_all); +} + +#ifdef __SIZEOF_INT128__ + +__noinline __u64 global_arg_i128_slots(u128 v, int a, int b, int c, int d) +{ + return (__u64)v + a + b + c + d; +} + +SEC("tc") +__failure __msg("global function global_arg_i128_slots() needs 6 > 5 argument slots") +__naked int aggregate_arg_i128_slots_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call %[global_arg_i128_slots];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_i128_slots) + : __clobber_all); +} + +#endif /* __SIZEOF_INT128__ */ + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c new file mode 100644 index 000000000000..c87e886423f4 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "../test_kmods/bpf_testmod_kfunc.h" +#include "bpf_misc.h" + +#ifdef __SIZEOF_INT128__ +typedef unsigned __int128 u128; +#endif + +#define MIX_A 0xdeadbeefcafef00dULL +#define MIX_B 0x0123456789abcdefULL + +#if defined(__clang__) + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_struct(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + + if (bpf_kfunc_call_test_pair_arg(1, s, 2) != 2 * a + 3 * b + 9) + return 1; + + return 0; +} + +#endif + +#ifdef __SIZEOF_INT128__ + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + + if (bpf_kfunc_call_test_i128_arg(1, 2, v) != 4 * a + 3 * b + 5) + return 1; + + return 0; +} + +/* + * arm64 rounds the register number up to an even one for an argument + * aligned to 16 bytes, so it wants this __int128 in x2 and x3, leaving x1 + * empty, and the last argument in x4. The x86-64 ABI has no such rule. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128_odd(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + + if (bpf_kfunc_call_test_i128_arg_odd(1, v, 2) != 1 + 2 * b + 3 * a + 4 * 2) + return 1; + + return 0; +} + +#endif /* __SIZEOF_INT128__ */ + +#if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT) + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_last_regs(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + + if (bpf_kfunc_call_test_pair_arg_nofit(1, 2, 3, 4, s) != 5 * a + 6 * b + 30) + return 1; + + return 0; +} + +/* + * The x86-64 ABI moves an argument its six remaining registers cannot hold + * wholly onto the stack. arm64, with eight argument registers, still has a + * pair for it. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_straddle(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_big_arg s = { .a = a, .b = b }; + + if (bpf_kfunc_call_stack_arg_big(1, 2, 3, 4, 5, s) != 6 * a + 7 * b + 55) + return 1; + + return 0; +} + +/* The same, with an argument after the struct to take the eighth register. */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_tail(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + + if (bpf_kfunc_call_test_pair_arg_tail(1, 2, 3, 4, 5, s, 6) != 6 * a + 7 * b + 103) + return 1; + + return 0; +} + +/* + * arm64 gives no register to an argument its eight registers cannot hold, + * nor to anything after it. Past its six registers the x86-64 ABI has both + * eightbytes on the stack either way. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_split8(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + + if (bpf_kfunc_call_test_pair_arg_split8(1, 2, 3, 4, 5, 6, 7, s) != 8 * a + 9 * b + 140) + return 1; + + return 0; +} + +#ifdef __SIZEOF_INT128__ + +/* + * The hole of aggregate_arg_kfunc_int128_odd, with arguments after it that + * arm64 shifts through x5 to x7, where its JIT keeps the BPF stack slots, + * and onto its stack. The second __int128 then leaves the second arm64 + * stack slot empty. x86-64 packs the eightbytes like the BPF convention, + * so nothing moves there. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128_odd_many(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + u128 w = ((u128)~a << 64) | ~b; + + if (bpf_kfunc_call_test_i128_arg_odd_many(1, v, 2, 3, 4, 5, 6, w) != + 1 + 2 * b + 3 * a + 4 * 2 + 5 * 3 + 6 * 4 + 7 * 5 + 8 * 6 + + 9 * ~b + 10 * ~a) + return 1; + + return 0; +} + +/* + * x86-64 moves s to the stack and gives R9 back to the seventh argument. + * w, aligned to 16 bytes by the __int128 it holds, then leaves a stack slot + * empty before it: the fourth on x86-64, the second on arm64. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128_stack_hole(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + struct prog_test_i128_arg w = { .v = ((u128)~a << 64) | ~b }; + + if (bpf_kfunc_call_test_i128_arg_stack_hole(1, 2, 3, 4, 5, s, 6, 7, w) != + 1 + 2 * 2 + 3 * 3 + 4 * 4 + 5 * 5 + 6 * a + 7 * b + 8 * 6 + 9 * 7 + + 10 * ~b + 11 * ~a) + return 1; + + return 0; +} + +/* + * Both conventions pad the stack to align this __int128, and the BPF + * convention pads for neither, so both JITs move it up an eightbyte. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128_pad(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + + if (bpf_kfunc_call_test_i128_arg_pad(1, 2, 3, 4, 5, 6, 7, v) != 9 * a + 8 * b + 140) + return 1; + + return 0; +} + +#endif /* __SIZEOF_INT128__ */ + +#endif + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("R1 type STRUCT is not composed of scalars") +int aggregate_arg_kfunc_ptr_member(struct __sk_buff *skb) +{ + struct prog_test_ptr_arg s = { .p = skb, .x = 1 }; + + return bpf_kfunc_call_test_ptr_arg(s); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_func.c b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c new file mode 100644 index 000000000000..f0077d951a25 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c @@ -0,0 +1,383 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf_arena_common.h> +#include "bpf_misc.h" + +typedef unsigned __int128 u128; + +__naked u128 global_agg_good(void) +{ + asm volatile ( + "r0 = 0x1234;" /* low 64 bits */ + "r2 = 0x5678;" /* high 64 bits */ + "exit;" + ); +} + +__naked u128 global_agg_bad(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +__naked u128 global_agg_bad_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = r10;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("R2 !read_ok") +__naked int aggregate_ret_global_fail(void) +{ + asm volatile ( + "call %[global_agg_bad];" + "r0 = r2;" + "exit;" + : + : __imm(global_agg_bad) + : __clobber_all); +} + +SEC("tc") +__load_if_JITed() +__failure __msg("At subprogram exit the register R2 is not a scalar value") +__naked int aggregate_ret_global_ptr_fail(void) +{ + asm volatile ( + "call %[global_agg_bad_ptr];" + "r0 = r2;" + "exit;" + : + : __imm(global_agg_bad_ptr) + : __clobber_all); +} + +static __naked __noinline u128 static_agg_bad_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = r10;" /* stack pointer placed in the second return register */ + "exit;" + ); +} + +/* + * R2 is a return register once the subprogram returns a pair, so a stack + * pointer left in it is rejected at the callee's exit exactly as one in R0 + * is: the callee frame is gone by the time the caller could use it. + */ +SEC("tc") +__load_if_JITed() +__failure __msg("cannot return stack pointer to the caller") +__naked int aggregate_ret_static_ptr_fail(void) +{ + asm volatile ( + "call %[static_agg_bad_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(static_agg_bad_ptr) + : __clobber_all); +} + +static __naked __noinline u128 static_agg_no_r2(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("R2 !read_ok") +__naked int aggregate_ret_static_uninit_fail(void) +{ + asm volatile ( + "call %[static_agg_no_r2];" + "r0 = r2;" + "exit;" + : + : __imm(static_agg_no_r2) + : __clobber_all); +} + +static __naked __noinline u128 static_agg_precise(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 4;" /* second half; its value is made precise below */ + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__log_level(2) +__msg("mark_precise: frame0: last_idx 5 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (57) r6 &= 7") +__msg("mark_precise: frame0: regs=r6 stack= before 1: (bf) r6 = r2") +__msg("mark_precise: frame0: regs=r2 stack= before 12: (95) exit") +__msg("mark_precise: frame1: regs=r2 stack= before 11: (b7) r2 = 4") +__naked int aggregate_ret_static_precise(void) +{ + asm volatile ( + "call %[static_agg_precise];" + "r6 = r2;" /* derived from the aggregate's second half */ + "r6 &= 7;" /* keep it in [0, 7] to index the stack */ + "r1 = r10;" + "r1 += -8;" + "r1 += r6;" /* ptr += scalar marks r6 (hence R2) precise */ + "r0 = 0;" + "*(u8 *)(r1 + 0) = r0;" + "r0 = 0;" + "exit;" + : + : __imm(static_agg_precise) + : __clobber_all); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__log_level(2) +__msg("mark_precise: frame0: last_idx 5 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (57) r6 &= 7") +__msg("mark_precise: frame0: regs=r6 stack= before 1: (bf) r6 = r2") +__msg("mark_precise: frame0: regs=r2 stack= before 0: (85) call pc+9") +__naked int aggregate_ret_global_precise(void) +{ + asm volatile ( + "call %[global_agg_good];" + "r6 = r2;" /* derived from the aggregate's second half */ + "r6 &= 7;" /* keep it in [0, 7] to index the stack */ + "r1 = r10;" + "r1 += -8;" + "r1 += r6;" /* ptr += scalar marks r6 (hence R2) precise */ + "r0 = 0;" + "*(u8 *)(r1 + 0) = r0;" + "r0 = 0;" + "exit;" + : + : __imm(global_agg_good) + : __clobber_all); +} + +#if defined(__clang_major__) && __clang_major__ >= 23 + +/* A by-value struct that smuggles a pointer, which must be rejected. */ +struct with_ptr { + void *p; + __u64 x; +}; + +/* A by-value union that smuggles a pointer, which must be rejected too. */ +union upair_with_ptr { + void *p; + __u64 halves[2]; +}; + +__naked struct with_ptr global_ret_struct_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("Global function global_ret_struct_ptr() has unsupported return type") +__naked int aggregate_ret_global_struct_ptr_fail(void) +{ + asm volatile ( + "call %[global_ret_struct_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_struct_ptr) + : __clobber_all); +} + +__naked union upair_with_ptr global_ret_union_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("Global function global_ret_union_ptr() has unsupported return type") +__naked int aggregate_ret_global_union_ptr_fail(void) +{ + asm volatile ( + "call %[global_ret_union_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_union_ptr) + : __clobber_all); +} + +#endif + +/* + * gcc returns a by-value struct through a hidden pointer, and emits the + * 'r0 = r1' returning it after the __naked body's exit, leaving the + * subprogram falling through. Build these with clang only. + */ +#if defined(__clang__) + +struct arena_pair { + void __arena *lo; + void __arena *hi; +}; + +struct arena_and_scalar { + void __arena *p; + __u64 x; +}; + +struct arena_array { + void __arena *p[2]; +}; + +struct arena_single { + void __arena *p; +}; + +union arena_upair { + void __arena *p; + __u64 halves[2]; +}; + +__naked struct arena_pair global_ret_arena_pair(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_pair(void) +{ + asm volatile ( + "call %[global_ret_arena_pair];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_pair) + : __clobber_all); +} + +__naked struct arena_and_scalar global_ret_arena_and_scalar(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_and_scalar(void) +{ + asm volatile ( + "call %[global_ret_arena_and_scalar];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_and_scalar) + : __clobber_all); +} + +__naked struct arena_array global_ret_arena_array(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_array(void) +{ + asm volatile ( + "call %[global_ret_arena_array];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_array) + : __clobber_all); +} + +__naked struct arena_single global_ret_arena_single(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +SEC("tc") +__success __retval(0) +__naked int aggregate_ret_global_arena_single(void) +{ + asm volatile ( + "call %[global_ret_arena_single];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_single) + : __clobber_all); +} + +__naked union arena_upair global_ret_arena_union(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_union(void) +{ + asm volatile ( + "call %[global_ret_arena_union];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_union) + : __clobber_all); +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c new file mode 100644 index 000000000000..0361a7b42c9b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +/* + * Reference kfunc addresses to force those BTF to be emitted. Taking the address + * (rather than calling) avoids any dependence on the compiler lowering an + * __int128 or struct return value, which the BPF backend only supports from + * LLVM 23 on. + */ +void __kfunc_btf_root(void) +{ + asm volatile ("" + : + : "r"(&bpf_kfunc_call_test_i128), + "r"(&bpf_kfunc_call_test_ret_fastcall), + "r"(&bpf_kfunc_call_test_ret_ptr), + "r"(&bpf_kfunc_call_test_ret_ii), + "r"(&bpf_kfunc_call_test_ret_nested), + "r"(&bpf_kfunc_call_test_ret_ptr_arr), + "r"(&bpf_kfunc_call_test_ret_deep), + "r"(&bpf_kfunc_call_test_ret_arr_struct), + "r"(&bpf_kfunc_call_test_ret_arr2d), + "r"(&bpf_kfunc_call_test_ret_big)); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +__log_level(2) +__msg("mark_precise: frame0: last_idx 7 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r6 stack= before 6: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 5: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (57) r6 &= 7") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r6 = r2") +__msg("mark_precise: frame0: regs=r2 stack= before 2: (85) call bpf_kfunc_call_test_i128") +__naked int aggregate_ret_kfunc_precise(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "call %[bpf_kfunc_call_test_i128];" + "r6 = r2;" /* second return half */ + "r6 &= 7;" /* keep it in [0, 7] to index the stack */ + "r1 = r10;" + "r1 += -8;" + "r1 += r6;" /* ptr += scalar marks r6 (hence R2) precise */ + "r0 = 0;" + "*(u8 *)(r1 + 0) = r0;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_i128) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("kfunc bpf_kfunc_call_test_ret_fastcall with >8-byte return is not supported with KF_FASTCALL") +__naked int aggregate_ret_kfunc_fastcall_fail(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "call %[bpf_kfunc_call_test_ret_fastcall];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_fastcall) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'p' has type PTR") +__naked int aggregate_ret_kfunc_ptr_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_ptr) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'in.p' has type PTR") +__naked int aggregate_ret_kfunc_nested_ptr_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_nested];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_nested) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'p[]' has type PTR") +__naked int aggregate_ret_kfunc_ptr_arr_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_ptr_arr];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_ptr_arr) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'in1[].in2.p' has type PTR") +__naked int aggregate_ret_kfunc_arr_struct_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_arr_struct];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_arr_struct) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'a[].p' has type PTR") +__naked int aggregate_ret_kfunc_arr2d_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_arr2d];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_arr2d) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("max struct nesting depth exceeded") +__naked int aggregate_ret_kfunc_too_deep_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_deep];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_deep) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("R2 !read_ok") +__naked int aggregate_ret_kfunc_small_no_r2(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "call %[bpf_kfunc_call_test_ret_ii];" + "r0 = r2;" /* R2 is not a return register for a <=8 byte struct */ + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_ii) + : __clobber_all); +} + +/* + * A return value larger than 16 bytes does not fit in R0:R2 and is rejected by + * btf_distill_func_proto() before the KF_FASTCALL and JIT-capability checks. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("The function bpf_kfunc_call_test_ret_big return type STRUCT is unsupported") +__naked int aggregate_ret_kfunc_too_big_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_big];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_big) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c new file mode 100644 index 000000000000..a0f263694a63 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf_arena_common.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +#if defined(__clang_major__) && __clang_major__ >= 23 + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 2); +} arena SEC(".maps"); + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_arena(void *ctx) +{ + u32 volatile __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + u32 volatile __arena *a, *b; + struct prog_test_ret_arena r; + + if (!page) + return 1; + + /* Both halves come back in R0:R2, pointing at page and page + 4. */ + r = bpf_kfunc_call_test_ret_arena((u64)page); + if (!r.a || !r.b) + return 2; + + a = (u32 __arena *)r.a; + b = (u32 __arena *)r.b; + *a = 1; + *b = 2; + if (*a != 1) + return 3; + if (*b != 2) + return 4; + + page[0] = 7; + if (*a != 7) + return 5; + page[1] = 9; + if (*b != 9) + return 6; + + return 0; +} + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_arena_mixed(void *ctx) +{ + u32 __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + struct prog_test_ret_arena_mixed r; + u32 volatile __arena *p; + + if (!page) + return 1; + + /* An arena pointer in R0 beside a scalar in R2. */ + r = bpf_kfunc_call_test_ret_arena_mixed((u64)page); + if (!r.p) + return 2; + if (r.tag != 0xbeef) + return 3; + + p = (u32 __arena *)r.p; + *p = 3; + if (*p != 3) + return 4; + + return 0; +} + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'b' has type PTR") +int aggregate_ret_kfunc_arena_untagged_fail(void *ctx) +{ + struct prog_test_ret_arena_untagged r; + + r = bpf_kfunc_call_test_ret_arena_untagged(0); + + return r.a == r.b; +} + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'b' has type PTR") +int aggregate_ret_kfunc_arena_union_fail(void *ctx) +{ + union prog_test_ret_arena_union r; + + r = bpf_kfunc_call_test_ret_arena_union(0); + + return r.a == r.b; +} + +#else + +SEC("socket") +__description("aggregate_ret_kfunc_arena: needs LLVM 23, dummy test") +__skip("needs LLVM 23") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_target.c b/tools/testing/selftests/bpf/progs/aggregate_ret_target.c new file mode 100644 index 000000000000..cffd8d7d3241 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_target.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* freplace target: a global subprogram returning 16 bytes in R0:R2. */ +__naked unsigned __int128 agg_ret_target_func(void) +{ + asm volatile ( + "r0 = 0x1234;" + "r2 = 0x5678;" + "exit;" + ); +} + +SEC("tc") +__naked int agg_ret_target(void) +{ + asm volatile ( + "call %[agg_ret_target_func];" + "r0 = 0;" + "exit;" + : + : __imm(agg_ret_target_func) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/arena_atomics.c b/tools/testing/selftests/bpf/progs/arena_atomics.c index 73bc2b835f3f..ce76a73d197b 100644 --- a/tools/testing/selftests/bpf/progs/arena_atomics.c +++ b/tools/testing/selftests/bpf/progs/arena_atomics.c @@ -91,13 +91,17 @@ int sub(const void *ctx) return 0; } -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING -_Atomic __u64 __arena_global and64_value = (0x110ull << 32); -_Atomic __u32 __arena_global and32_value = 0x110; -#else +/* + * A relaxed __c11_atomic_fetch_*() whose result is discarded is what selects + * the non-fetching insn, so each operation keeps a *_noreturn_value of each + * width to cover that lowering as well. + */ __u64 __arena_global and64_value = (0x110ull << 32); +__u64 __arena_global and64_result = 0; +_Atomic __u64 __arena_global and64_noreturn_value = (0x110ull << 32); __u32 __arena_global and32_value = 0x110; -#endif +__u32 __arena_global and32_result = 0; +_Atomic __u32 __arena_global and32_noreturn_value = 0x110; SEC("raw_tp/sys_enter") int and(const void *ctx) @@ -105,25 +109,22 @@ int and(const void *ctx) if (pid != (bpf_get_current_pid_tgid() >> 32)) return 0; #ifdef ENABLE_ATOMICS_TESTS -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING - __c11_atomic_fetch_and(&and64_value, 0x011ull << 32, memory_order_relaxed); - __c11_atomic_fetch_and(&and32_value, 0x011, memory_order_relaxed); -#else - __sync_fetch_and_and(&and64_value, 0x011ull << 32); - __sync_fetch_and_and(&and32_value, 0x011); -#endif + and64_result = __sync_fetch_and_and(&and64_value, 0x011ull << 32); + and32_result = __sync_fetch_and_and(&and32_value, 0x011); + __c11_atomic_fetch_and(&and64_noreturn_value, 0x011ull << 32, memory_order_relaxed); + __c11_atomic_fetch_and(&and32_noreturn_value, 0x011, memory_order_relaxed); #endif return 0; } -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING -_Atomic __u32 __arena_global or32_value = 0x110; -_Atomic __u64 __arena_global or64_value = (0x110ull << 32); -#else +/* u32s first: .addr_space.1 must not have holes, see the clang-17 note below */ __u32 __arena_global or32_value = 0x110; +__u32 __arena_global or32_result = 0; +_Atomic __u32 __arena_global or32_noreturn_value = 0x110; __u64 __arena_global or64_value = (0x110ull << 32); -#endif +__u64 __arena_global or64_result = 0; +_Atomic __u64 __arena_global or64_noreturn_value = (0x110ull << 32); SEC("raw_tp/sys_enter") int or(const void *ctx) @@ -131,25 +132,21 @@ int or(const void *ctx) if (pid != (bpf_get_current_pid_tgid() >> 32)) return 0; #ifdef ENABLE_ATOMICS_TESTS -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING - __c11_atomic_fetch_or(&or64_value, 0x011ull << 32, memory_order_relaxed); - __c11_atomic_fetch_or(&or32_value, 0x011, memory_order_relaxed); -#else - __sync_fetch_and_or(&or64_value, 0x011ull << 32); - __sync_fetch_and_or(&or32_value, 0x011); -#endif + or64_result = __sync_fetch_and_or(&or64_value, 0x011ull << 32); + or32_result = __sync_fetch_and_or(&or32_value, 0x011); + __c11_atomic_fetch_or(&or64_noreturn_value, 0x011ull << 32, memory_order_relaxed); + __c11_atomic_fetch_or(&or32_noreturn_value, 0x011, memory_order_relaxed); #endif return 0; } -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING -_Atomic __u64 __arena_global xor64_value = (0x110ull << 32); -_Atomic __u32 __arena_global xor32_value = 0x110; -#else __u64 __arena_global xor64_value = (0x110ull << 32); +__u64 __arena_global xor64_result = 0; +_Atomic __u64 __arena_global xor64_noreturn_value = (0x110ull << 32); __u32 __arena_global xor32_value = 0x110; -#endif +__u32 __arena_global xor32_result = 0; +_Atomic __u32 __arena_global xor32_noreturn_value = 0x110; SEC("raw_tp/sys_enter") int xor(const void *ctx) @@ -157,13 +154,10 @@ int xor(const void *ctx) if (pid != (bpf_get_current_pid_tgid() >> 32)) return 0; #ifdef ENABLE_ATOMICS_TESTS -#ifdef __BPF_FEATURE_ATOMIC_MEM_ORDERING - __c11_atomic_fetch_xor(&xor64_value, 0x011ull << 32, memory_order_relaxed); - __c11_atomic_fetch_xor(&xor32_value, 0x011, memory_order_relaxed); -#else - __sync_fetch_and_xor(&xor64_value, 0x011ull << 32); - __sync_fetch_and_xor(&xor32_value, 0x011); -#endif + xor64_result = __sync_fetch_and_xor(&xor64_value, 0x011ull << 32); + xor32_result = __sync_fetch_and_xor(&xor32_value, 0x011); + __c11_atomic_fetch_xor(&xor64_noreturn_value, 0x011ull << 32, memory_order_relaxed); + __c11_atomic_fetch_xor(&xor32_noreturn_value, 0x011, memory_order_relaxed); #endif return 0; @@ -213,23 +207,96 @@ int xchg(const void *ctx) return 0; } +__u64 __arena_global fetch_src_r0_value = 0x11000000110ull; +__u64 __arena_global fetch_src_r0_result = 0; +__u64 __arena_global fetch_dst_r0_value = 0x11000000110ull; +__u64 __arena_global fetch_dst_r0_result = 0; +__u64 __arena_global fetch_dst_r0_readback = 0; + +/* + * A fetching OR with the operand in r0, and one with the arena pointer in r0. + * The x86 JIT needs RAX for its CMPXCHG loop and substitutes BPF_REG_AX for + * whichever of the two is r0, so both have to keep working. Hand-written + * because clang picks its own registers and will not reliably emit either. + */ +SEC("raw_tp/sys_enter") +int fetch_r0(const void *ctx) +{ + if (pid != (bpf_get_current_pid_tgid() >> 32)) + return 0; +#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST) + asm volatile ( + "r1 = %[fetch_src_r0_value] ll;" + "r1 = addr_space_cast(r1, 0x0, 0x1);" + "r0 = 0x1100000011 ll;" + ".8byte %[fetch_src_r0_insn];" + "r2 = %[fetch_src_r0_result] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r0;" + : + : __imm_addr(fetch_src_r0_value), + __imm_insn(fetch_src_r0_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, BPF_REG_1, BPF_REG_0, 0)), + __imm_addr(fetch_src_r0_result) + : __clobber_all); + + asm volatile ( + "r0 = %[fetch_dst_r0_value] ll;" + "r0 = addr_space_cast(r0, 0x0, 0x1);" + "r1 = 0x1100000011 ll;" + ".8byte %[fetch_dst_r0_insn];" + "r2 = %[fetch_dst_r0_result] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r1;" + /* r0 is only read by the atomic, so it must still be the pointer. */ + "r3 = *(u64 *)(r0 + 0);" + "r2 = %[fetch_dst_r0_readback] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r3;" + : + : __imm_addr(fetch_dst_r0_value), + __imm_insn(fetch_dst_r0_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, BPF_REG_0, BPF_REG_1, 0)), + __imm_addr(fetch_dst_r0_result), + __imm_addr(fetch_dst_r0_readback) + : __clobber_all); +#endif + + return 0; +} + __u64 __arena_global uaf_sink; volatile __u64 __arena_global uaf_recovery_fails; +/* Register state after the r0 forms below fault, checked by the test */ +__u64 __arena_global uaf_r1_after_dst_r0; +__u64 __arena_global uaf_r0_before_dst_r0; +__u64 __arena_global uaf_r0_after_dst_r0; +__u64 __arena_global uaf_r0_after_src_r0; +/* What reading the faulted address again gives, see the end of uaf() */ +__u64 __arena_global uaf_reread; SEC("syscall") int uaf(const void *ctx) { if (pid != (bpf_get_current_pid_tgid() >> 32)) return 0; +/* + * __BPF_FEATURE_ADDR_SPACE_CAST is required by the asm below, which spells + * addr_space_cast textually. That and the mnemonic both arrived in LLVM 19. + */ #if defined(ENABLE_ATOMICS_TESTS) && !defined(__TARGET_ARCH_arm64) && \ - !defined(__TARGET_ARCH_x86) + defined(__BPF_FEATURE_ADDR_SPACE_CAST) __u32 __arena *page32; __u64 __arena *page64; void __arena *page; page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); bpf_arena_free_pages(&arena, page, 1); - uaf_recovery_fails = 24; + uaf_recovery_fails = 26; + uaf_r1_after_dst_r0 = 1; + uaf_r0_after_dst_r0 = 1; + uaf_r0_after_src_r0 = 1; + uaf_reread = 1; page32 = (__u32 __arena *)page; uaf_sink += __sync_fetch_and_add(page32, 1); @@ -282,6 +349,74 @@ int uaf(const void *ctx) uaf_recovery_fails -= 1; uaf_sink += __sync_lock_test_and_set(page64, 1); uaf_recovery_fails -= 1; + + /* + * The x86 JIT needs RAX for the CMPXCHG loop it lowers a fetching + * AND/OR/XOR into, and substitutes BPF_REG_AX for r0 when r0 is either + * operand. Only then does the fault fixup have to name that register, + * so spell both forms out: clang picks its own and never produces them. + * + * The freed page is the first of the arena, whose 32-bit address is 0, + * which would make "the address register was left alone" an empty + * check. Use the page after it, which is unmapped as well. + */ + asm volatile ( + "r0 = %[page];" + "r0 += 4096;" + "r0 = addr_space_cast(r0, 0x0, 0x1);" + "r1 = 1;" + "r2 = %[uaf_r0_before_dst_r0] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r0;" + ".8byte %[fetch_dst_r0];" + "r2 = %[uaf_r0_after_dst_r0] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r0;" + "r2 = %[uaf_r1_after_dst_r0] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r1;" + : : [page]"r"(page64), + __imm_insn(fetch_dst_r0, + BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, BPF_REG_0, BPF_REG_1, 0)), + __imm_addr(uaf_r0_before_dst_r0), + __imm_addr(uaf_r0_after_dst_r0), + __imm_addr(uaf_r1_after_dst_r0) + : "r0", "r1", "r2", "memory"); + uaf_recovery_fails -= 1; + + asm volatile ( + "r1 = %[page];" + "r1 += 4096;" + "r1 = addr_space_cast(r1, 0x0, 0x1);" + "r0 = 1;" + ".8byte %[fetch_src_r0];" + "r2 = %[uaf_r0_after_src_r0] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r0;" + : : [page]"r"(page64), + __imm_insn(fetch_src_r0, + BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, BPF_REG_1, BPF_REG_0, 0)), + __imm_addr(uaf_r0_after_src_r0) + : "r0", "r1", "r2", "memory"); + uaf_recovery_fails -= 1; + + /* + * If a fault were recovered by the arena mapping its zeroed scratch + * page rather than by the JIT's exception table entry, the access above + * would have succeeded and written there. The page has to still be + * unmapped, so that the read faults and gives 0. + */ + asm volatile ( + "r1 = %[page];" + "r1 += 4096;" + "r1 = addr_space_cast(r1, 0x0, 0x1);" + "r3 = *(u64 *)(r1 + 0);" + "r2 = %[uaf_reread] ll;" + "r2 = addr_space_cast(r2, 0x0, 0x1);" + "*(u64 *)(r2 + 0) = r3;" + : : [page]"r"(page64), + __imm_addr(uaf_reread) + : "r1", "r2", "r3", "memory"); #endif return 0; diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc.c b/tools/testing/selftests/bpf/progs/arena_kfunc.c index bf0d304e0e59..5d1cb1a7e8ee 100644 --- a/tools/testing/selftests/bpf/progs/arena_kfunc.c +++ b/tools/testing/selftests/bpf/progs/arena_kfunc.c @@ -192,20 +192,61 @@ int arena_arg_unpopulated(void *ctx) return 0; } +/* + * A subprogram whose only arena use is the argument of a kfunc needs the + * arena base too, the JIT rebases the argument on it. The address arrives + * as a number, so no cast marks the subprogram either. + */ +__noinline int arena_arg_from_subprog(unsigned long addr) +{ + return bpf_kfunc_arena_arg_test((u64 *)addr) != 41; +} + +/* + * A NULL constant for the __arena argument takes an early exit in the + * verifier; the call needs the arena base all the same, the JIT rebases + * the argument with a NULL test. + */ SEC("syscall") __arch_x86_64 __arch_arm64 -__failure __msg("arena pointer requires a program with an associated arena") -int arena_arg_no_arena(void *ctx) +__success __retval(0) +int arena_arg_null(void *ctx) { - bpf_kfunc_arena_arg_test((u64 *)1); + return bpf_kfunc_arena_cap_nullable_test(NULL) != 0; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_arg_subprog(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + int ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + if (arena_arg_from_subprog((unsigned long)val)) + ret = 2; + else + ret = *val != 42 ? 3 : 0; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); + return ret; +#else return 0; +#endif } SEC("syscall") __arch_x86_64 __arch_arm64 -__failure __msg("is not a pointer to arena or scalar") +__failure __msg("R1 type=fp expected=arena, scalar") int arena_arg_bad_reg(void *ctx) { u64 buf = 0; @@ -228,13 +269,29 @@ int arena_arg_stack(void *ctx) bpf_kfunc_arena_stack_arg_test(1, 2, 3, 4, 5, (u64 *)1); return 0; } + +#if defined(__clang__) +/* The struct takes two slots, so the arena pointer is the sixth. */ +SEC("syscall") +__arch_x86_64 __arch_arm64 +__failure __msg("arena pointer cannot be a stack argument") +int arena_arg_stack_after_pair(void *ctx) +{ + struct prog_test_pair_arg s = { .lo = 1, .hi = 2 }; + + bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + bpf_kfunc_call_test_pair_arena_arg(1, 2, 3, s, (u64 *)1); + return 0; +} +#endif #else SEC("syscall") __arch_x86_64 __arch_arm64 __description("arena_arg_stack: not supported, dummy test") +__skip("arena_arg_stack: not supported") __success -int arena_arg_stack(void *ctx) +int dummy_test(void *ctx) { return 0; } diff --git a/tools/testing/selftests/bpf/progs/arena_memcg.c b/tools/testing/selftests/bpf/progs/arena_memcg.c new file mode 100644 index 000000000000..aff73757e794 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_memcg.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_arena_common.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + /* + * Number of pages. Must cover ARENA_FAULT on the smallest page size + * (64M/4K = 16384) yet stay under the 4G arena limit on 64K pages + * (50000*64K = 3.2G). + */ + __uint(max_entries, 50000); +#ifdef __TARGET_ARCH_arm64 + __ulong(map_extra, 0x1ull << 32); /* start of mmap() region */ +#else + __ulong(map_extra, 0x1ull << 44); /* start of mmap() region */ +#endif +} arena SEC(".maps"); + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/arena_noarena.c b/tools/testing/selftests/bpf/progs/arena_noarena.c new file mode 100644 index 000000000000..edbd40cd5be8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_noarena.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +/* + * Programs without an arena. They cannot share an object with an arena map, + * libbpf binds the arena of an object to every program in it. + */ + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__failure __msg("arena pointer requires a program with an associated arena") +int arena_arg_no_arena(void *ctx) +{ + bpf_kfunc_arena_arg_test((u64 *)1); + return 0; +} + +SEC("syscall") +__flag(BPF_F_ARENA_SCALAR) +__arch_x86_64 +__arch_arm64 +__description("arena_scalar: no arena, no access through a number") +__failure __msg("R6 invalid mem access 'scalar'") +__naked void no_arena(void) +{ + asm volatile (" \ + r6 = 0x100000000000 ll; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" ::: __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/arena_race.c b/tools/testing/selftests/bpf/progs/arena_race.c new file mode 100644 index 000000000000..54dbd662ca70 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_race.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_experimental.h" +#include <bpf_arena_common.h> + +const volatile __u64 old_marker = 0x1111222233334444ULL; +const volatile __u64 new_marker = 0x5555666677778888ULL; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1024); +#ifdef __TARGET_ARCH_arm64 + __ulong(map_extra, 0x1ull << 32); +#else + __ulong(map_extra, 0x1ull << 44); +#endif +} arena SEC(".maps"); + +bool skip; +int page_idx; + +void __arena *ptr; +void __arena *realloc_ptr; +bool realloc_after_free; +__u64 current_marker; +__u64 marker_seq; + +int target_tid; +int pause_on_zap; +int zap_entered; +int release; +int timed_out; + +__u64 trigger_pid_tgid; +long trigger_syscall; +int deferred_free; +int worker_armed; +int worker_exited; + +static __always_inline void wait_for_release(void) +{ + while (!*(volatile int *)&release && can_loop) + ; + if (!*(volatile int *)&release) + timed_out = 1; +} + +SEC("syscall") +int alloc_old(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM) + __u64 __arena *p; + char __arena *base = arena_base(&arena); + + realloc_ptr = NULL; + ptr = bpf_arena_alloc_pages(&arena, base + (page_idx + 1) * __PAGE_SIZE, 1, + NUMA_NO_NODE, 0); + if (!ptr) + return 1; + p = (__u64 __arena *)ptr; + *p = old_marker; + current_marker = old_marker; +#else + skip = true; +#endif + return 0; +} + +SEC("syscall") +int free_page(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM) + __u64 __arena *p; + __u64 marker; + + if (!ptr) + return 1; + bpf_arena_free_pages(&arena, ptr, 1); + if (!realloc_after_free) + return 0; + + marker = new_marker + ++marker_seq; + realloc_ptr = bpf_arena_alloc_pages(&arena, ptr, 1, NUMA_NO_NODE, 0); + if (!realloc_ptr) + return 0; + ptr = realloc_ptr; + p = (__u64 __arena *)ptr; + *p = marker; + current_marker = marker; +#endif + return 0; +} + +SEC("syscall") +int try_realloc(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM) + __u64 __arena *p; + + realloc_ptr = bpf_arena_alloc_pages(&arena, ptr, 1, NUMA_NO_NODE, 0); + if (realloc_ptr) { + ptr = realloc_ptr; + p = (__u64 __arena *)realloc_ptr; + *p = new_marker; + current_marker = new_marker; + } +#endif + return 0; +} + +SEC("tp_btf/sys_enter") +int BPF_PROG(deferred_free_prog, struct pt_regs *regs, long id) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM) + if (!deferred_free || bpf_get_current_pid_tgid() != trigger_pid_tgid || + id != trigger_syscall) + return 0; + + deferred_free = 0; + /* The worker can run on another CPU before the kfunc returns. */ + worker_armed = 1; + bpf_arena_free_pages(&arena, ptr, 1); +#endif + return 0; +} + +SEC("fentry/arena_free_worker") +int BPF_PROG(trace_free_worker, struct work_struct *work) +{ + if (worker_armed && !target_tid) + target_tid = (__u32)bpf_get_current_pid_tgid(); + return 0; +} + +SEC("fexit/arena_free_worker") +int BPF_PROG(trace_free_worker_ret, struct work_struct *work) +{ + if ((__u32)bpf_get_current_pid_tgid() == target_tid) + worker_exited = 1; + return 0; +} + +/* + * Pause a free while it zaps user mappings: after it cleared the kernel PTEs + * and flushed the TLB, but before it releases the range. zap_vma_range() is + * generic MM code, so it is an fentry target even where + * flush_tlb_kernel_range() is inline (arm64). + */ +SEC("?fentry/zap_vma_range") +int BPF_PROG(trace_zap, struct vm_area_struct *vma, unsigned long address, + unsigned long size) +{ + if ((__u32)bpf_get_current_pid_tgid() != target_tid) + return 0; + zap_entered = 1; + if (pause_on_zap) + wait_for_release(); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/async_stack_depth.c b/tools/testing/selftests/bpf/progs/async_stack_depth.c index 36734683acbd..9cd874a90b39 100644 --- a/tools/testing/selftests/bpf/progs/async_stack_depth.c +++ b/tools/testing/selftests/bpf/progs/async_stack_depth.c @@ -29,7 +29,49 @@ static int bad_timer_cb(void *map, int *key, struct bpf_timer *timer) return buf[255] + timer_cb(NULL, NULL, NULL); } +/* + * The same shapes scaled to the 2 KiB budget of JITs with large stacks. The + * compiler caps a single function at 512 bytes, so the depth comes from a + * chain of 480-byte frames. + */ +__attribute__((noinline)) +static int timer_cb_large_0(void *map, int *key, struct bpf_timer *timer) +{ + volatile char buf[480] = {}; + return buf[69]; +} + +__attribute__((noinline)) +static int timer_cb_large_1(void *map, int *key, struct bpf_timer *timer) +{ + volatile char buf[480] = {}; + return buf[69] + timer_cb_large_0(map, key, timer); +} + +__attribute__((noinline)) +static int timer_cb_large_2(void *map, int *key, struct bpf_timer *timer) +{ + volatile char buf[480] = {}; + return buf[69] + timer_cb_large_1(map, key, timer); +} + +__attribute__((noinline)) +static int timer_cb_large_3(void *map, int *key, struct bpf_timer *timer) +{ + volatile char buf[480] = {}; + return buf[69] + timer_cb_large_2(map, key, timer); +} + +/* 5 * 480 = 2400 bytes on its own */ +__attribute__((noinline)) +static int bad_timer_cb_large(void *map, int *key, struct bpf_timer *timer) +{ + volatile char buf[480] = {}; + return buf[255] + timer_cb_large_3(map, key, timer); +} + SEC("tc") +__load_if_no_large_stack() __failure __msg("combined stack size of 2 calls is") int pseudo_call_check(struct __sk_buff *ctx) { @@ -44,7 +86,25 @@ int pseudo_call_check(struct __sk_buff *ctx) return bpf_timer_set_callback(&elem->timer, timer_cb) + buf[0]; } +/* main plus the four frames under timer_cb_large_3: 2400 bytes */ SEC("tc") +__load_if_large_stack() +__failure __msg("combined stack size of 5 calls is") +int pseudo_call_check_large(struct __sk_buff *ctx) +{ + struct hmap_elem *elem; + volatile char buf[480] = {}; + + elem = bpf_map_lookup_elem(&hmap, &(int){0}); + if (!elem) + return 0; + + timer_cb_large_3(NULL, NULL, NULL); + return bpf_timer_set_callback(&elem->timer, timer_cb_large_3) + buf[0]; +} + +SEC("tc") +__load_if_no_large_stack() __failure __msg("combined stack size of 2 calls is") int async_call_root_check(struct __sk_buff *ctx) { @@ -58,4 +118,19 @@ int async_call_root_check(struct __sk_buff *ctx) return bpf_timer_set_callback(&elem->timer, bad_timer_cb) + buf[0]; } +SEC("tc") +__load_if_large_stack() +__failure __msg("combined stack size of 5 calls is") +int async_call_root_check_large(struct __sk_buff *ctx) +{ + struct hmap_elem *elem; + volatile char buf[480] = {}; + + elem = bpf_map_lookup_elem(&hmap, &(int){0}); + if (!elem) + return 0; + + return bpf_timer_set_callback(&elem->timer, bad_timer_cb_large) + buf[0]; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_netlink.c b/tools/testing/selftests/bpf/progs/bpf_iter_netlink.c index 00b2ceae81fb..b9107d3ab603 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_netlink.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_netlink.c @@ -25,12 +25,12 @@ int dump_netlink(struct bpf_iter__netlink *ctx) return 0; if (ctx->meta->seq_num == 0) - BPF_SEQ_PRINTF(seq, "sk Eth Pid Groups " + BPF_SEQ_PRINTF(seq, "sk Eth Pid Groups " "Rmem Wmem Dump Locks Drops " "Inode\n"); s = &nlk->sk; - BPF_SEQ_PRINTF(seq, "%pK %-3d ", s, s->sk_protocol); + BPF_SEQ_PRINTF(seq, "0 %-3d ", s->sk_protocol); if (!nlk->groups) { group = 0; diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c b/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c index b1e509b231cd..d149cf0d07be 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c @@ -131,8 +131,7 @@ static int dump_tcp_sock(struct seq_file *seq, struct tcp_sock *tp, icsk->icsk_probes_out, sock_i_ino(sp), sp->sk_refcnt.refs.counter); - BPF_SEQ_PRINTF(seq, "%pK %lu %lu %u %u %d\n", - tp, + BPF_SEQ_PRINTF(seq, "0 %lu %lu %u %u %d\n", jiffies_to_clock_t(icsk->icsk_rto), jiffies_to_clock_t(icsk->icsk_ack.ato), (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(icsk), @@ -161,10 +160,10 @@ static int dump_tw_sock(struct seq_file *seq, struct tcp_timewait_sock *ttw, BPF_SEQ_PRINTF(seq, "%4d: %08X:%04X %08X:%04X ", seq_num, src, srcp, dest, destp); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0\n", tw->tw_substate, 0, 0, 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0, - tw->tw_refcnt.refs.counter, tw); + tw->tw_refcnt.refs.counter); return 0; } @@ -185,9 +184,9 @@ static int dump_req_sock(struct seq_file *seq, struct tcp_request_sock *treq, seq_num, irsk->ir_loc_addr, irsk->ir_num, irsk->ir_rmt_addr, bpf_ntohs(irsk->ir_rmt_port)); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0\n", TCP_SYN_RECV, 0, 0, 1, jiffies_to_clock_t(ttd), - req->num_timeout, uid, 0, 0, 0, req); + req->num_timeout, uid, 0, 0, 0); return 0; } diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c b/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c index dbc7166aee91..cd65fa81f121 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c @@ -135,8 +135,7 @@ static int dump_tcp6_sock(struct seq_file *seq, struct tcp6_sock *tp, icsk->icsk_probes_out, sock_i_ino(sp), sp->sk_refcnt.refs.counter); - BPF_SEQ_PRINTF(seq, "%pK %lu %lu %u %u %d\n", - tp, + BPF_SEQ_PRINTF(seq, "0 %lu %lu %u %u %d\n", jiffies_to_clock_t(icsk->icsk_rto), jiffies_to_clock_t(icsk->icsk_ack.ato), (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(icsk), @@ -170,10 +169,10 @@ static int dump_tw_sock(struct seq_file *seq, struct tcp_timewait_sock *ttw, dest->s6_addr32[0], dest->s6_addr32[1], dest->s6_addr32[2], dest->s6_addr32[3], destp); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0\n", tw->tw_substate, 0, 0, 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0, - tw->tw_refcnt.refs.counter, tw); + tw->tw_refcnt.refs.counter); return 0; } @@ -201,9 +200,9 @@ static int dump_req_sock(struct seq_file *seq, struct tcp_request_sock *treq, dest->s6_addr32[0], dest->s6_addr32[1], dest->s6_addr32[2], dest->s6_addr32[3], bpf_ntohs(irsk->ir_rmt_port)); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0\n", TCP_SYN_RECV, 0, 0, 1, jiffies_to_clock_t(ttd), - req->num_timeout, uid, 0, 0, 0, req); + req->num_timeout, uid, 0, 0, 0); return 0; } diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c b/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c index 23b2aa2604de..d4079a619da6 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c @@ -57,13 +57,13 @@ int dump_udp4(struct bpf_iter__udp *ctx) BPF_SEQ_PRINTF(seq, "%5d: %08X:%04X %08X:%04X ", ctx->bucket, src, srcp, dest, destp); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d 0 %u\n", inet->sk.sk_state, inet->sk.sk_wmem_alloc.refs.counter - 1, rqueue, 0, 0L, 0, ctx->uid, 0, sock_i_ino(&inet->sk), - inet->sk.sk_refcnt.refs.counter, udp_sk, + inet->sk.sk_refcnt.refs.counter, udp_sk->drop_counters.drops0.counter + udp_sk->drop_counters.drops1.counter); diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c b/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c index c48b05aa2a4b..dadc5f73ff09 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c @@ -65,13 +65,13 @@ int dump_udp6(struct bpf_iter__udp *ctx) dest->s6_addr32[0], dest->s6_addr32[1], dest->s6_addr32[2], dest->s6_addr32[3], destp); - BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u\n", + BPF_SEQ_PRINTF(seq, "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d 0 %u\n", inet->sk.sk_state, inet->sk.sk_wmem_alloc.refs.counter - 1, rqueue, 0, 0L, 0, ctx->uid, 0, sock_i_ino(&inet->sk), - inet->sk.sk_refcnt.refs.counter, udp_sk, + inet->sk.sk_refcnt.refs.counter, udp_sk->drop_counters.drops0.counter + udp_sk->drop_counters.drops1.counter); return 0; diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_unix.c b/tools/testing/selftests/bpf/progs/bpf_iter_unix.c index a2652c8c3616..3061b148f338 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_unix.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_unix.c @@ -42,10 +42,9 @@ int dump_unix(struct bpf_iter__unix *ctx) seq = ctx->meta->seq; seq_num = ctx->meta->seq_num; if (seq_num == 0) - BPF_SEQ_PRINTF(seq, "Num RefCount Protocol Flags Type St Inode Path\n"); + BPF_SEQ_PRINTF(seq, "Num RefCount Protocol Flags Type St Inode Path\n"); - BPF_SEQ_PRINTF(seq, "%pK: %08X %08X %08X %04X %02X %8lu", - unix_sk, + BPF_SEQ_PRINTF(seq, "%08u: %08X %08X %08X %04X %02X %8lu", 0, sk->sk_refcnt.refs.counter, 0, sk->sk_state == TCP_LISTEN ? __SO_ACCEPTCON : 0, @@ -71,7 +70,7 @@ int dump_unix(struct bpf_iter__unix *ctx) for (i = 1; i < len; i++) { /* unix_validate_addr() tests this upper bound. */ - if (i >= sizeof(struct sockaddr_un)) + if (i >= sizeof(unix_sk->addr->name->sun_path)) break; BPF_SEQ_PRINTF(seq, "%c", diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index 5eacf1b43252..d7f706c3bc4a 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -9,7 +9,8 @@ #define QUOTE(str) #str #define EXPAND_QUOTE(str) QUOTE(str) -/* This set of attributes controls behavior of the +/* + * This set of attributes controls behavior of the * test_loader.c:test_loader__run_subtests(). * * The test_loader sequentially loads each program in a skeleton. @@ -42,6 +43,10 @@ * __msg_unpriv Same as __msg but for unprivileged mode. * __not_msg_unpriv Same as __not_msg but for unprivileged mode. * + * __msg_next Same as __msg but expects the match to be on the next + * __msg_next_unpriv log line compared to the last match, not just some line + * after the last match. + * * __stderr Message expected to be found in bpf stderr stream. The * same regex rules apply like __msg. * __stderr_unpriv Same as __stderr but for unpriveleged mode. @@ -106,6 +111,11 @@ * __description Text to be used for display and as an additional filter * alias, while the original program name stays matchable. * + * __skip Report the test as SKIP with the given reason instead of + * running it. For placeholder programs that stand in for a + * feature the toolchain or JIT cannot provide, so a run does + * not report OK for coverage it never executed. + * * __log_level Log level to use for the program, numeric value expected. * * __flag Adds one flag use for the program, the following values are valid: @@ -126,6 +136,11 @@ * Several __arch_* annotations could be specified at once. * When test case is not run on current arch it is marked as skipped. * __caps_unpriv Specify the capabilities that should be set when running the test. + * __prepare_priv In unprivileged mode, prepare the object with the fixture's + * initial capabilities before dropping them for program loading. + * Preparation includes map creation and BTF/kfunc resolution; + * these operations are not tested at the reduced capabilities. + * Program loading uses the normal __caps_unpriv selection. * * __linear_size Specify the size of the linear area of non-linear skbs, or * 0 for linear skbs. @@ -134,13 +149,16 @@ #define __msg(msg) __test_tag("test_expect_msg=" msg) #define __not_msg(msg) __test_tag("test_expect_not_msg=" msg) +#define __msg_next(msg) __test_tag("test_expect_msg_next=" msg) #define __xlated(msg) __test_tag("test_expect_xlated=" msg) #define __jited(msg) __test_tag("test_jited=" msg) #define __failure __test_tag("test_expect_failure") #define __success __test_tag("test_expect_success") #define __description(desc) __test_tag("test_description=" desc) +#define __skip(reason) __test_tag("test_skip=" reason) #define __msg_unpriv(msg) __test_tag("test_expect_msg_unpriv=" msg) #define __not_msg_unpriv(msg) __test_tag("test_expect_not_msg_unpriv=" msg) +#define __msg_next_unpriv(msg) __test_tag("test_expect_msg_next_unpriv=" msg) #define __xlated_unpriv(msg) __test_tag("test_expect_xlated_unpriv=" msg) #define __jited_unpriv(msg) __test_tag("test_jited_unpriv=" msg) #define __failure_unpriv __test_tag("test_expect_failure_unpriv") @@ -160,8 +178,12 @@ #define __arch_s390x __arch("s390x") #define __arch_loongarch __arch("LOONGARCH") #define __caps_unpriv(caps) __test_tag("test_caps_unpriv=" EXPAND_QUOTE(caps)) +#define __prepare_priv __test_tag("test_prepare_priv") #define __load_if_JITed() __test_tag("load_mode=jited") #define __load_if_no_JITed() __test_tag("load_mode=no_jited") +/* Whether programs may use more than 512 bytes of stack on this kernel and JIT */ +#define __load_if_large_stack() __test_tag("stack_mode=large") +#define __load_if_no_large_stack() __test_tag("stack_mode=small") #define __stderr(msg) __test_tag("test_expect_stderr=" msg) #define __stderr_unpriv(msg) __test_tag("test_expect_stderr_unpriv=" msg) #define __stdout(msg) __test_tag("test_expect_stdout=" msg) @@ -255,11 +277,11 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 + (__clang_major__ >= 18 || defined(__BPF_FEATURE_GOTOL)) #define CAN_USE_GOTOL #endif -#if __clang_major__ >= 18 +#if __clang_major__ >= 18 || defined(__BPF_FEATURE_ST) #define CAN_USE_BPF_ST #endif diff --git a/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_returned_slice.c b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_returned_slice.c new file mode 100644 index 000000000000..8217f4c4c00c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_returned_slice.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include "bpf_experimental.h" +#include "bpf_qdisc_common.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +int proto; + +static __noinline struct ethhdr *slice_in_subprog(struct sk_buff *skb) +{ + struct bpf_dynptr ptr; + + bpf_dynptr_from_skb((struct __sk_buff *)skb, 0, &ptr); + return bpf_dynptr_slice(&ptr, 0, NULL, sizeof(struct ethhdr)); +} + +SEC("struct_ops") +__failure __msg("invalid mem access 'scalar'") +int BPF_PROG(invalid_dynptr_returned_slice, struct sk_buff *skb, + struct Qdisc *sch, struct bpf_sk_buff_ptr *to_free) +{ + struct ethhdr *hdr; + + hdr = slice_in_subprog(skb); + if (!hdr) { + bpf_qdisc_skb_drop(skb, to_free); + return NET_XMIT_DROP; + } + + /* this should fail */ + proto = hdr->h_proto; + + bpf_qdisc_skb_drop(skb, to_free); + + return NET_XMIT_DROP; +} + +SEC("struct_ops") +__auxiliary +struct sk_buff *BPF_PROG(bpf_qdisc_test_dequeue, struct Qdisc *sch) +{ + return NULL; +} + +SEC("struct_ops") +__auxiliary +int BPF_PROG(bpf_qdisc_test_init, struct Qdisc *sch, struct nlattr *opt, + struct netlink_ext_ack *extack) +{ + return 0; +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_reset, struct Qdisc *sch) +{ +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_destroy, struct Qdisc *sch) +{ +} + +SEC(".struct_ops") +struct Qdisc_ops test = { + .enqueue = (void *)invalid_dynptr_returned_slice, + .dequeue = (void *)bpf_qdisc_test_dequeue, + .init = (void *)bpf_qdisc_test_init, + .reset = (void *)bpf_qdisc_test_reset, + .destroy = (void *)bpf_qdisc_test_destroy, + .id = "bpf_qdisc_test", +}; diff --git a/tools/testing/selftests/bpf/progs/bpf_tcp_ops.c b/tools/testing/selftests/bpf/progs/bpf_tcp_ops.c new file mode 100644 index 000000000000..94a7f52573d5 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_tcp_ops.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +#define MAX_CGROUP_OPS 8 + +/* Call order for listen and connect, indexed by call sequence */ +u32 listen_order[MAX_CGROUP_OPS]; +u32 listen_cnt; + +u32 connect_order[MAX_CGROUP_OPS]; +u32 connect_cnt; + +static void record_listen(int id) +{ + u32 idx = listen_cnt; + + if (idx < MAX_CGROUP_OPS) { + listen_order[idx] = id; + listen_cnt = idx + 1; + } +} + +static void record_connect(int id) +{ + u32 idx = connect_cnt; + + if (idx < MAX_CGROUP_OPS) { + connect_order[idx] = id; + connect_cnt = idx + 1; + } +} + +/* struct_ops instance 1 */ + +SEC("struct_ops") +void BPF_PROG(tcp_ops1_listen, struct sock *sk) +{ + record_listen(1); +} + +SEC("struct_ops") +void BPF_PROG(tcp_ops1_connect, struct sock *sk) +{ + record_connect(1); +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops1 = { + .listen = (void *)tcp_ops1_listen, + .connect = (void *)tcp_ops1_connect, +}; + +/* struct_ops instance 2 */ + +SEC("struct_ops") +void BPF_PROG(tcp_ops2_listen, struct sock *sk) +{ + record_listen(2); +} + +SEC("struct_ops") +void BPF_PROG(tcp_ops2_connect, struct sock *sk) +{ + record_connect(2); +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops2 = { + .listen = (void *)tcp_ops2_listen, + .connect = (void *)tcp_ops2_connect, +}; + +/* struct_ops instance 3 */ + +SEC("struct_ops") +void BPF_PROG(tcp_ops3_listen, struct sock *sk) +{ + record_listen(3); +} + +SEC("struct_ops") +void BPF_PROG(tcp_ops3_connect, struct sock *sk) +{ + record_connect(3); +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops3 = { + .listen = (void *)tcp_ops3_listen, + .connect = (void *)tcp_ops3_connect, +}; + +#define OPS_RETVAL1 11 +#define OPS_RETVAL2 22 +#define OPS_RETVAL3 33 + +int retval_saw1; +int retval_saw2; +int retval_saw3; + +SEC("struct_ops") +int BPF_PROG(tcp_ops_retval1_timeout_init, struct sock *sk, struct request_sock *req) +{ + retval_saw1 = bpf_get_retval(); + return OPS_RETVAL1; +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops_retval1 = { + .timeout_init = (void *)tcp_ops_retval1_timeout_init, +}; + +SEC("struct_ops") +int BPF_PROG(tcp_ops_retval2_timeout_init, struct sock *sk, struct request_sock *req) +{ + retval_saw2 = bpf_get_retval(); + return OPS_RETVAL2; +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops_retval2 = { + .timeout_init = (void *)tcp_ops_retval2_timeout_init, +}; + +SEC("struct_ops") +int BPF_PROG(tcp_ops_retval3_timeout_init, struct sock *sk, struct request_sock *req) +{ + retval_saw3 = bpf_get_retval(); + return OPS_RETVAL3; +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_ops_retval3 = { + .timeout_init = (void *)tcp_ops_retval3_timeout_init, +}; + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/bpf_tcp_ops_hdr.c b/tools/testing/selftests/bpf/progs/bpf_tcp_ops_hdr.c new file mode 100644 index 000000000000..706e94d794e4 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_tcp_ops_hdr.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +/* Experimental option kind and payload written/parsed by this test. */ +#define TEST_OPT_KIND 0xFD +#define TEST_OPT_LEN 4 +#define TEST_OPT_D0 0xAB +#define TEST_OPT_D1 0xCD + +int hdr_opt_len_cnt; +int write_cnt; +int parse_cnt; +int found_cnt; +__u8 found_d0; +__u8 found_d1; + +SEC("struct_ops") +void BPF_PROG(test_listen, struct sock *sk) +{ + bpf_tcp_ops_set_flags((struct tcp_sock *)sk, + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_ALL | + BPF_TCP_OPS_FLAG_WRITE_HDR_OPT, + 0); +} + +SEC("struct_ops") +void BPF_PROG(test_connect, struct sock *sk) +{ + bpf_tcp_ops_set_flags((struct tcp_sock *)sk, + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN | + BPF_TCP_OPS_FLAG_WRITE_HDR_OPT, + 0); +} + +SEC("struct_ops") +void BPF_PROG(test_hdr_opt_len, struct sock *sk, struct sk_buff *skb, + struct request_sock *req, struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, unsigned int *remaining) +{ + hdr_opt_len_cnt++; + + /* + * Reserve TEST_OPT_LEN bytes; the helper decrements *remaining. Stacks + * with other progs in the cgroup hierarchy. + */ + bpf_reserve_hdr_opt(ctx, TEST_OPT_LEN, 0); +} + +SEC("struct_ops") +void BPF_PROG(test_write_hdr_opt, struct sock *sk, struct sk_buff *skb, + struct request_sock *req, struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, __u32 opt_off) +{ + __u8 opt[TEST_OPT_LEN] = { + TEST_OPT_KIND, TEST_OPT_LEN, TEST_OPT_D0, TEST_OPT_D1, + }; + + /* + * bpf_store_hdr_opt() takes the program ctx (the kernel reads the + * outgoing skb from it); it appends after any options already written + * in the reserved window, rejects duplicates, and confines the write to + * the header option scratch. Stacks across progs in the cgroup hierarchy. + */ + if (bpf_store_hdr_opt(ctx, opt, sizeof(opt), 0)) + return; + + write_cnt++; +} + +SEC("struct_ops") +void BPF_PROG(test_parse_hdr, struct sock *sk, struct sk_buff *skb) +{ + __u8 opt[TEST_OPT_LEN] = { + TEST_OPT_KIND, TEST_OPT_LEN, TEST_OPT_D0, TEST_OPT_D1, + }; + + parse_cnt++; + + /* + * Look up the experimental option written by test_write_hdr_opt() in + * the incoming skb. For an experimental kind the search matches on the + * 2-byte magic in opt[2..3]; on a match the found option is copied back + * into opt[]. + */ + if (bpf_load_hdr_opt(ctx, opt, sizeof(opt), 0) < 0) + return; + + found_d0 = opt[2]; + found_d1 = opt[3]; + found_cnt++; +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops test_hdr_ops = { + .listen = (void *)test_listen, + .connect = (void *)test_connect, + .hdr_opt_len = (void *)test_hdr_opt_len, + .write_hdr_opt = (void *)test_write_hdr_opt, + .parse_hdr = (void *)test_parse_hdr, +}; + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h index 593b38f90417..4c999d59cbbc 100644 --- a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h +++ b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h @@ -79,6 +79,8 @@ #define NEXTHDR_TCP 6 +#define TCPHDR_FIN 0x01 + #define TCPOPT_NOP 1 #define TCPOPT_EOL 0 #define TCPOPT_MSS 2 diff --git a/tools/testing/selftests/bpf/progs/bpftool_ringbuf.c b/tools/testing/selftests/bpf/progs/bpftool_ringbuf.c new file mode 100644 index 000000000000..bbbc93c30a83 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpftool_ringbuf.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); +} ringbuf SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); + __uint(max_entries, 1); +} perfbuf SEC(".maps"); + +/* + * Include the u32 raw size in perf's 8-byte alignment to avoid padding. + * Keep plain output large enough to flush stdio before the consumer exits. + */ +const unsigned char perf_payload[4092] = { 0x00, 0xff }; + +int record; +int output_err; + +SEC("socket") +int produce(struct __sk_buff *skb) +{ + unsigned char first[] = { 0x00, 0xff }; + unsigned char second[] = { 1, 2, 3, 4, 5 }; + unsigned char third[] = { + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, + }; + + if (record == 0) + output_err = bpf_ringbuf_output(&ringbuf, first, sizeof(first), 0); + else if (record == 1) + output_err = bpf_ringbuf_output(&ringbuf, second, sizeof(second), 0); + else if (record == 2) + output_err = bpf_ringbuf_output(&ringbuf, third, sizeof(third), 0); + else + output_err = bpf_perf_event_output(skb, &perfbuf, BPF_F_CURRENT_CPU, + (void *)perf_payload, sizeof(perf_payload)); + return 0; +} + +char LICENSE[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c b/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c index 8d38aafe66a2..ab12397a9b52 100644 --- a/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c +++ b/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c @@ -4,7 +4,8 @@ #include <bpf/bpf_helpers.h> #include "../test_kmods/bpf_testmod_kfunc.h" -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) long subprog_call_mem_kfunc(long a, long b, long c, long d, long e, long size) diff --git a/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c index 99bc115f8380..a980a3a399d3 100644 --- a/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c @@ -3,7 +3,8 @@ #include <vmlinux.h> #include <bpf/bpf_helpers.h> -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) int subprog_bad_order_6args(int a, int b, int c, int d, int e, int f) diff --git a/tools/testing/selftests/bpf/progs/btf_module_allowlist.c b/tools/testing/selftests/bpf/progs/btf_module_allowlist.c new file mode 100644 index 000000000000..bb3c38bee77d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/btf_module_allowlist.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +SEC("fentry/bpf_testmod_loop_test") +int BPF_PROG(test_btf_module_allowlist) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/call_rcu.c b/tools/testing/selftests/bpf/progs/call_rcu.c new file mode 100644 index 000000000000..505008647fd6 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/call_rcu.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +char _license[] SEC("license") = "GPL"; + +/* rh is not at offset 0, so the callback's value recovery is exercised. */ +struct elem { + __u64 pad; + struct bpf_rcu_head rh; + __u64 val; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 2); + __type(key, __u32); + __type(value, struct elem); +} arr SEC(".maps"); + +__u32 cb_key; +__u64 cb_keys; +__u64 cb_val; +__u32 cb_max_entries; +int callbacks; +int arm_err; +int busy_err; +int chain; /* set by userspace: number of times to re-arm from the callback */ +int chain_err; + +static int reclaim(struct bpf_map *map, void *key, void *value) +{ + struct elem *e = value; + + cb_key = *(__u32 *)key; + __sync_fetch_and_or(&cb_keys, 1ULL << cb_key); + cb_val = e->val; + cb_max_entries = map->max_entries; + e->val = 0; + + if (chain > 0) { + chain--; + chain_err = bpf_call_rcu(&e->rh, &arr, reclaim); + } + + __sync_fetch_and_add(&callbacks, 1); + return 0; +} + +SEC("syscall") +int arm(void *ctx) +{ + __u32 key = 1; + struct elem *e; + + e = bpf_map_lookup_elem(&arr, &key); + if (!e) + return 1; + + e->val = 0xdeadbeef; + /* Keep a grace period from elapsing between the two arms. */ + bpf_rcu_read_lock(); + arm_err = bpf_call_rcu(&e->rh, &arr, reclaim); + busy_err = bpf_call_rcu(&e->rh, &arr, reclaim); + bpf_rcu_read_unlock(); + return 0; +} + +SEC("syscall") +int arm_both(void *ctx) +{ + __u32 key0 = 0, key1 = 1; + struct elem *e0, *e1; + + e0 = bpf_map_lookup_elem(&arr, &key0); + e1 = bpf_map_lookup_elem(&arr, &key1); + if (!e0 || !e1) + return 1; + + e0->val = 0xdeadbeef; + e1->val = 0xdeadbeef; + arm_err = bpf_call_rcu(&e0->rh, &arr, reclaim); + arm_err |= bpf_call_rcu(&e1->rh, &arr, reclaim); + return 0; +} + +SEC("syscall") +int arm_trace(void *ctx) +{ + __u32 key = 1; + struct elem *e; + + e = bpf_map_lookup_elem(&arr, &key); + if (!e) + return 1; + + e->val = 0xdeadbeef; + /* No lock needed: the enclosing rcu_read_lock_trace() already blocks the grace period. */ + arm_err = bpf_call_rcu_tasks_trace(&e->rh, &arr, reclaim); + busy_err = bpf_call_rcu_tasks_trace(&e->rh, &arr, reclaim); + return 0; +} + +SEC("iter/bpf_map_elem") +int dump(struct bpf_iter__bpf_map_elem *ctx) +{ + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/call_rcu_fail.c b/tools/testing/selftests/bpf/progs/call_rcu_fail.c new file mode 100644 index 000000000000..bf60a27731fb --- /dev/null +++ b/tools/testing/selftests/bpf/progs/call_rcu_fail.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +const void *user_ptr = NULL; + +struct elem { + __u64 pad; + struct bpf_rcu_head rh; + __u64 val; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct elem); +} arr SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct elem); +} arr2 SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u64); +} plain SEC(".maps"); + +__u32 key = 0; + +static int reclaim(struct bpf_map *map, void *key, void *value) +{ + return 0; +} + +static int sleepable_reclaim(struct bpf_map *map, void *key, void *value) +{ + struct elem *e = value; + + bpf_copy_from_user(&e->val, sizeof(e->val), user_ptr); + return 0; +} + +SEC("syscall") +__failure __msg("bpf_rcu_head pointer in R1") __msg("doesn't match map pointer in R2") +int mismatch_map(void *ctx) +{ + struct elem *e; + + e = bpf_map_lookup_elem(&arr, &key); + if (!e) + return 0; + bpf_call_rcu(&e->rh, &arr2, reclaim); + return 0; +} + +SEC("syscall") +__failure __msg("map 'plain' has no valid bpf_rcu_head") +int no_rcu_head(void *ctx) +{ + __u64 *val; + + val = bpf_map_lookup_elem(&plain, &key); + if (!val) + return 0; + bpf_call_rcu((struct bpf_rcu_head *)val, &plain, reclaim); + return 0; +} + +SEC("syscall") +__failure __msg("doesn't point to 'struct bpf_rcu_head' that is at 8") +int wrong_offset(void *ctx) +{ + struct elem *e; + + e = bpf_map_lookup_elem(&arr, &key); + if (!e) + return 0; + bpf_call_rcu((struct bpf_rcu_head *)&e->pad, &arr, reclaim); + return 0; +} + +SEC("syscall") +__failure __msg("R1 type=fp expected=map_value") +int rcu_head_on_stack(void *ctx) +{ + struct bpf_rcu_head rh; + + bpf_call_rcu(&rh, &arr, reclaim); + return 0; +} + +SEC("syscall") +__failure __msg("sleepable helper bpf_copy_from_user") __msg("in non-sleepable prog") +int sleepable_callback(void *ctx) +{ + struct elem *e; + + e = bpf_map_lookup_elem(&arr, &key); + if (!e) + return 0; + bpf_call_rcu(&e->rh, &arr, sleepable_reclaim); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/callx_rodata.c b/tools/testing/selftests/bpf/progs/callx_rodata.c new file mode 100644 index 000000000000..e3f8335f80f8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/callx_rodata.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0 +/* callx through pointers to functions in .rodata, loaded by light skeleton */ + +#include <stdbool.h> +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +/* set by user space before the programs are loaded, it's in .rodata too */ +const volatile int bias = 1; + +int op_idx; + +#ifdef __clang__ + +const volatile bool skip = false; + +typedef int (*op_fn)(int); + +static __noinline int add_bias(int x) +{ + return x + bias; +} + +static __noinline int mul3(int x) +{ + return x * 3; +} + +/* volatile, so that the compiler doesn't replace the table with direct calls */ +static op_fn const volatile ops[] = { add_bias, mul3 }; + +SEC("socket") +int select_op(void *ctx) +{ + unsigned int i = op_idx; + + if (i >= sizeof(ops) / sizeof(ops[0])) + return -1; + return ops[i](10); +} + +/* functions have other offsets in this program */ +SEC("socket") +int both_ops(void *ctx) +{ + return ops[1](ops[0](4)); +} + +#else + +/* gcc doesn't support indirect calls */ +const volatile bool skip = true; + +SEC("socket") +int select_op(void *ctx) +{ + return 0; +} + +SEC("socket") +int both_ops(void *ctx) +{ + return 0; +} + +#endif + +/* no callx here */ +SEC("socket") +int read_bias(void *ctx) +{ + return bias; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/cb_refs.c b/tools/testing/selftests/bpf/progs/cb_refs.c index 5d6fc7f01ebb..d4a691db8a15 100644 --- a/tools/testing/selftests/bpf/progs/cb_refs.c +++ b/tools/testing/selftests/bpf/progs/cb_refs.c @@ -2,6 +2,7 @@ #include <vmlinux.h> #include <bpf/bpf_tracing.h> #include <bpf/bpf_helpers.h> +#include "bpf_misc.h" #include "../test_kmods/bpf_testmod_kfunc.h" struct map_value { @@ -24,6 +25,7 @@ static __noinline int cb1(void *map, void *key, void *value, void *ctx) } SEC("?tc") +__failure __msg("R1 type=scalar expected=ptr_, trusted_ptr_, rcu_ptr_") int underflow_prog(void *ctx) { struct prog_test_ref_kfunc *p; @@ -47,6 +49,7 @@ static __always_inline int cb2(void *map, void *key, void *value, void *ctx) } SEC("?tc") +__failure __msg("Unreleased reference id={{[0-9]+}} alloc_insn=3") /* alloc_insn=3{0,2,3} */ int leak_prog(void *ctx) { struct prog_test_ref_kfunc *p; @@ -83,6 +86,7 @@ static __always_inline int cb3(void *map, void *key, void *value, void *ctx) } SEC("?tc") +__failure __msg("Unreleased reference id={{[0-9]+}} alloc_insn=2") /* alloc_insn=2{2,4,5} */ int nested_cb(void *ctx) { struct prog_test_ref_kfunc *p; @@ -98,6 +102,7 @@ int nested_cb(void *ctx) } SEC("?tc") +__failure __msg("Unreleased reference id={{[0-9]+}} alloc_insn=1") /* alloc_insn=1{1,2} */ int non_cb_transfer_ref(void *ctx) { struct prog_test_ref_kfunc *p; diff --git a/tools/testing/selftests/bpf/cgroup_getset_retval_hooks.h b/tools/testing/selftests/bpf/progs/cgroup_getset_retval_hooks.h index a525d3544fd7..a525d3544fd7 100644 --- a/tools/testing/selftests/bpf/cgroup_getset_retval_hooks.h +++ b/tools/testing/selftests/bpf/progs/cgroup_getset_retval_hooks.h diff --git a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c index efe7bcae70f8..ede6a17d7da3 100644 --- a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c @@ -64,7 +64,7 @@ int BPF_PROG(cgrp_kfunc_acquire_no_null_check, struct cgroup *cgrp, const char * } SEC("tp_btf/cgroup_mkdir") -__failure __msg("R1 is fp expected STRUCT cgroup") +__failure __msg("R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_") int BPF_PROG(cgrp_kfunc_acquire_fp, struct cgroup *cgrp, const char *path) { struct cgroup *acquired, *stack_cgrp = (struct cgroup *)&path; @@ -154,7 +154,7 @@ int BPF_PROG(cgrp_kfunc_xchg_unreleased, struct cgroup *cgrp, const char *path) } SEC("tp_btf/cgroup_mkdir") -__failure __msg("release kfunc bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(cgrp_kfunc_rcu_get_release, struct cgroup *cgrp, const char *path) { struct cgroup *kptr; @@ -191,7 +191,7 @@ int BPF_PROG(cgrp_kfunc_release_untrusted, struct cgroup *cgrp, const char *path } SEC("tp_btf/cgroup_mkdir") -__failure __msg("release kfunc bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_") int BPF_PROG(cgrp_kfunc_release_fp, struct cgroup *cgrp, const char *path) { struct cgroup *acquired = (struct cgroup *)&path; @@ -237,7 +237,7 @@ int BPF_PROG(cgrp_kfunc_release_null, struct cgroup *cgrp, const char *path) } SEC("tp_btf/cgroup_mkdir") -__failure __msg("release kfunc bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(cgrp_kfunc_release_unacquired, struct cgroup *cgrp, const char *path) { /* Cannot release trusted cgroup pointer which was not acquired. */ diff --git a/tools/testing/selftests/bpf/progs/compute_live_registers.c b/tools/testing/selftests/bpf/progs/compute_live_registers.c index d055fc7b3b95..1ff24bbc7beb 100644 --- a/tools/testing/selftests/bpf/progs/compute_live_registers.c +++ b/tools/testing/selftests/bpf/progs/compute_live_registers.c @@ -299,7 +299,15 @@ __naked void gotol(void) "r3 = 24;" "if r1 > 0x7 goto +2;" "r0 = r2;" +#ifdef __clang__ "gotol +1;" +#else + /* + * gas mis-parses 'gotol +1' as 'goto l+1', same encoding + * without the sign (binutils PR gas/34558). + */ + "gotol 1;" +#endif "r0 = r3;" "exit;" : @@ -406,13 +414,42 @@ static __used __naked int aux1(void) ::: __clobber_all); } +static __noinline __used int static_two(int a, int b) +{ + return a; +} + +__noinline __used int global_two(int a, int b) +{ + return a; +} + +SEC("socket") +__log_level(2) +/* a static callee reads r1 only, a global callee its declared arguments */ +__msg("2: .1........ (85) call pc+") +__msg("5: .12....... (85) call pc+") +__naked void subprog_declared_args(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "call static_two;" + "r1 = 1;" + "r2 = 2;" + "call global_two;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + SEC("socket") __log_level(2) -__msg("0: ....45.... (b7) r1 = 1") -__msg("1: .1..45.... (b7) r2 = 2") -__msg("2: .12.45.... (b7) r3 = 3") -/* Conservative liveness for subprog parameters. */ -__msg("3: .12345.... (85) call pc+2") +__msg("0: .......... (b7) r1 = 1") +__msg("1: .1........ (b7) r2 = 2") +__msg("2: .12....... (b7) r3 = 3") +/* A static callee reads what its code reads: aux1 reads r1 and r2. */ +__msg("3: .12....... (85) call pc+2") __msg("4: .......... (b7) r0 = 0") __msg("5: 0......... (95) exit") __msg("6: .12....... (bf) r0 = r1") @@ -431,7 +468,36 @@ __naked void subprog1(void) ::: __clobber_all); } -#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) +static __used __naked unsigned __int128 aux2(void) +{ + asm volatile ( + "r0 = 1;" + "r2 = 2;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +/* A program observing the pair needs the JIT; see bpf_compute_subprog_ret_regs(). */ +__load_if_JITed() +__log_level(2) +__msg("0: .......... (85) call pc+2") +__msg("1: ..2....... (bf) r0 = r2") +__msg("2: 0......... (95) exit") +__msg("3: .......... (b7) r0 = 1") +__msg("4: 0......... (b7) r2 = 2") +__msg("5: 0.2....... (95) exit") +__naked void subprog_ret_reg_pair(void) +{ + asm volatile ( + "call aux2;" + "r0 = r2;" + "exit;" + ::: __clobber_all); +} + +#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) SEC("socket") __log_level(2) @@ -470,7 +536,7 @@ void gotox(void) : __clobber_all); } -#endif /* __TARGET_ARCH_x86 || __TARGET_ARCH_arm64 */ +#endif /* gotox: x86, arm64, riscv64 */ /* to retain debug info for BTF generation */ void kfunc_root(void) diff --git a/tools/testing/selftests/bpf/progs/copy_from_user_bprm.c b/tools/testing/selftests/bpf/progs/copy_from_user_bprm.c new file mode 100644 index 000000000000..00ecba3ea567 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/copy_from_user_bprm.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" + +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <errno.h> + +char _license[] SEC("license") = "GPL"; + +int monitored_pid; +int bprm_argc; +int bprm_envc; +int data_len; +int invalid_flags_ret; +int copy_ret; +int arg0_ret; +int arg1_ret; +int env0_ret; +int env1_ret; +char data[64] = {}; +char arg0[32] = {}; +char arg1[32] = {}; +char env0[32] = {}; +char env1[32] = {}; + +SEC("lsm.s/bprm_check_security") +int BPF_PROG(check_exec_args, struct linux_binprm *bprm) +{ + u32 pid = bpf_get_current_pid_tgid() >> 32; + struct mm_struct *mm; + u64 offset = 0; + + if (pid != monitored_pid) + return 0; + + mm = bprm->mm; + if (!mm) + return 0; + + bprm_argc = bprm->argc; + bprm_envc = bprm->envc; + + /* this is the total size of args and envs starting from bprm->p */ + data_len = bprm->exec - bprm->p; + + invalid_flags_ret = bpf_copy_from_user_mm(data, sizeof(data), (void *)bprm->p, mm, ~0ULL); + + copy_ret = bpf_copy_from_user_mm(data, sizeof(data), (void *)bprm->p, mm, 0); + if (copy_ret) + return 0; + + /* arg0 is at bprm->p */ + arg0_ret = bpf_copy_from_user_mm_str(arg0, sizeof(arg0), (void *)(bprm->p + offset), mm, BPF_F_PAD_ZEROS); + offset += arg0_ret; + + /* arg1 is at bprm->p + arg0_ret */ + arg1_ret = bpf_copy_from_user_mm_str(arg1, sizeof(arg1), (void *)(bprm->p + offset), mm, BPF_F_PAD_ZEROS); + offset += arg1_ret; + + /* env0 is at bprm->p + arg0_ret + arg1_ret */ + env0_ret = bpf_copy_from_user_mm_str(env0, sizeof(env0), (void *)(bprm->p + offset), mm, BPF_F_PAD_ZEROS); + offset += env0_ret; + + /* env1 is at bprm->p + arg0_ret + arg1_ret + env0_ret */ + env1_ret = bpf_copy_from_user_mm_str(env1, sizeof(env1), (void *)(bprm->p + offset), mm, BPF_F_PAD_ZEROS); + + return -EPERM; +} diff --git a/tools/testing/selftests/bpf/progs/cpumask_failure.c b/tools/testing/selftests/bpf/progs/cpumask_failure.c index 4628feb53d86..c89c88db39d1 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_failure.c +++ b/tools/testing/selftests/bpf/progs/cpumask_failure.c @@ -183,7 +183,7 @@ int BPF_PROG(test_global_mask_no_null_check, struct task_struct *task, u64 clone } SEC("tp_btf/task_newtask") -__failure __msg("Possibly NULL pointer passed to helper R2") +__failure __msg("release function bpf_kptr_xchg expects referenced PTR_TO_BTF_ID passed to R2") int BPF_PROG(test_global_mask_rcu_no_null_check, struct task_struct *task, u64 clone_flags) { struct bpf_cpumask *prev, *curr; @@ -243,7 +243,7 @@ int BPF_PROG(test_populate_invalid_destination, struct task_struct *task, u64 cl } SEC("tp_btf/task_newtask") -__failure __msg("leads to invalid memory access") +__failure __msg("R2 type=scalar expected=fp") int BPF_PROG(test_populate_invalid_source, struct task_struct *task, u64 clone_flags) { void *garbage = (void *)0x123456; diff --git a/tools/testing/selftests/bpf/progs/cpumask_success.c b/tools/testing/selftests/bpf/progs/cpumask_success.c index 36f77b9732d4..ddaba4318c57 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_success.c +++ b/tools/testing/selftests/bpf/progs/cpumask_success.c @@ -460,6 +460,9 @@ int BPF_PROG(test_insert_leave, struct task_struct *task, u64 clone_flags) { struct bpf_cpumask *cpumask; + if (!is_test_task()) + return 0; + cpumask = create_cpumask(); if (!cpumask) return 0; @@ -476,6 +479,9 @@ int BPF_PROG(test_insert_remove_release, struct task_struct *task, u64 clone_fla struct bpf_cpumask *cpumask; struct __cpumask_map_value *v; + if (!is_test_task()) + return 0; + cpumask = create_cpumask(); if (!cpumask) return 0; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena.c b/tools/testing/selftests/bpf/progs/data_in_arena.c new file mode 100644 index 000000000000..18f38103c0fc --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +int counter = 5; +long pair[2] = { 1, 2 }; +int x = 42; +long sum; +long table[4]; +struct { + long v[8]; +} aligned64 __attribute__((aligned(64))); +const volatile int ro = 7; +const volatile long ro_table[4] = { 10, 20, 30, 40 }; + +/* const strings stay in a map for helpers and kfuncs, a copy of them is in arena */ +const char hello[] SEC(".rodata.str.hello") = "hello"; + +/* pointers to data that are stored in data */ +int *px = &x; +const char *str = hello; +int *const volatile cpx SEC(".data.rel.ro") = &x; + +SEC("syscall") +int use_data(void *ctx) +{ + int i; + + for (i = 0; i < 4; i++) + table[i] = ro_table[i] + i; + sum = counter + ro + pair[0] + pair[1]; + counter++; + __sync_fetch_and_add(&pair[1], 3); + return sum; +} + +typedef int (*op_fn)(int); + +static __noinline int add1(int v) +{ + return v + 1; +} + +/* + * Pointers to functions and to data in read-only data of a program with callx. + * Volatile, so that the compiler doesn't replace the pointers with what + * they point to. + */ +static const volatile struct { + op_fn fn; + int *data; + const char *name; +} ops SEC(".data.rel.ro") = { add1, &x, hello }; + +SEC("syscall") +int use_ops(void *ctx) +{ + /* a program has the arena when its code refers to it */ + counter++; +#ifdef __clang__ + return ops.fn(*ops.data) + ops.name[1]; +#else + /* gcc doesn't support indirect calls */ + return add1(*ops.data) + ops.name[1]; +#endif +} + +SEC("syscall") +int use_ptrs(void *ctx) +{ + unsigned long addr = (unsigned long)&aligned64; + char local[4] = "abc"; + + /* hide the address from the compiler, it knows that '& 63' is 0 */ + asm volatile ("" : "+r"(addr)); + if (addr & 63) + return 1; + aligned64.v[7] = 7; + if (*px != 42) + return 2; + *px = 43; + if (x != 43 || *cpx != 43) + return 3; + if (str[0] != 'h' || str[4] != 'o' || str[5]) + return 4; + /* the literal is in a map */ + if (bpf_strncmp(local, sizeof(local), "abc")) + return 5; + return 0; +} + +char out[16]; + +/* format strings are in a map */ +SEC("syscall") +int use_printk(void *ctx) +{ + char buf[sizeof(out)]; + int i, n; + + bpf_printk("counter %d", counter); + n = BPF_SNPRINTF(buf, sizeof(buf), "%d-%d", x, ro); + for (i = 0; i < sizeof(out); i++) + out[i] = buf[i]; + return n; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_decl.c b/tools/testing/selftests/bpf/progs/data_in_arena_decl.c new file mode 100644 index 000000000000..01bc4fea8f0c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_decl.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_experimental.h" +#include <bpf_arena_common.h> + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 4); +} arena SEC(".maps"); + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +int counter = 5; +long sum; +const volatile int ro = 7; + +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) +int __arena avar = 11; +#else +int avar = 11; +#endif + +SEC("syscall") +int use_data(void *ctx) +{ + sum = counter + ro + avar; + counter++; + avar++; + return sum; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_extern.c b/tools/testing/selftests/bpf/progs/data_in_arena_extern.c new file mode 100644 index 000000000000..dc1ad5ca3bdd --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_extern.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +extern const int bpf_prog_active __ksym; + +/* the variable of the kernel is not in arena */ +const void *kp = &bpf_prog_active; + +SEC("syscall") +int ptr_to_extern(void *ctx) +{ + return *(int *)kp; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_fail.c b/tools/testing/selftests/bpf/progs/data_in_arena_fail.c new file mode 100644 index 000000000000..ad8afe9afc96 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_fail.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +int x = 42; +/* the table is read-only data with pointers. It's not in arena. */ +int *const volatile tbl[1] SEC(".data.rel.ro") = { &x }; +int *const volatile *pp = tbl; + +SEC("syscall") +int ptr_to_map(void *ctx) +{ + return **pp; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c b/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c new file mode 100644 index 000000000000..ccf66321eb36 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_arena_common.h" + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +int counter = 5; +/* needs an arena map that is declared. The one that libbpf creates won't do. */ +int __arena_global avar = 11; + +SEC("syscall") +int arena_var(void *ctx) +{ + return avar + counter; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs b/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs new file mode 100644 index 000000000000..3dd9d4208232 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Why .data, .bss and .rodata of a program in Rust are in arena. +// +// A reference in Rust is an address. It doesn't say what it points to and it +// can be stored in data. The list below has a node in each of the sections. +// The nodes are linked by references that are in the data: +// IN_BSS.next is stored by the program, +// IN_DATA.next is a relocation in .data against .rodata. +// sum() loads the references back and reads the three nodes with the same insn. +// +// When the sections are array maps libbpf skips the relocation in .data, and +// what sum() loads from a node is a number that can't be dereferenced: +// R1 invalid mem access 'scalar' +// In arena the address of a node is a number to begin with. + +#![no_std] +#![no_main] + +// Tell libbpf to keep .data, .bss and .rodata in arena. +#[used] +#[link_section = ".arena.data"] +static DATA_IN_ARENA: u8 = 0; + +#[used] +#[link_section = "license"] +static LICENSE: [u8; 4] = *b"GPL\0"; + +// panic=abort is a stop gap until panic=unwind is supported. +// Nothing here panics, so the handler is not a part of the program. +#[panic_handler] +fn panic(_info: &core::panic::PanicInfo) -> ! { + loop {} +} + +pub struct Node { + val: u32, + next: Option<&'static Node>, +} + +// no_mangle makes them visible outside, so LLVM can't fold the list into a constant. +#[no_mangle] +static IN_RODATA: Node = Node { val: 3, next: None }; +#[no_mangle] +static mut IN_DATA: Node = Node { + val: 20, + next: Some(&IN_RODATA), +}; +#[no_mangle] +static mut IN_BSS: Node = Node { val: 0, next: None }; + +#[inline(never)] +fn sum(mut node: Option<&Node>) -> u32 { + let mut sum = 0; + // The verifier wants a bound. + for _ in 0..8 { + let Some(n) = node else { break }; + sum += n.val; + node = n.next; + } + sum +} + +#[no_mangle] +#[link_section = "syscall"] +pub extern "C" fn list_in_data(_ctx: *mut u8) -> u32 { + unsafe { + let head = &mut *&raw mut IN_BSS; + head.val = 100; + head.next = Some(&*&raw const IN_DATA); + sum(Some(head)) + } +} diff --git a/tools/testing/selftests/bpf/progs/dynptr_fail.c b/tools/testing/selftests/bpf/progs/dynptr_fail.c index 1cd61d72c166..c2247b38c849 100644 --- a/tools/testing/selftests/bpf/progs/dynptr_fail.c +++ b/tools/testing/selftests/bpf/progs/dynptr_fail.c @@ -125,9 +125,9 @@ static int missing_release_callback_fn(__u32 index, void *data) return 0; } -/* Any dynptr initialized within a callback must have bpf_dynptr_put called */ +/* A callback cannot return with the last dynptr for a referenced resource. */ SEC("?raw_tp") -__failure __msg("Unreleased reference id") +__failure __msg("cannot overwrite referenced dynptr") int ringbuf_missing_release_callback(void *ctx) { bpf_loop(10, missing_release_callback_fn, NULL, 0); @@ -560,6 +560,9 @@ int global(void *ctx) /* A direct read should fail */ SEC("?raw_tp") __failure __msg("invalid read from stack") +__msg("Verification failed: Memory Safety: Direct read of dynptr stack state") +__msg("verifier-managed dynptr state") +__msg("Use dynptr helpers or kfuncs") int invalid_read1(void *ctx) { struct bpf_dynptr ptr; @@ -1892,6 +1895,101 @@ int clone_invalidate4(void *ctx) return 0; } +static __noinline void clone_slice_in_subprog(struct bpf_dynptr *ptr, int **data) +{ + struct bpf_dynptr clone; + + bpf_dynptr_clone(ptr, &clone); + *data = bpf_dynptr_data(&clone, 0, sizeof(val)); +} + +static __noinline void caller_slice_in_subprog(struct bpf_dynptr *ptr, int **data) +{ + struct bpf_dynptr clone; + + *data = bpf_dynptr_data(ptr, 0, sizeof(val)); + bpf_dynptr_clone(ptr, &clone); +} + +static __noinline void reserve_dynptr_in_subprog(void) +{ + struct bpf_dynptr ptr; + + bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr); +} + +/* A subprogram cannot lose the last dynptr that can release a resource. */ +SEC("?raw_tp") +__failure __msg("cannot overwrite referenced dynptr") +int referenced_dynptr_lost_on_subprog_return(void *ctx) +{ + reserve_dynptr_in_subprog(); + + return 0; +} + +/* + * Destroying a callee-local clone on return must not invalidate a slice whose + * source dynptr belongs to the caller. + */ +SEC("?raw_tp") +__success +int caller_dynptr_slice_across_subprog_valid(void *ctx) +{ + struct bpf_dynptr ptr; + int *data = NULL; + + bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr); + caller_slice_in_subprog(&ptr, &data); + if (data) + *data = 123; + bpf_ringbuf_submit_dynptr(&ptr, 0); + + return 0; +} + +/* + * A slice derived from a callee-local clone is invalid after the subprogram + * returns. + */ +SEC("?raw_tp") +__failure __msg("invalid mem access 'scalar'") +int callee_dynptr_slice_invalid_after_return(void *ctx) +{ + struct bpf_dynptr ptr; + int *data = NULL; + + bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr); + clone_slice_in_subprog(&ptr, &data); + if (data) + /* this should fail */ + *data = 123; + bpf_ringbuf_submit_dynptr(&ptr, 0); + + return 0; +} + +/* + * A slice from a caller-owned dynptr survives the subprogram return, but + * releasing the shared reservation must invalidate it. + */ +SEC("?raw_tp") +__failure __msg("invalid mem access 'scalar'") +int caller_dynptr_slice_release_after_subprog_invalid(void *ctx) +{ + struct bpf_dynptr ptr; + int *data = NULL; + + bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr); + caller_slice_in_subprog(&ptr, &data); + bpf_ringbuf_submit_dynptr(&ptr, 0); + if (data) + /* this should fail */ + *data = 123; + + return 0; +} + /* Invalidating a dynptr should invalidate any data slices * of its parent */ @@ -2050,19 +2148,18 @@ __noinline long global_call_bpf_dynptr(const struct bpf_dynptr *dynptr) /* Avoid leaving this global function empty to avoid having the compiler * optimize away the call to this global function. */ + __sink(dynptr); __sink(ret); return ret; } SEC("?raw_tp") -__failure __msg("R1 expected pointer to stack or const struct bpf_dynptr") +__failure __msg("R1 type=trusted_ptr_ expected=fp, dynptr_ptr") int test_dynptr_reg_type(void *ctx) { - struct task_struct *current = NULL; - /* R1 should be holding a PTR_TO_BTF_ID, so this shouldn't be a - * reg->type that can be passed to a function accepting a - * ARG_PTR_TO_DYNPTR | MEM_RDONLY. process_dynptr_func() should catch - * this. + struct task_struct *current = bpf_get_current_task_btf(); + /* R1 holds a PTR_TO_BTF_ID, which cannot be passed to a function + * accepting ARG_PTR_TO_DYNPTR | MEM_RDONLY. */ global_call_bpf_dynptr((const struct bpf_dynptr *)current); return 0; diff --git a/tools/testing/selftests/bpf/progs/exceptions_dead_subprog.c b/tools/testing/selftests/bpf/progs/exceptions_dead_subprog.c new file mode 100644 index 000000000000..4f8b82a85234 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/exceptions_dead_subprog.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_experimental.h" + +/* + * Read-only and frozen, so the verifier sees a constant zero, while clang + * cannot fold the volatile read and keeps the call below. + */ +const volatile int never; + +/* Only called from dead code, hence removed as a whole before the JIT runs. */ +static __noinline int dead_subprog(struct __sk_buff *ctx) +{ + return ctx->len; +} + +__noinline int exception_cb_behind_dead_subprog_cb(u64 cookie) +{ + return 42; +} + +/* + * libbpf appends the exception callback behind the subprograms reached by + * calls, so removing the dead one compacts subprog_info[] underneath the + * callback's index. + */ +SEC("?tc") +__exception_cb(exception_cb_behind_dead_subprog_cb) +__success __retval(42) +int exception_cb_behind_dead_subprog(struct __sk_buff *ctx) +{ + if (never) + return dead_subprog(ctx); + bpf_throw(0); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index ac44d60e5066..22503cf62e9f 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c @@ -60,7 +60,7 @@ __noinline int exception_cb_ok_arg_small(int a) SEC("?tc") __exception_cb(exception_cb_bad_ret_type1) -__failure __msg("Global function exception_cb_bad_ret_type1() return value not void or scalar.") +__failure __msg("Only void, scalar, arena pointer, or a struct/union of scalars and arena pointers up to 16 bytes is supported.") int reject_exception_cb_type_1(struct __sk_buff *ctx) { bpf_throw(0); diff --git a/tools/testing/selftests/bpf/progs/file_reader.c b/tools/testing/selftests/bpf/progs/file_reader.c index aa2c05cce2b3..8b972fd26d73 100644 --- a/tools/testing/selftests/bpf/progs/file_reader.c +++ b/tools/testing/selftests/bpf/progs/file_reader.c @@ -27,9 +27,14 @@ char tmp_buf[256000]; int pid = 0; int err, run_success = 0; +__u64 beyond_eof_offset; static int validate_file_read(struct file *file); static int task_work_callback(struct bpf_map *map, void *key, void *value); +static int sleepable_second_callback(struct bpf_map *map, void *key, void *value); + +void bpf_rcu_read_lock(void) __ksym; +void bpf_rcu_read_unlock(void) __ksym; SEC("lsm/file_open") int on_open_expect_fault(void *c) @@ -81,6 +86,101 @@ int on_open_validate_file_read(void *c) return 0; } +/* + * Exercise bpf_dynptr_from_file() first from a non-sleepable LSM program and + * then from its sleepable task-work callback. Reading beyond EOF makes the two + * backing implementations return different errors. + */ +SEC("lsm/file_open") +int on_open_non_sleepable_first(void *c) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct bpf_dynptr dynptr; + struct elem *work; + struct file *file; + int key = 0; + int ret; + + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 0; + + file = bpf_get_task_exe_file(task); + if (!file) { + err = 1; + return 0; + } + + /* The non-sleepable reader cannot fault in an uncached folio. */ + ret = bpf_dynptr_from_file(file, 0, &dynptr); + if (!ret) + ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0); + bpf_dynptr_file_discard(&dynptr); + bpf_put_file(file); + if (ret != -EFAULT) { + err = 2; + return 0; + } + + work = bpf_map_lookup_elem(&arrmap, &key); + if (!work) { + err = 3; + return 0; + } + + ret = bpf_task_work_schedule_signal(task, &work->tw, &arrmap, + sleepable_second_callback); + if (ret) + err = 4; + return 0; +} + +/* + * Exercise the opposite fixup order: the first call is made from a sleepable + * LSM program, while the RCU read-side section makes the second non-sleepable. + */ +SEC("lsm.s/file_open") +int on_open_sleepable_first(void *c) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct bpf_dynptr dynptr; + struct file *file; + int ret; + + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 0; + + file = bpf_get_task_exe_file(task); + if (!file) { + err = 7; + return 0; + } + + ret = bpf_dynptr_from_file(file, 0, &dynptr); + if (!ret) + ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0); + bpf_dynptr_file_discard(&dynptr); + if (ret != -EIO) { + err = 8; + goto out; + } + + bpf_rcu_read_lock(); + ret = bpf_dynptr_from_file(file, 0, &dynptr); + bpf_rcu_read_unlock(); + if (!ret) + ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0); + bpf_dynptr_file_discard(&dynptr); + if (ret != -EFAULT) { + err = 9; + goto out; + } + + run_success = 1; +out: + bpf_put_file(file); + return 0; +} + /* Called in a sleepable context, read 256K bytes, cross check with user space read data */ static int task_work_callback(struct bpf_map *map, void *key, void *value) { @@ -97,6 +197,35 @@ static int task_work_callback(struct bpf_map *map, void *key, void *value) return 0; } +/* Task-work callbacks are verified as sleepable. */ +static int sleepable_second_callback(struct bpf_map *map, void *key, void *value) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct bpf_dynptr dynptr; + struct file *file; + int ret; + + file = bpf_get_task_exe_file(task); + if (!file) { + err = 5; + return 0; + } + + /* freader_fetch() converts __kernel_read()'s short read at EOF to -EIO. */ + ret = bpf_dynptr_from_file(file, 0, &dynptr); + if (!ret) + ret = bpf_dynptr_read(tmp_buf, 1, &dynptr, beyond_eof_offset, 0); + bpf_dynptr_file_discard(&dynptr); + bpf_put_file(file); + if (ret != -EIO) { + err = 6; + return 0; + } + + run_success = 1; + return 0; +} + static int verify_dynptr_read(struct bpf_dynptr *ptr, u32 off, char *user_buf, u32 len) { int i; diff --git a/tools/testing/selftests/bpf/progs/freplace_ret_pair.c b/tools/testing/selftests/bpf/progs/freplace_ret_pair.c new file mode 100644 index 000000000000..12c15d293bd7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/freplace_ret_pair.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +SEC("freplace/agg_ret_target_func") +__u64 new_agg_ret_target_func(void) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/get_func_args_test.c b/tools/testing/selftests/bpf/progs/get_func_args_test.c index 1bf47f64d096..9b0beb58d88d 100644 --- a/tools/testing/selftests/bpf/progs/get_func_args_test.c +++ b/tools/testing/selftests/bpf/progs/get_func_args_test.c @@ -6,6 +6,40 @@ char _license[] SEC("license") = "GPL"; +/* Keep the upper index bits: the inlined expansion must truncate them. */ +static long (*bpf_get_func_arg_wide)(void *ctx, __u64 n, __u64 *value) = + (void *)BPF_FUNC_get_func_arg; + +/* Both traced functions are called with arguments 2 and 3. */ +static __always_inline int check_wide_index(void *ctx) +{ + __u64 a = 0, z = 0xdeadbeefdeadbeefULL; + __s64 err; + int ok = 1; + + /* The index is truncated to u32, so these select arguments 0 and 1. */ + err = bpf_get_func_arg_wide(ctx, 1ULL << 32, &a); + ok &= err == 0 && ((int)a == 2); + err = bpf_get_func_arg_wide(ctx, (1ULL << 32) | 1, &a); + ok &= err == 0 && a == 3; + err = bpf_get_func_arg_wide(ctx, 1ULL << 59, &a); + ok &= err == 0 && ((int)a == 2); + err = bpf_get_func_arg_wide(ctx, (1ULL << 59) | 1, &a); + ok &= err == 0 && a == 3; + + /* Out of range: -EINVAL and no store into the destination. */ + err = bpf_get_func_arg_wide(ctx, (1ULL << 32) | 2, &z); + ok &= err == -EINVAL && z == 0xdeadbeefdeadbeefULL; + err = bpf_get_func_arg_wide(ctx, (1ULL << 32) | 0xffffffffULL, &z); + ok &= err == -EINVAL && z == 0xdeadbeefdeadbeefULL; + err = bpf_get_func_arg_wide(ctx, (1ULL << 59) | 2, &z); + ok &= err == -EINVAL && z == 0xdeadbeefdeadbeefULL; + err = bpf_get_func_arg_wide(ctx, (1ULL << 59) | 0xffffffffULL, &z); + ok &= err == -EINVAL && z == 0xdeadbeefdeadbeefULL; + + return ok; +} + __u64 test1_result = 0; SEC("fentry/bpf_fentry_test1") int BPF_PROG(test1) @@ -63,6 +97,7 @@ int BPF_PROG(test2) /* return value */ err = bpf_get_func_ret(ctx, &ret); test2_result &= err == 0 && ret == 5; + test2_result &= check_wide_index(ctx); return 0; } @@ -162,6 +197,7 @@ int BPF_PROG(tp_test2) /* not valid argument */ err = bpf_get_func_arg(ctx, 2, &z); test6_result &= err == -EINVAL; + test6_result &= check_wide_index(ctx); return 0; } diff --git a/tools/testing/selftests/bpf/progs/irq.c b/tools/testing/selftests/bpf/progs/irq.c index a4a007866a33..50727fa9b11d 100644 --- a/tools/testing/selftests/bpf/progs/irq.c +++ b/tools/testing/selftests/bpf/progs/irq.c @@ -15,7 +15,20 @@ struct bpf_res_spin_lock lockA __hidden SEC(".data.A"); struct bpf_res_spin_lock lockB __hidden SEC(".data.B"); SEC("?tc") -__failure __msg("R1 doesn't point to an irq flag on stack") +__failure __msg("invalid read from stack") +__msg("Verification failed: Memory Safety: Direct read of IRQ flag stack state") +__msg("verifier-managed IRQ flag state") +__msg("Pass the saved IRQ flag to the matching restore kfunc") +int irq_flag_direct_read(struct __sk_buff *ctx) +{ + unsigned long flags; + + bpf_local_irq_save(&flags); + return flags; +} + +SEC("?tc") +__failure __msg("R1 type=map_value expected=fp") int irq_save_bad_arg(struct __sk_buff *ctx) { bpf_local_irq_save(&global_flags); @@ -23,7 +36,7 @@ int irq_save_bad_arg(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("R1 doesn't point to an irq flag on stack") +__failure __msg("R1 type=map_value expected=fp") int irq_restore_bad_arg(struct __sk_buff *ctx) { bpf_local_irq_restore(&global_flags); diff --git a/tools/testing/selftests/bpf/progs/iters.c b/tools/testing/selftests/bpf/progs/iters.c index c6699159dacd..ae845ad747bf 100644 --- a/tools/testing/selftests/bpf/progs/iters.c +++ b/tools/testing/selftests/bpf/progs/iters.c @@ -1688,7 +1688,7 @@ int iter_subprog_check_stacksafe(const void *ctx) struct bpf_iter_num global_it; SEC("raw_tp") -__failure __msg("R1 expected pointer to an iterator on stack") +__failure __msg("R1 type=map_value expected=fp") int iter_new_bad_arg(const void *ctx) { bpf_iter_num_new(&global_it, 0, 1); @@ -1696,7 +1696,7 @@ int iter_new_bad_arg(const void *ctx) } SEC("raw_tp") -__failure __msg("R1 expected pointer to an iterator on stack") +__failure __msg("R1 type=map_value expected=fp") int iter_next_bad_arg(const void *ctx) { bpf_iter_num_next(&global_it); @@ -1704,7 +1704,7 @@ int iter_next_bad_arg(const void *ctx) } SEC("raw_tp") -__failure __msg("R1 expected pointer to an iterator on stack") +__failure __msg("R1 type=map_value expected=fp") int iter_destroy_bad_arg(const void *ctx) { bpf_iter_num_destroy(&global_it); @@ -2188,4 +2188,386 @@ __naked void loop_counter_precision_2nd_iter(void) ); } +/* + * An iterator that is destroyed and created again inside its loop starts from + * the beginning and the loop never ends. In the first four progs the first + * prune point after bpf_iter_num_new() is the call of bpf_iter_num_next(). + */ +SEC("socket") +__description("iter: remake in the loop, jump to next") +__failure __msg("infinite loop detected") +__naked void iter_remake_jmp(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "call %[bpf_ktime_get_ns];" + "if r0 != 0 goto 3f;" + "r1 = r10;" + "r1 += -8;" +"2:" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" +"3:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -8;" + "goto 2b;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +__used __naked static void iter_remake(void) +{ + asm volatile ( + "r6 = r1;" + "call %[bpf_iter_num_destroy];" + "r1 = r6;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_destroy) + : __clobber_all); +} + +SEC("socket") +__description("iter: remake in a callee, jump to next") +__failure __msg("infinite loop detected") +__naked void iter_remake_callee_jmp(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "call %[bpf_ktime_get_ns];" + "if r0 != 0 goto 3f;" + "r1 = r10;" + "r1 += -8;" +"2:" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" +"3:" + "r1 = r10;" + "r1 += -8;" + "call iter_remake;" + "r1 = r10;" + "r1 += -8;" + "goto 2b;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("socket") +__description("iter: remake after next, jump to next") +__failure __msg("infinite loop detected") +__naked void iter_remake_after_next_jmp(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -8;" +"1:" + "call %[bpf_iter_num_next];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -8;" + "goto 1b;" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("socket") +__description("iter: two iterators, remake each after its next") +__failure __msg("infinite loop detected") +__naked void iter_remake_two_jmp(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -16;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -8;" +"1:" + "call %[bpf_iter_num_next];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -16;" + "call %[bpf_iter_num_next];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -16;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -16;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "r1 = r10;" + "r1 += -8;" + "goto 1b;" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -16;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +/* Same loops with a prune point in front of bpf_iter_num_next() */ +SEC("socket") +__description("iter: remake in the loop") +__failure __msg("infinite loop detected") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void iter_remake_inline(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "call %[bpf_ktime_get_ns];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("socket") +__description("iter: remake in a callee") +__failure __msg("infinite loop detected") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void iter_remake_callee(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "call %[bpf_ktime_get_ns];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call iter_remake;" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("socket") +__description("iter: remake after next") +__failure __msg("infinite loop detected") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void iter_remake_after_next(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_next];" + "if r0 == 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "goto 1b;" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("socket") +__description("iter: remake when drained") +__failure __msg("infinite loop detected") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void iter_remake_drained(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "call %[bpf_ktime_get_ns];" + "if r0 == 119 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r1 = r10;" + "r1 += -8;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" + "goto 1b;" +"2:" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +/* The inner iterator is created in every iteration of the outer loop */ +SEC("socket") +__description("iter: nested loops") +__success __retval(12) +int iter_nested_remake(void *ctx) +{ + int i, j, sum = 0; + + bpf_for(i, 0, 3) { + bpf_for(j, 0, 4) + sum++; + } + return sum; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/iters_state_safety.c b/tools/testing/selftests/bpf/progs/iters_state_safety.c index e5bb9fe6d5e5..b2ba4609a31e 100644 --- a/tools/testing/selftests/bpf/progs/iters_state_safety.c +++ b/tools/testing/selftests/bpf/progs/iters_state_safety.c @@ -354,6 +354,9 @@ int next_after_destroy_fail(void *ctx) SEC("?raw_tp") __failure __msg("invalid read from stack") +__msg("Verification failed: Memory Safety: Direct read of iterator stack state") +__msg("verifier-managed iterator state") +__msg("Use iterator kfuncs") int __naked read_from_iter_slot_fail(void) { asm volatile ( diff --git a/tools/testing/selftests/bpf/progs/iters_testmod.c b/tools/testing/selftests/bpf/progs/iters_testmod.c index 76012dbbdb41..145750bb64ef 100644 --- a/tools/testing/selftests/bpf/progs/iters_testmod.c +++ b/tools/testing/selftests/bpf/progs/iters_testmod.c @@ -46,6 +46,17 @@ int iter_next_trusted_or_null(const void *ctx) return 0; } +SEC("syscall") +__success +int trusted_nullable_kfunc(const void *ctx) +{ + struct task_struct *task; + + task = bpf_kfunc_trusted_null_test(); + bpf_kfunc_trusted_nullable_test(task); + return 0; +} + SEC("raw_tp/sys_enter") __success int iter_next_rcu(const void *ctx) @@ -105,8 +116,7 @@ out: } SEC("raw_tp/sys_enter") -__failure __msg("R1 cannot write into rdonly_mem") -/* Message should not be 'R1 cannot write into rdonly_trusted_mem' */ +__failure __msg("R1 type=rdonly_mem expected=fp") int iter_next_ptr_mem_not_trusted(const void *ctx) { struct bpf_iter_num num_it; @@ -135,7 +145,7 @@ int iter_ret_rcu_test_protected(const void *ctx) } SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") -__failure __msg("R1 type=rcu_ptr_or_null_ expected=") +__failure __msg("Possibly NULL pointer passed to trusted R1") int iter_ret_rcu_test_type(const void *ctx) { struct task_struct *p; @@ -158,7 +168,7 @@ int iter_ret_rcu_test_protected_nostruct(const void *ctx) } SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") -__failure __msg("R1 type=rdonly_rcu_mem_or_null expected=") +__failure __msg("Possibly NULL pointer passed to trusted R1") int iter_ret_rcu_test_type_nostruct(const void *ctx) { void *p; diff --git a/tools/testing/selftests/bpf/progs/jit_insns_arena.c b/tools/testing/selftests/bpf/progs/jit_insns_arena.c new file mode 100644 index 000000000000..95736eae2a58 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/jit_insns_arena.c @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * The arena accesses of jit_insns in their own object, so that a kernel + * without arena support fails only these programs. + */ +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_experimental.h" +#include "bpf_arena_common.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * The first bytes in memory of the test value and its sum with the slot + * after the u32 store depend on the byte order. + */ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define VAL_B0 0x88 +#define VAL_H0 0x7788 +#define VAL_W0 0x55667788 +#define VAL_SUM 0x22446688556689bc +#else +#define VAL_B0 0x11 +#define VAL_H0 0x1122 +#define VAL_W0 0x11223344 +#define VAL_SUM 0x11224578aaccef10 +#endif + +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) +SEC("syscall") +__success __retval(0) +int arena_zero_off(void *ctx) +{ + void __arena *page; + __u64 res; + + page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!page) + return 1; + res = (__u32)(__u64)page; + /* Accesses through two base registers */ + asm volatile (" \ + r7 = %[res]; \ + r7 = addr_space_cast(r7, 0x0, 0x1); \ + r1 = r7; \ + r8 = 0x1122334455667788 ll; \ + *(u64 *)(r7 + 0) = r8; \ + r9 = *(u64 *)(r7 + 0); \ + r9 ^= r8; \ + *(u64 *)(r1 + 0) = r8; \ + r2 = *(u64 *)(r1 + 0); \ + r2 ^= r8; \ + r9 |= r2; \ + r3 = *(u8 *)(r1 + 0); \ + r3 ^= %[b0]; \ + r9 |= r3; \ + r4 = *(u16 *)(r1 + 0); \ + r4 ^= %[h0]; \ + r9 |= r4; \ + r5 = *(u32 *)(r1 + 0); \ + r5 ^= %[w0]; \ + r9 |= r5; \ + *(u32 *)(r1 + 0) = 0x1234; \ + lock *(u64 *)(r7 + 0) += r8; \ + r2 = *(u64 *)(r1 + 0); \ + r8 = %[sum] ll; \ + r2 ^= r8; \ + r9 |= r2; \ + %[res] = r9; \ +" : [res]"+r"(res) + : __imm_const(b0, VAL_B0), __imm_const(h0, VAL_H0), + __imm_const(w0, VAL_W0), __imm_const(sum, VAL_SUM) + : "r1", "r2", "r3", "r4", "r5", "r7", "r8", "r9"); + return res != 0; +} + +SEC("syscall") +__success __retval(0) +int cast_to_user(void *ctx) +{ + void __arena *page; + __u64 res; + + page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!page) + return 1; + /* The user address comes back from the kernel one, NULL stays NULL */ + asm volatile (" \ + r7 = %[page]; \ + r1 = addr_space_cast(r7, 0x0, 0x1); \ + r1 = addr_space_cast(r1, 0x1, 0x0); \ + r9 = r1; \ + r9 ^= r7; \ + r2 = 0; \ + r2 = addr_space_cast(r2, 0x1, 0x0); \ + r9 |= r2; \ + %[res] = r9; \ +" : [res]"=r"(res) + : [page]"r"(page) + : "r1", "r2", "r7", "r9"); + return res != 0; +} +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/kasan.c b/tools/testing/selftests/bpf/progs/kasan.c new file mode 100644 index 000000000000..fe8e0dd228a0 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/kasan.c @@ -0,0 +1,502 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +#include <stdbool.h> +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" + +extern void bpf_kfunc_kasan_poison(void *mem, __u32 mem__sz) __ksym; +extern void bpf_kfunc_kasan_unpoison(void *mem, __u32 mem__sz) __ksym; + +struct bpf_testmod_oob { + __u8 data; + union { + __u8 redzone_1; + __u16 redzone_2; + __u32 redzone_4; + __u64 redzone_8; + }; +}; + +extern struct bpf_testmod_oob *bpf_testmod_oob_alloc(void) __ksym; +extern void bpf_testmod_oob_free(struct bpf_testmod_oob *oob) __ksym; + +int access_size; + +struct kasan_test_val { + __u8 data_1; + __u16 data_2; + __u32 data_4; + __u64 data_8; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct kasan_test_val); +} test_map SEC(".maps"); + +/* + * ST instructions are only emitted if the BPF cpu supports it (eg cpuv4), + * they are otherwise turned into MOV + STX, so compile and exercise ST + * only if supported. + */ +#ifdef __BPF_FEATURE_ST +SEC("tcx/ingress") +int st_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + val.data_1 = 0xAA; + break; + case 2: + val.data_2 = 0xAA; + break; + case 4: + val.data_4 = 0xAA; + break; + case 8: + val.data_8 = 0xAA; + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int st_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + val->data_1 = 0xAA; + break; + case 2: + val->data_2 = 0xAA; + break; + case 4: + val->data_4 = 0xAA; + break; + case 8: + val->data_8 = 0xAA; + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +bool skip_st_tests SEC(".data") = 0; +#else +bool skip_st_tests SEC(".data") = 1; +#endif + +SEC("tcx/ingress") +int stx_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + /* + * Unlike the st() programs above, the stored value comes from a + * runtime source (skb->len), so it cannot be constant-folded and + * clang always emits a genuine BPF_STX (register store) regardless + * of the target cpu version. + */ + switch (access_size) { + case 1: + val.data_1 = (__u8)skb->len; + break; + case 2: + val.data_2 = (__u16)skb->len; + break; + case 4: + val.data_4 = (__u32)skb->len; + break; + case 8: + val.data_8 = (__u64)skb->len; + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int stx_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + val->data_1 = (__u8)skb->len; + break; + case 2: + val->data_2 = (__u16)skb->len; + break; + case 4: + val->data_4 = (__u32)skb->len; + break; + case 8: + val->data_8 = (__u64)skb->len; + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __sink(val.data_1); + break; + case 2: + __sink(val.data_2); + break; + case 4: + __sink(val.data_4); + break; + case 8: + __sink(val.data_8); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __sink(val->data_1); + break; + case 2: + __sink(val->data_2); + break; + case 4: + __sink(val->data_4); + break; + case 8: + __sink(val->data_8); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_patched_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + __sink(val->data_4); + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + + return 0; +} + +SEC("tcx/ingress") +int ldx_patched_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + __sink(val.data_4); + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 4: + __sync_fetch_and_add(&val.data_4, 4); + break; + case 8: + __sync_fetch_and_add(&val.data_8, 8); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 4: + __sync_fetch_and_add(&val->data_4, 4); + break; + case 8: + __sync_fetch_and_add(&val->data_8, 8); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_fetch_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + __sync_fetch_and_or(&val.data_8, 8); + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_fetch_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + __sync_fetch_and_or(&val->data_8, 8); + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +#ifdef __BPF_FEATURE_LOAD_ACQ_STORE_REL +bool skip_load_acq_store_rel_tests SEC(".data") = 0; + +SEC("tcx/ingress") +int load_acquire_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __atomic_load_n(&val.data_1, __ATOMIC_ACQUIRE); + break; + case 2: + __atomic_load_n(&val.data_2, __ATOMIC_ACQUIRE); + break; + case 4: + __atomic_load_n(&val.data_4, __ATOMIC_ACQUIRE); + break; + case 8: + __atomic_load_n(&val.data_8, __ATOMIC_ACQUIRE); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int load_acquire_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __atomic_load_n(&val->data_1, __ATOMIC_ACQUIRE); + break; + case 2: + __atomic_load_n(&val->data_2, __ATOMIC_ACQUIRE); + break; + case 4: + __atomic_load_n(&val->data_4, __ATOMIC_ACQUIRE); + break; + case 8: + __atomic_load_n(&val->data_8, __ATOMIC_ACQUIRE); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int store_release_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __atomic_store_n(&val.data_1, 0xAA, __ATOMIC_RELEASE); + break; + case 2: + __atomic_store_n(&val.data_2, 0xBBBB, __ATOMIC_RELEASE); + break; + case 4: + __atomic_store_n(&val.data_4, 0xCCCCCCCC, __ATOMIC_RELEASE); + break; + case 8: + __atomic_store_n(&val.data_8, 0xDDDDDDDDDDDDDDDD, + __ATOMIC_RELEASE); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int store_release_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __atomic_store_n(&val->data_1, 0xAA, __ATOMIC_RELEASE); + break; + case 2: + __atomic_store_n(&val->data_2, 0xBBBB, __ATOMIC_RELEASE); + break; + case 4: + __atomic_store_n(&val->data_4, 0xCCCCCCCC, __ATOMIC_RELEASE); + break; + case 8: + __atomic_store_n(&val->data_8, 0xDDDDDDDDDDDDDDDD, + __ATOMIC_RELEASE); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} +#else +bool skip_load_acq_store_rel_tests SEC(".data") = 1; +#endif + +SEC("tcx/ingress") +int verifier_paths_stack_and_non_stack(struct __sk_buff *skb) +{ + struct kasan_test_val stack_val = {}; + struct kasan_test_val *val; + void *ptr; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + if (access_size) + ptr = val; + else + ptr = &stack_val; + + bpf_kfunc_kasan_poison(val, sizeof(*val)); + *(__u8 *)ptr = 0xAA; + bpf_kfunc_kasan_unpoison(val, sizeof(*val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_oob(struct __sk_buff *skb) +{ + struct bpf_testmod_oob *val; + struct kasan_test_val volatile tmp; + + val = bpf_testmod_oob_alloc(); + if (!val) + return 0; + + switch (access_size) { + case 1: + tmp.data_1 = (__u8)val->redzone_1; + break; + case 2: + tmp.data_2 = (__u16)val->redzone_2; + break; + case 4: + tmp.data_4 = (__u32)val->redzone_4; + break; + case 8: + tmp.data_8 = (__u64)val->redzone_8; + break; + } + bpf_testmod_oob_free(val); + return tmp.data_1; +} + +SEC("tcx/ingress") +int ldx_self_alias_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + __u64 addr; + + bpf_kfunc_kasan_poison(&val, sizeof(val)); + /* + * Check that a stack access with dst_reg == src_reg is correctly + * flagged as stack-only access + */ + addr = (__u64)&val; + asm volatile( + "r1 = %0\n" + "r1 = *(u64 *)(r1 + 0)\n" + : + : "r"(addr) + : "r1", "memory"); + bpf_kfunc_kasan_unpoison(&val, sizeof(val)); + + return 0; +} + +char LICENSE[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/kasan_harden.c b/tools/testing/selftests/bpf/progs/kasan_harden.c new file mode 100644 index 000000000000..8c9eb203419c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/kasan_harden.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +#include <stdbool.h> +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +extern void bpf_kfunc_kasan_poison(void *mem, __u32 mem__sz) __ksym; +extern void bpf_kfunc_kasan_unpoison(void *mem, __u32 mem__sz) __ksym; + +struct kasan_test_val { + __u8 data_1; + __u16 data_2; + __u32 data_4; + __u64 data_8; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct kasan_test_val); +} test_map SEC(".maps"); + +/* + * ST instructions are only emitted if the BPF cpu supports it (eg cpuv4), + * they are otherwise turned into MOV + STX, so compile and exercise ST + * only if supported. + */ +#ifdef __BPF_FEATURE_ST +SEC("tcx/ingress") +int st_blinded(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + val->data_1 = 0xAA; + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + + return 0; +} +bool skip_st_tests SEC(".data") = 0; +#else +bool skip_st_tests SEC(".data") = 1; +#endif + +char LICENSE[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/ksock_common.h b/tools/testing/selftests/bpf/progs/ksock_common.h index 01edaeb9fdd4..2a32cff2fbe9 100644 --- a/tools/testing/selftests/bpf/progs/ksock_common.h +++ b/tools/testing/selftests/bpf/progs/ksock_common.h @@ -16,6 +16,8 @@ int bpf_ksock_connect(struct bpf_ksock *ks, const union bpf_ksock_addr *addr, struct bpf_ksock *bpf_ksock_acquire(struct bpf_ksock *ks) __ksym; void bpf_ksock_release(struct bpf_ksock *ks) __ksym; int bpf_ksock_send(struct bpf_ksock *ks, const void *data, u32 data__sz) __ksym; +int bpf_ksock_sendto(struct bpf_ksock *ks, const void *data, u32 data__sz, + const union bpf_ksock_addr *addr__nullable, u32 addr__sz) __ksym; void bpf_rcu_read_lock(void) __ksym; void bpf_rcu_read_unlock(void) __ksym; diff --git a/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c b/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c index fd2ccfdb5802..9533570eb340 100644 --- a/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c +++ b/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c @@ -12,7 +12,7 @@ char send_data[11] = "dummy data"; SEC("lsm.s/socket_sendmsg") __description("bpf_ksock_send is rejected from socket_sendmsg LSM hook") __failure __msg("calling kernel function bpf_ksock_send is not allowed") -int BPF_PROG(ksock_socket_sendmsg, struct socket *sock, struct msghdr *msg, +int BPF_PROG(socket_sendmsg_ksock_send, struct socket *sock, struct msghdr *msg, int size, int ret) { struct __ksock_ctx_value *v; @@ -32,4 +32,27 @@ int BPF_PROG(ksock_socket_sendmsg, struct socket *sock, struct msghdr *msg, return ret; } +SEC("lsm.s/socket_sendmsg") +__description("bpf_ksock_sendto is rejected from socket_sendmsg LSM hook") +__failure __msg("calling kernel function bpf_ksock_sendto is not allowed") +int BPF_PROG(socket_sendmsg_ksock_sendto, struct socket *sock, + struct msghdr *msg, int size, int ret) +{ + struct __ksock_ctx_value *v; + struct bpf_ksock *ks; + + v = ksock_ctx_value_lookup(); + if (!v) + return ret; + + ks = bpf_kptr_xchg(&v->ctx, NULL); + if (!ks) + return ret; + + bpf_ksock_sendto(ks, send_data, sizeof(send_data), NULL, 0); + bpf_ksock_release(ks); + + return ret; +} + char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/ksock_sendto.c b/tools/testing/selftests/bpf/progs/ksock_sendto.c new file mode 100644 index 000000000000..b6eacf277fcb --- /dev/null +++ b/tools/testing/selftests/bpf/progs/ksock_sendto.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> +#include "bpf_tracing_net.h" +#include "ksock_common.h" + +char sendto_data[] = "hello from bpf sendto"; +char send_data[] = "hello from bpf send"; + +__be16 connected_port; +__be16 sendto_port; +bool connected; +bool use_sendto_addr; +int sendto_ret = -1; +int send_ret = -1; + +SEC("syscall") +int test_sendto(void *ctx) +{ + struct bpf_ksock_create_opts opts = { + .family = AF_INET, + .type = SOCK_DGRAM, + .protocol = IPPROTO_UDP, + }; + union bpf_ksock_addr addr = {}; + struct bpf_ksock *ks; + int err = 0; + + ks = bpf_ksock_create(&opts, sizeof(opts), &err); + if (!ks) + return err; + + addr.sin.sin_family = AF_INET; + addr.sin.sin_addr.s_addr = bpf_htonl(0x7f000001); + if (connected) { + addr.sin.sin_port = connected_port; + err = bpf_ksock_connect(ks, &addr, sizeof(addr)); + if (err) + goto out; + } + + addr.sin.sin_port = sendto_port; + if (use_sendto_addr) + sendto_ret = bpf_ksock_sendto(ks, sendto_data, sizeof(sendto_data), + &addr, sizeof(addr)); + else + sendto_ret = bpf_ksock_sendto(ks, sendto_data, sizeof(sendto_data), NULL, 0); + + /* sendto() must neither connect the socket nor change its default peer. */ + send_ret = bpf_ksock_send(ks, send_data, sizeof(send_data)); +out: + bpf_ksock_release(ks); + return err; +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/ksock_wq.c b/tools/testing/selftests/bpf/progs/ksock_wq.c index 16a1873d132e..fab356252d53 100644 --- a/tools/testing/selftests/bpf/progs/ksock_wq.c +++ b/tools/testing/selftests/bpf/progs/ksock_wq.c @@ -2,6 +2,7 @@ /* Copyright (c) 2026 Isovalent */ #include "vmlinux.h" +#include <bpf/bpf_endian.h> #include <bpf/bpf_helpers.h> #include "bpf_experimental.h" #include "bpf_tracing_net.h" @@ -14,15 +15,19 @@ struct ksock_wq_value { struct { __uint(type, BPF_MAP_TYPE_ARRAY); - __uint(max_entries, 1); + __uint(max_entries, 2); __type(key, u32); __type(value, struct ksock_wq_value); } work_map SEC(".maps"); -int create_err; +char send_data[] = "hello from bpf workqueue"; + +__be16 remote_port; +int connect_send_ret; +int create_ret; u32 callback_done; -static int ksock_wq_callback(void *map, int *key, void *value) +static int ksock_wq_create_cb(void *map, int *key, void *value) { struct bpf_ksock_create_opts opts = { .family = AF_INET, @@ -35,28 +40,85 @@ static int ksock_wq_callback(void *map, int *key, void *value) ks = bpf_ksock_create(&opts, sizeof(opts), &err); if (ks) bpf_ksock_release(ks); - create_err = err; + create_ret = err; __sync_fetch_and_add(&callback_done, 1); return 0; } -SEC("syscall") -int ksock_wq_start(void *ctx) +static int ksock_wq_connect_send_cb(void *map, int *key, void *value) +{ + union bpf_ksock_addr addr = { + .sin = { + .sin_family = AF_INET, + .sin_port = remote_port, + .sin_addr.s_addr = bpf_htonl(0x7f000001), + }, + }; + struct bpf_ksock *ks; + int err = -ENOENT; + + ks = ksock_ctx_get(); + if (!ks) + goto out; + err = bpf_ksock_connect(ks, &addr, sizeof(addr)); + if (!err) + err = bpf_ksock_send(ks, send_data, sizeof(send_data)); + bpf_ksock_release(ks); +out: + connect_send_ret = err; + __sync_fetch_and_add(&callback_done, 1); + return 0; +} + +static int wq_set_callback_start(int (*callback_fn)(void *, int *, void *), u32 key) { struct ksock_wq_value *value; - u32 key = 0; int err; value = bpf_map_lookup_elem(&work_map, &key); if (!value) return -ENOENT; + err = bpf_wq_init(&value->work, &work_map, 0); if (err) return err; - err = bpf_wq_set_callback(&value->work, ksock_wq_callback, 0); + err = bpf_wq_set_callback(&value->work, callback_fn, 0); + if (err) + return err; + err = bpf_wq_start(&value->work, 0); if (err) return err; - return bpf_wq_start(&value->work, 0); + + return 0; +} + +SEC("syscall") +int ksock_wq_start(void *ctx) +{ + struct bpf_ksock_create_opts opts = { + .family = AF_INET, + .type = SOCK_DGRAM, + .protocol = IPPROTO_UDP, + }; + struct bpf_ksock *ks; + int err = 0; + + ks = bpf_ksock_create(&opts, sizeof(opts), &err); + if (!ks) + return err; + err = ksock_ctx_insert(ks); + if (err) + return err; + + err = wq_set_callback_start(ksock_wq_connect_send_cb, 0); + if (err) + return err; + + err = wq_set_callback_start(ksock_wq_create_cb, 1); + if (err) + return err; + + return 0; } char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/linked_arena1.c b/tools/testing/selftests/bpf/progs/linked_arena1.c new file mode 100644 index 000000000000..5027b4ac2887 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/linked_arena1.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_arena_common.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); /* number of pages */ +} arena SEC(".maps"); + +/* + * Dereferencing an arena global needs the compiler to emit an + * addr_space_cast, which only clang does. Keep the variables so the arena + * is still populated and the skeleton still has its arena member, but skip + * the test elsewhere. + */ +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST +bool skip_tests __attribute((__section__(".data"))) = false; +#else +bool skip_tests = true; +#endif + +long __arena_global a_val = 1; +extern long __arena_global b_val; /* defined in linked_arena2.c */ + +SEC("syscall") +int sum1(void *ctx) +{ +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST + return a_val + b_val; +#else + /* + * Reference the extern without dereferencing it, so that the relink + * test still has an extern to resolve. + */ + return (long)&b_val; +#endif +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/linked_arena2.c b/tools/testing/selftests/bpf/progs/linked_arena2.c new file mode 100644 index 000000000000..db37198854a3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/linked_arena2.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_arena_common.h" + +long __arena_global b_val = 2; +extern long __arena_global a_val; /* defined in linked_arena1.c */ + +SEC("syscall") +int bump2(void *ctx) +{ +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST + a_val += 10; + b_val += 20; + return a_val + b_val; +#else /* see linked_arena1.c */ + return (long)&a_val; +#endif +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/lirc_mode2.c b/tools/testing/selftests/bpf/progs/lirc_mode2.c new file mode 100644 index 000000000000..98137f3c5c03 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lirc_mode2.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +// test ir decoder +// +// Copyright (C) 2018 Sean Young <sean@mess.org> + +#include <linux/bpf.h> +#include <linux/lirc.h> +#include <bpf/bpf_helpers.h> + +SEC("lirc_mode2") +int bpf_decoder(unsigned int *sample) +{ + if (LIRC_IS_PULSE(*sample)) { + unsigned int duration = LIRC_VALUE(*sample); + + /* + * Flag bits picked deliberately low: rc-loopback simulates + * a receiver overflow for any pulse over MS_TO_US(50) (see + * loop_tx_ir() in rc-loopback.c), which would silently + * swallow the sample before it ever reaches this decoder. + */ + if (duration & 0x8000) + bpf_rc_keydown(sample, 0x40, duration & 0x3fff, 0); + if (duration & 0x4000) + bpf_rc_pointer_rel(sample, (duration >> 7) & 0x7f, + duration & 0x7f); + } + + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/lsm.c b/tools/testing/selftests/bpf/progs/lsm.c index 7e32fbddbfad..e210e07d3a37 100644 --- a/tools/testing/selftests/bpf/progs/lsm.c +++ b/tools/testing/selftests/bpf/progs/lsm.c @@ -192,6 +192,7 @@ int BPF_PROG(test_void_hook, struct linux_binprm *bprm) { __u32 pid = bpf_get_current_pid_tgid() >> 32; struct inner_map *inner_map; + struct mm_struct *mm; char args[64]; __u32 key = 0; __u64 *value; @@ -200,7 +201,9 @@ int BPF_PROG(test_void_hook, struct linux_binprm *bprm) bprm_count++; bpf_copy_from_user(args, sizeof(args), (void *)bprm->vma->vm_mm->arg_start); - bpf_copy_from_user(args, sizeof(args), (void *)bprm->mm->arg_start); + mm = bprm->mm; + if (mm) + bpf_copy_from_user(args, sizeof(args), (void *)mm->arg_start); value = bpf_map_lookup_elem(&array, &key); if (value) diff --git a/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr.c b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr.c new file mode 100644 index 000000000000..77b37cf6165a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include <errno.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_helpers.h> +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +#define LABEL_MAX 32 + +struct label { + char v[LABEL_MAX]; + __u32 len; +}; + +struct { + __uint(type, BPF_MAP_TYPE_INODE_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct label); +} inode_zone SEC(".maps"); + +__u32 monitored_pid; + +const char xattr_zone[] = "security.bpf.zone"; +const char xattr_origin[] = "security.bpf.origin"; +const char xattr_overflow[] = "security.bpf.overflow"; +/* One byte over XATTR_NAME_MAX once "security." is prepended again. */ +const char xattr_toolong[] = "security.bpf." "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const char xattr_selinux[] = "security.selinux"; +const char xattr_user[] = "user.zone"; + +char zone_scratch[LABEL_MAX]; +char origin_value[] = "created"; + +__u32 hook_ran; +__s32 zone_err = 1; +__s32 origin_err = 1; +__s32 overflow_err = 1; +__s32 toolong_err = 1; +__s32 selinux_err = 1; +__s32 user_err = 1; +__u32 inherited; + +SEC("lsm/inode_init_security") +int BPF_PROG(init_label, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + struct label *parent; + int len = 0; + + if ((bpf_get_current_pid_tgid() >> 32) != monitored_pid) + return 0; + + hook_ran = 1; + if (!xattrs) { + /* The filesystem takes no xattrs at inode creation. */ + zone_err = -EOPNOTSUPP; + return 0; + } + __builtin_memset(zone_scratch, 0, LABEL_MAX); + + if (dir) { + parent = bpf_inode_storage_get(&inode_zone, dir, 0, 0); + if (parent && parent->len > 0 && parent->len <= LABEL_MAX) { + len = parent->len; + __builtin_memcpy(zone_scratch, parent->v, LABEL_MAX); + inherited = 1; + } + } + if (!len) { + __builtin_memcpy(zone_scratch, "default", sizeof("default")); + len = sizeof("default"); + } + + bpf_dynptr_from_mem(zone_scratch, len, 0, &value); + /* Refused before a slot is claimed, so the budget below is intact. */ + toolong_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_toolong, + &value); + zone_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_zone, + &value); + + bpf_dynptr_from_mem(origin_value, sizeof(origin_value), 0, &value); + origin_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_origin, + &value); + + overflow_err = bpf_inode_init_xattr(xattrs, xattr_count, + xattr_overflow, &value); + + selinux_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_selinux, + &value); + user_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_user, + &value); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_budget.c b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_budget.c new file mode 100644 index 000000000000..08dd93a1db11 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_budget.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include <errno.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_helpers.h> +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +const char xattr_name[32] = "security.bpf.slot"; + +__u32 monitored_pid; +__u32 hook_ran; +__s32 claim_err = 1; + +char claim_value[] = "v"; + +SEC("lsm/inode_init_security") +int BPF_PROG(claim_one, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if ((bpf_get_current_pid_tgid() >> 32) != monitored_pid) + return 0; + + hook_ran = 1; + if (!xattrs) { + claim_err = -EOPNOTSUPP; + return 0; + } + bpf_dynptr_from_mem(claim_value, sizeof(claim_value), 0, &value); + claim_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_name, + &value); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_value.c b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_value.c new file mode 100644 index 000000000000..6a879a758601 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_value.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include <errno.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_helpers.h> +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +#define VALUE_MAX (64 * 1024 + 1) + +const char xattr_value[] = "security.bpf.value"; + +/* Shape of the value handed to the kfunc, and what the hook returns after. */ +const volatile __u32 value_len; +const volatile __u64 dynptr_flags; +const volatile __s32 hook_retval; + +char value_src[VALUE_MAX]; + +__u32 monitored_pid; +__u32 hook_ran; +__s32 claim_err = 1; + +SEC("lsm/inode_init_security") +int BPF_PROG(claim_value, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + __u32 len = value_len; + + if ((bpf_get_current_pid_tgid() >> 32) != monitored_pid) + return 0; + if (len > VALUE_MAX) + return 0; + + hook_ran = 1; + if (!xattrs) { + claim_err = -EOPNOTSUPP; + return 0; + } + bpf_dynptr_from_mem(value_src, len, dynptr_flags, &value); + claim_err = bpf_inode_init_xattr(xattrs, xattr_count, xattr_value, + &value); + return hook_retval; +} diff --git a/tools/testing/selftests/bpf/progs/map_excl.c b/tools/testing/selftests/bpf/progs/map_excl.c index d461684728e4..90f127073dff 100644 --- a/tools/testing/selftests/bpf/progs/map_excl.c +++ b/tools/testing/selftests/bpf/progs/map_excl.c @@ -27,7 +27,8 @@ int should_have_access(void *ctx) SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") int should_not_have_access(void *ctx) { - int key = 0, value = 0xdeadbeef; + /* a different value, so that the two programs do not share a digest */ + int key = 0, value = 0xbaadf00d; bpf_map_update_elem(&excl_map, &key, &value, 0); return 0; diff --git a/tools/testing/selftests/bpf/progs/map_kptr_fail.c b/tools/testing/selftests/bpf/progs/map_kptr_fail.c index eee35d203b66..ac4003bfb8b0 100644 --- a/tools/testing/selftests/bpf/progs/map_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/map_kptr_fail.c @@ -149,7 +149,7 @@ int reject_bad_type_match(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("R1 type=untrusted_ptr_or_null_ expected=percpu_ptr_") +__failure __msg("Possibly NULL pointer passed to trusted R1") int marked_as_untrusted_or_null(struct __sk_buff *ctx) { struct map_value *v; @@ -217,7 +217,7 @@ int reject_kptr_xchg_on_unref(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("R1 type=rcu_ptr_or_null_ expected=percpu_ptr_") +__failure __msg("Possibly NULL pointer passed to trusted R1") int mark_ref_as_untrusted_or_null(struct __sk_buff *ctx) { struct map_value *v; @@ -252,7 +252,7 @@ int reject_untrusted_store_to_ref(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("release helper bpf_kptr_xchg expects referenced PTR_TO_BTF_ID passed to R2") +__failure __msg("release function bpf_kptr_xchg expects referenced PTR_TO_BTF_ID passed to R2") int reject_untrusted_xchg(struct __sk_buff *ctx) { struct prog_test_ref_kfunc *p; @@ -291,7 +291,7 @@ int reject_bad_type_xchg(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("invalid kptr access, R2 type=trusted_ptr_prog_test_ref_kfunc") +__failure __msg("R2 must have zero offset when passed to release func") int reject_member_of_ref_xchg(struct __sk_buff *ctx) { struct prog_test_ref_kfunc *ref_ptr; @@ -364,7 +364,7 @@ int kptr_xchg_ref_state(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("Possibly NULL pointer passed to helper R2") +__success int kptr_xchg_possibly_null(struct __sk_buff *ctx) { struct prog_test_ref_kfunc *p; diff --git a/tools/testing/selftests/bpf/progs/map_ptr_kern.c b/tools/testing/selftests/bpf/progs/map_ptr_kern.c index 373c8d17ea55..6bd4cb68c20c 100644 --- a/tools/testing/selftests/bpf/progs/map_ptr_kern.c +++ b/tools/testing/selftests/bpf/progs/map_ptr_kern.c @@ -114,7 +114,7 @@ static inline int check_hash(void) VERIFY(check_default_noinline(&hash->map, map)); VERIFY(hash->n_buckets == MAX_ENTRIES); - VERIFY(hash->elem_size == 64); + VERIFY(hash->elem_size == 32); VERIFY(hash->count.counter == 0); VERIFY(bpf_map_sum_elem_count(map) == 0); diff --git a/tools/testing/selftests/bpf/progs/may_goto_priv_stack.c b/tools/testing/selftests/bpf/progs/may_goto_priv_stack.c new file mode 100644 index 000000000000..5c2aa0aa5055 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/may_goto_priv_stack.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "../../../include/linux/filter.h" +#include "bpf_misc.h" + +/* r0-r5 after the loop and the number of iterations */ +__u64 regs_64[7]; +__u64 regs_128[7]; + +/* + * fentry prog with 64 or more bytes of stack uses private stack. + * Loop until may_goto expires. r0-r5 should stay zero. + */ +#define TIMED_MAY_GOTO_PRIV_STACK(size, func) \ +SEC("fentry/" #func) \ +__naked void priv_stack_##size(void) \ +{ \ + asm volatile ( \ + "r6 = 0;" \ + "*(u64 *)(r10 - " #size ") = r6;" \ + "r7 = %[regs] ll;" \ + "r0 = 0;" \ + "r1 = 0;" \ + "r2 = 0;" \ + "r3 = 0;" \ + "r4 = 0;" \ + "r5 = 0;" \ +"1:" \ + ".8byte %[may_goto];" \ + "r6 += 1;" \ + "goto 1b;" \ + "*(u64 *)(r7 + 0) = r0;" \ + "*(u64 *)(r7 + 8) = r1;" \ + "*(u64 *)(r7 + 16) = r2;" \ + "*(u64 *)(r7 + 24) = r3;" \ + "*(u64 *)(r7 + 32) = r4;" \ + "*(u64 *)(r7 + 40) = r5;" \ + "*(u64 *)(r7 + 48) = r6;" \ + "r0 = 0;" \ + "exit;" \ + : \ + : [regs]"i"(®s_##size), \ + __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 2, 0)) \ + : __clobber_all); \ +} + +TIMED_MAY_GOTO_PRIV_STACK(64, bpf_fentry_test1) +TIMED_MAY_GOTO_PRIV_STACK(128, bpf_fentry_test2) + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c b/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c index 3e0d4f687aaa..23019023511a 100644 --- a/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c +++ b/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c @@ -118,8 +118,7 @@ int atomic_rmw_not_ok(void *ctx) SEC("socket") __failure -__msg("invalid access to memory, mem_size=0 off=0 size=4") -__msg("R1 min value is outside of the allowed memory range") +__msg("R1 type=rdonly_untrusted_mem expected=fp") int kfunc_param_not_ok(void *ctx) { int *p; diff --git a/tools/testing/selftests/bpf/progs/memcg_proactive_reclaim.c b/tools/testing/selftests/bpf/progs/memcg_proactive_reclaim.c new file mode 100644 index 000000000000..b551192526fc --- /dev/null +++ b/tools/testing/selftests/bpf/progs/memcg_proactive_reclaim.c @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> + +struct reclaim_args { + __u64 cgroup_id; + __u64 size; +}; + +/* Signed, because bpf_test_run_opts.retval is a __u32. */ +__s64 reclaimed; + +SEC("syscall") +int memcg_proactive_reclaim(struct reclaim_args *ctx) +{ + struct mem_cgroup *memcg; + struct cgroup *cgrp; + + cgrp = bpf_cgroup_from_id(ctx->cgroup_id); + if (!cgrp) + return 0; + + memcg = bpf_get_mem_cgroup(&cgrp->self); + if (memcg) { + reclaimed = bpf_proactive_reclaim(memcg, ctx->size, -1); + bpf_put_mem_cgroup(memcg); + } + bpf_cgroup_release(cgrp); + + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c b/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c index 3701f4ea58c7..57615b0f0c25 100644 --- a/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c +++ b/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c @@ -24,6 +24,7 @@ struct val_600b_t { struct elem { long sum; struct val_t __percpu_kptr *pc; + struct val_t __percpu_kptr *pc2; }; struct { @@ -46,6 +47,8 @@ struct { struct task_struct *bpf_task_from_pid(s32 pid) __ksym; void bpf_task_release(struct task_struct *p) __ksym; +void bpf_rcu_read_lock(void) __ksym; +void bpf_rcu_read_unlock(void) __ksym; long ret; @@ -124,6 +127,38 @@ int BPF_PROG(test_array_map_3) } SEC("?fentry.s/bpf_fentry_test1") +__failure __msg("Possibly NULL pointer passed to trusted R1") +int BPF_PROG(reject_nullable_percpu_xchg_alias) +{ + struct val_t __percpu_kptr *p1, *p2, *old; + struct val_t *v; + struct elem *e; + int index = 0; + + e = bpf_map_lookup_elem(&array, &index); + if (!e) + return 0; + + p1 = bpf_percpu_obj_new(struct val_t); + p2 = bpf_percpu_obj_new(struct val_t); + + bpf_rcu_read_lock(); + old = bpf_kptr_xchg(&e->pc, p1); + if (old) + bpf_percpu_obj_drop(old); + old = bpf_kptr_xchg(&e->pc2, p2); + if (old) + bpf_percpu_obj_drop(old); + + if (p1) { + v = bpf_this_cpu_ptr(p2); + v->b = 1; + } + bpf_rcu_read_unlock(); + return 0; +} + +SEC("?fentry.s/bpf_fentry_test1") __failure __msg("R1 expected for bpf_percpu_obj_drop()") int BPF_PROG(test_array_map_4) { diff --git a/tools/testing/selftests/bpf/progs/preempt_lock.c b/tools/testing/selftests/bpf/progs/preempt_lock.c index 81c459435680..955391da326a 100644 --- a/tools/testing/selftests/bpf/progs/preempt_lock.c +++ b/tools/testing/selftests/bpf/progs/preempt_lock.c @@ -6,6 +6,8 @@ #include "bpf_experimental.h" extern int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void *unsafe_ptr__ign, u64 flags) __weak __ksym; +extern void bpf_rcu_read_lock(void) __ksym; +extern void bpf_rcu_read_unlock(void) __ksym; SEC("?tc") __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_preempt_disable-ed region") @@ -179,6 +181,30 @@ int preempt_sleepable_kfunc(void *ctx) return 0; } +SEC("?fentry/" SYS_PREFIX "sys_getpgid") +__failure __msg("program must be sleepable to call sleepable kfunc bpf_copy_from_user_str") +__msg("cannot be used in non-sleepable program") +int non_sleepable_kfunc(void *ctx) +{ + u32 data; + + bpf_copy_from_user_str(&data, sizeof(data), NULL, 0); + return 0; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__failure __msg("kernel func bpf_copy_from_user_str is sleepable within rcu_read_lock region") +__msg("cannot be used in RCU read lock region") +int sleepable_kfunc_in_rcu(void *ctx) +{ + u32 data; + + bpf_rcu_read_lock(); + bpf_copy_from_user_str(&data, sizeof(data), NULL, 0); + bpf_rcu_read_unlock(); + return 0; +} + int __noinline preempt_global_subprog(void) { preempt_balance_subprog(); diff --git a/tools/testing/selftests/bpf/progs/rbtree_fail.c b/tools/testing/selftests/bpf/progs/rbtree_fail.c index 4504608196ab..08709f23ec0f 100644 --- a/tools/testing/selftests/bpf/progs/rbtree_fail.c +++ b/tools/testing/selftests/bpf/progs/rbtree_fail.c @@ -180,7 +180,7 @@ err_out: } SEC("?tc") -__failure __msg("bpf_rbtree_remove can only take non-owning or refcounted bpf_rb_node pointer") +__failure __msg("R2 type=scalar expected=ptr_, rcu_ptr_, ptr_, rcu_ptr_") long rbtree_api_add_release_unlock_escape(void *ctx) { struct node_data *n; @@ -204,7 +204,7 @@ long rbtree_api_add_release_unlock_escape(void *ctx) } SEC("?tc") -__failure __msg("bpf_rbtree_remove can only take non-owning or refcounted bpf_rb_node pointer") +__failure __msg("R2 type=scalar expected=ptr_, rcu_ptr_, ptr_, rcu_ptr_") long rbtree_api_first_release_unlock_escape(void *ctx) { struct bpf_rb_node *res; diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c index 338e43822ffe..e80f78fae227 100644 --- a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c @@ -118,8 +118,8 @@ long refcount_acquire_maybe_null(void *ctx) } SEC("?tc") -__failure __msg("R1 is neither owning or non-owning ref") -__msg("expects a pointer to a BPF-managed refcounted object, but R1 is a context pointer") +__failure __msg("R1 type=ctx expected=ptr_, rcu_ptr_, ptr_, rcu_ptr_") +__msg("type ctx, but this argument accepts ptr_, rcu_ptr_, ptr_, rcu_ptr_") long refcount_acquire_non_object(void *ctx) { return bpf_refcount_acquire(ctx) != NULL; @@ -159,8 +159,7 @@ long refcount_acquire_rcu_map_kptr_unchecked_drop(void *ctx) SEC("?syscall") __failure -__msg("bpf_rbtree_remove can only take non-owning or refcounted " - "bpf_rb_node pointer") +__msg("R2 type=untrusted_ptr_ expected=ptr_, rcu_ptr_, ptr_, rcu_ptr_") long rbtree_remove_after_rcu_unlock(void *ctx) { struct map_value_rcu_graph *mapval; @@ -190,7 +189,7 @@ long rbtree_remove_after_rcu_unlock(void *ctx) } SEC("?syscall") -__failure __msg("R1 is neither owning or non-owning ref") +__failure __msg("R1 type=untrusted_ptr_ expected=ptr_, rcu_ptr_, ptr_, rcu_ptr_") long refcount_acquire_after_rcu_unlock(void *ctx) { struct map_value_refcount_only *mapval; diff --git a/tools/testing/selftests/bpf/progs/res_spin_lock_fail.c b/tools/testing/selftests/bpf/progs/res_spin_lock_fail.c index 330682a88c16..8fd591bd1f6c 100644 --- a/tools/testing/selftests/bpf/progs/res_spin_lock_fail.c +++ b/tools/testing/selftests/bpf/progs/res_spin_lock_fail.c @@ -24,7 +24,7 @@ struct bpf_spin_lock lock __hidden SEC(".data.A"); struct bpf_res_spin_lock res_lock __hidden SEC(".data.B"); SEC("?tc") -__failure __msg("point to map value or allocated object") +__failure __msg("R1 type=untrusted_ptr_ expected=map_value, ptr_") int res_spin_lock_arg(struct __sk_buff *ctx) { struct arr_elem *elem; diff --git a/tools/testing/selftests/bpf/progs/setget_sockopt.c b/tools/testing/selftests/bpf/progs/setget_sockopt.c index d96e99b67aeb..95e6a337ab71 100644 --- a/tools/testing/selftests/bpf/progs/setget_sockopt.c +++ b/tools/testing/selftests/bpf/progs/setget_sockopt.c @@ -61,8 +61,8 @@ static const struct sockopt_test sol_tcp_tests[] = { { .opt = TCP_NOTSENT_LOWAT, .new = 1314, .expected = 1314, }, { .opt = TCP_BPF_SOCK_OPS_CB_FLAGS, .new = BPF_SOCK_OPS_ALL_CB_FLAGS, .expected = BPF_SOCK_OPS_ALL_CB_FLAGS, }, - { .opt = TCP_BPF_DELACK_MAX, .new = 30000, .expected = 30000, }, - { .opt = TCP_BPF_RTO_MIN, .new = 30000, .expected = 30000, }, + { .opt = TCP_BPF_DELACK_MAX, .new = 40000, .expected = 40000, }, + { .opt = TCP_BPF_RTO_MIN, .new = 40000, .expected = 40000, }, { .opt = TCP_RTO_MAX_MS, .new = 2000, .expected = 2000, }, { .opt = 0, }, }; diff --git a/tools/testing/selftests/bpf/progs/skb_csum_complete.c b/tools/testing/selftests/bpf/progs/skb_csum_complete.c new file mode 100644 index 000000000000..644731a12dc8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/skb_csum_complete.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +SEC("tc") +int change_tail_trim(struct __sk_buff *skb) +{ + /* + * Drops CHECKSUM_COMPLETE. skb->csum still covers the trimmed bytes, + * which hold the non-zero tcp.urg_ptr of pkt_v4, so it no longer + * matches the packet. + */ + if (bpf_skb_change_tail(skb, skb->len - 4, 0)) + return 1; + return 0; +} + +SEC("tc") +int change_tail_grow_write(struct __sk_buff *skb) +{ + __u8 val = 0xab; + + /* + * Drops CHECKSUM_COMPLETE. The new bytes are zero and do not change + * the sum, but the write below can no longer update skb->csum. + */ + if (bpf_skb_change_tail(skb, skb->len + 4, 0)) + return 1; + if (bpf_skb_store_bytes(skb, skb->len - 1, &val, sizeof(val), + BPF_F_RECOMPUTE_CSUM)) + return 2; + return 0; +} + +SEC("tc") +int store_no_recompute(struct __sk_buff *skb) +{ + __u8 val = 0xab; + + /* skb stays CHECKSUM_COMPLETE, but skb->csum is not updated. */ + if (bpf_skb_store_bytes(skb, skb->len - 1, &val, sizeof(val), 0)) + return 1; + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/stack_arg.c b/tools/testing/selftests/bpf/progs/stack_arg.c index 944e3bb603e7..3e9ed37c57bb 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg.c +++ b/tools/testing/selftests/bpf/progs/stack_arg.c @@ -21,7 +21,8 @@ struct { int timer_result; -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) const volatile bool has_stack_arg = true; diff --git a/tools/testing/selftests/bpf/progs/stack_arg_fail.c b/tools/testing/selftests/bpf/progs/stack_arg_fail.c index ad9d4bfe15dc..fff2e947ea33 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_fail.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_fail.c @@ -3,20 +3,10 @@ #include <vmlinux.h> #include <bpf/bpf_helpers.h> -#include "../test_kmods/bpf_testmod_kfunc.h" #include "bpf_misc.h" #if defined(__BPF_FEATURE_STACK_ARGUMENT) -SEC("tc") -__failure __msg("Unrecognized *(R11-8) type STRUCT") -int test_stack_arg_big(struct __sk_buff *skb) -{ - struct prog_test_big_arg s = { .a = 1, .b = 2 }; - - return bpf_kfunc_call_stack_arg_big(1, 2, 3, 4, 5, s); -} - SEC("socket") __description("r11 in ALU instruction") __failure __msg("R11 is invalid") @@ -103,8 +93,9 @@ __naked void r11_store_zero_off(void) SEC("tc") __description("stack_arg_fail: not supported, dummy test") +__skip("stack_arg_fail: not supported") __success -int test_stack_arg_big(struct __sk_buff *skb) +int dummy_test(struct __sk_buff *skb) { return 0; } diff --git a/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c b/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c index 345f2da2e361..b6b1a8aa7f44 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c @@ -6,7 +6,8 @@ #include "bpf_kfuncs.h" #include "../test_kmods/bpf_testmod_kfunc.h" -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) const volatile bool has_stack_arg = true; diff --git a/tools/testing/selftests/bpf/progs/stack_arg_precision.c b/tools/testing/selftests/bpf/progs/stack_arg_precision.c index bee2eeec021d..c9564776d733 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_precision.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_precision.c @@ -6,7 +6,8 @@ #include "../test_kmods/bpf_testmod_kfunc.h" #include "bpf_misc.h" -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) /* Force kfunc extern BTF generation for inline asm call below. @@ -73,7 +74,7 @@ __msg("mark_precise: frame1: parent state regs= stack=: frame1: R10=fp0") __msg("mark_precise: frame0: parent state regs= stack=: R10=fp0") __msg("mark_precise: frame1: last_idx 11 first_idx 11 subseq_idx 13") __msg("mark_precise: frame1: regs= stack= before 11: (85) call pc+1") -__msg("mark_precise: frame0: parent state regs= stack=: R1=1 R2=2 R3=3 R4=4 R5=5 R10=fp0") +__msg("mark_precise: frame0: parent state regs= stack=: R10=fp0") __msg("mark_precise: frame0: last_idx 9 first_idx 7 subseq_idx 11") __msg("mark_precise: frame0: regs= stack= before 9: (05) goto pc+1") __msg("mark_precise: frame0: regs= stack= before 8: (7a) *(u64 *)(r11 -8) = 4") @@ -94,7 +95,7 @@ __msg("mark_precise: frame1: parent state regs= stack=: frame1: R10=fp0") __msg("mark_precise: frame0: parent state regs= stack=: R10=fp0") __msg("mark_precise: frame1: last_idx 11 first_idx 11 subseq_idx 13 ") __msg("mark_precise: frame1: regs= stack= before 11: (85) call pc+1") -__msg("mark_precise: frame0: parent state regs= stack=: R1=1 R2=2 R3=3 R4=4 R5=5 R10=fp0") +__msg("mark_precise: frame0: parent state regs= stack=: R10=fp0") __msg("mark_precise: frame0: last_idx 10 first_idx 10 subseq_idx 11 ") __msg("mark_precise: frame0: regs= stack= before 10: (7a) *(u64 *)(r11 -8) = 6") __naked void stack_arg_precision_bpf2bpf(void) @@ -124,6 +125,7 @@ __naked void stack_arg_precision_bpf2bpf(void) SEC("socket") __description("stack_arg_precision: not supported, dummy test") +__skip("stack_arg_precision: not supported") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/stream.c b/tools/testing/selftests/bpf/progs/stream.c index 8e8e1339dc74..1ae015c4e6b4 100644 --- a/tools/testing/selftests/bpf/progs/stream.c +++ b/tools/testing/selftests/bpf/progs/stream.c @@ -36,9 +36,15 @@ struct { } array SEC(".maps"); #define ENOSPC 28 +#define E2BIG 7 #define _STR "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +#define _X64 "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +/* 1024 bytes: truncated by bstr_printf, must return -E2BIG. */ +#define _BIG_STR (_X64 _X64 _X64 _X64 _X64 _X64 _X64 _X64 \ + _X64 _X64 _X64 _X64 _X64 _X64 _X64 _X64) int size; +int nmi_stream_prints; u64 fault_addr; void *arena_ptr; @@ -120,6 +126,48 @@ int stream_syscall(void *ctx) } SEC("syscall") +__success __retval(0) +int stream_empty(void *ctx) +{ + return bpf_stream_printk(BPF_STDOUT, ""); +} + +SEC("perf_event") +int stream_nmi(void *ctx) +{ + if (nmi_stream_prints) + return 0; + /* Retry on a later sample if the write failed, e.g. with -EBUSY. */ + if (bpf_stream_printk(BPF_STDOUT, "nmi")) + return 0; + nmi_stream_prints = 1; + return 0; +} + +SEC("syscall") +__success __retval(0) +int stream_oversize(void *ctx) +{ + int ret; + + ret = bpf_stream_printk(BPF_STDOUT, _BIG_STR); + if (ret != -E2BIG) + return ret ?: 1; + + /* The oversized output must not reduce the remaining stream capacity. */ + size = 0; + bpf_repeat(BPF_MAX_LOOPS) { + ret = bpf_stream_printk(BPF_STDOUT, _STR); + if (ret == -ENOSPC) + return size == 99954 ? 0 : 1; + if (ret) + return ret; + size += sizeof(_STR) - 1; + } + return 1; +} + +SEC("syscall") __arch_x86_64 __arch_arm64 __success __retval(0) diff --git a/tools/testing/selftests/bpf/progs/stream_fail.c b/tools/testing/selftests/bpf/progs/stream_fail.c index 21428bb1ee59..10ebb4a7f105 100644 --- a/tools/testing/selftests/bpf/progs/stream_fail.c +++ b/tools/testing/selftests/bpf/progs/stream_fail.c @@ -23,7 +23,7 @@ int stream_vprintk_scalar_arg(void *ctx) } SEC("syscall") -__failure __msg("R2 doesn't point to a const string") +__failure __msg("R2 type=ctx expected=map_value") int stream_vprintk_string_arg(void *ctx) { bpf_stream_vprintk(BPF_STDOUT, ctx, NULL, 0); diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_autobind.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_autobind.c new file mode 100644 index 000000000000..495faa0f4a8b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_autobind.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_experimental.h" +#include <bpf_arena_common.h> +#include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * Neither the callback nor the unrelated program refers to the arena. libbpf + * binds it to them all the same, so the callback can use the __arena argument + * the kernel hands it. + */ +SEC("struct_ops/test_arena") +int test_arena_cb(unsigned long long *ctx) +{ + u64 __arena *ptr = (u64 __arena *)ctx[0]; + + *ptr += 1; + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_autobind = { + .test_arena = (void *)test_arena_cb, +}; + +SEC("syscall") +int unrelated(void *ctx) +{ + return 0; +} + +SEC("syscall") +int trigger(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + if (bpf_testmod_ops3_call_test_arena((u64 *)val)) + return 2; + if (*val != 42) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member.c new file mode 100644 index 000000000000..a2218cd81619 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_experimental.h" +#include <bpf_arena_common.h> +#include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + /* page 0 hosts the arena globals, page 1 is for allocations */ + __uint(max_entries, 2); +} arena SEC(".maps"); + +/* also associates the callback with the arena */ +u64 __arena arena_touch; +/* the register value a load of args->ptr produced in the callback */ +u64 __arena member_ptr_val; + +/* + * The header is shared with the module, so the members are u64 __arena * on + * this side too and the loads need no cast. The verifier derives the type of + * the loaded value from the module BTF, where the members carry the tag. + */ +SEC("struct_ops/test_arena_member") +int BPF_PROG(test_arena_member_cb, struct bpf_testmod_arena_member_args *args) +{ + u64 __arena *ptr = args->ptr, *acq; + u64 raw; + + arena_touch++; + + /* the register right after the load, untouched by the compiler */ + asm volatile("%[raw] = *(u64 *)(%[args] + %[off])" + : [raw]"=r"(raw) + : [args]"r"(args), + [off]"i"(offsetof(struct bpf_testmod_arena_member_args, ptr))); + member_ptr_val = raw; + + *ptr += 1; + /* an acquire load of the member yields the same arena pointer */ + asm volatile("r1 = %[args];\n" + ".8byte 0x00000100000010db;\n" /* r0 = load_acquire((u64 *)(r1 + 0)) */ + "%[acq] = r0;" + : [acq]"=r"(acq) + : [args]"r"(args) + : "r0", "r1"); + if (acq != ptr) + return 0xbad; + *acq += 1; + if (args->null_ptr) + return 0xbad; + *args->nested.ptr += 1; + if (args->cookie != 0xc00c1e) + return 0xbad; + /* + * @faulting is a plain kernel pointer, so the member behind it is read + * with a probed load, which the JIT skips for a user address or which + * faults, and reads as NULL rather than as an arena pointer. + */ + if (args->faulting->ptr) + return 0xbad; + /* an arena pointer to kfuncs as well, the kfunc adds one */ + bpf_kfunc_arena_arg_test((u64 *)args->ptr); + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member = { + .test_arena_member = (void *)test_arena_member_cb, +}; + +SEC("syscall") +int trigger(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + int ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + ret = bpf_testmod_ops3_call_test_arena_member((u64 *)val); + if (ret) + return 2; + /* ptr, its acquire load, nested.ptr and the kfunc each added one */ + if (*val != 45) + return 3; + + /* + * The callback saw (u32)(kaddr - kern_vm_start), the arena offset of + * val with the upper 32 bits clear, the same as an __arena argument. + */ + if (member_ptr_val != (u32)(u64)val) + return 4; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch.c new file mode 100644 index 000000000000..0b4147894122 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf_arena_common.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * The same load reaches args->ptr on one path and args->cookie on the other, + * the branch on the unknown cookie value makes the verifier walk both. The + * rewrite is per insn, so the paths have to agree on what they load. + */ +SEC("struct_ops/test_arena_member") +__failure __msg("loads an arena pointer member on one path and something else on another") +__naked int mismatch_member(void) +{ + asm volatile ( + "r6 = *(u64 *)(r1 + 0);" + "r2 = %[arena] ll;" + "r0 = *(u64 *)(r6 + %[cookie_off]);" + "if r0 == 0 goto 1f;" + "r6 += %[cookie_off];" +"1: r0 = *(u64 *)(r6 + 0);" + "r0 = 0;" + "exit;" + : + : __imm_addr(arena), + __imm_const(cookie_off, offsetof(struct bpf_testmod_arena_member_args, cookie)) + : __clobber_all); +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)mismatch_member, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch_mem.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch_mem.c new file mode 100644 index 000000000000..ff6d4db47cfd --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch_mem.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf_arena_common.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * The same load reads args->ptr on one path and the memory argument on the + * other. The verifier lets a load insn serve a kernel struct pointer and a + * memory pointer, see save_aux_ptr_type(), but the rewrite of the arena + * member would corrupt the value the other path loads. A global function + * is the only place a struct_ops program of this type gets a memory pointer. + * The loaded value goes nowhere: an ALU insn on it would be an arena + * pointer on one path and a scalar on the other, which the verifier rejects + * on its own before the fixup pass gets to the load. + */ +__noinline int load_either(unsigned long long *ctx __arg_ctx, u64 *mem __arg_nonnull) +{ + struct bpf_testmod_arena_member_args *args = (void *)ctx[0]; + void *p = args; + + if (args->cookie == 0) + p = mem; + /* one load insn, reached with both pointers */ + asm volatile("r0 = *(u64 *)(%[p] + 0)" :: [p]"r"(p) : "r0"); + return 0; +} + +SEC("struct_ops/test_arena_member") +__failure __msg("loads an arena pointer member on one path and something else on another") +int BPF_PROG(mismatch_mem, struct bpf_testmod_arena_member_args *args) +{ + u64 local = 0; + + asm volatile("" :: "r"(&arena)); + return load_either(ctx, &local); +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)mismatch_mem, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__no_arena.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__no_arena.c new file mode 100644 index 000000000000..cdd8ea740478 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__no_arena.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* No arena in the object: the member cannot be translated into anything. */ +SEC("struct_ops/test_arena_member") +__failure __msg("arena pointer member ptr of struct bpf_testmod_arena_member_args " + "requires a program with an associated arena") +int BPF_PROG(no_arena_member, struct bpf_testmod_arena_member_args *args) +{ + return args->ptr != NULL; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)no_arena_member, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__union.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__union.c new file mode 100644 index 000000000000..d45666fd482c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__union.c @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf_arena_common.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * An offset into a union resolves to its first member, so a program reading + * raw there would get an arena pointer it did not ask for. + */ +SEC("struct_ops/test_arena_member") +__failure __msg("arena pointer member ptr of bpf_testmod_arena_union is inside a union") +int BPF_PROG(union_member, struct bpf_testmod_arena_member_args *args) +{ + asm volatile("" :: "r"(&arena)); + return *args->u.ptr != 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)union_member, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__write.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__write.c new file mode 100644 index 000000000000..ff0761980948 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__write.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf_arena_common.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * The kernel owns the member, a program only reads it. A struct_ops program + * cannot store through its arguments at all, so the store is refused before + * the tag comes into play; the exchange test reaches the arena member check. + */ +SEC("struct_ops/test_arena_member") +__failure __msg("only read is supported") +int BPF_PROG(write_member, struct bpf_testmod_arena_member_args *args) +{ + asm volatile("" :: "r"(&arena)); + args->ptr = NULL; + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)write_member, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__xchg.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__xchg.c new file mode 100644 index 000000000000..0d814166cabc --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__xchg.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf_arena_common.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* An atomic that writes the member is neither a plain nor an acquire load. */ +SEC("struct_ops/test_arena_member") +__failure __msg("arena pointer member ptr of struct " + "bpf_testmod_arena_member_args can only be read with a plain or an acquire load") +__naked int xchg_member(void) +{ + asm volatile ( + "r1 = *(u64 *)(r1 + 0);" + "r2 = %[arena] ll;" + "r0 = 0;" + /* r0 = xchg_64((u64 *)(r1 + 0), r0): BPF_STX | BPF_ATOMIC | BPF_DW, imm BPF_XCHG */ + ".8byte 0x000000e1000001db;" + "r0 = 0;" + "exit;" + : + : __imm_addr(arena) + : __clobber_all); +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)xchg_member, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c new file mode 100644 index 000000000000..09b4c740bdca --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_experimental.h" +#include <bpf_arena_common.h> + +char _license[] SEC("license") = "GPL"; + +/* + * The module types as a vmlinux.h generated by a libbpf that does not render + * the arena tag with its address space shows them: an attribute on a pointer + * in the kernel address space, so the program has to cast the member to the + * arena address space before using it. + */ +#define __arena_tag __attribute__((btf_type_tag("arena"))) +#define __nullable_tag __attribute__((btf_type_tag("nullable"))) + +struct bpf_testmod_arena_member_args { + __u64 __arena_tag *ptr; + __u64 __arena_tag __nullable_tag *null_ptr; + struct { + __u64 __arena_tag *ptr; + } nested; +} __attribute__((preserve_access_index)); + +struct bpf_testmod_ops3 { + int (*test_arena_member)(struct bpf_testmod_arena_member_args *args); +}; + +int bpf_testmod_ops3_call_test_arena_member(__u64 *ptr__arena) __ksym; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 2); +} arena SEC(".maps"); + +/* also associates the callback with the arena */ +u64 __arena arena_touch; + +SEC("struct_ops/test_arena_member") +int BPF_PROG(test_arena_member_render_cb, struct bpf_testmod_arena_member_args *args) +{ + u64 __arena *ptr = (u64 __arena *)args->ptr; + + arena_touch++; + *ptr += 1; + if (args->null_ptr) + return 0xbad; + *(u64 __arena *)args->nested.ptr += 1; + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_render = { + .test_arena_member = (void *)test_arena_member_render_cb, +}; + +SEC("syscall") +int trigger(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + int ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + ret = bpf_testmod_ops3_call_test_arena_member((u64 *)val); + if (ret) + return 2; + /* ptr and nested.ptr each added one */ + if (*val != 43) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/struct_ops_multi_args.c b/tools/testing/selftests/bpf/progs/struct_ops_multi_args.c index c62be15757f0..9d290f22b492 100644 --- a/tools/testing/selftests/bpf/progs/struct_ops_multi_args.c +++ b/tools/testing/selftests/bpf/progs/struct_ops_multi_args.c @@ -17,7 +17,6 @@ struct { } prog_array SEC(".maps"); SEC("struct_ops/test_refcounted_multi") -__failure __msg("program with __ref argument cannot tail call") int test_refcounted_multi(unsigned long long *ctx) { /* ctx[2] is used because the refcounted variable is the third argument */ @@ -29,7 +28,20 @@ int test_refcounted_multi(unsigned long long *ctx) return 0; } +__u64 trampoline_stack_arg9 = 0; + +SEC("struct_ops/test_trampoline_stack_args") +int BPF_PROG(test_trampoline_stack_args, int arg1, int arg2, int arg3, + int arg4, int arg5, int arg6, + int arg7, int arg8, int arg9) +{ + trampoline_stack_arg9 = arg9; + + return 0; +} + SEC(".struct_ops.link") struct bpf_testmod_ops testmod_ref_acquire = { .test_refcounted_multi = (void *)test_refcounted_multi, + .test_trampoline_stack_args = (void *)test_trampoline_stack_args, }; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable.c new file mode 100644 index 000000000000..929969e1e012 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +char _license[] SEC("license") = "GPL"; + +/* an untrusted pointer to the argument struct, never set, for the verifier */ +struct untrusted_val { + struct bpf_testmod_nullable_args __kptr_untrusted *args; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct untrusted_val); +} untrusted_map SEC(".maps"); + +SEC("struct_ops/test_nullable") +int BPF_PROG(test_nullable_cb, struct bpf_testmod_nullable_args *args) +{ + struct bpf_testmod_nullable_args *u; + struct untrusted_val *v; + struct task_struct *t; + int ret = 0, *i, zero = 0; + + /* trusted, so good for a kfunc as is */ + t = bpf_task_acquire(args->task); + if (!t) + return -1; + bpf_task_release(t); + + /* trusted but __nullable, so a check has to come first */ + if (args->task_or_null) { + t = bpf_task_acquire(args->task_or_null); + if (!t) + return -1; + bpf_task_release(t); + ret = 1; + } + /* per-CPU and __nullable: a check has to come first, per_cpu_ptr() has none */ + if (args->pcpu_or_null) { + struct bpf_testmod_pcpu *v = bpf_this_cpu_ptr(args->pcpu_or_null); + + if (v->val != 0x5ca1ab1e) + return -1; + } + /* through a plain pointer the loads are probed, __nullable still asks for a check */ + t = args->self->task_or_null; + if (t && t->pid != args->task->pid) + return -1; + /* a pointer to a scalar walks as untrusted memory, __nullable applies all the same */ + i = args->int_or_null; + if (i && *i != 7) + return -1; + /* through an untrusted pointer the member is probed too, and stays nullable */ + v = bpf_map_lookup_elem(&untrusted_map, &zero); + u = v ? v->args : NULL; + if (u) { + t = u->task_or_null; + if (t && t->pid != args->task->pid) + return -1; + } + return ret; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable = { + .test_nullable = (void *)test_nullable_cb, +}; + +SEC("syscall") +int trigger(void *ctx) +{ + /* one call without task_or_null, one with */ + return bpf_testmod_ops3_call_test_nullable() != 2; +} diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail.c new file mode 100644 index 000000000000..c15b3a8c8d39 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* The member is trusted, but __nullable, so it needs a check before the kfunc. */ +SEC("struct_ops/test_nullable") +__failure __msg("Possibly NULL pointer passed to trusted") +int BPF_PROG(unchecked_nullable, struct bpf_testmod_nullable_args *args) +{ + struct task_struct *t; + + t = bpf_task_acquire(args->task_or_null); + if (t) + bpf_task_release(t); + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_fail = { + .test_nullable = (void *)unchecked_nullable, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail__percpu.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail__percpu.c new file mode 100644 index 000000000000..652015189162 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail__percpu.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* + * The member is per-CPU and __nullable: per_cpu_ptr() shifts NULL like any + * other pointer, so a check has to come before the helper. + */ +SEC("struct_ops/test_nullable") +__failure __msg("Possibly NULL pointer passed to trusted R1") +int BPF_PROG(unchecked_percpu, struct bpf_testmod_nullable_args *args) +{ + struct bpf_testmod_pcpu *v = bpf_this_cpu_ptr(args->pcpu_or_null); + + return v->val; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_fail = { + .test_nullable = (void *)unchecked_percpu, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__plain.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__plain.c new file mode 100644 index 000000000000..c45f6d34f7d2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__plain.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* + * Through a plain pointer the member is probed, which tolerates NULL, so the + * tag asks for no check there, as the BTF_TYPE_SAFE_*_OR_NULL lists never did. + */ +SEC("struct_ops/test_nullable") +__success +int BPF_PROG(unchecked_plain, struct bpf_testmod_nullable_args *args) +{ + return args->self->task_or_null->pid; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_probed = { + .test_nullable = (void *)unchecked_plain, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__scalar.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__scalar.c new file mode 100644 index 000000000000..ff768746b0da --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__scalar.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* + * A pointer to a scalar walks as untrusted memory, which is probed, so the + * tag asks for no check there. + */ +SEC("struct_ops/test_nullable") +__success +int BPF_PROG(unchecked_scalar, struct bpf_testmod_nullable_args *args) +{ + return *args->int_or_null; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_probed = { + .test_nullable = (void *)unchecked_scalar, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__untrusted.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__untrusted.c new file mode 100644 index 000000000000..961999139841 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__untrusted.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct untrusted_val { + struct bpf_testmod_nullable_args __kptr_untrusted *args; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct untrusted_val); +} untrusted_map SEC(".maps"); + +/* + * Through an untrusted pointer the member is probed, which tolerates NULL, so + * the tag asks for no check there. + */ +SEC("struct_ops/test_nullable") +__success +int BPF_PROG(unchecked_untrusted, struct bpf_testmod_nullable_args *args) +{ + struct bpf_testmod_nullable_args *u; + struct untrusted_val *v; + int zero = 0; + + v = bpf_map_lookup_elem(&untrusted_map, &zero); + u = v ? v->args : NULL; + if (!u) + return 0; + return u->task_or_null->pid; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_probed = { + .test_nullable = (void *)unchecked_untrusted, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c index 1442728f5604..c8cb35b37867 100644 --- a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c +++ b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c @@ -4,6 +4,7 @@ #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> #include "../test_kmods/bpf_testmod.h" +#include "bpf_misc.h" char _license[] SEC("license") = "GPL"; @@ -25,6 +26,44 @@ __noinline static int subprog1(int *a) return subprog2(a, b); } +/* + * A chain of 480-byte frames under test_2, so that its call chain exceeds + * the 2 KiB budget of JITs with large stacks as well as the 512 bytes + * allowed elsewhere. The compiler caps a single function at 512 bytes, and + * the buffers are volatile so that it cannot shrink them. + */ +__noinline static int subprog_deep4(int *a) +{ + volatile char b[480] = {}; + + __sink(b[479]); + return a[10] + b[20]; +} + +__noinline static int subprog_deep3(int *a) +{ + volatile char b[480] = {}; + + __sink(b[479]); + return subprog_deep4(a) + b[20]; +} + +__noinline static int subprog_deep2(int *a) +{ + volatile char b[480] = {}; + + __sink(b[479]); + return subprog_deep3(a) + b[20]; +} + +__noinline static int subprog_deep1(int *a) +{ + volatile char b[480] = {}; + + __sink(b[479]); + return subprog_deep2(a) + b[20]; +} + SEC("struct_ops") int BPF_PROG(test_1) @@ -41,11 +80,13 @@ int BPF_PROG(test_1) SEC("struct_ops") int BPF_PROG(test_2) { - /* stack size 400 bytes */ - int a[100] = {}; + /* stack size 476 bytes, over 2 KiB with the four 480-byte deep subprogs */ + volatile char buf[376] = {}; + int a[25] = {}; + __sink(buf[375]); a[10] = 3; - val_j = subprog1(a); + val_j = subprog1(a) + subprog_deep1(a); return 0; } diff --git a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_large.c b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_large.c new file mode 100644 index 000000000000..94a25a2cff6e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_large.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "../test_kmods/bpf_testmod.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +long val; + +/* On a private stack every frame gets the whole 2 KiB budget. */ +__used __naked +static long frame_2048_leaf(void) +{ + asm volatile (" \ + r1 = 30; \ + *(u64 *)(r10 - 2048) = r1; \ + r1 = 12; \ + *(u64 *)(r10 - 8) = r1; \ + r0 = *(u64 *)(r10 - 2048); \ + r1 = *(u64 *)(r10 - 8); \ + r0 += r1; \ + exit; \ +" ::: __clobber_all); +} + +/* test_1 is the member bpf_testmod requests a private stack for */ +SEC("struct_ops") +__naked int test_1(void) +{ + asm volatile (" \ + r1 = 100; \ + *(u64 *)(r10 - 2048) = r1; \ + call frame_2048_leaf; \ + r1 = *(u64 *)(r10 - 2048); \ + r0 += r1; \ + r1 = %[val] ll; \ + *(u64 *)(r1 + 0) = r0; \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(val) + : __clobber_all); +} + +SEC(".struct_ops") +struct bpf_testmod_ops3 testmod_1 = { + .test_1 = (void *)test_1, +}; diff --git a/tools/testing/selftests/bpf/progs/tailcall_freplace_multi.c b/tools/testing/selftests/bpf/progs/tailcall_freplace_multi.c new file mode 100644 index 000000000000..ca764411dc0b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tailcall_freplace_multi.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 KylinSoft Corporation. */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +SEC("freplace") +int subprog_freplace(struct __sk_buff *skb) +{ + bpf_tail_call_static(skb, &jmp_table, 0); + return 0; +} + +SEC("freplace") +int entry_freplace(struct __sk_buff *skb) +{ + return 0; +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tailcall_large_stack.c b/tools/testing/selftests/bpf/progs/tailcall_large_stack.c new file mode 100644 index 000000000000..977197dac5d3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tailcall_large_stack.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +/* The tail call target sets up a 2 KiB frame of its own and uses both of its ends. */ +SEC("tc") +__naked int classifier_0(void) +{ + asm volatile (" \ + r1 = 42; \ + *(u64 *)(r10 - 2048) = r1; \ + r1 = 7; \ + *(u64 *)(r10 - 8) = r1; \ + r0 = *(u64 *)(r10 - 2048); \ + r1 = *(u64 *)(r10 - 8); \ + r0 += r1; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The frame of the subprog doing the tail call is unwound by it, so it may be + * large; only the frames of its callers stay behind and are limited to 256 + * bytes in total. Returns 1 when the tail call falls through. + */ +__used __naked +static int subprog_tail(void) +{ + asm volatile (" \ + r2 = 1; \ + *(u64 *)(r10 - 1536) = r2; \ + r2 = %[jmp_table] ll; \ + r3 = 0; \ + call %[bpf_tail_call]; \ + r0 = 1; \ + exit; \ +" : + : __imm(bpf_tail_call), + __imm_addr(jmp_table) + : __clobber_all); +} + +SEC("tc") +__naked int entry(void) +{ + asm volatile (" \ + r2 = 2; \ + *(u64 *)(r10 - 240) = r2; \ + call subprog_tail; \ + exit; \ +" ::: __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c index f96b0c13ed1a..12c8ac6099ca 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c @@ -50,7 +50,7 @@ int BPF_PROG(task_kfunc_acquire_untrusted, struct task_struct *task, u64 clone_f } SEC("tp_btf/task_newtask") -__failure __msg("R1 is fp expected STRUCT task_struct") +__failure __msg("R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_") int BPF_PROG(task_kfunc_acquire_fp, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired, *stack_task = (struct task_struct *)&clone_flags; @@ -179,7 +179,7 @@ int BPF_PROG(task_kfunc_release_untrusted, struct task_struct *task, u64 clone_f } SEC("tp_btf/task_newtask") -__failure __msg("release kfunc bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_") int BPF_PROG(task_kfunc_release_fp, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired = (struct task_struct *)&clone_flags; @@ -225,7 +225,7 @@ int BPF_PROG(task_kfunc_release_null, struct task_struct *task, u64 clone_flags) } SEC("tp_btf/task_newtask") -__failure __msg("release kfunc bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(task_kfunc_release_unacquired, struct task_struct *task, u64 clone_flags) { /* Cannot release trusted task pointer which was not acquired. */ @@ -333,7 +333,7 @@ int BPF_PROG(task_access_comm2, struct task_struct *task, u64 clone_flags) } SEC("tp_btf/task_newtask") -__failure __msg("write into memory") +__failure __msg("only read is supported") int BPF_PROG(task_access_comm3, struct task_struct *task, u64 clone_flags) { bpf_probe_read_kernel(task->comm, 16, task->comm); @@ -353,7 +353,7 @@ int BPF_PROG(task_access_comm4, struct task_struct *task, const char *buf, bool } SEC("tp_btf/task_newtask") -__failure __msg("release kfunc bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(task_kfunc_release_in_map, struct task_struct *task, u64 clone_flags) { struct task_struct *local; diff --git a/tools/testing/selftests/bpf/progs/task_work_busy.c b/tools/testing/selftests/bpf/progs/task_work_busy.c new file mode 100644 index 000000000000..9960011edadc --- /dev/null +++ b/tools/testing/selftests/bpf/progs/task_work_busy.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 KylinSoft Corporation. */ +#include <vmlinux.h> +#include <stdbool.h> +#include <errno.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_core_read.h> + +char _license[] SEC("license") = "GPL"; + +extern bool CONFIG_PREEMPT_RT __kconfig __weak; + +/* + * Bounds for the hardirq busy-wait that holds the scheduling handler + * inside the real window: task_work_add() has published the callback, + * but the handler's tail state transition has not run yet. + * irq_work_single() clears BUSY only after the handler returns, so + * while this wait runs, the context's scheduling irq_work is BUSY, + * even though the first callback may already have completed and + * returned the context to STANDBY. + * + * The wait is bounded both by a bpf_ktime_get_ns() check and by an + * iteration budget. The time bound is checked against a monotonic + * clock: host vCPU preemption can stretch the wall-clock duration of + * the window, which only makes the window longer, never shorter. + */ +#define BUSY_WAIT_NS 4000000ULL +#define BUSY_WAIT_ITERS 1000000ULL + +int trigger_tid; +int target_tid; + +/* + * High 32 bits of the result slots hold the sequence, low 32 bits the + * signed error. BUSY_WINDOW low bits: 1 open, 2 the in-window attempt + * was seen before the handler returned, 3 the wait bound was reached + * without it. Must match the test. + */ +enum task_work_busy_status { + BUSY_ARM_SEQ, + BUSY_WINDOW, + BUSY_ACK, + BUSY_GEN1_RESULT, + BUSY_GEN2_RESULT, + BUSY_CB1_SEQ, + BUSY_CB2_COUNT, + BUSY_CB2_SEQ, + BUSY_RECOVER_SEQ, + BUSY_LAST_WAIT_NS, + BUSY_STATUS_MAX, +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, BUSY_STATUS_MAX); + __type(key, int); + __type(value, __s64); +} status SEC(".maps"); + +struct task_work_busy_value { + __u32 seq; + char data[60]; + struct bpf_task_work tw; +}; + +/* + * max_entries must cover TASK_WORK_BUSY_MAX_ATTEMPTS in the test: each + * attempt schedules into its own fresh value. + */ +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(map_flags, BPF_F_NO_PREALLOC); + __uint(max_entries, 128); + __type(key, int); + __type(value, struct task_work_busy_value); +} hmap SEC(".maps"); + +static __always_inline void set_status(int key, __s64 value) +{ + __s64 *slot; + + slot = bpf_map_lookup_elem(&status, &key); + if (slot) + *slot = value; +} + +static __always_inline __s64 get_status(int key) +{ + __s64 *slot = bpf_map_lookup_elem(&status, &key); + + return slot ? *slot : 0; +} + +struct busy_wait_ctx { + __u64 seq; + __u64 start; + bool acked; +}; + +/* bpf_loop() callback: return 0 continues the loop, return 1 stops it. */ +static long busy_wait_cb(__u32 i, void *pctx) +{ + struct busy_wait_ctx *w = pctx; + + if (((__u64)get_status(BUSY_ACK) >> 32) == w->seq) { + w->acked = true; + return 1; + } + if (bpf_ktime_get_ns() - w->start > BUSY_WAIT_NS) + return 1; + return 0; +} + +static int busy_cb_gen1(struct bpf_map *map, void *key, void *value) +{ + struct task_work_busy_value *work = value; + __u64 we = (__u64)get_status(BUSY_WINDOW) >> 32; + + /* + * Backstop for deliveries from abandoned attempts: accept only + * while no window for a different sequence is published. The + * test waits for the old workers to disappear from PID lookup, + * after their exit task_work, before resetting the shared state. + * This callback may run before fexit publishes the window state. + */ + if (we && we != (__u64)work->seq) + return 0; + set_status(BUSY_CB1_SEQ, work->seq); + return 0; +} + +static int busy_cb_gen2(struct bpf_map *map, void *key, void *value) +{ + struct task_work_busy_value *work = value; + __s64 *count; + int key2 = BUSY_CB2_COUNT; + + /* + * Count only deliveries belonging to the sequence whose window is + * (or was) published: a late delivery from an abandoned attempt + * must not satisfy a later exactly-once check. + */ + if ((__u64)work->seq != ((__u64)get_status(BUSY_WINDOW) >> 32)) + return 0; + set_status(BUSY_CB2_SEQ, work->seq); + count = bpf_map_lookup_elem(&status, &key2); + if (count) + __sync_fetch_and_add(count, 1); + return 0; +} + +SEC("fexit/task_work_add") +int BPF_PROG(busy_fexit_task_work_add, struct task_struct *task, + struct callback_head *work, int notify, int retval) +{ + struct busy_wait_ctx w = {}; + __u64 seq, elapsed; + + seq = (__u64)get_status(BUSY_ARM_SEQ); + if (!seq) + return 0; + + /* + * One-shot arm. Only this test's own first-round add may consume + * it and hold the window: a successful TWA_SIGNAL add of the + * bpf_task_work callback, issued from the trigger thread's CPU by + * the self-IPI'd scheduling handler and targeting the target + * thread. Everything else leaves the arm untouched. + */ + if (retval != 0 || notify != TWA_SIGNAL || + BPF_CORE_READ(task, pid) != target_tid) + return 0; + if ((__u32)bpf_get_current_pid_tgid() != (__u32)trigger_tid) + return 0; + + set_status(BUSY_ARM_SEQ, 0); + set_status(BUSY_WINDOW, (seq << 32) | 1); + + w.seq = seq; + w.start = bpf_ktime_get_ns(); + bpf_loop(BUSY_WAIT_ITERS, busy_wait_cb, &w, 0); + elapsed = bpf_ktime_get_ns() - w.start; + + /* + * Publish elapsed time for logging after worker cleanup. Window + * validity depends on seeing ACK here, not on this diagnostic. + */ + set_status(BUSY_LAST_WAIT_NS, (__s64)elapsed); + if (w.acked) + set_status(BUSY_WINDOW, (seq << 32) | 2); + else + set_status(BUSY_WINDOW, (seq << 32) | 3); + return 0; +} + +/* + * Attempt the second scheduling round on the same context, targeting the + * trigger thread. The attempt uses the value keyed by its own sequence, + * so it can never touch another attempt's context. Returns true if a + * terminal result was published into res_key; a transient -EBUSY is left + * unpublished for the caller's retry policy, mirroring the race test's + * convention. + */ +static __always_inline bool busy_try_gen2(__u64 seq, int res_key) +{ + struct task_work_busy_value *work; + struct task_struct *task; + int key = (int)seq, err; + + task = bpf_task_from_pid(trigger_tid); + if (!task) { + set_status(res_key, ((__s64)seq << 32) | (__u32)-ESRCH); + return true; + } + work = bpf_map_lookup_elem(&hmap, &key); + if (!work) { + bpf_task_release(task); + set_status(res_key, ((__s64)seq << 32) | (__u32)-ENOENT); + return true; + } + work->seq = seq; + err = bpf_task_work_schedule_signal(task, &work->tw, &hmap, + busy_cb_gen2); + bpf_task_release(task); + if (err == -EBUSY) + return false; + set_status(res_key, ((__s64)seq << 32) | (__u32)err); + return true; +} + +SEC("tracepoint/syscalls/sys_enter_getppid") +int busy_sched_work(void *ctx) +{ + __u32 tid = (__u32)bpf_get_current_pid_tgid(); + struct task_work_busy_value *work; + struct task_struct *task; + __u64 window, seq; + int key, err; + + /* Also keep the kconfig extern in object BTF for the userspace skip. */ + if (CONFIG_PREEMPT_RT) + return 0; + + if (tid == (__u32)trigger_tid) { + /* + * Trigger thread: schedule the first round once per armed + * sequence, targeting the target thread. The kfunc queues + * the scheduling irq_work and returns; the handler then + * parks inside the fexit window above. A fresh value has no + * expected BUSY source, so publish every scheduling result. + */ + seq = (__u64)get_status(BUSY_ARM_SEQ); + if (!seq || + ((__u64)get_status(BUSY_GEN1_RESULT) >> 32) == seq) + return 0; + key = (int)seq; + task = bpf_task_from_pid(target_tid); + if (!task) { + set_status(BUSY_GEN1_RESULT, + ((__s64)seq << 32) | (__u32)-ESRCH); + set_status(BUSY_ARM_SEQ, 0); + return 0; + } + work = bpf_map_lookup_elem(&hmap, &key); + if (!work) { + bpf_task_release(task); + set_status(BUSY_GEN1_RESULT, + ((__s64)seq << 32) | (__u32)-ENOENT); + set_status(BUSY_ARM_SEQ, 0); + return 0; + } + work->seq = seq; + err = bpf_task_work_schedule_signal(task, &work->tw, &hmap, + busy_cb_gen1); + bpf_task_release(task); + set_status(BUSY_GEN1_RESULT, ((__s64)seq << 32) | (__u32)err); + return 0; + } + + if (tid != (__u32)target_tid) + return 0; + + window = (__u64)get_status(BUSY_WINDOW); + seq = window >> 32; + if (!seq || (__u32)window == 0) + return 0; + /* + * The first callback must have completed for this sequence. Being + * inside the target thread's own subsequent syscall proves the + * callback wrapper has returned and the context is STANDBY, which + * is stronger than observing the completion marker from another + * thread. + */ + if ((__u64)get_status(BUSY_CB1_SEQ) != seq) + return 0; + + if ((__u64)get_status(BUSY_ACK) >> 32 != seq) { + /* + * The in-window attempt, exactly once per sequence. On a + * fixed kernel it must be rejected with -EBUSY: the context + * is STANDBY, the map and program references are alive and + * no other scheduler exists, so only the BUSY gate can + * reject here. + */ + if (!busy_try_gen2(seq, BUSY_ACK)) + set_status(BUSY_ACK, + ((__s64)seq << 32) | (__u32)-EBUSY); + return 0; + } + + /* + * Recovery, gated by the test: the fexit program releasing the + * window does not mean the whole old handler has returned and + * BUSY has cleared yet, so retry until the schedule succeeds. + */ + if ((__u64)get_status(BUSY_RECOVER_SEQ) != seq) + return 0; + if ((__s32)get_status(BUSY_ACK) == 0) + return 0; /* scheduled in-window; nothing to recover */ + if (((__u64)get_status(BUSY_GEN2_RESULT) >> 32) == seq) + return 0; /* terminal recovery result published */ + busy_try_gen2(seq, BUSY_GEN2_RESULT); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/task_work_fail.c b/tools/testing/selftests/bpf/progs/task_work_fail.c index 3186e7b4b24e..bc56bdaca780 100644 --- a/tools/testing/selftests/bpf/progs/task_work_fail.c +++ b/tools/testing/selftests/bpf/progs/task_work_fail.c @@ -58,7 +58,7 @@ int mismatch_map(struct pt_regs *args) } SEC("perf_event") -__failure __msg("R2 doesn't point to a map value") +__failure __msg("R2 type=fp expected=map_value") int no_map_task_work(struct pt_regs *args) { struct task_struct *task; diff --git a/tools/testing/selftests/bpf/progs/task_work_race.c b/tools/testing/selftests/bpf/progs/task_work_race.c new file mode 100644 index 000000000000..c94eca510cb3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/task_work_race.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 KylinSoft Corporation. */ +#include <vmlinux.h> +#include <stdbool.h> +#include <errno.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +#define TASK_WORK_RACE_MAX_SEQ 2048 + +enum task_work_race_status { + RACE_ARM_SEQ, + RACE_READY_SEQ, + /* High 32 bits hold the sequence, low 32 bits the signed error. */ + RACE_RESULT, + RACE_STATUS_MAX, +}; + +int trigger_tid; +int target_tid; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, RACE_STATUS_MAX); + __type(key, int); + __type(value, __s64); +} status SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, TASK_WORK_RACE_MAX_SEQ); + __type(key, int); + __type(value, __u64); +} completed SEC(".maps"); + +struct task_work_race_value { + __u32 seq; + char data[60]; + struct bpf_task_work tw; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(map_flags, BPF_F_NO_PREALLOC); + __uint(max_entries, 4); + __type(key, int); + __type(value, struct task_work_race_value); +} hmap SEC(".maps"); + +static __always_inline void set_status(int key, __s64 value) +{ + __s64 *slot; + + slot = bpf_map_lookup_elem(&status, &key); + if (slot) + *slot = value; +} + +static int process_work(struct bpf_map *map, void *key, void *value) +{ + struct task_work_race_value *work = value; + __u64 *done; + int seq = work->seq; + + if (seq <= 0 || seq >= TASK_WORK_RACE_MAX_SEQ) + return 0; + done = bpf_map_lookup_elem(&completed, &seq); + if (done) + *done = 1; + return 0; +} + +SEC("tracepoint/syscalls/sys_enter_getppid") +int race_sched_work(void *ctx) +{ + struct task_work_race_value *work; + struct task_struct *task; + __s64 *arm, *result; + __u32 tid = (__u32)bpf_get_current_pid_tgid(); + int key = 0, err; + __u32 seq; + + if (tid != trigger_tid) + return 0; + + key = RACE_ARM_SEQ; + arm = bpf_map_lookup_elem(&status, &key); + key = RACE_RESULT; + result = bpf_map_lookup_elem(&status, &key); + if (!arm || !result || *arm <= 0 || + (__u32)((__u64)*result >> 32) == *arm) + return 0; + seq = *arm; + + task = bpf_task_from_pid(target_tid); + if (!task) { + err = -ESRCH; + goto out_done; + } + + key = 0; + work = bpf_map_lookup_elem(&hmap, &key); + if (!work) { + err = -ENOENT; + goto out_task; + } + + work->seq = seq; + set_status(RACE_READY_SEQ, seq); + err = bpf_task_work_schedule_signal(task, &work->tw, &hmap, + process_work); + if (err == -EBUSY) { + bpf_task_release(task); + return 0; + } + +out_task: + bpf_task_release(task); +out_done: + /* Publish the sequence and its error in one store. */ + *result = ((__u64)seq << 32) | (__u32)err; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/tcp_autolowat.c b/tools/testing/selftests/bpf/progs/tcp_autolowat.c new file mode 100644 index 000000000000..8a55f3cee260 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tcp_autolowat.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright 2026 Google LLC */ +#include "vmlinux.h" + +#include <string.h> +#include <limits.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_core_read.h> + +#include "bpf_tracing_net.h" + +#define SOL_BPF 0xdeadbeef +#define BPF_TCP_AUTOLOWAT 0x8badf00d + +//#define DEBUG /* For verbose output. */ + +struct rpc_descriptor { + u32 header_len; + u32 payload_len; +}; + +#define RPC_DESC_SIZE (sizeof(struct rpc_descriptor)) +#define MAX_RPC_DESC_PER_SKB 100 + +struct tcp_autolowat_cb { + /* Don't put this field at the end; BPF verifier complains. */ + char rpc_desc_buf[RPC_DESC_SIZE]; + u32 rpc_desc_seq; + u32 rpc_end_seq; +#ifdef DEBUG + u32 isn; +#endif + u8 rpc_desc_buff_len; +}; + +struct { + __uint(type, BPF_MAP_TYPE_SK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct tcp_autolowat_cb); +} tcp_autolowat_map SEC(".maps"); + +char test_name[64]; + +#ifdef DEBUG +#define LOG(str, ...) \ + bpf_printk("%s: " str, test_name, ##__VA_ARGS__) +#else +#define LOG(...) +#endif + +#define SEQ(val) \ + (val - cb->isn) +#define TP_SEQ(field) \ + (tp->field - cb->isn) +#define CB_SEQ(field) \ + (cb->field - cb->isn) + +static int tcp_parse_descriptor(struct tcp_autolowat_cb *cb, + struct bpf_dynptr *dptr, + u32 seq, u32 end_seq) +{ + struct rpc_descriptor *rpc_desc; + u32 rpc_copied_seq; + u64 copy_len; /* u32 should work, but not for no_alu32 :/ */ + u64 rpc_len; + int err; + + rpc_copied_seq = cb->rpc_desc_seq + cb->rpc_desc_buff_len; + + if (before(cb->rpc_desc_seq + RPC_DESC_SIZE, end_seq)) + copy_len = RPC_DESC_SIZE - cb->rpc_desc_buff_len; + else + copy_len = end_seq - rpc_copied_seq; + + if (copy_len == 0) + goto disable; /* FIN. */ + if (copy_len > RPC_DESC_SIZE) + goto disable; /* always false, only for verifier. */ + if (cb->rpc_desc_buff_len >= RPC_DESC_SIZE) + goto disable; /* always false, only for verifier. */ + + err = bpf_dynptr_read(cb->rpc_desc_buf + cb->rpc_desc_buff_len, + copy_len, dptr, rpc_copied_seq - seq, 0); + if (err) + goto disable; + + cb->rpc_desc_buff_len += copy_len; + + if (cb->rpc_desc_buff_len != RPC_DESC_SIZE) { + LOG("Copied %d bytes: rpc_desc_buff_len: %u", copy_len, cb->rpc_desc_buff_len); + goto partial; + } + + rpc_desc = (struct rpc_descriptor *)cb->rpc_desc_buf; + rpc_len = RPC_DESC_SIZE + rpc_desc->header_len + rpc_desc->payload_len; + + if (rpc_len > INT_MAX) + goto disable; + + cb->rpc_end_seq = cb->rpc_desc_seq + rpc_len; + + LOG("Copied full descriptor: rpc_desc_seq: %u, rpc_end_seq: %u, header_len: %u, payload_len: %u", + CB_SEQ(rpc_desc_seq), CB_SEQ(rpc_end_seq), + rpc_desc->header_len, rpc_desc->payload_len); + + return 0; +disable: + return -1; +partial: + return 1; +} + +static void tcp_set_autolowat(struct tcp_autolowat_cb *cb, + struct sock *sk) +{ + struct tcp_sock *tp = (struct tcp_sock *)sk; + u32 val; /* To handle wraparound. */ + + LOG("Setting rcvlowat: tp->copied_seq: %u, rpc_desc_seq: %u, rpc_end_seq: %u, rpc_desc_buff_len: %u", + TP_SEQ(copied_seq), CB_SEQ(rpc_desc_seq), + CB_SEQ(rpc_end_seq), cb->rpc_desc_buff_len); + + if (before(tp->copied_seq, cb->rpc_desc_seq)) + val = cb->rpc_desc_seq - tp->copied_seq; + else if (cb->rpc_desc_buff_len != RPC_DESC_SIZE) + val = RPC_DESC_SIZE; + else + val = cb->rpc_end_seq - tp->copied_seq; + + if (val != tp->inet_conn.icsk_inet.sk.sk_rcvlowat) { + bpf_tcp_ops_set_rcvlowat(sk, val); + + LOG("Set rcvlowat: expected: %u, actual: %d\n", + val, tp->inet_conn.icsk_inet.sk.sk_rcvlowat); + } else { + LOG("No need to set rcvlowat: %u\n", val); + } +} + +static void tcp_disable_autolowat(struct sock *sk) +{ + bpf_tcp_ops_set_flags((struct tcp_sock *)sk, 0, BPF_TCP_OPS_FLAG_RCVQ); + + bpf_tcp_ops_set_rcvlowat(sk, 1); + + LOG("Disabled autolowat"); +} + +static void tcp_do_autolowat(struct tcp_autolowat_cb *cb, + struct sock *sk, struct sk_buff *skb) +{ + struct bpf_dynptr dptr; + struct tcp_skb_cb *tcb; + u32 seq, end_seq; + int ret = 0, i; + + if (bpf_dynptr_from_skb((struct __sk_buff *)skb, 0, &dptr)) { + ret = -1; + goto update; + } + + tcb = bpf_core_cast(skb->cb, struct tcp_skb_cb); + seq = tcb->seq; + end_seq = tcb->end_seq - !!(tcb->tcp_flags & TCPHDR_FIN); + + LOG("Start parsing skb: seq: %u, end_seq: %u, len: %u, rpc_desc_seq: %u, rpc_end_seq: %u, rpc_buff_len: %u", + SEQ(seq), SEQ(end_seq), end_seq - seq, + CB_SEQ(rpc_desc_seq), CB_SEQ(rpc_end_seq), cb->rpc_desc_buff_len); + + if (cb->rpc_desc_buff_len != RPC_DESC_SIZE) { + ret = tcp_parse_descriptor(cb, &dptr, seq, end_seq); + if (ret) + goto update; + } + + i = 0; + + while (1) { + if (i++ > MAX_RPC_DESC_PER_SKB) { + ret = -1; + break; + } + + if (after(cb->rpc_end_seq, end_seq)) { + LOG("No more descriptor: rpc_end_seq: %u, end_seq: %u", + CB_SEQ(rpc_end_seq), SEQ(end_seq)); + break; + } + + cb->rpc_desc_seq = cb->rpc_end_seq; + cb->rpc_desc_buff_len = 0; + + if (cb->rpc_end_seq == end_seq) + break; + + LOG("Found next descriptor: rpc_end_seq: %u, end_seq: %u, len: %u", + CB_SEQ(rpc_end_seq), SEQ(end_seq), end_seq - cb->rpc_end_seq); + + ret = tcp_parse_descriptor(cb, &dptr, seq, end_seq); + if (ret) + break; + } + +update: + if (ret >= 0) + tcp_set_autolowat(cb, sk); + else + tcp_disable_autolowat(sk); +} + +SEC("struct_ops") +void BPF_PROG(tcp_autolowat_enqueue_rcvq, struct sock *sk, struct sk_buff *skb) +{ + struct tcp_autolowat_cb *cb; + + cb = bpf_sk_storage_get(&tcp_autolowat_map, sk, 0, 0); + if (!cb) + return; + + tcp_do_autolowat(cb, sk, skb); +} + +SEC("struct_ops") +void BPF_PROG(tcp_autolowat_dequeue_rcvq, struct sock *sk) +{ + struct tcp_autolowat_cb *cb; + + cb = bpf_sk_storage_get(&tcp_autolowat_map, sk, 0, 0); + if (!cb) + return; + + tcp_set_autolowat(cb, sk); +} + +SEC(".struct_ops.link") +struct bpf_tcp_ops tcp_autolowat_ops = { + .enqueue_rcvq = (void *)tcp_autolowat_enqueue_rcvq, + .dequeue_rcvq = (void *)tcp_autolowat_dequeue_rcvq, +}; + +static int tcp_init_autolowat_cb(struct bpf_tcp_sock *btp) +{ + struct tcp_autolowat_cb *cb; + struct tcp_sock *tp; + + cb = bpf_sk_storage_get(&tcp_autolowat_map, btp, 0, + BPF_SK_STORAGE_GET_F_CREATE); + if (!cb) + return -1; + + tp = bpf_core_cast(btp, struct tcp_sock); + + cb->rpc_desc_seq = tp->copied_seq; + cb->rpc_end_seq = tp->copied_seq; +#ifdef DEBUG + cb->isn = tp->copied_seq; +#endif + + return bpf_tcp_ops_set_flags((struct tcp_sock *)btp, + BPF_TCP_OPS_FLAG_RCVQ, 0); +} + +SEC("cgroup/setsockopt") +int tcp_autolowat_setsockopt(struct bpf_sockopt *ctx) +{ + void *optval_end = ctx->optval_end; + int *optval = ctx->optval; + struct bpf_tcp_sock *btp; + + if (ctx->level != SOL_BPF || ctx->optname != BPF_TCP_AUTOLOWAT) + goto out; + + if (optval + 1 > optval_end) + return 0; /* -EPERM */ + + btp = bpf_tcp_sock(ctx->sk); + if (!btp) + goto out; + + if (*optval && tcp_init_autolowat_cb(btp)) + return 0; /* -EPERM */ + + /* + * BPF has consumed this option, don't call kernel + * setsockopt handler. + */ + ctx->optlen = -1; +out: + return 1; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_global_func1.c b/tools/testing/selftests/bpf/progs/test_global_func1.c index fc69ff18880d..f0eca282e0d4 100644 --- a/tools/testing/selftests/bpf/progs/test_global_func1.c +++ b/tools/testing/selftests/bpf/progs/test_global_func1.c @@ -48,8 +48,73 @@ int f3(int val, struct __sk_buff *skb, int var) } SEC("tc") +__load_if_no_large_stack() __failure __msg("combined stack size of 3 calls is") int global_func1(struct __sk_buff *skb) { return f0(1, skb) + f1(skb) + f2(2, skb) + f3(3, skb, 4); } + +/* + * A chain of five frames that stay under 512 bytes each but add up to more + * than the 2 KiB budget of JITs with large stacks; the chain also exceeds + * 512 bytes after two frames, so it is rejected everywhere. + */ +#define MAX_STACK_LARGE 480 + +__attribute__ ((noinline)) +int g0(struct __sk_buff *skb) +{ + volatile char buf[MAX_STACK_LARGE] = {}; + + __sink(buf[MAX_STACK_LARGE - 1]); + + return skb->len; +} + +__attribute__ ((noinline)) +int g1(struct __sk_buff *skb) +{ + volatile char buf[MAX_STACK_LARGE] = {}; + + __sink(buf[MAX_STACK_LARGE - 1]); + + return g0(skb) + skb->len; +} + +__attribute__ ((noinline)) +int g2(struct __sk_buff *skb) +{ + volatile char buf[MAX_STACK_LARGE] = {}; + + __sink(buf[MAX_STACK_LARGE - 1]); + + return g1(skb) + skb->len; +} + +__attribute__ ((noinline)) +int g3(struct __sk_buff *skb) +{ + volatile char buf[MAX_STACK_LARGE] = {}; + + __sink(buf[MAX_STACK_LARGE - 1]); + + return g2(skb) + skb->len; +} + +__attribute__ ((noinline)) +int g4(struct __sk_buff *skb) +{ + volatile char buf[MAX_STACK_LARGE] = {}; + + __sink(buf[MAX_STACK_LARGE - 1]); + + return g3(skb) + skb->len; +} + +SEC("tc") +__failure __msg("combined stack size of {{[0-9]+}} calls is") +int global_func1_deep(struct __sk_buff *skb) +{ + return g4(skb); +} diff --git a/tools/testing/selftests/bpf/progs/test_global_func5.c b/tools/testing/selftests/bpf/progs/test_global_func5.c index 257c0569ff98..4573ecd96aa0 100644 --- a/tools/testing/selftests/bpf/progs/test_global_func5.c +++ b/tools/testing/selftests/bpf/progs/test_global_func5.c @@ -26,7 +26,7 @@ int f3(int val, struct __sk_buff *skb) } SEC("tc") -__failure __msg("expects pointer to ctx") +__failure __msg("R2 type=fp expected=ctx") int global_func5(struct __sk_buff *skb) { return f1(skb) + f2(2, skb) + f3(3, skb); diff --git a/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c b/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c index 1b634b543b62..edb8a223a3cb 100644 --- a/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c +++ b/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c @@ -67,12 +67,30 @@ int XCAT(f, n)(unsigned long a) \ #define F_31 F_30 FN(31, 30) #define F_32 F_31 FN(32, 31) +/* Same, with a 480-byte frame, to exceed the 2 KiB budget of large stacks. */ +#define FNB(n, prev) \ +__attribute__((noinline)) \ +int XCAT(f, n)(unsigned long a) \ +{ \ + volatile char buf[480] = {}; \ + volatile long b = XCAT(f, prev)(a - 1); \ + if (!b) \ + return 0; \ + return b + buf[479] + 1; \ +} + +#define F_33 F_32 FNB(33, 32) +#define F_34 F_33 FNB(34, 33) +#define F_35 F_34 FNB(35, 34) +#define F_36 F_35 FNB(36, 35) +#define F_37 F_36 FNB(37, 36) + #define CAT2(a, b) a ## b #define XCAT2(a, b) CAT2(a, b) #define F(n) XCAT2(F_, n) -F(32) +F(37) /* Ensure that even 32 levels deep, the function verifies. */ SEC("syscall") @@ -88,8 +106,21 @@ int global_func_deep_stack_success(struct __sk_buff *skb) * the size. */ SEC("syscall") +__load_if_no_large_stack() __failure __msg("combined stack size of 34 calls") int global_func_deep_stack_fail(struct __sk_buff *skb) { return f32(123); } + +/* + * Five 480-byte frames on top of the chain: 5 * 480 + 33 * 16 = 2928 bytes, + * more than the 2 KiB budget of JITs with large stacks, and more than 512 + * bytes after the second frame everywhere else. + */ +SEC("syscall") +__failure __msg("combined stack size of {{[0-9]+}} calls") +int global_func_deep_stack_fail_large(struct __sk_buff *skb) +{ + return f37(123); +} diff --git a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c index 5dc21b3b4cb5..2765cd46e9af 100644 --- a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c +++ b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c @@ -8,9 +8,12 @@ int loong SEC(".percpu.looooooooong"); int data3 SEC(".data.percpu"); int data2 SEC(".percpu.data"); +/* Used for testing bpf_map__set_value_size(). */ +int arr[1] SEC(".percpu.arr"); +int arr_sum; + int run; -/* cpu_id as array to verify map value resizing. */ -int cpu_id[1] SEC(".percpu"); +int cpu_id SEC(".percpu"); int data SEC(".percpu") = -1; int nums[7] SEC(".percpu"); bool set SEC(".percpu") = false; @@ -34,7 +37,8 @@ int update_percpu_data(void *ctx) data = 1; run++; set = true; - cpu_id[0] = bpf_get_smp_processor_id(); + cpu_id = bpf_get_smp_processor_id(); + arr_sum += arr[0]; return 0; } diff --git a/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c b/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c index bf48fc43c7ab..f7a83e502454 100644 --- a/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c +++ b/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c @@ -40,7 +40,7 @@ int BPF_PROG(not_valid_dynptr, int cmd, union bpf_attr *attr, unsigned int size, } SEC("?lsm.s/bpf") -__failure __msg("R1 expected pointer to stack or const struct bpf_dynptr") +__failure __msg("R1 type=map_value expected=fp, dynptr_ptr") int BPF_PROG(not_ptr_to_stack, int cmd, union bpf_attr *attr, unsigned int size, bool kernel) { static struct bpf_dynptr val; diff --git a/tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c b/tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c deleted file mode 100644 index cbe4284c032f..000000000000 --- a/tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// test ir decoder -// -// Copyright (C) 2018 Sean Young <sean@mess.org> - -#include <linux/bpf.h> -#include <linux/lirc.h> -#include <bpf/bpf_helpers.h> - -SEC("lirc_mode2") -int bpf_decoder(unsigned int *sample) -{ - if (LIRC_IS_PULSE(*sample)) { - unsigned int duration = LIRC_VALUE(*sample); - - if (duration & 0x1000) - bpf_rc_keydown(sample, 0x40, duration & 0xffff, 0); - if (duration & 0x2000) - bpf_rc_pointer_rel(sample, (duration >> 8) & 0xff, - duration & 0xff); - } - - return 0; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c b/tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c index 4cfa42aa9436..c5c643064eb5 100644 --- a/tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c +++ b/tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c @@ -29,7 +29,7 @@ int BPF_PROG(unix_listen, struct socket *sock, int backlog) len = unix_sk->addr->len - sizeof(short); path[0] = '@'; for (i = 1; i < len; i++) { - if (i >= (int)sizeof(struct sockaddr_un)) + if (i >= (int)sizeof(unix_sk->addr->name->sun_path)) break; path[i] = unix_sk->addr->name->sun_path[i]; diff --git a/tools/testing/selftests/bpf/progs/test_snprintf.c b/tools/testing/selftests/bpf/progs/test_snprintf.c index 8fda07544023..48a0025cf2ca 100644 --- a/tools/testing/selftests/bpf/progs/test_snprintf.c +++ b/tools/testing/selftests/bpf/progs/test_snprintf.c @@ -57,8 +57,8 @@ int handler(const void *ctx) sym_ret = BPF_SNPRINTF(sym_out, sizeof(sym_out), "%ps %pS %pB", &schedule, &schedule, &schedule); /* Kernel pointers */ - addr_ret = BPF_SNPRINTF(addr_out, sizeof(addr_out), "%pK %px %p", - 0, 0xFFFF00000ADD4E55, 0xFFFF00000ADD4E55); + addr_ret = BPF_SNPRINTF(addr_out, sizeof(addr_out), "%px %p", + 0xFFFF00000ADD4E55, 0xFFFF00000ADD4E55); /* Strings and single-byte character embedding */ str_ret = BPF_SNPRINTF(str_out, sizeof(str_out), "%s % 9c %+2c %-3c %04c %0c %+05s", str1, 'a', 'b', 'c', 'd', 'e', longstr); diff --git a/tools/testing/selftests/bpf/progs/test_tc_edt.c b/tools/testing/selftests/bpf/progs/test_tc_edt.c index 4f6f03122d61..69f4b16acbef 100644 --- a/tools/testing/selftests/bpf/progs/test_tc_edt.c +++ b/tools/testing/selftests/bpf/progs/test_tc_edt.c @@ -24,6 +24,7 @@ struct { } flow_map SEC(".maps"); __uint64_t target_rate; +__uint64_t tx_start_ns; static inline int throttle_flow(struct __sk_buff *skb) { @@ -62,7 +63,8 @@ static inline int throttle_flow(struct __sk_buff *skb) return TC_ACT_OK; } -static inline int handle_tcp(struct __sk_buff *skb, struct tcphdr *tcp) +static inline int handle_tcp(struct __sk_buff *skb, struct tcphdr *tcp, + uint32_t hdr_len) { void *data_end = (void *)(long)skb->data_end; @@ -70,8 +72,12 @@ static inline int handle_tcp(struct __sk_buff *skb, struct tcphdr *tcp) if ((void *)(tcp + 1) > data_end) return TC_ACT_SHOT; - if (tcp->source == bpf_htons(9000)) + if (tcp->source == bpf_htons(9000)) { + /* Note when the first packet with data goes out */ + if (!tx_start_ns && skb->len > hdr_len + tcp->doff * 4) + tx_start_ns = bpf_ktime_get_ns(); return throttle_flow(skb); + } return TC_ACT_OK; } @@ -94,7 +100,8 @@ static inline int handle_ipv4(struct __sk_buff *skb) return TC_ACT_SHOT; if (iph->protocol == IPPROTO_TCP) - return handle_tcp(skb, (struct tcphdr *)(((void *)iph) + ihl)); + return handle_tcp(skb, (struct tcphdr *)(((void *)iph) + ihl), + sizeof(struct ethhdr) + ihl); return TC_ACT_OK; } diff --git a/tools/testing/selftests/bpf/progs/test_tc_link.c b/tools/testing/selftests/bpf/progs/test_tc_link.c index 630f12e51b07..2a91793674dd 100644 --- a/tools/testing/selftests/bpf/progs/test_tc_link.c +++ b/tools/testing/selftests/bpf/progs/test_tc_link.c @@ -4,6 +4,8 @@ #include <linux/bpf.h> #include <linux/if_ether.h> +#include <linux/in.h> +#include <linux/ip.h> #include <linux/stddef.h> #include <linux/if_packet.h> #include <bpf/bpf_endian.h> @@ -119,11 +121,19 @@ SEC("tc/egress") int tc8(struct __sk_buff *skb) { struct net_device *dev = BPF_CORE_READ((struct sk_buff *)skb, dev); + struct iphdr iph = {}; + if (skb->protocol != __bpf_constant_htons(ETH_P_IP)) + goto out; + if (bpf_skb_load_bytes(skb, sizeof(struct ethhdr), &iph, sizeof(iph))) + goto out; + if (iph.protocol != IPPROTO_ICMP) + goto out; seen_tc8 = true; mark = skb->mark; prio = skb->priority; headroom = BPF_CORE_READ(dev, needed_headroom); tailroom = BPF_CORE_READ(dev, needed_tailroom); +out: return TCX_PASS; } diff --git a/tools/testing/selftests/bpf/progs/timer.c b/tools/testing/selftests/bpf/progs/timer.c index d6d5fefcd9b1..aa966dbed02e 100644 --- a/tools/testing/selftests/bpf/progs/timer.c +++ b/tools/testing/selftests/bpf/progs/timer.c @@ -6,6 +6,7 @@ #include <errno.h> #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> +#include "bpf_experimental.h" #define CLOCK_MONOTONIC 1 #define CLOCK_BOOTTIME 7 @@ -57,9 +58,12 @@ struct { __type(key, int); __type(value, struct elem); } abs_timer SEC(".maps"), soft_timer_pinned SEC(".maps"), abs_timer_pinned SEC(".maps"), - race_array SEC(".maps"); + race_array SEC(".maps"), loop_array SEC(".maps"); __u64 bss_data; +__u64 loop_sum; +int loop_rearm; /* number of times loop_cb() re-arms itself */ +__u64 zero; __u64 abs_data; __u64 err; __u64 ok; @@ -139,6 +143,60 @@ static int timer_cb1(void *map, int *key, struct bpf_timer *timer) return 0; } +/* + * Static and not inlined, so the loop is walked in a frame below the + * callback's rather than in a separately verified global subprog. + */ +static __noinline int sum_to(__u64 n) +{ + __u64 i, sum = 0; + + for (i = zero; i < n && can_loop; i++) + sum += i; + + return sum; +} + +static int loop_cb(void *map, int *key, struct elem *val); + +/* + * Re-arms from a frame below the callback's, where the caller frame carries + * no async_entry_cnt of its own. + */ +static __noinline void rearm(struct elem *val) +{ + if (loop_rearm > 0) { + loop_rearm--; + /* set_callback is what starts another async callback entry */ + bpf_timer_set_callback(&val->t, loop_cb); + bpf_timer_start(&val->t, 0, 0); + } +} + +/* Re-arms itself and reaches a bounded loop through a call. */ +static int loop_cb(void *map, int *key, struct elem *val) +{ + loop_sum += sum_to(16); + rearm(val); + return 0; +} + +SEC("fentry/bpf_fentry_test1") +int BPF_PROG2(test_loop_rearm, int, a) +{ + struct bpf_timer *timer; + int key = 0; + + timer = bpf_map_lookup_elem(&loop_array, &key); + if (!timer) + return 0; + + bpf_timer_init(timer, &loop_array, CLOCK_MONOTONIC); + bpf_timer_set_callback(timer, loop_cb); + bpf_timer_start(timer, 0, 0); + return 0; +} + SEC("fentry/bpf_fentry_test1") int BPF_PROG2(test1, int, a) { diff --git a/tools/testing/selftests/bpf/progs/timer_failure.c b/tools/testing/selftests/bpf/progs/timer_failure.c index 5a2e9dabf1c6..0538269101ca 100644 --- a/tools/testing/selftests/bpf/progs/timer_failure.c +++ b/tools/testing/selftests/bpf/progs/timer_failure.c @@ -66,3 +66,32 @@ long BPF_PROG2(test_bad_ret, int, a) return 0; } + +/* + * A real loop inside an async callback must still be rejected: both entries + * into the callback have the same async_entry_cnt, so telling entries apart + * does not apply here. + */ +static int timer_cb_infinite_loop(void *map, int *key, struct elem *val) +{ + for (;;) {} + + return 0; +} + +SEC("fentry/bpf_fentry_test1") +__failure __msg("infinite loop detected") +long BPF_PROG2(test_infinite_loop_cb, int, a) +{ + struct bpf_timer *timer; + int key = 0; + + timer = bpf_map_lookup_elem(&timer_map, &key); + if (timer) { + bpf_timer_init(timer, &timer_map, CLOCK_BOOTTIME); + bpf_timer_set_callback(timer, timer_cb_infinite_loop); + bpf_timer_start(timer, 1000, 0); + } + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tgt.c b/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tgt.c new file mode 100644 index 000000000000..5fde710c4e37 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tgt.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +__u64 base; +__u64 ret_stack; + +#if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT) + +const volatile bool has_stack_arg = true; + +/* g and h are on the stack, for the x86-64 kernel convention too. */ +static __noinline __u64 callee_stack(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, __u64 f, __u64 g, __u64 h) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + h * 8; +} + +/* No stack args of its own, so it can share a program with a tail call. */ +static __noinline __u64 call_stack(__u64 x) +{ + return callee_stack(x + 1, x + 2, x + 3, x + 4, x + 5, x + 6, x + 7, + x + 8); +} + +#else + +const volatile bool has_stack_arg = false; + +#endif + +/* Makes the program tail_call_reachable, though callee_stack() is not. */ +static __noinline int subprog_tail(void *ctx) +{ + int ret = 0; + + bpf_tail_call_static(ctx, &jmp_table, 0); + barrier_var(ret); + return ret; +} + +SEC("syscall") +int run(void *ctx) +{ +#if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT) + ret_stack = call_stack(base); +#endif + return subprog_tail(ctx); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tracer.c b/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tracer.c new file mode 100644 index 000000000000..1800e7b7afe2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tracer.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +long stack_g, stack_h, stack_ret; + +SEC("fentry/callee_stack") +int stack_entry(unsigned long long *ctx) +{ + stack_g = ctx[6]; + stack_h = ctx[7]; + return 0; +} + +SEC("fexit/callee_stack") +int stack_exit(unsigned long long *ctx) +{ + stack_ret = ctx[8]; + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128.c index 4638dfec1f38..5520ee500c01 100644 --- a/tools/testing/selftests/bpf/progs/tracing_struct_int128.c +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128.c @@ -15,4 +15,49 @@ int test_int128_arg_fexit(unsigned long long *ctx) return 0; } +/* + * The arguments come packed in the BPF convention whatever holes the + * kernel convention left between them: an eightbyte each, and two for the + * struct holding an __int128. + */ +long s_g, s_h_lo, s_h_hi, s_i, s_ret; + +SEC("fentry/bpf_testmod_test_int128_stack") +int test_int128_stack_fentry(unsigned long long *ctx) +{ + s_g = ctx[6]; + s_h_lo = ctx[7]; + s_h_hi = ctx[8]; + s_i = ctx[9]; + return 0; +} + +SEC("fexit/bpf_testmod_test_int128_stack") +int test_int128_stack_fexit(unsigned long long *ctx) +{ + s_ret = ctx[10]; + return 0; +} + +long b_f_a, b_f_b, b_g, b_h, b_i_lo, b_i_hi, b_ret; + +SEC("fentry/bpf_testmod_test_int128_backfill") +int test_int128_backfill_fentry(unsigned long long *ctx) +{ + b_f_a = ctx[5]; + b_f_b = ctx[6]; + b_g = ctx[7]; + b_h = ctx[8]; + b_i_lo = ctx[9]; + b_i_hi = ctx[10]; + return 0; +} + +SEC("fexit/bpf_testmod_test_int128_backfill") +int test_int128_backfill_fexit(unsigned long long *ctx) +{ + b_ret = ctx[11]; + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c new file mode 100644 index 000000000000..2561422c25dd --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +struct int128_arg { + __int128 v; +}; + +__u64 base; +__u64 ret_odd, ret_stack, ret_split, ret_unreliable; + +#if defined(__clang__) + +const volatile bool has_struct_arg = true; + +/* arm64 would want v in x2 and x3, and b in x4. */ +__noinline __u64 callee_odd(__u64 a, struct int128_arg v, __u64 b) +{ + return a + (__u64)v.v * 2 + (__u64)((unsigned __int128)v.v >> 64) * 3 + + b * 4; +} + +#else + +const volatile bool has_struct_arg = false; + +#endif + +#if defined(__clang__) && \ + (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) + +const volatile bool has_stack_arg = true; + +/* + * x86-64 would want h in its third and fourth stack slots, and arm64 in + * its first and second, leaving x7 empty. + */ +static __noinline __u64 callee_stack(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, __u64 f, __u64 g, + struct int128_arg h, __u64 i) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + + (__u64)h.v * 8 + (__u64)((unsigned __int128)h.v >> 64) * 9 + + i * 10; +} + +/* + * x86-64 BPF code passes h split between r9 and its first stack slot; + * arm64 passes it in x5 and x6, so nothing is split there. + */ +static __noinline __u64 callee_split(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, struct int128_arg h, __u64 i) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + (__u64)h.v * 6 + + (__u64)((unsigned __int128)h.v >> 64) * 7 + i * 8; +} + +/* + * A plain pointer makes the BTF of a static function unreliable, but f is + * still passed on the stack. The store keeps p a pointer. + */ +static __noinline __u64 callee_unreliable(__u64 *p, __u64 b, __u64 c, __u64 d, + __u64 e, __u64 f) +{ + *p += 1; + return *p + b * 2 + c * 3 + d * 4 + e * 5 + f * 6; +} + +#else + +const volatile bool has_stack_arg = false; + +#endif + +SEC("syscall") +int run(void *ctx) +{ +#if defined(__clang__) + struct int128_arg v = { .v = ((__int128)(base + 9) << 64) | (base + 8) }; + + ret_odd = callee_odd(base + 1, v, base + 2); +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) + ret_stack = callee_stack(base + 1, base + 2, base + 3, base + 4, + base + 5, base + 6, base + 7, v, base + 10); + ret_split = callee_split(base + 1, base + 2, base + 3, base + 4, + base + 5, v, base + 10); + { + __u64 x = base; + + ret_unreliable = callee_unreliable(&x, base + 2, base + 3, + base + 4, base + 5, base + 6); + } +#endif +#endif + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c new file mode 100644 index 000000000000..fc81c2eea4a8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +struct int128_arg { + __int128 v; +}; + +long odd_a, odd_lo, odd_hi, odd_b, odd_ret; + +SEC("fentry/callee_odd") +int BPF_PROG2(odd_entry, __u64, a, struct int128_arg, v, __u64, b) +{ + odd_a = a; + odd_lo = (__u64)v.v; + odd_hi = (unsigned __int128)v.v >> 64; + odd_b = b; + return 0; +} + +SEC("fexit/callee_odd") +int odd_exit(unsigned long long *ctx) +{ + odd_ret = ctx[4]; + return 0; +} + +long stack_g, stack_lo, stack_hi, stack_i, stack_ret; + +SEC("fentry/callee_stack") +int BPF_PROG2(stack_entry, __u64, a, __u64, b, __u64, c, __u64, d, __u64, e, + __u64, f, __u64, g, struct int128_arg, h, __u64, i) +{ + stack_g = g; + stack_lo = (__u64)h.v; + stack_hi = (unsigned __int128)h.v >> 64; + stack_i = i; + return 0; +} + +SEC("fexit/callee_stack") +int stack_exit(unsigned long long *ctx) +{ + stack_ret = ctx[10]; + return 0; +} + +long split_e, split_lo, split_hi, split_i, split_ret; + +SEC("fentry/callee_split") +int BPF_PROG2(split_entry, __u64, a, __u64, b, __u64, c, __u64, d, __u64, e, + struct int128_arg, h, __u64, i) +{ + split_e = e; + split_lo = (__u64)h.v; + split_hi = (unsigned __int128)h.v >> 64; + split_i = i; + return 0; +} + +SEC("fexit/callee_split") +int split_exit(unsigned long long *ctx) +{ + split_ret = ctx[8]; + return 0; +} + +/* Refused: the trampoline would lose the stack argument. */ +SEC("?fentry/callee_unreliable") +int unreliable_entry(unsigned long long *ctx) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/unwind_attach.c b/tools/testing/selftests/bpf/progs/unwind_attach.c new file mode 100644 index 000000000000..9aff7c6bda23 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/unwind_attach.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "../../../include/linux/filter.h" +#include "bpf_misc.h" + +/* The targets are global_unwinds() and global_never_unwinds() in verifier_unwind.c */ + +#define unwind_exit ".8byte %[insn_unwind_exit];" +#define __imm_unwind_exit \ + __imm_insn(insn_unwind_exit, \ + BPF_RAW_INSN(BPF_JMP | BPF_EXIT, 0, BPF_UNWIND_EXIT, 0, 0)) + +extern void bpf_throw(__u64 cookie) __ksym; + +int fentry_cnt; + +SEC("?fentry/global_unwinds") +int BPF_PROG(fentry_unwind) +{ + fentry_cnt++; + return 0; +} + +SEC("?fexit/global_unwinds") +int BPF_PROG(fexit_unwind) +{ + return 0; +} + +SEC("?freplace/global_unwinds") +unsigned long freplace_returns(void) +{ + return 9; +} + +SEC("?freplace/global_unwinds") +__naked unsigned long freplace_unwinds(void) +{ + asm volatile ( + "r0 = 3;" + unwind_exit + : + : __imm_unwind_exit + ); +} + +/* Not naked: the argument has to have a name in BTF */ +__noinline int throws_cb(__u64 cookie) +{ + if (cookie) + asm volatile ("r0 = 3;" unwind_exit :: __imm_unwind_exit : "r0"); + return 0; +} + +/* It has no unwind_exit, so it can replace a subprog that cannot unwind */ +SEC("?freplace/global_never_unwinds") +__attribute__((btf_decl_tag("exception_callback:throws_cb"))) +unsigned long freplace_throws(void) +{ + bpf_throw(1); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/unwind_cxx.cpp b/tools/testing/selftests/bpf/progs/unwind_cxx.cpp new file mode 100644 index 000000000000..3a113ffea29c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/unwind_cxx.cpp @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * C++ exceptions. llvm lowers unwinding to two insns: + * invoke foo unwind pad call foo; unwind_goto pad + * resume r0 = 0; unwind_exit + * so 'throw' is a jump or an exit, 'catch (...)' and the destructors that run on + * the way are ordinary blocks. + * + * vmlinux.h and bpf_helpers.h are not C++. + */ +#define SEC(name) __attribute__((section(name), used)) +#define __ksym __attribute__((section(".ksyms"))) +#define __noinline __attribute__((noinline)) +#define __always_inline inline __attribute__((always_inline)) + +#ifdef __BPF_FEATURE_UNWIND + +struct __sk_buff { + unsigned int len; +}; + +extern "C" { +void bpf_rcu_read_lock(void) noexcept __ksym; +void bpf_rcu_read_unlock(void) noexcept __ksym; +void bpf_preempt_disable(void) noexcept __ksym; +void bpf_preempt_enable(void) noexcept __ksym; + +/* Starts an unwind. Not a kfunc: llvm lowers the call and it doesn't come back. */ +int _Unwind_RaiseException(void *exc); + +/* All that is left of the C++ runtime. Nobody looks at the exception object. */ +static char exc_obj[16]; + +__always_inline void *__cxa_allocate_exception(unsigned long size) noexcept +{ + return exc_obj; +} + +__always_inline void __cxa_throw(void *exc, void *tinfo, void (*dtor)(void *)) +{ + _Unwind_RaiseException(exc); +} + +__always_inline void *__cxa_begin_catch(void *exc) noexcept +{ + return exc; +} + +__always_inline void __cxa_end_catch(void) noexcept +{ +} +} + +/* One decimal digit per destructor, in the order they ran */ +volatile int dtors SEC(".data.dtors"); + +/* The verifier insists that both are undone on every path out of the prog */ +struct rcu_guard { + __always_inline rcu_guard() + { + bpf_rcu_read_lock(); + } + __always_inline ~rcu_guard() + { + bpf_rcu_read_unlock(); + dtors = dtors * 10 + 1; + } +}; + +struct plain_rcu_guard { + __always_inline plain_rcu_guard() + { + bpf_rcu_read_lock(); + } + __always_inline ~plain_rcu_guard() + { + bpf_rcu_read_unlock(); + } +}; + +struct preempt_guard { + __always_inline preempt_guard() + { + bpf_preempt_disable(); + } + __always_inline ~preempt_guard() + { + bpf_preempt_enable(); + dtors = dtors * 10 + 2; + } +}; + +static __noinline int inner(int len) +{ + rcu_guard guard; + + if (len > 100) + throw 1; + return len; +} + +static __noinline int middle(int len) +{ + preempt_guard guard; + + return inner(len) + 1; +} + +using u128 = unsigned __int128; + +/* Global subprogs returning 16 bytes use R0:R2, including unwind exits. */ +extern "C" __noinline u128 aggregate_return(int len) +{ + plain_rcu_guard guard; + + if (len > 100) + throw 1; + return (static_cast<u128>(len) << 64) | 7; +} + +extern "C" SEC("tc") int catches(struct __sk_buff *skb) +{ + try { + return middle(skb->len); + } catch (...) { + return 42; + } +} + +/* The exception leaves the prog: it returns 0 */ +extern "C" SEC("tc") int does_not_catch(struct __sk_buff *skb) +{ + return middle(skb->len); +} + +extern "C" SEC("tc") int catches_aggregate(struct __sk_buff *skb) +{ + try { + u128 value = aggregate_return(skb->len); + + return static_cast<unsigned long>(value) + + static_cast<unsigned long>(value >> 64); + } catch (...) { + return 42; + } +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c b/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c new file mode 100644 index 000000000000..b0ecccede47e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +#define MIX_A 0xdeadbeefcafef00dULL +#define MIX_B 0x0123456789abcdefULL + +struct pair { + __u64 lo; + __u64 hi; +}; + +struct small { + __u32 a; + __u32 b; +}; + +union upair { + __u64 halves[2]; + struct { + __u64 lo; + __u64 hi; + } parts; +}; + +struct with_ptr { + void *p; + __u64 x; +}; + +static __noinline __u64 take_pair(int a, struct pair p, int c) +{ + return (__u64)a + p.lo + p.hi + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_static_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p = { .lo = a, .hi = b }; + + if (take_pair(1, p, 2) != a + b + 3) + return 1; + + return 0; +} + +#if defined(__clang__) + +__noinline __u64 take_pair_global(int a, struct pair p, int c) +{ + return (__u64)a + p.lo + p.hi + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_global_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p = { .lo = a, .hi = b }; + + if (take_pair_global(1, p, 2) != a + b + 3) + return 1; + + return 0; +} + +__noinline __u64 take_two_pairs_global(struct pair p, struct pair q, int d) +{ + return p.lo + p.hi + q.lo + q.hi + d; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_two_structs_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p = { .lo = a, .hi = b }; + struct pair q = { .lo = a + 1, .hi = b + 2 }; + + if (take_two_pairs_global(p, q, 3) != 2 * a + 2 * b + 6) + return 1; + + return 0; +} + +__noinline __u64 take_small_global(int a, struct small s, int c) +{ + return (__u64)a + s.a + s.b + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_small_struct_c_test(struct __sk_buff *skb) +{ + __u32 a = skb->len ^ (__u32)MIX_A; + __u32 b = skb->len ^ (__u32)MIX_B; + struct small s = { .a = a, .b = b }; + + if (take_small_global(1, s, 2) != (__u64)a + b + 3) + return 1; + + return 0; +} + +__noinline __u64 take_upair_global(int a, union upair u, int c) +{ + return (__u64)a + u.parts.lo + u.parts.hi + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_union_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + union upair u; + + u.halves[0] = a; + u.halves[1] = b; + if (take_upair_global(1, u, 2) != a + b + 3) + return 1; + + return 0; +} + +#endif + +#ifdef __SIZEOF_INT128__ + +typedef unsigned __int128 u128; + +__noinline __u64 take_i128_global(int a, u128 v, int c) +{ + return (__u64)a + (__u64)(v >> 64) + (__u64)v + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_int128_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + + if (take_i128_global(1, v, 2) != a + b + 3) + return 1; + + return 0; +} + +#endif /* __SIZEOF_INT128__ */ + +#if defined(__BPF_FEATURE_STACK_ARGUMENT) + +static __noinline __u64 take_spilled_pair(int a, int b, int c, int d, struct pair p) +{ + return (__u64)a + b + c + d + p.lo + p.hi; +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_spilled_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p = { .lo = a, .hi = b }; + int n = skb->len; + + if (take_spilled_pair(n, n + 1, n + 2, n + 3, p) != a + b + 4 * n + 6) + return 1; + + return 0; +} + +#endif + +__noinline __u64 take_with_ptr_global(struct with_ptr s) +{ + return s.x; +} + +SEC("tc") +__failure __msg("type STRUCT in take_with_ptr_global() is not composed of scalars") +int aggregate_arg_ptr_member_fail(struct __sk_buff *skb) +{ + struct with_ptr s = { .p = skb, .x = skb->len }; + + return take_with_ptr_global(s); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c b/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c new file mode 100644 index 000000000000..f5c1166027a1 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "../test_kmods/bpf_testmod_kfunc.h" +#include "bpf_misc.h" + +#if defined(__clang_major__) && __clang_major__ >= 23 + +#define MIX_A 0xdeadbeefcafef00dULL +#define MIX_B 0x0123456789abcdefULL + +typedef unsigned __int128 u128; + +struct pair { + __u64 lo; /* R0 */ + __u64 hi; /* R2 */ +}; + +union upair { + __u64 halves[2]; + struct { + __u64 lo; /* R0 */ + __u64 hi; /* R2 */ + } parts; +}; + +static __noinline u128 make_i128(__u64 a, __u64 b) +{ + return ((u128)(a + b) << 64) | (a - b); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_int128_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v; + + v = make_i128(a, b); + if ((__u64)(v >> 64) != a + b) + return 1; + if ((__u64)v != a - b) + return 2; + + return 0; +} + +static __noinline struct pair make_pair(__u64 a, __u64 b) +{ + struct pair p = { .lo = a + b, .hi = a - b }; + + return p; +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p; + + p = make_pair(a, b); + if (p.lo != a + b) + return 1; + if (p.hi != a - b) + return 2; + + return 0; +} + +__noinline struct pair make_pair_global(__u64 a, __u64 b) +{ + struct pair p = { .lo = a + b, .hi = a - b }; + + return p; +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_global_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p; + + p = make_pair_global(a, b); + if (p.lo != a + b) + return 1; + if (p.hi != a - b) + return 2; + + return 0; +} + +static __noinline union upair make_upair(__u64 a, __u64 b) +{ + union upair p; + + p.halves[0] = a + b; + p.halves[1] = a - b; + return p; +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_union_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + union upair p; + + p = make_upair(a, b); + if (p.parts.lo != a + b) + return 1; + if (p.parts.hi != a - b) + return 2; + + return 0; +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_int128_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v; + + v = bpf_kfunc_call_test_i128(a, b); + if ((__u64)(v >> 64) != a + b) + return 1; + if ((__u64)v != a - b) + return 2; + + return 0; +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_ret_pair p; + + p = bpf_kfunc_call_test_ret_pair(a, b); + if (p.lo != a + b) + return 1; + if (p.hi != a - b) + return 2; + + return 0; +} + +#else + +SEC("socket") +__description("verifier_aggregate_ret: needs LLVM 23, dummy test") +__skip("needs LLVM 23") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index 3e33766547c0..1ce9c4b78481 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c @@ -783,4 +783,52 @@ int check_arena_arg_ret(void *ctx) return 0; } +#if defined(__clang_major__) && __clang_major__ >= 23 + +struct arena_word_pair { + u32 __arena *first; + u32 __arena *second; +}; + +__weak struct arena_word_pair arena_word_pair(u32 __arena *page) +{ + struct arena_word_pair pair; + + pair.first = page; + pair.second = page + 1; + + return pair; +} + +SEC("syscall") +__load_if_JITed() +__success __retval(0) +int check_arena_struct_ret(void *ctx) +{ + u32 __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + u32 volatile __arena *first, *second; + struct arena_word_pair pair; + + if (!page) + return 1; + + pair = arena_word_pair(page); + if (!pair.first || !pair.second) + return 2; + + /* Both halves of the pair must still be usable as arena pointers. */ + first = pair.first; + second = pair.second; + *first = 1; + *second = 2; + if (*first != 1) + return 3; + if (*second != 2) + return 4; + + return 0; +} + +#endif + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_large.c b/tools/testing/selftests/bpf/progs/verifier_arena_large.c index 6ab8730d4878..dbdea14ca76f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena_large.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena_large.c @@ -10,6 +10,9 @@ #include <bpf_arena_common.h> #define ARENA_SIZE (1ull << 32) +#define LARGE_PAGE_CNT 1025 + +volatile int zero = 0; struct { __uint(type, BPF_MAP_TYPE_ARENA); @@ -284,6 +287,7 @@ int big_alloc2(void *ctx) return 0; } +/* Nonsleepable because it binds to a socket program. */ SEC("socket") __success __retval(0) int big_alloc3(void *ctx) @@ -291,24 +295,56 @@ int big_alloc3(void *ctx) #if defined(__BPF_FEATURE_ADDR_SPACE_CAST) char __arena *pages; u64 i; + int err = 0; - /* - * Allocate 2051 pages in one go to check how kmalloc_nolock() handles large requests. - * Since kmalloc_nolock() can allocate up to 1024 struct page * at a time, this call should - * result in three batches: two batches of 1024 pages each, followed by a final batch of 3 - * pages. - */ - pages = bpf_arena_alloc_pages(&arena, NULL, 2051, NUMA_NO_NODE, 0); + /* Verify that a nonsleepable allocation larger than 1024 pages succeeds. */ + pages = bpf_arena_alloc_pages(&arena, NULL, LARGE_PAGE_CNT, NUMA_NO_NODE, 0); if (!pages) - return 0; + return 1; + + for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) + pages[i * PAGE_SIZE] = 123; + + for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) { + if (pages[i * PAGE_SIZE] == 123) + continue; + err = 2; + break; + } + + bpf_arena_free_pages(&arena, pages, LARGE_PAGE_CNT); + return err; +#endif + return 0; +} - bpf_for(i, 0, 2051) - pages[i * PAGE_SIZE] = 123; - bpf_for(i, 0, 2051) - if (pages[i * PAGE_SIZE] != 123) - return i; +/* SYSCALL programs are always sleepable. */ +SEC("syscall") +__success __retval(0) +int big_alloc4(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + char __arena *pages; + u64 i; + int err = 0; + + /* Verify that a sleepable allocation larger than 1024 pages succeeds. */ + pages = bpf_arena_alloc_pages(&arena, NULL, LARGE_PAGE_CNT, NUMA_NO_NODE, 0); + if (!pages) + return 1; + + for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) + pages[i * PAGE_SIZE] = 123; + + for (i = zero; i < LARGE_PAGE_CNT && can_loop; i++) { + if (pages[i * PAGE_SIZE] == 123) + continue; + err = 2; + break; + } - bpf_arena_free_pages(&arena, pages, 2051); + bpf_arena_free_pages(&arena, pages, LARGE_PAGE_CNT); + return err; #endif return 0; } diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c new file mode 100644 index 000000000000..6f0efd9202ca --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c @@ -0,0 +1,1155 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "../../../include/linux/filter.h" +#include "bpf_misc.h" + +void *bpf_arena_alloc_pages(void *map, void *addr, __u32 page_cnt, int node_id, + __u64 flags) __ksym; +void bpf_arena_free_pages(void *map, void *ptr, __u32 page_cnt) __ksym; + +#ifdef __TARGET_ARCH_arm64 +#define ARENA_VM_START (1ull << 32) +#else +#define ARENA_VM_START (1ull << 44) +#endif + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 4); + __ulong(map_extra, ARENA_VM_START); +} arena SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1); + __type(key, int); + __type(value, long long); +} hash SEC(".maps"); + +/* JITs that take BPF_F_ARENA_SCALAR */ +#define __arena_scalar __flag(BPF_F_ARENA_SCALAR) __arch_x86_64 __arch_arm64 + +/* BTF FUNC records are not generated for kfuncs referenced from inline assembly */ +void __kfunc_btf_root(void) +{ + bpf_arena_alloc_pages(0, 0, 0, 0, 0); + bpf_arena_free_pages(0, 0, 0); +} + +/* Tests start with r6 = address of a new page as the user space sees it, a number */ + +SEC("syscall") +__arena_scalar +__description("arena_scalar: load and store of every size") +__success __retval(0) +__load_if_JITed() +__naked void ld_st_sizes(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = r6; \ + r1 = 0x1122334455667788 ll; \ + *(u64 *)(r6 + 0) = r1; \ + *(u32 *)(r6 + 8) = r1; \ + *(u16 *)(r6 + 12) = r1; \ + *(u8 *)(r6 + 14) = r1; \ + *(u64 *)(r6 + 16) = 0x1234; \ + *(u32 *)(r6 + 24) = 0x5678; \ + *(u16 *)(r6 + 28) = 0x9a; \ + *(u8 *)(r6 + 30) = 0xbc; \ + r0 = 1; \ + r2 = *(u64 *)(r6 + 0); \ + if r2 != r1 goto 9f; \ + r0 = 2; \ + r2 = *(u32 *)(r6 + 8); \ + if r2 != 0x55667788 goto 9f; \ + r0 = 3; \ + r2 = *(u16 *)(r6 + 12); \ + if r2 != 0x7788 goto 9f; \ + r0 = 4; \ + r2 = *(u8 *)(r6 + 14); \ + if r2 != 0x88 goto 9f; \ + r0 = 5; \ + r2 = *(u64 *)(r6 + 16); \ + if r2 != 0x1234 goto 9f; \ + r0 = 6; \ + r2 = *(u32 *)(r6 + 24); \ + if r2 != 0x5678 goto 9f; \ + r0 = 7; \ + r2 = *(u16 *)(r6 + 28); \ + if r2 != 0x9a goto 9f; \ + r0 = 8; \ + r2 = *(u8 *)(r6 + 30); \ + if r2 != 0xbc goto 9f; \ + /* the address is what it was */ \ + r0 = 9; \ + if r6 != r7 goto 9f; \ + r0 = 10; \ + r7 >>= 32; \ + if r7 == 0 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: load into the register that holds the address") +__success __retval(0) +__load_if_JITed() +__naked void ld_into_base(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = 77; \ + *(u64 *)(r6 + 0) = r1; \ + r6 = *(u64 *)(r6 + 0); \ + r0 = 1; \ + if r6 != 77 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: sign extending load") +__success __retval(0) +__load_if_JITed() +__naked void ldsx(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = r6; \ + *(u64 *)(r6 + 0) = 0x80; \ + .8byte %[ldsx_insn]; /* r2 = *(s8 *)(r6 + 0) */ \ + r0 = 1; \ + if r2 != -128 goto 9f; \ + r0 = 2; \ + if r6 != r7 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(ldsx_insn, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | BPF_B, + BPF_REG_2, BPF_REG_6, 0, 0)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: address loaded from arena") +__success __retval(0) +__load_if_JITed() +__naked void ptr_chase(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + /* page[0] = &page[64]; page[64] = 5; */ \ + r1 = r6; \ + r1 += 64; \ + *(u64 *)(r6 + 0) = r1; \ + *(u64 *)(r1 + 0) = 5; \ + r2 = *(u64 *)(r6 + 0); \ + r0 = 1; \ + if r2 != r1 goto 9f; \ + r3 = *(u64 *)(r2 + 0); \ + r0 = 2; \ + if r3 != 5 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: atomics") +__success __retval(0) +__load_if_JITed() +__naked void atomics(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = r6; \ + *(u64 *)(r6 + 8) = 1; \ + r1 = 2; \ + lock *(u64 *)(r6 + 8) += r1; \ + r1 = 4; \ + .8byte %[fetch_add_insn]; /* r1 = atomic_fetch_add((u64 *)(r6 + 8), r1) */ \ + r0 = 1; \ + if r1 != 3 goto 9f; \ + r1 = 8; \ + .8byte %[xchg_insn]; /* r1 = xchg_64(r6 + 8, r1) */ \ + r0 = 2; \ + if r1 != 7 goto 9f; \ + r0 = 8; \ + r1 = 16; \ + .8byte %[cmpxchg_insn]; /* r0 = cmpxchg_64(r6 + 8, r0, r1) */ \ + r2 = r0; \ + r0 = 3; \ + if r2 != 8 goto 9f; \ + r2 = *(u64 *)(r6 + 8); \ + r0 = 4; \ + if r2 != 16 goto 9f; \ + r0 = 5; \ + if r6 != r7 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(fetch_add_insn, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_6, BPF_REG_1, 8)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: fetching and/or/xor") +__success __retval(0) +__load_if_JITed() +__naked void fetch_and_or_xor(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = 0x11000000110 ll; \ + *(u64 *)(r6 + 8) = r7; \ + r0 = r6; \ + r0 ^= 0x55; \ + r1 = 0x1100000011 ll; \ + .8byte %[fetch_or_dw]; /* r1 = atomic_fetch_or((u64 *)(r6 + 8), r1) */ \ + r2 = r6; \ + r2 ^= 0x55; \ + if r0 == r2 goto 1f; \ + r0 = 99; \ + goto 9f; \ +1: r0 = 1; \ + if r1 != r7 goto 9f; \ + r7 = 0x11100000111 ll; \ + r1 = 0x11000000111 ll; \ + .8byte %[fetch_and_dw]; /* r1 = atomic_fetch_and((u64 *)(r6 + 8), r1) */ \ + r0 = 2; \ + if r1 != r7 goto 9f; \ + r7 = 0x11000000111 ll; \ + r1 = 0x101 ll; \ + .8byte %[fetch_xor_dw]; /* r1 = atomic_fetch_xor((u64 *)(r6 + 8), r1) */ \ + r0 = 3; \ + if r1 != r7 goto 9f; \ + r7 = 0x11000000010 ll; \ + r2 = *(u64 *)(r6 + 8); \ + r0 = 4; \ + if r2 != r7 goto 9f; \ + *(u32 *)(r6 + 16) = 0x110; \ + r1 = 0xdead000000000011 ll; \ + .8byte %[fetch_or_w]; /* r1 = atomic_fetch_or((u32 *)(r6 + 16), r1) */ \ + r0 = 5; \ + if r1 != 0x110 goto 9f; \ + r1 = 0xdead000000000101 ll; \ + .8byte %[fetch_and_w]; /* r1 = atomic_fetch_and((u32 *)(r6 + 16), r1) */ \ + r0 = 6; \ + if r1 != 0x111 goto 9f; \ + r1 = 0xdead000000000011 ll; \ + .8byte %[fetch_xor_w]; /* r1 = atomic_fetch_xor((u32 *)(r6 + 16), r1) */ \ + r0 = 7; \ + if r1 != 0x101 goto 9f; \ + r2 = *(u32 *)(r6 + 16); \ + r0 = 8; \ + if r2 != 0x110 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(fetch_and_dw, BPF_ATOMIC_OP(BPF_DW, BPF_AND | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(fetch_xor_dw, BPF_ATOMIC_OP(BPF_DW, BPF_XOR | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(fetch_or_w, BPF_ATOMIC_OP(BPF_W, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 16)), + __imm_insn(fetch_and_w, BPF_ATOMIC_OP(BPF_W, BPF_AND | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 16)), + __imm_insn(fetch_xor_w, BPF_ATOMIC_OP(BPF_W, BPF_XOR | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 16)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: fetching and/or/xor with the operand in r0") +__success __retval(0) +__load_if_JITed() +__naked void fetch_and_or_xor_r0(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = 0x11000000110 ll; \ + *(u64 *)(r6 + 8) = r7; \ + r0 = 0x1100000011 ll; \ + .8byte %[fetch_or_dw]; /* r0 = atomic_fetch_or((u64 *)(r6 + 8), r0) */ \ + r2 = r0; \ + r0 = 1; \ + if r2 != r7 goto 9f; \ + r7 = 0x11100000111 ll; \ + r0 = 0x11000000111 ll; \ + .8byte %[fetch_and_dw]; /* r0 = atomic_fetch_and((u64 *)(r6 + 8), r0) */ \ + r2 = r0; \ + r0 = 2; \ + if r2 != r7 goto 9f; \ + r7 = 0x11000000111 ll; \ + r0 = 0x101 ll; \ + .8byte %[fetch_xor_dw]; /* r0 = atomic_fetch_xor((u64 *)(r6 + 8), r0) */ \ + r2 = r0; \ + r0 = 3; \ + if r2 != r7 goto 9f; \ + r7 = 0x11000000010 ll; \ + r2 = *(u64 *)(r6 + 8); \ + r0 = 4; \ + if r2 != r7 goto 9f; \ + *(u32 *)(r6 + 16) = 0x110; \ + r0 = 0xdead000000000011 ll; \ + .8byte %[fetch_or_w]; /* r0 = atomic_fetch_or((u32 *)(r6 + 16), r0) */ \ + r2 = r0; \ + r0 = 5; \ + if r2 != 0x110 goto 9f; \ + r0 = 0xdead000000000101 ll; \ + .8byte %[fetch_and_w]; /* r0 = atomic_fetch_and((u32 *)(r6 + 16), r0) */ \ + r2 = r0; \ + r0 = 6; \ + if r2 != 0x111 goto 9f; \ + r0 = 0xdead000000000011 ll; \ + .8byte %[fetch_xor_w]; /* r0 = atomic_fetch_xor((u32 *)(r6 + 16), r0) */ \ + r2 = r0; \ + r0 = 7; \ + if r2 != 0x101 goto 9f; \ + r2 = *(u32 *)(r6 + 16); \ + r0 = 8; \ + if r2 != 0x110 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 8)), + __imm_insn(fetch_and_dw, BPF_ATOMIC_OP(BPF_DW, BPF_AND | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 8)), + __imm_insn(fetch_xor_dw, BPF_ATOMIC_OP(BPF_DW, BPF_XOR | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 8)), + __imm_insn(fetch_or_w, BPF_ATOMIC_OP(BPF_W, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 16)), + __imm_insn(fetch_and_w, BPF_ATOMIC_OP(BPF_W, BPF_AND | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 16)), + __imm_insn(fetch_xor_w, BPF_ATOMIC_OP(BPF_W, BPF_XOR | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 16)) + : __clobber_all); +} + +/* + * The page is gone, so the access faults. The fetched register has to come back + * 0 and everything else has to be left alone, r0 included. Reading the address + * again has to give 0 as well: had the fault been recovered by the arena mapping + * its zeroed scratch page instead of by the JIT's exception table entry, the + * access would have succeeded and left a 1 there. + */ +SEC("syscall") +__arena_scalar +__description("arena_scalar: fetching or through a number to a freed page") +__success __retval(0) +__load_if_JITed() +__naked void fetch_or_freed_page(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r6 + 8) = 0x55; \ + r1 = %[arena] ll; \ + r2 = r6; \ + r3 = 1; \ + call %[bpf_arena_free_pages]; \ + r0 = r6; \ + r0 ^= 0x55; \ + r1 = 1; \ + .8byte %[fetch_or_dw]; /* faults: r1 = atomic_fetch_or((u64 *)(r6 + 8), r1) */ \ + r2 = r6; \ + r2 ^= 0x55; \ + if r0 == r2 goto 1f; \ + r0 = 1; \ + goto 9f; \ +1: r0 = 2; \ + if r1 != 0 goto 9f; \ + r3 = *(u64 *)(r6 + 8); \ + r0 = 3; \ + if r3 != 0 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm(bpf_arena_free_pages), + __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 8)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: fetching or with the operand in r0 to a freed page") +__success __retval(0) +__load_if_JITed() +__naked void fetch_or_r0_freed_page(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r6 + 8) = 0x55; \ + r1 = %[arena] ll; \ + r2 = r6; \ + r3 = 1; \ + call %[bpf_arena_free_pages]; \ + r0 = 1; \ + .8byte %[fetch_or_dw]; /* faults: r0 = atomic_fetch_or((u64 *)(r6 + 8), r0) */ \ + r2 = r0; \ + r0 = 1; \ + if r2 != 0 goto 9f; \ + r3 = *(u64 *)(r6 + 8); \ + r0 = 2; \ + if r3 != 0 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm(bpf_arena_free_pages), + __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, + BPF_REG_6, BPF_REG_0, 8)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: cmpxchg through r0") +__success __retval(0) +__load_if_JITed() +__naked void cmpxchg_r0(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + /* The address is in r0. The value at the address is not equal to it. */ \ + *(u64 *)(r6 + 0) = 3; \ + r0 = r6; \ + r1 = 5; \ + .8byte %[cmpxchg_insn]; /* r0 = cmpxchg_64(r0 + 0, r0, r1) */ \ + r2 = r0; \ + r0 = 1; \ + if r2 != 3 goto 9f; \ + r2 = *(u64 *)(r6 + 0); \ + r0 = 2; \ + if r2 != 3 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_0, BPF_REG_1, 0)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: xchg into the register that holds the address") +__success __retval(0) +__load_if_JITed() +__naked void xchg_into_base(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r6 + 0) = 3; \ + r1 = r6; \ + .8byte %[xchg_insn]; /* r1 = xchg_64(r1 + 0, r1) */ \ + r0 = 1; \ + if r1 != 3 goto 9f; \ + r2 = *(u64 *)(r6 + 0); \ + r0 = 2; \ + if r2 != r6 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_1, 0)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: xchg into the register that holds a pointer to stack") +__failure __msg("misaligned access off (0x0; 0xffffffffffffffff)+0 size 8") +__naked void xchg_into_stack_ptr(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r1 = 0; \ + *(u64 *)(r10 - 8) = r1; \ + r1 = r10; \ + r1 += -8; \ + .8byte %[xchg_insn]; /* r1 = xchg_64(r1 + 0, r1) */ \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(arena), + __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_1, 0)) + : __clobber_all); +} + +#ifdef CAN_USE_LOAD_ACQ_STORE_REL + +SEC("syscall") +__arena_scalar +__description("arena_scalar: load-acquire and store-release") +__success __retval(0) +__load_if_JITed() +__naked void load_acq_store_rel(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = r6; \ + r1 = 0x1234; \ + .8byte %[store_release_insn]; /* store_release((u64 *)(r6 + 8), r1) */ \ + .8byte %[load_acquire_insn]; /* r2 = load_acquire((u64 *)(r6 + 8)) */ \ + r0 = 1; \ + if r2 != 0x1234 goto 9f; \ + .8byte %[load_acquire8_insn]; /* w2 = load_acquire((u8 *)(r6 + 8)) */ \ + r0 = 2; \ + if r2 != 0x34 goto 9f; \ + r0 = 3; \ + if r6 != r7 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(store_release_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(load_acquire_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_6, 8)), + __imm_insn(load_acquire8_insn, + BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_6, 8)) + : __clobber_all); +} + +#endif /* CAN_USE_LOAD_ACQ_STORE_REL */ + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number that is not an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void not_in_arena(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + /* nothing is allocated: all loads read 0, stores are dropped */ \ + r6 = 0xdeadbeef00000000 ll; \ + r0 = 1; \ + r2 = *(u64 *)(r6 + 0); \ + if r2 != 0 goto 9f; \ + r0 = 2; \ + r2 = *(u8 *)(r6 - 32768); \ + if r2 != 0 goto 9f; \ + r6 = 0x12345678ffffffff ll; \ + r0 = 3; \ + r2 = *(u64 *)(r6 + 32760); \ + if r2 != 0 goto 9f; \ + *(u64 *)(r6 + 32760) = 1; \ + *(u8 *)(r6 + 32767) = r2; \ + r1 = 1; \ + lock *(u64 *)(r6 + 32760) += r1; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: store through the address of the stack as a number") +__success __retval(0) +__load_if_JITed() +__naked void stack_addr_as_number(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + *(u64 *)(r10 - 8) = 5; \ + r6 = r10; \ + r6 |= 0; \ + /* r6 is a number now. The store goes to arena, not to the stack. */ \ + *(u64 *)(r6 - 8) = 7; \ + r1 = 9; \ + *(u64 *)(r6 - 8) = r1; \ + lock *(u64 *)(r6 - 8) += r1; \ + r2 = *(u64 *)(r10 - 8); \ + r0 = 1; \ + if r2 != 5 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: 64-bit math on the address stays 64-bit") +__success __retval(0) +__load_if_JITed() +__naked void alu64_after_access(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r2 = *(u64 *)(r6 + 0); \ + r7 = r6; \ + r7 += 8; \ + r7 -= r6; \ + r0 = 1; \ + if r7 != 8 goto 9f; \ + r7 = r6; \ + r7 >>= 32; \ + r0 = 2; \ + if r7 == 0 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: st of an immediate changes no register") +__success __retval(0) +__load_if_JITed() +__naked void st_keeps_regs(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r0 = r6; \ + r1 = 0x1111; \ + r2 = 0x2222; \ + *(u64 *)(r0 + 0) = 5; \ + r3 = r0; \ + r0 = 1; \ + if r3 != r6 goto 9f; \ + r0 = 2; \ + if r1 != 0x1111 goto 9f; \ + r0 = 3; \ + if r2 != 0x2222 goto 9f; \ + r0 = 0x3333; \ + r1 = r6; \ + *(u32 *)(r1 + 8) = -7; \ + r3 = r0; \ + r0 = 4; \ + if r3 != 0x3333 goto 9f; \ + r0 = 5; \ + if r1 != r6 goto 9f; \ + r0 = 6; \ + r3 = *(u64 *)(r6 + 0); \ + if r3 != 5 goto 9f; \ + r0 = 7; \ + r3 = *(u64 *)(r6 + 8); \ + r4 = 0xfffffff9 ll; \ + if r3 != r4 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: access through a number with garbage in the upper half") +__success __retval(0) +__load_if_JITed() +__naked void st_value_garbage(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = 0xdeadbeef00001000 ll; \ + *(u64 *)(r6 + 2048) = r1; \ + r7 = *(u64 *)(r6 + 2048); \ + r8 = *(u64 *)(r6 + 2048); \ + r1 = 3; \ + *(u64 *)(r8 + 0) = 1; \ + r0 = 1; \ + if r8 != r7 goto 9f; \ + *(u8 *)(r8 + 1) = 1; \ + r0 = 2; \ + if r8 != r7 goto 9f; \ + *(u32 *)(r8 + 4) = r1; \ + r0 = 3; \ + if r8 != r7 goto 9f; \ + r2 = *(u16 *)(r8 + 2); \ + r0 = 4; \ + if r8 != r7 goto 9f; \ + r0 = 5; \ + if r1 != 3 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: access through a number without the upper half") +__success __retval(0) +__load_if_JITed() +__naked void st_value_small(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = 0x1000 ll; \ + *(u64 *)(r6 + 2048) = r1; \ + r7 = *(u64 *)(r6 + 2048); \ + r8 = *(u64 *)(r6 + 2048); \ + r1 = 3; \ + *(u64 *)(r8 + 0) = 1; \ + r0 = 1; \ + if r8 != r7 goto 9f; \ + *(u8 *)(r8 + 1) = 1; \ + r0 = 2; \ + if r8 != r7 goto 9f; \ + *(u32 *)(r8 + 4) = r1; \ + r0 = 3; \ + if r8 != r7 goto 9f; \ + r2 = *(u16 *)(r8 + 2); \ + r0 = 4; \ + if r8 != r7 goto 9f; \ + r0 = 5; \ + if r1 != 3 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: atomics through a number that is not an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void atomic_value(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = 0xdeadbeef00001000 ll; \ + *(u64 *)(r6 + 2048) = r1; \ + r7 = *(u64 *)(r6 + 2048); \ + r8 = *(u64 *)(r6 + 2048); \ + r1 = 3; \ + lock *(u64 *)(r8 + 0) += r1; \ + r0 = 1; \ + if r8 != r7 goto 9f; \ + .8byte %[fetch_add_insn]; \ + r0 = 2; \ + if r8 != r7 goto 9f; \ + .8byte %[xchg_insn]; \ + r0 = 3; \ + if r8 != r7 goto 9f; \ + r0 = 0; \ + .8byte %[cmpxchg_insn]; \ + r0 = 4; \ + if r8 != r7 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(fetch_add_insn, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, + BPF_REG_8, BPF_REG_1, 0)), + __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_8, BPF_REG_1, 0)), + __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_8, BPF_REG_1, 0)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number and pointer to arena at the same insn") +__success __retval(0) +__load_if_JITed() +__naked void mixed_number_arena(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r6 + 8) = 0x1234; \ + /* a number that the verifier does not know, 0 at run time */ \ + r7 = *(u64 *)(r6 + 16); \ + r8 = r6; \ + if r7 != 0 goto 1f; \ + .8byte %[cast_kern_insn]; \ +1: *(u8 *)(r8 + 0) = 1; \ + *(u8 *)(r8 + 1) = r7; \ + r2 = *(u8 *)(r8 + 1); \ + lock *(u64 *)(r8 + 24) += r7; \ + r0 = 0; \ + if r7 != 0 goto 9f; \ + /* pointer to arena only: JIT adds all 64 bits of r8 to the base */ \ + r0 = 2; \ + r2 = *(u64 *)(r8 + 8); \ + if r2 != 0x1234 goto 9f; \ + r0 = 3; \ + r2 = *(u8 *)(r6 + 0); \ + if r2 != 1 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(cast_kern_insn, BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, + BPF_REG_8, BPF_REG_8, 1, 1)) + : __clobber_all); +} + +SEC("syscall") +__description("arena_scalar: no flag, no access through a number") +__failure __msg("R6 invalid mem access 'scalar'") +__naked void no_flag(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r6 = 0x100000000000 ll; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" : + : __imm_addr(arena) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: pointer that is NULL is an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void null_ptr(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r1 = 0; \ + *(u32 *)(r10 - 4) = r1; \ + r2 = r10; \ + r2 += -4; \ + r1 = %[hash] ll; \ + call %[bpf_map_lookup_elem]; \ + r1 = r0; \ + r0 = 1; \ + if r1 != 0 goto 9f; \ + /* nothing is allocated: the load reads 0 */ \ + r0 = *(u64 *)(r1 + 0); \ +9: exit; \ +" : + : __imm_addr(arena), + __imm_addr(hash), + __imm(bpf_map_lookup_elem) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: pointer that is NULL with an offset is an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void null_ptr_off(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r1 = 0; \ + *(u32 *)(r10 - 4) = r1; \ + r2 = r10; \ + r2 += -4; \ + r1 = %[hash] ll; \ + call %[bpf_map_lookup_elem]; \ + r1 = r0; \ + r0 = 1; \ + if r1 != 0 goto 9f; \ + r1 += 8; \ + *(u64 *)(r1 + 0) = 5; \ + r0 = *(u64 *)(r1 + 0); \ +9: exit; \ +" : + : __imm_addr(arena), + __imm_addr(hash), + __imm(bpf_map_lookup_elem) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number that is less than a page is an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void small_number(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + call %[bpf_get_prandom_u32]; \ + r1 = r0; \ + r1 &= 0xfff; \ + r0 = *(u64 *)(r1 + 0); \ + exit; \ +" : + : __imm_addr(arena), + __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number is not a pointer for a helper") +__failure __msg("R2 type=scalar expected=") +__naked void helper_arg(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = %[hash] ll; \ + r2 = r6; \ + call %[bpf_map_lookup_elem]; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm_addr(hash), + __imm(bpf_arena_alloc_pages), + __imm(bpf_map_lookup_elem) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number and pointer to stack at the same insn") +__failure __msg("same insn cannot be used with different pointers") +__load_if_JITed() +__naked void mixed_number_stack(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r10 - 8) = 0; \ + call %[bpf_get_prandom_u32]; \ + if w0 != 0 goto 1f; \ + r6 = r10; \ + r6 += -8; \ +1: r0 = *(u64 *)(r6 + 0); \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +static int st_cb(__u64 idx, void *ctx) +{ + volatile long *p = *(volatile long **)ctx; + + p[idx] = 7; + return 0; +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: store through a number in a callback") +__success __retval(0) +__load_if_JITed() +int st_in_callback(void *unused) +{ + volatile long *p = bpf_arena_alloc_pages(&arena, NULL, 1, -1, 0); + + if (!p) + return 100; + bpf_loop(4, st_cb, &p, 0); + return p[0] + p[3] + p[4] - 14; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds_step.c b/tools/testing/selftests/bpf/progs/verifier_bounds_step.c new file mode 100644 index 000000000000..d824a5d48a39 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_bounds_step.c @@ -0,0 +1,544 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* Tests for the linear "base + step * k" description tracked per scalar register. */ + +#define __no_step __not_msg("step=") __msg("\n") + +/* + * Safe ALU64 arithmetic preserves a non-power-of-two stride and scales its base: + * step=0+3 -> step=1+3 -> step=2+6 -> step=4+12. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 *= 3 {{.*}}step=0+3)") +__msg("r0 += 1 {{.*}}step=1+3)") +__msg("r0 *= 2 {{.*}}step=2+6)") +__msg("r0 <<= 1 {{.*}}step=4+12)") +__naked void step_arith64_no_overflow(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 0xff; \ + r0 *= 3; \ + r0 += 1; \ + r0 *= 2; \ + r0 <<= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Safe ALU32 multiplication and addition preserve the stride and scale its base: + * step=0+3 -> step=1+3 -> step=2+6. MOV32 preserves the result. + * LSH32 currently resets the stride, but must preserve the correct upper bound. + */ +SEC("socket") +__success __log_level(2) +__msg("w0 *= 3 {{.*}}step=0+3)") +__msg("w0 += 1 {{.*}}step=1+3)") +__msg("w0 *= 2 {{.*}}step=2+6)") +__msg("w1 = w0 {{.*}}R0=scalar({{.*}}step=2+6) R1=scalar({{.*}}step=2+6)") +__msg("w0 <<= 1 {{.*}}smax=umax=smax32=umax32=3064,") __no_step +__naked void step_arith32_no_overflow(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 0xff; \ + w0 *= 3; \ + w0 += 1; \ + w0 *= 2; \ + w1 = w0; \ + w0 <<= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Sign-crossing range with a non-power-of-2 step. After "*= 3; += -3" the value + * set is {-3, 0, 3, 6}. The line description is tracked in signed space, so the + * intersection keeps smax=6. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 += -3 {{.*}}smax=smax32=6,step=0+3)") +__naked void step_neg_value_range(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 3; \ + r0 *= 3; \ + r0 += -3; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * 0x49249249 * 7 wraps to U32_MAX; inferring step=0+7 would incorrectly + * cap the result at 0xfffffffc instead of U32_MAX. + * Adding 4 to {0xfffffffc, U32_MAX} wraps to {0, 3}; keeping step=1+3 + * incorrectly narrows the result to 1. + * Shifting the same input left by 1 gives {0xfffffff8, 0xfffffffe}; + * inferring step=0+6 incorrectly narrows the result to 0xfffffffc. + */ +SEC("socket") +__success __log_level(2) +__msg("w0 *= 7 {{.*}}umax=0xffffffff,") __no_step +__msg("r2 = r0 {{.*}}R0=scalar({{.*}}step=0+3) R2=scalar({{.*}}step=0+3)") +__msg("w0 += 4 {{.*}}smax=umax=smax32=umax32=3,") __no_step +__msg("w2 <<= 1 {{.*}}smax=umax=umax32=0xfffffffe,") __no_step +__naked void step_arith32_overflow(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w0 *= 7; \ + call %[bpf_get_prandom_u32]; \ + r0 &= 1; \ + r0 *= 3; \ + r1 = 0xfffffffc ll; \ + r0 += r1; \ + r2 = r0; \ + w0 += 4; \ + w2 <<= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Multiplying {0x2aaaaaaaaaaaaaab, 0x2aaaaaaaaaaaaaac} by 3 crosses S64_MAX + * and gives {S64_MIN + 1, S64_MIN + 4}. Keeping step=0+3 excludes both. + * + * Adding 5 to {S64_MAX - 4, S64_MAX - 1} gives {S64_MIN, S64_MIN + 3}. + * Updating the line to step=2+3 incorrectly narrows it to S64_MIN + 1. + * + * Adding -5 to {S64_MIN + 1, S64_MIN + 4} wraps to + * {S64_MAX - 3, S64_MAX}. The stale base 0 modulo 3 excludes S64_MAX. + * + * Shifting {0x4000000000000008, 0x400000000000000b} left by 1 gives + * {S64_MIN + 16, S64_MIN + 22}. Keeping step=0+6 excludes both. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 *= 3 {{.*}}smax=0x8000000000000004,") __no_step +__msg("r0 += r1 {{.*}}step=0+3)") +__msg("r0 += 5 {{.*}}smax=0x8000000000000003,") __no_step +__msg("r0 += r1 {{.*}}step=2+3)") +__msg("r0 += -5 {{.*}}umax=0x7fffffffffffffff,") __no_step +__msg("r0 += r1 {{.*}}step=0+3)") +__msg("r0 <<= 1 {{.*}}smax=0x8000000000000016,") __no_step +__naked void step_arith64_overflow(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 1; \ + r2 = r0; \ + r1 = 0x2aaaaaaaaaaaaaab ll; \ + r0 += r1; \ + r0 *= 3; \ + r2 *= 3; \ + r0 = r2; \ + r1 = 0x7ffffffffffffffb ll; \ + r0 += r1; \ + r0 += 5; \ + r0 = r2; \ + r1 = 0x8000000000000001 ll; \ + r0 += r1; \ + r0 += -5; \ + r0 = r2; \ + r1 = 0x4000000000000008 ll; \ + r0 += r1; \ + r0 <<= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * MOV32 maps {2^32, 2^32 + 3} to {0, 3}. The old base 1 modulo 3 + * excludes both results. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 += r1 {{.*}}step=1+3)") +__msg("w0 = w0 {{.*}}smax=umax=smax32=umax32=3,") __no_step +__naked void step_mov32_truncate(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 1; \ + r0 *= 3; \ + r1 = 0x100000000 ll; \ + r0 += r1; \ + w0 = w0; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * MOV32 maps {-4, -1} to {0xfffffffc, U32_MAX}. The old base 2 + * modulo 3 excludes U32_MAX. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 += -4 {{.*}}step=2+3)") +__msg("w0 = w0 {{.*}}umax=0xffffffff,") __no_step +__naked void step_mov32_negative(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 1; \ + r0 *= 3; \ + r0 += -4; \ + w0 = w0; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Sign-extending {128, 131} from s8 gives {-128, -125}. + * Keeping base 2 modulo 3 incorrectly narrows the result to -127. + * + * Sign-extending {32768, 32771} from s16 gives {-32768, -32765}. + * Keeping base 2 modulo 3 incorrectly narrows the result to -32767. + * + * Sign-extending {2^31, 2^31 + 3} from s32 gives {S32_MIN, S32_MIN + 3}. + * Keeping base 2 modulo 3 incorrectly narrows the result to S32_MIN + 1. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 += 128 {{.*}}step=2+3)") +__msg("r0 = (s8)r0 {{.*}}smax=smax32=-125,") __no_step +__msg("r0 += 32768 {{.*}}step=2+3)") +__msg("r0 = (s16)r0 {{.*}}smax=smax32=-32765,") __no_step +__msg("r0 += r1 {{.*}}step=2+3)") +__msg("r0 = (s32)r0 {{.*}}smax=0xffffffff80000003,") __no_step +__naked void step_movsx(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 1; \ + r0 *= 3; \ + r2 = r0; \ + r0 += 128; \ + r0 = (s8)r0; \ + r0 = r2; \ + r0 += 32768; \ + r0 = (s16)r0; \ + r0 = r2; \ + r1 = 0x80000000 ll; \ + r0 += r1; \ + r0 = (s32)r0; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * R1 = R0 + 1 shares R0's scalar ID. Refining R0 to {0, 3, 6} must + * translate its line as well as its bounds when updating R1 to {1, 4, 7}. + * Copying R0's base 0 modulo 3 excludes the reachable value 7. + */ +SEC("socket") +__success __log_level(2) +__msg("r1 += 1 {{.*}}step=1+3)") +__msg("if r0 > 0x6 {{.*}}R1=scalar({{.*}}smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=7,{{.*}}step=1+3)") +__naked void step_linked_regs_add64(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 3; \ + r0 *= 3; \ + r1 = r0; \ + r1 += 1; \ + if r0 > 6 goto l_out_%=; \ + r0 = r1; \ +l_out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * r0 &= 7 -> [0, 7] + * r0 *= 24 -> [0, 168] step=0+24 + * r0 += 8 -> [8, 176] step=8+24 + * r0 & 0x10 -> [32, 176] step=8+24 (var_off alone: umin=16) + * r0 > 80 -> [104, 176] step=8+24 (var_off alone: umin=88) + * r1 += 24 -> [128, 200] step=8+24 + * Bounds and step stay in sync after each refinement. + */ +SEC("socket") +__success __log_level(2) +__msg("r0 += 8 {{.*}}smin=umin=smin32=umin32=8,smax=umax=smax32=umax32=176,{{.*}}step=8+24)") +__msg("from 4 to 6: R0=scalar(smin=umin=smin32=umin32=32,smax=umax=smax32=umax32=176,{{.*}}step=8+24)") +__msg("r1 = r0 {{.*}}R1=scalar(id={{[0-9]+}},smin=umin=smin32=umin32=104,smax=umax=smax32=umax32=176,{{.*}}step=8+24)") +__msg("r1 += 24 {{.*}}R1=scalar({{.*}}smin=umin=smin32=umin32=128,smax=umax=smax32=umax32=200,{{.*}}step=8+24)") +__naked void step_bounds_resync_after_arith(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 7; \ + r0 *= 24; \ + r0 += 8; \ + if r0 & 0x10 goto l_set_%=; \ + goto l_out_%=; \ +l_set_%=: \ + if r0 <= 80 goto l_out_%=; \ + r1 = r0; \ + r1 += 24; \ + *(u64 *)(r10 - 8) = r1; \ +l_out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * r0 &= 7 -> [0, 7] + * r0 *= 24 -> [0, 168] step=0+24 + * r1 = (s16)r0 -> [0, 168] step=0+24 + * r0 += -24 -> [-24, 144] step=0+24 + * r2 = (s16)r0 -> [-24, 144] step=0+24 + * r3 = (s8)r0 -> [-128, 127] no step + * w4 = (s16)w1 -> [0, 168] step=0+24 + * w5 = (s8)w1 -> [-128, 127] no step + * Bounds and step stay in sync, the step is dropped when the value does not fit. + */ +SEC("socket") +__success __log_level(2) +__msg("r1 = (s16)r0 {{.*}}R1=scalar(id={{[0-9]+}},smin=smin32=0,smax=umax=smax32=umax32=168,{{.*}}step=0+24)") +__msg("r2 = (s16)r0 {{.*}}R2=scalar(smin=smin32=-24,smax=smax32=144,{{.*}}step=0+24)") +__msg("r3 = (s8)r0 {{.*}}R3=scalar(smin=smin32=-128,smax=smax32=127)") +__msg("w4 = (s16)w1 {{.*}}R4=scalar(id={{[0-9]+}},smin=smin32=0,smax=umax=smax32=umax32=168,{{.*}}step=0+24)") +__msg("w5 = (s8)w1 {{.*}}R5=scalar({{.*}}smin32=-128,smax32=127,var_off=(0x0; 0xffffffff))") +__naked void step_movsx_bounds_on_line(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 7; \ + r0 *= 24; \ + r1 = (s16)r0; \ + r0 += -24; \ + r2 = (s16)r0; \ + r3 = (s8)r0; \ + w4 = (s16)w1; \ + w5 = (s8)w1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +struct step_val { + __u8 data[1024]; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct step_val); +} step_map SEC(".maps"); + +/* Old register [4..130, step 2] should prune cur register [8..68, step 4]. */ +SEC("socket") +__success __log_level(2) +__msg("7: (27) r6 *= 4 ; R6=scalar({{.*}}umin32=8,{{.*}}umax32=68,{{.*}},step=0+4)") +__msg("10: (27) r7 *= 2 ; R7=scalar({{.*}}umin32=4,{{.*}}umax32=130,{{.*}},step=0+2)") +__msg("11: (25) if r0 > 0x2a goto pc+1") +__msg("from 11 to 13: safe") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void step_prune_hit_multiple(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + call %[bpf_get_prandom_u32]; \ + r7 = r0; \ + call %[bpf_get_prandom_u32]; \ + r6 &= 0x0f; \ + r6 += 2; \ + r6 *= 4; \ + r7 &= 0x3f; \ + r7 += 2; \ + r7 *= 2; \ + if r0 > 42 goto 1f; /* can't predict */ \ + r6 = r7; /* step=2 explored first, step=4 explored next */ \ +1: r0 = r10; \ + r6 = -r6; \ + r0 += r6; \ + *(u8 *)(r0 + 0) = 7; /* force r6 precise */ \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Old register [0..189, step 3] should not prune cur register [0..30, step 2]. + * Bounds and tnum containment allow pruning; step divisibility must prevent it. + */ +SEC("socket") +__success __log_level(2) +__msg("6: (27) r6 *= 2 ; R6=scalar({{.*}}smin32=0,{{.*}}umax32=30,{{.*}},step=0+2)") +__msg("8: (27) r7 *= 3 ; R7=scalar({{.*}}smin32=0,{{.*}}umax32=189,{{.*}},step=0+3)") +__msg("9: (25) if r0 > 0x2a goto pc+1") +__msg("17: (95) exit") +__msg("from 9 to 11: R6=scalar({{.*}},step=0+2)") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void step_prune_miss_non_multiple(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + call %[bpf_get_prandom_u32]; \ + r7 = r0; \ + call %[bpf_get_prandom_u32]; \ + r6 &= 0x0f; \ + r6 *= 2; \ + r7 &= 0x3f; \ + r7 *= 3; \ + if r0 > 42 goto 1f; /* can't predict */ \ + r6 = r7; /* step=3 explored first, step=2 explored next */ \ +1: \ + r2 = r10; \ + r6 += 1; \ + r6 = -r6; \ + r2 += r6; \ + *(u8 *)(r2 + 0) = 7; /* force r6 precise */ \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Constant current register lying on the cached line: cached is {0,3,6,...} + * (step 3, base 0), current is the constant 6. range_within() takes the + * constant branch: imod(6, 3) == base 0, so cur is on the line and the + * (precise) state is pruned -> "safe". + */ +SEC("socket") +__success __log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("14: (27) r1 *= 3") /* cached path: step 3 line */ +__msg("16: (b7) r1 = 6") /* current path: const 6, on the line */ +__msg("17: safe") /* pruned at join: imod(6, 3) == 0 */ +__naked void step_prune_hit_const_on_line(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + r1 = 0; \ + *(u32*)(r10 - 4) = r1; \ + r2 = r10; \ + r2 += -4; \ + r1 = %[step_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l_out_%=; \ + r7 = r0; \ + r1 = r6; \ + r1 &= 0xff; \ + if r6 > 0 goto l_cur_%=; \ + r1 *= 3; /* old: step 3 */ \ + goto l_join_%=; \ +l_cur_%=: \ + r1 = 6; /* cur: const on line */ \ +l_join_%=: \ + r0 = r7; \ + r0 += r1; /* r1 forced precise */ \ + r2 = *(u8 *)(r0 + 0); \ +l_out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32), + __imm(bpf_map_lookup_elem), + __imm_addr(step_map) + : __clobber_all); +} + +/* + * Constant current register NOT on the cached line: cached is {0,3,6,...} + * (step 3, base 0), current is the constant 7. imod(7, 3) == 1 != base 0, + * so range_within() fails and the join is traversed again. A power-of-two + * step is avoided on purpose: with step 3 the tnum is loose enough to admit + * 7, so imod() is + * the sole check that rejects it. No failure shape is possible here: 7 is + * within the line's bounds and tnum, and the cached line path already + * verifies the whole outro, so an eager prune could not miss an error. + */ +SEC("socket") +__success __log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("14: (27) r1 *= 3") /* cached path: step 3 line */ +__msg("16: (b7) r1 = 7") /* current path: const 7, off the line */ +/* not pruned: current continues past the join with the constant offset 7 */ +__msg("19: R0=map_value({{.*}}imm=7) R1=7") +__naked void step_prune_miss_const_off_line(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + r1 = 0; \ + *(u32*)(r10 - 4) = r1; \ + r2 = r10; \ + r2 += -4; \ + r1 = %[step_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l_out_%=; \ + r7 = r0; \ + r1 = r6; \ + r1 &= 0xff; \ + if r6 > 0 goto l_cur_%=; \ + r1 *= 3; /* old: step 3 */ \ + goto l_join_%=; \ +l_cur_%=: \ + r1 = 7; /* cur: const off line */ \ +l_join_%=: \ + r0 = r7; \ + r0 += r1; /* r1 forced precise */ \ + r2 = *(u8 *)(r0 + 0); \ +l_out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32), + __imm(bpf_map_lookup_elem), + __imm_addr(step_map) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c index a73b837553fb..e0c389102db9 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +++ b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c @@ -502,6 +502,175 @@ __naked void bad_helper_write(void) : __clobber_all); } +/* + * A helper buffer or a callee's pointer that reaches a fastcall spill slot + * must keep the spill/fill pair and the stack that covers it. Only load + * these programs: without the fix, they access kernel stack outside their + * frame. + * + * Uninitialized outputs without CAP_PERFMON have no explicit stack accesses. + */ +SEC("tc") +__log_level(4) +__msg_unpriv("subprog 0 (helper_uninit_stack) main {{.*}} stack 64") +__xlated_unpriv("*(u64 *)(r10 -8) = r1") +__xlated_unpriv("...") +__xlated_unpriv("r1 = *(u64 *)(r10 -8)") +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void helper_uninit_stack(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "r2 = 0;" + "r3 = r10;" + "r3 += -64;" + "r4 = 8;" + "call %[bpf_skb_load_bytes];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_smp_processor_id), + __imm(bpf_skb_load_bytes) + : __clobber_all); +} + +/* Same output in the caller's stack, passed to the helper by a callee. */ +static __used __naked void helper_uninit_stack_callee(void) +{ + asm volatile ( + "r3 = r2;" + "r2 = 0;" + "r4 = 8;" + "call %[bpf_skb_load_bytes];" + "exit;" + : + : __imm(bpf_skb_load_bytes) + : __clobber_all); +} + +SEC("tc") +__log_level(4) +__msg_unpriv("subprog 0 (helper_uninit_stack_caller) main {{.*}} stack 64") +__xlated_unpriv("*(u64 *)(r10 -8) = r1") +__xlated_unpriv("...") +__xlated_unpriv("r1 = *(u64 *)(r10 -8)") +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void helper_uninit_stack_caller(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "r2 = r10;" + "r2 += -64;" + "call helper_uninit_stack_callee;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_smp_processor_id) + : __clobber_all); +} + +/* A helper input whose only initialization is the fastcall spill. */ +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u64); +} fastcall_map SEC(".maps"); + +SEC("tc") +__log_level(4) +__msg("subprog 0 (helper_reads_fastcall_spill) main {{.*}} stack 8") +__xlated("*(u64 *)(r10 -8) = r1") +__xlated("...") +__xlated("r1 = *(u64 *)(r10 -8)") +__success +__naked void helper_reads_fastcall_spill(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "r1 = %[fastcall_map] ll;" + "r2 = r10;" + "r2 += -8;" + "call %[bpf_map_lookup_elem];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_smp_processor_id), + __imm(bpf_map_lookup_elem), + __imm_addr(fastcall_map) + : __clobber_all); +} + +/* A callee load from the caller's fastcall spill slot. */ +static __used __naked void read_caller_stack_callee(void) +{ + asm volatile ( + "r0 = *(u64 *)(r1 + 0);" + "exit;" + ::: __clobber_all); +} + +SEC("raw_tp") +__log_level(4) +__msg("subprog 0 (callee_reads_fastcall_spill) main {{.*}} stack 8") +__xlated("*(u64 *)(r10 -8) = r1") +__xlated("...") +__xlated("r1 = *(u64 *)(r10 -8)") +__success +__naked void callee_reads_fastcall_spill(void) +{ + asm volatile ( + "r1 = 1;" + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "r1 = r10;" + "r1 += -8;" + "call read_caller_stack_callee;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_smp_processor_id) + : __clobber_all); +} + +/* A zero-sized buffer touches no stack, the rewrite is still applied. */ +SEC("raw_tp") +__arch_x86_64 +__log_level(4) +__msg("subprog 0 (helper_zero_size_buffer) main {{.*}} stack 0") +__xlated("0: r1 = 1") +__xlated("1: r0 =") +__success +__naked void helper_zero_size_buffer(void) +{ + asm volatile ( + "r1 = 1;" + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "r1 = r10;" + "r1 += -64;" + "r2 = 0;" + "r3 = 0;" + "call %[bpf_probe_read_kernel];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_smp_processor_id), + __imm(bpf_probe_read_kernel) + : __clobber_all); +} + SEC("raw_tp") __arch_x86_64 /* main, not patched */ diff --git a/tools/testing/selftests/bpf/progs/verifier_bswap.c b/tools/testing/selftests/bpf/progs/verifier_bswap.c index cffaf36192bc..48ffb5b242d2 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bswap.c +++ b/tools/testing/selftests/bpf/progs/verifier_bswap.c @@ -8,7 +8,7 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 + (__clang_major__ >= 18 || defined(__BPF_FEATURE_BSWAP)) SEC("socket") __description("BSWAP, 16") @@ -117,6 +117,7 @@ l0_%=: \ SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_btf_array_access.c b/tools/testing/selftests/bpf/progs/verifier_btf_array_access.c new file mode 100644 index 000000000000..9dc2c9a5f8b7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_btf_array_access.c @@ -0,0 +1,577 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_core_read.h> +#include "bpf_experimental.h" +#include "bpf_misc.h" + +/* + * Variable-offset access into arrays within a BTF access chain, e.g. a[i].b, + * where 'i' is a register with a varying offset. The register's step has to + * match the stride of one of the arrays crossed while walking the access + * chain, and the maximal possible offset has to stay within that array. + * + * A program-allocated object (bpf_obj_new) is used as the source of a + * PTR_TO_BTF_ID with a shape we fully control. + */ +struct inner { + int a; + int b; +}; + +struct outer { + struct inner arr[8]; /* off 0, stride 8, size 64 */ + char bytes[32]; /* off 64, stride 1, size 32 */ + long longs[8]; /* off 96, stride 8, size 64 */ + int grid[4][4]; /* off 160, 16 ints, size 64 */ +}; + +/* arr[i].b, i in [0, 7], 4-byte read: stays within arr. */ +SEC("syscall") +__success +int arr_field_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 4); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* arr[i].b, i in [0, 15]: max offset runs past the end of arr. */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct outer") +int arr_field_out_of_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 15; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 4); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* longs[i], i in [0, 7], 8-byte read: scalar array with stride 8. */ +SEC("syscall") +__success +int long_array_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + long val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u64 *)(r1 + 96); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * bytes[i], i in [0, 31], read as 4 bytes: min_off is valid within the array, + * but the 4-byte read at max_off runs past the array end. Exercises the + * size-aware bound in btf_struct_access(). + */ +SEC("syscall") +__failure +__flag(BPF_F_ANY_ALIGNMENT) +__msg("invalid variable offset access into struct outer") +int byte_array_size_spanning(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 31; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 64); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* bytes[i], i in [0, 31], 1-byte read: step 1 matches the char array stride. */ +SEC("syscall") +__success +int byte_array_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 31; \ + r1 += r2; \ + %[val] = *(u8 *)(r1 + 64); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * arr[] accessed with step 4 (half of sizeof(struct inner)): the step is not + * a whole number of elements, so no crossed array has a matching stride. + */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct outer") +int arr_unaligned_step(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 4; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 0); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * grid[i][0], i in [0, 3]: steps along the outer dimension of a 2D array + * (stride 16). __btf_resolve_size() linearizes the array; the step is a + * multiple of the innermost element size (4). + */ +SEC("syscall") +__success +int grid_outer_dim(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 3; \ + r2 *= 16; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 160); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * grid[0][i], i in [0, 3]: steps along the inner dimension of a 2D array + * (stride 4, the innermost element size). + */ +SEC("syscall") +__success +int grid_inner_dim(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct outer *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 3; \ + r2 *= 4; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 160); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * An array of pointers within a program-allocated object. Reading a pointer + * member of a local object yields a SCALAR_VALUE, but the varying offset still + * has to be validated against the array bounds. + */ +struct with_ptrs { + struct inner *parr[8]; /* off 0, stride 8 (sizeof ptr), size 64 */ +}; + +/* parr[i], i in [0, 7], 8-byte read: stays within parr. */ +SEC("syscall") +__success +int ptr_array_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct with_ptrs *o; + long val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u64 *)(r1 + 0); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * parr[i], i in [0, 15]: max offset runs past the end of parr. Without the + * variable-offset check on the WALK_PTR path this would be wrongly accepted. + */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct with_ptrs") +int ptr_array_out_of_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct with_ptrs *o; + long val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 15; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u64 *)(r1 + 0); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * The cases above all use bpf_obj_new() (PTR_TO_BTF_ID | MEM_ALLOC). Exercise + * the trusted kernel-BTF path too, using a task_struct from + * bpf_get_current_task_btf() and its embedded char comm[] array. The comm + * offset is folded into the pointer as a constant before the varying index. + */ + +/* task->comm[i], i in [0, 15], 1-byte read: stays within comm. */ +SEC("syscall") +__success +int kernel_btf_array_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct task_struct *task; + int val = 0; + + task = bpf_get_current_task_btf(); + asm volatile (" \ + r1 = %[task]; \ + r1 += %[off]; \ + r2 = %[i]; \ + r2 &= 15; \ + r1 += r2; \ + %[val] = *(u8 *)(r1 + 0); \ +" : [val] "=r"(val) + : [task] "r"(task), [i] "r"(i), + [off] "i"(offsetof(struct task_struct, comm)) + : "r1", "r2"); + return val; +} + +/* task->comm[i], i in [0, 31]: max offset runs past comm. */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct task_struct") +int kernel_btf_array_out_of_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct task_struct *task; + int val = 0; + + task = bpf_get_current_task_btf(); + asm volatile (" \ + r1 = %[task]; \ + r1 += %[off]; \ + r2 = %[i]; \ + r2 &= 31; \ + r1 += r2; \ + %[val] = *(u8 *)(r1 + 0); \ +" : [val] "=r"(val) + : [task] "r"(task), [i] "r"(i), + [off] "i"(offsetof(struct task_struct, comm)) + : "r1", "r2"); + return val; +} + +/* + * Array nested below a struct member at a non-zero offset. The walk descends + * through 'm' (resetting the running offset) before reaching 'arr', exercising + * the array_start = min_off - arrays[i].off telescoping across a WALK_STRUCT + * dive. + */ +struct mid { + struct inner arr[8]; /* stride 8, size 64 */ +}; + +struct nest { + long pad; /* off 0 */ + struct mid m; /* off 8 */ +}; + +/* m.arr[i].b, i in [0, 7], 4-byte read: stays within m.arr. */ +SEC("syscall") +__success +int nested_struct_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct nest *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 12); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* m.arr[i].b, i in [0, 15]: max offset runs past the end of m.arr. */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct nest") +int nested_struct_out_of_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct nest *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 15; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 12); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +/* + * A trailing flexible array member makes the struct tail an unbounded region, + * so a varying access into it has no upper bound to exceed (mirrors how + * unix_address.name[] is accessed via sun_path[i]). + * + * A custom (program) BTF struct can only be reached as a PTR_TO_BTF_ID through + * bpf_obj_new(), which does not allow flexible array members. So use a kernel + * type via an untrusted PTR_TO_BTF_ID (bpf_core_cast()). struct vring_used ends + * with a flexible array 'ring[]' of struct vring_used_elem { __virtio32 id, len; }, + * mirroring a 'struct foo { int a; int b; }' flexible array. + */ + +/* ring[i].len, i in [0, 63], 4-byte read: the flexible array has no upper bound. */ +SEC("syscall") +__success +int flex_array_field(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct vring_used *o; + int val = 0; + + o = bpf_core_cast(bpf_get_current_task_btf(), struct vring_used); + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 63; \ + r2 *= 8; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 8); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + return val; +} + +/* + * ring[] accessed with step 4 (half of sizeof(struct vring_used_elem)): the + * step is not a whole number of elements, so the flexible array stride does + * not match. + */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct vring_used") +int flex_array_unaligned_step(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct vring_used *o; + int val = 0; + + o = bpf_core_cast(bpf_get_current_task_btf(), struct vring_used); + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 7; \ + r2 *= 4; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 4); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i) + : "r1", "r2"); + return val; +} + +/* + * Access either name[0].sun_path[0] or one byte past sun_path. + * Step 108 does not match sizeof(struct sockaddr_un), which is 110. + * It matches the inner char array's stride, but exceeds its bounds. + * The flexible-tail exemption must not apply to that fixed array. + */ +SEC("syscall") +__failure +__msg("invalid variable offset access into struct unix_address") +int flex_array_inner_array_out_of_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct unix_address *o; + int val = 0; + + o = bpf_core_cast(bpf_get_current_task_btf(), struct unix_address); + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 1; \ + r2 *= %[len]; \ + r1 += r2; \ + %[val] = *(u8 *)(r1 + %[off]); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i), + __imm_const(len, sizeof_field(struct sockaddr_un, sun_path)), + __imm_const(off, offsetof(struct unix_address, name) + + offsetof(struct sockaddr_un, sun_path)) + : "r1", "r2"); + return val; +} + +typedef int array_element_t; +typedef array_element_t array_type_t[4]; + +struct typedef_array { + array_type_t values; +}; + +/* + * Resolving values' size crosses both the array and element typedefs. + * Record the array ID, not the int ID left by element-type resolution. + * values[i], i in [0, 3], must pass the variable-offset array check. + */ +SEC("syscall") +__success +int typedef_array_in_bounds(void *ctx) +{ + unsigned long i = bpf_get_prandom_u32(); + struct typedef_array *o; + int val = 0; + + o = bpf_obj_new(typeof(*o)); + if (!o) + return 0; + asm volatile (" \ + r1 = %[o]; \ + r2 = %[i]; \ + r2 &= 3; \ + r2 *= %[elem_size]; \ + r1 += r2; \ + %[val] = *(u32 *)(r1 + 0); \ +" : [val] "=r"(val) + : [o] "r"(o), [i] "r"(i), + __imm_const(elem_size, sizeof(array_element_t)) + : "r1", "r2"); + bpf_obj_drop(o); + return val; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_callx.c b/tools/testing/selftests/bpf/progs/verifier_callx.c new file mode 100644 index 000000000000..7e59188d9fca --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_callx.c @@ -0,0 +1,1086 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Tests for callx: indirect calls of bpf subprogs */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "../../../include/linux/filter.h" + +/* gcc doesn't support indirect calls */ +#if defined(__clang__) && (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) + +#define CALLX_INSN(DST, SRC, OFF, IMM) \ + BPF_RAW_INSN(BPF_JMP | BPF_CALL | BPF_X, DST, SRC, OFF, IMM) + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, long long); +} map_array SEC(".maps"); + +struct val_with_lock { + struct bpf_spin_lock lock; + int cnt; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct val_with_lock); +} map_lock SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(int)); + __uint(value_size, sizeof(int)); +} map_prog SEC(".maps"); + +__naked __noinline __used +static unsigned long add1(void) +{ + asm volatile ( + "r0 = r1;" + "r0 += 1;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long add2(void) +{ + asm volatile ( + "r0 = r1;" + "r0 += 2;" + "exit;" + ); +} + +/* apply(fn, x) { return fn(x); } */ +__naked __noinline __used +static unsigned long apply(void) +{ + asm volatile ( + "r3 = r1;" + "r1 = r2;" + "callx r3;" + "exit;" + ); +} + +SEC("socket") +__success __retval(6) +__naked void callx_basic(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* callx is printed by the verifier log and xlated dump */ +SEC("socket") +__success __log_level(2) +__msg("(8d) callx r2") +__xlated("callx r2") +__naked void callx_disasm(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* different callees are called by the same callx on different paths */ +SEC("socket") +__success __retval(11) +__naked void callx_two_callees(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r2 = %[add1] ll;" + "if r6 == 0 goto +2;" + "r2 = %[add2] ll;" + "r1 = 10;" + "callx r2;" + /* add1(10) - 0 or add2(10) - 1 */ + "r0 -= r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(add1), + __imm_addr(add2) + : __clobber_all); +} + +/* pointer to a function is passed as an argument */ +SEC("socket") +__success __retval(45) +__naked void callx_fn_as_arg(void) +{ + asm volatile ( + "r1 = %[add2] ll;" + "r2 = 40;" + "call apply;" + "r6 = r0;" + "r1 = %[add1] ll;" + "r2 = 2;" + "call apply;" + "r0 += r6;" + "exit;" + : + : __imm_addr(add1), + __imm_addr(add2) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long get_add1(void) +{ + asm volatile ( + "r0 = %[add1] ll;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* pointer to a function is returned from a subprog and called via r0 */ +SEC("socket") +__success __retval(2) +__naked void callx_r0(void) +{ + asm volatile ( + "call get_add1;" + "r1 = 1;" + "callx r0;" + "exit;" + ::: __clobber_all); +} + +/* pointer to a function survives spill/fill */ +SEC("socket") +__success __retval(9) +__naked void callx_spill_fill(void) +{ + asm volatile ( + "r2 = %[add2] ll;" + "*(u64 *)(r10 - 8) = r2;" + "call %[bpf_get_prandom_u32];" + "r1 = 7;" + "r9 = *(u64 *)(r10 - 8);" + "callx r9;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(add2) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long clobber_callee_saved(void) +{ + asm volatile ( + "r6 = 100;" + "r7 = 100;" + "r8 = 100;" + "r9 = 100;" + "r0 = r1;" + "exit;" + ); +} + +/* r6-r9 are preserved across callx */ +SEC("socket") +__success __retval(11) +__naked void callx_callee_saved_regs(void) +{ + asm volatile ( + "r6 = 1;" + "r7 = 2;" + "r8 = 3;" + "r9 = 4;" + "r1 = 1;" + "r2 = %[clobber_callee_saved] ll;" + "callx r2;" + "r0 += r6;" + "r0 += r7;" + "r0 += r8;" + "r0 += r9;" + "exit;" + : + : __imm_addr(clobber_callee_saved) + : __clobber_all); +} + +/* r1-r5 are scratched by callx */ +SEC("socket") +__failure __msg("R1 !read_ok") +__naked void callx_scratches_args(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "callx r2;" + "r0 = r1;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* return value of the callee is tracked */ +SEC("socket") +__success __log_level(2) +__msg("to caller at 4:") +__msg("R0=6") +__naked void callx_retval_is_tracked(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long write42(void) +{ + asm volatile ( + "r2 = 42;" + "*(u64 *)(r1 + 0) = r2;" + "r0 = 0;" + "exit;" + ); +} + +/* callee writes into the stack of the caller */ +SEC("socket") +__success __retval(42) +__naked void callx_callee_writes_caller_stack(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "r1 = r10;" + "r1 += -8;" + "r2 = %[write42] ll;" + "callx r2;" + "r0 = *(u64 *)(r10 - 8);" + "exit;" + : + : __imm_addr(write42) + : __clobber_all); +} + +SEC("socket") +__failure __msg("R1 has type scalar, expected func") +__naked void callx_scalar(void) +{ + asm volatile ( + "r1 = 0;" + "callx r1;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("R2 !read_ok") +__naked void callx_uninit_reg(void) +{ + asm volatile ( + "callx r2;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("R10 has type fp, expected func") +__naked void callx_fp(void) +{ + asm volatile ( + "callx r10;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("R1 has type map_value, expected func") +__naked void callx_map_value(void) +{ + asm volatile ( + "r1 = %[map_array] ll;" + "r2 = r10;" + "r2 += -4;" + "r3 = 0;" + "*(u32 *)(r2 + 0) = r3;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 1f;" + "r1 = r0;" + "callx r1;" + "1:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_array) + : __clobber_all); +} + +/* the address of a subprog can't be modified before the call */ +SEC("socket") +__failure __msg("dereference of modified func ptr R2 off=8 disallowed") +__naked void callx_modified_ptr(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + "r2 += 8;" + "callx r2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +SEC("socket") +__failure __msg("variable func access var_off=") +__naked void callx_variable_ptr(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r0 &= 8;" + "r2 = %[add1] ll;" + "r2 += r0;" + "r1 = 5;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(add1) + : __clobber_all); +} + +__noinline __used +int global_add3(int x) +{ + return x + 3; +} + +/* only static subprogs can be called via callx */ +SEC("socket") +__failure __msg("callback function not static") +__naked void callx_global_func(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[global_add3] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(global_add3) + : __clobber_all); +} + +#define DEFINE_CALLX_RESERVED_FIELDS_PROG(NAME, SRC_REG, OFF, IMM) \ + SEC("socket") \ + __failure __msg("BPF_CALL|BPF_X uses reserved fields") \ + __naked void callx_reserved_field_ ## NAME(void) \ + { \ + asm volatile ( \ + "r1 = 5;" \ + "r2 = %[add1] ll;" \ + ".8byte %[callx_r2];" \ + "exit;" \ + : \ + : __imm_addr(add1), \ + __imm_insn(callx_r2, CALLX_INSN(BPF_REG_2, (SRC_REG), (OFF), (IMM))) \ + : __clobber_all); \ + } + +DEFINE_CALLX_RESERVED_FIELDS_PROG(src_reg, BPF_REG_1, 0, 0) +DEFINE_CALLX_RESERVED_FIELDS_PROG(off, BPF_REG_0, 1, 0) +DEFINE_CALLX_RESERVED_FIELDS_PROG(imm, BPF_REG_0, 0, 1) + +SEC("socket") +__failure __msg("unknown opcode 8e") +__naked void callx_jmp32(void) +{ + asm volatile ( + "r1 = 5;" + "r2 = %[add1] ll;" + ".8byte %[callx32_r2];" + "exit;" + : + : __imm_addr(add1), + __imm_insn(callx32_r2, + BPF_RAW_INSN(BPF_JMP32 | BPF_CALL | BPF_X, BPF_REG_2, 0, 0, 0)) + : __clobber_all); +} + +/* similar to calls of static subprogs callx is allowed under a lock */ +SEC("tc") +__success __retval(3) +__naked void callx_under_lock(void) +{ + asm volatile ( + "r1 = 0;" + "*(u32 *)(r10 - 4) = r1;" + "r2 = r10;" + "r2 += -4;" + "r1 = %[map_lock] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 != 0 goto 1f;" + "exit;" + "1:" + "r6 = r0;" + "r1 = r6;" + "call %[bpf_spin_lock];" + "r1 = 1;" + "r2 = %[add2] ll;" + "callx r2;" + "r7 = r0;" + "r1 = r6;" + "call %[bpf_spin_unlock];" + "r0 = r7;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm(bpf_spin_lock), + __imm(bpf_spin_unlock), + __imm_addr(map_lock), + __imm_addr(add2) + : __clobber_all); +} + +/* helpers are still not allowed under a lock in the callee */ +__naked __noinline __used +static unsigned long call_helper(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("tc") +__failure __msg("function calls are not allowed while holding a lock") +__naked void callx_helper_under_lock(void) +{ + asm volatile ( + "r1 = 0;" + "*(u32 *)(r10 - 4) = r1;" + "r2 = r10;" + "r2 += -4;" + "r1 = %[map_lock] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 != 0 goto 1f;" + "exit;" + "1:" + "r6 = r0;" + "r1 = r6;" + "call %[bpf_spin_lock];" + "r2 = %[call_helper] ll;" + "callx r2;" + "r1 = r6;" + "call %[bpf_spin_unlock];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm(bpf_spin_lock), + __imm(bpf_spin_unlock), + __imm_addr(map_lock), + __imm_addr(call_helper) + : __clobber_all); +} + +/* self(fn) { return fn(fn); } */ +__naked __noinline __used +static unsigned long self(void) +{ + asm volatile ( + "callx r1;" + "exit;" + ); +} + +/* unbounded recursion is caught by the main verification pass */ +SEC("socket") +__failure __msg("frames is too deep") +__naked void callx_unbounded_recursion(void) +{ + asm volatile ( + "r1 = %[self] ll;" + "call self;" + "exit;" + : + : __imm_addr(self) + : __clobber_all); +} + +/* countdown(fn, n) { return n ? fn(fn, n - 1) : 0; } */ +__naked __noinline __used +static unsigned long countdown(void) +{ + asm volatile ( + "r0 = 0;" + "if r2 == 0 goto 1f;" + "r2 += -1;" + "callx r1;" + "1:" + "exit;" + ); +} + +/* + * The depth of the recursion is known to the main verification pass, + * but recursive calls are not allowed regardless. + */ +SEC("socket") +__failure __msg("recursive call from countdown() to countdown()") +__naked void callx_bounded_recursion(void) +{ + asm volatile ( + "r1 = %[countdown] ll;" + "r2 = 2;" + "call countdown;" + "exit;" + : + : __imm_addr(countdown) + : __clobber_all); +} + +/* ping(fn1, fn2, n) { return n ? fn2(fn2, fn1, n - 1) : 0; } */ +__naked __noinline __used +static unsigned long ping(void) +{ + asm volatile ( + "r0 = 0;" + "if r3 == 0 goto 1f;" + "r3 += -1;" + "r4 = r1;" + "r1 = r2;" + "r2 = r4;" + "callx r1;" + "1:" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long pong(void) +{ + asm volatile ( + "r0 = 0;" + "if r3 == 0 goto 1f;" + "r3 += -1;" + "r4 = r1;" + "r1 = r2;" + "r2 = r4;" + "callx r1;" + "1:" + "exit;" + ); +} + +SEC("socket") +__failure __msg("recursive call from") +__naked void callx_mutual_recursion(void) +{ + asm volatile ( + "r1 = %[ping] ll;" + "r2 = %[pong] ll;" + "r3 = 3;" + "call ping;" + "exit;" + : + : __imm_addr(ping), + __imm_addr(pong) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long use_stack_304(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "exit;" + ); +} + + +/* Four 480-byte frames, deeper than any budget together with their caller */ +__naked __noinline __used +static unsigned long use_stack_480_0(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_1(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_0;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_2(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_1;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_3(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_2;" + "exit;" + ); +} + +/* stack of the callee of callx is accounted */ +SEC("socket") +__failure __msg("combined stack size of {{[0-9]+}} calls is") +__naked void callx_stack_depth(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "r2 = %[use_stack_480_3] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(use_stack_480_3) + : __clobber_all); +} + +/* apply_stack_304(fn) { char buf[304]; return fn(); } */ +__naked __noinline __used +static unsigned long apply_stack_304(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "callx r1;" + "exit;" + ); +} + +/* + * The address of use_stack_480_3() is taken by the main prog that doesn't + * use stack, but it is called from apply_stack_304(). + */ +SEC("socket") +__failure __msg("combined stack size of {{[0-9]+}} calls is") +__naked void callx_stack_depth_nested(void) +{ + asm volatile ( + "r1 = %[use_stack_480_3] ll;" + "call apply_stack_304;" + "exit;" + : + : __imm_addr(use_stack_480_3) + : __clobber_all); +} + +SEC("socket") +__success __retval(0) +__naked void callx_stack_depth_ok(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 200) = r0;" + "r2 = %[use_stack_304] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(use_stack_304) + : __clobber_all); +} + +__naked __noinline __used +static unsigned long do_tail_call(void) +{ + asm volatile ( + "r2 = %[map_prog] ll;" + "r3 = 0;" + "call %[bpf_tail_call];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_tail_call), + __imm_addr(map_prog) + : __clobber_all); +} + +SEC("socket") +__failure __msg("tail_calls are not allowed in functions called via callx") +__naked void callx_tail_call_in_callee(void) +{ + asm volatile ( + "r2 = %[do_tail_call] ll;" + "callx r2;" + "exit;" + : + : __imm_addr(do_tail_call) + : __clobber_all); +} + +/* tail call in the caller of callx is fine */ +SEC("socket") +__success __retval(3) +__naked void callx_tail_call_in_caller(void) +{ + asm volatile ( + "r6 = r1;" + "r1 = 2;" + "r2 = %[add1] ll;" + "callx r2;" + "r7 = r0;" + "r1 = r6;" + "r2 = %[map_prog] ll;" + "r3 = 0;" + "call %[bpf_tail_call];" + "r0 = r7;" + "exit;" + : + : __imm(bpf_tail_call), + __imm_addr(map_prog), + __imm_addr(add1) + : __clobber_all); +} + +/* read_idx(p) { return ((char *)map_value)[*p]; } */ +__naked __noinline __used +static unsigned long read_idx(void) +{ + asm volatile ( + "r6 = *(u64 *)(r1 + 0);" + "r1 = 0;" + "*(u32 *)(r10 - 4) = r1;" + "r2 = r10;" + "r2 += -4;" + "r1 = %[map_array] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 1f;" + "r0 += r6;" + "r0 = *(u8 *)(r0 + 0);" + "1:" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_array) + : __clobber_all); +} + +/* + * Stack slots of the caller that might be read by the callee of callx + * have to be considered alive at the checkpoints before callx and + * inside of the callee. Otherwise the state with fp-8 == 1000 is pruned + * and out of bounds access in read_idx() goes unnoticed. + */ +SEC("socket") +__failure __msg("invalid access to map value, value_size=8 off=1000 size=1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void callx_callee_reads_caller_stack(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1000;" + "*(u64 *)(r10 - 8) = r1;" + "if r0 == 0 goto 1f;" + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "1:" + "r1 = r10;" + "r1 += -8;" + "r2 = %[read_idx] ll;" + "callx r2;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(read_idx) + : __clobber_all); +} + +/* in bounds access in read_idx() is fine */ +SEC("socket") +__success __retval(0) +__flag(BPF_F_TEST_STATE_FREQ) +__naked void callx_callee_reads_caller_stack_ok(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 7;" + "*(u64 *)(r10 - 8) = r1;" + "if r0 == 0 goto 1f;" + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "1:" + "r1 = r10;" + "r1 += -8;" + "r2 = %[read_idx] ll;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(read_idx) + : __clobber_all); +} + +/* same as above, but the pointer to the stack is passed through one more frame */ +SEC("socket") +__failure __msg("invalid access to map value, value_size=8 off=1000 size=1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void callx_callee_reads_caller_stack_nested(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1000;" + "*(u64 *)(r10 - 8) = r1;" + "if r0 == 0 goto 1f;" + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "1:" + "r1 = %[read_idx] ll;" + "r2 = r10;" + "r2 += -8;" + "call apply;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(read_idx) + : __clobber_all); +} + +/* registers that are constant before callx are not constant after it */ +SEC("socket") +__success __retval(1) +__naked void callx_clobbers_const_regs(void) +{ + asm volatile ( + "r0 = 0;" + "r1 = 0;" + "r2 = %[add1] ll;" + "callx r2;" + /* dead branch pruning must not assume that r0 is still 0 */ + "if r0 == 0 goto 1f;" + "r0 = 1;" + "exit;" + "1:" + "r0 = 2;" + "exit;" + : + : __imm_addr(add1) + : __clobber_all); +} + +/* scalar argument passed through callx is tracked precisely */ +__naked __noinline __used +static unsigned long identity(void) +{ + asm volatile ( + "r0 = r1;" + "exit;" + ); +} + +long long vals[] SEC(".data.vals") = {1, 2, 3, 4}; + +SEC("socket") +__success __log_level(2) +__msg("mark_precise: frame0: regs=r0 stack= before 12: (95) exit") +__msg("mark_precise: frame1: regs=r0 stack= before 11: (bf) r0 = r1") +__msg("mark_precise: frame1: regs=r1 stack= before 4: (8d) callx r2") +__msg("mark_precise: frame0: regs=r1 stack= before 3: (bf) r1 = r6") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") +__retval(4) +__naked void callx_precision(void) +{ + asm volatile ( + "r2 = %[identity] ll;" + "r6 = 3;" + "r1 = r6;" + "callx r2;" + "r0 *= 8;" + "r1 = %[vals] ll;" + "r1 += r0;" + "r0 = *(u64 *)(r1 + 0);" + "exit;" + : + : __imm_addr(identity), + __imm_addr(vals) + : __clobber_all); +} + +/* + * The address of a function that is never called is valid. The function is + * the last one in the program. + */ +SEC("socket") +__success __retval(6) +__naked void callx_addr_of_dead_func(void) +{ + asm volatile ( + "r6 = %[add1] ll;" + "r7 = %[add2] ll;" + "r1 = 5;" + "callx r6;" + "if r7 != r6 goto +1;" + "r0 = 0;" + "exit;" + : + : __imm_addr(add1), __imm_addr(add2) + : __clobber_all); +} + +/* and when it's followed by another function */ +SEC("socket") +__success __retval(7) +__naked void callx_addr_of_dead_func_first(void) +{ + asm volatile ( + "r7 = %[add1] ll;" + "r6 = %[add2] ll;" + "r1 = 5;" + "callx r6;" + "if r7 != r6 goto +1;" + "r0 = 0;" + "exit;" + : + : __imm_addr(add1), __imm_addr(add2) + : __clobber_all); +} + +/* the same without callx */ +SEC("socket") +__success __retval(6) +__naked void call_addr_of_dead_func(void) +{ + asm volatile ( + "r1 = 5;" + "call add1;" + "r7 = %[add2] ll;" + "exit;" + : + : __imm_addr(add2) + : __clobber_all); +} + +/* function pointers in C */ + +typedef int (*op_fn)(int); + +static __noinline int mul3(int x) +{ + return x * 3; +} + +static __noinline int sub7(int x) +{ + return x - 7; +} + +static __noinline int apply_op(op_fn op, int x) +{ + return op(x); +} + +SEC("socket") +__success __retval(36) +int callx_c_fn_as_arg(void *ctx) +{ + /* (5 * 3) + (28 - 7) */ + return apply_op(mul3, 5) + apply_op(sub7, 28); +} + +SEC("socket") +__success __retval(30) +int callx_c_select(void *ctx) +{ + __u32 rnd = bpf_get_prandom_u32() & 1; + op_fn op = rnd ? mul3 : sub7; + int x = rnd ? 10 : 37; + + /* 10 * 3 or 37 - 7 */ + return op(x); +} + +struct ops { + op_fn first; + op_fn second; + int bias; +}; + +static __noinline int run_ops(const struct ops *ops, int x) +{ + return ops->second(ops->first(x)) + ops->bias; +} + +SEC("socket") +__success __retval(100) +int callx_c_ops_on_stack(void *ctx) +{ + struct ops a, b; + + /* avoid an initializer with function pointers in .rodata */ + a.first = mul3; + a.second = sub7; + a.bias = 10; + b.first = sub7; + b.second = mul3; + b.bias = 1; + + /* ((7 * 3) - 7 + 10) + ((32 - 7) * 3 + 1) */ + return run_ops(&a, 7) + run_ops(&b, 32); +} + +#else + +SEC("socket") +__success +int dummy(void *ctx) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c new file mode 100644 index 000000000000..6cf58ca9ac5b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_callx_rodata.c @@ -0,0 +1,903 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Tests for callx through pointers to functions in read-only data */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "../../../include/linux/filter.h" + +/* gcc doesn't support indirect calls */ +#if defined(__clang__) && (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) + +/* + * Read-only data with pointers to functions, where the compiler puts tables + * of functions, structures of operations and vtables. libbpf resolves + * a pointer to the offset of the function in the program and the kernel + * recognizes it by that value. + */ +#define DATA(SECTION, NAME, ...) \ + ".pushsection " SECTION ",@progbits;" \ + ".balign 8;" \ + #NAME "_%=:" \ + __VA_ARGS__ \ + ".type " #NAME "_%=, @object;" \ + ".size " #NAME "_%=, .-" #NAME "_%=;" \ + ".popsection;" + +#define RODATA(NAME, ...) DATA(".rodata,\"a\"", NAME, __VA_ARGS__) + +#define FUNC_TABLE2(NAME, F0, F1) RODATA(NAME, ".quad " #F0 "; .quad " #F1 ";") + +__naked __noinline __used +static unsigned long add1(void) +{ + asm volatile ( + "r0 = r1;" + "r0 += 1;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long add2(void) +{ + asm volatile ( + "r0 = r1;" + "r0 += 2;" + "exit;" + ); +} + +/* the second element of the table is called */ +SEC("socket") +__success __retval(12) +__naked void callx_rodata_const_index(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 8);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* + * The program reads the address of a function from the data, so pointers to + * functions in data are recognized only for programs that may leak pointers. + * Otherwise nothing refers to the functions. + */ +SEC("socket") +__success __retval(12) +__failure_unpriv __msg_unpriv("unreachable insn") +__caps_unpriv(CAP_BPF) +__naked void callx_rodata_needs_perfmon(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 8);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* the address of an element is used instead of the address of the table */ +SEC("socket") +__success __retval(12) +__naked void callx_rodata_elem_addr(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= + 8 ll;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* pointers to functions are mixed with other data like in a vtable */ +SEC("socket") +__success __retval(23) +__log_level(2) +__msg("r1 = *(u64 *)(r6 +0) ; R1=7") +__msg("r2 = *(u64 *)(r6 +8) ; R2=func()") +__naked void callx_rodata_mixed_with_data(void) +{ + asm volatile ( + RODATA(vt, ".quad 7; .quad add1; .quad 13; .quad add2;") + "r6 = vt_%= ll;" + "r1 = *(u64 *)(r6 + 0);" + "r2 = *(u64 *)(r6 + 8);" + /* add1(7) */ + "callx r2;" + "r7 = r0;" + "r1 = *(u64 *)(r6 + 16);" + "r2 = *(u64 *)(r6 + 24);" + /* add2(13) */ + "callx r2;" + "r0 += r7;" + "exit;" + ::: __clobber_all); +} + +/* + * Position independent code keeps constants with pointers in .data.rel.ro. + * libbpf treats it as read-only data. + */ +SEC("socket") +__success __retval(23) +__naked void callx_data_rel_ro(void) +{ + asm volatile ( + DATA(".data.rel.ro,\"aw\"", vt, ".quad 7; .quad add1; .quad 13; .quad add2;") + "r6 = vt_%= ll;" + "r1 = *(u64 *)(r6 + 0);" + "r2 = *(u64 *)(r6 + 8);" + "callx r2;" + "r7 = r0;" + "r1 = *(u64 *)(r6 + 16);" + "r2 = *(u64 *)(r6 + 24);" + "callx r2;" + "r0 += r7;" + "exit;" + ::: __clobber_all); +} + +/* an array of structures: the index selects one of the functions, not the data */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_array_of_structs(void) +{ + asm volatile ( + RODATA(arr, ".quad add1; .quad 0x1111; .quad add2; .quad 0x2222;") + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r3 = r6;" + "r3 <<= 4;" + "r2 = arr_%= ll;" + "r2 += r3;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + /* add1(10) - 0 or add2(10) - 1 */ + "r0 -= r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* and the data of the element that the index selects can be read */ +SEC("socket") +__success __retval(16) +__naked void callx_rodata_array_of_structs_data(void) +{ + asm volatile ( + RODATA(arr, ".quad add1; .quad 5; .quad add2; .quad 5;") + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r3 = r6;" + "r3 <<= 4;" + "r7 = arr_%= ll;" + "r7 += r3;" + "r2 = *(u64 *)(r7 + 0);" + "r1 = 10;" + "callx r2;" + /* add1(10) - 0 or add2(10) - 1 */ + "r0 -= r6;" + "r1 = *(u64 *)(r7 + 8);" + "r0 += r1;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* but not the data and the pointer with the same load */ +SEC("socket") +__failure __msg("overlaps with a pointer to a function") +__naked void callx_rodata_array_of_structs_any(void) +{ + asm volatile ( + RODATA(arr, ".quad add1; .quad 5; .quad add2; .quad 5;") + "call %[bpf_get_prandom_u32];" + "r0 &= 3;" + "r0 <<= 3;" + "r2 = arr_%= ll;" + "r2 += r0;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* dynamic dispatch: which vtable is used is not known until run time */ +SEC("socket") +__success __retval(2) +__naked void callx_rodata_two_vtables(void) +{ + asm volatile ( + RODATA(vt_a, ".quad 1; .quad add1;") + RODATA(vt_b, ".quad 0; .quad add2;") + "call %[bpf_get_prandom_u32];" + "r6 = vt_a_%= ll;" + "r0 &= 1;" + "if r0 == 0 goto +2;" + "r6 = vt_b_%= ll;" + "r1 = *(u64 *)(r6 + 0);" + "r2 = *(u64 *)(r6 + 8);" + /* add1(1) or add2(0) */ + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* both functions are verified, either of them is called */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_var_index(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r3 = r6;" + "r3 <<= 3;" + "r2 = tbl_%= ll;" + "r2 += r3;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + /* add1(10) - 0 or add2(10) - 1 */ + "r0 -= r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* a table that has no symbol, the compiler generates such for a switch statement */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_no_symbol(void) +{ + asm volatile ( + ".pushsection .rodata,\"a\",@progbits;" + ".balign 8;" + ".Lanon_%=:" + ".quad add1;" + ".quad add2;" + ".popsection;" + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r6 &= 1;" + "r3 = r6;" + "r3 <<= 3;" + "r2 = .Lanon_%= ll;" + "r2 += r3;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "r0 -= r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* the first instruction of the function is removed by the verifier */ +__naked __noinline __used +static unsigned long nop_add1(void) +{ + asm volatile ( + "goto +0;" + "r0 = r1;" + "r0 += 1;" + "exit;" + ); +} + +/* the pointer follows the function when instructions are removed */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_func_starts_with_nop(void) +{ + asm volatile ( + RODATA(tbl, ".quad nop_add1; .quad 0;") + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* can't be called with a scalar in r1 */ +__naked __noinline __used +static unsigned long deref_r1(void) +{ + asm volatile ( + "r0 = *(u64 *)(r1 + 0);" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long ret0(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +/* every function that might be called is verified */ +SEC("socket") +__failure __msg("R1 invalid mem access 'scalar'") +__naked void callx_rodata_all_callees_verified(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, ret0, deref_r1) + "call %[bpf_get_prandom_u32];" + "r0 &= 1;" + "r0 <<= 3;" + "r2 = tbl_%= ll;" + "r2 += r0;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 0;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* a function that is never called is not verified, it's dead code */ +SEC("socket") +__success __retval(0) +__naked void callx_rodata_unused_callee(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, ret0, deref_r1) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 0;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* a pointer that is read and never called doesn't keep the function, it's NULL */ +SEC("socket") +__success __retval(1) +__naked void callx_rodata_read_unused_callee(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, ret0, deref_r1) + "r6 = tbl_%= ll;" + "r2 = *(u64 *)(r6 + 0);" + "r1 = 0;" + "callx r2;" + "r2 = *(u64 *)(r6 + 8);" + "if r2 != 0 goto +1;" + "r0 = 1;" + "exit;" + ::: __clobber_all); +} + +/* the index may select what is not a pointer to a function */ +SEC("socket") +__failure __msg("overlaps with a pointer to a function") +__naked void callx_rodata_index_beyond_table(void) +{ + asm volatile ( + RODATA(tbl, ".quad add1; .quad add2; .quad 0x1234; .quad 0x5678;") + "call %[bpf_get_prandom_u32];" + "r0 &= 3;" + "r0 <<= 3;" + "r2 = tbl_%= ll;" + "r2 += r0;" + "r2 = *(u64 *)(r2 + 0);" + "r1 = 10;" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* the address of a function is not known until the program is jitted */ +SEC("socket") +__failure __msg("read of 4 bytes at offset") +__msg("overlaps with a pointer to a function") +__naked void callx_rodata_partial_read(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r6 = tbl_%= ll;" + "r2 = *(u64 *)(r6 + 8);" + "r1 = 10;" + "callx r2;" + "r0 = *(u32 *)(r6 + 0);" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("read of 8 bytes at offset") +__msg("overlaps with a pointer to a function") +__naked void callx_rodata_misaligned_read(void) +{ + asm volatile ( + RODATA(tbl, ".quad add1; .quad add2; .quad 0;") + "r6 = tbl_%= ll;" + "r2 = *(u64 *)(r6 + 0);" + "r1 = 10;" + "callx r2;" + "r0 = *(u64 *)(r6 + 4);" + "exit;" + ::: __clobber_all); +} + +/* + * The kernel doesn't look for pointers to functions in the data of a program + * that doesn't have callx. It's a number there. + */ +SEC("socket") +__success +__naked void callx_rodata_no_callx(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r0 = *(u32 *)(r2 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* the data next to a pointer is still known to the verifier */ +SEC("socket") +__success __retval(0x1234) +__log_level(2) +__msg("R0=4660") +__naked void callx_rodata_data_is_const(void) +{ + asm volatile ( + RODATA(tbl, ".quad add1; .quad 0x1234;") + "r6 = tbl_%= ll;" + "r2 = *(u64 *)(r6 + 0);" + "r1 = 10;" + "callx r2;" + "r0 = *(u64 *)(r6 + 8);" + "exit;" + ::: __clobber_all); +} + +/* + * The map has the offset of the function until the program is jitted. + * bpf_compute_const_regs() must not take it for a constant, otherwise + * bpf_prune_dead_branches() turns the branch below into the wrong one. + */ +SEC("socket") +__success __retval(11) +__naked void callx_rodata_ptr_is_not_const(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "r0 = 0;" + /* the offset is less, the address is greater */ + "if r2 < 0x100000 goto 1f;" + "r1 = 10;" + "callx r2;" + "1:" + "exit;" + ::: __clobber_all); +} + +/* the pointer read from the data can't be modified */ +SEC("socket") +__failure __msg("dereference of modified func ptr R2 off=8 disallowed") +__naked void callx_rodata_modified_ptr(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, add2) + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "r2 += 8;" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +__noinline __used +int global_add3(int x) +{ + return x + 3; +} + +/* + * A pointer to a global function is not recognized, it's a number. The function + * is called directly as well, so that libbpf adds it to the program and stores + * its offset into the table. + */ +SEC("socket") +__failure __msg("R2 has type scalar, expected func") +__naked void callx_rodata_global_func(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, add1, global_add3) + "r1 = 1;" + "call global_add3;" + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 8);" + "r1 = 10;" + "callx r2;" + "exit;" + ::: __clobber_all); +} + +/* selfcall(n) { return n ? tbl[0](n - 1) : 0; }, where tbl[0] == selfcall */ +__naked __noinline __used +static unsigned long selfcall(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, selfcall, ret0) + "r0 = 0;" + "if r1 == 0 goto 1f;" + "r1 += -1;" + "r2 = tbl_%= ll;" + "r2 = *(u64 *)(r2 + 0);" + "callx r2;" + "1:" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__failure __msg("recursive call from selfcall() to selfcall()") +__naked void callx_rodata_recursion(void) +{ + asm volatile ( + "r1 = 2;" + "call selfcall;" + "exit;" + ::: __clobber_all); +} + + +/* Four 480-byte frames, deeper than any budget together with their caller */ +__naked __noinline __used +static unsigned long use_stack_480_0(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_1(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_0;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_2(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_1;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long use_stack_480_3(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 480) = r0;" + "call use_stack_480_2;" + "exit;" + ); +} + +/* stack of all possible callees is accounted */ +SEC("socket") +__failure __msg("combined stack size of {{[0-9]+}} calls is") +__naked void callx_rodata_stack_depth(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, ret0, use_stack_480_3) + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "call %[bpf_get_prandom_u32];" + "r0 &= 1;" + "r0 <<= 3;" + "r2 = tbl_%= ll;" + "r2 += r0;" + "r2 = *(u64 *)(r2 + 0);" + "callx r2;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* stack of a callback that is read from the data is accounted too */ +SEC("socket") +__failure __msg("combined stack size of {{[0-9]+}} calls is") +__naked void callx_rodata_callback_stack_depth(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, use_stack_480_3, ret0) + "r0 = 0;" + "*(u64 *)(r10 - 304) = r0;" + "r6 = tbl_%= ll;" + "r1 = 1;" + "r2 = *(u64 *)(r6 + 0);" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "r2 = *(u64 *)(r6 + 8);" + "r1 = 0;" + "callx r2;" + "exit;" + : + : __imm(bpf_loop) + : __clobber_all); +} + +static __noinline __used int pull_data(struct __sk_buff *skb) +{ + return bpf_skb_pull_data(skb, 0); +} + +static __noinline __used int calls_pull_data(struct __sk_buff *skb) +{ + return pull_data(skb) + 1; +} + +__noinline __used int global_calls_pull_data(struct __sk_buff *skb) +{ + return calls_pull_data(skb) + 2; +} + +/* + * check_cfg() explores the functions that the data points to when it sees + * the address of the data. It must not lose the effects of the functions + * that call each other: global_calls_pull_data() changes packet data. + */ +SEC("tc") +__failure __msg("R7 invalid mem access 'scalar'") +__naked void callx_rodata_callee_effects(void) +{ + asm volatile ( + FUNC_TABLE2(tbl, pull_data, calls_pull_data) + "r6 = r1;" + "r7 = *(u32 *)(r6 + %[data]);" + "r8 = *(u32 *)(r6 + %[data_end]);" + "r2 = r7;" + "r2 += 8;" + "if r2 > r8 goto 1f;" + "r1 = r6;" + "call global_calls_pull_data;" + "r0 = *(u8 *)(r7 + 0);" + "1:" + "r9 = tbl_%= ll;" + "r2 = *(u64 *)(r9 + 8);" + "r1 = r6;" + "callx r2;" + "r0 = 0;" + "exit;" + : + : __imm_const(data, offsetof(struct __sk_buff, data)), + __imm_const(data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + +/* pointers to functions in C */ + +typedef int (*op_fn)(int); + +#define DEFINE_OP(N) static __noinline int op##N(int x) { return x * (N + 2) + N; } + +DEFINE_OP(0) DEFINE_OP(1) DEFINE_OP(2) DEFINE_OP(3) +DEFINE_OP(4) DEFINE_OP(5) DEFINE_OP(6) DEFINE_OP(7) +DEFINE_OP(8) DEFINE_OP(9) DEFINE_OP(10) DEFINE_OP(11) +DEFINE_OP(12) DEFINE_OP(13) DEFINE_OP(14) DEFINE_OP(15) + +static op_fn const ops[] = { + op0, op1, op2, op3, op4, op5, op6, op7, + op8, op9, op10, op11, op12, op13, op14, op15, +}; + +int op_idx = 11; + +SEC("socket") +__success __retval(50) +int callx_c_table(void *ctx) +{ + unsigned int i = op_idx; + + if (i >= sizeof(ops) / sizeof(ops[0])) + return -1; + /* op11(3) = 3 * 13 + 11 */ + return ops[i](3); +} + +SEC("socket") +__success __retval(50) +int callx_c_table_null_check(void *ctx) +{ + unsigned int i = op_idx; + op_fn op; + + if (i >= sizeof(ops) / sizeof(ops[0])) + return -1; + /* the address of a function is not known until the program is jitted */ + op = ops[i]; + if (!op) + return -2; + return op(3); +} + +/* a structure of operations, where pointers to functions are mixed with data */ +struct shape_ops { + int id; + op_fn area; + long scale; + op_fn perimeter; +}; + +static const struct shape_ops square_ops = { 1, op1, 10, op2 }; +static const struct shape_ops circle_ops = { 2, op3, 20, op1 }; + +static __noinline int use_shape(const struct shape_ops *ops, int x) +{ + return ops->area(x) * ops->scale + ops->perimeter(ops->id); +} + +SEC("socket") +__success __retval(433) +int callx_c_ops_mixed_with_data(void *ctx) +{ + /* + * square: op1(2) * 10 + op2(1) = 7 * 10 + 6 = 76 + * circle: op3(3) * 20 + op1(2) = 18 * 20 + 7 = 367 + * minus 10, since op_idx is 11 + */ + return use_shape(&square_ops, 2) + use_shape(&circle_ops, 3) - (op_idx == 11 ? 10 : 0); +} + +/* the ops are selected at run time */ +SEC("socket") +__success __retval(367) +int callx_c_ops_selected(void *ctx) +{ + const struct shape_ops *ops = op_idx == 11 ? &circle_ops : &square_ops; + + return use_shape(ops, 3); +} + +/* + * clang-22 with -mcpu=v4 drops the lookup table it builds for this switch + * ("Undefined temporary symbol .Lswitch.table.call_by_switch"). + * Fixed in clang-23 by llvm commit 9fc458a1d95b. + * + * https://github.com/llvm/llvm-project/commit/9fc458a1d95b + */ +#if __clang_major__ != 22 || __BPF_CPU_VERSION__ < 4 +/* the compiler might turn the switch into a table that has no symbol */ +static __noinline int call_by_switch(unsigned int idx, int x) +{ + op_fn op; + + switch (idx) { + case 0: + op = op8; + break; + case 1: + op = op9; + break; + case 2: + op = op10; + break; + case 3: + op = op11; + break; + case 4: + op = op12; + break; + case 5: + op = op13; + break; + case 6: + op = op14; + break; + case 7: + op = op15; + break; + default: + return -1; + } + return op(x); +} + +SEC("socket") +__success __retval(50) +int callx_c_switch_table(void *ctx) +{ + /* op11(3) = 3 * 13 + 11 */ + return call_by_switch(op_idx - 8, 3); +} + +#else + +SEC("socket") +__skip("clang-22 bug with -mcpu=v4, switch table test cannot be built") +__success +int callx_c_switch_table(void *ctx) +{ + return 0; +} + +#endif + +/* + * Misaligned pointers to functions are ignored, the rest of the data is + * accessible as before. The program doesn't have callx, so libbpf doesn't + * append the functions that .rodata points to and doesn't make a copy of + * .rodata for it. + */ +static const struct { + char tag; + op_fn op; +} __attribute__((packed)) packed_ops[] = { + { 5, op0 }, + { 7, op1 }, +}; + +SEC("socket") +__success __retval(7) +int callx_c_packed_struct(void *ctx) +{ + return packed_ops[op_idx & 1].tag; +} + +#else + +SEC("socket") +__success +int dummy(void *ctx) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_cfg.c b/tools/testing/selftests/bpf/progs/verifier_cfg.c index 3c3bb03e8217..b429fa7a08f5 100644 --- a/tools/testing/selftests/bpf/progs/verifier_cfg.c +++ b/tools/testing/selftests/bpf/progs/verifier_cfg.c @@ -56,6 +56,46 @@ __naked void out_of_range_jump2(void) " ::: __clobber_all); } +static __naked __noinline __used int cross_subprog_target(void) +{ + asm volatile (" \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("jump across subprogram boundary") +__failure __msg("jump out of range from insn 1") +__msg("jump_across_subprog_boundary @ verifier_cfg.c") +__naked void jump_across_subprog_boundary(void) +{ + asm volatile (" \ + call cross_subprog_target; \ + goto +1; \ + exit; \ +" ::: __clobber_all); +} + +static __naked __noinline __used int fallthrough_subprog(void) +{ + asm volatile (" \ + r0 = 0; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("subprogram fallthrough") +__failure __msg("last insn is not an exit or jmp") +__msg("fallthrough_subprog @ verifier_cfg.c") +__naked void subprog_fallthrough(void) +{ + asm volatile (" \ + call fallthrough_subprog; \ + exit; \ +" ::: __clobber_all); +} + SEC("socket") __description("invalid DW LDSX instruction in diagnostics") __failure __msg("BUG_ldx_99") @@ -173,4 +213,43 @@ l1_%=: exit; \ " ::: __clobber_all); } +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u32); +} map_array SEC(".maps"); + +extern void bpf_throw(__u64 cookie) __ksym; + +/* + * The subprogram leaves through the tail call or the exception and never + * reaches an exit insn, so the hidden edge of the tail call has no exit to + * point at. The stack access makes the liveness pass walk the subprogram, + * and the loop compiles to a single jump back onto itself. + */ +__noinline __used +static int tail_call_or_throw_subprog(void *ctx) +{ + volatile __u32 idx = 0; + + bpf_tail_call(ctx, &map_array, idx); + bpf_throw(0); + for (;;) {} + return 0; +} + +SEC("tc") +__description("tail call in a subprogram without an exit") +__arch_x86_64 +__arch_arm64 +__success +__naked void tail_call_in_subprog_without_exit(void) +{ + asm volatile (" \ + call tail_call_or_throw_subprog; \ + exit; \ +" ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_ctx.c b/tools/testing/selftests/bpf/progs/verifier_ctx.c index 7856dad3d1f3..9d42ba824408 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ctx.c +++ b/tools/testing/selftests/bpf/progs/verifier_ctx.c @@ -208,7 +208,7 @@ __naked void null_check_7_ctx_bind(void) SEC("cgroup/post_bind4") __description("pass ctx or null check, 8: null (bind)") -__failure __msg("R1 type=scalar expected=ctx") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void null_check_8_null_bind(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c index dcc2dd46751a..4c55b70df072 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c @@ -87,7 +87,7 @@ __weak int subprog_trusted_task_nonnull(struct task_struct *task __arg_trusted) SEC("?kprobe") __failure __log_level(2) -__msg("R1 type=scalar expected=ptr_, trusted_ptr_, rcu_ptr_") +__msg("Possibly NULL pointer passed to trusted R1") __msg("Caller passes invalid args into func#1 ('subprog_trusted_task_nonnull')") int trusted_task_arg_nonnull_fail1(void *ctx) { @@ -96,7 +96,7 @@ int trusted_task_arg_nonnull_fail1(void *ctx) SEC("?tp_btf/task_newtask") __failure __log_level(2) -__msg("R1 type=trusted_ptr_or_null_ expected=ptr_, trusted_ptr_, rcu_ptr_") +__msg("Possibly NULL pointer passed to trusted R1") __msg("Caller passes invalid args into func#1 ('subprog_trusted_task_nonnull')") int trusted_task_arg_nonnull_fail2(void *ctx) { @@ -177,7 +177,7 @@ __weak int subprog_trusted_destroy(struct task_struct *task __arg_trusted) SEC("?tp_btf/task_newtask") __failure __log_level(2) -__msg("release kfunc bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") +__msg("release function bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(trusted_destroy_fail, struct task_struct *task, u64 clone_flags) { return subprog_trusted_destroy(task); @@ -289,6 +289,41 @@ __weak int subprog_void_untrusted(void *p __arg_untrusted) return *(int *)p; } +__weak int subprog_trusted_bare(struct task_struct *task __arg_trusted) +{ + return task->pid; +} + +SEC("tp_btf/task_newtask") +__failure +__msg("R1 must be referenced or trusted") +__msg("Caller passes invalid args into func#{{.*}} ('subprog_trusted_bare')") +int bare_to_trusted(void *ctx) +{ + struct task_struct *cur = bpf_get_current_task_btf(); + struct task_struct *wakee; + + if (!cur) + return 0; + wakee = cur->last_wakee; + if (!wakee) + return 0; + return subprog_trusted_bare(wakee); +} + +SEC("tp_btf/task_newtask") +__failure +__msg("R1 must be referenced or trusted") +__msg("Caller passes invalid args into func#{{.*}}") +int memrcu_to_trusted(void *ctx) +{ + struct task_struct *cur = bpf_get_current_task_btf(); + + if (!cur) + return 0; + return subprog_trusted_task_nullable(cur->real_parent); +} + __weak int subprog_char_untrusted(char *p __arg_untrusted) { return *(int *)p; @@ -389,7 +424,8 @@ __weak int subprog_pkt_ptr_changes_data(struct __sk_buff *skb __arg_ctx, SEC("?tc") __failure __log_level(2) -__msg("R2 is a packet pointer, but func#{{[0-9]+}} may change packet data") +__msg("cannot pass packet pointer R2") +__msg("function may change packet data") __msg("Caller passes invalid args into func#{{[0-9]+}} ('subprog_pkt_ptr_changes_data')") int pkt_ptr_to_global_mem_arg_changes_data(struct __sk_buff *skb) { diff --git a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c index 966f49348787..574b26b5a7df 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c @@ -195,7 +195,7 @@ int arg_tag_nonnull_ptr_good(void *ctx) SEC("?raw_tp") __failure __log_level(2) -__msg("R1 is expected to be non-NULL") +__msg("Possibly NULL pointer passed to trusted R1") int arg_tag_nonnull_ptr_null_bad(void *ctx) { int y = 74; @@ -536,4 +536,35 @@ int return_from_void_global(struct __sk_buff *skb) return 0; } +int global_calls_loop(int x); + +static __noinline int static_calls_global(int x) +{ + return global_calls_loop(x); +} + +static __noinline int loop_cb_calls_static(u32 i, void *ctx) +{ + return static_calls_global(i); +} + +__noinline int global_calls_loop(int x) +{ + bpf_loop(1, loop_cb_calls_static, NULL, 0); + return 0; +} + +/* + * loop_cb_calls_static() -> static_calls_global() -> global_calls_loop() -> + * bpf_loop() -> loop_cb_calls_static() is an unbounded recursion that the + * main verification pass can't see, because it doesn't follow calls of global + * functions. None of the functions use stack. + */ +SEC("?raw_tp") +__failure __msg("recursive call from global_calls_loop() to loop_cb_calls_static()") +int recursion_via_global_func_and_callback(const void *ctx) +{ + return loop_cb_calls_static(0, NULL); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_gotol.c b/tools/testing/selftests/bpf/progs/verifier_gotol.c index d5d8f24df394..c0fd30a3a4c1 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotol.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotol.c @@ -52,6 +52,7 @@ __naked void gotol_large_imm(void) SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_gotox.c b/tools/testing/selftests/bpf/progs/verifier_gotox.c index 0e27c2c79c57..3d1ff7078a72 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotox.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotox.c @@ -6,7 +6,9 @@ #include "bpf_misc.h" #include "../../../include/linux/filter.h" -#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_powerpc) +#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_powerpc) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) #define DEFINE_SIMPLE_JUMP_TABLE_PROG(NAME, SRC_REG, OFF, IMM, OUTCOME) \ \ @@ -101,9 +103,11 @@ __naked void jump_table_terminal_gotox_subprog(void) */ SEC("socket") __failure __msg("no jump tables found for subprog starting at 0") +__msg(">>> 1 | (0d) gotox r0") __naked void jump_table_no_jump_table(void) { asm volatile (" \ + r0 = 0; \ .8byte %[gotox_r0]; \ r0 = 1; \ exit; \ @@ -366,7 +370,7 @@ __used static int test_subprog(void) } SEC("socket") -__failure __msg("jump table for insn 4 points outside of the subprog [0,10]") +__failure __msg("jump table of subprog starting at 0 spans multiple subprogs") __naked void jump_table_outside_subprog(void) { asm volatile (" \ @@ -457,6 +461,133 @@ __naked void spill_fill_ptr_to_insn(void) : __clobber_all); } -#endif /* __TARGET_ARCH_x86 || __TARGET_ARCH_arm64 || __TARGET_ARCH_powerpc*/ +/* + * Emit each table entry next to the instruction it targets. Numeric labels + * can be reused by .rept, while %= keeps table symbols unique per asm block. + * BASE is the code section: libbpf relocates section-relative byte offsets + * to instruction offsets in the loaded program. + */ +#define GOTOX_TABLE_BEGIN(COUNT) \ + ".pushsection .jumptables,\"\",@progbits;" \ + "jt_%=:;" \ + ".global jt_%=;" \ + ".size jt_%=, 8 * (" #COUNT ");" \ + ".popsection;" \ + "r1 = jt_%= ll;" + +#define GOTOX_TABLE_ENTRY(LABEL, BASE) \ + ".pushsection .jumptables,\"\",@progbits;" \ + ".quad " LABEL " - " BASE ";" \ + ".popsection;" + +/* N gotox instructions, each targeting all N gotox plus the final block. */ +#define GOTOX_SELF_TARGETS(N, BASE) \ + GOTOX_TABLE_BEGIN(N + 1) \ + "r0 = 0;" \ + ".rept " #N ";" \ + GOTOX_TABLE_ENTRY("1f", BASE) \ + "1:; .8byte %[gotox];" \ + ".endr;" \ + GOTOX_TABLE_ENTRY("2f", BASE) \ + "2:;" + +#define GOTOX_OPERAND \ + __imm_insn(gotox, BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_0, 0, 0, 0)) + +/* + * Count edges even when their targets are already discovered. These + * programs deliberately use a scalar: the CFG edge limit must reject them + * before symbolic verification reaches the invalid gotox operand. + */ +SEC("socket") +__description("too-many-gotox-edges") +__failure __msg("number of indirect jump edges in the program exceeds") +__naked void too_many_gotox_edges(void) +{ + asm volatile ( + /* 1000 * 1001 = 1,001,000 edges. */ + GOTOX_SELF_TARGETS(1000, "socket") + "r0 = 0; exit;" + : : GOTOX_OPERAND : __clobber_all); +} + +SEC("socket") +__description("gotox-edges-at-limit") +__failure __msg("Too many nested loops") +__naked void gotox_edges_at_limit(void) +{ + asm volatile ( + /* + * 1000 gotox * 1000 targets = 1,000,000 CFG edges. At run + * time, each block loads the next table entry and jumps to + * the following block, so the program terminates. + * + * Every block can jump to every later block, so the CFG is + * a nest of ~1000 irreducible loops, deeper than the + * verifier's loop stack; the program is rejected. + */ + GOTOX_TABLE_BEGIN(1000) + ".rept 1000;" + GOTOX_TABLE_ENTRY("1f", "socket") + "r0 = *(u64 *)(r1 + 0);" + "r1 += 8;" + ".8byte %[gotox];" + "1:;" + ".endr;" + "r0 = 0; exit;" + : : GOTOX_OPERAND : __clobber_all); +} + +static __noinline __used __naked void gotox_edges_subprog(void) +{ + asm volatile ( + GOTOX_SELF_TARGETS(750, ".text") + "r0 = 0; exit;" + : : GOTOX_OPERAND : __clobber_all); +} + +SEC("socket") +__description("gotox-edges-across-subprogs") +__failure __msg("number of indirect jump edges in the program exceeds") +__naked void gotox_edges_across_subprogs(void) +{ + asm volatile ( + /* Each subprog has 563,250 edges; together, 1,126,500. */ + GOTOX_SELF_TARGETS(750, "socket") + "call gotox_edges_subprog;" + "exit;" + : : GOTOX_OPERAND : __clobber_all); +} + +/* + * The gotox target is a ja +0, which the nop removal pass must keep as the + * jump table still names it. + */ +SEC("socket") +__success __retval(0) +__xlated("goto pc+0") +__naked void jump_table_target_nop(void) +{ + asm volatile (" \ + .pushsection .jumptables,\"\",@progbits; \ +jt0_%=: \ + .quad nop_%= - socket; \ + .size jt0_%=, 8; \ + .global jt0_%=; \ + .popsection; \ + \ + r0 = jt0_%= ll; \ + r0 = *(u64 *)(r0 + 0); \ + .8byte %[gotox_r0]; \ +nop_%=: \ + goto +0; \ + r0 = 0; \ + exit; \ +" : + : __imm_insn(gotox_r0, BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_0, 0, 0, 0)) + : __clobber_all); +} + +#endif /* gotox: x86, arm64, powerpc, riscv64 */ char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c index 343fc08d9747..2782faf0a528 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ /* Converted from tools/testing/selftests/bpf/verifier/helper_access_var_len.c */ #include <linux/bpf.h> @@ -621,7 +622,7 @@ l0_%=: exit; \ SEC("tracepoint") __description("helper access to variable memory: size = 0 not allowed on NULL (!ARG_PTR_TO_MEM_OR_NULL)") -__failure __msg("R1 type=scalar expected=fp") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void ptr_to_mem_or_null_8(void) { asm volatile (" \ @@ -637,7 +638,7 @@ __naked void ptr_to_mem_or_null_8(void) SEC("tracepoint") __description("helper access to variable memory: size > 0 not allowed on NULL (!ARG_PTR_TO_MEM_OR_NULL)") -__failure __msg("R1 type=scalar expected=fp") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void ptr_to_mem_or_null_9(void) { asm volatile (" \ @@ -822,4 +823,335 @@ __naked void bytes_no_leak_init_memory(void) : __clobber_all); } +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __uint(map_flags, BPF_F_WRONLY_PROG); + __type(key, __u32); + __type(value, struct bpf_fib_lookup); +} map_fib_wo SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct bpf_fib_lookup); +} map_fib_rw SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __uint(map_flags, BPF_F_RDONLY_PROG); + __type(key, __u32); + __type(value, struct bpf_fib_lookup); +} map_fib_ro SEC(".maps"); + +SEC("tc") +__failure __msg("read from map forbidden") +int writeonly_sized_input(struct __sk_buff *ctx) +{ + struct bpf_fib_lookup *params; + __u32 key = 0; + + params = bpf_map_lookup_elem(&map_fib_wo, &key); + if (params) + bpf_fib_lookup(ctx, params, sizeof(*params), 0); + return 0; +} + +SEC("tc") +__success +int readwrite_sized_input(struct __sk_buff *ctx) +{ + struct bpf_fib_lookup *params; + __u32 key = 0; + + params = bpf_map_lookup_elem(&map_fib_rw, &key); + if (params) + bpf_fib_lookup(ctx, params, sizeof(*params), 0); + return 0; +} + +SEC("tc") +__failure __msg("write into map forbidden") +int readonly_sized_output(struct __sk_buff *ctx) +{ + struct bpf_fib_lookup *params; + __u32 key = 0; + + params = bpf_map_lookup_elem(&map_fib_ro, &key); + if (params) + bpf_fib_lookup(ctx, params, sizeof(*params), 0); + return 0; +} + +SEC("xdp") +__failure __msg("read from map forbidden") +int xdp_writeonly_sized_input(struct xdp_md *ctx) +{ + struct bpf_fib_lookup *params; + __u32 key = 0; + + params = bpf_map_lookup_elem(&map_fib_wo, &key); + if (params) + bpf_fib_lookup(ctx, params, sizeof(*params), 0); + return XDP_PASS; +} + +SEC("sockops") +__failure __msg("read from map forbidden") +int writeonly_header_option(struct bpf_sock_ops *ctx) +{ + struct bpf_fib_lookup *buf; + __u32 key = 0; + + buf = bpf_map_lookup_elem(&map_fib_wo, &key); + if (buf) + bpf_load_hdr_opt(ctx, buf, sizeof(*buf), 0); + return 0; +} + +SEC("sockops") +__success +int readwrite_header_option(struct bpf_sock_ops *ctx) +{ + struct bpf_fib_lookup *buf; + __u32 key = 0; + + buf = bpf_map_lookup_elem(&map_fib_rw, &key); + if (buf) + bpf_load_hdr_opt(ctx, buf, sizeof(*buf), 0); + return 0; +} + +SEC("tc") +__success +int snprintf_writeonly_output(struct __sk_buff *ctx) +{ + void *buf; + __u32 key = 0; + + buf = bpf_map_lookup_elem(&map_fib_wo, &key); + if (buf) + bpf_snprintf(buf, 16, "ok", NULL, 0); + return 0; +} + +SEC("tc") +__failure __msg("write into map forbidden") +int snprintf_readonly_output(struct __sk_buff *ctx) +{ + void *buf; + __u32 key = 0; + + buf = bpf_map_lookup_elem(&map_fib_ro, &key); + if (buf) + bpf_snprintf(buf, 16, "ok", NULL, 0); + return 0; +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +int sysctl_writeonly_output(struct bpf_sysctl *ctx) +{ + void *buf; + __u32 key = 0; + + buf = bpf_map_lookup_elem(&map_fib_wo, &key); + if (buf) + bpf_sysctl_get_name(ctx, buf, 16, 0); + return 0; +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void sysctl_initialized_stack(void) +{ + asm volatile ( + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 8) = 0;" + "r2 = r10;" + "r2 += -16;" + "r3 = 16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u8 *)(r10 - 1);" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void sysctl_uninitialized_stack(void) +{ + asm volatile ( + "r2 = r10;" + "r2 += -16;" + "r3 = 16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = 0;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void sysctl_partial_initialized_bytes(void) +{ + asm volatile ( + "*(u32 *)(r10 - 16) = 0;" + "*(u8 *)(r10 - 1) = 1;" + "goto +0;" + "r2 = r10;" + "r2 += -16;" + "r3 = 16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u32 *)(r10 - 16);" + "r1 = *(u8 *)(r10 - 1);" + "r0 += r1;" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__failure_unpriv __msg_unpriv("invalid read from stack off -1+0 size 1") +__naked void sysctl_partial_invalid_bytes(void) +{ + asm volatile ( + "*(u32 *)(r10 - 16) = 0;" + "r2 = r10;" + "r2 += -16;" + "r3 = 16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u8 *)(r10 - 1);" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv +__naked void sysctl_partial_variable_size(void) +{ + asm volatile ( + "r3 = *(u32 *)(r1 + 0);" + "r3 &= 15;" + "r3 += 1;" + "*(u8 *)(r10 - 1) = 1;" + "goto +0;" + "r2 = r10;" + "r2 += -16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u8 *)(r10 - 1);" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__failure_unpriv __msg_unpriv("invalid read from stack off -1+0 size 1") +__naked void sysctl_partial_variable_invalid(void) +{ + asm volatile ( + "r3 = *(u32 *)(r1 + 0);" + "r3 &= 15;" + "r3 += 1;" + "r2 = r10;" + "r2 += -16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u8 *)(r10 - 1);" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__failure_unpriv __msg_unpriv("invalid read from stack R2") +__naked void sysctl_partial_spilled_pointer(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = r1;" + "goto +0;" + "r2 = r10;" + "r2 += -8;" + "r3 = 8;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = 0;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("cgroup/sysctl") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__failure_unpriv __msg_unpriv("invalid read from stack R2") +int sysctl_partial_dynptr(struct bpf_sysctl *ctx) +{ + struct bpf_dynptr ptr; + long long key = 0, *data; + + data = bpf_map_lookup_elem(&map_hash_8b, &key); + if (!data) + return 0; + bpf_dynptr_from_mem(data, sizeof(*data), 0, &ptr); + bpf_sysctl_get_name(ctx, (char *)&ptr, sizeof(ptr), 0); + return 0; +} + +SEC("cgroup/sysctl") +__success +__naked void sysctl_partial_variable_offset(void) +{ + asm volatile ( + "r2 = *(u32 *)(r1 + 0);" + "r2 &= 8;" + "r2 += r10;" + "r2 += -24;" + "r3 = 16;" + "r4 = 0;" + "call %[bpf_sysctl_get_name];" + "r0 = *(u8 *)(r10 - 16);" + "r0 &= 1;" + "exit;" + : : __imm(bpf_sysctl_get_name) : __clobber_all); +} + +SEC("tc") +__success __retval(42) +int snprintf_partial_output_runtime(struct __sk_buff *ctx) +{ + char buf[16]; + + buf[15] = 42; + bpf_snprintf(buf, sizeof(buf), "ok", NULL, 0); + return buf[15]; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_helper_packet_access.c index 71cee3f58324..12786b72c694 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_packet_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_packet_access.c @@ -258,7 +258,7 @@ l0_%=: r0 = 0; \ SEC("tc") __description("helper access to packet: test11, cls unsuitable helper 1") -__failure __msg("helper access to the packet") +__failure __msg("function access to the packet") __naked void test11_cls_unsuitable_helper_1(void) { asm volatile (" \ @@ -283,7 +283,7 @@ l0_%=: r0 = 0; \ SEC("tc") __description("helper access to packet: test12, cls unsuitable helper 2") -__failure __msg("helper access to the packet") +__failure __msg("function access to the packet") __naked void test12_cls_unsuitable_helper_2(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_idoms.c b/tools/testing/selftests/bpf/progs/verifier_idoms.c new file mode 100644 index 000000000000..fdee6bced77b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_idoms.c @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "../../../include/linux/filter.h" + +/* + * The immediate dominator is printed by log_program() as the numeric column + * immediately before the "<insn#>:" field of every "Program dump" line: + * + * Program dump (scc? loop_header? idom insn#: live_regs_before): + * -1 0: ....... (b7) r0 = 0 <- idom(0) = -1 (subprog entry) + * 0 1: ....... (07) r0 += 1 <- idom(1) = 0 + * ^^^^^^ + * idom insn# + */ + +/* Straight-line code, with an ldimm64 in the middle. */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 0 1: {{.*}} (18) r1 = 0x1122334455667788") +__msg("{{.*}} 1 3: {{.*}} (07) r0 += 1") +__msg("{{.*}} 3 4: {{.*}} (0f) r0 += r1") +__msg("{{.*}} 4 5: {{.*}} (95) exit") +__naked void straight_line_ldimm64(void) +{ + asm volatile (" \ + r0 = 0; \ + r1 = 0x1122334455667788 ll; \ + r0 += 1; \ + r0 += r1; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Asymmetric if-then-else diamond: the "then" arm is 5 instructions long, the + * "else" arm is a single instruction. The merge point (insn 8) is dominated by + * the branch (insn 1), not by either arm. This forces idoms_intersect() to walk + * the two predecessors up unequal postorder depths before they meet. + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (85) call bpf_get_prandom_u32#7") +__msg("{{.*}} 0 1: {{.*}} (25) if r0 > 0x0 goto pc+5") +__msg("{{.*}} 1 2: {{.*}} (b7) r1 = 1") +__msg("{{.*}} 2 3: {{.*}} (b7) r1 = 2") +__msg("{{.*}} 3 4: {{.*}} (b7) r1 = 3") +__msg("{{.*}} 4 5: {{.*}} (b7) r1 = 4") +__msg("{{.*}} 5 6: {{.*}} (05) goto pc+1") +__msg("{{.*}} 1 7: {{.*}} (b7) r1 = 9") +__msg("{{.*}} 1 8: {{.*}} (95) exit") +__naked void asymmetric_diamond(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + if r0 > 0 goto 1f; \ + r1 = 1; \ + r1 = 2; \ + r1 = 3; \ + r1 = 4; \ + goto 2f; \ +1: r1 = 9; \ +2: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Simple loop: the header (insn 1) is dominated by the pre-header (insn 0). The + * back edge (insn 2 -> insn 1) must not change idom(1); the back-edge + * predecessor folds up via idoms_intersect(). + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 0 1: {{.*}} (07) r0 += 1") +__msg("{{.*}} 1 2: {{.*}} (a5) if r0 < 0xa goto pc-2") +__msg("{{.*}} 2 3: {{.*}} (95) exit") +__naked void simple_loop(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Loop with an if-else in its body. The in-loop merge point (insn 5) is + * dominated by the in-loop branch (insn 3), combining a back-edge intersect + * with a forward-diamond intersect. + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 0 1: {{.*}} (07) r0 += 1") +__msg("{{.*}} 1 2: {{.*}} (bf) r1 = r0") +__msg("{{.*}} 2 3: {{.*}} (25) if r1 > 0x5 goto pc+1") +__msg("{{.*}} 3 4: {{.*}} (b7) r1 = 1") +__msg("{{.*}} 3 5: {{.*}} (0f) r0 += r1") +__msg("{{.*}} 5 6: {{.*}} (a5) if r0 < 0x64 goto pc-6") +__msg("{{.*}} 6 7: {{.*}} (95) exit") +__naked void loop_with_if_else_body(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = r0; \ + if r1 > 5 goto 2f; \ + r1 = 1; \ +2: r0 += r1; \ + if r0 < 100 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Nested loops: the idom chain runs inner-header -> outer-body -> outer-header + * -> pre-header. Exercises intersect across back edges at two nesting depths. + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 0 1: {{.*}} (07) r0 += 1") +__msg("{{.*}} 1 2: {{.*}} (b7) r1 = 0") +__msg("{{.*}} 2 3: {{.*}} (07) r1 += 1") +__msg("{{.*}} 3 4: {{.*}} (a5) if r1 < 0x5 goto pc-2") +__msg("{{.*}} 4 5: {{.*}} (a5) if r0 < 0xa goto pc-5") +__msg("{{.*}} 5 6: {{.*}} (95) exit") +__naked void nested_loops(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = 0; \ +2: r1 += 1; \ + if r1 < 5 goto 2b; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Loop header (insn 1) with two back edges (from insn 4 and insn 5), i.e. three + * predecessors. Both back edges route through the incrementing header, so the + * loop is bounded and verifies. Repeated idoms_intersect() at the header must + * stay stable and keep idom(1) at the pre-header (insn 0). + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r6 = 0") +__msg("{{.*}} 0 1: {{.*}} (07) r6 += 1") +__msg("{{.*}} 1 2: {{.*}} (25) if r6 > 0xa goto pc+3") +__msg("{{.*}} 2 3: {{.*}} (bf) r7 = r6") +__msg("{{.*}} 3 4: {{.*}} (a5) if r7 < 0x5 goto pc-4") +__msg("{{.*}} 4 5: {{.*}} (05) goto pc-5") +__msg("{{.*}} 2 6: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 6 7: {{.*}} (95) exit") +__naked void multi_backedge_header(void) +{ + asm volatile (" \ + r6 = 0; \ +1: r6 += 1; \ + if r6 > 10 goto 2f; \ + r7 = r6; \ + if r7 < 5 goto 1b; \ + goto 1b; \ +2: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Irreducible CFG (loop with two entries, insn 3 and insn 5, reached from the + * insn 2 branch). Dominators remain well-defined; this is a convergence test + * for the fixpoint under irreducibility. Note insn 6 is dominated by the branch + * (insn 2), not by either arm. + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (85) call bpf_get_prandom_u32#7") +__msg("{{.*}} 0 1: {{.*}} (b7) r1 = 0") +__msg("{{.*}} 1 2: {{.*}} (25) if r0 > 0x5 goto pc+2") +__msg("{{.*}} 2 3: {{.*}} (b7) r1 = 1") +__msg("{{.*}} 3 4: {{.*}} (05) goto pc+1") +__msg("{{.*}} 2 5: {{.*}} (b7) r1 = 2") +__msg("{{.*}} 2 6: {{.*}} (0f) r0 += r1") +__msg("{{.*}} 6 7: {{.*}} (a5) if r0 < 0x10 goto pc-5") +__msg("{{.*}} 7 8: {{.*}} (95) exit") +__naked void irreducible(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = 0; \ + if r0 > 5 goto 2f; \ +1: r1 = 1; \ + goto 3f; \ +2: r1 = 2; \ +3: r0 += r1; \ + if r0 < 16 goto 1b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Idoms are computed per subprog. Layout (libbpf preorder) is: + * main: 0..1, sub: 2..7 + * The callee entry (insn 2) must have idom -1 (subprog reset), and idoms inside + * the callee must reference only the callee's instructions, never main's. The + * in-callee merge (insn 7) is dominated by the in-callee branch (insn 3). The + * branch is on a prandom value so it is not constant-folded away. + */ +static __naked __noinline __used +unsigned long idoms_diamond_sub(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + if r0 > 0 goto 1f; \ + r0 = 1; \ + goto 2f; \ +1: r0 = 2; \ +2: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (85) call pc+1") +__msg("{{.*}} 0 1: {{.*}} (95) exit") +__msg("{{.*}} -1 2: {{.*}} (85) call bpf_get_prandom_u32#7") +__msg("{{.*}} 2 3: {{.*}} (25) if r0 > 0x0 goto pc+2") +__msg("{{.*}} 3 4: {{.*}} (b7) r0 = 1") +__msg("{{.*}} 4 5: {{.*}} (05) goto pc+1") +__msg("{{.*}} 3 6: {{.*}} (b7) r0 = 2") +__msg("{{.*}} 3 7: {{.*}} (95) exit") +__naked void multi_subprog(void) +{ + asm volatile (" \ + call idoms_diamond_sub; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The first instruction of a subprog (insn 3) is itself a loop header, i.e. it + * has an incoming back edge. Its idom must still be -1 (the subprog entry has no + * dominator). Exercises the idoms[start]=start ... idoms[start]=-1 handling in + * compute_subprog_idoms(). + */ +static __naked __noinline __used +unsigned long idoms_entry_header_sub(void) +{ + asm volatile (" \ +1: r1 += 1; \ + if r1 < 10 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (b7) r1 = 0") +__msg("{{.*}} 0 1: {{.*}} (85) call pc+1") +__msg("{{.*}} 1 2: {{.*}} (95) exit") +__msg("{{.*}} -1 3: {{.*}} (07) r1 += 1") +__msg("{{.*}} 3 4: {{.*}} (a5) if r1 < 0xa goto pc-2") +__msg("{{.*}} 4 5: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 5 6: {{.*}} (95) exit") +__naked void entry_is_loop_header(void) +{ + asm volatile (" \ + r1 = 0; \ + call idoms_entry_header_sub; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Self-loop: insn 1 is its own predecessor. idom(1) must be the pre-header + * (insn 0); the self-edge is ignored (idoms[pred] == -1 on the first pass, then + * idoms_intersect(a == b) short-circuits). Program is rejected later for an + * infinite loop, but the dump (and idoms) are printed before that. + */ +SEC("socket") +__failure +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (85) call bpf_get_prandom_u32#7") +__msg("{{.*}} 0 1: {{.*}} (a5) if r0 < 0xa goto pc-1") +__msg("{{.*}} 1 2: {{.*}} (95) exit") +__msg("infinite loop detected") +__naked void self_loop(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ +1: if r0 < 10 goto 1b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_powerpc) +/* + * Indirect jump (gotox) with a two-entry jump table. The gotox (insn 4) has two + * successors (insn 5 and insn 7), so both targets have the gotox as their only + * predecessor and idom. + */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg("{{.*}} -1 0: {{.*}} (18) r0 = {{0x[0-9a-f]+}}") +__msg("{{.*}} 0 2: {{.*}} (07) r0 += 8") +__msg("{{.*}} 2 3: {{.*}} (79) r0 = *(u64 *)(r0 +0)") +__msg("{{.*}} 3 4: {{.*}} (0d) gotox r0") +__msg("{{.*}} 4 5: {{.*}} (b7) r0 = 0") +__msg("{{.*}} 5 6: {{.*}} (95) exit") +__msg("{{.*}} 4 7: {{.*}} (b7) r0 = 1") +__msg("{{.*}} 7 8: {{.*}} (95) exit") +__naked void gotox_jump_table(void) +{ + asm volatile (" \ + .pushsection .jumptables,\"\",@progbits; \ +jt0_%=: \ + .quad ret0_%= - socket; \ + .quad ret1_%= - socket; \ + .size jt0_%=, 16; \ + .global jt0_%=; \ + .popsection; \ + \ + r0 = jt0_%= ll; \ + r0 += 8; \ + r0 = *(u64 *)(r0 + 0); \ + .8byte %[gotox_r0]; \ +ret0_%=: \ + r0 = 0; \ + exit; \ +ret1_%=: \ + r0 = 1; \ + exit; \ +" : + : __imm_insn(gotox_r0, BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_0, 0, 0, 0)) + : __clobber_all); +} +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_jit_convergence.c b/tools/testing/selftests/bpf/progs/verifier_jit_convergence.c index 9f3f2b7db450..4fd717334858 100644 --- a/tools/testing/selftests/bpf/progs/verifier_jit_convergence.c +++ b/tools/testing/selftests/bpf/progs/verifier_jit_convergence.c @@ -68,6 +68,13 @@ __naked void btf_jit_convergence_je_jmp(void) "r3 &= 0x1;" "r2 *= 0xa8;" "if r3 == 0x0 goto l2_%=;" + /* + * The x86-64 code from the je to l0 up to l0 has to be 125 bytes with + * the last goto l9 at two bytes and 128 with it at five, so that the + * je fits into an imm8 only while the jmp does not and the two keep + * flipping each other's size. The mov pads the region to that. + */ + "r3 = 0x1234;" "r1 += r2;" "r1 -= r7;" "r1 += 0x8;" diff --git a/tools/testing/selftests/bpf/progs/verifier_jit_inline.c b/tools/testing/selftests/bpf/progs/verifier_jit_inline.c index 02e562f56f9d..5b92da2ab75c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_jit_inline.c +++ b/tools/testing/selftests/bpf/progs/verifier_jit_inline.c @@ -11,7 +11,7 @@ __jited(" addq %gs:{{.*}}, %rax") __arch_arm64 __jited(" mrs x8, SP_EL0") __arch_riscv64 -__jited(" mv a5, tp") +__jited(" mv t6, tp") __arch_loongarch __jited(" move $a5, $tp") int inline_bpf_get_current_task(void) diff --git a/tools/testing/selftests/bpf/progs/verifier_kfunc_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_kfunc_packet_access.c new file mode 100644 index 000000000000..88009566d92f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_kfunc_packet_access.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +struct nf_conn *bpf_skb_ct_lookup(struct __sk_buff *skb_ctx, + struct bpf_sock_tuple *bpf_tuple, + u32 tuple__sz, struct bpf_ct_opts *opts, + u32 opts__sz) __ksym; +void bpf_ct_release(struct nf_conn *nfct) __ksym; + +char _license[] SEC("license") = "GPL"; + +SEC("tc") +__description("kfunc packet write requests writable skb") +__success +/* bpf_unclone_prologue() */ +__xlated("r6 = *(u8 *)(r1 +{{[0-9]+}})") +__xlated("...") +__xlated("w6 &= {{(1|128)}}") +__xlated("...") +__xlated("if r6 == 0x0 goto") +__xlated("r6 = r1") +__xlated("r2 ^= r2") +__xlated("call") +__xlated("if r0 == 0x0 goto") +__xlated("w0 = 2") +__xlated("...") +__xlated("exit") +__xlated("r1 = r6") +int kfunc_packet_write(struct __sk_buff *skb) +{ + void *data_end = (void *)(long)skb->data_end; + void *data = (void *)(long)skb->data; + struct bpf_sock_tuple tuple = {}; + struct nf_conn *nfct; + + if (data + sizeof(struct bpf_ct_opts) > data_end) + return 0; + + /* An invalid tuple size makes bpf_skb_ct_lookup() write opts->error. */ + nfct = bpf_skb_ct_lookup(skb, &tuple, 1, data, sizeof(struct bpf_ct_opts)); + if (nfct) + bpf_ct_release(nfct); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c new file mode 100644 index 000000000000..20ed71a54686 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +/* Keep the kfunc BTF records used by the inline assembly. */ +void __kfunc_btf_root(void) +{ + asm volatile ("" : + : "r"(&bpf_kfunc_test_uninit_struct), + "r"(&bpf_kfunc_test_uninit_mem), + "r"(&bpf_kfunc_test_uninit_partial), + "r"(&bpf_kfunc_test_uninit_alias)); +} + +SEC("tc") +__success __retval(10) +__log_level(2) +__msg("call bpf_kfunc_test_uninit_struct{{.*}}; def: fp0-8 fp0-16") +__msg_unpriv("call bpf_kfunc_test_uninit_struct{{.*}}; use: fp0-8 fp0-16") +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void struct_poisoned_at_checkpoint(void) +{ + asm volatile ( + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 8) = 0;" + "goto +0;" + "r1 = r10;" + "r1 += -16;" + "call %[bpf_kfunc_test_uninit_struct];" + "r0 = *(u32 *)(r10 - 16);" + "r1 = *(u32 *)(r10 - 12);" + "r0 += r1;" + "r1 = *(u32 *)(r10 - 8);" + "r0 += r1;" + "r1 = *(u32 *)(r10 - 4);" + "r0 += r1;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_struct) : __clobber_all); +} + +SEC("tc") +__success __retval(0x2a2a2a2a) +__log_level(2) +__msg("call bpf_kfunc_test_uninit_mem{{.*}}; def: fp0-8") +__msg_unpriv("call bpf_kfunc_test_uninit_mem{{.*}}; use: fp0-8") +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void sized_buffer_poisoned_at_checkpoint(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "goto +0;" + "r1 = r10;" + "r1 += -8;" + "r2 = 8;" + "call %[bpf_kfunc_test_uninit_mem];" + "r0 = *(u32 *)(r10 - 8);" + "r1 = *(u32 *)(r10 - 4);" + "if r0 == r1 goto +1;" + "r0 = 0;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_mem) : __clobber_all); +} + +SEC("tc") +__success __retval(7) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void initialized_input_alias(void) +{ + asm volatile ( + "*(u32 *)(r10 - 8) = 7;" + "r1 = r10;" + "r1 += -8;" + "r2 = r1;" + "call %[bpf_kfunc_test_uninit_alias];" + "exit;" + : : __imm(bpf_kfunc_test_uninit_alias) : __clobber_all); +} + +SEC("tc") +__success +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack") +__naked void uninitialized_input_alias(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = r1;" + "call %[bpf_kfunc_test_uninit_alias];" + "exit;" + : : __imm(bpf_kfunc_test_uninit_alias) : __clobber_all); +} + +SEC("tc") +__success __retval(0) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void uninitialized_struct_output_ignored(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -16;" + "call %[bpf_kfunc_test_uninit_struct];" + "r0 = 0;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_struct) : __clobber_all); +} + +SEC("tc") +__success __retval(0) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void variable_size_uninitialized_output_ignored(void) +{ + asm volatile ( + "r2 = *(u32 *)(r1 + 0);" + "r2 &= 7;" + "r2 += 1;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_kfunc_test_uninit_mem];" + "r0 = 0;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_mem) : __clobber_all); +} + +SEC("tc") +__success __retval(49) +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void partial_output_preserves_initialized_bytes(void) +{ + asm volatile ( + "*(u8 *)(r10 - 8) = 0;" + "*(u8 *)(r10 - 1) = 7;" + "goto +0;" + "r1 = r10;" + "r1 += -8;" + "r2 = 8;" + "call %[bpf_kfunc_test_uninit_partial];" + "r0 = *(u8 *)(r10 - 8);" + "r1 = *(u8 *)(r10 - 1);" + "r0 += r1;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_partial) : __clobber_all); +} + +SEC("tc") +__success __retval(42) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack off -8+0 size 1") +__naked void uninitialized_sized_output_read(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r2 = 8;" + "call %[bpf_kfunc_test_uninit_mem];" + "r0 = *(u8 *)(r10 - 8);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_mem) : __clobber_all); +} + +SEC("tc") +__success __retval(42) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack off -8+0 size 1") +__naked void variable_size_uninitialized_output_read(void) +{ + asm volatile ( + "r2 = *(u32 *)(r1 + 0);" + "r2 &= 7;" + "r2 += 1;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_kfunc_test_uninit_mem];" + "r0 = *(u8 *)(r10 - 8);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_mem) : __clobber_all); +} + +SEC("tc") +__success __retval(1) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack off -16+0 size 4") +__naked void fixed_struct_uninitialized_output_read(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -16;" + "call %[bpf_kfunc_test_uninit_struct];" + "r0 = *(u32 *)(r10 - 16);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_struct) : __clobber_all); +} + +SEC("tc") +__success __retval(49) +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void partial_output_variable_size(void) +{ + asm volatile ( + "r2 = *(u32 *)(r1 + 0);" + "r2 &= 7;" + "r2 += 1;" + "*(u8 *)(r10 - 8) = 0;" + "*(u8 *)(r10 - 1) = 7;" + "goto +0;" + "r1 = r10;" + "r1 += -8;" + "call %[bpf_kfunc_test_uninit_partial];" + "r0 = *(u8 *)(r10 - 8);" + "r1 = *(u8 *)(r10 - 1);" + "r0 += r1;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_partial) : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit_multi.c b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit_multi.c new file mode 100644 index 000000000000..ab3bb81a11fd --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_kfunc_uninit_multi.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +/* Keep the kfunc BTF records used by the inline assembly. */ +void __kfunc_btf_root(void) +{ + asm volatile ("" : + : "r"(&bpf_kfunc_test_uninit_multi), + "r"(&bpf_kfunc_test_uninit_pair), + "r"(&bpf_kfunc_test_uninit_stack)); +} + +SEC("tc") +__success __retval(42) +__flag(BPF_F_TEST_STATE_FREQ) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__success_unpriv +__naked void multiple_outputs(void) +{ + asm volatile ( + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 8) = 0;" + "goto +0;" + "r1 = r10;" + "r1 += -16;" + "r2 = r10;" + "r2 += -8;" + "r3 = 8;" + "call %[bpf_kfunc_test_uninit_multi];" + "r0 = *(u32 *)(r10 - 16);" + "r1 = *(u32 *)(r10 - 8);" + "if r1 != 0x2a2a2a2a goto 1f;" + "r1 = *(u32 *)(r10 - 4);" + "if r1 == 0x2a2a2a2a goto 2f;" + "1:;" + "r0 = 0;" + "2:;" + "exit;" + : : __imm(bpf_kfunc_test_uninit_multi) : __clobber_all); +} + +SEC("tc") +__success __retval(42) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack off -16+0 size 4") +__naked void variable_size_preserves_other_output(void) +{ + asm volatile ( + "r3 = *(u32 *)(r1 + 0);" + "r3 &= 7;" + "*(u64 *)(r10 - 8) = 0;" + "r1 = r10;" + "r1 += -16;" + "r2 = r10;" + "r2 += -8;" + "call %[bpf_kfunc_test_uninit_multi];" + "r0 = *(u32 *)(r10 - 16);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_multi) : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__success __retval(42) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack off -8+0 size 4") +__naked void output_after_by_value_argument(void) +{ + asm volatile ( + "r1 = 20;" + "r2 = 22;" + "r3 = r10;" + "r3 += -8;" + "call %[bpf_kfunc_test_uninit_pair];" + "r0 = *(u32 *)(r10 - 8);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_pair) : __clobber_all); +} + +#if defined(__BPF_FEATURE_STACK_ARGUMENT) +SEC("tc") +__arch_x86_64 __arch_arm64 __arch_riscv64 +__success __retval(15) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__prepare_priv +__failure_unpriv __msg_unpriv("invalid read from stack off -8+0 size 4") +__naked void output_passed_on_stack(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "r6 = r10;" + "r6 += -8;" + "*(u64 *)(r11 - 8) = r6;" + "call %[bpf_kfunc_test_uninit_stack];" + "r0 = *(u32 *)(r10 - 8);" + "exit;" + : : __imm(bpf_kfunc_test_uninit_stack) : __clobber_all); +} +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_large_stack.c b/tools/testing/selftests/bpf/progs/verifier_large_stack.c new file mode 100644 index 000000000000..c2a3ddf5ac91 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_large_stack.c @@ -0,0 +1,425 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* + * Programs may use MAX_BPF_STACK_JIT (2 KiB) of stack on JITs that support + * large stacks, combined over a call chain, with no separate limit on a + * single frame. Interpreted programs and other JITs keep 512 bytes. + */ + +SEC("socket") +__description("single frame of 2048 bytes") +__load_if_large_stack() +__success __success_unpriv __retval(42) +__naked void single_frame_2048(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2048; \ + r0 = 42; \ + *(u64*)(r1 + 0) = r0; \ + r0 = *(u64*)(r1 + 0); \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("single frame of 2048 bytes without large stack support") +__load_if_no_large_stack() +__failure __msg("invalid write to stack R1 off=-2048 size=8") +__naked void single_frame_2048_no_large_stack(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2048; \ + r0 = 42; \ + *(u64*)(r1 + 0) = r0; \ + exit; \ +" ::: __clobber_all); +} + +__used __naked +static void frame_512_leaf(void) +{ + asm volatile (" \ + r1 = 1; \ + *(u64 *)(r10 - 512) = r1; \ + exit; \ +" ::: __clobber_all); +} + +__used __naked +static void frame_512_depth_2(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 512) = r1; \ + call frame_512_leaf; \ + exit; \ +" ::: __clobber_all); +} + +__used __naked +static void frame_512_depth_3(void) +{ + asm volatile (" \ + r1 = 3; \ + *(u64 *)(r10 - 512) = r1; \ + call frame_512_depth_2; \ + exit; \ +" ::: __clobber_all); +} + +__used __naked +static void frame_512_depth_4(void) +{ + asm volatile (" \ + r1 = 4; \ + *(u64 *)(r10 - 512) = r1; \ + call frame_512_depth_3; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("four frames of 512 bytes fit the 2 KiB budget") +__load_if_large_stack() +__success __log_level(4) __msg("stack depth max 2048") +__naked void four_frames_of_512(void) +{ + asm volatile (" \ + call frame_512_depth_4; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("five frames of 512 bytes exceed the 2 KiB budget") +__load_if_large_stack() +__failure __msg("combined stack size of 5 calls is 2560. Too large") +__naked void five_frames_of_512(void) +{ + asm volatile (" \ + r1 = 5; \ + *(u64 *)(r10 - 512) = r1; \ + call frame_512_depth_4; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +__used __naked +static void frame_1536_leaf(void) +{ + asm volatile (" \ + r1 = 1; \ + *(u64 *)(r10 - 1536) = r1; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("512-byte frame calling a 1536-byte frame") +__load_if_large_stack() +__success __log_level(4) __msg("stack depth max 2048") +__naked void uneven_frames_fit(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 512) = r1; \ + call frame_1536_leaf; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("520-byte frame calling a 1536-byte frame") +__load_if_large_stack() +__failure __msg("combined stack size of 2 calls is 2064. Too large") +__naked void uneven_frames_exceed(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 520) = r1; \ + call frame_1536_leaf; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +#ifdef __BPF_FEATURE_MAY_GOTO +/* may_goto adds its counter below the frame; a JIT does not hold that against the budget */ +SEC("socket") +__description("frame of 2048 bytes with may_goto") +__load_if_large_stack() +__success __retval(42) +__naked void frame_2048_with_may_goto(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2048; \ + r0 = 42; \ + *(u32*)(r1 + 0) = r0; \ + may_goto l0_%=; \ + r2 = 100; \ + l0_%=: \ + exit; \ +" ::: __clobber_all); +} +#endif + +SEC("socket") +__description("variable offset write reaching 2048 bytes deep") +__load_if_large_stack() +__success +__naked void var_off_write_to_2048(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 8; \ + r2 = r10; \ + r2 += -2048; \ + r2 += r0; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__description("variable offset write reaching 2056 bytes deep") +__load_if_large_stack() +__failure __msg("invalid variable-offset write to stack R2") +__naked void var_off_write_to_2056(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 8; \ + r2 = r10; \ + r2 += -2056; \ + r2 += r0; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Each frame of a private stack gets the whole budget. */ +__used __naked +static void priv_stack_frame_2048(void) +{ + asm volatile (" \ + r1 = 1; \ + *(u64 *)(r10 - 2048) = r1; \ + exit; \ +" ::: __clobber_all); +} + +SEC("kprobe") +__description("private stack: two frames of 2048 bytes") +__load_if_large_stack() +__arch_x86_64 +__arch_arm64 +__success __log_level(4) +__msg("stack depth max 2048") +__msg("subprog 0 (private_stack_two_frames) main {{.*}} stack 2048") +__msg("subprog 1 (priv_stack_frame_2048) static {{.*}} stack 2048") +__naked void private_stack_two_frames(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 2048) = r1; \ + call priv_stack_frame_2048; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +/* + * A tail call unwinds the frame of the program doing it, so a large main + * frame is fine; the 256-byte rule only concerns the frames of callers of a + * subprog that tail calls. + */ +SEC("tc") +__description("tail call from a 1 KiB frame") +__load_if_large_stack() +__success +__naked void tail_call_from_large_frame(void) +{ + asm volatile (" \ + r2 = 42; \ + *(u64 *)(r10 - 1024) = r2; \ + r2 = %[jmp_table] ll; \ + r3 = 0; \ + call %[bpf_tail_call]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_tail_call), + __imm_addr(jmp_table) + : __clobber_all); +} + +/* + * bpf_clone_redirect() can run the program again on top of its own frame, + * ten frames deep, so a program calling it keeps the 512-byte budget. The + * redirect that happens after the program returns does not. + */ +SEC("tc") +__description("1 KiB frame with bpf_clone_redirect keeps the 512-byte budget") +__load_if_large_stack() +__failure __msg("invalid write to stack R1 off=-1024 size=8") +__naked void clone_redirect_keeps_512(void) +{ + asm volatile (" \ + r6 = r1; \ + r1 = r10; \ + r1 += -1024; \ + r0 = 42; \ + *(u64 *)(r1 + 0) = r0; \ + r1 = r6; \ + r2 = 1; \ + r3 = 0; \ + call %[bpf_clone_redirect]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_clone_redirect) + : __clobber_all); +} + +SEC("tc") +__description("1 KiB frame with bpf_redirect keeps the 2 KiB budget") +__load_if_large_stack() +__success +__naked void redirect_keeps_2048(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -1024; \ + r0 = 42; \ + *(u64 *)(r1 + 0) = r0; \ + r1 = 1; \ + r2 = 0; \ + call %[bpf_redirect]; \ + exit; \ +" : + : __imm(bpf_redirect) + : __clobber_all); +} + +/* Global subprogs are verified on their own but share the call chain budget. */ +__used __naked int global_frame_1536(void) +{ + asm volatile (" \ + r1 = 1; \ + *(u64 *)(r10 - 1536) = r1; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("512-byte frame calling a 1536-byte global subprog") +__load_if_large_stack() +__success __log_level(4) __msg("stack depth max 2048") +__naked void global_subprog_fits(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 512) = r1; \ + call global_frame_1536; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("520-byte frame calling a 1536-byte global subprog") +__load_if_large_stack() +__failure __msg("combined stack size of 2 calls is 2064. Too large") +__naked void global_subprog_exceeds(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 520) = r1; \ + call global_frame_1536; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* Callback frames are part of the chain of the helper that calls them. */ +static __naked int loop_cb_1536(void) +{ + asm volatile (" \ + r1 = 1; \ + *(u64 *)(r10 - 1536) = r1; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__description("512-byte frame with a 1536-byte bpf_loop callback") +__load_if_large_stack() +__success __log_level(4) __msg("stack depth max 2048") +__naked void loop_callback_fits(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 512) = r1; \ + r1 = 1; \ + r2 = %[loop_cb_1536]; \ + r3 = 0; \ + r4 = 0; \ + call %[bpf_loop]; \ + r0 = 0; \ + exit; \ +" : + : __imm_ptr(loop_cb_1536), + __imm(bpf_loop) + : __clobber_common); +} + +SEC("socket") +__description("520-byte frame with a 1536-byte bpf_loop callback") +__load_if_large_stack() +__failure __msg("combined stack size of 2 calls is 2064. Too large") +__naked void loop_callback_exceeds(void) +{ + asm volatile (" \ + r1 = 2; \ + *(u64 *)(r10 - 520) = r1; \ + r1 = 1; \ + r2 = %[loop_cb_1536]; \ + r3 = 0; \ + r4 = 0; \ + call %[bpf_loop]; \ + r0 = 0; \ + exit; \ +" : + : __imm_ptr(loop_cb_1536), + __imm(bpf_loop) + : __clobber_common); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_ldsx.c b/tools/testing/selftests/bpf/progs/verifier_ldsx.c index 41340877dc9d..49e8cd4bb4f9 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ldsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_ldsx.c @@ -9,13 +9,7 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 - -struct { - __uint(type, BPF_MAP_TYPE_ARENA); - __uint(map_flags, BPF_F_MMAPABLE); - __uint(max_entries, 1); -} arena SEC(".maps"); + (__clang_major__ >= 18 || defined(__BPF_FEATURE_LDSX)) SEC("socket") __description("LDSX, S8") @@ -263,6 +257,14 @@ __naked void ldsx_ctx_8(void) : __clobber_all); } +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + SEC("syscall") __description("Arena LDSX Disasm") __success @@ -286,6 +288,19 @@ __jited("add x11, x0, x28") __jited("ldrsh x22, [x11, #0x18]") __jited("add x11, x0, x28") __jited("ldrsb x22, [x11, #0x20]") +__arch_riscv64 +__jited("add t2, t6, s7") +__jited("lw s3, 0x10(t2)") +__jited("add t2, t6, s7") +__jited("lh s3, 0x18(t2)") +__jited("add t2, t6, s7") +__jited("lb s3, 0x20(t2)") +__jited("add t2, a0, s7") +__jited("lw s4, 0x10(t2)") +__jited("add t2, a0, s7") +__jited("lh s4, 0x18(t2)") +__jited("add t2, a0, s7") +__jited("lb s4, 0x20(t2)") __naked void arena_ldsx_disasm(void *ctx) { asm volatile ( @@ -317,6 +332,7 @@ __description("Arena LDSX Exception") __success __retval(0) __arch_x86_64 __arch_arm64 +__arch_riscv64 __naked void arena_ldsx_exception(void *ctx) { asm volatile ( @@ -338,6 +354,7 @@ __description("Arena LDSX, S8") __success __retval(-1) __arch_x86_64 __arch_arm64 +__arch_riscv64 __naked void arena_ldsx_s8(void *ctx) { asm volatile ( @@ -369,6 +386,7 @@ __description("Arena LDSX, S16") __success __retval(-1) __arch_x86_64 __arch_arm64 +__arch_riscv64 __naked void arena_ldsx_s16(void *ctx) { asm volatile ( @@ -400,6 +418,7 @@ __description("Arena LDSX, S32") __success __retval(-1) __arch_x86_64 __arch_arm64 +__arch_riscv64 __naked void arena_ldsx_s32(void *ctx) { asm volatile ( @@ -432,10 +451,13 @@ void kfunc_root(void) bpf_arena_alloc_pages(0, 0, 0, 0, 0); } +#endif /* __BPF_FEATURE_ADDR_SPACE_CAST */ + #else SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index 401152b2b64f..5f9b11a81260 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -21,8 +21,6 @@ struct { __type(value, __u64); } array_map_8b SEC(".maps"); -const char snprintf_u64_fmt[] = "%llu"; - SEC("socket") __log_level(2) __msg("0: (79) r1 = *(u64 *)(r10 -8) ; use: fp0-8") @@ -63,8 +61,12 @@ __naked void read_write_join(void) SEC("socket") __log_level(2) __msg("stack use/def subprog#0 must_write_not_same_slot (d0,cs0):") -__msg("6: (7b) *(u64 *)(r2 +0) = r0{{$}}") -__msg("Live regs before insn:") +/* + * 'r2 += r1' adds a scalar, so the offset is lost (off_cnt == 0): no def, + * but the write conservatively marks the whole frame as may_def. + */ +__msg("6: (7b) *(u64 *)(r2 +0) = r0 ; may_def: fp0-8..-{{(512|2048)}}") +__msg("Program dump") __naked void must_write_not_same_slot(void) { asm volatile ( @@ -83,6 +85,31 @@ __naked void must_write_not_same_slot(void) SEC("socket") __log_level(2) +__msg("stack use/def subprog#0 may_write_two_precise_offsets (d0,cs0):") +/* + * r1 is fp-8 or fp-16, with both offsets tracked precisely (off_cnt == 2). + * A write through it cannot set def, but marks both candidate slots as may_def. + */ +__msg("6: (7b) *(u64 *)(r1 +0) = r0 ; may_def: fp0-8 fp0-16") +__naked void may_write_two_precise_offsets(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = r10;" + "if r0 > 42 goto 1f;" + "r1 += -8;" + "goto 2f;" +"1:" + "r1 += -16;" +"2:" + "*(u64 *)(r1 + 0) = r0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__log_level(2) __msg("0: (7a) *(u64 *)(r10 -8) = 0 ; def: fp0-8") __msg("5: (85) call bpf_map_lookup_elem#1 ; use: fp0-8h") __naked void must_write_not_same_type(void) @@ -108,11 +135,11 @@ __naked void must_write_not_same_type(void) SEC("socket") __log_level(2) -/* Callee writes fp[0]-8: stack_use at call site has slots 0,1 live */ +/* Callee writes fp[0]-8: the def is summarized as may_def at the call site */ __msg("stack use/def subprog#0 caller_stack_write (d0,cs0):") -__msg("2: (85) call pc+1{{$}}") +__msg("2: (85) call pc+1 ; may_def: fp0-8") __msg("stack use/def subprog#1 write_first_param (d1,cs2):") -__msg("4: (7a) *(u64 *)(r1 +0) = 7 ; def: fp0-8") +__msg("4: (7a) *(u64 *)(r1 +0) = 7 ; def: fp0-8 may_def: fp0-8") __naked void caller_stack_write(void) { asm volatile ( @@ -132,6 +159,155 @@ static __used __naked void write_first_param(void) ::: __clobber_all); } +struct cnt8 { long v; }; + +__noinline int global_noop_arg(struct cnt8 *p) +{ + return 0; +} + +SEC("socket") +__log_level(2) +/* Call to a global function: the argument is described by its declared type. */ +__msg("stack use/def subprog#0 global_callee_arg (d0,cs0):") +__msg("3: (85) call pc+{{.*}}; use: fp0-8 def: fp0-8 may_def: fp0-8") +__naked void global_callee_arg(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r1 = r10;" + "r1 += -8;" + "call global_noop_arg;" + "exit;" + ::: __clobber_all); +} + +__noinline int global_int_arg(int x) +{ + return 0; +} + +SEC("socket") +__success __log_level(2) +__msg("stack use/def subprog#0 global_noop_int_arg (d0,cs0):") +__msg_next(" 0: (bf) r1 = r10{{$}}") +__msg_next(" 1: (07) r1 += -8{{$}}") +__msg_next(" 2: (1f) r1 -= r10{{$}}") +__msg_next(" 3: (85) call pc+1{{$}}") +__msg_next(" 4: (95) exit") +__naked void global_noop_int_arg(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "r1 -= r10;" /* This is a scalar now. */ + "call global_int_arg;" /* And should not be tracked as stack effect. */ + "exit;" + ::: __clobber_all); +} + +static char global_dynptr_buf[8]; + +__noinline int global_noop_dynptr_arg(struct bpf_dynptr *p) +{ + return 0; +} + +SEC("socket") +__success __log_level(2) +/* A global dynptr argument may read/write only its 16 bytes, with no definite write. */ +__msg("stack use/def subprog#0 global_callee_dynptr_arg (d0,cs0):") +__msg(" 7: (bf) r1 = r10{{$}}") +__msg_next(" 8: (07) r1 += -32{{$}}") +__msg_next(" 9: (85) call pc+1{{ +}}; use: fp0-24 fp0-32 may_def: fp0-24 fp0-32{{$}}") +__msg_next(" 10: (95) exit{{$}}") +__naked void global_callee_dynptr_arg(void) +{ + asm volatile ( + "r1 = %[global_dynptr_buf] ll;" + "r2 = 8;" + "r3 = 0;" + "r4 = r10;" + "r4 += -32;" + "call %[bpf_dynptr_from_mem];" + "r1 = r10;" + "r1 += -32;" + "call global_noop_dynptr_arg;" + "exit;" + :: __imm_addr(global_dynptr_buf), __imm(bpf_dynptr_from_mem) + : __clobber_all); +} + +struct task_struct { int pid; }; + +__noinline int global_noop_btf_arg(struct task_struct *p __arg_untrusted) +{ + return 0; +} + +SEC("socket") +__success __log_level(2) +/* Untrusted BTF pointer arguments need no caller stack state. */ +__msg("stack use/def subprog#0 global_callee_btf_arg (d0,cs0):") +__msg_next(" 0: (7a) *(u64 *)(r10 -8) = 0{{.*}}") +__msg_next(" 1: (bf) r1 = r10{{$}}") +__msg_next(" 2: (07) r1 += -8{{$}}") +__msg_next(" 3: (85) call pc+1{{$}}") +__msg_next(" 4: (95) exit{{$}}") +__naked void global_callee_btf_arg(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r1 = r10;" + "r1 += -8;" + "call global_noop_btf_arg;" + "exit;" + ::: __clobber_all); +} + +/* + * Cross-frame imprecise write: imprecise_frame_writer() receives a pointer + * into the caller's frame (frame 0) but conditionally replaces it with a + * pointer into its own frame (frame 1). At the store the two are joined into + * ARG_IMPRECISE (frame is unknown, mask = {0,1}), so the offset is dropped and + * the whole of both candidate frames is conservatively marked as may_def. + */ +SEC("socket") +__log_level(2) +/* The callee's write into the caller frame is summarized at the call site. */ +__msg("stack use/def subprog#0 imprecise_frame_write (d0,cs0):") +__msg("2: (85) call pc+2 ; may_def: fp0-8..-{{(512|2048)}}") +__msg("stack use/def subprog#1 imprecise_frame_writer (d1,cs2):") +__msg("12: (7b) *(u64 *)(r1 +0) = r2 ; may_def: fp1-8..-{{(512|2048)}} fp0-8..-{{(512|2048)}}") +__naked void imprecise_frame_write(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "call imprecise_frame_writer;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void imprecise_frame_writer(void) +{ + asm volatile ( + "r6 = r1;" /* save arg: pointer into frame 0 */ + "call %[bpf_get_prandom_u32];" + "r1 = r6;" /* r1 = frame0-8 (arg) */ + "if r0 == 0 goto 1f;" + "r1 = r10;" + "r1 += -8;" /* r1 = frame1-8 (own frame) */ +"1:" + "r2 = 0;" + "*(u64 *)(r1 + 0) = r2;" /* join frame0-8 | frame1-8 -> imprecise */ + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + SEC("socket") __log_level(2) __msg("stack use/def subprog#0 caller_stack_read (d0,cs0):") @@ -246,7 +422,7 @@ static __used __naked void read_first_param2(void) SEC("socket") __flag(BPF_F_TEST_STATE_FREQ) __failure -__msg("R1 type=scalar expected=map_ptr") +__msg("Possibly NULL pointer passed to trusted R1") __naked void caller_stack_pruning_callback(void) { asm volatile ( @@ -320,7 +496,7 @@ struct { } map_array SEC(".maps"); SEC("socket") -__failure __msg("invalid read from stack R2 off=-1024 size=8") +__failure __msg("invalid read from stack R2 off=-4096 size=8") __flag(BPF_F_TEST_STATE_FREQ) __naked unsigned long caller_stack_write_tail_call(void) { @@ -331,7 +507,7 @@ __naked unsigned long caller_stack_write_tail_call(void) "if r0 != 42 goto 1f;" "goto 2f;" "1:" - "*(u64 *)(r10 - 8) = -1024;" + "*(u64 *)(r10 - 8) = -4096;" "2:" "r1 = r6;" "r2 = r10;" @@ -806,9 +982,9 @@ void __kfunc_btf_root(void) */ SEC("socket") __success __log_level(2) -__msg(" 6: (85) call bpf_iter_num_new{{.*}} ; def: fp0-24{{$}}") -__msg(" 9: (85) call bpf_iter_num_next{{.*}} ; use: fp0-24{{$}}") -__msg("14: (85) call bpf_iter_num_destroy{{.*}} ; use: fp0-24{{$}}") +__msg(" 6: (85) call bpf_iter_num_new{{.*}} ; def: fp0-24 may_def: fp0-24{{$}}") +__msg(" 9: (85) call bpf_iter_num_next{{.*}} ; use: fp0-24 may_def: fp0-24{{$}}") +__msg("14: (85) call bpf_iter_num_destroy{{.*}} ; use: fp0-24 may_def: fp0-24{{$}}") __naked void kfunc_iter_stack_liveness(void) { asm volatile ( @@ -1009,8 +1185,9 @@ __naked void four_byte_read_upper_half(void) */ SEC("socket") __log_level(2) -__msg("0: (7a) *(u64 *)(r10 -8) = 0 ; def: fp0-8") -__msg("1: (6a) *(u16 *)(r10 -4) = 0{{$}}") +__msg("0: (7a) *(u64 *)(r10 -8) = 0 ; def: fp0-8 may_def: fp0-8") +/* 2-byte write only partially covers the upper half: may_def, but no def. */ +__msg("1: (6a) *(u16 *)(r10 -4) = 0 ; may_def: fp0-4h") __msg("2: (61) r0 = *(u32 *)(r10 -4) ; use: fp0-4h") __naked void two_byte_write_no_kill(void) { @@ -1357,9 +1534,12 @@ __naked void fp_spill_loses_precision_kills_liveness(void) */ SEC("socket") __log_level(2) -/* fp-8 live at call (callee conditionally writes → slot not killed) */ -__msg("1: (7b) *(u64 *)(r10 -8) = r1 ; def: fp0-8") -__msg("4: (85) call pc+2{{$}}") +/* + * fp-8 live at call: callee conditionally writes it, so the slot is not killed + * (no def), but the conditional write surfaces as may_def at the call site. + */ +__msg("1: (7b) *(u64 *)(r10 -8) = r1 ; def: fp0-8 may_def: fp0-8") +__msg("4: (85) call pc+2 ; may_def: fp0-8") __msg("5: (79) r0 = *(u64 *)(r10 -8) ; use: fp0-8") __naked void conditional_stx_in_subprog(void) { @@ -1464,7 +1644,7 @@ static __used __naked void inner_reader(void) SEC("socket") __log_level(2) /* fp-8 NOT live at call: callee writes before reading (parent_def kills it) */ -__msg("2: .12345.... (85) call pc+") +__msg("2: .1........ (85) call pc+") __naked void callee_must_write_before_read(void) { asm volatile ( @@ -1947,15 +2127,15 @@ static __used __naked void fwd_parent_key_to_helper(void) /* * Regression for keeping later helper args after a whole-stack fallback - * on an earlier local arg. The first bpf_snprintf() arg is a local + * on an earlier local arg. The bpf_map_update_elem() key is a local * frame-derived pointer with offset-imprecise tracking (`fp1 ?`), which - * conservatively marks the whole local stack live. The fourth arg still + * conservatively marks the whole local stack live. The value arg still * forwards &parent_fp-8 and must contribute nonlocal_use[0]=0:3. */ SEC("socket") __log_level(2) __success -__msg("call bpf_snprintf{{.*}} ; use: fp1-8..-512 fp0-8") +__msg("call bpf_map_update_elem{{.*}}; use: fp1-8..-{{(512|2048)}} fp0-8") __naked void helper_arg_fallback_keeps_scanning(void) { asm volatile ( @@ -1963,32 +2143,33 @@ __naked void helper_arg_fallback_keeps_scanning(void) "*(u64 *)(r10 - 8) = r1;" "r1 = r10;" "r1 += -8;" - "call helper_snprintf_parent_after_local_fallback;" + "call helper_update_parent_after_local_fallback;" "r0 = 0;" "exit;" ::: __clobber_all); } -static __used __naked void helper_snprintf_parent_after_local_fallback(void) +static __used __naked void helper_update_parent_after_local_fallback(void) { asm volatile ( "r6 = r1;" /* save &parent_fp-8 */ "call %[bpf_get_prandom_u32];" "r0 &= 8;" - "r1 = r10;" - "r1 += -16;" - "r1 += r0;" /* local fp, offset-imprecise */ - "r2 = 8;" - "r3 = %[snprintf_u64_fmt] ll;" - "r4 = r6;" /* later arg: parent fp-8 */ - "r5 = 8;" - "call %[bpf_snprintf];" + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 8) = 0;" + "r2 = r10;" + "r2 += -16;" + "r2 += r0;" /* local fp, offset-imprecise */ + "r1 = %[array_map_8b] ll;" + "r3 = r6;" /* later arg: parent fp-8 */ + "r4 = 0;" + "call %[bpf_map_update_elem];" "r0 = 0;" "exit;" : : __imm(bpf_get_prandom_u32), - __imm(bpf_snprintf), - __imm_addr(snprintf_u64_fmt) + __imm(bpf_map_update_elem), + __imm_addr(array_map_8b) : __clobber_all); } @@ -2268,7 +2449,11 @@ static __used __naked void merge_leaf_read(void) SEC("socket") __log_level(2) __success -__msg("call bpf_loop#181 ; use: fp2-8..-512 fp1-8..-512 fp0-8..-512") +__msg("stack use/def subprog#0 bpf_loop_two_callbacks (d0,cs0):") +__msg("5: (85) call pc+{{.*}};{{.*}} may_def: fp0-8..-{{(512|2048)}}{{$}}") +__msg("8: (85) call pc+{{.*}};{{.*}} may_def: fp0-8..-{{(512|2048)}}{{$}}") +__msg("call bpf_loop#181 ; use: fp2-8..-{{(512|2048)}} fp1-8..-{{(512|2048)}} fp0-8..-{{(512|2048)}}" + " may_def: fp2-8..-{{(512|2048)}} fp1-8..-{{(512|2048)}} fp0-8..-{{(512|2048)}}{{$}}") __naked void bpf_loop_two_callbacks(void) { asm volatile ( @@ -2386,8 +2571,27 @@ __flag(BPF_F_TEST_STATE_FREQ) __msg("subprog#2 write_first_read_second:") __msg("17: (7a) *(u64 *)(r1 +0) = 42{{$}}") __msg("18: (79) r0 = *(u64 *)(r2 +0) // r1=fp0-8 r2=fp0-16{{$}}") +/* + * The callee's write into the caller frame is summarized as may_def at the + * call sites, transitively across both frames (depth 2 -> 1 -> 0). The two + * call sites differ because the shared write_first_read_second instance + * accumulates the cross-pass union ({fp-8, fp-16}) at different points + * relative to each forwarding_rw's summarization. + */ +__msg("stack use/def subprog#0 shared_instance_must_write_overwrite (d0,cs0):") +__msg("7: (85) call pc+7 ; use: fp0-8 fp0-16 may_def: fp0-8 fp0-16") +__msg("12: (85) call pc+2 ; use: fp0-8 may_def: fp0-16") +__msg("stack use/def subprog#1 forwarding_rw (d1,cs7):") +__msg("15: (85) call pc+1 ; use: fp0-8 fp0-16 may_def: fp0-8 fp0-16") +__msg("stack use/def subprog#1 forwarding_rw (d1,cs12):") +__msg("15: (85) call pc+1 ; use: fp0-8 may_def: fp0-16") __msg("stack use/def subprog#2 write_first_read_second (d2,cs15):") -__msg("17: (7a) *(u64 *)(r1 +0) = 42{{$}}") +/* + * Shared across two callsites with swapped args (r1 is fp-8 on one pass, + * fp-16 on the other): must_write intersects to empty (no def), may_write + * unions to both slots. + */ +__msg("17: (7a) *(u64 *)(r1 +0) = 42 ; may_def: fp0-8 fp0-16") __msg("18: (79) r0 = *(u64 *)(r2 +0) ; use: fp0-8 fp0-16") __naked void shared_instance_must_write_overwrite(void) { @@ -2430,6 +2634,65 @@ static __used __naked void write_first_read_second(void) } /* + * merge_instances() must preserve may_write when one pass doesn't touch a frame. + * mvs_leaf is reached through a single callsite (mvs_mid) from two outer chains, + * so it is one shared (depth,callsite) instance analyzed twice and merged: + * - chain A passes r2 = map value, mvs_leaf's doesn't touch stack; + * - chain B passes r2 = caller stack slot fp-8. + * The merge must keep the stack pass's may_write even though the map pass left + * frame 0 untouched. + */ +SEC("socket") +__success +__log_level(2) +__msg("stack use/def subprog#2 mvs_leaf (d2,cs19):") +__msg("21: (7a) *(u64 *)(r2 +0) = 42 ; may_def: fp0-8") +__naked void merge_preserves_may_write(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 24) = r1;" /* map key */ + "r1 = %[map] ll;" + "r2 = r10;" + "r2 += -24;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 1f;" + /* chain A: r2 = map value (ARG_NONE, not stack) */ + "r1 = r10;" + "r1 += -16;" /* unused fp anchor */ + "r2 = r0;" + "call mvs_mid;" + /* chain B: r2 = caller stack slot fp-8 */ + "r1 = r10;" + "r1 += -16;" + "r2 = r10;" + "r2 += -8;" + "call mvs_mid;" +"1:" + "r0 = 0;" + "exit;" + : : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + +static __used __naked void mvs_mid(void) +{ + asm volatile ( + "call mvs_leaf;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void mvs_leaf(void) +{ + asm volatile ( + "*(u64 *)(r2 + 0) = 42;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* * Shared must_write when (callsite, depth) instance is reused. * Main calls fwd_to_stale_wr at two sites. fwd_to_stale_wr calls * stale_wr_leaf at a single internal callsite. Both calls share @@ -2514,6 +2777,153 @@ __naked void load_acquire_dont_clear_dst(void) #endif /* CAN_USE_LOAD_ACQ_STORE_REL */ SEC("socket") +__log_level(2) +__success +__msg("lock *(u64 *)(r10 -8) += r1{{.*}}use: fp0-8{{.*}}may_def: fp0-8") +__msg("r1 = atomic64_fetch_add((u64 *)(r10 -8), r1){{.*}}use: fp0-8{{.*}}may_def: fp0-8") +__msg("r1 = atomic64_xchg((u64 *)(r10 -8), r1){{.*}}use: fp0-8{{.*}}may_def: fp0-8") +__msg("r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r1){{.*}}use: fp0-8{{.*}}may_def: fp0-8") +__naked void atomic_rmw(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + ".8byte %[atomic_add];" + ".8byte %[atomic_fetch_add];" + ".8byte %[atomic_xchg];" + "r0 = 0;" + ".8byte %[atomic_cmpxchg];" + "exit;" + : + : __imm_insn(atomic_add, BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_fetch_add, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_xchg, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_cmpxchg, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_1, -8)) + : __clobber_all); +} + +#ifdef CAN_USE_LOAD_ACQ_STORE_REL + +/* + * An 8-byte load-acquire from a spill slot restores the spilled register. + * The store through the restored pointer is a definite write to fp-8. + */ +SEC("socket") +__log_level(2) +__success +__msg("r2 = load_acquire((u64 *)(r10 -16)){{.*}}use: fp0-16") +__msg("*(u64 *)(r2 +0) = 42{{.*}}def: fp0-8 may_def: fp0-8") +__naked void load_acquire_fills_spilled_ptr(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 16) = r1;" /* fp-16 = &fp-8 */ + ".8byte %[load_acquire_insn];" /* r2 = load_acquire(fp-16) */ + "*(u64 *)(r2 + 0) = 42;" + "r0 = 0;" + "exit;" + : + : __imm_insn(load_acquire_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -16)) + : __clobber_all); +} + +/* An 8-byte store-release spills the register, same as a plain store. */ +SEC("socket") +__log_level(2) +__success +__msg("store_release((u64 *)(r10 -16), r1){{.*}}def: fp0-16") +__msg("*(u64 *)(r2 +0) = 42{{.*}}def: fp0-8 may_def: fp0-8") +__naked void store_release_spills_ptr(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + ".8byte %[store_release_insn];" /* store_release(fp-16, r1): fp-16 = &fp-8 */ + "r2 = *(u64 *)(r10 - 16);" + "*(u64 *)(r2 + 0) = 42;" + "r0 = 0;" + "exit;" + : + : __imm_insn(store_release_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -16)) + : __clobber_all); +} + +#endif /* CAN_USE_LOAD_ACQ_STORE_REL */ + +/* Stack fills through xchg, fetch_add and cmpxchg instructions. */ +SEC("socket") +__log_level(2) +__success +__msg("r2 = atomic64_xchg((u64 *)(r10 -16), r2){{.*}}use: fp0-16{{.*}}may_def: fp0-16") +__msg("*(u64 *)(r2 +0) = 42{{.*}}def: fp0-8 may_def: fp0-8") +__msg("r3 = atomic64_fetch_add((u64 *)(r10 -16), r3){{.*}}use: fp0-16{{.*}}may_def: fp0-16") +__msg("*(u64 *)(r3 +0) = 42{{.*}}def: fp0-8 may_def: fp0-8") +__msg("r0 = atomic64_cmpxchg((u64 *)(r10 -16), r0, r4){{.*}}use: fp0-16{{.*}}may_def: fp0-16") +__msg("*(u64 *)(r0 +0) = 42{{.*}}def: fp0-8 may_def: fp0-8") +__naked void atomic_rmw_fills_spilled_ptr(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 16) = r1;" /* fp-16 = &fp-8 */ + "r2 = 0;" + ".8byte %[atomic_xchg];" /* r2 = xchg(fp-16, r2) */ + "*(u64 *)(r2 + 0) = 42;" + "*(u64 *)(r10 - 16) = r1;" + "r3 = 0;" + ".8byte %[atomic_fetch_add];" /* r3 = fetch_add(fp-16, r3) */ + "*(u64 *)(r3 + 0) = 42;" + "*(u64 *)(r10 - 16) = r1;" + "r0 = 0;" + "r4 = 0;" + ".8byte %[atomic_cmpxchg];" /* r0 = cmpxchg(fp-16, r0, r4) */ + "*(u64 *)(r0 + 0) = 42;" + "r0 = 0;" + "exit;" + : + : __imm_insn(atomic_xchg, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_10, BPF_REG_2, -16)), + __imm_insn(atomic_fetch_add, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_3, -16)), + __imm_insn(atomic_cmpxchg, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_4, -16)) + : __clobber_all); +} + +/* xchg returning caller's &fp-8. */ +SEC("socket") +__log_level(2) +__success +__msg("r2 = atomic64_xchg((u64 *)(r1 +0), r2){{.*}}use: fp0-16{{.*}}may_def: fp0-16") +__msg("*(u64 *)(r2 +0) = 42{{.*}}; may_def: fp0-8") +__naked void atomic_rmw_fills_parent_spilled_ptr(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 16) = r1;" /* fp-16 = &fp-8 */ + "r1 = r10;" + "r1 += -16;" + "call atomic_rmw_fills_parent_spilled_ptr_callee;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void atomic_rmw_fills_parent_spilled_ptr_callee(void) +{ + asm volatile ( + "r2 = 0;" + ".8byte %[atomic_xchg];" /* r2 = xchg(r1, r2) == caller's &fp-8 */ + "*(u64 *)(r2 + 0) = 42;" + "r0 = 0;" + "exit;" + : + : __imm_insn(atomic_xchg, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_2, 0)) + : __clobber_all); +} + +SEC("socket") __success __naked void imprecise_fill_loses_cross_frame(void) { @@ -2840,3 +3250,197 @@ static __used __naked void imprecise_dst_spill_join_sub(void) :: __imm(bpf_get_prandom_u32) : __clobber_all); } + +/* + * A store that does not fully cover a 4-byte half-slot defines nothing, so a + * narrow store at the top of the frame must not turn any slot into a "def", + * least of all every slot of the frame: the earlier data at fp-8 stays live. + */ +SEC("socket") +__log_level(2) +__msg("0: (79) r0 = *(u64 *)(r10 -8) ; use: fp0-8") +/* narrow stores define nothing, but they may write the half-slot they touch */ +__msg("1: (73) *(u8 *)(r10 -1) = r0 ; may_def: fp0-4h") +__msg("2: (6b) *(u16 *)(r10 -4) = r0 ; may_def: fp0-4h") +__msg("3: (79) r0 = *(u64 *)(r10 -8) ; use: fp0-8") +__naked void narrow_store_defines_nothing(void) +{ + asm volatile ( + "r0 = *(u64 *)(r10 - 8);" + "*(u8 *)(r10 - 1) = r0;" + "*(u16 *)(r10 - 4) = r0;" + "r0 = *(u64 *)(r10 - 8);" + "exit;" + ::: __clobber_all); +} + +/* + * The same callee instance is analyzed twice: the call sites are visited in + * postorder, so the second one goes first with a precise pointer 248 bytes + * into the main frame, and the first one then passes a pointer of unknown + * offset, which reads the whole frame. The precise pass stays within the + * first word of the masks on 64-bit, so the whole-frame pass is wider under + * every stack budget and merging the second into the first has to widen the + * masks while keeping the whole-frame read. + */ +SEC("socket") +__log_level(2) +__msg("stack use/def subprog#{{[0-9]+}} merge_read_all_callee (d2,cs{{[0-9]+}}):") +__msg("(79) r0 = *(u64 *)(r1 +0){{.*}}; use: fp0-8..-{{(512|2048)}}") +__naked void merge_keeps_whole_frame_read(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "*(u64 *)(r10 - 16) = r1;" + "*(u64 *)(r10 - 248) = r1;" + "call %[bpf_get_prandom_u32];" + "r0 &= 8;" + "r1 = r10;" + "r1 += -16;" + "r1 += r0;" + "call merge_read_all_mid;" + "r1 = r10;" + "r1 += -248;" + "call merge_read_all_mid;" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +static __used __naked void merge_read_all_mid(void) +{ + asm volatile ( + "call merge_read_all_callee;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void merge_read_all_callee(void) +{ + asm volatile ( + "r0 = *(u64 *)(r1 + 0);" + "exit;" + ::: __clobber_all); +} + +/* + * A frame pointer spilled below fp-512 is a spill like any other: the fill + * restores its identity, and a load through it reads only the slot it names + * instead of the whole frame. + */ +SEC("socket") +__log_level(2) +__load_if_large_stack() +__msg("(79) r0 = *(u64 *)(r1 +0){{.*}}; use: fp0-8{{$}}") +__naked void spill_below_512_stays_precise(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 520) = r1;" + "r1 = *(u64 *)(r10 - 520);" + "r0 = *(u64 *)(r1 + 0);" + "exit;" + ::: __clobber_all); +} + +/* Unknown-size outputs still carry a possible write, without a read or kill. */ +SEC("socket") +__success __log_level(2) +__msg("call bpf_probe_read_kernel{{.*}}; may_def: fp0-8{{$}}") +__naked void helper_output_unknown_size(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r0 &= 7;" + "r0 += 1;" + "r2 = r0;" + "r1 = r10;" + "r1 += -8;" + "r3 = 0;" + "call %[bpf_probe_read_kernel];" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32), __imm(bpf_probe_read_kernel) + : __clobber_all); +} + +struct { + __uint(type, BPF_MAP_TYPE_QUEUE); + __uint(max_entries, 1); + __type(value, __u64); +} queue_8b SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_QUEUE); + __uint(max_entries, 1); + __type(value, __u64[2]); +} queue_16b SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_BLOOM_FILTER); + __uint(max_entries, 16); + __type(value, __u64); +} bloom_8b SEC(".maps"); + +SEC("socket") +__success __log_level(2) +__msg("call bpf_map_pop_elem{{.*}}; def: fp0-16") +__naked void helper_map_output_known_size(void) +{ + asm volatile ( + "r1 = %[queue_8b] ll;" + "r2 = r10;" + "r2 += -16;" + "call %[bpf_map_pop_elem];" + "r0 = 0;" + "exit;" + :: __imm_addr(queue_8b), __imm(bpf_map_pop_elem) + : __clobber_all); +} + +/* The maximum map value size is not a guaranteed output extent. */ +SEC("socket") +__success __log_level(2) +__msg("call bpf_map_pop_elem{{.*}}; use: fp0-16 fp0-24 may_def: fp0-16 fp0-24{{$}}") +__naked void helper_map_output_merged_sizes(void) +{ + asm volatile ( + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 24) = 0;" + "call %[bpf_get_prandom_u32];" + "r1 = %[queue_8b] ll;" + "if r0 == 0 goto 1f;" + "r1 = %[queue_16b] ll;" +"1:" + "r2 = r10;" + "r2 += -24;" + "call %[bpf_map_pop_elem];" + "r0 = 0;" + "exit;" + :: __imm_addr(queue_8b), __imm_addr(queue_16b), + __imm(bpf_get_prandom_u32), __imm(bpf_map_pop_elem) + : __clobber_all); +} + +/* Bloom-filter peek treats buffer as an input despite the helper's output annotation. */ +SEC("socket") +__success __log_level(2) +__msg("call bpf_map_peek_elem{{.*}}; use: fp0-8{{$}}") +__naked void helper_bloom_peek_input(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r1 = %[bloom_8b] ll;" + "r2 = r10;" + "r2 += -8;" + "call %[bpf_map_peek_elem];" + "r0 = 0;" + "exit;" + :: __imm_addr(bloom_8b), __imm(bpf_map_peek_elem) + : __clobber_all); +} diff --git a/tools/testing/selftests/bpf/progs/verifier_load_acquire.c b/tools/testing/selftests/bpf/progs/verifier_load_acquire.c index d17026d7480d..bcc34e798c63 100644 --- a/tools/testing/selftests/bpf/progs/verifier_load_acquire.c +++ b/tools/testing/selftests/bpf/progs/verifier_load_acquire.c @@ -267,6 +267,7 @@ __naked void load_acquire_with_invalid_reg(void) SEC("socket") __description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test") +__skip("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_loop_hierarchy.c b/tools/testing/selftests/bpf/progs/verifier_loop_hierarchy.c new file mode 100644 index 000000000000..0608e983d7fc --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_loop_hierarchy.c @@ -0,0 +1,346 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* + * kernel/bpf/loops.c:compute_loops_in_subprog() distinguishes between + * the following cases: + * - B: backedge -> simple loop + * - C: cross edge to non-loop node -> no-op + * - D: edge to node whose header is in DFS path -> nested loop + * - E: edge to node whose header is NOT in DFS path -> irreducible + * + * Below test cases cover the above branches in various combinations. + */ + +/* Case B: single bounded loop. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 1 0 1: {{.*}} (07) r0 += 1") +__msg(" 1 1 1 2: {{.*}} (a5) if r0 < 0xa goto pc-2") +__msg(" 2 3: {{.*}} (95) exit") +__naked void loop_single(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Case B: two independent loops at the same nesting level. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("loop at 4{{$}}") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 2 0 1: {{.*}} (07) r0 += 1") +__msg(" 2 1 1 2: {{.*}} (a5) if r0 < 0xa goto pc-2") +__msg(" 2 3: {{.*}} (b7) r1 = 0") +__msg(" 1 3 4: {{.*}} (07) r1 += 1") +__msg(" 1 4 4 5: {{.*}} (a5) if r1 < 0xa goto pc-2") +__msg(" 5 6: {{.*}} (95) exit") +__naked void loop_two_independent(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 < 10 goto 1b; \ + r1 = 0; \ +2: r1 += 1; \ + if r1 < 10 goto 2b; \ + exit; \ +" ::: __clobber_all); +} + +/* Case B + D: nested loops. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("loop at 3, nested in 1") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 1 0 1: {{.*}} (07) r0 += 1") /* outer loop header */ +__msg(" 1 1 1 2: {{.*}} (b7) r1 = 0") /* outer loop insn */ +__msg(" 1 1 2 3: {{.*}} (07) r1 += 1") /* inner loop header */ +__msg(" 1 3 3 4: {{.*}} (a5) if r1 < 0x5 goto pc-2") /* inner loop insn */ +__msg(" 1 1 4 5: {{.*}} (a5) if r0 < 0xa goto pc-5") /* outer loop insn */ +__msg(" 5 6: {{.*}} (95) exit") +__naked void loop_nested(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = 0; \ +2: r1 += 1; \ + if r1 < 5 goto 2b; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Case C: forward edges, no loops. */ +SEC("socket") +__success +__log_level(2) +__msg("Program dump") +__msg(" -1 0: {{.*}} (85) call bpf_get_prandom_u32") +__msg(" 0 1: {{.*}} (25) if r0 > 0x0 goto pc+2") +__msg(" 1 2: {{.*}} (b7) r0 = 2") +__msg(" 2 3: {{.*}} (05) goto pc+1") +__msg(" 1 4: {{.*}} (b7) r0 = 3") +__msg(" 1 5: {{.*}} (95) exit") +__naked void fwd_edges_no_loop(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + if r0 > 0 goto 1f; \ + r0 = 2; \ + goto 2f; \ +1: r0 = 3; \ +2: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Case B + D: two sibling inner loops within one outer loop. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("loop at 3, nested in 1") +__msg("loop at 6, nested in 1") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 1 0 1: {{.*}} (07) r0 += 1") +__msg(" 1 1 1 2: {{.*}} (b7) r1 = 0") +__msg(" 1 1 2 3: {{.*}} (07) r1 += 1") +__msg(" 1 3 3 4: {{.*}} (a5) if r1 < 0x5 goto pc-2") +__msg(" 1 1 4 5: {{.*}} (b7) r2 = 0") +__msg(" 1 1 5 6: {{.*}} (07) r2 += 1") +__msg(" 1 6 6 7: {{.*}} (a5) if r2 < 0x5 goto pc-2") +__msg(" 1 1 7 8: {{.*}} (a5) if r0 < 0xa goto pc-8") +__msg(" 8 9: {{.*}} (95) exit") +__naked void loop_nested_siblings(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = 0; \ +2: r1 += 1; \ + if r1 < 5 goto 2b; \ + r2 = 0; \ +3: r2 += 1; \ + if r2 < 5 goto 3b; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Three levels of nesting. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("loop at 3, nested in 1") +__msg("loop at 5, nested in 3") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 1 0 1: {{.*}} (07) r0 += 1") +__msg(" 1 1 1 2: {{.*}} (b7) r1 = 0") +__msg(" 1 1 2 3: {{.*}} (07) r1 += 1") +__msg(" 1 3 3 4: {{.*}} (b7) r2 = 0") +__msg(" 1 3 4 5: {{.*}} (07) r2 += 1") +__msg(" 1 5 5 6: {{.*}} (a5) if r2 < 0x3 goto pc-2") +__msg(" 1 3 6 7: {{.*}} (a5) if r1 < 0x5 goto pc-5") +__msg(" 1 1 7 8: {{.*}} (a5) if r0 < 0xa goto pc-8") +__msg(" 8 9: {{.*}} (95) exit") +__naked void loop_three_levels(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = 0; \ +2: r1 += 1; \ + r2 = 0; \ +3: r2 += 1; \ + if r2 < 3 goto 3b; \ + if r1 < 5 goto 2b; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Loop with an if-else body (forward branch inside loop, Case D). */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1") +__msg("Program dump") +__msg(" -1 0: {{.*}} (b7) r0 = 0") +__msg(" 1 0 1: {{.*}} (07) r0 += 1") +__msg(" 1 1 1 2: {{.*}} (bf) r1 = r0") +__msg(" 1 1 2 3: {{.*}} (25) if r1 > 0x5 goto pc+1") +__msg(" 1 1 3 4: {{.*}} (b7) r1 = 1") +__msg(" 1 1 3 5: {{.*}} (0f) r0 += r1") +__msg(" 1 1 5 6: {{.*}} (a5) if r0 < 0x64 goto pc-6") +__msg(" 6 7: {{.*}} (95) exit") +__naked void loop_with_if_else(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = r0; \ + if r1 > 5 goto 2f; \ + r1 = 1; \ +2: r0 += r1; \ + if r0 < 100 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Case E: irreducible loop. */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 3, irreducible") +__msg("Program dump") +__msg(" -1 0: {{.*}} (85) call bpf_get_prandom_u32#7") +__msg(" 0 1: {{.*}} (b7) r1 = 0") +__msg(" 1 2: {{.*}} (25) if r0 > 0x5 goto pc+2") +__msg(" 1 2 3: {{.*}} (b7) r1 = 1") +__msg(" 1 3 3 4: {{.*}} (05) goto pc+1") +__msg(" 2 5: {{.*}} (b7) r1 = 2") +__msg(" 1 3 2 6: {{.*}} (0f) r0 += r1") +__msg(" 1 3 6 7: {{.*}} (a5) if r0 < 0x10 goto pc-5") +__msg(" 7 8: {{.*}} (95) exit") +__naked void loop_irreducible(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = 0; \ + if r0 > 5 goto 2f; \ +1: r1 = 1; \ + goto 3f; \ +2: r1 = 2; \ +3: r0 += r1; \ + if r0 < 16 goto 1b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__failure +__log_level(2) +__msg("loop at 1") +__msg("infinite loop detected at insn 1") +__naked void loop_self(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ +1: if r0 < 10 goto 1b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Loops at 5 and 7 are nested inside loops at 3 and 1. + * The edge 6 -> 7 exits loop at 5 and enters loop at 7. + * Check that it is logged only as an exit from loop at 5. + * + * 0: r0 = 0; + * do { + * 1: r0++; + * 2: r1 = 0; + * do { + * 3: r1++; + * 4: r2 = 0; + * do { + * 5: r2++; + * 6: } while (r2 < 3); + * do { + * 7: r2--; + * 8: } while (r2 != 0); + * 9: } while (r1 < 4); + * 10: } while (r0 < 5); + * 11: return r0; + */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg_next(" backedge from 10, latch at 10") +__msg_next(" exit from 10 to 11") +__msg_next("loop at 3, nested in 1") +__msg_next(" backedge from 9, latch at 9") +__msg_next(" exit from 9 to 10") +__msg_next("loop at 5, nested in 3") +__msg_next(" backedge from 6, latch at 6") +__msg_next(" exit from 6 to 7") +__msg_next("loop at 7, nested in 3") +__msg_next(" backedge from 8, latch at 8") +__msg_next(" exit from 8 to 9") +__naked void loop_nested_siblings_common_ancestors(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = 0; \ +2: r1 += 1; \ + r2 = 0; \ +3: r2 += 1; \ + if r2 < 3 goto 3b; \ +4: r2 += -1; \ + if r2 != 0 goto 4b; \ + if r1 < 4 goto 2b; \ + if r0 < 5 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The outer loop backedge (4 -> 2) originates in an inner loop. + * For now, latch should not be identified in such cases. + */ +SEC("socket") +__success +__log_level(2) +__msg("loop at 2{{$}}") +__msg_next(" backedge from 4{{$}}") +__msg_next(" exit from 5 to 7") +__msg_next("loop at 3, nested in 2") +__msg_next(" backedge from 6, latch at 5") +__msg_next(" exit from 4 to 2") +__msg_next(" exit from 5 to 7") +__naked void loop_outer_backedge_from_inner(void) +{ + asm volatile (" \ + r6 = 99; \ + r2 = 2; \ +2: r6 += 1; /* loop at 2 */ \ +3: r2 += 1; /* loop at 3 */ \ +4: if r2 == 3 goto 2b; \ +5: if r6 > 100 goto 7f; \ + goto 3b; \ +7: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_lsm.c b/tools/testing/selftests/bpf/progs/verifier_lsm.c index c724bf389f5c..8fbf7862265f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_lsm.c +++ b/tools/testing/selftests/bpf/progs/verifier_lsm.c @@ -212,4 +212,17 @@ __naked int retval_load_resets_bounds(void *ctx) ::: __clobber_all); } +SEC("lsm/file_mprotect") +__description("lsm ctx is read-only") +__failure __msg("invalid bpf_context access") +__naked int reject_ctx_write(void *ctx) +{ + asm volatile ( + "r6 = 0;" + "*(u64 *)(r1 + 0) = r6;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_lsm_init_xattr.c b/tools/testing/selftests/bpf/progs/verifier_lsm_init_xattr.c new file mode 100644 index 000000000000..b706bf17b556 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_lsm_init_xattr.c @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_helpers.h> +#include "bpf_kfuncs.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +const char xattr_zone[] = "security.bpf.zone"; +char value_buf[8] = "z"; +int scratch_count; + +SEC("lsm/inode_init_security") +__failure __msg("cannot write into rdonly_trusted_mem") +int BPF_PROG(reject_count_write, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + *xattr_count = 0; + return 0; +} + +SEC("lsm/inode_init_security") +__success +int BPF_PROG(allow_count_read, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + scratch_count = *xattr_count; + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("Possibly NULL pointer passed to trusted") +int BPF_PROG(reject_unchecked_xattrs, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, xattr_count, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("invalid mem access 'trusted_ptr_or_null_'") +int BPF_PROG(reject_unchecked_qstr, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + scratch_count = qstr->len; + return 0; +} + +SEC("lsm/inode_init_security") +__success +int BPF_PROG(allow_spilled_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + int *saved = xattr_count; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, saved, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("expected=rdonly_trusted_mem") +int BPF_PROG(reject_forged_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, &scratch_count, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("expected=rdonly_trusted_mem") +int BPF_PROG(reject_stack_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + int local = 0; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, &local, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("expected=rdonly_trusted_mem") +int BPF_PROG(reject_other_ctx_arg, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, (int *)xattrs, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("Possibly NULL pointer passed to trusted") +int BPF_PROG(reject_null_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, NULL, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("dereference of modified rdonly_trusted_mem ptr") +int BPF_PROG(reject_shifted_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, xattr_count + 1, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("variable rdonly_trusted_mem access var_off=") +int BPF_PROG(reject_var_shifted_count, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, xattr_count + (scratch_count & 1), + xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("invalid bpf_context access") +int reject_ctx_forged_count(unsigned long long *ctx) +{ + volatile unsigned long long *slot = ctx; + struct bpf_dynptr value; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + slot[4] = 0; + bpf_inode_init_xattr((struct xattr *)(long)slot[3], + (int *)(long)slot[4], xattr_zone, &value); + return 0; +} + +static __noinline int claim_via_subprog(struct xattr *xattrs, int *xattr_count, + struct bpf_dynptr *value) +{ + return bpf_inode_init_xattr(xattrs, xattr_count, xattr_zone, value); +} + +SEC("lsm/inode_init_security") +__success +int BPF_PROG(allow_count_via_subprog, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + claim_via_subprog(xattrs, xattr_count, &value); + return 0; +} + +SEC("lsm/inode_init_security") +__failure __msg("access beyond struct xattr at off 24") +int BPF_PROG(reject_shifted_xattrs, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs + 1, xattr_count, xattr_zone, &value); + return 0; +} + +SEC("lsm.s/inode_init_security") +__failure __msg("bpf_lsm_inode_init_security is not sleepable") +int BPF_PROG(reject_sleepable, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + return 0; +} + +SEC("lsm_cgroup/inode_init_security") +__failure __msg("calling kernel function bpf_inode_init_xattr is not allowed") +int BPF_PROG(reject_lsm_cgroup, struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct xattr *xattrs, int *xattr_count) +{ + struct bpf_dynptr value; + + if (!xattrs) + return 0; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &value); + bpf_inode_init_xattr(xattrs, xattr_count, xattr_zone, &value); + return 0; +} + +SEC("lsm/inode_setxattr") +__failure __msg("calling kernel function bpf_inode_init_xattr is not allowed") +int BPF_PROG(reject_wrong_hook, struct mnt_idmap *idmap, struct dentry *dentry, + const char *name, const void *value, size_t size, int flags) +{ + struct bpf_dynptr val; + + bpf_dynptr_from_mem(value_buf, sizeof(value_buf), 0, &val); + bpf_inode_init_xattr(NULL, &scratch_count, xattr_zone, &val); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/verifier_map_in_map.c b/tools/testing/selftests/bpf/progs/verifier_map_in_map.c index d3be69a9a755..621248a02a1f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_map_in_map.c +++ b/tools/testing/selftests/bpf/progs/verifier_map_in_map.c @@ -154,8 +154,7 @@ l0_%=: r0 = 0; \ SEC("socket") __description("forgot null checking on the inner map pointer") -__failure __msg("R1 type=map_ptr_or_null expected=map_ptr") -__msg("map_ptr_or_null, but this argument accepts map_ptr") +__failure __msg("Possibly NULL pointer passed to trusted R1") __failure_unpriv __naked void on_the_inner_map_pointer(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c b/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c index c01abf54923d..4b1eadddd89c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c +++ b/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c @@ -58,7 +58,7 @@ int mapofmaps_value_as_helper_mem_buf(struct __sk_buff *skb) } SEC("?tc") -__failure __msg("type=map_ptr_or_null expected=fp") +__failure __msg("Possibly NULL pointer passed to trusted R1") int mapofmaps_value_as_helper_fixed_mem(struct __sk_buff *skb) { char th[sizeof(struct tcphdr)] = {}; diff --git a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c index db7e30da234f..faf9eec0a63e 100644 --- a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c +++ b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c @@ -173,4 +173,125 @@ __naked void timed_may_goto_preserves_regs(void) : __clobber_all); } +__used __naked static void mg_ret1(void) +{ + asm volatile ( + "r0 = 1;" + "exit;" + ::: __clobber_all); +} + +__used __naked static void mg_ret2(void) +{ + asm volatile ( + "r0 = 2;" + "exit;" + ::: __clobber_all); +} + +__used __naked static void mg_call_first(void) +{ + asm volatile ( + "call mg_ret1;" + "r1 = 0;" +"1:" + ".8byte %[may_goto];" + "r1 += 1;" + "if r1 < 10 goto 1b;" + "exit;" + : + : __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 2, 0)) + : __clobber_all); +} + +/* + * mg_ret1 and mg_ret2 are placed in front of mg_call_first, + * so the offset of its call is negative. + */ +SEC("socket") +__description("may_goto in subprog that starts with a call") +__success +__retval(1) +__naked void may_goto_subprog_call_first(void) +{ + asm volatile ( + "call mg_ret1;" + "call mg_ret2;" + "call mg_call_first;" + "exit;" + ::: __clobber_all); +} + +__used __naked static void mg_cb_stop(void) +{ + asm volatile ( + "r0 = 1;" + "exit;" + ::: __clobber_all); +} + +__used __naked static void mg_cb_cont(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* r1 is zero, but not a constant, so that bpf_loop() is not inlined */ +__used __naked static void mg_func_first(void) +{ + asm volatile ( + "r2 = %[mg_cb_stop] ll;" + "r4 = r1;" + "r1 = 2;" + "r3 = 0;" + "call %[bpf_loop];" + "r1 = 0;" +"1:" + ".8byte %[may_goto];" + "r1 += 1;" + "if r1 < 10 goto 1b;" + "exit;" + : + : __imm_addr(mg_cb_stop), + __imm(bpf_loop), + __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 2, 0)) + : __clobber_all); +} + +/* + * mg_cb_stop stops bpf_loop() after the first iteration, + * mg_cb_cont lets it do both. + */ +SEC("socket") +__description("may_goto in subprog that starts with ld_imm64 of a func") +__success +__retval(1) +__naked void may_goto_subprog_func_first(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = %[mg_cb_stop] ll;" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "r1 = 1;" + "r2 = %[mg_cb_cont] ll;" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "call %[bpf_ktime_get_ns];" + "r1 = r0;" + "r1 >>= 63;" + "call mg_func_first;" + "exit;" + : + : __imm_addr(mg_cb_stop), + __imm_addr(mg_cb_cont), + __imm(bpf_loop), + __imm(bpf_ktime_get_ns) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_movsx.c b/tools/testing/selftests/bpf/progs/verifier_movsx.c index a4d8814eb5ed..195b27a51224 100644 --- a/tools/testing/selftests/bpf/progs/verifier_movsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_movsx.c @@ -8,7 +8,7 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 + (__clang_major__ >= 18 || defined(__BPF_FEATURE_MOVSX)) SEC("socket") __description("MOV32SX, S8") @@ -343,6 +343,7 @@ label_%=: \ SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_mtu.c b/tools/testing/selftests/bpf/progs/verifier_mtu.c index 256956ea1ac5..2f71f70302e4 100644 --- a/tools/testing/selftests/bpf/progs/verifier_mtu.c +++ b/tools/testing/selftests/bpf/progs/verifier_mtu.c @@ -17,4 +17,92 @@ int tc_uninit_mtu(struct __sk_buff *ctx) return TCX_PASS; } +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __uint(map_flags, BPF_F_WRONLY_PROG); + __type(key, __u32); + __type(value, __u32); +} map_mtu_wo SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u32); +} map_mtu_rw SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __uint(map_flags, BPF_F_RDONLY_PROG); + __type(key, __u32); + __type(value, __u32); +} map_mtu_ro SEC(".maps"); + +SEC("tc/ingress") +__failure __msg("read from map forbidden") +int tc_writeonly_mtu(struct __sk_buff *ctx) +{ + __u32 key = 0; + __u32 *mtu; + + mtu = bpf_map_lookup_elem(&map_mtu_wo, &key); + if (mtu) + bpf_check_mtu(ctx, 0, mtu, 0, 0); + return TCX_PASS; +} + +SEC("tc/ingress") +__success +int tc_readwrite_mtu(struct __sk_buff *ctx) +{ + __u32 key = 0; + __u32 *mtu; + + mtu = bpf_map_lookup_elem(&map_mtu_rw, &key); + if (mtu) + bpf_check_mtu(ctx, 0, mtu, 0, 0); + return TCX_PASS; +} + +SEC("xdp") +__failure __msg("read from map forbidden") +int xdp_writeonly_mtu(struct xdp_md *ctx) +{ + __u32 key = 0; + __u32 *mtu; + + mtu = bpf_map_lookup_elem(&map_mtu_wo, &key); + if (mtu) + bpf_check_mtu(ctx, 0, mtu, 0, 0); + return XDP_PASS; +} + +SEC("tc/ingress") +__failure __msg("write into map forbidden") +int tc_readonly_mtu(struct __sk_buff *ctx) +{ + __u32 key = 0; + __u32 *mtu; + + mtu = bpf_map_lookup_elem(&map_mtu_ro, &key); + if (mtu) + bpf_check_mtu(ctx, 0, mtu, 0, 0); + return TCX_PASS; +} + +SEC("tc/ingress") +__success +int tc_writeonly_output(struct __sk_buff *ctx) +{ + __u32 key = 0; + __u32 *out; + + out = bpf_map_lookup_elem(&map_mtu_wo, &key); + if (out) + bpf_get_current_comm(out, sizeof(*out)); + return TCX_PASS; +} + char LICENSE[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c b/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c index 967f4e6e3a49..fd418f27996f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c +++ b/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c @@ -61,6 +61,7 @@ __naked void percpu_addr(void) SEC("raw_tp") __description("percpu addr dummy") +__skip("percpu addr tests need x86") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_private_stack.c b/tools/testing/selftests/bpf/progs/verifier_private_stack.c index ea0a7e73331d..3d9994a52464 100644 --- a/tools/testing/selftests/bpf/progs/verifier_private_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_private_stack.c @@ -377,6 +377,7 @@ int private_stack_max_depth(void) SEC("kprobe") __description("private stack is not supported, use a dummy test") +__skip("private stack is not supported") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_raw_stack.c b/tools/testing/selftests/bpf/progs/verifier_raw_stack.c index c689665e07b9..0fe631411b9c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_raw_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_raw_stack.c @@ -84,6 +84,8 @@ __naked void skb_load_bytes_zero_len(void) SEC("tc") __description("raw_stack: skb_load_bytes, no init") __success __retval(0) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__failure_unpriv __msg_unpriv("invalid read from stack off -8+0 size 8") __naked void skb_load_bytes_no_init(void) { asm volatile (" \ @@ -103,6 +105,8 @@ __naked void skb_load_bytes_no_init(void) SEC("tc") __description("raw_stack: skb_load_bytes, init") __success __retval(0) +__caps_unpriv(CAP_BPF | CAP_NET_ADMIN) +__success_unpriv __naked void stack_skb_load_bytes_init(void) { asm volatile (" \ @@ -236,6 +240,7 @@ __naked void load_bytes_spilled_regs_data(void) SEC("tc") __description("raw_stack: skb_load_bytes, invalid access 1") +__load_if_no_large_stack() __failure __msg("invalid write to stack R3 off=-513 size=8") __naked void load_bytes_invalid_access_1(void) { @@ -254,6 +259,26 @@ __naked void load_bytes_invalid_access_1(void) } SEC("tc") +__description("raw_stack: skb_load_bytes, invalid access 1, large stack") +__load_if_large_stack() +__failure __msg("invalid write to stack R3 off=-2049 size=8") +__naked void load_bytes_invalid_access_1_large(void) +{ + asm volatile (" \ + r2 = 4; \ + r6 = r10; \ + r6 += -2049; \ + r3 = r6; \ + r4 = 8; \ + call %[bpf_skb_load_bytes]; \ + r0 = *(u64*)(r6 + 0); \ + exit; \ +" : + : __imm(bpf_skb_load_bytes) + : __clobber_all); +} + +SEC("tc") __description("raw_stack: skb_load_bytes, invalid access 2") __failure __msg("invalid write to stack R3 off=-1 size=8") __naked void load_bytes_invalid_access_2(void) diff --git a/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c b/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c index 199ad18f8eb5..799db6f5713b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c +++ b/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c @@ -344,7 +344,7 @@ __naked void potential_reference_to_system_key(void) SEC("tc") __description("reference tracking: release reference without check") -__failure __msg("type=sock_or_null expected=sock") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void tracking_release_reference_without_check(void) { asm volatile ( @@ -363,7 +363,7 @@ __naked void tracking_release_reference_without_check(void) SEC("tc") __description("reference tracking: release reference to sock_common without check") -__failure __msg("type=sock_common_or_null expected=sock") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void to_sock_common_without_check(void) { asm volatile ( @@ -1288,7 +1288,7 @@ l1_%=: r1 = r6; \ SEC("tc") __description("reference tracking: bpf_sk_release(listen_sk)") -__failure __msg("release helper bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") __naked void bpf_sk_release_listen_sk(void) { asm volatile ( diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c new file mode 100644 index 000000000000..661ce9ca8f93 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -0,0 +1,3451 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <linux/bpf.h> +#include <stdbool.h> +#include <stdint.h> +#include <bpf/bpf_helpers.h> +#include "../../../include/linux/filter.h" +#include "bpf_misc.h" +#include "bpf_kfuncs.h" + +#define COND_TEST(name, _initial, _step, _bound, msg, body) \ +SEC("xdp") \ +__success \ +__log_level(2) \ +__flag(BPF_F_TEST_STATE_FREQ) \ +__msg("loop header at {{[0-9]+}}, " msg) \ +__naked void name(void) \ +{ \ + asm volatile ( \ + "r6 = %[initial] ll;" \ + "r7 = %[bound] ll;" \ + "r9 = 0;" \ + body \ + "2: r0 = 0;" \ + "exit;" \ + : \ + : __imm_const(initial, _initial), \ + __imm_const(step, _step), \ + __imm_const(bound, _bound) \ + : __clobber_all); \ +} + +/* Optional assembly runs before the tested condition, preserving its latch. */ +#define PRE__COND_TEST(name, op, initial, step, bound, msg, ...) \ + COND_TEST(name, initial, step, bound, msg, \ + "1: r8 = %[step] ll;" __VA_ARGS__ \ + "if r6 " op " r7 goto 2f;" \ + "r6 += r8;" \ + "goto 1b;") + +#define POST_COND_TEST(name, op, initial, step, bound, msg, ...) \ + COND_TEST(name, initial, step, bound, msg, \ + "1: r8 = %[step] ll;" __VA_ARGS__ \ + "r6 += r8;" \ + "if r6 " op " r7 goto 1b;") + +/* + * Bound fallback verification of otherwise infinite or extremely long loops. + * JSET is not a linear condition, SCEV ignores it and the tested condition + * alone decides the iteration count. + */ +#define LIMIT_ITERATIONS "r9 += 1; if r9 & 8 goto 2f;" + +#define PRE__COND_TEST2(name, op, initial, step, bound, msg) \ + PRE__COND_TEST(name, op, initial, step, bound, msg, LIMIT_ITERATIONS) + +#define POST_COND_TEST32(name, op, initial, step, bound, msg) \ + COND_TEST(name, initial, step, bound, msg, \ + "1: r8 = %[step] ll;" LIMIT_ITERATIONS \ + "w6 += w8;" \ + "if w6 " op " w7 goto 1b;") + +#define ITER_U64_MAX (~0ULL) +#define ITER_S64_MAX ((1ULL << 63) - 1) +#define ITER_S64_MIN (1ULL << 63) +#define ITER_U32_MAX ((1ULL << 32) - 1) + +struct map_val { + char foo[1024]; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct map_val); +} map SEC(".maps"); + +typeof(map) other_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, __u64); +} small_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 4096); +} ringbuf SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __uint(max_entries, 1); + __type(key, int); + __type(value, int); + __array(values, struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(map_flags, BPF_F_INNER_MAP); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct map_val); + }); +} outer_map SEC(".maps"); + +struct bpf_iter__bpf_map_elem { + struct bpf_iter_meta *meta; + struct bpf_map *map; + void *key; + void *value; +}; + +SEC("xdp") +__success +__log_level(2) +__msg_next("scev at header 1:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__naked void simple_loop1(void) +{ + asm volatile (" \ + r0 = 0; \ +loop_%=: \ + if r0 == 10 goto exit_%=; \ + r0 += 1; \ + goto loop_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg("8: (7b) *(u64 *)(r1 +0) = r0 ; *fp-8 (+ *fp-8 1) -> ?") +__naked void indirect_write_invalidates_scev(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64 *)(r10 - 8) = r0; \ +loop_%=: \ + r0 = *(u64 *)(r10 - 8); \ + if r0 == 10 goto exit_%=; \ + r0 += 1; \ + *(u64 *)(r10 - 8) = r0; \ + r1 = r10; \ + r1 += -8; \ + *(u64 *)(r1 + 0) = r0; \ + goto loop_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* Even a no-op global callee can be freplaced with one that writes to *p. */ +__noinline int global_noop_counter(__u64 *p) +{ + return 0; +} + +SEC("xdp") +__success +__log_level(2) +__msg("8: (85) call pc+{{[0-9]+}} {{.*}}; *fp-8 (+ *fp-8 1) -> ?") +__naked void global_call_invalidates_scev(void) +{ + asm volatile ( + "r0 = 0;" + "*(u64 *)(r10 - 8) = r0;" + "r6 = 0;" +"1:" + "r0 = *(u64 *)(r10 - 8);" + "r0 += 1;" + "*(u64 *)(r10 - 8) = r0;" + "r1 = r10;" + "r1 += -8;" + "call global_noop_counter;" + "r6 += 1;" + "if r6 < 3 goto 1b;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg_next("scev at header 2:") +__msg_next(" *fp-8=(+ *fp-8 1) / (linear *fp-8 1)") +__msg_next(" scev at latch 3:") +__msg_next(" r0=*fp-8 / (linear *fp-8 1)") +__msg_next(" *fp-8=*fp-8 / (linear *fp-8 1)") +__naked void simple_loop2(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64 *)(r10 - 8) = r0; \ +loop_%=: \ + r0 = *(u64 *)(r10 - 8); \ + if r0 == 10 goto exit_%=; \ + r0 = *(u64 *)(r10 - 8); \ + r0 += 1; \ + *(u64 *)(r10 - 8) = r0; \ + goto loop_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg_next("scev at header 2:") +__msg_next(" r1=(+ r1 1) / (linear r1 1)") +__naked void meet_agrees(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = 0; \ +1: \ + if r1 == 2 goto 3f; \ + if r0 == 7 goto 2f; \ + r1 += 1; \ + goto 1b; \ +2: \ + r1 += 1; \ + goto 1b; \ +3: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("xdp") +__failure +__log_level(2) +__msg_next("scev at header 1:") +__msg_next(" r6=(any r6 (+ r6 1)) / ?") +__naked void meet_disagrees(void) +{ + asm volatile (" \ + r6 = 0; \ +1: \ + if r6 == 2 goto 3f; \ + call %[bpf_get_prandom_u32]; \ + if r0 == 7 goto 1b; \ + r6 += 1; \ + goto 1b; \ +3: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Both entry state values for r1 should be accounted for */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev at header 2:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg_next(" r1=42 / (any r1 42)") +__msg_next(" scev at latch 2:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg_next(" r1=42 / (any r1 42)") +__msg("loop header at 2, widening r0 to 0..10 step 1") +__msg_next("loop header at 2, widening r1 to 7..42 step 1") +__naked void entry_state_join(void) +{ + asm volatile (" \ + r0 = 0; \ + r1 = 7; \ +1: if r0 == 10 goto 2f; /* r1 in [7,42] */ \ + r0 += 1; \ + r1 = 42; \ + goto 1b; \ +2: r0 = r1; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__log_level(2) +__msg_next("scev at header 1:") +__msg_next(" r6=(bswap32 (bswap32 (zext32 (- (- (zext32 (+ (>>...) 1))))))) / ?") +__naked void expr_chain(void) +{ + asm volatile (" \ + r6 = 0; \ +1: \ + if r6 == 2 goto 2f; \ + r6 += 1; \ + r6 <<= 32; \ + r6 >>= 32; \ + w6 += 1; \ + r6 = -r6; \ + w6 = -w6; \ + r6 = bswap32 r6; \ + r6 = bswap32 r6; \ + goto 1b; \ +2: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +#define ALU_OP(insn) "r1 = r6; " insn "; r0 += r1;" + +SEC("xdp") +__success +__log_level(2) +__msg("r1 += 2 {{.*}}; r1 r6 -> (+ r6 2)") +__msg("r1 += r7 {{.*}}; r1 r6 -> (+ r6 r7)") +__msg("w1 += 2 {{.*}}; r1 r6 -> (zext32 (+ r6 2))") +__msg("w1 += w7 {{.*}}; r1 r6 -> (zext32 (+ r6 r7))") +__msg("r1 -= 2 {{.*}}; r1 r6 -> (- r6 2)") +__msg("r1 -= r7 {{.*}}; r1 r6 -> (- r6 r7)") +__msg("w1 -= 2 {{.*}}; r1 r6 -> (zext32 (- r6 2))") +__msg("w1 -= w7 {{.*}}; r1 r6 -> (zext32 (- r6 r7))") +__msg("r1 *= 2 {{.*}}; r1 r6 -> (* r6 2)") +__msg("r1 *= r7 {{.*}}; r1 r6 -> (* r6 r7)") +__msg("w1 *= 2 {{.*}}; r1 r6 -> (zext32 (* r6 2))") +__msg("w1 *= w7 {{.*}}; r1 r6 -> (zext32 (* r6 r7))") +__msg("r1 /= 2 {{.*}}; r1 r6 -> (/ r6 2)") +__msg("r1 /= r7 {{.*}}; r1 r6 -> (/ r6 r7)") +__msg("w1 /= 2 {{.*}}; r1 r6 -> (zext32 (/ r6 2))") +__msg("w1 /= w7 {{.*}}; r1 r6 -> (zext32 (/ r6 r7))") +__msg("r1 s/= 2 {{.*}}; r1 r6 -> (s/ r6 2)") +__msg("r1 s/= r7 {{.*}}; r1 r6 -> (s/ r6 r7)") +__msg("w1 s/= 2 {{.*}}; r1 r6 -> (zext32 (s/ r6 2))") +__msg("w1 s/= w7 {{.*}}; r1 r6 -> (zext32 (s/ r6 r7))") +__msg("r1 %= 2 {{.*}}; r1 r6 -> (% r6 2)") +__msg("r1 %= r7 {{.*}}; r1 r6 -> (% r6 r7)") +__msg("w1 %= 2 {{.*}}; r1 r6 -> (zext32 (% r6 2))") +__msg("w1 %= w7 {{.*}}; r1 r6 -> (zext32 (% r6 r7))") +__msg("r1 s%= 2 {{.*}}; r1 r6 -> (s% r6 2)") +__msg("r1 s%= r7 {{.*}}; r1 r6 -> (s% r6 r7)") +__msg("w1 s%= 2 {{.*}}; r1 r6 -> (zext32 (s% r6 2))") +__msg("w1 s%= w7 {{.*}}; r1 r6 -> (zext32 (s% r6 r7))") +__msg("r1 |= 2 {{.*}}; r1 r6 -> (| r6 2)") +__msg("r1 |= r7 {{.*}}; r1 r6 -> (| r6 r7)") +__msg("w1 |= 2 {{.*}}; r1 r6 -> (zext32 (| r6 2))") +__msg("w1 |= w7 {{.*}}; r1 r6 -> (zext32 (| r6 r7))") +__msg("r1 &= 2 {{.*}}; r1 r6 -> (& r6 2)") +__msg("r1 &= r7 {{.*}}; r1 r6 -> (& r6 r7)") +__msg("w1 &= 2 {{.*}}; r1 r6 -> (zext32 (& r6 2))") +__msg("w1 &= w7 {{.*}}; r1 r6 -> (zext32 (& r6 r7))") +__msg("r1 ^= 2 {{.*}}; r1 r6 -> (^ r6 2)") +__msg("r1 ^= r7 {{.*}}; r1 r6 -> (^ r6 r7)") +__msg("w1 ^= 2 {{.*}}; r1 r6 -> (zext32 (^ r6 2))") +__msg("w1 ^= w7 {{.*}}; r1 r6 -> (zext32 (^ r6 r7))") +__msg("r1 <<= 2 {{.*}}; r1 r6 -> (<< r6 2)") +__msg("r1 <<= r7 {{.*}}; r1 r6 -> (<< r6 r7)") +__msg("w1 <<= 2 {{.*}}; r1 r6 -> (zext32 (<< r6 2))") +__msg("w1 <<= w7 {{.*}}; r1 r6 -> (zext32 (<< r6 r7))") +__msg("r1 >>= 2 {{.*}}; r1 r6 -> (>> r6 2)") +__msg("r1 >>= r7 {{.*}}; r1 r6 -> (>> r6 r7)") +__msg("w1 >>= 2 {{.*}}; r1 r6 -> (zext32 (>> r6 2))") +__msg("w1 >>= w7 {{.*}}; r1 r6 -> (zext32 (>> r6 r7))") +__msg("r1 s>>= 2 {{.*}}; r1 r6 -> (s>> r6 2)") +__msg("r1 s>>= r7 {{.*}}; r1 r6 -> (s>> r6 r7)") +__msg("w1 s>>= 2 {{.*}}; r1 r6 -> (zext32 (s>> r6 2))") +__msg("w1 s>>= w7 {{.*}}; r1 r6 -> (zext32 (s>> r6 r7))") +__msg("r1 = -r1 {{.*}}; r1 r6 -> (- r6)") +__msg("w1 = -w1 {{.*}}; r1 r6 -> (zext32 (- r6))") +__msg("r1 = 42 {{.*}}; r1 r6 -> 42") +__msg("w1 = 42 {{.*}}; r1 r6 -> (zext32 42)") +__msg("r1 = r7 {{.*}}; r1 r6 -> r7") +__msg("w1 = w7 {{.*}}; r1 r6 -> (zext32 r7)") +__msg("r1 = (s8)r6 {{.*}}; r1 r6 -> (sext8 r6)") +__msg("r1 = (s16)r6 {{.*}}; r1 r6 -> (sext16 r6)") +__msg("r1 = (s32)r6 {{.*}}; r1 r6 -> (sext32 r6)") +__msg("w1 = (s8)w6 {{.*}}; r1 r6 -> (zext32 (sext8 r6))") +__msg("w1 = (s16)w6 {{.*}}; r1 r6 -> (zext32 (sext16 r6))") +__msg("r1 = bswap16 r1 {{.*}}; r1 r6 -> (bswap16 r6)") +__msg("r1 = bswap32 r1 {{.*}}; r1 r6 -> (bswap32 r6)") +__msg("r1 = bswap64 r1 {{.*}}; r1 r6 -> (bswap64 r6)") +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +__msg("r1 = le16 r1 {{.*}}; r1 r6 -> (zext16 r6)") +__msg("r1 = le32 r1 {{.*}}; r1 r6 -> (zext32 r6)") +__msg("r1 += 1 {{.*}}; r1 r6 -> (+ r6 1)") +__msg("r1 = be16 r1 {{.*}}; r1 r6 -> (bswap16 r6)") +__msg("r1 = be32 r1 {{.*}}; r1 r6 -> (bswap32 r6)") +__msg("r1 = be64 r1 {{.*}}; r1 r6 -> (bswap64 r6)") +#else +__msg("r1 = le16 r1 {{.*}}; r1 r6 -> (bswap16 r6)") +__msg("r1 = le32 r1 {{.*}}; r1 r6 -> (bswap32 r6)") +__msg("r1 = le64 r1 {{.*}}; r1 r6 -> (bswap64 r6)") +__msg("r1 = be16 r1 {{.*}}; r1 r6 -> (zext16 r6)") +__msg("r1 = be32 r1 {{.*}}; r1 r6 -> (zext32 r6)") +__msg("r1 += 1 {{.*}}; r1 r6 -> (+ r6 1)") +#endif +__naked void alu_ops_exprs(void) +{ + asm volatile ( + "r6 = 7;" + "r7 = 2;" + "r8 = 0;" +"1:" + "r0 = 0;" + ALU_OP("r1 += 2") + ALU_OP("r1 += r7") + ALU_OP("w1 += 2") + ALU_OP("w1 += w7") + ALU_OP("r1 -= 2") + ALU_OP("r1 -= r7") + ALU_OP("w1 -= 2") + ALU_OP("w1 -= w7") + ALU_OP("r1 *= 2") + ALU_OP("r1 *= r7") + ALU_OP("w1 *= 2") + ALU_OP("w1 *= w7") + ALU_OP("r1 /= 2") + ALU_OP("r1 /= r7") + ALU_OP("w1 /= 2") + ALU_OP("w1 /= w7") + ALU_OP("r1 s/= 2") + ALU_OP("r1 s/= r7") + ALU_OP("w1 s/= 2") + ALU_OP("w1 s/= w7") + ALU_OP("r1 %%= 2") + ALU_OP("r1 %%= r7") + ALU_OP("w1 %%= 2") + ALU_OP("w1 %%= w7") + ALU_OP("r1 s%%= 2") + ALU_OP("r1 s%%= r7") + ALU_OP("w1 s%%= 2") + ALU_OP("w1 s%%= w7") + ALU_OP("r1 |= 2") + ALU_OP("r1 |= r7") + ALU_OP("w1 |= 2") + ALU_OP("w1 |= w7") + ALU_OP("r1 &= 2") + ALU_OP("r1 &= r7") + ALU_OP("w1 &= 2") + ALU_OP("w1 &= w7") + ALU_OP("r1 ^= 2") + ALU_OP("r1 ^= r7") + ALU_OP("w1 ^= 2") + ALU_OP("w1 ^= w7") + ALU_OP("r1 <<= 2") + ALU_OP("r1 <<= r7") + ALU_OP("w1 <<= 2") + ALU_OP("w1 <<= w7") + ALU_OP("r1 >>= 2") + ALU_OP("r1 >>= r7") + ALU_OP("w1 >>= 2") + ALU_OP("w1 >>= w7") + ALU_OP("r1 s>>= 2") + ALU_OP("r1 s>>= r7") + ALU_OP("w1 s>>= 2") + ALU_OP("w1 s>>= w7") + ALU_OP("r1 = -r1") + ALU_OP("w1 = -w1") + ALU_OP("r1 = 42") + ALU_OP("w1 = 42") + ALU_OP("r1 = r7") + ALU_OP("w1 = w7") + ALU_OP("r1 = (s8)r6") + ALU_OP("r1 = (s16)r6") + ALU_OP("r1 = (s32)r6") + ALU_OP("w1 = (s8)w6") + ALU_OP("w1 = (s16)w6") + ALU_OP("r1 = bswap16 r1") + ALU_OP("r1 = bswap32 r1") + ALU_OP("r1 = bswap64 r1") + ALU_OP("r1 = le16 r1") + ALU_OP("r1 = le32 r1") + /* + * Expressions unchanged by transfer() do not appear in the log. + * Add r1 += 1 to touch the expression and make it appear in the log. + */ + ALU_OP("r1 = le64 r1; r1 += 1") + ALU_OP("r1 = be16 r1") + ALU_OP("r1 = be32 r1") + ALU_OP("r1 = be64 r1; r1 += 1") + "r8 += 1;" + "if r8 < 2 goto 1b;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +#undef ALU_OP + +/* Reset operands and consume results to keep transfer changes live. */ +#define ST_OP(stmt) \ + "r0 = 0; r1 = 1; r2 = 2;" \ + "*(u64 *)(r10 - 8) = -8;" \ + stmt ";" \ + "r0 += r1; r0 += r2;" \ + "r3 = *(u64 *)(r10 - 8); r0 += r3;" + +SEC("xdp") +__success +__log_level(2) +__msg("*(u64 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> 1") +__msg("*(u32 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> (spill32 1)") +__msg("*(u16 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> (spill16 1)") +__msg("*(u8 *)(r10 -8) = r1 {{.*}}; *fp-8 -8 -> (spill8 1)") +__msg("r1 = *(u64 *)(r10 -8) {{.*}}; r1 1 -> -8") +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +__msg("r1 = *(u32 *)(r10 -8) {{.*}}; r1 1 -> (zext32 -8)") +__msg("r1 = *(u16 *)(r10 -8) {{.*}}; r1 1 -> (zext16 -8)") +__msg("r1 = *(u8 *)(r10 -8) {{.*}}; r1 1 -> (zext8 -8)") +__msg("r1 = *(s32 *)(r10 -8) {{.*}}; r1 1 -> (sext32 -8)") +__msg("r1 = *(s16 *)(r10 -8) {{.*}}; r1 1 -> (sext16 -8)") +__msg("r1 = *(s8 *)(r10 -8) {{.*}}; r1 1 -> (sext8 -8)") +__msg("r1 = load_acquire((u8 *)(r10 -8)) {{.*}}; r1 1 -> (zext8 -8)") +__msg("r1 = load_acquire((u16 *)(r10 -8)) {{.*}}; r1 1 -> (zext16 -8)") +__msg("r1 = load_acquire((u32 *)(r10 -8)) {{.*}}; r1 1 -> (zext32 -8)") +#else +__msg("r1 = *(u32 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = *(u16 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = *(u8 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = *(s32 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = *(s16 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = *(s8 *)(r10 -8) {{.*}}; r1 1 -> _") +__msg("r1 = load_acquire((u8 *)(r10 -8)) {{.*}}; r1 1 -> _") +__msg("r1 = load_acquire((u16 *)(r10 -8)) {{.*}}; r1 1 -> _") +__msg("r1 = load_acquire((u32 *)(r10 -8)) {{.*}}; r1 1 -> _") +#endif +__msg("r2 = load_acquire((u64 *)(r10 -8)) {{.*}}; r2 2 -> -8") +__msg("store_release((u8 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> (spill8 1)") +__msg("store_release((u16 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> (spill16 1)") +__msg("store_release((u32 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> (spill32 1)") +__msg("store_release((u64 *)(r10 -8), r1) {{.*}}; *fp-8 -8 -> 1") +__msg("lock *(u64 *)(r10 -8) += r1 {{.*}}; *fp-8 -8 -> _") +__msg("r1 = atomic64_fetch_add((u64 *)(r10 -8), r1) {{.*}}; r1 1 -> _, *fp-8 -8 -> _") +__msg("r1 = atomic_fetch_add((u32 *)(r10 -8), r1) {{.*}}; r1 1 -> _, *fp-8 -8 -> _") +__msg("r1 = atomic64_xchg((u64 *)(r10 -8), r1) {{.*}}; r1 1 -> _, *fp-8 -8 -> _") +__msg("r0 = atomic64_cmpxchg((u64 *)(r10 -8), r0, r2) {{.*}}; r0 0 -> _, *fp-8 -8 -> _") +__msg("*(u64 *)(r10 -8) = 42 {{.*}}; *fp-8 -8 -> 42") +__msg("*(u8 *)(r10 -7) = r0 {{.*}}; *fp-8 -8 -> _") +__naked void store_load_exprs(void) +{ + asm volatile ( + "r7 = 0;" +"1:" + ST_OP("*(u64 *)(r10 - 8) = r1") + ST_OP("*(u32 *)(r10 - 8) = r1") + ST_OP("*(u16 *)(r10 - 8) = r1") + ST_OP("*(u8 *)(r10 - 8) = r1") + + ST_OP("r1 = *(u64 *)(r10 - 8)") + ST_OP("r1 = *(u32 *)(r10 - 8)") + ST_OP("r1 = *(u16 *)(r10 - 8)") + ST_OP("r1 = *(u8 *)(r10 - 8)") + ST_OP("r1 = *(s32 *)(r10 - 8)") + ST_OP("r1 = *(s16 *)(r10 - 8)") + ST_OP("r1 = *(s8 *)(r10 - 8)") + + ST_OP(".8byte %[load_acquire8]") + ST_OP(".8byte %[load_acquire16]") + ST_OP(".8byte %[load_acquire32]") + ST_OP(".8byte %[load_acquire64]") + ST_OP(".8byte %[store_release8]") + ST_OP(".8byte %[store_release16]") + ST_OP(".8byte %[store_release32]") + ST_OP(".8byte %[store_release64]") + + ST_OP(".8byte %[atomic_add64]") + ST_OP(".8byte %[atomic_fetch_add64]") + ST_OP(".8byte %[atomic_fetch_add32]") + ST_OP(".8byte %[atomic_xchg64]") + ST_OP(".8byte %[atomic_cmpxchg64]") + + ST_OP("*(u64 *)(r10 - 8) = 42") + ST_OP("*(u8 *)(r10 - 7) = r0") + + "r7 += 1;" + "if r7 < 2 goto 1b;" + "r0 = 0;" + "exit;" + : + : __imm_insn(atomic_add64, BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_xchg64, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(load_acquire8, BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_1, BPF_REG_10, -8)), + __imm_insn(load_acquire16, BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_1, BPF_REG_10, -8)), + __imm_insn(load_acquire32, BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_1, BPF_REG_10, -8)), + __imm_insn(load_acquire64, BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8)), + __imm_insn(store_release8, BPF_ATOMIC_OP(BPF_B, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(store_release16, BPF_ATOMIC_OP(BPF_H, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(store_release32, BPF_ATOMIC_OP(BPF_W, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(store_release64, BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_cmpxchg64, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_10, BPF_REG_2, -8)), + __imm_insn(atomic_fetch_add32, BPF_ATOMIC_OP(BPF_W, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_1, -8)), + __imm_insn(atomic_fetch_add64, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_1, -8)) + : __clobber_all); +} + +#undef ST_OP + +SEC("xdp") +__success +__log_level(2) +__msg_next("scev at header 1:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" r7=?") +__msg_next(" scev at latch 1:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" r7=?") +__msg_next("scev at header 4:") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg_next(" scev at latch 4:") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__naked void nested_loop1(void) +{ + asm volatile (" \ + r6 = 0; \ +1: \ + if r6 == 2 goto 2f; \ + r6 += 1; \ + r7 = 0; \ +3: \ + if r7 == 2 goto 4f; \ + r7 += 1; \ + goto 3b; \ +4: \ + goto 1b; \ +2: \ + r0 = r7; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("xdp") +__success +__msg("loop at 1") +__msg_next(" backedge from 4{{$}}") +__msg_next(" exit from 1 to 7") +__msg_next("loop at 4, nested in 1") +__msg_next(" backedge from 6, latch at 4") +__msg_next(" exit from 4 to 1") +__msg("scev at header 1:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" r7=?") +__msg_next("scev at header 4:") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg_next(" scev at latch 4:") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg("loop header at 1, latch not identified") +__msg("loop header at 4, widening r7 to 0..2 step 1") +__log_level(2) +__naked void nested_loop_hdr_backedge1(void) +{ + asm volatile (" \ + r6 = 0; \ +1: \ + if r6 == 2 goto 3f; \ + r6 += 1; \ + r7 = 0; \ +2: \ + if r7 == 2 goto 1b; \ + r7 += 1; \ + goto 2b; \ +3: \ + r0 = r7; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Outer loop {2, 3, 4, 5} contains inner loop {3, 4, 5}. + * The outer loop's only backedge is 4 -> 2, and its latch is also at 4, + * inside the inner loop. Outer loop's latch index is not computed in + * such a case, as there is no mechanism to build SCEVs for such latches + * at the moment. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop at 2{{$}}") +__msg_next(" backedge from 4{{$}}") +__msg_next(" exit from 5 to 6") +__msg_next("loop at 3, nested in 2") +__msg_next(" backedge from 5, latch at 5") +__msg_next(" exit from 4 to 2") +__msg_next(" exit from 5 to 6") +__msg("scev at header 2:") +__msg_next(" r6=?") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg_next("scev at header 3:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" scev at latch 5:") +__msg_next(" r6=(+ r6 1) / (linear (+ r6 1) 1)") +__msg("loop header at 2, latch not identified") +__msg("loop header at 3, header_count is [0..2] ") +__naked void nested_loop_hdr_backedge2(void) +{ + asm volatile (" \ + r6 = 0; \ + r7 = 0; \ +1: r7 += 1; \ +2: r6 += 1; \ + if r7 == 2 goto 1b; \ + if r6 < 2 goto 2b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Logical nest: for (r6 = 0; r6 < 3; r6++) + * for (r7 = 0; r7 < 4; r7++) r8++; + * Both backedges target header 3, so the CFG has one loop. r6 advances only + * on the outer backedge and r7 resets there; neither has a linear SCEV. + * r8 advances on both backedges and retains its linear SCEV. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop at 3{{$}}") +__msg("scev at header 3:") +__msg_next(" r6=(any r6 (+ r6 1)) / ?") +__msg_next(" r7=(any (+ r7 1) 0) / ?") +__msg_next(" r8=(+ r8 1) / (linear r8 1)") +__msg("loop header at 3, unsupported loop: multiple backedges") +__naked void nested_loops_shared_header(void) +{ + asm volatile (" \ + r6 = 0; \ + r7 = 0; \ + r8 = 0; \ +1: r8 += 1; \ + r7 += 1; \ + if r7 < 4 goto 1b; \ + r7 = 0; \ + r6 += 1; \ + if r6 < 3 goto 1b; \ + r0 = r8; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__success +__log_level(2) +__msg("loop header at 1, header_count is 10") +__msg("loop header at 5, header_count is 5") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void nested_loops_outer_latch_true_branch_to_inner(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; /* outer loop */ \ + r1 = 0; \ + if r0 < 10 goto 2f; /* outer latch true branch jumps into inner loop */ \ + goto 3f; \ +2: r1 += 1; /* inner loop */ \ + if r1 < 5 goto 2b; \ + goto 1b; \ +3: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Exact boundary hits for every continuation opcode. + * + * Example: pre_lt + * + * u64 r6 = 1; + * while (r6 < 7) { // r6 ∈ [1,3,5,7] + * r6 += 2; // r6 ∈ [3,5,7] + * } + */ +/* name op initial step bound expected message */ +PRE__COND_TEST (pre_lt, ">=", 1, 2, 7, "header_count is 4 ") +POST_COND_TEST (post_lt, "<", 1, 2, 7, "header_count is 3 ") +PRE__COND_TEST (pre_le, ">", 1, 2, 7, "header_count is 5 ") +POST_COND_TEST (post_le, "<=", 1, 2, 7, "header_count is 4 ") +PRE__COND_TEST (pre_gt, "<=", 9, -2, 3, "header_count is 4 ") +POST_COND_TEST (post_gt, ">", 9, -2, 3, "header_count is 3 ") +PRE__COND_TEST (pre_ge, "<", 9, -2, 3, "header_count is 5 ") +POST_COND_TEST (post_ge, ">=", 9, -2, 3, "header_count is 4 ") +PRE__COND_TEST (pre_slt, "s>=", -5, 2, 1, "header_count is 4 ") +POST_COND_TEST (post_slt, "s<", -5, 2, 1, "header_count is 3 ") +PRE__COND_TEST (pre_sle, "s>", -5, 2, 1, "header_count is 5 ") +POST_COND_TEST (post_sle, "s<=", -5, 2, 1, "header_count is 4 ") +PRE__COND_TEST (pre_sgt, "s<=", 5, -2, -1, "header_count is 4 ") +POST_COND_TEST (post_sgt, "s>", 5, -2, -1, "header_count is 3 ") +PRE__COND_TEST (pre_sge, "s<", 5, -2, -1, "header_count is 5 ") +POST_COND_TEST (post_sge, "s>=", 5, -2, -1, "header_count is 4 ") +PRE__COND_TEST (pre_ne, "==", 1, 2, 7, "header_count is 4 ") +POST_COND_TEST (post_ne, "!=", 1, 2, 7, "header_count is 3 ") +PRE__COND_TEST (pre_ne_down, "==", 9, -2, 3, "header_count is 4 ") +POST_COND_TEST (post_ne_down, "!=", 9, -2, 3, "header_count is 3 ") + +/* + * Bounds between two consecutive counter values. + * + * Example: pre_lt_round + * + * u64 r6 = 1; + * while (r6 < 8) { // r6 ∈ [1,3,5,7,9] + * r6 += 2; // r6 ∈ [3,5,7,9] + * } + */ +PRE__COND_TEST (pre_lt_round, ">=", 1, 2, 8, "header_count is 5 ") +PRE__COND_TEST (pre_le_round, ">", 1, 2, 8, "header_count is 5 ") +PRE__COND_TEST (pre_gt_round, "<=", 9, -2, 2, "header_count is 5 ") +PRE__COND_TEST (pre_ge_round, "<", 9, -2, 2, "header_count is 5 ") +PRE__COND_TEST (pre_slt_round, "s>=", -5, 2, 2, "header_count is 5 ") +PRE__COND_TEST (pre_sle_round, "s>", -5, 2, 2, "header_count is 5 ") +PRE__COND_TEST (pre_sgt_round, "s<=", 5, -2, -2, "header_count is 5 ") +PRE__COND_TEST (pre_sge_round, "s<", 5, -2, -2, "header_count is 5 ") + +/* + * Equality on the first non-strict comparison still takes a backedge. + * + * Example: pre_le_eq + * + * u64 r6 = 7; + * while (r6 <= 7) { // r6 ∈ [7,9] + * r6 += 2; // r6 ∈ [9] + * } + */ +PRE__COND_TEST (pre_le_eq, ">", 7, 2, 7, "header_count is 2 ") +PRE__COND_TEST (pre_ge_eq, "<", 3, -2, 3, "header_count is 2 ") +PRE__COND_TEST (pre_sle_eq, "s>", 1, 2, 1, "header_count is 2 ") +PRE__COND_TEST (pre_sge_eq, "s<", -1, -2, -1, "header_count is 2 ") + +/* + * Unsigned order across the sign bit. + * + * Example: pre_lt_sign + * + * u64 H = 1ULL << 63, r6 = H - 2; + * while (r6 < H + 1) { // r6 ∈ [H-2,H-1,H,H+1] + * r6 += 1; // r6 ∈ [H-1,H,H+1] + * } + */ +PRE__COND_TEST (pre_lt_sign, ">=", ITER_S64_MAX - 1, 1, ITER_S64_MIN + 1, "header_count is 4 ") +PRE__COND_TEST (pre_gt_sign, "<=", ITER_S64_MIN + 1, -1, ITER_S64_MAX - 1, "header_count is 4 ") + +/* + * The first latch comparison is false. + * + * Example: pre_lt_false + * + * u64 r6 = 7; + * while (r6 < 7) { // r6 ∈ [7] + * r6 += 2; // unreachable + * } + */ +PRE__COND_TEST (pre_lt_false, ">=", 7, 2, 7, "can't compute iterations count") +PRE__COND_TEST (pre_le_false, ">", 8, 2, 7, "can't compute iterations count") +PRE__COND_TEST (pre_gt_false, "<=", 3, -2, 3, "can't compute iterations count") +PRE__COND_TEST (pre_ge_false, "<", 2, -2, 3, "can't compute iterations count") +PRE__COND_TEST (pre_slt_false, "s>=", 1, 2, 1, "can't compute iterations count") +PRE__COND_TEST (pre_sle_false, "s>", 2, 2, 1, "can't compute iterations count") +PRE__COND_TEST (pre_sgt_false, "s<=", -1, -2, -1, "can't compute iterations count") +PRE__COND_TEST (pre_sge_false, "s<", -2, -2, -1, "can't compute iterations count") +PRE__COND_TEST (pre_ne_false, "==", 7, 2, 7, "can't compute iterations count") + +/* + * The counter moves away from the exit; bound fallback verification. + * + * Example: pre_lt_dir + * + * u64 r6 = 5, r9 = 0; + * while (++r9 <= 8 && r6 < 7) { // r6 ∈ [5,3,1,U64_MAX] + * r6 -= 2; // r6 ∈ [3,1,U64_MAX] + * } + */ +PRE__COND_TEST2(pre_lt_dir, ">=", 5, -2, 7, "can't compute iterations count") +PRE__COND_TEST2(pre_le_dir, ">", 5, -2, 7, "can't compute iterations count") +PRE__COND_TEST2(pre_gt_dir, "<=", 5, 2, 3, "can't compute iterations count") +PRE__COND_TEST2(pre_ge_dir, "<", 5, 2, 3, "can't compute iterations count") +PRE__COND_TEST2(pre_slt_dir, "s>=", -1, -2, 1, "can't compute iterations count") +PRE__COND_TEST2(pre_sle_dir, "s>", -1, -2, 1, "can't compute iterations count") +PRE__COND_TEST2(pre_sgt_dir, "s<=", 1, 2, -1, "can't compute iterations count") +PRE__COND_TEST2(pre_sge_dir, "s<", 1, 2, -1, "can't compute iterations count") + +/* + * The predicted exit would wrap the counter. + * + * Example: pre_lt_wrap + * + * u64 M = U64_MAX, r6 = M - 1, r9 = 0; + * while (++r9 <= 8 && r6 < M) { // r6 ∈ [M-1,0,2,...,14] + * r6 += 2; // r6 ∈ [0,2,...,14] + * } + */ +PRE__COND_TEST2(pre_lt_wrap, ">=", ITER_U64_MAX - 1, 2, ITER_U64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_le_wrap, ">", ITER_U64_MAX, 1, ITER_U64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_gt_wrap, "<=", 1, -2, 0, "can't compute iterations count") +PRE__COND_TEST2(pre_ge_wrap, "<", 0, -1, 0, "can't compute iterations count") +PRE__COND_TEST2(pre_slt_wrap, "s>=", ITER_S64_MAX - 1, 2, ITER_S64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_sle_wrap, "s>", ITER_S64_MAX, 1, ITER_S64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_sgt_wrap, "s<=", ITER_S64_MIN + 1, -2, ITER_S64_MIN, "can't compute iterations count") +PRE__COND_TEST2(pre_sge_wrap, "s<", ITER_S64_MIN, -1, ITER_S64_MIN, "can't compute iterations count") + +/* + * The non-strict count itself would overflow u64. + * + * Example: pre_le_count_ovf + * + * u64 r6 = 0, r9 = 0; + * while (++r9 <= 8 && r6 <= U64_MAX) { // r6 ∈ [0..8] + * r6 += 1; // r6 ∈ [1..8] + * } + */ +PRE__COND_TEST2(pre_le_count_ovf, ">", 0, 1, ITER_U64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_ge_count_ovf, "<", ITER_U64_MAX, -1, 0, "can't compute iterations count") +PRE__COND_TEST2(pre_sle_count_ovf, "s>", ITER_S64_MIN, 1, ITER_S64_MAX, "can't compute iterations count") +PRE__COND_TEST2(pre_sge_count_ovf, "s<", ITER_S64_MAX, -1, ITER_S64_MIN, "can't compute iterations count") + +/* + * JNE divisibility and equality across unsigned wraparound. + * + * Example: pre_ne_rem + * + * u64 r6 = 1, r9 = 0; + * while (++r9 <= 8 && r6 != 8) { // r6 ∈ [1,3,...,17] + * r6 += 2; // r6 ∈ [3,5,...,17] + * } + */ +PRE__COND_TEST2(pre_ne_rem, "==", 1, 2, 8, "can't compute iterations count") +PRE__COND_TEST2(pre_ne_rem_down, "==", 9, -2, 2, "can't compute iterations count") +PRE__COND_TEST (pre_ne_wrap_up, "==", ITER_U64_MAX - 1, 2, 0, "header_count is 2 ") +PRE__COND_TEST (pre_ne_wrap_down, "==", 1, -2, ITER_U64_MAX, "header_count is 2 ") + +/* + * Largest supported header count, sentinel, and u32 overflow. + * + * Example: pre_lt_max_hdrs + * + * u64 r6 = 0, r9 = 0; + * while (++r9 <= 8 && r6 < U32_MAX - 2) { // r6 ∈ [0..8] + * r6 += 1; // r6 ∈ [1..8] + * } + */ +PRE__COND_TEST2(pre_lt_max_hdrs, ">=", 0, 1, ITER_U32_MAX - 2, "header_count is [0..4294967294] ") +PRE__COND_TEST2(pre_lt_sentinel, ">=", 0, 1, ITER_U32_MAX - 1, "can't compute iterations count") +PRE__COND_TEST2(pre_lt_hdr_ovf, ">=", 0, 1, ITER_U32_MAX, "can't compute iterations count") + +/* + * The extreme signed step values remain valid unsigned magnitudes. + * + * Example: pre_lt_step_max + * + * u64 r6 = 0; + * while (r6 < S64_MAX) { // r6 ∈ [0,S64_MAX] + * r6 += S64_MAX; // r6 ∈ [S64_MAX] + * } + */ +PRE__COND_TEST (pre_lt_step_max, ">=", 0, ITER_S64_MAX, ITER_S64_MAX, "header_count is 2 ") +PRE__COND_TEST (pre_gt_step_min, "<=", ITER_S64_MIN, ITER_S64_MIN, 0, "header_count is 2 ") +PRE__COND_TEST (pre_ne_step_min, "==", 0, ITER_S64_MIN, ITER_S64_MIN, "header_count is 2 ") + +/* + * A zero step cannot establish a finite iteration count. + * + * Example: pre_lt_zero_step + * + * u64 r6 = 1, r9 = 0; + * while (++r9 <= 8 && r6 < 7) { // r6 ∈ [1], nine header visits + * r6 += 0; // r6 ∈ [1] + * } + */ +PRE__COND_TEST2(pre_lt_zero_step, ">=", 1, 0, 7, "can't compute iterations count") + +/* + * Signed JMP32 comparisons sign-extend the initial value, bound and step. + * Encode negative operands as positive u32 constants so the conversion matters. + * post32_sgt also checks that an exit exactly at S32_MIN is accepted. + */ +/* name op initial step bound expected message */ +POST_COND_TEST32(post32_slt, "s<", -7u, 2, -1u, "header_count is [0..3] ") +POST_COND_TEST32(post32_sle, "s<=", -7u, 2, -1u, "header_count is [0..4] ") +POST_COND_TEST32(post32_sgt, "s>", (__u32)INT32_MIN + 6, -2u, (__u32)INT32_MIN, "header_count is [0..3] ") +POST_COND_TEST32(post32_sge, "s>=", -1u, -2u, -7u, "header_count is [0..4] ") + +/* + * When computing 32-bit iteration bounds, scev.c:compute_max_iters() does + * the computation in the 64-bit domain first. For unsigned JMP32 comparisons, + * it zero-extends the initial value and bound, but sign-extends the step. + * Hence pick negative 64-bit initial values and bounds, and encode negative + * steps as positive u32 constants, so both conversions affect the computation. + * post32_lt also checks that an exit exactly at U32_MAX is accepted. + */ +POST_COND_TEST32(post32_lt, "<", -7, 2, -1, "header_count is [0..3] ") +POST_COND_TEST32(post32_le, "<=", -9, 2, -3, "header_count is [0..4] ") +POST_COND_TEST32(post32_gt, ">", -1, -2u, -7, "header_count is [0..3] ") +POST_COND_TEST32(post32_ge, ">=", -1, -2u, -7, "header_count is [0..4] ") +POST_COND_TEST32(post32_ne, "!=", -1, -2u, -7, "header_count is [0..3] ") + +/* An exit exactly at S32_MAX is accepted as well. */ +POST_COND_TEST32(post32_slt_max, "s<", INT32_MAX - 6, 2, INT32_MAX, "header_count is [0..3] ") + +/* + * The first iteration fits in 32 bits, but the predicted exit overshoots the bound + * by one and wraps. + */ +POST_COND_TEST32(post32_slt_wrap, "s<", INT32_MAX - 3, 2, INT32_MAX, "can't compute iterations count") +POST_COND_TEST32(post32_sgt_wrap, "s>", (__u32)INT32_MIN + 3, -2u, (__u32)INT32_MIN, "can't compute iterations count") +POST_COND_TEST32(post32_lt_wrap, "<", -4, 2, -1, "can't compute iterations count") + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 10 ") +__msg("loop header at 1, widening r0 to 0..9 step 1") +__msg("processed 5 insns") +__naked void post_cond_jlt(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 < 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Explore with r7=10 first, r7=4096 next. + * Should trigger invalid stack read if r7 is marked precise. + * + * r0 = 0; + * r6 = 0; + * r7 = bpf_get_prandom_u32() == 42 ? 10 : 4096; + * do { + * r0++; + * r6++; + * r3 = *(fp - r0); + * } while (r6 < r7); + * return r0; + */ +SEC("xdp") +__failure +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 6, header_count is 10 ") +__msg("mark_precise: frame0: parent state regs=r6,r7 stack=:") +__msg("loop header at 6, widening r0 to 0..9 step 1") +__msg("loop header at 6, header_count is 4096 ") +__msg("loop header at 6, widening r0 to 0..4095 step 1") +__msg("invalid variable-offset read from stack R2") +__naked void mark_bound_precise(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r7 = 4096; \ + if r0 != 42 goto 1f; \ + r7 = 10; \ +1: r0 = 0; \ + r6 = 0; \ +2: r0 += 1; \ + r6 += 1; \ + r2 = r0; \ + r2 = -r2; \ + r2 += r10; \ + r3 = *(u8 *)(r2 + 0); \ + if r6 < r7 goto 2b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Explore with r6=4086 first, r6=0 next. + * Should trigger invalid stack read if r6 is marked precise. + * + * r0 = 0; + * r6 = bpf_get_prandom_u32() == 42 ? 4086 : 0; + * do { + * r0++; + * r6++; + * r3 = *(fp - r0); + * } while (r6 < 4096); + * return r0; + */ +SEC("xdp") +__failure +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 5, header_count is 10 ") +__msg("mark_precise: frame0: parent state regs=r6 stack=:") +__msg("loop header at 5, widening r0 to 0..9 step 1") +__msg("loop header at 5, header_count is 4096 ") +__msg("loop header at 5, widening r0 to 0..4095 step 1") +__msg("invalid variable-offset read from stack R2") +__naked void mark_base_precise(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = 0; \ + if r0 != 42 goto 1f; \ + r6 = 4086; \ +1: r0 = 0; \ +2: r0 += 1; \ + r6 += 1; \ + r2 = r0; \ + r2 = -r2; \ + r2 += r10; \ + r3 = *(u8 *)(r2 + 0); \ + if r6 < 4096 goto 2b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Spill size should be taken into account when evaluating bound scev. + * + * r0 = 256; + * *(u8 *)(r10 - 8) = r0; + * r0 = 0; + * do { + * r0++; + * r1 = *(u64 *)(r10 - 8); + * if (r0 & 0x200) break; // hidden from scev, limit main pass iterations. + * } while (r0 < r1); + * return r0; + */ +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 3:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg("loop header at 3, can't compute iterations count") +__naked void eval_expr_small_spill_bound(void) +{ + asm volatile (" \ + r0 = 256; \ + *(u8 *)(r10 - 8) = r0; \ + r0 = 0; \ +1: r0 += 1; \ + r1 = *(u64 *)(r10 - 8); \ + if r0 & 0x200 goto 2f; \ + if r0 < r1 goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +/* Even when the saved value fits, the upper bytes of the slot are unknown. */ +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 3:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg("loop header at 3, can't compute iterations count") +__naked void eval_expr_small_spill_bound_fits(void) +{ + asm volatile (" \ + r0 = 10; \ + *(u8 *)(r10 - 8) = r0; \ + r0 = 0; \ +1: r0 += 1; \ + r1 = *(u64 *)(r10 - 8); \ + if r0 & 0x80 goto 2f; \ + if r0 < r1 goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +/* + * r0 = 255; + * *(u8 *)(r10 - 8) = r0; // narrow spill + * do { + * r0 = *(u64 *)(r10 - 8); // wide load, widening should bail out here. + * r0++; + * *(u64 *)(r10 - 8) = r0; + * } while (r0 < 265); + * return r0; + */ +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 2:") +__msg_next(" *fp-8=(+ *fp-8 1) / (linear *fp-8 1)") +__msg("loop header at 2, can't compute iterations count") +__not_msg("widening") +__naked void no_widen_small_spill_counter(void) +{ + asm volatile (" \ + r0 = 255; \ + *(u8 *)(r10 - 8) = r0; \ +1: r0 = *(u64 *)(r10 - 8); \ + r0 += 1; \ + *(u64 *)(r10 - 8) = r0; \ + if r0 < 265 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* Full-width counterpart: the counter can be widened from 5 to 14. */ +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 2:") +__msg_next(" *fp-8=(+ *fp-8 1) / (linear *fp-8 1)") +__msg("loop header at 2, widening *fp-8 to 5..14 step 1") +__naked void widen_full_spill_counter(void) +{ + asm volatile (" \ + r0 = 5; \ + *(u64 *)(r10 - 8) = r0; \ +1: r0 = *(u64 *)(r10 - 8); \ + r0 += 1; \ + *(u64 *)(r10 - 8) = r0; \ + if r0 < 15 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 2:") +__msg_next(" *fp-8=(+ *fp-8 1) / (linear *fp-8 1)") +__msg("loop header at 2, widening *fp-8 to 5..14 step 1") +__naked void widen_acquire_release_counter(void) +{ + asm volatile ( + "r0 = 5;" + "*(u64 *)(r10 - 8) = r0;" +"1:" + ".8byte %[load_acquire];" + "r0 += 1;" + ".8byte %[store_release];" + "if r0 < 15 goto 1b;" + "exit;" + : + : __imm_insn(load_acquire, BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -8)), + __imm_insn(store_release, BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_0, -8)) + : __clobber_all); +} + +/* + * r6 scev is (any r6 *fp-8). The byte store leaves memory at zero, + * but the spill metadata retains 256. Widen r6 to an unknown scalar + * instead of [1,256], which would exclude the zero loaded from memory. + * + * r0 = 0; + * *(u64 *)(r10 - 8) = r0; + * r0 = 256; + * *(u8 *)(r10 - 8) = r0; // narrow spill + * r6 = 1; + * r7 = 0; + * do { + * r0 = r6; + * r6 = *(u64 *)(r10 - 8); + * r7++; + * } while (r7 < 2); + * return r0; + */ +SEC("xdp") +__success __retval(0) +__log_level(2) +__msg("scev at header 6:") +__msg_next(" r6=*fp-8 / (any r6 *fp-8)") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg("loop header at 6, widening r6 to 0x8000000000000000..0x7fffffffffffffff step 1") +__naked void widen_any_small_spill_leaf(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64 *)(r10 - 8) = r0; \ + r0 = 256; \ + *(u8 *)(r10 - 8) = r0; \ + r6 = 1; \ + r7 = 0; \ +1: r0 = r6; \ + r6 = *(u64 *)(r10 - 8); \ + r7 += 1; \ + if r7 < 2 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Widening (any *fp-8 7) must allocate the slot before writing its state. + * The unallocated entry value makes the union an unknown scalar. + * + * r6 = 0; + * do { + * if (r6 == 1) + * *(u64 *)(r10 - 8) = 7; + * r0 = *(u64 *)(r10 - 8); + * r6++; + * } while (r6 < 3); + * return r0; + */ +SEC("xdp") +__success __retval(7) +__log_level(2) +__msg("scev at header 1:") +__msg_next("r6=(+ r6 1) / (linear r6 1)") +__msg_next("*fp-8=(any *fp-8 7) / (any *fp-8 7)") +__msg("loop header at 1, widening r6 to 0..2 step 1") +__msg("2: (7a) *(u64 *)(r10 -8) = 7 ; R10=fp0 fp-8=7 loop_stack=1") +__naked void widen_any_unallocated_slot(void) +{ + asm volatile (" \ + r6 = 0; \ +1: if r6 != 1 goto 2f; \ + *(u64 *)(r10 - 8) = 7; \ +2: r0 = *(u64 *)(r10 - 8); \ + r6 += 1; \ + if r6 < 3 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* A spilled zero entry contributes zero to the union. */ +SEC("xdp") +__success __retval(7) +__log_level(2) +__msg("loop header at 2, widening *fp-8 to 0..7 step 1") +__naked void widen_any_zero_spill(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r6 = 0; \ +1: if r6 != 1 goto 2f; \ + *(u64 *)(r10 - 8) = 7; \ +2: r0 = *(u64 *)(r10 - 8); \ + r6 += 1; \ + if r6 < 3 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* A byte store inside the slot leaves STACK_MISC and an unknown scalar value. */ +SEC("xdp") +__success __retval(7) +__log_level(2) +__msg("scev at header 2:") +__msg_next("r6=(+ r6 1) / (linear r6 1)") +__msg_next("*fp-8=(any *fp-8 7) / (any *fp-8 7)") +__msg("loop header at 2, widening r6 to 0..2 step 1") +__msg("3: (79) r1 = *(u64 *)(r10 -8) ; R1=scalar() R10=fp0 fp-8=m???????") +__naked void widen_any_misc_slot(void) +{ + asm volatile (" \ + *(u8 *)(r10 - 1) = 1; \ + r6 = 0; \ +1: if r6 != 1 goto 2f; \ + r1 = *(u64 *)(r10 - 8); \ + *(u64 *)(r10 - 8) = 7; \ +2: r0 = *(u64 *)(r10 - 8); \ + r6 += 1; \ + if r6 < 3 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +__naked __noinline __used static int probe_read_stack(void) +{ + asm volatile ( + "r2 = 8;" + "r3 = 0;" + "call %[bpf_probe_read_kernel];" + "exit;" + : + : __imm(bpf_probe_read_kernel) + : __clobber_all); +} + +/* The unknown SCEV of an unbound STACK_MISC slot must not prevent widening. */ +SEC("tracepoint") +__success +__log_level(2) +__msg("scev at header 4:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" *fp-8=?") +__msg("loop header at 4, widening r6 to 0..9 step 1") +__naked void widen_subprog_probe_read(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "call probe_read_stack;" + "r6 = 0;" +"1:" + "r0 = *(u64 *)(r10 - 8);" + "r1 = r10;" + "r1 += -8;" + "call probe_read_stack;" + "r6 += 1;" + "if r6 < 10 goto 1b;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* An induction variable whose range overflows s64 is widened to the full range. */ +SEC("socket") +__success +__log_level(2) +__msg("loop header at 4, header_count is 10") +__msg("loop header at 4, widening r1 to 0..9 step 1") +__msg("loop header at 4, widening r6 to 0x8000000000000000..0x7fffffffffffffff step 1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void widen_overflowing_range(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + r6 |= 1; /* not an unknown scalar, those need no widening */ \ + r1 = 0; \ +1: r6 += 24; \ + r1 += 1; \ + if r1 < 10 goto 1b; \ + r0 = r6; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * The bound slot is unallocated at the first loop header. SCEV must + * treat it as an unknown scalar. JSET limits main-pass iterations. + * + * r0 = 0; + * do { + * r0++; + * r1 = *(u64 *)(r10 - 8); + * if (r0 & 0x80) break; + * } while (r0 < r1); + * return 0; + */ +SEC("xdp") +__success __retval(0) +__log_level(2) +__msg("scev at header 1:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg("loop header at 1, can't compute iterations count") +__not_msg("widening") +__naked void eval_expr_unallocated_stack_bound(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + r1 = *(u64 *)(r10 - 8); \ + if r0 & 0x80 goto 2f; \ + if r0 < r1 goto 1b; \ +2: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 3, header_count is 8") +__msg("loop header at 3, widening r0 to 2..9 step 1") +__msg("3: R0=scalar(smin=umin=smin32=umin32=2,smax=umax=smax32=umax32=9,{{.*}})") +__msg("processed 6 insns") +__naked void post_cond_jlt_with_base(void) +{ + asm volatile (" \ + r7 = 10 ll; /* ldimm64 for a twist */ \ + r0 = 2; \ +1: r0 += 1; \ + if r0 < r7 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 4 ") +__naked void latch_base_differs_from_step(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r1 = r0; \ + r1 += 1; \ + if r1 >= 10 goto 2f; \ + r0 += 4; \ + goto 1b; \ +2: r0 = r1; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 11") +__msg("loop header at 1, widening r0 to 0..10 step 1") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=10,{{.*}})") +__msg("processed 5 insns") +__naked void post_cond_jle(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 <= 10 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 10") +__msg("loop header at 1, widening r0 to 0..9 step 1") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=9,{{.*}})") +__msg("processed 6 insns") +__naked void post_cond_jge(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 >= 10 goto 2f; \ + goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 11") +__msg("loop header at 1, widening r0 to 0..10 step 1") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=10,{{.*}})") +__msg("processed 6 insns") +__naked void pre_cond_jge(void) +{ + asm volatile (" \ + r0 = 0; \ +1: if r0 >= 10 goto 2f; \ + r0 += 1; \ + goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg_next("scev at header 1:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg_next(" scev at latch 2:") +__msg_next(" r0=(+ r0 1) / (linear (+ r0 1) 1)") +__msg("loop header at 1, widening r0") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,{{.*}})") +__msg("1: (07) r0 += 1 ; R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,{{.*}})") +__msg("2: (55) if r0 != 0x3 goto pc-2") +__msg("3: (95) exit") +__msg("loop header at 1, clamping r0") +__msg("from 2 to 1: safe") +__msg("processed 5 insns") +__naked void post_cond_jne(void) +{ + asm volatile (" \ + r0 = 0; \ +1: r0 += 1; \ + if r0 != 3 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg_next("scev at header 1:") +__msg_next(" r0=(+ r0 -1) / (linear r0 -1)") +__msg_next(" scev at latch 2:") +__msg_next(" r0=(+ r0 -1) / (linear (+ r0 -1) -1)") +__msg("loop header at 1, header_count is 3 ") +__msg("loop header at 1, widening r0 to 1..3 step 1") +__msg("1: R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,var_off=(0x0; 0x3)) loop_stack=1") +__msg("loop header at 1, clamping r0 to 1..2 (s32 1..2) step 1") +__msg("processed 5 insns") +__naked void post_cond_jne_neg_step(void) +{ + asm volatile (" \ + r0 = 3; \ +1: r0 += -1; \ + if r0 != 0 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg_next("scev at header 1:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg_next(" scev at latch 1:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg("loop header at 1, widening r0") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=3,{{.*}})") +__msg("1: (15) if r0 == 0x3 goto pc+2 ; R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,{{.*}})") +__msg("2: (07) r0 += 1 ; R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,{{.*}})") +__msg("3: (05) goto pc-3") +__msg("loop header at 1, clamping r0") +__msg("1: safe") +__msg("from 1 to 4: R0=3") +__msg("4: R0=3") +__msg("4: (95) exit") +__msg("processed 6 insns") +__naked void pre_cond_je1(void) +{ + asm volatile (" \ + r0 = 0; \ +1: if r0 == 3 goto 2f; \ + r0 += 1; \ + goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg_next("scev at header 1:") +__msg_next(" r0=(zext32 (+ r0 1)) / (zext32 (linear r0 1))") +__msg_next(" scev at latch 2:") +__msg_next(" r0=(zext32 (+ r0 1)) / (zext32 (linear (+ r0 1) 1))") +__msg("loop header at 1, widening r0") +__msg("1: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,{{.*}})") +__msg("1: (04) w0 += 1 ; R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,{{.*}})") +__msg("2: (56) if w0 != 0x3 goto pc-2") +__msg("3: (95) exit") +__msg("loop header at 1, clamping r0") +__msg("from 2 to 1: safe") +__msg("processed 5 insns") +__naked void post_cond_jne32(void) +{ + asm volatile (" \ + w0 = 0; \ +1: w0 += 1; \ + if w0 != 3 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* w0 ∈ {0xffffffff, 2, 5}, w0 % 3 ∈ {0, 2, 2} ⇒ step 3 can't be preserved */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev at header 1:") +__msg_next(" r0=(zext32 (+ r0 3)) / (zext32 (linear r0 3))") +__msg_next(" scev at latch 2:") +__msg_next(" r0=(zext32 (+ r0 3)) / (zext32 (linear (+ r0 3) 3))") +__msg("entering loop 1") +__msg_next("loop header at 1, header_count is 2") +__msg("loop header at 1, widening r0 to 0..0xffffffff (s32 -1..2) step 1") +__msg("1: R0=scalar(smin=0,smax=umax=0xffffffff,smin32=-1,smax32=2,{{.*}})") +__naked void post_cond_jne32_overflow(void) +{ + asm volatile (" \ + w0 = 0xffffffff; \ +1: w0 += 3; \ + if w0 != 5 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * w0 ∈ {0, 3, ..., 0xffffffff, 2, 5}, w0 % 3 ∈ {0, 0, ..., 0, 2, 2} + * ⇒ step 3 can't be preserved. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, header_count is 1431655767") +__msg("loop header at 2, widening r0 to 0..0xffffffff step 1") +__naked void post_cond_jne32_full_range(void) +{ + asm volatile (" \ + w0 = 0; \ + w1 = 0; \ +1: w0 += 3; \ + w1 += 1; \ + if w1 != 1431655767 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* r0 ∈ {-6, 0xfffffff7, 0xfffffff4}, r0 % 3 ∈ {0, 1, 1} ⇒ step 3 can't be preserved. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 2") +__msg("loop header at 1, widening r0 to -6..0xfffffffa (s32 -9..-6) step 1") +__naked void post_cond_jne32_base_mismatch(void) +{ + asm volatile (" \ + r0 = -6; \ +1: w0 += -3; \ + if w0 != -12 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * r0 ∈ {0, 0xffffffff, 0xfffffffe}, + * ⇒ the 64-bit header bounds are [0, U32_MAX], 32-bit header bounds are [-1, 0]. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 2") +__msg("loop header at 1, widening r0 to 0..0xffffffff (s32 -1..0) step 1") +__naked void post_cond_jne32_neg_step(void) +{ + asm volatile (" \ + w0 = 0; \ +1: w0 += -1; \ + if w0 != -2 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* + * r0 ∈ {S64_MIN + 1, 3, 5}, r0 % 2 ∈ {1, 1, 1} + * ⇒ step 2 and base 1 can be preserved. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, header_count is 2") +__msg("loop header at 2, widening r0 to 0x8000000000000000..0x7fffffffffffffff (s32 1..3) step 2") +__msg("2: R0=scalar({{.*}}step=1+2)") +__naked void post_cond_jne32_signed_boundary(void) +{ + asm volatile (" \ + r0 = %[initial] ll; \ +1: w0 += 2; \ + if w0 != 5 goto 1b; \ + exit; \ +" : + : __imm_const(initial, ITER_S64_MIN + 1) + : __clobber_all); +} + +/* The pre-condition also visits the header with the terminating value 3. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 4") +__msg("loop header at 1, widening r0 to 0..3 (s32 0..3) step 1") +__msg("loop header at 1, clamping r0") +__msg("4: R0=3") +__naked void pre_cond_je32(void) +{ + asm volatile (" \ + w0 = 0; \ +1: if w0 == 3 goto 2f; \ + w0 += 1; \ + goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +/* ALU32 keeps r0 in [0, U32_MAX], so the 64-bit comparison never becomes false. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 4, can't compute iterations count") +__naked void post_cond_alu32_jmp64(void) +{ + asm volatile (" \ + w0 = 0; \ + r1 = 0x100000003 ll; \ + r9 = 0; \ +1: " + LIMIT_ITERATIONS + "w0 += 1; \ + if r0 != r1 goto 1b; \ +2: exit; \ +" ::: __clobber_all); +} + +/* The signed 32-bit comparison is always true, even when r0 exceeds INT32_MAX. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, can't compute iterations count") +__naked void post_cond_alu64_jmp32(void) +{ + asm volatile (" \ + r0 = %[initial]; \ + r9 = 0; \ +1: " + LIMIT_ITERATIONS + "r0 += 1; \ + if w0 s<= %[bound] goto 1b; \ +2: exit; \ +" : + : __imm_const(initial, INT32_MAX - 1), + __imm_const(bound, INT32_MAX) + : __clobber_all); +} + +/* ALU32 values above INT32_MAX remain positive in a signed 64-bit comparison. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 3, header_count is 3") +__msg("loop header at 3, widening r0 to 0xfffffffc..0xfffffffe (s32 -4..-2) step 1") +__naked void post_cond_alu32_jmp64_bounded(void) +{ + asm volatile (" \ + w0 = -4; \ + r1 = 0xffffffff ll; \ +1: w0 += 1; \ + if r0 s< r1 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +/* The negative 64-bit counter and its exit value fit in the signed 32-bit range. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is 3") +__msg("loop header at 1, widening r0 to -6..-2 step 2") +__naked void post_cond_alu64_jmp32_bounded(void) +{ + asm volatile (" \ + r0 = -6; \ +1: r0 += 2; \ + if w0 s< 0 goto 1b; \ + exit; \ +" ::: __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 1, header_count is [0..3] ") +__naked void one_backedge_two_exits(void) +{ + asm volatile (" \ + r6 = 0; \ +1: call %[bpf_get_prandom_u32]; \ + if r0 == 0 goto 2f; \ + r6 += 1; \ + if r6 != 3 goto 1b; \ +2: r0 = r6; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg_next("scev at header 11:") +__msg_next(" r0=(+ r0 1) / (linear r0 1)") +__msg_next(" r1=(+ r1 2) / (linear r1 2)") +__msg_next(" scev at latch 16:") +__msg_next(" r0=(+ r0 1) / (linear (+ r0 1) 1)") +__msg_next(" r1=(+ r1 2) / (linear (+ r1 2) 2)") +__msg("loop header at 11, widening r0") +__msg("loop header at 11, widening r1") +__msg("11: R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=7,var_off=(0x0; 0x7)) R1=scalar(smin=smin32=0,smax=umax=smax32=umax32=14,var_off=(0x0; 0xe),step=0+2)") +/* loop exit */ +__msg("16: (a5) if r0 < 0x8 goto pc-6") +__msg("exiting loop 11") +__msg("17: (95) exit") +/* second iteration */ +__msg("loop header at 11, clamping r0") +__msg("loop header at 11, clamping r1") +/* iteration convergence */ +__msg("from 16 to 11: safe") +__not_msg("{{^}}11:") +/* map lookup error path */ +__msg("from 7 to 17: safe") +__naked void correlated_regs(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64*)(r10 - 8) = r1; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r0 = 0; \ + r1 = 0; \ +1: r2 = r6; \ + r2 += r1; \ + *(u8 *)(r2 + 0) = 1; \ + r0 += 1; \ + r1 += 2; \ + if r0 < 8 goto 1b; \ +2: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +/* + * k = 0 + * for (i = 0; i < 4; i++): + * for (j = 0; j < 4; j++): + * k += 1 + * k <<= 1 // make SCEV construction not possible + * k >>= 1 + * map[k] = 1 // make k precise + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("19: (72) *(u8 *)(r4 +0) = 1 ; R4=map_value(id={{.*}},map=map,ks=4,vs=1024,imm=16)") +__not_msg("19: ") +__msg("processed 106 insns") +__naked void nested_loops_precise_var1(void) +{ + asm volatile (" \ + *(u64*)(r10 - 8) = 0; \ + r1 = %[map] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 3f; \ + r1 = 0; \ + r3 = 0; \ + /* outer loop */ \ +1: r2 = 0; \ + /* inner loop */ \ +2: r2 += 1; \ + r3 += 1; \ + r3 <<= 1; \ + r3 >>= 1; \ + if r2 < 4 goto 2b; \ + r1 += 1; \ + if r1 < 4 goto 1b; \ + r4 = r0; \ + r4 += r3; \ + *(u8 *)(r4 + 0) = 1; \ + r0 = 0; \ +3: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg("loop header at 2, header_count is 100 ") +__msg("loop header at 4, header_count is [0..100] ") +__msg("loop header at 6, header_count is [0..100] ") +__msg("processed 20 insns") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void nested_loop_with_two_exits(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = 0; \ +1: r6 += 1; \ + r7 = 0; \ +2: r7 += 1; \ + r8 = 0; \ +3: r8 += 1; \ + call %[bpf_get_prandom_u32]; \ + if r0 == 42 goto +1; \ + goto 4f; \ + if r8 < 100 goto 3b; \ + if r7 < 100 goto 2b; \ +4: if r6 < 100 goto 1b; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__naked void exit_loop_into_loop_header(void) +{ + asm volatile (" \ + r1 = 0; \ + r2 = 0; \ +loop_a_%=: \ + r1 += 1; \ + if r1 < 10 goto loop_a_%=; \ +loop_b_%=: \ + r2 += 1; \ + if r2 < 10 goto loop_b_%=; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * This exercises verifier.c:loop_stack_{pop,push}() implementation, + * at 'goto d' loops 'b' and 'a' have to be popped from stack, + * while loops 'c' and 'd' have to be pushed to stack. + * + * loop a: // header 5 + * loop b: // header 6 + * if (rand) goto d; // 8 -> 12, side entry into inner loop d + * ... + * loop c: // header 11 + * loop d: // header 12 + * ... + */ +SEC("xdp") +__log_level(2) +__msg("loop at 5") +__msg("loop at 6, nested in 5") +__msg("loop at 11, irreducible") +__msg("loop at 12, nested in 11") +/* entry via if r0 == 5 goto d_%= false branch */ +__msg("loop header at 12, header_count is 3 ") +__msg("loop header at 12, widening r9 to 0..2 step 1") +__msg("12: R8=1 R9=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,var_off=(0x0; 0x3)) loop_stack=11,12") +/* entry via if r0 == 5 goto d_%= true branch */ +__msg("loop header at 12, header_count is 3 ") +__msg("loop header at 12, widening r9 to 0..2 step 1") +__msg("from 8 to 12: R8=0 R9=scalar(smin=smin32=0,smax=umax=smax32=umax32=2,var_off=(0x0; 0x3)) R10=fp0 loop_stack=11,12") +__naked void enter_nested_loop_from_side(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = 0; \ + r7 = 0; \ + r8 = 0; \ + r9 = 0; \ +a_%=: r6 += 1; \ +b_%=: r7 += 1; \ + call %[bpf_get_prandom_u32]; \ + if r0 == 5 goto d_%=; \ + if r7 < 3 goto b_%=; \ + if r6 < 3 goto a_%=; \ +c_%=: r8 += 1; \ +d_%=: r9 += 1; \ + if r9 < 3 goto d_%=; \ + if r8 < 3 goto c_%=; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Loop A (counter in r6) contains irreducible loop B (header 5), + * which contains C (counter in r9). + * Entering B at 'body' saves and restores r6, making it appear invariant. + * Entering B at 'alternate' skips the save and modifies r6 instead. + * Hence A must not infer a SCEV expression for r6. + * SCEV expression for r9 in C should still be computed. + * + * 0: r6 = 0; + * do { // A + * 1: r6++; + * 2: r0 = bpf_get_prandom_u32(); + * 3: r7 = 0; + * 4: if (r0 > 5) goto alternate; + * 5: B: r8 = r6; // B + * 6: goto body; + * 7: alternate: + * r8 = r6; + * 8: r8++; + * 9: body: + * r6 = r8; + * 10: r9 = 0; + * do { // C + * 11: r9++; + * 12: } while (r9 < 3); + * 13: r7++; + * 14: if (r7 < 4) goto B; + * 15: } while (r6 < 4); + * 16: r0 = 0; + * 17: return r0; + */ +SEC("xdp") +__success +__log_level(2) +__msg("loop at 1{{$}}") +__msg("loop at 5, nested in 1, irreducible") +__msg("loop at 11, nested in 5") +__msg("scev at header 1:") +__msg_next(" r6=?") +__msg("scev at header 11:") +__msg_next(" r9=(+ r9 1) / (linear r9 1)") +__msg("loop header at 11, widening r9 to 0..2 step 1") +__not_msg("loop header at 1, widening r6") +__naked void nested_irreducible_loop(void) +{ + asm volatile (" \ + r6 = 0; \ +a_%=: r6 += 1; \ + call %[bpf_get_prandom_u32]; \ + r7 = 0; \ + if r0 > 5 goto alternate_%=; \ +b_%=: r8 = r6; \ + goto body_%=; \ +alternate_%=: \ + r8 = r6; \ + r8 += 1; \ +body_%=: \ + r6 = r8; \ + r9 = 0; \ +c_%=: r9 += 1; \ + if r9 < 3 goto c_%=; \ + r7 += 1; \ + if r7 < 4 goto b_%=; \ + if r6 < 4 goto a_%=; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Loop A (counter in r6) contains irreducible loop B. + * Loop B has an exit edge from A. This is a corner case + * observed during testing: insn_aux_data[l1].need_scev == true, + * but SCEV environment is not constructed. + * + * r6 = 0; + * do { // A + * r6++; + * r0 = bpf_get_prandom_u32(); + * r7 = 0; + * if (r0 > 5) goto alternate; + * do { // B + * r7++; + * alt: r0 = bpf_get_prandom_u32(); + * l1: if (r0 == 42) + * goto exit; + * r7++; + * } while (r7 < 4); + * } while (r6 < 4); + * r0 = 0; + * exit: return r0; + */ +SEC("xdp") +__success +__log_level(2) +__naked void nested_irreducible_loop2(void) +{ + asm volatile (" \ + r6 = 0; \ +a_%=: r6 += 1; \ + call %[bpf_get_prandom_u32]; \ + r7 = 0; \ + if r0 > 5 goto alternate_%=; \ +b_%=: r7 += 1; \ +alternate_%=: \ + call %[bpf_get_prandom_u32]; \ + if r0 == 42 goto exit_%=; \ + r7 += 1; \ + if r7 < 4 goto b_%=; \ + if r6 < 4 goto a_%=; \ + r0 = 0; \ +exit_%=:exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Induction variable seeded from a non-constant value. r7 enters the loop as a + * range aligned to 2 (prandom & 0x6 -> {0,2,4,6}) and is incremented by a + * non-power-of-2 slope of 6. Since the entry value is not a single point, only + * the power-of-two alignment shared by the entry value and the slope can be + * guaranteed, so the widened step is 2. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 4, widening r7 to 0..18 step 2") +__msg("R7=scalar(smin=smin32=0,smax=umax=smax32=umax32=18,var_off=(0x0; 0x1e),step=0+2)") +__naked void widen_nonconst_base(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r7 = r0; \ + r7 &= 0x6; \ + r6 = 0; \ +1: r7 += 6; \ + r6 += 1; \ + if r6 < 3 goto 1b; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * After excluding zero from {0,4,8,12}, the interval starts at 1 while the + * values remain multiples of 4. Widening over three iterations must preserve + * base 0 and include 20. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 5, header_count is 3 ") +__msg("5: R7=scalar({{.*}}smax=umax=smax32=umax32=20,{{.*}}step=0+4)") +__naked void widen_nonconst_base_refined(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r7 = r0; \ + r7 &= 0xc; \ + if r7 < 1 goto 2f; \ + r8 = 0; \ +1: r7 += 4; \ + r8 += 1; \ + if r8 < 3 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Two nested loops with (1) exiting directly to (2): + * + * for (r6 = 0; r6 < 4; r6++) { + * r7 = 0; + * for (; r7 < 3; r7++) {} // (1) + * for (; r7 != 0; r7--) {} // (2) + * } + */ +SEC("xdp") +__success +__log_level(2) +__msg("loop header at 1, widening r6 to 0..3 step 1") +__msg("loop header at 2, widening r7 to 0..2 step 1") +__msg("exiting loop 2") +__msg("entering loop 4") +__msg("loop header at 4, widening r7 to 1..3 step 1") +__naked void sibling_inner_loops(void) +{ + asm volatile (" \ + r6 = 0; \ +1: r7 = 0; \ +2: r7 += 1; \ + if r7 < 3 goto 2b; \ +3: r7 += -1; \ + if r7 != 0 goto 3b; \ + r6 += 1; \ + if r6 < 4 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__success +__log_level(2) +__msg("loop header at 0, can't compute iterations count") +__naked void uninit_slot_counter(void) +{ + asm volatile (" \ +1: r0 = *(u64 *)(r10 - 8); \ + r0 += 1; \ + *(u64 *)(r10 - 8) = r0; \ + if r0 < 10 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The assignment happens only on the second iteration: + * + * r7 = 5; + * for (r6 = 0; r6 < 3; r6++) + * if (r6 == 1) + * r7 = 10; + * if (r7 != 10) + * invalid_stack_read(); + * + * R7 is always 10 at the real exit. Widening loses the correlation between + * R6 and R7, leaving R7 in [5, 10] both at the loop header and after the loop. + * The verifier therefore rejects the possible invalid stack read. + */ +SEC("xdp") +__failure +__log_level(2) +__msg("2: {{.*}}R7=scalar(smin=umin=smin32=umin32=5,smax=umax=smax32=umax32=10,var_off=(0x0; 0xf))") +__msg("7: R7=scalar(smin=umin=smin32=umin32=5,smax=umax=smax32=umax32=10,var_off=(0x0; 0xf))") +__msg("invalid read from stack R10 off=0 size=8") +__naked void conditional_assignment_on_second_iteration(void) +{ + asm volatile (" \ + r7 = 5; \ + r6 = 0; \ +1: if r6 >= 3 goto 3f; \ + if r6 != 1 goto 2f; \ + r7 = 10; \ +2: r6 += 1; \ + goto 1b; \ +3: if r7 == 10 goto 4f; \ + r0 = *(u64 *)(r10 + 0); \ +4: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The latch bound has header SCEV (any r7 100), so it is not invariant: + * + * u64 i = 0, n = 3; + * for (;;) { + * ++i; + * if (i >= n) + * break; + * if (i == 1) + * n = 100; + * } + * + * The ANY expression must also appear at the latch, rather than a bare r7 + * that would incorrectly be treated as a loop-invariant bound. + */ +SEC("xdp") +__success +__log_level(2) +__msg("scev at header 2:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" r7=(any r7 100) / (any r7 100)") +__msg_next(" scev at latch 3:") +__msg_next(" r6=(+ r6 1) / (linear (+ r6 1) 1)") +__msg_next(" r7=r7 / (any r7 100)") +__naked void no_widen_changing_latch_bound(void) +{ + asm volatile (" \ + r6 = 0; \ + r7 = 3; \ +loop_%=: \ + r6 += 1; \ + if r6 >= r7 goto exit_%=; \ + if r6 != 1 goto next_%=; \ + r7 = 100; \ +next_%=: \ + goto loop_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * Join two stack pointer offsets via (any r7 r8), with r8 loop-invariant. + * The joined range [-16, -8] still points to initialized stack memory, + * so dereferencing r7 after the loop is safe. + */ +SEC("xdp") +__success __retval(0) +__log_level(2) +__msg("loop header at 7, widening r7 to -16..-8 step 1") +__msg("7: {{.*}}R7=fp(smin=smin32=-16,smax=smax32=-8,") +__msg("12: R7=fp(smin=smin32=-16,smax=smax32=-8,") +__naked void conditional_stack_pointer_assignment(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 16) = 0; \ + *(u64 *)(r10 - 8) = 0; \ + r7 = r10; \ + r7 += -16; \ + r8 = r10; \ + r8 += -8; \ + r6 = 0; \ +1: if r6 >= 3 goto 3f; \ + if r6 != 1 goto 2f; \ + r7 = r8; \ +2: r6 += 1; \ + goto 1b; \ +3: r0 = *(u64 *)(r7 + 0); \ + exit; \ +" ::: __clobber_all); +} + +/* + * Both choices are valid, but combining their offsets while retaining + * small_map's metadata would invent small_map + 32, outside its value. + * + * r6 = lookup(small_map, 0); + * r7 = lookup(map, 0) + 32; + * for (r8 = 0; r8 < 3; r8++) { + * if (r8 == 1) + * r6 = r7; + * r0 = *(u32 *)(r6 + 0); + * } + */ +SEC("xdp") +__success __retval(0) +__log_level(2) +__msg("loop header at {{[0-9]+}}, can't widen r6, expr is") +__not_msg("widening") +__naked void no_widen_any_map_origin(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[small_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 3f; \ + r6 = r0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 3f; \ + r7 = r0; \ + r7 += 32; \ + r8 = 0; \ +1: if r8 != 1 goto 2f; \ + r6 = r7; \ +2: r0 = *(u32 *)(r6 + 0); \ + r8 += 1; \ + if r8 < 3 goto 1b; \ +3: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(small_map), __imm_addr(map) + : __clobber_all); +} + +__naked __noinline __used static int any_stack_origin_callee(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r6 = r1; \ + r7 = r10; \ + r7 += -8; \ + r8 = 0; \ +1: if r8 != 1 goto 2f; \ + r6 = r7; \ +2: r8 += 1; \ + if r8 < 3 goto 1b; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" ::: __clobber_all); +} + +/* + * The callee chooses between caller fp-16 and its own fp-8: + * + * r6 = r1; + * r7 = fp - 8; + * for (r8 = 0; r8 < 3; r8++) + * if (r8 == 1) + * r6 = r7; + * return *(u64 *)r6; + * + * Both locations hold zero. Their offsets belong to different frames + * and cannot be combined into one stack pointer. + */ +SEC("xdp") +__success __retval(0) +__log_level(2) +__msg("loop header at {{[0-9]+}}, can't widen r6, expr is") +__not_msg("widening") +__naked void no_widen_any_stack_origin(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 16) = 0; \ + r1 = r10; \ + r1 += -16; \ + call any_stack_origin_callee; \ + exit; \ +" ::: __clobber_all); +} + +/* + * r7 starts as 1 + 6*k and invariant r8 as 1 + 9*k (0 <= k <= 3). + * The (any r7 r8) union must preserve base 1 and step gcd(6, 9) = 3. + */ +SEC("xdp") +__success __retval(1) +__log_level(2) +__msg("r7 += 1 {{.*}}step=1+6)") +__msg("r8 += 1 {{.*}}step=1+9)") +__msg("loop header at 9, widening r7 to 1..28 step 3") +__msg("9: {{.*}}R7=scalar({{.*}},step=1+3)") +__msg("14: R7=scalar({{.*}},step=1+3)") +__naked void conditional_scalar_assignment_gcd(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 3; \ + r7 = r0; \ + r7 *= 6; \ + r7 += 1; \ + r8 = r0; \ + r8 *= 9; \ + r8 += 1; \ + r6 = 0; \ +1: if r6 >= 3 goto 3f; \ + if r6 != 1 goto 2f; \ + r7 = r8; \ +2: r6 += 1; \ + goto 1b; \ +3: r0 = r7; \ + r0 %%= 3; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * r7 = ctx->data; + * r8 = r7 + * for (r6 = 0; r6 < 10 && random() != 42; r7++, r6++); + * if (r7 >= ctx->data_end) + * return; + * *(r8 + 4); // At this point the loop executed unknown number of times. + * // Hence, r7 range gives no information about r8. + */ +SEC("tc") +__failure +__msg("R8 min value is outside of the allowed memory range") +__naked void break_pkt_pointers_id(void) +{ + asm volatile (" \ + r7 = *(u32*)(r1 + %[__sk_buff_data]); \ + r8 = r7; \ + r9 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r6 = 0; \ +1: call %[bpf_get_prandom_u32]; \ + if r0 == 42 goto 3f; \ + r6 += 1; \ + r7 += 1; \ + if r6 < 10 goto 1b; \ +3: if r7 >= r9 goto 2f; \ + r0 = *(u8*)(r8 + 4); \ +2: r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), + __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * A loop induction variable used to compute the base address of a store to the + * stack must not be widened: the spill offset would become varying, which the + * verifier does not track. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, can't widen r2, expr is (+ r2 8), requires exact stack-offset tracking") +__naked void no_widen_stack_spill(void) +{ + asm volatile (" \ + r0 = 0; \ + r2 = 0; \ +1: r3 = r10; \ + r3 += -64; \ + r3 += r2; \ + *(u64 *)(r3 + 0) = r0; \ + r0 += 1; \ + r2 += 8; \ + if r0 < 4 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} +/* + * Same hazard across a loop nest: the outer induction variable r2 addresses a + * stack store performed inside the inner loop. The dependency is pulled up from + * the inner loop, so the outer loop must not widen r2. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, can't widen r2, expr is (+ r2 8), requires exact stack-offset tracking") +__msg("loop header at 3, widening r1 to 0..1 step 1") +__naked void no_widen_stack_spill_nested(void) +{ + asm volatile (" \ + r0 = 0; \ + r2 = 0; \ +1: r1 = 0; \ +2: r3 = r10; \ + r3 += -64; \ + r3 += r2; \ + *(u64 *)(r3 + 0) = r1; \ + r1 += 1; \ + if r1 < 2 goto 2b; \ + r0 += 1; \ + r2 += 8; \ + if r0 < 4 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * The inner loop restores r6 on its backedge but exits with r6 = -1000 on break. + * Outer loop must account for the exit value instead of deriving r6 = r6_entry + n. + * On the first outer backedge r6 is -999, so the next map access is invalid. + * + * u8 *r7 = map_value; + * s64 r6 = 0; + * do { + * 1: r9 = 0; + * r0 = r7[r6]; + * while (true) { + * 2: r8 = r6; + * r6 = -1000; + * if (++r9 > 5) + * break; + * r6 = r8; + * 3: } + * r6++; + * } while (r6 < 3); + */ +SEC("xdp") +__failure +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev at header 9:") +__msg_next(" r6=(+ ? 1) / ?") +__msg_next(" scev at latch 20:") +__msg_next(" r6=(+ ? 1) / (+ ? 1)") +__msg_next("scev at header 13:") +__msg_next(" r9=(+ r9 1) / (linear r9 1)") +__msg_next(" scev at latch 16:") +__msg_next(" r6=-1000 / -1000") +__msg_next(" r8=r6 / r6") +__msg_next(" r9=(+ r9 1) / (linear (+ r9 1) 1)") +__msg("R6=-999") +__msg("R1 min value is negative") +__naked void nested_loop_exit_clobbers_reg(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r2 = r10;" + "r2 += -8;" + "r1 = %[map] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 4f;" + "r7 = r0;" + "r6 = 0;" +"1:" + "r9 = 0;" + "r1 = r7;" + "r1 += r6;" + "r0 = *(u8 *)(r1 + 0);" +"2:" + "r8 = r6;" + "r6 = -1000;" + "r9 += 1;" + "if r9 > 5 goto 3f;" + "r6 = r8;" + "goto 2b;" +"3:" + "r6 += 1;" + "if r6 s< 3 goto 1b;" +"4:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +/* + * The inner loop exits both nested loops with r6 unchanged, or exits only itself with r6 = -1000. + * The middle loop repairs the latter exit with a stack fill. It therefore preserves r6, + * so the outermost loop must retain r6_entry + n. + * + * r6 = 0; + * do { + * r7 = 0; + * do { + * spill = r6; + * r9 = 0; + * do { + * r8 = r6; + * if (random() & 1) + * goto next; + * r6 = -1000; + * if (++r9 >= 2) + * break; + * r6 = r8; + * } while (true); + * r6 = spill; + * } while (++r7 < 2); + * next: + * r6++; + * } while (r6 < 3); + */ +SEC("xdp") +__success __retval(3) +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("r6 = *(u64 *)(r10 -8) {{.*}}; r6 ? -> r6") +__msg("scev at header 1:") +__msg_next(" r6=(+ r6 1) / (linear r6 1)") +__msg_next(" scev at latch 16:") +__msg_next(" r6=(+ r6 1) / (linear (+ r6 1) 1)") +__msg("scev at header 2:") +__msg_next(" r7=(+ r7 1) / (linear r7 1)") +__msg("scev at header 4:") +__msg_next(" r9=(+ r9 1) / (linear r9 1)") +__msg(" scev at latch 9:") +__msg_next(" r8=r6 / r6") +__msg_next(" r9=(+ r9 1) / (linear (+ r9 1) 1)") +__naked void nested_loop_exit_preserves_reg(void) +{ + asm volatile ( + "r6 = 0;" +"1:" + "r7 = 0;" +"2:" + "*(u64 *)(r10 - 8) = r6;" + "r9 = 0;" +"3:" + "r8 = r6;" + "call %[bpf_get_prandom_u32];" + "if r0 & 1 goto 5f;" + "r6 = -1000;" + "r9 += 1;" + "if r9 >= 2 goto 4f;" + "r6 = r8;" + "goto 3b;" +"4:" + "r6 = *(u64 *)(r10 - 8);" + "r7 += 1;" + "if r7 < 2 goto 2b;" +"5:" + "r6 += 1;" + "if r6 < 3 goto 1b;" + "r0 = r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Complement of no_widen_stack_spill: a sub-register (1-byte) store to the stack + * lands as STACK_MISC and carries no tracked value, so the induction variable + * addressing it (r2) is still widened. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("widening r2 to 0..24 step 8") +__naked void widen_byte_stack_store(void) +{ + asm volatile (" \ + r0 = 0; \ + r2 = 0; \ +1: r3 = r10; \ + r3 += -64; \ + r3 += r2; \ + *(u8 *)(r3 + 0) = r0; \ + r0 += 1; \ + r2 += 8; \ + if r0 < 4 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * A fill (BPF_LDX) at a varying stack offset loses precision just like a spill, + * so the induction variable computing the load base (r2) must not be widened. + * The slots are initialized up front so the fill itself is a valid read. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 6, can't widen r2, expr is (+ r2 8), requires exact stack-offset tracking") +__naked void no_widen_stack_fill(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64 *)(r10 - 64) = r0; \ + *(u64 *)(r10 - 56) = r0; \ + *(u64 *)(r10 - 48) = r0; \ + *(u64 *)(r10 - 40) = r0; \ + r2 = 0; \ +1: r3 = r10; \ + r3 += -64; \ + r3 += r2; \ + r4 = *(u64 *)(r3 + 0); \ + r0 += 1; \ + r2 += 8; \ + if r0 < 4 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * struct xdp_md *slots[2] = { ctx, ctx }; + * + * r7 = 0; + * for (r8 = 0; r8 < 2; r8++) { + * value = slots[r7]->data; + * if (i == 0) + * r7 = 1; + * } + * + * SCEV expression for r7 is (any r7 8) and r7 is used to + * address the stack memory. Avoid widening it, otherwise + * verifier won't know the type of the slots[r7] expression. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev at header 4:") +__msg_next("r7=(any r7 8) / (any r7 8)") +__msg("loop header at 4, can't widen r7, expr is {{.*}}, requires exact stack-offset tracking") +__naked void no_widen_stack_fill_any(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 16) = r1; \ + *(u64 *)(r10 - 8) = r1; \ + r7 = 0; \ + r8 = 0; \ +1: r2 = r10; \ + r2 += -16; \ + r2 += r7; \ + r3 = *(u64 *)(r2 + 0); /* slots[r7] */ \ + r0 = *(u32 *)(r3 + 0); /* slots[r7]->data */ \ + if r8 != 0 goto 2f; \ + r7 = 8; /* conditionally update r7 */ \ +2: r8 += 1; \ + if r8 < 2 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* + * A dynptr/iter/irq/res_spin_lock call initializes a stack object through a + * pointer argument, which acts like a spill base: the induction variable + * computing that argument's varying stack offset must not be widened, otherwise + * the slot can't be resolved. Here each iteration constructs an xdp dynptr at + * &dptrs[i]. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("can't widen {{.*}}, requires exact stack-offset tracking") +#ifndef __clang__ +/* The issue is unrelated to SCEV */ +__skip("GCC emits a stack-pointer loop condition the verifier cannot resolve") +#endif +int no_widen_dynptr_kfunc_arg(struct xdp_md *ctx) +{ + struct bpf_dynptr dptrs[4]; + int i; + +#ifdef __clang__ +#pragma clang loop unroll(disable) +#else +#pragma GCC unroll 0 +#endif + for (i = 0; i < 4; i++) + bpf_dynptr_from_xdp(ctx, 0, &dptrs[i]); + + return 0; +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("header_count is 4 ") +__msg("widening r6 to -16..-13 step 1") +__naked void ptr_stack(void) +{ + asm volatile (" \ + r6 = r10; \ + r6 += -16; \ + r7 = r6; \ + r7 += 4; \ +1: r6 += 1; \ + if r6 < r7 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +__naked __noinline __used static int ptr_stack_callee(void) +{ + asm volatile (" \ + r6 = r10; \ + r6 += -16; \ + r7 = r1; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 4f; /* bound main pass iteration, but hide it from SCEV */ \ + if r6 < r7 goto 1b; /* r6 and r7 are from different frames, can't be widened */ \ +4: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* The end pointer belongs to the caller's frame. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (fp), r7 (fp)") +__not_msg("widening r6") +__naked void ptr_stack_other_frame(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -12; \ + call ptr_stack_callee; \ + exit; \ +" ::: __clobber_all); +} + +/* Metadata and packet data have different origins. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (pkt_meta), r7 (pkt)") +__not_msg("widening r6") +__naked void ptr_packet_meta_other_origin(void) +{ + asm volatile (" \ + r6 = *(u32 *)(r1 + %[data_meta]); \ + r7 = *(u32 *)(r1 + %[data]); \ + r7 += 4; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 4f; \ + if r6 < r7 goto 1b; \ +4: r0 = 0; \ + exit; \ +" : + : __imm_const(data, offsetof(struct xdp_md, data)), + __imm_const(data_meta, offsetof(struct xdp_md, data_meta)) + : __clobber_all); +} + +/* Two lookups in a 1-element array count as the same memory origin. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("header_count is 4 ") +__msg("widening r6 to 0..3 step 1") +__naked void ptr_map_value_two_lookups(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[small_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[small_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r7 = r0; \ + r7 += 4; \ +1: r6 += 1; \ + if r6 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(small_map) + : __clobber_all); +} + +/* + * For inner array maps with BPF_F_INNER_MAP flag the actual array size + * might differ from metadata in map declaration. Hence, r6 and r7 below + * cannot be directly compared / iterations count can't be inferred. + * + * r8 = bpf_map_lookup_elem(outer_map, ...); // inner map + * r6 = bpf_map_lookup_elem(r8, ...); + * r7 = bpf_map_lookup_elem(r8, ...); + * r7 += 4; + * do { r6++; ... } while (r6 < r7); + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (map_value), r7 (map_value)") +__not_msg("widening r6") +__naked void ptr_map_value_two_lookups_inner_map(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[outer_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r8 = r0; \ + r2 = r10; \ + r2 += -8; \ + r1 = r8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r1 = r8; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r7 = r0; \ + r7 += 4; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 2f; /* main pass iteration guard */ \ + if r6 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(outer_map) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("header_count is 4 ") +__msg("widening r6 to 0..3 step 1") +__naked void ptr_map_value(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r7 = r6; \ + r7 += 4; \ +1: r6 += 1; \ + if r6 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + +/* Equal offsets into different maps do not establish a common origin. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (map_value), r7 (map_value)") +__not_msg("widening r6") +__naked void ptr_map_value_other_map(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[other_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r7 = r0; \ + r7 += 4; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 4f; \ + if r6 < r7 goto 1b; \ +4: \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map), __imm_addr(other_map) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("header_count is 4 ") +__msg("widening r6 to 0..3 step 1") +__naked void ptr_mem(void) +{ + asm volatile (" \ + r1 = %[ringbuf] ll; \ + r2 = 8; \ + r3 = 0; \ + call %[bpf_ringbuf_reserve]; \ + if r0 == 0 goto 2f; \ + r8 = r0; \ + r6 = r0; \ + r7 = r6; \ + r7 += 4; \ +1: r6 += 1; \ + if r6 < r7 goto 1b; \ + r1 = r8; \ + r2 = 0; \ + call %[bpf_ringbuf_discard]; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_ringbuf_reserve), __imm(bpf_ringbuf_discard), __imm_addr(ringbuf) + : __clobber_all); +} + +/* Separate reservations have different pointer IDs. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (ringbuf_mem), r7 (ringbuf_mem)") +__not_msg("widening r6") +__naked void ptr_mem_other_reservation(void) +{ + asm volatile (" \ + r1 = %[ringbuf] ll; \ + r2 = 8; \ + r3 = 0; \ + call %[bpf_ringbuf_reserve]; \ + if r0 == 0 goto 3f; \ + r8 = r0; \ + r6 = r0; \ + r1 = %[ringbuf] ll; \ + r2 = 8; \ + r3 = 0; \ + call %[bpf_ringbuf_reserve]; \ + if r0 == 0 goto 2f; \ + r9 = r0; \ + r7 = r0; \ + r7 += 4; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 4f; \ + if r6 < r7 goto 1b; \ +4: r1 = r9; \ + r2 = 0; \ + call %[bpf_ringbuf_discard]; \ +2: r1 = r8; \ + r2 = 0; \ + call %[bpf_ringbuf_discard]; \ +3: r0 = 0; \ + exit; \ +" : + : __imm(bpf_ringbuf_reserve), __imm(bpf_ringbuf_discard), __imm_addr(ringbuf) + : __clobber_all); +} + +SEC("iter/bpf_map_elem") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("header_count is 4 ") +__msg("widening r6 to 0..3 step 1") +__naked void ptr_buf(void) +{ + asm volatile (" \ + r6 = *(u64 *)(r1 + %[value]); \ + if r6 == 0 goto 2f; \ + r7 = r6; \ + r7 += 4; \ +1: r6 += 1; \ + if r6 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm_const(value, offsetof(struct bpf_iter__bpf_map_elem, value)) + : __clobber_all); +} + +/* Separate context loads receive different pointer IDs. */ +SEC("iter/bpf_map_elem") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r6 (buf), r7 (buf)") +__not_msg("widening r6") +__naked void ptr_buf_other_load(void) +{ + asm volatile (" \ + r6 = *(u64 *)(r1 + %[value]); \ + if r6 == 0 goto 2f; \ + r7 = *(u64 *)(r1 + %[value]); \ + if r7 == 0 goto 2f; \ + r7 += 4; \ + r0 = 0; \ +1: r6 += 1; \ + r0 += 1; \ + if r0 & 8 goto 4f; \ + if r6 < r7 goto 1b; \ +4: \ +2: r0 = 0; \ + exit; \ +" : + : __imm_const(value, offsetof(struct bpf_iter__bpf_map_elem, value)) + : __clobber_all); +} + +/* A pointer-versus-scalar latch must not establish an iteration count. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("scev: incompatible latch operands r1 (map_value), scalar 4") +__not_msg("widening r1") +__naked void ptr_vs_scalar_no_widen(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto out_%=; \ + r1 = r0; \ + r0 = 0; \ +loop_%=: \ + r1 += 1; \ + r0 += 1; \ + if r0 & 8 goto out_%=; \ + if r1 < 4 goto loop_%=; \ +out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +/* A nullable pointer must not be widened before its null check. */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 9, can't widen r1, expr is (+ r1 8)") +__naked void no_widen_maybe_null(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64 *)(r10 - 8) = r1; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + r1 = r0; \ + r6 = 0; \ +loop_%=: \ + if r1 == 0 goto exit_%=; \ + r1 += 8; \ + r6 += 1; \ + if r6 < 3 goto loop_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +/* + * A terminating inner loop does not prove termination of the outer loop. + * + * u64 r7 = 0; + * while (bpf_get_prandom_u32() != 42) { + * r7 = (r7 + 1) & 0xf; + * u64 r8 = 0; + * do { + * r8++; + * } while (r8 < 3); + * } + * return 0; + */ +SEC("xdp") +__failure +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("infinite loop detected at insn 6") +__naked void nested_terminating_prune(void) +{ + asm volatile (" \ + r7 = 0; \ +outer_%=: \ + r7 += 1; \ + r7 &= 0xf; \ + call %[bpf_get_prandom_u32]; \ + if r0 == 42 goto out_%=; \ + r8 = 0; \ +inner_%=: \ + r8 += 1; \ + if r8 < 3 goto inner_%=; \ + goto outer_%=; \ +out_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Returning directly from the inner loop to the outer header changes the + * active stack from [1, 4] to [1]. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop at 1{{$}}") +__msg("loop at 4, nested in 1") +__msg("entering loop 1") +__msg("entering loop 4") +__msg("exiting loop 4") +__not_msg("entering loop 1") +__msg("from 4 to 1:") +__msg("loop_stack=1{{$}}") +__naked void nested_loop_backedge_to_outer_header(void) +{ + asm volatile (" \ + r6 = 0; \ +outer_%=: \ + if r6 == 3 goto exit_%=; \ + r6 += 1; \ + r7 = 0; \ +inner_%=: \ + if r7 == 2 goto outer_%=; \ + r7 += 1; \ + goto inner_%=; \ +exit_%=: \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* Induction variable as the src operand of the latch compare. */ +SEC("socket") +__success +__log_level(2) +__msg("loop header at 2, header_count is 10") +__msg("loop header at 2, widening r1 to 0..9 step 1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void post_cond_ivar_src(void) +{ + asm volatile (" \ + r1 = 0; \ + r2 = 10; \ +1: r1 += 1; \ + if r2 != r1 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* Early exit dominates the latch and gives a tighter bound. */ +SEC("socket") +__success +__log_level(2) +__msg(" backedge from 4, latch at 4, dominating exits at 2") +__msg("loop header at 2, header_count is [0..15]") +__msg("loop header at 2, widening r1 to 0..14 step 1") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void dominating_exit_bounds_count(void) +{ + asm volatile (" \ + r1 = 0; \ + r2 = 14; \ +1: if r2 == r1 goto 2f; \ + r1 += 1; \ + if r1 != 63 goto 1b; \ +2: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_sdiv.c b/tools/testing/selftests/bpf/progs/verifier_sdiv.c index 95f3239ce228..d6cbe1e27fd6 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sdiv.c +++ b/tools/testing/selftests/bpf/progs/verifier_sdiv.c @@ -9,7 +9,7 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 + (__clang_major__ >= 18 || defined(__BPF_FEATURE_SDIV_SMOD)) SEC("socket") __description("SDIV32, non-zero imm divisor, check 1") @@ -1271,6 +1271,7 @@ __naked void smod32_int_min_mod_neg2_imm(void) SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_sock.c b/tools/testing/selftests/bpf/progs/verifier_sock.c index bf9f6fb6582c..d8e65938c35c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sock.c +++ b/tools/testing/selftests/bpf/progs/verifier_sock.c @@ -148,7 +148,7 @@ l0_%=: r0 = *(u32*)(r1 + %[bpf_sock_type]); \ SEC("cgroup/skb") __description("bpf_sk_fullsock(skb->sk): no !skb->sk check") -__failure __msg("type=sock_common_or_null expected=sock_common") +__failure __msg("Possibly NULL pointer passed to trusted R1") __failure_unpriv __naked void sk_no_skb_sk_check_1(void) { @@ -504,7 +504,7 @@ l1_%=: r0 = *(u32*)(r0 + %[bpf_sock_rx_queue_mapping__end]);\ SEC("cgroup/skb") __description("bpf_tcp_sock(skb->sk): no !skb->sk check") -__failure __msg("type=sock_common_or_null expected=sock_common") +__failure __msg("Possibly NULL pointer passed to trusted R1") __failure_unpriv __naked void sk_no_skb_sk_check_2(void) { @@ -612,6 +612,54 @@ l1_%=: r0 = *(u64*)(r0 + %[bpf_tcp_sock_bytes_acked__end]);\ } SEC("cgroup/skb") +__description("bpf_tcp_sock(skb->sk): narrow load of tp->bytes_received") +__failure __msg("invalid tcp_sock access") +__failure_unpriv +__naked void skb_sk_tp_bytes_received_narrow(void) +{ + asm volatile (" \ + r1 = *(u64*)(r1 + %[__sk_buff_sk]); \ + if r1 != 0 goto l0_%=; \ + r0 = 0; \ + exit; \ +l0_%=: call %[bpf_tcp_sock]; \ + if r0 != 0 goto l1_%=; \ + exit; \ +l1_%=: r0 = *(u32*)(r0 + %[off]); \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_tcp_sock), + __imm_const(__sk_buff_sk, offsetof(struct __sk_buff, sk)), + __imm_const(off, offsetof(struct bpf_tcp_sock, bytes_received) + 4) + : __clobber_all); +} + +SEC("cgroup/skb") +__description("bpf_tcp_sock(skb->sk): narrow load of tp->bytes_acked") +__failure __msg("invalid tcp_sock access") +__failure_unpriv +__naked void skb_sk_tp_bytes_acked_narrow(void) +{ + asm volatile (" \ + r1 = *(u64*)(r1 + %[__sk_buff_sk]); \ + if r1 != 0 goto l0_%=; \ + r0 = 0; \ + exit; \ +l0_%=: call %[bpf_tcp_sock]; \ + if r0 != 0 goto l1_%=; \ + exit; \ +l1_%=: r0 = *(u32*)(r0 + %[off]); \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_tcp_sock), + __imm_const(__sk_buff_sk, offsetof(struct __sk_buff, sk)), + __imm_const(off, offsetof(struct bpf_tcp_sock, bytes_acked) + 4) + : __clobber_all); +} + +SEC("cgroup/skb") __description("bpf_tcp_sock(bpf_sk_fullsock(skb->sk)): tp->snd_cwnd") __success __success_unpriv __retval(0) __naked void skb_sk_tp_snd_cwnd_2(void) @@ -641,7 +689,7 @@ l2_%=: r0 = *(u32*)(r0 + %[bpf_tcp_sock_snd_cwnd]); \ SEC("tc") __description("bpf_sk_release(skb->sk)") -__failure __msg("release helper bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") __naked void bpf_sk_release_skb_sk(void) { asm volatile (" \ @@ -658,7 +706,7 @@ l0_%=: r0 = 0; \ SEC("tc") __description("bpf_sk_release(bpf_sk_fullsock(skb->sk))") -__failure __msg("release helper bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") __naked void bpf_sk_fullsock_skb_sk(void) { asm volatile (" \ @@ -682,7 +730,7 @@ l1_%=: r1 = r0; \ SEC("tc") __description("bpf_sk_release(bpf_tcp_sock(skb->sk))") -__failure __msg("release helper bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") __naked void bpf_tcp_sock_skb_sk(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c index 39a1766dae3f..4d1374885f6d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c +++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c @@ -1277,6 +1277,8 @@ __description("stack_noperfmon: reject read of invalid slots") __success __caps_unpriv(CAP_BPF) __failure_unpriv __msg_unpriv("invalid read from stack off -8+1 size 8") +__msg_unpriv("Verification failed: Memory Safety: Uninitialized stack read") +__msg_unpriv("Initialize every byte in the stack range before reading it") __naked void stack_noperfmon_reject_invalid_read(void) { asm volatile (" \ @@ -1344,6 +1346,47 @@ __naked void old_imprecise_scalar32_vs_cur_stack_misc(void) : __clobber_all); } +SEC("socket") +__description("stack_noperfmon: reject non-fetch atomic on narrow spill") +__success +__caps_unpriv(CAP_BPF) +__failure_unpriv __msg_unpriv("invalid read from stack off -8+4 size 8") +__naked void stack_noperfmon_reject_atomic_on_narrow_spill(void) +{ + asm volatile ( + "r1 = 1;" + "*(u32 *)(r10 - 8) = r1;" + /* A non-fetch atomic reads all 8 bytes of the slot. */ + "lock *(u64 *)(r10 - 8) += r1;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("stack_noperfmon: reject helper read of narrow spill") +__success +__caps_unpriv(CAP_BPF) +__failure_unpriv __msg_unpriv("invalid read from stack R2 off -8+4 size 8") +__naked void stack_noperfmon_reject_helper_read_of_narrow_spill(void) +{ + asm volatile ( + "r1 = 1;" + "*(u32 *)(r10 - 8) = r1;" + "r1 = %[map_ringbuf] ll;" + "r2 = r10;" + "r2 += -8;" + "r3 = 8;" + "r4 = 0;" + "call %[bpf_ringbuf_output];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_ringbuf_output), + __imm_addr(map_ringbuf) + : __clobber_all); +} + SEC("raw_tp") __success __naked void var_off_write_over_scalar_spill(void) diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c index 7e0ce5db28a0..b9f799f6e405 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c @@ -12,7 +12,8 @@ struct { __type(value, long long); } map_hash_8b SEC(".maps"); -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) __noinline __used @@ -116,8 +117,8 @@ SEC("tc") __description("stack_arg: pruning with different stack arg types") __failure __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) -__msg("arg JOIN insn 9 -> 10 r1: fp0-8 + _ => fp0-8|fp0+0") -__msg("arg JOIN insn 9 -> 10 sa0: fp0-8 + _ => fp0-8|fp0+0") +__msg("arg JOIN insn 8 -> 10 r1: _ + fp0-8 => fp0-8|fp0+0") +__msg("arg JOIN insn 8 -> 10 sa0: _ + fp0-8 => fp0-8|fp0+0") __msg("R{{[0-9]}} invalid mem access 'scalar'") __naked void stack_arg_pruning_type_mismatch(void) { @@ -436,6 +437,7 @@ __naked void stack_arg_sequential_calls(void) SEC("socket") __description("stack_arg is not supported by compiler or jit, use a dummy test") +__skip("stack_arg is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c index c9fe4857da3f..ab1955852233 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c @@ -5,7 +5,8 @@ #include <bpf/bpf_helpers.h> #include "bpf_misc.h" -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) __noinline __used __naked @@ -115,8 +116,8 @@ __naked void stack_arg_pruning_load_after_call(void) /* * "bad_ptr": the first arg is 'long *', which is not a recognized pointer * type for static subprogs (not ctx, dynptr, or tagged). btf_prepare_func_args() - * sets arg_cnt = 7 / stack_arg_cnt = 2, then fails with -EINVAL. The subprog - * is marked unreliable but the call still proceeds for static subprogs. + * sets arg_slot_cnt = 7 / stack_arg_cnt = 2, then fails with -EINVAL. The + * subprog is marked unreliable but the call still proceeds for static subprogs. */ __noinline __used __naked static void subprog_bad_ptr_7args(long *a, int b, int c, int d, int e, int f, int g) @@ -174,6 +175,7 @@ __naked void stack_arg_read_without_write_2(void) SEC("socket") __description("stack_arg order is not supported by compiler or jit, use a dummy test") +__skip("stack_arg order is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c index 8e8cf8232255..7fa7bc207850 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c @@ -235,6 +235,7 @@ __naked void to_stack_check_low_1(void) SEC("socket") __description("PTR_TO_STACK check low 2") +__load_if_no_large_stack() __success __failure_unpriv __msg_unpriv("R1 stack pointer arithmetic goes out of range") __retval(42) @@ -251,7 +252,26 @@ __naked void to_stack_check_low_2(void) } SEC("socket") +__description("PTR_TO_STACK check low 2, large stack") +__load_if_large_stack() +__success __failure_unpriv +__msg_unpriv("R1 stack pointer arithmetic goes out of range") +__retval(42) +__naked void to_stack_check_low_2_large(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2049; \ + r0 = 42; \ + *(u8*)(r1 + 1) = r0; \ + r0 = *(u8*)(r1 + 1); \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") __description("PTR_TO_STACK check low 3") +__load_if_no_large_stack() __failure __msg("invalid write to stack R1 off=-513 size=1") __msg_unpriv("R1 stack pointer arithmetic goes out of range") __naked void to_stack_check_low_3(void) @@ -267,6 +287,23 @@ __naked void to_stack_check_low_3(void) } SEC("socket") +__description("PTR_TO_STACK check low 3, large stack") +__load_if_large_stack() +__failure __msg("invalid write to stack R1 off=-2049 size=1") +__msg_unpriv("R1 stack pointer arithmetic goes out of range") +__naked void to_stack_check_low_3_large(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2049; \ + r0 = 42; \ + *(u8*)(r1 + 0) = r0; \ + r0 = *(u8*)(r1 + 0); \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") __description("PTR_TO_STACK check low 4") __failure __msg("math between fp pointer") __failure_unpriv @@ -483,6 +520,7 @@ l1_%=: r0 = 42; \ SEC("socket") __description("PTR_TO_STACK stack size > 512") +__load_if_no_large_stack() __failure __msg("invalid write to stack R1 off=-520 size=8") __naked void stack_check_size_gt_512(void) { @@ -495,6 +533,21 @@ __naked void stack_check_size_gt_512(void) " ::: __clobber_all); } +SEC("socket") +__description("PTR_TO_STACK stack size > 2048") +__load_if_large_stack() +__failure __msg("invalid write to stack R1 off=-2056 size=8") +__naked void stack_check_size_gt_2048(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -2056; \ + r0 = 42; \ + *(u64*)(r1 + 0) = r0; \ + exit; \ +" ::: __clobber_all); +} + #ifdef __BPF_FEATURE_MAY_GOTO SEC("socket") __description("PTR_TO_STACK stack size 512 with may_goto with jit") @@ -533,4 +586,40 @@ __naked void stack_check_size_512_with_may_goto(void) } #endif +/* + * Verify that old PTR_TO_STACK state is considered a super-set of + * new PTR_TO_STACK state when new variable range is a sub-range + * of the old range, e.g. old [-72, -16] vs new [-64, -16]. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +/* + * r7 is widened to [-72, -16] at the loop header (insn 3), + * the loop body sees [-64, -8] after 'r7 += 8' + */ +__msg("loop header at 3, widening r7 to -72..-16 step 8") +__msg("R7=fp(smin=smin32=-64,smax=smax32=-8") +/* + * back-edge state is clamped to the remaining iterations and pruned + * at the header, because [-64, -16] is within the widened [-72, -16] + */ +__msg("loop header at 3, clamping r7 to -64..-16 (s32 -64..-8) step 8") +__msg("from 5 to 3: safe") +__msg("processed 9 insns") +__naked void stack_ptr_subrange_in_loop(void) +{ + asm volatile (" \ + r7 = r10; \ + r7 += -72; \ + r6 = 0; \ + r7 += 8; \ + r6 += 1; \ + if r6 < 8 goto -3; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_store_release.c b/tools/testing/selftests/bpf/progs/verifier_store_release.c index 72f1eb006074..0abbee5ce109 100644 --- a/tools/testing/selftests/bpf/progs/verifier_store_release.c +++ b/tools/testing/selftests/bpf/progs/verifier_store_release.c @@ -290,6 +290,7 @@ __naked void store_release_with_invalid_reg(void) SEC("socket") __description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test") +__skip("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_tailcall.c b/tools/testing/selftests/bpf/progs/verifier_tailcall.c index b4acce60fb9b..51687da97225 100644 --- a/tools/testing/selftests/bpf/progs/verifier_tailcall.c +++ b/tools/testing/selftests/bpf/progs/verifier_tailcall.c @@ -28,4 +28,61 @@ __naked void invalid_map_for_tail_call(void) : __clobber_all); } +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +__used __naked +static int subprog_tail_call(void) +{ + asm volatile (" \ + r2 = %[jmp_table] ll; \ + r3 = 0; \ + call %[bpf_tail_call]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_tail_call), + __imm_addr(jmp_table) + : __clobber_all); +} + +/* + * A tail call unwinds only the frame of the subprog doing it, so the + * frames of its callers stay on the stack. With up to 33 tail calls in + * a chain the verifier caps the stack those frames may add up to at + * 256 bytes. + */ +SEC("tc") +__description("tail call from subprog with 240 bytes of caller stack") +__success +__naked void tail_call_caller_stack_ok(void) +{ + asm volatile (" \ + r2 = 42; \ + *(u64 *)(r10 - 240) = r2; \ + call subprog_tail_call; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("tc") +__description("tail call from subprog with 256 bytes of caller stack") +__failure +__msg("tail_calls are not allowed when call stack of previous frames is 256 bytes. Too large") +__naked void tail_call_caller_stack_too_large(void) +{ + asm volatile (" \ + r2 = 42; \ + *(u64 *)(r10 - 256) = r2; \ + call subprog_tail_call; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_unpriv.c b/tools/testing/selftests/bpf/progs/verifier_unpriv.c index 3069e70fbcbd..a54b5a58b945 100644 --- a/tools/testing/selftests/bpf/progs/verifier_unpriv.c +++ b/tools/testing/selftests/bpf/progs/verifier_unpriv.c @@ -1035,4 +1035,41 @@ __naked void stack_write_nospec_slot_index(void) " ::: __clobber_all); } +__noinline int nospec_global_func(int x) +{ + return x + 1; +} + +__used __naked static void nospec_call_subprog(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r0 &= 0xff; \ + r1 = 0; \ + if r0 < 0x100 goto l0_%=; \ + /* executed only speculatively */ \ + r1 = r10; \ +l0_%=: call nospec_global_func; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* nospec_global_func is placed in front of nospec_call_subprog */ +SEC("socket") +__description("unpriv: nospec in front of a call") +__success __success_unpriv +__caps_unpriv(CAP_BPF) +__retval(1) +__naked void nospec_backward_call(void) +{ + asm volatile (" \ + r1 = 0; \ + call nospec_global_func; \ + call nospec_call_subprog; \ + exit; \ +" ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_unwind.c b/tools/testing/selftests/bpf/progs/verifier_unwind.c new file mode 100644 index 000000000000..5a21345c34d4 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_unwind.c @@ -0,0 +1,1240 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "../../../include/linux/filter.h" +#include "bpf_misc.h" + +/* + * llvm lowers unwinding to two insns: + * invoke foo unwind pad call foo; unwind_goto pad + * resume unwind_exit + * The tests spell them out as raw insns, so they don't depend on llvm version. + * The argument of unwind_goto() is the number of insns to skip. + */ +#define unwind_goto(off) ".8byte %[insn_unwind_goto_" #off "];" +#define unwind_exit ".8byte %[insn_unwind_exit];" + +#define __imm_unwind_goto(off) \ + __imm_insn(insn_unwind_goto_##off, \ + BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, BPF_UNWIND_GOTO, off, 0)) +#define __imm_unwind_exit \ + __imm_insn(insn_unwind_exit, \ + BPF_RAW_INSN(BPF_JMP | BPF_EXIT, 0, BPF_UNWIND_EXIT, 0, 0)) + +extern void bpf_rcu_read_lock(void) __ksym; +extern void bpf_rcu_read_unlock(void) __ksym; +extern void bpf_throw(__u64 cookie) __ksym; + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(int)); + __uint(value_size, sizeof(int)); +} prog_array SEC(".maps"); + +struct timer_elem { + struct bpf_timer t; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct timer_elem); +} timer_map SEC(".maps"); + +/* The verifier doesn't know what is in there */ +unsigned long zero; +unsigned long one = 1; + +/* + * The verifier follows a static subprog, so it knows which way it leaves when + * it knows the argument. To have both ways at run time the tests pass skb->len, + * which is 64, or skb->len - 64. + * + * r1 == 0: returns 7, otherwise unwinds with r0 = 5 + */ +__naked __noinline __used +static unsigned long may_unwind(void) +{ + asm volatile ( + "if r1 != 0 goto 1f;" + "r0 = 7;" + "exit;" + "1:" + "r0 = 5;" + unwind_exit + : + : __imm_unwind_exit + ); +} + +/* owns nothing: its pad only carries on */ +__naked __noinline __used +static unsigned long passes_on(void) +{ + asm volatile ( + "call may_unwind;" + unwind_goto(2) + "r0 += 1;" + "exit;" + unwind_exit + : + : __imm_unwind_goto(2), + __imm_unwind_exit + ); +} + +__naked __noinline __used +static unsigned long always_unwinds(void) +{ + asm volatile ( + "r0 = 5;" + unwind_exit + : + : __imm_unwind_exit + ); +} + +__naked __noinline __used +static unsigned long never_unwinds(void) +{ + asm volatile ( + "r0 = 7;" + "exit;" + ); +} + +/* Both can unwind. This one returns 7. */ +__naked __noinline __used +unsigned long global_returns(void) +{ + asm volatile ( + "r1 = %[zero] ll;" + "r1 = *(u64 *)(r1 + 0);" + "if r1 != 0 goto 1f;" + "r0 = 7;" + "exit;" + "1:" + "r0 = 5;" + unwind_exit + : + : __imm_addr(zero), + __imm_unwind_exit + ); +} + +/* This one unwinds with r0 = 5 */ +__naked __noinline __used +unsigned long global_unwinds(void) +{ + asm volatile ( + "r1 = %[one] ll;" + "r1 = *(u64 *)(r1 + 0);" + "if r1 != 0 goto 1f;" + "r0 = 7;" + "exit;" + "1:" + "r0 = 5;" + unwind_exit + : + : __imm_addr(one), + __imm_unwind_exit + ); +} + +SEC("tc") +__description("unwind: the callee returns") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int callee_returns(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "r1 -= 64;" + "call may_unwind;" + unwind_goto(1) + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: the callee unwinds") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(42) +__naked int callee_unwinds(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "call may_unwind;" + unwind_goto(1) + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: return through two frames") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(8) +__naked int two_frames_return(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "r1 -= 64;" + "call passes_on;" + unwind_goto(1) + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +/* unwind_exit returns r0 like exit does */ +SEC("tc") +__description("unwind: unwind through two frames, r0 reaches the pad") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(15) +__naked int two_frames_unwind(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "call passes_on;" + unwind_goto(1) + "exit;" + "r0 += 10;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: r6-r9 and the stack are intact in the pad") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(1111) +__naked int pad_sees_callee_saved(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "r6 = 1;" + "r7 = 10;" + "r8 = 100;" + "r9 = 400;" + "*(u64 *)(r10 - 8) = 600;" + "call passes_on;" + unwind_goto(2) + "r0 = 0;" + "exit;" + "r0 = r6;" + "r0 += r7;" + "r0 += r8;" + "r0 += r9;" + "r1 = *(u64 *)(r10 - 8);" + "r0 += r1;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_goto(2) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: r1-r5 are not readable in the pad") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("R3 !read_ok") +__naked int pad_reads_r3(void) +{ + asm volatile ( + "r3 = 0;" + "call always_unwinds;" + unwind_goto(1) + "exit;" + "r0 = r3;" + "exit;" + : + : __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: unwind_exit of the main prog is exit") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(3) +__naked int main_prog_unwinds(void) +{ + asm volatile ( + "r0 = 3;" + unwind_exit + : + : __imm_unwind_exit + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: unwind_exit of the main prog needs r0") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("R0 !read_ok") +__naked int main_prog_unwinds_no_r0(void) +{ + asm volatile ( + unwind_exit + : + : __imm_unwind_exit + : __clobber_all + ); +} + +/* + * What follows unwind_goto is dead code. unwind_exit still has to find the pad, + * and the cpu predicts that the callee returns: there is a trap for that. + */ +SEC("tc") +__description("unwind: the callee never returns") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(42) +__xlated("0: call pc+4") +__xlated("1: unwind_goto pc+1") +__xlated("2: goto pc-1") +__xlated("3: r0 = 42") +__xlated("4: exit") +__naked int callee_never_returns(void) +{ + asm volatile ( + "call always_unwinds;" + unwind_goto(2) + "r0 = 1;" + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_unwind_goto(2) + : __clobber_all + ); +} + +/* The pad is dead code and goes away together with unwind_goto */ +SEC("tc") +__description("unwind: the callee never unwinds") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__xlated("0: call pc+1") +__xlated("1: exit") +__xlated("2: r0 = 7") +__xlated("3: exit") +__naked int callee_never_unwinds(void) +{ + asm volatile ( + "call never_unwinds;" + unwind_goto(1) + "exit;" + "r0 = *(u64 *)(r0 + 0);" + "exit;" + : + : __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: the pad drops what the frame owns") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(42) +__naked int pad_unlocks(void) +{ + asm volatile ( + "r6 = *(u32 *)(r1 + %[len]);" + "call %[bpf_rcu_read_lock];" + "r1 = r6;" + "call may_unwind;" + unwind_goto(3) + "call %[bpf_rcu_read_unlock];" + "r0 = 0;" + "exit;" + "call %[bpf_rcu_read_unlock];" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm(bpf_rcu_read_lock), + __imm(bpf_rcu_read_unlock), + __imm_unwind_goto(3) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: the pad is checked like any other code") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("in main prog cannot be used inside bpf_rcu_read_lock-ed region") +__naked int pad_does_not_unlock(void) +{ + asm volatile ( + "r6 = *(u32 *)(r1 + %[len]);" + "call %[bpf_rcu_read_lock];" + "r1 = r6;" + "call may_unwind;" + unwind_goto(3) + "call %[bpf_rcu_read_unlock];" + "r0 = 0;" + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm(bpf_rcu_read_lock), + __imm(bpf_rcu_read_unlock), + __imm_unwind_goto(3) + : __clobber_all + ); +} + +/* Leaves either way with the same registers */ +__naked __noinline __used +static unsigned long same_regs(void) +{ + asm volatile ( + "r0 = 0;" + "if r1 != 0 goto 1f;" + "exit;" + "1:" + unwind_exit + : + : __imm_unwind_exit + ); +} + +/* + * After the call the two states differ only in the way the callee left. + * The second one must not be pruned, or nobody looks at the pad. + */ +SEC("tc") +__description("unwind: the way the callee left is a part of the state") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("R0 invalid mem access 'scalar'") +__flag(BPF_F_TEST_STATE_FREQ) +__naked int no_pruning_after_the_call(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "call same_regs;" + unwind_goto(1) + "exit;" + "r0 = *(u64 *)(r0 + 0);" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +/* Returns ctx, or unwinds with a number in r0 */ +__naked __noinline __used +static unsigned long ctx_or_unwind(void) +{ + asm volatile ( + "r2 = *(u32 *)(r1 + %[len]);" + "if r2 > 100 goto 1f;" + "r0 = r1;" + "exit;" + "1:" + "r0 = 0;" + unwind_exit + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_exit + ); +} + +/* + * unwind_exit changes its return address, so the cpu mispredicts the return and + * runs what follows unwind_goto with r0 of unwind_exit. + */ +SEC("socket") +__description("unwind: what follows unwind_goto runs speculatively") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(50) /* 64 without the ethernet header */ +__success_unpriv __retval_unpriv(50) +__caps_unpriv(CAP_BPF) +#ifdef SPEC_V1 +__xlated_unpriv("unwind_goto pc+{{.*}}") +__xlated_unpriv("nospec") /* inserted to prevent `R0 invalid mem access 'scalar'` */ +__xlated_unpriv("r0 = *(u32 *)(r0 +{{.*}})") +#endif +__naked int speculative_fall_through(void) +{ + asm volatile ( + "call ctx_or_unwind;" + unwind_goto(2) + "r0 = *(u32 *)(r0 + %[len]);" + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_goto(2) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: a global subprog returns") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int calls_global_returns(void) +{ + asm volatile ( + "call global_returns;" + unwind_goto(1) + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: a global subprog unwinds") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(15) +__naked int calls_global_unwinds(void) +{ + asm volatile ( + "call global_unwinds;" + unwind_goto(1) + "exit;" + "r0 += 10;" + "exit;" + : + : __imm_unwind_goto(1) + : __clobber_all + ); +} + +/* The verifier doesn't look inside */ +SEC("tc") +__description("unwind: the pad of a call of a global subprog is always checked") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("R0 invalid mem access 'scalar'") +__naked int global_pad_is_checked(void) +{ + asm volatile ( + "call global_returns;" + unwind_goto(1) + "exit;" + "r0 = *(u64 *)(r0 + 0);" + "exit;" + : + : __imm_unwind_goto(1) + : __clobber_all + ); +} + +__naked __noinline __used +unsigned long global_never_unwinds(void) +{ + asm volatile ( + "r0 = 7;" + "exit;" + ); +} + +SEC("tc") +__description("unwind: a global subprog that cannot unwind") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int calls_global_never_unwinds(void) +{ + asm volatile ( + "call global_never_unwinds;" + "exit;" + ::: __clobber_all + ); +} + +/* + * The verifier gets to the pad from the first call only, with r6 = 1. The second + * unwind_goto stays, since the pad is not dead code. + */ +SEC("tc") +__description("unwind: a pad shared with a call of a subprog that cannot unwind") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int shared_pad(void) +{ + asm volatile ( + "r6 = 1;" + "call global_returns;" + unwind_goto(4) + "r6 = 2;" + "call global_never_unwinds;" + unwind_goto(1) + "exit;" + "r0 = r6;" + "exit;" + : + : __imm_unwind_goto(4), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: no unwind_goto after a call of a global subprog") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("no unwind_goto after the call of global_returns()") +__naked int global_no_unwind_goto(void) +{ + asm volatile ( + "call global_returns;" + "exit;" + ::: __clobber_all + ); +} + +SEC("tc") +__description("unwind: no unwind_goto after a call of a static subprog") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("unwind_exit, but no unwind_goto after the call at insn 1") +__naked int static_no_unwind_goto(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "call may_unwind;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)) + : __clobber_all + ); +} + +/* The verifier sees that it doesn't get to unwind_exit */ +SEC("tc") +__description("unwind: no unwind_goto is fine when the callee doesn't unwind") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int static_does_not_get_there(void) +{ + asm volatile ( + "r1 = 0;" + "call may_unwind;" + "exit;" + ::: __clobber_all + ); +} + +SEC("tc") +__description("unwind: callx, the callee returns") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int callx_returns(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "r1 -= 64;" + "r6 = %[may_unwind] ll;" + "callx r6;" + unwind_goto(1) + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_addr(may_unwind), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: callx, the callee unwinds") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(42) +__naked int callx_unwinds(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "r6 = %[may_unwind] ll;" + "callx r6;" + unwind_goto(1) + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_addr(may_unwind), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: unwind_goto after a plain insn") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("unwind_goto at insn 1 doesn't follow a call of a subprog") +__naked int unwind_goto_after_mov(void) +{ + asm volatile ( + "r0 = 0;" + unwind_goto(0) + "exit;" + : + : __imm_unwind_goto(0) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: unwind_goto is the first insn") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("unwind_goto at insn 0 doesn't follow a call of a subprog") +__naked int unwind_goto_first(void) +{ + asm volatile ( + unwind_goto(0) + "r0 = 0;" + "exit;" + : + : __imm_unwind_goto(0) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: unwind_goto after a call of a helper") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("unwind_goto at insn 1 doesn't follow a call of a subprog") +__naked int unwind_goto_after_helper(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + unwind_goto(0) + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_unwind_goto(0) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: unwind_goto after a call of a kfunc") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("unwind_goto at insn 1 doesn't follow a call of a subprog") +__naked int unwind_goto_after_kfunc(void) +{ + asm volatile ( + "call %[bpf_rcu_read_lock];" + unwind_goto(0) + "call %[bpf_rcu_read_unlock];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_rcu_read_lock), + __imm(bpf_rcu_read_unlock), + __imm_unwind_goto(0) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: jump to unwind_goto") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("jump to unwind_goto at insn 3") +__naked int jump_to_unwind_goto(void) +{ + asm volatile ( + "r0 = 0;" + "if r1 != 0 goto 1f;" + "call never_unwinds;" + "1:" + unwind_goto(0) + "exit;" + : + : __imm_unwind_goto(0) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: reserved fields of unwind_goto") +__failure __msg("invalid may_goto imm 1") +__naked int unwind_goto_reserved(void) +{ + asm volatile ( + "call never_unwinds;" + ".8byte %[bad];" + "exit;" + : + : __imm_insn(bad, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, BPF_UNWIND_GOTO, 0, 1)) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: unknown kind of jcond") +__failure __msg("invalid may_goto imm 0") +__naked int jcond_unknown(void) +{ + asm volatile ( + "call never_unwinds;" + ".8byte %[bad];" + "exit;" + : + : __imm_insn(bad, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, BPF_UNWIND_GOTO + 1, 0, 0)) + : __clobber_all + ); +} + +SEC("tc") +__description("unwind: unknown kind of exit") +__failure __msg("BPF_EXIT uses reserved fields") +__naked int exit_unknown(void) +{ + asm volatile ( + "r0 = 0;" + ".8byte %[bad];" + : + : __imm_insn(bad, BPF_RAW_INSN(BPF_JMP | BPF_EXIT, 0, BPF_UNWIND_EXIT + 1, 0, 0)) + : __clobber_all + ); +} + +/* r1 is the index: unwinds in the second iteration */ +__naked __noinline __used +static unsigned long cb_unwinds(void) +{ + asm volatile ( + "r0 = 0;" + "if r1 != 0 goto 1f;" + "exit;" + "1:" + unwind_exit + : + : __imm_unwind_exit + ); +} + +SEC("tc") +__description("unwind: callback unwinds") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__failure __msg("callback cannot unwind") +__naked int callback_unwinds(void) +{ + asm volatile ( + "r1 = 3;" + "r2 = %[cb_unwinds];" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "exit;" + : + : __imm_ptr(cb_unwinds), + __imm(bpf_loop) + : __clobber_common + ); +} + +__naked __noinline __used +static unsigned long async_cb_unwinds(void) +{ + asm volatile ( + "r0 = 0;" + unwind_exit + : + : __imm_unwind_exit + ); +} + +/* The kernel calls it. It's the top frame like the main prog: unwind_exit is exit. */ +SEC("tc") +__description("unwind: async callback unwinds") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(0) +int async_callback_unwinds(void *ctx) +{ + struct timer_elem *val; + int key = 0; + + val = bpf_map_lookup_elem(&timer_map, &key); + if (!val) + return 0; + + bpf_timer_init(&val->t, &timer_map, 0); + bpf_timer_set_callback(&val->t, (void *)async_cb_unwinds); + bpf_timer_start(&val->t, 0, 0); + return 0; +} + +/* Not naked: the argument has to have a name in BTF */ +__noinline int exception_cb_unwinds(__u64 cookie) +{ + if (cookie) + asm volatile ("r0 = 7;" unwind_exit :: __imm_unwind_exit : "r0"); + return 0; +} + +SEC("tc") +__description("unwind: exception callback unwinds") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__attribute__((btf_decl_tag("exception_callback:exception_cb_unwinds"))) +__success __retval(7) +int exception_callback_unwinds(struct __sk_buff *skb) +{ + bpf_throw(skb->len); + return 0; +} + +/* What it calls can unwind as long as it doesn't let it out */ +__naked __noinline __used +static unsigned long cb_catches(void) +{ + asm volatile ( + "call may_unwind;" + unwind_goto(0) + "r0 = 0;" + "exit;" + : + : __imm_unwind_goto(0) + ); +} + +SEC("tc") +__description("unwind: callback catches") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(3) +__naked int callback_catches(void) +{ + asm volatile ( + "r1 = 3;" + "r2 = %[cb_catches];" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "exit;" + : + : __imm_ptr(cb_catches), + __imm(bpf_loop) + : __clobber_common + ); +} + +/* Both leave the subprog like exit does */ +__naked __noinline __used +static unsigned long does_tail_call(void) +{ + asm volatile ( + "r6 = *(u32 *)(r1 + %[len]);" + "r2 = %[prog_array] ll;" + "r3 = 0;" + "call %[bpf_tail_call];" + "r0 = 7;" + "if r6 == 64 goto 1f;" + unwind_exit + "1:" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm(bpf_tail_call), + __imm_addr(prog_array), + __imm_unwind_exit + ); +} + +SEC("tc") +__description("unwind: tail call in a subprog that can unwind") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int tail_call_in_subprog(void) +{ + asm volatile ( + "call does_tail_call;" + unwind_goto(1) + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_unwind_goto(1) + : __clobber_all + ); +} + +/* The packet is shorter than that: ld_abs leaves the subprog with r0 = 0 */ +__naked __noinline __used +static unsigned long does_ld_abs(void) +{ + asm volatile ( + "r6 = r1;" + "r0 = *(u8 *)skb[1000];" + "r0 = 5;" + unwind_exit + : + : __imm_unwind_exit + ); +} + +SEC("tc") +__description("unwind: ld_abs in a subprog that can unwind") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int ld_abs_in_subprog(void) +{ + asm volatile ( + "call does_ld_abs;" + unwind_goto(2) + "r0 += 7;" + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_unwind_goto(2) + : __clobber_all + ); +} + +/* + * JITs don't emit anything for unwind_goto. It's an extable entry, where + * unwind_exit looks its return address up. + * With KASAN x86-64 JIT wraps a load in pushes, a call and pops. 'r2 = 0x1234' + * is where the patterns start, so that they don't take that call for the one + * of the subprog. + */ +SEC("tc") +__description("unwind: jit") +__success __retval(42) +__load_if_JITed() +__arch_x86_64 +__jited(" movl $0x1234, %esi") +__jited("...") /* call depth accounting */ +__jited(" callq 0x{{.*}}") +__jited(" movl $0x7, %eax") +__jited("...") +__jited(" movl $0x5, %eax") +__jited(" pushq %rax") +__jited(" pushq %rsi") +__jited(" movq 0x8(%rbp), %rdi") +__jited("...") /* call depth accounting */ +__jited(" callq 0x{{.*}}") +__jited(" movq %rax, 0x8(%rbp)") +__jited(" popq %rsi") +__jited(" popq %rax") +__arch_arm64 +__jited(" mov x1, #0x1234") +__jited(" bl {{.*}}") +__jited(" mov x8, x0") +__jited(" mov x8, #0x7") +__jited("...") +__jited(" mov x8, #0x5") +__jited(" stp x8, x1, [sp, #-0x10]!") +__jited(" ldr x0, [x29, #0x8]") +__jited("...") +__jited(" mov x10, x0") +__jited(" ldp x8, x1, [sp], #0x10") +__jited("...") +__jited(" add x30, x30, x10") +__jited(" ret") +__naked int jit(void) +{ + asm volatile ( + "r1 = *(u32 *)(r1 + %[len]);" + "r2 = 0x1234;" + "call may_unwind;" + unwind_goto(2) + "r0 = 7;" + "exit;" + "r0 = 42;" + "exit;" + : + : __imm_const(len, offsetof(struct __sk_buff, len)), + __imm_unwind_goto(2) + : __clobber_all + ); +} + +/* r1 == 0: returns 7, otherwise unwinds with r0 = 5 */ +__naked __noinline __used +static unsigned long may_unwind_big_stack(void) +{ + asm volatile ( + "*(u64 *)(r10 - 200) = r1;" + "if r1 != 0 goto 1f;" + "r0 = 7;" + "exit;" + "1:" + "r0 = 5;" + unwind_exit + : + : __imm_unwind_exit + ); +} + +/* + * x86-64 keeps the pointer to the private stack in r9 and saves it around a call. + * The value that is read back in the pad is there only when r9 is restored. + */ +SEC("raw_tp") +__description("unwind: private stack, the callee unwinds") +__success __retval(47) +__load_if_JITed() +__arch_x86_64 +__jited(" movl $0x1234, %esi") /* past the load, see above */ +__jited(" pushq %r9") +__jited("...") /* call depth accounting */ +__jited(" callq 0x{{.*}}") +__jited(" popq %r9") +__jited(" jmp L{{.*}}") +__jited(" popq %r9") +__jited(" jmp L{{.*}}") +__arch_arm64 +__naked int private_stack_unwinds(void) +{ + asm volatile ( + "*(u64 *)(r10 - 200) = 42;" + "r1 = %[one] ll;" + "r1 = *(u64 *)(r1 + 0);" + "r2 = 0x1234;" + "call may_unwind_big_stack;" + unwind_goto(1) + "exit;" + "r1 = *(u64 *)(r10 - 200);" + "r0 += r1;" + "exit;" + : + : __imm_addr(one), + __imm_unwind_goto(1) + : __clobber_all + ); +} + +SEC("raw_tp") +__description("unwind: private stack, the callee returns") +__success __retval(49) +__load_if_JITed() +__arch_x86_64 +__arch_arm64 +__naked int private_stack_returns(void) +{ + asm volatile ( + "*(u64 *)(r10 - 200) = 42;" + "r1 = %[zero] ll;" + "r1 = *(u64 *)(r1 + 0);" + "call may_unwind_big_stack;" + unwind_goto(3) + "r1 = *(u64 *)(r10 - 200);" + "r0 += r1;" + "exit;" + "r0 = 0;" + "exit;" + : + : __imm_addr(zero), + __imm_unwind_goto(3) + : __clobber_all + ); +} + +/* + * BTF FUNC records are not generated for kfuncs referenced + * from inline assembly. These records are necessary for + * libbpf to link the program. The function below is a hack + * to ensure that BTF FUNC records are generated. + */ +void __kfunc_btf_root(void) +{ + bpf_rcu_read_lock(); + bpf_rcu_read_unlock(); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c b/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c index 4d8273c258d5..31663338866d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c +++ b/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c @@ -22,8 +22,8 @@ struct { SEC("socket") __description("map element value illegal alu op, 1") -__failure __msg("R0 bitwise operator &= on pointer") -__failure_unpriv +__failure __msg("R0 invalid mem access 'scalar'") +__failure_unpriv __msg_unpriv("R0 bitwise operator &= on pointer") __naked void value_illegal_alu_op_1(void) { asm volatile (" \ @@ -70,8 +70,8 @@ l0_%=: exit; \ SEC("socket") __description("map element value illegal alu op, 3") -__failure __msg("R0 pointer arithmetic with /= operator") -__failure_unpriv +__failure __msg("R0 invalid mem access 'scalar'") +__failure_unpriv __msg_unpriv("R0 pointer arithmetic with /= operator") __naked void value_illegal_alu_op_3(void) { asm volatile (" \ @@ -165,6 +165,171 @@ __naked void map_ptr_illegal_alu_op(void) : __clobber_all); } +SEC("socket") +__description("tag in the low bit of a pointer, and, shift") +__success __retval(0) +__failure_unpriv __msg_unpriv("R1 bitwise operator &= on pointer") +__naked void ptr_tag_and_shift(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r1 = r0; \ + r1 &= 1; \ + r2 = r0; \ + r2 >>= 1; \ + r3 = r0; \ + r3 |= 1; \ + r3 ^= 1; \ + r0 = *(u32*)(r0 + 0); \ + r0 = 0; \ +l0_%=: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("tag in the low bit of a pointer, CAP_BPF without CAP_PERFMON") +__success __retval(0) +__failure_unpriv __msg_unpriv("R1 bitwise operator &= on pointer") +__caps_unpriv(CAP_BPF) +__naked void ptr_tag_cap_bpf(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r1 = r0; \ + r1 &= 1; \ + r0 = 0; \ +l0_%=: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("number op= pointer") +__success __retval(0) +__failure_unpriv __msg_unpriv("R1 pointer arithmetic with *= operator") +__naked void number_mul_ptr(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r1 = 7; \ + r1 *= r0; \ + r0 = 0; \ +l0_%=: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("pointer with the tag cleared is a number") +__failure __msg("R0 invalid mem access 'scalar'") +__failure_unpriv __msg_unpriv("R0 bitwise operator |= on pointer") +__naked void ptr_tag_cleared_deref(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r0 |= 1; \ + r0 ^= 1; \ + r0 = *(u32*)(r0 + 0); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("shift of a pointer that may be NULL") +__failure __msg("R0 pointer arithmetic on map_value_or_null prohibited, null-check it first") +__failure_unpriv +__naked void ptr_or_null_shift(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + r0 >>= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("and of a pointer to map") +__failure __msg("R0 pointer arithmetic on map_ptr prohibited") +__failure_unpriv +__naked void map_ptr_and(void) +{ + asm volatile (" \ + r0 = %[map_hash_48b] ll; \ + r0 &= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("32-bit and of a pointer") +__success __retval(0) +__failure_unpriv __msg_unpriv("R0 32-bit pointer arithmetic prohibited") +__naked void ptr_and32(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + w0 &= 1; \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + SEC("flow_dissector") __description("flow_keys illegal alu op with variable offset") __failure __msg("R7 pointer arithmetic on flow_keys prohibited") diff --git a/tools/testing/selftests/bpf/progs/verifier_var_off.c b/tools/testing/selftests/bpf/progs/verifier_var_off.c index a63e33675091..399884911ea5 100644 --- a/tools/testing/selftests/bpf/progs/verifier_var_off.c +++ b/tools/testing/selftests/bpf/progs/verifier_var_off.c @@ -406,6 +406,7 @@ __naked void zero_sized_access_max_out_of_bound(void) SEC("lwt_in") __description("indirect variable-offset stack access, min out of bound") +__load_if_no_large_stack() __failure __msg("invalid variable-offset read from stack R2") __naked void access_min_out_of_bound(void) { @@ -433,6 +434,37 @@ __naked void access_min_out_of_bound(void) : __clobber_all); } +SEC("lwt_in") +__description("indirect variable-offset stack access, min out of bound, large stack") +__load_if_large_stack() +__failure __msg("invalid variable-offset read from stack R2") +__naked void access_min_out_of_bound_large(void) +{ + asm volatile (" \ + /* Fill the top 8 bytes of the stack */ \ + r2 = 0; \ + *(u64*)(r10 - 8) = r2; \ + /* Get an unknown value */ \ + r2 = *(u32*)(r1 + 0); \ + /* Make it small and 4-byte aligned */ \ + r2 &= 4; \ + r2 -= 2052; \ + /* \ + * add it to fp. We now have either fp-2052 or fp-2048, but\ + * we don't know which \ + */ \ + r2 += r10; \ + /* dereference it indirectly */ \ + r1 = %[map_hash_8b] ll; \ + call %[bpf_map_lookup_elem]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b) + : __clobber_all); +} + SEC("cgroup/skb") __description("indirect variable-offset stack access, min_off < min_initialized") __success diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c index 8f0c45421f89..b5f456d57669 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c @@ -28,7 +28,7 @@ int BPF_PROG(get_task_exe_file_kfunc_null) } SEC("lsm.s/inode_getxattr") -__failure __msg("R1 is fp expected STRUCT task_struct") +__failure __msg("R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_") int BPF_PROG(get_task_exe_file_kfunc_fp) { u64 x; @@ -80,7 +80,7 @@ int BPF_PROG(get_task_exe_file_kfunc_unreleased) } SEC("lsm.s/file_open") -__failure __msg("release kfunc bpf_put_file expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("release function bpf_put_file expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(put_file_kfunc_unacquired, struct file *file) { /* Can't release an unacquired pointer. */ @@ -128,7 +128,7 @@ int BPF_PROG(path_d_path_kfunc_untrusted_from_current) } SEC("lsm.s/file_open") -__failure __msg("kernel function bpf_path_d_path R1 expected pointer to STRUCT path but R1 has a pointer to STRUCT file") +__failure __msg("bpf_path_d_path R1 expected pointer to STRUCT path but R1 has a pointer to STRUCT file") int BPF_PROG(path_d_path_kfunc_type_mismatch, struct file *file) { bpf_path_d_path((struct path *)&file->f_task_work, buf, sizeof(buf)); diff --git a/tools/testing/selftests/bpf/progs/verifier_xadd.c b/tools/testing/selftests/bpf/progs/verifier_xadd.c index 05a0a55adb45..7bde3da2f36e 100644 --- a/tools/testing/selftests/bpf/progs/verifier_xadd.c +++ b/tools/testing/selftests/bpf/progs/verifier_xadd.c @@ -121,4 +121,31 @@ l0_%=: r0 = 42; \ " ::: __clobber_all); } +SEC("tc") +__description("xadd with variable stack offset") +__failure +__msg("variable offset stack pointer cannot be passed into helper function") +__msg("Verification failed: Memory Safety: Variable-offset stack access") +__msg("The instruction would access the stack") +__msg("Use a fixed stack offset for the instruction") +__naked void xadd_variable_stack_offset(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64 *)(r10 - 16) = r1; \ + *(u64 *)(r10 - 8) = r1; \ + call %[bpf_get_prandom_u32]; \ + r0 &= 8; \ + r1 = r10; \ + r1 += -16; \ + r1 += r0; \ + r2 = 1; \ + lock *(u64 *)(r1 + 0) += r2; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/wakeup_source_fail.c b/tools/testing/selftests/bpf/progs/wakeup_source_fail.c index d4d0f1610853..ec4e0f3ff792 100644 --- a/tools/testing/selftests/bpf/progs/wakeup_source_fail.c +++ b/tools/testing/selftests/bpf/progs/wakeup_source_fail.c @@ -42,7 +42,7 @@ int wakeup_source_access_lock_fields(void *ctx) } SEC("syscall") -__failure __msg("release kfunc bpf_wakeup_sources_read_unlock expects referenced PTR_TO_BTF_ID passed to R1") +__failure __msg("R1 type=scalar expected=ptr_, trusted_ptr_, rcu_ptr_") int wakeup_source_unlock_no_lock(void *ctx) { struct bpf_ws_lock *lock = (void *)0x1; diff --git a/tools/testing/selftests/bpf/progs/wq_failures.c b/tools/testing/selftests/bpf/progs/wq_failures.c index 32dc8827e128..bd30217579d4 100644 --- a/tools/testing/selftests/bpf/progs/wq_failures.c +++ b/tools/testing/selftests/bpf/progs/wq_failures.c @@ -48,7 +48,7 @@ __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) __failure __msg(": (85) call bpf_wq_init#") /* anchor message */ -__msg("pointer in R2 isn't map pointer") +__msg("R2 type=fp expected=map_ptr") long test_wq_init_nomap(void *ctx) { struct bpf_wq *wq; @@ -98,7 +98,7 @@ __failure * is a correct bpf_wq pointer. */ __msg(": (85) call bpf_wq_set_callback#") /* anchor message */ -__msg("R1 doesn't point to a map value") +__msg("R1 type=fp expected=map_value") long test_wrong_wq_pointer(void *ctx) { int key = 0; diff --git a/tools/testing/selftests/bpf/test_btf.h b/tools/testing/selftests/bpf/test_btf.h index e65889ab4adf..e7bc78108374 100644 --- a/tools/testing/selftests/bpf/test_btf.h +++ b/tools/testing/selftests/bpf/test_btf.h @@ -7,7 +7,7 @@ #define BTF_END_RAW 0xdeadbeef #define BTF_INFO_ENC(kind, kind_flag, vlen) \ - ((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) + (((__u32)(!!(kind_flag)) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) #define BTF_TYPE_ENC(name, info, size_or_type) \ (name), (info), (size_or_type) diff --git a/tools/testing/selftests/bpf/test_kmod.sh b/tools/testing/selftests/bpf/test_kmod.sh deleted file mode 100755 index 50dca53ac536..000000000000 --- a/tools/testing/selftests/bpf/test_kmod.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0 - -# Usage: -# ./test_kmod.sh [module_param]... -# Ex.: ./test_kmod.sh test_range=1,3 -# All the parameters are passed to the kernel module. - -# Kselftest framework requirement - SKIP code is 4. -ksft_skip=4 - -msg="skip all tests:" -if [ "$(id -u)" != "0" ]; then - echo $msg please run this as root >&2 - exit $ksft_skip -fi - -if [ "$building_out_of_srctree" ]; then - # We are in linux-build/kselftest/bpf - OUTPUT=../../ -else - # We are in linux/tools/testing/selftests/bpf - OUTPUT=../../../../ -fi - -test_run() -{ - sysctl -w net.core.bpf_jit_enable=$1 2>&1 > /dev/null - sysctl -w net.core.bpf_jit_harden=$2 2>&1 > /dev/null - - echo "[ JIT enabled:$1 hardened:$2 ]" - shift 2 - dmesg -C - if [ -f ${OUTPUT}/lib/test_bpf.ko ]; then - insmod ${OUTPUT}/lib/test_bpf.ko "$@" 2> /dev/null - if [ $? -ne 0 ]; then - rc=1 - fi - else - # Use modprobe dry run to check for missing test_bpf module - if ! /sbin/modprobe -q -n test_bpf "$@"; then - echo "test_bpf: [SKIP]" - elif /sbin/modprobe -q test_bpf "$@"; then - echo "test_bpf: ok" - else - echo "test_bpf: [FAIL]" - rc=1 - fi - fi - rmmod test_bpf 2> /dev/null - dmesg | grep FAIL -} - -test_save() -{ - JE=`sysctl -n net.core.bpf_jit_enable` - JH=`sysctl -n net.core.bpf_jit_harden` -} - -test_restore() -{ - sysctl -w net.core.bpf_jit_enable=$JE 2>&1 > /dev/null - sysctl -w net.core.bpf_jit_harden=$JH 2>&1 > /dev/null -} - -rc=0 -test_save -test_run 0 0 "$@" -test_run 1 0 "$@" -test_run 1 1 "$@" -test_run 1 2 "$@" -test_restore -exit $rc diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 9366a3c578f1..6380f52f4dd2 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -17,6 +17,7 @@ #include <linux/in.h> #include <linux/in6.h> #include <linux/un.h> +#include <linux/unaligned.h> #include <linux/filter.h> #include <linux/rcupdate_trace.h> #include <net/sock.h> @@ -75,6 +76,16 @@ union bpf_testmod_union_arg_2 { struct bpf_testmod_struct_arg_2 arg; }; +struct bpf_testmod_oob { + __u8 data; + union { + __u8 redzone_1; + __u16 redzone_2; + __u32 redzone_4; + __u64 redzone_8; + }; +}; + __bpf_hook_start(); noinline int @@ -186,6 +197,43 @@ bpf_testmod_test_int128_arg(__int128 a, int b, long c) bpf_testmod_test_struct_arg_result = (long)a + b + c; return bpf_testmod_test_struct_arg_result; } + +struct bpf_testmod_struct_int128 { + __int128 a; +}; + +/* + * x86-64 puts g in the first stack slot and h, aligned to 16 bytes, in + * the third and fourth, leaving a hole. arm64 leaves x7 empty and puts h + * in the first two stack slots. Either way h and i are not where the BPF + * convention packs them, right after g. + */ +noinline long +bpf_testmod_test_int128_stack(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, + u64 g, struct bpf_testmod_struct_int128 h, u64 i) +{ + bpf_testmod_test_struct_arg_result = a + b * 2 + c * 3 + d * 4 + e * 5 + + f * 6 + g * 7 + (u64)h.a * 8 + + (u64)((unsigned __int128)h.a >> 64) * 9 + i * 10; + return bpf_testmod_test_struct_arg_result; +} + +/* + * x86-64 has no register left for f after e, so f goes to the stack and g + * takes R9 back; h goes to the third stack slot and i, aligned to 16 + * bytes, to the fifth and sixth. arm64 has registers up to g, puts h in + * the first stack slot and i in the third and fourth. + */ +noinline long +bpf_testmod_test_int128_backfill(u64 a, u64 b, u64 c, u64 d, u64 e, + struct bpf_testmod_struct_arg_2 f, u64 g, u64 h, + struct bpf_testmod_struct_int128 i) +{ + bpf_testmod_test_struct_arg_result = a + b * 2 + c * 3 + d * 4 + e * 5 + + f.a * 6 + f.b * 7 + g * 8 + h * 9 + (u64)i.a * 10 + + (u64)((unsigned __int128)i.a >> 64) * 11; + return bpf_testmod_test_struct_arg_result; +} #endif __weak noinline void bpf_testmod_looooooooooooooooooooooooooooooong_name(void) @@ -302,6 +350,15 @@ __bpf_kfunc void bpf_kfunc_trusted_task_test(struct task_struct *ptr) { } +__bpf_kfunc struct task_struct *bpf_kfunc_trusted_null_test(void) +{ + return NULL; +} + +__bpf_kfunc void bpf_kfunc_trusted_nullable_test(struct task_struct *ptr__nullable) +{ +} + __bpf_kfunc void bpf_kfunc_trusted_num_test(int *ptr) { } @@ -336,6 +393,47 @@ __bpf_kfunc void bpf_kfunc_put_default_trusted_ptr_test(struct prog_test_member */ } +#ifdef CONFIG_BPF_JIT_KASAN + +extern void kasan_poison(const void *addr, size_t size, u8 value, bool init); + +#define KASAN_SLAB_FREE 0xFB + +__bpf_kfunc void bpf_kfunc_kasan_poison(void *mem, u32 mem__sz) +{ + kasan_poison(mem, mem__sz, KASAN_SLAB_FREE, false); +} + +__bpf_kfunc void bpf_kfunc_kasan_unpoison(void *mem, u32 mem__sz) +{ + kasan_poison(mem, mem__sz, 0x00, false); +} +#else +__bpf_kfunc void bpf_kfunc_kasan_poison(void *mem, u32 mem__sz) { } +__bpf_kfunc void bpf_kfunc_kasan_unpoison(void *mem, u32 mem__sz) { } +#endif + +__bpf_kfunc struct bpf_testmod_oob *bpf_testmod_oob_alloc(void) +{ + struct bpf_testmod_oob *p; + + /* + * Only allocate size of data (and so, voluntarily use kmalloc + * instead of kmalloc_obj), not the rest of the structure, so + * that programs under test trying to access the rest of the + * structure trigger OoB accesses + */ + p = kmalloc(sizeof(p->data), GFP_ATOMIC); + if (!p) + return NULL; + return p; +} + +__bpf_kfunc void bpf_testmod_oob_free(struct bpf_testmod_oob *oob) +{ + kfree(oob); +} + __bpf_kfunc struct bpf_testmod_ctx * bpf_testmod_ctx_create(int *err) { @@ -408,6 +506,16 @@ static int bpf_testmod_ops3__test_arena_multislot(struct bpf_testmod_arena_pair return 0; } +static int bpf_testmod_ops3__test_arena_member(struct bpf_testmod_arena_member_args *args) +{ + return 0; +} + +static int bpf_testmod_ops3__test_nullable(struct bpf_testmod_nullable_args *args) +{ + return 0; +} + static struct bpf_testmod_ops3 __bpf_testmod_ops3 = { .test_1 = bpf_testmod_test_3, .test_2 = bpf_testmod_test_4, @@ -415,6 +523,8 @@ static struct bpf_testmod_ops3 __bpf_testmod_ops3 = { .test_arena_nullable = bpf_testmod_ops3__test_arena_nullable, .test_arena_stack = bpf_testmod_ops3__test_arena_stack, .test_arena_multislot = bpf_testmod_ops3__test_arena_multislot, + .test_arena_member = bpf_testmod_ops3__test_arena_member, + .test_nullable = bpf_testmod_ops3__test_nullable, }; static void bpf_testmod_test_struct_ops3(void) @@ -455,6 +565,58 @@ __bpf_kfunc int bpf_testmod_ops3_call_test_arena_multislot(u64 *ptr__arena) return st_ops3->test_arena_multislot(p, ptr__arena); } +/* The callback gets the kernel address of @ptr__arena behind each __arena member. */ +__bpf_kfunc int bpf_testmod_ops3_call_test_arena_member(u64 *ptr__arena) +{ + struct bpf_testmod_arena_member_args args = { + .ptr = ptr__arena, + .nested.ptr = ptr__arena, + .u.ptr = ptr__arena, + .cookie = 0xc00c1e, + /* in the first page, which is never mapped */ + .faulting = (void *)16, + }; + + if (!st_ops3 || !st_ops3->test_arena_member) + return -EOPNOTSUPP; + return st_ops3->test_arena_member(&args); +} + +/* + * The members of struct bpf_testmod_nullable_args are trusted the way the + * verifier's BTF_TYPE_SAFE_TRUSTED() lists vmlinux members: through a struct + * of the same name with the __safe_trusted suffix in the BTF of the module. + */ +struct bpf_testmod_nullable_args__safe_trusted { + struct task_struct *task; + struct task_struct *task_or_null; +}; + +static DEFINE_PER_CPU(struct bpf_testmod_pcpu, bpf_testmod_nullable_pcpu); + +/* + * Call test_nullable() with the __nullable members unset, then set. The + * callback returns 1 when it found a task in @task_or_null, so the expected + * result is 2. + */ +__bpf_kfunc int bpf_testmod_ops3_call_test_nullable(void) +{ + struct bpf_testmod_nullable_args args = { .task = current }; + int seven = 7, ret; + + BTF_TYPE_EMIT(struct bpf_testmod_nullable_args__safe_trusted); + + if (!st_ops3 || !st_ops3->test_nullable) + return -EOPNOTSUPP; + args.self = &args; + ret = st_ops3->test_nullable(&args); + args.task_or_null = current; + this_cpu_write(bpf_testmod_nullable_pcpu.val, 0x5ca1ab1e); + args.pcpu_or_null = &bpf_testmod_nullable_pcpu; + args.int_or_null = &seven; + return ret | st_ops3->test_nullable(&args) << 1; +} + struct bpf_testmod_btf_type_tag_1 { int a; }; @@ -631,6 +793,12 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, #ifdef __SIZEOF_INT128__ (void)bpf_testmod_test_int128_ret(i); (void)bpf_testmod_test_int128_arg((__int128)1, 2, 3); + (void)bpf_testmod_test_int128_stack(1, 2, 3, 4, 5, 6, 7, + (struct bpf_testmod_struct_int128){ .a = ((__int128)9 << 64) | 8 }, + 10); + (void)bpf_testmod_test_int128_backfill(1, 2, 3, 4, 5, + (struct bpf_testmod_struct_arg_2){ .a = 6, .b = 7 }, 8, 9, + (struct bpf_testmod_struct_int128){ .a = ((__int128)11 << 64) | 10 }); #endif (void)trace_bpf_testmod_test_raw_tp_null_tp(NULL); @@ -683,6 +851,13 @@ out: EXPORT_SYMBOL(bpf_testmod_test_read); ALLOW_ERROR_INJECTION(bpf_testmod_test_read, ERRNO); +static __always_inline size_t testmod_inline_len(size_t len) +{ + /* Keep an instruction in the inline site without changing len. */ + asm volatile("" : "+m"(len)); + return len; +} + noinline ssize_t bpf_testmod_test_write(struct file *file, struct kobject *kobj, const struct bin_attribute *bin_attr, @@ -691,7 +866,7 @@ bpf_testmod_test_write(struct file *file, struct kobject *kobj, struct bpf_testmod_test_write_ctx ctx = { .buf = buf, .off = off, - .len = len, + .len = testmod_inline_len(len), }; trace_bpf_testmod_test_write_bare_tp(current, &ctx); @@ -750,7 +925,7 @@ static struct testmod_uprobe uprobe = { .consumer.ret_handler = uprobe_ret_handler, }; -static int testmod_register_uprobe(loff_t offset) +static __always_inline int testmod_register_uprobe(loff_t offset) { int err = -EBUSY; @@ -855,6 +1030,8 @@ BTF_ID_FLAGS(func, bpf_kfunc_nested_acquire_zero_offset_test, KF_ACQUIRE) BTF_ID_FLAGS(func, bpf_kfunc_nested_release_test, KF_RELEASE) BTF_ID_FLAGS(func, bpf_kfunc_trusted_vma_test) BTF_ID_FLAGS(func, bpf_kfunc_trusted_task_test) +BTF_ID_FLAGS(func, bpf_kfunc_trusted_null_test, KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_kfunc_trusted_nullable_test) BTF_ID_FLAGS(func, bpf_kfunc_trusted_num_test) BTF_ID_FLAGS(func, bpf_kfunc_rcu_task_test, KF_RCU) BTF_ID_FLAGS(func, bpf_kfunc_ret_rcu_test, KF_RET_NULL | KF_RCU_PROTECTED) @@ -867,8 +1044,14 @@ BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_nullable) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_stack) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_multislot) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_member) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_nullable) BTF_ID_FLAGS(func, bpf_kfunc_get_default_trusted_ptr_test); BTF_ID_FLAGS(func, bpf_kfunc_put_default_trusted_ptr_test); +BTF_ID_FLAGS(func, bpf_kfunc_kasan_poison) +BTF_ID_FLAGS(func, bpf_kfunc_kasan_unpoison) +BTF_ID_FLAGS(func, bpf_testmod_oob_alloc, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_testmod_oob_free, KF_RELEASE) BTF_KFUNCS_END(bpf_testmod_common_kfunc_ids) BTF_ID_LIST(bpf_testmod_dtor_ids) @@ -954,6 +1137,188 @@ __bpf_kfunc int bpf_kfunc_call_test5(u8 a, u16 b, u32 c) return 0; } +#if defined(__x86_64__) || defined(__aarch64__) +__bpf_kfunc __int128 bpf_kfunc_call_test_i128(u64 a, u64 b) +{ + return (__int128)(((unsigned __int128)(a + b) << 64) | (a - b)); +} + +__bpf_kfunc struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(u64 a, u64 b) +{ + struct prog_test_ret_pair r = { .lo = a + b, .hi = a - b }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(u64 a, u64 b) +{ + struct prog_test_ret_pair r = { .lo = a + b, .hi = a - b }; + + return r; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arg(u64 a, struct prog_test_pair_arg s, u64 b) +{ + return a + s.lo * 2 + s.hi * 3 + b * 4; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg(u64 a, u64 b, __int128 v) +{ + return a + b * 2 + (u64)v * 3 + (u64)((unsigned __int128)v >> 64) * 4; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_odd(u64 a, __int128 v, u64 b) +{ + return a + (u64)v * 2 + (u64)((unsigned __int128)v >> 64) * 3 + b * 4; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_odd_many(u64 a, __int128 v, u64 b, u64 c, + u64 d, u64 e, u64 f, __int128 w) +{ + return a + (u64)v * 2 + (u64)((unsigned __int128)v >> 64) * 3 + + b * 4 + c * 5 + d * 6 + e * 7 + f * 8 + + (u64)w * 9 + (u64)((unsigned __int128)w >> 64) * 10; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_stack_hole(u64 a, u64 b, u64 c, u64 d, u64 e, + struct prog_test_pair_arg s, + u64 f, u64 g, + struct prog_test_i128_arg w) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + s.lo * 6 + s.hi * 7 + + f * 8 + g * 9 + (u64)w.v * 10 + + (u64)((unsigned __int128)w.v >> 64) * 11; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_pad(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 f, u64 g, __int128 v) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + + (u64)v * 8 + (u64)((unsigned __int128)v >> 64) * 9; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arg_nofit(u64 a, u64 b, u64 c, u64 d, + struct prog_test_pair_arg s) +{ + return a + b * 2 + c * 3 + d * 4 + s.lo * 5 + s.hi * 6; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arg_tail(u64 a, u64 b, u64 c, u64 d, u64 e, + struct prog_test_pair_arg s, u64 f) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + s.lo * 6 + s.hi * 7 + f * 8; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arg_split8(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 f, u64 g, + struct prog_test_pair_arg s) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + + s.lo * 8 + s.hi * 9; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_ptr_arg(struct prog_test_ptr_arg s) +{ + return s.x; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arena_arg(u64 a, u64 b, u64 c, + struct prog_test_pair_arg s, + u64 *f__arena) +{ + return a + b + c + s.lo + s.hi + *f__arena; +} + +__bpf_kfunc struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(u64 tag) +{ + struct prog_test_ret_ptr r = { .p = NULL, .tag = tag }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arena bpf_kfunc_call_test_ret_arena(u64 addr) +{ + struct prog_test_ret_arena r = { .a = (void *)addr, .b = (void *)(addr + 4) }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arena_mixed bpf_kfunc_call_test_ret_arena_mixed(u64 addr) +{ + struct prog_test_ret_arena_mixed r = { .p = (void *)addr, .tag = 0xbeef }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arena_untagged bpf_kfunc_call_test_ret_arena_untagged(u64 addr) +{ + struct prog_test_ret_arena_untagged r = { .a = (void *)addr, .b = NULL }; + + return r; +} + +__bpf_kfunc union prog_test_ret_arena_union bpf_kfunc_call_test_ret_arena_union(u64 addr) +{ + union prog_test_ret_arena_union r = { .a = (void *)addr }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(u64 tag) +{ + struct prog_test_ret_nested r = { .in = { .p = NULL }, .tag = tag }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_ptr_arr bpf_kfunc_call_test_ret_ptr_arr(void) +{ + struct prog_test_ret_ptr_arr r = { .p = { NULL, NULL } }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arr_struct bpf_kfunc_call_test_ret_arr_struct(void) +{ + struct prog_test_ret_arr_struct r = {}; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arr2d bpf_kfunc_call_test_ret_arr2d(void) +{ + struct prog_test_ret_arr2d r = {}; + + return r; +} + +__bpf_kfunc struct prog_test_ret_deep bpf_kfunc_call_test_ret_deep(u64 v) +{ + struct prog_test_ret_deep r = { .l1 = { .l2 = { .l3 = { .l4 = { .v = v } } } } }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) +{ + struct prog_test_ret_ii r = { .a = a, .b = b }; + + return r; +} +#endif /* __x86_64__ || __aarch64__ */ + +/* + * Takes no argument on purpose: with no arguments there is nothing for the sret + * pointer to displace, so this needs no architecture guard even though it + * returns 24 bytes. + */ +__bpf_kfunc struct prog_test_ret_big bpf_kfunc_call_test_ret_big(void) +{ + struct prog_test_ret_big r = { .a = 1, .b = 2, .c = 3 }; + + return r; +} + __bpf_kfunc u64 bpf_kfunc_call_stack_arg(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, u64 g, u64 h, u64 i, u64 j) @@ -1021,7 +1386,7 @@ __bpf_kfunc u64 bpf_kfunc_call_stack_arg_timer(u64 a, u64 b, u64 c, u64 d, u64 e __bpf_kfunc u64 bpf_kfunc_call_stack_arg_big(u64 a, u64 b, u64 c, u64 d, u64 e, struct prog_test_big_arg s) { - return a + b + c + d + e + s.a + s.b; + return a + b * 2 + c * 3 + d * 4 + e * 5 + s.a * 6 + s.b * 7; } static struct prog_test_ref_kfunc prog_test_struct = { @@ -1102,6 +1467,50 @@ __bpf_kfunc void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p) { } +__bpf_kfunc void bpf_kfunc_test_uninit_struct(struct prog_test_pass1 *out__uninit) +{ + out__uninit->x0 = 1; + out__uninit->x1 = 2; + out__uninit->x2 = 3; + out__uninit->x3 = 4; +} + +__bpf_kfunc void bpf_kfunc_test_uninit_mem(void *out__uninit, u32 out__sz) +{ + memset(out__uninit, 0x2a, out__sz); +} + +__bpf_kfunc void bpf_kfunc_test_uninit_partial(void *out__uninit, u32 out__sz) +{ + if (out__sz) + *(u8 *)out__uninit = 42; +} + +__bpf_kfunc int bpf_kfunc_test_uninit_alias(int *out__uninit, const int *in) +{ + int value = get_unaligned(in); + + put_unaligned(42, out__uninit); + return value; +} + +__bpf_kfunc void bpf_kfunc_test_uninit_multi(int *a__uninit, void *b__uninit, u32 b__sz) +{ + put_unaligned(42, a__uninit); + memset(b__uninit, 0x2a, b__sz); +} + +__bpf_kfunc void bpf_kfunc_test_uninit_pair(struct prog_test_pair_arg p, int *out__uninit) +{ + put_unaligned((int)(p.lo + p.hi), out__uninit); +} + +__bpf_kfunc void bpf_kfunc_test_uninit_stack(u64 a, u64 b, u64 c, u64 d, u64 e, + int *out__uninit) +{ + put_unaligned((int)(a + b + c + d + e), out__uninit); +} + __bpf_kfunc void bpf_kfunc_call_test_fail1(struct prog_test_fail1 *p) { } @@ -1487,6 +1896,34 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test2) BTF_ID_FLAGS(func, bpf_kfunc_call_test3) BTF_ID_FLAGS(func, bpf_kfunc_call_test4) BTF_ID_FLAGS(func, bpf_kfunc_call_test5) +#if defined(__x86_64__) || defined(__aarch64__) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_pair) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_fastcall, KF_FASTCALL) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena_mixed) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena_untagged) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena_union) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_nested) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr_arr) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arr_struct) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arr2d) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_deep) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ii) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_odd) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_odd_many) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_stack_hole) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_pad) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_nofit) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_tail) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_split8) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ptr_arg) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arena_arg) +#endif +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_big) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_ptr) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_mix) @@ -1508,6 +1945,13 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_int_mem_release, KF_RELEASE) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass_ctx) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass1) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass2) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_struct) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_mem) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_partial) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_alias) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_multi) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_pair) +BTF_ID_FLAGS(func, bpf_kfunc_test_uninit_stack) BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail1) BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail2) BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail3) @@ -1578,6 +2022,7 @@ static const struct bpf_verifier_ops bpf_testmod_verifier_ops = { }; static const struct bpf_verifier_ops bpf_testmod_verifier_ops3 = { + .get_func_proto = bpf_base_func_proto, .is_valid_access = bpf_testmod_ops_is_valid_access, }; @@ -1592,6 +2037,8 @@ static int bpf_dummy_reg(void *kdata, struct bpf_link *link) */ if (ops->test_2) ops->test_2(4, ops->data); + if (ops->test_trampoline_stack_args) + ops->test_trampoline_stack_args(1, 2, 3, 4, 5, 6, 7, 8, 9999); return 0; } @@ -1639,6 +2086,13 @@ bpf_testmod_ops__test_return_ref_kptr(int dummy, struct task_struct *task__ref, return NULL; } +static int bpf_testmod_ops__test_trampoline_stack_args(int arg1, int arg2, int arg3, + int arg4, int arg5, int arg6, + int arg7, int arg8, int arg9) +{ + return arg9; +} + static struct bpf_testmod_ops __bpf_testmod_ops = { .test_1 = bpf_testmod_test_1, .test_2 = bpf_testmod_test_2, @@ -1646,6 +2100,7 @@ static struct bpf_testmod_ops __bpf_testmod_ops = { .test_refcounted = bpf_testmod_ops__test_refcounted, .test_refcounted_multi = bpf_testmod_ops__test_refcounted_multi, .test_return_ref_kptr = bpf_testmod_ops__test_return_ref_kptr, + .test_trampoline_stack_args = bpf_testmod_ops__test_trampoline_stack_args, }; struct bpf_struct_ops bpf_bpf_testmod_ops = { diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h index 210b919290cc..4ba9def74404 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h @@ -46,6 +46,11 @@ struct bpf_testmod_ops { struct task_struct *(*test_return_ref_kptr)(int dummy, struct task_struct *task, struct cgroup *cgrp); + /* Used to test indirect struct_ops trampoline with stack-passed arguments. */ + int (*test_trampoline_stack_args)(int arg1, int arg2, int arg3, + int arg4, int arg5, int arg6, + int arg7, int arg8, int arg9); + /* The following fields are used to test shadow copies. */ char onebyte; struct { @@ -109,6 +114,76 @@ struct bpf_testmod_arena_pair { u64 b; }; +/* + * The qualifiers are BTF type tags for the verifier. The module gets __arena + * and __nullable from bpf_compiler_types.h. A BPF program gets __arena from + * bpf_arena_common.h, and the same definition here when it includes this + * header first, and needs __nullable and __percpu empty. User space needs all + * of them empty. + */ +#ifdef __KERNEL__ +#include <linux/bpf_compiler_types.h> +#endif +#if defined(__BPF__) && !defined(__arena) +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) && !defined(BPF_ARENA_FORCE_ASM) +#define __arena __attribute__((address_space(1))) __attribute__((btf_type_tag("arena"))) +#else +#define __arena __attribute__((btf_type_tag("arena"))) +#endif +#endif +#ifndef __arena +#define __arena +#endif +#ifndef __nullable +#define __nullable +#endif +#ifndef __percpu +#define __percpu +#endif + +struct bpf_testmod_pcpu { + int val; +}; + +struct bpf_testmod_arena_nested { + u64 __arena *ptr; +}; + +union bpf_testmod_arena_union { + u64 __arena *ptr; + u64 raw; +}; + +/* Arguments of test_arena_member(), the pointers hold kernel addresses of arena memory */ +struct bpf_testmod_arena_member_args { + /* the value the caller passed, never NULL */ + u64 __arena *ptr; + /* always NULL */ + u64 __arena __nullable *null_ptr; + /* the same value again, behind a nested struct */ + struct bpf_testmod_arena_nested nested; + /* the same value again, but a load from a union is rejected */ + union bpf_testmod_arena_union u; + /* a scalar, loaded by the same insn as @ptr in a verifier test */ + u64 cookie; + /* a bad pointer to a struct like this one, a load through it faults */ + struct bpf_testmod_arena_member_args *faulting; +}; + +/* Arguments of test_nullable(), trusted through the __safe_trusted alias in the module */ +struct bpf_testmod_nullable_args { + /* never NULL */ + struct task_struct *task; + /* NULL on the first call, set on the second */ + struct task_struct __nullable *task_or_null; + /* NULL on the first call, a per-CPU variable on the second */ + struct bpf_testmod_pcpu __percpu __nullable *pcpu_or_null; + /* NULL on the first call, set on the second */ + int __nullable *int_or_null; + /* this struct again, through a plain pointer, which the verifier probes */ + struct bpf_testmod_nullable_args *self; +}; + struct bpf_testmod_ops3 { int (*test_1)(void); int (*test_2)(void); @@ -120,6 +195,10 @@ struct bpf_testmod_ops3 { u64 g, u64 h, u64 *ptr); /* a multi-slot leading arg, so @ptr is not at the slot its arg index suggests */ int (*test_arena_multislot)(struct bpf_testmod_arena_pair p, u64 *ptr); + /* Used to test __arena pointer members. */ + int (*test_arena_member)(struct bpf_testmod_arena_member_args *args); + /* Used to test __nullable pointer members. */ + int (*test_nullable)(struct bpf_testmod_nullable_args *args); }; struct st_ops_args { diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 7d81070eefe7..3b810f1e438f 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -26,6 +26,12 @@ struct prog_test_ref_kfunc { }; #endif +#if __has_attribute(btf_type_tag) +#define __arena_tag __attribute__((btf_type_tag("arena"))) +#else +#define __arena_tag +#endif + struct bpf_iter_testmod_seq; struct prog_test_pass1 { @@ -55,6 +61,100 @@ struct prog_test_big_arg { __u64 b; }; +struct prog_test_pair_arg { /* 16 bytes: two argument registers */ + __u64 lo; + __u64 hi; +}; + +#ifdef __SIZEOF_INT128__ +struct prog_test_i128_arg { /* 16 bytes, aligned to 16 like its member */ + __int128 v; +}; +#endif + +struct prog_test_ptr_arg { /* 16 bytes, but holds a pointer */ + void *p; + __u64 x; +}; + +struct prog_test_ret_pair { /* 16 bytes: R0:R2 */ + __u64 lo; + __u64 hi; +}; + +struct prog_test_ret_ii { /* 8 bytes: R0 only */ + int a; + int b; +}; + +struct prog_test_ret_ptr { /* 16 bytes: contains a pointer */ + void *p; + __u64 tag; +}; + +struct prog_test_ret_arena { /* 16 bytes: two arena pointers */ + void __arena_tag *a; + void __arena_tag *b; +}; + +struct prog_test_ret_arena_mixed { /* 16 bytes: an arena pointer and a scalar */ + void __arena_tag *p; + __u64 tag; +}; + +struct prog_test_ret_arena_untagged { /* 16 bytes: 'b' lacks the arena tag */ + void __arena_tag *a; + void *b; +}; + +union prog_test_ret_arena_union { /* 8 bytes: 'b' lacks the arena tag */ + void __arena_tag *a; + void *b; +}; + +struct prog_test_ret_nested { /* 16 bytes: the pointer hides one level down */ + struct { + void *p; + } in; + __u64 tag; +}; + +struct prog_test_ret_ptr_arr { /* 16 bytes: an array of pointers */ + void *p[2]; +}; + +struct prog_test_ret_arr_struct { /* 16 bytes: the pointer is under an array of structs */ + struct { + struct { + void *p; + } in2; + } in1[2]; +}; + +struct prog_test_ret_arr2d { /* 16 bytes: a two dimensional array of structs */ + struct { + void *p; + } a[1][2]; +}; + +struct prog_test_ret_deep { /* 8 bytes, but nested past the 4-level walk limit */ + struct { + struct { + struct { + struct { + __u64 v; + } l4; + } l3; + } l2; + } l1; +}; + +struct prog_test_ret_big { /* 24 bytes: too large for R0:R2 */ + __u64 a; + __u64 b; + __u64 c; +}; + struct prog_test_fail1 { void *p; int x; @@ -124,6 +224,8 @@ int bpf_testmod_ops3_call_test_arena(__u64 *ptr__arena) __ksym; int bpf_testmod_ops3_call_test_arena_nullable(__u64 *ptr__arena__nullable) __ksym; int bpf_testmod_ops3_call_test_arena_stack(__u64 *ptr__arena) __ksym; int bpf_testmod_ops3_call_test_arena_multislot(__u64 *ptr__arena) __ksym; +int bpf_testmod_ops3_call_test_arena_member(__u64 *ptr__arena) __ksym; +int bpf_testmod_ops3_call_test_nullable(void) __ksym; __u64 bpf_kfunc_call_test1(struct sock *sk, __u32 a, __u64 b, __u32 c, __u64 d) __ksym; @@ -131,6 +233,47 @@ int bpf_kfunc_call_test2(struct sock *sk, __u32 a, __u32 b) __ksym; struct sock *bpf_kfunc_call_test3(struct sock *sk) __ksym; long bpf_kfunc_call_test4(signed char a, short b, int c, long d) __ksym; int bpf_kfunc_call_test5(__u8 a, __u16 b, __u32 c) __ksym; +#ifdef __SIZEOF_INT128__ +__int128 bpf_kfunc_call_test_i128(__u64 a, __u64 b) __ksym; +#endif +struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(__u64 a, __u64 b) __ksym; +struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(__u64 a, __u64 b) __ksym; +struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) __ksym; +__u64 bpf_kfunc_call_test_pair_arg(__u64 a, struct prog_test_pair_arg s, __u64 b) __ksym; +#ifdef __SIZEOF_INT128__ +__u64 bpf_kfunc_call_test_i128_arg(__u64 a, __u64 b, __int128 v) __ksym; +__u64 bpf_kfunc_call_test_i128_arg_odd(__u64 a, __int128 v, __u64 b) __ksym; +__u64 bpf_kfunc_call_test_i128_arg_odd_many(__u64 a, __int128 v, __u64 b, __u64 c, + __u64 d, __u64 e, __u64 f, __int128 w) __ksym; +__u64 bpf_kfunc_call_test_i128_arg_stack_hole(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, struct prog_test_pair_arg s, + __u64 f, __u64 g, + struct prog_test_i128_arg w) __ksym; +__u64 bpf_kfunc_call_test_i128_arg_pad(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 f, __u64 g, __int128 v) __ksym; +#endif +__u64 bpf_kfunc_call_test_pair_arg_nofit(__u64 a, __u64 b, __u64 c, __u64 d, + struct prog_test_pair_arg s) __ksym; +__u64 bpf_kfunc_call_test_pair_arg_tail(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + struct prog_test_pair_arg s, __u64 f) __ksym; +__u64 bpf_kfunc_call_test_pair_arg_split8(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 f, __u64 g, + struct prog_test_pair_arg s) __ksym; +__u64 bpf_kfunc_call_test_ptr_arg(struct prog_test_ptr_arg s) __ksym; +__u64 bpf_kfunc_call_test_pair_arena_arg(__u64 a, __u64 b, __u64 c, + struct prog_test_pair_arg s, + __u64 *f__arena) __ksym; +struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(__u64 tag) __ksym; +struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(__u64 tag) __ksym; +struct prog_test_ret_ptr_arr bpf_kfunc_call_test_ret_ptr_arr(void) __ksym; +struct prog_test_ret_arr_struct bpf_kfunc_call_test_ret_arr_struct(void) __ksym; +struct prog_test_ret_arr2d bpf_kfunc_call_test_ret_arr2d(void) __ksym; +struct prog_test_ret_arena bpf_kfunc_call_test_ret_arena(__u64 addr) __ksym; +struct prog_test_ret_arena_mixed bpf_kfunc_call_test_ret_arena_mixed(__u64 addr) __ksym; +struct prog_test_ret_arena_untagged bpf_kfunc_call_test_ret_arena_untagged(__u64 addr) __ksym; +union prog_test_ret_arena_union bpf_kfunc_call_test_ret_arena_union(__u64 addr) __ksym; +struct prog_test_ret_deep bpf_kfunc_call_test_ret_deep(__u64 v) __ksym; +struct prog_test_ret_big bpf_kfunc_call_test_ret_big(void) __ksym; __u64 bpf_kfunc_call_stack_arg(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, __u64 g, __u64 h, __u64 i, __u64 j) __ksym; @@ -161,6 +304,14 @@ __u64 bpf_kfunc_call_stack_arg_big(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, void bpf_kfunc_call_test_pass_ctx(struct __sk_buff *skb) __ksym; void bpf_kfunc_call_test_pass1(struct prog_test_pass1 *p) __ksym; void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p) __ksym; +void bpf_kfunc_test_uninit_struct(struct prog_test_pass1 *out__uninit) __ksym; +void bpf_kfunc_test_uninit_mem(void *out__uninit, __u32 out__sz) __ksym; +void bpf_kfunc_test_uninit_partial(void *out__uninit, __u32 out__sz) __ksym; +int bpf_kfunc_test_uninit_alias(int *out__uninit, const int *in) __ksym; +void bpf_kfunc_test_uninit_multi(int *a__uninit, void *b__uninit, __u32 b__sz) __ksym; +void bpf_kfunc_test_uninit_pair(struct prog_test_pair_arg p, int *out__uninit) __ksym; +void bpf_kfunc_test_uninit_stack(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + int *out__uninit) __ksym; void bpf_kfunc_call_test_mem_len_fail2(__u64 *mem, int len) __ksym; void bpf_kfunc_call_test_destructive(void) __ksym; @@ -204,6 +355,8 @@ int bpf_kfunc_st_ops_inc10(struct st_ops_args *args) __ksym; void bpf_kfunc_trusted_vma_test(struct vm_area_struct *ptr) __ksym; void bpf_kfunc_trusted_task_test(struct task_struct *ptr) __ksym; +struct task_struct *bpf_kfunc_trusted_null_test(void) __ksym; +void bpf_kfunc_trusted_nullable_test(struct task_struct *ptr__nullable) __ksym; void bpf_kfunc_trusted_num_test(int *ptr) __ksym; void bpf_kfunc_rcu_task_test(struct task_struct *ptr) __ksym; struct task_struct *bpf_kfunc_ret_rcu_test(void) __ksym; diff --git a/tools/testing/selftests/bpf/test_lirc_mode2.sh b/tools/testing/selftests/bpf/test_lirc_mode2.sh deleted file mode 100755 index 5252b91f48a1..000000000000 --- a/tools/testing/selftests/bpf/test_lirc_mode2.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: GPL-2.0 - -# Kselftest framework requirement - SKIP code is 4. -ksft_skip=4 -ret=$ksft_skip - -msg="skip all tests:" -if [ $UID != 0 ]; then - echo $msg please run this as root >&2 - exit $ksft_skip -fi - -GREEN='\033[0;92m' -RED='\033[0;31m' -NC='\033[0m' # No Color - -modprobe rc-loopback - -for i in /sys/class/rc/rc* -do - if grep -q DRV_NAME=rc-loopback $i/uevent - then - LIRCDEV=$(grep DEVNAME= $i/lirc*/uevent | sed sQDEVNAME=Q/dev/Q) - INPUTDEV=$(grep DEVNAME= $i/input*/event*/uevent | sed sQDEVNAME=Q/dev/Q) - fi -done - -if [ -n "$LIRCDEV" ]; -then - TYPE=lirc_mode2 - ./test_lirc_mode2_user $LIRCDEV $INPUTDEV - ret=$? - if [ $ret -ne 0 ]; then - echo -e ${RED}"FAIL: $TYPE"${NC} - else - echo -e ${GREEN}"PASS: $TYPE"${NC} - fi -fi - -exit $ret diff --git a/tools/testing/selftests/bpf/test_lirc_mode2_user.c b/tools/testing/selftests/bpf/test_lirc_mode2_user.c deleted file mode 100644 index cd191da20d14..000000000000 --- a/tools/testing/selftests/bpf/test_lirc_mode2_user.c +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// test ir decoder -// -// Copyright (C) 2018 Sean Young <sean@mess.org> - -// A lirc chardev is a device representing a consumer IR (cir) device which -// can receive infrared signals from remote control and/or transmit IR. -// -// IR is sent as a series of pulses and space somewhat like morse code. The -// BPF program can decode this into scancodes so that rc-core can translate -// this into input key codes using the rc keymap. -// -// This test works by sending IR over rc-loopback, so the IR is processed by -// BPF and then decoded into scancodes. The lirc chardev must be the one -// associated with rc-loopback, see the output of ir-keytable(1). -// -// The following CONFIG options must be enabled for the test to succeed: -// CONFIG_RC_CORE=y -// CONFIG_BPF_RAWIR_EVENT=y -// CONFIG_RC_LOOPBACK=y - -// Steps: -// 1. Open the /dev/lircN device for rc-loopback (given on command line) -// 2. Attach bpf_lirc_mode2 program which decodes some IR. -// 3. Send some IR to the same IR device; since it is loopback, this will -// end up in the bpf program -// 4. bpf program should decode IR and report keycode -// 5. We can read keycode from same /dev/lirc device - -#include <linux/bpf.h> -#include <linux/input.h> -#include <errno.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> -#include <poll.h> -#include <sys/types.h> -#include <sys/ioctl.h> -#include <sys/stat.h> -#include <fcntl.h> - -#include "bpf_util.h" -#include <bpf/bpf.h> -#include <bpf/libbpf.h> - -#include "testing_helpers.h" - -int main(int argc, char **argv) -{ - struct bpf_object *obj; - int ret, lircfd, progfd, inputfd; - int testir1 = 0x1ead; - int testir2 = 0x2101; - u32 prog_ids[10], prog_flags[10], prog_cnt; - - if (argc != 3) { - printf("Usage: %s /dev/lircN /dev/input/eventM\n", argv[0]); - return 2; - } - - ret = bpf_prog_test_load("test_lirc_mode2_kern.bpf.o", - BPF_PROG_TYPE_LIRC_MODE2, &obj, &progfd); - if (ret) { - printf("Failed to load bpf program\n"); - return 1; - } - - lircfd = open(argv[1], O_RDWR | O_NONBLOCK); - if (lircfd == -1) { - printf("failed to open lirc device %s: %m\n", argv[1]); - return 1; - } - - /* Let's try detach it before it was ever attached */ - ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); - if (ret != -ENOENT) { - printf("bpf_prog_detach2 not attached should fail: %m\n"); - return 1; - } - - inputfd = open(argv[2], O_RDONLY | O_NONBLOCK); - if (inputfd == -1) { - printf("failed to open input device %s: %m\n", argv[1]); - return 1; - } - - prog_cnt = 10; - ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, - &prog_cnt); - if (ret) { - printf("Failed to query bpf programs on lirc device: %m\n"); - return 1; - } - - if (prog_cnt != 0) { - printf("Expected nothing to be attached\n"); - return 1; - } - - ret = bpf_prog_attach(progfd, lircfd, BPF_LIRC_MODE2, 0); - if (ret) { - printf("Failed to attach bpf to lirc device: %m\n"); - return 1; - } - - /* Write raw IR */ - ret = write(lircfd, &testir1, sizeof(testir1)); - if (ret != sizeof(testir1)) { - printf("Failed to send test IR message: %m\n"); - return 1; - } - - struct pollfd pfd = { .fd = inputfd, .events = POLLIN }; - struct input_event event; - - for (;;) { - poll(&pfd, 1, 100); - - /* Read decoded IR */ - ret = read(inputfd, &event, sizeof(event)); - if (ret != sizeof(event)) { - printf("Failed to read decoded IR: %m\n"); - return 1; - } - - if (event.type == EV_MSC && event.code == MSC_SCAN && - event.value == 0x1ead) { - break; - } - } - - /* Write raw IR */ - ret = write(lircfd, &testir2, sizeof(testir2)); - if (ret != sizeof(testir2)) { - printf("Failed to send test IR message: %m\n"); - return 1; - } - - for (;;) { - poll(&pfd, 1, 100); - - /* Read decoded IR */ - ret = read(inputfd, &event, sizeof(event)); - if (ret != sizeof(event)) { - printf("Failed to read decoded IR: %m\n"); - return 1; - } - - if (event.type == EV_REL && event.code == REL_Y && - event.value == 1 ) { - break; - } - } - - prog_cnt = 10; - ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, - &prog_cnt); - if (ret) { - printf("Failed to query bpf programs on lirc device: %m\n"); - return 1; - } - - if (prog_cnt != 1) { - printf("Expected one program to be attached\n"); - return 1; - } - - /* Let's try detaching it now it is actually attached */ - ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); - if (ret) { - printf("bpf_prog_detach2: returned %m\n"); - return 1; - } - - return 0; -} diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 07807757b518..824e5912ebcb 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -33,6 +33,7 @@ static inline const char *str_has_pfx(const char *str, const char *pfx) #endif static int sysctl_unpriv_disabled = -1; +static int unpriv_mitigations_disabled = -1; enum mode { PRIV = 1, @@ -44,6 +45,11 @@ enum load_mode { NO_JITED = 1 << 1, }; +enum stack_mode { + LARGE_STACK = 1 << 0, + SMALL_STACK = 1 << 1, +}; + struct test_subspec { char *name; char *description; @@ -69,7 +75,10 @@ struct test_spec { int mode_mask; int arch_mask; int load_mask; + int stack_mask; int linear_sz; + const char *skip_reason; + bool prepare_priv; bool auxiliary; bool valid; }; @@ -422,6 +431,7 @@ static int parse_test_spec(struct test_loader *tester, int err = 0; u32 arch_mask = 0; u32 load_mask = 0; + u32 stack_mask = 0; struct btf *btf; enum arch arch; @@ -456,6 +466,8 @@ static int parse_test_spec(struct test_loader *tester, continue; if ((val = str_has_pfx(s, "test_description="))) { description = val; + } else if ((val = str_has_pfx(s, "test_skip="))) { + spec->skip_reason = val; } else if (strcmp(s, "test_expect_failure") == 0) { spec->priv.expect_failure = true; spec->mode_mask |= PRIV; @@ -496,6 +508,16 @@ static int parse_test_spec(struct test_loader *tester, if (err) goto cleanup; spec->mode_mask |= UNPRIV; + } else if ((msg = str_has_pfx(s, "test_expect_msg_next="))) { + err = __push_msg(msg, true, false, &spec->priv.expect_msgs); + if (err) + goto cleanup; + spec->mode_mask |= PRIV; + } else if ((msg = str_has_pfx(s, "test_expect_msg_next_unpriv="))) { + err = __push_msg(msg, true, false, &spec->unpriv.expect_msgs); + if (err) + goto cleanup; + spec->mode_mask |= UNPRIV; } else if ((msg = str_has_pfx(s, "test_jited="))) { if (arch_mask == 0) { PRINT_FAIL("__jited used before __arch_*"); @@ -568,6 +590,8 @@ static int parse_test_spec(struct test_loader *tester, update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear); } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) { update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear); + } else if (strcmp(val, "BPF_F_ARENA_SCALAR") == 0) { + update_flags(&spec->prog_flags, BPF_F_ARENA_SCALAR, clear); } else /* assume numeric value */ { err = parse_int(val, &flags, "test prog flags"); if (err) @@ -603,6 +627,8 @@ static int parse_test_spec(struct test_loader *tester, if (err) goto cleanup; spec->mode_mask |= UNPRIV; + } else if (strcmp(s, "test_prepare_priv") == 0) { + spec->prepare_priv = true; } else if ((val = str_has_pfx(s, "load_mode="))) { if (strcmp(val, "jited") == 0) { load_mask = JITED; @@ -613,6 +639,16 @@ static int parse_test_spec(struct test_loader *tester, err = -EINVAL; goto cleanup; } + } else if ((val = str_has_pfx(s, "stack_mode="))) { + if (strcmp(val, "large") == 0) { + stack_mask = LARGE_STACK; + } else if (strcmp(val, "small") == 0) { + stack_mask = SMALL_STACK; + } else { + PRINT_FAIL("bad stack spec: '%s'", val); + err = -EINVAL; + goto cleanup; + } } else if ((msg = str_has_pfx(s, "test_expect_stderr="))) { err = push_disasm_msg(msg, &stderr_on_next_line, &spec->priv.stderr); @@ -652,6 +688,7 @@ static int parse_test_spec(struct test_loader *tester, spec->arch_mask = arch_mask ?: -1; spec->load_mask = load_mask ?: (JITED | NO_JITED); + spec->stack_mask = stack_mask ?: (LARGE_STACK | SMALL_STACK); if (spec->mode_mask == 0) spec->mode_mask = PRIV; @@ -745,7 +782,7 @@ static void prepare_case(struct test_loader *tester, struct bpf_object *obj, struct bpf_program *prog) { - int min_log_level = 0, prog_flags; + int min_log_level = 0; if (env.verbosity > VERBOSE_NONE) min_log_level = 1; @@ -763,8 +800,7 @@ static void prepare_case(struct test_loader *tester, else bpf_program__set_log_level(prog, spec->log_level); - prog_flags = bpf_program__flags(prog); - bpf_program__set_flags(prog, prog_flags | spec->prog_flags); + bpf_program__add_flags(prog, spec->prog_flags); tester->log_buf[0] = '\0'; } @@ -972,6 +1008,7 @@ void validate_msgs(const char *log_buf, struct expected_msgs *msgs, no_match = !msg->negative && !match->start; wrong_line = !msg->negative && + match->start && msg->on_next_line && prev_match && prev_match->line + 1 != match->line; unexpected = msg->negative && match->start; @@ -1013,10 +1050,10 @@ struct cap_state { bool initialized; }; -static int drop_capabilities(struct cap_state *caps) +static int drop_capabilities(struct cap_state *caps, __u64 keep_caps) { const __u64 caps_to_drop = (1ULL << CAP_SYS_ADMIN | 1ULL << CAP_NET_ADMIN | - 1ULL << CAP_PERFMON | 1ULL << CAP_BPF); + 1ULL << CAP_PERFMON | 1ULL << CAP_BPF) & ~keep_caps; int err; err = cap_disable_effective(caps_to_drop, &caps->old_caps); @@ -1026,6 +1063,13 @@ static int drop_capabilities(struct cap_state *caps) } caps->initialized = true; + if (keep_caps) { + err = cap_enable_effective(keep_caps, NULL); + if (err) { + PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(-err)); + return err; + } + } return 0; } @@ -1046,8 +1090,12 @@ static int restore_capabilities(struct cap_state *caps) static bool can_execute_unpriv(struct test_loader *tester, struct test_spec *spec) { if (sysctl_unpriv_disabled < 0) - sysctl_unpriv_disabled = get_unpriv_disabled() ? 1 : 0; - if (sysctl_unpriv_disabled) + sysctl_unpriv_disabled = get_unpriv_sysctl_disabled(); + if (sysctl_unpriv_disabled && !(spec->unpriv.caps & (1ULL << CAP_BPF))) + return false; + if (unpriv_mitigations_disabled < 0) + unpriv_mitigations_disabled = get_unpriv_mitigations_disabled(); + if (unpriv_mitigations_disabled) return false; if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) return false; @@ -1314,6 +1362,7 @@ void run_subtest(struct test_loader *tester, { struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv; int current_runtime = is_jit_enabled() ? JITED : NO_JITED; + int current_stack = is_large_stack_supported() ? LARGE_STACK : SMALL_STACK; struct bpf_program *tprog = NULL, *tprog_iter; struct bpf_link *link, *links[32] = {}; struct test_spec *spec_iter; @@ -1327,6 +1376,12 @@ void run_subtest(struct test_loader *tester, if (!test__start_subtest_with_desc(subspec->name, subspec->description)) return; + if (spec->skip_reason) { + printf("%s:SKIP: %s\n", __func__, spec->skip_reason); + test__skip(); + return; + } + if ((get_current_arch() & spec->arch_mask) == 0) { test__skip(); return; @@ -1337,23 +1392,19 @@ void run_subtest(struct test_loader *tester, return; } + if ((current_stack & spec->stack_mask) == 0) { + test__skip(); + return; + } + if (unpriv) { if (!can_execute_unpriv(tester, spec)) { test__skip(); test__end_subtest(); return; } - if (drop_capabilities(&caps)) { - test__end_subtest(); - return; - } - if (subspec->caps) { - err = cap_enable_effective(subspec->caps, NULL); - if (err) { - PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(-err)); - goto subtest_cleanup; - } - } + if (!spec->prepare_priv && drop_capabilities(&caps, subspec->caps)) + goto subtest_cleanup; } /* Implicitly reset to NULL if next test case doesn't specify. @@ -1406,6 +1457,18 @@ void run_subtest(struct test_loader *tester, bpf_object__for_each_map(map, tobj) bpf_map__set_autocreate(map, !unpriv || is_unpriv_capable_map(map)); + if (unpriv && spec->prepare_priv) { + /* + * Module BTF lookup needs CAP_SYS_ADMIN. Allow tests to prepare + * their objects first, then verify programs with the requested caps. + */ + err = bpf_object__prepare(tobj); + if (!ASSERT_OK(err, "obj_prepare")) + goto tobj_cleanup; + if (drop_capabilities(&caps, subspec->caps)) + goto tobj_cleanup; + } + err = bpf_object__load(tobj); if (subspec->expect_failure) { if (!ASSERT_ERR(err, "unexpected_load_success")) { @@ -1483,7 +1546,8 @@ void run_subtest(struct test_loader *tester, } err = do_prog_test_run(bpf_program__fd(tprog), &retval, - bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL ? true : false, + bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL || + bpf_program__type(tprog) == BPF_PROG_TYPE_RAW_TRACEPOINT, spec->linear_sz); if (!err && retval != subspec->retval && subspec->retval != POINTER_VALUE) { PRINT_FAIL("Unexpected retval: %d != %d\n", retval, subspec->retval); diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 46eb201b96a3..168d84cf4eb7 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -2065,8 +2065,11 @@ int main(int argc, char **argv) libbpf_set_print(libbpf_print_fn); err = register_session_key((const char *)test_progs_verification_cert, test_progs_verification_cert_len); - if (err < 0) + if (err < 0) { + fprintf(stderr, "Failed to add libbpf_session_key to the session keyring: %s\n", + strerror(errno)); return err; + } traffic_monitor_set_print(traffic_monitor_print_fn); diff --git a/tools/testing/selftests/bpf/test_progs.h b/tools/testing/selftests/bpf/test_progs.h index ea493c477fbd..adc18f5eceb6 100644 --- a/tools/testing/selftests/bpf/test_progs.h +++ b/tools/testing/selftests/bpf/test_progs.h @@ -349,6 +349,9 @@ void hexdump(const char *prefix, const void *buf, size_t len); ___ok; \ }) +#define ASSERT_TEXT_EQ(actual, expected, name) \ + ASSERT_TRUE(compare_text_to_expected(actual, expected) == 0, name) + #define ASSERT_MEMEQ(actual, expected, len, name) ({ \ static int duration = 0; \ const void *__act = actual; \ diff --git a/tools/testing/selftests/bpf/testing_helpers.c b/tools/testing/selftests/bpf/testing_helpers.c index 9f672d0c9dda..1e3e31ac36eb 100644 --- a/tools/testing/selftests/bpf/testing_helpers.c +++ b/tools/testing/selftests/bpf/testing_helpers.c @@ -8,6 +8,7 @@ #include <sys/mman.h> #include <alloca.h> #include <bpf/bpf.h> +#include <bpf/btf.h> #include <bpf/libbpf.h> #include "disasm.h" #include "test_progs.h" @@ -293,7 +294,6 @@ int bpf_prog_test_load(const char *file, enum bpf_prog_type type, ); struct bpf_object *obj; struct bpf_program *prog; - __u32 flags; int err; obj = bpf_object__open_file(file, &opts); @@ -309,8 +309,7 @@ int bpf_prog_test_load(const char *file, enum bpf_prog_type type, if (type != BPF_PROG_TYPE_UNSPEC && bpf_program__type(prog) != type) bpf_program__set_type(prog, type); - flags = bpf_program__flags(prog) | testing_prog_flags(); - bpf_program__set_flags(prog, flags); + bpf_program__add_flags(prog, testing_prog_flags()); err = bpf_object__load(obj); if (err) @@ -547,6 +546,47 @@ bool is_jit_enabled(void) return enabled; } +/* + * Whether the kernel accepts a program using more than 512 bytes of stack, + * which depends on the JIT in use. Probed once with a program that stores + * at the 2 KiB depth. Only the verifier's verdict on that store is cached: + * a load that fails for another reason, such as a missing capability, is + * reported and probed again on the next call. + */ +bool is_large_stack_supported(void) +{ + static int supported = -1; + struct bpf_insn insns[] = { + BPF_ST_MEM(BPF_DW, BPF_REG_10, -2048, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char log[1024] = {}; + LIBBPF_OPTS(bpf_prog_load_opts, opts, + .log_buf = log, + .log_size = sizeof(log), + .log_level = 1, + ); + int fd; + + if (supported >= 0) + return supported; + + fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, ARRAY_SIZE(insns), + &opts); + if (fd >= 0) { + close(fd); + supported = 1; + } else if (strstr(log, "invalid write to stack")) { + supported = 0; + } else { + fprintf(stderr, "%s: probe failed with errno %d, assuming 512 bytes:\n%s", + __func__, errno, log); + return false; + } + return supported; +} + int stack_mprotect(void) { void *buf; @@ -562,3 +602,90 @@ int stack_mprotect(void) PROT_READ | PROT_WRITE | PROT_EXEC); return ret; } + +int compare_text_to_expected(const char *actual, const char *expected) +{ + char exp_path[] = "/tmp/selftest_expected.XXXXXX"; + char act_path[] = "/tmp/selftest_actual.XXXXXX"; + char buf[512], cmd[128]; + int exp_fd, act_fd; + FILE *p; + + if (!strcmp(actual, expected)) + return 0; + + exp_fd = mkstemp(exp_path); + act_fd = mkstemp(act_path); + if (exp_fd < 0 || act_fd < 0) { + fprintf(stdout, "output differs, no temp file for a diff\n"); + goto out; + } + + dprintf(exp_fd, "%s", expected); + dprintf(act_fd, "%s", actual); + + snprintf(cmd, sizeof(cmd), "diff -u '%s' '%s'", exp_path, act_path); + p = popen(cmd, "r"); + if (!p) { + fprintf(stdout, "output differs, '%s' did not run\n", cmd); + goto out; + } + while (fgets(buf, sizeof(buf), p)) + fputs(buf, stdout); + pclose(p); + +out: + if (exp_fd >= 0) { + close(exp_fd); + unlink(exp_path); + } + if (act_fd >= 0) { + close(act_fd); + unlink(act_path); + } + return -1; +} + +int testmod_member_has_type_tag(const char *struct_name, const char *member, const char *tag) +{ + struct btf *vmlinux_btf, *module_btf = NULL; + const struct btf_member *m; + const struct btf_type *t; + int err = 0, id, i; + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!vmlinux_btf) + return -errno; + module_btf = btf__load_module_btf("bpf_testmod", vmlinux_btf); + if (!module_btf) { + err = -errno; + goto out; + } + id = btf__find_by_name_kind_own(module_btf, struct_name, BTF_KIND_STRUCT); + if (id < 0) { + err = id; + goto out; + } + t = btf__type_by_id(module_btf, id); + err = -ENOENT; + for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { + if (strcmp(btf__name_by_offset(module_btf, m->name_off), member)) + continue; + err = 0; + /* the tag sits on the pointee, behind the pointer and other modifiers */ + for (id = m->type; id; id = t->type) { + t = btf__type_by_id(module_btf, id); + if (btf_is_type_tag(t) && !strcmp(btf__name_by_offset(module_btf, t->name_off), tag)) { + err = 1; + break; + } + if (!btf_is_ptr(t) && !btf_is_mod(t) && !btf_is_typedef(t)) + break; + } + break; + } +out: + btf__free(module_btf); + btf__free(vmlinux_btf); + return err; +} diff --git a/tools/testing/selftests/bpf/testing_helpers.h b/tools/testing/selftests/bpf/testing_helpers.h index 1f83b6080716..36a75158495b 100644 --- a/tools/testing/selftests/bpf/testing_helpers.h +++ b/tools/testing/selftests/bpf/testing_helpers.h @@ -61,6 +61,17 @@ struct bpf_insn; int get_xlated_program(int fd_prog, struct bpf_insn **buf, __u32 *cnt); int testing_prog_flags(void); bool is_jit_enabled(void); +bool is_large_stack_supported(void); int stack_mprotect(void); +/* Runs diff(1) on mismatch */ +int compare_text_to_expected(const char *actual, const char *expected); + +/* + * Whether @member of struct @struct_name in the BTF of bpf_testmod carries + * the BTF type tag @tag: 1 if so, 0 if not, negative on a BTF error or when + * the struct or the member is missing. + */ +int testmod_member_has_type_tag(const char *struct_name, const char *member, const char *tag); + #endif /* __TESTING_HELPERS_H */ diff --git a/tools/testing/selftests/bpf/trace_helpers.c b/tools/testing/selftests/bpf/trace_helpers.c index 679008b310d9..6d7a4b8c4ab8 100644 --- a/tools/testing/selftests/bpf/trace_helpers.c +++ b/tools/testing/selftests/bpf/trace_helpers.c @@ -244,6 +244,45 @@ out: return err; } +unsigned long long module_get_base_addr(const char *module) +{ + unsigned long long addr, base_addr = 0; + char name[500]; + FILE *f; + + f = fopen("/proc/modules", "r"); + if (!f) + return 0; + + while (fscanf(f, "%499s %*s %*s %*s %*s 0x%llx", name, &addr) == 2) { + if (strcmp(module, name) == 0) { + base_addr = addr; + break; + } + } + fclose(f); + return base_addr; +} + +unsigned long long module_get_base_addr_sec(const char *module, const char *sec) +{ + unsigned long long addr = 0; + char *path; + FILE *f; + + if (asprintf(&path, "/sys/module/%s/sections/%s", module, sec) < 0) + return 0; + f = fopen(path, "r"); + free(path); + if (!f) + return 0; + + if (fscanf(f, "%llx", &addr) != 1) + addr = 0; + fclose(f); + return addr; +} + #ifdef PROCMAP_QUERY int env_verbosity __weak = 0; diff --git a/tools/testing/selftests/bpf/trace_helpers.h b/tools/testing/selftests/bpf/trace_helpers.h index 01c8ecc45627..0f8d99746084 100644 --- a/tools/testing/selftests/bpf/trace_helpers.h +++ b/tools/testing/selftests/bpf/trace_helpers.h @@ -38,6 +38,8 @@ typedef int (*ksym_search_cmp_t)(const void *p1, const struct ksym *p2); int load_kallsyms(void); struct ksym *ksym_search(long key); long ksym_get_addr(const char *name); +unsigned long long module_get_base_addr(const char *module); +unsigned long long module_get_base_addr_sec(const char *module, const char *sec); struct ksyms *load_kallsyms_local(void); struct ksym *ksym_search_local(struct ksyms *ksyms, long key); diff --git a/tools/testing/selftests/bpf/unpriv_helpers.c b/tools/testing/selftests/bpf/unpriv_helpers.c index f997d7ec8fd0..c8dd5d848584 100644 --- a/tools/testing/selftests/bpf/unpriv_helpers.c +++ b/tools/testing/selftests/bpf/unpriv_helpers.c @@ -72,8 +72,8 @@ static int config_contains(const char *pat) static bool cmdline_contains(const char *pat) { + int fd, cnt, ret = false; char cmdline[4096], *c; - int fd, ret = false; fd = open("/proc/cmdline", O_RDONLY); if (fd < 0) { @@ -81,14 +81,15 @@ static bool cmdline_contains(const char *pat) return false; } - if (read(fd, cmdline, sizeof(cmdline) - 1) < 0) { + cnt = read(fd, cmdline, sizeof(cmdline) - 1); + if (cnt < 0) { perror("read /proc/cmdline"); goto out; } - cmdline[sizeof(cmdline) - 1] = '\0'; + cmdline[cnt] = '\0'; for (c = strtok(cmdline, " \n"); c; c = strtok(NULL, " \n")) { - if (strncmp(c, pat, strlen(c))) + if (strcmp(c, pat)) continue; ret = true; break; @@ -110,9 +111,8 @@ static int get_mitigations_off(void) return !enabled_in_config; } -bool get_unpriv_disabled(void) +bool get_unpriv_sysctl_disabled(void) { - int mitigations_off; bool disabled; char buf[2]; FILE *fd; @@ -126,8 +126,12 @@ bool get_unpriv_disabled(void) disabled = true; } - if (disabled) - return true; + return disabled; +} + +bool get_unpriv_mitigations_disabled(void) +{ + int mitigations_off; /* * Some unpriv tests rely on spectre mitigations being on. @@ -142,3 +146,18 @@ bool get_unpriv_disabled(void) } return mitigations_off; } + +bool get_unpriv_disabled(void) +{ + return get_unpriv_sysctl_disabled() || get_unpriv_mitigations_disabled(); +} + +bool get_kasan_jit_enabled(void) +{ + return config_contains("CONFIG_BPF_JIT_KASAN=y") == 1; +} + +bool get_kasan_multi_shot_enabled(void) +{ + return cmdline_contains("kasan_multi_shot"); +} diff --git a/tools/testing/selftests/bpf/unpriv_helpers.h b/tools/testing/selftests/bpf/unpriv_helpers.h index 151f67329665..c24d53e14f3a 100644 --- a/tools/testing/selftests/bpf/unpriv_helpers.h +++ b/tools/testing/selftests/bpf/unpriv_helpers.h @@ -5,3 +5,7 @@ #define UNPRIV_SYSCTL "kernel/unprivileged_bpf_disabled" bool get_unpriv_disabled(void); +bool get_unpriv_sysctl_disabled(void); +bool get_unpriv_mitigations_disabled(void); +bool get_kasan_jit_enabled(void); +bool get_kasan_multi_shot_enabled(void); diff --git a/tools/testing/selftests/bpf/usdt_2.c b/tools/testing/selftests/bpf/usdt_2.c index 5e38f8605b02..3b7024b9b08b 100644 --- a/tools/testing/selftests/bpf/usdt_2.c +++ b/tools/testing/selftests/bpf/usdt_2.c @@ -13,6 +13,22 @@ void usdt_2(void) USDT(optimized_attach, usdt_2); } +/* + * Force the nop1,nop10 combo of the USDT probe to a spot where the nop10 + * crosses a page boundary: .balign starts the padding exactly at a page + * start regardless of the compiler-generated prologue size, and the 4086 + * one-byte nops put the nop1 at page offset 4086, so the following nop10 + * occupies the last 9 bytes of that page and 1 byte of the next one. + * The kernel can't optimize such nop10, so libbpf must keep the uprobe + * on the 1-byte nop. + */ +__attribute__((noinline)) +void usdt_2_cross_page(void) +{ + asm volatile (".balign 4096, 0x90\n\t.skip 4086, 0x90"); + USDT(optimized_attach, usdt_2_cross_page); +} + static volatile unsigned long usdt_red_zone_arg1 = 0xDEADBEEF; static volatile unsigned long usdt_red_zone_arg2 = 0xCAFEBABE; static volatile unsigned long usdt_red_zone_arg3 = 0xFEEDFACE; diff --git a/tools/testing/selftests/bpf/verifier/basic_call.c b/tools/testing/selftests/bpf/verifier/basic_call.c index a8c6ab4c1622..0f93c4551f23 100644 --- a/tools/testing/selftests/bpf/verifier/basic_call.c +++ b/tools/testing/selftests/bpf/verifier/basic_call.c @@ -4,7 +4,7 @@ BPF_RAW_INSN(BPF_JMP | BPF_CALL | BPF_X, 0, 0, 0, 0), BPF_EXIT_INSN(), }, - .errstr = "unknown opcode 8d", + .errstr = "R0 !read_ok", .result = REJECT, }, { diff --git a/tools/testing/selftests/bpf/verifier/calls.c b/tools/testing/selftests/bpf/verifier/calls.c index eb6e3baef412..0af237c02ddf 100644 --- a/tools/testing/selftests/bpf/verifier/calls.c +++ b/tools/testing/selftests/bpf/verifier/calls.c @@ -31,7 +31,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "R1 is fp expected STRUCT prog_test_fail1", + .errstr = "R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail1", 2 }, }, @@ -46,7 +46,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "max struct nesting depth exceeded\nR1 is fp expected STRUCT prog_test_fail2", + .errstr = "max struct nesting depth exceeded\nR1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail2", 2 }, }, @@ -61,7 +61,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "R1 is fp expected STRUCT prog_test_fail3", + .errstr = "R1 type=fp expected=ptr_, trusted_ptr_, rcu_ptr_", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail3", 2 }, }, @@ -76,7 +76,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "R1 expected pointer to ctx, but got fp", + .errstr = "R1 type=fp expected=ctx", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_pass_ctx", 2 }, }, @@ -152,7 +152,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "kernel function bpf_kfunc_call_memb1_release R1 expected pointer", + .errstr = "bpf_kfunc_call_memb1_release R1 expected pointer", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_memb_acquire", 1 }, { "bpf_kfunc_call_memb1_release", 5 }, @@ -1037,15 +1037,34 @@ .result = ACCEPT, }, { - "calls: stack overflow using two frames (pre-call access)", + /* + * Five 480-byte frames exceed the 2 KiB budget of JITs with large + * stacks, and two of them the 512 bytes allowed elsewhere. + */ + "calls: stack overflow using five frames (pre-call access)", .insns = { /* prog 1 */ - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), BPF_EXIT_INSN(), /* prog 2 */ - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), + BPF_EXIT_INSN(), + + /* prog 3 */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), + BPF_EXIT_INSN(), + + /* prog 4 */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), + BPF_EXIT_INSN(), + + /* prog 5 */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }, @@ -1054,15 +1073,30 @@ .result = REJECT, }, { - "calls: stack overflow using two frames (post-call access)", + "calls: stack overflow using five frames (post-call access)", .insns = { /* prog 1 */ BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_EXIT_INSN(), /* prog 2 */ - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + + /* prog 3 */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + + /* prog 4 */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + + /* prog 5 */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }, @@ -1127,7 +1161,7 @@ .result = ACCEPT, }, { - "calls: stack depth check using three frames. test3", + "calls: stack depth check using five frames. test3", .insns = { /* main */ BPF_MOV64_REG(BPF_REG_6, BPF_REG_1), @@ -1135,66 +1169,104 @@ BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 8), /* call B */ BPF_JMP_IMM(BPF_JGE, BPF_REG_6, 0, 1), - BPF_ST_MEM(BPF_B, BPF_REG_10, -64, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), /* A */ BPF_JMP_IMM(BPF_JLT, BPF_REG_1, 10, 1), BPF_EXIT_INSN(), - BPF_ST_MEM(BPF_B, BPF_REG_10, -224, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_JMP_IMM(BPF_JA, 0, 0, -3), /* B */ BPF_JMP_IMM(BPF_JGT, BPF_REG_1, 2, 1), - BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, -6), /* call A */ - BPF_ST_MEM(BPF_B, BPF_REG_10, -256, 0), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), /* call C */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + /* C */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2), /* call D */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + /* D */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, -12), /* call A */ + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_EXIT_INSN(), }, .prog_type = BPF_PROG_TYPE_XDP, - /* stack_main=64, stack_A=224, stack_B=256 - * and max(main+A, main+A+B) > 512 + /* + * every frame is 480 bytes, main+A = 960 > 512 and + * max(main+A, main+B+C+D+A) = 2400 > 2048 */ .errstr = "combined stack", .result = REJECT, }, { - "calls: stack depth check using three frames. test4", - /* void main(void) { + "calls: stack depth check using five frames. test4", + /* + * void main(void) { * func1(0); * func1(1); * func2(1); * } - * void func1(int alloc_or_recurse) { + * void funcN(int alloc_or_recurse) { N = 1..4 * if (alloc_or_recurse) { - * frame_pointer[-300] = 1; + * frame_pointer[-480] = 1; * } else { - * func2(alloc_or_recurse); + * funcN+1(alloc_or_recurse); * } * } - * void func2(int alloc_or_recurse) { + * void func5(int alloc_or_recurse) { * if (alloc_or_recurse) { - * frame_pointer[-300] = 1; + * frame_pointer[-480] = 1; * } * } + * main also calls func2 to func5 with 1 so that every function has a + * path allocating its 480 bytes, and the chain adds up to 2400 bytes, + * more than the 2 KiB budget of JITs with large stacks, and to 960 + * bytes after two frames, more than the 512 bytes allowed elsewhere. */ .insns = { /* main */ BPF_MOV64_IMM(BPF_REG_1, 0), - BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 6), /* call A */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 12), /* call A */ BPF_MOV64_IMM(BPF_REG_1, 1), - BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 4), /* call A */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 10), /* call A */ + BPF_MOV64_IMM(BPF_REG_1, 1), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 13), /* call B */ + BPF_MOV64_IMM(BPF_REG_1, 1), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 16), /* call C */ BPF_MOV64_IMM(BPF_REG_1, 1), - BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 7), /* call B */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 19), /* call D */ + BPF_MOV64_IMM(BPF_REG_1, 1), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 22), /* call E */ BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), /* A */ BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2), - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_EXIT_INSN(), BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call B */ BPF_EXIT_INSN(), /* B */ + BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call C */ + BPF_EXIT_INSN(), + /* C */ + BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call D */ + BPF_EXIT_INSN(), + /* D */ + BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), + BPF_EXIT_INSN(), + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call E */ + BPF_EXIT_INSN(), + /* E */ BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 1), - BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0), + BPF_ST_MEM(BPF_B, BPF_REG_10, -480, 0), BPF_EXIT_INSN(), }, .prog_type = BPF_PROG_TYPE_XDP, diff --git a/tools/testing/selftests/bpf/verifier/map_kptr.c b/tools/testing/selftests/bpf/verifier/map_kptr.c index 4b39f8472f9b..345cecc722a3 100644 --- a/tools/testing/selftests/bpf/verifier/map_kptr.c +++ b/tools/testing/selftests/bpf/verifier/map_kptr.c @@ -311,7 +311,7 @@ .prog_type = BPF_PROG_TYPE_SCHED_CLS, .fixup_map_kptr = { 1 }, .result = REJECT, - .errstr = "R1 type=rcu_ptr_or_null_ expected=percpu_ptr_", + .errstr = "Possibly NULL pointer passed to trusted R1", }, { "map_kptr: ref: reject off != 0", @@ -342,7 +342,7 @@ .prog_type = BPF_PROG_TYPE_SCHED_CLS, .fixup_map_kptr = { 1 }, .result = REJECT, - .errstr = "invalid kptr access, R2 type=ptr_prog_test_ref_kfunc expected=ptr_prog_test_member", + .errstr = "R2 must have zero offset when passed to release func", }, { "map_kptr: ref: reference state created and released on xchg", diff --git a/tools/testing/selftests/bpf/verify_sig_setup.sh b/tools/testing/selftests/bpf/verify_sig_setup.sh index 09179fb551f0..ba5921e4b12d 100755 --- a/tools/testing/selftests/bpf/verify_sig_setup.sh +++ b/tools/testing/selftests/bpf/verify_sig_setup.sh @@ -28,7 +28,7 @@ authorityKeyIdentifier=keyid usage() { - echo "Usage: $0 <setup|cleanup <existing_tmp_dir>" + echo "Usage: $0 <setup-rsa|setup-mldsa|cleanup <existing_tmp_dir>" exit 1 } @@ -47,7 +47,7 @@ genkey() ${tmp_dir}/signing_key.der -outform der } -setup() +setup_rsa() { local tmp_dir="$1" @@ -57,6 +57,57 @@ setup() keyctl link $key_id $keyring_id } +mldsa_supported() +{ + local tmp_dir="$1" + + genkey_mldsa "${tmp_dir}" || return 1 + : > ${tmp_dir}/probe + # Same digest as the caller signs with, see sign_buf_digest(). + ./sign-file -d sha512 ${tmp_dir}/signing_key.pem \ + ${tmp_dir}/signing_key.pem ${tmp_dir}/probe || return 1 + rm -f ${tmp_dir}/probe ${tmp_dir}/probe.p7s +} + +genkey_mldsa() +{ + local tmp_dir="$1" + + echo "${x509_genkey_content}" > ${tmp_dir}/x509.genkey + + # No -<digest> here: ML-DSA hashes the message itself, so openssl + # ignores an explicit digest for it. + openssl req -new -nodes -utf8 -days 36500 \ + -batch -x509 -newkey ML-DSA-87 \ + -config ${tmp_dir}/x509.genkey \ + -outform PEM -out ${tmp_dir}/signing_key.pem \ + -keyout ${tmp_dir}/signing_key.pem 2>&1 + + openssl x509 -in ${tmp_dir}/signing_key.pem -out \ + ${tmp_dir}/signing_key.der -outform der +} + +mldsa_skip() +{ + local tmp_dir="$1" + + rm -f ${tmp_dir}/x509.genkey ${tmp_dir}/signing_key.pem \ + ${tmp_dir}/signing_key.der ${tmp_dir}/probe \ + ${tmp_dir}/probe.p7s + exit 77 +} + +setup_mldsa() +{ + local tmp_dir="$1" + + mldsa_supported "${tmp_dir}" || mldsa_skip "${tmp_dir}" + key_id=$(cat ${tmp_dir}/signing_key.der | + keyctl padd asymmetric ebpf_testing_key @s) + keyring_id=$(keyctl newring ebpf_testing_keyring @s) + keyctl link $key_id $keyring_id +} + cleanup() { local tmp_dir="$1" @@ -91,7 +142,7 @@ catch() local exit_code="$1" local log_file="$2" - if [[ "${exit_code}" -ne 0 ]]; then + if [[ "${exit_code}" -ne 0 && "${exit_code}" -ne 77 ]]; then cat "${log_file}" >&3 fi @@ -108,8 +159,10 @@ main() [[ ! -d "${tmp_dir}" ]] && echo "Directory ${tmp_dir} doesn't exist" && exit 1 - if [[ "${action}" == "setup" ]]; then - setup "${tmp_dir}" + if [[ "${action}" == "setup-rsa" ]]; then + setup_rsa "${tmp_dir}" + elif [[ "${action}" == "setup-mldsa" ]]; then + setup_mldsa "${tmp_dir}" elif [[ "${action}" == "genkey" ]]; then genkey "${tmp_dir}" elif [[ "${action}" == "cleanup" ]]; then diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c index e70741c6b9b7..9cfc9b4b41c1 100644 --- a/tools/testing/selftests/bpf/veristat.c +++ b/tools/testing/selftests/bpf/veristat.c @@ -1722,9 +1722,9 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf fixup_obj(obj, prog, base_filename); if (env.force_checkpoints) - bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_STATE_FREQ); + bpf_program__add_flags(prog, BPF_F_TEST_STATE_FREQ); if (env.force_reg_invariants) - bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_REG_INVARIANTS); + bpf_program__add_flags(prog, BPF_F_TEST_REG_INVARIANTS); opts.log_buf = buf; opts.log_size = buf_sz; diff --git a/tools/testing/selftests/bpf/vmtest.sh b/tools/testing/selftests/bpf/vmtest.sh index 6a3d026d76bd..c4663cad74f5 100755 --- a/tools/testing/selftests/bpf/vmtest.sh +++ b/tools/testing/selftests/bpf/vmtest.sh @@ -35,11 +35,11 @@ aarch64) ARCH="arm64" ;; riscv64) - # required qemu version v7.2.0+ + # required qemu version v8.2.0+ QEMU_BINARY=qemu-system-riscv64 QEMU_CONSOLE="ttyS0,115200" HOST_FLAGS=(-M virt -cpu host -enable-kvm -smp 8) - CROSS_FLAGS=(-M virt -cpu rv64,sscofpmf=true -smp 8) + CROSS_FLAGS=(-M virt -cpu max,sscofpmf=false -smp 8) BZIMAGE="arch/riscv/boot/Image" ARCH="riscv" ;; @@ -65,7 +65,7 @@ OUTPUT_DIR="$HOME/.bpf_selftests" KCONFIG_REL_PATHS=("tools/testing/selftests/bpf/config" "tools/testing/selftests/bpf/config.vm" "tools/testing/selftests/bpf/config.${PLATFORM}") -INDEX_URL="https://raw.githubusercontent.com/libbpf/ci/master/INDEX" +INDEX_URL="https://raw.githubusercontent.com/libbpf/ci/main/INDEX" NUM_COMPILE_JOBS="$(nproc)" LOG_FILE_BASE="$(date +"bpf_selftests.%Y-%m-%d_%H-%M-%S")" LOG_FILE="${LOG_FILE_BASE}.log" @@ -107,6 +107,14 @@ Options: -s) Instead of powering off the VM, start an interactive shell. If <command> is specified, the shell runs after the command finishes executing + +Environment variables: + + KERNEL_CMDLINE_EXTRA + Extra arguments to append to the guest kernel command + line, for tests that need a boot-time setting. e.g: + + KERNEL_CMDLINE_EXTRA="bpf.keyring_unsealed=1" $0 -- ./test_progs -t signed_loader EOF } @@ -286,6 +294,12 @@ EOF QEMU_FLAGS=("${HOST_FLAGS[@]}") fi + local kernel_cmdline="root=/dev/vda rw console=${QEMU_CONSOLE}" + + if [[ -n "${KERNEL_CMDLINE_EXTRA:-}" ]]; then + kernel_cmdline+=" ${KERNEL_CMDLINE_EXTRA}" + fi + ${QEMU_BINARY} \ -nodefaults \ -display none \ @@ -294,7 +308,7 @@ EOF -m 4G \ -drive file="${rootfs_img}",format=raw,index=1,media=disk,if=virtio,cache=none \ -kernel "${kernel_bzimage}" \ - -append "root=/dev/vda rw console=${QEMU_CONSOLE}" + -append "${kernel_cmdline}" } copy_logs() diff --git a/tools/testing/selftests/bpf/xdp_hw_metadata.c b/tools/testing/selftests/bpf/xdp_hw_metadata.c index 6db3b5555a22..ab3f5fc56e86 100644 --- a/tools/testing/selftests/bpf/xdp_hw_metadata.c +++ b/tools/testing/selftests/bpf/xdp_hw_metadata.c @@ -845,7 +845,7 @@ int main(int argc, char *argv[]) prog = bpf_object__find_program_by_name(bpf_obj->obj, "rx"); bpf_program__set_ifindex(prog, ifindex); - bpf_program__set_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__add_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY); printf("load bpf program...\n"); ret = xdp_hw_metadata__load(bpf_obj); |