diff options
Diffstat (limited to 'tools/testing/selftests/bpf')
531 files changed, 44075 insertions, 4693 deletions
diff --git a/tools/testing/selftests/bpf/.gitignore b/tools/testing/selftests/bpf/.gitignore index a3ea98211ea6..986a6389186b 100644 --- a/tools/testing/selftests/bpf/.gitignore +++ b/tools/testing/selftests/bpf/.gitignore @@ -21,7 +21,6 @@ test_lirc_mode2_user flow_dissector_load test_tcpnotify_user test_libbpf -xdping test_cpp *.d *.subskel.h @@ -47,3 +46,5 @@ verification_cert.h *.BTF *.BTF_ids *.BTF.base +usdt_1 +usdt_2 diff --git a/tools/testing/selftests/bpf/DENYLIST.riscv64 b/tools/testing/selftests/bpf/DENYLIST.riscv64 index 4fc4dfdde293..ca1beae7fe8f 100644 --- a/tools/testing/selftests/bpf/DENYLIST.riscv64 +++ b/tools/testing/selftests/bpf/DENYLIST.riscv64 @@ -1,3 +1,2 @@ # riscv64 deny list for BPF CI and local vmtest exceptions # JIT does not support exceptions -tailcalls/tailcall_bpf2bpf* # JIT does not support mixing bpf2bpf and tailcalls diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 65485967c968..93c707116fad 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -3,7 +3,6 @@ include ../../../build/Build.include include ../../../scripts/Makefile.arch include ../../../scripts/Makefile.include -CXX ?= $(CROSS_COMPILE)g++ OBJCOPY ?= $(CROSS_COMPILE)objcopy CURDIR := $(abspath .) @@ -41,6 +40,14 @@ LIBELF_LIBS := $(shell $(PKG_CONFIG) libelf --libs 2>/dev/null || echo -lelf) SKIP_DOCS ?= SKIP_LLVM ?= +SKIP_LIBBFD ?= +SKIP_CRYPTO ?= + +# When BPF_STRICT_BUILD is 1, any BPF object, skeleton, test object, or +# benchmark compilation failure is fatal. Set to 0 to tolerate failures +# and continue building the remaining tests. +BPF_STRICT_BUILD ?= 1 +PERMISSIVE := $(filter 0,$(BPF_STRICT_BUILD)) ifeq ($(srctree),) srctree := $(patsubst %/,%,$(dir $(CURDIR))) @@ -49,19 +56,20 @@ srctree := $(patsubst %/,%,$(dir $(srctree))) srctree := $(patsubst %/,%,$(dir $(srctree))) endif -CFLAGS += -g $(OPT_FLAGS) -rdynamic -std=gnu11 \ +COMMON_CFLAGS = -g $(OPT_FLAGS) -rdynamic -std=gnu11 \ -Wall -Werror -fno-omit-frame-pointer \ -Wno-unused-but-set-variable \ $(GENFLAGS) $(SAN_CFLAGS) $(LIBELF_CFLAGS) \ -I$(CURDIR) -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ - -I$(TOOLSINCDIR) -I$(TOOLSARCHINCDIR) -I$(APIDIR) -I$(OUTPUT) + -I$(TOOLSINCDIR) -I$(TOOLSARCHINCDIR) -I$(APIDIR) -I$(OUTPUT) \ + -I$(CURDIR)/libarena/include LDFLAGS += $(SAN_LDFLAGS) LDLIBS += $(LIBELF_LIBS) -lz -lrt -lpthread PCAP_CFLAGS := $(shell $(PKG_CONFIG) --cflags libpcap 2>/dev/null && echo "-DTRAFFIC_MONITOR=1") PCAP_LIBS := $(shell $(PKG_CONFIG) --libs libpcap 2>/dev/null) LDLIBS += $(PCAP_LIBS) -CFLAGS += $(PCAP_CFLAGS) +CFLAGS += $(COMMON_CFLAGS) $(PCAP_CFLAGS) # Some utility functions use LLVM libraries jit_disasm_helpers.c-CFLAGS = $(LLVM_CFLAGS) @@ -76,6 +84,12 @@ ifneq ($(shell $(CLANG) --target=bpf -mcpu=help 2>&1 | grep 'v4'),) CLANG_CPUV4 := 1 endif +# Check whether clang supports BPF address sanitizer (requires LLVM 22+) +CLANG_HAS_ARENA_ASAN := $(shell echo 'int x;' | \ + $(CLANG) --target=bpf -fsanitize=kernel-address \ + -mllvm -asan-shadow-addr-space=1 \ + -x c -c - -o /dev/null 2>/dev/null && echo 1) + # Order correspond to 'make run_tests' order TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_progs \ test_sockmap \ @@ -109,16 +123,13 @@ TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c) # Order correspond to 'make run_tests' order TEST_PROGS := test_kmod.sh \ test_lirc_mode2.sh \ - test_xdping.sh \ test_bpftool_build.sh \ - test_bpftool.sh \ test_doc_build.sh \ test_xsk.sh \ test_xdp_features.sh TEST_PROGS_EXTENDED := \ - ima_setup.sh verify_sig_setup.sh \ - test_bpftool.py + ima_setup.sh verify_sig_setup.sh TEST_KMODS := bpf_testmod.ko bpf_test_no_cfi.ko bpf_test_modorder_x.ko \ bpf_test_modorder_y.ko bpf_test_rqspinlock.ko @@ -134,7 +145,6 @@ TEST_GEN_PROGS_EXTENDED = \ xdp_features \ xdp_hw_metadata \ xdp_synproxy \ - xdping \ xskxceiver TEST_GEN_FILES += $(TEST_KMODS) liburandom_read.so urandom_read sign-file uprobe_multi @@ -153,12 +163,13 @@ override define CLEAN $(Q)$(RM) -r $(TEST_KMODS) $(Q)$(RM) -r $(EXTRA_CLEAN) $(Q)$(MAKE) -C test_kmods clean + $(Q)$(MAKE) -C libarena clean $(Q)$(MAKE) docs-clean endef include ../lib.mk -NON_CHECK_FEAT_TARGETS := clean docs-clean +NON_CHECK_FEAT_TARGETS := clean docs-clean emit_tests CHECK_FEAT := $(filter-out $(NON_CHECK_FEAT_TARGETS),$(or $(MAKECMDGOALS), "none")) ifneq ($(CHECK_FEAT),) FEATURE_USER := .selftests @@ -182,8 +193,15 @@ ifeq ($(feature-llvm),1) LLVM_CONFIG_LIB_COMPONENTS := mcdisassembler all-targets # both llvm-config and lib.mk add -D_GNU_SOURCE, which ends up as conflict LLVM_CFLAGS += $(filter-out -D_GNU_SOURCE,$(shell $(LLVM_CONFIG) --cflags)) - # Prefer linking statically if it's available, otherwise fallback to shared - ifeq ($(shell $(LLVM_CONFIG) --link-static --libs >/dev/null 2>&1 && echo static),static) + # Cross compilation must use dynamic linking to avoid unresolved library + # dependencies. For native build, prefer linking statically if it's + # available, otherwise fallback to shared. + ifneq ($(ARCH), $(HOSTARCH)) + LLVM_LINK_STATIC := + else + LLVM_LINK_STATIC := $(shell $(LLVM_CONFIG) --link-static --libs >/dev/null 2>&1 && echo y) + endif + ifeq ($(LLVM_LINK_STATIC),y) LLVM_LDLIBS += $(shell $(LLVM_CONFIG) --link-static --libs $(LLVM_CONFIG_LIB_COMPONENTS)) LLVM_LDLIBS += $(filter-out -lxml2,$(shell $(LLVM_CONFIG) --link-static --system-libs $(LLVM_CONFIG_LIB_COMPONENTS))) LLVM_LDLIBS += -lstdc++ @@ -255,7 +273,7 @@ endif $(OUTPUT)/liburandom_read.so: urandom_read_lib1.c urandom_read_lib2.c liburandom_read.map $(call msg,LIB,,$@) $(Q)$(CLANG) $(CLANG_TARGET_ARCH) \ - $(filter-out -static,$(CFLAGS) $(LDFLAGS)) \ + $(filter-out -static,$(COMMON_CFLAGS) $(LDFLAGS)) \ $(filter %.c,$^) $(filter-out -static,$(LDLIBS)) \ -Wno-unused-command-line-argument \ -fuse-ld=$(LLD) -Wl,-znoseparate-code -Wl,--build-id=sha1 \ @@ -265,7 +283,7 @@ $(OUTPUT)/liburandom_read.so: urandom_read_lib1.c urandom_read_lib2.c liburandom $(OUTPUT)/urandom_read: urandom_read.c urandom_read_aux.c $(OUTPUT)/liburandom_read.so $(call msg,BINARY,,$@) $(Q)$(CLANG) $(CLANG_TARGET_ARCH) \ - $(filter-out -static,$(CFLAGS) $(LDFLAGS)) $(filter %.c,$^) \ + $(filter-out -static,$(COMMON_CFLAGS) $(LDFLAGS)) $(filter %.c,$^) \ -Wno-unused-command-line-argument \ -lurandom_read $(filter-out -static,$(LDLIBS)) -L$(OUTPUT) \ -fuse-ld=$(LLD) -Wl,-znoseparate-code -Wl,--build-id=sha1 \ @@ -274,6 +292,7 @@ $(OUTPUT)/urandom_read: urandom_read.c urandom_read_aux.c $(OUTPUT)/liburandom_r $(OUTPUT)/sign-file: ../../../../scripts/sign-file.c $(call msg,SIGN-FILE,,$@) $(Q)$(CC) $(shell $(PKG_CONFIG) --cflags libcrypto 2> /dev/null) \ + -I$(srctree)/tools/include/uapi/ \ $< -o $@ \ $(shell $(PKG_CONFIG) --libs libcrypto 2> /dev/null || echo -lcrypto) @@ -283,13 +302,15 @@ $(OUTPUT)/sign-file: ../../../../scripts/sign-file.c # subst() turns the rule into a pattern matching rule $(addprefix test_kmods/,$(subst .ko,%ko,$(TEST_KMODS))): $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard test_kmods/Makefile test_kmods/*.[ch]) $(Q)$(RM) test_kmods/*.ko test_kmods/*.mod.o # force re-compilation - $(Q)$(MAKE) $(submake_extras) -C test_kmods \ - RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) \ + $(Q)$(MAKE) $(submake_extras) -C test_kmods \ + $(if $(O),O=$(abspath $(O))) \ + $(if $(KBUILD_OUTPUT),KBUILD_OUTPUT=$(abspath $(KBUILD_OUTPUT)))\ + RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) \ EXTRA_CFLAGS='' EXTRA_LDFLAGS='' $(TEST_KMOD_TARGETS): $(addprefix test_kmods/,$(TEST_KMODS)) $(call msg,MOD,,$@) - $(Q)cp test_kmods/$(@F) $@ + $(Q)$(if $(PERMISSIVE),if [ -f test_kmods/$(@F) ]; then )cp test_kmods/$(@F) $@$(if $(PERMISSIVE),; fi) DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool @@ -302,8 +323,6 @@ TRUNNER_BPFTOOL := $(DEFAULT_BPFTOOL) USE_BOOTSTRAP := "bootstrap/" endif -TEST_GEN_PROGS_EXTENDED += $(TRUNNER_BPFTOOL) - $(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED): $(BPFOBJ) TESTING_HELPERS := $(OUTPUT)/testing_helpers.o @@ -319,7 +338,6 @@ $(OUTPUT)/test_tcpnotify_user: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(TRACE_HELP $(OUTPUT)/test_sock_fields: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(OUTPUT)/test_tag: $(TESTING_HELPERS) $(OUTPUT)/test_lirc_mode2_user: $(TESTING_HELPERS) -$(OUTPUT)/xdping: $(TESTING_HELPERS) $(OUTPUT)/flow_dissector_load: $(TESTING_HELPERS) $(OUTPUT)/test_maps: $(TESTING_HELPERS) $(OUTPUT)/test_verifier: $(TESTING_HELPERS) $(CAP_HELPERS) $(UNPRIV_HELPERS) @@ -335,6 +353,9 @@ $(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \ OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \ LIBBPF_OUTPUT=$(HOST_BUILD_DIR)/libbpf/ \ LIBBPF_DESTDIR=$(HOST_SCRATCH_DIR)/ \ + SKIP_LLVM=$(SKIP_LLVM) \ + SKIP_LIBBFD=$(SKIP_LIBBFD) \ + SKIP_CRYPTO=$(SKIP_CRYPTO) \ prefix= DESTDIR=$(HOST_SCRATCH_DIR)/ install-bin ifneq ($(CROSS_COMPILE),) @@ -347,6 +368,9 @@ $(CROSS_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \ OUTPUT=$(BUILD_DIR)/bpftool/ \ LIBBPF_OUTPUT=$(BUILD_DIR)/libbpf/ \ LIBBPF_DESTDIR=$(SCRATCH_DIR)/ \ + SKIP_LLVM=$(SKIP_LLVM) \ + SKIP_LIBBFD=$(SKIP_LIBBFD) \ + SKIP_CRYPTO=$(SKIP_CRYPTO) \ prefix= DESTDIR=$(SCRATCH_DIR)/ install-bin endif @@ -439,6 +463,7 @@ endif CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH)) BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN) \ -I$(INCLUDE_DIR) -I$(CURDIR) -I$(APIDIR) \ + -I$(TOOLSINCDIR) -I$(CURDIR)/libarena/include \ -I$(abspath $(OUTPUT)/../usr/include) \ -std=gnu11 \ -fno-strict-aliasing \ @@ -464,22 +489,26 @@ $(OUTPUT)/cgroup_getset_retval_hooks.o: cgroup_getset_retval_hooks.h # $4 - binary name define CLANG_BPF_BUILD_RULE $(call msg,CLNG-BPF,$4,$2) - $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v3 -o $2 + $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v3 -o $2 $(if $(PERMISSIVE),|| \ + ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) endef # Similar to CLANG_BPF_BUILD_RULE, but with disabled alu32 define CLANG_NOALU32_BPF_BUILD_RULE $(call msg,CLNG-BPF,$4,$2) - $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v2 -o $2 + $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v2 -o $2 $(if $(PERMISSIVE),|| \ + ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) endef # Similar to CLANG_BPF_BUILD_RULE, but with cpu-v4 define CLANG_CPUV4_BPF_BUILD_RULE $(call msg,CLNG-BPF,$4,$2) - $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v4 -o $2 + $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v4 -o $2 $(if $(PERMISSIVE),|| \ + ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) endef # Build BPF object using GCC define GCC_BPF_BUILD_RULE $(call msg,GCC-BPF,$4,$2) - $(Q)$(BPF_GCC) $3 -DBPF_NO_PRESERVE_ACCESS_INDEX -Wno-attributes -O2 -c $1 -o $2 + $(Q)$(BPF_GCC) $3 -DBPF_NO_PRESERVE_ACCESS_INDEX -Wno-attributes -O2 -c $1 -o $2 $(if $(PERMISSIVE),|| \ + ($(RM) $2; printf ' %-12s %s\n' 'SKIP-BPF' '$(notdir $2)' 1>&2)) endef SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c @@ -487,7 +516,10 @@ SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \ linked_vars.skel.h linked_maps.skel.h \ test_subskeleton.skel.h test_subskeleton_lib.skel.h \ - test_usdt.skel.h + test_usdt.skel.h tracing_multi.skel.h \ + tracing_multi_module.skel.h \ + tracing_multi_intersect.skel.h \ + tracing_multi_session.skel.h LSKELS := fexit_sleep.c trace_printk.c trace_vprintk.c map_ptr_kern.c \ core_kern.c core_kern_overflow.c test_ringbuf.c \ @@ -498,7 +530,7 @@ LSKELS_SIGNED := fentry_test.c fexit_test.c atomics.c # Generate both light skeleton and libbpf skeleton for these LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c kfunc_call_test.c \ - kfunc_call_test_subprog.c + kfunc_call_test_subprog.c test_global_percpu_data.c SKEL_BLACKLIST += $$(LSKELS) $$(LSKELS_SIGNED) test_static_linked.skel.h-deps := test_static_linked1.bpf.o test_static_linked2.bpf.o @@ -513,11 +545,16 @@ test_usdt.skel.h-deps := test_usdt.bpf.o test_usdt_multispec.bpf.o xsk_xdp_progs.skel.h-deps := xsk_xdp_progs.bpf.o xdp_hw_metadata.skel.h-deps := xdp_hw_metadata.bpf.o xdp_features.skel.h-deps := xdp_features.bpf.o +tracing_multi.skel.h-deps := tracing_multi_attach.bpf.o tracing_multi_check.bpf.o +tracing_multi_module.skel.h-deps := tracing_multi_attach_module.bpf.o tracing_multi_check.bpf.o +tracing_multi_intersect.skel.h-deps := tracing_multi_intersect_attach.bpf.o tracing_multi_check.bpf.o +tracing_multi_session.skel.h-deps := tracing_multi_session_attach.bpf.o tracing_multi_check.bpf.o LINKED_BPF_OBJS := $(foreach skel,$(LINKED_SKELS),$($(skel)-deps)) LINKED_BPF_SRCS := $(patsubst %.bpf.o,%.c,$(LINKED_BPF_OBJS)) HEADERS_FOR_BPF_OBJS := $(wildcard $(BPFDIR)/*.bpf.h) \ + $(wildcard $(CURDIR)/libarena/include/*.[ch]) \ $(addprefix $(BPFDIR)/, bpf_core_read.h \ bpf_endian.h \ bpf_helpers.h \ @@ -562,6 +599,12 @@ endef # $2 - test runner extra "flavor" (e.g., no_alu32, cpuv4, bpf_gcc, etc) define DEFINE_TEST_RUNNER_RULES +# Permissive build behaviour (skip-on-failure compile, partial-link) only +# applies to test_progs and its flavors; runners that use strong cross-object +# references (e.g. test_maps) keep strict semantics even when permissive. +# The check is inlined per-runner so $1 is substituted at $(call) time and +# the result is baked into each rule's recipe. + ifeq ($($(TRUNNER_OUTPUT)-dir),) $(TRUNNER_OUTPUT)-dir := y $(TRUNNER_OUTPUT): @@ -585,47 +628,81 @@ $(TRUNNER_BPF_OBJS): $(TRUNNER_OUTPUT)/%.bpf.o: \ $$($$<-$2-CFLAGS),$(TRUNNER_BINARY)) $(TRUNNER_BPF_SKELS): %.skel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) - $$(call msg,GEN-SKEL,$(TRUNNER_BINARY),$$@) - $(Q)$$(BPFTOOL) gen object $$(<:.o=.linked1.o) $$< - $(Q)$$(BPFTOOL) gen object $$(<:.o=.linked2.o) $$(<:.o=.linked1.o) - $(Q)$$(BPFTOOL) gen object $$(<:.o=.linked3.o) $$(<:.o=.linked2.o) - $(Q)diff $$(<:.o=.linked2.o) $$(<:.o=.linked3.o) - $(Q)$$(BPFTOOL) gen skeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$@ - $(Q)$$(BPFTOOL) gen subskeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$(@:.skel.h=.subskel.h) - $(Q)rm -f $$(<:.o=.linked1.o) $$(<:.o=.linked2.o) $$(<:.o=.linked3.o) + $(Q)$(if $(PERMISSIVE),if [ ! -f $$< ]; then \ + $$(RM) $$@ $$(@:.skel.h=.subskel.h); \ + printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ + exit 0; \ + fi;) \ + printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2; \ + $$(BPFTOOL) gen object $$(<:.o=.linked1.o) $$< && \ + $$(BPFTOOL) gen object $$(<:.o=.linked2.o) $$(<:.o=.linked1.o) && \ + $$(BPFTOOL) gen object $$(<:.o=.linked3.o) $$(<:.o=.linked2.o) && \ + diff $$(<:.o=.linked2.o) $$(<:.o=.linked3.o) && \ + $$(BPFTOOL) gen skeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$@ && \ + $$(BPFTOOL) gen subskeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$(@:.skel.h=.subskel.h) $(if $(PERMISSIVE),|| { \ + $$(RM) $$@ $$(@:.skel.h=.subskel.h); \ + printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ + }) && \ + rm -f $$(<:.o=.linked1.o) $$(<:.o=.linked2.o) $$(<:.o=.linked3.o) $(TRUNNER_BPF_LSKELS): %.lskel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) - $$(call msg,GEN-SKEL,$(TRUNNER_BINARY),$$@) - $(Q)$$(BPFTOOL) gen object $$(<:.o=.llinked1.o) $$< - $(Q)$$(BPFTOOL) gen object $$(<:.o=.llinked2.o) $$(<:.o=.llinked1.o) - $(Q)$$(BPFTOOL) gen object $$(<:.o=.llinked3.o) $$(<:.o=.llinked2.o) - $(Q)diff $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) - $(Q)$$(BPFTOOL) gen skeleton -L $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@ - $(Q)rm -f $$(<:.o=.llinked1.o) $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) + $(Q)$(if $(PERMISSIVE),if [ ! -f $$< ]; then \ + $$(RM) $$@; \ + printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ + exit 0; \ + fi;) \ + printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2; \ + $$(BPFTOOL) gen object $$(<:.o=.llinked1.o) $$< && \ + $$(BPFTOOL) gen object $$(<:.o=.llinked2.o) $$(<:.o=.llinked1.o) && \ + $$(BPFTOOL) gen object $$(<:.o=.llinked3.o) $$(<:.o=.llinked2.o) && \ + diff $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) && \ + $$(BPFTOOL) gen skeleton -L $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@ $(if $(PERMISSIVE),|| { \ + $$(RM) $$@; \ + printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ + }) && \ + rm -f $$(<:.o=.llinked1.o) $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) $(TRUNNER_BPF_LSKELS_SIGNED): %.lskel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) - $$(call msg,GEN-SKEL,$(TRUNNER_BINARY) (signed),$$@) - $(Q)$$(BPFTOOL) gen object $$(<:.o=.llinked1.o) $$< - $(Q)$$(BPFTOOL) gen object $$(<:.o=.llinked2.o) $$(<:.o=.llinked1.o) - $(Q)$$(BPFTOOL) gen object $$(<:.o=.llinked3.o) $$(<:.o=.llinked2.o) - $(Q)diff $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) - $(Q)$$(BPFTOOL) gen skeleton $(LSKEL_SIGN) $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@ - $(Q)rm -f $$(<:.o=.llinked1.o) $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) + $(Q)$(if $(PERMISSIVE),if [ ! -f $$< ]; then \ + $$(RM) $$@; \ + printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ + exit 0; \ + fi;) \ + printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY) (signed)] $$(notdir $$@)' 1>&2; \ + $$(BPFTOOL) gen object $$(<:.o=.llinked1.o) $$< && \ + $$(BPFTOOL) gen object $$(<:.o=.llinked2.o) $$(<:.o=.llinked1.o) && \ + $$(BPFTOOL) gen object $$(<:.o=.llinked3.o) $$(<:.o=.llinked2.o) && \ + diff $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) && \ + $$(BPFTOOL) gen skeleton $(LSKEL_SIGN) $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@ $(if $(PERMISSIVE),|| { \ + $$(RM) $$@; \ + printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ + }) && \ + rm -f $$(<:.o=.llinked1.o) $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) $(LINKED_BPF_OBJS): %: $(TRUNNER_OUTPUT)/% # .SECONDEXPANSION here allows to correctly expand %-deps variables as prerequisites .SECONDEXPANSION: $(TRUNNER_BPF_SKELS_LINKED): $(TRUNNER_OUTPUT)/%: $$$$(%-deps) $(BPFTOOL) | $(TRUNNER_OUTPUT) - $$(call msg,LINK-BPF,$(TRUNNER_BINARY),$$(@:.skel.h=.bpf.o)) - $(Q)$$(BPFTOOL) gen object $$(@:.skel.h=.linked1.o) $$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps)) - $(Q)$$(BPFTOOL) gen object $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked1.o) - $(Q)$$(BPFTOOL) gen object $$(@:.skel.h=.linked3.o) $$(@:.skel.h=.linked2.o) - $(Q)diff $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked3.o) - $$(call msg,GEN-SKEL,$(TRUNNER_BINARY),$$@) - $(Q)$$(BPFTOOL) gen skeleton $$(@:.skel.h=.linked3.o) name $$(notdir $$(@:.skel.h=)) > $$@ - $(Q)$$(BPFTOOL) gen subskeleton $$(@:.skel.h=.linked3.o) name $$(notdir $$(@:.skel.h=)) > $$(@:.skel.h=.subskel.h) - $(Q)rm -f $$(@:.skel.h=.linked1.o) $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked3.o) + $(Q)$(if $(PERMISSIVE),for f in $$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps)); do \ + if [ ! -f $$$$f ]; then \ + $$(RM) $$@ $$(@:.skel.h=.subskel.h); \ + printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ + exit 0; \ + fi; \ + done;) \ + printf ' %-12s %s\n' 'LINK-BPF' '[$(TRUNNER_BINARY)] $$(notdir $$(@:.skel.h=.bpf.o))' 1>&2; \ + $$(BPFTOOL) gen object $$(@:.skel.h=.linked1.o) $$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps)) && \ + $$(BPFTOOL) gen object $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked1.o) && \ + $$(BPFTOOL) gen object $$(@:.skel.h=.linked3.o) $$(@:.skel.h=.linked2.o) && \ + diff $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked3.o) && \ + printf ' %-12s %s\n' 'GEN-SKEL' '[$(TRUNNER_BINARY)] $$(notdir $$@)' 1>&2 && \ + $$(BPFTOOL) gen skeleton $$(@:.skel.h=.linked3.o) name $$(notdir $$(@:.skel.h=)) > $$@ && \ + $$(BPFTOOL) gen subskeleton $$(@:.skel.h=.linked3.o) name $$(notdir $$(@:.skel.h=)) > $$(@:.skel.h=.subskel.h) $(if $(PERMISSIVE),|| { \ + $$(RM) $$@ $$(@:.skel.h=.subskel.h); \ + printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ + }) && \ + rm -f $$(@:.skel.h=.linked1.o) $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked3.o) # When the compiler generates a %.d file, only skel basenames (not # full paths) are specified as prerequisites for corresponding %.o @@ -657,22 +734,25 @@ $(TRUNNER_TEST_OBJS): $(TRUNNER_OUTPUT)/%.test.o: \ $(TRUNNER_TESTS_DIR)/%.c \ | $(TRUNNER_OUTPUT)/%.test.d $$(call msg,TEST-OBJ,$(TRUNNER_BINARY),$$@) - $(Q)cd $$(@D) && $$(CC) -I. $$(CFLAGS) -MMD -MT $$@ -c $(CURDIR)/$$< $$(LDLIBS) -o $$(@F) + $(Q)(cd $$(@D) && $$(CC) -I. $$(CFLAGS) -MMD -MT $$@ -c $(CURDIR)/$$< $$(LDLIBS) -o $$(@F)) $(if $(filter test_progs%,$1),$(if $(PERMISSIVE),|| \ + ($(RM) $$@; printf ' %-12s %s\n' 'SKIP-TEST' '$$(notdir $$@)' 1>&2))) $$(if $$(TEST_NEEDS_BTFIDS), \ - $$(call msg,BTFIDS,$(TRUNNER_BINARY),$$@) \ + $(Q)if [ -f $$@ ]; then \ + $(if $(filter 1,$(V)),true,printf ' %-8s%s %s\n' "BTFIDS" " [$(TRUNNER_BINARY)]" "$$(notdir $$@)"); \ $(RESOLVE_BTFIDS) --btf $(TRUNNER_OUTPUT)/btf_data.bpf.o $$@; \ - $(RESOLVE_BTFIDS) --patch_btfids $$@.BTF_ids $$@) + $(RESOLVE_BTFIDS) --patch_btfids $$@.BTF_ids $$@; \ + fi) $(TRUNNER_TEST_OBJS:.o=.d): $(TRUNNER_OUTPUT)/%.test.d: \ $(TRUNNER_TESTS_DIR)/%.c \ $(TRUNNER_EXTRA_HDRS) \ + $$(BPFOBJ) | $(TRUNNER_OUTPUT) \ $(TRUNNER_BPF_SKELS) \ $(TRUNNER_BPF_LSKELS) \ $(TRUNNER_BPF_LSKELS_SIGNED) \ - $(TRUNNER_BPF_SKELS_LINKED) \ - $$(BPFOBJ) | $(TRUNNER_OUTPUT) + $(TRUNNER_BPF_SKELS_LINKED) -ifeq ($(filter clean docs-clean,$(MAKECMDGOALS)),) +ifeq ($(filter clean docs-clean emit_tests,$(MAKECMDGOALS)),) include $(wildcard $(TRUNNER_TEST_OBJS:.o=.d)) endif @@ -698,20 +778,21 @@ $(TRUNNER_LIB_OBJS): $(TRUNNER_OUTPUT)/%.o:$(TOOLSDIR)/lib/%.c $(TRUNNER_BINARY)-extras: $(TRUNNER_EXTRA_FILES) | $(TRUNNER_OUTPUT) ifneq ($2:$(OUTPUT),:$(shell pwd)) $$(call msg,EXT-COPY,$(TRUNNER_BINARY),$(TRUNNER_EXTRA_FILES)) - $(Q)rsync -aq $$^ $(TRUNNER_OUTPUT)/ + $(Q)rsync -aq $(if $(PERMISSIVE),--ignore-missing-args) $$^ $(TRUNNER_OUTPUT)/ endif # some X.test.o files have runtime dependencies on Y.bpf.o files $(OUTPUT)/$(TRUNNER_BINARY): | $(TRUNNER_BPF_OBJS) -$(OUTPUT)/$(TRUNNER_BINARY): $(TRUNNER_TEST_OBJS) \ +$(OUTPUT)/$(TRUNNER_BINARY): $(if $(filter test_progs%,$1),$(if $(PERMISSIVE),$$(wildcard $(TRUNNER_TEST_OBJS)),$(TRUNNER_TEST_OBJS)),$(TRUNNER_TEST_OBJS)) \ $(TRUNNER_EXTRA_OBJS) $$(BPFOBJ) \ $(TRUNNER_LIB_OBJS) \ $(TRUNNER_BPFTOOL) \ $(OUTPUT)/veristat \ - | $(TRUNNER_BINARY)-extras + | $(TRUNNER_BINARY)-extras \ + $(if $(filter test_progs%,$1),$(if $(PERMISSIVE),$(TRUNNER_TEST_OBJS))) $$(call msg,BINARY,,$$@) - $(Q)$$(CC) $$(CFLAGS) $$(filter %.a %.o,$$^) $$(LDLIBS) $$(LLVM_LDLIBS) $$(LDFLAGS) $$(LLVM_LDFLAGS) -o $$@ + $(Q)$$(CC) $$(CFLAGS) $(if $(filter test_progs%,$1),$(if $(PERMISSIVE),$$(filter %.a %.o,$$(wildcard $(TRUNNER_TEST_OBJS)) $$(filter-out $(TRUNNER_TEST_OBJS),$$^)),$$(filter %.a %.o,$$^)),$$(filter %.a %.o,$$^)) $$(LDLIBS) $$(LLVM_LDLIBS) $$(LDFLAGS) $$(LLVM_LDFLAGS) -o $$@ $(Q)ln -sf $(if $2,..,.)/tools/build/bpftool/$(USE_BOOTSTRAP)bpftool \ $(OUTPUT)/$(if $2,$2/)bpftool @@ -733,6 +814,37 @@ $(VERIFY_SIG_HDR): $(VERIFICATION_CERT) echo "};"; \ echo "unsigned int test_progs_verification_cert_len = $$(wc -c < $<);") > $@ +LIBARENA_MAKE_ARGS = \ + BPFTOOL="$(BPFTOOL)" \ + INCLUDE_DIR="$(INCLUDE_DIR)" \ + LIBBPF_INCLUDE="$(HOST_INCLUDE_DIR)" \ + BPFOBJ="$(BPFOBJ)" \ + LDLIBS="$(LDLIBS) -lzstd" \ + CLANG="$(CLANG)" \ + BPF_CFLAGS="$(BPF_CFLAGS) $(CLANG_CFLAGS)" \ + BPF_TARGET_ENDIAN="$(BPF_TARGET_ENDIAN)" \ + Q="$(Q)" + +LIBARENA_BPF_DEPS := $(wildcard libarena/Makefile \ + libarena/include/* \ + libarena/include/libarena/* \ + libarena/src/* \ + libarena/selftests/* \ + libarena/*.bpf.o) + +LIBARENA_SKEL := libarena/libarena.skel.h + +$(LIBARENA_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BPF_DEPS) + +$(MAKE) -C libarena libarena.skel.h $(LIBARENA_MAKE_ARGS) + +ifneq ($(CLANG_HAS_ARENA_ASAN),) +LIBARENA_ASAN_SKEL := libarena/libarena_asan.skel.h +CFLAGS += -DHAS_BPF_ARENA_ASAN + +$(LIBARENA_ASAN_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BPF_DEPS) + +$(MAKE) -C libarena libarena_asan.skel.h $(LIBARENA_MAKE_ARGS) +endif + # Define test_progs test runner. TRUNNER_TESTS_DIR := prog_tests TRUNNER_BPF_PROGS_DIR := progs @@ -744,6 +856,7 @@ TRUNNER_EXTRA_SOURCES := test_progs.c \ btf_helpers.c \ cap_helpers.c \ unpriv_helpers.c \ + sysctl_helpers.c \ netlink_helpers.c \ jit_disasm_helpers.c \ io_helpers.c \ @@ -755,7 +868,10 @@ TRUNNER_EXTRA_SOURCES := test_progs.c \ $(VERIFY_SIG_HDR) \ flow_dissector_load.h \ ip_check_defrag_frags.h \ - bpftool_helpers.c + bpftool_helpers.c \ + usdt_1.c usdt_2.c \ + $(LIBARENA_SKEL) \ + $(LIBARENA_ASAN_SKEL) TRUNNER_LIB_SOURCES := find_bit.c TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \ $(OUTPUT)/liburandom_read.so \ @@ -815,17 +931,26 @@ $(OUTPUT)/test_verifier: test_verifier.c verifier/tests.h $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ -# Include find_bit.c to compile xskxceiver. -EXTRA_SRC := $(TOOLSDIR)/lib/find_bit.c prog_tests/test_xsk.c prog_tests/test_xsk.h -$(OUTPUT)/xskxceiver: $(EXTRA_SRC) xskxceiver.c xskxceiver.h $(OUTPUT)/network_helpers.o $(OUTPUT)/xsk.o $(OUTPUT)/xsk_xdp_progs.skel.h $(BPFOBJ) | $(OUTPUT) +# Keep xskxceiver independent from test_progs object dependencies. +$(OUTPUT)/xskxceiver: xskxceiver.c xsk.c network_helpers.c \ + $(TOOLSDIR)/lib/find_bit.c prog_tests/test_xsk.c \ + xskxceiver.h xsk.h network_helpers.h \ + prog_tests/test_xsk.h test_progs.h bpf_util.h \ + $(OUTPUT)/xsk_xdp_progs.skel.h $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ -$(OUTPUT)/xdp_hw_metadata: xdp_hw_metadata.c $(OUTPUT)/network_helpers.o $(OUTPUT)/xsk.o $(OUTPUT)/xdp_hw_metadata.skel.h | $(OUTPUT) +$(OUTPUT)/xdp_hw_metadata: xdp_hw_metadata.c xsk.c network_helpers.c \ + $(TOOLSDIR)/lib/find_bit.c xdp_metadata.h \ + xsk.h network_helpers.h test_progs.h bpf_util.h \ + $(OUTPUT)/xdp_hw_metadata.skel.h $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ -$(OUTPUT)/xdp_features: xdp_features.c $(OUTPUT)/network_helpers.o $(OUTPUT)/xdp_features.skel.h | $(OUTPUT) +$(OUTPUT)/xdp_features: xdp_features.c network_helpers.c xdp_features.h \ + network_helpers.h \ + test_progs.h bpf_util.h $(OUTPUT)/xdp_features.skel.h \ + $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ @@ -840,13 +965,15 @@ $(OUTPUT)/test_cpp: test_cpp.cpp $(OUTPUT)/test_core_extern.skel.h $(BPFOBJ) # Benchmark runner $(OUTPUT)/bench_%.o: benchs/bench_%.c bench.h $(BPFOBJ) $(call msg,CC,,$@) - $(Q)$(CC) $(CFLAGS) -O2 -c $(filter %.c,$^) $(LDLIBS) -o $@ + $(Q)$(CC) $(CFLAGS) -O2 -c $(filter %.c,$^) $(LDLIBS) -o $@ $(if $(PERMISSIVE),|| \ + ($(RM) $@; printf ' %-12s %s\n' 'SKIP-BENCH' '$(notdir $@)' 1>&2)) $(OUTPUT)/bench_rename.o: $(OUTPUT)/test_overhead.skel.h $(OUTPUT)/bench_trigger.o: $(OUTPUT)/trigger_bench.skel.h $(OUTPUT)/bench_ringbufs.o: $(OUTPUT)/ringbuf_bench.skel.h \ $(OUTPUT)/perfbuf_bench.skel.h $(OUTPUT)/bench_bloom_filter_map.o: $(OUTPUT)/bloom_filter_bench.skel.h $(OUTPUT)/bench_bpf_loop.o: $(OUTPUT)/bpf_loop_bench.skel.h +$(OUTPUT)/bench_bpf_for.o: $(OUTPUT)/bpf_for_bench.skel.h $(OUTPUT)/bench_strncmp.o: $(OUTPUT)/strncmp_bench.skel.h $(OUTPUT)/bench_bpf_hashmap_full_update.o: $(OUTPUT)/bpf_hashmap_full_update_bench.skel.h $(OUTPUT)/bench_local_storage.o: $(OUTPUT)/local_storage_bench.skel.h @@ -857,6 +984,9 @@ $(OUTPUT)/bench_htab_mem.o: $(OUTPUT)/htab_mem_bench.skel.h $(OUTPUT)/bench_bpf_crypto.o: $(OUTPUT)/crypto_bench.skel.h $(OUTPUT)/bench_sockmap.o: $(OUTPUT)/bench_sockmap_prog.skel.h $(OUTPUT)/bench_lpm_trie_map.o: $(OUTPUT)/lpm_trie_bench.skel.h $(OUTPUT)/lpm_trie_map.skel.h +$(OUTPUT)/bench_bpf_nop.o: $(OUTPUT)/bpf_nop_bench.skel.h bench_bpf_timing.h +$(OUTPUT)/bench_xdp_lb.o: $(OUTPUT)/xdp_lb_bench.skel.h bench_bpf_timing.h +$(OUTPUT)/bench_bpf_timing.o: bench_bpf_timing.h $(OUTPUT)/bench.o: bench.h testing_helpers.h $(BPFOBJ) $(OUTPUT)/bench: LDLIBS += -lm $(OUTPUT)/bench: $(OUTPUT)/bench.o \ @@ -869,6 +999,7 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \ $(OUTPUT)/bench_ringbufs.o \ $(OUTPUT)/bench_bloom_filter_map.o \ $(OUTPUT)/bench_bpf_loop.o \ + $(OUTPUT)/bench_bpf_for.o \ $(OUTPUT)/bench_strncmp.o \ $(OUTPUT)/bench_bpf_hashmap_full_update.o \ $(OUTPUT)/bench_local_storage.o \ @@ -879,9 +1010,15 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \ $(OUTPUT)/bench_bpf_crypto.o \ $(OUTPUT)/bench_sockmap.o \ $(OUTPUT)/bench_lpm_trie_map.o \ + $(OUTPUT)/bench_bpf_timing.o \ + $(OUTPUT)/bench_bpf_nop.o \ + $(OUTPUT)/bench_xdp_lb.o \ + $(OUTPUT)/usdt_1.o \ + $(OUTPUT)/usdt_2.o \ # $(call msg,BINARY,,$@) - $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@ + $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@ $(if $(PERMISSIVE),|| \ + ($(RM) $@; printf ' %-12s %s\n' 'SKIP-LINK' '$(notdir $@) (some benchmarks may have been skipped)' 1>&2)) # This works around GCC warning about snprintf truncating strings like: # @@ -914,11 +1051,31 @@ EXTRA_CLEAN := $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) \ # Delete partially updated (corrupted) files on error .DELETE_ON_ERROR: +# When permissive, tell rsync to ignore missing source arguments so that +# partial builds do not abort installation. +ifneq ($(PERMISSIVE),) +override define INSTALL_SINGLE_RULE + $(if $(INSTALL_LIST),@mkdir -p $(INSTALL_PATH)) + $(if $(INSTALL_LIST),rsync -a --copy-unsafe-links --ignore-missing-args $(INSTALL_LIST) $(INSTALL_PATH)/) +endef +endif + DEFAULT_INSTALL_RULE := $(INSTALL_RULE) override define INSTALL_RULE $(DEFAULT_INSTALL_RULE) - @for DIR in $(TEST_INST_SUBDIRS); do \ - mkdir -p $(INSTALL_PATH)/$$DIR; \ - rsync -a $(OUTPUT)/$$DIR/*.bpf.o $(INSTALL_PATH)/$$DIR;\ + @mkdir -p $(INSTALL_PATH)/tools/sbin + @rsync -a $(if $(PERMISSIVE),--ignore-missing-args) $(TRUNNER_BPFTOOL) $(INSTALL_PATH)/tools/sbin/ + @rsync -a $(if $(PERMISSIVE),--ignore-missing-args) $(OUTPUT)/*.BTF $(INSTALL_PATH)/ + @for DIR in $(TEST_INST_SUBDIRS); do \ + mkdir -p $(INSTALL_PATH)/$$DIR; \ + rsync -a $(if $(PERMISSIVE),--ignore-missing-args) \ + $(OUTPUT)/$$DIR/*.bpf.o $(OUTPUT)/$$DIR/*.BTF \ + $(INSTALL_PATH)/$$DIR; \ done endef + +libarena: $(LIBARENA_SKEL) + +ifneq ($(CLANG_HAS_ARENA_ASAN),) +libarena_asan: $(LIBARENA_ASAN_SKEL) +endif diff --git a/tools/testing/selftests/bpf/README.rst b/tools/testing/selftests/bpf/README.rst index 776fbe3cb8f9..07c834433b38 100644 --- a/tools/testing/selftests/bpf/README.rst +++ b/tools/testing/selftests/bpf/README.rst @@ -77,7 +77,7 @@ In case of linker errors when running selftests, try using static linking: .. code-block:: console - $ LDLIBS=-static PKG_CONFIG='pkg-config --static' vmtest.sh + $ LDLIBS=-static EXTRA_LDFLAGS=-static PKG_CONFIG='pkg-config --static' vmtest.sh .. note:: Some distros may not support static linking. @@ -107,12 +107,12 @@ Docker container and local rootfs image. The overall steps are as follows: tools/testing/selftests/bpf/vmtest.sh \ -l <path of local rootfs image> -- \ ./test_progs -d \ - \"$(cat tools/testing/selftests/bpf/DENYLIST.riscv64 \ + "$(cat tools/testing/selftests/bpf/DENYLIST.riscv64 \ | cut -d'#' -f1 \ | sed -e 's/^[[:space:]]*//' \ -e 's/[[:space:]]*$//' \ | tr -s '\n' ',' \ - )\" + )" Link: https://github.com/pulehui/riscv-bpf-vmtest.git [0] Link: https://github.com/libbpf/ci/blob/main/rootfs/mkrootfs_debian.sh [1] diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c index 8368bd3a0665..465233ea5f79 100644 --- a/tools/testing/selftests/bpf/bench.c +++ b/tools/testing/selftests/bpf/bench.c @@ -276,6 +276,7 @@ static const struct argp_option opts[] = { extern struct argp bench_ringbufs_argp; extern struct argp bench_bloom_map_argp; extern struct argp bench_bpf_loop_argp; +extern struct argp bench_bpf_for_argp; extern struct argp bench_local_storage_argp; extern struct argp bench_local_storage_rcu_tasks_trace_argp; extern struct argp bench_strncmp_argp; @@ -286,11 +287,13 @@ extern struct argp bench_trigger_batch_argp; extern struct argp bench_crypto_argp; extern struct argp bench_sockmap_argp; extern struct argp bench_lpm_trie_map_argp; +extern struct argp bench_xdp_lb_argp; static const struct argp_child bench_parsers[] = { { &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 }, { &bench_bloom_map_argp, 0, "Bloom filter map benchmark", 0 }, { &bench_bpf_loop_argp, 0, "bpf_loop helper benchmark", 0 }, + { &bench_bpf_for_argp, 0, "bpf_for loop benchmark", 0 }, { &bench_local_storage_argp, 0, "local_storage benchmark", 0 }, { &bench_strncmp_argp, 0, "bpf_strncmp helper benchmark", 0 }, { &bench_local_storage_rcu_tasks_trace_argp, 0, @@ -302,6 +305,7 @@ static const struct argp_child bench_parsers[] = { { &bench_crypto_argp, 0, "bpf crypto benchmark", 0 }, { &bench_sockmap_argp, 0, "bpf sockmap benchmark", 0 }, { &bench_lpm_trie_map_argp, 0, "LPM trie map benchmark", 0 }, + { &bench_xdp_lb_argp, 0, "XDP load-balancer benchmark", 0 }, {}, }; @@ -537,10 +541,12 @@ extern const struct bench bench_trig_uretprobe_multi_push; extern const struct bench bench_trig_uprobe_multi_ret; extern const struct bench bench_trig_uretprobe_multi_ret; #ifdef __x86_64__ -extern const struct bench bench_trig_uprobe_nop5; -extern const struct bench bench_trig_uretprobe_nop5; -extern const struct bench bench_trig_uprobe_multi_nop5; -extern const struct bench bench_trig_uretprobe_multi_nop5; +extern const struct bench bench_trig_uprobe_nop10; +extern const struct bench bench_trig_uretprobe_nop10; +extern const struct bench bench_trig_uprobe_multi_nop10; +extern const struct bench bench_trig_uretprobe_multi_nop10; +extern const struct bench bench_trig_usdt_nop; +extern const struct bench bench_trig_usdt_nop10; #endif extern const struct bench bench_rb_libbpf; @@ -553,16 +559,20 @@ extern const struct bench bench_bloom_false_positive; extern const struct bench bench_hashmap_without_bloom; extern const struct bench bench_hashmap_with_bloom; extern const struct bench bench_bpf_loop; +extern const struct bench bench_bpf_for; extern const struct bench bench_strncmp_no_helper; extern const struct bench bench_strncmp_helper; extern const struct bench bench_bpf_hashmap_full_update; +extern const struct bench bench_bpf_rhashmap_full_update; extern const struct bench bench_local_storage_cache_seq_get; extern const struct bench bench_local_storage_cache_interleaved_get; extern const struct bench bench_local_storage_cache_hashmap_control; extern const struct bench bench_local_storage_tasks_trace; extern const struct bench bench_bpf_hashmap_lookup; +extern const struct bench bench_bpf_rhashmap_lookup; extern const struct bench bench_local_storage_create; extern const struct bench bench_htab_mem; +extern const struct bench bench_rhtab_mem; extern const struct bench bench_crypto_encrypt; extern const struct bench bench_crypto_decrypt; extern const struct bench bench_sockmap; @@ -573,6 +583,8 @@ extern const struct bench bench_lpm_trie_insert; extern const struct bench bench_lpm_trie_update; extern const struct bench bench_lpm_trie_delete; extern const struct bench bench_lpm_trie_free; +extern const struct bench bench_bpf_nop; +extern const struct bench bench_xdp_lb; static const struct bench *benchs[] = { &bench_count_global, @@ -613,10 +625,12 @@ static const struct bench *benchs[] = { &bench_trig_uprobe_multi_ret, &bench_trig_uretprobe_multi_ret, #ifdef __x86_64__ - &bench_trig_uprobe_nop5, - &bench_trig_uretprobe_nop5, - &bench_trig_uprobe_multi_nop5, - &bench_trig_uretprobe_multi_nop5, + &bench_trig_uprobe_nop10, + &bench_trig_uretprobe_nop10, + &bench_trig_uprobe_multi_nop10, + &bench_trig_uretprobe_multi_nop10, + &bench_trig_usdt_nop, + &bench_trig_usdt_nop10, #endif /* ringbuf/perfbuf benchmarks */ &bench_rb_libbpf, @@ -629,16 +643,20 @@ static const struct bench *benchs[] = { &bench_hashmap_without_bloom, &bench_hashmap_with_bloom, &bench_bpf_loop, + &bench_bpf_for, &bench_strncmp_no_helper, &bench_strncmp_helper, &bench_bpf_hashmap_full_update, + &bench_bpf_rhashmap_full_update, &bench_local_storage_cache_seq_get, &bench_local_storage_cache_interleaved_get, &bench_local_storage_cache_hashmap_control, &bench_local_storage_tasks_trace, &bench_bpf_hashmap_lookup, + &bench_bpf_rhashmap_lookup, &bench_local_storage_create, &bench_htab_mem, + &bench_rhtab_mem, &bench_crypto_encrypt, &bench_crypto_decrypt, &bench_sockmap, @@ -649,6 +667,8 @@ static const struct bench *benchs[] = { &bench_lpm_trie_update, &bench_lpm_trie_delete, &bench_lpm_trie_free, + &bench_bpf_nop, + &bench_xdp_lb, }; static void find_benchmark(void) @@ -737,6 +757,13 @@ static void setup_benchmark(void) static pthread_mutex_t bench_done_mtx = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t bench_done = PTHREAD_COND_INITIALIZER; +void bench_force_done(void) +{ + pthread_mutex_lock(&bench_done_mtx); + pthread_cond_signal(&bench_done); + pthread_mutex_unlock(&bench_done_mtx); +} + static void collect_measurements(long delta_ns) { int iter = state.res_cnt++; struct bench_res *res = &state.results[iter]; @@ -746,11 +773,8 @@ static void collect_measurements(long delta_ns) { if (bench->report_progress) bench->report_progress(iter, res, delta_ns); - if (iter == env.duration_sec + env.warmup_sec) { - pthread_mutex_lock(&bench_done_mtx); - pthread_cond_signal(&bench_done); - pthread_mutex_unlock(&bench_done_mtx); - } + if (iter == env.duration_sec + env.warmup_sec) + bench_force_done(); } int main(int argc, char **argv) diff --git a/tools/testing/selftests/bpf/bench.h b/tools/testing/selftests/bpf/bench.h index 7cf21936e7ed..89a3fc72f70e 100644 --- a/tools/testing/selftests/bpf/bench.h +++ b/tools/testing/selftests/bpf/bench.h @@ -70,6 +70,7 @@ extern struct env env; extern const struct bench *bench; void setup_libbpf(void); +void bench_force_done(void); void hits_drops_report_progress(int iter, struct bench_res *res, long delta_ns); void hits_drops_report_final(struct bench_res res[], int res_cnt); void false_hits_report_progress(int iter, struct bench_res *res, long delta_ns); diff --git a/tools/testing/selftests/bpf/bench_bpf_timing.h b/tools/testing/selftests/bpf/bench_bpf_timing.h new file mode 100644 index 000000000000..6ef23b6d6639 --- /dev/null +++ b/tools/testing/selftests/bpf/bench_bpf_timing.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#ifndef __BENCH_BPF_TIMING_H__ +#define __BENCH_BPF_TIMING_H__ + +#include <stdbool.h> +#include <linux/types.h> +#include "bench.h" + +#ifndef BENCH_NR_SAMPLES +#define BENCH_NR_SAMPLES 4096 +#endif +#ifndef BENCH_NR_CPUS +#define BENCH_NR_CPUS 256 +#endif + +typedef void (*bpf_bench_run_fn)(void *ctx); + +struct bpf_bench_timing { + __u64 (*samples)[BENCH_NR_SAMPLES]; /* skel->bss->timing_samples */ + __u32 *idx; /* skel->bss->timing_idx */ + volatile __u32 *timing_enabled; /* &skel->bss->timing_enabled */ + volatile __u32 *batch_iters_bss; /* &skel->bss->batch_iters */ + __u32 batch_iters; + __u32 target_samples; + __u32 nr_cpus; + int warmup_ticks; + bool done; + bool machine_readable; +}; + +#define BENCH_TIMING_INIT(t, skel, iters) do { \ + (t)->samples = (skel)->bss->timing_samples; \ + (t)->idx = (skel)->bss->timing_idx; \ + (t)->timing_enabled = &(skel)->bss->timing_enabled; \ + (t)->batch_iters_bss = &(skel)->bss->batch_iters; \ + (t)->batch_iters = (iters); \ + (t)->target_samples = 200; \ + (t)->nr_cpus = env.nr_cpus; \ + (t)->warmup_ticks = 0; \ + (t)->done = false; \ + (t)->machine_readable = false; \ +} while (0) + +void bpf_bench_timing_measure(struct bpf_bench_timing *t, struct bench_res *res); +void bpf_bench_timing_report(struct bpf_bench_timing *t, const char *name, const char *desc); +void bpf_bench_calibrate(struct bpf_bench_timing *t, bpf_bench_run_fn run_fn, void *ctx); + +#endif /* __BENCH_BPF_TIMING_H__ */ diff --git a/tools/testing/selftests/bpf/benchs/bench_bpf_for.c b/tools/testing/selftests/bpf/benchs/bench_bpf_for.c new file mode 100644 index 000000000000..730c51ad2dec --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/bench_bpf_for.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <argp.h> +#include "bench.h" +#include "bpf_for_bench.skel.h" + +/* BPF triggering benchmarks */ +static struct ctx { + struct bpf_for_bench *skel; +} ctx; + +static struct { + __u32 nr_loops; +} args = { + /* + * Default to a large loop count so the per-iteration bpf_iter_num_next() cost dominates + * the one-time bpf_iter_num_new()/destroy() setup and teardown. + */ + .nr_loops = 1000, +}; + +enum { + ARG_NR_LOOPS = 4000, +}; + +static const struct argp_option opts[] = { + { "nr_loops", ARG_NR_LOOPS, "nr_loops", 0, + "Set number of iterations for the bpf_for() loop"}, + {}, +}; + +static error_t parse_arg(int key, char *arg, struct argp_state *state) +{ + switch (key) { + case ARG_NR_LOOPS: + args.nr_loops = strtol(arg, NULL, 10); + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +/* exported into benchmark runner */ +const struct argp bench_bpf_for_argp = { + .options = opts, + .parser = parse_arg, +}; + +static void validate(void) +{ + if (env.consumer_cnt != 0) { + fprintf(stderr, "benchmark doesn't support consumer!\n"); + exit(1); + } +} + +static void *producer(void *input) +{ + while (true) + /* trigger the bpf program */ + syscall(__NR_getpgid); + + return NULL; +} + +static void measure(struct bench_res *res) +{ + res->hits = atomic_swap(&ctx.skel->bss->hits, 0); +} + +static void setup(void) +{ + struct bpf_link *link; + + setup_libbpf(); + + ctx.skel = bpf_for_bench__open_and_load(); + if (!ctx.skel) { + fprintf(stderr, "failed to open skeleton\n"); + exit(1); + } + + link = bpf_program__attach(ctx.skel->progs.benchmark); + if (!link) { + fprintf(stderr, "failed to attach program!\n"); + exit(1); + } + + ctx.skel->bss->nr_loops = args.nr_loops; +} + +const struct bench bench_bpf_for = { + .name = "bpf-for", + .argp = &bench_bpf_for_argp, + .validate = validate, + .setup = setup, + .producer_thread = producer, + .measure = measure, + .report_progress = ops_report_progress, + .report_final = ops_report_final, +}; diff --git a/tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_full_update.c b/tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_full_update.c index ee1dc12c5e5e..7278fa860397 100644 --- a/tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_full_update.c +++ b/tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_full_update.c @@ -34,19 +34,29 @@ static void measure(struct bench_res *res) { } -static void setup(void) +static void hashmap_full_update_setup(enum bpf_map_type map_type) { struct bpf_link *link; int map_fd, i, max_entries; setup_libbpf(); - ctx.skel = bpf_hashmap_full_update_bench__open_and_load(); + ctx.skel = bpf_hashmap_full_update_bench__open(); if (!ctx.skel) { fprintf(stderr, "failed to open skeleton\n"); exit(1); } + bpf_map__set_type(ctx.skel->maps.hash_map_bench, map_type); + if (map_type == BPF_MAP_TYPE_RHASH) + bpf_map__set_map_flags(ctx.skel->maps.hash_map_bench, + BPF_F_NO_PREALLOC); + + if (bpf_hashmap_full_update_bench__load(ctx.skel)) { + fprintf(stderr, "failed to load skeleton\n"); + exit(1); + } + ctx.skel->bss->nr_loops = MAX_LOOP_NUM; link = bpf_program__attach(ctx.skel->progs.benchmark); @@ -62,6 +72,16 @@ static void setup(void) bpf_map_update_elem(map_fd, &i, &i, BPF_ANY); } +static void setup(void) +{ + hashmap_full_update_setup(BPF_MAP_TYPE_HASH); +} + +static void rhash_setup(void) +{ + hashmap_full_update_setup(BPF_MAP_TYPE_RHASH); +} + static void hashmap_report_final(struct bench_res res[], int res_cnt) { unsigned int nr_cpus = bpf_num_possible_cpus(); @@ -87,3 +107,13 @@ const struct bench bench_bpf_hashmap_full_update = { .report_progress = NULL, .report_final = hashmap_report_final, }; + +const struct bench bench_bpf_rhashmap_full_update = { + .name = "bpf-rhashmap-full-update", + .validate = validate, + .setup = rhash_setup, + .producer_thread = producer, + .measure = measure, + .report_progress = NULL, + .report_final = hashmap_report_final, +}; diff --git a/tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_lookup.c b/tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_lookup.c index 279ff1b8b5b2..5264b7b20e39 100644 --- a/tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_lookup.c +++ b/tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_lookup.c @@ -148,9 +148,10 @@ static inline void patch_key(u32 i, u32 *key) /* the rest of key is random */ } -static void setup(void) +static void hashmap_lookup_setup(enum bpf_map_type map_type) { struct bpf_link *link; + __u32 map_flags; int map_fd; int ret; int i; @@ -163,10 +164,15 @@ static void setup(void) exit(1); } + map_flags = args.map_flags; + if (map_type == BPF_MAP_TYPE_RHASH) + map_flags |= BPF_F_NO_PREALLOC; + + bpf_map__set_type(ctx.skel->maps.hash_map_bench, map_type); bpf_map__set_max_entries(ctx.skel->maps.hash_map_bench, args.max_entries); bpf_map__set_key_size(ctx.skel->maps.hash_map_bench, args.key_size); bpf_map__set_value_size(ctx.skel->maps.hash_map_bench, 8); - bpf_map__set_map_flags(ctx.skel->maps.hash_map_bench, args.map_flags); + bpf_map__set_map_flags(ctx.skel->maps.hash_map_bench, map_flags); ctx.skel->bss->nr_entries = args.nr_entries; ctx.skel->bss->nr_loops = args.nr_loops / args.nr_entries; @@ -197,6 +203,16 @@ static void setup(void) } } +static void setup(void) +{ + hashmap_lookup_setup(BPF_MAP_TYPE_HASH); +} + +static void rhash_setup(void) +{ + hashmap_lookup_setup(BPF_MAP_TYPE_RHASH); +} + static inline double events_from_time(u64 time) { if (time) @@ -275,3 +291,14 @@ const struct bench bench_bpf_hashmap_lookup = { .report_progress = NULL, .report_final = hashmap_report_final, }; + +const struct bench bench_bpf_rhashmap_lookup = { + .name = "bpf-rhashmap-lookup", + .argp = &bench_hashmap_lookup_argp, + .validate = validate, + .setup = rhash_setup, + .producer_thread = producer, + .measure = measure, + .report_progress = NULL, + .report_final = hashmap_report_final, +}; diff --git a/tools/testing/selftests/bpf/benchs/bench_bpf_nop.c b/tools/testing/selftests/bpf/benchs/bench_bpf_nop.c new file mode 100644 index 000000000000..e2d8c2ccf384 --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/bench_bpf_nop.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include "bench.h" +#include "bench_bpf_timing.h" +#include "bpf_nop_bench.skel.h" +#include "bpf_util.h" + +static struct ctx { + struct bpf_nop_bench *skel; + struct bpf_bench_timing timing; + int prog_fd; +} ctx; + +static void nop_validate(void) +{ + if (env.consumer_cnt != 0) { + fprintf(stderr, "benchmark doesn't support consumers\n"); + exit(1); + } +} + +static void nop_run_once(void *unused __always_unused) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + + bpf_prog_test_run_opts(ctx.prog_fd, &topts); +} + +static void nop_setup(void) +{ + struct bpf_nop_bench *skel; + int err; + + setup_libbpf(); + + skel = bpf_nop_bench__open(); + if (!skel) { + fprintf(stderr, "failed to open skeleton\n"); + exit(1); + } + + err = bpf_nop_bench__load(skel); + if (err) { + fprintf(stderr, "failed to load skeleton: %s\n", strerror(-err)); + bpf_nop_bench__destroy(skel); + exit(1); + } + + ctx.skel = skel; + ctx.prog_fd = bpf_program__fd(skel->progs.bench_nop); + + BENCH_TIMING_INIT(&ctx.timing, skel, 0); + bpf_bench_calibrate(&ctx.timing, nop_run_once, NULL); + + env.duration_sec = 600; +} + +static void *nop_producer(void *input) +{ + while (true) + nop_run_once(NULL); + + return NULL; +} + +static void nop_measure(struct bench_res *res) +{ + bpf_bench_timing_measure(&ctx.timing, res); +} + +static void nop_report_final(struct bench_res res[], int res_cnt) +{ + bpf_bench_timing_report(&ctx.timing, "bpf-nop", NULL); +} + +const struct bench bench_bpf_nop = { + .name = "bpf-nop", + .validate = nop_validate, + .setup = nop_setup, + .producer_thread = nop_producer, + .measure = nop_measure, + .report_final = nop_report_final, +}; diff --git a/tools/testing/selftests/bpf/benchs/bench_bpf_timing.c b/tools/testing/selftests/bpf/benchs/bench_bpf_timing.c new file mode 100644 index 000000000000..e02ad324f7bc --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/bench_bpf_timing.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <math.h> +#include "bench_bpf_timing.h" +#include "bpf_util.h" + +struct timing_stats { + double min, max; + double median, p99; + double mean, stddev; + int count; +}; + +static int cmp_double(const void *a, const void *b) +{ + double da = *(const double *)a; + double db = *(const double *)b; + + if (da < db) + return -1; + if (da > db) + return 1; + return 0; +} + +static double percentile(const double *sorted, int n, double pct) +{ + int idx = (int)(n * pct / 100.0); + + if (idx >= n) + idx = n - 1; + return sorted[idx]; +} + +static int collect_samples(struct bpf_bench_timing *t, + double *out, int max_out) +{ + unsigned int nr_cpus = bpf_num_possible_cpus(); + __u32 timed_iters = t->batch_iters; + int total = 0; + + if (nr_cpus > BENCH_NR_CPUS) + nr_cpus = BENCH_NR_CPUS; + + for (unsigned int cpu = 0; cpu < nr_cpus; cpu++) { + __u32 count = t->idx[cpu]; + + if (count > BENCH_NR_SAMPLES) + count = BENCH_NR_SAMPLES; + + for (__u32 i = 0; i < count && total < max_out; i++) { + __u64 sample = t->samples[cpu][i]; + + if (sample == 0) + continue; + out[total++] = (double)sample / timed_iters; + } + } + + qsort(out, total, sizeof(double), cmp_double); + return total; +} + +static int filter_outliers_iqr(double *sorted, int n) +{ + double q1, q3, iqr, lo, hi; + int start = 0, end = n; + + if (n < 8) + return n; + + q1 = sorted[n / 4]; + q3 = sorted[3 * n / 4]; + iqr = q3 - q1; + lo = q1 - 1.5 * iqr; + hi = q3 + 1.5 * iqr; + + while (start < end && sorted[start] < lo) + start++; + while (end > start && sorted[end - 1] > hi) + end--; + + if (start > 0) + memmove(sorted, sorted + start, (end - start) * sizeof(double)); + + return end - start; +} + +static void compute_stats(const double *sorted, int n, + struct timing_stats *s) +{ + double sum = 0, var_sum = 0; + + memset(s, 0, sizeof(*s)); + s->count = n; + + if (n == 0) + return; + + s->min = sorted[0]; + s->max = sorted[n - 1]; + s->median = sorted[n / 2]; + s->p99 = percentile(sorted, n, 99); + + for (int i = 0; i < n; i++) + sum += sorted[i]; + s->mean = sum / n; + + for (int i = 0; i < n; i++) { + double d = sorted[i] - s->mean; + + var_sum += d * d; + } + s->stddev = n > 1 ? sqrt(var_sum / (n - 1)) : 0; +} + +void bpf_bench_timing_measure(struct bpf_bench_timing *t, struct bench_res *res) +{ + unsigned int nr_cpus; + __u32 total_samples; + int i; + + t->warmup_ticks++; + + if (t->warmup_ticks < env.warmup_sec) + return; + + if (t->warmup_ticks == env.warmup_sec) { + *t->timing_enabled = 1; + return; + } + + nr_cpus = bpf_num_possible_cpus(); + if (nr_cpus > BENCH_NR_CPUS) + nr_cpus = BENCH_NR_CPUS; + + total_samples = 0; + for (i = 0; i < (int)nr_cpus; i++) { + __u32 cnt = t->idx[i]; + + if (cnt > BENCH_NR_SAMPLES) + cnt = BENCH_NR_SAMPLES; + total_samples += cnt; + } + + if (total_samples >= (__u32)env.producer_cnt * t->target_samples && !t->done) { + t->done = true; + *t->timing_enabled = 0; + bench_force_done(); + } +} + +void bpf_bench_timing_report(struct bpf_bench_timing *t, const char *name, const char *description) +{ + int max_out = BENCH_NR_CPUS * BENCH_NR_SAMPLES; + struct timing_stats s; + double *all; + int total; + + all = calloc(max_out, sizeof(*all)); + if (!all) { + fprintf(stderr, "failed to allocate timing buffer\n"); + return; + } + + total = collect_samples(t, all, max_out); + + if (total == 0) { + printf("No timing samples collected.\n"); + free(all); + return; + } + + total = filter_outliers_iqr(all, total); + compute_stats(all, total, &s); + + if (t->machine_readable) { + printf("RESULT scenario=%s samples=%d median=%.2f stddev=%.2f cv=%.2f min=%.2f " + "p99=%.2f max=%.2f\n", name, total, s.median, s.stddev, + s.mean > 0 ? s.stddev / s.mean * 100.0 : 0.0, s.min, s.p99, s.max); + } else { + printf("%s: median %.2f ns/op, stddev %.2f, p99 %.2f (%d samples)\n", name, + s.median, s.stddev, s.p99, total); + } + + free(all); +} + +#define CALIBRATE_SEED_BATCH 100 +#define CALIBRATE_MIN_BATCH 100 +#define CALIBRATE_MAX_BATCH 10000000 +#define CALIBRATE_TARGET_MS 10 +#define CALIBRATE_RUNS 5 +#define PROPORTIONALITY_TOL 0.05 /* 5% */ + +static void reset_timing(struct bpf_bench_timing *t) +{ + *t->timing_enabled = 0; + memset(t->samples, 0, sizeof(__u64) * BENCH_NR_CPUS * BENCH_NR_SAMPLES); + memset(t->idx, 0, sizeof(__u32) * BENCH_NR_CPUS); +} + +static __u64 measure_elapsed(struct bpf_bench_timing *t, bpf_bench_run_fn run_fn, void *run_ctx, + __u32 iters, int runs) +{ + __u64 buf[CALIBRATE_RUNS]; + int n = 0, i, j; + + reset_timing(t); + *t->batch_iters_bss = iters; + *t->timing_enabled = 1; + + for (i = 0; i < runs; i++) + run_fn(run_ctx); + + *t->timing_enabled = 0; + + for (i = 0; i < BENCH_NR_CPUS && n < runs; i++) { + __u32 cnt = t->idx[i]; + + for (j = 0; j < (int)cnt && n < runs; j++) + buf[n++] = t->samples[i][j]; + } + + if (n == 0) + return 0; + + for (i = 1; i < n; i++) { + __u64 key = buf[i]; + + j = i - 1; + while (j >= 0 && buf[j] > key) { + buf[j + 1] = buf[j]; + j--; + } + buf[j + 1] = key; + } + + return buf[n / 2]; +} + +static __u32 compute_batch_iters(__u64 per_op_ns) +{ + __u64 target_ns = (__u64)CALIBRATE_TARGET_MS * 1000000ULL; + __u32 iters; + + if (per_op_ns == 0) + return CALIBRATE_MIN_BATCH; + + iters = target_ns / per_op_ns; + + if (iters < CALIBRATE_MIN_BATCH) + iters = CALIBRATE_MIN_BATCH; + if (iters > CALIBRATE_MAX_BATCH) + iters = CALIBRATE_MAX_BATCH; + + return iters; +} + +void bpf_bench_calibrate(struct bpf_bench_timing *t, bpf_bench_run_fn run_fn, void *run_ctx) +{ + __u64 elapsed, per_op_ns; + __u64 time_n, time_2n; + double ratio; + + elapsed = measure_elapsed(t, run_fn, run_ctx, CALIBRATE_SEED_BATCH, CALIBRATE_RUNS); + if (elapsed == 0) { + fprintf(stderr, "calibration: no timing samples, using default\n"); + t->batch_iters = 10000; + *t->batch_iters_bss = t->batch_iters; + reset_timing(t); + return; + } + + per_op_ns = elapsed / CALIBRATE_SEED_BATCH; + t->batch_iters = compute_batch_iters(per_op_ns); + + time_n = measure_elapsed(t, run_fn, run_ctx, t->batch_iters, CALIBRATE_RUNS); + time_2n = measure_elapsed(t, run_fn, run_ctx, t->batch_iters * 2, CALIBRATE_RUNS); + + if (time_n > 0 && time_2n > 0) { + ratio = (double)time_2n / (double)time_n; + + if (fabs(ratio - 2.0) / 2.0 > PROPORTIONALITY_TOL) + fprintf(stderr, + "WARNING: proportionality check failed (2N/N ratio=%.3f, " + "expected=2.000, error=%.1f%%)\n System noise may be affecting " + "results.\n", + ratio, fabs(ratio - 2.0) / 2.0 * 100.0); + } + + *t->batch_iters_bss = t->batch_iters; + reset_timing(t); +} diff --git a/tools/testing/selftests/bpf/benchs/bench_htab_mem.c b/tools/testing/selftests/bpf/benchs/bench_htab_mem.c index 297e32390cd1..1ee217d97434 100644 --- a/tools/testing/selftests/bpf/benchs/bench_htab_mem.c +++ b/tools/testing/selftests/bpf/benchs/bench_htab_mem.c @@ -152,7 +152,7 @@ static const struct htab_mem_use_case *htab_mem_find_use_case_or_exit(const char exit(1); } -static void htab_mem_setup(void) +static void htab_mem_setup_impl(enum bpf_map_type map_type) { struct bpf_map *map; const char **names; @@ -178,10 +178,11 @@ static void htab_mem_setup(void) } map = ctx.skel->maps.htab; + bpf_map__set_type(map, map_type); bpf_map__set_value_size(map, args.value_size); /* Ensure that different CPUs can operate on different subset */ bpf_map__set_max_entries(map, MAX(8192, 64 * env.nr_cpus)); - if (args.preallocated) + if (map_type != BPF_MAP_TYPE_RHASH && args.preallocated) bpf_map__set_map_flags(map, bpf_map__map_flags(map) & ~BPF_F_NO_PREALLOC); names = ctx.uc->progs; @@ -220,6 +221,16 @@ cleanup: exit(1); } +static void htab_mem_setup(void) +{ + htab_mem_setup_impl(BPF_MAP_TYPE_HASH); +} + +static void rhtab_mem_setup(void) +{ + htab_mem_setup_impl(BPF_MAP_TYPE_RHASH); +} + static void htab_mem_add_fn(pthread_barrier_t *notify) { while (true) { @@ -338,6 +349,15 @@ static void htab_mem_report_final(struct bench_res res[], int res_cnt) cleanup_cgroup_environment(); } +static void rhtab_mem_validate(void) +{ + if (args.preallocated) { + fprintf(stderr, "rhash map does not support preallocation\n"); + exit(1); + } + htab_mem_validate(); +} + const struct bench bench_htab_mem = { .name = "htab-mem", .argp = &bench_htab_mem_argp, @@ -348,3 +368,14 @@ const struct bench bench_htab_mem = { .report_progress = htab_mem_report_progress, .report_final = htab_mem_report_final, }; + +const struct bench bench_rhtab_mem = { + .name = "rhtab-mem", + .argp = &bench_htab_mem_argp, + .validate = rhtab_mem_validate, + .setup = rhtab_mem_setup, + .producer_thread = htab_mem_producer, + .measure = htab_mem_measure, + .report_progress = htab_mem_report_progress, + .report_final = htab_mem_report_final, +}; diff --git a/tools/testing/selftests/bpf/benchs/bench_local_storage_create.c b/tools/testing/selftests/bpf/benchs/bench_local_storage_create.c index e2ff8ea1cb79..71e38000ee06 100644 --- a/tools/testing/selftests/bpf/benchs/bench_local_storage_create.c +++ b/tools/testing/selftests/bpf/benchs/bench_local_storage_create.c @@ -101,11 +101,6 @@ static void setup(void) } } - if (!bpf_program__attach(skel->progs.kmalloc)) { - fprintf(stderr, "Error attaching bpf program\n"); - exit(1); - } - threads = calloc(env.producer_cnt, sizeof(*threads)); if (!threads) { @@ -140,7 +135,6 @@ static void setup(void) static void measure(struct bench_res *res) { res->hits = atomic_swap(&skel->bss->create_cnts, 0); - res->drops = atomic_swap(&skel->bss->kmalloc_cnts, 0); } static void *sk_producer(void *input) @@ -203,28 +197,25 @@ static void *producer(void *input) static void report_progress(int iter, struct bench_res *res, long delta_ns) { - double creates_per_sec, kmallocs_per_create; + double creates_per_sec; creates_per_sec = res->hits / 1000.0 / (delta_ns / 1000000000.0); - kmallocs_per_create = (double)res->drops / res->hits; printf("Iter %3d (%7.3lfus): ", iter, (delta_ns - 1000000000) / 1000.0); - printf("creates %8.3lfk/s (%7.3lfk/prod), ", + printf("creates %8.3lfk/s (%7.3lfk/prod)\n", creates_per_sec, creates_per_sec / env.producer_cnt); - printf("%3.2lf kmallocs/create\n", kmallocs_per_create); } static void report_final(struct bench_res res[], int res_cnt) { double creates_mean = 0.0, creates_stddev = 0.0; - long total_creates = 0, total_kmallocs = 0; + long total_creates = 0; int i; for (i = 0; i < res_cnt; i++) { creates_mean += res[i].hits / 1000.0 / (0.0 + res_cnt); total_creates += res[i].hits; - total_kmallocs += res[i].drops; } if (res_cnt > 1) { @@ -234,9 +225,9 @@ static void report_final(struct bench_res res[], int res_cnt) (res_cnt - 1.0); creates_stddev = sqrt(creates_stddev); } - printf("Summary: creates %8.3lf \u00B1 %5.3lfk/s (%7.3lfk/prod), ", - creates_mean, creates_stddev, creates_mean / env.producer_cnt); - printf("%4.2lf kmallocs/create\n", (double)total_kmallocs / total_creates); + printf("Summary: creates %8.3lf \u00B1 %5.3lfk/s (%7.3lfk/prod), %ld total\n", + creates_mean, creates_stddev, creates_mean / env.producer_cnt, + total_creates); if (create_owner_errs || skel->bss->create_errs) printf("%s() errors %ld create_errs %ld\n", storage_type == BPF_MAP_TYPE_SK_STORAGE ? diff --git a/tools/testing/selftests/bpf/benchs/bench_trigger.c b/tools/testing/selftests/bpf/benchs/bench_trigger.c index f74b313d6ae4..61513efc167a 100644 --- a/tools/testing/selftests/bpf/benchs/bench_trigger.c +++ b/tools/testing/selftests/bpf/benchs/bench_trigger.c @@ -396,15 +396,32 @@ static void *uprobe_producer_ret(void *input) } #ifdef __x86_64__ -__nocf_check __weak void uprobe_target_nop5(void) +__nocf_check __weak void uprobe_target_nop10(void) { - asm volatile (".byte 0x0f, 0x1f, 0x44, 0x00, 0x00"); + asm volatile (".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00"); } -static void *uprobe_producer_nop5(void *input) +static void *uprobe_producer_nop10(void *input) { while (true) - uprobe_target_nop5(); + uprobe_target_nop10(); + return NULL; +} + +void usdt_1(void); +void usdt_2(void); + +static void *uprobe_producer_usdt_nop(void *input) +{ + while (true) + usdt_1(); + return NULL; +} + +static void *uprobe_producer_usdt_nop10(void *input) +{ + while (true) + usdt_2(); return NULL; } #endif @@ -525,24 +542,65 @@ static void uretprobe_multi_ret_setup(void) } #ifdef __x86_64__ -static void uprobe_nop5_setup(void) +static void uprobe_nop10_setup(void) +{ + usetup(false, false /* !use_multi */, &uprobe_target_nop10); +} + +static void uretprobe_nop10_setup(void) +{ + usetup(true, false /* !use_multi */, &uprobe_target_nop10); +} + +static void uprobe_multi_nop10_setup(void) +{ + usetup(false, true /* use_multi */, &uprobe_target_nop10); +} + +static void uretprobe_multi_nop10_setup(void) { - usetup(false, false /* !use_multi */, &uprobe_target_nop5); + usetup(true, true /* use_multi */, &uprobe_target_nop10); } -static void uretprobe_nop5_setup(void) +static void usdt_setup(const char *name) { - usetup(true, false /* !use_multi */, &uprobe_target_nop5); + struct bpf_link *link; + int err; + + setup_libbpf(); + + ctx.skel = trigger_bench__open(); + if (!ctx.skel) { + fprintf(stderr, "failed to open skeleton\n"); + exit(1); + } + + bpf_program__set_autoload(ctx.skel->progs.bench_trigger_usdt, true); + + err = trigger_bench__load(ctx.skel); + if (err) { + fprintf(stderr, "failed to load skeleton\n"); + exit(1); + } + + link = bpf_program__attach_usdt(ctx.skel->progs.bench_trigger_usdt, + 0 /*self*/, "/proc/self/exe", + "optimized_attach", name, NULL); + if (libbpf_get_error(link)) { + fprintf(stderr, "failed to attach optimized_attach:%s usdt probe\n", name); + exit(1); + } + ctx.skel->links.bench_trigger_usdt = link; } -static void uprobe_multi_nop5_setup(void) +static void usdt_nop_setup(void) { - usetup(false, true /* use_multi */, &uprobe_target_nop5); + usdt_setup("usdt_1"); } -static void uretprobe_multi_nop5_setup(void) +static void usdt_nop10_setup(void) { - usetup(true, true /* use_multi */, &uprobe_target_nop5); + usdt_setup("usdt_2"); } #endif @@ -607,8 +665,10 @@ BENCH_TRIG_USERMODE(uretprobe_multi_nop, nop, "uretprobe-multi-nop"); BENCH_TRIG_USERMODE(uretprobe_multi_push, push, "uretprobe-multi-push"); BENCH_TRIG_USERMODE(uretprobe_multi_ret, ret, "uretprobe-multi-ret"); #ifdef __x86_64__ -BENCH_TRIG_USERMODE(uprobe_nop5, nop5, "uprobe-nop5"); -BENCH_TRIG_USERMODE(uretprobe_nop5, nop5, "uretprobe-nop5"); -BENCH_TRIG_USERMODE(uprobe_multi_nop5, nop5, "uprobe-multi-nop5"); -BENCH_TRIG_USERMODE(uretprobe_multi_nop5, nop5, "uretprobe-multi-nop5"); +BENCH_TRIG_USERMODE(uprobe_nop10, nop10, "uprobe-nop10"); +BENCH_TRIG_USERMODE(uretprobe_nop10, nop10, "uretprobe-nop10"); +BENCH_TRIG_USERMODE(uprobe_multi_nop10, nop10, "uprobe-multi-nop10"); +BENCH_TRIG_USERMODE(uretprobe_multi_nop10, nop10, "uretprobe-multi-nop10"); +BENCH_TRIG_USERMODE(usdt_nop, usdt_nop, "usdt-nop"); +BENCH_TRIG_USERMODE(usdt_nop10, usdt_nop10, "usdt-nop10"); #endif diff --git a/tools/testing/selftests/bpf/benchs/bench_xdp_lb.c b/tools/testing/selftests/bpf/benchs/bench_xdp_lb.c new file mode 100644 index 000000000000..8e25bccbde92 --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/bench_xdp_lb.c @@ -0,0 +1,1124 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <argp.h> +#include <string.h> +#include <arpa/inet.h> +#include <linux/if_ether.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/in.h> +#include <linux/tcp.h> +#include <linux/udp.h> +#include "bench.h" +#include "bench_bpf_timing.h" +#include "xdp_lb_bench.skel.h" +#include "xdp_lb_bench_common.h" +#include "bpf_util.h" + +#define IP4(a, b, c, d) (((__u32)(a) << 24) | ((__u32)(b) << 16) | ((__u32)(c) << 8) | (__u32)(d)) + +#define IP6(a, b, c, d) { (__u32)(a), (__u32)(b), (__u32)(c), (__u32)(d) } + +#define TNL_DST IP4(192, 168, 1, 2) +#define REAL_INDEX 1 +#define REAL_INDEX_V6 2 +#define MAX_PKT_SIZE 256 +#define IP_MF 0x2000 + +static const __u32 tnl_dst_v6[4] = { 0xfd000000, 0, 0, 2 }; + +static const __u8 lb_mac[ETH_ALEN] = {0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}; +static const __u8 client_mac[ETH_ALEN] = {0x11, 0x22, 0x33, 0x44, 0x55, 0x66}; +static const __u8 router_mac[ETH_ALEN] = {0xde, 0xad, 0xbe, 0xef, 0x00, 0x01}; + +enum scenario_id { + S_TCP_V4_LRU_HIT, + S_TCP_V4_CH, + S_TCP_V6_LRU_HIT, + S_TCP_V6_CH, + S_UDP_V4_LRU_HIT, + S_UDP_V6_LRU_HIT, + S_TCP_V4V6_LRU_HIT, + S_TCP_V4_LRU_DIVERSE, + S_TCP_V4_CH_DIVERSE, + S_TCP_V6_LRU_DIVERSE, + S_TCP_V6_CH_DIVERSE, + S_UDP_V4_LRU_DIVERSE, + S_TCP_V4_LRU_MISS, + S_UDP_V4_LRU_MISS, + S_TCP_V4_LRU_WARMUP, + S_TCP_V4_SYN, + S_TCP_V4_RST_MISS, + S_PASS_V4_NO_VIP, + S_PASS_V6_NO_VIP, + S_PASS_V4_ICMP, + S_PASS_NON_IP, + S_DROP_V4_FRAG, + S_DROP_V4_OPTIONS, + S_DROP_V6_FRAG, + NUM_SCENARIOS, +}; + +enum lru_miss_type { + LRU_MISS_AUTO = 0, /* compute from scenario flags (default) */ + LRU_MISS_NONE, /* 0 misses (all LRU hits) */ + LRU_MISS_ALL, /* batch_iters+1 misses (every op misses) */ + LRU_MISS_FIRST, /* 1 miss (first miss, then hits) */ +}; + +#define S_BASE_ENCAP_V4 \ + .expected_retval = XDP_TX, .expect_encap = true, \ + .tunnel_dst = TNL_DST + +#define S_BASE_ENCAP_V6 \ + .expected_retval = XDP_TX, .expect_encap = true, \ + .is_v6 = true, .encap_v6_outer = true, \ + .tunnel_dst_v6 = { 0xfd000000, 0, 0, 2 } + +#define S_BASE_ENCAP_V4V6 \ + .expected_retval = XDP_TX, .expect_encap = true, \ + .encap_v6_outer = true, \ + .tunnel_dst_v6 = { 0xfd000000, 0, 0, 2 } + +struct test_scenario { + const char *name; + const char *description; + int expected_retval; + bool expect_encap; + bool is_v6; + __u32 vip_addr; + __u32 src_addr; + __u32 tunnel_dst; + __u32 vip_addr_v6[4]; + __u32 src_addr_v6[4]; + __u32 tunnel_dst_v6[4]; + __u16 dst_port; + __u16 src_port; + __u8 ip_proto; + __u32 vip_flags; + __u32 vip_num; + bool prepopulate_lru; + bool set_frag; + __u16 eth_proto; + bool encap_v6_outer; + __u32 flow_mask; + bool cold_lru; + bool set_syn; + bool set_rst; + bool set_ip_options; + __u32 fixed_batch_iters; /* 0 = auto-calibrate, >0 = use this value */ + enum lru_miss_type lru_miss; /* expected LRU miss pattern */ +}; + +static const struct test_scenario scenarios[NUM_SCENARIOS] = { + /* Single-flow baseline */ + [S_TCP_V4_LRU_HIT] = { + S_BASE_ENCAP_V4, .ip_proto = IPPROTO_TCP, + .name = "tcp-v4-lru-hit", + .description = "IPv4 TCP, LRU hit, IPIP encap", + .vip_addr = IP4(10, 10, 1, 1), .dst_port = 80, + .src_addr = IP4(10, 10, 2, 1), .src_port = 12345, + .prepopulate_lru = true, .lru_miss = LRU_MISS_NONE, + }, + [S_TCP_V4_CH] = { + S_BASE_ENCAP_V4, .ip_proto = IPPROTO_TCP, + .name = "tcp-v4-ch", + .description = "IPv4 TCP, CH (LRU bypass), IPIP encap", + .vip_addr = IP4(10, 10, 1, 2), .dst_port = 80, + .src_addr = IP4(10, 10, 2, 2), .src_port = 54321, + .vip_flags = F_LRU_BYPASS, .vip_num = 1, + .lru_miss = LRU_MISS_ALL, + }, + [S_TCP_V6_LRU_HIT] = { + S_BASE_ENCAP_V6, .ip_proto = IPPROTO_TCP, + .name = "tcp-v6-lru-hit", + .description = "IPv6 TCP, LRU hit, IP6IP6 encap", + .vip_addr_v6 = IP6(0xfd000100, 0, 0, 1), .dst_port = 80, + .src_addr_v6 = IP6(0xfd000200, 0, 0, 1), .src_port = 12345, + .vip_num = 10, + .prepopulate_lru = true, .lru_miss = LRU_MISS_NONE, + }, + [S_TCP_V6_CH] = { + S_BASE_ENCAP_V6, .ip_proto = IPPROTO_TCP, + .name = "tcp-v6-ch", + .description = "IPv6 TCP, CH (LRU bypass), IP6IP6 encap", + .vip_addr_v6 = IP6(0xfd000100, 0, 0, 2), .dst_port = 80, + .src_addr_v6 = IP6(0xfd000200, 0, 0, 2), .src_port = 54321, + .vip_flags = F_LRU_BYPASS, .vip_num = 12, + .lru_miss = LRU_MISS_ALL, + }, + [S_UDP_V4_LRU_HIT] = { + S_BASE_ENCAP_V4, .ip_proto = IPPROTO_UDP, + .name = "udp-v4-lru-hit", + .description = "IPv4 UDP, LRU hit, IPIP encap", + .vip_addr = IP4(10, 10, 1, 1), .dst_port = 443, + .src_addr = IP4(10, 10, 3, 1), .src_port = 11111, + .vip_num = 2, + .prepopulate_lru = true, .lru_miss = LRU_MISS_NONE, + }, + [S_UDP_V6_LRU_HIT] = { + S_BASE_ENCAP_V6, .ip_proto = IPPROTO_UDP, + .name = "udp-v6-lru-hit", + .description = "IPv6 UDP, LRU hit, IP6IP6 encap", + .vip_addr_v6 = IP6(0xfd000100, 0, 0, 1), .dst_port = 443, + .src_addr_v6 = IP6(0xfd000200, 0, 0, 3), .src_port = 22222, + .vip_num = 14, + .prepopulate_lru = true, .lru_miss = LRU_MISS_NONE, + }, + [S_TCP_V4V6_LRU_HIT] = { + S_BASE_ENCAP_V4V6, .ip_proto = IPPROTO_TCP, + .name = "tcp-v4v6-lru-hit", + .description = "IPv4 TCP, LRU hit, IPv4-in-IPv6 encap", + .vip_addr = IP4(10, 10, 1, 4), .dst_port = 80, + .src_addr = IP4(10, 10, 2, 4), .src_port = 12347, + .vip_num = 13, + .prepopulate_lru = true, .lru_miss = LRU_MISS_NONE, + }, + + /* Diverse flows (4K src addrs) */ + [S_TCP_V4_LRU_DIVERSE] = { + S_BASE_ENCAP_V4, .ip_proto = IPPROTO_TCP, + .name = "tcp-v4-lru-diverse", + .description = "IPv4 TCP, diverse flows, warm LRU", + .vip_addr = IP4(10, 10, 1, 1), .dst_port = 80, + .src_addr = IP4(10, 10, 2, 1), .src_port = 12345, + .prepopulate_lru = true, .flow_mask = 0xFFF, + .lru_miss = LRU_MISS_NONE, + }, + [S_TCP_V4_CH_DIVERSE] = { + S_BASE_ENCAP_V4, .ip_proto = IPPROTO_TCP, + .name = "tcp-v4-ch-diverse", + .description = "IPv4 TCP, diverse flows, CH (LRU bypass)", + .vip_addr = IP4(10, 10, 1, 2), .dst_port = 80, + .src_addr = IP4(10, 10, 2, 2), .src_port = 54321, + .vip_flags = F_LRU_BYPASS, .vip_num = 1, + .flow_mask = 0xFFF, .lru_miss = LRU_MISS_ALL, + }, + [S_TCP_V6_LRU_DIVERSE] = { + S_BASE_ENCAP_V6, .ip_proto = IPPROTO_TCP, + .name = "tcp-v6-lru-diverse", + .description = "IPv6 TCP, diverse flows, warm LRU", + .vip_addr_v6 = IP6(0xfd000100, 0, 0, 1), .dst_port = 80, + .src_addr_v6 = IP6(0xfd000200, 0, 0, 1), .src_port = 12345, + .vip_num = 10, + .prepopulate_lru = true, .flow_mask = 0xFFF, + .lru_miss = LRU_MISS_NONE, + }, + [S_TCP_V6_CH_DIVERSE] = { + S_BASE_ENCAP_V6, .ip_proto = IPPROTO_TCP, + .name = "tcp-v6-ch-diverse", + .description = "IPv6 TCP, diverse flows, CH (LRU bypass)", + .vip_addr_v6 = IP6(0xfd000100, 0, 0, 2), .dst_port = 80, + .src_addr_v6 = IP6(0xfd000200, 0, 0, 2), .src_port = 54321, + .vip_flags = F_LRU_BYPASS, .vip_num = 12, + .flow_mask = 0xFFF, .lru_miss = LRU_MISS_ALL, + }, + [S_UDP_V4_LRU_DIVERSE] = { + S_BASE_ENCAP_V4, .ip_proto = IPPROTO_UDP, + .name = "udp-v4-lru-diverse", + .description = "IPv4 UDP, diverse flows, warm LRU", + .vip_addr = IP4(10, 10, 1, 1), .dst_port = 443, + .src_addr = IP4(10, 10, 3, 1), .src_port = 11111, + .vip_num = 2, + .prepopulate_lru = true, .flow_mask = 0xFFF, + .lru_miss = LRU_MISS_NONE, + }, + + /* LRU stress */ + [S_TCP_V4_LRU_MISS] = { + S_BASE_ENCAP_V4, .ip_proto = IPPROTO_TCP, + .name = "tcp-v4-lru-miss", + .description = "IPv4 TCP, LRU miss (16M flow space), CH lookup", + .vip_addr = IP4(10, 10, 1, 1), .dst_port = 80, + .src_addr = IP4(10, 10, 2, 1), .src_port = 12345, + .flow_mask = 0xFFFFFF, .cold_lru = true, + .lru_miss = LRU_MISS_FIRST, + }, + [S_UDP_V4_LRU_MISS] = { + S_BASE_ENCAP_V4, .ip_proto = IPPROTO_UDP, + .name = "udp-v4-lru-miss", + .description = "IPv4 UDP, LRU miss (16M flow space), CH lookup", + .vip_addr = IP4(10, 10, 1, 1), .dst_port = 443, + .src_addr = IP4(10, 10, 3, 1), .src_port = 11111, + .vip_num = 2, + .flow_mask = 0xFFFFFF, .cold_lru = true, + .lru_miss = LRU_MISS_FIRST, + }, + [S_TCP_V4_LRU_WARMUP] = { + S_BASE_ENCAP_V4, .ip_proto = IPPROTO_TCP, + .name = "tcp-v4-lru-warmup", + .description = "IPv4 TCP, 4K flows, ~50% LRU miss", + .vip_addr = IP4(10, 10, 1, 1), .dst_port = 80, + .src_addr = IP4(10, 10, 2, 1), .src_port = 12345, + .flow_mask = 0xFFF, .cold_lru = true, + .fixed_batch_iters = 6500, + .lru_miss = LRU_MISS_FIRST, + }, + + /* TCP flags */ + [S_TCP_V4_SYN] = { + S_BASE_ENCAP_V4, .ip_proto = IPPROTO_TCP, + .name = "tcp-v4-syn", + .description = "IPv4 TCP SYN, skip LRU, CH + LRU insert", + .vip_addr = IP4(10, 10, 1, 1), .dst_port = 80, + .src_addr = IP4(10, 10, 8, 2), .src_port = 60001, + .set_syn = true, .lru_miss = LRU_MISS_ALL, + }, + [S_TCP_V4_RST_MISS] = { + S_BASE_ENCAP_V4, .ip_proto = IPPROTO_TCP, + .name = "tcp-v4-rst-miss", + .description = "IPv4 TCP RST, CH lookup, no LRU insert", + .vip_addr = IP4(10, 10, 1, 1), .dst_port = 80, + .src_addr = IP4(10, 10, 8, 1), .src_port = 60000, + .flow_mask = 0xFFFFFF, .cold_lru = true, + .set_rst = true, .lru_miss = LRU_MISS_ALL, + }, + + /* Early exits */ + [S_PASS_V4_NO_VIP] = { + .name = "pass-v4-no-vip", + .description = "IPv4 TCP, unknown VIP, XDP_PASS", + .expected_retval = XDP_PASS, + .ip_proto = IPPROTO_TCP, + .vip_addr = IP4(10, 10, 9, 9), .dst_port = 80, + .src_addr = IP4(10, 10, 4, 1), .src_port = 33333, + }, + [S_PASS_V6_NO_VIP] = { + .name = "pass-v6-no-vip", + .description = "IPv6 TCP, unknown VIP, XDP_PASS", + .expected_retval = XDP_PASS, .is_v6 = true, + .ip_proto = IPPROTO_TCP, + .vip_addr_v6 = IP6(0xfd009900, 0, 0, 1), .dst_port = 80, + .src_addr_v6 = IP6(0xfd000400, 0, 0, 1), .src_port = 33333, + }, + [S_PASS_V4_ICMP] = { + .name = "pass-v4-icmp", + .description = "IPv4 ICMP, non-TCP/UDP protocol, XDP_PASS", + .expected_retval = XDP_PASS, + .ip_proto = IPPROTO_ICMP, + .vip_addr = IP4(10, 10, 1, 1), + .src_addr = IP4(10, 10, 6, 1), + }, + [S_PASS_NON_IP] = { + .name = "pass-non-ip", + .description = "Non-IP (ARP), earliest XDP_PASS exit", + .expected_retval = XDP_PASS, + .eth_proto = ETH_P_ARP, + }, + [S_DROP_V4_FRAG] = { + .name = "drop-v4-frag", + .description = "IPv4 fragmented, XDP_DROP", + .expected_retval = XDP_DROP, .ip_proto = IPPROTO_TCP, + .vip_addr = IP4(10, 10, 1, 1), .dst_port = 80, + .src_addr = IP4(10, 10, 5, 1), .src_port = 44444, + .set_frag = true, + }, + [S_DROP_V4_OPTIONS] = { + .name = "drop-v4-options", + .description = "IPv4 with IP options (ihl>5), XDP_DROP", + .expected_retval = XDP_DROP, .ip_proto = IPPROTO_TCP, + .vip_addr = IP4(10, 10, 1, 1), .dst_port = 80, + .src_addr = IP4(10, 10, 7, 1), .src_port = 55555, + .set_ip_options = true, + }, + [S_DROP_V6_FRAG] = { + .name = "drop-v6-frag", + .description = "IPv6 fragment extension header, XDP_DROP", + .expected_retval = XDP_DROP, .is_v6 = true, + .ip_proto = IPPROTO_TCP, + .vip_addr_v6 = IP6(0xfd000100, 0, 0, 1), .dst_port = 80, + .src_addr_v6 = IP6(0xfd000500, 0, 0, 1), .src_port = 44444, + .set_frag = true, + }, +}; + +#define MAX_ENCAP_SIZE (MAX_PKT_SIZE + sizeof(struct ipv6hdr)) + +static __u8 pkt_buf[NUM_SCENARIOS][MAX_PKT_SIZE]; +static __u32 pkt_len[NUM_SCENARIOS]; +static __u8 expected_buf[NUM_SCENARIOS][MAX_ENCAP_SIZE]; +static __u32 expected_len[NUM_SCENARIOS]; + +static int lru_inner_fds[BENCH_NR_CPUS]; +static int nr_inner_maps; + +static struct ctx { + struct xdp_lb_bench *skel; + struct bpf_bench_timing timing; + int prog_fd; +} ctx; + +static struct { + int scenario; + bool machine_readable; +} args = { + .scenario = -1, +}; + +static __u16 ip_checksum(const void *hdr, int len) +{ + const __u16 *p = hdr; + __u32 csum = 0; + int i; + + for (i = 0; i < len / 2; i++) + csum += p[i]; + + while (csum >> 16) + csum = (csum & 0xffff) + (csum >> 16); + + return ~csum; +} + +static void htonl_v6(__be32 dst[4], const __u32 src[4]) +{ + int i; + + for (i = 0; i < 4; i++) + dst[i] = htonl(src[i]); +} + +static void build_flow_key(struct flow_key *fk, const struct test_scenario *sc) +{ + memset(fk, 0, sizeof(*fk)); + if (sc->is_v6) { + htonl_v6(fk->srcv6, sc->src_addr_v6); + htonl_v6(fk->dstv6, sc->vip_addr_v6); + } else { + fk->src = htonl(sc->src_addr); + fk->dst = htonl(sc->vip_addr); + } + fk->proto = sc->ip_proto; + fk->port16[0] = htons(sc->src_port); + fk->port16[1] = htons(sc->dst_port); +} + +static void build_l4(const struct test_scenario *sc, __u8 *p, __u32 *off) +{ + if (sc->ip_proto == IPPROTO_TCP) { + struct tcphdr tcp = {}; + + tcp.source = htons(sc->src_port); + tcp.dest = htons(sc->dst_port); + tcp.doff = 5; + tcp.syn = sc->set_syn ? 1 : 0; + tcp.rst = sc->set_rst ? 1 : 0; + tcp.window = htons(8192); + memcpy(p + *off, &tcp, sizeof(tcp)); + *off += sizeof(tcp); + } else if (sc->ip_proto == IPPROTO_UDP) { + struct udphdr udp = {}; + + udp.source = htons(sc->src_port); + udp.dest = htons(sc->dst_port); + udp.len = htons(sizeof(udp) + 16); + memcpy(p + *off, &udp, sizeof(udp)); + *off += sizeof(udp); + } +} + +static void build_packet(int idx) +{ + const struct test_scenario *sc = &scenarios[idx]; + __u8 *p = pkt_buf[idx]; + struct ethhdr eth = {}; + __u16 proto; + __u32 off = 0; + + memcpy(eth.h_dest, lb_mac, ETH_ALEN); + memcpy(eth.h_source, client_mac, ETH_ALEN); + + if (sc->eth_proto) + proto = sc->eth_proto; + else if (sc->is_v6) + proto = ETH_P_IPV6; + else + proto = ETH_P_IP; + + eth.h_proto = htons(proto); + memcpy(p, ð, sizeof(eth)); + off += sizeof(eth); + + if (proto != ETH_P_IP && proto != ETH_P_IPV6) { + memcpy(p + off, "bench___payload!", 16); + off += 16; + pkt_len[idx] = off; + return; + } + + if (sc->is_v6) { + struct ipv6hdr ip6h = {}; + __u32 ip6_off = off; + + ip6h.version = 6; + ip6h.nexthdr = sc->set_frag ? 44 : sc->ip_proto; + ip6h.hop_limit = 64; + htonl_v6((__be32 *)&ip6h.saddr, sc->src_addr_v6); + htonl_v6((__be32 *)&ip6h.daddr, sc->vip_addr_v6); + off += sizeof(ip6h); + + if (sc->set_frag) { + memset(p + off, 0, 8); + p[off] = sc->ip_proto; + off += 8; + } + + build_l4(sc, p, &off); + + memcpy(p + off, "bench___payload!", 16); + off += 16; + + ip6h.payload_len = htons(off - ip6_off - sizeof(ip6h)); + memcpy(p + ip6_off, &ip6h, sizeof(ip6h)); + } else { + struct iphdr iph = {}; + __u32 ip_off = off; + + iph.version = 4; + iph.ihl = sc->set_ip_options ? 6 : 5; + iph.ttl = 64; + iph.protocol = sc->ip_proto; + iph.saddr = htonl(sc->src_addr); + iph.daddr = htonl(sc->vip_addr); + iph.frag_off = sc->set_frag ? htons(IP_MF) : 0; + off += sizeof(iph); + + if (sc->set_ip_options) { + /* NOP option padding (4 bytes = 1 word) */ + __u32 nop = htonl(0x01010101); + + memcpy(p + off, &nop, sizeof(nop)); + off += sizeof(nop); + } + + build_l4(sc, p, &off); + + memcpy(p + off, "bench___payload!", 16); + off += 16; + + iph.tot_len = htons(off - ip_off); + iph.check = ip_checksum(&iph, sizeof(iph)); + memcpy(p + ip_off, &iph, sizeof(iph)); + } + + pkt_len[idx] = off; +} + +static void populate_vip(struct xdp_lb_bench *skel, const struct test_scenario *sc) +{ + struct vip_definition key = {}; + struct vip_meta val = {}; + int err; + + if (sc->is_v6) + htonl_v6(key.vipv6, sc->vip_addr_v6); + else + key.vip = htonl(sc->vip_addr); + key.port = htons(sc->dst_port); + key.proto = sc->ip_proto; + val.flags = sc->vip_flags; + val.vip_num = sc->vip_num; + + err = bpf_map_update_elem(bpf_map__fd(skel->maps.vip_map), &key, &val, BPF_ANY); + if (err) { + fprintf(stderr, "vip_map [%s]: %s\n", sc->name, strerror(errno)); + exit(1); + } +} + +static void create_per_cpu_lru_maps(struct xdp_lb_bench *skel) +{ + int outer_fd = bpf_map__fd(skel->maps.lru_mapping); + unsigned int nr_cpus = bpf_num_possible_cpus(); + int i, inner_fd, err; + __u32 cpu; + + if (nr_cpus > BENCH_NR_CPUS) + nr_cpus = BENCH_NR_CPUS; + + for (i = 0; i < (int)nr_cpus; i++) { + LIBBPF_OPTS(bpf_map_create_opts, opts); + + inner_fd = bpf_map_create(BPF_MAP_TYPE_LRU_HASH, "lru_inner", + sizeof(struct flow_key), + sizeof(struct real_pos_lru), + DEFAULT_LRU_SIZE, &opts); + if (inner_fd < 0) { + fprintf(stderr, "lru_inner[%d]: %s\n", i, strerror(errno)); + exit(1); + } + + cpu = i; + err = bpf_map_update_elem(outer_fd, &cpu, &inner_fd, BPF_ANY); + if (err) { + fprintf(stderr, "lru_mapping[%d]: %s\n", i, strerror(errno)); + close(inner_fd); + exit(1); + } + + lru_inner_fds[i] = inner_fd; + } + + nr_inner_maps = nr_cpus; +} + +static __u64 ktime_get_ns(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + return (__u64)ts.tv_sec * 1000000000ULL + ts.tv_nsec; +} + +static void populate_lru(const struct test_scenario *sc, __u32 real_idx) +{ + struct real_pos_lru lru = { .pos = real_idx }; + struct flow_key fk; + int i, err; + + if (sc->ip_proto == IPPROTO_UDP) + lru.atime = ktime_get_ns(); + + build_flow_key(&fk, sc); + + /* Insert into every per-CPU inner LRU so the entry is found + * regardless of which CPU runs the BPF program. + */ + for (i = 0; i < nr_inner_maps; i++) { + err = bpf_map_update_elem(lru_inner_fds[i], &fk, &lru, BPF_ANY); + if (err) { + fprintf(stderr, "lru_inner[%d] [%s]: %s\n", i, sc->name, + strerror(errno)); + exit(1); + } + } +} + +static void populate_maps(struct xdp_lb_bench *skel) +{ + struct real_definition real_v4 = {}; + struct real_definition real_v6 = {}; + struct ctl_value cval = {}; + __u32 key, real_idx = REAL_INDEX; + int ch_fd, err, i; + + if (scenarios[args.scenario].expect_encap) + populate_vip(skel, &scenarios[args.scenario]); + + ch_fd = bpf_map__fd(skel->maps.ch_rings); + for (i = 0; i < CH_RINGS_SIZE; i++) { + __u32 k = i; + + err = bpf_map_update_elem(ch_fd, &k, &real_idx, BPF_ANY); + if (err) { + fprintf(stderr, "ch_rings[%d]: %s\n", i, strerror(errno)); + exit(1); + } + } + + memcpy(cval.mac, router_mac, ETH_ALEN); + key = 0; + err = bpf_map_update_elem(bpf_map__fd(skel->maps.ctl_array), &key, &cval, BPF_ANY); + if (err) { + fprintf(stderr, "ctl_array: %s\n", strerror(errno)); + exit(1); + } + + key = REAL_INDEX; + real_v4.dst = htonl(TNL_DST); + htonl_v6(real_v4.dstv6, tnl_dst_v6); + err = bpf_map_update_elem(bpf_map__fd(skel->maps.reals), &key, &real_v4, BPF_ANY); + if (err) { + fprintf(stderr, "reals[%d]: %s\n", REAL_INDEX, strerror(errno)); + exit(1); + } + + key = REAL_INDEX_V6; + htonl_v6(real_v6.dstv6, tnl_dst_v6); + real_v6.flags = F_IPV6; + err = bpf_map_update_elem(bpf_map__fd(skel->maps.reals), &key, &real_v6, BPF_ANY); + if (err) { + fprintf(stderr, "reals[%d]: %s\n", REAL_INDEX_V6, strerror(errno)); + exit(1); + } + + create_per_cpu_lru_maps(skel); + + if (scenarios[args.scenario].prepopulate_lru) { + const struct test_scenario *sc = &scenarios[args.scenario]; + __u32 ridx = sc->encap_v6_outer ? REAL_INDEX_V6 : REAL_INDEX; + + populate_lru(sc, ridx); + } + + if (scenarios[args.scenario].expect_encap) { + const struct test_scenario *sc = &scenarios[args.scenario]; + struct vip_definition miss_vip = {}; + + if (sc->is_v6) + htonl_v6(miss_vip.vipv6, sc->vip_addr_v6); + else + miss_vip.vip = htonl(sc->vip_addr); + miss_vip.port = htons(sc->dst_port); + miss_vip.proto = sc->ip_proto; + + key = 0; + err = bpf_map_update_elem(bpf_map__fd(skel->maps.vip_miss_stats), + &key, &miss_vip, BPF_ANY); + if (err) { + fprintf(stderr, "vip_miss_stats: %s\n", strerror(errno)); + exit(1); + } + } +} + +static void build_expected_packet(int idx) +{ + const struct test_scenario *sc = &scenarios[idx]; + __u8 *p = expected_buf[idx]; + struct ethhdr eth = {}; + const __u8 *in = pkt_buf[idx]; + __u32 in_len = pkt_len[idx]; + __u32 off = 0; + __u32 inner_len = in_len - sizeof(struct ethhdr); + + if (sc->expected_retval == XDP_DROP) { + expected_len[idx] = 0; + return; + } + + if (sc->expected_retval == XDP_PASS) { + memcpy(p, in, in_len); + expected_len[idx] = in_len; + return; + } + + memcpy(eth.h_dest, router_mac, ETH_ALEN); + memcpy(eth.h_source, lb_mac, ETH_ALEN); + eth.h_proto = htons(sc->encap_v6_outer ? ETH_P_IPV6 : ETH_P_IP); + memcpy(p, ð, sizeof(eth)); + off += sizeof(eth); + + if (sc->encap_v6_outer) { + struct ipv6hdr ip6h = {}; + __u8 nexthdr = sc->is_v6 ? IPPROTO_IPV6 : IPPROTO_IPIP; + + ip6h.version = 6; + ip6h.nexthdr = nexthdr; + ip6h.payload_len = htons(inner_len); + ip6h.hop_limit = 64; + + create_encap_ipv6_src(htons(sc->src_port), + sc->is_v6 ? htonl(sc->src_addr_v6[0]) + : htonl(sc->src_addr), + (__be32 *)&ip6h.saddr); + htonl_v6((__be32 *)&ip6h.daddr, sc->tunnel_dst_v6); + + memcpy(p + off, &ip6h, sizeof(ip6h)); + off += sizeof(ip6h); + } else { + struct iphdr iph = {}; + + iph.version = 4; + iph.ihl = sizeof(iph) >> 2; + iph.protocol = IPPROTO_IPIP; + iph.tot_len = htons(inner_len + sizeof(iph)); + iph.ttl = 64; + iph.saddr = create_encap_ipv4_src(htons(sc->src_port), + htonl(sc->src_addr)); + iph.daddr = htonl(sc->tunnel_dst); + iph.check = ip_checksum(&iph, sizeof(iph)); + + memcpy(p + off, &iph, sizeof(iph)); + off += sizeof(iph); + } + + memcpy(p + off, in + sizeof(struct ethhdr), inner_len); + off += inner_len; + + expected_len[idx] = off; +} + +static void print_hex_diff(const char *name, const __u8 *got, __u32 got_len, const __u8 *exp, + __u32 exp_len) +{ + __u32 max_len = got_len > exp_len ? got_len : exp_len; + __u32 i, ndiffs = 0; + + fprintf(stderr, " [%s] got %u bytes, expected %u bytes\n", + name, got_len, exp_len); + + for (i = 0; i < max_len && ndiffs < 8; i++) { + __u8 g = i < got_len ? got[i] : 0; + __u8 e = i < exp_len ? exp[i] : 0; + + if (g != e || i >= got_len || i >= exp_len) { + fprintf(stderr, " offset 0x%03x: got 0x%02x expected 0x%02x\n", + i, g, e); + ndiffs++; + } + } + + if (ndiffs >= 8 && i < max_len) + fprintf(stderr, " ... (more differences)\n"); +} + +static void read_stat(int stats_fd, __u32 key, __u64 *v1_out, __u64 *v2_out) +{ + struct lb_stats values[BENCH_NR_CPUS]; + unsigned int nr_cpus = bpf_num_possible_cpus(); + __u64 v1 = 0, v2 = 0; + unsigned int i; + + if (nr_cpus > BENCH_NR_CPUS) + nr_cpus = BENCH_NR_CPUS; + + if (bpf_map_lookup_elem(stats_fd, &key, values) == 0) { + for (i = 0; i < nr_cpus; i++) { + v1 += values[i].v1; + v2 += values[i].v2; + } + } + + *v1_out = v1; + *v2_out = v2; +} + +static void reset_stats(int stats_fd) +{ + struct lb_stats zeros[BENCH_NR_CPUS]; + __u32 key; + + memset(zeros, 0, sizeof(zeros)); + for (key = 0; key < STATS_SIZE; key++) + bpf_map_update_elem(stats_fd, &key, zeros, BPF_ANY); +} + +static bool validate_counters(int idx) +{ + const struct test_scenario *sc = &scenarios[idx]; + int stats_fd = bpf_map__fd(ctx.skel->maps.stats); + __u64 xdp_tx, xdp_pass, xdp_drop, lru_pkts, lru_misses, tcp_misses; + __u64 expected_misses; + __u64 dummy; + /* + * BENCH_BPF_LOOP runs batch_iters timed + 1 untimed iteration. + * Each iteration calls process_packet -> count_action, so all + * counters are incremented (batch_iters + 1) times. + */ + __u64 n = ctx.timing.batch_iters + 1; + bool pass = true; + + read_stat(stats_fd, STATS_XDP_TX, &xdp_tx, &dummy); + read_stat(stats_fd, STATS_XDP_PASS, &xdp_pass, &dummy); + read_stat(stats_fd, STATS_XDP_DROP, &xdp_drop, &dummy); + read_stat(stats_fd, STATS_LRU, &lru_pkts, &lru_misses); + read_stat(stats_fd, STATS_LRU_MISS, &tcp_misses, &dummy); + + if (sc->expected_retval == XDP_TX && xdp_tx != n) { + fprintf(stderr, " [%s] COUNTER FAIL: STATS_XDP_TX=%llu, expected %llu\n", sc->name, + (unsigned long long)xdp_tx, (unsigned long long)n); + pass = false; + } + if (sc->expected_retval == XDP_PASS && xdp_pass != n) { + fprintf(stderr, " [%s] COUNTER FAIL: STATS_XDP_PASS=%llu, expected %llu\n", + sc->name, (unsigned long long)xdp_pass, (unsigned long long)n); + pass = false; + } + if (sc->expected_retval == XDP_DROP && xdp_drop != n) { + fprintf(stderr, " [%s] COUNTER FAIL: STATS_XDP_DROP=%llu, expected %llu\n", + sc->name, (unsigned long long)xdp_drop, (unsigned long long)n); + pass = false; + } + + if (!sc->expect_encap) + goto out; + + if (lru_pkts != n) { + fprintf(stderr, " [%s] COUNTER FAIL: STATS_LRU.v1=%llu, expected %llu\n", + sc->name, (unsigned long long)lru_pkts, (unsigned long long)n); + pass = false; + } + + switch (sc->lru_miss) { + case LRU_MISS_NONE: + expected_misses = 0; + break; + case LRU_MISS_ALL: + expected_misses = n; + break; + case LRU_MISS_FIRST: + expected_misses = 1; + break; + default: + /* LRU_MISS_AUTO: compute from scenario flags */ + if (sc->prepopulate_lru && !sc->set_syn) + expected_misses = 0; + else if (sc->set_syn || sc->set_rst || + (sc->vip_flags & F_LRU_BYPASS)) + expected_misses = n; + else if (sc->cold_lru) + expected_misses = 1; + else + expected_misses = n; + break; + } + + if (lru_misses != expected_misses) { + fprintf(stderr, " [%s] COUNTER FAIL: LRU misses=%llu, expected %llu\n", + sc->name, (unsigned long long)lru_misses, + (unsigned long long)expected_misses); + pass = false; + } + + if (sc->ip_proto == IPPROTO_TCP && lru_misses > 0) { + if (tcp_misses != lru_misses) { + fprintf(stderr, " [%s] COUNTER FAIL: TCP LRU misses=%llu, expected %llu\n", + sc->name, (unsigned long long)tcp_misses, + (unsigned long long)lru_misses); + pass = false; + } + } + +out: + reset_stats(stats_fd); + return pass; +} + +static const char *xdp_action_str(int action) +{ + switch (action) { + case XDP_DROP: return "XDP_DROP"; + case XDP_PASS: return "XDP_PASS"; + case XDP_TX: return "XDP_TX"; + default: return "UNKNOWN"; + } +} + +static bool validate_scenario(int idx) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + const struct test_scenario *sc = &scenarios[idx]; + __u8 out[MAX_ENCAP_SIZE]; + int err; + + topts.data_in = pkt_buf[idx]; + topts.data_size_in = pkt_len[idx]; + topts.data_out = out; + topts.data_size_out = sizeof(out); + topts.repeat = 1; + + err = bpf_prog_test_run_opts(ctx.prog_fd, &topts); + if (err) { + fprintf(stderr, " [%s] FAIL: test_run: %s\n", sc->name, strerror(errno)); + return false; + } + + if ((int)topts.retval != sc->expected_retval) { + fprintf(stderr, " [%s] FAIL: retval %s, expected %s\n", sc->name, + xdp_action_str(topts.retval), xdp_action_str(sc->expected_retval)); + return false; + } + + /* + * Compare output packet when it's deterministic. + * Skip for XDP_DROP (no output) and cold_lru (source IP poisoned). + */ + if (sc->expected_retval != XDP_DROP && !sc->cold_lru) { + if (topts.data_size_out != expected_len[idx] || + memcmp(out, expected_buf[idx], expected_len[idx]) != 0) { + fprintf(stderr, " [%s] FAIL: output packet mismatch\n", sc->name); + print_hex_diff(sc->name, out, topts.data_size_out, expected_buf[idx], + expected_len[idx]); + return false; + } + } + + if (!validate_counters(idx)) + return false; + return true; +} + +static int find_scenario(const char *name) +{ + int i; + + for (i = 0; i < NUM_SCENARIOS; i++) { + if (strcmp(scenarios[i].name, name) == 0) + return i; + } + return -1; +} + +static void xdp_lb_validate(void) +{ + if (env.consumer_cnt != 0) { + fprintf(stderr, "benchmark doesn't support consumers\n"); + exit(1); + } + if (bpf_num_possible_cpus() > BENCH_NR_CPUS) { + fprintf(stderr, "too many CPUs (%d > %d), increase BENCH_NR_CPUS\n", + bpf_num_possible_cpus(), BENCH_NR_CPUS); + exit(1); + } +} + +static void xdp_lb_run_once(void *unused __always_unused) +{ + int idx = args.scenario; + + LIBBPF_OPTS(bpf_test_run_opts, topts, + .data_in = pkt_buf[idx], + .data_size_in = pkt_len[idx], + .repeat = 1, + ); + + bpf_prog_test_run_opts(ctx.prog_fd, &topts); +} + +static void xdp_lb_setup(void) +{ + struct xdp_lb_bench *skel; + int err; + + if (args.scenario < 0) { + fprintf(stderr, "--scenario is required. Use --list-scenarios to see options.\n"); + exit(1); + } + + setup_libbpf(); + + skel = xdp_lb_bench__open(); + if (!skel) { + fprintf(stderr, "failed to open skeleton\n"); + exit(1); + } + + err = xdp_lb_bench__load(skel); + if (err) { + fprintf(stderr, "failed to load skeleton: %s\n", strerror(-err)); + xdp_lb_bench__destroy(skel); + exit(1); + } + + ctx.skel = skel; + ctx.prog_fd = bpf_program__fd(skel->progs.xdp_lb_bench); + + build_packet(args.scenario); + build_expected_packet(args.scenario); + + populate_maps(skel); + + BENCH_TIMING_INIT(&ctx.timing, skel, 0); + ctx.timing.machine_readable = args.machine_readable; + + if (scenarios[args.scenario].fixed_batch_iters) { + ctx.timing.batch_iters = scenarios[args.scenario].fixed_batch_iters; + skel->bss->batch_iters = ctx.timing.batch_iters; + } else { + bpf_bench_calibrate(&ctx.timing, xdp_lb_run_once, NULL); + } + + env.duration_sec = 600; + + /* + * Enable cold_lru before validation so LRU miss counters are + * correct. Seed the LRU with one run so the original flow is + * present; validation then sees exactly 1 miss (the poisoned + * flow) regardless of whether calibration ran. + */ + if (scenarios[args.scenario].cold_lru) { + skel->bss->cold_lru = 1; + xdp_lb_run_once(NULL); + } + + reset_stats(bpf_map__fd(skel->maps.stats)); + + if (!validate_scenario(args.scenario)) { + fprintf(stderr, "Validation FAILED - aborting benchmark\n"); + exit(1); + } + + if (scenarios[args.scenario].flow_mask) + skel->bss->flow_mask = scenarios[args.scenario].flow_mask; +} + +static void *xdp_lb_producer(void *input) +{ + while (true) + xdp_lb_run_once(NULL); + + return NULL; +} + +static void xdp_lb_measure(struct bench_res *res) +{ + bpf_bench_timing_measure(&ctx.timing, res); +} + +static void xdp_lb_report_final(struct bench_res res[], int res_cnt) +{ + bpf_bench_timing_report(&ctx.timing, scenarios[args.scenario].name, + scenarios[args.scenario].description); +} + +enum { + ARG_SCENARIO = 9001, + ARG_LIST_SCENARIOS = 9002, + ARG_MACHINE_READABLE = 9003, +}; + +static const struct argp_option opts[] = { + { "scenario", ARG_SCENARIO, "NAME", 0, + "Scenario to benchmark (required)" }, + { "list-scenarios", ARG_LIST_SCENARIOS, NULL, 0, + "List available scenarios and exit" }, + { "machine-readable", ARG_MACHINE_READABLE, NULL, 0, + "Print only a machine-readable RESULT line" }, + {}, +}; + +static error_t parse_arg(int key, char *arg, struct argp_state *state) +{ + int i; + + switch (key) { + case ARG_SCENARIO: + args.scenario = find_scenario(arg); + if (args.scenario < 0) { + fprintf(stderr, "unknown scenario: '%s'\n", arg); + fprintf(stderr, "use --list-scenarios to see options\n"); + argp_usage(state); + } + break; + case ARG_LIST_SCENARIOS: + printf("Available scenarios:\n"); + for (i = 0; i < NUM_SCENARIOS; i++) + printf(" %-20s %s\n", scenarios[i].name, scenarios[i].description); + exit(0); + case ARG_MACHINE_READABLE: + args.machine_readable = true; + env.quiet = true; + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +const struct argp bench_xdp_lb_argp = { + .options = opts, + .parser = parse_arg, +}; + +const struct bench bench_xdp_lb = { + .name = "xdp-lb", + .argp = &bench_xdp_lb_argp, + .validate = xdp_lb_validate, + .setup = xdp_lb_setup, + .producer_thread = xdp_lb_producer, + .measure = xdp_lb_measure, + .report_final = xdp_lb_report_final, +}; diff --git a/tools/testing/selftests/bpf/benchs/run_bench_bpf_for.sh b/tools/testing/selftests/bpf/benchs/run_bench_bpf_for.sh new file mode 100755 index 000000000000..7da6453920da --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/run_bench_bpf_for.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +source ./benchs/run_common.sh + +set -eufo pipefail + +for t in 1 4 8 12 16; do +for i in 10 100 500 1000 5000 10000 50000 100000 500000 1000000; do +subtitle "nr_loops: $i, nr_threads: $t" + summarize_ops "bpf_for: " \ + "$($RUN_BENCH -p $t --nr_loops $i bpf-for)" + printf "\n" +done +done diff --git a/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh b/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh index 03f55405484b..e490b337e960 100755 --- a/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh +++ b/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh @@ -2,7 +2,7 @@ set -eufo pipefail -for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop5} +for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop10} usdt-nop usdt-nop10 do summary=$(sudo ./bench -w2 -d5 -a trig-$i | tail -n1 | cut -d'(' -f1 | cut -d' ' -f3-) printf "%-15s: %s\n" $i "$summary" diff --git a/tools/testing/selftests/bpf/benchs/run_bench_xdp_lb.sh b/tools/testing/selftests/bpf/benchs/run_bench_xdp_lb.sh new file mode 100755 index 000000000000..f65cf46214a3 --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/run_bench_xdp_lb.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +source ./benchs/run_common.sh + +set -eufo pipefail + +WARMUP=${WARMUP:-3} + +RUN="sudo ./bench -q -w${WARMUP} -a xdp-lb --machine-readable" + +SEP=" +----------------------------------+----------+---------+----------+" +HDR=" | %-32s | %8s | %7s | %8s |\n" +ROW=" | %-32s | %8s | %7s | %8s |\n" + +function group_header() +{ + printf "%s\n" "$SEP" + printf "$HDR" "$1" "p50" "stddev" "p99" + printf "%s\n" "$SEP" +} + +function rval() +{ + echo "$1" | sed -nE "s/.*$2=([^ ]+).*/\1/p" +} + +function run_scenario() +{ + local sc="$1" + shift + local output rline + + output=$($RUN --scenario "$sc" "$@" 2>&1) || true + rline=$(echo "$output" | grep '^RESULT ' || true) + + if [ -z "$rline" ]; then + printf "$ROW" "$sc" "ERR" "-" "-" + return + fi + + printf "$ROW" "$sc" \ + "$(rval "$rline" median)" \ + "$(rval "$rline" stddev)" \ + "$(rval "$rline" p99)" +} + +header "XDP load-balancer benchmark" + +group_header "Single-flow baseline" +for sc in tcp-v4-lru-hit tcp-v4-ch \ + tcp-v6-lru-hit tcp-v6-ch \ + udp-v4-lru-hit udp-v6-lru-hit \ + tcp-v4v6-lru-hit; do + run_scenario "$sc" +done + +group_header "Diverse flows (4K src addrs)" +for sc in tcp-v4-lru-diverse tcp-v4-ch-diverse \ + tcp-v6-lru-diverse tcp-v6-ch-diverse \ + udp-v4-lru-diverse; do + run_scenario "$sc" +done + +group_header "TCP flags" +run_scenario tcp-v4-syn +run_scenario tcp-v4-rst-miss + +group_header "LRU stress" +run_scenario tcp-v4-lru-miss +run_scenario udp-v4-lru-miss +run_scenario tcp-v4-lru-warmup + +group_header "Early exits" +for sc in pass-v4-no-vip pass-v6-no-vip pass-v4-icmp pass-non-ip drop-v4-frag drop-v4-options \ + drop-v6-frag; do + run_scenario "$sc" +done +printf "%s\n" "$SEP" diff --git a/tools/testing/selftests/bpf/bpf_arena_alloc.h b/tools/testing/selftests/bpf/bpf_arena_alloc.h index c27678299e0c..cda147fd9d25 100644 --- a/tools/testing/selftests/bpf/bpf_arena_alloc.h +++ b/tools/testing/selftests/bpf/bpf_arena_alloc.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ /* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ #pragma once -#include "bpf_arena_common.h" +#include <bpf_arena_common.h> #ifndef __round_mask #define __round_mask(x, y) ((__typeof__(x))((y)-1)) diff --git a/tools/testing/selftests/bpf/bpf_arena_htab.h b/tools/testing/selftests/bpf/bpf_arena_htab.h index acc01a876668..d7ba86362d86 100644 --- a/tools/testing/selftests/bpf/bpf_arena_htab.h +++ b/tools/testing/selftests/bpf/bpf_arena_htab.h @@ -14,9 +14,8 @@ struct htab { htab_bucket_t *buckets; int n_buckets; }; -typedef struct htab __arena htab_t; -static inline htab_bucket_t *__select_bucket(htab_t *htab, __u32 hash) +static inline htab_bucket_t *__select_bucket(struct htab __arena *htab, __u32 hash) { htab_bucket_t *b = htab->buckets; @@ -24,7 +23,7 @@ static inline htab_bucket_t *__select_bucket(htab_t *htab, __u32 hash) return &b[hash & (htab->n_buckets - 1)]; } -static inline arena_list_head_t *select_bucket(htab_t *htab, __u32 hash) +static inline arena_list_head_t *select_bucket(struct htab __arena *htab, __u32 hash) { return &__select_bucket(htab, hash)->head; } @@ -53,7 +52,7 @@ static int htab_hash(int key) return key; } -__weak int htab_lookup_elem(htab_t *htab __arg_arena, int key) +__weak int htab_lookup_elem(struct htab __arena *htab, int key) { hashtab_elem_t *l_old; arena_list_head_t *head; @@ -66,7 +65,7 @@ __weak int htab_lookup_elem(htab_t *htab __arg_arena, int key) return 0; } -__weak int htab_update_elem(htab_t *htab __arg_arena, int key, int value) +__weak int htab_update_elem(struct htab __arena *htab, int key, int value) { hashtab_elem_t *l_new = NULL, *l_old; arena_list_head_t *head; @@ -90,7 +89,7 @@ __weak int htab_update_elem(htab_t *htab __arg_arena, int key, int value) return 0; } -void htab_init(htab_t *htab) +void htab_init(struct htab __arena *htab) { void __arena *buckets = bpf_arena_alloc_pages(&arena, NULL, 2, NUMA_NO_NODE, 0); diff --git a/tools/testing/selftests/bpf/bpf_arena_list.h b/tools/testing/selftests/bpf/bpf_arena_list.h index e16fa7d95fcf..1af2ffc27d9c 100644 --- a/tools/testing/selftests/bpf/bpf_arena_list.h +++ b/tools/testing/selftests/bpf/bpf_arena_list.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ /* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ #pragma once -#include "bpf_arena_common.h" +#include <bpf_arena_common.h> struct arena_list_node; diff --git a/tools/testing/selftests/bpf/bpf_arena_strsearch.h b/tools/testing/selftests/bpf/bpf_arena_strsearch.h index c1b6eaa905bb..10a70667c8bf 100644 --- a/tools/testing/selftests/bpf/bpf_arena_strsearch.h +++ b/tools/testing/selftests/bpf/bpf_arena_strsearch.h @@ -1,9 +1,9 @@ /* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ #pragma once -#include "bpf_arena_common.h" +#include <bpf_arena_common.h> -__noinline int bpf_arena_strlen(const char __arena *s __arg_arena) +__noinline int bpf_arena_strlen(const char __arena *s) { const char __arena *sc; @@ -40,7 +40,7 @@ __noinline int bpf_arena_strlen(const char __arena *s __arg_arena) * * An opening bracket without a matching close is matched literally. */ -__noinline bool glob_match(char const __arena *pat __arg_arena, char const __arena *str __arg_arena) +__noinline bool glob_match(char const __arena *pat, char const __arena *str) { /* * Backtrack to previous * on mismatch and retry starting one diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h index 4b7210c318dd..2893bf06ff25 100644 --- a/tools/testing/selftests/bpf/bpf_experimental.h +++ b/tools/testing/selftests/bpf/bpf_experimental.h @@ -5,159 +5,15 @@ #include <bpf/bpf_tracing.h> #include <bpf/bpf_helpers.h> #include <bpf/bpf_core_read.h> +#include <bpf_may_goto.h> #define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) -/* Description - * Allocates an object of the type represented by 'local_type_id' in - * program BTF. User may use the bpf_core_type_id_local macro to pass the - * type ID of a struct in program BTF. - * - * The 'local_type_id' parameter must be a known constant. - * The 'meta' parameter is rewritten by the verifier, no need for BPF - * program to set it. - * Returns - * A pointer to an object of the type corresponding to the passed in - * 'local_type_id', or NULL on failure. - */ -extern void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; - -/* Convenience macro to wrap over bpf_obj_new_impl */ -#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) - -/* Description - * Free an allocated object. All fields of the object that require - * destruction will be destructed before the storage is freed. - * - * The 'meta' parameter is rewritten by the verifier, no need for BPF - * program to set it. - * Returns - * Void. - */ -extern void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; +/* Convenience macro to wrap over bpf_obj_new */ +#define bpf_obj_new(type) ((type *)bpf_obj_new(bpf_core_type_id_local(type))) -/* Convenience macro to wrap over bpf_obj_drop_impl */ -#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) - -/* Description - * Increment the refcount on a refcounted local kptr, turning the - * non-owning reference input into an owning reference in the process. - * - * The 'meta' parameter is rewritten by the verifier, no need for BPF - * program to set it. - * Returns - * An owning reference to the object pointed to by 'kptr' - */ -extern void *bpf_refcount_acquire_impl(void *kptr, void *meta) __ksym; - -/* Convenience macro to wrap over bpf_refcount_acquire_impl */ -#define bpf_refcount_acquire(kptr) bpf_refcount_acquire_impl(kptr, NULL) - -/* Description - * Add a new entry to the beginning of the BPF linked list. - * - * The 'meta' and 'off' parameters are rewritten by the verifier, no need - * for BPF programs to set them - * Returns - * 0 if the node was successfully added - * -EINVAL if the node wasn't added because it's already in a list - */ -extern int bpf_list_push_front_impl(struct bpf_list_head *head, - struct bpf_list_node *node, - void *meta, __u64 off) __ksym; - -/* Convenience macro to wrap over bpf_list_push_front_impl */ -#define bpf_list_push_front(head, node) bpf_list_push_front_impl(head, node, NULL, 0) - -/* Description - * Add a new entry to the end of the BPF linked list. - * - * The 'meta' and 'off' parameters are rewritten by the verifier, no need - * for BPF programs to set them - * Returns - * 0 if the node was successfully added - * -EINVAL if the node wasn't added because it's already in a list - */ -extern int bpf_list_push_back_impl(struct bpf_list_head *head, - struct bpf_list_node *node, - void *meta, __u64 off) __ksym; - -/* Convenience macro to wrap over bpf_list_push_back_impl */ -#define bpf_list_push_back(head, node) bpf_list_push_back_impl(head, node, NULL, 0) - -/* Description - * Remove the entry at the beginning of the BPF linked list. - * Returns - * Pointer to bpf_list_node of deleted entry, or NULL if list is empty. - */ -extern struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; - -/* Description - * Remove the entry at the end of the BPF linked list. - * Returns - * Pointer to bpf_list_node of deleted entry, or NULL if list is empty. - */ -extern struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; - -/* Description - * Remove 'node' from rbtree with root 'root' - * Returns - * Pointer to the removed node, or NULL if 'root' didn't contain 'node' - */ -extern struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, - struct bpf_rb_node *node) __ksym; - -/* Description - * Add 'node' to rbtree with root 'root' using comparator 'less' - * - * The 'meta' and 'off' parameters are rewritten by the verifier, no need - * for BPF programs to set them - * Returns - * 0 if the node was successfully added - * -EINVAL if the node wasn't added because it's already in a tree - */ -extern int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node, - bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b), - void *meta, __u64 off) __ksym; - -/* Convenience macro to wrap over bpf_rbtree_add_impl */ -#define bpf_rbtree_add(head, node, less) bpf_rbtree_add_impl(head, node, less, NULL, 0) - -/* Description - * Return the first (leftmost) node in input tree - * Returns - * Pointer to the node, which is _not_ removed from the tree. If the tree - * contains no nodes, returns NULL. - */ -extern struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; - -/* Description - * Allocates a percpu object of the type represented by 'local_type_id' in - * program BTF. User may use the bpf_core_type_id_local macro to pass the - * type ID of a struct in program BTF. - * - * The 'local_type_id' parameter must be a known constant. - * The 'meta' parameter is rewritten by the verifier, no need for BPF - * program to set it. - * Returns - * A pointer to a percpu object of the type corresponding to the passed in - * 'local_type_id', or NULL on failure. - */ -extern void *bpf_percpu_obj_new_impl(__u64 local_type_id, void *meta) __ksym; - -/* Convenience macro to wrap over bpf_percpu_obj_new_impl */ -#define bpf_percpu_obj_new(type) ((type __percpu_kptr *)bpf_percpu_obj_new_impl(bpf_core_type_id_local(type), NULL)) - -/* Description - * Free an allocated percpu object. All fields of the object that require - * destruction will be destructed before the storage is freed. - * - * The 'meta' parameter is rewritten by the verifier, no need for BPF - * program to set it. - * Returns - * Void. - */ -extern void bpf_percpu_obj_drop_impl(void *kptr, void *meta) __ksym; +/* Convenience macro to wrap over bpf_percpu_obj_new */ +#define bpf_percpu_obj_new(type) ((type __percpu_kptr *)bpf_percpu_obj_new(bpf_core_type_id_local(type))) struct bpf_iter_task_vma; @@ -167,9 +23,6 @@ extern int bpf_iter_task_vma_new(struct bpf_iter_task_vma *it, extern struct vm_area_struct *bpf_iter_task_vma_next(struct bpf_iter_task_vma *it) __ksym; extern void bpf_iter_task_vma_destroy(struct bpf_iter_task_vma *it) __ksym; -/* Convenience macro to wrap over bpf_obj_drop_impl */ -#define bpf_percpu_obj_drop(kptr) bpf_percpu_obj_drop_impl(kptr, NULL) - /* Description * Throw a BPF exception from the program, immediately terminating its * execution and unwinding the stack. The supplied 'cookie' parameter @@ -352,89 +205,6 @@ l_true: \ }) #endif -/* - * Note that cond_break can only be portably used in the body of a breakable - * construct, whereas can_loop can be used anywhere. - */ -#ifdef __BPF_FEATURE_MAY_GOTO -#define can_loop \ - ({ __label__ l_break, l_continue; \ - bool ret = true; \ - asm volatile goto("may_goto %l[l_break]" \ - :::: l_break); \ - goto l_continue; \ - l_break: ret = false; \ - l_continue:; \ - ret; \ - }) - -#define __cond_break(expr) \ - ({ __label__ l_break, l_continue; \ - asm volatile goto("may_goto %l[l_break]" \ - :::: l_break); \ - goto l_continue; \ - l_break: expr; \ - l_continue:; \ - }) -#else -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define can_loop \ - ({ __label__ l_break, l_continue; \ - bool ret = true; \ - asm volatile goto("1:.byte 0xe5; \ - .byte 0; \ - .long ((%l[l_break] - 1b - 8) / 8) & 0xffff; \ - .short 0" \ - :::: l_break); \ - goto l_continue; \ - l_break: ret = false; \ - l_continue:; \ - ret; \ - }) - -#define __cond_break(expr) \ - ({ __label__ l_break, l_continue; \ - asm volatile goto("1:.byte 0xe5; \ - .byte 0; \ - .long ((%l[l_break] - 1b - 8) / 8) & 0xffff; \ - .short 0" \ - :::: l_break); \ - goto l_continue; \ - l_break: expr; \ - l_continue:; \ - }) -#else -#define can_loop \ - ({ __label__ l_break, l_continue; \ - bool ret = true; \ - asm volatile goto("1:.byte 0xe5; \ - .byte 0; \ - .long (((%l[l_break] - 1b - 8) / 8) & 0xffff) << 16; \ - .short 0" \ - :::: l_break); \ - goto l_continue; \ - l_break: ret = false; \ - l_continue:; \ - ret; \ - }) - -#define __cond_break(expr) \ - ({ __label__ l_break, l_continue; \ - asm volatile goto("1:.byte 0xe5; \ - .byte 0; \ - .long (((%l[l_break] - 1b - 8) / 8) & 0xffff) << 16; \ - .short 0" \ - :::: l_break); \ - goto l_continue; \ - l_break: expr; \ - l_continue:; \ - }) -#endif -#endif - -#define cond_break __cond_break(break) -#define cond_break_label(label) __cond_break(goto label) - #ifndef bpf_nop_mov #define bpf_nop_mov(var) \ asm volatile("%[reg]=%[reg]"::[reg]"r"((short)var)) @@ -594,19 +364,25 @@ extern void bpf_iter_dmabuf_destroy(struct bpf_iter_dmabuf *it) __weak __ksym; extern int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__str, struct bpf_dynptr *value_p) __weak __ksym; +extern int bpf_sock_read_xattr(struct socket *sock, const char *name__str, + struct bpf_dynptr *value_p) __weak __ksym; + #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 +#define HARDIRQ_DISABLE_BITS 8 #define HARDIRQ_BITS 4 -#define NMI_BITS 4 +#define NMI_BITS 1 #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) -#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS) #define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS) #define __IRQ_MASK(x) ((1UL << (x))-1) #define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_DISABLE_SHIFT) #define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT) #define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT) @@ -627,6 +403,10 @@ struct task_struct___preempt_rt { int softirq_disable_cnt; } __attribute__((preserve_access_index)); +#ifdef bpf_target_s390 +extern struct lowcore *bpf_get_lowcore(void) __weak __ksym; +#endif + static inline int get_preempt_count(void) { #if defined(bpf_target_x86) @@ -645,6 +425,14 @@ static inline int get_preempt_count(void) bpf_this_cpu_ptr(&pcpu_hot))->preempt_count; #elif defined(bpf_target_arm64) return bpf_get_current_task_btf()->thread_info.preempt.count; +#elif defined(bpf_target_powerpc) + return bpf_get_current_task_btf()->thread_info.preempt_count; +#elif defined(bpf_target_s390) + return bpf_get_lowcore()->preempt_count; +#elif defined(bpf_target_loongarch) + return bpf_get_current_task_btf()->thread_info.preempt_count; +#elif defined(bpf_target_riscv) + return bpf_get_current_task_btf()->thread_info.preempt_count; #endif return 0; } @@ -653,6 +441,10 @@ static inline int get_preempt_count(void) * Report whether it is in interrupt context. Only works on the following archs: * * x86 * * arm64 + * * powerpc64 + * * s390x + * * loongarch + * * riscv */ static inline int bpf_in_interrupt(void) { @@ -672,6 +464,10 @@ static inline int bpf_in_interrupt(void) * Report whether it is in NMI context. Only works on the following archs: * * x86 * * arm64 + * * powerpc64 + * * s390x + * * loongarch + * * riscv */ static inline int bpf_in_nmi(void) { @@ -682,6 +478,10 @@ static inline int bpf_in_nmi(void) * Report whether it is in hard IRQ context. Only works on the following archs: * * x86 * * arm64 + * * powerpc64 + * * s390x + * * loongarch + * * riscv */ static inline int bpf_in_hardirq(void) { @@ -692,6 +492,10 @@ static inline int bpf_in_hardirq(void) * Report whether it is in softirq context. Only works on the following archs: * * x86 * * arm64 + * * powerpc64 + * * s390x + * * loongarch + * * riscv */ static inline int bpf_in_serving_softirq(void) { @@ -710,6 +514,10 @@ static inline int bpf_in_serving_softirq(void) * Report whether it is in task context. Only works on the following archs: * * x86 * * arm64 + * * powerpc64 + * * s390x + * * loongarch + * * riscv */ static inline int bpf_in_task(void) { diff --git a/tools/testing/selftests/bpf/bpf_kfuncs.h b/tools/testing/selftests/bpf/bpf_kfuncs.h index 7dad01439391..ae71e9b69051 100644 --- a/tools/testing/selftests/bpf/bpf_kfuncs.h +++ b/tools/testing/selftests/bpf/bpf_kfuncs.h @@ -40,7 +40,7 @@ extern void *bpf_dynptr_slice(const struct bpf_dynptr *ptr, __u64 offset, extern void *bpf_dynptr_slice_rdwr(const struct bpf_dynptr *ptr, __u64 offset, void *buffer, __u64 buffer__szk) __ksym __weak; -extern int bpf_dynptr_adjust(const struct bpf_dynptr *ptr, __u64 start, __u64 end) __ksym __weak; +extern int bpf_dynptr_adjust(struct bpf_dynptr *ptr, __u64 start, __u64 end) __ksym __weak; extern bool bpf_dynptr_is_null(const struct bpf_dynptr *ptr) __ksym __weak; extern bool bpf_dynptr_is_rdonly(const struct bpf_dynptr *ptr) __ksym __weak; extern __u64 bpf_dynptr_size(const struct bpf_dynptr *ptr) __ksym __weak; @@ -70,13 +70,13 @@ extern void *bpf_rdonly_cast(const void *obj, __u32 btf_id) __ksym __weak; extern int bpf_get_file_xattr(struct file *file, const char *name, struct bpf_dynptr *value_ptr) __ksym; -extern int bpf_get_fsverity_digest(struct file *file, struct bpf_dynptr *digest_ptr) __ksym; +extern int bpf_get_fsverity_digest(struct file *file, const struct bpf_dynptr *digest_ptr) __ksym; extern struct bpf_key *bpf_lookup_user_key(__s32 serial, __u64 flags) __ksym; extern struct bpf_key *bpf_lookup_system_key(__u64 id) __ksym; extern void bpf_key_put(struct bpf_key *key) __ksym; -extern int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_ptr, - struct bpf_dynptr *sig_ptr, +extern int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_ptr, + const struct bpf_dynptr *sig_ptr, struct bpf_key *trusted_keyring) __ksym; struct dentry; diff --git a/tools/testing/selftests/bpf/bpftool_helpers.c b/tools/testing/selftests/bpf/bpftool_helpers.c index 929fc257f431..0a2a4f0a2794 100644 --- a/tools/testing/selftests/bpf/bpftool_helpers.c +++ b/tools/testing/selftests/bpf/bpftool_helpers.c @@ -2,18 +2,18 @@ #include <unistd.h> #include <string.h> #include <stdbool.h> +#include <limits.h> #include "bpf_util.h" #include "bpftool_helpers.h" -#define BPFTOOL_PATH_MAX_LEN 64 -#define BPFTOOL_FULL_CMD_MAX_LEN 512 +#define BPFTOOL_FULL_CMD_MAX_LEN (PATH_MAX * 2) #define BPFTOOL_DEFAULT_PATH "tools/sbin/bpftool" static int detect_bpftool_path(char *buffer, size_t size) { - char tmp[BPFTOOL_PATH_MAX_LEN]; + char tmp[PATH_MAX]; const char *env_path; /* First, check if BPFTOOL environment variable is set */ @@ -29,7 +29,7 @@ static int detect_bpftool_path(char *buffer, size_t size) /* Check default bpftool location (will work if we are running the * default flavor of test_progs) */ - snprintf(tmp, BPFTOOL_PATH_MAX_LEN, "./%s", BPFTOOL_DEFAULT_PATH); + snprintf(tmp, sizeof(tmp), "./%s", BPFTOOL_DEFAULT_PATH); if (access(tmp, X_OK) == 0) { strscpy(buffer, tmp, size); return 0; @@ -38,7 +38,7 @@ static int detect_bpftool_path(char *buffer, size_t size) /* Check alternate bpftool location (will work if we are running a * specific flavor of test_progs, e.g. cpuv4 or no_alu32) */ - snprintf(tmp, BPFTOOL_PATH_MAX_LEN, "../%s", BPFTOOL_DEFAULT_PATH); + snprintf(tmp, sizeof(tmp), "../%s", BPFTOOL_DEFAULT_PATH); if (access(tmp, X_OK) == 0) { strscpy(buffer, tmp, size); return 0; @@ -50,7 +50,7 @@ static int detect_bpftool_path(char *buffer, size_t size) static int run_command(char *args, char *output_buf, size_t output_max_len) { - static char bpftool_path[BPFTOOL_PATH_MAX_LEN] = {0}; + static char bpftool_path[PATH_MAX] = {}; bool suppress_output = !(output_buf && output_max_len); char command[BPFTOOL_FULL_CMD_MAX_LEN]; FILE *f; @@ -60,7 +60,7 @@ static int run_command(char *args, char *output_buf, size_t output_max_len) if (bpftool_path[0] == 0 && detect_bpftool_path(bpftool_path, sizeof(bpftool_path))) return 1; - ret = snprintf(command, BPFTOOL_FULL_CMD_MAX_LEN, "%s %s%s", + ret = snprintf(command, sizeof(command), "%s %s%s", bpftool_path, args, suppress_output ? " > /dev/null 2>&1" : ""); @@ -84,4 +84,3 @@ int get_bpftool_command_output(char *args, char *output_buf, size_t output_max_l { return run_command(args, output_buf, output_max_len); } - diff --git a/tools/testing/selftests/bpf/cgroup_iter_memcg.h b/tools/testing/selftests/bpf/cgroup_iter_memcg.h index 3f59b127943b..ff20ec537164 100644 --- a/tools/testing/selftests/bpf/cgroup_iter_memcg.h +++ b/tools/testing/selftests/bpf/cgroup_iter_memcg.h @@ -9,8 +9,6 @@ struct memcg_query { unsigned long nr_shmem; unsigned long nr_file_pages; unsigned long nr_file_mapped; - /* some memcg_stat_item */ - unsigned long memcg_kmem; /* some vm_event_item */ unsigned long pgfault; }; diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config index 24855381290d..ea7044f30adc 100644 --- a/tools/testing/selftests/bpf/config +++ b/tools/testing/selftests/bpf/config @@ -45,13 +45,16 @@ CONFIG_IPV6=y CONFIG_IPV6_FOU=y CONFIG_IPV6_FOU_TUNNEL=y CONFIG_IPV6_GRE=y +CONFIG_IPV6_IOAM6_LWTUNNEL=y CONFIG_IPV6_SEG6_BPF=y +CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_IPV6_SIT=y CONFIG_IPV6_TUNNEL=y CONFIG_KEYS=y CONFIG_LIRC=y CONFIG_LIVEPATCH=y CONFIG_LWTUNNEL=y +CONFIG_LWTUNNEL_BPF=y CONFIG_MODULE_SIG=y CONFIG_MODULE_SRCVERSION_ALL=y CONFIG_MODULE_UNLOAD=y @@ -79,6 +82,7 @@ CONFIG_NET_SCH_BPF=y CONFIG_NET_SCH_FQ=y CONFIG_NET_SCH_INGRESS=y CONFIG_NET_SCH_HTB=y +CONFIG_NET_SCH_RED=y CONFIG_NET_SCHED=y CONFIG_NETDEVSIM=y CONFIG_NETFILTER=y @@ -130,4 +134,5 @@ CONFIG_INFINIBAND=y CONFIG_SMC=y CONFIG_SMC_HS_CTRL_BPF=y CONFIG_DIBS=y -CONFIG_DIBS_LO=y
\ No newline at end of file +CONFIG_DIBS_LO=y +CONFIG_PM_WAKELOCKS=y diff --git a/tools/testing/selftests/bpf/config.aarch64 b/tools/testing/selftests/bpf/config.aarch64 index 7efad36ceb26..fc85257701dc 100644 --- a/tools/testing/selftests/bpf/config.aarch64 +++ b/tools/testing/selftests/bpf/config.aarch64 @@ -71,7 +71,6 @@ CONFIG_INPUT_EVDEV=y CONFIG_IP_ADVANCED_ROUTER=y CONFIG_IP_MULTICAST=y CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_IPVLAN=y CONFIG_JUMP_LABEL=y CONFIG_KERNEL_UNCOMPRESSED=y diff --git a/tools/testing/selftests/bpf/config.ppc64el b/tools/testing/selftests/bpf/config.ppc64el index b53afb5e0b71..5685fa4ee82b 100644 --- a/tools/testing/selftests/bpf/config.ppc64el +++ b/tools/testing/selftests/bpf/config.ppc64el @@ -39,7 +39,6 @@ CONFIG_INET=y CONFIG_IP_ADVANCED_ROUTER=y CONFIG_IP_MULTICAST=y CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_JUMP_LABEL=y CONFIG_KALLSYMS_ALL=y CONFIG_KPROBES=y diff --git a/tools/testing/selftests/bpf/config.riscv64 b/tools/testing/selftests/bpf/config.riscv64 index 7bee24a79a71..655cb05a7689 100644 --- a/tools/testing/selftests/bpf/config.riscv64 +++ b/tools/testing/selftests/bpf/config.riscv64 @@ -30,7 +30,6 @@ CONFIG_HARDLOCKUP_DETECTOR=y CONFIG_HIGH_RES_TIMERS=y CONFIG_HUGETLBFS=y CONFIG_INET=y -CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_IP_ADVANCED_ROUTER=y CONFIG_IP_MULTICAST=y CONFIG_IP_MULTIPLE_TABLES=y diff --git a/tools/testing/selftests/bpf/config.s390x b/tools/testing/selftests/bpf/config.s390x index db61878148e4..755d1cfcd9e0 100644 --- a/tools/testing/selftests/bpf/config.s390x +++ b/tools/testing/selftests/bpf/config.s390x @@ -56,7 +56,6 @@ CONFIG_INET=y CONFIG_IP_ADVANCED_ROUTER=y CONFIG_IP_MULTICAST=y CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_IPVLAN=y CONFIG_JUMP_LABEL=y CONFIG_KERNEL_UNCOMPRESSED=y diff --git a/tools/testing/selftests/bpf/config.x86_64 b/tools/testing/selftests/bpf/config.x86_64 index 42ad817b00ae..523e0d29bbd4 100644 --- a/tools/testing/selftests/bpf/config.x86_64 +++ b/tools/testing/selftests/bpf/config.x86_64 @@ -114,7 +114,6 @@ CONFIG_IP_ROUTE_VERBOSE=y CONFIG_IPV6_MIP6=y CONFIG_IPV6_ROUTE_INFO=y CONFIG_IPV6_ROUTER_PREF=y -CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_IPV6_SUBTREES=y CONFIG_IRQ_POLL=y CONFIG_JUMP_LABEL=y diff --git a/tools/testing/selftests/bpf/default.profraw b/tools/testing/selftests/bpf/default.profraw Binary files differnew file mode 100644 index 000000000000..e865e87829f8 --- /dev/null +++ b/tools/testing/selftests/bpf/default.profraw diff --git a/tools/testing/selftests/bpf/disasm_helpers.c b/tools/testing/selftests/bpf/disasm_helpers.c index f529f1c8c171..30221352568d 100644 --- a/tools/testing/selftests/bpf/disasm_helpers.c +++ b/tools/testing/selftests/bpf/disasm_helpers.c @@ -55,10 +55,9 @@ struct bpf_insn *disasm_insn(struct bpf_insn *insn, char *buf, size_t buf_sz) * for each instruction (FF stands for instruction `code` byte). * Remove the prefix inplace, and also simplify call instructions. * E.g.: "(85) call foo#10" -> "call foo". - * Also remove newline in the end (the 'max(strlen(buf) - 1, 0)' thing). */ pfx_end = buf + 5; - sfx_start = buf + max((int)strlen(buf) - 1, 0); + sfx_start = buf + (int)strlen(buf); if (strncmp(pfx_end, "call ", 5) == 0 && (tmp = strrchr(buf, '#'))) sfx_start = tmp; len = sfx_start - pfx_end; diff --git a/tools/testing/selftests/bpf/jit_disasm_helpers.c b/tools/testing/selftests/bpf/jit_disasm_helpers.c index 364c557c5115..3558fe10e28c 100644 --- a/tools/testing/selftests/bpf/jit_disasm_helpers.c +++ b/tools/testing/selftests/bpf/jit_disasm_helpers.c @@ -96,10 +96,19 @@ static int disasm_one_func(FILE *text_out, uint8_t *image, __u32 len) __u32 *label_pc, pc; int i, cnt, err = 0; char buf[64]; + char *cpu, *features; triple = LLVMGetDefaultTargetTriple(); - ctx = LLVMCreateDisasm(triple, &labels, 0, NULL, lookup_symbol); - if (!ASSERT_OK_PTR(ctx, "LLVMCreateDisasm")) { + + cpu = LLVMGetHostCPUName(); + features = LLVMGetHostCPUFeatures(); + + ctx = LLVMCreateDisasmCPUFeatures(triple, cpu, features, &labels, 0, NULL, lookup_symbol); + + LLVMDisposeMessage(cpu); + LLVMDisposeMessage(features); + + if (!ASSERT_OK_PTR(ctx, "LLVMCreateDisasmCPUFeatures")) { err = -EINVAL; goto out; } diff --git a/tools/testing/selftests/bpf/libarena/Makefile b/tools/testing/selftests/bpf/libarena/Makefile new file mode 100644 index 000000000000..5e2ab514805e --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/Makefile @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +# Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + +.PHONY: clean + +# Defaults for standalone builds + +CLANG ?= clang +BPFTOOL ?= bpftool +LDLIBS ?= -lbpf -lelf -lz -lrt -lpthread -lzstd + +ifeq ($(V),1) +Q = +msg = +else +Q ?= @ +msg = @printf ' %-8s%s %s%s\n' "$(1)" "$(if $(2), [$(2)])" "$(notdir $(3))" "$(if $(4), $(4))"; +endif + +IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null | \ + grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__') +BPF_TARGET_ENDIAN ?= $(if $(IS_LITTLE_ENDIAN),--target=bpfel,--target=bpfeb) + +LIBARENA=$(abspath .) +BPFDIR=$(abspath $(LIBARENA)/..) + +INCLUDE_DIR ?= $(BPFDIR)/tools/include +LIBBPF_INCLUDE ?= $(INCLUDE_DIR) + +# Scan src/ and selftests/ to generate the final binaries +LIBARENA_SOURCES = $(wildcard $(LIBARENA)/src/*.bpf.c) $(wildcard $(LIBARENA)/selftests/*.bpf.c) +LIBARENA_OBJECTS = $(notdir $(LIBARENA_SOURCES:.bpf.c=.bpf.o)) +LIBARENA_OBJECTS_ASAN = $(notdir $(LIBARENA_SOURCES:.bpf.c=_asan.bpf.o)) + +INCLUDES = -I$(LIBARENA)/include -I$(BPFDIR) +ifneq ($(INCLUDE_DIR),) +INCLUDES += -I$(INCLUDE_DIR) +endif +ifneq ($(LIBBPF_INCLUDE),) +INCLUDES += -I$(LIBBPF_INCLUDE) +endif + +ASAN_FLAGS = -fsanitize=kernel-address -fno-stack-protector -fno-builtin +ASAN_FLAGS += -mllvm -asan-instrument-address-spaces=1 -mllvm -asan-shadow-addr-space=1 +ASAN_FLAGS += -mllvm -asan-use-stack-safety=0 -mllvm -asan-stack=0 +ASAN_FLAGS += -mllvm -asan-kernel=1 +ASAN_FLAGS += -mllvm -asan-constructor-kind=none +ASAN_FLAGS += -mllvm -asan-destructor-kind=none + +# ENABLE_ATOMICS_TESTS required because we use arena spinlocks +override BPF_CFLAGS += -DENABLE_ATOMICS_TESTS +override BPF_CFLAGS += -O2 -g +override BPF_CFLAGS += -Wno-incompatible-pointer-types-discards-qualifiers +# Required for suppressing harmless vmlinux.h-related warnings. +override BPF_CFLAGS += -Wno-missing-declarations +override BPF_CFLAGS += $(INCLUDES) + +CFLAGS = -O2 -no-pie +CFLAGS += $(INCLUDES) + +vpath %.bpf.c $(LIBARENA)/src $(LIBARENA)/selftests +vpath %.c $(LIBARENA)/src $(LIBARENA)/selftests + +skeletons: libarena.skel.h libarena_asan.skel.h +.PHONY: skeletons + +libarena_asan.skel.h: libarena_asan.bpf.o + $(call msg,GEN-SKEL,libarena,$@) + $(Q)$(BPFTOOL) gen skeleton $< name "libarena_asan" > $@ + +libarena.skel.h: libarena.bpf.o + $(call msg,GEN-SKEL,libarena,$@) + $(Q)$(BPFTOOL) gen skeleton $< name "libarena" > $@ + +libarena_asan.bpf.o: $(LIBARENA_OBJECTS_ASAN) + $(call msg,GEN-OBJ,libarena,$@) + $(Q)$(BPFTOOL) gen object $@ $^ + +libarena.bpf.o: $(LIBARENA_OBJECTS) + $(call msg,GEN-OBJ,libarena,$@) + $(Q)$(BPFTOOL) gen object $@ $^ + +%_asan.bpf.o: %.bpf.c + $(call msg,CLNG-BPF,libarena,$@) + $(Q)$(CLANG) $(BPF_CFLAGS) $(ASAN_FLAGS) -DBPF_ARENA_ASAN $(BPF_TARGET_ENDIAN) -c $< -o $@ + +%.bpf.o: %.bpf.c + $(call msg,CLNG-BPF,libarena,$@) + $(Q)$(CLANG) $(BPF_CFLAGS) $(BPF_TARGET_ENDIAN) -c $< -o $@ + +clean: + $(Q)rm -f *.skel.h *.bpf.o *.linked*.o diff --git a/tools/testing/selftests/bpf/bpf_arena_common.h b/tools/testing/selftests/bpf/libarena/include/bpf_arena_common.h index 16f8ce832004..82aafe879fae 100644 --- a/tools/testing/selftests/bpf/bpf_arena_common.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_arena_common.h @@ -33,12 +33,12 @@ #endif #if defined(__BPF_FEATURE_ADDR_SPACE_CAST) && !defined(BPF_ARENA_FORCE_ASM) -#define __arena __attribute__((address_space(1))) +#define __arena __attribute__((address_space(1))) __attribute__((btf_type_tag("arena"))) #define __arena_global __attribute__((address_space(1))) #define cast_kern(ptr) /* nop for bpf prog. emitted by LLVM */ #define cast_user(ptr) /* nop for bpf prog. emitted by LLVM */ #else -#define __arena +#define __arena __attribute__((btf_type_tag("arena"))) #define __arena_global SEC(".addr_space.1") #define cast_kern(ptr) bpf_addr_space_cast(ptr, 0, 1) #define cast_user(ptr) bpf_addr_space_cast(ptr, 1, 0) @@ -54,7 +54,6 @@ void bpf_arena_free_pages(void *map, void __arena *ptr, __u32 page_cnt) __ksym _ #else /* when compiled as user space code */ #define __arena -#define __arg_arena #define cast_kern(ptr) /* nop for user space */ #define cast_user(ptr) /* nop for user space */ __weak char arena[1]; diff --git a/tools/testing/selftests/bpf/progs/bpf_arena_spin_lock.h b/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h index f90531cf3ee5..71d9db610263 100644 --- a/tools/testing/selftests/bpf/progs/bpf_arena_spin_lock.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h @@ -5,7 +5,7 @@ #include <vmlinux.h> #include <bpf/bpf_helpers.h> -#include "bpf_atomic.h" +#include <bpf_atomic.h> #define arch_mcs_spin_lock_contended_label(l, label) smp_cond_load_acquire_label(l, VAL, label) #define arch_mcs_spin_unlock_contended(l) smp_store_release((l), 1) @@ -16,10 +16,6 @@ #define EOPNOTSUPP 95 #define ETIMEDOUT 110 -#ifndef __arena -#define __arena __attribute__((address_space(1))) -#endif - extern unsigned long CONFIG_NR_CPUS __kconfig; /* @@ -107,7 +103,7 @@ struct arena_qnode { #define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET) #define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET) -struct arena_qnode __arena qnodes[_Q_MAX_CPUS][_Q_MAX_NODES]; +extern struct arena_qnode __arena __hidden qnodes[_Q_MAX_CPUS][_Q_MAX_NODES]; static inline u32 encode_tail(int cpu, int idx) { @@ -240,8 +236,8 @@ static __always_inline int arena_spin_trylock(arena_spinlock_t __arena *lock) return likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL)); } -__noinline -int arena_spin_lock_slowpath(arena_spinlock_t __arena __arg_arena *lock, u32 val) +__noinline __weak +int arena_spin_lock_slowpath(arena_spinlock_t __arena *lock, u32 val) { struct arena_mcs_spinlock __arena *prev, *next, *node0, *node; int ret = -ETIMEDOUT; diff --git a/tools/testing/selftests/bpf/bpf_atomic.h b/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h index c550e5711967..43c306e17f19 100644 --- a/tools/testing/selftests/bpf/bpf_atomic.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h @@ -5,7 +5,7 @@ #include <vmlinux.h> #include <bpf/bpf_helpers.h> -#include "bpf_experimental.h" +#include <bpf_may_goto.h> extern bool CONFIG_X86_64 __kconfig __weak; @@ -42,7 +42,9 @@ extern bool CONFIG_X86_64 __kconfig __weak; #define READ_ONCE(x) (*(volatile typeof(x) *)&(x)) +#ifndef WRITE_ONCE #define WRITE_ONCE(x, val) ((*(volatile typeof(x) *)&(x)) = (val)) +#endif #define cmpxchg(p, old, new) __sync_val_compare_and_swap((p), old, new) @@ -84,6 +86,25 @@ extern bool CONFIG_X86_64 __kconfig __weak; /* Control dependency provides LOAD->STORE, provide LOAD->LOAD */ #define smp_acquire__after_ctrl_dep() ({ smp_rmb(); }) +#if defined(__BPF_FEATURE_LOAD_ACQ_STORE_REL) +/* + * Clang advertises this feature when it can lower acquire/release atomic + * builtins to BPF_LOAD_ACQ/BPF_STORE_REL. Older compilers keep using the + * barrier-based fallback below. The generated instructions require kernel + * verifier/JIT support added in Linux 6.15; compile for an older BPF CPU to + * keep using the fallback when targeting older kernels. + */ +#define smp_load_acquire(p) \ + ({ \ + __unqual_typeof(*(p)) ___p1 = __atomic_load_n((p), __ATOMIC_ACQUIRE); \ + (typeof(*(p)))___p1; \ + }) + +#define smp_store_release(p, val) \ + ({ \ + __atomic_store_n((p), (val), __ATOMIC_RELEASE); \ + }) +#else #define smp_load_acquire(p) \ ({ \ __unqual_typeof(*(p)) __v = READ_ONCE(*(p)); \ @@ -100,6 +121,7 @@ extern bool CONFIG_X86_64 __kconfig __weak; barrier(); \ WRITE_ONCE(*(p), val); \ }) +#endif #define smp_cond_load_relaxed_label(p, cond_expr, label) \ ({ \ diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h b/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h new file mode 100644 index 000000000000..9ba90689d6ba --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h @@ -0,0 +1,84 @@ +#pragma once + +/* + * Note that cond_break can only be portably used in the body of a breakable + * construct, whereas can_loop can be used anywhere. + */ +#ifdef __BPF_FEATURE_MAY_GOTO +#define can_loop \ + ({ __label__ l_break, l_continue; \ + bool ret = true; \ + asm volatile goto("may_goto %l[l_break]" \ + :::: l_break); \ + goto l_continue; \ + l_break: ret = false; \ + l_continue:; \ + ret; \ + }) + +#define __cond_break(expr) \ + ({ __label__ l_break, l_continue; \ + asm volatile goto("may_goto %l[l_break]" \ + :::: l_break); \ + goto l_continue; \ + l_break: expr; \ + l_continue:; \ + }) +#else +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define can_loop \ + ({ __label__ l_break, l_continue; \ + bool ret = true; \ + asm volatile goto("1:.byte 0xe5; \ + .byte 0; \ + .long ((%l[l_break] - 1b - 8) / 8) & 0xffff; \ + .short 0" \ + :::: l_break); \ + goto l_continue; \ + l_break: ret = false; \ + l_continue:; \ + ret; \ + }) + +#define __cond_break(expr) \ + ({ __label__ l_break, l_continue; \ + asm volatile goto("1:.byte 0xe5; \ + .byte 0; \ + .long ((%l[l_break] - 1b - 8) / 8) & 0xffff; \ + .short 0" \ + :::: l_break); \ + goto l_continue; \ + l_break: expr; \ + l_continue:; \ + }) +#else +#define can_loop \ + ({ __label__ l_break, l_continue; \ + bool ret = true; \ + asm volatile goto("1:.byte 0xe5; \ + .byte 0; \ + .long (((%l[l_break] - 1b - 8) / 8) & 0xffff) << 16; \ + .short 0" \ + :::: l_break); \ + goto l_continue; \ + l_break: ret = false; \ + l_continue:; \ + ret; \ + }) + +#define __cond_break(expr) \ + ({ __label__ l_break, l_continue; \ + asm volatile goto("1:.byte 0xe5; \ + .byte 0; \ + .long (((%l[l_break] - 1b - 8) / 8) & 0xffff) << 16; \ + .short 0" \ + :::: l_break); \ + goto l_continue; \ + l_break: expr; \ + l_continue:; \ + }) +#endif +#endif + +#define cond_break __cond_break(break) +#define cond_break_label(label) __cond_break(goto label) diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/asan.h b/tools/testing/selftests/bpf/libarena/include/libarena/asan.h new file mode 100644 index 000000000000..900267159292 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/include/libarena/asan.h @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#pragma once + +struct asan_init_args { + u64 arena_all_pages; + u64 arena_globals_pages; +}; + +int asan_init(struct asan_init_args *args); + +extern volatile u64 __asan_shadow_memory_dynamic_address; +extern volatile u32 asan_reported; +extern volatile bool asan_inited; +extern volatile bool asan_report_once; + +#ifdef __BPF__ + +#define ASAN_SHADOW_SHIFT 3 +#define ASAN_SHADOW_SCALE (1ULL << ASAN_SHADOW_SHIFT) +#define ASAN_GRANULE_MASK ((1ULL << ASAN_SHADOW_SHIFT) - 1) +#define ASAN_GRANULE(addr) ((s8)((u32)(u64)((addr)) & ASAN_GRANULE_MASK)) + +#define __noasan __attribute__((no_sanitize("address"))) + +#ifdef BPF_ARENA_ASAN + +static inline +s8 __arena *mem_to_shadow(void __arena *addr) +{ + return (s8 __arena *)(((u32)(u64)addr >> ASAN_SHADOW_SHIFT) + + __asan_shadow_memory_dynamic_address); +} + +__weak __noasan +bool asan_ready(void) +{ + return __asan_shadow_memory_dynamic_address; +} + +int asan_poison(void __arena *addr, s8 val, size_t size); +int asan_unpoison(void __arena *addr, size_t size); +bool asan_shadow_set(void __arena *addr); + +/* + * Dummy calls to ensure the ASAN runtime's BTF information is present + * in every object file when compiling the runtime and local BPF code + * separately. The runtime calls are injected into the LLVM IR file + */ +#define DECLARE_ASAN_LOAD_STORE_SIZE(size) \ + void __asan_store##size(intptr_t addr); \ + void __asan_store##size##_noabort(intptr_t addr); \ + void __asan_load##size(intptr_t addr); \ + void __asan_load##size##_noabort(intptr_t addr); \ + void __asan_report_store##size(intptr_t addr); \ + void __asan_report_store##size##_noabort(intptr_t addr); \ + void __asan_report_load##size(intptr_t addr); \ + void __asan_report_load##size##_noabort(intptr_t addr); + +DECLARE_ASAN_LOAD_STORE_SIZE(1); +DECLARE_ASAN_LOAD_STORE_SIZE(2); +DECLARE_ASAN_LOAD_STORE_SIZE(4); +DECLARE_ASAN_LOAD_STORE_SIZE(8); + +void __asan_storeN(intptr_t addr, ssize_t size); +void __asan_storeN_noabort(intptr_t addr, ssize_t size); +void __asan_loadN(intptr_t addr, ssize_t size); +void __asan_loadN_noabort(intptr_t addr, ssize_t size); + +/* + * Force LLVM to emit BTF information for the stubs, + * because the ASAN pass in LLVM by itself doesn't. + */ +#define ASAN_LOAD_STORE_SIZE(size) \ + __asan_store##size, \ + __asan_store##size##_noabort, \ + __asan_load##size, \ + __asan_load##size##_noabort, \ + __asan_report_store##size, \ + __asan_report_store##size##_noabort, \ + __asan_report_load##size, \ + __asan_report_load##size##_noabort + +__attribute__((used)) +static void (*__asan_btf_anchors[])(intptr_t) = { + ASAN_LOAD_STORE_SIZE(1), + ASAN_LOAD_STORE_SIZE(2), + ASAN_LOAD_STORE_SIZE(4), + ASAN_LOAD_STORE_SIZE(8), +}; + +#else /* BPF_ARENA_ASAN */ + +static inline int asan_poison(void __arena *addr, s8 val, size_t size) { return 0; } +static inline int asan_unpoison(void __arena *addr, size_t size) { return 0; } +static inline bool asan_shadow_set(void __arena *addr) { return 0; } +__weak bool asan_ready(void) { return true; } + +#endif /* BPF_ARENA_ASAN */ + +#endif /* __BPF__ */ diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h new file mode 100644 index 000000000000..e2431ea6fdd6 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h @@ -0,0 +1,34 @@ +#pragma once + +#define BITS_PER_BYTE 8 +#define BYTES_TO_BITS(nb) ((nb) * BITS_PER_BYTE) + +#define BITS_PER_LONG_LONG (sizeof(long long) * BITS_PER_BYTE) +#define BITS_TO_LONG_LONGS(nr) (((nr) + BITS_PER_LONG_LONG - 1) / BITS_PER_LONG_LONG) +#define BIT_MASK(nr) (1ULL << ((nr) % BITS_PER_LONG_LONG)) +#define BIT_WORD(nr) ((nr) / BITS_PER_LONG_LONG) + +struct arena_bitmap { + u64 bits[0]; +}; + +struct arena_bitmap __arena *bmp_alloc(size_t bits); +void bmp_free(struct arena_bitmap __arena *bmp); + +void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp); +void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); +void bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp); +void bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); +bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp); +bool bmp_test_and_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); +bool bmp_test_and_set_bit(u32 bit, struct arena_bitmap __arena *bmp); + +void bmp_clear(size_t bits, struct arena_bitmap __arena *bmp); +void bmp_and(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src1, struct arena_bitmap __arena *src2); +void bmp_or(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src1, struct arena_bitmap __arena *src2); +bool bmp_empty(size_t bits, struct arena_bitmap __arena *bmp); +void bmp_copy(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src); + +bool bmp_intersects(size_t bits, struct arena_bitmap __arena *arg1, struct arena_bitmap __arena *arg2); +bool bmp_subset(size_t bits, struct arena_bitmap __arena *big, struct arena_bitmap __arena *small); +void bmp_print(size_t bits, struct arena_bitmap __arena *bmp); diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/buddy.h b/tools/testing/selftests/bpf/libarena/include/libarena/buddy.h new file mode 100644 index 000000000000..528c69a1f38e --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/include/libarena/buddy.h @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#pragma once + +enum buddy_consts { + /* + * Minimum allocation is 1 << BUDDY_MIN_ALLOC_SHIFT. + * Larger sizes increase internal fragmentation, but smaller + * sizes increase the space overhead of the block metadata. + */ + BUDDY_MIN_ALLOC_SHIFT = 4, + BUDDY_MIN_ALLOC_BYTES = 1 << BUDDY_MIN_ALLOC_SHIFT, + + /* + * How many orders the buddy allocator can serve. Minimum block + * size is 1 << BUDDY_MIN_ALLOC_SHIFT, maximum block size is + * 1 << (BUDDY_MIN_ALLOC_SHIFT + BUDDY_CHUNK_NUM_ORDERS - 1): + * Each block has size 1 << BUDDY_MIN_ALLOC_SHIFT, and the + * allocation orders are in [0, BUDDY_CHUNK_NUM_ORDERS). + * We keep two blocks of the maximum size to retain the + * property in the code that all blocks have a buddy. + * Higher values increase the maximum allocation size, + * but also the size of the metadata for each block. + */ + BUDDY_CHUNK_NUM_ORDERS = 1 << 4, + BUDDY_CHUNK_BYTES = BUDDY_MIN_ALLOC_BYTES << (BUDDY_CHUNK_NUM_ORDERS), + + /* Offset of the buddy header within a free block, see buddy.bpf.c for details */ + BUDDY_HEADER_OFF = 8, + + /* The maximum number of blocks a chunk may have to track. */ + BUDDY_CHUNK_ITEMS = 1 << (BUDDY_CHUNK_NUM_ORDERS), + BUDDY_CHUNK_OFFSET_MASK = BUDDY_CHUNK_BYTES - 1, + + /* + * Alignment for chunk allocations based on bpf_arena_alloc_pages. + * The arena allocation kfunc does not have an alignment argument, + * but that is required for all block calculations in the chunk to + * work. + */ + BUDDY_VADDR_OFFSET = BUDDY_CHUNK_BYTES, + + /* Total arena virtual address space the allocator can consume. */ + BUDDY_VADDR_SIZE = BUDDY_CHUNK_BYTES << 10 +}; + +struct buddy_header { + u32 prev_index; /* "Pointer" to the previous available allocation of the same size. */ + u32 next_index; /* Same for the next allocation. */ +}; + +/* + * We bring memory into the allocator 1 MiB at a time. + */ +struct buddy_chunk { + /* The order of the current allocation for a item. 4 bits per order. */ + u8 orders[BUDDY_CHUNK_ITEMS / 2]; + /* + * Bit to denote whether chunk is allocated. Size of the allocated/free + * chunk found from the orders array. + */ + u8 allocated[BUDDY_CHUNK_ITEMS / 8]; + /* Freelists for O(1) allocation. */ + u64 freelists[BUDDY_CHUNK_NUM_ORDERS]; + struct buddy_chunk __arena *next; +}; + +struct buddy { + struct buddy_chunk __arena *first_chunk; /* Pointer to the chunk linked list. */ + arena_spinlock_t lock; /* Allocator lock */ + u64 vaddr; /* Allocation into reserved vaddr */ +}; + +#ifdef __BPF__ + +int buddy_init(struct buddy __arena *buddy); +int buddy_destroy(struct buddy __arena *buddy); +int buddy_free(struct buddy __arena *buddy, void __arena *free); +void __arena *buddy_alloc(struct buddy __arena *buddy, size_t size); + +#endif /* __BPF__ */ diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/common.h b/tools/testing/selftests/bpf/libarena/include/libarena/common.h new file mode 100644 index 000000000000..931ace9a49e2 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/include/libarena/common.h @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#pragma once + +#ifdef __BPF__ + +#include <vmlinux.h> + +#include <bpf_arena_common.h> +#include <bpf_arena_spin_lock.h> + +#include <asm-generic/errno.h> + +#ifndef __BPF_FEATURE_ADDR_SPACE_CAST +#error "Arena allocators require bpf_addr_space_cast feature" +#endif + +#define arena_stdout(fmt, ...) bpf_stream_printk(1, (fmt), ##__VA_ARGS__) +#define arena_stderr(fmt, ...) bpf_stream_printk(2, (fmt), ##__VA_ARGS__) + +#ifndef __maybe_unused +#define __maybe_unused __attribute__((__unused__)) +#endif + +#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) + +#define ARENA_PAGES (1UL << (32 - __builtin_ffs(__PAGE_SIZE) + 1)) + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, ARENA_PAGES); /* number of pages */ +#if defined(__TARGET_ARCH_arm64) || defined(__aarch64__) + __ulong(map_extra, (1ull << 32)); /* start of mmap() region */ +#else + __ulong(map_extra, (1ull << 44)); /* start of mmap() region */ +#endif +} arena __weak SEC(".maps"); + +/* + * This is a variable used to aid verification. The may_goto directive + * permits open-coded for loops, but requires that the index variable is + * imprecise. To force the variable to be imprecise, initialize it with + * the opaque volatile variable 0 instead of the constant 0. + */ +volatile u32 zero __weak; +extern volatile u64 asan_violated; + +int arena_fls(__u64 word); + +void __arena *arena_malloc(size_t size); +void arena_free(void __arena *ptr); + +/* + * The verifier associates arenas with programs by checking LD.IMM + * instruction operands for an arena and populating the program state + * with the first instance it finds. This requires accessing our global + * arena variable, but subprogs do not necessarily do so while still + * using pointers from that arena. Insert an LD.IMM instruction to + * access the arena and help the verifier. + */ +#define arena_subprog_init() do { asm volatile ("" :: "r"(&arena)); } while (0) + +#else /* ! __BPF__ */ + +#include <stdint.h> + +#define __arena + +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +typedef int8_t s8; +typedef int16_t s16; +typedef int32_t s32; +typedef int64_t s64; + +/* Dummy "definition" for userspace. */ +#define arena_spinlock_t int + +#endif /* __BPF__ */ + +struct arena_get_info_args { + void __arena *arena_base; +}; + +struct arena_alloc_reserve_args { + u64 nr_pages; +}; + +/* Reasonable default number of pages reserved by arena_alloc_reserve. */ +#define ARENA_RESERVE_PAGES_DFL (8) diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/rbtree.h b/tools/testing/selftests/bpf/libarena/include/libarena/rbtree.h new file mode 100644 index 000000000000..486428911d96 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/include/libarena/rbtree.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause */ + +#pragma once + +#define RB_MAXLVL_PRINT (16) + +struct rbnode; + +struct rbnode { + struct rbnode __arena *parent; + union { + struct { + struct rbnode __arena *left; + struct rbnode __arena *right; + }; + + struct rbnode __arena *child[2]; + }; + uint64_t key; + /* Used as a linked list or to store KV pairs. */ + union { + struct rbnode __arena *next; + uint64_t value; + }; + bool is_red; +}; + +/* + * Does the rbtree allocate its own nodes, or do they get + * allocated by the caller? + */ +enum rbtree_alloc { + RB_ALLOC, + RB_NOALLOC, +}; + +/* + * Specify the behavior of rbtree insertions when the key is + * already present in the tree. + * + * RB_DEFAULT: Default behavior, reject the new insert. + * + * RB_UPDATE: Update the existing value in the rbtree. + * This updates the node itself, not just the value in + * the existing node. + * + * RB_DUPLICATE: Allow nodes with identical keys in the rbtree. + * Finding/popping/removing a key acts on any of the nodes + * with the appropriate key - there is no ordering by time + * of insertion. + */ +enum rbtree_insert_mode { + RB_DEFAULT, + RB_UPDATE, + RB_DUPLICATE, +}; + +struct rbtree { + struct rbnode __arena *root; + enum rbtree_alloc alloc; + enum rbtree_insert_mode insert; +}; + +#ifdef __BPF__ +struct rbtree __arena *rb_create(enum rbtree_alloc alloc, enum rbtree_insert_mode insert); + +int rb_destroy(struct rbtree __arena *rbtree); +int rb_insert(struct rbtree __arena *rbtree, u64 key, u64 value); +int rb_remove(struct rbtree __arena *rbtree, u64 key); +int rb_find(struct rbtree __arena *rbtree, u64 key, u64 *value); +int rb_print(struct rbtree __arena *rbtree); +int rb_least(struct rbtree __arena *rbtree, u64 *key, u64 *value); +int rb_pop(struct rbtree __arena *rbtree, u64 *key, u64 *value); + +int rb_insert_node(struct rbtree __arena *rbtree, struct rbnode __arena *node); +int rb_remove_node(struct rbtree __arena *rbtree, struct rbnode __arena *node); + +struct rbnode __arena *rb_node_alloc(u64 key, u64 value); +void rb_node_free(struct rbnode __arena *rbnode); + +int rb_integrity_check(struct rbtree __arena *rbtree); + +#endif /* __BPF__ */ diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/spmc.h b/tools/testing/selftests/bpf/libarena/include/libarena/spmc.h new file mode 100644 index 000000000000..75611276ce13 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/include/libarena/spmc.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause */ + +#pragma once + +struct spmc_arr; + +#define SPMC_ARR_BASESZ 128 +#define SPMC_ARR_ORDERS 10 + +struct spmc_arr { + u64 __arena *data; + u64 order; +}; + +struct spmc { + volatile struct spmc_arr __arena *cur; + volatile u64 top; + volatile u64 bottom; + struct spmc_arr arr[SPMC_ARR_ORDERS]; +}; + +int spmc_owned_add(struct spmc __arena *spmc, u64 val); +int spmc_owned_remove(struct spmc __arena *spmc, u64 *val); +int spmc_steal(struct spmc __arena *spmc, u64 *val); + +struct spmc __arena *spmc_create(void); +int spmc_destroy(struct spmc __arena *spmc); diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h b/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h new file mode 100644 index 000000000000..b6676dd67bc0 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#pragma once + +#include <errno.h> +#include <stdlib.h> +#include <unistd.h> +#include <sys/mman.h> + +#include <bpf/libbpf.h> +#include <bpf/bpf.h> + +static inline int libarena_run_prog(int prog_fd) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int ret; + + ret = bpf_prog_test_run_opts(prog_fd, &opts); + if (ret) + return ret; + + return opts.retval; +} + +static inline bool libarena_is_test_prog(const char *name) +{ + return strstr(name, "test_") == name; +} + +static inline bool libarena_is_asan_test_prog(const char *name) +{ + return strstr(name, "asan_test") == name; +} + +static inline bool libarena_is_parallel_test_prog(const char *name) +{ + return strstr(name, "parallel_test") == name; +} + + +static inline int libarena_run_prog_args(int prog_fd, void *args, size_t argsize) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int ret; + + opts.ctx_in = args; + opts.ctx_size_in = argsize; + + ret = bpf_prog_test_run_opts(prog_fd, &opts); + + return ret ?: opts.retval; +} + +static inline int libarena_get_arena_base(int arena_get_info_fd, + void **arena_base) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct arena_get_info_args args = { .arena_base = NULL }; + int ret; + + opts.ctx_in = &args; + opts.ctx_size_in = sizeof(args); + + ret = bpf_prog_test_run_opts(arena_get_info_fd, &opts); + if (ret) + return ret; + if (opts.retval) + return opts.retval; + + *arena_base = args.arena_base; + return 0; +} + +static inline int libarena_get_globals_pages(int arena_get_globals_fd, + size_t arena_all_pages, + u64 *globals_pages) +{ + size_t pgsize = sysconf(_SC_PAGESIZE); + void *arena_base; + ssize_t i; + u8 *vec; + int ret; + + ret = libarena_get_arena_base(arena_get_globals_fd, &arena_base); + if (ret) + return ret; + + if (!arena_base) + return -EINVAL; + + vec = calloc(arena_all_pages, sizeof(*vec)); + if (!vec) + return -ENOMEM; + + if (mincore(arena_base, arena_all_pages * pgsize, vec) < 0) { + ret = -errno; + free(vec); + return ret; + } + + *globals_pages = 0; + for (i = arena_all_pages - 1; i >= 0; i--) { + if (!(vec[i] & 0x1)) + break; + *globals_pages += 1; + } + + free(vec); + return 0; +} + +static inline int libarena_asan_init(int arena_asan_init_fd, + int asan_init_fd, + size_t arena_all_pages) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct asan_init_args args; + u64 globals_pages = 0; + int ret; + + ret = libarena_get_globals_pages(arena_asan_init_fd, + arena_all_pages, &globals_pages); + if (ret) + return ret; + + args = (struct asan_init_args){ + .arena_all_pages = arena_all_pages, + .arena_globals_pages = globals_pages, + }; + + opts.ctx_in = &args; + opts.ctx_size_in = sizeof(args); + + ret = bpf_prog_test_run_opts(asan_init_fd, &opts); + if (ret) + return ret; + return opts.retval; +} diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_asan_buddy.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_asan_buddy.bpf.c new file mode 100644 index 000000000000..3266a28f53d7 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/selftests/test_asan_buddy.bpf.c @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <libarena/common.h> +#include <libarena/asan.h> +#include <libarena/buddy.h> + +/* Required for parsing the ASAN call stacks. */ +#include "test_progs_compat.h" + +extern struct buddy __arena buddy; + +#ifdef BPF_ARENA_ASAN + +#include "test_asan_common.h" + +static __always_inline int asan_test_buddy_oob_single(size_t alloc_size) +{ + u8 __arena *mem; + int ret, i; + + ret = asan_validate(); + if (ret < 0) + return ret; + + mem = buddy_alloc(&buddy, alloc_size); + if (!mem) { + arena_stdout("buddy_alloc failed for size %lu", alloc_size); + return -ENOMEM; + } + + ret = asan_validate(); + if (ret < 0) + return ret; + + for (i = zero; i < alloc_size && can_loop; i++) { + mem[i] = 0xba; + ret = asan_validate_addr(false, &mem[i]); + if (ret < 0) + return ret; + } + + mem[alloc_size] = 0xba; + ret = asan_validate_addr(true, &mem[alloc_size]); + if (ret < 0) + return ret; + + buddy_free(&buddy, mem); + + return 0; +} + +/* + * Factored out because asan_validate_addr is complex enough to cause + * verification failures if verified with the rest of asan_test_buddy_uaf_single. + */ +__weak int asan_test_buddy_byte(u8 __arena *mem, int i, bool freed) +{ + int ret; + + /* The header in freed blocks doesn't get poisoned. */ + if (freed && BUDDY_HEADER_OFF <= i && + i < BUDDY_HEADER_OFF + sizeof(struct buddy_header)) + return 0; + + mem[i] = 0xba; + ret = asan_validate_addr(freed, &mem[i]); + if (ret < 0) + return ret; + + return 0; +} + +__weak int asan_test_buddy_uaf_single(size_t alloc_size) +{ + u8 __arena *mem; + int ret; + int i; + + mem = buddy_alloc(&buddy, alloc_size); + if (!mem) { + arena_stdout("buddy_alloc failed for size %lu", alloc_size); + return -ENOMEM; + } + + ret = asan_validate(); + if (ret < 0) + return ret; + + for (i = zero; i < alloc_size && can_loop; i++) { + ret = asan_test_buddy_byte(mem, i, false); + if (ret) + return ret; + } + + ret = asan_validate(); + if (ret < 0) + return ret; + + buddy_free(&buddy, mem); + + for (i = zero; i < alloc_size && can_loop; i++) { + ret = asan_test_buddy_byte(mem, i, true); + if (ret) + return ret; + } + + return 0; +} + +struct buddy_blob { + volatile u8 mem[48]; + u8 oob; +}; + +static __always_inline int asan_test_buddy_blob_single(void) +{ + volatile struct buddy_blob __arena *blob; + const size_t alloc_size = sizeof(struct buddy_blob) - 1; + int ret; + + blob = buddy_alloc(&buddy, alloc_size); + if (!blob) + return -ENOMEM; + + blob->mem[0] = 0xba; + ret = asan_validate_addr(false, &blob->mem[0]); + if (ret < 0) + return ret; + + blob->mem[47] = 0xba; + ret = asan_validate_addr(false, &blob->mem[47]); + if (ret < 0) + return ret; + + blob->oob = 0; + ret = asan_validate_addr(true, &blob->oob); + if (ret < 0) + return ret; + + buddy_free(&buddy, (void __arena *)blob); + + return 0; +} + +SEC("syscall") +__stderr("Memory violation for address {{.*}} for write of size 1") +__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}") +__stderr("Call trace:\n" +"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n" +"|[ \t]+[^\n]+\n)*}}") +__weak int asan_test_buddy_oob(void) +{ + size_t sizes[] = { + 7, 8, 17, 18, 64, 256, 317, 512, 1024, + }; + int ret; + u32 i; + + ret = buddy_init(&buddy); + if (ret) { + arena_stdout("buddy_init failed with %d", ret); + return ret; + } + + for (i = zero; i < sizeof(sizes) / sizeof(sizes[0]) && can_loop; i++) { + barrier_var(i); + ret = asan_test_buddy_oob_single(sizes[i]); + if (ret) { + arena_stdout("%s:%d Failed for size %lu", __func__, + __LINE__, sizes[i]); + buddy_destroy(&buddy); + return ret; + } + } + + buddy_destroy(&buddy); + + ret = asan_validate(); + if (ret < 0) + return ret; + + return 0; +} + +SEC("syscall") +__stderr("Memory violation for address {{.*}} for write of size 1") +__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}") +__stderr("Call trace:\n" +"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n" +"|[ \t]+[^\n]+\n)*}}") +__weak int asan_test_buddy_uaf(void) +{ + size_t sizes[] = { 16, 32, 64, 128, 256, 512, 1024, 16384 }; + int ret; + u32 i; + + ret = buddy_init(&buddy); + if (ret) { + arena_stdout("buddy_init failed with %d", ret); + return ret; + } + + for (i = zero; i < sizeof(sizes) / sizeof(sizes[0]) && can_loop; i++) { + barrier_var(i); + ret = asan_test_buddy_uaf_single(sizes[i]); + if (ret) { + arena_stdout("%s:%d Failed for size %lu", __func__, + __LINE__, sizes[i]); + buddy_destroy(&buddy); + return ret; + } + } + + buddy_destroy(&buddy); + + ret = asan_validate(); + if (ret < 0) + return ret; + + return 0; +} + +SEC("syscall") +__stderr("Memory violation for address {{.*}} for write of size 1") +__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}") +__stderr("Call trace:\n" +"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n" +"|[ \t]+[^\n]+\n)*}}") +__weak int asan_test_buddy_blob(void) +{ + const int iters = 10; + int ret, i; + + ret = buddy_init(&buddy); + if (ret) { + arena_stdout("buddy_init failed with %d", ret); + return ret; + } + + for (i = zero; i < iters && can_loop; i++) { + ret = asan_test_buddy_blob_single(); + if (ret) { + arena_stdout("%s:%d Failed on iteration %d", __func__, + __LINE__, i); + buddy_destroy(&buddy); + return ret; + } + } + + buddy_destroy(&buddy); + + ret = asan_validate(); + if (ret < 0) + return ret; + + return 0; +} + +#endif + +__weak char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_asan_common.h b/tools/testing/selftests/bpf/libarena/selftests/test_asan_common.h new file mode 100644 index 000000000000..34a7918cb4cf --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/selftests/test_asan_common.h @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#pragma once + +#define ST_PAGES 64 + +static inline void print_asan_map_state(void __arena *addr) +{ + arena_stdout("%s:%d ASAN %p -> (val: %x gran: %x set: [%s])", + __func__, __LINE__, addr, + *(s8 __arena *)(addr), ASAN_GRANULE(addr), + asan_shadow_set(addr) ? "yes" : "no"); +} + +/* + * Emit an error and force the current function to exit if the ASAN + * violation state is unexpected. Reset the violation state after. + */ +static inline int asan_validate_addr(bool cond, void __arena *addr) +{ + if ((asan_violated != 0) == cond) { + asan_violated = 0; + return 0; + } + + arena_stdout("%s:%d ASAN asan_violated %lx", __func__, __LINE__, + (u64)asan_violated); + print_asan_map_state(addr); + + asan_violated = 0; + + return -EINVAL; +} + +static inline int asan_validate(void) +{ + if (!asan_violated) + return 0; + + arena_stdout("%s:%d Found ASAN violation at %lx", __func__, __LINE__, + asan_violated); + + asan_violated = 0; + + return -EINVAL; +} + +struct blob { + volatile u8 mem[59]; + u8 oob; +}; diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c new file mode 100644 index 000000000000..76319a529f02 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c @@ -0,0 +1,394 @@ +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/bitmap.h> + +#define TEST_BITS (2 * BITS_PER_LONG_LONG) +#define TEST_WORDS BITS_TO_LONG_LONGS(TEST_BITS) +#define MID_BIT (BITS_PER_LONG_LONG + 1) +#define LAST_BIT (TEST_BITS - 1) + +static void test_bmp_setall(struct arena_bitmap __arena *bmp) +{ + volatile u32 i; + + for (i = zero; i < TEST_WORDS && can_loop; i++) + bmp->bits[i] = ~0ULL; +} + +SEC("syscall") +__weak int test_bitmap_alloc_free(void) +{ + struct arena_bitmap __arena *bmp; + + bmp = bmp_alloc(TEST_BITS); + if (!bmp) + return -ENOMEM; + + if (!bmp_empty(TEST_BITS, bmp)) + goto err; + + __bmp_set_bit(LAST_BIT, bmp); + if (!bmp_test_bit(LAST_BIT, bmp)) + goto err; + + __bmp_clear_bit(LAST_BIT, bmp); + if (bmp_test_bit(LAST_BIT, bmp)) + goto err; + + bmp_free(bmp); + return 0; + +err: + bmp_free(bmp); + return -EINVAL; +} + +SEC("syscall") +__weak int test_bitmap_bit_ops(void) +{ + struct arena_bitmap __arena *bmp; + + bmp = bmp_alloc(TEST_BITS); + if (!bmp) + return -ENOMEM; + + __bmp_set_bit(0, bmp); + if (!bmp_test_bit(0, bmp)) + goto err; + + __bmp_set_bit(MID_BIT, bmp); + if (!bmp_test_bit(MID_BIT, bmp)) + goto err; + + __bmp_set_bit(LAST_BIT, bmp); + if (!bmp_test_bit(LAST_BIT, bmp)) + goto err; + + if (bmp_test_bit(MID_BIT - 1, bmp)) + goto err; + + __bmp_clear_bit(MID_BIT, bmp); + if (bmp_test_bit(MID_BIT, bmp)) + goto err; + + if (!bmp_test_bit(0, bmp)) + goto err; + + if (!bmp_test_bit(LAST_BIT, bmp)) + goto err; + + __bmp_clear_bit(0, bmp); + __bmp_clear_bit(LAST_BIT, bmp); + if (!bmp_empty(TEST_BITS, bmp)) + goto err; + + if (bmp->bits[0]) + goto err; + + if (bmp->bits[1]) + goto err; + + bmp_free(bmp); + return 0; + +err: + bmp_free(bmp); + return -EINVAL; +} + +static bool test_bitmap_test_and_clear_single(struct arena_bitmap __arena *bmp, size_t ind) +{ + if (bmp_test_and_clear_bit(ind, bmp)) + return false; + + __bmp_set_bit(ind, bmp); + + if (!bmp_test_and_clear_bit(ind, bmp)) + return false; + + if (bmp_test_bit(ind, bmp)) + return false; + + if (bmp_test_and_clear_bit(ind, bmp)) + return false; + + return true; +} + +static bool test_bitmap_test_and_set_single(struct arena_bitmap __arena *bmp, size_t ind) +{ + if (bmp_test_and_set_bit(ind, bmp)) + return false; + + if (!bmp_test_and_set_bit(ind, bmp)) + return false; + + if (!bmp_test_bit(ind, bmp)) + return false; + + __bmp_clear_bit(ind, bmp); + + if (bmp_test_and_set_bit(ind, bmp)) + return false; + + return true; +} + +SEC("syscall") +__weak int test_bitmap_test_and_clear_bit(void) +{ + struct arena_bitmap __arena *bmp; + + bmp = bmp_alloc(TEST_BITS); + if (!bmp) + return -ENOMEM; + + if (!test_bitmap_test_and_clear_single(bmp, 0)) + goto err; + + if (!test_bitmap_test_and_clear_single(bmp, MID_BIT)) + goto err; + + if (!test_bitmap_test_and_clear_single(bmp, LAST_BIT)) + goto err; + + if (!bmp_empty(TEST_BITS, bmp)) + goto err; + + bmp_free(bmp); + return 0; + +err: + bmp_free(bmp); + return -EINVAL; +} + +SEC("syscall") +__weak int test_bitmap_test_and_set_bit(void) +{ + struct arena_bitmap __arena *bmp; + + bmp = bmp_alloc(TEST_BITS); + if (!bmp) + return -ENOMEM; + + if (!test_bitmap_test_and_set_single(bmp, 0)) + goto err; + + if (!test_bitmap_test_and_set_single(bmp, MID_BIT)) + goto err; + + if (!test_bitmap_test_and_set_single(bmp, LAST_BIT)) + goto err; + + bmp_free(bmp); + return 0; + +err: + bmp_free(bmp); + return -EINVAL; +} + + +SEC("syscall") +__weak int test_bitmap_and(void) +{ + struct arena_bitmap __arena *src1 = NULL, *src2 = NULL, *dst = NULL; + + src1 = bmp_alloc(TEST_BITS); + src2 = bmp_alloc(TEST_BITS); + dst = bmp_alloc(TEST_BITS); + if (!src1 || !src2 || !dst) + goto err; + + test_bmp_setall(dst); + + __bmp_set_bit(0, src1); + __bmp_set_bit(MID_BIT, src1); + __bmp_set_bit(LAST_BIT, src1); + + __bmp_set_bit(MID_BIT, src2); + __bmp_set_bit(LAST_BIT, src2); + + bmp_and(TEST_BITS, dst, src1, src2); + + if (bmp_test_bit(0, dst)) + goto err; + if (!bmp_test_bit(MID_BIT, dst)) + goto err; + if (!bmp_test_bit(LAST_BIT, dst)) + goto err; + + if (dst->bits[0]) + goto err; + if (dst->bits[1] != (BIT_MASK(MID_BIT) | BIT_MASK(LAST_BIT))) + goto err; + + bmp_free(src1); + bmp_free(src2); + bmp_free(dst); + return 0; + +err: + bmp_free(src1); + bmp_free(src2); + bmp_free(dst); + return -EINVAL; +} + +SEC("syscall") +__weak int test_bitmap_or(void) +{ + struct arena_bitmap __arena *src1 = NULL, *src2 = NULL, *dst = NULL; + + src1 = bmp_alloc(TEST_BITS); + src2 = bmp_alloc(TEST_BITS); + dst = bmp_alloc(TEST_BITS); + if (!src1 || !src2 || !dst) + goto err; + + test_bmp_setall(dst); + + __bmp_set_bit(0, src1); + __bmp_set_bit(LAST_BIT, src1); + + __bmp_set_bit(MID_BIT, src2); + __bmp_set_bit(LAST_BIT, src2); + + bmp_or(TEST_BITS, dst, src1, src2); + + if (!bmp_test_bit(0, dst)) + goto err; + if (!bmp_test_bit(MID_BIT, dst)) + goto err; + if (!bmp_test_bit(LAST_BIT, dst)) + goto err; + + if (dst->bits[0] != BIT_MASK(0)) + goto err; + if (dst->bits[1] != (BIT_MASK(MID_BIT) | BIT_MASK(LAST_BIT))) + goto err; + + bmp_free(src1); + bmp_free(src2); + bmp_free(dst); + return 0; + +err: + bmp_free(src1); + bmp_free(src2); + bmp_free(dst); + return -EINVAL; +} + +SEC("syscall") +__weak int test_bitmap_subset(void) +{ + struct arena_bitmap __arena *big = NULL, *small = NULL; + + big = bmp_alloc(TEST_BITS); + small = bmp_alloc(TEST_BITS); + if (!big || !small) + goto err; + + if (!bmp_subset(TEST_BITS, big, small)) + goto err; + + __bmp_set_bit(0, small); + if (bmp_subset(TEST_BITS, big, small)) + goto err; + + __bmp_set_bit(0, big); + if (!bmp_subset(TEST_BITS, big, small)) + goto err; + + __bmp_set_bit(LAST_BIT, small); + if (bmp_subset(TEST_BITS, big, small)) + goto err; + + __bmp_set_bit(LAST_BIT, big); + __bmp_set_bit(MID_BIT, big); + if (!bmp_subset(TEST_BITS, big, small)) + goto err; + + if (bmp_subset(TEST_BITS, small, big)) + goto err; + + bmp_free(big); + bmp_free(small); + return 0; + +err: + bmp_free(big); + bmp_free(small); + return -EINVAL; + +} + +SEC("syscall") +__weak int test_bitmap_intersects(void) +{ + struct arena_bitmap __arena *arg1 = NULL, *arg2 = NULL; + + arg1 = bmp_alloc(TEST_BITS); + arg2 = bmp_alloc(TEST_BITS); + if (!arg1 || !arg2) + goto err; + + if (bmp_intersects(TEST_BITS, arg1, arg2)) + goto err; + + __bmp_set_bit(0, arg1); + __bmp_set_bit(MID_BIT, arg2); + if (bmp_intersects(TEST_BITS, arg1, arg2)) + goto err; + + __bmp_set_bit(LAST_BIT, arg1); + __bmp_set_bit(LAST_BIT, arg2); + if (!bmp_intersects(TEST_BITS, arg1, arg2)) + goto err; + + bmp_free(arg1); + bmp_free(arg2); + return 0; + +err: + bmp_free(arg1); + bmp_free(arg2); + return -EINVAL; +} + +SEC("syscall") +__weak int test_bitmap_copy(void) +{ + struct arena_bitmap __arena *arg1 = NULL, *arg2 = NULL; + + arg1 = bmp_alloc(TEST_BITS); + arg2 = bmp_alloc(TEST_BITS); + if (!arg1 || !arg2) + goto err; + + __bmp_set_bit(0, arg1); + __bmp_set_bit(MID_BIT, arg1); + + /* Make sure those get overwritten. */ + __bmp_set_bit(1, arg2); + __bmp_set_bit(MID_BIT + 2, arg2); + + bmp_copy(TEST_BITS, arg2, arg1); + + /* Bitmaps are equal if a subset of each other. */ + if (!bmp_subset(TEST_BITS, arg1, arg2) || + !bmp_subset(TEST_BITS, arg2, arg1)) + goto err; + + bmp_free(arg1); + bmp_free(arg2); + return 0; + +err: + bmp_free(arg1); + bmp_free(arg2); + return -EINVAL; +} diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_buddy.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_buddy.bpf.c new file mode 100644 index 000000000000..5628f0987012 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/selftests/test_buddy.bpf.c @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/buddy.h> + +extern struct buddy __arena buddy; + +struct segarr_entry { + u8 __arena *block; + size_t sz; + u8 poison; +}; + +#define SEGARRLEN (512) +static struct segarr_entry __arena segarr[SEGARRLEN]; +static void __arena *ptrs[17]; +size_t __arena alloc_sizes[] = { 3, 17, 1025, 129, 16350, 333, 9, 517 }; +size_t __arena alloc_multiple_sizes[] = { 3, 17, 1025, 129, 16350, 333, 9, 517, 2099 }; +size_t __arena alloc_free_sizes[] = { 3, 17, 64, 129, 256, 333, 512, 517 }; +size_t __arena alignment_sizes[] = { 1, 3, 7, 8, 9, 15, 16, 17, 31, + 32, 64, 100, 128, 255, 256, 512, 1000 }; + +SEC("syscall") +__weak int test_buddy_create(void) +{ + const int iters = 10; + int ret, i; + + for (i = zero; i < iters && can_loop; i++) { + ret = buddy_init(&buddy); + if (ret) + return ret; + + ret = buddy_destroy(&buddy); + if (ret) + return ret; + } + + return 0; +} + +SEC("syscall") +__weak int test_buddy_alloc(void) +{ + void __arena *mem; + int ret, i; + + for (i = zero; i < 8 && can_loop; i++) { + ret = buddy_init(&buddy); + if (ret) + return ret; + + mem = buddy_alloc(&buddy, alloc_sizes[i]); + if (!mem) { + buddy_destroy(&buddy); + return -ENOMEM; + } + + buddy_destroy(&buddy); + } + + return 0; +} + +SEC("syscall") +__weak int test_buddy_alloc_free(void) +{ + const int iters = 800; + void __arena *mem; + int ret, i; + + ret = buddy_init(&buddy); + if (ret) + return ret; + + for (i = zero; i < iters && can_loop; i++) { + mem = buddy_alloc(&buddy, alloc_free_sizes[(i * 5) % 8]); + if (!mem) { + buddy_destroy(&buddy); + return -ENOMEM; + } + + buddy_free(&buddy, mem); + } + + buddy_destroy(&buddy); + + return 0; +} + +SEC("syscall") +__weak int test_buddy_alloc_multiple(void) +{ + int ret, j; + u32 i, idx; + u8 __arena *mem; + size_t sz; + u8 poison; + + ret = buddy_init(&buddy); + if (ret) + return ret; + + /* + * Cycle through each size, allocating an entry in the + * segarr. Continue for SEGARRLEN iterations. For every + * allocation write down the size, use the current index + * as a poison value, and log it with the pointer in the + * segarr entry. Use the poison value to poison the entire + * allocated memory according to the size given. + */ + for (i = zero; i < SEGARRLEN && can_loop; i++) { + sz = alloc_multiple_sizes[i % 9]; + poison = (u8)i; + + mem = buddy_alloc(&buddy, sz); + if (!mem) { + buddy_destroy(&buddy); + arena_stdout("%s:%d", __func__, __LINE__); + return -ENOMEM; + } + + segarr[i].block = mem; + segarr[i].sz = sz; + segarr[i].poison = poison; + + for (j = zero; j < sz && can_loop; j++) { + mem[j] = poison; + if (mem[j] != poison) { + buddy_destroy(&buddy); + return -EINVAL; + } + } + } + + /* + * Go to (i * 17) % SEGARRLEN, and free the block pointed to. + * Before freeing, check all bytes have the poisoned value + * corresponding to the element. If any values are unexpected, + * return an error. Skip some elements to test destroying the + * buddy allocator while data is still allocated. + */ + for (i = 10; i < SEGARRLEN && can_loop; i++) { + idx = (i * 17) % SEGARRLEN; + + mem = segarr[idx].block; + sz = segarr[idx].sz; + poison = segarr[idx].poison; + + for (j = zero; j < sz && can_loop; j++) { + if (mem[j] != poison) { + buddy_destroy(&buddy); + arena_stdout("%s:%d %lx %u vs %u", __func__, + __LINE__, (uintptr_t)&mem[j], + mem[j], poison); + return -EINVAL; + } + } + + buddy_free(&buddy, mem); + } + + buddy_destroy(&buddy); + + return 0; +} + +SEC("syscall") +__weak int test_buddy_alignment(void) +{ + int ret; + u32 i; + + ret = buddy_init(&buddy); + if (ret) + return ret; + + /* Allocate various sizes and check alignment */ + for (i = zero; i < 17 && can_loop; i++) { + barrier_var(i); + ptrs[i] = buddy_alloc(&buddy, alignment_sizes[i]); + if (!ptrs[i]) { + arena_stdout("alignment test: alloc failed for size %lu", + alignment_sizes[i]); + buddy_destroy(&buddy); + return -ENOMEM; + } + + /* Check 8-byte alignment */ + if ((u64)ptrs[i] & 0x7) { + arena_stdout( + "alignment test: ptr %llx not 8-byte aligned (size %lu)", + (u64)ptrs[i], alignment_sizes[i]); + buddy_destroy(&buddy); + return -EINVAL; + } + } + + /* Free all allocations */ + for (i = zero; i < 17 && can_loop; i++) { + barrier_var(i); + buddy_free(&buddy, ptrs[i]); + } + + buddy_destroy(&buddy); + + return 0; +} + +__weak char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c new file mode 100644 index 000000000000..ea1fac95b461 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause + +#include <bpf_atomic.h> + +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/bitmap.h> + +#define TEST_BITMAP_THREADS 2 +#define TEST_BITMAP_BITS (2 * BITS_PER_LONG_LONG) +#define TEST_BITMAP_SYNC_SPINS BPF_MAX_LOOPS +#define TEST_BITMAP_ITERS 10 * 1000 * 1000 + +static struct arena_bitmap __arena *bitmap; +static volatile u64 started; +static volatile bool test_abort; + +/* + * The test needs cmpxchg atomics on arena memory. + */ +#if defined(ENABLE_ATOMICS_TESTS) && \ + (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ + defined(__TARGET_ARCH_s390) || \ + defined(__TARGET_ARCH_powerpc) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) +static bool bitmap_tests_enabled(void) +{ + return true; +} +#else +static bool bitmap_tests_enabled(void) +{ + return false; +} +#endif + +__weak +int bitmap_wait_for_start(void) +{ + u64 i; + + __sync_fetch_and_add(&started, 1); + + for (i = zero; i < TEST_BITMAP_SYNC_SPINS && can_loop; i++) { + if (test_abort) + return -EINTR; + if (smp_load_acquire(&started) >= TEST_BITMAP_THREADS) + return 0; + } + + test_abort = true; + return -ETIMEDOUT; +} + +/* + * The test makes sure writes don't clobber each other by overwriting + * the same word. One thread always writes on even bits, the other on + * odds. Both should be able to operate on the bitmap oblivious of the + * other's operations. + */ +__weak +int bitmap_test_bit_sequence(u32 bit) +{ + if (bmp_test_and_clear_bit(bit, bitmap)) + return -EINVAL; + + if (bmp_test_and_set_bit(bit, bitmap)) + return -EINVAL; + if (!bmp_test_bit(bit, bitmap)) + return -EINVAL; + + if (!bmp_test_and_set_bit(bit, bitmap)) + return -EINVAL; + if (!bmp_test_bit(bit, bitmap)) + return -EINVAL; + + if (!bmp_test_and_clear_bit(bit, bitmap)) + return -EINVAL; + if (bmp_test_bit(bit, bitmap)) + return -EINVAL; + + if (bmp_test_and_clear_bit(bit, bitmap)) + return -EINVAL; + + bmp_set_bit(bit, bitmap); + if (!bmp_test_bit(bit, bitmap)) + return -EINVAL; + + bmp_clear_bit(bit, bitmap); + if (bmp_test_bit(bit, bitmap)) + return -EINVAL; + + bmp_set_bit(bit, bitmap); + if (!bmp_test_bit(bit, bitmap)) + return -EINVAL; + + return 0; + +} + +static void bitmap_test_reset_single(int parity) +{ + u32 bit; + + for (bit = parity; bit < TEST_BITMAP_BITS && can_loop; bit += 2) + bmp_clear_bit(bit, bitmap); + +} + +static int bitmap_test_common_single(int parity) +{ + u32 bit; + int ret; + + for (bit = parity; bit < TEST_BITMAP_BITS && can_loop; bit += 2) { + if (test_abort) + return -EINTR; + + ret = bitmap_test_bit_sequence(bit); + if (ret) { + test_abort = true; + return ret; + } + } + + return 0; +} + +static int bitmap_test_common(int parity) +{ + int ret; + u32 i; + + arena_subprog_init(); + + ret = bitmap_wait_for_start(); + if (ret) + return ret; + + for (i = zero; i < TEST_BITMAP_ITERS && can_loop; i++) { + ret = bitmap_test_common_single(parity); + if (ret) + return ret; + + if (test_abort) + break; + + bitmap_test_reset_single(parity); + } + + return 0; +} + +SEC("syscall") int parallel_test_bitmap__enabled(void) +{ + return bitmap_tests_enabled() ? 0 : -EOPNOTSUPP; +} + +SEC("syscall") int parallel_test_bitmap__init(void) +{ + bitmap = bmp_alloc(TEST_BITMAP_BITS); + if (!bitmap) + return -ENOMEM; + + return 0; +} + +SEC("syscall") int parallel_test_bitmap__fini(void) +{ + int ret = 0; + + if (!bitmap) + return -EINVAL; + + bmp_free(bitmap); + bitmap = NULL; + + return ret; +} + +SEC("syscall") int parallel_test_bitmap__0(void) +{ + return bitmap_test_common(0); +} + +SEC("syscall") int parallel_test_bitmap__1(void) +{ + return bitmap_test_common(1); +} diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_parallel_spmc.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_spmc.bpf.c new file mode 100644 index 000000000000..5fa96eb74095 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_spmc.bpf.c @@ -0,0 +1,668 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause + +#include <bpf_atomic.h> + +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/spmc.h> + +#define TEST_SPMC_THREADS 3 +#define TEST_SPMC_STEALERS (TEST_SPMC_THREADS - 1) + +/* + * The test requires the stealers/owners to sometimes quiesce + * before continuing the benchmark. Normally we'd use something + * like a condition variable, but since the benchmark is short-lived + * and operations are wait-free we just spin around the quiescence + * point instead. If we time out, we just fail the benchmark. + */ +#define TEST_SPMC_SYNC_SPINS BPF_MAX_LOOPS + +/* + * We track all the values we retrieve from the queue + * to get some guarantee we're, not corrupting data, + * e.g., accidentally reusing a past value from a slot. + */ +#define TEST_SPMC_MAX_VALUES (1024) +static u64 __arena seen[TEST_SPMC_MAX_VALUES]; + +/* The single spmc queue for the benchmark. */ +static struct spmc __arena *spmc; + +/* Owner and stealer epochs. We define the , */ +static volatile u64 owner_epoch; +static volatile u64 stealer_epoch; + +/* Map owner epochs to stealer epochs (simply scale by # of stealers). */ +#define STEALER_EPOCH(owner_epoch) ((owner_epoch) * TEST_SPMC_STEALERS) + +/* Global abort switch. If any thread fails, all others exit ASAP. */ +static volatile bool test_abort; + +/* + * Counters useful for ensuring conservation of pushes/pops of unique values + * (we're not stealing/popping more/fewer items than were pushed). + */ +static volatile u64 expected_total; +static volatile u64 total_seen; + +/* Measure how many pops and steals we've made (irrespective of retrieved value). */ +static volatile u64 pops; +static volatile u64 steals; + +/* Used for the resize selftest, see below. */ +static volatile u64 stealers_started; + +/* Used for the mixed selftest, see below. */ +static volatile u64 round_steals; + +/* + * We have multiple stealers and a single owner. We sometimes want the owner + * to successfully outproduce the stealers, we add a busy loop in them. + */ +#define TEST_SPMC_WASTE_ROUNDS (1UL << 12) + +/* + * The spmc data structure depends on the runtime fully + * supporting acquire/release semantics, which is not + * the case for all architectures. + */ +#if defined(ENABLE_ATOMICS_TESTS) && \ + (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) +static bool spmc_tests_enabled(void) +{ + return true; +} +#else +static bool spmc_tests_enabled(void) +{ + return false; +} +#endif + +/* + * Scaffolding for each parallel test. Each test has setup/teardown, + * a single owner thread that owns the queue, and TEST_SPMC_STEALER + * threads that try to steal. + */ +#define DEFINE_PARALLEL_SPMC_TEST(prefix, expected_total) \ + SEC("syscall") int parallel_test_spmc_##prefix##__enabled(void) \ + { \ + return spmc_tests_enabled() ? 0 : -EOPNOTSUPP; \ + } \ + SEC("syscall") int parallel_test_spmc_##prefix##__init(void) \ + { \ + return spmc_common_init(expected_total); \ + } \ + SEC("syscall") int parallel_test_spmc_##prefix##__fini(void) \ + { \ + return spmc_common_fini(); \ + } \ + SEC("syscall") int parallel_test_spmc_##prefix##__0(void) \ + { \ + return spmc_##prefix##_owner(); \ + } \ + SEC("syscall") int parallel_test_spmc_##prefix##__1(void) \ + { \ + return spmc_##prefix##_stealer(); \ + } \ + SEC("syscall") int parallel_test_spmc_##prefix##__2(void) \ + { \ + return spmc_##prefix##_stealer(); \ + } \ + +static int spmc_common_init(u64 total) +{ + u64 i; + + if (total > TEST_SPMC_MAX_VALUES) + return -E2BIG; + + owner_epoch = 0; + stealer_epoch = 0; + test_abort = false; + expected_total = total; + total_seen = 0; + pops = 0; + steals = 0; + stealers_started = 0; + round_steals = 0; + + for (i = zero; i < TEST_SPMC_MAX_VALUES && can_loop; i++) + seen[i] = 0; + + spmc = spmc_create(); + if (!spmc) + return -ENOMEM; + + return 0; +} + +static int spmc_common_fini(void) +{ + int ret; + + ret = spmc_destroy(spmc); + spmc = NULL; + + return ret; +} + +__weak +int spmc_quiesce_on_owner(u64 epoch) +{ + u64 i; + + for (i = zero; i < TEST_SPMC_SYNC_SPINS && can_loop; i++) { + if (test_abort) + return -EINTR; + if (smp_load_acquire(&owner_epoch) >= epoch) + return 0; + } + + test_abort = true; + + return -ETIMEDOUT; +} + +__weak +int spmc_quiesce_on_stealer(u64 epoch) +{ + u64 target, cur; + unsigned int i; + int err = -ETIMEDOUT; + + target = STEALER_EPOCH(epoch); + for (i = zero; i < TEST_SPMC_SYNC_SPINS && can_loop; i++) { + if (test_abort) { + err = -EINTR; + break; + } + + cur = smp_load_acquire(&stealer_epoch); + if (cur > target) { + err = -EINVAL; + test_abort = true; + break; + } + + if (cur == target) + return 0; + } + + test_abort = true; + + return err; +} + +static int spmc_update_stats(u64 val, bool owner) +{ + u64 total; + + total = expected_total; + if (val >= total || val >= TEST_SPMC_MAX_VALUES) { + test_abort = true; + return -EINVAL; + } + + if (__sync_fetch_and_add(&seen[val], 1) != 0) { + test_abort = true; + return -EINVAL; + } + + __sync_fetch_and_add(&total_seen, 1); + if (owner) + __sync_fetch_and_add(&pops, 1); + else + __sync_fetch_and_add(&steals, 1); + + return 0; +} + +static int spmc_validate_owner_empty(void) +{ + u64 val; + int ret; + + ret = spmc_owned_remove(spmc, &val); + if (ret != -ENOENT) { + test_abort = true; + /* Change a 0 return value into -EINVAL. */ + return ret ?: -EINVAL; + } + + return 0; +} + +__weak +int spmc_validate_all_seen(void) +{ + u64 i, total; + + total = expected_total; + if (total_seen != total) + goto err; + + if (pops + steals != total) + goto err; + + for (i = zero; i < total && can_loop; i++) { + if (seen[i % TEST_SPMC_MAX_VALUES] != 1) + goto err; + } + + return 0; + +err: + test_abort = true; + + return -EINVAL; +} + +/* + * Single value benchmark. The owner adds an item then races with + * the stealers for it. This way directly race between owner and + * stealers on the same slot. + */ + + +#define TEST_SPMC_SINGLEVAL_ITERS (64) + +__weak +int spmc_singleval_tryconsume(u64 expected, bool steal) +{ + u64 val; + int ret; + + while (can_loop) { + if (steal) + ret = spmc_steal(spmc, &val); + else + ret = spmc_owned_remove(spmc, &val); + + /* Success. Update and validate. */ + if (!ret) { + if (val != expected) + return -EINVAL; + + ret = spmc_update_stats(val, !steal); + if (ret) + return ret; + + return 0; + } + + /* + * If we got -ENOENT, the queue is empty + * and we're good to go. + */ + if (ret != -EAGAIN) + return (ret == -ENOENT) ? 0 : ret; + } + + /* Impossible. */ + return -EINVAL; +} + +static int spmc_singleval_owner(void) +{ + int ret; + u64 i; + + for (i = zero; i < TEST_SPMC_SINGLEVAL_ITERS && can_loop; i++) { + ret = spmc_quiesce_on_stealer(i); + if (ret) + goto err; + + ret = spmc_owned_add(spmc, i); + if (ret) + goto err; + + __sync_fetch_and_add(&owner_epoch, 1); + + ret = spmc_singleval_tryconsume(i, false); + if (ret) + goto err; + + ret = spmc_quiesce_on_stealer(i + 1); + if (ret) + goto err; + } + + ret = spmc_validate_owner_empty(); + if (ret) + return ret; + + return spmc_validate_all_seen(); + +err: + test_abort = true; + return -EINVAL; +} + +static int spmc_singleval_stealer(void) +{ + int ret; + u64 i; + + for (i = zero; i < TEST_SPMC_SINGLEVAL_ITERS && can_loop; i++) { + ret = spmc_quiesce_on_owner(i + 1); + if (ret) + goto err; + + ret = spmc_singleval_tryconsume(i, true); + if (ret) + goto err; + + __sync_fetch_and_add(&stealer_epoch, 1); + } + + return 0; + +err: + test_abort = true; + return -EINVAL; +} + +DEFINE_PARALLEL_SPMC_TEST(singleval, TEST_SPMC_SINGLEVAL_ITERS) + +/* + * The resize test. Force a resize from the owner even while the stealers + * are trying to consume. Then make sure the queue is still consistent + * after the resize. + * + * The owner _doesn't_ consume from the queue. The test makes sure that + * switching the array from underneath the stealers works. + */ + +/* Force 2 resizes (since the rate of resize is logarithmic). */ +#define TEST_SPMC_RESIZE_ORDER (2) +#define TEST_SPMC_RESIZE_PREFILL ((SPMC_ARR_BASESZ << TEST_SPMC_RESIZE_ORDER) - 1) + +/* */ +#define TEST_SPMC_RESIZE_TAIL (SPMC_ARR_BASESZ << TEST_SPMC_RESIZE_ORDER) +#define TEST_SPMC_RESIZE_TOTAL (TEST_SPMC_RESIZE_PREFILL + TEST_SPMC_RESIZE_TAIL) + +__weak +int spmc_wait_for_stealers_to_start(u64 target) +{ + u64 i; + + for (i = zero; i < TEST_SPMC_SYNC_SPINS && can_loop; i++) { + if (test_abort) + return -EINTR; + if (READ_ONCE(stealers_started) >= target) + return 0; + } + + test_abort = true; + + return -ETIMEDOUT; +} + +__weak +void spmc_waste_time(void) +{ + int i; + int j; + + for (i = zero; i < TEST_SPMC_WASTE_ROUNDS && can_loop; i++) { + /* Random computation. */ + WRITE_ONCE(j, i * 17 + 23); + } +} + +static int spmc_resize_owner(void) +{ + bool resized = false; + u64 i; + int ret; + + /* Get a head start vs the consumers. */ + for (i = zero; i < TEST_SPMC_RESIZE_PREFILL && can_loop; i++) { + ret = spmc_owned_add(spmc, i); + if (ret) { + test_abort = true; + return ret; + } + } + + __sync_fetch_and_add(&owner_epoch, 1); + + /* Wait for stealers to start then start racing. */ + ret = spmc_wait_for_stealers_to_start(TEST_SPMC_STEALERS); + if (ret) + return ret; + + for (i = TEST_SPMC_RESIZE_PREFILL; i < TEST_SPMC_RESIZE_TOTAL && can_loop; i++) { + ret = spmc_owned_add(spmc, i); + if (ret) { + test_abort = true; + return ret; + } + + if (spmc->cur->order > TEST_SPMC_RESIZE_ORDER) + resized = true; + } + + /* Did we get to resize while racing? */ + if (!resized) { + test_abort = true; + return -EINVAL; + } + + /* + * Wait for the stealers to drain and make sure + * we didn't lose any items along the way. + */ + __sync_fetch_and_add(&owner_epoch, 1); + + ret = spmc_quiesce_on_stealer(1); + if (ret) + return ret; + + ret = spmc_validate_owner_empty(); + if (ret) + return ret; + + return spmc_validate_all_seen(); +} + +static int spmc_resize_stealer(void) +{ + bool owner_done = false; + u64 val; + int ret; + + arena_subprog_init(); + + ret = spmc_quiesce_on_owner(1); + if (ret) + return ret; + + __sync_fetch_and_add(&stealers_started, 1); + + while (can_loop) { + spmc_waste_time(); + if (test_abort) + return -EINTR; + + ret = spmc_steal(spmc, &val); + if (!ret) { + ret = spmc_update_stats(val, false); + if (ret) + return ret; + continue; + } + + if (ret == -EAGAIN) + continue; + + if (ret == -ENOENT) { + if (owner_done) + break; + owner_done = owner_epoch >= 2; + continue; + } + + test_abort = true; + return ret; + } + + __sync_fetch_and_add(&stealer_epoch, 1); + + return 0; +} + +DEFINE_PARALLEL_SPMC_TEST(resize, TEST_SPMC_RESIZE_TOTAL) + +/* + * The burst benchmark. The owner generates data all at once, + * then waits for the stealers to steal half then starts removing + * items until the queue empties. The owner also makes sure the + * item order is not jumbled. + */ + +#define TEST_SPMC_BURST_ROUNDS (4) +#define TEST_SPMC_BURST_BURST (64) +#define TEST_SPMC_BURST_TOTAL (TEST_SPMC_BURST_ROUNDS * TEST_SPMC_BURST_BURST) +#define TEST_SPMC_BURST_STEAL_TARGET (TEST_SPMC_BURST_BURST / 2) + +static int spmc_wait_for_round_steals(u64 target) +{ + u64 i; + + arena_subprog_init(); + + for (i = zero; i < TEST_SPMC_SYNC_SPINS && can_loop; i++) { + if (test_abort) + return -EINTR; + if (round_steals >= target) + return 0; + } + + test_abort = true; + + return -ETIMEDOUT; +} + +__weak int +spmc_burst_owner_round(u64 round) +{ + u64 i, base, stolen, expected, val; + int ret; + + base = round * TEST_SPMC_BURST_BURST; + round_steals = 0; + + for (i = zero; i < TEST_SPMC_BURST_BURST && can_loop; i++) { + ret = spmc_owned_add(spmc, base + i); + if (ret) + return ret; + } + + __sync_fetch_and_add(&owner_epoch, 1); + + ret = spmc_wait_for_round_steals(TEST_SPMC_BURST_STEAL_TARGET); + if (ret == -EINTR || ret == -ETIMEDOUT) + return ret; + + __sync_fetch_and_add(&owner_epoch, 1); + + ret = spmc_quiesce_on_stealer(round + 1); + if (ret) + return ret; + + stolen = round_steals; + if (stolen > TEST_SPMC_BURST_BURST) + return -EINVAL; + + for (i = zero; i < TEST_SPMC_BURST_BURST - stolen && can_loop; i++) { + ret = spmc_owned_remove(spmc, &val); + if (ret) + return ret; + + expected = base + TEST_SPMC_BURST_BURST - 1 - i; + if (val != expected) + return -EINVAL; + + ret = spmc_update_stats(val, true); + if (ret) { + test_abort = true; + return -EINVAL; + } + } + + ret = spmc_validate_owner_empty(); + if (ret) + return ret; + + return 0; +} + +static int spmc_burst_owner(void) +{ + u64 round; + int ret; + + arena_subprog_init(); + + for (round = zero; round < TEST_SPMC_BURST_ROUNDS && can_loop; round++) { + ret = spmc_burst_owner_round(round); + if (ret) + goto err; + } + + return spmc_validate_all_seen(); + +err: + test_abort = true; + return -EINVAL; +} + +static int spmc_burst_stealer(void) +{ + u64 round, val, active_epoch; + int ret; + + arena_subprog_init(); + + for (round = zero; round < TEST_SPMC_BURST_ROUNDS && can_loop; round++) { + active_epoch = round * 2 + 1; + + /* + * Wait till the owner prefills the queue then + * start stealing. + */ + ret = spmc_quiesce_on_owner(active_epoch); + if (ret) + return ret; + + while (owner_epoch == active_epoch && can_loop) { + if (test_abort) + return -EINTR; + + ret = spmc_steal(spmc, &val); + if (!ret) { + ret = spmc_update_stats(val, false); + if (ret) + return ret; + __sync_fetch_and_add(&round_steals, 1); + continue; + } + if (ret == -EAGAIN || ret == -ENOENT) + continue; + + test_abort = true; + return ret; + } + + __sync_fetch_and_add(&stealer_epoch, 1); + } + + return 0; +} + +DEFINE_PARALLEL_SPMC_TEST(burst, TEST_SPMC_BURST_TOTAL) diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_progs_compat.h b/tools/testing/selftests/bpf/libarena/selftests/test_progs_compat.h new file mode 100644 index 000000000000..9d431376c42f --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/selftests/test_progs_compat.h @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#pragma once + +#ifdef __BPF__ + +/* Selftests use these tags for compatibility with test_progs. */ +#define __test_tag(tag) __attribute__((btf_decl_tag("comment:" XSTR(__COUNTER__) ":" tag))) +#define __stderr(msg) __test_tag("test_expect_stderr=" msg) +#define __stderr_unpriv(msg) __test_tag("test_expect_stderr_unpriv=" msg) + +#define XSTR(s) STR(s) +#define STR(s) #s + +#endif diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_rbtree.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_rbtree.bpf.c new file mode 100644 index 000000000000..856c484a009a --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/selftests/test_rbtree.bpf.c @@ -0,0 +1,968 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause + +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/rbtree.h> + +typedef struct node_ctx __arena *node_ctx; + +struct node_ctx { + struct rbnode rbnode; + node_ctx next; +}; + +static const u64 keys[] = { 51, 43, 37, 3, 301, 46, 383, 990, 776, 729, 871, 96, 189, 213, + 376, 167, 131, 939, 626, 119, 374, 700, 772, 154, 883, 620, 641, 5, + 428, 516, 105, 622, 988, 811, 931, 973, 246, 690, 934, 744, 210, 311, + 32, 255, 960, 830, 523, 429, 541, 738, 705, 774, 715, 446, 98, 578, + 777, 191, 279, 91, 767 }; + +static const u64 morekeys[] = { 173, 636, 1201, 8642, 5957, 3617, 4586, 8053, 6551, 7592, 1748, 1589, 8644, 9918, 6977, + 4448, 5852, 4640, 9717, 2303, 7424, 7695, 2334, 8876, 8618, 5745, 7134, 2178, 5280, 2140, 1138, + 5083, 8922, 1516, 2437, 2488, 4307, 4329, 5088, 8456, 5938, 1441, 1684, 5750, 721, 1107, 2089, + 9737, 4687, 5016, 4849, 8193, 9603, 9147, 5992, 166, 6721, 812, 4144, 6237, 6509, 3466, 9255, + 7767, 3960, 6759, 2968, 6046, 9784, 8395, 2619, 1711, 528, 6424, 9084, 3179, 1342, 5676, 9445, + 5691, 6678, 8487, 1627, 998, 6178, 2229, 1987, 3319, 572, 169, 2161, 3018, 5439, 7287, 7265, 5995, + 5003, 5857, 2836, 5634, 4735, 9261, 8287, 5359, 533, 1406, 9573, 4026, 714, 3956, 1722, 6395, + 9648, 3887, 7185, 470, 4482, 4997, 841, 8913, 9946, 3999, 9357, 9847, 277, 8184, 8704, 6766, 3323, + 5468, 8638, 7905, 8858, 6142, 3685, 3452, 4689, 8878, 8836, 158, 831, 7914, 3031, 8374, 4921, + 4207, 3460, 5547, 3358, 1083, 4619, 7818, 2962, 4879, 4583, 2172, 8819, 9830, 1194, 2666, 9812, + 5704, 8432, 5916, 6007, 6609, 4791, 1985, 3226, 2478, 9605, 5236, 8079, 3042, 1965, 3539, 9704, + 4267, 6416, 760, 9968, 2983, 1190, 1964, 3211, 2870, 3106, 2794, 1542, 6916, 5986, 9096, 441, + 5894, 8353, 7765, 3757, 5732, 88, 3091, 5637, 6042, 8447, 4073, 6923, 5491, 7010, 3663, 5029, + 6162, 822, 4874, 7491, 5100, 3461, 6983, 2170, 1458, 1856, 648, 6272, 4887, 976, 2369, 5909, 4274, + 3324, 6968, 2312, 2271, 8891, 6268, 6581, 1610, 8880, 6194, 6144, 9764, 6915, 829, 3774, 2265, + 1752, 1314, 6377, 8760, 8004, 501, 4912, 9278, 1425, 9578, 7337, 307, 1885, 3151, 9617, 1647, + 2458, 3702, 6091, 8902, 5663, 9378, 7640, 3336, 557, 1644, 6848, 1559, 8821, 266, 4330, 9790, + 5920, 4222, 1143, 6248, 5792, 4847, 9726, 6303, 821, 6839, 6062, 7133, 3649, 9888, 2528, 1966, + 5456, 4914, 3615, 1543, 3206, 3353, 6097, 2800, 1424, 9094, 7920, 7243, 1394, 5464, 1707, 576, + 6524, 4261, 4187, 7889, 5336, 3377, 2921, 7244, 2766, 6584, 5514, 1387, 2957, 2258, 1077, 9979, + 1128, 876, 4056, 4668, 4532, 1982, 7093, 4184, 5460, 7588, 4704, 6717, 61, 3959, 1826, 2294, 18, + 8170, 9394, 8796, 7288, 7285, 7143, 148, 6676, 6603, 1051, 8225, 4169, 3230, 7697, 6971, 3454, + 7501, 9514, 394, 2339, 4993, 5606, 6060, 1297, 8273, 3012, 157, 8181, 6765, 7207, 1005, 8833, 1914, + 7456, 1846, 8375, 2741, 2074, 1712, 5286 }; + +SEC("syscall") +__weak int test_rbtree_find_nonexistent(void) +{ + u64 key = 0xdeadbeef; + u64 value = 0; + int ret; + + struct rbtree __arena *rbtree; + + rbtree = rb_create(RB_ALLOC, RB_DEFAULT); + if (!rbtree) + return 1; + + /* Should return -EINVAL */ + ret = rb_find(rbtree, key, &value); + if (!ret) + return 2; + + return rb_destroy(rbtree); +} + +SEC("syscall") +__weak int test_rbtree_insert_existing(void) +{ + u64 key = 525252; + u64 value = 24; + int ret; + + struct rbtree __arena *rbtree; + + rbtree = rb_create(RB_ALLOC, RB_DEFAULT); + if (!rbtree) + return 1; + + ret = rb_insert(rbtree, key, value); + if (ret) + return 2; + + /* Should return -EALREADY. */ + ret = rb_insert(rbtree, key, value); + if (ret != -EALREADY) { + return 3; + } + + return rb_destroy(rbtree); +} + +SEC("syscall") +__weak int test_rbtree_update_existing(void) +{ + u64 key = 33333; + u64 value; + int ret; + + struct rbtree __arena *rbtree; + + rbtree = rb_create(RB_ALLOC, RB_UPDATE); + if (!rbtree) + return 1; + + value = 52; + ret = rb_insert(rbtree, key, value); + if (ret) + return 2; + + ret = rb_find(rbtree, key, &value); + if (ret) + return 3; + + if (value != 52) + return 4; + + value = 65; + + /* Should succeed. */ + ret = rb_insert(rbtree, key, value); + if (ret) + return 5; + + /* Should be updated. */ + ret = rb_find(rbtree, key, &value); + if (ret) + return 6; + + if (value != 65) + return 7; + + return rb_destroy(rbtree); +} + +SEC("syscall") +__weak int test_rbtree_insert_one(void) +{ + u64 key = 202020; + u64 value = 0xbadcafe; + int ret; + + struct rbtree __arena *rbtree; + + rbtree = rb_create(RB_ALLOC, RB_UPDATE); + if (!rbtree) + return 1; + + ret = rb_insert(rbtree, key, value); + if (ret) + return 2; + + ret = rb_find(rbtree, key, &value); + if (ret) + return 3; + + if (value != 0xbadcafe) + return 4; + + return rb_destroy(rbtree); +} + +SEC("syscall") +__weak int test_rbtree_insert_ten(void) +{ + u64 key, value; + int ret, i; + + struct rbtree __arena *rbtree; + + rbtree = rb_create(RB_ALLOC, RB_UPDATE); + if (!rbtree) + return 1; + + for (i = 0; i < 10 && can_loop; i++) { + key = keys[i]; + ret = rb_insert(rbtree, key, 2 * key); + if (ret) + return 2 + 3 * i; + + /* Read it back. */ + ret = rb_find(rbtree, key, &value); + if (ret) + return 2 + 3 * i + 1; + + if (value != 2 * key) + return 2 + 3 * i + 2; + } + + /* Go find all inserted pairs. */ + for (i = 0; i < 10 && can_loop; i++) { + key = keys[i]; + + ret = rb_find(rbtree, key, &value); + if (ret) + return 35 + 2 * i; + + if (value != 2 * key) + return 35 + 2 * i + 1; + } + + return rb_destroy(rbtree); +} + +SEC("syscall") +__weak int test_rbtree_duplicate(void) +{ + u64 key = 0x121212; + u64 value; + int ret, i; + + struct rbtree __arena *rbtree; + + rbtree = rb_create(RB_ALLOC, RB_DUPLICATE); + if (!rbtree) + return 1; + + for (i = 0; i < 10 && can_loop; i++) { + ret = rb_insert(rbtree, key, 2 * key); + if (ret) + return 2 + 3 * i; + + /* Read it back. */ + ret = rb_find(rbtree, key, &value); + if (ret) + return 2 + 3 * i + 1; + + if (value != 2 * key) + return 2 + 3 * i + 2; + } + + /* Go find all inserted copies and remove them. */ + for (i = 0; i < 10 && can_loop; i++) { + ret = rb_find(rbtree, key, &value); + if (ret) { + rb_print(rbtree); + return 35 + 3 * i; + } + + if (value != 2 * key) + return 35 + 3 * i + 1; + + ret = rb_remove(rbtree, key); + if (ret) + return 35 + 3 * i + 2; + } + + return rb_destroy(rbtree); +} + +static inline int +clean_up_noalloc_tree(struct rbtree __arena *rbtree) +{ + node_ctx nodec; + int ret; + + if (rbtree->alloc != RB_NOALLOC) + return -EINVAL; + + /* Can't destroy an RB_NOALLOC tree that still has nodes. */ + if (rb_destroy(rbtree) != -EBUSY) + return -EINVAL; + + while (rbtree->root && can_loop) { + nodec = (node_ctx)arena_container_of(rbtree->root, struct node_ctx, rbnode); + ret = rb_remove_node(rbtree, &nodec->rbnode); + if (ret) + return ret; + + arena_free(nodec); + } + + return 0; +} + +int insert_many(enum rbtree_alloc alloc, enum rbtree_insert_mode insert) +{ + const size_t numkeys = sizeof(keys) / sizeof(keys[0]); + node_ctx nodec; + u64 key, value; + int ret; + int i; + + struct rbtree __arena *rbtree; + + rbtree = rb_create(alloc, insert); + if (!rbtree) + return 1; + + for (i = 0; i < numkeys && can_loop; i++) { + key = keys[i]; + if (rbtree->alloc != RB_ALLOC) { + nodec = arena_malloc(sizeof(*nodec)); + if (!nodec) { + arena_stderr("out of memory\n"); + return -ENOMEM; + } + nodec->rbnode.key = key; + nodec->rbnode.value = 2 * key; + ret = rb_insert_node(rbtree, &nodec->rbnode); + } else { + ret = rb_insert(rbtree, key, 2 * key); + } + if (ret) + return 2 + 3 * i; + + /* Read it back. */ + ret = rb_find(rbtree, key, &value); + if (ret) + return 2 + 3 * i + 1; + + if (value != 2 * key) + return 2 + 3 * i + 2; + } + + /* Go find all inserted pairs. */ + for (i = 0; i < numkeys && can_loop; i++) { + key = keys[i]; + + ret = rb_find(rbtree, key, &value); + if (ret) + return 302 + 2 * i; + + if (value != 2 * key) + return 302 + 2 * i + 1; + } + + /* RB_ALLOC trees are destroyed while still having elements. */ + if (rbtree->alloc == RB_ALLOC) + return rb_destroy(rbtree); + + /* Otherwise manually clean up the tree. */ + if (clean_up_noalloc_tree(rbtree)) + return 5; + + return rb_destroy(rbtree); +} + +SEC("syscall") +__weak int test_rbtree_remove_one(void) +{ + u64 key = 20, value = 5, newvalue; + int ret; + + struct rbtree __arena *rbtree; + + rbtree = rb_create(RB_ALLOC, RB_DEFAULT); + if (!rbtree) + return 1; + + ret = rb_find(rbtree, key, &newvalue); + if (!ret) + return 2; + + ret = rb_insert(rbtree, key, value); + if (ret) + return 3; + + ret = rb_find(rbtree, key, &newvalue); + if (ret || value != newvalue) + return 4; + + ret = rb_remove(rbtree, key); + if (ret) + return 5; + + ret = rb_find(rbtree, key, &newvalue); + if (!ret) + return 6; + + return rb_destroy(rbtree); +} + +static __always_inline int remove_many_verify_all_present(struct rbtree __arena *rbtree) +{ + const size_t numkeys = sizeof(morekeys) / sizeof(morekeys[0]); + u64 value; + int ret; + int i; + + for (i = 0; i < numkeys && can_loop; i++) { + u64 key = morekeys[i]; + + ret = rb_find(rbtree, key, &value); + if (ret) + return -1; + + if (value != 2 * key) + return -1; + } + + return 0; +} + +static __always_inline int remove_many_verify_remaining(struct rbtree __arena *rbtree) +{ + const size_t numkeys = sizeof(morekeys) / sizeof(morekeys[0]); + u64 value; + int ret; + int i; + + for (i = 0; i < numkeys && can_loop; i += 2) { + u64 key = morekeys[i]; + + ret = rb_find(rbtree, key, &value); + if (!ret) + return -1; + + if (i + 1 >= numkeys) + break; + + key = morekeys[i + 1]; + ret = rb_find(rbtree, key, &value); + if (ret) + return -1; + + if (value != 2 * key) + return -1; + } + + for (i = 1; i < numkeys && can_loop; i += 2) { + u64 key = morekeys[i]; + + ret = rb_find(rbtree, key, &value); + if (ret) + return -1; + + if (value != 2 * key) + return -1; + } + + return 0; +} + +static __noinline int remove_many_alloc(struct rbtree __arena *rbtree) +{ + const size_t numkeys = sizeof(morekeys) / sizeof(morekeys[0]); + u64 value; + int ret; + int i; + + for (i = 0; i < numkeys && can_loop; i++) { + u64 key = morekeys[i]; + + ret = rb_insert(rbtree, key, 2 * key); + if (ret) + return -1; + + if (rb_integrity_check(rbtree)) { + arena_stderr("iteration %d\n", i); + return -EINVAL; + } + + ret = rb_find(rbtree, key, &value); + if (ret) + return -1; + + if (value != 2 * key) + return -1; + } + + ret = remove_many_verify_all_present(rbtree); + if (ret) + return ret; + + for (i = 0; i < numkeys && can_loop; i += 2) { + u64 key = morekeys[i]; + + ret = rb_remove(rbtree, key); + if (ret) { + arena_stderr("Failed to remove %ld\n", key); + return -1; + } + + ret = rb_find(rbtree, key, &value); + if (!ret) + return -1; + } + + return remove_many_verify_remaining(rbtree); +} + +static __noinline int remove_many_noalloc(struct rbtree __arena *rbtree) +{ + const size_t numkeys = sizeof(morekeys) / sizeof(morekeys[0]); + node_ctx first = NULL, last = NULL; + u64 value; + int ret; + int i; + + for (i = 0; i < numkeys && can_loop; i++) { + u64 key = morekeys[i]; + node_ctx nodec = arena_malloc(sizeof(*nodec)); + + if (!nodec) { + arena_stderr("out of memory\n"); + return -ENOMEM; + } + nodec->rbnode.key = key; + nodec->rbnode.value = 2 * key; + nodec->next = NULL; + + if (!first) + first = nodec; + + if (last) + last->next = nodec; + last = nodec; + + ret = rb_insert_node(rbtree, &nodec->rbnode); + if (ret) + return -1; + + if (rb_integrity_check(rbtree)) { + arena_stderr("iteration %d\n", i); + return -EINVAL; + } + + ret = rb_find(rbtree, key, &value); + if (ret) + return -1; + + if (value != 2 * key) + return -1; + } + + ret = remove_many_verify_all_present(rbtree); + if (ret) + return ret; + + for (i = 0; i < numkeys && can_loop; i += 2) { + u64 key = morekeys[i]; + node_ctx nodec = first; + + if (!nodec || key != nodec->rbnode.key) + return -1; + + first = nodec->next ? nodec->next->next : NULL; + ret = rb_remove_node(rbtree, &nodec->rbnode); + if (ret) { + arena_stderr("Failed to remove %ld\n", key); + return -1; + } + + ret = rb_find(rbtree, key, &value); + if (!ret) + return -1; + } + + return remove_many_verify_remaining(rbtree); +} + +static inline int remove_many(enum rbtree_alloc alloc, + enum rbtree_insert_mode insert) +{ + int ret; + struct rbtree __arena *rbtree; + + rbtree = rb_create(alloc, insert); + if (!rbtree) + return -ENOMEM; + + ret = (alloc == RB_ALLOC) ? remove_many_alloc(rbtree) + : remove_many_noalloc(rbtree); + if (ret) + return ret; + + if (alloc == RB_ALLOC) + return rb_destroy(rbtree); + + ret = clean_up_noalloc_tree(rbtree); + if (ret) + return ret; + + return rb_destroy(rbtree); +} + +SEC("syscall") +__weak int test_rbtree_insert_many_update(void) +{ + return insert_many(RB_ALLOC, RB_UPDATE); +} + +SEC("syscall") +__weak int test_rbtree_insert_many_noalloc(void) +{ + return insert_many(RB_NOALLOC, RB_DUPLICATE); +} + +SEC("syscall") +__weak int test_rbtree_remove_many_update(void) +{ + return remove_many(RB_ALLOC, RB_UPDATE); +} + +SEC("syscall") +__weak int test_rbtree_remove_many_noalloc(void) +{ + return remove_many(RB_NOALLOC, RB_DUPLICATE); +} + +SEC("syscall") +__weak int test_rbtree_add_remove_circular(void) +{ + const size_t iters = 60; + const size_t prefill = 10; + const size_t numkeys = 50; + const size_t prefix = 400000; + u64 value, rmval; + int errval = 1; + u64 key; + int ret; + int i; + + struct rbtree __arena *rbtree; + + rbtree = rb_create(RB_ALLOC, RB_UPDATE); + if (!rbtree) + return 1; + + for (i = 0; i < prefill && can_loop; i++) { + ret = rb_insert(rbtree, prefix + (i % numkeys), i); + if (ret) + return errval; + + errval += 1; + } + + errval = 2 * 1000 * 1000; + + for (i = 0; i < prefill && can_loop; i++) { + /* Read it back. */ + ret = rb_find(rbtree, prefix + (i % numkeys), &value); + if (ret) + return errval; + + if (value != i) + return errval; + } + + errval = 3 * 1000 * 1000; + + for (i = prefill; i < iters && can_loop; i++) { + key = prefix + (i % numkeys); + + ret = rb_find(rbtree, key, &value); + if (!ret) { + arena_stderr("Key %d already present\n", key); + return errval; + } + + errval += 1; + + ret = rb_insert(rbtree, key, i); + if (ret) { + arena_stderr("ITERATION %d\n", i); + rb_print(rbtree); + return errval; + } + + rmval = i - prefill; + + errval += 1; + + ret = rb_find(rbtree, prefix + (rmval % numkeys), &value); + if (ret) + return errval; + + errval += 1; + + if (value != rmval) + return errval; + + errval += 1; + + ret = rb_remove(rbtree, prefix + (rmval % numkeys)); + if (ret) { + arena_stderr("ITERATION %d\n", i); + return errval; + } + + errval += 1; + } + + for (i = 0; i < numkeys && can_loop; i++) { + rb_remove(rbtree, prefix + i); + } + + return rb_destroy(rbtree); +} + +SEC("syscall") +__weak int test_rbtree_add_remove_circular_reverse(void) +{ + const size_t iters = 110; + const size_t prefill = 10; + const size_t numkeys = 50; + const size_t prefix = 500000; + u64 value, rmval; + int errval = 1; + u64 key; + int ret; + int i; + + struct rbtree __arena *rbtree; + + rbtree = rb_create(RB_ALLOC, RB_UPDATE); + if (!rbtree) + return 1; + + for (i = 0; i < prefill && can_loop; i++) { + ret = rb_insert(rbtree, prefix - (i % numkeys), i); + if (ret) + return errval; + + errval += 1; + } + + errval = 2 * 1000 * 1000; + + for (i = 0; i < prefill && can_loop; i++) { + /* Read it back. */ + ret = rb_find(rbtree, prefix - (i % numkeys), &value); + if (ret) + return errval; + + if (value != i) + return errval; + } + + errval = 3 * 1000 * 1000; + + for (i = prefill; i < iters && can_loop; i++) { + key = prefix - (i % numkeys); + + ret = rb_find(rbtree, key, &value); + if (!ret) { + arena_stderr("Key %d already present\n", key); + return errval; + } + + errval += 1; + + ret = rb_insert(rbtree, key, i); + if (ret) { + arena_stderr("error %d on insert\n", ret); + rb_print(rbtree); + return errval; + } + + rmval = i - prefill; + + errval += 1; + + ret = rb_find(rbtree, prefix - (rmval % numkeys), &value); + if (ret) + return errval; + + errval += 1; + + if (value != rmval) + return errval; + + errval += 1; + + ret = rb_remove(rbtree, prefix - (rmval % numkeys)); + if (ret) + return errval; + + errval += 1; + } + + + errval = 4 * 1000 * 1000; + for (i = 0; i < prefill && can_loop; i++) { + ret = rb_remove(rbtree, prefix - i); + if (ret) { + arena_stderr("Did not remove %d, error %d\n", prefix - i, ret); + return errval + i; + } + } + + return rb_destroy(rbtree); +} + +SEC("syscall") +__weak int test_rbtree_least_pop(void) +{ + const size_t keys = 10; + u64 key, value; + int errval = 1; + int ret, i; + + struct rbtree __arena *rbtree; + + rbtree = rb_create(RB_ALLOC, RB_DEFAULT); + if (!rbtree) + return errval; + + errval += 1; + + for (i = 0; i < keys / 2 && can_loop; i++) { + ret = rb_insert(rbtree, i, i); + if (ret) + return errval; + + errval += 1; + + ret = rb_insert(rbtree, keys - 1 - i, keys - 1 - i); + if (ret) + return errval; + + errval += 1; + + ret = rb_least(rbtree, &key, &value); + if (ret) + return errval; + + errval += 1; + + if (key != 0 || value != 0) + return errval; + + errval += 1; + } + + errval = 1000; + + for (i = 0; i < keys && can_loop; i++) { + ret = rb_least(rbtree, &key, &value); + if (ret) { + arena_stderr("rb_least failed with %d\n", ret); + return errval; + } + + errval += 1; + + if (key != i || value != i) { + arena_stderr("Got KV %ld/%ld expected %d\n", key, value, i); + return errval; + } + + errval += 1; + + ret = rb_pop(rbtree, &key, &value); + if (ret) { + arena_stderr("Error %d during pop on iter %d\n", ret, i); + return errval; + } + + errval += 1; + + if (key != i || value != i) + return errval; + } + + return rb_destroy(rbtree); +} + +/* Reject rb_pop() for RB_NOALLOC trees. */ +SEC("syscall") +__weak int test_rbtree_noalloc_pop(void) +{ + const u64 expect_value = 1; + const u64 expect_key = 0; + struct rbtree __arena *rbtree; + struct rbnode __arena *node; + u64 value = 0; + int ret; + + rbtree = rb_create(RB_NOALLOC, RB_DEFAULT); + if (!rbtree) + return 1; + + node = rb_node_alloc(expect_key, expect_value); + if (!node) { + rb_destroy(rbtree); + return 2; + } + + ret = rb_insert_node(rbtree, node); + if (ret) { + rb_node_free(node); + rb_destroy(rbtree); + return 3; + } + + ret = rb_pop(rbtree, NULL, &value); + if (ret != -EINVAL) + return 4; + + ret = rb_find(rbtree, expect_key, &value); + if (ret) + return 5; + + if (value != expect_value) + return 6; + + ret = rb_remove_node(rbtree, node); + if (ret) + return 7; + + rb_node_free(node); + + return rb_destroy(rbtree); +} + +SEC("syscall") +__weak int test_rbtree_alloc_check(void) +{ + struct rbtree __arena *alloc, *noalloc; + struct rbnode __arena *node; + int ret; + + alloc = rb_create(RB_ALLOC, RB_DEFAULT); + if (!alloc) + return 1; + + noalloc = rb_create(RB_NOALLOC, RB_DEFAULT); + if (!noalloc) + return 2; + + + node = rb_node_alloc(0, 0); + if (!node) + return 3; + + /* + * RB_ALLOC trees can use rb_insert, RB_NOALLOC trees can + * use rb_insert_node. RB_ALLOC and RB_NOALLOC trees cannot + * use each other's APIs. + * + * NOTE: This begs the question, why not different types? We + * want to partially share the API and that would require us + * to duplicate it. + */ + if (rb_insert(alloc, 0, 0)) + return 4; + + if (!rb_insert_node(alloc, node)) + return 5; + + if (!rb_remove_node(alloc, node)) + return 6; + + if (rb_remove(alloc, 0)) + return 7; + + if (rb_insert_node(noalloc, node)) + return 8; + + if (!rb_insert(noalloc, 0, 0)) + return 9; + + if (!rb_remove(noalloc, 0)) + return 10; + + if (rb_remove_node(noalloc, node)) + return 11; + + rb_node_free(node); + + ret = rb_destroy(alloc); + if (ret) + return ret; + + return rb_destroy(noalloc); +} diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_spmc.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_spmc.bpf.c new file mode 100644 index 000000000000..4d7a520115d1 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/selftests/test_spmc.bpf.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause + +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/spmc.h> + +/* + * NOTE: These selftests only test for the single-threaded use case, which for + * Lev-Chase queues is obviously the simplest one. Still, it is important to + * exercise the API to ensure it passes verification and basic checks. + */ + +SEC("syscall") +int test_spmc_remove_empty(void) +{ + u64 val; + int ret; + + struct spmc __arena *spmc = spmc_create(); + + if (!spmc) + return 1; + + ret = spmc_owned_remove(spmc, &val); + if (ret != -ENOENT) + return 1; + + spmc_destroy(spmc); + + return 0; +} + +SEC("syscall") +int test_spmc_steal_empty(void) +{ + u64 val; + int ret; + + struct spmc __arena *spmc = spmc_create(); + + if (!spmc) + return 1; + + ret = spmc_steal(spmc, &val); + if (ret != -ENOENT) + return 1; + + spmc_destroy(spmc); + + return 0; +} + +SEC("syscall") +int test_spmc_steal_one(void) +{ + u64 val, newval; + int ret, i; + + struct spmc __arena *spmc = spmc_create(); + + if (!spmc) + return 1; + + for (i = 0; i < 10 && can_loop; i++) { + val = i; + + ret = spmc_owned_add(spmc, val); + if (ret) + return 1; + + ret = spmc_steal(spmc, &newval); + if (ret) + return 2; + + if (val != newval) + return 3; + } + + spmc_destroy(spmc); + + return 0; +} + +SEC("syscall") +int test_spmc_remove_one(void) +{ + u64 val, newval; + int ret, i; + + struct spmc __arena *spmc = spmc_create(); + + if (!spmc) + return 1; + + for (i = 0; i < 10 && can_loop; i++) { + val = i; + + ret = spmc_owned_add(spmc, val); + if (ret) + return 1; + + ret = spmc_owned_remove(spmc, &newval); + if (ret) + return 2; + + if (val != newval) + return 3; + } + + spmc_destroy(spmc); + + return 0; +} + +SEC("syscall") +int test_spmc_remove_many(void) +{ + u64 val, newval; + int ret, i; + u64 expected; + + struct spmc __arena *spmc = spmc_create(); + + if (!spmc) + return 1; + + for (i = 0; i < 500 && can_loop; i++) { + val = i; + + ret = spmc_owned_add(spmc, val); + if (ret) { + arena_stderr("%s:%d error %d\n", __func__, __LINE__, ret); + return 1; + } + } + + for (i = 0; i < 500 && can_loop; i++) { + ret = spmc_owned_remove(spmc, &newval); + if (ret) { + arena_stderr("%s:%d error %d\n", __func__, __LINE__, ret); + return 1; + } + + expected = 500 - 1 - i; + if (newval != expected) { + arena_stderr("%s:%d expected %llu found %llu\n", __func__, __LINE__, expected, newval); + return 1; + } + } + + spmc_destroy(spmc); + + return 0; +} + +SEC("syscall") +int test_spmc_steal_many(void) +{ + u64 val, newval; + int ret, i; + + struct spmc __arena *spmc = spmc_create(); + + if (!spmc) + return 1; + + for (i = 0; i < 500 && can_loop; i++) { + val = i; + + ret = spmc_owned_add(spmc, val); + if (ret) { + arena_stderr("%s:%d error %d\n", __func__, __LINE__, ret); + return 1; + } + } + + for (i = 0; i < 500 && can_loop; i++) { + ret = spmc_steal(spmc, &newval); + if (ret) { + arena_stderr("%s:%d error %d\n", __func__, __LINE__, ret); + return 1; + } + + if (newval != i) { + arena_stderr("%s:%d expected %d found %llu\n", __func__, __LINE__, i, newval); + return 1; + } + } + + spmc_destroy(spmc); + + return 0; +} diff --git a/tools/testing/selftests/bpf/libarena/src/asan.bpf.c b/tools/testing/selftests/bpf/libarena/src/asan.bpf.c new file mode 100644 index 000000000000..5135d5c72a46 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/src/asan.bpf.c @@ -0,0 +1,553 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <libarena/common.h> +#include <libarena/asan.h> + + +enum { + /* + * Is the access checked by check_region_inline + * a read or a write? + */ + ASAN_READ = 0x0U, + ASAN_WRITE = 0x1U, +}; + +/* + * Address sanitizer (ASAN) for arena-based BPF programs, inspired + * by KASAN. + * + * The API + * ------- + * + * The implementation includes two kinds of components: Implementation + * of ASAN hooks injected by LLVM into the program, and API calls that + * allocators use to mark memory as valid or invalid. The full list is: + * + * LLVM stubs: + * + * void __asan_{load, store}<size>(intptr_t addr) + * Checks whether an access is valid. All variations covered + * by check_region_inline(). + * + * void __asan_{store, load}((intptr_t addr, ssize_t size) + * + * void __asan_report_{load, store}<size>(intptr_t addr) + * Report an access violation for the program. Used when LLVM + * uses direct code generation for shadow map checks. + * + * void *__asan_memcpy(void *d, const void *s, size_t n) + * void *__asan_memmove(void *d, const void *s, size_t n) + * void *__asan_memset(void *p, int c, size_t n) + * Hooks for ASAN instrumentation of the LLVM mem* builtins. + * Currently unimplemented just like the builtins themselves. + * + * API methods: + * + * asan_init() + * Initialize the ASAN map for the arena. + * + * asan_poison() + * Mark a region of memory as poisoned. Accessing poisoned memory + * causes asan_report() to fire. Invoked during free(). + * + * asan_unpoison() + * Mark a region as unpoisoned after alloc(). + * + * asan_shadow_set() + * Check a byte's validity directly. + * + * The Algorithm In Brief + * ---------------------- + * Each group of 8 bytes is mapped to a "granule" in the shadow map. This + * granule is the size of the byte and describes which bytes are valid. + * Possible values are: + * + * 0: All bytes are valid. Makes checks in the middle of an allocated region + * (most of them) fast. + * (0, 7]: How many consecutive bytes are valid, starting from the lowest one. + * The tradeoff is that we can't poison individual bytes in the middle of a + * valid region. + * [0x80, 0xff]: Special poison values, can be used to denote specific error + * modes (e.g., recently freed vs uninitialized memory). + * + * The mapping between a memory location and its shadow is: + * shadow_addr = shadow_base + (addr >> 3). We retain the 8:1 data:shadow + * ratio of existing ASAN implementations as a compromise between tracking + * granularity and space usage/scan overhead. + */ + +#ifdef BPF_ARENA_ASAN + +#pragma clang attribute push(__attribute__((no_sanitize("address"))), \ + apply_to = function) + +#define SHADOW_ALL_ZEROES ((u64)-1) + +/* + * Canary variable for ASAN violations. Set to the offending address. + */ +volatile u64 asan_violated = 0; + +/* + * Shadow map occupancy map. + */ +volatile u64 __asan_shadow_memory_dynamic_address; + +volatile u32 asan_reported = false; +volatile bool asan_inited = false; + +/* + * Set during program load. + */ +volatile bool asan_report_once = false; + +/* + * BPF does not currently support the memset/memcpy/memcmp intrinsics. + * For large sequential copies, or assignments of large data structures, + * the frontend will generate an intrinsic that causes the BPF backend + * to exit due to a missing implementation. Provide a simple implementation + * just for memset to use it for poisoning/unpoisoning the map. + */ +__weak int asan_memset(s8 __arena *dst, s8 val, size_t size) +{ + size_t i; + + for (i = zero; i < size && can_loop; i++) + dst[i] = val; + + return 0; +} + +/* Validate a 1-byte access, always within a single byte. */ +static __always_inline bool memory_is_poisoned_1(s8 __arena *addr) +{ + s8 shadow_value = *(s8 __arena *)mem_to_shadow(addr); + + /* Byte is 0, access is valid. */ + if (likely(!shadow_value)) + return false; + + /* + * Byte is non-zero. Access is valid if granule offset in [0, shadow_value), + * so the memory is poisoned if shadow_value is negative or smaller than + * the granule's value. + */ + + return ASAN_GRANULE(addr) >= shadow_value; +} + +/* Validate a 2- 4-, 8-byte access, shadow spans up to 2 bytes. */ +static __always_inline bool memory_is_poisoned_2_4_8(s8 __arena *addr, u64 size) +{ + u64 end = (u64)addr + size - 1; + + /* + * Region fully within a single byte (addition didn't + * overflow above ASAN_GRANULE). + */ + if (likely(ASAN_GRANULE(end) >= size - 1)) + return memory_is_poisoned_1((s8 __arena *)end); + + /* + * Otherwise first byte must be fully unpoisoned, and second byte + * must be unpoisoned up to the end of the accessed region. + */ + + return *(s8 __arena *)mem_to_shadow(addr) || memory_is_poisoned_1((s8 __arena *)end); +} + +__weak bool asan_shadow_set(void __arena *addr) +{ + return memory_is_poisoned_1(addr); +} + +static __always_inline u64 first_nonzero_byte(u64 addr, size_t size) +{ + while (size && can_loop) { + if (unlikely(*(s8 __arena *)addr)) + return addr; + addr += 1; + size -= 1; + } + + return SHADOW_ALL_ZEROES; +} + +static __always_inline bool memory_is_poisoned_n(s8 __arena *addr, u64 size) +{ + u64 ret; + u64 start; + u64 end; + + /* Size of [start, end] is end - start + 1. */ + start = (u64)mem_to_shadow(addr); + end = (u64)mem_to_shadow(addr + size - 1); + + ret = first_nonzero_byte(start, (end - start) + 1); + if (likely(ret == SHADOW_ALL_ZEROES)) + return false; + + return unlikely(ret != end || ASAN_GRANULE(addr + size - 1) >= *(s8 __arena *)end); +} + +__weak int asan_report(s8 __arena *addr, size_t sz, u32 flags) +{ + u32 reported = __sync_val_compare_and_swap(&asan_reported, false, true); + + /* Only report the first ASAN violation. */ + if (reported && asan_report_once) + return 0; + + asan_violated = (u64)addr; + + arena_stderr("Memory violation for address %p (0x%lx) for %s of size %ld\n", + addr, (u64)addr, + (flags & ASAN_WRITE) ? "write" : "read", + sz); + bpf_stream_print_stack(BPF_STDERR); + + return 0; +} + +static __always_inline bool check_asan_args(s8 __arena *addr, size_t size, + bool *result) +{ + bool valid = true; + + /* Size 0 accesses are valid even if the address is invalid. */ + if (unlikely(size == 0)) + goto confirmed_valid; + + /* + * Wraparound is possible for values close to the the edge of the + * 4GiB boundary of the arena (last valid address is 1UL << 32 - 1). + * + * + * The wraparound detection below works for small sizes. check_asan_args is + * always called from the builtin ASAN checks, so 1 <= size <= 64. Even + * for storeN/loadN that we do not expect to encounter the intrinsics will + * not have a large enough size that: + * + * - addr + size > MAX_U32 + * - (u32)(addr + size) > (u32) addr + * + * which would defeat wraparound detection. + */ + if (unlikely((u32)(u64)(addr + size) < (u32)(u64)addr)) + goto confirmed_invalid; + + return false; + +confirmed_invalid: + valid = false; + + /* FALLTHROUGH */ +confirmed_valid: + *result = valid; + + return true; +} + +static __always_inline bool check_region_inline(intptr_t ptr, size_t size, + u32 flags) +{ + s8 __arena *addr = (s8 __arena *)(u64)ptr; + bool is_poisoned, is_valid; + + if (check_asan_args(addr, size, &is_valid)) { + if (!is_valid) + asan_report(addr, size, flags); + return is_valid; + } + + switch (size) { + case 1: + is_poisoned = memory_is_poisoned_1(addr); + break; + case 2: + case 4: + case 8: + is_poisoned = memory_is_poisoned_2_4_8(addr, size); + break; + default: + is_poisoned = memory_is_poisoned_n(addr, size); + } + + if (is_poisoned) { + asan_report(addr, size, flags); + return false; + } + + return true; +} + +/* + * __alias is not supported for BPF so define *__noabort() variants as wrappers. + */ +#define DEFINE_ASAN_LOAD_STORE(size) \ + __hidden void __asan_store##size(intptr_t addr) \ + { \ + check_region_inline(addr, size, ASAN_WRITE); \ + } \ + __hidden void __asan_store##size##_noabort(intptr_t addr) \ + { \ + check_region_inline(addr, size, ASAN_WRITE); \ + } \ + __hidden void __asan_load##size(intptr_t addr) \ + { \ + check_region_inline(addr, size, ASAN_READ); \ + } \ + __hidden void __asan_load##size##_noabort(intptr_t addr) \ + { \ + check_region_inline(addr, size, ASAN_READ); \ + } \ + __hidden void __asan_report_store##size(intptr_t addr) \ + { \ + asan_report((s8 __arena *)addr, size, ASAN_WRITE); \ + } \ + __hidden void __asan_report_store##size##_noabort(intptr_t addr) \ + { \ + asan_report((s8 __arena *)addr, size, ASAN_WRITE); \ + } \ + __hidden void __asan_report_load##size(intptr_t addr) \ + { \ + asan_report((s8 __arena *)addr, size, ASAN_READ); \ + } \ + __hidden void __asan_report_load##size##_noabort(intptr_t addr) \ + { \ + asan_report((s8 __arena *)addr, size, ASAN_READ); \ + } + +DEFINE_ASAN_LOAD_STORE(1); +DEFINE_ASAN_LOAD_STORE(2); +DEFINE_ASAN_LOAD_STORE(4); +DEFINE_ASAN_LOAD_STORE(8); + +void __asan_storeN(intptr_t addr, ssize_t size) +{ + check_region_inline(addr, size, ASAN_WRITE); +} + +void __asan_storeN_noabort(intptr_t addr, ssize_t size) +{ + check_region_inline(addr, size, ASAN_WRITE); +} + +void __asan_loadN(intptr_t addr, ssize_t size) +{ + check_region_inline(addr, size, ASAN_READ); +} + +void __asan_loadN_noabort(intptr_t addr, ssize_t size) +{ + check_region_inline(addr, size, ASAN_READ); +} + +/* + * We currently do not sanitize globals. + */ +void __asan_register_globals(intptr_t globals, size_t n) +{ +} + +void __asan_unregister_globals(intptr_t globals, size_t n) +{ +} + +/* + * We do not currently have memcpy/memmove/memset intrinsics + * in LLVM. Do not implement sanitization. + */ +void *__asan_memcpy(void *d, const void *s, size_t n) +{ + arena_stderr("ASAN: Unexpected %s call", __func__); + return NULL; +} + +void *__asan_memmove(void *d, const void *s, size_t n) +{ + arena_stderr("ASAN: Unexpected %s call", __func__); + return NULL; +} + +void *__asan_memset(void *p, int c, size_t n) +{ + arena_stderr("ASAN: Unexpected %s call", __func__); + return NULL; +} + +/* + * Poisoning code, used when we add more freed memory to the allocator by: + * a) pulling memory from the arena segment using bpf_arena_alloc_pages() + * b) freeing memory from application code + */ +__hidden __noasan int asan_poison(void __arena *addr, s8 val, size_t size) +{ + s8 __arena *shadow; + size_t len; + + /* + * Poisoning from a non-granule address makes no sense: We can only allocate + * memory to the application that has a granule-aligned starting address, + * and bpf_arena_alloc_pages returns page-aligned memory. A non-aligned + * addr then implies we're freeing a different address than the one we + * allocated. + */ + if (unlikely((u64)addr & ASAN_GRANULE_MASK)) + return -EINVAL; + + /* + * We cannot free an unaligned region because it'd be possible that we + * cannot describe the resulting poisoning state of the granule in + * the ASAN encoding. + * + * Every granule represents a region of memory that looks like the + * following (P for poisoned bytes, C for clear): + * + * <Clear> <Poisoned> + * [ C C C ... P P ] + * + * The value of the granule's shadow map is the number of clear bytes in + * it. We cannot represent granules with the following state: + * + * [ P P ... C C ... P P ] + * + * That would be possible if we could free unaligned regions, so prevent that. + */ + if (unlikely(size & ASAN_GRANULE_MASK)) + return -EINVAL; + + shadow = mem_to_shadow(addr); + len = size >> ASAN_SHADOW_SHIFT; + + asan_memset(shadow, val, len); + + return 0; +} + +/* + * Unpoisoning code for marking memory as valid during allocation calls. + * + * Very similar to asan_poison, except we need to round up instead of + * down, then partially poison the last granule if necessary. + * + * Partial poisoning is useful for keeping the padding poisoned. Allocations + * are granule-aligned, so we we're reserving granule-aligned sizes for the + * allocation. However, we want to still treat accesses to the padding as + * invalid. Partial poisoning takes care of that. Freeing and poisoning the + * memory is still done in granule-aligned sizes and repoisons the already + * poisoned padding. + */ +__hidden __noasan int asan_unpoison(void __arena *addr, size_t size) +{ + size_t partial = size & ASAN_GRANULE_MASK; + s8 __arena *shadow; + size_t len; + + /* + * We cannot allocate in the middle of the granule. The ASAN shadow + * map encoding only describes regions of memory where every granule + * follows this format (P for poisoned, C for clear): + * + * <Clear> <Poisoned> + * [ C C C ... P P ] + * + * This is so we can use a single number in [0, ASAN_SHADOW_SCALE) + * to represent the poison state of the granule. + */ + if (unlikely((u64)addr & ASAN_GRANULE_MASK)) + return -EINVAL; + + shadow = mem_to_shadow(addr); + len = size >> ASAN_SHADOW_SHIFT; + + asan_memset(shadow, 0, len); + + /* + * If we are allocating a non-granule aligned region, we need to adjust + * the last byte of the shadow map to list how many bytes in the granule + * are unpoisoned. If the region is aligned, then the memset call above + * was enough. + */ + if (partial) + shadow[len] = partial; + + return 0; +} + +/* + * Initialize ASAN state when necessary. Triggered from userspace before + * allocator startup. + */ +SEC("syscall") +__weak __noasan int asan_init(struct asan_init_args *args) +{ + u64 globals_pages = args->arena_globals_pages; + u64 all_pages = args->arena_all_pages; + u64 shadow_map, shadow_pgoff; + u64 shadow_pages; + + if (asan_inited) + return 0; + + /* + * Round up the shadow map size to the nearest page. + */ + shadow_pages = all_pages >> ASAN_SHADOW_SHIFT; + if ((all_pages & ((1 << ASAN_SHADOW_SHIFT) - 1))) + shadow_pages += 1; + + if (all_pages > (1ULL << 32) / __PAGE_SIZE) { + arena_stderr("error: arena size %lx too large", all_pages); + return -EINVAL; + } + + if (globals_pages > all_pages) { + arena_stderr("error: globals %lx do not fit in arena %lx", + globals_pages, all_pages); + return -EINVAL; + } + + if (globals_pages + shadow_pages >= all_pages) { + arena_stderr("error: globals %lx do not leave room for shadow map %lx " + "(arena pages %lx)", + globals_pages, shadow_pages, all_pages); + return -EINVAL; + } + + shadow_pgoff = all_pages - shadow_pages - globals_pages; + __asan_shadow_memory_dynamic_address = shadow_pgoff * __PAGE_SIZE; + + /* + * Allocate the last (1/ASAN_SHADOW_SCALE)th of an arena's pages for the map + * We find the offset and size from the arena map. + * + * The allocated map pages are zeroed out, meaning all memory is marked as valid + * even if it's not allocated already. This is expected: Since the actual memory + * pages are not allocated, accesses to it will trigger page faults and will be + * reported through BPF streams. Any pages allocated through bpf_arena_alloc_pages + * should be poisoned by the allocator right after the call succeeds. + */ + shadow_map = (u64)bpf_arena_alloc_pages( + &arena, (void __arena *)__asan_shadow_memory_dynamic_address, + shadow_pages, NUMA_NO_NODE, 0); + if (!shadow_map) { + arena_stderr("Could not allocate shadow map\n"); + + __asan_shadow_memory_dynamic_address = 0; + + return -ENOMEM; + } + + asan_inited = true; + + return 0; +} + +#pragma clang attribute pop + +#endif /* BPF_ARENA_ASAN */ + +__weak char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c new file mode 100644 index 000000000000..5ff8e688ddc7 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* + * Copyright (c) 2025-2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2025-2026 Emil Tsalapatis <emil@etsalapatis.com> + */ + +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/bitmap.h> + +__weak +struct arena_bitmap __arena *bmp_alloc(size_t bits) +{ + struct arena_bitmap __arena *bmp; + size_t size = BITS_TO_LONG_LONGS(bits) * sizeof(bmp->bits[0]); + + /* Assume long-aligned masks. */ + if (bits % BITS_PER_LONG_LONG) + return NULL; + + bmp = (struct arena_bitmap __arena *)arena_malloc(size); + if (!bmp) + return NULL; + + bmp_clear(bits, bmp); + + return bmp; +} + +__weak +void bmp_free(struct arena_bitmap __arena *bmp) +{ + arena_free(bmp); +} + +__weak +void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + bmp->bits[BIT_WORD(bit)] |= BIT_MASK(bit); +} + +__weak +void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + bmp->bits[BIT_WORD(bit)] &= ~BIT_MASK(bit); +} + +__weak +bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + return bmp->bits[BIT_WORD(bit)] & BIT_MASK(bit); +} + +__weak +bool bmp_test_and_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + u64 val = BIT_MASK(bit); + u32 idx = BIT_WORD(bit); + u64 old, new, actual; + + do { + old = bmp->bits[idx]; + + if (!(old & val)) + return false; + + new = old & ~val; + actual = cmpxchg(&bmp->bits[idx], old, new); + + if (actual == old) + return true; + + } while (can_loop); + + return false; +} + +__weak +bool bmp_test_and_set_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + u64 val = BIT_MASK(bit); + u32 idx = BIT_WORD(bit); + u64 old, new, actual; + + do { + old = bmp->bits[idx]; + + if ((old & val)) + return true; + + new = old | val; + actual = cmpxchg(&bmp->bits[idx], old, new); + + if (actual == old) + return false; + + } while (can_loop); + + return false; +} + +__weak +void bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + u64 val = BIT_MASK(bit); + u32 idx = BIT_WORD(bit); + u64 old, new, actual; + + do { + old = bmp->bits[idx]; + new = old & ~val; + actual = cmpxchg(&bmp->bits[idx], old, new); + + } while (actual != old && can_loop); +} + +__weak +void bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + u64 val = BIT_MASK(bit); + u32 idx = BIT_WORD(bit); + u64 old, new, actual; + + do { + old = bmp->bits[idx]; + new = old | val; + actual = cmpxchg(&bmp->bits[idx], old, new); + + } while (actual != old && can_loop); +} + +__weak +void bmp_clear(size_t bits, struct arena_bitmap __arena *bmp) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) + bmp->bits[i] = 0; +} + +static __always_inline u64 bmp_last_word_mask(size_t bits) +{ + u32 rem = bits % BITS_PER_LONG_LONG; + + return rem ? (1ULL << rem) - 1 : ~0ULL; +} + +__weak +void bmp_and(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src1, struct arena_bitmap __arena *src2) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) + dst->bits[i] = src1->bits[i] & src2->bits[i]; + + if (nwords && bits % BITS_PER_LONG_LONG) + dst->bits[nwords - 1] &= bmp_last_word_mask(bits); +} + +__weak +void bmp_or(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src1, struct arena_bitmap __arena *src2) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) + dst->bits[i] = src1->bits[i] | src2->bits[i]; + + if (nwords && bits % BITS_PER_LONG_LONG) + dst->bits[nwords - 1] &= bmp_last_word_mask(bits); +} + +__weak +bool bmp_empty(size_t bits, struct arena_bitmap __arena *bmp) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) { + u64 mask = (i == nwords - 1) ? bmp_last_word_mask(bits) : ~0ULL; + + if (bmp->bits[i] & mask) + return false; + } + + return true; +} + +__weak +void bmp_copy(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) + dst->bits[i] = src->bits[i]; + + if (nwords && bits % BITS_PER_LONG_LONG) + dst->bits[nwords - 1] &= bmp_last_word_mask(bits); +} + +__weak +bool bmp_subset(size_t bits, struct arena_bitmap __arena *big, struct arena_bitmap __arena *small) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) { + u64 mask = (i == nwords - 1) ? bmp_last_word_mask(bits) : ~0ULL; + + if (~big->bits[i] & small->bits[i] & mask) + return false; + } + + return true; +} + +__weak +bool bmp_intersects(size_t bits, struct arena_bitmap __arena *arg1, struct arena_bitmap __arena *arg2) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) { + u64 mask = (i == nwords - 1) ? bmp_last_word_mask(bits) : ~0ULL; + + if (arg1->bits[i] & arg2->bits[i] & mask) + return true; + } + + return false; +} + +__weak +void bmp_print(size_t bits, struct arena_bitmap __arena *bmp) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) + arena_stderr("%016llx ", bmp->bits[i]); +} diff --git a/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c b/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c new file mode 100644 index 000000000000..c674ee5cfcc1 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c @@ -0,0 +1,903 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <libarena/common.h> +#include <libarena/asan.h> +#include <libarena/buddy.h> + +/* + * Buddy allocator arena-based implementation. + * + * Memory is organized into chunks. These chunks + * cannot be coalesced or split. Allocating + * chunks allocates their memory eagerly. + * + * Internally, each chunk is organized into blocks. + * Blocks _can_ be coalesced/split, but only inside + * the chunk. Each block can be allocated or + * unallocated. If allocated, the entire block holds + * user data. If unallocated, the block is mostly + * invalid memory, with the exception of a header + * used for freelist tracking. + * + * The header is placed at an offset inside the block + * to prevent off-by-one errors from the previous block + * from trivially overwriting the header. Such an error + * is also not catchable by ASAN, since the header remains + * valid memory even after the block is freed. It is still + * theoretically possible for the header to be corrupted + * without being caught by ASAN, but harder. + * + * Since the allocator needs to track order information for + * both allocated and free blocks, and allocated blocks cannot + * store a header, the allocator also stores per-chunk order + * information in a reserved region at the beginning of the + * chunk. The header includes a bitmap with the order of blocks + * and their allocation state. It also includes the freelist + * heads for the allocation itself. + */ + + +enum { + BUDDY_POISONED = (s8)0xef, + + /* Number of pages to be allocated per chunk. */ + BUDDY_CHUNK_PAGES = BUDDY_CHUNK_BYTES / __PAGE_SIZE +}; + +static inline int buddy_lock(struct buddy __arena *buddy) +{ + return arena_spin_lock(&buddy->lock); +} + +static inline void buddy_unlock(struct buddy __arena *buddy) +{ + arena_spin_unlock(&buddy->lock); +} + +/* + * Reserve part of the arena address space for the allocator. We use + * this to get aligned addresses for the chunks, since the arena + * page alloc kfuncs do not support aligning to a boundary (in this + * case 1 MiB, see buddy.h on how this is derived). + */ +static int buddy_reserve_arena_vaddr(struct buddy __arena *buddy) +{ + buddy->vaddr = 0; + + return bpf_arena_reserve_pages(&arena, + (void __arena *)BUDDY_VADDR_OFFSET, + BUDDY_VADDR_SIZE / __PAGE_SIZE); +} + +/* + * Free up any unused address space. Used only during teardown. + */ +static void buddy_unreserve_arena_vaddr(struct buddy __arena *buddy) +{ + bpf_arena_free_pages( + &arena, (void __arena *)(BUDDY_VADDR_OFFSET + buddy->vaddr), + (BUDDY_VADDR_SIZE - buddy->vaddr) / __PAGE_SIZE); + + buddy->vaddr = 0; +} + +/* + * Carve out part of the reserved address space and hand it over + * to the buddy allocator. + * + * We are assuming the buddy allocator is the only allocator in the + * system, so there is no race between this function reserving a + * page range and some other allocator actually making the BPF call + * to really create and reserve it. + * + * However, bump allocation must still be atomic because this function + * is called without the buddy lock from multiple threads concurrently. + */ +__weak int buddy_alloc_arena_vaddr(struct buddy __arena *buddy, u64 *vaddrp) +{ + u64 vaddr, old, new; + + if (!buddy || !vaddrp) + return -EINVAL; + + do { + vaddr = buddy->vaddr; + new = vaddr + BUDDY_CHUNK_BYTES; + + if (new > BUDDY_VADDR_SIZE) + return -EINVAL; + + old = __sync_val_compare_and_swap(&buddy->vaddr, vaddr, new); + } while (old != vaddr && can_loop); + + if (old != vaddr) + return -EINVAL; + + *vaddrp = BUDDY_VADDR_OFFSET + vaddr; + + return 0; +} + +static u64 arena_next_pow2(__u64 n) +{ + n--; + n |= n >> 1; + n |= n >> 2; + n |= n >> 4; + n |= n >> 8; + n |= n >> 16; + n |= n >> 32; + n++; + + return n; +} + +__weak +int idx_set_allocated(struct buddy_chunk __arena *chunk, u64 idx, bool allocated) +{ + bool already_allocated; + + if (unlikely(idx >= BUDDY_CHUNK_ITEMS)) { + arena_stderr("setting state of invalid idx (%ld, max %d)\n", idx, + BUDDY_CHUNK_ITEMS); + return -EINVAL; + } + + already_allocated = chunk->allocated[idx / 8] & (1 << (idx % 8)); + if (unlikely(already_allocated == allocated)) { + arena_stderr("Double %s of idx %ld for chunk %p", + allocated ? "alloc" : "free", + idx, chunk); + return -EINVAL; + } + + if (allocated) + chunk->allocated[idx / 8] |= 1 << (idx % 8); + else + chunk->allocated[idx / 8] &= ~(1 << (idx % 8)); + + return 0; +} + +static int idx_is_allocated(struct buddy_chunk __arena *chunk, u64 idx, bool *allocated) +{ + if (unlikely(idx >= BUDDY_CHUNK_ITEMS)) { + arena_stderr("getting state of invalid idx (%llu, max %d)\n", idx, + BUDDY_CHUNK_ITEMS); + return -EINVAL; + } + + *allocated = chunk->allocated[idx / 8] & (1 << (idx % 8)); + return 0; +} + +__weak +int idx_set_order(struct buddy_chunk __arena *chunk, u64 idx, u8 order) +{ + u8 prev_order; + + if (unlikely(order >= BUDDY_CHUNK_NUM_ORDERS)) { + arena_stderr("setting invalid order %u\n", order); + return -EINVAL; + } + + if (unlikely(idx >= BUDDY_CHUNK_ITEMS)) { + arena_stderr("setting order of invalid idx (%d, max %d)\n", idx, + BUDDY_CHUNK_ITEMS); + return -EINVAL; + } + + /* + * We store two order instances per byte, one per nibble. + * Retain the existing nibble. + */ + prev_order = chunk->orders[idx / 2]; + if (idx & 0x1) { + order &= 0xf; + order |= (prev_order & 0xf0); + } else { + order <<= 4; + order |= (prev_order & 0xf); + } + + chunk->orders[idx / 2] = order; + + return 0; +} + +static u8 idx_get_order(struct buddy_chunk __arena *chunk, u64 idx) +{ + u8 result; + + _Static_assert(BUDDY_CHUNK_NUM_ORDERS <= 16, + "order must fit in 4 bits"); + + if (unlikely(idx >= BUDDY_CHUNK_ITEMS)) { + arena_stderr("getting order of invalid idx %u\n", idx); + return BUDDY_CHUNK_NUM_ORDERS; + } + + result = chunk->orders[idx / 2]; + + return (idx & 0x1) ? (result & 0xf) : (result >> 4); +} + +static void __arena *idx_to_addr(struct buddy_chunk __arena *chunk, size_t idx) +{ + u64 address; + + if (unlikely(idx >= BUDDY_CHUNK_ITEMS)) { + arena_stderr("translating invalid idx %u\n", idx); + return NULL; + } + + /* + * The data blocks start in the chunk after the metadata block. + * We find the actual address by indexing into the region at an + * BUDDY_MIN_ALLOC_BYTES granularity, the minimum allowed. + * The index number already accounts for the fact that the first + * blocks in the chunk are occupied by the metadata, so we do + * not need to offset it. + */ + + address = (u64)chunk + (idx * BUDDY_MIN_ALLOC_BYTES); + + return (void __arena *)address; +} + +static struct buddy_header __arena *idx_to_header(struct buddy_chunk __arena *chunk, size_t idx) +{ + bool allocated; + u64 address; + + if (unlikely(idx_is_allocated(chunk, idx, &allocated))) { + arena_stderr("accessing invalid idx 0x%lx\n", idx); + return NULL; + } + + if (unlikely(allocated)) { + arena_stderr("accessing allocated idx 0x%lx as header\n", idx); + return NULL; + } + + address = (u64)idx_to_addr(chunk, idx); + if (!address) + return NULL; + + /* + * Offset the header within the block. This avoids accidental overwrites + * to the header because of off-by-one errors when using adjacent blocks. + * + * The offset has been chosen as a compromise between ASAN effectiveness + * and allocator granularity: + * 1) ASAN dictates valid data runs are 8-byte aligned. + * 2) We want to keep a low minimum allocation size (currently 16). + * + * As a result, we have only two possible positions for the header: Bytes + * 0 and 8. Keeping the header in byte 0 means off-by-ones from the previous + * block touch the header, and, since the header must be accessible, ASAN + * will not trigger. Keeping the header on byte 8 means off-by-one errors from + * the previous block are caught by ASAN. Negative offsets are rarer, so + * while accesses into the block from the next block are possible, they are + * less probable. + */ + + return (struct buddy_header __arena *)(address + BUDDY_HEADER_OFF); +} + +static void header_add_freelist(struct buddy_chunk __arena *chunk, struct buddy_header __arena *header, + u64 idx, u8 order) +{ + struct buddy_header __arena *tmp_header; + + idx_set_order(chunk, idx, order); + + header->next_index = chunk->freelists[order]; + header->prev_index = BUDDY_CHUNK_ITEMS; + + if (header->next_index != BUDDY_CHUNK_ITEMS) { + tmp_header = idx_to_header(chunk, header->next_index); + tmp_header->prev_index = idx; + } + + chunk->freelists[order] = idx; +} + +static void header_remove_freelist(struct buddy_chunk __arena *chunk, + struct buddy_header __arena *header, u8 order) +{ + struct buddy_header __arena *tmp_header; + + if (header->prev_index != BUDDY_CHUNK_ITEMS) { + tmp_header = idx_to_header(chunk, header->prev_index); + tmp_header->next_index = header->next_index; + } + + if (header->next_index != BUDDY_CHUNK_ITEMS) { + tmp_header = idx_to_header(chunk, header->next_index); + tmp_header->prev_index = header->prev_index; + } + + /* Pop off the list head if necessary. */ + if (idx_to_header(chunk, chunk->freelists[order]) == header) + chunk->freelists[order] = header->next_index; + + header->prev_index = BUDDY_CHUNK_ITEMS; + header->next_index = BUDDY_CHUNK_ITEMS; +} + +static u64 size_to_order(size_t size) +{ + u64 order; + + /* + * Legal sizes are [1, 4GiB] (the biggest possible arena). + * Of course, sizes close to GiB are practically impossible + * to fulfill and allocation will fail, but that's taken care + * of by the caller. + */ + + if (unlikely(size == 0 || size > (1UL << 32))) { + arena_stderr("illegal size request %lu\n", size); + return 64; + } + /* + * To find the order of the allocation we find the first power of two + * >= the requested size, take the log2, then adjust it for the minimum + * allocation size by removing the minimum shift from it. Requests + * smaller than the minimum allocation size are rounded up. + */ + order = arena_fls(arena_next_pow2(size)) - 1; + if (order < BUDDY_MIN_ALLOC_SHIFT) + return 0; + + return order - BUDDY_MIN_ALLOC_SHIFT; +} + +__weak +int add_leftovers_to_freelist(struct buddy_chunk __arena *chunk, u32 cur_idx, + u64 min_order, u64 max_order) +{ + struct buddy_header __arena *header; + u64 ord; + u32 idx; + + for (ord = min_order; ord < max_order && can_loop; ord++) { + /* Mark the buddy as free and add it to the freelists. */ + idx = cur_idx + (1 << ord); + + header = idx_to_header(chunk, idx); + if (unlikely(!header)) { + arena_stderr("idx %u has no header", idx); + return -EINVAL; + } + + asan_unpoison(header, sizeof(*header)); + + header_add_freelist(chunk, header, idx, ord); + } + + return 0; +} + +static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) +{ + u64 order, ord, min_order, max_order; + struct buddy_chunk __arena *chunk; + size_t left; + int power2; + u64 vaddr; + u32 idx; + int ret; + + /* + * Step 1: Allocate a properly aligned chunk, and + * prep it for insertion into the buddy allocator. + * We don't need the allocator lock until step 2. + */ + + ret = buddy_alloc_arena_vaddr(buddy, &vaddr); + if (ret) + return NULL; + + /* Addresses must be aligned to the chunk boundary. */ + if (vaddr % BUDDY_CHUNK_BYTES) + return NULL; + + /* Unreserve the address space. */ + bpf_arena_free_pages(&arena, (void __arena *)vaddr, + BUDDY_CHUNK_PAGES); + + chunk = bpf_arena_alloc_pages(&arena, (void __arena *)vaddr, + BUDDY_CHUNK_PAGES, NUMA_NO_NODE, 0); + if (!chunk) { + arena_stderr("[ALLOC FAILED]"); + return NULL; + } + + if (buddy_lock(buddy)) { + /* + * We cannot reclaim the vaddr space, but that is ok - this + * operation should always succeed. The error path is to catch + * accidental deadlocks that will cause -ENOMEMs to the program as + * the allocator fails to refill itself, in which case vaddr usage + * is the least of our worries. + */ + bpf_arena_free_pages(&arena, (void __arena *)vaddr, BUDDY_CHUNK_PAGES); + return NULL; + } + + asan_poison(chunk, BUDDY_POISONED, BUDDY_CHUNK_PAGES * __PAGE_SIZE); + + /* Unpoison the chunk itself. */ + asan_unpoison(chunk, sizeof(*chunk)); + + /* Mark all freelists as empty. */ + for (ord = zero; ord < BUDDY_CHUNK_NUM_ORDERS && can_loop; ord++) + chunk->freelists[ord] = BUDDY_CHUNK_ITEMS; + + /* + * Initialize the chunk by carving out a page range to hold the metadata + * struct above, then dumping the rest of the pages into the allocator. + */ + + _Static_assert(BUDDY_CHUNK_PAGES * __PAGE_SIZE >= + BUDDY_MIN_ALLOC_BYTES * + BUDDY_CHUNK_ITEMS, + "chunk must fit within the allocation"); + + /* + * Step 2: Reserve a chunk for the chunk metadata, then breaks + * the rest of the full allocation into the different buckets. + * We allocating the memory by grabbing blocks of progressively + * smaller sizes from the allocator, which are guaranteed to be + * continuous. + * + * This operation also populates the allocator. + * + * Algorithm: + * + * - max_order: The last order allocation we made + * - left: How many bytes are left to allocate + * - cur_index: Current index into the top-level block we are + * allocating from. + * + * Step 3: + * - Find the largest power-of-2 allocation still smaller than left (infimum) + * - Reserve a chunk of that size, along with its buddy + * - For every order from [infimum + 1, last order), carve out a block + * and put it into the allocator. + * + * Example: Chunk size 0b1010000 (80 bytes) + * + * Step 1: + * + * idx infimum 1 << max_order + * 0 64 128 1 << 20 + * |________|_________|______________________| + * + * Blocks set aside: + * [0, 64) - Completely allocated + * [64, 128) - Will be further split in the next iteration + * + * Blocks added to the allocator: + * [128, 256) + * [256, 512) + * ... + * [1 << 18, 1 << 19) + * [1 << 19, 1 << 20) + * + * Step 2: + * + * idx infimum idx + 1 << max_order + * 64 80 96 64 + 1 << 6 = 128 + * |________|_________|______________________| + * + * Blocks set aside: + * [64, 80) - Completely allocated + * + * Blocks added to the allocator: + * [80, 96) - left == 0 so the buddy is unused and marked as freed + * [96, 128) + */ + max_order = BUDDY_CHUNK_NUM_ORDERS; + left = sizeof(*chunk); + idx = 0; + while (left && can_loop) { + power2 = arena_fls(left) - 1; + /* + * Note: The condition below only triggers to catch serious bugs + * early. There is no sane way to undo any block insertions from + * the allocated chunk, so just leak any leftover allocations, + * emit a diagnostic, unlock and exit. + * + */ + if (unlikely(power2 >= BUDDY_CHUNK_NUM_ORDERS)) { + arena_stderr( + "buddy chunk metadata require allocation of order %d\n", + power2); + arena_stderr( + "chunk has size of 0x%lx bytes (left %lx bytes)\n", + sizeof(*chunk), left); + buddy_unlock(buddy); + + return NULL; + } + + /* Round up allocations that are too small. */ + + left -= (power2 >= BUDDY_MIN_ALLOC_SHIFT) ? 1 << power2 : left; + order = (power2 >= BUDDY_MIN_ALLOC_SHIFT) ? power2 - BUDDY_MIN_ALLOC_SHIFT : 0; + + if (idx_set_allocated(chunk, idx, true)) { + buddy_unlock(buddy); + return NULL; + } + + /* + * Starting an order above the one we allocated, populate + * the allocator with free blocks. If this is the last + * allocation (left == 0), also mark the buddy as free. + * + * See comment above about error handling: The error path + * is only there as a way to mitigate deeply buggy allocator + * states by emitting a diagnostic in add_leftovers_to_freelist() + * and leaking any memory not added in the freelists. + */ + min_order = left ? order + 1 : order; + if (add_leftovers_to_freelist(chunk, idx, min_order, max_order)) { + buddy_unlock(buddy); + return NULL; + } + + /* Adjust the index. */ + idx += 1 << order; + max_order = order; + } + + buddy_unlock(buddy); + + return chunk; +} + +__weak int buddy_init(struct buddy __arena *buddy) +{ + struct buddy_chunk __arena *chunk; + int ret; + + if (!asan_ready()) + return -EINVAL; + + /* Reserve enough address space to ensure allocations are aligned. */ + ret = buddy_reserve_arena_vaddr(buddy); + if (ret) + return ret; + + _Static_assert(BUDDY_CHUNK_PAGES > 0, + "chunk must use one or more pages"); + + chunk = buddy_chunk_get(buddy); + + if (buddy_lock(buddy)) { + bpf_arena_free_pages(&arena, chunk, BUDDY_CHUNK_PAGES); + return -EINVAL; + } + + /* Chunk is already properly unpoisoned if allocated. */ + if (chunk) + chunk->next = buddy->first_chunk; + + /* Put the chunk at the beginning of the list. */ + buddy->first_chunk = chunk; + + buddy_unlock(buddy); + + return chunk ? 0 : -ENOMEM; +} + +/* + * Destroy the allocator. This does not check whether there are any allocations + * currently in use, so any pages being accessed will start taking arena faults. + * We do not take a lock because we are freeing arena pages, and nobody should + * be using the allocator at that point in the execution. + */ +__weak int buddy_destroy(struct buddy __arena *buddy) +{ + struct buddy_chunk __arena *chunk, *next; + + if (!buddy) + return -EINVAL; + + /* + * Traverse all buddy chunks and free them back to the arena + * with the same granularity they were allocated with. + */ + for (chunk = buddy->first_chunk; chunk && can_loop; chunk = next) { + next = chunk->next; + + /* Wholesale poison the entire block. */ + asan_poison(chunk, BUDDY_POISONED, + BUDDY_CHUNK_PAGES * __PAGE_SIZE); + bpf_arena_free_pages(&arena, chunk, BUDDY_CHUNK_PAGES); + } + + /* Free up any part of the address space that did not get used. */ + buddy_unreserve_arena_vaddr(buddy); + + /* Clear all fields. */ + buddy->first_chunk = NULL; + + return 0; +} + +__weak u64 buddy_chunk_alloc(struct buddy_chunk __arena *chunk, int order_req) +{ + struct buddy_header __arena *header, *tmp_header, *next_header; + u32 idx, tmpidx, retidx; + u64 address; + u64 order = 0; + u64 i; + + for (order = order_req; order < BUDDY_CHUNK_NUM_ORDERS && can_loop; order++) { + if (chunk->freelists[order] != BUDDY_CHUNK_ITEMS) + break; + } + + if (order >= BUDDY_CHUNK_NUM_ORDERS) + return (u64)NULL; + + retidx = chunk->freelists[order]; + header = idx_to_header(chunk, retidx); + if (unlikely(!header)) + return (u64) NULL; + + chunk->freelists[order] = header->next_index; + + if (header->next_index != BUDDY_CHUNK_ITEMS) { + next_header = idx_to_header(chunk, header->next_index); + next_header->prev_index = BUDDY_CHUNK_ITEMS; + } + + header->prev_index = BUDDY_CHUNK_ITEMS; + header->next_index = BUDDY_CHUNK_ITEMS; + if (idx_set_order(chunk, retidx, order_req)) + return (u64)NULL; + + if (idx_set_allocated(chunk, retidx, true)) + return (u64)NULL; + + /* + * Do not unpoison the address yet, will be done by the caller + * because the caller has the exact allocation size requested. + */ + address = (u64)idx_to_addr(chunk, retidx); + if (!address) + return (u64)NULL; + + /* If we allocated from a larger-order chunk, split the buddies. */ + for (i = order_req; i < order && can_loop; i++) { + /* + * Flip the bit for the current order (the bit is guaranteed + * to be 0, so just add 1 << i). + */ + idx = retidx + (1 << i); + + /* Add the buddy of the allocation to the free list. */ + header = idx_to_header(chunk, idx); + /* Unpoison the buddy header */ + asan_unpoison(header, sizeof(*header)); + + if (idx_set_order(chunk, idx, i)) + return (u64)NULL; + + /* Push the header to the beginning of the freelists list. */ + tmpidx = chunk->freelists[i]; + + header->prev_index = BUDDY_CHUNK_ITEMS; + header->next_index = tmpidx; + + if (tmpidx != BUDDY_CHUNK_ITEMS) { + tmp_header = idx_to_header(chunk, tmpidx); + tmp_header->prev_index = idx; + } + + chunk->freelists[i] = idx; + } + + return address; +} + +/* Scan the existing chunks for available memory. */ +static u64 buddy_alloc_from_existing_chunks(struct buddy __arena *buddy, int order) +{ + struct buddy_chunk __arena *chunk; + u64 address; + + for (chunk = buddy->first_chunk; chunk != NULL && can_loop; + chunk = chunk->next) { + address = buddy_chunk_alloc(chunk, order); + if (address) + return address; + } + + return (u64)NULL; +} + +/* + * Try an allocation from a newly allocated chunk. Also + * incorporate the chunk into the linked list. + */ +static u64 buddy_alloc_from_new_chunk(struct buddy __arena *buddy, struct buddy_chunk __arena *chunk, int order) +{ + u64 address; + + if (buddy_lock(buddy)) + return (u64)NULL; + + + /* + * Add the chunk into the allocator and try + * to allocate specifically from that chunk. + */ + chunk->next = buddy->first_chunk; + buddy->first_chunk = chunk; + + address = buddy_chunk_alloc(buddy->first_chunk, order); + + buddy_unlock(buddy); + + return (u64)address; +} +__weak +void __arena *buddy_alloc(struct buddy __arena *buddy, size_t size) +{ + void __arena *address = NULL; + struct buddy_chunk __arena *chunk; + int order; + + if (!buddy) + return NULL; + + order = size_to_order(size); + if (order >= BUDDY_CHUNK_NUM_ORDERS || order < 0) { + arena_stderr("invalid order %d (sz %lu)\n", order, size); + return NULL; + } + + if (buddy_lock(buddy)) + return NULL; + + address = (u8 __arena *)buddy_alloc_from_existing_chunks(buddy, order); + buddy_unlock(buddy); + if (address) + goto done; + + /* Get a new chunk. */ + chunk = buddy_chunk_get(buddy); + if (chunk) + address = (u8 __arena *)buddy_alloc_from_new_chunk(buddy, chunk, order); + +done: + /* If we failed to allocate memory, return NULL. */ + if (!address) + return NULL; + + /* + * Unpoison exactly the amount of bytes requested. If the + * data is smaller than the header, we must poison any + * unused bytes that were part of the header. + */ + if (size < BUDDY_HEADER_OFF + sizeof(struct buddy_header __arena)) + asan_poison(address + BUDDY_HEADER_OFF, BUDDY_POISONED, + sizeof(struct buddy_header __arena)); + + asan_unpoison(address, size); + + return address; +} + +static __always_inline int buddy_free_unlocked(struct buddy __arena *buddy, u64 addr) +{ + struct buddy_header __arena *header, *buddy_header; + u64 idx, buddy_idx, tmp_idx; + struct buddy_chunk __arena *chunk; + bool allocated; + u8 order; + int ret; + + if (!buddy) + return -EINVAL; + + if (addr & (BUDDY_MIN_ALLOC_BYTES - 1)) { + arena_stderr("Freeing unaligned address %llx\n", addr); + return -EINVAL; + } + + /* Get (chunk, idx) out of the address. */ + chunk = (void __arena *)(addr & ~BUDDY_CHUNK_OFFSET_MASK); + idx = (addr & BUDDY_CHUNK_OFFSET_MASK) / BUDDY_MIN_ALLOC_BYTES; + + /* Mark the block as unallocated so we can access the header. */ + ret = idx_set_allocated(chunk, idx, false); + if (ret) + return ret; + + order = idx_get_order(chunk, idx); + header = idx_to_header(chunk, idx); + + /* The header is in the block itself, keep it unpoisoned. */ + asan_poison((u8 __arena *)addr, BUDDY_POISONED, + BUDDY_MIN_ALLOC_BYTES << order); + asan_unpoison(header, sizeof(*header)); + + /* + * Coalescing loop. Merge with free buddies of equal order. + * For every coalescing step, keep the left buddy and + * drop the right buddy's header. + */ + for (; order < BUDDY_CHUNK_NUM_ORDERS && can_loop; order++) { + buddy_idx = idx ^ (1 << order); + + /* Check if the buddy is actually free. */ + idx_is_allocated(chunk, buddy_idx, &allocated); + if (allocated) + break; + + /* + * If buddy is not the same order as the chunk + * being freed, then we're done coalescing. + */ + if (idx_get_order(chunk, buddy_idx) != order) + break; + + buddy_header = idx_to_header(chunk, buddy_idx); + header_remove_freelist(chunk, buddy_header, order); + + /* Keep the left header out of the two buddies, drop the other one. */ + if (buddy_idx < idx) { + tmp_idx = idx; + idx = buddy_idx; + buddy_idx = tmp_idx; + } + + /* Remove the buddy from the freelists so that we can merge it. */ + idx_set_order(chunk, buddy_idx, order); + + buddy_header = idx_to_header(chunk, buddy_idx); + asan_poison(buddy_header, BUDDY_POISONED, + sizeof(*buddy_header)); + } + + /* Header properly freed but not in any freelists yet .*/ + idx_set_order(chunk, idx, order); + + header = idx_to_header(chunk, idx); + header_add_freelist(chunk, header, idx, order); + + return 0; +} + +__weak int buddy_free(struct buddy __arena *buddy, void __arena *addr) +{ + int ret; + + if (!buddy) + return -EINVAL; + + /* Freeing NULL is a valid no-op. */ + if (!addr) + return 0; + + ret = buddy_lock(buddy); + if (ret) + return ret; + + ret = buddy_free_unlocked(buddy, (u64)addr); + + buddy_unlock(buddy); + + return ret; +} + +__weak char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/libarena/src/common.bpf.c b/tools/testing/selftests/bpf/libarena/src/common.bpf.c new file mode 100644 index 000000000000..41b1de3452fe --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/src/common.bpf.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <libarena/common.h> +#include <libarena/asan.h> +#include <libarena/buddy.h> + +struct buddy __arena buddy; +volatile u32 zero = 0; + +/* + * Storage for the queue nodes declared by bpf_arena_spin_lock.h. Each program + * linking the arena spinlock provides exactly one definition, so that the array + * is emitted once rather than once per translation unit. + */ +struct arena_qnode __arena __hidden qnodes[_Q_MAX_CPUS][_Q_MAX_NODES]; + +int arena_fls(__u64 word) +{ + if (!word) + return 0; + + return 64 - __builtin_clzll(word); +} + +SEC("syscall") +__weak int arena_get_info(struct arena_get_info_args *args) +{ + args->arena_base = arena_base(&arena); + + return 0; +} + +SEC("syscall") +__weak int arena_alloc_reserve(struct arena_alloc_reserve_args *args) +{ + return bpf_arena_reserve_pages(&arena, NULL, args->nr_pages); +} + +SEC("syscall") +__weak int arena_buddy_reset(void) +{ + buddy_destroy(&buddy); + + return buddy_init(&buddy); +} + +SEC("syscall") +__weak int arena_buddy_destroy(void) +{ + return buddy_destroy(&buddy); +} + +__weak void __arena *arena_malloc(size_t size) +{ + return buddy_alloc(&buddy, size); +} + +__weak void arena_free(void __arena *ptr) +{ + buddy_free(&buddy, ptr); +} + + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/libarena/src/rbtree.bpf.c b/tools/testing/selftests/bpf/libarena/src/rbtree.bpf.c new file mode 100644 index 000000000000..7f0f6dc3e17d --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/src/rbtree.bpf.c @@ -0,0 +1,1047 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* + * Copyright (c) 2025-2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2025-2026 Emil Tsalapatis <emil@etsalapatis.com> + */ + +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/rbtree.h> + +int rb_integrity_check(struct rbtree __arena *rbtree); +void rbnode_print(size_t depth, struct rbnode __arena *rbn); +static int rbnode_replace(struct rbtree __arena *rbtree, + struct rbnode __arena *existing, + struct rbnode __arena *replacement); + +struct rbtree __arena *rb_create(enum rbtree_alloc alloc, + enum rbtree_insert_mode insert) +{ + struct rbtree __arena *rbtree; + + rbtree = arena_malloc(sizeof(*rbtree)); + if (unlikely(!rbtree)) + return NULL; + + /* + * RB_UPDATE overwrites existing values in the nodes, but RB_NOALLOC + * trees manage the tree nodes directly (including holding pointers + * to them). Disallow mixing the two modes to avoid dealing with + * unintuitive semantics. + */ + if (alloc == RB_NOALLOC && insert == RB_UPDATE) { + arena_stderr("WARNING: Cannot combine RB_NOALLOC and RB_UPDATE"); + arena_free(rbtree); + return NULL; + } + + rbtree->alloc = alloc; + rbtree->insert = insert; + rbtree->root = NULL; + + return rbtree; +} + +__weak +int rb_destroy(struct rbtree __arena *rbtree) +{ + int ret = 0; + + arena_subprog_init(); + + if (unlikely(!rbtree)) + return -EINVAL; + + if (rbtree->alloc == RB_NOALLOC) { + /* + * We cannot do anything about RB_NOALLOC nodes. The whole + * point of RB_NOALLOC is that the nodes are directly owned + * by the caller that allocates and inserts them. We could + * unilaterally grab all nodes and free them anyway, but that + * would almost certainly cause UAF as the callers keep accessing + * the now freed nodes. Throw an error instead. + */ + if (rbtree->root) { + arena_stderr("WARNING: Destroying RB_NOALLOC tree with > 0 nodes"); + return -EBUSY; + } + + goto out; + } + + while (rbtree->root && can_loop) { + ret = rb_remove(rbtree, rbtree->root->key); + if (ret) + break; + } + +out: + arena_free(rbtree); + return ret; +} + +static inline int rbnode_dir(struct rbnode __arena *node) +{ + /* Arbitrarily choose a direction for the root. */ + if (unlikely(!node->parent)) + return 0; + + return (node->parent->left == node) ? 0 : 1; +} + +/* + * The __noinline is to prevent inlining from bloating the add + * remove calls, in turn causing register splits and increasing + * stack usage above what is permitted. + */ +__noinline +int rbnode_rotate(struct rbtree __arena *rbtree, + struct rbnode __arena *node, int dir) +{ + struct rbnode __arena *tmp, *parent; + int parentdir; + + parent = node->parent; + if (parent) + parentdir = rbnode_dir(node); + + /* If we're doing a root change, are we the root? */ + if (unlikely(!parent && rbtree->root != node)) + return -EINVAL; + + /* + * Does the node we're turning into the root into exist? + * Note that the new root is on the opposite side of the + * rotation's direction. + */ + tmp = node->child[1 - dir]; + if (unlikely(!tmp)) + return -EINVAL; + + /* Steal the closest child of the new root. */ + node->child[1 - dir] = tmp->child[dir]; + if (node->child[1 - dir]) + node->child[1 - dir]->parent = node; + + /* Put the node below the new root.*/ + tmp->child[dir] = node; + node->parent = tmp; + + tmp->parent = parent; + if (parent) + parent->child[parentdir] = tmp; + else + rbtree->root = tmp; + + return 0; +} + +static +struct rbnode __arena *rbnode_find(struct rbnode __arena *subtree, u64 key) +{ + struct rbnode __arena *node = subtree; + int dir; + + if (!subtree) + return NULL; + + while (can_loop) { + if (node->key == key) + break; + + dir = (key < node->key) ? 0 : 1; + + if (!node->child[dir]) + break; + + node = node->child[dir]; + } + + return node; +} + +static +struct rbnode __arena *rbnode_least_upper_bound(struct rbnode __arena *subtree, uint64_t key) +{ + struct rbnode __arena *node = subtree; + int dir; + + if (!subtree) + return NULL; + + while (can_loop) { + dir = (key <= node->key) ? 0 : 1; + + if (!node->child[dir]) + break; + + node = node->child[dir]; + } + + return node; +} + +__weak +int rb_find(struct rbtree __arena *rbtree, u64 key, u64 *value) +{ + struct rbnode __arena *node; + + if (unlikely(!rbtree)) + return -EINVAL; + + if (unlikely(!value)) + return -EINVAL; + + node = rbnode_find(rbtree->root, key); + if (!node || node->key != key) + return -ENOENT; + + *value = node->value; + + return 0; +} + +__weak +struct rbnode __arena *rb_node_alloc(u64 key, u64 value) +{ + struct rbnode __arena *rbnode = NULL; + + rbnode = (struct rbnode __arena *)arena_malloc(sizeof(*rbnode)); + if (!rbnode) + return NULL; + + /* + * WARNING: The order of assignments is weird on purpose. + * See comment in rb_insert_node() for more context. + * TL;DR: Prevent consecutive 0 assignments from being + * promoted into an unverifiable memset by the compiler. + */ + + rbnode->key = key; + rbnode->parent = NULL; + rbnode->value = value; + rbnode->left = NULL; + rbnode->is_red = true; + rbnode->right = NULL; + + return rbnode; +} + +__weak +void rb_node_free(struct rbnode __arena *rbnode) +{ + arena_free(rbnode); +} + +static +int rb_node_insert(struct rbtree __arena *rbtree, + struct rbnode __arena *node) +{ + struct rbnode __arena *grandparent, *parent = rbtree->root; + u64 key = node->key; + struct rbnode __arena *uncle; + int dir; + int ret; + + if (unlikely(!rbtree)) + return -EINVAL; + + if (!parent) { + rbtree->root = node; + return 0; + } + + if (rbtree->insert != RB_DUPLICATE) + parent = rbnode_find(parent, key); + else + parent = rbnode_least_upper_bound(parent, key); + + if (key == parent->key && rbtree->insert != RB_DUPLICATE) { + if (rbtree->insert == RB_UPDATE) { + /* + * Replace the old node with the new one. + * Free up the old node. + */ + ret = rbnode_replace(rbtree, parent, node); + if (ret) + return ret; + + if (rbtree->alloc == RB_ALLOC) + rb_node_free(parent); + + return 0; + } + + /* Otherwise it's RB_DEFAULT. */ + return -EALREADY; + } + + node->parent = parent; + /* Also works if key == parent->key. */ + if (key <= parent->key) + parent->left = node; + else + parent->right = node; + + while (can_loop) { + parent = node->parent; + if (!parent) + return 0; + + if (!parent->is_red) + return 0; + + grandparent = parent->parent; + if (!grandparent) { + parent->is_red = false; + return 0; + } + + dir = rbnode_dir(parent); + uncle = grandparent->child[1 - dir]; + + if (!uncle || !uncle->is_red) { + if (node == parent->child[1 - dir]) { + rbnode_rotate(rbtree, parent, dir); + node = parent; + parent = grandparent->child[dir]; + } + + rbnode_rotate(rbtree, grandparent, 1 - dir); + parent->is_red = false; + grandparent->is_red = true; + + return 0; + } + + /* Uncle is red. */ + + parent->is_red = false; + uncle->is_red = false; + grandparent->is_red = true; + + node = grandparent; + } + + return 0; +} + +int rb_insert_node(struct rbtree __arena *rbtree, + struct rbnode __arena *node) +{ + if (unlikely(!rbtree)) + return -EINVAL; + + if (unlikely(rbtree->alloc == RB_ALLOC)) + return -EINVAL; + + node->left = NULL; + + /* + * Workaround to break an optimization that causes + * verification failures on some compilers. Assignments + * of the kind + * + * *(r0 + 0) = 0; + * *(r0 + 8) = 0; + * *(r0 + 16) = 0; + * + * get promoted into a memset, and that in turn is not + * handled properly for arena memory by LLVM 21 and GCC 15. + * Add a barrier for now to prevent the assignments from being fused. + */ + barrier(); + + node->parent = NULL; + node->right = NULL; + + node->is_red = true; + + return rb_node_insert(rbtree, node); +} + +__weak +int rb_insert(struct rbtree __arena *rbtree, u64 key, u64 value) +{ + struct rbnode __arena *node; + int ret; + + if (unlikely(!rbtree)) + return -EINVAL; + + if (unlikely(rbtree->alloc != RB_ALLOC)) + return -EINVAL; + + node = rb_node_alloc(key, value); + if (!node) + return -ENOMEM; + + ret = rb_node_insert(rbtree, node); + if (ret) { + rb_node_free(node); + return ret; + } + + return 0; +} + +static inline struct rbnode __arena *rbnode_least(struct rbnode __arena *subtree) +{ + while (subtree->left && can_loop) + subtree = subtree->left; + + return subtree; +} + +__weak int rb_least(struct rbtree __arena *rbtree, u64 *key, u64 *value) +{ + struct rbnode __arena *least; + + if (unlikely(!rbtree)) + return -EINVAL; + + if (!rbtree->root) + return -ENOENT; + + least = rbnode_least(rbtree->root); + if (key) + *key = least->key; + if (value) + *value = least->value; + + return 0; +} + + +/* + * If we are referencing ourselves, a and b have a parent-child relation, + * and we should be pointing at the other node instead. + */ +static inline void rbnode_fixup_pointers(struct rbnode __arena *a, + struct rbnode __arena *b) +{ +#define fixup(n1, n2, member) do { if (n1->member == n1) n1->member = n2; } while (0) + fixup(a, b, left); + fixup(a, b, right); + fixup(a, b, parent); +#undef fixup +} + +static inline void rbnode_swap_values(struct rbnode __arena *a, + struct rbnode __arena *b) +{ +#define swap(n1, n2, tmp) do { (tmp) = (n1); (n1) = (n2); (n2) = (tmp); } while (0) + struct rbnode __arena *tmpnode; + u64 tmp; + + /* Swap the pointers. */ + swap(a->is_red, b->is_red, tmp); + + swap(a->left, b->left, tmpnode); + swap(a->right, b->right, tmpnode); + swap(a->parent, b->parent, tmpnode); +#undef swap + + /* Account for the nodes being parent and child. */ + rbnode_fixup_pointers(b, a); + rbnode_fixup_pointers(a, b); +} + +static inline void rbnode_adjust_neighbors(struct rbtree __arena *rbtree, + struct rbnode __arena *node, int dir) +{ + if (node->left) + node->left->parent = node; + if (node->right) + node->right->parent = node; + + if (node->parent) { + node->parent->child[dir] = node; + return; + } + + rbtree->root = node; +} + +/* + * Directly replace an existing node with a replacement. The replacement node + * should not already be in the tree. + */ +static int rbnode_replace(struct rbtree __arena *rbtree, + struct rbnode __arena *existing, + struct rbnode __arena *replacement) +{ + int dir = 0; + + if (unlikely(replacement->parent || replacement->left || replacement->right)) + return -EINVAL; + + if (existing->parent) + dir = rbnode_dir(existing); + + replacement->is_red = existing->is_red; + replacement->left = existing->left; + replacement->right = existing->right; + replacement->parent = existing->parent; + + /* Fix up the new node's neighbors. */ + rbnode_adjust_neighbors(rbtree, replacement, dir); + + return 0; +} + +/* + * Switch two nodes in the tree in place. This is useful during node deletion. + * This is more involved than switching the values of the two nodes because we + * must update all tree pointers. + */ +static void rbnode_switch(struct rbtree __arena *rbtree, + struct rbnode __arena *a, + struct rbnode __arena *b) +{ + int adir = 0, bdir = 0; + + /* + * Store the direction in the parent because we will not + * be able to recompute it once we start swapping values. + */ + if (a->parent) + adir = rbnode_dir(a); + + if (b->parent) + bdir = rbnode_dir(b); + + rbnode_swap_values(a, b); + + /* + * Fix up the pointers from the children/parent to the + * new nodes. + */ + rbnode_adjust_neighbors(rbtree, a, bdir); + rbnode_adjust_neighbors(rbtree, b, adir); +} + +static inline int rbnode_remove_node_single_child(struct rbtree __arena *rbtree, + struct rbnode __arena *node, + bool free) +{ + struct rbnode __arena *child; + int dir; + + if (unlikely(node->is_red)) { + arena_stderr("Node unexpectedly red\n"); + return -EINVAL; + } + + child = node->left ? node->left : node->right; + if (unlikely(!child->is_red)) { + arena_stderr("Only child is black\n"); + return -EINVAL; + } + + /* + * Since it's the immediate child, we can just + * remove the parent. + */ + child->parent = node->parent; + + if (node->parent) { + dir = rbnode_dir(node); + node->parent->child[dir] = child; + } else { + rbtree->root = child; + } + + /* Color the child black. */ + child->is_red = false; + + /* Only free if called from rb_remove. */ + if (free) + rb_node_free(node); + + return 0; +} + +static inline bool rbnode_has_red_children(struct rbnode __arena *node) +{ + if (node->left && node->left->is_red) + return true; + + return node->right && node->right->is_red; +} + +static +int rb_node_remove(struct rbtree __arena *rbtree, + struct rbnode __arena *node) +{ + struct rbnode __arena *parent, *sibling, *close_nephew, *distant_nephew; + bool free = (rbtree->alloc == RB_ALLOC); + struct rbnode __arena *replace, *initial; + bool is_red; + int dir; + + /* Both children present, replace with next largest key. */ + if (node->left && node->right) { + /* + * Swap the node itself instead of just the + * key/value pair to account for nodes embedded + * in other structs. + */ + + replace = rbnode_least(node->right); + rbnode_switch(rbtree, replace, node); + + /* + * FALLTHROUGH: We moved the node we are removing to + * the leftmost position of the subtree. We can now + * remove it as if it was always where we moved it to. + */ + } + + initial = node; + + /* Only one child present, replace with child and paint it black. */ + if (!node->left != !node->right) + return rbnode_remove_node_single_child(rbtree, node, free); + + /* (!node->left && !node->right) */ + + parent = node->parent; + if (!parent) { + /* Check that we're _actually_ the root. */ + if (rbtree->root == node) + rbtree->root = NULL; + else + arena_stderr("WARNING: Attempting to remove detached node from rbtree\n"); + + if (free) + rb_node_free(node); + return 0; + } + + dir = rbnode_dir(node); + parent->child[dir] = NULL; + is_red = node->is_red; + + if (free) + rb_node_free(node); + + /* If we removed a red node, we did not unbalance the tree.*/ + if (is_red) + return 0; + + sibling = parent->child[1 - dir]; + if (unlikely(!sibling)) { + arena_stderr("rbtree: removed black node has no sibling\n"); + return -EINVAL; + } + + /* + * We removed a black node, causing a change in path + * weight. Start rebalancing. The invariant is that + * all paths going through the node are shortened + * by one, and the current node is black. + */ + while (can_loop) { + + /* Balancing reached the root, there can be no imbalance. */ + if (!parent) + return 0; + + /* + * We already determined the dir, either above or + * at the end of the loop. + */ + + /* + * If we have no sibling, the tree was + * already unbalanced. + */ + sibling = parent->child[1 - dir]; + if (unlikely(!sibling)) { + arena_stderr("rbtree: removed black node has no sibling\n"); + return -EINVAL; + } + + /* Sibling is red, turn it into the grandparent. */ + if (sibling->is_red) { + /* + * Sibling is red. Transform the tree to turn + * the sibling into the parent's position, and + * repaint them. This does not balance the tree + * but makes it so we know the sibling is black + * and so can use the transformations to balance. + */ + rbnode_rotate(rbtree, parent, dir); + parent->is_red = true; + sibling->is_red = false; + + /* Our new sibling is now the close nephew. */ + sibling = parent->child[1 - dir]; + /* If sibling has any red siblings, break out. */ + if (rbnode_has_red_children(sibling)) + break; + + /* We can repaint the sibling and parent, we're done. */ + sibling->is_red = true; + parent->is_red = false; + + return 0; + } + + /* Sibling guaranteed to be black. If it has red children, break out. */ + if (rbnode_has_red_children(sibling)) + break; + + /* + * Both sibling and children are black. If parent is red, swap + * colors with the sibling. Otherwise + */ + if (parent->is_red) { + parent->is_red = false; + sibling->is_red = true; + return 0; + } + + /* + * Parent, sibling, and all its children are black. Repaint the sibling. + * This shortens the paths through it, so pop up a level in the + * tree and repeat the balancing. + */ + sibling->is_red = true; + node = parent; + parent = node->parent; + dir = rbnode_dir(node); + } + + if (node != initial) { + dir = rbnode_dir(node); + parent = node->parent; + sibling = parent->child[1-dir]; + } + /* + * Almost there. We know between the parent, sibling, + * and nephews only one or two of the nephews are red. If + * it is the close one, rotate it to the sibling position, + * paint it black, and paint the previous sibling red. + */ + + close_nephew = sibling->child[dir]; + distant_nephew = sibling->child[1 - dir]; + + /* + * If the distant red nephew is not red, rotate + * and repaint. We need the distant nephew + * to be red. We know the close nephew is red + * because at least one of them are, so the + * distant one is black if it exists. + */ + if (!distant_nephew || !distant_nephew->is_red) { + rbnode_rotate(rbtree, sibling, 1 - dir); + sibling->is_red = true; + close_nephew->is_red = false; + distant_nephew = sibling; + sibling = close_nephew; + } + + /* + * We now know it's the distant nephew that's red. + * Rotate the sibling into our parent's position + * and paint both black. + */ + + rbnode_rotate(rbtree, parent, dir); + sibling->is_red = parent->is_red; + parent->is_red = false; + distant_nephew->is_red = false; + + return 0; +} + +__weak +int rb_remove_node(struct rbtree __arena *rbtree, + struct rbnode __arena *node) +{ + if (unlikely(!rbtree)) + return -EINVAL; + + if (unlikely(rbtree->alloc == RB_ALLOC)) + return -EINVAL; + + return rb_node_remove(rbtree, node); +} + +__weak +int rb_remove(struct rbtree __arena *rbtree, u64 key) +{ + struct rbnode __arena *node; + + if (unlikely(!rbtree)) + return -EINVAL; + + if (unlikely(rbtree->alloc != RB_ALLOC)) + return -EINVAL; + + if (!rbtree->root) + return -ENOENT; + + node = rbnode_find(rbtree->root, key); + if (!node || node->key != key) + return -ENOENT; + + return rb_node_remove(rbtree, node); +} + +__weak +int rb_pop(struct rbtree __arena *rbtree, u64 *key, u64 *value) +{ + struct rbnode __arena *node; + + if (unlikely(!rbtree)) + return -EINVAL; + + if (!rbtree->root) + return -ENOENT; + + if (rbtree->alloc != RB_ALLOC) + return -EINVAL; + + node = rbnode_least(rbtree->root); + if (unlikely(!node)) + return -ENOENT; + + if (key) + *key = node->key; + if (value) + *value = node->value; + + return rb_node_remove(rbtree, node); +} + +inline void rbnode_print(size_t depth, struct rbnode __arena *rbn) +{ + arena_stderr("[DEPTH %d] %p (%s)\n PARENT %p", depth, rbn, rbn->is_red ? "red" : "black", rbn->parent); + arena_stderr("\tKV (%ld, %ld)\n LEFT %p RIGHT %p]\n", rbn->key, rbn->value, rbn->left, rbn->right); +} + +enum rb_print_state { + RB_NONE_VISITED, + RB_LEFT_VISITED, + RB_RIGHT_VISITED, +}; + +__weak +enum rb_print_state rb_print_next_state(struct rbnode __arena *rbnode, + enum rb_print_state state, u64 *next) +{ + if (unlikely(!next)) + return RB_NONE_VISITED; + + switch (state) { + case RB_NONE_VISITED: + if (rbnode->left) { + *next = (u64)rbnode->left; + state = RB_LEFT_VISITED; + break; + } + + /* FALLTHROUGH */ + + case RB_LEFT_VISITED: + if (rbnode->right) { + *next = (u64)rbnode->right; + state = RB_RIGHT_VISITED; + break; + } + + /* FALLTHROUGH */ + + default: + *next = 0; + state = RB_RIGHT_VISITED; + } + + return state; +} + +__weak +int rb_print_pop_up(struct rbnode __arena **rbnodep, u8 *depthp, enum rb_print_state (*stack)[RB_MAXLVL_PRINT], enum rb_print_state *state) +{ + struct rbnode __arena *rbnode; + volatile u8 depth; + int j; + + if (unlikely(!rbnodep || !depthp || !stack || !state)) + return -EINVAL; + + rbnode = *rbnodep; + depth = *depthp; + + for (j = 0; j < RB_MAXLVL_PRINT && can_loop; j++) { + if (*state != RB_RIGHT_VISITED) + break; + + depth -= 1; + if (depth < 0 || depth >= RB_MAXLVL_PRINT) + break; + + *state = (*stack)[depth % RB_MAXLVL_PRINT]; + rbnode = rbnode->parent; + } + + *rbnodep = rbnode; + *depthp = depth; + + return 0; +} + +__weak +int rb_print(struct rbtree __arena *rbtree) +{ + enum rb_print_state stack[RB_MAXLVL_PRINT]; + struct rbnode __arena *rbnode = rbtree->root; + enum rb_print_state state; + struct rbnode __arena *next; + u64 next_addr; + u8 depth; + int ret; + + if (unlikely(!rbtree)) + return -EINVAL; + + depth = 0; + state = RB_NONE_VISITED; + + arena_stderr("=== RB TREE START ===\n"); + + if (!rbtree->root) + goto out; + + /* Even with can_loop, the verifier doesn't like infinite loops. */ + while (can_loop) { + if (state == RB_NONE_VISITED) + rbnode_print(depth, rbnode); + + /* Find which child to traverse next. */ + state = rb_print_next_state(rbnode, state, &next_addr); + next = (struct rbnode __arena *)next_addr; + + /* Child found. Store the node state and go on. */ + if (next) { + if (depth < 0 || depth >= RB_MAXLVL_PRINT) + return 0; + + stack[depth++] = state; + + rbnode = next; + state = RB_NONE_VISITED; + + continue; + } + + /* Otherwise, go as far up as possible. */ + ret = rb_print_pop_up(&rbnode, &depth, &stack, &state); + if (ret) + return -EINVAL; + + if (depth < 0 || depth >= RB_MAXLVL_PRINT) { + arena_stderr("=== RB TREE END (depth %d\n)===", depth); + return 0; + } + + } + +out: + arena_stderr("=== RB TREE END ===\n"); + + return 0; +} + +__weak +int rb_integrity_check(struct rbtree __arena *rbtree) +{ + enum rb_print_state stack[RB_MAXLVL_PRINT]; + struct rbnode __arena *rbnode = rbtree->root; + enum rb_print_state state; + struct rbnode __arena *next; + u64 next_addr; + u8 depth; + int ret; + + if (unlikely(!rbtree)) + return -EINVAL; + + if (!rbtree->root) + return 0; + + depth = 0; + state = RB_NONE_VISITED; + + /* Even with can_loop, the verifier doesn't like infinite loops. */ + while (can_loop) { + if (rbnode->parent && rbnode->parent->left != rbnode + && rbnode->parent->right != rbnode) { + arena_stderr("WARNING: Inconsistent tree. Parent %p has no child %p\n", rbnode->parent, rbnode); + return -EINVAL; + } + + if (rbnode->parent == rbnode) { + arena_stderr("WARNING: Inconsistent tree, node %p is its own parent\n", rbnode); + return -EINVAL; + } + + if (rbnode->left == rbnode) { + arena_stderr("WARNING: Inconsistent tree, node %p is its own left child\n", rbnode); + return -EINVAL; + } + + if (rbnode->right == rbnode) { + arena_stderr("WARNING: Inconsistent tree, node %p is its own right child\n", rbnode); + return -EINVAL; + } + + if (rbnode->is_red) { + if (rbnode->left && rbnode->left->is_red) { + arena_stderr("WARNING: Inconsistent tree. Parent has %p has red child %p\n", rbnode, rbnode->left); + return -EINVAL; + } + if (rbnode->right && rbnode->right->is_red) { + arena_stderr("WARNING: Inconsistent tree. Parent has %p has red child %p\n", rbnode, rbnode->right); + return -EINVAL; + } + } else if (rbnode->parent && rbnode->parent->child[1 - rbnode_dir(rbnode)] == NULL) { + arena_stderr("WARNING: Inconsistent tree. Black node %p has no sibling\n", rbnode); + return -EINVAL; + } + + /* Find which child to traverse next. */ + state = rb_print_next_state(rbnode, state, &next_addr); + next = (struct rbnode __arena *)next_addr; + + /* Child found. Store the node state and go on. */ + if (next) { + if (depth < 0 || depth >= RB_MAXLVL_PRINT) + return 0; + + stack[depth++] = state; + + rbnode = next; + state = RB_NONE_VISITED; + + continue; + } + + /* Otherwise, go as far up as possible. */ + ret = rb_print_pop_up(&rbnode, &depth, &stack, &state); + if (ret) + return -EINVAL; + + if (depth < 0 || depth >= RB_MAXLVL_PRINT) { + return 0; + } + + } + + return 0; +} diff --git a/tools/testing/selftests/bpf/libarena/src/spmc.bpf.c b/tools/testing/selftests/bpf/libarena/src/spmc.bpf.c new file mode 100644 index 000000000000..42732b7d29a6 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/src/spmc.bpf.c @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* + * Copyright (c) 2025-2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2025-2026 Emil Tsalapatis <etsal@meta.com> + */ + +#include <bpf_atomic.h> + +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/spmc.h> + +static inline +u64 spmc_arr_size(volatile struct spmc_arr __arena *spmc_arr) +{ + return SPMC_ARR_BASESZ << spmc_arr->order; +} + +static inline +u64 spmc_arr_get(volatile struct spmc_arr __arena *spmc_arr, u64 ind) +{ + u64 ret = READ_ONCE(spmc_arr->data[ind % spmc_arr_size(spmc_arr)]); + + return ret; +} + +static inline +void spmc_arr_put(volatile struct spmc_arr __arena *spmc_arr, u64 ind, u64 value) +{ + WRITE_ONCE(spmc_arr->data[ind % spmc_arr_size(spmc_arr)], value); +} + +static inline +void spmc_arr_copy(volatile struct spmc_arr __arena *dst, + volatile struct spmc_arr __arena *src, u64 b, u64 t) +{ + u64 i; + + for (i = t; i < b && can_loop; i++) + spmc_arr_put(dst, i, spmc_arr_get(src, i)); +} + +static inline +int spmc_order_init(struct spmc __arena *spmc, int order) +{ + volatile struct spmc_arr __arena *arr = &spmc->arr[order]; + + if (unlikely(!spmc)) + return -EINVAL; + + if (order >= SPMC_ARR_ORDERS) + return -E2BIG; + + /* Already allocated? */ + if (arr->data) + return 0; + + arr->data = arena_malloc((SPMC_ARR_BASESZ << order) * sizeof(*arr->data)); + if (!arr->data) + return -ENOMEM; + + return 0; +} + +__weak +int spmc_owned_add(struct spmc __arena *spmc, u64 val) +{ + volatile struct spmc_arr __arena *newarr; + volatile struct spmc_arr __arena *arr; + ssize_t sz; + u64 b, t; + int ret; + + if (unlikely(!spmc)) + return -EINVAL; + + /* + * Bottom must always be read first, also + * see spmc_steal(). + */ + b = smp_load_acquire(&spmc->bottom); + t = READ_ONCE(spmc->top); + arr = READ_ONCE(spmc->cur); + + sz = b - t; + if (sz >= spmc_arr_size(arr) - 1) { + ret = spmc_order_init(spmc, arr->order + 1); + if (ret) + return ret; + + newarr = &spmc->arr[arr->order + 1]; + + spmc_arr_copy(newarr, arr, b, t); + smp_store_release(&spmc->cur, newarr); + arr = newarr; + } + + spmc_arr_put(arr, b, val); + smp_store_release(&spmc->bottom, b + 1); + + return 0; +} + + +__weak +int spmc_owned_remove(struct spmc __arena *spmc, u64 *val) +{ + volatile struct spmc_arr __arena *arr; + int ret = 0; + ssize_t sz; + u64 value; + u64 b, t; + + if (unlikely(!spmc || !val)) + return -EINVAL; + + b = READ_ONCE(spmc->bottom) - 1; + WRITE_ONCE(spmc->bottom, b); + smp_mb(); + + t = READ_ONCE(spmc->top); + arr = READ_ONCE(spmc->cur); + + sz = b - t; + if (sz < 0) { + WRITE_ONCE(spmc->bottom, t); + return -ENOENT; + } + + value = spmc_arr_get(arr, b); + if (sz > 0) { + *val = value; + return 0; + } + + if (cmpxchg(&spmc->top, t, t + 1) != t) + ret = -EAGAIN; + + WRITE_ONCE(spmc->bottom, t + 1); + + if (ret) + return ret; + + *val = value; + + return 0; +} + +__weak +int spmc_steal(struct spmc __arena *spmc, u64 *val) +{ + volatile struct spmc_arr __arena *arr; + ssize_t sz; + u64 value; + u64 b, t; + + if (unlikely(!spmc || !val)) + return -EINVAL; + + /* + * It is important that t is read before b for + * stealers to avoid racing with the owner. + * Races between stealers are dealt with using + * CAS to increment the top value below. + */ + t = smp_load_acquire(&spmc->top); + b = smp_load_acquire(&spmc->bottom); + + sz = b - t; + if (sz <= 0) + return -ENOENT; + + arr = smp_load_acquire(&spmc->cur); + value = spmc_arr_get(arr, t); + + if (cmpxchg(&spmc->top, t, t + 1) != t) + return -EAGAIN; + + *val = value; + + return 0; +} + + +__weak +struct spmc __arena *spmc_create(void) +{ + /* + * Marked as volatile because otherwise the array + * reference in the internal loop gets demoted to + * scalar and the program fails verification. + */ + struct spmc __arena *volatile spmc; + int ret, i; + + spmc = arena_malloc(sizeof(*spmc)); + if (!spmc) + return NULL; + + spmc->bottom = 0; + spmc->top = 0; + + for (i = 0; i < SPMC_ARR_ORDERS && can_loop; i++) { + spmc->arr[i].data = NULL; + spmc->arr[i].order = i; + } + + ret = spmc_order_init((struct spmc __arena *)spmc, 0); + if (ret) { + arena_free(spmc); + return NULL; + } + + spmc->cur = &spmc->arr[0]; + + return (struct spmc __arena *)spmc; +} + +__weak +int spmc_destroy(struct spmc __arena *spmc) +{ + int i; + + if (unlikely(!spmc)) + return -EINVAL; + + for (i = 0; i < SPMC_ARR_ORDERS && can_loop; i++) + arena_free(spmc->arr[i].data); + + arena_free(spmc); + + return 0; +} diff --git a/tools/testing/selftests/bpf/network_helpers.c b/tools/testing/selftests/bpf/network_helpers.c index b82f572641b7..cdf2d7d3ab32 100644 --- a/tools/testing/selftests/bpf/network_helpers.c +++ b/tools/testing/selftests/bpf/network_helpers.c @@ -111,7 +111,7 @@ int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t a if (settimeo(fd, opts->timeout_ms)) goto error_close; - if (type == SOCK_STREAM && + if ((type & SOCK_TYPE_MASK) == SOCK_STREAM && setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on))) { log_err("Failed to enable SO_REUSEADDR"); goto error_close; @@ -128,7 +128,7 @@ int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t a goto error_close; } - if (type == SOCK_STREAM) { + if ((type & SOCK_TYPE_MASK) == SOCK_STREAM) { if (listen(fd, opts->backlog ? MAX(opts->backlog, 0) : 1) < 0) { log_err("Failed to listed on socket"); goto error_close; @@ -424,7 +424,8 @@ int make_sockaddr(int family, const char *addr_str, __u16 port, *len = sizeof(*sin6); return 0; } else if (family == AF_UNIX) { - /* Note that we always use abstract unix sockets to avoid having + /* + * Note that we always use abstract unix sockets to avoid having * to clean up leftover files. */ struct sockaddr_un *sun = (void *)addr; @@ -865,7 +866,8 @@ static bool is_ethernet(const u_char *packet) memcpy(&arphdr_type, packet + 8, 2); arphdr_type = ntohs(arphdr_type); - /* Except the following cases, the protocol type contains the + /* + * Except the following cases, the protocol type contains the * Ethernet protocol type for the packet. * * https://www.tcpdump.org/linktypes/LINKTYPE_LINUX_SLL2.html @@ -1033,19 +1035,22 @@ static void *traffic_monitor_thread(void *arg) if (!packet) continue; - /* According to the man page of pcap_dump(), first argument + /* + * According to the man page of pcap_dump(), first argument * is the pcap_dumper_t pointer even it's argument type is * u_char *. */ pcap_dump((u_char *)dumper, &header, packet); - /* Not sure what other types of packets look like. Here, we + /* + * Not sure what other types of packets look like. Here, we * parse only Ethernet and compatible packets. */ if (!is_ethernet(packet)) continue; - /* Skip SLL2 header + /* + * Skip SLL2 header * https://www.tcpdump.org/linktypes/LINKTYPE_LINUX_SLL2.html * * Although the document doesn't mention that, the payload @@ -1079,7 +1084,8 @@ static void *traffic_monitor_thread(void *arg) return NULL; } -/* Prepare the pcap handle to capture packets. +/* + * Prepare the pcap handle to capture packets. * * This pcap is non-blocking and immediate mode is enabled to receive * captured packets as soon as possible. The snaplen is set to 1024 bytes @@ -1150,7 +1156,8 @@ static void encode_test_name(char *buf, size_t len, const char *test_name, const #define PCAP_DIR "/tmp/tmon_pcap" -/* Start to monitor the network traffic in the given network namespace. +/* + * Start to monitor the network traffic in the given network namespace. * * netns: the name of the network namespace to monitor. If NULL, the * current network namespace is monitored. @@ -1255,7 +1262,8 @@ static void traffic_monitor_release(struct tmonitor_ctx *ctx) free(ctx); } -/* Stop the network traffic monitor. +/* + * Stop the network traffic monitor. * * ctx: the context returned by traffic_monitor_start() */ diff --git a/tools/testing/selftests/bpf/network_helpers.h b/tools/testing/selftests/bpf/network_helpers.h index 79a010c88e11..75133119c04a 100644 --- a/tools/testing/selftests/bpf/network_helpers.h +++ b/tools/testing/selftests/bpf/network_helpers.h @@ -25,6 +25,11 @@ typedef __u16 __sum16; #define VIP_NUM 5 #define MAGIC_BYTES 123 +/* include/linux/net.h */ +#ifndef SOCK_TYPE_MASK +#define SOCK_TYPE_MASK 0xf +#endif + struct network_helper_opts { int timeout_ms; int proto; diff --git a/tools/testing/selftests/bpf/prog_tests/access_variable_array.c b/tools/testing/selftests/bpf/prog_tests/access_variable_array.c deleted file mode 100644 index 08131782437c..000000000000 --- a/tools/testing/selftests/bpf/prog_tests/access_variable_array.c +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2022 Bytedance */ - -#include <test_progs.h> -#include "test_access_variable_array.skel.h" - -void test_access_variable_array(void) -{ - struct test_access_variable_array *skel; - - skel = test_access_variable_array__open_and_load(); - if (!ASSERT_OK_PTR(skel, "test_access_variable_array__open_and_load")) - return; - - test_access_variable_array__destroy(skel); -} diff --git a/tools/testing/selftests/bpf/prog_tests/align.c b/tools/testing/selftests/bpf/prog_tests/align.c deleted file mode 100644 index 24c509ce4e5b..000000000000 --- a/tools/testing/selftests/bpf/prog_tests/align.c +++ /dev/null @@ -1,712 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include <test_progs.h> - -#define MAX_INSNS 512 -#define MAX_MATCHES 24 - -struct bpf_reg_match { - unsigned int line; - const char *reg; - const char *match; -}; - -struct bpf_align_test { - const char *descr; - struct bpf_insn insns[MAX_INSNS]; - enum { - UNDEF, - ACCEPT, - REJECT - } result; - enum bpf_prog_type prog_type; - /* Matches must be in order of increasing line */ - struct bpf_reg_match matches[MAX_MATCHES]; -}; - -static struct bpf_align_test tests[] = { - /* Four tests of known constants. These aren't staggeringly - * interesting since we track exact values now. - */ - { - .descr = "mov", - .insns = { - BPF_MOV64_IMM(BPF_REG_3, 2), - BPF_MOV64_IMM(BPF_REG_3, 4), - BPF_MOV64_IMM(BPF_REG_3, 8), - BPF_MOV64_IMM(BPF_REG_3, 16), - BPF_MOV64_IMM(BPF_REG_3, 32), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - {0, "R1", "ctx()"}, - {0, "R10", "fp0"}, - {0, "R3", "2"}, - {1, "R3", "4"}, - {2, "R3", "8"}, - {3, "R3", "16"}, - {4, "R3", "32"}, - }, - }, - { - .descr = "shift", - .insns = { - BPF_MOV64_IMM(BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_3, 4), - BPF_MOV64_IMM(BPF_REG_4, 32), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - {0, "R1", "ctx()"}, - {0, "R10", "fp0"}, - {0, "R3", "1"}, - {1, "R3", "2"}, - {2, "R3", "4"}, - {3, "R3", "8"}, - {4, "R3", "16"}, - {5, "R3", "1"}, - {6, "R4", "32"}, - {7, "R4", "16"}, - {8, "R4", "8"}, - {9, "R4", "4"}, - {10, "R4", "2"}, - }, - }, - { - .descr = "addsub", - .insns = { - BPF_MOV64_IMM(BPF_REG_3, 4), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, 4), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, 2), - BPF_MOV64_IMM(BPF_REG_4, 8), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 2), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - {0, "R1", "ctx()"}, - {0, "R10", "fp0"}, - {0, "R3", "4"}, - {1, "R3", "8"}, - {2, "R3", "10"}, - {3, "R4", "8"}, - {4, "R4", "12"}, - {5, "R4", "14"}, - }, - }, - { - .descr = "mul", - .insns = { - BPF_MOV64_IMM(BPF_REG_3, 7), - BPF_ALU64_IMM(BPF_MUL, BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_MUL, BPF_REG_3, 2), - BPF_ALU64_IMM(BPF_MUL, BPF_REG_3, 4), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - {0, "R1", "ctx()"}, - {0, "R10", "fp0"}, - {0, "R3", "7"}, - {1, "R3", "7"}, - {2, "R3", "14"}, - {3, "R3", "56"}, - }, - }, - - /* Tests using unknown values */ -#define PREP_PKT_POINTERS \ - BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, \ - offsetof(struct __sk_buff, data)), \ - BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, \ - offsetof(struct __sk_buff, data_end)) - -#define LOAD_UNKNOWN(DST_REG) \ - PREP_PKT_POINTERS, \ - BPF_MOV64_REG(BPF_REG_0, BPF_REG_2), \ - BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 8), \ - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_0, 1), \ - BPF_EXIT_INSN(), \ - BPF_LDX_MEM(BPF_B, DST_REG, BPF_REG_2, 0) - - { - .descr = "unknown shift", - .insns = { - LOAD_UNKNOWN(BPF_REG_3), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1), - LOAD_UNKNOWN(BPF_REG_4), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_4, 5), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - {6, "R0", "pkt(off=8,r=8)"}, - {6, "R3", "var_off=(0x0; 0xff)"}, - {7, "R3", "var_off=(0x0; 0x1fe)"}, - {8, "R3", "var_off=(0x0; 0x3fc)"}, - {9, "R3", "var_off=(0x0; 0x7f8)"}, - {10, "R3", "var_off=(0x0; 0xff0)"}, - {12, "R3", "pkt_end()"}, - {17, "R4", "var_off=(0x0; 0xff)"}, - {18, "R4", "var_off=(0x0; 0x1fe0)"}, - {19, "R4", "var_off=(0x0; 0xff0)"}, - {20, "R4", "var_off=(0x0; 0x7f8)"}, - {21, "R4", "var_off=(0x0; 0x3fc)"}, - {22, "R4", "var_off=(0x0; 0x1fe)"}, - }, - }, - { - .descr = "unknown mul", - .insns = { - LOAD_UNKNOWN(BPF_REG_3), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_3), - BPF_ALU64_IMM(BPF_MUL, BPF_REG_4, 1), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_3), - BPF_ALU64_IMM(BPF_MUL, BPF_REG_4, 2), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_3), - BPF_ALU64_IMM(BPF_MUL, BPF_REG_4, 4), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_3), - BPF_ALU64_IMM(BPF_MUL, BPF_REG_4, 8), - BPF_ALU64_IMM(BPF_MUL, BPF_REG_4, 2), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - {6, "R3", "var_off=(0x0; 0xff)"}, - {7, "R4", "var_off=(0x0; 0xff)"}, - {8, "R4", "var_off=(0x0; 0xff)"}, - {9, "R4", "var_off=(0x0; 0xff)"}, - {10, "R4", "var_off=(0x0; 0x1fe)"}, - {11, "R4", "var_off=(0x0; 0xff)"}, - {12, "R4", "var_off=(0x0; 0x3fc)"}, - {13, "R4", "var_off=(0x0; 0xff)"}, - {14, "R4", "var_off=(0x0; 0x7f8)"}, - {15, "R4", "var_off=(0x0; 0xff0)"}, - }, - }, - { - .descr = "packet const offset", - .insns = { - PREP_PKT_POINTERS, - BPF_MOV64_REG(BPF_REG_5, BPF_REG_2), - - BPF_MOV64_IMM(BPF_REG_0, 0), - - /* Skip over ethernet header. */ - BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 14), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - - BPF_LDX_MEM(BPF_B, BPF_REG_4, BPF_REG_5, 0), - BPF_LDX_MEM(BPF_B, BPF_REG_4, BPF_REG_5, 1), - BPF_LDX_MEM(BPF_B, BPF_REG_4, BPF_REG_5, 2), - BPF_LDX_MEM(BPF_B, BPF_REG_4, BPF_REG_5, 3), - BPF_LDX_MEM(BPF_H, BPF_REG_4, BPF_REG_5, 0), - BPF_LDX_MEM(BPF_H, BPF_REG_4, BPF_REG_5, 2), - BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_5, 0), - - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - {2, "R5", "pkt(r=0)"}, - {4, "R5", "pkt(off=14,r=0)"}, - {5, "R4", "pkt(off=14,r=0)"}, - {9, "R2", "pkt(r=18)"}, - {10, "R5", "pkt(off=14,r=18)"}, - {10, "R4", "var_off=(0x0; 0xff)"}, - {13, "R4", "var_off=(0x0; 0xffff)"}, - {14, "R4", "var_off=(0x0; 0xffff)"}, - }, - }, - { - .descr = "packet variable offset", - .insns = { - LOAD_UNKNOWN(BPF_REG_6), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_6, 2), - - /* First, add a constant to the R5 packet pointer, - * then a variable with a known alignment. - */ - BPF_MOV64_REG(BPF_REG_5, BPF_REG_2), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 14), - BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_5, 0), - - /* Now, test in the other direction. Adding first - * the variable offset to R5, then the constant. - */ - BPF_MOV64_REG(BPF_REG_5, BPF_REG_2), - BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 14), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_5, 0), - - /* Test multiple accumulations of unknown values - * into a packet pointer. - */ - BPF_MOV64_REG(BPF_REG_5, BPF_REG_2), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 14), - BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 4), - BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_5, 0), - - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - /* Calculated offset in R6 has unknown value, but known - * alignment of 4. - */ - {6, "R2", "pkt(r=8)"}, - {7, "R6", "var_off=(0x0; 0x3fc)"}, - /* Offset is added to packet pointer R5, resulting in - * known fixed offset, and variable offset from R6. - */ - {11, "R5", "pkt(id=1,off=14,"}, - /* At the time the word size load is performed from R5, - * it's total offset is NET_IP_ALIGN + reg->off (0) + - * reg->aux_off (14) which is 16. Then the variable - * offset is considered using reg->aux_off_align which - * is 4 and meets the load's requirements. - */ - {15, "R4", "var_off=(0x0; 0x3fc)"}, - {15, "R5", "var_off=(0x0; 0x3fc)"}, - /* Variable offset is added to R5 packet pointer, - * resulting in auxiliary alignment of 4. To avoid BPF - * verifier's precision backtracking logging - * interfering we also have a no-op R4 = R5 - * instruction to validate R5 state. We also check - * that R4 is what it should be in such case. - */ - {18, "R4", "var_off=(0x0; 0x3fc)"}, - {18, "R5", "var_off=(0x0; 0x3fc)"}, - /* Constant offset is added to R5, resulting in - * reg->off of 14. - */ - {19, "R5", "pkt(id=2,off=14,"}, - /* At the time the word size load is performed from R5, - * its total fixed offset is NET_IP_ALIGN + reg->off - * (14) which is 16. Then the variable offset is 4-byte - * aligned, so the total offset is 4-byte aligned and - * meets the load's requirements. - */ - {24, "R4", "var_off=(0x0; 0x3fc)"}, - {24, "R5", "var_off=(0x0; 0x3fc)"}, - /* Constant offset is added to R5 packet pointer, - * resulting in reg->off value of 14. - */ - {26, "R5", "pkt(off=14,r=8)"}, - /* Variable offset is added to R5, resulting in a - * variable offset of (4n). See comment for insn #18 - * for R4 = R5 trick. - */ - {28, "R4", "var_off=(0x0; 0x3fc)"}, - {28, "R5", "var_off=(0x0; 0x3fc)"}, - /* Constant is added to R5 again, setting reg->off to 18. */ - {29, "R5", "pkt(id=3,off=18,"}, - /* And once more we add a variable; resulting var_off - * is still (4n), fixed offset is not changed. - * Also, we create a new reg->id. - */ - {31, "R4", "var_off=(0x0; 0x7fc)"}, - {31, "R5", "var_off=(0x0; 0x7fc)"}, - /* At the time the word size load is performed from R5, - * its total fixed offset is NET_IP_ALIGN + reg->off (18) - * which is 20. Then the variable offset is (4n), so - * the total offset is 4-byte aligned and meets the - * load's requirements. - */ - {35, "R4", "var_off=(0x0; 0x7fc)"}, - {35, "R5", "var_off=(0x0; 0x7fc)"}, - }, - }, - { - .descr = "packet variable offset 2", - .insns = { - /* Create an unknown offset, (4n+2)-aligned */ - LOAD_UNKNOWN(BPF_REG_6), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_6, 2), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, 14), - /* Add it to the packet pointer */ - BPF_MOV64_REG(BPF_REG_5, BPF_REG_2), - BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6), - /* Check bounds and perform a read */ - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_5, 0), - /* Make a (4n) offset from the value we just read */ - BPF_ALU64_IMM(BPF_AND, BPF_REG_6, 0xff), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_6, 2), - /* Add it to the packet pointer */ - BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6), - /* Check bounds and perform a read */ - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_5, 0), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - /* Calculated offset in R6 has unknown value, but known - * alignment of 4. - */ - {6, "R2", "pkt(r=8)"}, - {7, "R6", "var_off=(0x0; 0x3fc)"}, - /* Adding 14 makes R6 be (4n+2) */ - {8, "R6", "var_off=(0x2; 0x7fc)"}, - /* Packet pointer has (4n+2) offset */ - {11, "R5", "var_off=(0x2; 0x7fc)"}, - {12, "R4", "var_off=(0x2; 0x7fc)"}, - /* At the time the word size load is performed from R5, - * its total fixed offset is NET_IP_ALIGN + reg->off (0) - * which is 2. Then the variable offset is (4n+2), so - * the total offset is 4-byte aligned and meets the - * load's requirements. - */ - {15, "R5", "var_off=(0x2; 0x7fc)"}, - /* Newly read value in R6 was shifted left by 2, so has - * known alignment of 4. - */ - {17, "R6", "var_off=(0x0; 0x3fc)"}, - /* Added (4n) to packet pointer's (4n+2) var_off, giving - * another (4n+2). - */ - {19, "R5", "var_off=(0x2; 0xffc)"}, - {20, "R4", "var_off=(0x2; 0xffc)"}, - /* At the time the word size load is performed from R5, - * its total fixed offset is NET_IP_ALIGN + reg->off (0) - * which is 2. Then the variable offset is (4n+2), so - * the total offset is 4-byte aligned and meets the - * load's requirements. - */ - {23, "R5", "var_off=(0x2; 0xffc)"}, - }, - }, - { - .descr = "dubious pointer arithmetic", - .insns = { - PREP_PKT_POINTERS, - BPF_MOV64_IMM(BPF_REG_0, 0), - /* (ptr - ptr) << 2 */ - BPF_MOV64_REG(BPF_REG_5, BPF_REG_3), - BPF_ALU64_REG(BPF_SUB, BPF_REG_5, BPF_REG_2), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_5, 2), - /* We have a (4n) value. Let's make a packet offset - * out of it. First add 14, to make it a (4n+2) - */ - BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 14), - /* Then make sure it's nonnegative */ - BPF_JMP_IMM(BPF_JSGE, BPF_REG_5, 0, 1), - BPF_EXIT_INSN(), - /* Add it to packet pointer */ - BPF_MOV64_REG(BPF_REG_6, BPF_REG_2), - BPF_ALU64_REG(BPF_ADD, BPF_REG_6, BPF_REG_5), - /* Check bounds and perform a read */ - BPF_MOV64_REG(BPF_REG_4, BPF_REG_6), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_6, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .result = REJECT, - .matches = { - {3, "R5", "pkt_end()"}, - /* (ptr - ptr) << 2 == unknown, (4n) */ - {5, "R5", "var_off=(0x0; 0xfffffffffffffffc)"}, - /* (4n) + 14 == (4n+2). We blow our bounds, because - * the add could overflow. - */ - {6, "R5", "var_off=(0x2; 0xfffffffffffffffc)"}, - /* Checked s>=0 */ - {9, "R5", "var_off=(0x2; 0x7ffffffffffffffc)"}, - /* packet pointer + nonnegative (4n+2) */ - {11, "R6", "var_off=(0x2; 0x7ffffffffffffffc)"}, - {12, "R4", "var_off=(0x2; 0x7ffffffffffffffc)"}, - /* NET_IP_ALIGN + (4n+2) == (4n), alignment is fine. - * We checked the bounds, but it might have been able - * to overflow if the packet pointer started in the - * upper half of the address space. - * So we did not get a 'range' on R6, and the access - * attempt will fail. - */ - {15, "R6", "var_off=(0x2; 0x7ffffffffffffffc)"}, - } - }, - { - .descr = "variable subtraction", - .insns = { - /* Create an unknown offset, (4n+2)-aligned */ - LOAD_UNKNOWN(BPF_REG_6), - BPF_MOV64_REG(BPF_REG_7, BPF_REG_6), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_6, 2), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, 14), - /* Create another unknown, (4n)-aligned, and subtract - * it from the first one - */ - BPF_ALU64_IMM(BPF_LSH, BPF_REG_7, 2), - BPF_ALU64_REG(BPF_SUB, BPF_REG_6, BPF_REG_7), - /* Bounds-check the result */ - BPF_JMP_IMM(BPF_JSGE, BPF_REG_6, 0, 1), - BPF_EXIT_INSN(), - /* Add it to the packet pointer */ - BPF_MOV64_REG(BPF_REG_5, BPF_REG_2), - BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6), - /* Check bounds and perform a read */ - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_5, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - /* Calculated offset in R6 has unknown value, but known - * alignment of 4. - */ - {6, "R2", "pkt(r=8)"}, - {8, "R6", "var_off=(0x0; 0x3fc)"}, - /* Adding 14 makes R6 be (4n+2) */ - {9, "R6", "var_off=(0x2; 0x7fc)"}, - /* New unknown value in R7 is (4n) */ - {10, "R7", "var_off=(0x0; 0x3fc)"}, - /* Subtracting it from R6 blows our unsigned bounds */ - {11, "R6", "var_off=(0x2; 0xfffffffffffffffc)"}, - /* Checked s>= 0 */ - {14, "R6", "var_off=(0x2; 0x7fc)"}, - /* At the time the word size load is performed from R5, - * its total fixed offset is NET_IP_ALIGN + reg->off (0) - * which is 2. Then the variable offset is (4n+2), so - * the total offset is 4-byte aligned and meets the - * load's requirements. - */ - {20, "R5", "var_off=(0x2; 0x7fc)"}, - }, - }, - { - .descr = "pointer variable subtraction", - .insns = { - /* Create an unknown offset, (4n+2)-aligned and bounded - * to [14,74] - */ - LOAD_UNKNOWN(BPF_REG_6), - BPF_MOV64_REG(BPF_REG_7, BPF_REG_6), - BPF_ALU64_IMM(BPF_AND, BPF_REG_6, 0xf), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_6, 2), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, 14), - /* Subtract it from the packet pointer */ - BPF_MOV64_REG(BPF_REG_5, BPF_REG_2), - BPF_ALU64_REG(BPF_SUB, BPF_REG_5, BPF_REG_6), - /* Create another unknown, (4n)-aligned and >= 74. - * That in fact means >= 76, since 74 % 4 == 2 - */ - BPF_ALU64_IMM(BPF_LSH, BPF_REG_7, 2), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_7, 76), - /* Add it to the packet pointer */ - BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_7), - /* Check bounds and perform a read */ - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_5, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - /* Calculated offset in R6 has unknown value, but known - * alignment of 4. - */ - {6, "R2", "pkt(r=8)"}, - {9, "R6", "var_off=(0x0; 0x3c)"}, - /* Adding 14 makes R6 be (4n+2) */ - {10, "R6", "var_off=(0x2; 0x7c)"}, - /* Subtracting from packet pointer overflows ubounds */ - {13, "R5", "var_off=(0xffffffffffffff82; 0x7c)"}, - /* New unknown value in R7 is (4n), >= 76 */ - {14, "R7", "var_off=(0x0; 0x7fc)"}, - /* Adding it to packet pointer gives nice bounds again */ - {16, "R5", "var_off=(0x2; 0x7fc)"}, - /* At the time the word size load is performed from R5, - * its total fixed offset is NET_IP_ALIGN + reg->off (0) - * which is 2. Then the variable offset is (4n+2), so - * the total offset is 4-byte aligned and meets the - * load's requirements. - */ - {20, "R5", "var_off=(0x2; 0x7fc)"}, - }, - }, -}; - -static int probe_filter_length(const struct bpf_insn *fp) -{ - int len; - - for (len = MAX_INSNS - 1; len > 0; --len) - if (fp[len].code != 0 || fp[len].imm != 0) - break; - return len + 1; -} - -static char bpf_vlog[32768]; - -static int do_test_single(struct bpf_align_test *test) -{ - struct bpf_insn *prog = test->insns; - int prog_type = test->prog_type; - char bpf_vlog_copy[32768]; - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .prog_flags = BPF_F_STRICT_ALIGNMENT, - .log_buf = bpf_vlog, - .log_size = sizeof(bpf_vlog), - .log_level = 2, - ); - const char *main_pass_start = "0: R1=ctx() R10=fp0"; - const char *line_ptr; - int cur_line = -1; - int prog_len, i; - char *start; - int fd_prog; - int ret; - - prog_len = probe_filter_length(prog); - fd_prog = bpf_prog_load(prog_type ? : BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", - prog, prog_len, &opts); - if (fd_prog < 0 && test->result != REJECT) { - printf("Failed to load program.\n"); - printf("%s", bpf_vlog); - ret = 1; - } else if (fd_prog >= 0 && test->result == REJECT) { - printf("Unexpected success to load!\n"); - printf("%s", bpf_vlog); - ret = 1; - close(fd_prog); - } else { - ret = 0; - /* We make a local copy so that we can strtok() it */ - strncpy(bpf_vlog_copy, bpf_vlog, sizeof(bpf_vlog_copy)); - start = strstr(bpf_vlog_copy, main_pass_start); - if (!start) { - ret = 1; - printf("Can't find initial line '%s'\n", main_pass_start); - goto out; - } - line_ptr = strtok(start, "\n"); - for (i = 0; i < MAX_MATCHES; i++) { - struct bpf_reg_match m = test->matches[i]; - const char *p; - int tmp; - - if (!m.match) - break; - while (line_ptr) { - cur_line = -1; - sscanf(line_ptr, "%u: ", &cur_line); - if (cur_line == -1) - sscanf(line_ptr, "from %u to %u: ", &tmp, &cur_line); - if (cur_line == m.line) - break; - line_ptr = strtok(NULL, "\n"); - } - if (!line_ptr) { - printf("Failed to find line %u for match: %s=%s\n", - m.line, m.reg, m.match); - ret = 1; - printf("%s", bpf_vlog); - break; - } - /* Check the next line as well in case the previous line - * did not have a corresponding bpf insn. Example: - * func#0 @0 - * 0: R1=ctx() R10=fp0 - * 0: (b7) r3 = 2 ; R3_w=2 - * - * Sometimes it's actually two lines below, e.g. when - * searching for "6: R3_w=scalar(umax=255,var_off=(0x0; 0xff))": - * from 4 to 6: R0_w=pkt(off=8,r=8) R1=ctx() R2_w=pkt(r=8) R3_w=pkt_end() R10=fp0 - * 6: R0_w=pkt(off=8,r=8) R1=ctx() R2_w=pkt(r=8) R3_w=pkt_end() R10=fp0 - * 6: (71) r3 = *(u8 *)(r2 +0) ; R2_w=pkt(r=8) R3_w=scalar(umax=255,var_off=(0x0; 0xff)) - */ - while (!(p = strstr(line_ptr, m.reg)) || !strstr(p, m.match)) { - cur_line = -1; - line_ptr = strtok(NULL, "\n"); - sscanf(line_ptr ?: "", "%u: ", &cur_line); - if (!line_ptr || cur_line != m.line) - break; - } - if (cur_line != m.line || !line_ptr || !(p = strstr(line_ptr, m.reg)) || !strstr(p, m.match)) { - printf("Failed to find match %u: %s=%s\n", m.line, m.reg, m.match); - ret = 1; - printf("%s", bpf_vlog); - break; - } - } -out: - if (fd_prog >= 0) - close(fd_prog); - } - return ret; -} - -void test_align(void) -{ - unsigned int i; - - for (i = 0; i < ARRAY_SIZE(tests); i++) { - struct bpf_align_test *test = &tests[i]; - - if (!test__start_subtest(test->descr)) - continue; - - ASSERT_OK(do_test_single(test), test->descr); - } -} diff --git a/tools/testing/selftests/bpf/prog_tests/arena_atomics.c b/tools/testing/selftests/bpf/prog_tests/arena_atomics.c index d98577a6babc..1ad5d03d07ad 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_atomics.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_atomics.c @@ -222,7 +222,7 @@ static void test_store_release(struct arena_atomics *skel) "store_release64_result"); } -void test_arena_atomics(void) +void serial_test_arena_atomics(void) { struct arena_atomics *skel; int err; diff --git a/tools/testing/selftests/bpf/prog_tests/arena_direct_value.c b/tools/testing/selftests/bpf/prog_tests/arena_direct_value.c new file mode 100644 index 000000000000..01fcf4965ea4 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_direct_value.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include <bpf/bpf.h> +#include <errno.h> +#include <sys/mman.h> +#include <unistd.h> + +#define ARENA_PAGES 32 + +static char log_buf[16384]; + +static void test_arena_direct_value_one_past_end(void) +{ + char expected[128]; + __u32 arena_sz = ARENA_PAGES * getpagesize(); + struct bpf_insn insns[] = { + BPF_LD_IMM64_RAW(BPF_REG_1, BPF_PSEUDO_MAP_VALUE, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_map_create_opts, map_opts); + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts); + void *arena; + int map_fd, prog_fd; + + map_opts.map_flags = BPF_F_MMAPABLE; + prog_opts.log_buf = log_buf; + prog_opts.log_size = sizeof(log_buf); + prog_opts.log_level = 1; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARENA, "arena_direct_value", + 0, 0, ARENA_PAGES, &map_opts); + if (map_fd < 0) { + if (errno == EOPNOTSUPP) { + test__skip(); + return; + } + ASSERT_GE(map_fd, 0, "bpf_map_create"); + return; + } + + arena = mmap(NULL, arena_sz, PROT_READ | PROT_WRITE, MAP_SHARED, map_fd, 0); + if (!ASSERT_NEQ(arena, MAP_FAILED, "arena_mmap")) + goto cleanup; + + insns[0].imm = map_fd; + insns[1].imm = arena_sz; + + prog_fd = bpf_prog_load(BPF_PROG_TYPE_RAW_TRACEPOINT, + "arena_direct_value", "GPL", insns, + ARRAY_SIZE(insns), &prog_opts); + if (!ASSERT_LT(prog_fd, 0, "prog_load")) { + close(prog_fd); + goto cleanup; + } + + snprintf(expected, sizeof(expected), + "invalid access to map value pointer, value_size=0 off=%u", + arena_sz); + ASSERT_HAS_SUBSTR(log_buf, expected, "verifier_log"); + +cleanup: + if (arena != MAP_FAILED) + munmap(arena, arena_sz); + close(map_fd); +} + +void serial_test_arena_direct_value(void) +{ + if (test__start_subtest("one_past_end")) + test_arena_direct_value_one_past_end(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/arena_htab.c b/tools/testing/selftests/bpf/prog_tests/arena_htab.c index d69fd2465f53..91ccf0402980 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_htab.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_htab.c @@ -81,7 +81,7 @@ static void test_arena_htab_asm(void) arena_htab_asm__destroy(skel); } -void test_arena_htab(void) +void serial_test_arena_htab(void) { if (test__start_subtest("arena_htab_llvm")) test_arena_htab_llvm(); diff --git a/tools/testing/selftests/bpf/prog_tests/arena_list.c b/tools/testing/selftests/bpf/prog_tests/arena_list.c index 4f2866a615ce..2648e06f53d0 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_list.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_list.c @@ -68,7 +68,7 @@ out: arena_list__destroy(skel); } -void test_arena_list(void) +void serial_test_arena_list(void) { if (test__start_subtest("arena_list_1")) test_arena_list_add_del(1, false); diff --git a/tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c b/tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c new file mode 100644 index 000000000000..14c2d1a1d673 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include <sys/user.h> +#ifndef PAGE_SIZE /* on some archs it comes in sys/user.h */ +#include <unistd.h> +#define PAGE_SIZE getpagesize() +#endif + +#include "arena_mem_usage.skel.h" + +/* + * arena_map_mem_usage() is surfaced to user space through the map's + * /proc/<pid>/fdinfo/<fd> "memlock:" line (the same value bpftool map show + * prints). Read it directly so the test has no external dependency. + */ +static long map_memlock(int map_fd) +{ + char path[64], line[128]; + long memlock = -1; + FILE *f; + + snprintf(path, sizeof(path), "/proc/self/fdinfo/%d", map_fd); + f = fopen(path, "r"); + if (!ASSERT_OK_PTR(f, "open_fdinfo")) + return -1; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "memlock:\t%ld", &memlock) == 1) + break; + } + fclose(f); + ASSERT_NEQ(memlock, -1, "parse_memlock"); + return memlock; +} + +static int run(struct bpf_program *prog, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + + if (!ASSERT_OK(err, name)) + return -1; + if (!ASSERT_OK(opts.retval, name)) + return -1; + return 0; +} + +void serial_test_arena_mem_usage(void) +{ + struct arena_mem_usage *skel; + const long ps = PAGE_SIZE; + char *base; + size_t sz; + int fd, i; + + skel = arena_mem_usage__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_load")) + return; + fd = bpf_map__fd(skel->maps.arena); + + /* Fresh arena: no data pages, and the scratch page is not counted. */ + ASSERT_EQ(map_memlock(fd), 0, "initial"); + + /* BPF-side allocation of 17 pages. */ + skel->bss->alloc_cnt = 17; + if (run(skel->progs.alloc, "alloc")) + goto out; + /* + * A NULL ptr means bpf_arena_alloc_pages() itself failed (e.g. the host + * is under memory pressure), not a miscount -- flag it distinctly so a + * red CI run is not mistaken for a counting bug. + */ + if (!ASSERT_OK_PTR(skel->bss->ptr, "arena_alloc_pages")) + goto out; + ASSERT_EQ(map_memlock(fd), 17 * ps, "after_alloc"); + + /* Free a single page (arena_free_pages page_cnt==1 path). */ + skel->bss->free_byte_off = 0; + skel->bss->free_cnt = 1; + if (run(skel->progs.free_pages, "free_one")) + goto out; + ASSERT_EQ(map_memlock(fd), 16 * ps, "after_free_one"); + + /* Free ten pages in one call (bulk path); only the freed pages count. */ + skel->bss->free_byte_off = 1 * ps; + skel->bss->free_cnt = 10; + if (run(skel->progs.free_pages, "free_bulk")) + goto out; + ASSERT_EQ(map_memlock(fd), 6 * ps, "after_free_bulk"); + + /* Free the remaining six -> arena empty again. */ + skel->bss->free_byte_off = 11 * ps; + skel->bss->free_cnt = 6; + if (run(skel->progs.free_pages, "free_rest")) + goto out; + ASSERT_EQ(map_memlock(fd), 0, "after_free_rest"); + + /* + * User-space fault-in: touching unallocated arena pages allocates them + * through arena_vm_fault(). libbpf mmap()s the arena at map_extra during + * load, so bpf_map__initial_value() hands back that base. + */ + base = bpf_map__initial_value(skel->maps.arena, &sz); + if (!ASSERT_OK_PTR(base, "arena_base")) + goto out; + for (i = 0; i < 8; i++) + base[i * ps] = 1; + ASSERT_EQ(map_memlock(fd), 8 * ps, "after_faultin"); + + /* + * Free the faulted-in pages from BPF. They are mapped into the user vma + * (elevated refcount), so this also exercises the zap path. + */ + skel->bss->ptr = base; + skel->bss->free_byte_off = 0; + skel->bss->free_cnt = 8; + if (run(skel->progs.free_pages, "free_faulted")) + goto out; + ASSERT_EQ(map_memlock(fd), 0, "after_free_faulted"); +out: + arena_mem_usage__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c b/tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c index 693fd86fbde6..545b05d7a0aa 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c @@ -5,13 +5,6 @@ #include <sys/sysinfo.h> struct __qspinlock { int val; }; -typedef struct __qspinlock arena_spinlock_t; - -struct arena_qnode { - unsigned long next; - int count; - int locked; -}; #include "arena_spin_lock.skel.h" @@ -108,7 +101,7 @@ end: return; } -void test_arena_spin_lock(void) +void serial_test_arena_spin_lock(void) { repeat = 1000; if (test__start_subtest("arena_spin_lock_1")) diff --git a/tools/testing/selftests/bpf/prog_tests/arena_strsearch.c b/tools/testing/selftests/bpf/prog_tests/arena_strsearch.c index f81a0c066505..0c1c6cbfa0f1 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_strsearch.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_strsearch.c @@ -23,7 +23,7 @@ static void test_arena_str(void) arena_strsearch__destroy(skel); } -void test_arena_strsearch(void) +void serial_test_arena_strsearch(void) { if (test__start_subtest("arena_strsearch")) test_arena_str(); diff --git a/tools/testing/selftests/bpf/prog_tests/attach_probe.c b/tools/testing/selftests/bpf/prog_tests/attach_probe.c index 9e77e5da7097..e8c1a619e330 100644 --- a/tools/testing/selftests/bpf/prog_tests/attach_probe.c +++ b/tools/testing/selftests/bpf/prog_tests/attach_probe.c @@ -123,6 +123,140 @@ cleanup: test_attach_probe_manual__destroy(skel); } +/* manual attach address-based kprobe/kretprobe testings */ +static void test_attach_kprobe_by_addr(enum probe_attach_mode attach_mode) +{ + LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts); + struct test_attach_probe_manual *skel; + unsigned long func_addr; + + if (!ASSERT_OK(load_kallsyms(), "load_kallsyms")) + return; + + func_addr = ksym_get_addr(SYS_NANOSLEEP_KPROBE_NAME); + if (!ASSERT_NEQ(func_addr, 0UL, "func_addr")) + return; + + skel = test_attach_probe_manual__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_kprobe_manual_open_and_load")) + return; + + kprobe_opts.attach_mode = attach_mode; + kprobe_opts.retprobe = false; + kprobe_opts.offset = func_addr; + skel->links.handle_kprobe = + bpf_program__attach_kprobe_opts(skel->progs.handle_kprobe, + NULL, &kprobe_opts); + if (!ASSERT_OK_PTR(skel->links.handle_kprobe, "attach_kprobe_by_addr")) + goto cleanup; + + kprobe_opts.retprobe = true; + skel->links.handle_kretprobe = + bpf_program__attach_kprobe_opts(skel->progs.handle_kretprobe, + NULL, &kprobe_opts); + if (!ASSERT_OK_PTR(skel->links.handle_kretprobe, "attach_kretprobe_by_addr")) + goto cleanup; + + /* trigger & validate kprobe && kretprobe */ + usleep(1); + + ASSERT_EQ(skel->bss->kprobe_res, 1, "check_kprobe_res"); + ASSERT_EQ(skel->bss->kretprobe_res, 2, "check_kretprobe_res"); + +cleanup: + test_attach_probe_manual__destroy(skel); +} + +/* reject legacy address-based kprobe attach */ +static void test_attach_kprobe_legacy_by_addr_reject(void) +{ + LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts); + struct test_attach_probe_manual *skel; + unsigned long func_addr; + + if (!ASSERT_OK(load_kallsyms(), "load_kallsyms")) + return; + + func_addr = ksym_get_addr(SYS_NANOSLEEP_KPROBE_NAME); + if (!ASSERT_NEQ(func_addr, 0UL, "func_addr")) + return; + + skel = test_attach_probe_manual__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_kprobe_manual_open_and_load")) + return; + + kprobe_opts.attach_mode = PROBE_ATTACH_MODE_LEGACY; + kprobe_opts.offset = func_addr; + skel->links.handle_kprobe = + bpf_program__attach_kprobe_opts(skel->progs.handle_kprobe, + NULL, &kprobe_opts); + ASSERT_ERR_PTR(skel->links.handle_kprobe, "attach_kprobe_legacy_by_addr"); + ASSERT_EQ(libbpf_get_error(skel->links.handle_kprobe), + -EOPNOTSUPP, "attach_kprobe_legacy_by_addr_err"); + + test_attach_probe_manual__destroy(skel); +} + +/* + * bpf_fentry_shadow_test exists in both vmlinux (net/bpf/test_run.c) and + * bpf_testmod (bpf_testmod.c). When bpf_testmod is loaded the symbol is + * duplicated. Test that kprobe attachment handles this correctly: + * - Unqualified name ("bpf_fentry_shadow_test") attaches to vmlinux. + * - MOD:SYM name ("bpf_testmod:bpf_fentry_shadow_test") attaches to module. + * + * Note: bpf_fentry_shadow_test is not invoked via test_run, so we only + * verify that attach and detach succeed without triggering the probe. + */ +static void test_attach_probe_dup_sym(enum probe_attach_mode attach_mode) +{ + DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts); + struct bpf_link *kprobe_link, *kretprobe_link; + struct test_attach_probe_manual *skel; + + skel = test_attach_probe_manual__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_dup_sym_open_and_load")) + return; + + kprobe_opts.attach_mode = attach_mode; + + /* Unqualified: should attach to vmlinux symbol */ + kprobe_opts.retprobe = false; + kprobe_link = bpf_program__attach_kprobe_opts(skel->progs.handle_kprobe, + "bpf_fentry_shadow_test", + &kprobe_opts); + if (!ASSERT_OK_PTR(kprobe_link, "attach_kprobe_vmlinux")) + goto cleanup; + bpf_link__destroy(kprobe_link); + + kprobe_opts.retprobe = true; + kretprobe_link = bpf_program__attach_kprobe_opts(skel->progs.handle_kretprobe, + "bpf_fentry_shadow_test", + &kprobe_opts); + if (!ASSERT_OK_PTR(kretprobe_link, "attach_kretprobe_vmlinux")) + goto cleanup; + bpf_link__destroy(kretprobe_link); + + /* MOD:SYM qualified: should attach to module symbol */ + kprobe_opts.retprobe = false; + kprobe_link = bpf_program__attach_kprobe_opts(skel->progs.handle_kprobe, + "bpf_testmod:bpf_fentry_shadow_test", + &kprobe_opts); + if (!ASSERT_OK_PTR(kprobe_link, "attach_kprobe_module")) + goto cleanup; + bpf_link__destroy(kprobe_link); + + kprobe_opts.retprobe = true; + kretprobe_link = bpf_program__attach_kprobe_opts(skel->progs.handle_kretprobe, + "bpf_testmod:bpf_fentry_shadow_test", + &kprobe_opts); + if (!ASSERT_OK_PTR(kretprobe_link, "attach_kretprobe_module")) + goto cleanup; + bpf_link__destroy(kretprobe_link); + +cleanup: + test_attach_probe_manual__destroy(skel); +} + /* attach uprobe/uretprobe long event name testings */ static void test_attach_uprobe_long_event_name(void) { @@ -220,11 +354,73 @@ static void test_attach_kprobe_write_ctx(void) kprobe_write_ctx__destroy(skel); } + +static void test_freplace_kprobe_write_ctx(void) +{ + struct bpf_program *prog_kprobe, *prog_ext, *prog_fentry; + struct kprobe_write_ctx *skel_kprobe, *skel_ext = NULL; + struct bpf_link *link_kprobe = NULL, *link_ext = NULL; + int err, prog_fd; + LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts); + LIBBPF_OPTS(bpf_test_run_opts, topts); + + skel_kprobe = kprobe_write_ctx__open(); + if (!ASSERT_OK_PTR(skel_kprobe, "kprobe_write_ctx__open kprobe")) + return; + + prog_kprobe = skel_kprobe->progs.kprobe_dummy; + bpf_program__set_autoload(prog_kprobe, true); + + prog_fentry = skel_kprobe->progs.fentry; + bpf_program__set_autoload(prog_fentry, true); + + err = kprobe_write_ctx__load(skel_kprobe); + if (!ASSERT_OK(err, "kprobe_write_ctx__load kprobe")) + goto out; + + skel_ext = kprobe_write_ctx__open(); + if (!ASSERT_OK_PTR(skel_ext, "kprobe_write_ctx__open ext")) + goto out; + + prog_ext = skel_ext->progs.freplace_kprobe; + bpf_program__set_autoload(prog_ext, true); + + prog_fd = bpf_program__fd(skel_kprobe->progs.kprobe_write_ctx); + bpf_program__set_attach_target(prog_ext, prog_fd, "kprobe_write_ctx"); + + err = kprobe_write_ctx__load(skel_ext); + if (!ASSERT_OK(err, "kprobe_write_ctx__load ext")) + goto out; + + prog_fd = bpf_program__fd(prog_kprobe); + link_ext = bpf_program__attach_freplace(prog_ext, prog_fd, "kprobe_dummy"); + ASSERT_ERR_PTR(link_ext, "bpf_program__attach_freplace link"); + ASSERT_EQ(libbpf_get_error(link_ext), -EINVAL, "bpf_program__attach_freplace error"); + + link_kprobe = bpf_program__attach_kprobe_opts(prog_kprobe, "bpf_fentry_test1", + &kprobe_opts); + if (!ASSERT_OK_PTR(link_kprobe, "bpf_program__attach_kprobe_opts")) + goto out; + + err = bpf_prog_test_run_opts(bpf_program__fd(prog_fentry), &topts); + ASSERT_OK(err, "bpf_prog_test_run_opts"); + +out: + bpf_link__destroy(link_ext); + bpf_link__destroy(link_kprobe); + kprobe_write_ctx__destroy(skel_ext); + kprobe_write_ctx__destroy(skel_kprobe); +} #else static void test_attach_kprobe_write_ctx(void) { test__skip(); } + +static void test_freplace_kprobe_write_ctx(void) +{ + test__skip(); +} #endif static void test_attach_probe_auto(struct test_attach_probe *skel) @@ -416,6 +612,21 @@ void test_attach_probe(void) test_attach_probe_manual(PROBE_ATTACH_MODE_PERF); if (test__start_subtest("manual-link")) test_attach_probe_manual(PROBE_ATTACH_MODE_LINK); + if (test__start_subtest("kprobe-perf-by-addr")) + test_attach_kprobe_by_addr(PROBE_ATTACH_MODE_PERF); + if (test__start_subtest("kprobe-link-by-addr")) + test_attach_kprobe_by_addr(PROBE_ATTACH_MODE_LINK); + if (test__start_subtest("kprobe-legacy-by-addr-reject")) + test_attach_kprobe_legacy_by_addr_reject(); + + if (test__start_subtest("dup-sym-default")) + test_attach_probe_dup_sym(PROBE_ATTACH_MODE_DEFAULT); + if (test__start_subtest("dup-sym-legacy")) + test_attach_probe_dup_sym(PROBE_ATTACH_MODE_LEGACY); + if (test__start_subtest("dup-sym-perf")) + test_attach_probe_dup_sym(PROBE_ATTACH_MODE_PERF); + if (test__start_subtest("dup-sym-link")) + test_attach_probe_dup_sym(PROBE_ATTACH_MODE_LINK); if (test__start_subtest("auto")) test_attach_probe_auto(skel); @@ -434,6 +645,8 @@ void test_attach_probe(void) test_attach_kprobe_long_event_name(); if (test__start_subtest("kprobe-write-ctx")) test_attach_kprobe_write_ctx(); + if (test__start_subtest("freplace-kprobe-write-ctx")) + test_freplace_kprobe_write_ctx(); cleanup: test_attach_probe__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_attr_size.c b/tools/testing/selftests/bpf/prog_tests/bpf_attr_size.c new file mode 100644 index 000000000000..87842c4347a6 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpf_attr_size.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Google LLC */ +#include <linux/bpf.h> +#include <unistd.h> +#include <sys/syscall.h> +#include <test_progs.h> +#include <cgroup_helpers.h> +#include "cgroup_skb_direct_packet_access.skel.h" + +#define OLD_QUERY_SIZE offsetofend(union bpf_attr, query.prog_cnt) +#define FULL_QUERY_SIZE offsetofend(union bpf_attr, query.revision) + +static void test_query_size_boundaries(void) +{ + struct cgroup_skb_direct_packet_access *skel; + struct bpf_link *link = NULL; + union bpf_attr attr; + int cg_fd = -1; + int err; + + skel = cgroup_skb_direct_packet_access__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + return; + + cg_fd = test__join_cgroup("/attr_size_cg"); + if (!ASSERT_GE(cg_fd, 0, "join_cgroup")) + goto cleanup; + + link = bpf_program__attach_cgroup(skel->progs.direct_packet_access, + cg_fd); + if (!ASSERT_OK_PTR(link, "cg_attach")) + goto cleanup; + + memset(&attr, 0, sizeof(attr)); + attr.query.target_fd = cg_fd; + attr.query.attach_type = BPF_CGROUP_INET_INGRESS; + attr.query.revision = 0xdeadbeefdeadbeefULL; + + err = syscall(__NR_bpf, BPF_PROG_QUERY, &attr, OLD_QUERY_SIZE); + if (ASSERT_OK(err, "query_old_size")) { + ASSERT_EQ(attr.query.prog_cnt, 1, "prog_cnt_written_old"); + ASSERT_EQ(attr.query.revision, 0xdeadbeefdeadbeefULL, + "revision_not_written_old"); + } + + memset(&attr, 0, sizeof(attr)); + attr.query.target_fd = cg_fd; + attr.query.attach_type = BPF_CGROUP_INET_INGRESS; + + err = syscall(__NR_bpf, BPF_PROG_QUERY, &attr, FULL_QUERY_SIZE); + if (!ASSERT_OK(err, "query_full_size")) + goto cleanup; + + ASSERT_EQ(attr.query.prog_cnt, 1, "prog_cnt_written"); + ASSERT_GT(attr.query.revision, 0, "revision_written"); + +cleanup: + if (link) + bpf_link__destroy(link); + if (cg_fd >= 0) + close(cg_fd); + cgroup_skb_direct_packet_access__destroy(skel); +} + +static void test_map_info_tail_zero(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, map_opts); + struct bpf_map_info_fake { + __u8 info[offsetofend(struct bpf_map_info, hash_size)]; + __u32 pad; + } info = { + .pad = 1, + }; + int map_fd, err; + __u32 info_len; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "arr", sizeof(int), 1, 1, &map_opts); + if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) + return; + + info_len = sizeof(info); + err = bpf_obj_get_info_by_fd(map_fd, &info, &info_len); + ASSERT_EQ(err, -E2BIG, "bpf_obj_get_info_by_fd"); + + close(map_fd); +} + +static void test_prog_info_tail_zero(void) +{ + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts); + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + struct bpf_prog_info_fake { + __u8 info[offsetofend(struct bpf_prog_info, attach_btf_id)]; + __u32 pad; + } info = { + .pad = 1, + }; + int prog_fd, err; + __u32 info_len; + + prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "test_prog", "GPL", insns, + ARRAY_SIZE(insns), &prog_opts); + if (!ASSERT_GE(prog_fd, 0, "bpf_prog_load")) + return; + + info_len = sizeof(info); + err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); + ASSERT_EQ(err, -E2BIG, "bpf_obj_get_info_by_fd"); + + close(prog_fd); +} + +void test_bpf_attr_size(void) +{ + if (test__start_subtest("query_size_boundaries")) + test_query_size_boundaries(); + if (test__start_subtest("map_info_tail_zero")) + test_map_info_tail_zero(); + if (test__start_subtest("prog_info_tail_zero")) + test_prog_info_tail_zero(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_cookie.c b/tools/testing/selftests/bpf/prog_tests/bpf_cookie.c index 75f4dff7d042..fa484d00a7a5 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_cookie.c @@ -6,6 +6,7 @@ #include <sys/syscall.h> #include <sys/mman.h> #include <unistd.h> +#include <linux/compiler.h> #include <test_progs.h> #include <network_helpers.h> #include <bpf/btf.h> @@ -105,6 +106,11 @@ static void kprobe_multi_link_api_subtest(void) unsigned long long addrs[8]; __u64 cookies[8]; + if (!env.has_testmod) { + test__skip(); + return; + } + if (!ASSERT_OK(load_kallsyms(), "load_kallsyms")) goto cleanup; @@ -192,6 +198,11 @@ static void kprobe_multi_attach_api_subtest(void) }; __u64 cookies[8]; + if (!env.has_testmod) { + test__skip(); + return; + } + skel = kprobe_multi__open_and_load(); if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load")) goto cleanup; @@ -241,10 +252,17 @@ cleanup: kprobe_multi__destroy(skel); } -/* defined in prog_tests/uprobe_multi_test.c */ -void uprobe_multi_func_1(void); -void uprobe_multi_func_2(void); -void uprobe_multi_func_3(void); +/* + * Weak uprobe target stubs. noinline is required because + * uprobe_multi_test_run() takes their addresses to configure the BPF + * program's attachment points; an inlined function has no stable + * address in the binary to probe. The strong definitions in + * uprobe_multi_test.c take precedence when that translation unit is + * linked. + */ +noinline __weak void uprobe_multi_func_1(void) { asm volatile (""); } +noinline __weak void uprobe_multi_func_2(void) { asm volatile (""); } +noinline __weak void uprobe_multi_func_3(void) { asm volatile (""); } static void uprobe_multi_test_run(struct uprobe_multi *skel) { @@ -421,11 +439,12 @@ cleanup: bpf_link__destroy(link3); } -static void burn_cpu(void) +static void burn_cpu(long loops) { - volatile int j = 0; + long j = 0; cpu_set_t cpu_set; - int i, err; + long i; + int err; /* generate some branches on cpu 0 */ CPU_ZERO(&cpu_set); @@ -433,9 +452,10 @@ static void burn_cpu(void) err = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set), &cpu_set); ASSERT_OK(err, "set_thread_affinity"); - /* spin the loop for a while (random high number) */ - for (i = 0; i < 1000000; ++i) + for (i = 0; i < loops; ++i) { ++j; + barrier(); + } } static void pe_subtest(struct test_bpf_cookie *skel) @@ -451,7 +471,7 @@ static void pe_subtest(struct test_bpf_cookie *skel) attr.type = PERF_TYPE_SOFTWARE; attr.config = PERF_COUNT_SW_CPU_CLOCK; attr.sample_period = 100000; - pfd = syscall(__NR_perf_event_open, &attr, -1, 0, -1, PERF_FLAG_FD_CLOEXEC); + pfd = syscall(__NR_perf_event_open, &attr, 0, -1, -1, PERF_FLAG_FD_CLOEXEC); if (!ASSERT_GE(pfd, 0, "perf_fd")) goto cleanup; @@ -460,7 +480,7 @@ static void pe_subtest(struct test_bpf_cookie *skel) if (!ASSERT_OK_PTR(link, "link1")) goto cleanup; - burn_cpu(); /* trigger BPF prog */ + burn_cpu(100000000L); /* trigger BPF prog */ ASSERT_EQ(skel->bss->pe_res, 0x100000, "pe_res1"); @@ -479,7 +499,7 @@ static void pe_subtest(struct test_bpf_cookie *skel) if (!ASSERT_OK_PTR(link, "link2")) goto cleanup; - burn_cpu(); /* trigger BPF prog */ + burn_cpu(100000000L); /* trigger BPF prog */ ASSERT_EQ(skel->bss->pe_res, 0x200000, "pe_res2"); @@ -561,8 +581,6 @@ cleanup: close(fmod_ret_fd); } -int stack_mprotect(void); - static void lsm_subtest(struct test_bpf_cookie *skel) { __u64 cookie; diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_gotox.c b/tools/testing/selftests/bpf/prog_tests/bpf_gotox.c index 75b0cf2467ab..73dc63882b7d 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_gotox.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_gotox.c @@ -317,7 +317,7 @@ static void check_ldimm64_off_load(struct bpf_gotox *skel __always_unused) static int __check_ldimm64_gotox_prog_load(struct bpf_insn *insns, __u32 insn_cnt, - __u32 off1, __u32 off2) + int off1, int off2, int off3) { const __u32 values[] = {5, 7, 9, 11, 13, 15}; const __u32 max_entries = ARRAY_SIZE(values); @@ -349,16 +349,46 @@ static int __check_ldimm64_gotox_prog_load(struct bpf_insn *insns, /* r1 += off2 */ insns[2].imm = off2; + /* r1 = *(r1 + off3) */ + insns[3].off = off3; + ret = prog_load(insns, insn_cnt); close(map_fd); return ret; } -static void reject_offsets(struct bpf_insn *insns, __u32 insn_cnt, __u32 off1, __u32 off2) +static void +allow_offsets(struct bpf_insn *insns, __u32 insn_cnt, int off1, int off2, int off3) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + int prog_fd, err; + char s[128] = ""; + + prog_fd = __check_ldimm64_gotox_prog_load(insns, insn_cnt, off1, off2, off3); + snprintf(s, sizeof(s), "__check_ldimm64_gotox_prog_load(%d,%d,%d)", off1, off2, off3); + if (!ASSERT_GE(prog_fd, 0, s)) + return; + + err = bpf_prog_test_run_opts(prog_fd, &topts); + if (!ASSERT_OK(err, "test_run_opts err")) { + close(prog_fd); + return; + } + + if (!ASSERT_EQ(topts.retval, (off1 + off2 + off3) / 8, "test_run_opts retval")) { + close(prog_fd); + return; + } + + close(prog_fd); +} + +static void +reject_offsets(struct bpf_insn *insns, __u32 insn_cnt, int off1, int off2, int off3) { int prog_fd; - prog_fd = __check_ldimm64_gotox_prog_load(insns, insn_cnt, off1, off2); + prog_fd = __check_ldimm64_gotox_prog_load(insns, insn_cnt, off1, off2, off3); if (!ASSERT_EQ(prog_fd, -EACCES, "__check_ldimm64_gotox_prog_load")) close(prog_fd); } @@ -376,7 +406,7 @@ static void check_ldimm64_off_gotox(struct bpf_gotox *skel __always_unused) * The program rewrites the offsets in the instructions below: * r1 = &map + offset1 * r1 += offset2 - * r1 = *r1 + * r1 = *(r1 + offset3) * gotox r1 */ BPF_LD_IMM64_RAW(BPF_REG_1, BPF_PSEUDO_MAP_VALUE, 0), @@ -403,43 +433,55 @@ static void check_ldimm64_off_gotox(struct bpf_gotox *skel __always_unused) BPF_MOV64_IMM(BPF_REG_0, 5), BPF_EXIT_INSN(), }; - int prog_fd, err; - __u32 off1, off2; - - /* allow all combinations off1 + off2 < 6 */ - for (off1 = 0; off1 < 6; off1++) { - for (off2 = 0; off1 + off2 < 6; off2++) { - LIBBPF_OPTS(bpf_test_run_opts, topts); - - prog_fd = __check_ldimm64_gotox_prog_load(insns, ARRAY_SIZE(insns), - off1 * 8, off2 * 8); - if (!ASSERT_GE(prog_fd, 0, "__check_ldimm64_gotox_prog_load")) - return; - - err = bpf_prog_test_run_opts(prog_fd, &topts); - if (!ASSERT_OK(err, "test_run_opts err")) { - close(prog_fd); - return; - } - - if (!ASSERT_EQ(topts.retval, off1 + off2, "test_run_opts retval")) { - close(prog_fd); - return; - } - - close(prog_fd); - } - } + int off1, off2, off3; + + /* allow all combinations off1 + off2 + off3 < 6 */ + for (off1 = 0; off1 < 6; off1++) + for (off2 = 0; off1 + off2 < 6; off2++) + for (off3 = 0; off1 + off2 + off3 < 6; off3++) + allow_offsets(insns, ARRAY_SIZE(insns), + off1 * 8, off2 * 8, off3 * 8); + + /* allow for some offsets to be negative */ + allow_offsets(insns, ARRAY_SIZE(insns), 8 * 3, 0, -(8 * 3)); + allow_offsets(insns, ARRAY_SIZE(insns), 8 * 3, -(8 * 3), 0); + allow_offsets(insns, ARRAY_SIZE(insns), 0, 8 * 3, -(8 * 3)); + allow_offsets(insns, ARRAY_SIZE(insns), 8 * 4, 0, -(8 * 2)); + allow_offsets(insns, ARRAY_SIZE(insns), 8 * 4, -(8 * 2), 0); + allow_offsets(insns, ARRAY_SIZE(insns), 0, 8 * 4, -(8 * 2)); + + /* disallow negative sums of offsets */ + reject_offsets(insns, ARRAY_SIZE(insns), 8 * 3, 0, -(8 * 4)); + reject_offsets(insns, ARRAY_SIZE(insns), 8 * 3, -(8 * 4), 0); + reject_offsets(insns, ARRAY_SIZE(insns), 0, 8 * 3, -(8 * 4)); + + /* disallow the off1 to be negative in any case */ + reject_offsets(insns, ARRAY_SIZE(insns), -8 * 1, 0, 0); + reject_offsets(insns, ARRAY_SIZE(insns), -8 * 1, 8 * 1, 0); + reject_offsets(insns, ARRAY_SIZE(insns), -8 * 1, 8 * 1, 8 * 1); + + /* reject off1 + off2 + off3 >= 6 */ + reject_offsets(insns, ARRAY_SIZE(insns), 8 * 3, 8 * 3, 8 * 0); + reject_offsets(insns, ARRAY_SIZE(insns), 8 * 7, 8 * 0, 8 * 0); + reject_offsets(insns, ARRAY_SIZE(insns), 8 * 0, 8 * 7, 8 * 0); + reject_offsets(insns, ARRAY_SIZE(insns), 8 * 3, 8 * 0, 8 * 3); + reject_offsets(insns, ARRAY_SIZE(insns), 8 * 0, 8 * 3, 8 * 3); + + /* reject (off1 + off2) % 8 != 0, off3 % 8 != 0 */ + reject_offsets(insns, ARRAY_SIZE(insns), 3, 3, 0); + reject_offsets(insns, ARRAY_SIZE(insns), 7, 0, 0); + reject_offsets(insns, ARRAY_SIZE(insns), 0, 7, 0); + reject_offsets(insns, ARRAY_SIZE(insns), 0, 0, 7); +} - /* reject off1 + off2 >= 6 */ - reject_offsets(insns, ARRAY_SIZE(insns), 8 * 3, 8 * 3); - reject_offsets(insns, ARRAY_SIZE(insns), 8 * 7, 8 * 0); - reject_offsets(insns, ARRAY_SIZE(insns), 8 * 0, 8 * 7); +static void check_ldimm64_off_gotox_llvm(struct bpf_gotox *skel) +{ + __u64 in[] = {0, 1, 2, 3, 4}; + __u64 out[] = {1, 1, 5, 1, 1}; + int i; - /* reject (off1 + off2) % 8 != 0 */ - reject_offsets(insns, ARRAY_SIZE(insns), 3, 3); - reject_offsets(insns, ARRAY_SIZE(insns), 7, 0); - reject_offsets(insns, ARRAY_SIZE(insns), 0, 7); + for (i = 0; i < ARRAY_SIZE(in); i++) + check_simple(skel, skel->progs.load_with_nonzero_offset, in[i], out[i]); } void test_bpf_gotox(void) @@ -496,5 +538,8 @@ void test_bpf_gotox(void) if (test__start_subtest("check-ldimm64-off-gotox")) __subtest(skel, check_ldimm64_off_gotox); + if (test__start_subtest("check-ldimm64-off-gotox-llvm")) + __subtest(skel, check_ldimm64_off_gotox_llvm); + bpf_gotox__destroy(skel); } diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c index 269870bec941..0222a9a5d076 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c @@ -3,7 +3,7 @@ #include <bpf/bpf.h> #include <test_progs.h> -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(__powerpc__) || defined(__aarch64__) static int map_create(__u32 map_type, __u32 max_entries) { const char *map_name = "insn_array"; diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c index 215878ea04de..14d4c1793aed 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c @@ -5,24 +5,26 @@ #include "test_bpf_nf.skel.h" #include "test_bpf_nf_fail.skel.h" +#define CT_OPTS_ERROR_GUARD 0x12345678 + static char log_buf[1024 * 1024]; struct { const char *prog_name; const char *err_msg; } test_bpf_nf_fail_tests[] = { - { "alloc_release", "kernel function bpf_ct_release args#0 expected pointer to STRUCT nf_conn but" }, - { "insert_insert", "kernel function bpf_ct_insert_entry args#0 expected pointer to STRUCT nf_conn___init but" }, - { "lookup_insert", "kernel function bpf_ct_insert_entry args#0 expected pointer to STRUCT nf_conn___init but" }, - { "set_timeout_after_insert", "kernel function bpf_ct_set_timeout args#0 expected pointer to STRUCT nf_conn___init but" }, - { "set_status_after_insert", "kernel function bpf_ct_set_status args#0 expected pointer to STRUCT nf_conn___init but" }, - { "change_timeout_after_alloc", "kernel function bpf_ct_change_timeout args#0 expected pointer to STRUCT nf_conn but" }, - { "change_status_after_alloc", "kernel function bpf_ct_change_status args#0 expected pointer to STRUCT nf_conn but" }, + { "alloc_release", "kernel function bpf_ct_release R1 expected pointer to STRUCT nf_conn but" }, + { "insert_insert", "kernel function bpf_ct_insert_entry R1 expected pointer to STRUCT nf_conn___init but" }, + { "lookup_insert", "kernel function bpf_ct_insert_entry R1 expected pointer to STRUCT nf_conn___init but" }, + { "set_timeout_after_insert", "kernel function bpf_ct_set_timeout R1 expected pointer to STRUCT nf_conn___init but" }, + { "set_status_after_insert", "kernel function bpf_ct_set_status R1 expected pointer to STRUCT nf_conn___init but" }, + { "change_timeout_after_alloc", "kernel function bpf_ct_change_timeout R1 expected pointer to STRUCT nf_conn but" }, + { "change_status_after_alloc", "kernel function bpf_ct_change_status R1 expected pointer to STRUCT nf_conn but" }, { "write_not_allowlisted_field", "no write support to nf_conn at off" }, - { "lookup_null_bpf_tuple", "Possibly NULL pointer passed to trusted arg1" }, - { "lookup_null_bpf_opts", "Possibly NULL pointer passed to trusted arg3" }, - { "xdp_lookup_null_bpf_tuple", "Possibly NULL pointer passed to trusted arg1" }, - { "xdp_lookup_null_bpf_opts", "Possibly NULL pointer passed to trusted arg3" }, + { "lookup_null_bpf_tuple", "Possibly NULL pointer passed to trusted R2" }, + { "lookup_null_bpf_opts", "Possibly NULL pointer passed to trusted R4" }, + { "xdp_lookup_null_bpf_tuple", "Possibly NULL pointer passed to trusted R2" }, + { "xdp_lookup_null_bpf_opts", "Possibly NULL pointer passed to trusted R4" }, }; enum { @@ -119,6 +121,10 @@ static void test_bpf_nf_ct(int mode) ASSERT_EQ(skel->bss->test_einval_reserved_new, -EINVAL, "Test EINVAL for reserved in new struct not set to 0"); ASSERT_EQ(skel->bss->test_einval_netns_id, -EINVAL, "Test EINVAL for netns_id < -1"); ASSERT_EQ(skel->bss->test_einval_len_opts, -EINVAL, "Test EINVAL for len__opts != NF_BPF_CT_OPTS_SZ"); + ASSERT_EQ(skel->bss->test_einval_len_opts_small_lookup, CT_OPTS_ERROR_GUARD, + "Test no error write for lookup opts__sz before error field"); + ASSERT_EQ(skel->bss->test_einval_len_opts_small_alloc, CT_OPTS_ERROR_GUARD, + "Test no error write for alloc opts__sz before error field"); ASSERT_EQ(skel->bss->test_eproto_l4proto, -EPROTO, "Test EPROTO for l4proto != TCP or UDP"); ASSERT_EQ(skel->bss->test_enonet_netns_id, -ENONET, "Test ENONET for bad but valid netns_id"); ASSERT_EQ(skel->bss->test_enoent_lookup, -ENOENT, "Test ENOENT for failed lookup"); diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c b/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c index 730357cd0c9a..6dbd1487343c 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c @@ -8,6 +8,11 @@ #include "bpf_qdisc_fifo.skel.h" #include "bpf_qdisc_fq.skel.h" #include "bpf_qdisc_fail__incompl_ops.skel.h" +#include "bpf_qdisc_fail__invalid_dynptr.skel.h" +#include "bpf_qdisc_fail__invalid_dynptr_slice.skel.h" +#include "bpf_qdisc_fail__invalid_dynptr_cross_frame.skel.h" +#include "bpf_qdisc_fail__untrusted_write.skel.h" +#include "bpf_qdisc_dynptr_use_after_invalidate_clone.skel.h" #define LO_IFINDEX 1 @@ -223,6 +228,11 @@ void test_ns_bpf_qdisc(void) test_qdisc_attach_to_non_root(); if (test__start_subtest("incompl_ops")) test_incompl_ops(); + RUN_TESTS(bpf_qdisc_fail__invalid_dynptr); + RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_cross_frame); + RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_slice); + RUN_TESTS(bpf_qdisc_fail__untrusted_write); + RUN_TESTS(bpf_qdisc_dynptr_use_after_invalidate_clone); } void serial_test_bpf_qdisc_default(void) diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c index f829b6f09bc9..eb05fc82f81b 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c @@ -14,6 +14,7 @@ #include "tcp_ca_incompl_cong_ops.skel.h" #include "tcp_ca_unsupp_cong_op.skel.h" #include "tcp_ca_kfunc.skel.h" +#include "tcp_ca_untrusted_btf_write.skel.h" #include "bpf_cc_cubic.skel.h" static const unsigned int total_bytes = 10 * 1024 * 1024; @@ -112,6 +113,10 @@ static void test_cubic(void) ASSERT_EQ(cubic_skel->bss->bpf_cubic_acked_called, 1, "pkts_acked called"); + ASSERT_TRUE(cubic_skel->bss->nodelay_init_reject, "init reject nodelay option"); + ASSERT_TRUE(cubic_skel->bss->nodelay_cwnd_event_tx_start_reject, + "cwnd_event_tx_start reject nodelay option"); + bpf_link__destroy(link); bpf_cubic__destroy(cubic_skel); } @@ -575,6 +580,15 @@ static void test_tcp_ca_kfunc(void) tcp_ca_kfunc__destroy(skel); } +static void test_untrusted_btf_write(void) +{ + struct tcp_ca_untrusted_btf_write *skel; + + skel = tcp_ca_untrusted_btf_write__open_and_load(); + ASSERT_ERR_PTR(skel, "tcp_ca_untrusted_btf_write__open_and_load"); + tcp_ca_untrusted_btf_write__destroy(skel); +} + static void test_cc_cubic(void) { struct cb_opts cb_opts = { @@ -633,6 +647,8 @@ void test_bpf_tcp_ca(void) test_link_replace(); if (test__start_subtest("tcp_ca_kfunc")) test_tcp_ca_kfunc(); + if (test__start_subtest("untrusted_btf_write")) + test_untrusted_btf_write(); if (test__start_subtest("cc_cubic")) test_cc_cubic(); if (test__start_subtest("dctcp_autoattach_map")) diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 054ecb6b1e9f..67b9015cbd98 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -1924,11 +1924,11 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "invalid BTF_INFO", + .descr = "invalid BTF kind", .raw_types = { /* int */ /* [1] */ BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), - BTF_TYPE_ENC(0, 0x20000000, 4), + BTF_TYPE_ENC(0, 0x7f000000, 4), BTF_END_RAW, }, .str_sec = "", @@ -1941,7 +1941,7 @@ static struct btf_raw_test raw_tests[] = { .value_type_id = 1, .max_entries = 4, .btf_load_err = true, - .err_str = "Invalid btf_info", + .err_str = "Invalid kind", }, { @@ -4121,8 +4121,6 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 1, .value_type_id = 1, .max_entries = 1, - .btf_load_err = true, - .err_str = "Type tags don't precede modifiers", }, { .descr = "type_tag test #3, type tag order", @@ -4141,8 +4139,6 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 1, .value_type_id = 1, .max_entries = 1, - .btf_load_err = true, - .err_str = "Type tags don't precede modifiers", }, { .descr = "type_tag test #4, type tag order", @@ -4161,8 +4157,6 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 1, .value_type_id = 1, .max_entries = 1, - .btf_load_err = true, - .err_str = "Type tags don't precede modifiers", }, { .descr = "type_tag test #5, type tag order", @@ -4198,11 +4192,9 @@ static struct btf_raw_test raw_tests[] = { .map_name = "tag_type_check_btf", .key_size = sizeof(int), .value_size = 4, - .key_type_id = 1, - .value_type_id = 1, + .key_type_id = 4, + .value_type_id = 4, .max_entries = 1, - .btf_load_err = true, - .err_str = "Type tags don't precede modifiers", }, { .descr = "type_tag test #7, tag with kflag", @@ -4258,6 +4250,91 @@ static struct btf_raw_test raw_tests[] = { .max_entries = 1, }, +/* + * struct inner { + * struct bpf_spin_lock lock; + * }; + * + * struct value { + * struct bpf_spin_lock lock; + * struct inner nested; + * }; + */ +{ + .descr = "struct test duplicate nested unique fields", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_TBD, 1, 4), /* [2] */ + BTF_MEMBER_ENC(NAME_TBD, 1, 0), + BTF_STRUCT_ENC(NAME_TBD, 1, 4), /* [3] */ + BTF_MEMBER_ENC(NAME_TBD, 2, 0), + BTF_STRUCT_ENC(NAME_TBD, 2, 8), /* [4] */ + BTF_MEMBER_ENC(NAME_TBD, 2, 0), + BTF_MEMBER_ENC(NAME_TBD, 3, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0bpf_spin_lock\0val\0inner\0lock\0value\0lock\0nested"), + .btf_load_err = true, +}, + +/* + * struct value { + * struct bpf_refcount a; + * struct bpf_refcount b; + * }; + */ +{ + .descr = "struct test duplicate bpf_refcount fields", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_TBD, 1, 4), /* [2] */ + BTF_MEMBER_ENC(NAME_TBD, 1, 0), + BTF_STRUCT_ENC(NAME_TBD, 2, 8), /* [3] */ + BTF_MEMBER_ENC(NAME_TBD, 2, 0), + BTF_MEMBER_ENC(NAME_TBD, 2, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0bpf_refcount\0refs\0value\0a\0b"), + .btf_load_err = true, +}, + +{ + .descr = "struct test repeated fields count overflow", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_TBD, 0, 0), /* [2] */ + BTF_TYPE_TAG_ENC(NAME_TBD, 2), /* [3] */ + BTF_PTR_ENC(3), /* [4] */ + BTF_TYPE_ARRAY_ENC(4, 1, 1), /* [5] */ + BTF_STRUCT_ENC(NAME_TBD, 10, 8), /* [6] */ + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_TYPE_ARRAY_ENC(6, 1, 0x1999999aU), /* [7] */ + BTF_STRUCT_ENC(NAME_TBD, 2, 8 + 8 * 0x1999999aU), /* [8] */ + BTF_MEMBER_ENC(NAME_TBD, 4, 0), + BTF_MEMBER_ENC(NAME_TBD, 7, 64), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0prog_test_ref_kfunc\0kptr_untrusted\0elem" + "\0p0\0p1\0p2\0p3\0p4\0p5\0p6\0p7\0p8\0p9" + "\0outer\0trigger\0elems"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "repeat_fields", + .key_size = sizeof(int), + .value_size = 8 + 8 * 0x1999999aU, + .key_type_id = 1, + .value_type_id = 8, + .max_entries = 1, + .btf_load_err = true, +}, }; /* struct btf_raw_test raw_tests[] */ static const char *get_next_str(const char *start, const char *end) @@ -8092,7 +8169,7 @@ static struct btf_dedup_test dedup_tests[] = { static int btf_type_size(const struct btf_type *t) { int base_size = sizeof(struct btf_type); - __u16 vlen = BTF_INFO_VLEN(t->info); + __u32 vlen = BTF_INFO_VLEN(t->info); __u16 kind = BTF_INFO_KIND(t->info); switch (kind) { diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c b/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c index 5bc15bb6b7ce..9d6161151593 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c @@ -20,18 +20,22 @@ static void test_split_simple() { btf__add_struct(btf1, "s1", 4); /* [3] struct s1 { */ btf__add_field(btf1, "f1", 1, 0, 0); /* int f1; */ /* } */ + btf__add_typedef(btf1, "t1", 1); /* [4] typedef int */ VALIDATE_RAW_BTF( btf1, "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", "[2] PTR '(anon)' type_id=1", "[3] STRUCT 's1' size=4 vlen=1\n" - "\t'f1' type_id=1 bits_offset=0"); + "\t'f1' type_id=1 bits_offset=0", + "[4] TYPEDEF 't1' type_id=1"); ASSERT_STREQ(btf_type_c_dump(btf1), "\ struct s1 {\n\ int f1;\n\ -};\n\n", "c_dump"); +};\n\ +\n\ +typedef int t1;\n\n", "c_dump"); btf2 = btf__new_empty_split(btf1); if (!ASSERT_OK_PTR(btf2, "empty_split_btf")) @@ -49,39 +53,46 @@ struct s1 {\n\ ASSERT_EQ(btf_is_int(t), true, "int_kind"); ASSERT_STREQ(btf__str_by_offset(btf2, t->name_off), "int", "int_name"); - btf__add_struct(btf2, "s2", 16); /* [4] struct s2 { */ - btf__add_field(btf2, "f1", 6, 0, 0); /* struct s1 f1; */ - btf__add_field(btf2, "f2", 5, 32, 0); /* int f2; */ + btf__add_struct(btf2, "s2", 16); /* [5] struct s2 { */ + btf__add_field(btf2, "f1", 7, 0, 0); /* struct s1 f1; */ + btf__add_field(btf2, "f2", 6, 32, 0); /* int f2; */ btf__add_field(btf2, "f3", 2, 64, 0); /* int *f3; */ /* } */ /* duplicated int */ - btf__add_int(btf2, "int", 4, BTF_INT_SIGNED); /* [5] int */ + btf__add_int(btf2, "int", 4, BTF_INT_SIGNED); /* [6] int */ /* duplicated struct s1 */ - btf__add_struct(btf2, "s1", 4); /* [6] struct s1 { */ - btf__add_field(btf2, "f1", 5, 0, 0); /* int f1; */ + btf__add_struct(btf2, "s1", 4); /* [7] struct s1 { */ + btf__add_field(btf2, "f1", 6, 0, 0); /* int f1; */ /* } */ + /* duplicated typedef t1 */ + btf__add_typedef(btf2, "t1", 6); /* [8] typedef int */ + VALIDATE_RAW_BTF( btf2, "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", "[2] PTR '(anon)' type_id=1", "[3] STRUCT 's1' size=4 vlen=1\n" "\t'f1' type_id=1 bits_offset=0", - "[4] STRUCT 's2' size=16 vlen=3\n" - "\t'f1' type_id=6 bits_offset=0\n" - "\t'f2' type_id=5 bits_offset=32\n" + "[4] TYPEDEF 't1' type_id=1", + "[5] STRUCT 's2' size=16 vlen=3\n" + "\t'f1' type_id=7 bits_offset=0\n" + "\t'f2' type_id=6 bits_offset=32\n" "\t'f3' type_id=2 bits_offset=64", - "[5] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", - "[6] STRUCT 's1' size=4 vlen=1\n" - "\t'f1' type_id=5 bits_offset=0"); + "[6] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[7] STRUCT 's1' size=4 vlen=1\n" + "\t'f1' type_id=6 bits_offset=0", + "[8] TYPEDEF 't1' type_id=6"); ASSERT_STREQ(btf_type_c_dump(btf2), "\ struct s1 {\n\ int f1;\n\ };\n\ \n\ +typedef int t1;\n\ +\n\ struct s1___2 {\n\ int f1;\n\ };\n\ @@ -90,7 +101,9 @@ struct s2 {\n\ struct s1___2 f1;\n\ int f2;\n\ int *f3;\n\ -};\n\n", "c_dump"); +};\n\ +\n\ +typedef int t1___2;\n\n", "c_dump"); err = btf__dedup(btf2, NULL); if (!ASSERT_OK(err, "btf_dedup")) @@ -102,7 +115,8 @@ struct s2 {\n\ "[2] PTR '(anon)' type_id=1", "[3] STRUCT 's1' size=4 vlen=1\n" "\t'f1' type_id=1 bits_offset=0", - "[4] STRUCT 's2' size=16 vlen=3\n" + "[4] TYPEDEF 't1' type_id=1", + "[5] STRUCT 's2' size=16 vlen=3\n" "\t'f1' type_id=3 bits_offset=0\n" "\t'f2' type_id=1 bits_offset=32\n" "\t'f3' type_id=2 bits_offset=64"); @@ -112,6 +126,8 @@ struct s1 {\n\ int f1;\n\ };\n\ \n\ +typedef int t1;\n\ +\n\ struct s2 {\n\ struct s1 f1;\n\ int f2;\n\ @@ -487,9 +503,8 @@ static void test_split_module(void) for (i = 0; i < ARRAY_SIZE(mod_funcs); i++) { const struct btf_param *p; const struct btf_type *t; - __u16 vlen; + __u32 vlen, j; __u32 id; - int j; id = btf__find_by_name_kind(btf1, mod_funcs[i], BTF_KIND_FUNC); if (!ASSERT_GE(id, nr_base_types, "func_id")) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index f1642794f70e..9f1b50e07a29 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -1027,8 +1027,8 @@ static void test_btf_dump_datasec_data(char *str) char license[4] = "GPL"; struct btf_dump *d; - btf = btf__parse("xdping_kern.bpf.o", NULL); - if (!ASSERT_OK_PTR(btf, "xdping_kern.bpf.o BTF not found")) + btf = btf__parse("xdp_dummy.bpf.o", NULL); + if (!ASSERT_OK_PTR(btf, "xdp_dummy.bpf.o BTF not found")) return; d = btf_dump__new(btf, btf_dump_snprintf, str, NULL); diff --git a/tools/testing/selftests/bpf/prog_tests/btf_kind.c b/tools/testing/selftests/bpf/prog_tests/btf_kind.c new file mode 100644 index 000000000000..f61afe6a79a5 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_kind.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026, Oracle and/or its affiliates. */ + +#include <test_progs.h> +#include <bpf/btf.h> +#include <bpf/libbpf.h> + +/* Verify kind encoding exists for each kind */ +static void test_btf_kind_encoding(void) +{ + LIBBPF_OPTS(btf_new_opts, opts); + const struct btf_header *hdr; + const void *raw_btf; + struct btf *btf; + __u32 raw_size; + + opts.add_layout = true; + btf = btf__new_empty_opts(&opts); + if (!ASSERT_OK_PTR(btf, "btf_new")) + return; + + raw_btf = btf__raw_data(btf, &raw_size); + if (!ASSERT_OK_PTR(raw_btf, "btf__raw_data")) + return; + + hdr = raw_btf; + + ASSERT_EQ(hdr->layout_off % 4, 0, "layout_aligned"); + ASSERT_EQ(hdr->layout_len, sizeof(struct btf_layout) * NR_BTF_KINDS, + "layout_len"); + ASSERT_EQ(hdr->str_off, hdr->layout_off + hdr->layout_len, "str_after_layout"); + btf__free(btf); + + opts.add_layout = false; + btf = btf__new_empty_opts(&opts); + if (!ASSERT_OK_PTR(btf, "btf_new")) + return; + + raw_btf = btf__raw_data(btf, &raw_size); + if (!ASSERT_OK_PTR(raw_btf, "btf__raw_data")) + return; + + hdr = raw_btf; + + ASSERT_EQ(hdr->layout_off, 0, "no_layout_off"); + ASSERT_EQ(hdr->layout_len, 0, "no_layout_len"); + ASSERT_EQ(hdr->str_off, hdr->type_off + hdr->type_len, "strs_after_types"); + btf__free(btf); +} + +static int write_raw_btf(void *raw_btf, size_t raw_size, char *file) +{ + int fd = mkstemp(file); + ssize_t n; + + if (!ASSERT_OK_FD(fd, "open_raw_btf")) + return -1; + n = write(fd, raw_btf, raw_size); + close(fd); + if (!ASSERT_EQ(n, (ssize_t)raw_size, "write_raw_btf")) + return -1; + return 0; +} + +/* + * Fabricate an unrecognized kind at BTF_KIND_MAX + 1, and after adding + * the appropriate struct/typedefs to the BTF such that it recognizes + * this kind, ensure that parsing of BTF containing the unrecognized kind + * can succeed. + */ +void test_btf_kind_decoding(void) +{ + char btf_kind_file1[] = "/tmp/test_btf_kind.XXXXXX"; + char btf_kind_file2[] = "/tmp/test_btf_kind.XXXXXX"; + char btf_kind_file3[] = "/tmp/test_btf_kind.XXXXXX"; + struct btf *btf = NULL, *new_btf = NULL; + __s32 int_id, unrec_id, id, id2; + LIBBPF_OPTS(btf_new_opts, opts); + struct btf_layout *l; + struct btf_header *hdr; + const void *raw_btf; + struct btf_type *t; + void *new_raw_btf; + void *str_data; + __u32 raw_size; + + opts.add_layout = true; + btf = btf__new_empty_opts(&opts); + if (!ASSERT_OK_PTR(btf, "btf_new")) + return; + + int_id = btf__add_int(btf, "test_char", 1, BTF_INT_CHAR); + if (!ASSERT_GT(int_id, 0, "add_int_id")) + return; + + /* + * Create our type with unrecognized kind by adding a typedef kind + * we will overwrite it with our unrecognized kind value. + */ + unrec_id = btf__add_typedef(btf, "unrec_kind", int_id); + if (!ASSERT_GT(unrec_id, 0, "add_unrec_id")) + return; + + /* + * Add an id after it that we will look up to verify we can parse + * beyond unrecognized kinds. + */ + id = btf__add_typedef(btf, "test_lookup", int_id); + if (!ASSERT_GT(id, 0, "add_test_lookup_id")) + return; + id2 = btf__add_typedef(btf, "test_lookup2", int_id); + if (!ASSERT_GT(id2, 0, "add_test_lookup_id2")) + return; + + raw_btf = (void *)btf__raw_data(btf, &raw_size); + if (!ASSERT_OK_PTR(raw_btf, "btf__raw_data")) + return; + + new_raw_btf = calloc(1, raw_size + sizeof(*l)); + if (!ASSERT_OK_PTR(new_raw_btf, "calloc_raw_btf")) + return; + memcpy(new_raw_btf, raw_btf, raw_size); + + hdr = new_raw_btf; + + /* Move strings to make space for one new layout description */ + raw_size += sizeof(*l); + str_data = new_raw_btf + hdr->hdr_len + hdr->str_off; + memmove(str_data + sizeof(*l), str_data, hdr->str_len); + hdr->str_off += sizeof(*l); + + /* Add new layout description */ + hdr->layout_len += sizeof(*l); + l = new_raw_btf + hdr->hdr_len + hdr->layout_off; + l[NR_BTF_KINDS].info_sz = 0; + l[NR_BTF_KINDS].elem_sz = 0; + l[NR_BTF_KINDS].flags = 0; + + /* Now modify typedef added above to be an unrecognized kind. */ + t = (void *)hdr + hdr->hdr_len + hdr->type_off + sizeof(struct btf_type) + + sizeof(__u32); + t->info = (NR_BTF_KINDS << 24); + + /* Write BTF to a raw file, ready for parsing. */ + if (write_raw_btf(new_raw_btf, raw_size, btf_kind_file1)) + goto out; + + /* + * Verify parsing succeeds, and that we can read type info past + * the unrecognized kind. + */ + new_btf = btf__parse_raw(btf_kind_file1); + if (ASSERT_OK_PTR(new_btf, "btf__parse_raw")) { + ASSERT_EQ(btf__find_by_name(new_btf, "unrec_kind"), unrec_id, + "unrec_kind_found"); + ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup", + BTF_KIND_TYPEDEF), id, + "verify_id_lookup"); + ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup2", + BTF_KIND_TYPEDEF), id2, + "verify_id2_lookup"); + } + btf__free(new_btf); + new_btf = NULL; + + /* + * Next, change info_sz to equal sizeof(struct btf_type); this means the + * "test_lookup" kind will be reinterpreted as a singular info element + * following the unrecognized kind. + */ + l[NR_BTF_KINDS].info_sz = sizeof(struct btf_type); + if (write_raw_btf(new_raw_btf, raw_size, btf_kind_file2)) + goto out; + + new_btf = btf__parse_raw(btf_kind_file2); + if (ASSERT_OK_PTR(new_btf, "btf__parse_raw")) { + ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup", + BTF_KIND_TYPEDEF), -ENOENT, + "verify_id_not_found"); + /* id of "test_lookup2" will be id2 -1 as we have removed one type */ + ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup2", + BTF_KIND_TYPEDEF), id2 - 1, + "verify_id_lookup2"); + + } + btf__free(new_btf); + new_btf = NULL; + + /* + * Change elem_sz to equal sizeof(struct btf_type) and set vlen + * associated with unrecognized type to 1; this allows us to verify + * vlen-specified BTF can still be parsed. + */ + l[NR_BTF_KINDS].info_sz = 0; + l[NR_BTF_KINDS].elem_sz = sizeof(struct btf_type); + t->info |= 1; + if (write_raw_btf(new_raw_btf, raw_size, btf_kind_file3)) + goto out; + + new_btf = btf__parse_raw(btf_kind_file3); + if (ASSERT_OK_PTR(new_btf, "btf__parse_raw")) { + ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup", + BTF_KIND_TYPEDEF), -ENOENT, + "verify_id_not_found"); + /* id of "test_lookup2" will be id2 -1 as we have removed one type */ + ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup2", + BTF_KIND_TYPEDEF), id2 - 1, + "verify_id_lookup2"); + + } +out: + btf__free(new_btf); + free(new_raw_btf); + unlink(btf_kind_file1); + unlink(btf_kind_file2); + unlink(btf_kind_file3); + btf__free(btf); +} + +void test_btf_kind(void) +{ + if (test__start_subtest("btf_kind_encoding")) + test_btf_kind_encoding(); + if (test__start_subtest("btf_kind_decoding")) + test_btf_kind_decoding(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/btf_map_keyless.c b/tools/testing/selftests/bpf/prog_tests/btf_map_keyless.c new file mode 100644 index 000000000000..3248bccc3557 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_map_keyless.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <bpf/btf.h> + +/* + * A hash map with a key-less BTF (btf_key_type_id == 0) used to be accepted + * and then NULL-deref in btf_type_show() when dumped through bpffs. A fixed + * kernel rejects it at creation; verify that rejection, with a keyed positive + * control so the -EINVAL is about the missing key type and not some unrelated + * failure. + */ +static void check_keyless(int map_type, __u32 map_flags, int btf_fd, int val_id) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + int map_fd; + + opts.map_flags = map_flags; + opts.btf_fd = btf_fd; + opts.btf_value_type_id = val_id; + + /* Positive control: the same map with a real key type is accepted. */ + opts.btf_key_type_id = val_id; + map_fd = bpf_map_create(map_type, "keyed_map", 4, 4, 8, &opts); + if (!ASSERT_GE(map_fd, 0, "keyed create is accepted")) + return; + close(map_fd); + + /* A key-less BTF must be rejected. */ + opts.btf_key_type_id = 0; + map_fd = bpf_map_create(map_type, "keyless_map", 4, 4, 8, &opts); + ASSERT_EQ(map_fd, -EINVAL, "key-less create is rejected"); + if (map_fd >= 0) + close(map_fd); +} + +void test_btf_map_keyless(void) +{ + int btf_fd, val_id; + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf__new_empty")) + return; + + val_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_GT(val_id, 0, "btf__add_int")) + goto out; + + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf__load_into_kernel")) + goto out; + btf_fd = btf__fd(btf); + + if (test__start_subtest("hash")) + check_keyless(BPF_MAP_TYPE_HASH, 0, btf_fd, val_id); + if (test__start_subtest("rhash")) + check_keyless(BPF_MAP_TYPE_RHASH, BPF_F_NO_PREALLOC, btf_fd, val_id); +out: + btf__free(btf); +} diff --git a/tools/testing/selftests/bpf/prog_tests/btf_sanitize.c b/tools/testing/selftests/bpf/prog_tests/btf_sanitize.c new file mode 100644 index 000000000000..652b51efafc2 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_sanitize.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026, Oracle and/or its affiliates. */ +#include <test_progs.h> +#include <linux/btf.h> +#include "bpf/libbpf_internal.h" +#include "../test_btf.h" +#include "kfree_skb.skel.h" + +#define TYPE_LEN (sizeof(struct btf_type) + sizeof(__u32)) +#define MAX_NR_LAYOUT 2 +#define LAYOUT_LEN (sizeof(struct btf_layout) * MAX_NR_LAYOUT) +#define STR_LEN sizeof("\0int") + +struct layout_btf { + struct btf_header hdr; + __u32 types[TYPE_LEN/sizeof(__u32)]; + struct btf_layout layout[MAX_NR_LAYOUT]; + char strs[STR_LEN]; +}; + +static const struct layout_btf layout_btf = { + .hdr = { + .magic = BTF_MAGIC, + .version = BTF_VERSION, + .hdr_len = sizeof(struct btf_header), + .type_off = 0, + .type_len = TYPE_LEN, + .str_off = TYPE_LEN + LAYOUT_LEN, + .str_len = STR_LEN, + .layout_off = TYPE_LEN, + .layout_len = LAYOUT_LEN, + }, + .types = { + BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), + }, + .layout = { + { .info_sz = 0, .elem_sz = 0, .flags = 0 }, + { .info_sz = sizeof(__u32), .elem_sz = 0, .flags = 0 }, + }, + .strs = "\0int", +}; + +void test_btf_sanitize_layout(void) +{ + struct btf *orig = NULL, *sanitized = NULL; + struct kern_feature_cache *cache = NULL; + struct kfree_skb *skel = NULL; + const struct btf_header *hdr; + const void *raw; + __u32 raw_sz; + + skel = kfree_skb__open(); + if (!ASSERT_OK_PTR(skel, "kfree_skb_skel")) + return; + orig = btf__new(&layout_btf, sizeof(layout_btf)); + if (!ASSERT_OK_PTR(orig, "btf_new_layout")) + goto out; + raw = btf__raw_data(orig, &raw_sz); + if (!ASSERT_OK_PTR(raw, "btf__raw_data_orig")) + goto out; + hdr = (struct btf_header *)raw; + ASSERT_EQ(hdr->layout_off, TYPE_LEN, "layout_off_nonzero"); + ASSERT_EQ(hdr->layout_len, LAYOUT_LEN, "layout_len_nonzero"); + + cache = calloc(1, sizeof(*cache)); + if (!ASSERT_OK_PTR(cache, "alloc_feat_cache")) + goto out; + for (int i = 0; i < __FEAT_CNT; i++) + cache->res[i] = FEAT_SUPPORTED; + cache->res[FEAT_BTF_LAYOUT] = FEAT_MISSING; + + bpf_object_set_feat_cache(skel->obj, cache); + + if (!ASSERT_FALSE(kernel_supports(skel->obj, FEAT_BTF_LAYOUT), "layout_feature_missing")) + goto out; + if (!ASSERT_TRUE(kernel_supports(skel->obj, FEAT_BTF_FUNC), "other_feature_allowed")) + goto out; + + sanitized = bpf_object__sanitize_btf(skel->obj, orig); + if (!ASSERT_OK_PTR(sanitized, "bpf_object__sanitize_btf")) + goto out; + + raw = btf__raw_data(sanitized, &raw_sz); + if (!ASSERT_OK_PTR(raw, "btf__raw_data_sanitized")) + goto out; + hdr = (struct btf_header *)raw; + ASSERT_EQ(hdr->layout_off, 0, "layout_off_zero"); + ASSERT_EQ(hdr->layout_len, 0, "layout_len_zero"); + ASSERT_EQ(hdr->str_off, TYPE_LEN, "strs_after_types"); + ASSERT_EQ(hdr->str_len, STR_LEN, "strs_len_unchanged"); + ASSERT_EQ(raw_sz, hdr->hdr_len + hdr->type_len + hdr->str_len, "btf_raw_sz_reduced"); +out: + /* This will free the cache we allocated above */ + kfree_skb__destroy(skel); + btf__free(sanitized); + btf__free(orig); +} diff --git a/tools/testing/selftests/bpf/prog_tests/btf_write.c b/tools/testing/selftests/bpf/prog_tests/btf_write.c index 6e36de1302fc..5c84723cf254 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_write.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_write.c @@ -497,10 +497,121 @@ cleanup: btf__free(btf2); } +static void test_btf_add_btf_split() +{ + struct btf *base = NULL, *split1 = NULL, *split2 = NULL; + struct btf *combined = NULL; + int id, err; + + /* Create a base BTF with an INT and a PTR to it */ + base = btf__new_empty(); + if (!ASSERT_OK_PTR(base, "base")) + return; + + id = btf__add_int(base, "int", 4, BTF_INT_SIGNED); + ASSERT_EQ(id, 1, "base_int_id"); + id = btf__add_ptr(base, 1); + ASSERT_EQ(id, 2, "base_ptr_id"); + + /* base has 2 types, type IDs 1..2 */ + ASSERT_EQ(btf__type_cnt(base), 3, "base_type_cnt"); + + /* Create split1 on base: a STRUCT referencing base's int (ID 1) */ + split1 = btf__new_empty_split(base); + if (!ASSERT_OK_PTR(split1, "split1")) + goto cleanup; + + id = btf__add_struct(split1, "s1", 4); + /* split types start at base_type_cnt = 3 */ + ASSERT_EQ(id, 3, "split1_struct_id"); + btf__add_field(split1, "x", 1, 0, 0); /* refers to base int */ + + id = btf__add_ptr(split1, 3); + ASSERT_EQ(id, 4, "split1_ptr_id"); /* ptr to the struct (split self-ref) */ + + /* Add a typedef "int_alias" -> base int in split1, which will be + * duplicated in split2 to test that btf__dedup() merges them. + */ + id = btf__add_typedef(split1, "int_alias", 1); + ASSERT_EQ(id, 5, "split1_typedef_id"); + + /* Create split2 on base: a TYPEDEF referencing base's ptr (ID 2) */ + split2 = btf__new_empty_split(base); + if (!ASSERT_OK_PTR(split2, "split2")) + goto cleanup; + + id = btf__add_typedef(split2, "int_ptr", 2); /* refers to base ptr */ + ASSERT_EQ(id, 3, "split2_typedef_id"); + + id = btf__add_struct(split2, "s2", 8); + ASSERT_EQ(id, 4, "split2_struct_id"); + btf__add_field(split2, "p", 3, 0, 0); /* refers to split2's own typedef */ + + /* Same "int_alias" typedef as split1 - should be deduped away */ + id = btf__add_typedef(split2, "int_alias", 1); + ASSERT_EQ(id, 5, "split2_dup_typedef_id"); + + /* Create combined split BTF on same base and merge both */ + combined = btf__new_empty_split(base); + if (!ASSERT_OK_PTR(combined, "combined")) + goto cleanup; + + /* Merge split1: its types (3,4,5) should land at IDs 3,4,5 */ + id = btf__add_btf(combined, split1); + if (!ASSERT_GE(id, 0, "add_split1")) + goto cleanup; + ASSERT_EQ(id, 3, "split1_first_id"); + + /* Merge split2: its types (3,4,5) should be remapped to 6,7,8 */ + id = btf__add_btf(combined, split2); + if (!ASSERT_GE(id, 0, "add_split2")) + goto cleanup; + ASSERT_EQ(id, 6, "split2_first_id"); + + /* Before dedup: base (2) + split1 (3) + split2 (3) = 8 types + void */ + ASSERT_EQ(btf__type_cnt(combined), 9, "pre_dedup_type_cnt"); + + VALIDATE_RAW_BTF( + combined, + /* base types (IDs 1-2) */ + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] PTR '(anon)' type_id=1", + + /* split1 types (IDs 3-5): base refs unchanged */ + "[3] STRUCT 's1' size=4 vlen=1\n" + "\t'x' type_id=1 bits_offset=0", /* refers to base int=1 */ + "[4] PTR '(anon)' type_id=3", /* refers to split1's struct=3 */ + "[5] TYPEDEF 'int_alias' type_id=1", /* refers to base int=1 */ + + /* split2 types (IDs 6-8): remapped from 3,4,5 to 6,7,8 */ + "[6] TYPEDEF 'int_ptr' type_id=2", /* base ptr=2, unchanged */ + "[7] STRUCT 's2' size=8 vlen=1\n" + "\t'p' type_id=6 bits_offset=0", /* split2 typedef: 3->6 */ + "[8] TYPEDEF 'int_alias' type_id=1"); /* dup of [5] */ + + /* Dedup to mirror the bpftool merge flow; should remove the + * duplicate "int_alias" typedef. + */ + err = btf__dedup(combined, NULL); + if (!ASSERT_OK(err, "dedup")) + goto cleanup; + + /* After dedup: one int_alias removed, so 7 types + void */ + ASSERT_EQ(btf__type_cnt(combined), 8, "dedup_type_cnt"); + +cleanup: + btf__free(combined); + btf__free(split2); + btf__free(split1); + btf__free(base); +} + void test_btf_write() { if (test__start_subtest("btf_add")) test_btf_add(); if (test__start_subtest("btf_add_btf")) test_btf_add_btf(); + if (test__start_subtest("btf_add_btf_split")) + test_btf_add_btf_split(); } diff --git a/tools/testing/selftests/bpf/prog_tests/cb_refs.c b/tools/testing/selftests/bpf/prog_tests/cb_refs.c index c40df623a8f7..490e15e7126d 100644 --- a/tools/testing/selftests/bpf/prog_tests/cb_refs.c +++ b/tools/testing/selftests/bpf/prog_tests/cb_refs.c @@ -11,9 +11,9 @@ struct { const char *prog_name; const char *err_msg; } cb_refs_tests[] = { - { "underflow_prog", "must point to scalar, or struct with scalar" }, - { "leak_prog", "Possibly NULL pointer passed to helper arg2" }, - { "nested_cb", "Unreleased reference id=4 alloc_insn=2" }, /* alloc_insn=2{4,5} */ + { "underflow_prog", "release kfunc bpf_kfunc_call_test_release expects referenced PTR_TO_BTF_ID passed to R1" }, + { "leak_prog", "Possibly NULL pointer passed to helper R2" }, + { "nested_cb", "Unreleased reference id=5 alloc_insn=2" }, /* alloc_insn=2{4,5} */ { "non_cb_transfer_ref", "Unreleased reference id=4 alloc_insn=1" }, /* alloc_insn=1{1,2} */ }; diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c b/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c index a5afd16705f0..5a1e08d39a06 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c @@ -10,6 +10,17 @@ #include "cgroup_iter_memcg.h" #include "cgroup_iter_memcg.skel.h" +/* + * memcg stats are cached per-cpu and only become visible once the periodic + * flusher runs (FLUSH_TIME, 2s), or once pending updates cross + * MEMCG_CHARGE_BATCH * num_online_cpus(). That threshold grows with the CPU + * count, so on a large machine a single pass does not reach it and + * bpf_mem_cgroup_flush_stats() returns without flushing anything. Retry for + * long enough to cover a flusher cycle. + */ +#define MEMCG_STAT_RETRIES 16 +#define MEMCG_STAT_RETRY_DELAY_US (250 * 1000) + static int read_stats(struct bpf_link *link) { int fd, ret = 0; @@ -35,11 +46,13 @@ static int read_stats(struct bpf_link *link) static void test_anon(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; len = sysconf(_SC_PAGESIZE) * 1024; +retry: /* * Increase memcg anon usage by mapping and writing * to a new anon region. @@ -53,6 +66,12 @@ static void test_anon(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; + if (!memcg_query->nr_anon_mapped && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->nr_anon_mapped, 0, "final anon mapped val"); cleanup: @@ -61,6 +80,7 @@ cleanup: static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; char *path; @@ -76,6 +96,7 @@ static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) fd = open(path, O_CREAT | O_RDWR, 0644); if (!ASSERT_OK_FD(fd, "open fd")) return; +retry: if (!ASSERT_OK(ftruncate(fd, len), "ftruncate")) goto cleanup_fd; @@ -88,6 +109,13 @@ static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup_map; + if ((!memcg_query->nr_file_pages || !memcg_query->nr_file_mapped) && + ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->nr_file_pages, 0, "final file value"); ASSERT_GT(memcg_query->nr_file_mapped, 0, "final file mapped value"); @@ -100,6 +128,7 @@ cleanup_fd: static void test_shmem(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; size_t len; int fd; @@ -107,58 +136,39 @@ static void test_shmem(struct bpf_link *link, struct memcg_query *memcg_query) /* * Increase memcg shmem usage by creating and writing - * to a shmem object. + * to a memfd backed by shmem/tmpfs. */ - fd = shm_open("/tmp_shmem", O_CREAT | O_RDWR, 0644); - if (!ASSERT_OK_FD(fd, "shm_open")) + fd = memfd_create("tmp_shmem", 0); + if (!ASSERT_OK_FD(fd, "memfd_create")) return; +retry: if (!ASSERT_OK(fallocate(fd, 0, 0, len), "fallocate")) goto cleanup; if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; - ASSERT_GT(memcg_query->nr_shmem, 0, "final shmem value"); - -cleanup: - close(fd); - shm_unlink("/tmp_shmem"); -} - -#define NR_PIPES 64 -static void test_kmem(struct bpf_link *link, struct memcg_query *memcg_query) -{ - int fds[NR_PIPES][2], i; - - /* - * Increase kmem value by creating pipes which will allocate some - * kernel buffers. - */ - for (i = 0; i < NR_PIPES; i++) { - if (!ASSERT_OK(pipe(fds[i]), "pipe")) - goto cleanup; + if (!memcg_query->nr_shmem && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + goto retry; } - if (!ASSERT_OK(read_stats(link), "read stats")) - goto cleanup; - - ASSERT_GT(memcg_query->memcg_kmem, 0, "kmem value"); + ASSERT_GT(memcg_query->nr_shmem, 0, "final shmem value"); cleanup: - for (i = i - 1; i >= 0; i--) { - close(fds[i][0]); - close(fds[i][1]); - } + close(fd); } static void test_pgfault(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; len = sysconf(_SC_PAGESIZE) * 1024; +retry: /* Create region to use for triggering a page fault. */ map = mmap(NULL, len, PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (!ASSERT_NEQ(map, MAP_FAILED, "mmap anon")) @@ -170,6 +180,12 @@ static void test_pgfault(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; + if (!memcg_query->pgfault && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->pgfault, 0, "final pgfault val"); cleanup: @@ -209,8 +225,6 @@ void test_cgroup_iter_memcg(void) test_shmem(link, &skel->data_query->memcg_query); if (test__start_subtest("cgroup_iter_memcg__file")) test_file(link, &skel->data_query->memcg_query); - if (test__start_subtest("cgroup_iter_memcg__kmem")) - test_kmem(link, &skel->data_query->memcg_query); if (test__start_subtest("cgroup_iter_memcg__pgfault")) test_pgfault(link, &skel->data_query->memcg_query); diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_preorder.c b/tools/testing/selftests/bpf/prog_tests/cgroup_preorder.c index d4d583872fa2..d2ccf409dfe3 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_preorder.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_preorder.c @@ -102,6 +102,82 @@ close_skel: return err; } +/* + * Replacing a link's program (bpf_link_update) must target the correct slot in + * the effective array even when a BPF_F_PREORDER program is attached to the + * same cgroup. All programs here are attached to a single cgroup; "parent" is + * reused only as a third distinct program. + * + * Attach child(1) normally and child_2(2) with BPF_F_PREORDER, so the effective + * order is [2, 1]. Then replace child(1)'s program with parent(3): only the + * non-preorder slot changes, giving [2, 3]. + */ +static int run_link_replace_test(int cgroup_fd, int sock_fd) +{ + LIBBPF_OPTS(bpf_link_create_opts, create_opts); + int err = 0, normal_link = -1, preorder_link = -1; + struct cgroup_preorder *skel = NULL; + enum bpf_attach_type atype; + __u8 *result, buf = 0x00; + socklen_t optlen = 1; + + skel = cgroup_preorder__open_and_load(); + if (!ASSERT_OK_PTR(skel, "cgroup_preorder__open_and_load")) + return -1; + + err = setsockopt(sock_fd, SOL_IP, IP_TOS, &buf, 1); + if (!ASSERT_OK(err, "setsockopt")) + goto close_skel; + + atype = bpf_program__expected_attach_type(skel->progs.child); + + create_opts.flags = 0; + normal_link = bpf_link_create(bpf_program__fd(skel->progs.child), + cgroup_fd, atype, &create_opts); + if (!ASSERT_GE(normal_link, 0, "create_normal_link")) { + err = normal_link; + goto close_skel; + } + + create_opts.flags = BPF_F_PREORDER; + preorder_link = bpf_link_create(bpf_program__fd(skel->progs.child_2), + cgroup_fd, atype, &create_opts); + if (!ASSERT_GE(preorder_link, 0, "create_preorder_link")) { + err = preorder_link; + goto close_links; + } + + result = skel->bss->result; + skel->bss->idx = 0; + memset(result, 0, 4); + + err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen); + if (!ASSERT_OK(err, "getsockopt-before")) + goto close_links; + ASSERT_TRUE(result[0] == 2 && result[1] == 1, "order before update"); + + /* Replace the normal link's program child(1) -> parent(3). */ + err = bpf_link_update(normal_link, bpf_program__fd(skel->progs.parent), NULL); + if (!ASSERT_OK(err, "bpf_link_update")) + goto close_links; + + skel->bss->idx = 0; + memset(result, 0, 4); + + err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen); + if (!ASSERT_OK(err, "getsockopt-after")) + goto close_links; + ASSERT_TRUE(result[0] == 2 && result[1] == 3, "order after update"); + +close_links: + if (preorder_link >= 0) + close(preorder_link); + close(normal_link); +close_skel: + cgroup_preorder__destroy(skel); + return err; +} + void test_cgroup_preorder(void) { int cg_parent = -1, cg_child = -1, sock_fd = -1; @@ -120,6 +196,7 @@ void test_cgroup_preorder(void) ASSERT_OK(run_getsockopt_test(cg_parent, cg_child, sock_fd, false), "getsockopt_test_1"); ASSERT_OK(run_getsockopt_test(cg_parent, cg_child, sock_fd, true), "getsockopt_test_2"); + ASSERT_OK(run_link_replace_test(cg_child, sock_fd), "link_replace_test"); out: close(sock_fd); diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_storage.c b/tools/testing/selftests/bpf/prog_tests/cgroup_storage.c index cf395715ced4..8dab655db342 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_storage.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_storage.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 +#include <unistd.h> +#include <sys/socket.h> #include <test_progs.h> #include "cgroup_helpers.h" #include "network_helpers.h" @@ -86,6 +88,11 @@ void test_cgroup_storage(void) err = SYS_NOFAIL(PING_CMD); ASSERT_OK(err, "sixth ping"); + err = bpf_map__get_next_key(skel->maps.cgroup_storage, &key, &key, + sizeof(key)); + ASSERT_ERR(err, "bpf_map__get_next_key should fail"); + ASSERT_EQ(errno, ENOENT, "no second key"); + cleanup_progs: cgroup_storage__destroy(skel); cleanup_network: @@ -94,3 +101,43 @@ cleanup_cgroup: close(cgroup_fd); cleanup_cgroup_environment(); } + +void test_cgroup_storage_oob(void) +{ + struct cgroup_storage *skel; + int cgroup_fd, sock_fd; + + cgroup_fd = cgroup_setup_and_join(TEST_CGROUP); + if (!ASSERT_OK_FD(cgroup_fd, "create cgroup")) + return; + + /* Load and attach BPF program */ + skel = cgroup_storage__open_and_load(); + if (!ASSERT_OK_PTR(skel, "cgroup_storage__open_and_load")) + goto cleanup_cgroup; + + skel->links.trigger_oob = bpf_program__attach_cgroup(skel->progs.trigger_oob, + cgroup_fd); + if (!ASSERT_OK_PTR(skel->links.trigger_oob, "attach_cgroup")) + goto cleanup_skel; + + /* Create a socket to trigger cgroup/sock_create hook. + * This will execute our BPF program and trigger the OOB read + * if the bug is present (before the fix). + */ + sock_fd = socket(AF_INET, SOCK_DGRAM, 0); + if (!ASSERT_OK_FD(sock_fd, "create socket")) + goto cleanup_skel; + + close(sock_fd); + + /* If we reach here without a kernel panic or KASAN report, + * the test passes (the fix is working). + */ + +cleanup_skel: + cgroup_storage__destroy(skel); +cleanup_cgroup: + close(cgroup_fd); + cleanup_cgroup_environment(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/cgrp_local_storage.c b/tools/testing/selftests/bpf/prog_tests/cgrp_local_storage.c index 478a77cb67e6..c4398ccf3493 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgrp_local_storage.c +++ b/tools/testing/selftests/bpf/prog_tests/cgrp_local_storage.c @@ -176,7 +176,7 @@ static void test_cgroup_iter_sleepable(int cgroup_fd, __u64 cgroup_id) DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); union bpf_iter_link_info linfo; struct cgrp_ls_sleepable *skel; - struct bpf_link *link; + struct bpf_link *link, *fexit_link; int err, iter_fd; char buf[16]; @@ -200,16 +200,27 @@ static void test_cgroup_iter_sleepable(int cgroup_fd, __u64 cgroup_id) if (!ASSERT_OK_PTR(link, "attach_iter")) goto out; + fexit_link = bpf_program__attach(skel->progs.fexit_update); + if (!ASSERT_OK_PTR(fexit_link, "attach_fexit")) + goto out_link; + iter_fd = bpf_iter_create(bpf_link__fd(link)); if (!ASSERT_GE(iter_fd, 0, "iter_create")) - goto out_link; + goto out_fexit_link; + + skel->bss->target_pid = sys_gettid(); /* trigger the program run */ (void)read(iter_fd, buf, sizeof(buf)); + skel->bss->target_pid = 0; + + ASSERT_EQ(skel->bss->update_err, 0, "update_err"); ASSERT_EQ(skel->bss->cgroup_id, cgroup_id, "cgroup_id"); close(iter_fd); +out_fexit_link: + bpf_link__destroy(fexit_link); out_link: bpf_link__destroy(link); out: diff --git a/tools/testing/selftests/bpf/prog_tests/clone_attach_btf_id.c b/tools/testing/selftests/bpf/prog_tests/clone_attach_btf_id.c new file mode 100644 index 000000000000..1c3e28e74606 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/clone_attach_btf_id.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2025 Meta */ +#include <test_progs.h> +#include "clone_attach_btf_id.skel.h" + +/* + * Test that bpf_program__clone() respects caller-provided attach_btf_id + * override via bpf_prog_load_opts. + * + * The BPF program has SEC("fentry/bpf_fentry_test1"). Clone it twice + * from the same prepared object: first with no opts (callback resolves + * attach_btf_id from sec_name), then with attach_btf_id overridden to + * bpf_fentry_test2. Verify each loaded program's attach_btf_id via + * bpf_prog_get_info_by_fd(). + */ + +static int get_prog_attach_btf_id(int prog_fd) +{ + struct bpf_prog_info info = {}; + __u32 info_len = sizeof(info); + int err; + + err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len); + if (err) + return err; + return info.attach_btf_id; +} + +void test_clone_attach_btf_id(void) +{ + struct clone_attach_btf_id *skel; + int fd1 = -1, fd2 = -1, err; + int btf_id_test1, btf_id_test2; + + btf_id_test1 = libbpf_find_vmlinux_btf_id("bpf_fentry_test1", BPF_TRACE_FENTRY); + if (!ASSERT_GT(btf_id_test1, 0, "find_btf_id_test1")) + return; + + btf_id_test2 = libbpf_find_vmlinux_btf_id("bpf_fentry_test2", BPF_TRACE_FENTRY); + if (!ASSERT_GT(btf_id_test2, 0, "find_btf_id_test2")) + return; + + skel = clone_attach_btf_id__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + + err = bpf_object__prepare(skel->obj); + if (!ASSERT_OK(err, "obj_prepare")) + goto out; + + /* Clone with no opts — callback resolves BTF from sec_name */ + fd1 = bpf_program__clone(skel->progs.fentry_handler, NULL); + if (!ASSERT_GE(fd1, 0, "clone_default")) + goto out; + ASSERT_EQ(get_prog_attach_btf_id(fd1), btf_id_test1, + "attach_btf_id_default"); + + /* + * Clone with attach_btf_id override pointing to a different + * function. The BPF program never accesses arguments, so the + * load succeeds regardless of signature mismatch. + */ + LIBBPF_OPTS(bpf_prog_load_opts, opts, + .attach_btf_id = btf_id_test2, + ); + fd2 = bpf_program__clone(skel->progs.fentry_handler, &opts); + if (!ASSERT_GE(fd2, 0, "clone_override")) + goto out; + ASSERT_EQ(get_prog_attach_btf_id(fd2), btf_id_test2, + "attach_btf_id_override"); + +out: + if (fd1 >= 0) + close(fd1); + if (fd2 >= 0) + close(fd2); + clone_attach_btf_id__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/connect_force_port.c b/tools/testing/selftests/bpf/prog_tests/connect_force_port.c index 24d553109f8d..dfb7f6cf3ee1 100644 --- a/tools/testing/selftests/bpf/prog_tests/connect_force_port.c +++ b/tools/testing/selftests/bpf/prog_tests/connect_force_port.c @@ -53,6 +53,9 @@ static int run_test(int cgroup_fd, int server_fd, int family, int type) __u16 expected_peer_port = 60000; struct bpf_program *prog; struct bpf_object *obj; + struct bpf_map *map; + __u16 *port_ptr; + size_t port_size; const char *obj_file = v4 ? "connect_force_port4.bpf.o" : "connect_force_port6.bpf.o"; int fd, err; __u32 duration = 0; @@ -61,6 +64,21 @@ static int run_test(int cgroup_fd, int server_fd, int family, int type) if (!ASSERT_OK_PTR(obj, "bpf_obj_open")) return -1; + map = bpf_object__find_map_by_name(obj, ".bss"); + if (!ASSERT_OK_PTR(map, "find bss map")) { + err = -EIO; + goto close_bpf_object; + } + + port_ptr = bpf_map__initial_value(map, &port_size); + if (!ASSERT_OK_PTR(port_ptr, "get bss initial value")) { + err = -EIO; + goto close_bpf_object; + } + + /* Auto assigns the port according to availability */ + *port_ptr = ntohs(get_socket_local_port(server_fd)); + err = bpf_object__load(obj); if (!ASSERT_OK(err, "bpf_obj_load")) { err = -EIO; @@ -138,25 +156,25 @@ void test_connect_force_port(void) if (CHECK_FAIL(cgroup_fd < 0)) return; - server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 60123, 0); + server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0); if (CHECK_FAIL(server_fd < 0)) goto close_cgroup_fd; CHECK_FAIL(run_test(cgroup_fd, server_fd, AF_INET, SOCK_STREAM)); close(server_fd); - server_fd = start_server(AF_INET6, SOCK_STREAM, NULL, 60124, 0); + server_fd = start_server(AF_INET6, SOCK_STREAM, NULL, 0, 0); if (CHECK_FAIL(server_fd < 0)) goto close_cgroup_fd; CHECK_FAIL(run_test(cgroup_fd, server_fd, AF_INET6, SOCK_STREAM)); close(server_fd); - server_fd = start_server(AF_INET, SOCK_DGRAM, NULL, 60123, 0); + server_fd = start_server(AF_INET, SOCK_DGRAM, NULL, 0, 0); if (CHECK_FAIL(server_fd < 0)) goto close_cgroup_fd; CHECK_FAIL(run_test(cgroup_fd, server_fd, AF_INET, SOCK_DGRAM)); close(server_fd); - server_fd = start_server(AF_INET6, SOCK_DGRAM, NULL, 60124, 0); + server_fd = start_server(AF_INET6, SOCK_DGRAM, NULL, 0, 0); if (CHECK_FAIL(server_fd < 0)) goto close_cgroup_fd; CHECK_FAIL(run_test(cgroup_fd, server_fd, AF_INET6, SOCK_DGRAM)); diff --git a/tools/testing/selftests/bpf/prog_tests/core_reloc_raw.c b/tools/testing/selftests/bpf/prog_tests/core_reloc_raw.c index a18d3680fb16..51f42b02a267 100644 --- a/tools/testing/selftests/bpf/prog_tests/core_reloc_raw.c +++ b/tools/testing/selftests/bpf/prog_tests/core_reloc_raw.c @@ -14,6 +14,197 @@ static char log[16 * 1024]; +static int load_core_relo_insns(int btf_fd, struct bpf_insn *insns, int insn_cnt, + struct bpf_func_info *funcs, int func_cnt, + int enum_id, int access_str_off, int insn_idx, + bool relocate) +{ + struct bpf_core_relo relo = { + .insn_off = insn_idx * sizeof(struct bpf_insn), + .type_id = enum_id, + .access_str_off = access_str_off, + .kind = BPF_CORE_ENUMVAL_VALUE, + }; + union bpf_attr attr = { + .prog_type = BPF_PROG_TYPE_SOCKET_FILTER, + .insn_cnt = insn_cnt, + .insns = (__u64)insns, + .license = (__u64)"GPL", + .log_buf = (__u64)log, + .log_size = sizeof(log), + .log_level = 2, + .prog_btf_fd = btf_fd, + .func_info_rec_size = sizeof(struct bpf_func_info), + .func_info = (__u64)funcs, + .func_info_cnt = func_cnt, + }; + + if (relocate) { + attr.core_relo_cnt = 1; + attr.core_relos = (__u64)&relo; + attr.core_relo_rec_size = sizeof(relo); + } + memset(log, 0, sizeof(log)); + return sys_bpf_prog_load(&attr, sizeof(attr), 1); +} + +static void test_early_core_relo(void) +{ + static const char unrecognized[] = "trying to relocate unrecognized insn #2"; + static const struct { + const char *name; + struct bpf_insn insns[2]; + const char *err_msg; + } tests[] = { + { "poison_exit", { BPF_EXIT_INSN() }, unrecognized }, + { "poison_ja", { BPF_JMP_A(1) }, unrecognized }, + { "poison_jmp", { BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1) }, unrecognized }, + { "poison_jmp32", { BPF_JMP32_IMM(BPF_JEQ, BPF_REG_0, 0, 1) }, unrecognized }, + { "poison_call", { BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32) }, unrecognized }, + { "poison_alu_reg", { BPF_MOV32_REG(BPF_REG_0, BPF_REG_1) }, unrecognized }, + { "poison_alu64_reg", { BPF_MOV64_REG(BPF_REG_0, BPF_REG_1) }, unrecognized }, + { "poison_ld_abs", { BPF_LD_ABS(BPF_W, 0) }, + "insn #2 (LDIMM64) has unexpected form" }, + { "poison_alu_imm", { BPF_MOV32_IMM(BPF_REG_0, 0) } }, + { "poison_alu64_imm", { BPF_MOV64_IMM(BPF_REG_0, 0) } }, + { "poison_ldx", { BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0) } }, + { "poison_st", { BPF_ST_MEM(BPF_W, BPF_REG_10, -4, 0) } }, + { "poison_stx", { BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -4) } }, + { "poison_ldimm64", { BPF_LD_IMM64(BPF_REG_0, 0) } }, + }; + struct test_btf { + struct btf_header hdr; + __u32 types[18]; + char strings[64]; + } raw_btf = { + .hdr = { + .magic = BTF_MAGIC, + .version = BTF_VERSION, + .hdr_len = sizeof(struct btf_header), + .type_off = 0, + .type_len = sizeof(raw_btf.types), + .str_off = offsetof(struct test_btf, strings) - + offsetof(struct test_btf, types), + .str_len = sizeof(raw_btf.strings), + }, + .types = { + BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] int */ + BTF_FUNC_PROTO_ENC(1, 0), /* [2] int (*)(void) */ + BTF_FUNC_ENC(5, 2), /* [3] main_fn */ + BTF_FUNC_ENC(13, 2), /* [4] sub_fn */ + BTF_TYPE_ENC(20, BTF_INFO_ENC(BTF_KIND_ENUM, 0, 1), 4), /* [5] enum */ + BTF_ENUM_ENC(45, 0), /* value = 0 */ + }, + .strings = "\0int\0main_fn\0sub_fn\0core_relo_poison_missing\0value\0" "0", + }; + struct bpf_func_info funcs[] = { + { .insn_off = 0, .type_id = 3 }, + { .insn_off = 3, .type_id = 4 }, + }; + struct bpf_insn core_only[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + struct bpf_insn subprog[] = { + BPF_CALL_REL(2), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + struct bpf_insn truncated_ldimm64[] = { + BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, 0, 0, 0, 0), + }; + int access_str_off = 51; /* offset of "0" */ + int enum_id = 5; + int btf_fd, prog_fd = -1, i; + + btf_fd = bpf_btf_load(&raw_btf, sizeof(raw_btf), NULL); + if (!ASSERT_GE(btf_fd, 0, "btf_load")) + goto cleanup; + + if (test__start_subtest("without_func_info")) { + prog_fd = load_core_relo_insns(btf_fd, core_only, ARRAY_SIZE(core_only), NULL, 0, + enum_id, access_str_off, 2, false); + if (!ASSERT_GE(prog_fd, 0, "control_load")) + goto cleanup; + close(prog_fd); + prog_fd = load_core_relo_insns(btf_fd, core_only, ARRAY_SIZE(core_only), NULL, 0, + enum_id, access_str_off, 2, true); + if (!ASSERT_GE(prog_fd, 0, "poisoned_load")) + goto cleanup; + ASSERT_HAS_SUBSTR(log, "substituting insn #2", "poison_log"); + close(prog_fd); + prog_fd = -1; + } + + if (test__start_subtest("before_subprog_validation")) { + prog_fd = load_core_relo_insns(btf_fd, subprog, ARRAY_SIZE(subprog), funcs, 2, + enum_id, access_str_off, 2, true); + if (!ASSERT_LT(prog_fd, 0, "poisoned_load")) + goto cleanup; + ASSERT_HAS_SUBSTR(log, "substituting insn #2", "poison_log"); + ASSERT_HAS_SUBSTR(log, "last insn is not an exit or jmp", "poisoned_load_log"); + } + + if (test__start_subtest("truncated_ldimm64")) { + prog_fd = load_core_relo_insns(btf_fd, truncated_ldimm64, + ARRAY_SIZE(truncated_ldimm64), NULL, 0, + enum_id, access_str_off, 0, true); + if (!ASSERT_LT(prog_fd, 0, "truncated_load")) + goto cleanup; + ASSERT_HAS_SUBSTR(log, "invalid bpf_ld_imm64 insn", "truncated_load_log"); + } + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1), + tests[i].insns[0], + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + bool is_ldimm64 = insns[2].code == (BPF_LD | BPF_DW | BPF_IMM); + + if (!test__start_subtest(tests[i].name)) + continue; + if (is_ldimm64) { + insns[1].off = 2; + insns[3] = tests[i].insns[1]; + } + prog_fd = load_core_relo_insns(btf_fd, insns, ARRAY_SIZE(insns), funcs, 1, + enum_id, access_str_off, 2, false); + if (!ASSERT_GE(prog_fd, 0, "control_load")) + goto cleanup; + close(prog_fd); + prog_fd = load_core_relo_insns(btf_fd, insns, ARRAY_SIZE(insns), funcs, 1, + enum_id, access_str_off, 2, true); + if (!tests[i].err_msg) { + ASSERT_GE(prog_fd, 0, "dead_poison_load"); + ASSERT_HAS_SUBSTR(log, "substituting insn #2", "poison_log"); + if (is_ldimm64) + ASSERT_HAS_SUBSTR(log, "substituting insn #3", "poison_ldimm64_log"); + } else { + ASSERT_LT(prog_fd, 0, "invalid_poison_load"); + ASSERT_HAS_SUBSTR(log, tests[i].err_msg, "invalid_poison_log"); + ASSERT_NULL(strstr(log, "substituting insn"), "invalid_poison_substitution"); + } + close(prog_fd); + prog_fd = -1; + } + +cleanup: + if (env.verbosity > VERBOSE_NORMAL && log[0]) { + printf("-------- program load log start --------\n"); + printf("%s", log); + printf("-------- program load log end ----------\n"); + } + close(prog_fd); + close(btf_fd); +} + /* Check that verifier rejects BPF program containing relocation * pointing to non-existent BTF type. */ @@ -120,6 +311,7 @@ out: void test_core_reloc_raw(void) { + test_early_core_relo(); if (test__start_subtest("bad_local_id")) test_bad_local_id(); } diff --git a/tools/testing/selftests/bpf/prog_tests/ctx_rewrite.c b/tools/testing/selftests/bpf/prog_tests/ctx_rewrite.c index 469e92869523..2c3124092b73 100644 --- a/tools/testing/selftests/bpf/prog_tests/ctx_rewrite.c +++ b/tools/testing/selftests/bpf/prog_tests/ctx_rewrite.c @@ -69,19 +69,19 @@ static struct test_case test_cases[] = { #if defined(__x86_64__) || defined(__aarch64__) { N(SCHED_CLS, struct __sk_buff, tstamp), - .read = "r11 = *(u8 *)($ctx + sk_buff::__mono_tc_offset);" - "if w11 & 0x4 goto pc+1;" + .read = "r12 = *(u8 *)($ctx + sk_buff::__mono_tc_offset);" + "if w12 & 0x4 goto pc+1;" "goto pc+4;" - "if w11 & 0x3 goto pc+1;" + "if w12 & 0x3 goto pc+1;" "goto pc+2;" "$dst = 0;" "goto pc+1;" "$dst = *(u64 *)($ctx + sk_buff::tstamp);", - .write = "r11 = *(u8 *)($ctx + sk_buff::__mono_tc_offset);" - "if w11 & 0x4 goto pc+1;" + .write = "r12 = *(u8 *)($ctx + sk_buff::__mono_tc_offset);" + "if w12 & 0x4 goto pc+1;" "goto pc+2;" - "w11 &= -4;" - "*(u8 *)($ctx + sk_buff::__mono_tc_offset) = r11;" + "w12 &= -4;" + "*(u8 *)($ctx + sk_buff::__mono_tc_offset) = r12;" "*(u64 *)($ctx + sk_buff::tstamp) = $src;", }, #endif @@ -253,8 +253,7 @@ static int find_field_offset_aux(struct btf *btf, int btf_id, char *field_name, { const struct btf_type *type = btf__type_by_id(btf, btf_id); const struct btf_member *m; - __u16 mnum; - int i; + __u32 mnum, i; if (!type) { PRINT_FAIL("Can't find btf_type for id %d\n", btf_id); diff --git a/tools/testing/selftests/bpf/prog_tests/dynptr.c b/tools/testing/selftests/bpf/prog_tests/dynptr.c index 5fda11590708..4396560365e8 100644 --- a/tools/testing/selftests/bpf/prog_tests/dynptr.c +++ b/tools/testing/selftests/bpf/prog_tests/dynptr.c @@ -9,6 +9,7 @@ enum test_setup_type { SETUP_SYSCALL_SLEEP, SETUP_SKB_PROG, + SETUP_SKB_PROG_NONLINEAR, SETUP_SKB_PROG_TP, SETUP_XDP_PROG, }; @@ -32,6 +33,7 @@ static struct { {"test_ringbuf", SETUP_SYSCALL_SLEEP}, {"test_skb_readonly", SETUP_SKB_PROG}, {"test_dynptr_skb_data", SETUP_SKB_PROG}, + {"test_dynptr_skb_slice_non_linear", SETUP_SKB_PROG_NONLINEAR}, {"test_dynptr_skb_meta_data", SETUP_SKB_PROG}, {"test_dynptr_skb_meta_flags", SETUP_SKB_PROG}, {"test_adjust", SETUP_SYSCALL_SLEEP}, @@ -94,7 +96,9 @@ static void verify_success(const char *prog_name, enum test_setup_type setup_typ bpf_link__destroy(link); break; case SETUP_SKB_PROG: + case SETUP_SKB_PROG_NONLINEAR: { + struct __sk_buff ctx = {}; int prog_fd; char buf[64]; @@ -106,6 +110,12 @@ static void verify_success(const char *prog_name, enum test_setup_type setup_typ .repeat = 1, ); + if (setup_type == SETUP_SKB_PROG_NONLINEAR) { + ctx.data_end = ETH_HLEN + sizeof(struct iphdr); + topts.ctx_in = &ctx; + topts.ctx_size_in = sizeof(ctx); + } + prog_fd = bpf_program__fd(prog); if (!ASSERT_GE(prog_fd, 0, "prog_fd")) goto cleanup; diff --git a/tools/testing/selftests/bpf/prog_tests/empty_skb.c b/tools/testing/selftests/bpf/prog_tests/empty_skb.c index 438583e1f2d1..c9fcb70cbafb 100644 --- a/tools/testing/selftests/bpf/prog_tests/empty_skb.c +++ b/tools/testing/selftests/bpf/prog_tests/empty_skb.c @@ -10,8 +10,8 @@ void test_empty_skb(void) struct empty_skb *bpf_obj = NULL; struct nstoken *tok = NULL; struct bpf_program *prog; + struct ethhdr eth_hlen; char eth_hlen_pp[15]; - char eth_hlen[14]; int veth_ifindex; int ipip_ifindex; int err; @@ -25,7 +25,9 @@ void test_empty_skb(void) int err; int ret; int lwt_egress_ret; /* expected retval at lwt/egress */ + __be16 h_proto; bool success_on_tc; + bool adjust_room; } tests[] = { /* Empty packets are always rejected. */ @@ -46,6 +48,28 @@ void test_empty_skb(void) .err = -EINVAL, }, + /* ETH_HLEN-sized packets with IPv4/IPv6 EtherType but + * no L3 header are rejected. + */ + { + .msg = "veth short IPv4 ingress packet", + .data_in = ð_hlen, + .data_size_in = sizeof(eth_hlen), + .ifindex = &veth_ifindex, + .err = -EINVAL, + .h_proto = htons(ETH_P_IP), + .adjust_room = true, + }, + { + .msg = "veth short IPv6 ingress packet", + .data_in = ð_hlen, + .data_size_in = sizeof(eth_hlen), + .ifindex = &veth_ifindex, + .err = -EINVAL, + .h_proto = htons(ETH_P_IPV6), + .adjust_room = true, + }, + /* ETH_HLEN-sized packets: * - can not be redirected at LWT_XMIT * - can be redirected at TC to non-tunneling dest @@ -54,7 +78,7 @@ void test_empty_skb(void) { /* __bpf_redirect_common */ .msg = "veth ETH_HLEN packet ingress", - .data_in = eth_hlen, + .data_in = ð_hlen, .data_size_in = sizeof(eth_hlen), .ifindex = &veth_ifindex, .ret = -ERANGE, @@ -68,7 +92,7 @@ void test_empty_skb(void) * tc: skb->len=14 <= skb_network_offset=14 */ .msg = "ipip ETH_HLEN packet ingress", - .data_in = eth_hlen, + .data_in = ð_hlen, .data_size_in = sizeof(eth_hlen), .ifindex = &ipip_ifindex, .ret = -ERANGE, @@ -108,17 +132,27 @@ void test_empty_skb(void) SYS(out, "ip addr add 192.168.1.1/16 dev ipip0"); ipip_ifindex = if_nametoindex("ipip0"); + memset(eth_hlen_pp, 0, sizeof(eth_hlen_pp)); + memset(ð_hlen, 0, sizeof(eth_hlen)); + bpf_obj = empty_skb__open_and_load(); if (!ASSERT_OK_PTR(bpf_obj, "open skeleton")) goto out; for (i = 0; i < ARRAY_SIZE(tests); i++) { + if (tests[i].data_in == ð_hlen) + eth_hlen.h_proto = tests[i].h_proto; + bpf_object__for_each_program(prog, bpf_obj->obj) { bool at_egress = strstr(bpf_program__name(prog), "egress") != NULL; bool at_tc = !strncmp(bpf_program__section_name(prog), "tc", 2); + bool is_adjust_room = !strcmp(bpf_program__name(prog), "tc_adjust_room"); int expected_ret; char buf[128]; + if (tests[i].adjust_room != is_adjust_room) + continue; + expected_ret = at_egress && !at_tc ? tests[i].lwt_egress_ret : tests[i].ret; tattr.data_in = tests[i].data_in; diff --git a/tools/testing/selftests/bpf/prog_tests/exceptions.c b/tools/testing/selftests/bpf/prog_tests/exceptions.c index 516f4a13013c..639866ce09a9 100644 --- a/tools/testing/selftests/bpf/prog_tests/exceptions.c +++ b/tools/testing/selftests/bpf/prog_tests/exceptions.c @@ -55,6 +55,7 @@ static void test_exceptions_success(void) RUN_SUCCESS(exception_ext, 0); RUN_SUCCESS(exception_ext_mod_cb_runtime, 35); RUN_SUCCESS(exception_throw_subprog, 1); + RUN_SUCCESS(exception_throw_subprog_stack_cb, 0x1234); RUN_SUCCESS(exception_assert_nz_gfunc, 1); RUN_SUCCESS(exception_assert_zero_gfunc, 1); RUN_SUCCESS(exception_assert_neg_gfunc, 1); @@ -83,6 +84,14 @@ static void test_exceptions_success(void) RUN_SUCCESS(exception_assert_range_with, 1); RUN_SUCCESS(exception_bad_assert_range, 0); RUN_SUCCESS(exception_bad_assert_range_with, 10); + RUN_SUCCESS(exception_throw_from_void_global, 11); + + if (skel->rodata->has_stack_arg) { + RUN_SUCCESS(exception_throw_stack_arg, 56); + RUN_SUCCESS(exception_throw_after_stack_arg, 56); + RUN_SUCCESS(exception_throw_subprog_stack_arg, 56); + RUN_SUCCESS(exception_throw_subprog_after_stack_arg, 56); + } #define RUN_EXT(load_ret, attach_err, expr, msg, after_link) \ { \ @@ -127,7 +136,7 @@ static void test_exceptions_success(void) bpf_program__fd(skel->progs.exception_ext_mod_cb_runtime), "exception_cb_mod"), "set_attach_target")) goto done; - }), "FENTRY/FEXIT programs cannot attach to exception callback", 0); + }), "Tracing programs cannot attach to exception callback", 0); if (test__start_subtest("throwing fentry -> exception_cb")) RUN_EXT(-EINVAL, true, ({ @@ -137,7 +146,7 @@ static void test_exceptions_success(void) bpf_program__fd(skel->progs.exception_ext_mod_cb_runtime), "exception_cb_mod"), "set_attach_target")) goto done; - }), "FENTRY/FEXIT programs cannot attach to exception callback", 0); + }), "Tracing programs cannot attach to exception callback", 0); if (test__start_subtest("non-throwing fexit -> exception_cb")) RUN_EXT(-EINVAL, true, ({ @@ -147,7 +156,7 @@ static void test_exceptions_success(void) bpf_program__fd(skel->progs.exception_ext_mod_cb_runtime), "exception_cb_mod"), "set_attach_target")) goto done; - }), "FENTRY/FEXIT programs cannot attach to exception callback", 0); + }), "Tracing programs cannot attach to exception callback", 0); if (test__start_subtest("throwing fexit -> exception_cb")) RUN_EXT(-EINVAL, true, ({ @@ -157,7 +166,7 @@ static void test_exceptions_success(void) bpf_program__fd(skel->progs.exception_ext_mod_cb_runtime), "exception_cb_mod"), "set_attach_target")) goto done; - }), "FENTRY/FEXIT programs cannot attach to exception callback", 0); + }), "Tracing programs cannot attach to exception callback", 0); if (test__start_subtest("throwing extension (with custom cb) -> exception_cb")) RUN_EXT(-EINVAL, true, ({ diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c index f29fc789c14b..2523c07a16c6 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c @@ -5,6 +5,7 @@ #include <bpf/btf.h> #include "bind4_prog.skel.h" #include "freplace_progmap.skel.h" +#include "fentry_sleepable.skel.h" #include "xdp_dummy.skel.h" typedef int (*test_cb)(struct bpf_object *obj); @@ -111,7 +112,7 @@ static void test_fexit_bpf2bpf_common(const char *obj_file, struct bpf_link_info link_info; struct bpf_program *pos; const char *pos_sec_name; - char *tgt_name; + const char *tgt_name; __s32 btf_id; tgt_name = strstr(prog_name[i], "/"); @@ -335,14 +336,13 @@ out: bpf_object__close(pkt_obj); } - -static void test_func_sockmap_update(void) +static void test_func_replace_void(void) { const char *prog_name[] = { - "freplace/cls_redirect", + "freplace/foo", }; - test_fexit_bpf2bpf_common("./freplace_cls_redirect.bpf.o", - "./test_cls_redirect.bpf.o", + test_fexit_bpf2bpf_common("./freplace_void.bpf.o", + "./test_global_func7.bpf.o", ARRAY_SIZE(prog_name), prog_name, false, NULL); } @@ -432,6 +432,15 @@ static void test_func_replace_global_func(void) prog_name, false, NULL); } +static void test_func_replace_int_with_void(void) +{ + /* Make sure we can't freplace with the wrong type */ + test_obj_load_failure_common("freplace_int_with_void.bpf.o", + "./test_global_func2.bpf.o", + "Return type UNKNOWN of test_freplace_int_with_void()" + " doesn't match type INT of global_func2()"); +} + static int find_prog_btf_id(const char *name, __u32 attach_prog_fd) { struct bpf_prog_info info = {}; @@ -568,6 +577,60 @@ out: freplace_progmap__destroy(skel); } +static void test_sleepable_fentry_to_xdp(void) +{ + struct fentry_sleepable *skel = NULL; + struct xdp_dummy *skel_xdp = NULL; + int ifindex, prog_fd, err; + char buff[64] = {}; + +#ifndef __x86_64__ + test__skip(); + return; +#endif + + ifindex = if_nametoindex("lo"); + if (!ASSERT_GT(ifindex, 0, "if_nametoindex")) + return; + + skel_xdp = xdp_dummy__open_and_load(); + if (!ASSERT_OK_PTR(skel_xdp, "xdp_dummy__open_and_load")) + return; + + skel = fentry_sleepable__open(); + if (!ASSERT_OK_PTR(skel, "fentry_sleepable__open")) + goto out; + + skel->bss->user_ptr = buff; + + prog_fd = bpf_program__fd(skel_xdp->progs.__x64_sys_nop); + err = bpf_program__set_attach_target(skel->progs.fentry_xdp, prog_fd, "__x64_sys_nop"); + if (!ASSERT_OK(err, "bpf_program__set_attach_target")) + goto out; + + err = fentry_sleepable__load(skel); + ASSERT_ERR(err, "fentry_sleepable__load"); + if (err) + goto out; + + skel->links.fentry_xdp = bpf_program__attach_trace(skel->progs.fentry_xdp); + if (!ASSERT_OK_PTR(skel->links.fentry_xdp, "bpf_program__attach_trace")) + goto out; + + skel_xdp->links.__x64_sys_nop = bpf_program__attach_xdp(skel_xdp->progs.__x64_sys_nop, + ifindex); + if (!ASSERT_OK_PTR(skel_xdp->links.__x64_sys_nop, "bpf_program__attach_xdp")) + goto out; + + err = system("ping -q -c 1 -W 1 127.0.0.1 > /dev/null"); + ASSERT_OK(err, "ping"); + ASSERT_ERR(skel->bss->retval, "retval"); + +out: + fentry_sleepable__destroy(skel); + xdp_dummy__destroy(skel_xdp); +} + /* NOTE: affect other tests, must run in serial mode */ void serial_test_fexit_bpf2bpf(void) { @@ -579,8 +642,6 @@ void serial_test_fexit_bpf2bpf(void) test_func_replace(); if (test__start_subtest("func_replace_verify")) test_func_replace_verify(); - if (test__start_subtest("func_sockmap_update")) - test_func_sockmap_update(); if (test__start_subtest("func_replace_return_code")) test_func_replace_return_code(); if (test__start_subtest("func_map_prog_compatibility")) @@ -597,4 +658,10 @@ void serial_test_fexit_bpf2bpf(void) test_fentry_to_cgroup_bpf(); if (test__start_subtest("func_replace_progmap")) test_func_replace_progmap(); + if (test__start_subtest("freplace_int_with_void")) + test_func_replace_int_with_void(); + if (test__start_subtest("freplace_void")) + test_func_replace_void(); + if (test__start_subtest("sleepable_fentry_to_xdp")) + test_sleepable_fentry_to_xdp(); } diff --git a/tools/testing/selftests/bpf/prog_tests/fib_lookup.c b/tools/testing/selftests/bpf/prog_tests/fib_lookup.c index bd7658958004..8f4779dd802e 100644 --- a/tools/testing/selftests/bpf/prog_tests/fib_lookup.c +++ b/tools/testing/selftests/bpf/prog_tests/fib_lookup.c @@ -2,6 +2,7 @@ /* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ #include <linux/rtnetlink.h> +#include <linux/if_ether.h> #include <sys/types.h> #include <net/if.h> @@ -23,6 +24,7 @@ #define IPV4_TBID_ADDR "172.0.0.254" #define IPV4_TBID_NET "172.0.0.0" #define IPV4_TBID_DST "172.0.0.2" +#define IPV4_TBID_NONEIGH_DST "172.0.0.5" #define IPV6_TBID_ADDR "fd00::FFFF" #define IPV6_TBID_NET "fd00::" #define IPV6_TBID_DST "fd00::2" @@ -37,6 +39,41 @@ #define IPV6_LOCAL "fd01::3" #define IPV6_GW1 "fd01::1" #define IPV6_GW2 "fd01::2" +#define VLAN_ID 100 +#define VLAN_IFACE "veth1.100" +#define VLAN_ID_DOWN 102 +#define VLAN_IFACE_DOWN "veth1.102" +#define QINQ_OUTER_IFACE "veth1.200" +#define QINQ_INNER_IFACE "veth1.200.300" +#define VLAN_TABLE "300" +#define IPV4_VLAN_IFACE_ADDR "10.5.0.254" +#define IPV4_VLAN_EGRESS_DST "10.5.0.2" +#define IPV4_QINQ_DST "10.7.0.2" +#define IPV4_VLAN_DST "10.6.0.2" +#define IPV4_VLAN_GW "10.5.0.1" +#define IPV6_VLAN_IFACE_ADDR "fd02::254" +#define IPV6_VLAN_EGRESS_DST "fd02::2" +#define IPV6_VLAN_DST "fd03::2" +#define IPV6_VLAN_GW "fd02::1" +#define VLAN_VID_UNUSED 999 +#define VRF_IFACE "vrf-blue" +#define VRF_TABLE "1000" +#define VRF_VLAN_ID 101 +#define VRF_VLAN_IFACE "veth1.101" +#define IPV4_VRF_IFACE_ADDR "10.8.0.254" +#define IPV4_VRF_GW "10.8.0.1" +#define IPV4_VRF_DST "10.9.0.2" +#define TBID_VLAN_ID 50 +#define TBID_VLAN_IFACE "veth2.50" +#define IPV4_TBID_VLAN_DST "172.2.0.2" +#define IPV4_BOND_VLAN_DST "10.11.0.2" +#define IPV4_VLAN_MTU_DST "10.5.9.2" +#define QINQ_AD_VLAN_ID 200 +#define QINQ_INNER_VLAN_ID 300 +#define BOND_IFACE "bond99" +#define BOND_PORT "veth3" +#define BOND_PORT_PEER "veth4" +#define BOND_VLAN_ID 500 #define DMAC "11:11:11:11:11:11" #define DMAC_INIT { 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, } #define DMAC2 "01:01:01:01:01:01" @@ -52,6 +89,17 @@ struct fib_lookup_test { __u32 tbid; __u8 dmac[6]; __u32 mark; + /* + * input tag with BPF_FIB_LOOKUP_VLAN_INPUT; expected output tag + * with BPF_FIB_LOOKUP_VLAN (checked when check_vlan is set) + */ + __u16 vlan_proto; + __u16 vlan_id; + bool check_vlan; + const char *expected_dev; /* expected params->ifindex after lookup */ + const char *iif; /* override the default veth1 input device */ + __u16 tot_len; /* triggers the in-lookup mtu check when set */ + __u16 expected_mtu; /* expected mtu_result (union with tot_len) */ }; static const struct fib_lookup_test tests[] = { @@ -79,6 +127,17 @@ static const struct fib_lookup_test tests[] = { .daddr = IPV4_TBID_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, .lookup_flags = BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID, .tbid = 100, .dmac = DMAC_INIT2, }, + /* + * An error that returns after the egress device is resolved must + * report the egress ifindex, not the input. This routes from input + * veth1 via veth2 (table 100) to a dst with no neighbour, so + * input != egress, pinning NO_NEIGH to the egress device. + */ + { .desc = "IPv4 NO_NEIGH reports the egress ifindex, not the input", + .daddr = IPV4_TBID_NONEIGH_DST, + .expected_ret = BPF_FIB_LKUP_RET_NO_NEIGH, + .lookup_flags = BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID, .tbid = 100, + .expected_dev = "veth2", }, { .desc = "IPv6 TBID lookup failure", .daddr = IPV6_TBID_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, .lookup_flags = BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID, @@ -142,12 +201,237 @@ static const struct fib_lookup_test tests[] = { .expected_dst = IPV6_GW1, .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, .mark = MARK, }, + /* vlan egress resolution */ + /* + * Invariant the VLAN-egress arms jointly enforce: a + * BPF_FIB_LOOKUP_VLAN SUCCESS always carries a physical, + * xmit-capable ifindex; no SUCCESS ever returns a VLAN-device + * ifindex. Reducible arms pin ifindex == the physical parent; the + * QinQ and foreign-netns arms pin VLAN_FAILURE with params->ifindex + * left at the input, so a regression to best-effort (SUCCESS + the + * VLAN ifindex) fails one. + */ + { .desc = "IPv4 VLAN egress, no flag", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = VLAN_IFACE, .check_vlan = true, }, + { .desc = "IPv4 VLAN egress, single VLAN", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + /* + * skb path without tot_len: mtu_result is the VLAN device's mtu + * (1400), not the parent's (1500) + */ + { .desc = "IPv4 VLAN egress, skb-path mtu is the VLAN device's without the flag", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = VLAN_IFACE, .check_vlan = true, .expected_mtu = 1400, }, + { .desc = "IPv4 VLAN egress, flag set but egress is not a VLAN", + .daddr = IPV4_NUD_FAILED_ADDR, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, }, + { .desc = "IPv4 VLAN egress, QinQ not reducible (VLAN_FAILURE)", + .daddr = IPV4_QINQ_DST, + .expected_ret = BPF_FIB_LKUP_RET_VLAN_FAILURE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, }, + { .desc = "IPv4 QinQ egress without the flag (escape hatch)", + .daddr = IPV4_QINQ_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = QINQ_INNER_IFACE, }, + { .desc = "IPv6 VLAN egress, single VLAN", + .daddr = IPV6_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN egress, neighbour on the VLAN device", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, .dmac = DMAC_INIT, }, + { .desc = "IPv4 VLAN egress in OUTPUT mode", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .iif = VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_OUTPUT | BPF_FIB_LOOKUP_VLAN | + BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN egress over a bond", + .daddr = IPV4_BOND_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = BOND_IFACE, .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = BOND_VLAN_ID, }, + { .desc = "IPv4 VLAN egress via TBID table", + .daddr = IPV4_TBID_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID | + BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .tbid = 100, + .expected_dev = "veth2", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = TBID_VLAN_ID, }, + { .desc = "IPv4 VLAN egress, success writes mtu_result with the swap", + .daddr = IPV4_VLAN_MTU_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .tot_len = 500, .expected_mtu = 1000, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN egress, FRAG_NEEDED reports mtu, swap unwritten", + .daddr = IPV4_VLAN_MTU_DST, .expected_ret = BPF_FIB_LKUP_RET_FRAG_NEEDED, + .tot_len = 1400, .expected_mtu = 1000, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, }, + /* vlan tag as lookup input */ + { .desc = "IPv4 VLAN input, no flag", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_GW1, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, }, + { .desc = "IPv4 VLAN input, tag selects subinterface route", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VLAN_GW, .expected_dev = VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv6 VLAN input, tag selects subinterface route", + .daddr = IPV6_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV6_VLAN_GW, .expected_dev = VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input and egress combined", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VLAN_GW, .expected_dev = "veth1", + .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_VLAN | + BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input, neighbour resolved on the route", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VLAN_GW, .expected_dev = VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, .dmac = DMAC_INIT2, }, + { .desc = "IPv4 VLAN input, source address from the subinterface", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_src = IPV4_VLAN_IFACE_ADDR, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SRC | + BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + /* + * VRF: the resolved subinterface is enslaved, so the l3mdev rule + * (full lookup) and l3mdev_fib_table_rcu() (DIRECT) must select + * the VRF table from the resolved ingress + */ + { .desc = "IPv4 VLAN input, VRF subinterface, no flag", + .daddr = IPV4_VRF_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_GW1, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, }, + { .desc = "IPv4 VLAN input, tag selects VRF table", + .daddr = IPV4_VRF_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VRF_GW, .expected_dev = VRF_VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VRF_VLAN_ID, }, + { .desc = "IPv4 VLAN input, DIRECT uses VRF table from resolved ingress", + .daddr = IPV4_VRF_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VRF_GW, .expected_dev = VRF_VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_DIRECT | + BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VRF_VLAN_ID, }, + /* + * failure arms also assert params is left untouched: ifindex still + * names the physical device and the input tag bytes survive + */ + { .desc = "IPv4 VLAN input, invalid proto", + .daddr = IPV4_VLAN_DST, .expected_ret = -EINVAL, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = 0x1234, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input, unmatched VID", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_VID_UNUSED, }, + { .desc = "IPv4 VLAN input, subinterface down", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID_DOWN, }, + /* + * the resolver runs before the forwarding check, so on devices + * with forwarding off FWD_DISABLED (not NOT_FWDED) proves the tag + * resolved to that device and the lookup used it as ingress + */ + { .desc = "IPv4 VLAN input, 802.1ad tag", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_FWD_DISABLED, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021AD, .vlan_id = QINQ_AD_VLAN_ID, }, + { .desc = "IPv4 VLAN input, PCP and DEI bits ignored in TCI", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VLAN_GW, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = 0xe000 | VLAN_ID, }, + { .desc = "IPv4 VLAN input, inner QinQ device from VLAN ifindex", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_FWD_DISABLED, + .iif = QINQ_OUTER_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = QINQ_INNER_VLAN_ID, }, + /* + * bonding: the VLANs live on the master, as on receive, where the + * frame is steered to the master before VLAN processing; a port + * ifindex does not match (ports carry vid state but no VLAN devs) + */ + { .desc = "IPv4 VLAN input, tag on bond master resolves", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_FWD_DISABLED, + .iif = BOND_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = BOND_VLAN_ID, }, + { .desc = "IPv4 VLAN input, tag on bond port does not match", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .iif = BOND_PORT, .expected_dev = BOND_PORT, .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = BOND_VLAN_ID, }, + { .desc = "IPv6 VLAN input, invalid proto", + .daddr = IPV6_VLAN_DST, .expected_ret = -EINVAL, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = 0x1234, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input, VID 0 priority tag fails closed", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = 0, }, + { .desc = "IPv6 VLAN input, unmatched VID", + .daddr = IPV6_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_VID_UNUSED, }, + { .desc = "unknown flag bit rejected", + .daddr = IPV4_VLAN_DST, .expected_ret = -EINVAL, + .lookup_flags = (1 << 14) | BPF_FIB_LOOKUP_SKIP_NEIGH, }, + { .desc = "IPv4 VLAN input rejected with TBID", + .daddr = IPV4_VLAN_DST, .expected_ret = -EINVAL, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_TBID, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input rejected with OUTPUT", + .daddr = IPV4_VLAN_DST, .expected_ret = -EINVAL, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_OUTPUT, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, }; static int setup_netns(void) { int err; + /* + * a new netns copies the IPv4 conf from init_net, so on a host with + * forwarding enabled the arms that expect FWD_DISABLED would see the + * lookup succeed instead; pin it off here and enable it per device + */ + err = write_sysctl("/proc/sys/net/ipv4/conf/all/forwarding", "0"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.conf.all.forwarding)")) + goto fail; + + err = write_sysctl("/proc/sys/net/ipv4/conf/default/forwarding", "0"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.conf.default.forwarding)")) + goto fail; + SYS(fail, "ip link add veth1 type veth peer name veth2"); SYS(fail, "ip link set dev veth1 up"); SYS(fail, "ip link set dev veth2 up"); @@ -204,6 +488,105 @@ static int setup_netns(void) SYS(fail, "ip rule add prio 2 fwmark %d lookup %s", MARK, MARK_TABLE); SYS(fail, "ip -6 rule add prio 2 fwmark %d lookup %s", MARK, MARK_TABLE); + /* + * Setup for vlan tests: a subinterface for egress resolution and + * tag-as-input, a QinQ stack, and an iif rule so the input tests + * observe which device the lookup used as ingress. + */ + SYS(fail, "ip link add link veth1 name %s type vlan id %d", + VLAN_IFACE, VLAN_ID); + SYS(fail, "ip link set dev %s up", VLAN_IFACE); + /* + * lower than the veth1 parent (1500): the skb-path mtu check uses the + * FIB result (VLAN) device, so mtu_result is this value, which the + * no-flag arm below pins + */ + SYS(fail, "ip link set dev %s mtu 1400", VLAN_IFACE); + SYS(fail, "ip addr add %s/24 dev %s", IPV4_VLAN_IFACE_ADDR, VLAN_IFACE); + SYS(fail, "ip addr add %s/64 dev %s nodad", IPV6_VLAN_IFACE_ADDR, VLAN_IFACE); + + /* + * stays down: the input flag must treat its tag the way real + * ingress treats a frame arriving on a down VLAN device (drop) + */ + SYS(fail, "ip link add link veth1 name %s type vlan id %d", + VLAN_IFACE_DOWN, VLAN_ID_DOWN); + + err = write_sysctl("/proc/sys/net/ipv4/conf/" VLAN_IFACE "/forwarding", "1"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.conf." VLAN_IFACE ".forwarding)")) + goto fail; + + err = write_sysctl("/proc/sys/net/ipv6/conf/" VLAN_IFACE "/forwarding", "1"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv6.conf." VLAN_IFACE ".forwarding)")) + goto fail; + + SYS(fail, "ip link add link veth1 name %s type vlan proto 802.1ad id 200", + QINQ_OUTER_IFACE); + SYS(fail, "ip link add link %s name %s type vlan id 300", + QINQ_OUTER_IFACE, QINQ_INNER_IFACE); + SYS(fail, "ip link set dev %s up", QINQ_OUTER_IFACE); + SYS(fail, "ip link set dev %s up", QINQ_INNER_IFACE); + SYS(fail, "ip route add %s/32 dev %s", IPV4_QINQ_DST, QINQ_INNER_IFACE); + + SYS(fail, "ip route add %s/32 via %s", IPV4_VLAN_DST, IPV4_GW1); + SYS(fail, "ip route add table %s %s/32 via %s", + VLAN_TABLE, IPV4_VLAN_DST, IPV4_VLAN_GW); + SYS(fail, "ip rule add prio 3 iif %s lookup %s", VLAN_IFACE, VLAN_TABLE); + SYS(fail, "ip -6 route add %s/128 via %s", IPV6_VLAN_DST, IPV6_GW1); + SYS(fail, "ip -6 route add table %s %s/128 via %s", + VLAN_TABLE, IPV6_VLAN_DST, IPV6_VLAN_GW); + SYS(fail, "ip -6 rule add prio 3 iif %s lookup %s", VLAN_IFACE, VLAN_TABLE); + + /* a bond with one port and a VLAN on the bond */ + SYS(fail, "ip link add %s type bond", BOND_IFACE); + SYS(fail, "ip link add %s type veth peer name %s", BOND_PORT, BOND_PORT_PEER); + SYS(fail, "ip link set %s master %s", BOND_PORT, BOND_IFACE); + SYS(fail, "ip link set dev %s up", BOND_IFACE); + SYS(fail, "ip link set dev %s up", BOND_PORT); + SYS(fail, "ip link add link %s name %s.%d type vlan id %d", + BOND_IFACE, BOND_IFACE, BOND_VLAN_ID, BOND_VLAN_ID); + SYS(fail, "ip link set dev %s.%d up", BOND_IFACE, BOND_VLAN_ID); + SYS(fail, "ip route add %s/32 dev %s.%d", + IPV4_BOND_VLAN_DST, BOND_IFACE, BOND_VLAN_ID); + + /* + * a VRF with its own dedicated subinterface (the iif rules above + * must not see it), for the table-selection-by-ingress cases + */ + SYS(fail, "ip link add %s type vrf table %s", VRF_IFACE, VRF_TABLE); + SYS(fail, "ip link set dev %s up", VRF_IFACE); + SYS(fail, "ip link add link veth1 name %s type vlan id %d", + VRF_VLAN_IFACE, VRF_VLAN_ID); + SYS(fail, "ip link set %s master %s", VRF_VLAN_IFACE, VRF_IFACE); + SYS(fail, "ip link set dev %s up", VRF_VLAN_IFACE); + SYS(fail, "ip addr add %s/24 dev %s", IPV4_VRF_IFACE_ADDR, VRF_VLAN_IFACE); + err = write_sysctl("/proc/sys/net/ipv4/conf/" VRF_VLAN_IFACE "/forwarding", "1"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.conf." VRF_VLAN_IFACE ".forwarding)")) + goto fail; + SYS(fail, "ip route add %s/32 via %s", IPV4_VRF_DST, IPV4_GW1); + SYS(fail, "ip route add table %s %s/32 via %s", + VRF_TABLE, IPV4_VRF_DST, IPV4_VRF_GW); + + /* neighbours on the VLAN subinterface for the non-SKIP_NEIGH cases */ + err = write_sysctl("/proc/sys/net/ipv4/neigh/" VLAN_IFACE "/gc_stale_time", "900"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.neigh." VLAN_IFACE ".gc_stale_time)")) + goto fail; + SYS(fail, "ip neigh add %s dev %s lladdr %s nud stale", + IPV4_VLAN_EGRESS_DST, VLAN_IFACE, DMAC); + SYS(fail, "ip neigh add %s dev %s lladdr %s nud stale", + IPV4_VLAN_GW, VLAN_IFACE, DMAC2); + + /* a VLAN on veth2 with a route in the tbid test table */ + SYS(fail, "ip link add link veth2 name %s type vlan id %d", + TBID_VLAN_IFACE, TBID_VLAN_ID); + SYS(fail, "ip link set dev %s up", TBID_VLAN_IFACE); + SYS(fail, "ip route add table 100 %s/32 dev %s", + IPV4_TBID_VLAN_DST, TBID_VLAN_IFACE); + + /* a locked-mtu route via the subinterface for the FRAG_NEEDED case */ + SYS(fail, "ip route add %s/32 dev %s mtu lock 1000", + IPV4_VLAN_MTU_DST, VLAN_IFACE); + return 0; fail: return -1; @@ -218,9 +601,16 @@ static int set_lookup_params(struct bpf_fib_lookup *params, memset(params, 0, sizeof(*params)); params->l4_protocol = IPPROTO_TCP; - params->ifindex = ifindex; + params->ifindex = test->iif ? if_nametoindex(test->iif) : ifindex; params->tbid = test->tbid; params->mark = test->mark; + params->tot_len = test->tot_len; + + /* h_vlan_proto/h_vlan_TCI union with tbid */ + if (test->lookup_flags & BPF_FIB_LOOKUP_VLAN_INPUT) { + params->h_vlan_proto = htons(test->vlan_proto); + params->h_vlan_TCI = htons(test->vlan_id); + } if (inet_pton(AF_INET6, test->daddr, params->ipv6_dst) == 1) { params->family = AF_INET6; @@ -298,7 +688,7 @@ void test_fib_lookup(void) struct nstoken *nstoken = NULL; struct __sk_buff skb = { }; struct fib_lookup *skel; - int prog_fd, err, ret, i; + int prog_fd, xdp_fd, err, ret, i; /* The test does not use the skb->data, so * use pkt_v6 for both v6 and v4 test. @@ -309,11 +699,16 @@ void test_fib_lookup(void) .ctx_in = &skb, .ctx_size_in = sizeof(skb), ); + LIBBPF_OPTS(bpf_test_run_opts, xdp_opts, + .data_in = &pkt_v6, + .data_size_in = sizeof(pkt_v6), + ); skel = fib_lookup__open_and_load(); if (!ASSERT_OK_PTR(skel, "skel open_and_load")) return; prog_fd = bpf_program__fd(skel->progs.fib_lookup); + xdp_fd = bpf_program__fd(skel->progs.fib_lookup_xdp); SYS(fail, "ip netns add %s", NS_TEST); @@ -343,6 +738,16 @@ void test_fib_lookup(void) if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) continue; + /* + * BPF_FIB_LOOKUP_VLAN is XDP-only; the tc helper rejects it. + * These cases are exercised on the XDP path below. + */ + if (tests[i].lookup_flags & BPF_FIB_LOOKUP_VLAN) { + ASSERT_EQ(skel->bss->fib_lookup_ret, -EINVAL, + "tc rejects BPF_FIB_LOOKUP_VLAN"); + continue; + } + ASSERT_EQ(skel->bss->fib_lookup_ret, tests[i].expected_ret, "fib_lookup_ret"); @@ -352,6 +757,21 @@ void test_fib_lookup(void) if (tests[i].expected_dst) assert_dst_ip(fib_params, tests[i].expected_dst); + if (tests[i].expected_dev) + ASSERT_EQ(fib_params->ifindex, + if_nametoindex(tests[i].expected_dev), "ifindex"); + + if (tests[i].expected_mtu) + ASSERT_EQ(fib_params->mtu_result, tests[i].expected_mtu, + "mtu_result"); + + if (tests[i].check_vlan) { + ASSERT_EQ(fib_params->h_vlan_proto, + htons(tests[i].vlan_proto), "h_vlan_proto"); + ASSERT_EQ(fib_params->h_vlan_TCI, + htons(tests[i].vlan_id), "h_vlan_TCI"); + } + ret = memcmp(tests[i].dmac, fib_params->dmac, sizeof(tests[i].dmac)); if (!ASSERT_EQ(ret, 0, "dmac not match")) { char expected[18], actual[18]; @@ -361,17 +781,330 @@ void test_fib_lookup(void) printf("dmac expected %s actual %s ", expected, actual); } - // ensure tbid is zero'd out after fib lookup. - if (tests[i].lookup_flags & BPF_FIB_LOOKUP_DIRECT) { + /* + * ensure tbid is zero'd out after fib lookup. With + * BPF_FIB_LOOKUP_VLAN the union holds the packed vlan + * fields instead, so skip the check for those. + */ + if ((tests[i].lookup_flags & BPF_FIB_LOOKUP_DIRECT) && + !(tests[i].lookup_flags & BPF_FIB_LOOKUP_VLAN)) { if (!ASSERT_EQ(skel->bss->fib_params.tbid, 0, "expected fib_params.tbid to be zero")) goto fail; } } + /* + * Re-run the cases through bpf_xdp_fib_lookup(). test_run uses the + * current netns' loopback for ctx->rxq->dev, so dev_net() is NS_TEST + * and the lookup runs against its FIB. The path-independent results + * (return code, swapped ifindex, vlan tag, gateway) must match the skb + * path; the no-tot_len mtu_result is skb-specific and not rechecked. + */ + for (i = 0; i < ARRAY_SIZE(tests); i++) { + if (set_lookup_params(fib_params, &tests[i], skb.ifindex)) + continue; + + skel->bss->fib_lookup_ret = -1; + skel->bss->lookup_flags = tests[i].lookup_flags; + + err = bpf_prog_test_run_opts(xdp_fd, &xdp_opts); + if (!ASSERT_OK(err, "xdp test_run")) + continue; + + if (!ASSERT_EQ(skel->bss->fib_lookup_ret, tests[i].expected_ret, + "xdp fib_lookup_ret")) + printf("(xdp) %s\n", tests[i].desc); + + if (tests[i].expected_dev) + ASSERT_EQ(fib_params->ifindex, + if_nametoindex(tests[i].expected_dev), + "xdp ifindex"); + + if (tests[i].expected_dst) + assert_dst_ip(fib_params, tests[i].expected_dst); + + if (tests[i].check_vlan) { + ASSERT_EQ(fib_params->h_vlan_proto, + htons(tests[i].vlan_proto), "xdp h_vlan_proto"); + ASSERT_EQ(fib_params->h_vlan_TCI, + htons(tests[i].vlan_id), "xdp h_vlan_TCI"); + } + + ret = memcmp(tests[i].dmac, fib_params->dmac, sizeof(tests[i].dmac)); + ASSERT_EQ(ret, 0, "xdp dmac"); + + /* + * mtu_result from a tot_len lookup is the route mtu and is + * path-independent; the no-tot_len arm reads dev->mtu and is + * skb-only, so gate on tot_len + */ + if (tests[i].expected_mtu && tests[i].tot_len) + ASSERT_EQ(fib_params->mtu_result, tests[i].expected_mtu, + "xdp mtu_result"); + } + fail: if (nstoken) close_netns(nstoken); SYS_NOFAIL("ip netns del " NS_TEST); fib_lookup__destroy(skel); } + +#define NS_VLAN_A "fib_lookup_vlan_ns_a" +#define NS_VLAN_B "fib_lookup_vlan_ns_b" +#define IPV4_VLAN_NETNS_ADDR "10.66.0.1" +#define IPV4_VLAN_NETNS_DST "10.66.0.2" + +/* + * A VLAN device can be moved to another netns while staying registered + * on its parent. Neither direction may then cross the boundary: the + * egress flag must not publish the foreign parent's ifindex, and the + * input flag must fail closed rather than use a foreign ingress. + */ +void test_fib_lookup_vlan_netns(void) +{ + struct bpf_fib_lookup *fib_params; + struct nstoken *nstoken = NULL; + struct __sk_buff skb = { }; + struct fib_lookup *skel = NULL; + int prog_fd, xdp_fd, err, parent_idx, vlan_idx; + + LIBBPF_OPTS(bpf_test_run_opts, run_opts, + .data_in = &pkt_v6, + .data_size_in = sizeof(pkt_v6), + .ctx_in = &skb, + .ctx_size_in = sizeof(skb), + ); + LIBBPF_OPTS(bpf_test_run_opts, xdp_opts, + .data_in = &pkt_v6, + .data_size_in = sizeof(pkt_v6), + ); + + skel = fib_lookup__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + return; + prog_fd = bpf_program__fd(skel->progs.fib_lookup); + xdp_fd = bpf_program__fd(skel->progs.fib_lookup_xdp); + fib_params = &skel->bss->fib_params; + + SYS(fail, "ip netns add %s", NS_VLAN_A); + SYS(fail, "ip netns add %s", NS_VLAN_B); + + nstoken = open_netns(NS_VLAN_A); + if (!ASSERT_OK_PTR(nstoken, "open_netns(a)")) + goto fail; + + SYS(fail, "ip link add veth7 type veth peer name veth8"); + SYS(fail, "ip link set dev veth7 up"); + SYS(fail, "ip link add link veth7 name veth7.66 type vlan id 66"); + SYS(fail, "ip link set veth7.66 netns %s", NS_VLAN_B); + /* + * up it in B before the input lookup: the move closed it, and a + * down device fails the resolver on IFF_UP before reaching the + * netns check this subtest exists to pin + */ + SYS(fail, "ip -n %s link set dev veth7.66 up", NS_VLAN_B); + + parent_idx = if_nametoindex("veth7"); + if (!ASSERT_NEQ(parent_idx, 0, "if_nametoindex(veth7)")) + goto fail; + + /* + * give this netns a route to the destination: the lookup below runs + * against this FIB, so without the route a kernel that resolved the + * moved device anyway would still return NOT_FWDED and the arm would + * pass for the wrong reason + */ + SYS(fail, "ip route add %s/32 dev veth7", IPV4_VLAN_NETNS_DST); + + /* + * input: the moved device is still in veth7's VLAN group, but it + * lives in another netns, so the lookup must fail closed + */ + skb.ifindex = parent_idx; + memset(fib_params, 0, sizeof(*fib_params)); + fib_params->family = AF_INET; + fib_params->l4_protocol = IPPROTO_TCP; + fib_params->ifindex = parent_idx; + fib_params->h_vlan_proto = htons(ETH_P_8021Q); + fib_params->h_vlan_TCI = htons(66); + if (!ASSERT_EQ(inet_pton(AF_INET, IPV4_VLAN_NETNS_DST, &fib_params->ipv4_dst), + 1, "inet_pton(dst)")) + goto fail; + + skel->bss->fib_lookup_ret = -1; + skel->bss->lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | + BPF_FIB_LOOKUP_SKIP_NEIGH; + err = bpf_prog_test_run_opts(prog_fd, &run_opts); + if (!ASSERT_OK(err, "test_run(input)")) + goto fail; + ASSERT_EQ(skel->bss->fib_lookup_ret, BPF_FIB_LKUP_RET_NOT_FWDED, + "input across netns fails closed"); + ASSERT_EQ(fib_params->ifindex, parent_idx, "ifindex untouched"); + ASSERT_EQ(fib_params->h_vlan_TCI, htons(66), "tag untouched"); + + close_netns(nstoken); + nstoken = open_netns(NS_VLAN_B); + if (!ASSERT_OK_PTR(nstoken, "open_netns(b)")) + goto fail; + + /* + * egress: the fib result is the VLAN device here, but its parent + * is in the other netns, so the swap must not happen + */ + SYS(fail, "ip addr add %s/24 dev veth7.66", IPV4_VLAN_NETNS_ADDR); + err = write_sysctl("/proc/sys/net/ipv4/conf/veth7.66/forwarding", "1"); + if (!ASSERT_OK(err, "write_sysctl(forwarding)")) + goto fail; + + vlan_idx = if_nametoindex("veth7.66"); + if (!ASSERT_NEQ(vlan_idx, 0, "if_nametoindex(veth7.66)")) + goto fail; + + memset(fib_params, 0, sizeof(*fib_params)); + fib_params->family = AF_INET; + fib_params->l4_protocol = IPPROTO_TCP; + fib_params->ifindex = vlan_idx; + if (!ASSERT_EQ(inet_pton(AF_INET, IPV4_VLAN_NETNS_DST, &fib_params->ipv4_dst), + 1, "inet_pton(dst)") || + !ASSERT_EQ(inet_pton(AF_INET, IPV4_VLAN_NETNS_ADDR, &fib_params->ipv4_src), + 1, "inet_pton(src)")) + goto fail; + + skel->bss->fib_lookup_ret = -1; + skel->bss->lookup_flags = BPF_FIB_LOOKUP_VLAN | + BPF_FIB_LOOKUP_SKIP_NEIGH; + err = bpf_prog_test_run_opts(xdp_fd, &xdp_opts); + if (!ASSERT_OK(err, "test_run(egress)")) + goto fail; + ASSERT_EQ(skel->bss->fib_lookup_ret, BPF_FIB_LKUP_RET_VLAN_FAILURE, + "egress returns VLAN_FAILURE"); + ASSERT_EQ(fib_params->ifindex, vlan_idx, + "foreign parent not published"); + ASSERT_EQ(fib_params->h_vlan_TCI, 0, "vlan fields zero"); + +fail: + if (nstoken) + close_netns(nstoken); + SYS_NOFAIL("ip netns del " NS_VLAN_A); + SYS_NOFAIL("ip netns del " NS_VLAN_B); + fib_lookup__destroy(skel); +} + +#define REDIRECT_NPKTS 1000 +#define NS_REDIRECT "fib_lookup_redirect_ns" + +/* + * The egress flag exists so an XDP program can redirect to the physical + * parent. A redirect that lands on a VLAN device is dropped at + * xdp_do_flush(), because a VLAN device has no ndo_xdp_xmit. Drive real + * frames with BPF_F_TEST_XDP_LIVE_FRAMES, which runs the native + * xdp_do_redirect() + xdp_do_flush() path: a reducible VLAN egress + * resolves to veth1 and is delivered to its peer veth2, while a QinQ + * egress returns VLAN_FAILURE and is passed to the stack instead of + * redirected to a device that would silently drop it. + */ +void test_fib_lookup_vlan_redirect(void) +{ + int redirect_fd, err, veth1_idx, veth2_idx = -1; + struct bpf_fib_lookup *fib_params; + struct nstoken *nstoken = NULL; + struct fib_lookup *skel = NULL; + bool xdp_attached = false; + + LIBBPF_OPTS(bpf_test_run_opts, lf_opts, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .flags = BPF_F_TEST_XDP_LIVE_FRAMES, + .repeat = REDIRECT_NPKTS, + ); + + skel = fib_lookup__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + return; + redirect_fd = bpf_program__fd(skel->progs.fib_lookup_redirect); + fib_params = &skel->bss->fib_params; + + SYS(fail, "ip netns add %s", NS_REDIRECT); + nstoken = open_netns(NS_REDIRECT); + if (!ASSERT_OK_PTR(nstoken, "open_netns")) + goto fail; + if (setup_netns()) + goto fail; + + veth1_idx = if_nametoindex("veth1"); + veth2_idx = if_nametoindex("veth2"); + if (!ASSERT_NEQ(veth1_idx, 0, "if_nametoindex(veth1)") || + !ASSERT_NEQ(veth2_idx, 0, "if_nametoindex(veth2)")) + goto fail; + + /* + * A redirect to veth1 is delivered to its peer veth2. veth_xdp_xmit() + * only accepts the frame if veth2's NAPI is up, which on veth means + * veth2 carries an XDP program; xdp_count tallies what arrives. + */ + err = bpf_xdp_attach(veth2_idx, bpf_program__fd(skel->progs.xdp_count), + XDP_FLAGS_DRV_MODE, NULL); + if (!ASSERT_OK(err, "attach xdp_count on veth2")) + goto fail; + xdp_attached = true; + + /* reducible VLAN egress: resolves to the physical parent veth1 */ + memset(fib_params, 0, sizeof(*fib_params)); + fib_params->family = AF_INET; + fib_params->l4_protocol = IPPROTO_TCP; + fib_params->ifindex = veth1_idx; + if (!ASSERT_EQ(inet_pton(AF_INET, IPV4_IFACE_ADDR, &fib_params->ipv4_src), + 1, "inet_pton(src)") || + !ASSERT_EQ(inet_pton(AF_INET, IPV4_VLAN_EGRESS_DST, &fib_params->ipv4_dst), + 1, "inet_pton(reducible dst)")) + goto fail; + skel->bss->lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH; + skel->bss->redirected = 0; + skel->bss->passed = 0; + skel->bss->delivered = 0; + + err = bpf_prog_test_run_opts(redirect_fd, &lf_opts); + if (!ASSERT_OK(err, "test_run(reducible egress)")) + goto fail; + ASSERT_EQ(skel->bss->redirected, REDIRECT_NPKTS, "reducible egress redirected"); + ASSERT_EQ(skel->bss->passed, 0, "reducible egress not passed"); + ASSERT_GT(skel->bss->delivered, 0, "reducible egress delivered to veth2"); + + /* + * QinQ egress: not reducible, so the lookup returns VLAN_FAILURE and + * the program passes the frame instead of redirecting to the inner + * VLAN device. redirected == 0 is the assertion that matters: the + * program did not redirect to a device that would drop the frame at + * xdp_do_flush(). veth2's delivered count is not checked here, since + * a passed frame can still reach veth2 through the stack's forwarding + * path, which is unrelated to the redirect under test. + */ + memset(fib_params, 0, sizeof(*fib_params)); + fib_params->family = AF_INET; + fib_params->l4_protocol = IPPROTO_TCP; + fib_params->ifindex = veth1_idx; + if (!ASSERT_EQ(inet_pton(AF_INET, IPV4_IFACE_ADDR, &fib_params->ipv4_src), + 1, "inet_pton(src)") || + !ASSERT_EQ(inet_pton(AF_INET, IPV4_QINQ_DST, &fib_params->ipv4_dst), + 1, "inet_pton(qinq dst)")) + goto fail; + skel->bss->lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH; + skel->bss->redirected = 0; + skel->bss->passed = 0; + + err = bpf_prog_test_run_opts(redirect_fd, &lf_opts); + if (!ASSERT_OK(err, "test_run(qinq egress)")) + goto fail; + ASSERT_EQ(skel->bss->passed, REDIRECT_NPKTS, "qinq egress passed"); + ASSERT_EQ(skel->bss->redirected, 0, "qinq egress not redirected"); + +fail: + if (xdp_attached) + bpf_xdp_detach(veth2_idx, XDP_FLAGS_DRV_MODE, NULL); + if (nstoken) + close_netns(nstoken); + SYS_NOFAIL("ip netns del " NS_REDIRECT); + fib_lookup__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/file_reader.c b/tools/testing/selftests/bpf/prog_tests/file_reader.c index 5cde32b35da4..48aae7ea0e4b 100644 --- a/tools/testing/selftests/bpf/prog_tests/file_reader.c +++ b/tools/testing/selftests/bpf/prog_tests/file_reader.c @@ -10,6 +10,7 @@ const char *user_ptr = "hello world"; char file_contents[256000]; +void *addr; void *get_executable_base_addr(void) { @@ -26,8 +27,7 @@ void *get_executable_base_addr(void) static int initialize_file_contents(void) { int fd, page_sz = sysconf(_SC_PAGESIZE); - ssize_t n = 0, cur, off; - void *addr; + ssize_t n = 0, cur; fd = open("/proc/self/exe", O_RDONLY); if (!ASSERT_OK_FD(fd, "Open /proc/self/exe\n")) @@ -52,16 +52,6 @@ static int initialize_file_contents(void) /* page-align base file address */ addr = (void *)((unsigned long)addr & ~(page_sz - 1)); - /* - * Page out range 0..512K, use 0..256K for positive tests and - * 256K..512K for negative tests expecting page faults - */ - for (off = 0; off < sizeof(file_contents) * 2; off += page_sz) { - if (!ASSERT_OK(madvise(addr + off, page_sz, MADV_PAGEOUT), - "madvise pageout")) - return errno; - } - return 0; } @@ -90,6 +80,14 @@ static void run_test(const char *prog_name) if (!ASSERT_OK(err, "file_reader__load")) goto cleanup; + /* + * Page out range 0..512K, use 0..256K for positive tests and + * 256K..512K for negative tests expecting page faults + */ + if (!ASSERT_OK(madvise(addr, sizeof(file_contents) * 2, MADV_PAGEOUT), + "madvise pageout")) + goto cleanup; + err = file_reader__attach(skel); if (!ASSERT_OK(err, "file_reader__attach")) goto cleanup; diff --git a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c index e40114620751..0918321c8e63 100644 --- a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c +++ b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c @@ -4,6 +4,7 @@ #include <string.h> #include <linux/bpf.h> #include <linux/limits.h> +#include <bpf/btf.h> #include <test_progs.h> #include "trace_helpers.h" #include "test_fill_link_info.skel.h" @@ -24,6 +25,22 @@ static __u64 kmulti_cookies[] = { 3, 1, 2 }; #define KPROBE_FUNC "bpf_fentry_test1" static __u64 kprobe_addr; +static const char * const tmulti_syms[] = { + "bpf_fentry_test2", + "bpf_fentry_test1", + "bpf_fentry_test3", +}; + +static __u64 tmulti_cookies[] = { 30, 10, 20 }; +#define TRACING_MULTI_CNT ARRAY_SIZE(tmulti_syms) + +struct tmulti_target { + const char *name; + __u64 addr; + __u64 cookie; + __u32 id; +}; + #define UPROBE_FILE "/proc/self/exe" static ssize_t uprobe_offset; /* uprobe attach point */ @@ -396,6 +413,224 @@ static void test_kprobe_multi_fill_link_info(struct test_fill_link_info *skel, bpf_link__destroy(link); } +static int tmulti_target_cmp(const void *a, const void *b) +{ + const struct tmulti_target *ta = a; + const struct tmulti_target *tb = b; + + return (ta->id > tb->id) - (ta->id < tb->id); +} + +static int setup_tmulti_targets(const struct bpf_program *prog, + struct tmulti_target *targets, + __u32 *btf_obj_id) +{ + struct bpf_prog_info prog_info; + __u32 len = sizeof(prog_info); + struct btf *btf; + int err, i; + __s32 id; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "btf__load_vmlinux_btf")) + return -1; + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + id = btf__find_by_name_kind(btf, tmulti_syms[i], BTF_KIND_FUNC); + if (!ASSERT_GT(id, 0, "btf__find_by_name_kind")) + goto error; + + targets[i].name = tmulti_syms[i]; + targets[i].addr = ksym_get_addr(tmulti_syms[i]); + targets[i].cookie = tmulti_cookies[i]; + targets[i].id = id; + } + + memset(&prog_info, 0, len); + err = bpf_prog_get_info_by_fd(bpf_program__fd(prog), &prog_info, &len); + if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd")) + goto error; + if (!ASSERT_GT(prog_info.attach_btf_obj_id, 0, "attach_btf_obj_id")) + goto error; + *btf_obj_id = prog_info.attach_btf_obj_id; + + /* + * The kernel tracing multi attach sorts ids. We sort as well, + * so we can easily compare ids and cookies later. + */ + qsort(targets, TRACING_MULTI_CNT, sizeof(targets[0]), tmulti_target_cmp); + btf__free(btf); + return 0; + +error: + btf__free(btf); + return -1; +} + +static int verify_tracing_multi_link_info(int fd, const struct bpf_program *prog, + const struct tmulti_target *targets, + __u32 btf_obj_id, bool has_cookies) +{ + enum bpf_attach_type attach_type = bpf_program__expected_attach_type(prog); + __u64 addrs[TRACING_MULTI_CNT], cookies[TRACING_MULTI_CNT]; + __u32 ids[TRACING_MULTI_CNT]; + struct bpf_link_info info; + __u32 len = sizeof(info); + int err, i; + + memset(&info, 0, sizeof(info)); + err = bpf_link_get_info_by_fd(fd, &info, &len); + if (!ASSERT_OK(err, "bpf_link_get_info_by_fd")) + return -1; + + if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type")) + return -1; + + ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type"); + ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count"); + + memset(ids, 0, sizeof(ids)); + memset(cookies, 0, sizeof(cookies)); + memset(addrs, 0, sizeof(addrs)); + + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.addrs = ptr_to_u64(addrs); + info.tracing_multi.cookies = has_cookies ? ptr_to_u64(cookies) : 0; + info.tracing_multi.count = TRACING_MULTI_CNT; + + err = bpf_link_get_info_by_fd(fd, &info, &len); + if (!ASSERT_OK(err, "bpf_link_get_info_by_fd")) + return -1; + + if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type")) + return -1; + + ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type"); + ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count"); + ASSERT_EQ(info.tracing_multi.btf_obj_id, btf_obj_id, "tracing_multi.btf_obj_id"); + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids"); + ASSERT_EQ(cookies[i], has_cookies ? targets[i].cookie : 0, "tracing_multi.cookies"); + + if (targets[i].addr) { + struct ksym *ksym; + + if (!ASSERT_NEQ(addrs[i], 0, "tracing_multi.addrs")) + return -1; + ksym = ksym_search(addrs[i]); + if (!ASSERT_OK_PTR(ksym, "ksym_search")) + return -1; + ASSERT_STREQ(ksym->name, targets[i].name, "tracing_multi.addr_name"); + } else { + ASSERT_EQ(addrs[i], 0, "tracing_multi.addrs"); + } + } + + return 0; +} + +static void verify_tracing_multi_invalid_user_buffer(int fd, const struct tmulti_target *targets) +{ + __u32 ids[TRACING_MULTI_CNT] = {}; + struct bpf_link_info info; + __u32 len = sizeof(info); + int err, i; + + /* Wrong info setup (ids != NULL and cnt == 0) -> EINVAL */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EINVAL, "tracing_multi.invalid_count"); + + /* Smaller than actual count provided -> ENOSPC */ + memset(ids, 0, sizeof(ids)); + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.count = TRACING_MULTI_CNT - 1; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -ENOSPC, "tracing_multi.small_count"); + for (i = 0; i < TRACING_MULTI_CNT - 1; i++) + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.partial_ids"); + /* check that the last entry is not populated */ + ASSERT_EQ(ids[i], 0, "tracing_multi.partial_ids"); + + /* Bigger than actual count provided -> OK */ + memset(ids, 0, sizeof(ids)); + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.count = TRACING_MULTI_CNT + 1; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_OK(err, "tracing_multi.big_count"); + for (i = 0; i < TRACING_MULTI_CNT; i++) + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids"); + + /* Invalid ids pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_btf_ids"); + + /* Invalid cookies pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.cookies = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_cookies"); + + /* Invalid addrs pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.addrs = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_addrs"); +} + +static void test_tracing_multi_fill_link_info(struct test_fill_link_info *skel, + bool has_cookies, bool invalid) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tmulti_target targets[TRACING_MULTI_CNT]; + __u32 ids[TRACING_MULTI_CNT], btf_obj_id; + __u64 cookies[TRACING_MULTI_CNT]; + struct bpf_link *link; + int link_fd, err, i; + +#ifndef __x86_64__ + test__skip(); + return; +#endif + + if (setup_tmulti_targets(skel->progs.tmulti_run, targets, &btf_obj_id)) + return; + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + ids[i] = targets[i].id; + cookies[i] = targets[i].cookie; + } + + opts.ids = ids; + opts.cnt = TRACING_MULTI_CNT; + if (has_cookies) + opts.cookies = cookies; + + link = bpf_program__attach_tracing_multi(skel->progs.tmulti_run, NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_tracing_multi")) + return; + + link_fd = bpf_link__fd(link); + if (invalid) { + verify_tracing_multi_invalid_user_buffer(link_fd, targets); + } else { + err = verify_tracing_multi_link_info(link_fd, skel->progs.tmulti_run, + targets, btf_obj_id, has_cookies); + ASSERT_OK(err, "verify_tracing_multi_link_info"); + } + + bpf_link__destroy(link); +} + #define SEC(name) __attribute__((section(name), used)) static short uprobe_link_info_sema_1 SEC(".probes"); @@ -469,7 +704,7 @@ verify_umulti_link_info(int fd, bool retprobe, __u64 *offsets, ASSERT_EQ(info.uprobe_multi.pid, getpid(), "info.uprobe_multi.pid"); ASSERT_EQ(info.uprobe_multi.count, 3, "info.uprobe_multi.count"); - ASSERT_EQ(info.uprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN, + ASSERT_EQ(info.uprobe_multi.flags & BPF_F_UPROBE_MULTI_RETURN, retprobe, "info.uprobe_multi.flags.retprobe"); ASSERT_EQ(info.uprobe_multi.path_size, strlen(path) + 1, "info.uprobe_multi.path_size"); ASSERT_STREQ(path_buf, path, "info.uprobe_multi.path"); @@ -640,6 +875,13 @@ void test_fill_link_info(void) if (test__start_subtest("kprobe_multi_invalid_ubuff")) test_kprobe_multi_fill_link_info(skel, true, true, true); + if (test__start_subtest("tracing_multi_link_info")) { + test_tracing_multi_fill_link_info(skel, false, false); + test_tracing_multi_fill_link_info(skel, true, false); + } + if (test__start_subtest("tracing_multi_invalid_ubuff")) + test_tracing_multi_fill_link_info(skel, true, true); + if (test__start_subtest("uprobe_multi_link_info")) test_uprobe_multi_fill_link_info(skel, false, false); if (test__start_subtest("uretprobe_multi_link_info")) diff --git a/tools/testing/selftests/bpf/prog_tests/get_func_args_test.c b/tools/testing/selftests/bpf/prog_tests/get_func_args_test.c index 96b27de05524..7bf8adc41e99 100644 --- a/tools/testing/selftests/bpf/prog_tests/get_func_args_test.c +++ b/tools/testing/selftests/bpf/prog_tests/get_func_args_test.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> #include "get_func_args_test.skel.h" +#include "get_func_args_fsession_test.skel.h" void test_get_func_args_test(void) { @@ -41,8 +42,30 @@ void test_get_func_args_test(void) ASSERT_EQ(skel->bss->test4_result, 1, "test4_result"); ASSERT_EQ(skel->bss->test5_result, 1, "test5_result"); ASSERT_EQ(skel->bss->test6_result, 1, "test6_result"); - ASSERT_EQ(skel->bss->test7_result, 1, "test7_result"); cleanup: get_func_args_test__destroy(skel); } + +void test_get_func_args_fsession_test(void) +{ + struct get_func_args_fsession_test *skel = NULL; + int err; + LIBBPF_OPTS(bpf_test_run_opts, topts); + + skel = get_func_args_fsession_test__open_and_load(); + if (!ASSERT_OK_PTR(skel, "get_func_args_fsession_test__open_and_load")) + return; + + err = get_func_args_fsession_test__attach(skel); + if (!ASSERT_OK(err, "get_func_args_fsession_test__attach")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test1), &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "test_run"); + + ASSERT_EQ(skel->bss->test1_result, 1, "test1_result"); +cleanup: + get_func_args_fsession_test__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c b/tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c index 7772a0f288d3..357fdedfea93 100644 --- a/tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c +++ b/tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c @@ -2,6 +2,7 @@ #include <test_progs.h> #include "get_func_ip_test.skel.h" #include "get_func_ip_uprobe_test.skel.h" +#include "get_func_ip_fsession_test.skel.h" static noinline void uprobe_trigger(void) { @@ -46,8 +47,6 @@ static void test_function_entry(void) ASSERT_EQ(skel->bss->test5_result, 1, "test5_result"); ASSERT_EQ(skel->bss->test7_result, 1, "test7_result"); ASSERT_EQ(skel->bss->test8_result, 1, "test8_result"); - ASSERT_EQ(skel->bss->test9_entry_result, 1, "test9_entry_result"); - ASSERT_EQ(skel->bss->test9_exit_result, 1, "test9_exit_result"); cleanup: get_func_ip_test__destroy(skel); @@ -139,3 +138,28 @@ void test_get_func_ip_test(void) test_function_entry(); test_function_body(); } + +void test_get_func_ip_fsession_test(void) +{ + struct get_func_ip_fsession_test *skel = NULL; + int err; + LIBBPF_OPTS(bpf_test_run_opts, topts); + + skel = get_func_ip_fsession_test__open_and_load(); + if (!ASSERT_OK_PTR(skel, "get_func_ip_fsession_test__open_and_load")) + return; + + err = get_func_ip_fsession_test__attach(skel); + if (!ASSERT_OK(err, "get_func_ip_fsession_test__attach")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test1), &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "test_run"); + + ASSERT_EQ(skel->bss->test1_entry_result, 1, "test1_entry_result"); + ASSERT_EQ(skel->bss->test1_exit_result, 1, "test1_exit_result"); + +cleanup: + get_func_ip_fsession_test__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/get_smp_processor_id.c b/tools/testing/selftests/bpf/prog_tests/get_smp_processor_id.c new file mode 100644 index 000000000000..1b5c738ab81f --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/get_smp_processor_id.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "bpf/libbpf_internal.h" +#include "get_smp_processor_id.skel.h" + +void test_get_smp_processor_id(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts, + .flags = BPF_F_TEST_RUN_ON_CPU, + .cpu = 0, + ); + struct get_smp_processor_id *skel; + int prog_fd, err, online_cpu_nr, i; + bool *online = NULL; + + err = parse_cpu_mask_file("/sys/devices/system/cpu/online", + &online, &online_cpu_nr); + if (!ASSERT_OK(err, "parse_cpu_mask_file")) + return; + + skel = get_smp_processor_id__open_and_load(); + if (!ASSERT_OK_PTR(skel, "get_smp_processor_id__open_and_load")) + goto cleanup; + + prog_fd = bpf_program__fd(skel->progs.call_bpf_get_smp_processor_id); + + for (i = 0; i < online_cpu_nr; i++) { + if (!online[i]) + continue; + + opts.cpu = i; + skel->bss->cpu_nr_result = -1; + + err = bpf_prog_test_run_opts(prog_fd, &opts); + if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) + goto cleanup; + + ASSERT_EQ(skel->bss->cpu_nr_result, opts.cpu, "cpu_nr_result"); + } + +cleanup: + free(online); + get_smp_processor_id__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/global_data_init.c b/tools/testing/selftests/bpf/prog_tests/global_data_init.c index 8466332d7406..5671c31085cd 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_data_init.c +++ b/tools/testing/selftests/bpf/prog_tests/global_data_init.c @@ -1,5 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> +#include "bpf/libbpf_internal.h" +#include "test_global_percpu_data.skel.h" +#include "test_global_percpu_data.lskel.h" void test_global_data_init(void) { @@ -60,3 +63,336 @@ out: free(newval); bpf_object__close(obj); } + +static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_fd, int *runp) +{ + struct test_global_percpu_data__percpu *data = NULL; + int i, err, key = 0, num_online, run = 0; + __u64 args[2] = {0x1234ULL, 0x5678ULL}; + size_t data_sz; + bool *online; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .ctx_in = args, + .ctx_size_in = sizeof(args), + .flags = BPF_F_TEST_RUN_ON_CPU, + ); + + err = parse_cpu_mask_file("/sys/devices/system/cpu/online", &online, &num_online); + if (!ASSERT_OK(err, "parse_cpu_mask_file")) + return; + + data_sz = map ? bpf_map__value_size(map) : sizeof(*data); + data = calloc(1, data_sz); + if (!ASSERT_OK_PTR(data, "calloc percpu data")) + goto out; + + /* run on every online-CPU */ + for (i = 0; i < num_online; i++) { + __u64 flags; + + if (!online[i]) + continue; + + topts.cpu = i; + topts.retval = -1; + err = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(err, "bpf_prog_test_run_opts"); + ASSERT_EQ(topts.retval, 0, "bpf_prog_test_run_opts retval"); + + memset(data, 0, data_sz); + flags = ((__u64) i << 32) | BPF_F_CPU; + if (map) + err = bpf_map__lookup_elem(map, &key, sizeof(key), data, data_sz, flags); + else + err = bpf_map_lookup_elem_flags(map_fd, &key, data, flags); + if (!ASSERT_OK(err, "lookup_elem on cpu")) + break; + + ASSERT_EQ(*runp, ++run, "run"); + ASSERT_EQ(data->cpu_id[0], i, "cpu_id"); + ASSERT_EQ(data->data, 1, "data"); + ASSERT_TRUE(data->set, "set"); + ASSERT_EQ(data->nums[6], 0xc0de, "nums[6]"); + ASSERT_EQ(data->struct_data.i, 1, "struct_data.i"); + ASSERT_TRUE(data->struct_data.set, "struct_data.set"); + ASSERT_EQ(data->struct_data.nums[6], 0xc0de, "struct_data.nums[6]"); + } + +out: + free(data); + free(online); +} + +static void test_global_percpu_data_init(void) +{ + struct test_global_percpu_data__percpu init_value = {}; + struct test_global_percpu_data__percpu *init_data; + const __u32 desired_sz = sysconf(_SC_PAGE_SIZE); + struct test_global_percpu_data *skel = NULL; + size_t init_data_sz; + struct bpf_map *map; + int prog_fd, err; + + skel = test_global_percpu_data__open(); + if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu, "skel->percpu")) + goto out; + if (!ASSERT_OK_PTR(skel->data_percpu, "skel->data_percpu")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu_data, "skel->percpu_data")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu_looooooooong, "skel->percpu_looooooooong")) + goto out; + + ASSERT_STREQ(bpf_map__name(skel->maps.percpu_data), ".percpu.data", + ".percpu.data map name"); + ASSERT_STREQ(bpf_map__name(skel->maps.data_percpu), ".data.percpu", + ".data.percpu map name"); + ASSERT_STREQ(bpf_map__name(skel->maps.percpu_looooooooong), ".percpu.looooooooong", + "long map name"); + ASSERT_STREQ(bpf_map__name(skel->maps.percpu), ".percpu", "map name"); + ASSERT_EQ(skel->percpu->data, -1, "skel->percpu->data"); + ASSERT_FALSE(skel->percpu->set, "skel->percpu->set"); + ASSERT_EQ(skel->percpu->nums[6], 0, "skel->percpu->nums[6]"); + ASSERT_EQ(skel->percpu->struct_data.i, -1, "struct_data.i"); + ASSERT_FALSE(skel->percpu->struct_data.set, "struct_data.set"); + ASSERT_EQ(skel->percpu->struct_data.nums[6], 0, "struct_data.nums[6]"); + + map = skel->maps.percpu; + if (!ASSERT_EQ(bpf_map__type(map), BPF_MAP_TYPE_PERCPU_ARRAY, "bpf_map__type")) + goto out; + + init_value.data = 2; + init_value.nums[6] = -1; + init_value.struct_data.i = 2; + init_value.struct_data.nums[6] = -1; + err = bpf_map__set_initial_value(map, &init_value, sizeof(init_value)); + if (!ASSERT_OK(err, "bpf_map__set_initial_value")) + goto out; + + init_data = bpf_map__initial_value(map, &init_data_sz); + if (!ASSERT_OK_PTR(init_data, "bpf_map__initial_value")) + goto out; + + ASSERT_EQ(init_data->data, init_value.data, "init_value data"); + ASSERT_EQ(init_data->set, init_value.set, "init_value set"); + ASSERT_EQ(init_data->struct_data.i, init_value.struct_data.i, "init_value struct_data.i"); + ASSERT_EQ(init_data->struct_data.nums[6], init_value.struct_data.nums[6], + "init_value struct_data.nums[6]"); + ASSERT_EQ(init_data_sz, sizeof(init_value), "init_value size"); + ASSERT_EQ((void *) init_data, (void *) skel->percpu, "skel->percpu eq init_data"); + ASSERT_EQ(skel->percpu->data, init_value.data, "skel->percpu->data"); + ASSERT_EQ(skel->percpu->set, init_value.set, "skel->percpu->set"); + ASSERT_EQ(skel->percpu->struct_data.i, init_value.struct_data.i, + "skel->percpu->struct_data.i"); + ASSERT_EQ(skel->percpu->struct_data.nums[6], init_value.struct_data.nums[6], + "skel->percpu->struct_data.nums[6]"); + + ASSERT_GT(desired_sz, sizeof(init_value), "desired_sz"); + err = bpf_map__set_value_size(map, desired_sz); + if (!ASSERT_OK(err, "bpf_map__set_value_size")) + goto out; + if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "percpu value size")) + goto out; + if (!ASSERT_NEQ(bpf_map__btf_value_type_id(map), 0, "percpu BTF value type")) + goto out; + + init_data = bpf_map__initial_value(map, &init_data_sz); + if (!ASSERT_OK_PTR(init_data, "resized bpf_map__initial_value")) + goto out; + if (!ASSERT_EQ(init_data_sz, desired_sz, "resized initial value size")) + goto out; + if (!ASSERT_EQ(init_data->data, init_value.data, "resized initial value data")) + goto out; + + err = test_global_percpu_data__load(skel); + if (!ASSERT_OK(err, "test_global_percpu_data__load")) + goto out; + + ASSERT_OK_PTR(skel->percpu, "skel->percpu"); + + prog_fd = bpf_program__fd(skel->progs.update_percpu_data); + test_percpu_data_on_cpus(map, bpf_map__fd(map), prog_fd, &skel->bss->run); + +out: + test_global_percpu_data__destroy(skel); +} + +static void test_global_percpu_data_lskel(void) +{ + struct test_global_percpu_data_lskel *lskel = NULL; + int prog_fd, map_fd; + + lskel = test_global_percpu_data_lskel__open_and_load(); + if (!ASSERT_OK_PTR(lskel, "test_global_percpu_data_lskel__open_and_load")) + goto out; + + map_fd = lskel->maps.percpu.map_fd; + prog_fd = lskel->progs.update_percpu_data.prog_fd; + test_percpu_data_on_cpus(NULL, map_fd, prog_fd, &lskel->bss->run); + +out: + test_global_percpu_data_lskel__destroy(lskel); +} + +static int create_rdonly_percpu_array(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, map_opts, + .map_flags = BPF_F_RDONLY_PROG, + ); + int key = 0, map_fd, err; + __u64 value = 0; + + map_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "percpu_ro_map", sizeof(int), + sizeof(__u64), 1, &map_opts); + if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) + return -1; + + err = bpf_map_update_elem(map_fd, &key, &value, BPF_F_ALL_CPUS); + if (!ASSERT_OK(err, "bpf_map_update_elem")) + goto out; + + err = bpf_map_freeze(map_fd); + if (!ASSERT_OK(err, "bpf_map_freeze")) + goto out; + + return map_fd; + +out: + close(map_fd); + return -1; +} + +static void test_global_percpu_data_rdonly_direct_read(void) +{ + /* + * Raw instructions with manually prepared rdonly percpu_array map + * for testing direct-read global percpu data, because libbpf + * doesn't have rdonly internal percpu_array map support for + * global percpu data. + */ + struct bpf_insn insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0), + BPF_EXIT_INSN(), + }; + int map_fd, prog_fd; + + map_fd = create_rdonly_percpu_array(); + if (map_fd < 0) + return; + + insns[0].imm = map_fd; + prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "percpu_ro_prog", "GPL", insns, + ARRAY_SIZE(insns), NULL); + if (ASSERT_GE(prog_fd, 0, "bpf_prog_load")) + close(prog_fd); + close(map_fd); +} + +static void test_global_percpu_data_rdonly_direct_write(void) +{ + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts); + /* See the comment in test_global_percpu_data_rdonly_direct_read() */ + struct bpf_insn insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0), + BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0), + BPF_EXIT_INSN(), + }; + char log_buf[256] = {}; + int map_fd, prog_fd; + + prog_opts.log_buf = log_buf; + prog_opts.log_size = sizeof(log_buf); + prog_opts.log_level = 1; + + map_fd = create_rdonly_percpu_array(); + if (map_fd < 0) + return; + + insns[0].imm = map_fd; + prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "percpu_ro_prog", "GPL", insns, + ARRAY_SIZE(insns), &prog_opts); + if (!ASSERT_LT(prog_fd, 0, "bpf_prog_load")) + close(prog_fd); + else + ASSERT_HAS_SUBSTR(log_buf, "write into map forbidden", "verifier log"); + close(map_fd); +} + +static void test_global_percpu_data_verifier_log(void) +{ + RUN_TESTS(test_global_percpu_data); +} + +static void test_global_percpu_data_iter(void) +{ + DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); + struct test_global_percpu_data *skel; + union bpf_iter_link_info linfo = {}; + struct bpf_link *link = NULL; + int fd, num_cpus, len, err; + char buf[16]; + + num_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(num_cpus, 0, "libbpf_num_possible_cpus")) + return; + + skel = test_global_percpu_data__open(); + if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) + return; + + skel->rodata->num_cpus = num_cpus; + skel->rodata->num_off = offsetof(struct test_global_percpu_data__percpu, + struct_data.nums[6]); + skel->rodata->elem_sz = roundup(sizeof(struct test_global_percpu_data__percpu), 8); + skel->percpu->struct_data.nums[6] = 0xc0de; + + err = test_global_percpu_data__load(skel); + if (!ASSERT_OK(err, "test_global_percpu_data__load")) + goto out; + + linfo.map.map_fd = bpf_map__fd(skel->maps.percpu); + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + link = bpf_program__attach_iter(skel->progs.dump_percpu_data, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_iter")) + goto out; + + fd = bpf_iter_create(bpf_link__fd(link)); + if (!ASSERT_GE(fd, 0, "bpf_iter_create")) + goto out; + + while ((len = read(fd, buf, sizeof(buf))) > 0) + do { } while (0); + ASSERT_EQ(len, 0, "read iter"); + ASSERT_TRUE(skel->bss->run_iter, "run_iter"); + ASSERT_EQ(skel->bss->sum, 0xc0de * num_cpus, "sum"); + + close(fd); +out: + bpf_link__destroy(link); + test_global_percpu_data__destroy(skel); +} + +void test_global_percpu_data(void) +{ + if (!feat_supported(NULL, FEAT_PERCPU_DATA)) { + test__skip(); + return; + } + + if (test__start_subtest("init")) + test_global_percpu_data_init(); + if (test__start_subtest("lskel")) + test_global_percpu_data_lskel(); + if (test__start_subtest("rdonly_direct_read")) + test_global_percpu_data_rdonly_direct_read(); + if (test__start_subtest("rdonly_direct_write")) + test_global_percpu_data_rdonly_direct_write(); + test_global_percpu_data_verifier_log(); + if (test__start_subtest("iter")) + test_global_percpu_data_iter(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c index 56b5baef35c8..602ce30f1720 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c +++ b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c @@ -23,6 +23,7 @@ static void global_map_resize_bss_subtest(void) struct bpf_map *map; const __u32 desired_sz = sizeof(skel->bss->sum) + sysconf(_SC_PAGE_SIZE) * 2; size_t array_len, actual_sz, new_sz; + int *array; skel = test_global_map_resize__open(); if (!ASSERT_OK_PTR(skel, "test_global_map_resize__open")) @@ -58,10 +59,13 @@ static void global_map_resize_bss_subtest(void) goto teardown; /* fill the newly resized array with ones, - * skipping the first element which was previously set + * skipping the first element which was previously set; + * access through a plain pointer to avoid -Warray-bounds + * since the array was resized beyond its declared length. */ + array = skel->bss->array; for (int i = 1; i < array_len; i++) - skel->bss->array[i] = 1; + array[i] = 1; /* set global const values before loading */ skel->rodata->pid = getpid(); diff --git a/tools/testing/selftests/bpf/prog_tests/htab_reuse.c b/tools/testing/selftests/bpf/prog_tests/htab_reuse.c index a742dd994d60..d7c3df165adc 100644 --- a/tools/testing/selftests/bpf/prog_tests/htab_reuse.c +++ b/tools/testing/selftests/bpf/prog_tests/htab_reuse.c @@ -59,7 +59,7 @@ static void *htab_update_fn(void *arg) return NULL; } -void test_htab_reuse(void) +static void test_htab_reuse_basic(void) { unsigned int i, wr_nr = 1, rd_nr = 4; pthread_t tids[wr_nr + rd_nr]; @@ -99,3 +99,170 @@ reap: } htab_reuse__destroy(skel); } + +/* + * Writes consistency test for BPF_F_LOCK update + * + * The race: + * 1. Thread A: BPF_F_LOCK|BPF_EXIST update + * 2. Thread B: delete element then update it with BPF_ANY + */ + +struct htab_val_large { + struct bpf_spin_lock lock; + __u32 seq; + __u64 data[256]; +}; + +struct consistency_ctx { + int fd; + int start_fd; + int loop; + volatile bool torn_write; +}; + +static void wait_for_start(int fd) +{ + char buf; + + read(fd, &buf, 1); +} + +static void *locked_update_fn(void *arg) +{ + struct consistency_ctx *ctx = arg; + struct htab_val_large value; + unsigned int key = 1; + int i; + + memset(&value, 0xAA, sizeof(value)); + wait_for_start(ctx->start_fd); + + for (i = 0; i < ctx->loop; i++) { + value.seq = i; + bpf_map_update_elem(ctx->fd, &key, &value, + BPF_F_LOCK | BPF_EXIST); + } + + return NULL; +} + +/* Delete + update: removes the element then re-creates it with BPF_ANY. */ +static void *delete_update_fn(void *arg) +{ + struct consistency_ctx *ctx = arg; + struct htab_val_large value; + unsigned int key = 1; + int i; + + memset(&value, 0xBB, sizeof(value)); + + wait_for_start(ctx->start_fd); + + for (i = 0; i < ctx->loop; i++) { + value.seq = i; + bpf_map_delete_elem(ctx->fd, &key); + bpf_map_update_elem(ctx->fd, &key, &value, BPF_ANY | BPF_F_LOCK); + } + + return NULL; +} + +static void *locked_lookup_fn(void *arg) +{ + struct consistency_ctx *ctx = arg; + struct htab_val_large value; + unsigned int key = 1; + int i, j; + + wait_for_start(ctx->start_fd); + + for (i = 0; i < ctx->loop && !ctx->torn_write; i++) { + if (bpf_map_lookup_elem_flags(ctx->fd, &key, &value, BPF_F_LOCK)) + continue; + + for (j = 0; j < 256; j++) { + if (value.data[j] != value.data[0]) { + ctx->torn_write = true; + return NULL; + } + } + } + + return NULL; +} + +static void test_htab_reuse_consistency(void) +{ + int threads_total = 6, threads = 2; + pthread_t tids[threads_total]; + struct consistency_ctx ctx; + struct htab_val_large seed; + struct htab_reuse *skel; + unsigned int key = 1, i; + int pipefd[2]; + int err; + + skel = htab_reuse__open_and_load(); + if (!ASSERT_OK_PTR(skel, "htab_reuse__open_and_load")) + return; + + if (!ASSERT_OK(pipe(pipefd), "pipe")) + goto out; + + ctx.fd = bpf_map__fd(skel->maps.htab_lock_consistency); + ctx.start_fd = pipefd[0]; + ctx.loop = 100000; + ctx.torn_write = false; + + /* Seed the element so locked updaters have something to find */ + memset(&seed, 0xBB, sizeof(seed)); + err = bpf_map_update_elem(ctx.fd, &key, &seed, BPF_ANY); + if (!ASSERT_OK(err, "seed_element")) + goto close_pipe; + + memset(tids, 0, sizeof(tids)); + for (i = 0; i < threads; i++) { + err = pthread_create(&tids[i], NULL, locked_update_fn, &ctx); + if (!ASSERT_OK(err, "pthread_create")) + goto stop; + } + for (i = 0; i < threads; i++) { + err = pthread_create(&tids[threads + i], NULL, delete_update_fn, &ctx); + if (!ASSERT_OK(err, "pthread_create")) + goto stop; + } + for (i = 0; i < threads; i++) { + err = pthread_create(&tids[threads * 2 + i], NULL, locked_lookup_fn, &ctx); + if (!ASSERT_OK(err, "pthread_create")) + goto stop; + } + + /* Release all threads simultaneously */ + close(pipefd[1]); + pipefd[1] = -1; + +stop: + for (i = 0; i < threads_total; i++) { + if (!tids[i]) + continue; + pthread_join(tids[i], NULL); + } + + ASSERT_FALSE(ctx.torn_write, "no torn writes detected"); + +close_pipe: + if (pipefd[1] >= 0) + close(pipefd[1]); + close(pipefd[0]); +out: + htab_reuse__destroy(skel); +} + +void test_htab_reuse(void) +{ + if (test__start_subtest("basic")) + test_htab_reuse_basic(); + if (test__start_subtest("consistency")) + test_htab_reuse_consistency(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/htab_update.c b/tools/testing/selftests/bpf/prog_tests/htab_update.c index ea1a6766fbe9..0a28d4346924 100644 --- a/tools/testing/selftests/bpf/prog_tests/htab_update.c +++ b/tools/testing/selftests/bpf/prog_tests/htab_update.c @@ -23,7 +23,7 @@ static void test_reenter_update(void) if (!ASSERT_OK_PTR(skel, "htab_update__open")) return; - bpf_program__set_autoload(skel->progs.bpf_obj_free_fields, true); + bpf_program__set_autoload(skel->progs.bpf_obj_cancel_fields, true); err = htab_update__load(skel); if (!ASSERT_TRUE(!err, "htab_update__load") || err) goto out; @@ -50,7 +50,7 @@ static void test_reenter_update(void) /* * Second update: replace existing element with same key and trigger * the reentrancy of bpf_map_update_elem(). - * check_and_free_fields() calls bpf_obj_free_fields() on the old + * check_and_cancel_fields() calls bpf_obj_cancel_fields() on the old * value, which is where fentry program runs and performs a nested * bpf_map_update_elem(), triggering -EDEADLK. */ diff --git a/tools/testing/selftests/bpf/prog_tests/icmp_send_kfunc.c b/tools/testing/selftests/bpf/prog_tests/icmp_send_kfunc.c new file mode 100644 index 000000000000..9318d4bc7ce8 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/icmp_send_kfunc.c @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <network_helpers.h> +#include <cgroup_helpers.h> +#include <linux/errqueue.h> +#include <poll.h> +#include <unistd.h> +#include "icmp_send.skel.h" + +#define TIMEOUT_MS 1000 + +#define ICMP_DEST_UNREACH 3 +#define ICMPV6_DEST_UNREACH 1 + +#define ICMP_HOST_UNREACH 1 +#define ICMP_FRAG_NEEDED 4 +#define NR_ICMP_UNREACH 15 +#define ICMPV6_REJECT_ROUTE 6 + +#define KFUNC_RET_UNSET -1 + +static int connect_to_fd_nonblock(int server_fd) +{ + struct sockaddr_storage addr; + socklen_t len = sizeof(addr); + int fd, err, on = 1; + + if (getsockname(server_fd, (struct sockaddr *)&addr, &len)) + return -1; + + fd = socket(addr.ss_family, SOCK_STREAM | SOCK_NONBLOCK, 0); + if (fd < 0) + return -1; + + if (addr.ss_family == AF_INET6 && + setsockopt(fd, IPPROTO_IPV6, IPV6_RECVERR, &on, sizeof(on)) < 0) { + close(fd); + return -1; + } + + err = connect(fd, (struct sockaddr *)&addr, len); + if (err < 0 && errno != EINPROGRESS) { + close(fd); + return -1; + } + + return fd; +} + +static void read_icmp_errqueue(int sockfd, int expected_code, int af) +{ + int expected_ee_type = (af == AF_INET) ? ICMP_DEST_UNREACH : + ICMPV6_DEST_UNREACH; + int expected_origin = (af == AF_INET) ? SO_EE_ORIGIN_ICMP : + SO_EE_ORIGIN_ICMP6; + int expected_level = (af == AF_INET) ? IPPROTO_IP : IPPROTO_IPV6; + int expected_type = (af == AF_INET) ? IP_RECVERR : IPV6_RECVERR; + struct sock_extended_err *sock_err; + char ctrl_buf[512]; + struct msghdr msg = { + .msg_control = ctrl_buf, + .msg_controllen = sizeof(ctrl_buf), + }; + struct pollfd pfd = { + .fd = sockfd, + .events = POLLERR, + }; + struct cmsghdr *cm; + ssize_t n; + + if (!ASSERT_GE(poll(&pfd, 1, TIMEOUT_MS), 1, "poll_errqueue")) + return; + + n = recvmsg(sockfd, &msg, MSG_ERRQUEUE); + if (!ASSERT_GE(n, 0, "recvmsg_errqueue")) + return; + + cm = CMSG_FIRSTHDR(&msg); + if (!ASSERT_NEQ(cm, NULL, "cm_firsthdr_null")) + return; + + for (; cm; cm = CMSG_NXTHDR(&msg, cm)) { + if (cm->cmsg_level != expected_level || + cm->cmsg_type != expected_type) + continue; + + sock_err = (struct sock_extended_err *)CMSG_DATA(cm); + + if (!ASSERT_EQ(sock_err->ee_origin, expected_origin, + "sock_err_origin")) + return; + if (!ASSERT_EQ(sock_err->ee_type, expected_ee_type, + "sock_err_type_dest_unreach")) + return; + ASSERT_EQ(sock_err->ee_code, expected_code, "sock_err_code"); + return; + } + + ASSERT_FAIL("no IP_RECVERR/IPV6_RECVERR control message found"); +} + +static bool valid_unreach_code(int code, int af) +{ + if (code < 0) + return false; + + if (af == AF_INET) + return code <= NR_ICMP_UNREACH && code != ICMP_FRAG_NEEDED; + + return code <= ICMPV6_REJECT_ROUTE; +} + +static void trigger_prog_read_icmp_errqueue(struct icmp_send *skel, int code, + int af, const char *ip) +{ + int srv_fd = -1, client_fd = -1; + int port; + + srv_fd = start_server(af, SOCK_STREAM, ip, 0, TIMEOUT_MS); + if (!ASSERT_OK_FD(srv_fd, "start_server")) + return; + + port = get_socket_local_port(srv_fd); + if (!ASSERT_GE(port, 0, "get_socket_local_port")) { + close(srv_fd); + return; + } + + skel->bss->server_port = ntohs(port); + skel->bss->unreach_type = (af == AF_INET) ? ICMP_DEST_UNREACH : + ICMPV6_DEST_UNREACH; + skel->bss->unreach_code = code; + skel->data->kfunc_ret = KFUNC_RET_UNSET; + + client_fd = connect_to_fd_nonblock(srv_fd); + if (!ASSERT_OK_FD(client_fd, "client_connect_nonblock")) { + close(srv_fd); + return; + } + + if (valid_unreach_code(code, af)) + read_icmp_errqueue(client_fd, code, af); + + close(client_fd); + close(srv_fd); +} + +static void run_icmp_test(struct icmp_send *skel, int af, const char *ip, + int max_code) +{ + for (int code = 0; code <= max_code; code++) { + if (af == AF_INET && code == ICMP_FRAG_NEEDED) + continue; + + trigger_prog_read_icmp_errqueue(skel, code, af, ip); + ASSERT_EQ(skel->data->kfunc_ret, 0, "kfunc_ret"); + } + + /* Test invalid codes */ + trigger_prog_read_icmp_errqueue(skel, -1, af, ip); + ASSERT_EQ(skel->data->kfunc_ret, -EINVAL, "kfunc_ret"); + + trigger_prog_read_icmp_errqueue(skel, max_code + 1, af, ip); + ASSERT_EQ(skel->data->kfunc_ret, -EINVAL, "kfunc_ret"); + + if (af == AF_INET) { + trigger_prog_read_icmp_errqueue(skel, ICMP_FRAG_NEEDED, af, ip); + ASSERT_EQ(skel->data->kfunc_ret, -EINVAL, "kfunc_ret"); + } +} + +static void run_icmp_no_route_test(struct icmp_send *skel, int af) +{ + union { + struct ipv4_packet v4; + struct ipv6_packet v6; + } pkt; + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, + .data_in = &pkt, + ); + int err; + + switch (af) { + case AF_INET: + pkt.v4 = pkt_v4; + pkt.v4.iph.version = 4; + pkt.v4.iph.daddr = htonl(INADDR_LOOPBACK); + pkt.v4.tcp.dest = htons(80); + opts.data_size_in = sizeof(pkt.v4); + skel->bss->unreach_type = ICMP_DEST_UNREACH; + break; + case AF_INET6: + pkt.v6 = pkt_v6; + pkt.v6.iph.version = 6; + pkt.v6.iph.daddr = in6addr_loopback; + pkt.v6.tcp.dest = htons(80); + opts.data_size_in = sizeof(pkt.v6); + skel->bss->unreach_type = ICMPV6_DEST_UNREACH; + break; + default: + ASSERT_FAIL("af_not_supported"); + return; + } + + skel->bss->server_port = 80; + skel->data->kfunc_ret = KFUNC_RET_UNSET; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.egress), &opts); + if (!ASSERT_OK(err, "test_run")) + return; + + ASSERT_EQ(skel->data->kfunc_ret, -ENETUNREACH, "kfunc_ret_no_route"); +} + +void test_icmp_send_unreach_cgroup(void) +{ + struct icmp_send *skel; + int cgroup_fd = -1; + + skel = icmp_send__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + cgroup_fd = test__join_cgroup("/icmp_send_unreach_cgroup"); + if (!ASSERT_OK_FD(cgroup_fd, "join_cgroup")) + goto cleanup; + + skel->links.egress = + bpf_program__attach_cgroup(skel->progs.egress, cgroup_fd); + if (!ASSERT_OK_PTR(skel->links.egress, "prog_attach_cgroup")) + goto cleanup; + + if (test__start_subtest("ipv4")) + run_icmp_test(skel, AF_INET, "127.0.0.1", NR_ICMP_UNREACH); + + if (test__start_subtest("ipv6")) + run_icmp_test(skel, AF_INET6, "::1", ICMPV6_REJECT_ROUTE); + + if (test__start_subtest("no_route_ipv4")) + run_icmp_no_route_test(skel, AF_INET); + + if (test__start_subtest("no_route_ipv6")) + run_icmp_no_route_test(skel, AF_INET6); + +cleanup: + icmp_send__destroy(skel); + if (cgroup_fd >= 0) + close(cgroup_fd); +} + +void test_icmp_send_unreach_recursion(void) +{ + struct icmp_send *skel; + int cgroup_fd = -1; + int err; + + err = setup_cgroup_environment(); + if (!ASSERT_OK(err, "setup_cgroup_environment")) + return; + + skel = icmp_send__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + cgroup_fd = get_root_cgroup(); + if (!ASSERT_OK_FD(cgroup_fd, "get_root_cgroup")) + goto cleanup; + + skel->data->target_pid = getpid(); + skel->links.recursion = + bpf_program__attach_cgroup(skel->progs.recursion, cgroup_fd); + if (!ASSERT_OK_PTR(skel->links.recursion, "prog_attach_cgroup")) + goto cleanup; + + trigger_prog_read_icmp_errqueue(skel, ICMP_HOST_UNREACH, AF_INET, + "127.0.0.1"); + + /* + * Because there's recursion involved, the first call will return at + * index 1 since it will return the second, and the second call will + * return at index 0 since it will return the first. + */ + ASSERT_EQ(skel->bss->rec_count, 2, "rec_count"); + ASSERT_EQ(skel->data->rec_kfunc_rets[0], -EBUSY, "kfunc_rets[0]"); + ASSERT_EQ(skel->data->rec_kfunc_rets[1], 0, "kfunc_rets[1]"); + +cleanup: + icmp_send__destroy(skel); + if (cgroup_fd >= 0) + close(cgroup_fd); + cleanup_cgroup_environment(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/iter_buf_null_fail.c b/tools/testing/selftests/bpf/prog_tests/iter_buf_null_fail.c new file mode 100644 index 000000000000..ea97787b870d --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/iter_buf_null_fail.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "iter_buf_null_fail.skel.h" + +void test_iter_buf_null_fail(void) +{ + RUN_TESTS(iter_buf_null_fail); +} diff --git a/tools/testing/selftests/bpf/prog_tests/iters.c b/tools/testing/selftests/bpf/prog_tests/iters.c index a539980a2fbe..c0b6082f345a 100644 --- a/tools/testing/selftests/bpf/prog_tests/iters.c +++ b/tools/testing/selftests/bpf/prog_tests/iters.c @@ -202,8 +202,6 @@ cleanup: iters_task__destroy(skel); } -extern int stack_mprotect(void); - static void subtest_css_task_iters(void) { struct iters_css_task *skel = NULL; diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c index f79c8e53cb3e..2b39cc1b09f9 100644 --- a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c +++ b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c @@ -66,14 +66,20 @@ static struct kfunc_test_params kfunc_tests[] = { TC_FAIL(kfunc_call_test_get_mem_fail_rdonly, 0, "R0 cannot write into rdonly_mem"), TC_FAIL(kfunc_call_test_get_mem_fail_use_after_free, 0, "invalid mem access 'scalar'"), TC_FAIL(kfunc_call_test_get_mem_fail_oob, 0, "min value is outside of the allowed memory range"), + TC_FAIL(kfunc_call_test_get_mem_fail_zero_size, 0, "min value is outside of the allowed memory range"), + TC_FAIL(kfunc_call_test_get_mem_fail_oversized, 0, "allocation size exceeds u32 max"), TC_FAIL(kfunc_call_test_get_mem_fail_not_const, 0, "is not a const"), TC_FAIL(kfunc_call_test_mem_acquire_fail, 0, "acquire kernel function does not return PTR_TO_BTF_ID"), - TC_FAIL(kfunc_call_test_pointer_arg_type_mismatch, 0, "arg#0 expected pointer to ctx, but got scalar"), + TC_FAIL(kfunc_call_test_pointer_arg_type_mismatch, 0, "R1 expected pointer to ctx, but got scalar"), + TC_FAIL(kfunc_call_test_spin_lock_unsafe, 0, "function calls are not allowed while holding a lock"), /* success cases */ + TC_TEST(kfunc_call_test_spin_lock_safe, 0), TC_TEST(kfunc_call_test1, 12), TC_TEST(kfunc_call_test2, 3), TC_TEST(kfunc_call_test4, -1234), + TC_TEST(kfunc_call_test5, 0), + TC_TEST(kfunc_call_test5_asm, 0), TC_TEST(kfunc_call_test_ref_btf_id, 0), TC_TEST(kfunc_call_test_get_mem, 42), SYSCALL_TEST(kfunc_syscall_test, 0), diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c b/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c index 8cd298b78e44..04aaf4c9cf5e 100644 --- a/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c +++ b/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c @@ -14,7 +14,7 @@ static struct { const char *prog_name; int expected_runtime_err; } kfunc_dynptr_tests[] = { - {"dynptr_data_null", -EBADMSG}, + {"dynptr_data_null", -EINVAL}, }; static bool kfunc_not_supported; diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_implicit_args_tracing.c b/tools/testing/selftests/bpf/prog_tests/kfunc_implicit_args_tracing.c new file mode 100644 index 000000000000..61cc5aaba025 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/kfunc_implicit_args_tracing.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <test_progs.h> +#include "kfunc_implicit_args_tracing.skel.h" + +void test_kfunc_implicit_args_tracing(void) +{ + struct kfunc_implicit_args_tracing *skel; + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, fd; + + skel = kfunc_implicit_args_tracing__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + err = kfunc_implicit_args_tracing__attach(skel); + if (!ASSERT_OK(err, "attach")) + goto cleanup; + + fd = bpf_program__fd(skel->progs.trigger_implicit_arg); + err = bpf_prog_test_run_opts(fd, &topts); + if (!ASSERT_OK(err, "test_run")) + goto cleanup; + + ASSERT_EQ(topts.retval, 5, "kfunc_retval"); + ASSERT_EQ(skel->bss->fentry_arg_cnt, 2, "fentry_arg_cnt"); + ASSERT_NEQ(skel->bss->fentry_aux_arg, 0, "fentry_aux_arg"); + ASSERT_EQ(skel->bss->fentry_result, 1, "fentry_result"); + ASSERT_EQ(skel->bss->fexit_arg_cnt, 2, "fexit_arg_cnt"); + ASSERT_NEQ(skel->bss->fexit_aux_arg, 0, "fexit_aux_arg"); + ASSERT_EQ(skel->bss->fexit_result, 1, "fexit_result"); + +cleanup: + kfunc_implicit_args_tracing__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c index f81dcd609ee9..2e0ddef77ba5 100644 --- a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c +++ b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c @@ -10,6 +10,7 @@ #include "kprobe_multi_session_cookie.skel.h" #include "kprobe_multi_verifier.skel.h" #include "kprobe_write_ctx.skel.h" +#include "kprobe_multi_sleepable.skel.h" #include "bpf/libbpf_internal.h" #include "bpf/hashmap.h" @@ -220,7 +221,9 @@ static void test_attach_api_syms(void) static void test_attach_api_fails(void) { LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); + LIBBPF_OPTS(bpf_test_run_opts, topts); struct kprobe_multi *skel = NULL; + struct kprobe_multi_sleepable *sl_skel = NULL; struct bpf_link *link = NULL; unsigned long long addrs[2]; const char *syms[2] = { @@ -228,7 +231,7 @@ static void test_attach_api_fails(void) "bpf_fentry_test2", }; __u64 cookies[2]; - int saved_error; + int saved_error, err; addrs[0] = ksym_get_addr("bpf_fentry_test1"); addrs[1] = ksym_get_addr("bpf_fentry_test2"); @@ -327,9 +330,63 @@ static void test_attach_api_fails(void) if (!ASSERT_EQ(saved_error, -E2BIG, "fail_6_error")) goto cleanup; + /* fail_7 - non-existent wildcard pattern (slow path) */ + LIBBPF_OPTS_RESET(opts); + + link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, + "__nonexistent_func_xyz_*", + &opts); + saved_error = -errno; + if (!ASSERT_ERR_PTR(link, "fail_7")) + goto cleanup; + + if (!ASSERT_EQ(saved_error, -ENOENT, "fail_7_error")) + goto cleanup; + + /* fail_8 - non-existent exact name (fast path), same error as wildcard */ + link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, + "__nonexistent_func_xyz_123", + &opts); + saved_error = -errno; + if (!ASSERT_ERR_PTR(link, "fail_8")) + goto cleanup; + + if (!ASSERT_EQ(saved_error, -ENOENT, "fail_8_error")) + goto cleanup; + + /* fail_9 - sleepable kprobe multi should not attach */ + sl_skel = kprobe_multi_sleepable__open(); + if (!ASSERT_OK_PTR(sl_skel, "sleep_skel_open")) + goto cleanup; + + sl_skel->bss->user_ptr = sl_skel; + + err = bpf_program__set_flags(sl_skel->progs.handle_kprobe_multi_sleepable, + BPF_F_SLEEPABLE); + if (!ASSERT_OK(err, "sleep_skel_set_flags")) + goto cleanup; + + err = kprobe_multi_sleepable__load(sl_skel); + if (!ASSERT_OK(err, "sleep_skel_load")) + goto cleanup; + + link = bpf_program__attach_kprobe_multi_opts(sl_skel->progs.handle_kprobe_multi_sleepable, + "bpf_fentry_test1", NULL); + saved_error = -errno; + + if (!ASSERT_ERR_PTR(link, "fail_9")) + goto cleanup; + + if (!ASSERT_EQ(saved_error, -EINVAL, "fail_9_error")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(sl_skel->progs.fentry), &topts); + ASSERT_OK(err, "bpf_prog_test_run_opts"); + cleanup: bpf_link__destroy(link); kprobe_multi__destroy(skel); + kprobe_multi_sleepable__destroy(sl_skel); } static void test_session_skel_api(void) @@ -355,8 +412,13 @@ static void test_session_skel_api(void) ASSERT_OK(err, "test_run"); ASSERT_EQ(topts.retval, 0, "test_run"); - /* bpf_fentry_test1-4 trigger return probe, result is 2 */ - for (i = 0; i < 4; i++) + /* + * bpf_fentry_test1 is hit by both the wildcard probe and the exact + * name probe (test_kprobe_syms), so entry + return fires twice: 4. + * bpf_fentry_test2-4 are hit only by the wildcard probe: 2. + */ + ASSERT_EQ(skel->bss->kprobe_session_result[0], 4, "kprobe_session_result"); + for (i = 1; i < 4; i++) ASSERT_EQ(skel->bss->kprobe_session_result[i], 2, "kprobe_session_result"); /* bpf_fentry_test5-8 trigger only entry probe, result is 1 */ @@ -604,6 +666,44 @@ static void test_attach_write_ctx(void) } #endif +/* + * Test kprobe_multi handles shadow symbols (vmlinux + module duplicate). + * bpf_fentry_shadow_test exists in both vmlinux and bpf_testmod. + * kprobe_multi resolves via ftrace_lookup_symbols() which finds the + * vmlinux symbol first and stops, so this should always succeed. + */ +static void test_attach_probe_dup_sym(void) +{ + LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); + const char *syms[1] = { "bpf_fentry_shadow_test" }; + struct kprobe_multi *skel = NULL; + struct bpf_link *link1 = NULL, *link2 = NULL; + + skel = kprobe_multi__open_and_load(); + if (!ASSERT_OK_PTR(skel, "kprobe_multi__open_and_load")) + goto cleanup; + + skel->bss->pid = getpid(); + opts.syms = syms; + opts.cnt = ARRAY_SIZE(syms); + + link1 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, + NULL, &opts); + if (!ASSERT_OK_PTR(link1, "attach_kprobe_multi_dup_sym")) + goto cleanup; + + opts.retprobe = true; + link2 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kretprobe_manual, + NULL, &opts); + if (!ASSERT_OK_PTR(link2, "attach_kretprobe_multi_dup_sym")) + goto cleanup; + +cleanup: + bpf_link__destroy(link2); + bpf_link__destroy(link1); + kprobe_multi__destroy(skel); +} + void serial_test_kprobe_multi_bench_attach(void) { if (test__start_subtest("kernel")) @@ -647,5 +747,7 @@ void test_kprobe_multi_test(void) test_unique_match(); if (test__start_subtest("attach_write_ctx")) test_attach_write_ctx(); + if (test__start_subtest("dup_sym")) + test_attach_probe_dup_sym(); RUN_TESTS(kprobe_multi_verifier); } diff --git a/tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c b/tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c index 7def158da9eb..1215d6edd590 100644 --- a/tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c +++ b/tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c @@ -14,7 +14,8 @@ void test_kptr_xchg_inline(void) int err; #if !(defined(__x86_64__) || defined(__aarch64__) || \ - (defined(__riscv) && __riscv_xlen == 64)) + (defined(__riscv) && __riscv_xlen == 64) || \ + (defined(__loongarch__) && __loongarch_grlen == 64)) test__skip(); return; #endif diff --git a/tools/testing/selftests/bpf/prog_tests/ksock.c b/tools/testing/selftests/bpf/prog_tests/ksock.c new file mode 100644 index 000000000000..dd6b167623d9 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/ksock.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include <arpa/inet.h> + +#include "test_progs.h" +#include "network_helpers.h" +#include "ksock_lsm.skel.h" +#include "ksock_lsm_verifier.skel.h" + +#define NS_TEST "ksock_lsm_ns" +#define RECV_PORT 7777 +#define RECV_TIMEOUT_SEC 5 + +struct ksock_test_env { + struct nstoken *nstoken; + int rfd; +}; + +static bool ksock_test_env_setup(struct ksock_test_env *env) +{ + struct sockaddr_in addr = { + .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK), + .sin_port = htons(RECV_PORT), + }; + struct timeval tv = { .tv_sec = RECV_TIMEOUT_SEC }; + int err; + + memset(env, 0, sizeof(*env)); + env->rfd = -1; + + if (!ASSERT_OK(make_netns(NS_TEST), "make_netns")) + goto fail; + + env->nstoken = open_netns(NS_TEST); + if (!ASSERT_OK_PTR(env->nstoken, "open_netns")) + goto fail; + + env->rfd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if (!ASSERT_OK_FD(env->rfd, "receiver socket")) + goto fail; + + err = bind(env->rfd, (struct sockaddr *)&addr, sizeof(addr)); + if (!ASSERT_OK(err, "bind receiver")) + goto fail; + + err = setsockopt(env->rfd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + if (!ASSERT_OK(err, "set rcvtimeo")) + goto fail; + + return true; + +fail: + return false; +} + +void test_ksock_lsm(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct ksock_test_env env; + struct sockaddr_in trigger_addr = { + .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK), + }; + struct ksock_lsm *skel; + char recv_data[sizeof(skel->data->send_data)] = {}; + ssize_t n; + int tfd = -1; + int err; + + skel = ksock_lsm__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + return; + + if (!ksock_test_env_setup(&env)) + goto fail; + + /* Step 1: Run the setup SYSCALL prog to create the ksock */ + skel->bss->ipv4_remote = htonl(INADDR_LOOPBACK); + skel->bss->remote_port = RECV_PORT; + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.ksock_setup), + &opts); + if (!ASSERT_OK(err, "ksock_setup run")) + goto fail; + if (!ASSERT_OK(opts.retval, "ksock_setup retval")) + goto fail; + + /* Step 2: Attach LSM prog and trigger socket_bind from userspace */ + skel->links.ksock_socket_bind = + bpf_program__attach_lsm(skel->progs.ksock_socket_bind); + if (!ASSERT_OK_PTR(skel->links.ksock_socket_bind, + "attach socket_bind lsm")) + goto fail; + + tfd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if (!ASSERT_OK_FD(tfd, "trigger socket")) + goto fail; + + skel->bss->target_pid = getpid(); + err = bind(tfd, (struct sockaddr *)&trigger_addr, sizeof(trigger_addr)); + skel->bss->target_pid = 0; + if (!ASSERT_OK(err, "trigger bind")) + goto fail; + + /* Step 3: Verify the LSM hook sent the notification */ + if (!ASSERT_EQ(skel->data->send_ret, sizeof(skel->data->send_data), + "LSM send bytes")) + goto fail; + + n = recvfrom(env.rfd, recv_data, sizeof(recv_data), 0, NULL, NULL); + if (ASSERT_EQ(n, sizeof(recv_data), "recvfrom len")) + ASSERT_MEMEQ(recv_data, skel->data->send_data, sizeof(recv_data), + "payload match"); + +fail: + if (tfd >= 0) + close(tfd); + if (env.rfd >= 0) + close(env.rfd); + if (env.nstoken) + close_netns(env.nstoken); + remove_netns(NS_TEST); + ksock_lsm__destroy(skel); +} + +void test_ksock_lsm_verifier(void) +{ + RUN_TESTS(ksock_lsm_verifier); +} diff --git a/tools/testing/selftests/bpf/prog_tests/ksock_wq.c b/tools/testing/selftests/bpf/prog_tests/ksock_wq.c new file mode 100644 index 000000000000..d6dc20b8f95b --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/ksock_wq.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include <unistd.h> + +#include "test_progs.h" +#include "ksock_wq.skel.h" + +#define CALLBACK_WAIT_RETRIES 1000 +#define CALLBACK_WAIT_US 1000 + +void test_ksock_wq(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct ksock_wq *skel; + u32 callback_done; + int err, i; + + skel = ksock_wq__open_and_load(); + if (!ASSERT_OK_PTR(skel, "ksock_wq open and load")) + return; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.ksock_wq_start), + &opts); + if (!ASSERT_OK(err, "run ksock_wq_start")) + goto out; + if (!ASSERT_OK(opts.retval, "ksock_wq_start retval")) + goto out; + + for (i = 0; i < CALLBACK_WAIT_RETRIES; i++) { + if (__atomic_load_n(&skel->bss->callback_done, __ATOMIC_ACQUIRE)) + break; + usleep(CALLBACK_WAIT_US); + } + callback_done = __atomic_load_n(&skel->bss->callback_done, + __ATOMIC_ACQUIRE); + if (!ASSERT_EQ(callback_done, 1, "workqueue callback completed")) + goto out; + + ASSERT_EQ(skel->bss->create_err, -EOPNOTSUPP, + "workqueue create rejected"); + +out: + ksock_wq__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/libarena.c b/tools/testing/selftests/bpf/prog_tests/libarena.c new file mode 100644 index 000000000000..daade4150af6 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/libarena.c @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> +#include <unistd.h> + +#include <libarena/common.h> +#include <libarena/asan.h> +#include <libarena/buddy.h> +#include <libarena/userspace.h> + +#include "libarena/libarena.skel.h" + +static void run_libarena_test(struct libarena *skel, struct bpf_program *prog, + const char *name) +{ + int ret; + + if (strstr(name, "test_buddy")) { + /* Buddy tests initialize the allocator directly. */ + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_destroy)); + if (!ASSERT_OK(ret, "arena_buddy_destroy")) + return; + } else { + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_reset)); + if (!ASSERT_OK(ret, "arena_buddy_reset")) + return; + } + + ret = libarena_run_prog(bpf_program__fd(prog)); + + ASSERT_OK(ret, name); +} + +static void *run_libarena_parallel_prog(void *arg) +{ + struct bpf_program *prog = arg; + + return (void *)(long)libarena_run_prog(bpf_program__fd(prog)); +} + +/* Max suffix is ceil((lg 2^32) / (lg 10)) + sizeof("__") = 10 + 2 = 12. */ +#define MAX_PARTEST_SUFFIX (12) +#define MAX_PARTEST_NAME (1024) +#define MAX_PARTEST_PREFIX (MAX_PARTEST_NAME - MAX_PARTEST_SUFFIX) + +static int run_libarena_parallel_fini(struct libarena *skel, const char *name, + size_t prefixlen) +{ + char tdname[MAX_PARTEST_NAME]; + struct bpf_program *fini_prog; + int ret; + + ret = snprintf(tdname, sizeof(tdname), "%.*s__fini", (int)prefixlen, name); + if (!ASSERT_LT(ret, sizeof(tdname), "partest fini name")) + return -ENAMETOOLONG; + + fini_prog = bpf_object__find_program_by_name(skel->obj, tdname); + if (!ASSERT_TRUE(fini_prog, "partest fini prog")) + return -ENOENT; + + ret = libarena_run_prog(bpf_program__fd(fini_prog)); + ASSERT_OK(ret, tdname); + + return ret; +} + +static int run_libarena_parallel_test_workers(struct libarena *skel, + const char *name, size_t prefixlen) +{ + pthread_t *threads = NULL, *tmp_threads; + char tdname[MAX_PARTEST_NAME]; + struct bpf_program *tdprog; + uint32_t nthreads; + void *thread_ret; + int ret, err = 0; + int worker_err; + int i; + + for (nthreads = 0; nthreads < UINT_MAX; nthreads++) { + ret = snprintf(tdname, sizeof(tdname), "%.*s__%u", (int)prefixlen, + name, nthreads); + if (!ASSERT_LT(ret, sizeof(tdname), "test worker name")) { + err = -ENAMETOOLONG; + break; + } + + /* + * We enumerate the worker threads for a given test with __0, __1, + * and so on. The suffixes always start from 0 and are contiguous, + * so if we don't find a program with the requested name we have + * discovered all available worker programs. + */ + tdprog = bpf_object__find_program_by_name(skel->obj, tdname); + if (!tdprog) + break; + + /* Bump the alloc array to accommodate the new thread. */ + tmp_threads = realloc(threads, (nthreads + 1) * sizeof(*threads)); + if (!ASSERT_TRUE(tmp_threads, "realloc")) { + err = -ENOMEM; + break; + } + threads = tmp_threads; + + ret = pthread_create(&threads[nthreads], NULL, + run_libarena_parallel_prog, + tdprog); + if (!ASSERT_OK(ret, "pthread_create")) { + err = ret; + break; + } + } + + + for (i = 0; i < nthreads; i++) { + ret = pthread_join(threads[i], &thread_ret); + if (!ASSERT_OK(ret, "pthread_join")) { + err = err ?: ret; + continue; + } + + worker_err = (long)thread_ret; + + /* + * A worker that bails out because another one already gave up + * reports -EINTR. It is collateral damage that carries no + * information, so skip it entirely: never let it become the + * reported error, and don't log it either. + */ + if (!worker_err || worker_err == -EINTR) + continue; + + if (!err) + err = worker_err; + + fprintf(stdout, "%.*s__%d returned %d\n", (int)prefixlen, name, + i, worker_err); + } + + free(threads); + + return err; +} + +static bool libarena_parallel_test_enabled(struct libarena *skel, + const char *prefix, + size_t prefixlen) +{ + struct bpf_program *prog; + char progname[MAX_PARTEST_NAME]; + int ret; + + ret = snprintf(progname, sizeof(progname), "%.*s__enabled", (int)prefixlen, + prefix); + if (!ASSERT_LT(ret, sizeof(progname), "partest enabled name")) + return false; + + prog = bpf_object__find_program_by_name(skel->obj, progname); + if (!prog) + return true; + + ret = libarena_run_prog(bpf_program__fd(prog)); + if (ret == -EOPNOTSUPP) + return false; + if (!ASSERT_OK(ret, progname)) + return false; + return true; +} + +static void run_libarena_parallel_test(struct libarena *skel, struct bpf_program *prog, + const char *name) +{ + char testname[MAX_PARTEST_NAME]; + size_t prefixlen; + const char *pos; + int ret; + + /* + * We annotate the initialization prog with __init. If the current prog does + * not match, it is one of the parallel threads instead and is ignored. + * + * We assume the test writer knows what they are doing and do not add __init + * randomly in the middle of a test name. + */ + pos = strstr(name, "__init"); + if (!pos) + return; + + prefixlen = pos - name; + if (!ASSERT_LT(prefixlen, MAX_PARTEST_PREFIX, "partest prefix too long")) + return; + + /* The name of the test without the __init suffix. Looks nicer in the test log. */ + ret = snprintf(testname, sizeof(testname), "%.*s", (int)prefixlen, name); + if (!ASSERT_LT(ret, sizeof(testname), "partest test name")) + return; + + if (!test__start_subtest(testname)) + return; + + if (!libarena_parallel_test_enabled(skel, testname, prefixlen)) { + test__skip(); + return; + } + + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_reset)); + if (!ASSERT_OK(ret, "arena_buddy_reset")) + return; + + ret = libarena_run_prog(bpf_program__fd(prog)); + if (!ASSERT_OK(ret, testname)) + return; + + ret = run_libarena_parallel_test_workers(skel, name, prefixlen); + + ASSERT_OK(ret, testname); + + run_libarena_parallel_fini(skel, name, prefixlen); +} + +void serial_test_libarena(void) +{ + struct arena_alloc_reserve_args args; + struct libarena *skel; + struct bpf_program *prog; + int ret; + + skel = libarena__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + ret = libarena__attach(skel); + if (!ASSERT_OK(ret, "attach")) + goto out; + + args.nr_pages = ARENA_RESERVE_PAGES_DFL; + + ret = libarena_run_prog_args(bpf_program__fd(skel->progs.arena_alloc_reserve), + &args, sizeof(args)); + if (!ASSERT_OK(ret, "arena_alloc_reserve")) + goto out; + + bpf_object__for_each_program(prog, skel->obj) { + const char *name = bpf_program__name(prog); + + /* + * Handle parallel test progs separately. For those + * progs it's not a matter of test/skip, because each + * parallel test prog includes an initialization prog + * and a set of progs to be run in parallel. For the + * latter we do not record them as skipped or run, + * because we run them all at once when we come across + * the initialization prog. For more details on how we + * discover the progs see the comment on + * run_libarena_parallel_test. + */ + if (libarena_is_parallel_test_prog(name)) { + run_libarena_parallel_test(skel, prog, name); + continue; + } + + if (!libarena_is_test_prog(name)) + continue; + + if (!test__start_subtest(name)) + continue; + + run_libarena_test(skel, prog, name); + } + +out: + libarena__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/libarena_asan.c b/tools/testing/selftests/bpf/prog_tests/libarena_asan.c new file mode 100644 index 000000000000..9c31b17dbf39 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/libarena_asan.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> + +#ifdef HAS_BPF_ARENA_ASAN +#include <unistd.h> + +#include <libarena/common.h> +#include <libarena/asan.h> +#include <libarena/buddy.h> +#include <libarena/userspace.h> + +#include "libarena/libarena_asan.skel.h" + +static void run_libarena_asan_test(struct libarena_asan *skel, + struct bpf_program *prog, const char *name) +{ + int ret; + + if (strstr(name, "test_buddy")) { + /* Buddy tests initialize the allocator directly. */ + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_destroy)); + if (!ASSERT_OK(ret, "arena_buddy_destroy")) + return; + } else { + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_reset)); + if (!ASSERT_OK(ret, "arena_buddy_reset")) + return; + } + + ret = libarena_run_prog(bpf_program__fd(prog)); + ASSERT_OK(ret, name); + + verify_test_stderr(skel->obj, prog); +} + +static void run_test(void) +{ + struct arena_alloc_reserve_args args; + struct libarena_asan *skel; + struct bpf_program *prog; + int ret; + + skel = libarena_asan__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + ret = libarena_asan__attach(skel); + if (!ASSERT_OK(ret, "attach")) + goto out; + + args.nr_pages = ARENA_RESERVE_PAGES_DFL; + + ret = libarena_run_prog_args(bpf_program__fd(skel->progs.arena_alloc_reserve), + &args, sizeof(args)); + if (!ASSERT_OK(ret, "arena_alloc_reserve")) + goto out; + + ret = libarena_asan_init( + bpf_program__fd(skel->progs.arena_get_info), + bpf_program__fd(skel->progs.asan_init), + (1ULL << 32) / sysconf(_SC_PAGESIZE)); + if (!ASSERT_OK(ret, "libarena_asan_init")) + goto out; + + bpf_object__for_each_program(prog, skel->obj) { + const char *name = bpf_program__name(prog); + + if (!libarena_is_asan_test_prog(name)) + continue; + + if (!test__start_subtest(name)) + continue; + + run_libarena_asan_test(skel, prog, name); + } + +out: + libarena_asan__destroy(skel); +} + +#endif /* HAS_BPF_ARENA_ASAN */ + +/* + * Run the test depending on whether LLVM can compile arena ASAN + * programs. + */ +void serial_test_libarena_asan(void) +{ +#ifdef HAS_BPF_ARENA_ASAN + run_test(); +#else + test__skip(); +#endif + + return; +} diff --git a/tools/testing/selftests/bpf/prog_tests/linked_list.c b/tools/testing/selftests/bpf/prog_tests/linked_list.c index 14c5a7ef0e87..52fabbee3dd5 100644 --- a/tools/testing/selftests/bpf/prog_tests/linked_list.c +++ b/tools/testing/selftests/bpf/prog_tests/linked_list.c @@ -68,6 +68,7 @@ static struct { { "obj_type_id_oor", "local type ID argument must be in range [0, U32_MAX]" }, { "obj_new_no_composite", "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct" }, { "obj_new_no_struct", "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct" }, + { "obj_new_flex_array", "access beyond struct obj_new_flex" }, { "obj_drop_non_zero_off", "R1 must have zero offset when passed to release func" }, { "new_null_ret", "R0 invalid mem access 'ptr_or_null_'" }, { "obj_new_acq", "Unreleased reference id=" }, @@ -81,18 +82,18 @@ static struct { { "direct_write_node", "direct access to bpf_list_node is disallowed" }, { "use_after_unlock_push_front", "invalid mem access 'scalar'" }, { "use_after_unlock_push_back", "invalid mem access 'scalar'" }, - { "double_push_front", "arg#1 expected pointer to allocated object" }, - { "double_push_back", "arg#1 expected pointer to allocated object" }, + { "double_push_front", "R2 expected pointer to allocated object" }, + { "double_push_back", "R2 expected pointer to allocated object" }, { "no_node_value_type", "bpf_list_node not found at offset=0" }, { "incorrect_value_type", "operation on bpf_list_head expects arg#1 bpf_list_node at offset=48 in struct foo, " "but arg is at offset=0 in struct bar" }, - { "incorrect_node_var_off", "variable ptr_ access var_off=(0x0; 0xffffffff) disallowed" }, + { "incorrect_node_var_off", "variable ptr_ access var_off=(0x0; 0x1ffffffff) disallowed" }, { "incorrect_node_off1", "bpf_list_node not found at offset=49" }, { "incorrect_node_off2", "arg#1 offset=0, but expected bpf_list_node at offset=48 in struct foo" }, { "no_head_type", "bpf_list_head not found at offset=0" }, { "incorrect_head_var_off1", "R1 doesn't have constant offset" }, - { "incorrect_head_var_off2", "variable ptr_ access var_off=(0x0; 0xffffffff) disallowed" }, + { "incorrect_head_var_off2", "variable ptr_ access var_off=(0x0; 0x1ffffffff) disallowed" }, { "incorrect_head_off1", "bpf_list_head not found at offset=25" }, { "incorrect_head_off2", "bpf_list_head not found at offset=1" }, { "pop_front_off", "off 48 doesn't point to 'struct bpf_spin_lock' that is at 40" }, @@ -131,13 +132,14 @@ end: linked_list_fail__destroy(skel); } -static void clear_fields(struct bpf_map *map) +static void clear_fields(struct bpf_program *prog) { - char buf[24]; - int key = 0; + LIBBPF_OPTS(bpf_test_run_opts, opts); + int ret; - memset(buf, 0xff, sizeof(buf)); - ASSERT_OK(bpf_map__update_elem(map, &key, sizeof(key), buf, sizeof(buf), 0), "check_and_free_fields"); + ret = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + ASSERT_OK(ret, "clear_fields"); + ASSERT_OK(opts.retval, "clear_fields retval"); } enum { @@ -170,31 +172,31 @@ static void test_linked_list_success(int mode, bool leave_in_map) ASSERT_OK(ret, "map_list_push_pop"); ASSERT_OK(opts.retval, "map_list_push_pop retval"); if (!leave_in_map) - clear_fields(skel->maps.array_map); + clear_fields(skel->progs.clear_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.inner_map_list_push_pop), &opts); ASSERT_OK(ret, "inner_map_list_push_pop"); ASSERT_OK(opts.retval, "inner_map_list_push_pop retval"); if (!leave_in_map) - clear_fields(skel->maps.inner_map); + clear_fields(skel->progs.clear_inner_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.global_list_push_pop), &opts); ASSERT_OK(ret, "global_list_push_pop"); ASSERT_OK(opts.retval, "global_list_push_pop retval"); if (!leave_in_map) - clear_fields(skel->maps.bss_A); + clear_fields(skel->progs.clear_global_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.global_list_push_pop_nested), &opts); ASSERT_OK(ret, "global_list_push_pop_nested"); ASSERT_OK(opts.retval, "global_list_push_pop_nested retval"); if (!leave_in_map) - clear_fields(skel->maps.bss_A); + clear_fields(skel->progs.clear_global_nested_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.global_list_array_push_pop), &opts); ASSERT_OK(ret, "global_list_array_push_pop"); ASSERT_OK(opts.retval, "global_list_array_push_pop retval"); if (!leave_in_map) - clear_fields(skel->maps.bss_A); + clear_fields(skel->progs.clear_global_array_list); if (mode == PUSH_POP) goto end; @@ -204,19 +206,19 @@ ppm: ASSERT_OK(ret, "map_list_push_pop_multiple"); ASSERT_OK(opts.retval, "map_list_push_pop_multiple retval"); if (!leave_in_map) - clear_fields(skel->maps.array_map); + clear_fields(skel->progs.clear_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.inner_map_list_push_pop_multiple), &opts); ASSERT_OK(ret, "inner_map_list_push_pop_multiple"); ASSERT_OK(opts.retval, "inner_map_list_push_pop_multiple retval"); if (!leave_in_map) - clear_fields(skel->maps.inner_map); + clear_fields(skel->progs.clear_inner_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.global_list_push_pop_multiple), &opts); ASSERT_OK(ret, "global_list_push_pop_multiple"); ASSERT_OK(opts.retval, "global_list_push_pop_multiple retval"); if (!leave_in_map) - clear_fields(skel->maps.bss_A); + clear_fields(skel->progs.clear_global_list); if (mode == PUSH_POP_MULT) goto end; @@ -226,19 +228,19 @@ lil: ASSERT_OK(ret, "map_list_in_list"); ASSERT_OK(opts.retval, "map_list_in_list retval"); if (!leave_in_map) - clear_fields(skel->maps.array_map); + clear_fields(skel->progs.clear_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.inner_map_list_in_list), &opts); ASSERT_OK(ret, "inner_map_list_in_list"); ASSERT_OK(opts.retval, "inner_map_list_in_list retval"); if (!leave_in_map) - clear_fields(skel->maps.inner_map); + clear_fields(skel->progs.clear_inner_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.global_list_in_list), &opts); ASSERT_OK(ret, "global_list_in_list"); ASSERT_OK(opts.retval, "global_list_in_list retval"); if (!leave_in_map) - clear_fields(skel->maps.bss_A); + clear_fields(skel->progs.clear_global_list); end: linked_list__destroy(skel); } @@ -712,7 +714,7 @@ static void test_btf(void) break; err = btf__load_into_kernel(btf); - ASSERT_EQ(err, -ELOOP, "check btf"); + ASSERT_EQ(err, 0, "check btf"); btf__free(btf); break; } @@ -771,7 +773,7 @@ static void test_btf(void) break; err = btf__load_into_kernel(btf); - ASSERT_EQ(err, -ELOOP, "check btf"); + ASSERT_EQ(err, 0, "check btf"); btf__free(btf); break; } diff --git a/tools/testing/selftests/bpf/prog_tests/livepatch_trampoline.c b/tools/testing/selftests/bpf/prog_tests/livepatch_trampoline.c index 72aa5376c30e..0a12af924a99 100644 --- a/tools/testing/selftests/bpf/prog_tests/livepatch_trampoline.c +++ b/tools/testing/selftests/bpf/prog_tests/livepatch_trampoline.c @@ -5,6 +5,8 @@ #include "testing_helpers.h" #include "livepatch_trampoline.skel.h" +#define LIVEPATCH_ENABLED_PATH "/sys/kernel/livepatch/livepatch_sample/enabled" + static int load_livepatch(void) { char path[4096]; @@ -19,7 +21,8 @@ static int load_livepatch(void) static void unload_livepatch(void) { /* Disable the livepatch before unloading the module */ - system("echo 0 > /sys/kernel/livepatch/livepatch_sample/enabled"); + if (!access(LIVEPATCH_ENABLED_PATH, F_OK)) + system("echo 0 > " LIVEPATCH_ENABLED_PATH); unload_module("livepatch_sample", env_verbosity > VERBOSE_NONE); } @@ -81,9 +84,22 @@ out: void test_livepatch_trampoline(void) { int retry_cnt = 0; + int err; + + /* Skip if kernel was built without CONFIG_LIVEPATCH */ + if (access("/sys/kernel/livepatch", F_OK)) { + test__skip(); + return; + } retry: - if (load_livepatch()) { + err = load_livepatch(); + if (err) { + if (err == -ENOENT) { + test__skip(); + return; + } + if (retry_cnt) { ASSERT_OK(1, "load_livepatch"); goto out; diff --git a/tools/testing/selftests/bpf/prog_tests/local_kptr_ownership.c b/tools/testing/selftests/bpf/prog_tests/local_kptr_ownership.c new file mode 100644 index 000000000000..a487aa68f2ee --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/local_kptr_ownership.c @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <bpf/btf.h> +#include <linux/btf.h> +#include <test_progs.h> + +#define SPIN_LOCK 2 +#define LIST_HEAD 3 +#define LIST_NODE 4 +/* Keep in sync with BTF_MAX_OWNERSHIP_DEPTH. */ +#define MAX_OWNERSHIP_DEPTH 8 + +static struct btf *init_btf(void) +{ + struct btf *btf; + int id; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf__new_empty")) + return NULL; + id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_EQ(id, 1, "btf__add_int")) + goto err_out; + id = btf__add_struct(btf, "bpf_spin_lock", 4); + if (!ASSERT_EQ(id, SPIN_LOCK, "btf__add_struct bpf_spin_lock")) + goto err_out; + id = btf__add_struct(btf, "bpf_list_head", 16); + if (!ASSERT_EQ(id, LIST_HEAD, "btf__add_struct bpf_list_head")) + goto err_out; + id = btf__add_struct(btf, "bpf_list_node", 24); + if (!ASSERT_EQ(id, LIST_NODE, "btf__add_struct bpf_list_node")) + goto err_out; + return btf; + +err_out: + btf__free(btf); + return NULL; +} + +static int add_local_kptr(struct btf *btf, int pointee_id, const char *tag) +{ + int id; + + id = btf__add_type_tag(btf, tag, pointee_id); + if (!ASSERT_GT(id, 0, "btf__add_type_tag")) + return id; + id = btf__add_ptr(btf, id); + ASSERT_GT(id, 0, "btf__add_ptr"); + return id; +} + +static void test_self_cycle(const char *tag, int expected_err) +{ + struct btf *btf; + int id, err; + + btf = init_btf(); + if (!ASSERT_OK_PTR(btf, "init_btf")) + return; + id = add_local_kptr(btf, 7, tag); + if (id <= 0) + goto out; + id = btf__add_struct(btf, "self_cycle", 8); + if (!ASSERT_EQ(id, 7, "btf__add_struct self_cycle")) + goto out; + err = btf__add_field(btf, "next", 6, 0, 0); + if (!ASSERT_OK(err, "btf__add_field self_cycle::next")) + goto out; + + err = btf__load_into_kernel(btf); + ASSERT_EQ(err, expected_err, "check btf"); +out: + btf__free(btf); +} + +static void test_aba_cycle(void) +{ + struct btf *btf; + int id, err; + + btf = init_btf(); + if (!ASSERT_OK_PTR(btf, "init_btf")) + return; + id = add_local_kptr(btf, 10, "kptr"); + if (id <= 0) + goto out; + id = add_local_kptr(btf, 9, "kptr"); + if (id <= 0) + goto out; + id = btf__add_struct(btf, "cycle_a", 8); + if (!ASSERT_EQ(id, 9, "btf__add_struct cycle_a")) + goto out; + err = btf__add_field(btf, "b", 6, 0, 0); + if (!ASSERT_OK(err, "btf__add_field cycle_a::b")) + goto out; + id = btf__add_struct(btf, "cycle_b", 8); + if (!ASSERT_EQ(id, 10, "btf__add_struct cycle_b")) + goto out; + err = btf__add_field(btf, "a", 8, 0, 0); + if (!ASSERT_OK(err, "btf__add_field cycle_b::a")) + goto out; + + err = btf__load_into_kernel(btf); + ASSERT_EQ(err, -ELOOP, "check btf"); +out: + btf__free(btf); +} + +static void test_mixed_cycle(void) +{ + struct btf *btf; + int id, err; + + btf = init_btf(); + if (!ASSERT_OK_PTR(btf, "init_btf")) + return; + id = add_local_kptr(btf, 7, "kptr"); + if (id <= 0) + goto out; + id = btf__add_struct(btf, "mixed_owner", 20); + if (!ASSERT_EQ(id, 7, "btf__add_struct mixed_owner")) + goto out; + err = btf__add_field(btf, "root", LIST_HEAD, 0, 0); + if (!ASSERT_OK(err, "btf__add_field mixed_owner::root")) + goto out; + err = btf__add_field(btf, "lock", SPIN_LOCK, 128, 0); + if (!ASSERT_OK(err, "btf__add_field mixed_owner::lock")) + goto out; + id = btf__add_decl_tag(btf, "contains:mixed_node:node", 7, 0); + if (!ASSERT_EQ(id, 8, "btf__add_decl_tag mixed_owner")) + goto out; + id = btf__add_struct(btf, "mixed_node", 32); + if (!ASSERT_EQ(id, 9, "btf__add_struct mixed_node")) + goto out; + err = btf__add_field(btf, "node", LIST_NODE, 0, 0); + if (!ASSERT_OK(err, "btf__add_field mixed_node::node")) + goto out; + err = btf__add_field(btf, "owner", 6, 192, 0); + if (!ASSERT_OK(err, "btf__add_field mixed_node::owner")) + goto out; + + err = btf__load_into_kernel(btf); + ASSERT_EQ(err, -ELOOP, "check btf"); +out: + btf__free(btf); +} + +static void test_acyclic_depth(int depth, bool child_first, bool shared_suffix, int expected_err) +{ + int ptr_id[MAX_OWNERSHIP_DEPTH + 1]; + int first_struct_id; + struct btf *btf; + int id, err, i, n, pointee_id; + + btf = init_btf(); + if (!ASSERT_OK_PTR(btf, "init_btf")) + return; + first_struct_id = 5 + 2 * depth; + for (i = 0; i < depth; i++) { + if (i == depth - 1) + pointee_id = first_struct_id + depth; + else + pointee_id = first_struct_id + (child_first ? depth - 2 - i : i + 1); + ptr_id[i] = add_local_kptr(btf, pointee_id, "kptr"); + if (ptr_id[i] <= 0) + goto out; + } + for (n = 0; n < depth; n++) { + char name[32]; + int offset = 0; + + i = child_first ? depth - 1 - n : n; + snprintf(name, sizeof(name), "owner_%d", i); + id = btf__add_struct(btf, name, shared_suffix && !i ? 16 : 8); + if (!ASSERT_EQ(id, first_struct_id + n, "btf__add_struct owner")) + goto out; + if (shared_suffix && !i) { + /* + * Visit the shared suffix through the shorter path before + * reaching it again with less remaining depth. + */ + err = btf__add_field(btf, "suffix", ptr_id[1], 0, 0); + if (!ASSERT_OK(err, "btf__add_field owner::suffix")) + goto out; + offset = 64; + } + err = btf__add_field(btf, "next", ptr_id[i], offset, 0); + if (!ASSERT_OK(err, "btf__add_field owner::next")) + goto out; + } + id = btf__add_struct(btf, "plain_leaf", 4); + if (!ASSERT_EQ(id, first_struct_id + depth, "btf__add_struct plain_leaf")) + goto out; + + err = btf__load_into_kernel(btf); + ASSERT_EQ(err, expected_err, "check btf"); +out: + btf__free(btf); +} + +static void test_graph_depth(bool rbtree, int depth, int expected_err) +{ + int root_type = LIST_HEAD, node_type = LIST_NODE, node_size = 24; + int id, err, i, lock_off, root_off, size; + struct btf *btf; + + btf = init_btf(); + if (!ASSERT_OK_PTR(btf, "init_btf")) + return; + if (rbtree) { + root_type = btf__add_struct(btf, "bpf_rb_root", 16); + if (!ASSERT_GT(root_type, 0, "btf__add_struct bpf_rb_root")) + goto out; + node_type = btf__add_struct(btf, "bpf_rb_node", 32); + if (!ASSERT_GT(node_type, 0, "btf__add_struct bpf_rb_node")) + goto out; + node_size = 32; + } + + for (i = 0; i < depth; i++) { + char name[32], tag[64]; + + lock_off = i ? node_size : 0; + root_off = lock_off + 8; + size = i == depth - 1 ? node_size : root_off + 16; + snprintf(name, sizeof(name), "graph_owner_%d", i); + id = btf__add_struct(btf, name, size); + if (!ASSERT_GT(id, 0, "btf__add_struct graph_owner")) + goto out; + if (i) { + err = btf__add_field(btf, "node", node_type, 0, 0); + if (!ASSERT_OK(err, "btf__add_field graph_owner::node")) + goto out; + } + if (i == depth - 1) + continue; + err = btf__add_field(btf, "lock", SPIN_LOCK, lock_off * 8, 0); + if (!ASSERT_OK(err, "btf__add_field graph_owner::lock")) + goto out; + err = btf__add_field(btf, "root", root_type, root_off * 8, 0); + if (!ASSERT_OK(err, "btf__add_field graph_owner::root")) + goto out; + snprintf(tag, sizeof(tag), "contains:graph_owner_%d:node", i + 1); + err = btf__add_decl_tag(btf, tag, id, i ? 2 : 1); + if (!ASSERT_GT(err, 0, "btf__add_decl_tag graph_owner")) + goto out; + } + + err = btf__load_into_kernel(btf); + ASSERT_EQ(err, expected_err, "check btf"); +out: + btf__free(btf); +} + +void test_local_kptr_ownership(void) +{ + if (test__start_subtest("self_cycle")) + test_self_cycle("kptr", -ELOOP); + if (test__start_subtest("untrusted_self_cycle")) + test_self_cycle("kptr_untrusted", 0); + if (test__start_subtest("percpu_self_cycle")) + test_self_cycle("percpu_kptr", -ELOOP); + if (test__start_subtest("ABA_cycle")) + test_aba_cycle(); + if (test__start_subtest("mixed_graph_root_cycle")) + test_mixed_cycle(); + if (test__start_subtest("max_acyclic")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH, false, false, 0); + if (test__start_subtest("too_deep_acyclic")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH + 1, false, false, -ELOOP); + if (test__start_subtest("max_acyclic_child_first")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH, true, false, 0); + if (test__start_subtest("too_deep_acyclic_child_first")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH + 1, true, false, -ELOOP); + if (test__start_subtest("max_acyclic_shared_suffix")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH, false, true, 0); + if (test__start_subtest("too_deep_acyclic_shared_suffix")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH + 1, false, true, -ELOOP); + if (test__start_subtest("list_three_types")) + test_graph_depth(false, 3, 0); + if (test__start_subtest("list_four_types")) + test_graph_depth(false, 4, 0); + if (test__start_subtest("list_max_depth")) + test_graph_depth(false, MAX_OWNERSHIP_DEPTH, 0); + if (test__start_subtest("list_too_deep")) + test_graph_depth(false, MAX_OWNERSHIP_DEPTH + 1, -ELOOP); + if (test__start_subtest("rbtree_three_types")) + test_graph_depth(true, 3, 0); + if (test__start_subtest("rbtree_four_types")) + test_graph_depth(true, 4, 0); + if (test__start_subtest("rbtree_max_depth")) + test_graph_depth(true, MAX_OWNERSHIP_DEPTH, 0); + if (test__start_subtest("rbtree_too_deep")) + test_graph_depth(true, MAX_OWNERSHIP_DEPTH + 1, -ELOOP); +} diff --git a/tools/testing/selftests/bpf/prog_tests/lru_lock_nmi.c b/tools/testing/selftests/bpf/prog_tests/lru_lock_nmi.c new file mode 100644 index 000000000000..60666a9ba41f --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/lru_lock_nmi.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Stress every LRU lock-failure and orphan-recovery. + * perf_event NMI BPF on every online CPU does + * update+delete on a small LRU map; userspace threads on every CPU do + * the same from syscall context. + */ +#define _GNU_SOURCE +#include <pthread.h> +#include <sched.h> +#include <sys/syscall.h> +#include <linux/perf_event.h> +#include <test_progs.h> +#include "testing_helpers.h" +#include "lru_lock_nmi.skel.h" + +#define MAP_ENTRIES 64 +#define KEY_RANGE (MAP_ENTRIES * 2) +#define STRESS_NS (500 * 1000 * 1000ULL) + +struct hammer_arg { + int map_fd; + int cpu; + __u64 deadline_ns; +}; + +struct refill_arg { + int map_fd; + int cpu; + int per_cpu_quota; + int update_errors; +}; + +/* + * Pin the calling thread to @cpu. Uses dynamically-allocated CPU sets so + * we stay correct on hosts with @cpu >= CPU_SETSIZE (default 1024). + */ +static int pin_to_cpu(int cpu) +{ + cpu_set_t *cs; + size_t cs_size; + int err; + + cs = CPU_ALLOC(cpu + 1); + if (!cs) + return -ENOMEM; + cs_size = CPU_ALLOC_SIZE(cpu + 1); + + CPU_ZERO_S(cs_size, cs); + CPU_SET_S(cpu, cs_size, cs); + err = pthread_setaffinity_np(pthread_self(), cs_size, cs); + CPU_FREE(cs); + return err; +} + +static void *hammer_thread(void *p) +{ + struct hammer_arg *a = p; + int nr_possible_cpus = libbpf_num_possible_cpus(); + __u64 val[nr_possible_cpus]; + unsigned int seed; + __u32 key; + + memset(val, 0, sizeof(val)); + pin_to_cpu(a->cpu); + + seed = (unsigned int)a->cpu ^ (unsigned int)(uintptr_t)pthread_self(); + + while (get_time_ns() < a->deadline_ns) { + bool do_update = rand_r(&seed) & 1; + + key = rand_r(&seed) % KEY_RANGE; + if (do_update) + bpf_map_update_elem(a->map_fd, &key, val, BPF_ANY); + else + bpf_map_delete_elem(a->map_fd, &key); + } + return NULL; +} + +static void *refill_thread(void *p) +{ + struct refill_arg *a = p; + int nr_possible_cpus = libbpf_num_possible_cpus(); + __u64 val[nr_possible_cpus]; + __u32 start, end, key; + + memset(val, 0, sizeof(val)); + pin_to_cpu(a->cpu); + + start = (__u32)a->cpu * (__u32)a->per_cpu_quota; + end = start + (__u32)a->per_cpu_quota; + for (key = start; key < end; key++) + if (bpf_map_update_elem(a->map_fd, &key, val, BPF_ANY)) + a->update_errors++; + return NULL; +} + +/* + * Drain the map, then refill it with each CPU inserting only its own + * quota of keys. + * After refill, lookup every key we inserted - a stranded node on any + * CPU's pool would have forced eviction. + */ +static int drain_then_verify_capacity(int map_fd, int nr_cpus) +{ + int per_cpu_quota = MAP_ENTRIES / nr_cpus; + int total = per_cpu_quota * nr_cpus; + int nr_possible_cpus = libbpf_num_possible_cpus(); + pthread_t threads[nr_cpus]; + struct refill_arg args[nr_cpus]; + __u64 val[nr_possible_cpus]; + int i, hits = 0, nthreads = 0; + __u32 key; + + memset(val, 0, sizeof(val)); + + for (key = 0; key < KEY_RANGE; key++) + bpf_map_delete_elem(map_fd, &key); + + for (i = 0; i < nr_cpus; i++) { + args[i] = (struct refill_arg){ + .map_fd = map_fd, + .cpu = i, + .per_cpu_quota = per_cpu_quota, + }; + if (pthread_create(&threads[nthreads], NULL, refill_thread, &args[i]) == 0) + nthreads++; + } + for (i = 0; i < nthreads; i++) + pthread_join(threads[i], NULL); + + for (i = 0; i < nr_cpus; i++) + if (args[i].update_errors) + return -ENOMEM; + + for (key = 0; key < (__u32)total; key++) + if (bpf_map_lookup_elem(map_fd, &key, val) == 0) + hits++; + + return hits == total ? 0 : -EIO; +} + +static void run_variant(enum bpf_map_type type, __u32 map_flags, const char *name) +{ + struct perf_event_attr attr = { + .size = sizeof(attr), + .type = PERF_TYPE_HARDWARE, + .config = PERF_COUNT_HW_CPU_CYCLES, + .freq = 1, + }; + int nr_cpus, max_cpus = 64; + struct bpf_link *links[max_cpus]; + pthread_t threads[max_cpus]; + struct hammer_arg args[max_cpus]; + struct lru_lock_nmi *skel = NULL; + int map_fd, i, err, nr_threads = 0, pmu_fd = -1; + __u64 deadline; + + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "num_cpus")) + return; + + if (nr_cpus > max_cpus) + nr_cpus = max_cpus; + + if (!test__start_subtest(name)) + return; + + memset(links, 0, sizeof(links)); + skel = lru_lock_nmi__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + err = bpf_map__set_type(skel->maps.lru_map, type); + if (!ASSERT_OK(err, "set_type")) + goto cleanup; + err = bpf_map__set_map_flags(skel->maps.lru_map, map_flags); + if (!ASSERT_OK(err, "set_flags")) + goto cleanup; + err = bpf_map__set_max_entries(skel->maps.lru_map, MAP_ENTRIES); + if (!ASSERT_OK(err, "set_max_entries")) + goto cleanup; + + err = lru_lock_nmi__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + skel->bss->hits = 0; + map_fd = bpf_map__fd(skel->maps.lru_map); + attr.sample_freq = read_perf_max_sample_freq(); + + for (i = 0; i < nr_cpus; i++) { + pmu_fd = syscall(__NR_perf_event_open, &attr, -1, i, -1, 0); + if (pmu_fd < 0) { + if (i == 0 && + (errno == ENOENT || errno == EOPNOTSUPP)) { + test__skip(); + goto cleanup; + } + continue; + } + /* libbpf takes ownership of pfd on success */ + links[i] = bpf_program__attach_perf_event(skel->progs.oncpu, pmu_fd); + if (!links[i]) + close(pmu_fd); + } + + deadline = get_time_ns() + STRESS_NS; + for (i = 0; i < nr_cpus; i++) { + args[i].map_fd = map_fd; + args[i].cpu = i; + args[i].deadline_ns = deadline; + if (pthread_create(&threads[nr_threads], NULL, hammer_thread, &args[i]) == 0) + nr_threads++; + } + for (i = 0; i < nr_threads; i++) + pthread_join(threads[i], NULL); + + for (i = 0; i < nr_cpus; i++) { + if (links[i]) { + bpf_link__destroy(links[i]); + links[i] = NULL; + } + } + + ASSERT_GT(skel->bss->hits, 0, "nmi_bpf_ran"); + ASSERT_OK(drain_then_verify_capacity(map_fd, nr_cpus), "drain_then_verify_capacity"); + +cleanup: + for (i = 0; i < nr_cpus; i++) { + if (links[i]) + bpf_link__destroy(links[i]); + } + lru_lock_nmi__destroy(skel); +} + +void serial_test_lru_lock_nmi(void) +{ + run_variant(BPF_MAP_TYPE_LRU_HASH, 0, "common_lru"); + run_variant(BPF_MAP_TYPE_LRU_HASH, BPF_F_NO_COMMON_LRU, "no_common_lru"); + run_variant(BPF_MAP_TYPE_LRU_PERCPU_HASH, 0, "percpu_lru"); +} diff --git a/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c b/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c new file mode 100644 index 000000000000..28bc4b117f41 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Christian Brauner <brauner@kernel.org> */ + +/* + * Test BPF LSM block device integrity hooks with dm-verity. + * + * Creates a dm-verity device over loopback, which triggers + * security_bdev_setintegrity() during verity_preresume(). + * Verifies that the BPF program correctly tracks the integrity + * metadata in its hashmap. + */ + +#define _GNU_SOURCE +#include <test_progs.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/sysmacros.h> +#include <sys/types.h> +#include <unistd.h> +#include "lsm_bdev.skel.h" + +/* Must match the definition in progs/lsm_bdev.c. */ +struct verity_info { + __u8 has_roothash; + __u8 sig_valid; + __u32 setintegrity_cnt; +}; + +#define DATA_SIZE_MB 8 +#define HASH_SIZE_MB 1 +#define DM_NAME "bpf_test_verity" +#define DM_DEV_PATH "/dev/mapper/" DM_NAME + +/* Run a command and optionally capture the first line of stdout. */ +static int run_cmd(const char *cmd, char *out, size_t out_sz) +{ + FILE *fp; + int ret; + + fp = popen(cmd, "r"); + if (!fp) + return -1; + + if (out && out_sz > 0) { + if (!fgets(out, out_sz, fp)) + out[0] = '\0'; + /* strip trailing newline */ + out[strcspn(out, "\n")] = '\0'; + } + + ret = pclose(fp); + return WIFEXITED(ret) ? WEXITSTATUS(ret) : -1; +} + +static bool has_prerequisites(void) +{ + if (getuid() != 0) { + printf("SKIP: must be root\n"); + return false; + } + + if (run_cmd("modprobe loop 2>/dev/null", NULL, 0) && + run_cmd("ls /dev/loop-control 2>/dev/null", NULL, 0)) { + printf("SKIP: no loop device support\n"); + return false; + } + + if (run_cmd("modprobe dm-verity 2>/dev/null", NULL, 0) && + run_cmd("dmsetup targets 2>/dev/null | grep -q verity", NULL, 0)) { + printf("SKIP: dm-verity module not available\n"); + return false; + } + + if (run_cmd("which veritysetup >/dev/null 2>&1", NULL, 0)) { + printf("SKIP: veritysetup not found\n"); + return false; + } + + return true; +} + +void test_lsm_bdev(void) +{ + char data_img[] = "/tmp/bpf_verity_data_XXXXXX"; + char hash_img[] = "/tmp/bpf_verity_hash_XXXXXX"; + char data_loop[64] = {}; + char hash_loop[64] = {}; + char roothash[256] = {}; + char cmd[512]; + int data_fd = -1, hash_fd = -1; + struct lsm_bdev *skel = NULL; + struct verity_info val; + struct stat st; + __u32 dev_key; + int err; + + if (!has_prerequisites()) { + test__skip(); + return; + } + + /* Clean up any stale device from a previous crashed run. */ + snprintf(cmd, sizeof(cmd), "dmsetup remove %s 2>/dev/null", DM_NAME); + run_cmd(cmd, NULL, 0); + + /* Create temporary image files. */ + data_fd = mkstemp(data_img); + if (!ASSERT_OK_FD(data_fd, "mkstemp data")) + return; + + hash_fd = mkstemp(hash_img); + if (!ASSERT_OK_FD(hash_fd, "mkstemp hash")) + goto cleanup; + + if (!ASSERT_OK(ftruncate(data_fd, DATA_SIZE_MB * 1024 * 1024), + "truncate data")) + goto cleanup; + + if (!ASSERT_OK(ftruncate(hash_fd, HASH_SIZE_MB * 1024 * 1024), + "truncate hash")) + goto cleanup; + + close(data_fd); + data_fd = -1; + close(hash_fd); + hash_fd = -1; + + /* Set up loop devices. */ + snprintf(cmd, sizeof(cmd), + "losetup --find --show %s 2>/dev/null", data_img); + if (!ASSERT_OK(run_cmd(cmd, data_loop, sizeof(data_loop)), + "losetup data")) + goto teardown; + + snprintf(cmd, sizeof(cmd), + "losetup --find --show %s 2>/dev/null", hash_img); + if (!ASSERT_OK(run_cmd(cmd, hash_loop, sizeof(hash_loop)), + "losetup hash")) + goto teardown; + + /* Format the dm-verity device and capture the root hash. */ + snprintf(cmd, sizeof(cmd), + "veritysetup format %s %s 2>/dev/null | " + "grep -i 'root hash' | awk '{print $NF}'", + data_loop, hash_loop); + if (!ASSERT_OK(run_cmd(cmd, roothash, sizeof(roothash)), + "veritysetup format")) + goto teardown; + + if (!ASSERT_GT((int)strlen(roothash), 0, "roothash not empty")) + goto teardown; + + /* Load and attach BPF program before activating dm-verity. */ + skel = lsm_bdev__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + goto teardown; + + err = lsm_bdev__attach(skel); + if (!ASSERT_OK(err, "skel attach")) + goto teardown; + + /* Activate dm-verity — triggers verity_preresume() hooks. */ + snprintf(cmd, sizeof(cmd), + "veritysetup open %s %s %s %s 2>/dev/null", + data_loop, DM_NAME, hash_loop, roothash); + if (!ASSERT_OK(run_cmd(cmd, NULL, 0), "veritysetup open")) + goto teardown; + + /* Get the dm device's dev_t. */ + if (!ASSERT_OK(stat(DM_DEV_PATH, &st), "stat dm dev")) + goto remove_dm; + + dev_key = (major(st.st_rdev) << 20) | minor(st.st_rdev); + + /* Look up the device in the BPF map and verify. */ + err = bpf_map__lookup_elem(skel->maps.verity_devices, + &dev_key, sizeof(dev_key), + &val, sizeof(val), 0); + if (!ASSERT_OK(err, "map lookup")) + goto remove_dm; + + ASSERT_EQ(val.has_roothash, 1, "has_roothash"); + ASSERT_EQ(val.sig_valid, 0, "sig_valid (unsigned)"); + /* + * verity_preresume() always calls security_bdev_setintegrity() + * for the roothash. The signature-validity call only happens + * when CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG is enabled. + */ + ASSERT_GE(val.setintegrity_cnt, 1, "setintegrity_cnt min"); + ASSERT_LE(val.setintegrity_cnt, 2, "setintegrity_cnt max"); + + /* Verify that the alloc hook fired at least once. */ + ASSERT_GT(skel->bss->alloc_count, 0, "alloc_count"); + +remove_dm: + snprintf(cmd, sizeof(cmd), "dmsetup remove %s 2>/dev/null", DM_NAME); + run_cmd(cmd, NULL, 0); + +teardown: + if (data_loop[0]) { + snprintf(cmd, sizeof(cmd), "losetup -d %s 2>/dev/null", + data_loop); + run_cmd(cmd, NULL, 0); + } + if (hash_loop[0]) { + snprintf(cmd, sizeof(cmd), "losetup -d %s 2>/dev/null", + hash_loop); + run_cmd(cmd, NULL, 0); + } + +cleanup: + lsm_bdev__destroy(skel); + if (data_fd >= 0) + close(data_fd); + if (hash_fd >= 0) + close(hash_fd); + unlink(data_img); + unlink(hash_img); +} diff --git a/tools/testing/selftests/bpf/prog_tests/lsm_cgroup.c b/tools/testing/selftests/bpf/prog_tests/lsm_cgroup.c index 6df25de8f080..41e867467f6c 100644 --- a/tools/testing/selftests/bpf/prog_tests/lsm_cgroup.c +++ b/tools/testing/selftests/bpf/prog_tests/lsm_cgroup.c @@ -2,6 +2,7 @@ #include <sys/types.h> #include <sys/socket.h> +#include <sys/xattr.h> #include <test_progs.h> #include <bpf/btf.h> @@ -309,11 +310,89 @@ static void test_lsm_cgroup_nonvoid(void) lsm_cgroup_nonvoid__destroy(skel); } +static void test_lsm_cgroup_retval(void) +{ + struct lsm_cgroup *skel = NULL; + int skipcap_prog_fd1, skipcap_prog_fd2, socket_prog_fd1, socket_prog_fd2; + int cgroup_fd = -1; + int err, fd; + char tmpfile[] = "/tmp/test_lsm_cgroup_retval.XXXXXX"; + + fd = mkstemp(tmpfile); + if (!ASSERT_OK_FD(fd, "mkstemp")) + return; + close(fd); + + cgroup_fd = test__join_cgroup("/default_retval"); + if (!ASSERT_OK_FD(cgroup_fd, "join_cgroup")) + goto cleanup_tmpfile; + + skel = lsm_cgroup__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto cleanup_cgroup; + + skipcap_prog_fd1 = bpf_program__fd(skel->progs.skipcap_first); + skipcap_prog_fd2 = bpf_program__fd(skel->progs.skipcap_second); + socket_prog_fd1 = bpf_program__fd(skel->progs.socket_first); + socket_prog_fd2 = bpf_program__fd(skel->progs.socket_second); + + err = bpf_prog_attach(skipcap_prog_fd1, cgroup_fd, BPF_LSM_CGROUP, BPF_F_ALLOW_MULTI); + if (err == -ENOTSUPP) { + test__skip(); + goto cleanup_skeleton; + } + if (!ASSERT_OK(err, "attach first skipcap prog")) + goto cleanup_skeleton; + + err = bpf_prog_attach(skipcap_prog_fd2, cgroup_fd, BPF_LSM_CGROUP, BPF_F_ALLOW_MULTI); + if (!ASSERT_OK(err, "attach second skipcap prog")) + goto cleanup_skipcap1; + + err = bpf_prog_attach(socket_prog_fd1, cgroup_fd, BPF_LSM_CGROUP, BPF_F_ALLOW_MULTI); + if (!ASSERT_OK(err, "attach first sock_create prog")) + goto cleanup_skipcap2; + + err = bpf_prog_attach(socket_prog_fd2, cgroup_fd, BPF_LSM_CGROUP, BPF_F_ALLOW_MULTI); + if (!ASSERT_OK(err, "attach second sock_create prog")) + goto cleanup_sock_create1; + + /* trigger the bool hook by setxattr */ + err = setxattr(tmpfile, "user.test", "value", 5, 0); + if (!ASSERT_OK(err, "setxattr")) + goto cleanup_sock_create2; + + /* trigger the errno hook by creating a socket */ + fd = socket(AF_INET, SOCK_STREAM, 0); + if (!ASSERT_OK_FD(fd, "socket")) + goto cleanup_sock_create2; + close(fd); + + ASSERT_EQ(skel->data->skipcap_retval, 0, "bool_hook_retval_should_be_0"); + ASSERT_EQ(skel->data->socket_retval, -EPERM, "errno_hook_retval_should_be_EPERM"); + +cleanup_sock_create2: + bpf_prog_detach2(socket_prog_fd2, cgroup_fd, BPF_LSM_CGROUP); +cleanup_sock_create1: + bpf_prog_detach2(socket_prog_fd1, cgroup_fd, BPF_LSM_CGROUP); +cleanup_skipcap2: + bpf_prog_detach2(skipcap_prog_fd2, cgroup_fd, BPF_LSM_CGROUP); +cleanup_skipcap1: + bpf_prog_detach2(skipcap_prog_fd1, cgroup_fd, BPF_LSM_CGROUP); +cleanup_skeleton: + lsm_cgroup__destroy(skel); +cleanup_cgroup: + close(cgroup_fd); +cleanup_tmpfile: + unlink(tmpfile); +} + void test_lsm_cgroup(void) { if (test__start_subtest("functional")) test_lsm_cgroup_functional(); if (test__start_subtest("nonvoid")) test_lsm_cgroup_nonvoid(); + if (test__start_subtest("retval")) + test_lsm_cgroup_retval(); btf__free(btf); } diff --git a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c index b6391af5f6f9..39e8a3b8b6af 100644 --- a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c +++ b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c @@ -3,6 +3,7 @@ #include "network_helpers.h" #include "test_progs.h" +#include "test_lwt_ip_encap.skel.h" #define BPF_FILE "test_lwt_ip_encap.bpf.o" @@ -32,6 +33,9 @@ #define IP6_ADDR_8 "fb08::1" #define IP6_ADDR_GRE "fb10::1" +#define IP4_ADDR_VXLAN "172.16.17.100" +#define IP6_ADDR_VXLAN "fb11::1" + #define IP6_ADDR_SRC IP6_ADDR_1 #define IP6_ADDR_DST IP6_ADDR_4 @@ -406,7 +410,8 @@ close_netns: static int check_ping_ok(const char *ns1) { SYS(fail, "ip netns exec %s ping -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP4_ADDR_DST); - SYS(fail, "ip netns exec %s ping6 -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP6_ADDR_DST); + SYS(fail, "ip netns exec %s %s -c 1 -W1 -I veth1 %s > /dev/null", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); return 0; fail: return -1; @@ -420,7 +425,8 @@ static int check_ping_fails(const char *ns1) if (!ret) return -1; - ret = SYS_NOFAIL("ip netns exec %s ping6 -c 1 -W1 -I veth1 %s", ns1, IP6_ADDR_DST); + ret = SYS_NOFAIL("ip netns exec %s %s -c 1 -W1 -I veth1 %s", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); if (!ret) return -1; @@ -538,3 +544,145 @@ void test_lwt_ip_encap_ipv4(void) if (test__start_subtest("ingress")) lwt_ip_encap(IPV4_ENCAP, INGRESS, ""); } + +/* + * VxLAN Setup/topology: + * + * NS1 (IP*_ADDR_1) NS2 NS3 (IP*_ADDR_4) + * [ping src] + * | top route + * veth1 (LWT encap) <<-- veth2 veth3 <<-- veth4 (ping dst) + * | ^ + * (bottom route) | (inner pkt) + * v bottom route | + * veth5 -->> veth6 veth7 -->> veth8 (vxlan decap) + * (IP*_ADDR_VXLAN) + * + * Add the VxLAN endpoint addresses to NS3's veth8, create standard + * VxLAN decap devices bound to those addresses, and install routes so + * NS1/NS2 can reach the endpoints via the bottom route. NS2 here is to + * make sure the LWT-encap VxLAN packets are routed to NS3 correctly. + */ +static int setup_vxlan_routes(const char *ns3, const char *ns1, const char *ns2) +{ + struct nstoken *nstoken; + + nstoken = open_netns(ns3); + if (!ASSERT_OK_PTR(nstoken, "open ns3 for vxlan")) + return -1; + + SYS(fail_close, "ip a add %s/32 dev veth8", IP4_ADDR_VXLAN); + SYS(fail_close, "ip -6 a add %s/128 dev veth8", IP6_ADDR_VXLAN); + /* + * Standard VxLAN devices to decap the encapsulated packets. The inner + * Ethernet frame uses a broadcast dst MAC so the IP stack accepts it + * without ARP or FDB configuration. + */ + SYS(fail_close, "ip link add vxlan4 type vxlan id 1 dstport 4789 local %s dev veth8 nolearning noudpcsum", + IP4_ADDR_VXLAN); + SYS(fail_close, "ip link set vxlan4 up"); + SYS(fail_close, "ip link add vxlan6 type vxlan id 1 dstport 4789 local %s dev veth8 nolearning udp6zerocsumrx", + IP6_ADDR_VXLAN); + SYS(fail_close, "ip link set vxlan6 up"); + close_netns(nstoken); + + SYS(fail, "ip -n %s route add %s/32 dev veth5 via %s", + ns1, IP4_ADDR_VXLAN, IP4_ADDR_6); + SYS(fail, "ip -n %s route add %s/32 dev veth7 via %s", + ns2, IP4_ADDR_VXLAN, IP4_ADDR_8); + SYS(fail, "ip -n %s -6 route add %s/128 dev veth5 via %s", + ns1, IP6_ADDR_VXLAN, IP6_ADDR_6); + SYS(fail, "ip -n %s -6 route add %s/128 dev veth7 via %s", + ns2, IP6_ADDR_VXLAN, IP6_ADDR_8); + return 0; + +fail_close: + close_netns(nstoken); +fail: + return -1; +} + +static void lwt_ip_encap_vxlan(bool ipv4_encap) +{ + char ns1[NETNS_NAME_SIZE] = NETNS_BASE "-1-"; + char ns2[NETNS_NAME_SIZE] = NETNS_BASE "-2-"; + char ns3[NETNS_NAME_SIZE] = NETNS_BASE "-3-"; + const char *sec = ipv4_encap ? "encap_vxlan" : "encap_vxlan6"; + int expected_offset = ipv4_encap ? (int)sizeof(struct iphdr) + : (int)sizeof(struct ipv6hdr); + struct test_lwt_ip_encap *skel = NULL; + int thdr_offset, err; + + if (!ASSERT_OK(create_ns(ns1, NETNS_NAME_SIZE), "create ns1")) + goto out; + if (!ASSERT_OK(create_ns(ns2, NETNS_NAME_SIZE), "create ns2")) + goto out; + if (!ASSERT_OK(create_ns(ns3, NETNS_NAME_SIZE), "create ns3")) + goto out; + + if (!ASSERT_OK(setup_network(ns1, ns2, ns3, ""), "setup network")) + goto out; + + if (!ASSERT_OK(setup_vxlan_routes(ns3, ns1, ns2), "setup vxlan routes")) + goto out; + + skel = test_lwt_ip_encap__open(); + if (!ASSERT_OK_PTR(skel, "test_lwt_ip_encap__open")) + goto out; + + bpf_program__set_autoload(skel->progs.bpf_lwt_encap_gre, false); + bpf_program__set_autoload(skel->progs.bpf_lwt_encap_gre6, false); + bpf_program__set_autoload(skel->progs.bpf_lwt_encap_vxlan, false); + bpf_program__set_autoload(skel->progs.bpf_lwt_encap_vxlan6, false); + bpf_program__set_autoload(skel->progs.fexit_lwt_push_ip_encap, true); + skel->rodata->tgt_ip_version = ipv4_encap ? 4 : 6; + + err = test_lwt_ip_encap__load(skel); + if (!ASSERT_OK(err, "test_lwt_ip_encap__load")) + goto out; + + err = test_lwt_ip_encap__attach(skel); + if (!ASSERT_OK(err, "test_lwt_ip_encap__attach")) + goto out; + + /* Remove the direct NS2->DST route so packets must go via LWT encap. */ + SYS(out, "ip -n %s route del %s/32 dev veth3", ns2, IP4_ADDR_DST); + SYS(out, "ip -n %s -6 route del %s/128 dev veth3", ns2, IP6_ADDR_DST); + + if (ipv4_encap) + SYS(out, "ip -n %s route add %s encap bpf xmit obj %s sec %s dev veth1", + ns1, IP4_ADDR_DST, BPF_FILE, sec); + else + SYS(out, "ip -n %s -6 route add %s encap bpf xmit obj %s sec %s dev veth1", + ns1, IP6_ADDR_DST, BPF_FILE, sec); + + skel->bss->fexit_triggered = false; + + if (ipv4_encap) + SYS(out, "ip netns exec %s ping -c 1 -W1 %s", ns1, IP4_ADDR_DST); + else + SYS(out, "ip netns exec %s %s -c 1 -W1 %s", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); + + if (!ASSERT_TRUE(skel->bss->fexit_triggered, "fexit_triggered")) + goto out; + + thdr_offset = (int)skel->bss->transport_hdr - (int)skel->bss->network_hdr; + ASSERT_EQ(thdr_offset, expected_offset, "transport_hdr offset"); + +out: + test_lwt_ip_encap__destroy(skel); + SYS_NOFAIL("ip netns del %s", ns1); + SYS_NOFAIL("ip netns del %s", ns2); + SYS_NOFAIL("ip netns del %s", ns3); +} + +void test_lwt_ip_encap_vxlan_ipv4(void) +{ + lwt_ip_encap_vxlan(IPV4_ENCAP); +} + +void test_lwt_ip_encap_vxlan_ipv6(void) +{ + lwt_ip_encap_vxlan(IPV6_ENCAP); +} diff --git a/tools/testing/selftests/bpf/prog_tests/lwt_misc.c b/tools/testing/selftests/bpf/prog_tests/lwt_misc.c new file mode 100644 index 000000000000..6940fca38512 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/lwt_misc.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "lwt_misc.skel.h" + +void test_lwt_misc(void) +{ + RUN_TESTS(lwt_misc); +} diff --git a/tools/testing/selftests/bpf/prog_tests/map_excl.c b/tools/testing/selftests/bpf/prog_tests/map_excl.c index 6bdc6d6de0da..3f4422b9ffa6 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_excl.c +++ b/tools/testing/selftests/bpf/prog_tests/map_excl.c @@ -7,6 +7,11 @@ #include <bpf/btf.h> #include "map_excl.skel.h" +#include "bpf_iter_bpf_array_map.skel.h" + +#ifndef SHA256_DIGEST_SIZE +#define SHA256_DIGEST_SIZE 32 +#endif static void test_map_excl_allowed(void) { @@ -45,10 +50,127 @@ out: } +static void test_map_excl_no_map_in_map(void) +{ + __u8 hash[SHA256_DIGEST_SIZE] = {}; + LIBBPF_OPTS(bpf_map_create_opts, excl_opts, + .excl_prog_hash = hash, + .excl_prog_hash_size = sizeof(hash)); + LIBBPF_OPTS(bpf_map_create_opts, outer_opts); + int excl_fd, tmpl_fd = -1, outer_fd = -1, err; + __u32 key = 0; + + excl_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "excl_inner", 4, 4, 1, &excl_opts); + if (!ASSERT_OK_FD(excl_fd, "create exclusive map")) + return; + + outer_opts.inner_map_fd = excl_fd; + err = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "outer_from_excl", + 4, 4, 1, &outer_opts); + if (err >= 0) + close(err); + ASSERT_EQ(err, -ENOTSUPP, "reject exclusive map as map-in-map template"); + + tmpl_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "tmpl", 4, 4, 1, NULL); + if (!ASSERT_OK_FD(tmpl_fd, "create inner template")) + goto out; + + outer_opts.inner_map_fd = tmpl_fd; + outer_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "outer", 4, 4, 1, &outer_opts); + if (!ASSERT_OK_FD(outer_fd, "create map-of-maps")) + goto out; + + err = bpf_map_update_elem(outer_fd, &key, &excl_fd, 0); + ASSERT_EQ(err, -ENOTSUPP, "reject exclusive map as map-in-map element"); +out: + if (outer_fd >= 0) + close(outer_fd); + if (tmpl_fd >= 0) + close(tmpl_fd); + close(excl_fd); +} + +static void test_map_excl_no_map_iter(void) +{ + __u8 hash[SHA256_DIGEST_SIZE] = {}; + LIBBPF_OPTS(bpf_map_create_opts, excl_opts, + .excl_prog_hash = hash, + .excl_prog_hash_size = sizeof(hash)); + DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); + struct bpf_iter_bpf_array_map *skel = NULL; + union bpf_iter_link_info linfo; + struct bpf_link *link; + int excl_fd; + + excl_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "excl_iter", 4, 8, 3, &excl_opts); + if (!ASSERT_OK_FD(excl_fd, "create exclusive map")) + return; + + skel = bpf_iter_bpf_array_map__open_and_load(); + if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load")) + goto out; + + memset(&linfo, 0, sizeof(linfo)); + linfo.map.map_fd = excl_fd; + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + + link = bpf_program__attach_iter(skel->progs.dump_bpf_array_map, &opts); + if (!ASSERT_ERR_PTR(link, "reject exclusive map as iter target")) { + bpf_link__destroy(link); + goto out; + } + ASSERT_EQ(libbpf_get_error(link), -EPERM, "iter attach errno"); +out: + bpf_iter_bpf_array_map__destroy(skel); + close(excl_fd); +} + +static void test_map_excl_create_validation(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, o); + __u8 hash[SHA256_DIGEST_SIZE] = {}; + int fd; + + o.excl_prog_hash = hash; + o.excl_prog_hash_size = SHA256_DIGEST_SIZE / 2; + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "excl", 4, 4, 1, &o); + if (fd >= 0) + close(fd); + ASSERT_EQ(fd, -EINVAL, "reject short excl_prog_hash_size"); + + o.excl_prog_hash = hash; + o.excl_prog_hash_size = SHA256_DIGEST_SIZE * 2; + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "excl", 4, 4, 1, &o); + if (fd >= 0) + close(fd); + ASSERT_EQ(fd, -EINVAL, "reject long excl_prog_hash_size"); + + o.excl_prog_hash = hash; + o.excl_prog_hash_size = 0; + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "excl", 4, 4, 1, &o); + if (fd >= 0) + close(fd); + ASSERT_EQ(fd, -EINVAL, "reject hash pointer with zero size"); + + o.excl_prog_hash = NULL; + o.excl_prog_hash_size = SHA256_DIGEST_SIZE; + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "excl", 4, 4, 1, &o); + if (fd >= 0) + close(fd); + ASSERT_EQ(fd, -EINVAL, "reject size with NULL hash pointer"); +} + void test_map_excl(void) { if (test__start_subtest("map_excl_allowed")) test_map_excl_allowed(); if (test__start_subtest("map_excl_denied")) test_map_excl_denied(); + if (test__start_subtest("map_excl_no_map_in_map")) + test_map_excl_no_map_in_map(); + if (test__start_subtest("map_excl_no_map_iter")) + test_map_excl_no_map_iter(); + if (test__start_subtest("map_excl_create_validation")) + test_map_excl_create_validation(); } diff --git a/tools/testing/selftests/bpf/prog_tests/map_init.c b/tools/testing/selftests/bpf/prog_tests/map_init.c index 14a31109dd0e..c804c3ce9be9 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_init.c +++ b/tools/testing/selftests/bpf/prog_tests/map_init.c @@ -212,3 +212,195 @@ void test_map_init(void) if (test__start_subtest("pcpu_lru_map_init")) test_pcpu_lru_map_init(); } + +static void test_map_create(enum bpf_map_type map_type, const char *map_name, + struct bpf_map_create_opts *opts, const char *exp_msg) +{ + const int key_size = 4, value_size = 4, max_entries = 1; + char log_buf[128]; + int fd; + LIBBPF_OPTS(bpf_log_opts, log_opts); + + log_buf[0] = '\0'; + log_opts.buf = log_buf; + log_opts.size = sizeof(log_buf); + log_opts.level = 1; + opts->log_opts = &log_opts; + fd = bpf_map_create(map_type, map_name, key_size, value_size, max_entries, opts); + if (!ASSERT_LT(fd, 0, "bpf_map_create")) { + close(fd); + return; + } + + ASSERT_STREQ(log_buf, exp_msg, "log_buf"); + ASSERT_EQ(log_opts.true_size, strlen(exp_msg) + 1, "true_size"); +} + +static void test_map_create_array(struct bpf_map_create_opts *opts, const char *exp_msg) +{ + test_map_create(BPF_MAP_TYPE_ARRAY, "test_map_create", opts, exp_msg); +} + +static void test_invalid_vmlinux_value_type_id_struct_ops(void) +{ + const char *msg = "btf_vmlinux_value_type_id can only be used with struct_ops maps.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .btf_vmlinux_value_type_id = 1, + ); + + test_map_create_array(&opts, msg); +} + +static void test_invalid_vmlinux_value_type_id_kv_type_id(void) +{ + const char *msg = "btf_vmlinux_value_type_id is mutually exclusive with btf_key_type_id and btf_value_type_id.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .btf_vmlinux_value_type_id = 1, + .btf_key_type_id = 1, + ); + + test_map_create(BPF_MAP_TYPE_STRUCT_OPS, "test_map_create", &opts, msg); +} + +static void test_invalid_value_type_id(void) +{ + const char *msg = "Invalid btf_value_type_id.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .btf_key_type_id = 1, + ); + + test_map_create_array(&opts, msg); +} + +static void test_invalid_map_extra(void) +{ + const char *msg = "Invalid map_extra.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .map_extra = 1, + ); + + test_map_create_array(&opts, msg); +} + +static void test_invalid_numa_node(void) +{ + const char *msg = "Invalid numa_node.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .map_flags = BPF_F_NUMA_NODE, + .numa_node = 0xFF, + ); + + test_map_create_array(&opts, msg); +} + +static void test_invalid_map_type(void) +{ + const char *msg = "Invalid map_type.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts); + + test_map_create(__MAX_BPF_MAP_TYPE, "test_map_create", &opts, msg); +} + +static void test_invalid_token_fd(void) +{ + const char *msg = "Invalid map_token_fd.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .map_flags = BPF_F_TOKEN_FD, + .token_fd = -1, + ); + + test_map_create_array(&opts, msg); +} + +static void test_invalid_map_name(void) +{ + const char *msg = "Invalid map_name.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts); + + test_map_create(BPF_MAP_TYPE_ARRAY, "test-!@#", &opts, msg); +} + +static void test_invalid_btf_fd(void) +{ + const char *msg = "Invalid btf_fd.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .btf_fd = -1, + .btf_key_type_id = 1, + .btf_value_type_id = 1, + ); + + test_map_create_array(&opts, msg); +} + +static void test_excl_prog_hash_size_1(void) +{ + const char *msg = "Invalid excl_prog_hash_size.\n"; + const char *hash = "DEADCODE"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .excl_prog_hash = hash, + ); + + test_map_create_array(&opts, msg); +} + +static void test_excl_prog_hash_size_2(void) +{ + const char *msg = "Invalid excl_prog_hash_size.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .excl_prog_hash_size = 1, + ); + + test_map_create_array(&opts, msg); +} + +static void test_common_attr_padding(void) +{ + struct bpf_common_attr_fake { + __u8 attrs[offsetofend(struct bpf_common_attr, log_true_size)]; + __u32 pad; + } attr_common = { + .pad = 1, + }; + union bpf_attr attr = { + .map_type = BPF_MAP_TYPE_ARRAY, + .key_size = 4, + .value_size = 4, + .max_entries = 1, + }; + int fd; + + fd = syscall(__NR_bpf, BPF_MAP_CREATE | BPF_COMMON_ATTRS, &attr, sizeof(attr), &attr_common, + sizeof(attr_common)); + if (!ASSERT_LT(fd, 0, "syscall")) + close(fd); + else + ASSERT_EQ(errno, E2BIG, "errno"); +} + +void test_map_create_failure(void) +{ + if (test__start_subtest("invalid_vmlinux_value_type_id_struct_ops")) + test_invalid_vmlinux_value_type_id_struct_ops(); + if (test__start_subtest("invalid_vmlinux_value_type_id_kv_type_id")) + test_invalid_vmlinux_value_type_id_kv_type_id(); + if (test__start_subtest("invalid_value_type_id")) + test_invalid_value_type_id(); + if (test__start_subtest("invalid_map_extra")) + test_invalid_map_extra(); + if (test__start_subtest("invalid_numa_node")) + test_invalid_numa_node(); + if (test__start_subtest("invalid_map_type")) + test_invalid_map_type(); + if (test__start_subtest("invalid_token_fd")) + test_invalid_token_fd(); + if (test__start_subtest("invalid_map_name")) + test_invalid_map_name(); + if (test__start_subtest("invalid_btf_fd")) + test_invalid_btf_fd(); + if (test__start_subtest("invalid_excl_prog_hash_size_1")) + test_excl_prog_hash_size_1(); + if (test__start_subtest("invalid_excl_prog_hash_size_2")) + test_excl_prog_hash_size_2(); + if (test__start_subtest("common_attr_padding")) + test_common_attr_padding(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/map_kptr.c b/tools/testing/selftests/bpf/prog_tests/map_kptr.c index 03b46f17cf53..17e707dddda8 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_kptr.c +++ b/tools/testing/selftests/bpf/prog_tests/map_kptr.c @@ -51,7 +51,6 @@ static void test_map_kptr_success(bool test_run) ret = bpf_map__update_elem(skel->maps.array_map, &key, sizeof(key), buf, sizeof(buf), 0); ASSERT_OK(ret, "array_map update"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); @@ -59,49 +58,42 @@ static void test_map_kptr_success(bool test_run) ret = bpf_map__update_elem(skel->maps.pcpu_array_map, &key, sizeof(key), pbuf, cpu * sizeof(buf), 0); ASSERT_OK(ret, "pcpu_array_map update"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.hash_map, &key, sizeof(key), 0); ASSERT_OK(ret, "hash_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.pcpu_hash_map, &key, sizeof(key), 0); ASSERT_OK(ret, "pcpu_hash_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.hash_malloc_map, &key, sizeof(key), 0); ASSERT_OK(ret, "hash_malloc_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.pcpu_hash_malloc_map, &key, sizeof(key), 0); ASSERT_OK(ret, "pcpu_hash_malloc_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.lru_hash_map, &key, sizeof(key), 0); ASSERT_OK(ret, "lru_hash_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.lru_pcpu_hash_map, &key, sizeof(key), 0); ASSERT_OK(ret, "lru_pcpu_hash_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); @@ -151,12 +143,68 @@ static void wait_for_map_release(void) map_kptr__destroy(skel); } +enum map_update_kptr_case { + MAP_UPDATE_KPTR_ARRAY, + MAP_UPDATE_KPTR_HASH, + MAP_UPDATE_KPTR_HASH_MALLOC, +}; + +static struct bpf_program *map_update_kptr_prog(struct map_kptr *skel, + enum map_update_kptr_case test) +{ + switch (test) { + case MAP_UPDATE_KPTR_ARRAY: + return skel->progs.test_array_map_update_kptr; + case MAP_UPDATE_KPTR_HASH: + return skel->progs.test_hash_map_update_kptr; + case MAP_UPDATE_KPTR_HASH_MALLOC: + return skel->progs.test_hash_malloc_map_update_kptr; + } + + return NULL; +} + +static void test_map_update_kptr(enum map_update_kptr_case test) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct map_kptr *skel; + struct bpf_program *prog; + int ret; + + skel = map_kptr__open_and_load(); + if (!ASSERT_OK_PTR(skel, "map_kptr__open_and_load")) + return; + + prog = map_update_kptr_prog(skel, test); + if (!ASSERT_OK_PTR(prog, "map_update_kptr_prog")) + goto out; + + ret = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + if (!ASSERT_OK(ret, "map_update_kptr")) + goto out; + if (!ASSERT_OK(opts.retval, "map_update_kptr retval")) + goto out; + + ASSERT_EQ(skel->bss->num_of_refs, 3, "refs_after_update"); + +out: + map_kptr__destroy(skel); + wait_for_map_release(); +} + void serial_test_map_kptr(void) { struct rcu_tasks_trace_gp *skel; RUN_TESTS(map_kptr_fail); + if (test__start_subtest("update_array_map_kptr")) + test_map_update_kptr(MAP_UPDATE_KPTR_ARRAY); + if (test__start_subtest("update_hash_map_kptr")) + test_map_update_kptr(MAP_UPDATE_KPTR_HASH); + if (test__start_subtest("update_hash_malloc_map_kptr")) + test_map_update_kptr(MAP_UPDATE_KPTR_HASH_MALLOC); + skel = rcu_tasks_trace_gp__open_and_load(); if (!ASSERT_OK_PTR(skel, "rcu_tasks_trace_gp__open_and_load")) return; @@ -175,7 +223,7 @@ void serial_test_map_kptr(void) ASSERT_OK(kern_sync_rcu(), "sync rcu"); wait_for_map_release(); - /* Observe refcount dropping to 1 on synchronous delete elem */ + /* Observe refcount dropping to 1 on map release. */ test_map_kptr_success(true); } diff --git a/tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c b/tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c index 653b0a20fab9..c62907732c19 100644 --- a/tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c +++ b/tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c @@ -7,24 +7,29 @@ * 3. call listen() for 1 server socket. (migration target) * 4. update a map to migrate all child sockets * to the last server socket (migrate_map[cookie] = 4) - * 5. call shutdown() for first 4 server sockets + * 5. for TCP_ESTABLISHED and TCP_SYN_RECV cases, verify via epoll + * that the last server socket is not ready before migration. + * 6. call shutdown() for first 4 server sockets * and migrate the requests in the accept queue * to the last server socket. - * 6. call listen() for the second server socket. - * 7. call shutdown() for the last server + * 7. for TCP_ESTABLISHED and TCP_SYN_RECV cases, verify via epoll + * that the last server socket is ready after migration. + * 8. call listen() for the second server socket. + * 9. call shutdown() for the last server * and migrate the requests in the accept queue * to the second server socket. - * 8. call listen() for the last server. - * 9. call shutdown() for the second server + * 10. call listen() for the last server. + * 11. call shutdown() for the second server * and migrate the requests in the accept queue * to the last server socket. - * 10. call accept() for the last server socket. + * 12. call accept() for the last server socket. * * Author: Kuniyuki Iwashima <kuniyu@amazon.co.jp> */ #include <bpf/bpf.h> #include <bpf/libbpf.h> +#include <sys/epoll.h> #include "test_progs.h" #include "test_migrate_reuseport.skel.h" @@ -350,21 +355,51 @@ static int update_maps(struct migrate_reuseport_test_case *test_case, static int migrate_dance(struct migrate_reuseport_test_case *test_case) { + struct epoll_event ev = { + .events = EPOLLIN, + }; + int epoll = -1, nfds; int i, err; + if (test_case->state != BPF_TCP_NEW_SYN_RECV) { + epoll = epoll_create1(0); + if (!ASSERT_NEQ(epoll, -1, "epoll_create1")) + return -1; + + ev.data.fd = test_case->servers[MIGRATED_TO]; + if (!ASSERT_OK(epoll_ctl(epoll, EPOLL_CTL_ADD, + test_case->servers[MIGRATED_TO], &ev), + "epoll_ctl")) + goto close_epoll; + + nfds = epoll_wait(epoll, &ev, 1, 0); + if (!ASSERT_EQ(nfds, 0, "epoll_wait 1")) + goto close_epoll; + } + /* Migrate TCP_ESTABLISHED and TCP_SYN_RECV requests * to the last listener based on eBPF. */ for (i = 0; i < MIGRATED_TO; i++) { err = shutdown(test_case->servers[i], SHUT_RDWR); if (!ASSERT_OK(err, "shutdown")) - return -1; + goto close_epoll; } /* No dance for TCP_NEW_SYN_RECV to migrate based on eBPF */ if (test_case->state == BPF_TCP_NEW_SYN_RECV) return 0; + nfds = epoll_wait(epoll, &ev, 1, 0); + if (!ASSERT_EQ(nfds, 1, "epoll_wait 2")) { +close_epoll: + if (epoll >= 0) + close(epoll); + return -1; + } + + close(epoll); + /* Note that we use the second listener instead of the * first one here. * diff --git a/tools/testing/selftests/bpf/prog_tests/modify_return.c b/tools/testing/selftests/bpf/prog_tests/modify_return.c index a70c99c2f8c8..4661d77ebdfc 100644 --- a/tools/testing/selftests/bpf/prog_tests/modify_return.c +++ b/tools/testing/selftests/bpf/prog_tests/modify_return.c @@ -5,6 +5,7 @@ */ #include <test_progs.h> +#include <unistd.h> #include "modify_return.skel.h" #define LOWER(x) ((x) & 0xffff) @@ -23,11 +24,13 @@ static void run_test(__u32 input_retval, __u16 want_side_effect, __s16 want_ret) if (!ASSERT_OK_PTR(skel, "skel_load")) goto cleanup; + skel->bss->input_retval = input_retval; + skel->bss->test_pid = getpid(); + err = modify_return__attach(skel); if (!ASSERT_OK(err, "modify_return__attach failed")) goto cleanup; - skel->bss->input_retval = input_retval; prog_fd = bpf_program__fd(skel->progs.fmod_ret_test); err = bpf_prog_test_run_opts(prog_fd, &topts); ASSERT_OK(err, "test_run"); @@ -49,8 +52,7 @@ cleanup: modify_return__destroy(skel); } -/* TODO: conflict with get_func_ip_test */ -void serial_test_modify_return(void) +void test_modify_return(void) { run_test(0 /* input_retval */, 2 /* want_side_effect */, diff --git a/tools/testing/selftests/bpf/prog_tests/module_attach.c b/tools/testing/selftests/bpf/prog_tests/module_attach.c index 70fa7ae93173..92c336333fcb 100644 --- a/tools/testing/selftests/bpf/prog_tests/module_attach.c +++ b/tools/testing/selftests/bpf/prog_tests/module_attach.c @@ -6,7 +6,22 @@ #include "test_module_attach.skel.h" #include "testing_helpers.h" -static int duration; +static const char * const read_tests[] = { + "handle_raw_tp", + "handle_tp_btf", + "handle_fentry", + "handle_fentry_explicit", + "handle_fmod_ret", +}; + +static const char * const detach_tests[] = { + "handle_fentry", + "handle_fexit", + "kprobe_multi", +}; + +static const int READ_SZ = 456; +static const int WRITE_SZ = 457; static int trigger_module_test_writable(int *val) { @@ -33,53 +48,73 @@ static int trigger_module_test_writable(int *val) return 0; } -void test_module_attach(void) +static void test_module_attach_prog(const char *prog_name, int sz, + const char *attach_target, int ret) { - const int READ_SZ = 456; - const int WRITE_SZ = 457; - struct test_module_attach* skel; - struct test_module_attach__bss *bss; - struct bpf_link *link; + struct test_module_attach *skel; + struct bpf_program *prog; int err; - int writable_val = 0; skel = test_module_attach__open(); - if (CHECK(!skel, "skel_open", "failed to open skeleton\n")) + if (!ASSERT_OK_PTR(skel, "module_attach open")) return; - err = bpf_program__set_attach_target(skel->progs.handle_fentry_manual, - 0, "bpf_testmod_test_read"); - ASSERT_OK(err, "set_attach_target"); + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "module_attach find_program")) + goto cleanup; + bpf_program__set_autoload(prog, true); - err = bpf_program__set_attach_target(skel->progs.handle_fentry_explicit_manual, - 0, "bpf_testmod:bpf_testmod_test_read"); - ASSERT_OK(err, "set_attach_target_explicit"); + if (attach_target) { + err = bpf_program__set_attach_target(prog, 0, attach_target); + if (!ASSERT_OK(err, attach_target)) + goto cleanup; + } err = test_module_attach__load(skel); - if (CHECK(err, "skel_load", "failed to load skeleton\n")) + if (!ASSERT_OK(err, "module_attach load")) + goto cleanup; + + err = test_module_attach__attach(skel); + if (!ASSERT_OK(err, "module_attach attach")) + goto cleanup; + + if (sz) { + /* trigger both read and write though each test uses only one */ + ASSERT_OK(trigger_module_test_read(sz), "trigger_read"); + ASSERT_OK(trigger_module_test_write(sz), "trigger_write"); + + ASSERT_EQ(skel->bss->sz, sz, prog_name); + } + + if (ret) + ASSERT_EQ(skel->bss->retval, ret, "ret"); +cleanup: + test_module_attach__destroy(skel); +} + +static void test_module_attach_writable(void) +{ + struct test_module_attach__bss *bss; + struct test_module_attach *skel; + int writable_val = 0; + int err; + + skel = test_module_attach__open(); + if (!ASSERT_OK_PTR(skel, "module_attach open")) return; + bpf_program__set_autoload(skel->progs.handle_raw_tp_writable_bare, true); + + err = test_module_attach__load(skel); + if (!ASSERT_OK(err, "module_attach load")) + goto cleanup; + bss = skel->bss; err = test_module_attach__attach(skel); - if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err)) + if (!ASSERT_OK(err, "module_attach attach")) goto cleanup; - /* trigger tracepoint */ - ASSERT_OK(trigger_module_test_read(READ_SZ), "trigger_read"); - ASSERT_OK(trigger_module_test_write(WRITE_SZ), "trigger_write"); - - ASSERT_EQ(bss->raw_tp_read_sz, READ_SZ, "raw_tp"); - ASSERT_EQ(bss->raw_tp_bare_write_sz, WRITE_SZ, "raw_tp_bare"); - ASSERT_EQ(bss->tp_btf_read_sz, READ_SZ, "tp_btf"); - ASSERT_EQ(bss->fentry_read_sz, READ_SZ, "fentry"); - ASSERT_EQ(bss->fentry_manual_read_sz, READ_SZ, "fentry_manual"); - ASSERT_EQ(bss->fentry_explicit_read_sz, READ_SZ, "fentry_explicit"); - ASSERT_EQ(bss->fentry_explicit_manual_read_sz, READ_SZ, "fentry_explicit_manual"); - ASSERT_EQ(bss->fexit_read_sz, READ_SZ, "fexit"); - ASSERT_EQ(bss->fexit_ret, -EIO, "fexit_tet"); - ASSERT_EQ(bss->fmod_ret_read_sz, READ_SZ, "fmod_ret"); - bss->raw_tp_writable_bare_early_ret = true; bss->raw_tp_writable_bare_out_val = 0xf1f2f3f4; ASSERT_OK(trigger_module_test_writable(&writable_val), @@ -87,31 +122,73 @@ void test_module_attach(void) ASSERT_EQ(bss->raw_tp_writable_bare_in_val, 1024, "writable_test_in"); ASSERT_EQ(bss->raw_tp_writable_bare_out_val, writable_val, "writable_test_out"); +cleanup: + test_module_attach__destroy(skel); +} - test_module_attach__detach(skel); - - /* attach fentry/fexit and make sure it gets module reference */ - link = bpf_program__attach(skel->progs.handle_fentry); - if (!ASSERT_OK_PTR(link, "attach_fentry")) - goto cleanup; +static void test_module_attach_detach(const char *prog_name) +{ + struct test_module_attach *skel; + struct bpf_program *prog; + struct bpf_link *link; + int err; - ASSERT_ERR(unload_bpf_testmod(false), "unload_bpf_testmod"); - bpf_link__destroy(link); + skel = test_module_attach__open(); + if (!ASSERT_OK_PTR(skel, "module_attach open")) + return; - link = bpf_program__attach(skel->progs.handle_fexit); - if (!ASSERT_OK_PTR(link, "attach_fexit")) + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "module_attach find_program")) goto cleanup; + bpf_program__set_autoload(prog, true); - ASSERT_ERR(unload_bpf_testmod(false), "unload_bpf_testmod"); - bpf_link__destroy(link); + err = test_module_attach__load(skel); + if (!ASSERT_OK(err, "module_attach load")) + goto cleanup; - link = bpf_program__attach(skel->progs.kprobe_multi); - if (!ASSERT_OK_PTR(link, "attach_kprobe_multi")) + /* attach and make sure it gets module reference */ + link = bpf_program__attach(prog); + if (!ASSERT_OK_PTR(link, "module_attach attach")) goto cleanup; - ASSERT_ERR(unload_bpf_testmod(false), "unload_bpf_testmod"); + ASSERT_ERR(try_unload_module("bpf_testmod", 1, false), "try_unload_module"); bpf_link__destroy(link); - cleanup: test_module_attach__destroy(skel); } + +void test_module_attach(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(read_tests); i++) { + if (!test__start_subtest(read_tests[i])) + continue; + test_module_attach_prog(read_tests[i], READ_SZ, NULL, 0); + } + if (test__start_subtest("handle_raw_tp_bare")) + test_module_attach_prog("handle_raw_tp_bare", WRITE_SZ, NULL, 0); + if (test__start_subtest("handle_raw_tp_writable_bare")) + test_module_attach_writable(); + if (test__start_subtest("handle_fentry_manual")) { + test_module_attach_prog("handle_fentry_manual", READ_SZ, + "bpf_testmod_test_read", 0); + } + if (test__start_subtest("handle_fentry_explicit_manual")) { + test_module_attach_prog("handle_fentry_explicit_manual", + READ_SZ, + "bpf_testmod:bpf_testmod_test_read", 0); + } + if (test__start_subtest("handle_fexit")) + test_module_attach_prog("handle_fexit", READ_SZ, NULL, -EIO); + if (test__start_subtest("handle_fexit_ret")) + test_module_attach_prog("handle_fexit_ret", 0, NULL, 0); + for (i = 0; i < ARRAY_SIZE(detach_tests); i++) { + char test_name[50]; + + snprintf(test_name, sizeof(test_name), "%s_detach", detach_tests[i]); + if (!test__start_subtest(test_name)) + continue; + test_module_attach_detach(detach_tests[i]); + } +} diff --git a/tools/testing/selftests/bpf/prog_tests/mptcp.c b/tools/testing/selftests/bpf/prog_tests/mptcp.c index 8fade8bdc451..32dfc1c511af 100644 --- a/tools/testing/selftests/bpf/prog_tests/mptcp.c +++ b/tools/testing/selftests/bpf/prog_tests/mptcp.c @@ -264,7 +264,7 @@ static int verify_mptcpify(int server_fd, int client_fd) return err; } -static int run_mptcpify(int cgroup_fd) +static int run_mptcpify(int cgroup_fd, int type) { int server_fd, client_fd, err = 0; struct mptcpify *mptcpify_skel; @@ -280,7 +280,7 @@ static int run_mptcpify(int cgroup_fd) goto out; /* without MPTCP */ - server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0); + server_fd = start_server(AF_INET, type, NULL, 0, 0); if (!ASSERT_GE(server_fd, 0, "start_server")) { err = -EIO; goto out; @@ -317,7 +317,14 @@ static void test_mptcpify(void) if (!ASSERT_OK_PTR(netns, "netns_new")) goto fail; - ASSERT_OK(run_mptcpify(cgroup_fd), "run_mptcpify"); + ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM), "run_mptcpify"); + /* userspace sets flags such as SOCK_CLOEXEC together with the type; + * the BPF prog must still upgrade the socket to MPTCP. See + * update_socket_protocol() in net/socket.c, which runs before the + * type is masked with SOCK_TYPE_MASK. + */ + ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM | SOCK_CLOEXEC), + "run_mptcpify_cloexec"); fail: netns_free(netns); diff --git a/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c b/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c index a72ae0b29f6e..7b4a1e24363b 100644 --- a/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c +++ b/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE +#include <sched.h> #include <test_progs.h> #include "cgroup_helpers.h" #include "percpu_alloc_array.skel.h" @@ -350,6 +352,103 @@ static void test_lru_percpu_hash_cpu_flag(void) test_percpu_map_cpu_flag(BPF_MAP_TYPE_LRU_PERCPU_HASH); } +/* + * A BPF_F_CPU update that creates an element must zero the value on the other + * cpus, rather than leave them holding whatever the recycled element last + * contained. max_entries is 1 so the second key can only reuse the element + * the first one released. + */ +static void test_percpu_map_cpu_flag_create(enum bpf_map_type map_type, __u32 map_flags) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = map_flags); + const u32 stale = 0xDEADC0DE, fresh = 0xC0FFEE; + int nr_cpus, cpu, map_fd, err, key; + int pinned_cpu, value_cpu; + cpu_set_t old_mask, new_mask; + bool restore_mask = false; + u32 value; + u64 flags; + + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) + return; + + if (nr_cpus < 2) { + test__skip(); + return; + } + + map_fd = bpf_map_create(map_type, "cpu_flag_create", sizeof(key), sizeof(value), 1, &opts); + if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) + return; + + /* NO_PREALLOC recycles per cpu, so keep the delete and the create on one cpu. */ + err = sched_getaffinity(0, sizeof(old_mask), &old_mask); + if (!ASSERT_OK(err, "sched_getaffinity")) + goto out; + + pinned_cpu = sched_getcpu(); + if (!ASSERT_GE(pinned_cpu, 0, "sched_getcpu")) + goto out; + + CPU_ZERO(&new_mask); + CPU_SET(pinned_cpu, &new_mask); + err = sched_setaffinity(0, sizeof(new_mask), &new_mask); + if (!ASSERT_OK(err, "sched_setaffinity")) + goto out; + restore_mask = true; + + value_cpu = pinned_cpu ? 0 : 1; + + key = 1; + value = stale; + err = bpf_map_update_elem(map_fd, &key, &value, BPF_F_ALL_CPUS); + if (!ASSERT_OK(err, "bpf_map_update_elem all_cpus")) + goto out; + + err = bpf_map_delete_elem(map_fd, &key); + if (!ASSERT_OK(err, "bpf_map_delete_elem")) + goto out; + + key = 2; + value = fresh; + flags = (u64)value_cpu << 32 | BPF_F_CPU; + err = bpf_map_update_elem(map_fd, &key, &value, flags); + if (!ASSERT_OK(err, "bpf_map_update_elem specified cpu")) + goto out; + + for (cpu = 0; cpu < nr_cpus; cpu++) { + value = 0; + flags = (u64)cpu << 32 | BPF_F_CPU; + err = bpf_map_lookup_elem_flags(map_fd, &key, &value, flags); + if (!ASSERT_OK(err, "bpf_map_lookup_elem_flags specified cpu")) + goto out; + if (!ASSERT_EQ(value, cpu == value_cpu ? fresh : 0, "value on specified cpu")) + goto out; + } + +out: + if (restore_mask) + sched_setaffinity(0, sizeof(old_mask), &old_mask); + close(map_fd); +} + +static void test_percpu_hash_cpu_flag_create(void) +{ + test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_PERCPU_HASH, 0); +} + +static void test_percpu_hash_cpu_flag_create_malloc(void) +{ + test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_PERCPU_HASH, BPF_F_NO_PREALLOC); +} + +static void test_lru_percpu_hash_cpu_flag_create(void) +{ + /* lru without prealloc is -ENOTSUPP, so there is no malloc variant */ + test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_LRU_PERCPU_HASH, 0); +} + static void test_percpu_cgroup_storage_cpu_flag(void) { struct percpu_alloc_array *skel = NULL; @@ -454,6 +553,12 @@ void test_percpu_alloc(void) test_percpu_hash_cpu_flag(); if (test__start_subtest("cpu_flag_lru_percpu_hash")) test_lru_percpu_hash_cpu_flag(); + if (test__start_subtest("cpu_flag_create_percpu_hash")) + test_percpu_hash_cpu_flag_create(); + if (test__start_subtest("cpu_flag_create_percpu_hash_malloc")) + test_percpu_hash_cpu_flag_create_malloc(); + if (test__start_subtest("cpu_flag_create_lru_percpu_hash")) + test_lru_percpu_hash_cpu_flag_create(); if (test__start_subtest("cpu_flag_percpu_cgroup_storage")) test_percpu_cgroup_storage_cpu_flag(); if (test__start_subtest("cpu_flag_array")) diff --git a/tools/testing/selftests/bpf/prog_tests/percpu_array_inner_map.c b/tools/testing/selftests/bpf/prog_tests/percpu_array_inner_map.c new file mode 100644 index 000000000000..2a8b2381306b --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/percpu_array_inner_map.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> + +/* + * Test that replacing an inner percpu array map with one that has different + * max_entries is rejected. percpu_array_map_gen_lookup() inlines the + * template's index_mask, so allowing a smaller replacement would cause OOB. + */ +void test_percpu_array_inner_map(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + int outer_fd, tmpl_fd, good_fd, bad_fd, err; + int zero = 0; + + /* Create template: percpu array with 8 entries */ + tmpl_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "tmpl", + sizeof(int), sizeof(long), 8, NULL); + if (!ASSERT_OK_FD(tmpl_fd, "create_tmpl")) + return; + + /* Create outer array-of-maps using template */ + opts.inner_map_fd = tmpl_fd; + outer_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "outer", + sizeof(int), sizeof(int), 1, &opts); + if (!ASSERT_OK_FD(outer_fd, "create_outer")) + goto close_tmpl; + + /* Insert template as initial inner map */ + err = bpf_map_update_elem(outer_fd, &zero, &tmpl_fd, 0); + if (!ASSERT_OK(err, "insert_tmpl")) + goto close_outer; + + /* Replacement with same max_entries should succeed */ + good_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "good", + sizeof(int), sizeof(long), 8, NULL); + if (!ASSERT_OK_FD(good_fd, "create_good")) + goto close_outer; + + err = bpf_map_update_elem(outer_fd, &zero, &good_fd, 0); + ASSERT_OK(err, "replace_same_max_entries"); + close(good_fd); + + /* Replacement with fewer max_entries must fail */ + bad_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "bad", + sizeof(int), sizeof(long), 2, NULL); + if (!ASSERT_OK_FD(bad_fd, "create_bad")) + goto close_outer; + + err = bpf_map_update_elem(outer_fd, &zero, &bad_fd, 0); + ASSERT_ERR(err, "replace_smaller_max_entries"); + close(bad_fd); + +close_outer: + close(outer_fd); +close_tmpl: + close(tmpl_fd); +} diff --git a/tools/testing/selftests/bpf/prog_tests/perf_link.c b/tools/testing/selftests/bpf/prog_tests/perf_link.c index d940ff87fa08..9e3a0d217af8 100644 --- a/tools/testing/selftests/bpf/prog_tests/perf_link.c +++ b/tools/testing/selftests/bpf/prog_tests/perf_link.c @@ -1,8 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2021 Facebook */ #define _GNU_SOURCE -#include <pthread.h> -#include <sched.h> +#include <linux/compiler.h> #include <test_progs.h> #include "testing_helpers.h" #include "test_perf_link.skel.h" @@ -12,23 +11,14 @@ static void burn_cpu(void) { - volatile int j = 0; - cpu_set_t cpu_set; - int i, err; - - /* generate some branches on cpu 0 */ - CPU_ZERO(&cpu_set); - CPU_SET(0, &cpu_set); - err = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set), &cpu_set); - ASSERT_OK(err, "set_thread_affinity"); + int i; /* spin the loop for a while (random high number) */ for (i = 0; i < 1000000; ++i) - ++j; + barrier(); } -/* TODO: often fails in concurrent mode */ -void serial_test_perf_link(void) +void test_perf_link(void) { struct test_perf_link *skel = NULL; struct perf_event_attr attr; @@ -45,7 +35,7 @@ void serial_test_perf_link(void) attr.config = PERF_COUNT_SW_CPU_CLOCK; attr.freq = 1; attr.sample_freq = 1000; - pfd = syscall(__NR_perf_event_open, &attr, -1, 0, -1, PERF_FLAG_FD_CLOEXEC); + pfd = syscall(__NR_perf_event_open, &attr, 0, -1, -1, PERF_FLAG_FD_CLOEXEC); if (!ASSERT_GE(pfd, 0, "perf_fd")) goto cleanup; diff --git a/tools/testing/selftests/bpf/prog_tests/probe_user.c b/tools/testing/selftests/bpf/prog_tests/probe_user.c index 8721671321de..7a9d13aa2c87 100644 --- a/tools/testing/selftests/bpf/prog_tests/probe_user.c +++ b/tools/testing/selftests/bpf/prog_tests/probe_user.c @@ -1,8 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> -/* TODO: corrupts other tests uses connect() */ -void serial_test_probe_user(void) +void test_probe_user(void) { static const char *const prog_names[] = { "handle_sys_connect", @@ -20,6 +19,11 @@ void serial_test_probe_user(void) struct bpf_program *kprobe_progs[prog_count]; struct bpf_object *obj; static const int zero = 0; + struct test_pro_bss { + struct sockaddr_in old; + __u32 test_pid; + }; + struct test_pro_bss results = {}; size_t i; obj = bpf_object__open_file(obj_file, &opts); @@ -34,6 +38,23 @@ void serial_test_probe_user(void) goto cleanup; } + { + struct bpf_map *bss_map; + struct test_pro_bss bss_init = {}; + + bss_init.test_pid = getpid(); + bss_map = bpf_object__find_map_by_name(obj, "test_pro.bss"); + if (!ASSERT_OK_PTR(bss_map, "find_bss_map")) + goto cleanup; + if (!ASSERT_EQ(bpf_map__value_size(bss_map), sizeof(bss_init), + "bss_size")) + goto cleanup; + err = bpf_map__set_initial_value(bss_map, &bss_init, + sizeof(bss_init)); + if (!ASSERT_OK(err, "set_bss_init")) + goto cleanup; + } + err = bpf_object__load(obj); if (CHECK(err, "obj_load", "err %d\n", err)) goto cleanup; @@ -62,11 +83,13 @@ void serial_test_probe_user(void) connect(sock_fd, &curr, sizeof(curr)); close(sock_fd); - err = bpf_map_lookup_elem(results_map_fd, &zero, &tmp); + err = bpf_map_lookup_elem(results_map_fd, &zero, &results); if (CHECK(err, "get_kprobe_res", "failed to get kprobe res: %d\n", err)) goto cleanup; + memcpy(&tmp, &results.old, sizeof(tmp)); + in = (struct sockaddr_in *)&tmp; if (CHECK(memcmp(&tmp, &orig, sizeof(orig)), "check_kprobe_res", "wrong kprobe res from probe read: %s:%u\n", diff --git a/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_bad_access.c b/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_bad_access.c new file mode 100644 index 000000000000..b8538fc4fc3f --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_bad_access.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "test_kmods/bpf_testmod.h" +#include "bpf_util.h" + +static void check_attach_reject(const struct bpf_insn *program, size_t prog_len) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts); + char error[4096]; + int bpf_fd, tp_fd; + + opts.log_level = 2; + opts.log_buf = error; + opts.log_size = sizeof(error); + + bpf_fd = bpf_prog_load(BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, NULL, "GPL v2", + program, prog_len, &opts); + if (!ASSERT_GE(bpf_fd, 0, "prog_load")) + return; + + tp_fd = bpf_raw_tracepoint_open("bpf_testmod_test_writable_bare_tp", bpf_fd); + ASSERT_EQ(tp_fd, -EINVAL, "bpf_raw_tracepoint_open"); + if (tp_fd >= 0) + close(tp_fd); + + close(bpf_fd); +} + +void test_raw_tp_writable_reject_bad_access(void) +{ + const struct bpf_insn program[] = { + /* r6 is our tp buffer */ + BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 0), + /* one byte beyond the end of the writable context */ + BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_6, + sizeof(struct bpf_testmod_test_writable_ctx)), + BPF_EXIT_INSN(), + }; + + const struct bpf_insn negative_var_off_program[] = { + BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 0), + /* make var_off negative, but keep the effective access offset non-negative */ + BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, -8), + /* one byte beyond the end of the writable context */ + BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_6, + sizeof(struct bpf_testmod_test_writable_ctx) + 8), + BPF_EXIT_INSN(), + }; + + if (test__start_subtest("past_end")) + check_attach_reject(program, ARRAY_SIZE(program)); + + if (test__start_subtest("negative_var_off_past_end")) + check_attach_reject(negative_var_off_program, + ARRAY_SIZE(negative_var_off_program)); +} diff --git a/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c b/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c deleted file mode 100644 index 216b0dfac0fe..000000000000 --- a/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include <test_progs.h> -#include <linux/nbd.h> -#include "bpf_util.h" - -void test_raw_tp_writable_reject_nbd_invalid(void) -{ - __u32 duration = 0; - char error[4096]; - int bpf_fd = -1, tp_fd = -1; - - const struct bpf_insn program[] = { - /* r6 is our tp buffer */ - BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 0), - /* one byte beyond the end of the nbd_request struct */ - BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_6, - sizeof(struct nbd_request)), - BPF_EXIT_INSN(), - }; - - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .log_level = 2, - .log_buf = error, - .log_size = sizeof(error), - ); - - bpf_fd = bpf_prog_load(BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, NULL, "GPL v2", - program, ARRAY_SIZE(program), - &opts); - if (CHECK(bpf_fd < 0, "bpf_raw_tracepoint_writable load", - "failed: %d errno %d\n", bpf_fd, errno)) - return; - - tp_fd = bpf_raw_tracepoint_open("nbd_send_request", bpf_fd); - if (CHECK(tp_fd >= 0, "bpf_raw_tracepoint_writable open", - "erroneously succeeded\n")) - goto out_bpffd; - - close(tp_fd); -out_bpffd: - close(bpf_fd); -} diff --git a/tools/testing/selftests/bpf/prog_tests/rbtree.c b/tools/testing/selftests/bpf/prog_tests/rbtree.c index d8f3d7a45fe9..a854fb38e418 100644 --- a/tools/testing/selftests/bpf/prog_tests/rbtree.c +++ b/tools/testing/selftests/bpf/prog_tests/rbtree.c @@ -9,6 +9,7 @@ #include "rbtree_btf_fail__wrong_node_type.skel.h" #include "rbtree_btf_fail__add_wrong_type.skel.h" #include "rbtree_search.skel.h" +#include "rbtree_search_kptr.skel.h" static void test_rbtree_add_nodes(void) { @@ -193,3 +194,8 @@ void test_rbtree_search(void) { RUN_TESTS(rbtree_search); } + +void test_rbtree_search_kptr(void) +{ + RUN_TESTS(rbtree_search_kptr); +} diff --git a/tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c b/tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c index 246eb259c08a..6a07b2b418d1 100644 --- a/tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c @@ -34,6 +34,8 @@ static void test_success(void) bpf_program__set_autoload(skel->progs.rcu_read_lock_global_subprog, true); bpf_program__set_autoload(skel->progs.rcu_read_lock_subprog_lock, true); bpf_program__set_autoload(skel->progs.rcu_read_lock_subprog_unlock, true); + bpf_program__set_autoload(skel->progs.non_own_ref_untrusted_ld, true); + bpf_program__set_autoload(skel->progs.rcu_untrusted_union_ld, true); err = rcu_read_lock__load(skel); if (!ASSERT_OK(err, "skel_load")) goto out; diff --git a/tools/testing/selftests/bpf/prog_tests/refcounted_kptr.c b/tools/testing/selftests/bpf/prog_tests/refcounted_kptr.c index d2c0542716a8..1737eba34323 100644 --- a/tools/testing/selftests/bpf/prog_tests/refcounted_kptr.c +++ b/tools/testing/selftests/bpf/prog_tests/refcounted_kptr.c @@ -57,6 +57,7 @@ void test_percpu_hash_refcounted_kptr_refcount_leak(void) .data_size_in = sizeof(pkt_v4), .repeat = 1, ); + LIBBPF_OPTS(bpf_test_run_opts, syscall_opts); cpu_nr = libbpf_num_possible_cpus(); if (!ASSERT_GT(cpu_nr, 0, "libbpf_num_possible_cpus")) @@ -87,8 +88,11 @@ void test_percpu_hash_refcounted_kptr_refcount_leak(void) if (!ASSERT_EQ(opts.retval, 2, "opts.retval")) goto out; - err = bpf_map__update_elem(map, &key, sizeof(key), values, values_sz, 0); - if (!ASSERT_OK(err, "bpf_map__update_elem")) + fd = bpf_program__fd(skel->progs.clear_percpu_hash_kptr); + err = bpf_prog_test_run_opts(fd, &syscall_opts); + if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) + goto out; + if (!ASSERT_EQ(syscall_opts.retval, 1, "syscall_opts.retval")) goto out; fd = bpf_program__fd(skel->progs.check_percpu_hash_refcount); diff --git a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c index cb8dd2f63296..7f170a69d1d8 100644 --- a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c +++ b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c @@ -478,6 +478,52 @@ static struct range range_refine_in_halves(enum num_t x_t, struct range x, } +static __always_inline u64 next_u32_block(u64 x) { return x + (1ULL << 32); } +static __always_inline u64 prev_u32_block(u64 x) { return x - (1ULL << 32); } + +/* Is v within the circular u64 range [base, base + len]? */ +static __always_inline bool u64_range_contains(u64 v, u64 base, u64 len) +{ + return v - base <= len; +} + +/* Is v within the circular u32 range [base, base + len]? */ +static __always_inline bool u32_range_contains(u32 v, u32 base, u32 len) +{ + return v - base <= len; +} + +static bool range64_range32_intersect(enum num_t a_t, + struct range a /* 64 */, + struct range b /* 32 */, + struct range *out /* 64 */) +{ + u64 b_len = (u32)(b.b - b.a); + u64 a_len = a.b - a.a; + u64 lo, hi; + + if (u32_range_contains((u32)a.a, (u32)b.a, b_len)) { + lo = a.a; + } else { + lo = swap_low32(a.a, (u32)b.a); + if (!u64_range_contains(lo, a.a, a_len)) + lo = next_u32_block(lo); + if (!u64_range_contains(lo, a.a, a_len)) + return false; + } + if (u32_range_contains(a.b, (u32)b.a, b_len)) { + hi = a.b; + } else { + hi = swap_low32(a.b, (u32)b.b); + if (!u64_range_contains(hi, a.a, a_len)) + hi = prev_u32_block(hi); + if (!u64_range_contains(hi, a.a, a_len)) + return false; + } + *out = range(a_t, lo, hi); + return true; +} + static struct range range_refine(enum num_t x_t, struct range x, enum num_t y_t, struct range y) { struct range y_cast; @@ -500,23 +546,45 @@ static struct range range_refine(enum num_t x_t, struct range x, enum num_t y_t, (s64)x.a >= S32_MIN && (s64)x.b <= S32_MAX) return range_intersection(x_t, x, y_cast); - /* the case when new range knowledge, *y*, is a 32-bit subregister - * range, while previous range knowledge, *x*, is a full register - * 64-bit range, needs special treatment to take into account upper 32 - * bits of full register range - */ + if (y_t == U32 && x_t == U64) { + u64 xmin_swap, xmax_swap, xmin_lower32, xmax_lower32; + + xmin_lower32 = x.a & 0xffffffff; + xmax_lower32 = x.b & 0xffffffff; + if (xmin_lower32 < y.a || xmin_lower32 > y.b) { + /* The 32 lower bits of the umin64 are outside the u32 + * range. Let's update umin64 to match the u32 range. + * We want to *increase* the umin64 to the *minimum* + * value that matches the u32 range. + */ + xmin_swap = swap_low32(x.a, y.a); + /* We should always only increase the minimum, so if + * the new value is lower than before, we need to + * increase the 32 upper bits by 1. + */ + if (xmin_swap < x.a) + xmin_swap += 0x100000000; + if (xmin_swap == x.b) + return range(x_t, x.b, x.b); + } else if (xmax_lower32 < y.a || xmax_lower32 > y.b) { + /* Same for the umax64, but we want to *decrease* + * umax64 to the *maximum* value that matches the u32 + * range. + */ + xmax_swap = swap_low32(x.b, y.b); + if (xmax_swap > x.b) + xmax_swap -= 0x100000000; + if (xmax_swap == x.a) + return range(x_t, x.a, x.a); + } + } + if (t_is_32(y_t) && !t_is_32(x_t)) { - struct range x_swap; + struct range x1; - /* some combinations of upper 32 bits and sign bit can lead to - * invalid ranges, in such cases it's easier to detect them - * after cast/swap than try to enumerate all the conditions - * under which transformation and knowledge transfer is valid - */ - x_swap = range(x_t, swap_low32(x.a, y_cast.a), swap_low32(x.b, y_cast.b)); - if (!is_valid_range(x_t, x_swap)) - return x; - return range_intersection(x_t, x, x_swap); + if (range64_range32_intersect(x_t, x, y, &x1)) + return x1; + return x; } /* otherwise, plain range cast and intersection works */ @@ -1217,7 +1285,23 @@ static int parse_range_cmp_log(const char *log_buf, struct case_spec spec, spec.compare_subregs ? "w0" : "r0", spec.compare_subregs ? "w" : "r", specs[i].reg_idx); - q = strstr(p, buf); + /* + * In the verifier log look for lines: + * 18: (bf) r0 = r6 ; R0=... R6=... + * Different verifier passes may print + * 18: (bf) r0 = r6 + * as well, but never followed by ';'. + */ + q = p; + while ((q = strstr(q, buf)) != NULL) { + const char *s = q + strlen(buf); + + while (*s == ' ' || *s == '\t') + s++; + if (*s == ';') + break; + q = s; + } if (!q) { *specs[i].state = (struct reg_state){.valid = false}; continue; @@ -1251,6 +1335,26 @@ static bool assert_range_eq(enum num_t t, struct range x, struct range y, return false; } +/* For a pair of signed/unsigned t1/t2 checks if r1/r2 intersect in two intervals. */ +static bool needs_two_arcs(enum num_t t1, struct range r1, + enum num_t t2, struct range r2) +{ + u64 lo = cast_t(t1, r2.a); + u64 hi = cast_t(t1, r2.b); + + /* does r2 wrap in t1's domain: [0, hi] ∪ [lo, MAX]? */ + return lo > hi && r1.a <= hi && r1.b >= lo; +} + +static bool reg_state_needs_two_arcs(struct reg_state *s) +{ + if (!s->valid) + return false; + + return needs_two_arcs(U64, s->r[U64], S64, s->r[S64]) || + needs_two_arcs(U32, s->r[U32], S32, s->r[S32]); +} + /* Validate that register states match, and print details if they don't */ static bool assert_reg_state_eq(struct reg_state *r, struct reg_state *e, const char *ctx) { @@ -1475,6 +1579,11 @@ static int verify_case_op(enum num_t init_t, enum num_t cond_t, !assert_reg_state_eq(&fr2, &fe2, "false_reg2") || !assert_reg_state_eq(&tr1, &te1, "true_reg1") || !assert_reg_state_eq(&tr2, &te2, "true_reg2")) { + if (reg_state_needs_two_arcs(&fe1) || reg_state_needs_two_arcs(&fe2) || + reg_state_needs_two_arcs(&te1) || reg_state_needs_two_arcs(&te2)) { + test__skip(); + return 0; + } failed = true; } @@ -2129,6 +2238,8 @@ static struct subtest_case crafted_cases[] = { {U64, S64, {0x7fffffff00000001ULL, 0xffffffff00000000ULL}, {0, 0}}, {U64, S64, {0, 0xffffffffULL}, {1, 1}}, {U64, S64, {0, 0xffffffffULL}, {0x7fffffff, 0x7fffffff}}, + {U64, S32, {0xfffffffe00000001, 0xffffffff00000000}, {S64_MIN, S64_MIN}}, + {U64, U32, {0xfffffffe00000000, U64_MAX - 1}, {U64_MAX, U64_MAX}}, {U64, U32, {0, 0x100000000}, {0, 0}}, {U64, U32, {0xfffffffe, 0x300000000}, {0x80000000, 0x80000000}}, diff --git a/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c b/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c index f0a8c828f8f1..7541f4966abc 100644 --- a/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c @@ -3,6 +3,8 @@ #include <test_progs.h> #include <network_helpers.h> #include <sys/sysinfo.h> +#include <sys/syscall.h> +#include <linux/perf_event.h> #include "res_spin_lock.skel.h" #include "res_spin_lock_fail.skel.h" @@ -102,11 +104,29 @@ end: void serial_test_res_spin_lock_stress(void) { + struct perf_event_attr attr = { + .size = sizeof(attr), + .type = PERF_TYPE_HARDWARE, + .config = PERF_COUNT_HW_CPU_CYCLES, + }; + int pmu_fd; + if (libbpf_num_possible_cpus() < 3) { test__skip(); return; } + pmu_fd = syscall(__NR_perf_event_open, &attr, 0, -1, -1, 0); + if (pmu_fd < 0) { + if (errno == ENOENT || errno == EOPNOTSUPP) { + test__skip(); + return; + } + ASSERT_OK(-errno, "perf_event_open pmu probe"); + return; + } + close(pmu_fd); + ASSERT_OK(load_module("bpf_test_rqspinlock.ko", false), "load module AA"); sleep(5); unload_module("bpf_test_rqspinlock", false); diff --git a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c index 41dfaaabb73f..3f9949e8227d 100644 --- a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c +++ b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c @@ -10,7 +10,25 @@ #include <linux/btf_ids.h> #include "test_progs.h" -static int duration; +#define BTF_DATA_FILE "resolve_btfids.test.o.BTF" + +#define DECL_TAG_FASTCALL "bpf_fastcall" +#define DECL_TAG_KFUNC "bpf_kfunc" +#define TYPE_ATTR_ARENA "address_space(1)" +#define ARENA_ARG(n) (1U << (n)) + +#ifndef KF_FASTCALL +#define KF_FASTCALL (1 << 12) +#endif +#ifndef KF_ARENA_RET +#define KF_ARENA_RET (1 << 13) +#endif +#ifndef KF_ARENA_ARG1 +#define KF_ARENA_ARG1 (1 << 14) +#endif +#ifndef KF_ARENA_ARG2 +#define KF_ARENA_ARG2 (1 << 15) +#endif struct symbol { const char *name; @@ -28,12 +46,50 @@ struct symbol test_symbols[] = { { "func", BTF_KIND_FUNC, -1 }, }; +struct kfunc_symbol { + const char *name; + s32 id; + u32 flags; + u32 arena_args; + bool arena_ret; +}; + +static struct kfunc_symbol kfunc_symbols[] = { + { "kfunc_a", -1, 0, 0, false }, + { "kfunc_b", -1, KF_FASTCALL, 0, false }, + { "kfunc_c", -1, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2, + ARENA_ARG(0) | ARENA_ARG(1), true }, + { "kfunc_d", -1, KF_ARENA_ARG2, ARENA_ARG(1), false }, + { "kfunc_e", -1, 0, ARENA_ARG(0) | ARENA_ARG(1) | ARENA_ARG(2) | + ARENA_ARG(3) | ARENA_ARG(4), false }, + { "kfunc_f", -1, 0, ARENA_ARG(1), false }, + { "kfunc_g", -1, KF_ARENA_RET, ARENA_ARG(0) | ARENA_ARG(1), true }, +}; + /* Align the .BTF_ids section to 4 bytes */ asm ( ".pushsection " BTF_IDS_SECTION " ,\"a\"; \n" ".balign 4, 0; \n" ".popsection; \n"); +/* + * test_list_local, test_set and test_kfunc_set are .local symbols placed + * in .BTF_ids by inline asm, and are read here directly by C name. To the + * compiler they are plain, default-visibility extern objects. + * + * When test_progs is linked as a position-independent executable (PIE), + * taking the address of such an extern is routed through the GOT. The + * GNU assembler on aarch64 unconditionally converts references to .local + * symbols into section + addend form (".BTF_ids + <offset>"), but a GOT + * slot cannot carry an addend (the AArch64 ELF spec mandates zero), so + * the linker resolves it to the .BTF_ids base. + * + * Mark them hidden so the compiler treats them as non-interposable and + * emits a direct, addend-preserving PC-relative access instead of a GOT + * load, in both PIE and non-PIE builds. test_list_global is .globl and + * not affected, so it is left at default visibility. + */ +#pragma GCC visibility push(hidden) BTF_ID_LIST(test_list_local) BTF_ID_UNUSED BTF_ID(typedef, S) @@ -43,24 +99,49 @@ BTF_ID(struct, S) BTF_ID(union, U) BTF_ID(func, func) -extern __u32 test_list_global[]; -BTF_ID_LIST_GLOBAL(test_list_global, 1) -BTF_ID_UNUSED +BTF_SET_START(test_set) BTF_ID(typedef, S) BTF_ID(typedef, T) BTF_ID(typedef, U) BTF_ID(struct, S) BTF_ID(union, U) BTF_ID(func, func) +BTF_SET_END(test_set) -BTF_SET_START(test_set) +BTF_KFUNCS_START(test_kfunc_set) +BTF_ID_FLAGS(func, kfunc_a) +BTF_ID_FLAGS(func, kfunc_b, KF_FASTCALL) +BTF_ID_FLAGS(func, kfunc_c, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_d, KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_e) +BTF_ID_FLAGS(func, kfunc_f) +BTF_ID_FLAGS(func, kfunc_g, KF_ARENA_RET) +BTF_KFUNCS_END(test_kfunc_set) + +/* + * Same kfuncs in reverse declaration order, so resolve_btfids has to + * actually sort at least one of the two sets. + */ +BTF_KFUNCS_START(test_kfunc_set_rev) +BTF_ID_FLAGS(func, kfunc_g, KF_ARENA_RET) +BTF_ID_FLAGS(func, kfunc_f) +BTF_ID_FLAGS(func, kfunc_e) +BTF_ID_FLAGS(func, kfunc_d, KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_c, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_b, KF_FASTCALL) +BTF_ID_FLAGS(func, kfunc_a) +BTF_KFUNCS_END(test_kfunc_set_rev) +#pragma GCC visibility pop + +extern __u32 test_list_global[]; +BTF_ID_LIST_GLOBAL(test_list_global, 1) +BTF_ID_UNUSED BTF_ID(typedef, S) BTF_ID(typedef, T) BTF_ID(typedef, U) BTF_ID(struct, S) BTF_ID(union, U) BTF_ID(func, func) -BTF_SET_END(test_set) static int __resolve_symbol(struct btf *btf, int type_id) @@ -70,10 +151,10 @@ __resolve_symbol(struct btf *btf, int type_id) unsigned int i; type = btf__type_by_id(btf, type_id); - if (!type) { - PRINT_FAIL("Failed to get type for ID %d\n", type_id); + if (!ASSERT_OK_PTR(type, "btf__type_by_id")) return -1; - } + + str = btf__name_by_offset(btf, type->name_off); for (i = 0; i < ARRAY_SIZE(test_symbols); i++) { if (test_symbols[i].id >= 0) @@ -82,64 +163,118 @@ __resolve_symbol(struct btf *btf, int type_id) if (BTF_INFO_KIND(type->info) != test_symbols[i].type) continue; - str = btf__name_by_offset(btf, type->name_off); - if (!str) { - PRINT_FAIL("Failed to get name for BTF ID %d\n", type_id); - return -1; - } - if (!strcmp(str, test_symbols[i].name)) test_symbols[i].id = type_id; } + if (!btf_is_func(type)) + return 0; + + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + if (kfunc_symbols[i].id >= 0) + continue; + if (!strcmp(str, kfunc_symbols[i].name)) + kfunc_symbols[i].id = type_id; + } + return 0; } -static int resolve_symbols(void) +static int resolve_symbols(struct btf *btf) { - struct btf *btf; + __u32 nr = btf__type_cnt(btf); int type_id; - __u32 nr; - btf = btf__parse_raw("resolve_btfids.test.o.BTF"); - if (CHECK(libbpf_get_error(btf), "resolve", - "Failed to load BTF from resolve_btfids.test.o.BTF\n")) - return -1; + for (type_id = 1; type_id < nr; type_id++) { + if (__resolve_symbol(btf, type_id)) + return -1; + } + return 0; +} + +static bool btf_has_decl_tag(struct btf *btf, const char *tag_name, s32 target_id) +{ + const struct btf_type *t; + const char *name; + int nr, id; nr = btf__type_cnt(btf); + for (id = 1; id < nr; id++) { + t = btf__type_by_id(btf, id); + if (!btf_is_decl_tag(t)) + continue; + if (t->type != (__u32)target_id) + continue; + if (btf_decl_tag(t)->component_idx != -1) + continue; + name = btf__name_by_offset(btf, t->name_off); + if (strcmp(name, tag_name) == 0) + return true; + } + return false; +} - for (type_id = 1; type_id < nr; type_id++) { - if (__resolve_symbol(btf, type_id)) - break; +static void check_kfunc_set(struct btf_id_set8 *set) +{ + unsigned int i, j; + + ASSERT_EQ(set->flags, BTF_SET8_KFUNCS, "kfunc_set_flags"); + ASSERT_EQ(set->cnt, ARRAY_SIZE(kfunc_symbols), "kfunc_set_cnt"); + + for (i = 0; i < set->cnt; i++) { + for (j = 0; j < ARRAY_SIZE(kfunc_symbols); j++) { + if (kfunc_symbols[j].id == (s32)set->pairs[i].id) { + ASSERT_EQ(set->pairs[i].flags, + kfunc_symbols[j].flags, "kfunc_flags_check"); + break; + } + } + + ASSERT_TRUE(j < ARRAY_SIZE(kfunc_symbols), "kfunc_id_found"); + + if (i > 0) { + ASSERT_LE(set->pairs[i - 1].id, + set->pairs[i].id, "kfunc_sort_check"); + } } +} - btf__free(btf); - return 0; +/* True if @id is PTR -> TYPE_TAG(kflag=1, "address_space(1)") -> pointee */ +static bool is_arena_tagged_ptr(struct btf *btf, __u32 id) +{ + const struct btf_type *ptr, *tag; + const char *name; + + ptr = btf__type_by_id(btf, id); + if (!btf_is_ptr(ptr)) + return false; + tag = btf__type_by_id(btf, ptr->type); + if (!btf_is_type_tag(tag) || !btf_kflag(tag)) + return false; + name = btf__name_by_offset(btf, tag->name_off); + return strcmp(name, TYPE_ATTR_ARENA) == 0; } void test_resolve_btfids(void) { __u32 *test_list, *test_lists[] = { test_list_local, test_list_global }; unsigned int i, j; - int ret = 0; + struct btf *btf; - if (resolve_symbols()) + btf = btf__parse_raw(BTF_DATA_FILE); + if (!ASSERT_OK_PTR(btf, "btf_parse")) return; + if (resolve_symbols(btf)) + goto out; + /* Check BTF_ID_LIST(test_list_local) and * BTF_ID_LIST_GLOBAL(test_list_global) IDs */ for (j = 0; j < ARRAY_SIZE(test_lists); j++) { test_list = test_lists[j]; - for (i = 0; i < ARRAY_SIZE(test_symbols); i++) { - ret = CHECK(test_list[i] != test_symbols[i].id, - "id_check", - "wrong ID for %s (%d != %d)\n", - test_symbols[i].name, - test_list[i], test_symbols[i].id); - if (ret) - return; - } + for (i = 0; i < ARRAY_SIZE(test_symbols); i++) + ASSERT_EQ(test_list[i], test_symbols[i].id, test_symbols[i].name); } /* Check BTF_SET_START(test_set) IDs */ @@ -153,15 +288,60 @@ void test_resolve_btfids(void) break; } - ret = CHECK(!found, "id_check", - "ID %d not found in test_symbols\n", - test_set.ids[i]); - if (ret) + if (!ASSERT_TRUE(found, "id_in_test_symbols")) break; - if (i > 0) { - if (!ASSERT_LE(test_set.ids[i - 1], test_set.ids[i], "sort_check")) - return; + if (i > 0) + ASSERT_LE(test_set.ids[i - 1], test_set.ids[i], "sort_check"); + } + + check_kfunc_set(&test_kfunc_set); + check_kfunc_set(&test_kfunc_set_rev); + + /* Check resolve_btfids emitted a bpf_kfunc decl_tag for each kfunc */ + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + ASSERT_TRUE(btf_has_decl_tag(btf, DECL_TAG_KFUNC, + kfunc_symbols[i].id), + kfunc_symbols[i].name); + } + + /* Check resolve_btfids emitted bpf_fastcall for KF_FASTCALL kfuncs */ + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + if (kfunc_symbols[i].flags & KF_FASTCALL) { + ASSERT_TRUE(btf_has_decl_tag(btf, DECL_TAG_FASTCALL, + kfunc_symbols[i].id), + kfunc_symbols[i].name); } } + + /* + * Check resolve_btfids wrapped exactly the arena-flagged or suffixed + * return/args with the address_space(1) type attribute, and left other + * pointers/returns untouched. + */ + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + const struct btf_type *fn, *proto; + const struct btf_param *params; + const char *name = kfunc_symbols[i].name; + u32 arena_args = kfunc_symbols[i].arena_args; + __u32 nr; + + fn = btf__type_by_id(btf, kfunc_symbols[i].id); + if (!ASSERT_TRUE(btf_is_func(fn), name)) + continue; + proto = btf__type_by_id(btf, fn->type); + if (!ASSERT_TRUE(btf_is_func_proto(proto), name)) + continue; + params = btf_params(proto); + nr = btf_vlen(proto); + + ASSERT_EQ(is_arena_tagged_ptr(btf, proto->type), + kfunc_symbols[i].arena_ret, name); + for (j = 0; j < nr; j++) + ASSERT_EQ(is_arena_tagged_ptr(btf, params[j].type), + !!(arena_args & ARENA_ARG(j)), name); + } + +out: + btf__free(btf); } diff --git a/tools/testing/selftests/bpf/prog_tests/rhash.c b/tools/testing/selftests/bpf/prog_tests/rhash.c new file mode 100644 index 000000000000..0641bd5b0a9e --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/rhash.c @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> +#include <string.h> +#include <stdio.h> +#include "rhash.skel.h" +#include "bpf_iter_bpf_rhash_map.skel.h" +#include <linux/bpf.h> +#include <linux/perf_event.h> +#include <sys/syscall.h> + +static void rhash_run(const char *prog_name) +{ + struct rhash *skel; + struct bpf_program *prog; + LIBBPF_OPTS(bpf_test_run_opts, opts); + int err; + + skel = rhash__open(); + if (!ASSERT_OK_PTR(skel, "rhash__open")) + return; + + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name")) + goto cleanup; + bpf_program__set_autoload(prog, true); + + err = rhash__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + if (!ASSERT_OK(err, "prog run")) + goto cleanup; + + if (!ASSERT_OK(opts.retval, "prog retval")) + goto cleanup; + + if (!ASSERT_OK(skel->bss->err, "bss->err")) + goto cleanup; + +cleanup: + rhash__destroy(skel); +} + +static int rhash_map_create(__u32 max_entries, __u64 map_extra) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, + .map_flags = BPF_F_NO_PREALLOC, + .map_extra = map_extra); + + return bpf_map_create(BPF_MAP_TYPE_RHASH, "rhash_extra", + sizeof(__u32), sizeof(__u64), max_entries, &opts); +} + +static void rhash_map_extra_presize(void) +{ + const __u32 max_entries = 1024; + const __u32 nelem_hint = 256; + struct bpf_map_info info = {}; + __u32 info_len = sizeof(info); + __u64 val = 0; + __u32 key; + int fd, i; + + fd = rhash_map_create(max_entries, nelem_hint); + if (!ASSERT_GE(fd, 0, "rhash_map_create presize")) + return; + + if (!ASSERT_OK(bpf_map_get_info_by_fd(fd, &info, &info_len), "info")) + goto close; + ASSERT_EQ(info.map_extra, nelem_hint, "info.map_extra"); + + for (i = 0; i < (int)nelem_hint; i++) { + key = i; + if (!ASSERT_OK(bpf_map_update_elem(fd, &key, &val, BPF_NOEXIST), + "update")) + goto close; + } +close: + close(fd); +} + +static void rhash_map_extra_too_big(void) +{ + int fd; + + fd = rhash_map_create(1U << 20, 0x10000); + if (!ASSERT_LT(fd, 0, "rhash_map_create hint > U16_MAX")) + close(fd); +} + +static void rhash_iter_test(void) +{ + DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); + struct bpf_iter_bpf_rhash_map *skel; + int err, i, len, map_fd, iter_fd; + union bpf_iter_link_info linfo; + u32 expected_key_sum = 0, key; + struct bpf_link *link; + u64 val = 0; + char buf[64]; + + skel = bpf_iter_bpf_rhash_map__open(); + if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_rhash_map__open")) + return; + + err = bpf_iter_bpf_rhash_map__load(skel); + if (!ASSERT_OK(err, "bpf_iter_bpf_rhash_map__load")) + goto out; + + map_fd = bpf_map__fd(skel->maps.rhashmap); + + /* Populate map with test data */ + for (i = 0; i < 64; i++) { + key = i + 1; + expected_key_sum += key; + + err = bpf_map_update_elem(map_fd, &key, &val, BPF_NOEXIST); + if (!ASSERT_OK(err, "map_update")) + goto out; + } + + memset(&linfo, 0, sizeof(linfo)); + linfo.map.map_fd = map_fd; + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + + link = bpf_program__attach_iter(skel->progs.dump_bpf_rhash_map, &opts); + if (!ASSERT_OK_PTR(link, "attach_iter")) + goto out; + + iter_fd = bpf_iter_create(bpf_link__fd(link)); + if (!ASSERT_GE(iter_fd, 0, "create_iter")) + goto free_link; + + do { + len = read(iter_fd, buf, sizeof(buf)); + } while (len > 0); + + ASSERT_EQ(skel->bss->key_sum, expected_key_sum, "key_sum"); + ASSERT_EQ(skel->bss->elem_count, 64, "elem_count"); + + close(iter_fd); + +free_link: + bpf_link__destroy(link); +out: + bpf_iter_bpf_rhash_map__destroy(skel); +} + +void test_rhash(void) +{ + if (test__start_subtest("test_rhash_lookup_update")) + rhash_run("test_rhash_lookup_update"); + + if (test__start_subtest("test_rhash_update_delete")) + rhash_run("test_rhash_update_delete"); + + if (test__start_subtest("test_rhash_update_elements")) + rhash_run("test_rhash_update_elements"); + + if (test__start_subtest("test_rhash_update_exist")) + rhash_run("test_rhash_update_exist"); + + if (test__start_subtest("test_rhash_update_any")) + rhash_run("test_rhash_update_any"); + + if (test__start_subtest("test_rhash_noexist_duplicate")) + rhash_run("test_rhash_noexist_duplicate"); + + if (test__start_subtest("test_rhash_delete_nonexistent")) + rhash_run("test_rhash_delete_nonexistent"); + + if (test__start_subtest("test_rhash_kptr_update")) + rhash_run("test_rhash_kptr_update"); + + if (test__start_subtest("test_rhash_kptr_delete")) + rhash_run("test_rhash_kptr_delete"); + + if (test__start_subtest("test_rhash_map_extra_presize")) + rhash_map_extra_presize(); + + if (test__start_subtest("test_rhash_map_extra_too_big")) + rhash_map_extra_too_big(); + + if (test__start_subtest("test_rhash_iter")) + rhash_iter_test(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/rhash_timer.c b/tools/testing/selftests/bpf/prog_tests/rhash_timer.c new file mode 100644 index 000000000000..3aad9fc02e06 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/rhash_timer.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE +#include <sched.h> + +#include <test_progs.h> +#include "rhash_timer.skel.h" + +#define MAX_ATTEMPTS 256 +#define RCU_SYNC_INTERVAL 64 + +static int pin_to_first_cpu(cpu_set_t *old_mask) +{ + cpu_set_t new_mask; + int cpu; + + if (sched_getaffinity(0, sizeof(*old_mask), old_mask)) + return -errno; + + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) + if (CPU_ISSET(cpu, old_mask)) + break; + if (cpu == CPU_SETSIZE) + return -EINVAL; + + CPU_ZERO(&new_mask); + CPU_SET(cpu, &new_mask); + if (sched_setaffinity(0, sizeof(new_mask), &new_mask)) + return -errno; + return 0; +} + +static int update_timer_map(int map_fd, __u64 key) +{ + __u64 value[3] = {}; + + return bpf_map_update_elem(map_fd, &key, value, BPF_NOEXIST); +} + +static int run_prog(int prog_fd, struct bpf_test_run_opts *opts) +{ + int err; + + err = bpf_prog_test_run_opts(prog_fd, opts); + if (err) + return err; + return opts->retval; +} + +void test_rhash_timer(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct rhash_timer *skel = NULL; + cpu_set_t old_mask; + int map_fd = -1, arm_fd, cancel_fd; + bool affinity_set = false; + __u64 key = 1; + int attempt, err; + + err = pin_to_first_cpu(&old_mask); + if (!ASSERT_OK(err, "pin_to_first_cpu")) + return; + affinity_set = true; + + skel = rhash_timer__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto out; + + map_fd = bpf_map__fd(skel->maps.timer_map); + if (!ASSERT_GE(map_fd, 0, "timer_map fd")) + goto out; + arm_fd = bpf_program__fd(skel->progs.arm_deleted_timer); + if (!ASSERT_GE(arm_fd, 0, "arm_deleted_timer fd")) + goto out; + cancel_fd = bpf_program__fd(skel->progs.cancel_recycled_timer); + if (!ASSERT_GE(cancel_fd, 0, "cancel_recycled_timer fd")) + goto out; + + err = update_timer_map(map_fd, key); + if (!ASSERT_OK(err, "seed_timer_map")) + goto out; + + for (attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { + err = run_prog(arm_fd, &opts); + if (err) { + ASSERT_OK(err, "arm_deleted_timer"); + goto out; + } + if (skel->bss->armed != attempt + 1) { + ASSERT_EQ(skel->bss->armed, attempt + 1, "armed"); + goto out; + } + if (skel->bss->timer_init_err) { + ASSERT_OK(skel->bss->timer_init_err, "timer_init_err"); + goto out; + } + if (skel->bss->timer_set_callback_err) { + ASSERT_OK(skel->bss->timer_set_callback_err, + "timer_set_callback_err"); + goto out; + } + if (skel->bss->timer_start_err) { + ASSERT_OK(skel->bss->timer_start_err, "timer_start_err"); + goto out; + } + + if ((attempt + 1) % RCU_SYNC_INTERVAL == 0) { + err = kern_sync_rcu(); + if (err) { + ASSERT_OK(err, "kern_sync_rcu"); + goto out; + } + } + + err = update_timer_map(map_fd, ++key); + if (err) { + ASSERT_OK(err, "replace_timer_map"); + goto out; + } + + err = run_prog(cancel_fd, &opts); + if (err) { + ASSERT_OK(err, "cancel_recycled_timer"); + goto out; + } + if (skel->bss->timer_cancel_err) { + ASSERT_OK(skel->bss->timer_cancel_err, "timer_cancel_err"); + goto out; + } + if (skel->bss->cancelled) + break; + } + + ASSERT_GT(skel->bss->cancelled, 0, "preserved timer"); +out: + if (map_fd >= 0) + bpf_map_delete_elem(map_fd, &key); + rhash_timer__destroy(skel); + if (affinity_set) + sched_setaffinity(0, sizeof(old_mask), &old_mask); +} diff --git a/tools/testing/selftests/bpf/prog_tests/setget_sockopt.c b/tools/testing/selftests/bpf/prog_tests/setget_sockopt.c index 77fe1bfb7504..4e91d9b615ce 100644 --- a/tools/testing/selftests/bpf/prog_tests/setget_sockopt.c +++ b/tools/testing/selftests/bpf/prog_tests/setget_sockopt.c @@ -199,6 +199,83 @@ err_out: bpf_link__destroy(getsockopt_link); } +static int connect_to_v4mapped_v6_fd(int server_fd) +{ + struct sockaddr_storage addr; + struct sockaddr_in *addr4 = (void *)&addr; + socklen_t addrlen = sizeof(addr); + struct sockaddr_in6 addr6 = {}; + int fd = -1, v6only = 0, err; + + err = getsockname(server_fd, (struct sockaddr *)&addr, &addrlen); + if (!ASSERT_OK(err, "getsockname")) + return -1; + + fd = socket(AF_INET6, SOCK_STREAM, 0); + if (!ASSERT_GE(fd, 0, "socket")) + return -1; + + err = settimeo(fd, 0); + if (!ASSERT_OK(err, "settimeo")) + goto err_out; + + err = setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &v6only, sizeof(v6only)); + if (!ASSERT_OK(err, "clear_v6only")) + goto err_out; + + addr6.sin6_family = AF_INET6; + addr6.sin6_port = addr4->sin_port; + addr6.sin6_addr.s6_addr[10] = 0xff; + addr6.sin6_addr.s6_addr[11] = 0xff; + memcpy(&addr6.sin6_addr.s6_addr[12], &addr4->sin_addr, sizeof(addr4->sin_addr)); + + err = connect(fd, (struct sockaddr *)&addr6, sizeof(addr6)); + if (!ASSERT_OK(err, "connect")) + goto err_out; + + return fd; + +err_out: + close(fd); + return -1; +} + +static void test_v4mapped_v6_ip_tos(void) +{ + struct setget_sockopt__bss *bss = skel->bss; + int sfd = -1, fd = -1, got = 0, exp = 0x1c; + socklen_t optlen; + + memset(bss, 0, sizeof(*bss)); + bss->v4mapped_v6_ip_tos_enable = 1; + bss->v4mapped_v6_ip_tos_ret = -1; + bss->v4mapped_v6_ip_tos_val = exp; + + sfd = start_server(AF_INET, SOCK_STREAM, addr4_str, 0, 0); + if (!ASSERT_GE(sfd, 0, "start_server")) + goto err_out; + + fd = connect_to_v4mapped_v6_fd(sfd); + if (!ASSERT_GE(fd, 0, "connect_to_v4mapped_v6_fd")) + goto err_out; + + ASSERT_GT(bss->v4mapped_v6_ip_tos_cnt, 0, "v4mapped_v6_ip_tos_cnt"); + ASSERT_EQ(bss->v4mapped_v6_ip_tos_ret, 0, "v4mapped_v6_ip_tos_ret"); + + optlen = sizeof(got); + if (!ASSERT_OK(getsockopt(fd, SOL_IP, IP_TOS, &got, &optlen), "getsockopt_ip_tos")) + goto err_out; + + ASSERT_EQ(got, exp, "ip_tos"); + +err_out: + bss->v4mapped_v6_ip_tos_enable = 0; + if (fd >= 0) + close(fd); + if (sfd >= 0) + close(sfd); +} + void test_setget_sockopt(void) { cg_fd = test__join_cgroup(CG_NAME); @@ -238,6 +315,7 @@ void test_setget_sockopt(void) test_ktls(AF_INET); test_nonstandard_opt(AF_INET); test_nonstandard_opt(AF_INET6); + test_v4mapped_v6_ip_tos(); done: setget_sockopt__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/sha256.c b/tools/testing/selftests/bpf/prog_tests/sha256.c index 604a0b1423d5..5edbc6194b07 100644 --- a/tools/testing/selftests/bpf/prog_tests/sha256.c +++ b/tools/testing/selftests/bpf/prog_tests/sha256.c @@ -25,10 +25,10 @@ void test_sha256(void) size_t i; data = malloc(MAX_LEN); - if (!ASSERT_OK_PTR(data, "malloc")) + if (!ASSERT_NEQ(data, NULL, "malloc")) goto out; digests = malloc((MAX_LEN + 1) * SHA256_DIGEST_LENGTH); - if (!ASSERT_OK_PTR(digests, "malloc")) + if (!ASSERT_NEQ(digests, NULL, "malloc")) goto out; /* Generate MAX_LEN bytes of "random" data deterministically. */ diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c new file mode 100644 index 000000000000..77381d345435 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -0,0 +1,1840 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include <test_progs.h> +#include <sys/syscall.h> +#include <sys/mman.h> +#include <sys/wait.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <limits.h> +#include <linux/keyctl.h> +#include <linux/bpf.h> + +#include <bpf/btf.h> + +#include "bpf/libbpf_internal.h" /* for libbpf_sha256() */ +#include "bpf/skel_internal.h" /* for loader ctx layout (bpf_loader_ctx etc) */ + +#include "test_signed_loader.skel.h" +#include "test_signed_loader_map.skel.h" +#include "test_signed_loader_data.skel.h" +#include "test_signed_loader_lsm.skel.h" + +enum { + BPF_SIG_UNSIGNED = 0, + BPF_SIG_VERIFIED, +}; + +enum { + BPF_SIG_KEYRING_NONE = 0, + BPF_SIG_KEYRING_BUILTIN, + BPF_SIG_KEYRING_SECONDARY, + BPF_SIG_KEYRING_PLATFORM, + BPF_SIG_KEYRING_USER, +}; + +static int load_loader(const void *insns, __u32 insns_sz, int map_fd, + const void *sig, __u32 sig_sz, __s32 keyring_id, + __u32 fd_array_cnt) +{ + union bpf_attr attr; + int fd; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = insns_sz / sizeof(struct bpf_insn); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(&map_fd); + if (sig) { + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = keyring_id; + } + attr.fd_array_cnt = fd_array_cnt; + memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); + fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + return fd < 0 ? -errno : fd; +} + +static int run_gen_loader(const void *insns, __u32 insns_sz, + const void *data, __u32 data_sz, + const void *excl, __u32 excl_sz, + const void *sig, __u32 sig_sz, + void *ctx, __u32 ctx_sz, bool *loader_ran) +{ + LIBBPF_OPTS(bpf_map_create_opts, mopts, + .excl_prog_hash = excl, + .excl_prog_hash_size = excl_sz); + __u32 key = 0; + union bpf_attr attr; + int map_fd, prog_fd, ret; + + *loader_ran = false; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", + 4, data_sz, 1, &mopts); + if (map_fd < 0) + return -errno; + if (bpf_map_update_elem(map_fd, &key, data, 0)) { + ret = -errno; + goto out_map; + } + if (bpf_map_freeze(map_fd)) { + ret = -errno; + goto out_map; + } + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = insns_sz / sizeof(struct bpf_insn); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(&map_fd); + if (sig) { + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + attr.fd_array_cnt = 1; + } + memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + if (prog_fd < 0) { + ret = -errno; + goto out_map; + } + + memset(&attr, 0, sizeof(attr)); + attr.test.prog_fd = prog_fd; + attr.test.ctx_in = ptr_to_u64(ctx); + attr.test.ctx_size_in = ctx_sz; + if (syscall(__NR_bpf, BPF_PROG_RUN, &attr, + offsetofend(union bpf_attr, test)) < 0) { + ret = -errno; + goto out_prog; + } + *loader_ran = true; + ret = (int)attr.test.retval; +out_prog: + close(prog_fd); +out_map: + close(map_fd); + return ret; +} + +static void close_loader_ctx_fds(void *ctx, int nr_maps, int nr_progs) +{ + struct bpf_map_desc *md = (struct bpf_map_desc *)((char *)ctx + + sizeof(struct bpf_loader_ctx)); + struct bpf_prog_desc *pd = (struct bpf_prog_desc *)(md + nr_maps); + int i; + + for (i = 0; i < nr_maps; i++) + if (md[i].map_fd > 0) + close(md[i].map_fd); + for (i = 0; i < nr_progs; i++) + if (pd[i].prog_fd > 0) + close(pd[i].prog_fd); +} + +static int run_setup(const char *cmd, const char *dir) +{ + int pid, status; + + pid = fork(); + if (pid < 0) + return -errno; + if (pid == 0) { + execlp("./verify_sig_setup.sh", "./verify_sig_setup.sh", + cmd, dir, NULL); + exit(1); + } + if (waitpid(pid, &status, 0) < 0) + return -errno; + return (WIFEXITED(status) && + WEXITSTATUS(status) == 0) ? 0 : -EINVAL; +} + +static int sign_buf(const char *dir, const void *buf, __u32 len, + void *sig, __u32 *sig_sz) +{ + char data_tmpl[PATH_MAX], key[PATH_MAX]; + char sigpath[PATH_MAX + sizeof(".p7s")]; + int fd, pid, status, ret; + struct stat st; + + ret = snprintf(data_tmpl, sizeof(data_tmpl), "%s/dataXXXXXX", dir); + if (ret < 0 || ret >= (int)sizeof(data_tmpl)) + return -ENAMETOOLONG; + ret = 0; + + fd = mkstemp(data_tmpl); + if (fd < 0) + return -errno; + if (write(fd, buf, len) != (ssize_t)len) { + close(fd); + ret = -EIO; + goto out; + } + close(fd); + + pid = fork(); + if (pid < 0) { + ret = -errno; + goto out; + } + if (pid == 0) { + snprintf(key, sizeof(key), "%s/signing_key.pem", dir); + execlp("./sign-file", "./sign-file", "-d", "sha256", + key, key, data_tmpl, NULL); + exit(1); + } + if (waitpid(pid, &status, 0) < 0 || + !WIFEXITED(status) || WEXITSTATUS(status)) { + ret = -EINVAL; + goto out; + } + + snprintf(sigpath, sizeof(sigpath), "%s.p7s", data_tmpl); + if (stat(sigpath, &st) < 0) { + ret = -errno; + goto out; + } + if (st.st_size > (off_t)*sig_sz) { + ret = -E2BIG; + goto out_sig; + } + fd = open(sigpath, O_RDONLY); + if (fd < 0) { + ret = -errno; + goto out_sig; + } + if (read(fd, sig, st.st_size) != st.st_size) { + close(fd); + ret = -EIO; + goto out_sig; + } + close(fd); + *sig_sz = st.st_size; +out_sig: + unlink(sigpath); +out: + unlink(data_tmpl); + return ret; +} + +struct gen_loader_fixture { + struct test_signed_loader *skel; + struct gen_loader_opts gopts; + unsigned char *blob; + void *ctx; + __u32 data_sz; + __u32 ctx_sz; + int nr_maps; + int nr_progs; + __u8 excl[SHA256_DIGEST_LENGTH]; +}; + +static int gen_loader_fixture_init(struct gen_loader_fixture *f) +{ + LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true); + int nr_maps = 0, nr_progs = 0; + struct bpf_program *p; + struct bpf_map *m; + + memset(f, 0, sizeof(*f)); + f->skel = test_signed_loader__open(); + if (!ASSERT_OK_PTR(f->skel, "skel_open")) + return -1; + if (!ASSERT_OK(bpf_object__gen_loader(f->skel->obj, &gopts), "gen_loader")) + return -1; + if (!ASSERT_OK(bpf_object__load(f->skel->obj), "gen_load")) + return -1; + f->gopts = gopts; + + bpf_object__for_each_program(p, f->skel->obj) + nr_progs++; + bpf_object__for_each_map(m, f->skel->obj) + nr_maps++; + f->nr_maps = nr_maps; + f->nr_progs = nr_progs; + f->ctx_sz = sizeof(struct bpf_loader_ctx) + + nr_maps * sizeof(struct bpf_map_desc) + + nr_progs * sizeof(struct bpf_prog_desc); + f->ctx = calloc(1, f->ctx_sz); + if (!ASSERT_OK_PTR(f->ctx, "ctx_alloc")) + return -1; + ((struct bpf_loader_ctx *)f->ctx)->sz = f->ctx_sz; + + f->data_sz = gopts.data_sz; + f->blob = malloc(f->data_sz); + if (!ASSERT_OK_PTR(f->blob, "blob_alloc")) + return -1; + memcpy(f->blob, gopts.data, f->data_sz); + + /* excl_prog_hash = SHA256(loader insns) == the loader's prog->digest. */ + libbpf_sha256(gopts.insns, gopts.insns_sz, f->excl); + return 0; +} + +static void gen_loader_fixture_fini(struct gen_loader_fixture *f) +{ + if (f->ctx) + close_loader_ctx_fds(f->ctx, f->nr_maps, f->nr_progs); + free(f->blob); + free(f->ctx); + test_signed_loader__destroy(f->skel); +} + +static void metadata_match(void) +{ + struct gen_loader_fixture f; + bool ran; + int r; + + if (gen_loader_fixture_init(&f) == 0) { + r = run_gen_loader(f.gopts.insns, f.gopts.insns_sz, f.blob, + f.data_sz, f.excl, sizeof(f.excl), NULL, 0, + f.ctx, f.ctx_sz, &ran); + ASSERT_TRUE(ran, "loader ran"); + ASSERT_EQ(r, 0, "honest loader retval"); + } + gen_loader_fixture_fini(&f); +} + +static void signature_enforced(void) +{ + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + struct gen_loader_fixture f; + int fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * A present-but-invalid signature (the cert bytes are not a + * PKCS#7 signature) must be rejected at load: the signature + * path is honored, not ignored. (The valid path is covered by + * the signed lskels.) Pin -EBADMSG, the PKCS#7 parse failure: + * a looser fd < 0 check could also be satisfied by the sparse + * fd_array rejection (-EACCES) that the loader's map reference + * would trip even if the signature were silently ignored. + */ + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, + sizeof(junk), KEY_SPEC_SESSION_KEYRING, 0); + ASSERT_EQ(fd, -EBADMSG, "invalid signature rejected at load"); + if (fd >= 0) + close(fd); + } + gen_loader_fixture_fini(&f); +} + +static void signed_nonexcl_fd_array_rejected(void) +{ + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + struct gen_loader_fixture f; + int map_fd, fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * A signed program may only bind exclusive maps through fd_array + * (their contents are folded into the signature). Binding a + * non-exclusive map is rejected, before the signature is even + * examined. + */ + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "nonexcl", 4, + f.data_sz, 1, NULL); + if (ASSERT_OK_FD(map_fd, "nonexcl_map")) { + if (ASSERT_OK(bpf_map_freeze(map_fd), "freeze")) { + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, + map_fd, junk, sizeof(junk), + KEY_SPEC_SESSION_KEYRING, 1); + ASSERT_EQ(fd, -EPERM, + "non-exclusive map in signed fd_array rejected"); + if (fd >= 0) + close(fd); + } + close(map_fd); + } + } + gen_loader_fixture_fini(&f); +} + +static void signed_unfrozen_fd_array_rejected(void) +{ + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + LIBBPF_OPTS(bpf_map_create_opts, mopts); + struct gen_loader_fixture f; + __u32 key = 0; + int map_fd, fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * The metadata map must be frozen before a signed load so the + * folded bytes cannot change afterwards. Bind an exclusive map + * with matching contents but skip the freeze: the load must be + * rejected by the frozen check with -EPERM. The exclusivity + * check right after it would pass, so the errno uniquely pins + * the freeze requirement. + */ + mopts.excl_prog_hash = f.excl; + mopts.excl_prog_hash_size = sizeof(f.excl); + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "unfrozen", 4, + f.data_sz, 1, &mopts); + if (ASSERT_OK_FD(map_fd, "unfrozen_map")) { + if (ASSERT_OK(bpf_map_update_elem(map_fd, &key, f.blob, 0), + "update")) { + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, + map_fd, junk, sizeof(junk), + KEY_SPEC_SESSION_KEYRING, 1); + ASSERT_EQ(fd, -EPERM, + "unfrozen map in signed fd_array rejected"); + if (fd >= 0) + close(fd); + } + close(map_fd); + } + } + gen_loader_fixture_fini(&f); +} + +static void signed_nonarray_fd_array_rejected(void) +{ + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + LIBBPF_OPTS(bpf_map_create_opts, mopts); + struct gen_loader_fixture f; + int map_fd, fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * Only a plain BPF_MAP_TYPE_ARRAY may be folded into the + * signature. An exclusive map of any other type is rejected + * (-EINVAL) rather than folded - this is the type gate that + * keeps arena maps (map_direct_value_addr() returns a user + * address) and insn-array maps (buffer smaller than value_size) + * out of the hashed region, where the old code would have + * memcpy()'d from them. A hash map stands in here: it is + * exclusive (bound to the loader digest) but not an array. + */ + mopts.excl_prog_hash = f.excl; + mopts.excl_prog_hash_size = sizeof(f.excl); + map_fd = bpf_map_create(BPF_MAP_TYPE_HASH, "excl_hash", 4, 4, 1, + &mopts); + if (ASSERT_OK_FD(map_fd, "excl_hash_map")) { + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, + junk, sizeof(junk), + KEY_SPEC_SESSION_KEYRING, 1); + ASSERT_EQ(fd, -EINVAL, + "non-array map in signed fd_array rejected"); + if (fd >= 0) + close(fd); + close(map_fd); + } + } + gen_loader_fixture_fini(&f); +} + +static int setup_meta_map(const struct gen_loader_fixture *f); + +static void signed_btf_fd_array_rejected(void) +{ + char dir_tmpl[] = "/tmp/signed_loader_btfXXXXXX", *dir = NULL; + __u32 sig_sz = 8192; + int map_fd = -1, prog_fd = -1; + unsigned char *buf = NULL; + struct gen_loader_fixture f; + bool have_fixture = false; + struct btf *btf = NULL; + union bpf_attr attr; + int fds[2]; + __u8 sig[8192]; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + have_fixture = true; + if (gen_loader_fixture_init(&f) != 0) + goto out; + + /* + * fd_array binds maps and BTFs alike, but only exclusive array maps are + * folded into the signature. Build an otherwise genuinely signed load - + * insns || metadata, exclusive frozen map at fd_array[0] - then smuggle + * an extra BTF into fd_array[1]. A signed program may not bind any BTF, + * so resolving the fd_array entries rejects the BTF with -EACCES (in + * __add_used_btf(), before the signature is even verified). + */ + buf = malloc((size_t)f.gopts.insns_sz + f.data_sz); + if (!ASSERT_OK_PTR(buf, "signbuf")) + goto out; + memcpy(buf, f.gopts.insns, f.gopts.insns_sz); + memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz); + if (!ASSERT_OK(sign_buf(dir, buf, f.gopts.insns_sz + f.data_sz, sig, + &sig_sz), "sign insns||metadata")) + goto out; + + map_fd = setup_meta_map(&f); + if (!ASSERT_OK_FD(map_fd, "meta_map")) + goto out; + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf_new_empty")) + goto out; + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf_load")) + goto out; + + fds[0] = map_fd; + fds[1] = btf__fd(btf); + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(f.gopts.insns); + attr.insn_cnt = f.gopts.insns_sz / sizeof(struct bpf_insn); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(fds); + attr.fd_array_cnt = 2; + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + ASSERT_EQ(prog_fd < 0 ? -errno : prog_fd, -EACCES, + "BTF in signed fd_array rejected"); + if (prog_fd >= 0) + close(prog_fd); +out: + if (btf) + btf__free(btf); + if (map_fd >= 0) + close(map_fd); + if (have_fixture) + gen_loader_fixture_fini(&f); + if (dir) + run_setup("cleanup", dir); + free(buf); +} + +static void signature_failure_logs(void) +{ + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + char log_buf[1024] = {}; + struct gen_loader_fixture f; + union bpf_attr attr; + int fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * Signature verification now runs inside bpf_check(), so a + * failure is reported through the verifier log. A present-but- + * invalid signature is rejected and the log says why. + */ + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(f.gopts.insns); + attr.insn_cnt = f.gopts.insns_sz / sizeof(struct bpf_insn); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.signature = ptr_to_u64(junk); + attr.signature_size = sizeof(junk); + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + attr.log_level = 1; + attr.log_buf = ptr_to_u64(log_buf); + attr.log_size = sizeof(log_buf); + memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); + + fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + ASSERT_LT(fd, 0, "invalid signature rejected at load"); + if (fd >= 0) + close(fd); + ASSERT_HAS_SUBSTR(log_buf, "signature verification failed", + "verifier logs signature failure"); + } + gen_loader_fixture_fini(&f); +} + +static void signature_too_large(void) +{ + static const __u8 junk[64] = {}; + struct gen_loader_fixture f; + int fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * signature_size beyond the kernel's bound (KMALLOC_MAX_CACHE_SIZE) + * is rejected before the buffer is read. + */ + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, + 64 << 20, KEY_SPEC_SESSION_KEYRING, 0); + ASSERT_EQ(fd, -EINVAL, "oversized signature rejected"); + if (fd >= 0) + close(fd); + } + gen_loader_fixture_fini(&f); +} + +static void signature_zero_size(void) +{ + static const __u8 junk[64] = {}; + struct gen_loader_fixture f; + int fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * A present signature with signature_size == 0 is rejected + * up front, before the keyring is resolved or the signature + * buffer is read. + */ + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, + 0, KEY_SPEC_SESSION_KEYRING, 0); + ASSERT_EQ(fd, -EINVAL, "zero-size signature rejected"); + if (fd >= 0) + close(fd); + } + gen_loader_fixture_fini(&f); +} + +static void signature_bad_keyring(void) +{ + static const __u8 junk[64] = {}; + struct gen_loader_fixture f; + int fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * A present signature with a keyring_id that resolves to no key is + * rejected up front: bpf_prog_verify_signature() fails the keyring + * lookup (-EINVAL) before it ever looks at the signature bytes. A + * large positive serial takes the user-keyring path and won't exist. + */ + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, + sizeof(junk), INT_MAX, 0); + ASSERT_EQ(fd, -EINVAL, "signature with bad keyring_id rejected"); + if (fd >= 0) + close(fd); + } + gen_loader_fixture_fini(&f); +} + +/* + * A signed loader must ignore ctx-supplied map dimensions: the host cannot + * resize a signed program's maps via the loader ctx. Drive a one-map program + * through gen_loader, ask (via ctx) for every map to be resized to a bogus + * value, and confirm the created maps keep their attested size. + */ +#define GATING_BOGUS_MAX 0x4000 + +static void metadata_ctx_max_entries_ignored(void) +{ + LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true); + struct test_signed_loader_map *skel; + __u8 excl[SHA256_DIGEST_LENGTH]; + int nr_maps = 0, nr_progs = 0, i, checked = 0, r; + struct bpf_program *p; + struct bpf_map *m; + struct bpf_map_desc *md; + unsigned char *blob; + __u32 ctx_sz, data_sz; + void *ctx; + bool ran; + + skel = test_signed_loader_map__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + if (!ASSERT_OK(bpf_object__gen_loader(skel->obj, &gopts), "gen_loader")) + goto destroy; + if (!ASSERT_OK(bpf_object__load(skel->obj), "gen_load")) + goto destroy; + + bpf_object__for_each_program(p, skel->obj) + nr_progs++; + bpf_object__for_each_map(m, skel->obj) + nr_maps++; + ctx_sz = sizeof(struct bpf_loader_ctx) + + nr_maps * sizeof(struct bpf_map_desc) + + nr_progs * sizeof(struct bpf_prog_desc); + ctx = calloc(1, ctx_sz); + if (!ASSERT_OK_PTR(ctx, "ctx_alloc")) + goto destroy; + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + + md = (struct bpf_map_desc *)((char *)ctx + sizeof(struct bpf_loader_ctx)); + for (i = 0; i < nr_maps; i++) + md[i].max_entries = GATING_BOGUS_MAX; + + libbpf_sha256(gopts.insns, gopts.insns_sz, excl); + data_sz = gopts.data_sz; + blob = malloc(data_sz); + if (!ASSERT_OK_PTR(blob, "blob_alloc")) + goto free_ctx; + memcpy(blob, gopts.data, data_sz); + + r = run_gen_loader(gopts.insns, gopts.insns_sz, blob, data_sz, + excl, sizeof(excl), NULL, 0, ctx, ctx_sz, &ran); + if (!ASSERT_TRUE(ran, "loader ran") || + !ASSERT_EQ(r, 0, "loader retval")) + goto free_blob; + + for (i = 0; i < nr_maps; i++) { + struct bpf_map_info info; + __u32 ilen = sizeof(info); + int fd = md[i].map_fd; + + if (fd <= 0) + continue; + memset(&info, 0, sizeof(info)); + if (ASSERT_OK(bpf_map_get_info_by_fd(fd, &info, &ilen), "map_info")) { + ASSERT_NEQ(info.max_entries, GATING_BOGUS_MAX, + "ctx max_entries ignored for signed loader"); + checked++; + } + } + ASSERT_GT(checked, 0, "inspected a created map"); + +free_blob: + free(blob); +free_ctx: + close_loader_ctx_fds(ctx, nr_maps, nr_progs); + free(ctx); +destroy: + test_signed_loader_map__destroy(skel); +} + +/* + * A signed loader must also ignore ctx-supplied initial_value: the host cannot + * re-seed a signed program's map contents through the loader ctx. Drive a + * program with one initialized global (a .data map) through gen_loader, point + * every map's ctx initial_value at an adversarial buffer, and confirm the + * created map still holds the attested value, never the ctx bytes. + */ +#define DATA_MAGIC 0x5eed1234abad1deaULL + +static void metadata_ctx_initial_value_ignored(void) +{ + LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true); + struct test_signed_loader_data *skel; + __u8 excl[SHA256_DIGEST_LENGTH], evil[64]; + int nr_maps = 0, nr_progs = 0, i, found = 0, r; + struct bpf_program *p; + struct bpf_map *m; + struct bpf_map_desc *md; + unsigned char *blob; + __u32 ctx_sz, data_sz; + void *ctx; + bool ran; + + skel = test_signed_loader_data__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + if (!ASSERT_OK(bpf_object__gen_loader(skel->obj, &gopts), "gen_loader")) + goto destroy; + if (!ASSERT_OK(bpf_object__load(skel->obj), "gen_load")) + goto destroy; + + bpf_object__for_each_program(p, skel->obj) + nr_progs++; + bpf_object__for_each_map(m, skel->obj) + nr_maps++; + ctx_sz = sizeof(struct bpf_loader_ctx) + + nr_maps * sizeof(struct bpf_map_desc) + + nr_progs * sizeof(struct bpf_prog_desc); + ctx = calloc(1, ctx_sz); + if (!ASSERT_OK_PTR(ctx, "ctx_alloc")) + goto destroy; + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + + memset(evil, 0xAA, sizeof(evil)); + md = (struct bpf_map_desc *)((char *)ctx + sizeof(struct bpf_loader_ctx)); + for (i = 0; i < nr_maps; i++) + md[i].initial_value = ptr_to_u64(evil); + + libbpf_sha256(gopts.insns, gopts.insns_sz, excl); + data_sz = gopts.data_sz; + blob = malloc(data_sz); + if (!ASSERT_OK_PTR(blob, "blob_alloc")) + goto free_ctx; + memcpy(blob, gopts.data, data_sz); + + r = run_gen_loader(gopts.insns, gopts.insns_sz, blob, data_sz, + excl, sizeof(excl), NULL, 0, ctx, ctx_sz, &ran); + if (!ASSERT_TRUE(ran, "loader ran") || + !ASSERT_EQ(r, 0, "loader retval")) + goto free_blob; + + for (i = 0; i < nr_maps; i++) { + struct bpf_map_info info; + __u32 ilen = sizeof(info), key = 0; + __u8 value[64] = {}; + __u64 got; + int fd = md[i].map_fd; + + if (fd <= 0) + continue; + memset(&info, 0, sizeof(info)); + if (!ASSERT_OK(bpf_map_get_info_by_fd(fd, &info, &ilen), "map_info")) + continue; + if (info.value_size <= sizeof(value) && + bpf_map_lookup_elem(fd, &key, value) == 0) { + memcpy(&got, value, sizeof(got)); + /* attested .data survives; ctx bytes (0xAA..) ignored */ + if (got == DATA_MAGIC) + found = 1; + ASSERT_NEQ(got, 0xAAAAAAAAAAAAAAAAULL, + "ctx initial_value ignored for signed loader"); + } + } + ASSERT_EQ(found, 1, "attested .data value preserved"); + +free_blob: + free(blob); +free_ctx: + close_loader_ctx_fds(ctx, nr_maps, nr_progs); + free(ctx); +destroy: + test_signed_loader_data__destroy(skel); +} + +/* + * The load-time signature must authenticate the loader instructions: a valid + * signature loads, and the very same signature over one-byte-tampered insns is + * rejected. Uses ./verify_sig_setup.sh + ./sign-file at runtime, like + * verify_pkcs7_sig, and verifies against the session keyring the key was added + * to. (signature_enforced/_too_large only cover a malformed signature.) + */ +static void signature_authenticates_insns(void) +{ + LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true); + char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir; + struct test_signed_loader *skel = NULL; + __u8 excl[SHA256_DIGEST_LENGTH], sig[8192]; + __u32 sig_sz = sizeof(sig), insns_sz, data_sz, ctx_sz; + unsigned char *insns = NULL, *tampered = NULL, *blob = NULL; + unsigned char *signbuf = NULL; + int nr_maps = 0, nr_progs = 0, r; + struct bpf_program *p; + struct bpf_map *m; + void *ctx = NULL; + bool ran; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + + skel = test_signed_loader__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + if (!ASSERT_OK(bpf_object__gen_loader(skel->obj, &gopts), "gen_loader")) + goto cleanup; + if (!ASSERT_OK(bpf_object__load(skel->obj), "gen_load")) + goto cleanup; + + bpf_object__for_each_program(p, skel->obj) + nr_progs++; + bpf_object__for_each_map(m, skel->obj) + nr_maps++; + ctx_sz = sizeof(struct bpf_loader_ctx) + + nr_maps * sizeof(struct bpf_map_desc) + + nr_progs * sizeof(struct bpf_prog_desc); + insns_sz = gopts.insns_sz; + data_sz = gopts.data_sz; + ctx = calloc(1, ctx_sz); + insns = malloc(insns_sz); + tampered = malloc(insns_sz); + blob = malloc(data_sz); + if (!ASSERT_OK_PTR(ctx, "ctx") || + !ASSERT_OK_PTR(insns, "insns") || + !ASSERT_OK_PTR(tampered, "tampered") || + !ASSERT_OK_PTR(blob, "blob")) + goto cleanup; + memcpy(insns, gopts.insns, insns_sz); + memcpy(blob, gopts.data, data_sz); + libbpf_sha256(insns, insns_sz, excl); + + signbuf = malloc((size_t)insns_sz + data_sz); + if (!ASSERT_OK_PTR(signbuf, "signbuf")) + goto cleanup; + memcpy(signbuf, insns, insns_sz); + memcpy(signbuf + insns_sz, blob, data_sz); + if (!ASSERT_OK(sign_buf(dir, signbuf, insns_sz + data_sz, sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(ctx, 0, ctx_sz); + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + r = run_gen_loader(insns, insns_sz, blob, data_sz, excl, sizeof(excl), + sig, sig_sz, ctx, ctx_sz, &ran); + ASSERT_TRUE(ran, "valid signature: loader loaded and ran"); + ASSERT_EQ(r, 0, "valid signature accepted"); + close_loader_ctx_fds(ctx, nr_maps, nr_progs); + + memcpy(tampered, insns, insns_sz); + tampered[insns_sz / 2] ^= 0xff; + /* + * Bind the metadata map to the tampered loader's own digest, so the + * verifier's exclusive-map check (excl_prog_sha == prog->digest) passes + * and the signature - verified after the maps are resolved - is what + * rejects the load. This is the attacker's best case: even after + * re-binding the exclusive map to their tampered loader, the signature + * over the original insns || metadata still fails. (Leaving the map + * bound to the original digest would instead trip the excl check first.) + */ + libbpf_sha256(tampered, insns_sz, excl); + memset(ctx, 0, ctx_sz); + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + r = run_gen_loader(tampered, insns_sz, blob, data_sz, excl, sizeof(excl), + sig, sig_sz, ctx, ctx_sz, &ran); + ASSERT_FALSE(ran, "tampered loader rejected before run"); + ASSERT_EQ(r, -EKEYREJECTED, "signature is bound to the instructions"); +cleanup: + free(insns); + free(tampered); + free(blob); + free(signbuf); + free(ctx); + test_signed_loader__destroy(skel); + run_setup("cleanup", dir); +} + +static void signature_authenticates_metadata(void) +{ + LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true); + char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir; + struct test_signed_loader *skel = NULL; + __u8 excl[SHA256_DIGEST_LENGTH], sig[8192]; + __u32 sig_sz = sizeof(sig), insns_sz, data_sz, ctx_sz; + unsigned char *insns = NULL, *blob = NULL; + unsigned char *signbuf = NULL; + int nr_maps = 0, nr_progs = 0, r; + struct bpf_program *p; + struct bpf_map *m; + void *ctx = NULL; + bool ran; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + + skel = test_signed_loader__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + if (!ASSERT_OK(bpf_object__gen_loader(skel->obj, &gopts), "gen_loader")) + goto cleanup; + if (!ASSERT_OK(bpf_object__load(skel->obj), "gen_load")) + goto cleanup; + + bpf_object__for_each_program(p, skel->obj) + nr_progs++; + bpf_object__for_each_map(m, skel->obj) + nr_maps++; + ctx_sz = sizeof(struct bpf_loader_ctx) + + nr_maps * sizeof(struct bpf_map_desc) + + nr_progs * sizeof(struct bpf_prog_desc); + insns_sz = gopts.insns_sz; + data_sz = gopts.data_sz; + ctx = calloc(1, ctx_sz); + insns = malloc(insns_sz); + blob = malloc(data_sz); + if (!ASSERT_OK_PTR(ctx, "ctx") || + !ASSERT_OK_PTR(insns, "insns") || + !ASSERT_OK_PTR(blob, "blob")) + goto cleanup; + memcpy(insns, gopts.insns, insns_sz); + memcpy(blob, gopts.data, data_sz); + libbpf_sha256(insns, insns_sz, excl); + + signbuf = malloc((size_t)insns_sz + data_sz); + if (!ASSERT_OK_PTR(signbuf, "signbuf")) + goto cleanup; + memcpy(signbuf, insns, insns_sz); + memcpy(signbuf + insns_sz, blob, data_sz); + if (!ASSERT_OK(sign_buf(dir, signbuf, insns_sz + data_sz, sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(ctx, 0, ctx_sz); + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + r = run_gen_loader(insns, insns_sz, blob, data_sz, excl, sizeof(excl), + sig, sig_sz, ctx, ctx_sz, &ran); + ASSERT_TRUE(ran, "valid signature: loader loaded and ran"); + ASSERT_EQ(r, 0, "valid signature accepted"); + close_loader_ctx_fds(ctx, nr_maps, nr_progs); + + /* + * Tamper the metadata after signing while leaving the instructions + * and thus the exclusive hash binding untouched: the map freezes + * fine and excl_prog_sha still matches the loader's digest, so the + * load reaches signature verification, which folds the live frozen + * map bytes into the checked payload and must reject the modified + * blob. A kernel folding anything but the map contents themselves + * would wrongly accept this load. + */ + blob[data_sz / 2] ^= 0xff; + memset(ctx, 0, ctx_sz); + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + r = run_gen_loader(insns, insns_sz, blob, data_sz, excl, sizeof(excl), + sig, sig_sz, ctx, ctx_sz, &ran); + ASSERT_FALSE(ran, "tampered metadata rejected before run"); + ASSERT_EQ(r, -EKEYREJECTED, "signature is bound to the metadata"); +cleanup: + free(insns); + free(blob); + free(signbuf); + free(ctx); + test_signed_loader__destroy(skel); + run_setup("cleanup", dir); +} + +static int make_excl_map(__u32 flags, __u32 value_size) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + __u8 hash[SHA256_DIGEST_LENGTH] = { 1 }; /* any 32-byte value */ + + opts.excl_prog_hash = hash; + opts.excl_prog_hash_size = sizeof(hash); + opts.map_flags = flags; + return bpf_map_create(BPF_MAP_TYPE_ARRAY, "md", 4, value_size, 1, &opts); +} + +static void hash_requires_frozen(void) +{ + __u8 hbuf[SHA256_DIGEST_LENGTH], val[64] = {}; + struct bpf_map_info info; + __u32 ilen, key = 0; + int fd; + + fd = make_excl_map(0, sizeof(val)); + if (!ASSERT_OK_FD(fd, "excl_map")) + return; + ASSERT_OK(bpf_map_update_elem(fd, &key, val, 0), "update"); + + memset(&info, 0, sizeof(info)); + info.hash = ptr_to_u64(hbuf); + info.hash_size = sizeof(hbuf); + ilen = sizeof(info); + ASSERT_EQ(bpf_map_get_info_by_fd(fd, &info, &ilen), -EPERM, + "hash of unfrozen map rejected"); + close(fd); +} + +static void no_update_after_freeze(void) +{ + __u8 val[64] = {}; + __u32 key = 0; + int fd; + + fd = make_excl_map(0, sizeof(val)); + if (!ASSERT_OK_FD(fd, "excl_map")) + return; + ASSERT_OK(bpf_map_update_elem(fd, &key, val, 0), "update"); + ASSERT_OK(bpf_map_freeze(fd), "freeze"); + ASSERT_EQ(bpf_map_update_elem(fd, &key, val, 0), -EPERM, + "update after freeze rejected"); + close(fd); +} + +static void freeze_writable_mmap(void) +{ + void *w; + int fd; + + fd = make_excl_map(BPF_F_MMAPABLE, 4096); + if (!ASSERT_OK_FD(fd, "excl_mmapable_map")) + return; + w = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (ASSERT_OK_PTR(w, "writable_mmap")) { + ASSERT_EQ(bpf_map_freeze(fd), -EBUSY, + "freeze rejected while writable mmap held"); + munmap(w, 4096); + } + close(fd); +} + +static void no_writable_mmap_frozen(void) +{ + void *w; + int fd; + + fd = make_excl_map(BPF_F_MMAPABLE, 4096); + if (!ASSERT_OK_FD(fd, "excl_mmapable_map")) + return; + ASSERT_OK(bpf_map_freeze(fd), "freeze"); + w = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + ASSERT_EQ(w, MAP_FAILED, "writable mmap of frozen map rejected"); + if (w != MAP_FAILED) + munmap(w, 4096); + close(fd); +} + +static void map_hash_matches_libbpf(void) +{ + __u8 kbuf[SHA256_DIGEST_LENGTH], lbuf[SHA256_DIGEST_LENGTH], val[64] = {}; + struct bpf_map_info info; + __u32 ilen, key = 0; + int fd, i; + + /* + * The signing scheme assumes the kernel's map hash equals what libbpf + * computes over the same bytes (gen_loader bakes libbpf_sha256(blob); + * the kernel recomputes via array_map_get_hash). Pin that they agree. + */ + for (i = 0; i < (int)sizeof(val); i++) + val[i] = i * 7 + 1; + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "h", 4, sizeof(val), 1, NULL); + if (!ASSERT_OK_FD(fd, "array_map")) + return; + ASSERT_OK(bpf_map_update_elem(fd, &key, val, 0), "update"); + ASSERT_OK(bpf_map_freeze(fd), "freeze"); + memset(&info, 0, sizeof(info)); + info.hash = ptr_to_u64(kbuf); + info.hash_size = sizeof(kbuf); + ilen = sizeof(info); + if (ASSERT_OK(bpf_map_get_info_by_fd(fd, &info, &ilen), "get_hash")) { + libbpf_sha256(val, sizeof(val), lbuf); + ASSERT_EQ(memcmp(kbuf, lbuf, sizeof(kbuf)), 0, + "kernel map hash matches libbpf_sha256"); + } + close(fd); +} + +static void map_hash_multi_element(void) +{ + const __u32 nr = 8, value_size = 64; + __u8 kbuf[SHA256_DIGEST_LENGTH], lbuf[SHA256_DIGEST_LENGTH]; + struct bpf_map_info info; + __u32 ilen, i, j; + __u8 *full; + int fd; + + /* + * array_map_get_hash() hashes elem_size * max_entries (the whole value + * area), not just element 0. With an 8-aligned value_size elem_size has + * no padding, so pin that a >1-entry array's kernel hash equals + * libbpf_sha256() over the full, concatenated element contents. + */ + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "h", 4, value_size, nr, NULL); + if (!ASSERT_OK_FD(fd, "array_map")) + return; + full = calloc(nr, value_size); + if (!ASSERT_OK_PTR(full, "buf")) + goto close_fd; + for (i = 0; i < nr; i++) { + __u8 *v = full + i * value_size; + + for (j = 0; j < value_size; j++) + v[j] = i * 31 + j * 7 + 1; + ASSERT_OK(bpf_map_update_elem(fd, &i, v, 0), "update"); + } + ASSERT_OK(bpf_map_freeze(fd), "freeze"); + memset(&info, 0, sizeof(info)); + info.hash = ptr_to_u64(kbuf); + info.hash_size = sizeof(kbuf); + ilen = sizeof(info); + if (ASSERT_OK(bpf_map_get_info_by_fd(fd, &info, &ilen), "get_hash")) { + libbpf_sha256(full, (size_t)nr * value_size, lbuf); + ASSERT_EQ(memcmp(kbuf, lbuf, sizeof(kbuf)), 0, + "kernel hash covers full multi-element value area"); + } + free(full); +close_fd: + close(fd); +} + +static void map_hash_bad_size(void) +{ + __u8 kbuf[SHA256_DIGEST_LENGTH], val[64] = {}; + struct bpf_map_info info; + __u32 ilen, key = 0; + int fd; + + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "h", 4, sizeof(val), 1, NULL); + if (!ASSERT_OK_FD(fd, "array_map")) + return; + ASSERT_OK(bpf_map_update_elem(fd, &key, val, 0), "update"); + ASSERT_OK(bpf_map_freeze(fd), "freeze"); + memset(&info, 0, sizeof(info)); + info.hash = ptr_to_u64(kbuf); + info.hash_size = sizeof(kbuf) / 2; + ilen = sizeof(info); + ASSERT_EQ(bpf_map_get_info_by_fd(fd, &info, &ilen), -EINVAL, + "wrong hash_size rejected"); + close(fd); +} + +static void map_hash_unsupported_type(void) +{ + __u8 kbuf[SHA256_DIGEST_LENGTH]; + struct bpf_map_info info; + __u32 ilen; + int fd; + + /* Only arrays implement map_get_hash; a hash map must be refused. */ + fd = bpf_map_create(BPF_MAP_TYPE_HASH, "h", 4, 8, 4, NULL); + if (!ASSERT_OK_FD(fd, "hash_map")) + return; + memset(&info, 0, sizeof(info)); + info.hash = ptr_to_u64(kbuf); + info.hash_size = sizeof(kbuf); + ilen = sizeof(info); + ASSERT_EQ(bpf_map_get_info_by_fd(fd, &info, &ilen), -EINVAL, + "hash unsupported for non-array map"); + close(fd); +} + +static int setup_meta_map(const struct gen_loader_fixture *f) +{ + LIBBPF_OPTS(bpf_map_create_opts, mopts, + .excl_prog_hash = f->excl, + .excl_prog_hash_size = sizeof(f->excl)); + __u32 key = 0; + int fd; + + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, + f->data_sz, 1, &mopts); + if (fd < 0) + return -errno; + if (bpf_map_update_elem(fd, &key, f->blob, 0) || bpf_map_freeze(fd)) { + close(fd); + return -errno; + } + return fd; +} + +static void lsm_signature_verdict(void) +{ + char dir_tmpl[] = "/tmp/signed_loader_lsmXXXXXX", *dir = NULL; + struct test_signed_loader_lsm *lsm = NULL; + __u32 sig_sz = 8192, msig_sz = 8192; + int map_fd = -1, prog_fd = -1; + bool have_fixture = false; + struct gen_loader_fixture f; + unsigned char *buf; + __s32 ses_serial; + __u8 sig[8192]; + + lsm = test_signed_loader_lsm__open_and_load(); + if (!ASSERT_OK_PTR(lsm, "lsm_skel_load")) + return; + lsm->bss->monitored_tid = sys_gettid(); + if (!ASSERT_OK(test_signed_loader_lsm__attach(lsm), "lsm_attach")) + goto out; + + have_fixture = true; + if (gen_loader_fixture_init(&f) != 0) + goto out; + + map_fd = setup_meta_map(&f); + if (!ASSERT_OK_FD(map_fd, "meta_map_unsigned")) + goto out; + lsm->bss->seen = 0; + prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, NULL, 0, 0, 0); + close(map_fd); + map_fd = -1; + if (!ASSERT_OK_FD(prog_fd, "unsigned loader load")) + goto out; + close(prog_fd); + prog_fd = -1; + if (!ASSERT_NEQ(lsm->bss->seen, 0, "bpf LSM in the active LSM set")) + goto out; + ASSERT_EQ(lsm->bss->seen, 1, "unsigned: one observed load"); + ASSERT_EQ(lsm->bss->sig_verdict, BPF_SIG_UNSIGNED, "unsigned verdict"); + ASSERT_EQ(lsm->bss->sig_keyring_type, BPF_SIG_KEYRING_NONE, "unsigned keyring type"); + ASSERT_EQ(lsm->bss->sig_keyring_serial, 0, "unsigned: no keyring serial"); + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + goto out; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + dir = NULL; + goto out; + } + if (!ASSERT_OK(sign_buf(dir, f.gopts.insns, f.gopts.insns_sz, sig, + &sig_sz), "sign-file")) + goto out; + + map_fd = setup_meta_map(&f); + if (!ASSERT_OK_FD(map_fd, "meta_map_signed")) + goto out; + lsm->bss->seen = 0; + prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, sig, + sig_sz, KEY_SPEC_SESSION_KEYRING, 0); + close(map_fd); + map_fd = -1; + ASSERT_EQ(prog_fd, -EACCES, "unfolded metadata rejected"); + if (prog_fd >= 0) + close(prog_fd); + prog_fd = -1; + + ses_serial = syscall(__NR_keyctl, KEYCTL_GET_KEYRING_ID, + KEY_SPEC_SESSION_KEYRING, 0); + ASSERT_EQ(lsm->bss->seen, 1, "signed: one observed load"); + ASSERT_EQ(lsm->bss->sig_verdict, BPF_SIG_VERIFIED, + "admission saw a valid signature"); + ASSERT_EQ(lsm->bss->sig_keyring_type, BPF_SIG_KEYRING_USER, "signed keyring type"); + ASSERT_GT(ses_serial, 0, "session keyring serial resolved"); + ASSERT_EQ(lsm->bss->sig_keyring_serial, ses_serial, + "signed: validated against session keyring"); + + buf = malloc((size_t)f.gopts.insns_sz + f.data_sz); + if (!ASSERT_OK_PTR(buf, "meta_signbuf")) + goto out; + memcpy(buf, f.gopts.insns, f.gopts.insns_sz); + memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz); + if (!ASSERT_OK(sign_buf(dir, buf, f.gopts.insns_sz + f.data_sz, + sig, &msig_sz), "sign insns||metadata")) { + free(buf); + goto out; + } + free(buf); + + map_fd = setup_meta_map(&f); + if (!ASSERT_OK_FD(map_fd, "meta_map_bound")) + goto out; + lsm->bss->seen = 0; + prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, sig, + msig_sz, KEY_SPEC_SESSION_KEYRING, 1); + close(map_fd); + map_fd = -1; + if (!ASSERT_OK_FD(prog_fd, "metadata-bound loader load")) + goto out; + close(prog_fd); + prog_fd = -1; + ASSERT_EQ(lsm->bss->seen, 1, "metadata: one observed load"); + ASSERT_EQ(lsm->bss->sig_verdict, BPF_SIG_VERIFIED, + "metadata-bound verdict"); +out: + if (map_fd >= 0) + close(map_fd); + if (prog_fd >= 0) + close(prog_fd); + if (have_fixture) + gen_loader_fixture_fini(&f); + if (dir) + run_setup("cleanup", dir); + test_signed_loader_lsm__destroy(lsm); +} + +/* + * Load-time metadata verification: the kernel folds the frozen metadata map + * into the signature (insns || metadata) and checks it at BPF_PROG_LOAD via + * fd_array_cnt, rather than the loader checking from within BPF. Sign that + * concatenation, hand the kernel the map, and confirm the signed loader loads, + * runs, and installs its target. + */ +static int loadtime_drive(const char *dir, const void *insns, __u32 insns_sz, + const void *data, __u32 data_sz, const __u8 *excl, + void *ctx, __u32 ctx_sz, int *load_ret, bool *ran) +{ + LIBBPF_OPTS(bpf_map_create_opts, mopts, + .excl_prog_hash = excl, + .excl_prog_hash_size = SHA256_DIGEST_LENGTH); + __u32 sig_sz = 8192, key = 0; + unsigned char *buf = NULL; + int map_fd, prog_fd, ret = 0; + union bpf_attr attr; + __u8 sig[8192]; + + *ran = false; + *load_ret = 0; + + /* + * Metadata map, bound to the loader digest and frozen, exactly as + * skel_internal.h's bpf_load_and_run() sets it up. + */ + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, + data_sz, 1, &mopts); + if (map_fd < 0) { + ret = -errno; + goto out_load; + } + if (bpf_map_update_elem(map_fd, &key, data, 0) || bpf_map_freeze(map_fd)) { + ret = -errno; + goto out_load; + } + + /* Sign insns || metadata, the same bytes the kernel reconstructs. */ + buf = malloc((size_t)insns_sz + data_sz); + if (!buf) { + ret = -ENOMEM; + goto out_load; + } + memcpy(buf, insns, insns_sz); + memcpy(buf + insns_sz, data, data_sz); + ret = sign_buf(dir, buf, insns_sz + data_sz, sig, &sig_sz); + if (ret) + goto out_load; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = insns_sz / sizeof(struct bpf_insn); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(&map_fd); + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + attr.fd_array_cnt = 1; + memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + if (prog_fd < 0) { + ret = -errno; + goto out_load; + } + + memset(&attr, 0, sizeof(attr)); + attr.test.prog_fd = prog_fd; + attr.test.ctx_in = ptr_to_u64(ctx); + attr.test.ctx_size_in = ctx_sz; + if (syscall(__NR_bpf, BPF_PROG_RUN, &attr, + offsetofend(union bpf_attr, test)) < 0) { + ret = -errno; + goto out_prog; + } + *ran = true; + ret = (int)attr.test.retval; +out_prog: + close(prog_fd); + goto out_map; +out_load: + *load_ret = ret; +out_map: + free(buf); + if (map_fd >= 0) + close(map_fd); + return ret; +} + +static void loadtime_verify(struct bpf_object *obj, int expect_maps) +{ + LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true); + char dir_tmpl[] = "/tmp/signed_loader_ltXXXXXX", *dir = NULL; + int nr_maps = 0, nr_progs = 0, load_ret = 0, r; + __u8 excl[SHA256_DIGEST_LENGTH]; + struct bpf_prog_desc *pd; + struct bpf_map_desc *md; + unsigned char *blob = NULL; + struct bpf_program *p; + struct bpf_map *m; + __u32 ctx_sz, data_sz; + void *ctx = NULL; + bool ran = false; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + + if (!ASSERT_OK(bpf_object__gen_loader(obj, &gopts), "gen_loader")) + goto out; + if (!ASSERT_OK(bpf_object__load(obj), "gen_load")) + goto out; + + bpf_object__for_each_program(p, obj) + nr_progs++; + bpf_object__for_each_map(m, obj) + nr_maps++; + if (!ASSERT_EQ(nr_maps, expect_maps, "fixture map count")) + goto out; + + ctx_sz = sizeof(struct bpf_loader_ctx) + + nr_maps * sizeof(struct bpf_map_desc) + + nr_progs * sizeof(struct bpf_prog_desc); + ctx = calloc(1, ctx_sz); + if (!ASSERT_OK_PTR(ctx, "ctx_alloc")) + goto out; + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + + data_sz = gopts.data_sz; + blob = malloc(data_sz); + if (!ASSERT_OK_PTR(blob, "blob_alloc")) + goto out; + memcpy(blob, gopts.data, data_sz); + + /* excl_prog_hash = SHA256(loader insns) == the loader's prog->digest. */ + libbpf_sha256(gopts.insns, gopts.insns_sz, excl); + + r = loadtime_drive(dir, gopts.insns, gopts.insns_sz, blob, data_sz, + excl, ctx, ctx_sz, &load_ret, &ran); + ASSERT_OK(load_ret, "signed loader loaded (insns || metadata)"); + ASSERT_TRUE(ran, "loader ran"); + ASSERT_EQ(r, 0, "loader installed its target"); + + md = (struct bpf_map_desc *)((char *)ctx + sizeof(struct bpf_loader_ctx)); + pd = (struct bpf_prog_desc *)(md + nr_maps); + ASSERT_GT(pd[0].prog_fd, 0, "target program installed"); + if (nr_maps) + ASSERT_GT(md[0].map_fd, 0, "target map installed"); + + close_loader_ctx_fds(ctx, nr_maps, nr_progs); +out: + free(blob); + free(ctx); + if (dir) + run_setup("cleanup", dir); +} + +static void loadtime_no_map(void) +{ + struct test_signed_loader *skel = test_signed_loader__open(); + + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + loadtime_verify(skel->obj, 0); + test_signed_loader__destroy(skel); +} + +static void loadtime_with_map(void) +{ + struct test_signed_loader_map *skel = test_signed_loader_map__open(); + + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + loadtime_verify(skel->obj, 1); + test_signed_loader_map__destroy(skel); +} + +/* + * A signed program need not bind any map. A plain BPF_PROG_TYPE_SYSCALL + * program with no fd_array is signed over its instructions alone: the kernel + * verifies the signature, folds no metadata, and the program loads. Exercise + * the fd_array == NULL / fd_array_cnt == 0 path, and confirm the signature + * still authenticates the instructions (a tampered copy is rejected). + */ +static void signed_no_fd_array(void) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir; + __u32 sig_sz = 8192; + union bpf_attr attr; + __u8 sig[8192]; + int prog_fd, err; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + + /* No metadata map: the signed payload is the instructions alone. */ + if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + /* fd_array and fd_array_cnt deliberately left NULL/0. */ + memcpy(attr.prog_name, "signed_nomap", sizeof("signed_nomap")); + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + if (!ASSERT_GE(prog_fd, 0, "map-less signed program loaded")) { + if (prog_fd >= 0) + close(prog_fd); + goto cleanup; + } + close(prog_fd); + + /* The signature covers the instructions, so tampering must be rejected. */ + insns[0].imm = 1; + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + err = prog_fd < 0 ? -errno : prog_fd; + ASSERT_EQ(err, -EKEYREJECTED, "tampered map-less program rejected"); + if (prog_fd >= 0) + close(prog_fd); +cleanup: + run_setup("cleanup", dir); +} + +/* + * A signed program may reach maps only through fd_array indices, so the kernel + * folds (and thus attests) them. A direct BPF_PSEUDO_MAP_FD reference - a raw, + * unfolded fd baked into the signed instructions - is rejected by the verifier. + */ +static void signed_map_by_fd_rejected(void) +{ + struct bpf_insn insns[] = { + BPF_LD_MAP_FD(BPF_REG_1, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir; + __u32 sig_sz = 8192; + union bpf_attr attr; + __u8 sig[8192]; + int map_fd, prog_fd, err; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "sig_mapfd", 4, 4, 1, NULL); + if (!ASSERT_GE(map_fd, 0, "map_create")) + return; + insns[0].imm = map_fd; /* bake the raw map fd into the ld_imm64 */ + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + goto out_map; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + goto out_map; + } + + /* Sign the instructions, raw map fd and all. */ + if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + /* No fd_array: the map is reached by a raw fd in the instructions. */ + memcpy(attr.prog_name, "signed_mapfd", sizeof("signed_mapfd")); + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + err = prog_fd < 0 ? -errno : prog_fd; + ASSERT_EQ(err, -EINVAL, "signed program referencing a map by fd rejected"); + if (prog_fd >= 0) + close(prog_fd); +cleanup: + run_setup("cleanup", dir); +out_map: + close(map_fd); +} + +/* + * A signed program may reach maps only through the continuous fd_array, so the + * kernel folds (and thus attests) them. Referencing a map by fd_array *index* + * while leaving fd_array_cnt at 0 selects the sparse path, which resolves a map + * the signature never covered; the verifier rejects it up front with -EACCES. + */ +static void signed_sparse_fd_array_rejected(void) +{ + struct bpf_insn insns[] = { + BPF_LD_IMM64_RAW(BPF_REG_1, BPF_PSEUDO_MAP_IDX, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char dir_tmpl[] = "/tmp/signed_loader_spXXXXXX", *dir; + __u32 sig_sz = 8192; + union bpf_attr attr; + __u8 sig[8192]; + int map_fd, prog_fd, err; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "sig_sparse", 4, 4, 1, NULL); + if (!ASSERT_GE(map_fd, 0, "map_create")) + return; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + goto out_map; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + goto out_map; + } + + /* Sign the instructions alone; the sparse map is not folded. */ + if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(&map_fd); + attr.fd_array_cnt = 0; /* sparse: force lazy map resolution */ + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + memcpy(attr.prog_name, "signed_sparse", sizeof("signed_sparse")); + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + err = prog_fd < 0 ? -errno : prog_fd; + ASSERT_EQ(err, -EACCES, "signed program binding a sparse fd_array map rejected"); + if (prog_fd >= 0) + close(prog_fd); +cleanup: + run_setup("cleanup", dir); +out_map: + close(map_fd); +} + +static void signed_module_kfunc_rejected(void) +{ + struct bpf_insn insns[] = { + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 1, 1), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char dir_tmpl[] = "/tmp/signed_loader_kfnXXXXXX", *dir; + int prog_fd, err, fds[2]; + struct btf *btf = NULL; + __u32 sig_sz = 8192; + union bpf_attr attr; + __u8 sig[8192]; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz), + "sign-file")) + goto cleanup; + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf_new_empty")) + goto cleanup; + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf_load")) + goto cleanup; + fds[0] = -1; + fds[1] = btf__fd(btf); + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(fds); + attr.fd_array_cnt = 0; /* sparse: force lazy kfunc BTF resolution */ + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + memcpy(attr.prog_name, "signed_kfunc", sizeof("signed_kfunc")); + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + err = prog_fd < 0 ? -errno : prog_fd; + if (prog_fd >= 0) + close(prog_fd); + + ASSERT_EQ(err, -EACCES, "module kfunc BTF in signed program rejected"); +cleanup: + if (btf) + btf__free(btf); + run_setup("cleanup", dir); +} + +void test_signed_loader(void) +{ + if (test__start_subtest("loadtime_no_map")) + loadtime_no_map(); + if (test__start_subtest("loadtime_with_map")) + loadtime_with_map(); + if (test__start_subtest("metadata_match")) + metadata_match(); + if (test__start_subtest("signature_enforced")) + signature_enforced(); + if (test__start_subtest("signed_nonexcl_fd_array_rejected")) + signed_nonexcl_fd_array_rejected(); + if (test__start_subtest("signed_unfrozen_fd_array_rejected")) + signed_unfrozen_fd_array_rejected(); + if (test__start_subtest("signed_nonarray_fd_array_rejected")) + signed_nonarray_fd_array_rejected(); + if (test__start_subtest("signed_btf_fd_array_rejected")) + signed_btf_fd_array_rejected(); + if (test__start_subtest("signed_module_kfunc_rejected")) + signed_module_kfunc_rejected(); + if (test__start_subtest("signature_failure_logs")) + signature_failure_logs(); + if (test__start_subtest("signature_too_large")) + signature_too_large(); + if (test__start_subtest("signature_zero_size")) + signature_zero_size(); + if (test__start_subtest("signature_bad_keyring")) + signature_bad_keyring(); + if (test__start_subtest("metadata_ctx_max_entries_ignored")) + metadata_ctx_max_entries_ignored(); + if (test__start_subtest("metadata_ctx_initial_value_ignored")) + metadata_ctx_initial_value_ignored(); + if (test__start_subtest("signature_authenticates_insns")) + signature_authenticates_insns(); + if (test__start_subtest("signature_authenticates_metadata")) + signature_authenticates_metadata(); + if (test__start_subtest("hash_requires_frozen")) + hash_requires_frozen(); + if (test__start_subtest("no_update_after_freeze")) + no_update_after_freeze(); + if (test__start_subtest("freeze_writable_mmap")) + freeze_writable_mmap(); + if (test__start_subtest("no_writable_mmap_frozen")) + no_writable_mmap_frozen(); + if (test__start_subtest("map_hash_matches_libbpf")) + map_hash_matches_libbpf(); + if (test__start_subtest("map_hash_multi_element")) + map_hash_multi_element(); + if (test__start_subtest("map_hash_bad_size")) + map_hash_bad_size(); + if (test__start_subtest("map_hash_unsupported_type")) + map_hash_unsupported_type(); + if (test__start_subtest("lsm_signature_verdict")) + lsm_signature_verdict(); + if (test__start_subtest("signed_no_fd_array")) + signed_no_fd_array(); + if (test__start_subtest("signed_map_by_fd_rejected")) + signed_map_by_fd_rejected(); + if (test__start_subtest("signed_sparse_fd_array_rejected")) + signed_sparse_fd_array_rejected(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/sleepable_tracepoints.c b/tools/testing/selftests/bpf/prog_tests/sleepable_tracepoints.c new file mode 100644 index 000000000000..19500b785ee3 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/sleepable_tracepoints.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ + +#include <test_progs.h> +#include <unistd.h> +#include "test_sleepable_tracepoints.skel.h" +#include "test_sleepable_tracepoints_fail.skel.h" + +static void run_test(struct test_sleepable_tracepoints *skel) +{ + char buf[PATH_MAX] = "/"; + + skel->bss->target_pid = getpid(); + skel->bss->prog_triggered = 0; + skel->bss->err = 0; + skel->bss->copied_byte = 0; + + syscall(__NR_getcwd, buf, sizeof(buf)); + + ASSERT_EQ(skel->bss->prog_triggered, 1, "prog_triggered"); + ASSERT_EQ(skel->bss->err, 0, "err"); + ASSERT_EQ(skel->bss->copied_byte, '/', "copied_byte"); +} + +static void run_auto_attach_test(struct bpf_program *prog, + struct test_sleepable_tracepoints *skel) +{ + struct bpf_link *link; + + link = bpf_program__attach(prog); + if (!ASSERT_OK_PTR(link, "prog_attach")) + return; + + run_test(skel); + bpf_link__destroy(link); +} + +static void test_attach_only(struct bpf_program *prog) +{ + struct bpf_link *link; + + link = bpf_program__attach(prog); + if (ASSERT_OK_PTR(link, "attach")) + bpf_link__destroy(link); +} + +static void test_attach_reject(struct bpf_program *prog) +{ + struct bpf_link *link; + + link = bpf_program__attach(prog); + if (!ASSERT_ERR_PTR(link, "attach_should_fail")) + bpf_link__destroy(link); +} + +static void test_raw_tp_bare(struct test_sleepable_tracepoints *skel) +{ + struct bpf_link *link; + + link = bpf_program__attach_raw_tracepoint(skel->progs.handle_raw_tp_bare, + "sys_enter"); + if (ASSERT_OK_PTR(link, "attach")) + bpf_link__destroy(link); +} + +static void test_tp_bare(struct test_sleepable_tracepoints *skel) +{ + struct bpf_link *link; + + link = bpf_program__attach_tracepoint(skel->progs.handle_tp_bare, + "syscalls", "sys_enter_getcwd"); + if (ASSERT_OK_PTR(link, "attach")) + bpf_link__destroy(link); +} + +static void test_test_run(struct test_sleepable_tracepoints *skel) +{ + __u64 args[2] = {0x1234ULL, 0x5678ULL}; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .ctx_in = args, + .ctx_size_in = sizeof(args), + ); + int fd, err; + + fd = bpf_program__fd(skel->progs.handle_test_run); + err = bpf_prog_test_run_opts(fd, &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, args[0] + args[1], "test_run_retval"); +} + +static void test_test_run_on_cpu_reject(struct test_sleepable_tracepoints *skel) +{ + __u64 args[2] = {}; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .ctx_in = args, + .ctx_size_in = sizeof(args), + .flags = BPF_F_TEST_RUN_ON_CPU, + ); + int fd, err; + + fd = bpf_program__fd(skel->progs.handle_test_run); + err = bpf_prog_test_run_opts(fd, &topts); + ASSERT_ERR(err, "test_run_on_cpu_reject"); +} + +void test_sleepable_tracepoints(void) +{ + struct test_sleepable_tracepoints *skel; + + skel = test_sleepable_tracepoints__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + if (test__start_subtest("tp_btf")) + run_auto_attach_test(skel->progs.handle_sys_enter_tp_btf, skel); + if (test__start_subtest("raw_tp")) + run_auto_attach_test(skel->progs.handle_sys_enter_raw_tp, skel); + if (test__start_subtest("tracepoint")) + run_auto_attach_test(skel->progs.handle_sys_enter_tp, skel); + if (test__start_subtest("sys_exit")) + run_auto_attach_test(skel->progs.handle_sys_exit_tp, skel); + if (test__start_subtest("tracepoint_alias")) + test_attach_only(skel->progs.handle_sys_enter_tp_alias); + if (test__start_subtest("raw_tracepoint_alias")) + test_attach_only(skel->progs.handle_sys_enter_raw_tp_alias); + if (test__start_subtest("raw_tp_bare")) + test_raw_tp_bare(skel); + if (test__start_subtest("tp_bare")) + test_tp_bare(skel); + if (test__start_subtest("test_run")) + test_test_run(skel); + if (test__start_subtest("test_run_on_cpu_reject")) + test_test_run_on_cpu_reject(skel); + if (test__start_subtest("raw_tp_non_faultable")) + test_attach_reject(skel->progs.handle_raw_tp_non_faultable); + if (test__start_subtest("tp_non_syscall")) + test_attach_reject(skel->progs.handle_tp_non_syscall); + if (test__start_subtest("tp_btf_non_faultable_reject")) + RUN_TESTS(test_sleepable_tracepoints_fail); + + test_sleepable_tracepoints__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/snprintf.c b/tools/testing/selftests/bpf/prog_tests/snprintf.c index 594441acb707..4e4a82d54f79 100644 --- a/tools/testing/selftests/bpf/prog_tests/snprintf.c +++ b/tools/testing/selftests/bpf/prog_tests/snprintf.c @@ -114,7 +114,8 @@ static void test_snprintf_negative(void) ASSERT_ERR(load_single_snprintf("%--------"), "invalid specifier 5"); ASSERT_ERR(load_single_snprintf("%lc"), "invalid specifier 6"); ASSERT_ERR(load_single_snprintf("%llc"), "invalid specifier 7"); - ASSERT_ERR(load_single_snprintf("\x80"), "non ascii character"); + ASSERT_OK(load_single_snprintf("\x80"), "non ascii plain text"); + ASSERT_ERR(load_single_snprintf("%\x80"), "non ascii in specifier"); ASSERT_ERR(load_single_snprintf("\x1"), "non printable character"); ASSERT_ERR(load_single_snprintf("%p%"), "invalid specifier 8"); ASSERT_ERR(load_single_snprintf("%s%"), "invalid specifier 9"); diff --git a/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c b/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c index dd41b826be30..edce9c1b54fb 100644 --- a/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c +++ b/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> #include <linux/btf.h> +#include <bpf/btf.h> #include "netif_receive_skb.skel.h" +#include "snprintf_btf_void.skel.h" /* Demonstrate that bpf_snprintf_btf succeeds and that various data types * are formatted correctly. @@ -58,3 +60,80 @@ void serial_test_snprintf_btf(void) cleanup: netif_receive_skb__destroy(skel); } + +/* + * bpf_snprintf_btf() renders a type_id taken straight from the vmlinux BTF. + * Two such type_ids used to NULL-deref in the BTF show path: + * - a "const void" (a modifier resolving to void) in btf_modifier_show() + * - a BTF_KIND_VAR in btf_var_show() (base BTF has no resolved_ids) + * A fixed kernel renders both without crashing. + */ +static long run(struct snprintf_btf_void *skel, __u32 type_id) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + char ctx[8] = {}; + + skel->bss->type_id = type_id; + topts.ctx_in = ctx; + topts.ctx_size_in = sizeof(ctx); + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.dump_type), + &topts), "test_run")) + return -1; + return skel->bss->ret; +} + +void test_snprintf_btf_void(void) +{ + const struct btf_type *t; + struct snprintf_btf_void *skel; + int i, n, cv = 0, var = 0; + struct btf *btf; + + btf = btf__parse("/sys/kernel/btf/vmlinux", NULL); + if (!btf) { + test__skip(); + return; + } + + skel = snprintf_btf_void__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + goto out_btf; + + n = btf__type_cnt(btf); + for (i = 1; i < n && !(cv && var); i++) { + t = btf__type_by_id(btf, i); + if (!cv && btf_kind(t) == BTF_KIND_CONST && t->type == 0) + cv = i; + /* Pick a VAR small enough to render from the program's buffer. */ + if (!var && btf_kind(t) == BTF_KIND_VAR) { + long sz = btf__resolve_size(btf, t->type); + + if (sz > 0 && sz <= (long)sizeof(skel->bss->obj)) + var = i; + } + } + + /* "const void" renders the "<unsupported kind:0>" placeholder. */ + if (test__start_subtest("const_void")) { + if (cv) { + ASSERT_EQ(run(skel, cv), + sizeof("<unsupported kind:0>") - 1, "ret"); + ASSERT_STREQ(skel->bss->out, "<unsupported kind:0>", + "placeholder"); + } else { + test__skip(); + } + } + + /* A BTF_KIND_VAR must resolve and render without error. */ + if (test__start_subtest("var")) { + if (var) + ASSERT_GT(run(skel, var), 0, "ret"); + else + test__skip(); + } + + snprintf_btf_void__destroy(skel); +out_btf: + btf__free(btf); +} diff --git a/tools/testing/selftests/bpf/prog_tests/sock_destroy.c b/tools/testing/selftests/bpf/prog_tests/sock_destroy.c index 9c11938fe597..78d642a02bdb 100644 --- a/tools/testing/selftests/bpf/prog_tests/sock_destroy.c +++ b/tools/testing/selftests/bpf/prog_tests/sock_destroy.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include <poll.h> #include <test_progs.h> #include <bpf/bpf_endian.h> @@ -110,6 +111,122 @@ cleanup: close(serv); } +static void test_tcp_listen_pending(struct sock_destroy_prog *skel) +{ + int serv = -1, clien = -1, accept_serv = -1, n, serv_port; + struct pollfd pfd = { .events = POLLIN }; + char buf[1]; + + serv = start_server(AF_INET6, SOCK_STREAM, NULL, 0, 0); + if (!ASSERT_GE(serv, 0, "start_server")) + goto cleanup; + serv_port = get_socket_local_port(serv); + if (!ASSERT_GE(serv_port, 0, "get_sock_local_port")) + goto cleanup; + skel->bss->serv_port = (__be16)serv_port; + + /* + * Connect but never accept, so the child sits in the accept queue + * of the listener. Wait until it's actually there. + */ + clien = connect_to_fd(serv, 0); + if (!ASSERT_GE(clien, 0, "connect_to_fd")) + goto cleanup; + pfd.fd = serv; + if (!ASSERT_EQ(poll(&pfd, 1, -1), 1, "poll listener")) + goto cleanup; + + /* Run iterator program that destroys server sockets. */ + start_iter_sockets(skel->progs.iter_tcp6_server); + + accept_serv = accept(serv, NULL, NULL); + if (!ASSERT_LT(accept_serv, 0, "accept on destroyed listener")) + goto cleanup; + ASSERT_EQ(errno, EINVAL, "error code on destroyed listener"); + + /* The unaccepted child was reset along with the listener. */ + n = recv(clien, buf, sizeof(buf), 0); + if (!ASSERT_LT(n, 0, "client recv on reset child")) + goto cleanup; + ASSERT_EQ(errno, ECONNRESET, "error code on reset child"); + +cleanup: + if (clien != -1) + close(clien); + if (accept_serv != -1) + close(accept_serv); + if (serv != -1) + close(serv); +} + +static void test_tcp_timewait(struct sock_destroy_prog *skel) +{ + int serv = -1, clien = -1, accept_serv = -1, n; + struct timeval tv = {}; + char buf[1]; + + serv = start_server(AF_INET6, SOCK_STREAM, NULL, 0, 0); + if (!ASSERT_GE(serv, 0, "start_server")) + goto cleanup; + + clien = connect_to_fd(serv, 0); + if (!ASSERT_GE(clien, 0, "connect_to_fd")) + goto cleanup; + + accept_serv = accept(serv, NULL, NULL); + if (!ASSERT_GE(accept_serv, 0, "serv accept")) + goto cleanup; + + /* + * Active close from the client, then close the server side. Once + * recv() sees EOF the server FIN has been processed and the client + * sock is in TIME_WAIT. Block without timeout so a loaded CI box + * can't race us. + */ + if (!ASSERT_OK(setsockopt(clien, SOL_SOCKET, SO_RCVTIMEO, &tv, + sizeof(tv)), "clear rcvtimeo")) + goto cleanup; + if (!ASSERT_OK(shutdown(clien, SHUT_WR), "client shutdown")) + goto cleanup; + + /* + * Make sure the server has seen the client FIN before it closes, + * so the two FINs never cross. + */ + n = recv(accept_serv, buf, sizeof(buf), 0); + if (!ASSERT_EQ(n, 0, "server recv EOF")) + goto cleanup; + + close(accept_serv); + accept_serv = -1; + + /* block until return EOF */ + n = recv(clien, buf, sizeof(buf), 0); + if (!ASSERT_EQ(n, 0, "client recv EOF")) + goto cleanup; + + /* Run iterator program that destroys the timewait client sock. */ + skel->bss->tw_found = 0; + start_iter_sockets(skel->progs.iter_tcp6_timewait); + if (!ASSERT_EQ(skel->bss->tw_found, 1, "timewait sock found")) + goto cleanup; + + ASSERT_OK(skel->bss->tw_destroy_err, "destroy timewait sock"); + + /* The destroyed timewait sock must be gone. */ + skel->bss->tw_found = 0; + start_iter_sockets(skel->progs.iter_tcp6_timewait); + ASSERT_EQ(skel->bss->tw_found, 0, "timewait sock destroyed"); + +cleanup: + if (clien != -1) + close(clien); + if (accept_serv != -1) + close(accept_serv); + if (serv != -1) + close(serv); +} + static void test_udp_client(struct sock_destroy_prog *skel) { int serv = -1, clien = -1, n = 0; @@ -204,6 +321,10 @@ void test_sock_destroy(void) test_tcp_client(skel); if (test__start_subtest("tcp_server")) test_tcp_server(skel); + if (test__start_subtest("tcp_listen_pending")) + test_tcp_listen_pending(skel); + if (test__start_subtest("tcp_timewait")) + test_tcp_timewait(skel); if (test__start_subtest("udp_client")) test_udp_client(skel); if (test__start_subtest("udp_server")) diff --git a/tools/testing/selftests/bpf/prog_tests/sock_ops_get_sk.c b/tools/testing/selftests/bpf/prog_tests/sock_ops_get_sk.c new file mode 100644 index 000000000000..343d92c4df30 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/sock_ops_get_sk.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "cgroup_helpers.h" +#include "network_helpers.h" +#include "sock_ops_get_sk.skel.h" + +/* See progs/sock_ops_get_sk.c for the bug description. */ +static void run_sock_ops_test(int cgroup_fd, int prog_fd) +{ + int server_fd, client_fd, err; + + err = bpf_prog_attach(prog_fd, cgroup_fd, BPF_CGROUP_SOCK_OPS, 0); + if (!ASSERT_OK(err, "prog_attach")) + return; + + server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0); + if (!ASSERT_OK_FD(server_fd, "start_server")) + goto detach; + + /* Trigger TCP handshake which causes TCP_NEW_SYN_RECV state where + * is_fullsock == 0 and is_locked_tcp_sock == 0. + */ + client_fd = connect_to_fd(server_fd, 0); + if (!ASSERT_OK_FD(client_fd, "connect_to_fd")) + goto close_server; + + close(client_fd); + +close_server: + close(server_fd); +detach: + bpf_prog_detach(cgroup_fd, BPF_CGROUP_SOCK_OPS); +} + +void test_ns_sock_ops_get_sk(void) +{ + struct sock_ops_get_sk *skel; + int cgroup_fd; + + cgroup_fd = test__join_cgroup("/sock_ops_get_sk"); + if (!ASSERT_OK_FD(cgroup_fd, "join_cgroup")) + return; + + skel = sock_ops_get_sk__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_load")) + goto close_cgroup; + + /* Test SOCK_OPS_GET_SK with same src/dst register */ + if (test__start_subtest("get_sk")) { + run_sock_ops_test(cgroup_fd, + bpf_program__fd(skel->progs.sock_ops_get_sk_same_reg)); + ASSERT_EQ(skel->bss->null_seen, 1, "null_seen"); + ASSERT_EQ(skel->bss->bug_detected, 0, "bug_not_detected"); + } + + /* Test SOCK_OPS_GET_FIELD with same src/dst register */ + if (test__start_subtest("get_field")) { + run_sock_ops_test(cgroup_fd, + bpf_program__fd(skel->progs.sock_ops_get_field_same_reg)); + ASSERT_EQ(skel->bss->field_null_seen, 1, "field_null_seen"); + ASSERT_EQ(skel->bss->field_bug_detected, 0, "field_bug_not_detected"); + } + + /* Test SOCK_OPS_GET_SK with different src/dst register */ + if (test__start_subtest("get_sk_diff_reg")) { + run_sock_ops_test(cgroup_fd, + bpf_program__fd(skel->progs.sock_ops_get_sk_diff_reg)); + ASSERT_EQ(skel->bss->diff_reg_null_seen, 1, "diff_reg_null_seen"); + ASSERT_EQ(skel->bss->diff_reg_bug_detected, 0, "diff_reg_bug_not_detected"); + } + + sock_ops_get_sk__destroy(skel); +close_cgroup: + close(cgroup_fd); +} diff --git a/tools/testing/selftests/bpf/prog_tests/sock_xattr.c b/tools/testing/selftests/bpf/prog_tests/sock_xattr.c new file mode 100644 index 000000000000..b5816e90f01a --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/sock_xattr.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Christian Brauner */ + +#include <errno.h> +#include <string.h> +#include <unistd.h> +#include <sys/xattr.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <test_progs.h> + +#include "sock_read_xattr.skel.h" + +static const char xattr_value[] = "bpf_sock_value"; +static const char xattr_name[] = "user.bpf_test"; + +static void test_read_sock_xattr(void) +{ + struct sockaddr_in addr = {}; + struct sock_read_xattr *skel = NULL; + struct bpf_link *link = NULL; + int sock_fd = -1, err; + + sock_fd = socket(AF_INET, SOCK_STREAM, 0); + if (!ASSERT_OK_FD(sock_fd, "socket")) + return; + + err = fsetxattr(sock_fd, xattr_name, xattr_value, sizeof(xattr_value), 0); + if (!ASSERT_OK(err, "fsetxattr")) + goto out; + + skel = sock_read_xattr__open_and_load(); + if (!ASSERT_OK_PTR(skel, "sock_read_xattr__open_and_load")) + goto out; + + skel->bss->monitored_pid = sys_gettid(); + + /* Only attach the functional program; the verifier-only programs + * above are not pid-gated and would clobber the shared globals. + */ + link = bpf_program__attach(skel->progs.read_sock_xattr); + if (!ASSERT_OK_PTR(link, "attach read_sock_xattr")) + goto out; + + addr.sin_family = AF_INET; + addr.sin_port = htons(1234); + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + /* Only the lsm/socket_connect hook matters; the connect may fail. */ + connect(sock_fd, (struct sockaddr *)&addr, sizeof(addr)); + + ASSERT_EQ(skel->data->read_ret, sizeof(xattr_value), "read_ret"); + ASSERT_STREQ(skel->bss->value, xattr_value, "value"); + +out: + bpf_link__destroy(link); + if (sock_fd >= 0) + close(sock_fd); + sock_read_xattr__destroy(skel); +} + +void test_sock_xattr(void) +{ + RUN_TESTS(sock_read_xattr); + + if (test__start_subtest("read_sock_xattr")) + test_read_sock_xattr(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c index dd3c757859f6..1fef6ec2ba7a 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c @@ -7,13 +7,13 @@ #include "test_progs.h" #include "test_skmsg_load_helpers.skel.h" -#include "test_sockmap_update.skel.h" #include "test_sockmap_invalid_update.skel.h" #include "test_sockmap_skb_verdict_attach.skel.h" #include "test_sockmap_progs_query.skel.h" #include "test_sockmap_pass_prog.skel.h" #include "test_sockmap_drop_prog.skel.h" #include "test_sockmap_change_tail.skel.h" +#include "test_sockmap_msg_pop_data.skel.h" #include "bpf_iter_sockmap.skel.h" #include "sockmap_helpers.h" @@ -234,53 +234,6 @@ out: test_skmsg_load_helpers__destroy(skel); } -static void test_sockmap_update(enum bpf_map_type map_type) -{ - int err, prog, src; - struct test_sockmap_update *skel; - struct bpf_map *dst_map; - const __u32 zero = 0; - char dummy[14] = {0}; - LIBBPF_OPTS(bpf_test_run_opts, topts, - .data_in = dummy, - .data_size_in = sizeof(dummy), - .repeat = 1, - ); - __s64 sk; - - sk = connected_socket_v4(); - if (!ASSERT_NEQ(sk, -1, "connected_socket_v4")) - return; - - skel = test_sockmap_update__open_and_load(); - if (!ASSERT_OK_PTR(skel, "open_and_load")) - goto close_sk; - - prog = bpf_program__fd(skel->progs.copy_sock_map); - src = bpf_map__fd(skel->maps.src); - if (map_type == BPF_MAP_TYPE_SOCKMAP) - dst_map = skel->maps.dst_sock_map; - else - dst_map = skel->maps.dst_sock_hash; - - err = bpf_map_update_elem(src, &zero, &sk, BPF_NOEXIST); - if (!ASSERT_OK(err, "update_elem(src)")) - goto out; - - err = bpf_prog_test_run_opts(prog, &topts); - if (!ASSERT_OK(err, "test_run")) - goto out; - if (!ASSERT_NEQ(topts.retval, 0, "test_run retval")) - goto out; - - compare_cookies(skel->maps.src, dst_map); - -out: - test_sockmap_update__destroy(skel); -close_sk: - close(sk); -} - static void test_sockmap_invalid_update(void) { struct test_sockmap_invalid_update *skel; @@ -666,6 +619,51 @@ out: test_sockmap_change_tail__destroy(skel); } +static void test_sockmap_msg_verdict_pop_data(void) +{ + struct test_sockmap_msg_pop_data *skel; + int err, map, verdict; + int c1 = -1, p1 = -1, sent; + int zero = 0; + char *buf; + const size_t len = 32 * 1024; + + skel = test_sockmap_msg_pop_data__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + verdict = bpf_program__fd(skel->progs.prog_msg_pop_data); + map = bpf_map__fd(skel->maps.sock_map); + + err = bpf_prog_attach(verdict, map, BPF_SK_MSG_VERDICT, 0); + if (!ASSERT_OK(err, "bpf_prog_attach")) + goto out; + + err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1); + if (!ASSERT_OK(err, "create_pair")) + goto out; + + err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); + if (!ASSERT_OK(err, "bpf_map_update_elem")) + goto out_close; + + buf = calloc(len, 1); + if (!ASSERT_OK_PTR(buf, "calloc")) + goto out_close; + + sent = xsend(c1, buf, len, 0); + ASSERT_EQ(sent, (ssize_t)len, "xsend"); + ASSERT_EQ(skel->data->pop_data_ret, -EINVAL, "pop_data_rejects overflow"); + + free(buf); + +out_close: + close(c1); + close(p1); +out: + test_sockmap_msg_pop_data__destroy(skel); +} + static void test_sockmap_skb_verdict_peek_helper(int map) { int err, c1, p1, zero = 0, sent, recvd, avail; @@ -807,7 +805,7 @@ static void test_sockmap_many_socket(void) return; } - udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); + udp = socket_loopback(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK); if (udp < 0) { close(dgram); close(tcp); @@ -876,7 +874,7 @@ static void test_sockmap_many_maps(void) return; } - udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); + udp = socket_loopback(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK); if (udp < 0) { close(dgram); close(tcp); @@ -947,7 +945,7 @@ static void test_sockmap_same_sock(void) return; } - udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); + udp = socket_loopback(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK); if (udp < 0) { close(dgram); close(tcp); @@ -1298,10 +1296,23 @@ static void test_sockmap_multi_channels(int sotype) avail = wait_for_fionread(p1, expected, IO_TIMEOUT_SEC); ASSERT_EQ(avail, expected, "ioctl(FIONREAD) full return"); - recvd = recv_timeout(p1, rcv, sizeof(rcv), MSG_DONTWAIT, 1); - if (!ASSERT_EQ(recvd, sizeof(buf), "recv_timeout(p1)") || + recvd = recv_timeout(p1, rcv, expected, MSG_DONTWAIT, 1); + if (!ASSERT_EQ(recvd, expected, "recv_timeout(p1)") || !ASSERT_OK(memcmp(buf, rcv, recvd), "data mismatch")) goto end; + + /* process remaining data for udp if secondary data is available */ + expected = sizeof(buf) - expected; + if (expected) { + avail = wait_for_fionread(p1, expected, IO_TIMEOUT_SEC); + ASSERT_EQ(avail, expected, "second ioctl(FIONREAD) full return"); + + recvd = recv_timeout(p1, rcv, expected, MSG_DONTWAIT, 1); + if (!ASSERT_EQ(recvd, expected, "second recv_timeout(p1)") || + !ASSERT_OK(memcmp(buf + sizeof(buf) - expected, rcv, recvd), + "second data mismatch")) + goto end; + } end: if (c0 >= 0) close(c0); @@ -1314,6 +1325,43 @@ end: test_sockmap_pass_prog__destroy(skel); } +/* A socket in a sockmap without a verdict program keeps its ingress data + * in sk_receive_queue: FIONREAD must account for it. + */ +static void test_sockmap_no_verdict_fionread(void) +{ + int err, map, zero = 0, sent, avail; + int c0 = -1, c1 = -1, p0 = -1, p1 = -1; + struct test_sockmap_pass_prog *skel; + char buf[256] = "0123456789"; + + skel = test_sockmap_pass_prog__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + map = bpf_map__fd(skel->maps.sock_map_rx); + + err = create_socket_pairs(AF_INET, SOCK_STREAM, &c0, &c1, &p0, &p1); + if (!ASSERT_OK(err, "create_socket_pairs()")) + goto out; + + err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); + if (!ASSERT_OK(err, "bpf_map_update_elem(c1)")) + goto out_close; + + sent = xsend(p1, &buf, sizeof(buf), 0); + ASSERT_EQ(sent, sizeof(buf), "xsend(p1)"); + avail = wait_for_fionread(c1, sizeof(buf), IO_TIMEOUT_SEC); + ASSERT_EQ(avail, sizeof(buf), "ioctl(FIONREAD)"); + +out_close: + close(c0); + close(p0); + close(c1); + close(p1); +out: + test_sockmap_pass_prog__destroy(skel); +} + void test_sockmap_basic(void) { if (test__start_subtest("sockmap create_update_free")) @@ -1326,10 +1374,6 @@ void test_sockmap_basic(void) test_skmsg_helpers(BPF_MAP_TYPE_SOCKMAP); if (test__start_subtest("sockhash sk_msg load helpers")) test_skmsg_helpers(BPF_MAP_TYPE_SOCKHASH); - if (test__start_subtest("sockmap update")) - test_sockmap_update(BPF_MAP_TYPE_SOCKMAP); - if (test__start_subtest("sockhash update")) - test_sockmap_update(BPF_MAP_TYPE_SOCKHASH); if (test__start_subtest("sockmap update in unsafe context")) test_sockmap_invalid_update(); if (test__start_subtest("sockmap copy")) @@ -1356,10 +1400,14 @@ void test_sockmap_basic(void) test_sockmap_skb_verdict_shutdown(); if (test__start_subtest("sockmap skb_verdict fionread")) test_sockmap_skb_verdict_fionread(true); + if (test__start_subtest("sockmap no_verdict fionread")) + test_sockmap_no_verdict_fionread(); if (test__start_subtest("sockmap skb_verdict fionread on drop")) test_sockmap_skb_verdict_fionread(false); if (test__start_subtest("sockmap skb_verdict change tail")) test_sockmap_skb_verdict_change_tail(); + if (test__start_subtest("sockmap msg_verdict pop_data overflow")) + test_sockmap_msg_verdict_pop_data(); if (test__start_subtest("sockmap skb_verdict msg_f_peek")) test_sockmap_skb_verdict_peek(); if (test__start_subtest("sockmap skb_verdict msg_f_peek with link")) diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c b/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c index b87e7f39e15a..34737e8df6ea 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c @@ -9,7 +9,6 @@ #include "test_progs.h" #include "sockmap_helpers.h" #include "test_skmsg_load_helpers.skel.h" -#include "test_sockmap_ktls.skel.h" #define MAX_TEST_NAME 80 #define TCP_ULP 31 @@ -117,6 +116,68 @@ close: close(s); } +static void test_sockmap_ktls_enable_fails_when_in_sockmap(int family, int map) +{ + struct tls12_crypto_info_aes_gcm_128 crypto = { + .info = { + .version = TLS_1_2_VERSION, + .cipher_type = TLS_CIPHER_AES_GCM_128, + }, + }; + struct sockaddr_storage addr = {}; + socklen_t len = sizeof(addr); + struct sockaddr_in6 *v6; + struct sockaddr_in *v4; + int err, s, zero = 0; + + switch (family) { + case AF_INET: + v4 = (struct sockaddr_in *)&addr; + v4->sin_family = AF_INET; + break; + case AF_INET6: + v6 = (struct sockaddr_in6 *)&addr; + v6->sin6_family = AF_INET6; + break; + default: + PRINT_FAIL("unsupported socket family %d", family); + return; + } + + s = socket(family, SOCK_STREAM, 0); + if (!ASSERT_GE(s, 0, "socket")) + return; + + err = bind(s, (struct sockaddr *)&addr, len); + if (!ASSERT_OK(err, "bind")) + goto close; + + err = getsockname(s, (struct sockaddr *)&addr, &len); + if (!ASSERT_OK(err, "getsockname")) + goto close; + + err = connect(s, (struct sockaddr *)&addr, len); + if (!ASSERT_OK(err, "connect")) + goto close; + + /* Add the socket to the sockmap, attaching a psock. */ + err = bpf_map_update_elem(map, &zero, &s, BPF_ANY); + if (!ASSERT_OK(err, "sockmap update elem")) + goto close; + + /* Installing the TLS ULP is allowed, it does not touch the datapath. */ + err = setsockopt(s, IPPROTO_TCP, TCP_ULP, "tls", strlen("tls")); + if (!ASSERT_OK(err, "setsockopt(TCP_ULP)")) + goto close; + + /* Enabling the TLS crypto datapath must be rejected. */ + err = setsockopt(s, SOL_TLS, TLS_TX, &crypto, sizeof(crypto)); + ASSERT_ERR(err, "setsockopt(TLS_TX)"); + +close: + close(s); +} + static const char *fmt_test_name(const char *subtest_name, int family, enum bpf_map_type map_type) { @@ -160,249 +221,6 @@ out: close(p); } -static void test_sockmap_ktls_tx_cork(int family, int sotype, bool push) -{ - int err, off; - int i, j; - int start_push = 0, push_len = 0; - int c = 0, p = 0, one = 1, sent, recvd; - int prog_fd, map_fd; - char msg[12] = "hello world\0"; - char rcv[20] = {0}; - struct test_sockmap_ktls *skel; - - skel = test_sockmap_ktls__open_and_load(); - if (!ASSERT_TRUE(skel, "open ktls skel")) - return; - - err = create_pair(family, sotype, &c, &p); - if (!ASSERT_OK(err, "create_pair()")) - goto out; - - prog_fd = bpf_program__fd(skel->progs.prog_sk_policy); - map_fd = bpf_map__fd(skel->maps.sock_map); - - err = bpf_prog_attach(prog_fd, map_fd, BPF_SK_MSG_VERDICT, 0); - if (!ASSERT_OK(err, "bpf_prog_attach sk msg")) - goto out; - - err = bpf_map_update_elem(map_fd, &one, &c, BPF_NOEXIST); - if (!ASSERT_OK(err, "bpf_map_update_elem(c)")) - goto out; - - err = init_ktls_pairs(c, p); - if (!ASSERT_OK(err, "init_ktls_pairs(c, p)")) - goto out; - - skel->bss->cork_byte = sizeof(msg); - if (push) { - start_push = 1; - push_len = 2; - } - skel->bss->push_start = start_push; - skel->bss->push_end = push_len; - - off = sizeof(msg) / 2; - sent = send(c, msg, off, 0); - if (!ASSERT_EQ(sent, off, "send(msg)")) - goto out; - - recvd = recv_timeout(p, rcv, sizeof(rcv), MSG_DONTWAIT, 1); - if (!ASSERT_EQ(-1, recvd, "expected no data")) - goto out; - - /* send remaining msg */ - sent = send(c, msg + off, sizeof(msg) - off, 0); - if (!ASSERT_EQ(sent, sizeof(msg) - off, "send remaining data")) - goto out; - - recvd = recv_timeout(p, rcv, sizeof(rcv), MSG_DONTWAIT, 1); - if (!ASSERT_OK(err, "recv(msg)") || - !ASSERT_EQ(recvd, sizeof(msg) + push_len, "check length mismatch")) - goto out; - - for (i = 0, j = 0; i < recvd;) { - /* skip checking the data that has been pushed in */ - if (i >= start_push && i <= start_push + push_len - 1) { - i++; - continue; - } - if (!ASSERT_EQ(rcv[i], msg[j], "data mismatch")) - goto out; - i++; - j++; - } -out: - if (c) - close(c); - if (p) - close(p); - test_sockmap_ktls__destroy(skel); -} - -static void test_sockmap_ktls_tx_no_buf(int family, int sotype, bool push) -{ - int c = -1, p = -1, one = 1, two = 2; - struct test_sockmap_ktls *skel; - unsigned char *data = NULL; - struct msghdr msg = {0}; - struct iovec iov[2]; - int prog_fd, map_fd; - int txrx_buf = 1024; - int iov_length = 8192; - int err; - - skel = test_sockmap_ktls__open_and_load(); - if (!ASSERT_TRUE(skel, "open ktls skel")) - return; - - err = create_pair(family, sotype, &c, &p); - if (!ASSERT_OK(err, "create_pair()")) - goto out; - - err = setsockopt(c, SOL_SOCKET, SO_RCVBUFFORCE, &txrx_buf, sizeof(int)); - err |= setsockopt(p, SOL_SOCKET, SO_SNDBUFFORCE, &txrx_buf, sizeof(int)); - if (!ASSERT_OK(err, "set buf limit")) - goto out; - - prog_fd = bpf_program__fd(skel->progs.prog_sk_policy_redir); - map_fd = bpf_map__fd(skel->maps.sock_map); - - err = bpf_prog_attach(prog_fd, map_fd, BPF_SK_MSG_VERDICT, 0); - if (!ASSERT_OK(err, "bpf_prog_attach sk msg")) - goto out; - - err = bpf_map_update_elem(map_fd, &one, &c, BPF_NOEXIST); - if (!ASSERT_OK(err, "bpf_map_update_elem(c)")) - goto out; - - err = bpf_map_update_elem(map_fd, &two, &p, BPF_NOEXIST); - if (!ASSERT_OK(err, "bpf_map_update_elem(p)")) - goto out; - - skel->bss->apply_bytes = 1024; - - err = init_ktls_pairs(c, p); - if (!ASSERT_OK(err, "init_ktls_pairs(c, p)")) - goto out; - - data = calloc(iov_length, sizeof(char)); - if (!data) - goto out; - - iov[0].iov_base = data; - iov[0].iov_len = iov_length; - iov[1].iov_base = data; - iov[1].iov_len = iov_length; - msg.msg_iov = iov; - msg.msg_iovlen = 2; - - for (;;) { - err = sendmsg(c, &msg, MSG_DONTWAIT); - if (err <= 0) - break; - } - -out: - if (data) - free(data); - if (c != -1) - close(c); - if (p != -1) - close(p); - - test_sockmap_ktls__destroy(skel); -} - -static void test_sockmap_ktls_tx_pop(int family, int sotype) -{ - char msg[37] = "0123456789abcdefghijklmnopqrstuvwxyz\0"; - int c = 0, p = 0, one = 1, sent, recvd; - struct test_sockmap_ktls *skel; - int prog_fd, map_fd; - char rcv[50] = {0}; - int err; - int i, m, r; - - skel = test_sockmap_ktls__open_and_load(); - if (!ASSERT_TRUE(skel, "open ktls skel")) - return; - - err = create_pair(family, sotype, &c, &p); - if (!ASSERT_OK(err, "create_pair()")) - goto out; - - prog_fd = bpf_program__fd(skel->progs.prog_sk_policy); - map_fd = bpf_map__fd(skel->maps.sock_map); - - err = bpf_prog_attach(prog_fd, map_fd, BPF_SK_MSG_VERDICT, 0); - if (!ASSERT_OK(err, "bpf_prog_attach sk msg")) - goto out; - - err = bpf_map_update_elem(map_fd, &one, &c, BPF_NOEXIST); - if (!ASSERT_OK(err, "bpf_map_update_elem(c)")) - goto out; - - err = init_ktls_pairs(c, p); - if (!ASSERT_OK(err, "init_ktls_pairs(c, p)")) - goto out; - - struct { - int pop_start; - int pop_len; - } pop_policy[] = { - /* trim the start */ - {0, 2}, - {0, 10}, - {1, 2}, - {1, 10}, - /* trim the end */ - {35, 2}, - /* New entries should be added before this line */ - {-1, -1}, - }; - - i = 0; - while (pop_policy[i].pop_start >= 0) { - skel->bss->pop_start = pop_policy[i].pop_start; - skel->bss->pop_end = pop_policy[i].pop_len; - - sent = send(c, msg, sizeof(msg), 0); - if (!ASSERT_EQ(sent, sizeof(msg), "send(msg)")) - goto out; - - recvd = recv_timeout(p, rcv, sizeof(rcv), MSG_DONTWAIT, 1); - if (!ASSERT_EQ(recvd, sizeof(msg) - pop_policy[i].pop_len, "pop len mismatch")) - goto out; - - /* verify the data - * msg: 0123456789a bcdefghij klmnopqrstuvwxyz - * | | - * popped data - */ - for (m = 0, r = 0; m < sizeof(msg);) { - /* skip checking the data that has been popped */ - if (m >= pop_policy[i].pop_start && - m <= pop_policy[i].pop_start + pop_policy[i].pop_len - 1) { - m++; - continue; - } - - if (!ASSERT_EQ(msg[m], rcv[r], "data mismatch")) - goto out; - m++; - r++; - } - i++; - } -out: - if (c) - close(c); - if (p) - close(p); - test_sockmap_ktls__destroy(skel); -} - static void run_tests(int family, enum bpf_map_type map_type) { int map; @@ -414,6 +232,9 @@ static void run_tests(int family, enum bpf_map_type map_type) if (test__start_subtest(fmt_test_name("update_fails_when_sock_has_ulp", family, map_type))) test_sockmap_ktls_update_fails_when_sock_has_ulp(family, map); + if (test__start_subtest(fmt_test_name("enable_fails_when_in_sockmap", family, map_type))) + test_sockmap_ktls_enable_fails_when_in_sockmap(family, map); + close(map); } @@ -421,14 +242,6 @@ static void run_ktls_test(int family, int sotype) { if (test__start_subtest("tls simple offload")) test_sockmap_ktls_offload(family, sotype); - if (test__start_subtest("tls tx cork")) - test_sockmap_ktls_tx_cork(family, sotype, false); - if (test__start_subtest("tls tx cork with push")) - test_sockmap_ktls_tx_cork(family, sotype, true); - if (test__start_subtest("tls tx egress with no buf")) - test_sockmap_ktls_tx_no_buf(family, sotype, true); - if (test__start_subtest("tls tx with pop")) - test_sockmap_ktls_tx_pop(family, sotype); } void test_sockmap_ktls(void) diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c index cc0c68bab907..1c96a3cf4b97 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c @@ -53,8 +53,8 @@ static void test_insert_opened(struct test_sockmap_listen *skel __always_unused, int family, int sotype, int mapfd) { u32 key = 0; - u64 value; int err, s; + u64 value; s = xsocket(family, sotype, 0); if (s == -1) @@ -63,11 +63,8 @@ static void test_insert_opened(struct test_sockmap_listen *skel __always_unused, errno = 0; value = s; err = bpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); - if (sotype == SOCK_STREAM) { - if (!err || errno != EOPNOTSUPP) - FAIL_ERRNO("map_update: expected EOPNOTSUPP"); - } else if (err) - FAIL_ERRNO("map_update: expected success"); + ASSERT_ERR(err, "map_update"); + ASSERT_EQ(errno, EOPNOTSUPP, "errno"); xclose(s); } @@ -77,8 +74,8 @@ static void test_insert_bound(struct test_sockmap_listen *skel __always_unused, struct sockaddr_storage addr; socklen_t len = 0; u32 key = 0; - u64 value; int err, s; + u64 value; init_addr_loopback(family, &addr, &len); @@ -93,8 +90,12 @@ static void test_insert_bound(struct test_sockmap_listen *skel __always_unused, errno = 0; value = s; err = bpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); - if (!err || errno != EOPNOTSUPP) - FAIL_ERRNO("map_update: expected EOPNOTSUPP"); + if (sotype == SOCK_STREAM) { + ASSERT_ERR(err, "map_update"); + ASSERT_EQ(errno, EOPNOTSUPP, "errno"); + } else { + ASSERT_OK(err, "map_update"); + } close: xclose(s); } @@ -1289,7 +1290,7 @@ static void test_ops(struct test_sockmap_listen *skel, struct bpf_map *map, /* insert */ TEST(test_insert_invalid), TEST(test_insert_opened), - TEST(test_insert_bound, SOCK_STREAM), + TEST(test_insert_bound), TEST(test_insert), /* delete */ TEST(test_delete_after_insert), diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_strp.c b/tools/testing/selftests/bpf/prog_tests/sockmap_strp.c index 621b3b71888e..1d7231728eaf 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_strp.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_strp.c @@ -431,6 +431,35 @@ out: test_sockmap_strp__destroy(strp); } +static void test_sockmap_strp_parser_reject(void) +{ + struct test_sockmap_strp *strp = NULL; + int parser_mod, parser_ro, link; + int err, map; + + strp = test_sockmap_strp__open_and_load(); + if (!ASSERT_OK_PTR(strp, "test_sockmap_strp__open_and_load")) + return; + + map = bpf_map__fd(strp->maps.sock_map); + parser_mod = bpf_program__fd(strp->progs.prog_skb_parser_resize); + parser_ro = bpf_program__fd(strp->progs.prog_skb_parser); + + err = bpf_prog_attach(parser_mod, map, BPF_SK_SKB_STREAM_PARSER, 0); + ASSERT_ERR(err, "bpf_prog_attach parser_mod"); + + link = bpf_link_create(parser_ro, map, BPF_SK_SKB_STREAM_PARSER, NULL); + if (!ASSERT_GE(link, 0, "bpf_link_create parser_ro")) + goto out; + + err = bpf_link_update(link, parser_mod, NULL); + ASSERT_ERR(err, "bpf_link_update parser_mod"); +out: + if (link >= 0) + close(link); + test_sockmap_strp__destroy(strp); +} + void test_sockmap_strp(void) { if (test__start_subtest("sockmap strp tcp pass")) @@ -451,4 +480,6 @@ void test_sockmap_strp(void) test_sockmap_strp_multiple_pkt(AF_INET, SOCK_STREAM); if (test__start_subtest("sockmap strp tcp dispatch")) test_sockmap_strp_dispatch_pkt(AF_INET, SOCK_STREAM); + if (test__start_subtest("sockmap strp parser reject pkt mod")) + test_sockmap_strp_parser_reject(); } diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt.c b/tools/testing/selftests/bpf/prog_tests/sockopt.c index eaac83a7f388..6c96f2d9fccf 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockopt.c +++ b/tools/testing/selftests/bpf/prog_tests/sockopt.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include <linux/tcp.h> #include <test_progs.h> #include <io_uring/mini_liburing.h> #include "cgroup_helpers.h" @@ -284,6 +285,27 @@ static struct sockopt_test { .io_uring_support = true, }, { + .descr = "getsockopt: deny negative ctx->optlen in TCP_ZEROCOPY_RECEIVE", + .insns = { + /* ctx->optlen = -1 */ + BPF_MOV64_IMM(BPF_REG_0, -1), + BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, + offsetof(struct bpf_sockopt, optlen)), + + /* return 1 */ + BPF_MOV64_IMM(BPF_REG_0, 1), + BPF_EXIT_INSN(), + }, + .attach_type = BPF_CGROUP_GETSOCKOPT, + .expected_attach_type = BPF_CGROUP_GETSOCKOPT, + + .get_level = IPPROTO_TCP, + .get_optname = TCP_ZEROCOPY_RECEIVE, + .get_optlen = sizeof(struct tcp_zerocopy_receive), + + .error = EFAULT_GETSOCKOPT, + }, + { .descr = "getsockopt: ignore >PAGE_SIZE optlen", .insns = { /* write 0xFF to the first optval byte */ diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c b/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c index 53637431ec5d..3a41c517b918 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c +++ b/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c @@ -190,7 +190,7 @@ static int getsetsockopt(void) fd = socket(AF_NETLINK, SOCK_RAW, 0); if (fd < 0) { log_err("Failed to create AF_NETLINK socket"); - return -1; + goto err; } buf.u32 = 1; @@ -211,6 +211,21 @@ static int getsetsockopt(void) } ASSERT_EQ(optlen, 8, "Unexpected NETLINK_LIST_MEMBERSHIPS value"); + /* Trick bpf_tcp_sock() with IPPROTO_TCP */ + close(fd); + fd = socket(AF_INET, SOCK_RAW, IPPROTO_TCP); + if (!ASSERT_OK_FD(fd, "socket")) + goto err; + + /* The BPF prog intercepts this before the kernel sees it, any + * optlen works. Go with 4 bytes for simplicity. + */ + buf.u32 = 1; + optlen = sizeof(buf.u32); + err = setsockopt(fd, SOL_TCP, TCP_SAVED_SYN, &buf, optlen); + if (!ASSERT_ERR(err, "setsockopt(TCP_SAVED_SYN)")) + goto err; + free(big_buf); close(fd); return 0; diff --git a/tools/testing/selftests/bpf/prog_tests/spin_lock.c b/tools/testing/selftests/bpf/prog_tests/spin_lock.c index 254fbfeab06a..e368370262c8 100644 --- a/tools/testing/selftests/bpf/prog_tests/spin_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/spin_lock.c @@ -13,8 +13,9 @@ static struct { const char *err_msg; } spin_lock_fail_tests[] = { { "lock_id_kptr_preserve", - "5: (bf) r1 = r0 ; R0=ptr_foo(id=2,ref_obj_id=2) " - "R1=ptr_foo(id=2,ref_obj_id=2) refs=2\n6: (85) call bpf_this_cpu_ptr#154\n" + "[0-9]\\+: (bf) r1 = r0 ; R0=ptr_foo(id=2)" + " R1=ptr_foo(id=2) refs=2\n" + "[0-9]\\+: (85) call bpf_this_cpu_ptr#154\n" "R1 type=ptr_ expected=percpu_ptr_" }, { "lock_id_global_zero", "; R1=map_value(map=.data.A,ks=4,vs=4)\n2: (85) call bpf_this_cpu_ptr#154\n" @@ -53,6 +54,8 @@ static struct { { "lock_global_sleepable_helper_subprog", "global function calls are not allowed while holding a lock" }, { "lock_global_sleepable_kfunc_subprog", "global function calls are not allowed while holding a lock" }, { "lock_global_sleepable_subprog_indirect", "global function calls are not allowed while holding a lock" }, + { "callback_value_lock_identity", "bpf_spin_unlock of different lock" }, + { "callback_inner_map_value_lock_identity", "bpf_spin_unlock of different lock" }, }; static int match_regex(const char *pattern, const char *string) diff --git a/tools/testing/selftests/bpf/prog_tests/stack_arg.c b/tools/testing/selftests/bpf/prog_tests/stack_arg.c new file mode 100644 index 000000000000..57193543f260 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/stack_arg.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <test_progs.h> +#include <network_helpers.h> +#include "stack_arg.skel.h" +#include "stack_arg_kfunc.skel.h" + +static void run_subtest(struct bpf_program *prog, int expected) +{ + int err, prog_fd; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .repeat = 1, + ); + + prog_fd = bpf_program__fd(prog); + err = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, expected, "retval"); +} + +static void test_global_many(void) +{ + struct stack_arg *skel; + + skel = stack_arg__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + if (!skel->rodata->has_stack_arg) { + test__skip(); + goto out; + } + + if (!ASSERT_OK(stack_arg__load(skel), "load")) + goto out; + + run_subtest(skel->progs.test_global_many_args, 55); + +out: + stack_arg__destroy(skel); +} + +static void test_async_cb_many(void) +{ + struct stack_arg *skel; + + skel = stack_arg__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + if (!skel->rodata->has_stack_arg) { + test__skip(); + goto out; + } + + if (!ASSERT_OK(stack_arg__load(skel), "load")) + goto out; + + run_subtest(skel->progs.test_async_cb_many_args, 0); + + /* Wait for the timer callback to fire and verify the result. + * 10+20+30+40+50+60+70+80+90+100 = 550 + */ + usleep(50); + ASSERT_EQ(skel->bss->timer_result, 550, "timer_result"); + +out: + stack_arg__destroy(skel); +} + +static void test_bpf2bpf(void) +{ + struct stack_arg *skel; + + skel = stack_arg__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + if (!skel->rodata->has_stack_arg) { + test__skip(); + goto out; + } + + if (!ASSERT_OK(stack_arg__load(skel), "load")) + goto out; + + run_subtest(skel->progs.test_bpf2bpf_ptr_stack_arg, 75); + run_subtest(skel->progs.test_bpf2bpf_mix_stack_args, 66); + run_subtest(skel->progs.test_bpf2bpf_nesting_stack_arg, 84); + run_subtest(skel->progs.test_bpf2bpf_dynptr_stack_arg, 99); + run_subtest(skel->progs.test_two_callees, 133); + +out: + stack_arg__destroy(skel); +} + +static void test_kfunc(void) +{ + struct stack_arg_kfunc *skel; + + skel = stack_arg_kfunc__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + if (!skel->rodata->has_stack_arg) { + test__skip(); + goto out; + } + + if (!ASSERT_OK(stack_arg_kfunc__load(skel), "load")) + goto out; + + run_subtest(skel->progs.test_stack_arg_scalar, 55); + run_subtest(skel->progs.test_stack_arg_ptr, 75); + run_subtest(skel->progs.test_stack_arg_mix, 66); + run_subtest(skel->progs.test_stack_arg_dynptr, 99); + run_subtest(skel->progs.test_stack_arg_mem, 151); + run_subtest(skel->progs.test_stack_arg_iter, 145); + run_subtest(skel->progs.test_stack_arg_const_str, 45); + run_subtest(skel->progs.test_stack_arg_timer, 45); + +out: + stack_arg_kfunc__destroy(skel); +} + +void test_stack_arg(void) +{ + if (test__start_subtest("global_many_args")) + test_global_many(); + if (test__start_subtest("async_cb_many_args")) + test_async_cb_many(); + if (test__start_subtest("bpf2bpf")) + test_bpf2bpf(); + if (test__start_subtest("kfunc")) + test_kfunc(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/stack_arg_fail.c b/tools/testing/selftests/bpf/prog_tests/stack_arg_fail.c new file mode 100644 index 000000000000..090af1330953 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/stack_arg_fail.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <test_progs.h> +#include "stack_arg_fail.skel.h" + +void test_stack_arg_fail(void) +{ + RUN_TESTS(stack_arg_fail); +} diff --git a/tools/testing/selftests/bpf/prog_tests/stack_arg_precision.c b/tools/testing/selftests/bpf/prog_tests/stack_arg_precision.c new file mode 100644 index 000000000000..1ab041d66de3 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/stack_arg_precision.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <test_progs.h> +#include "stack_arg_precision.skel.h" + +void test_stack_arg_precision(void) +{ + RUN_TESTS(stack_arg_precision); +} diff --git a/tools/testing/selftests/bpf/prog_tests/stream.c b/tools/testing/selftests/bpf/prog_tests/stream.c index c3cce5c292bd..e4e9374309e2 100644 --- a/tools/testing/selftests/bpf/prog_tests/stream.c +++ b/tools/testing/selftests/bpf/prog_tests/stream.c @@ -103,6 +103,12 @@ void test_stream_arena_fault_address(void) test_address(skel->progs.stream_arena_read_fault, &skel->bss->fault_addr); if (test__start_subtest("write_fault")) test_address(skel->progs.stream_arena_write_fault, &skel->bss->fault_addr); + if (test__start_subtest("load_acquire_fault")) + test_address(skel->progs.stream_arena_load_acquire_fault, &skel->bss->fault_addr); + if (test__start_subtest("xchg_fault")) + test_address(skel->progs.stream_arena_xchg_fault, &skel->bss->fault_addr); + if (test__start_subtest("cmpxchg_fault")) + test_address(skel->progs.stream_arena_cmpxchg_fault, &skel->bss->fault_addr); stream__destroy(skel); } diff --git a/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c b/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c index d42123a0fb16..98db9bafa44b 100644 --- a/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c +++ b/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c @@ -5,6 +5,7 @@ #include "struct_ops_private_stack_fail.skel.h" #include "struct_ops_private_stack_recur.skel.h" +#if defined(__x86_64__) || defined(__aarch64__) || defined(__powerpc64__) static void test_private_stack(void) { struct struct_ops_private_stack *skel; @@ -15,11 +16,6 @@ static void test_private_stack(void) if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack__open")) return; - if (skel->data->skip) { - test__skip(); - goto cleanup; - } - err = struct_ops_private_stack__load(skel); if (!ASSERT_OK(err, "struct_ops_private_stack__load")) goto cleanup; @@ -48,15 +44,9 @@ static void test_private_stack_fail(void) if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack_fail__open")) return; - if (skel->data->skip) { - test__skip(); - goto cleanup; - } - err = struct_ops_private_stack_fail__load(skel); ASSERT_ERR(err, "struct_ops_private_stack_fail__load"); -cleanup: struct_ops_private_stack_fail__destroy(skel); } @@ -70,11 +60,6 @@ static void test_private_stack_recur(void) if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack_recur__open")) return; - if (skel->data->skip) { - test__skip(); - goto cleanup; - } - err = struct_ops_private_stack_recur__load(skel); if (!ASSERT_OK(err, "struct_ops_private_stack_recur__load")) goto cleanup; @@ -93,7 +78,7 @@ cleanup: struct_ops_private_stack_recur__destroy(skel); } -void test_struct_ops_private_stack(void) +static void __test_struct_ops_private_stack(void) { if (test__start_subtest("private_stack")) test_private_stack(); @@ -102,3 +87,14 @@ void test_struct_ops_private_stack(void) if (test__start_subtest("private_stack_recur")) test_private_stack_recur(); } +#else +static void __test_struct_ops_private_stack(void) +{ + test__skip(); +} +#endif + +void test_struct_ops_private_stack(void) +{ + __test_struct_ops_private_stack(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/summarization.c b/tools/testing/selftests/bpf/prog_tests/summarization.c index 5dd6c120a838..6951786044ca 100644 --- a/tools/testing/selftests/bpf/prog_tests/summarization.c +++ b/tools/testing/selftests/bpf/prog_tests/summarization.c @@ -58,7 +58,7 @@ static void test_aux(const char *main_prog_name, * this particular combination can be enabled. */ if (!strcmp("might_sleep", replacement) && err) { - ASSERT_HAS_SUBSTR(log, "helper call might sleep in a non-sleepable prog", "error log"); + ASSERT_HAS_SUBSTR(log, "sleepable helper bpf_copy_from_user#", "error log"); ASSERT_EQ(err, -EINVAL, "err"); test__skip(); goto out; diff --git a/tools/testing/selftests/bpf/prog_tests/tailcalls.c b/tools/testing/selftests/bpf/prog_tests/tailcalls.c index 7d534fde0af9..c5c9d6c359bb 100644 --- a/tools/testing/selftests/bpf/prog_tests/tailcalls.c +++ b/tools/testing/selftests/bpf/prog_tests/tailcalls.c @@ -8,7 +8,13 @@ #include "tailcall_freplace.skel.h" #include "tc_bpf2bpf.skel.h" #include "tailcall_fail.skel.h" +#include "tailcall_cgrp_storage_owner.skel.h" +#include "tailcall_cgrp_storage_no_storage.skel.h" +#include "tailcall_cgrp_storage.skel.h" #include "tailcall_sleepable.skel.h" +#include "tailcall_callback.skel.h" +#include "tailcall_bpf2bpf2.skel.h" +#include "tailcall_bpf2bpf_fexit.skel.h" /* test_tailcall_1 checks basic functionality by patching multiple locations * in a single program for a single tail call slot with nop->jmp, jmp->nop @@ -1654,6 +1660,179 @@ static void test_tailcall_failure() RUN_TESTS(tailcall_fail); } +static void test_tailcall_cgrp_storage(void) +{ + struct tailcall_cgrp_storage_owner *owner_skel = NULL; + struct tailcall_cgrp_storage *skel = NULL; + int err, key = 0, prog_array_fd, prog_fd, storage_map_fd; + + owner_skel = tailcall_cgrp_storage_owner__open_and_load(); + if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load")) + return; + + prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array); + storage_map_fd = bpf_map__fd(owner_skel->maps.storage_map); + + skel = tailcall_cgrp_storage__open(); + if (!ASSERT_OK_PTR(skel, "tailcall_cgrp_storage__open")) + goto out; + + err = bpf_map__reuse_fd(skel->maps.prog_array, prog_array_fd); + if (!ASSERT_OK(err, "reuse_prog_array")) + goto out; + + err = bpf_map__reuse_fd(skel->maps.storage_map, storage_map_fd); + if (!ASSERT_OK(err, "reuse_storage_map")) + goto out; + + err = bpf_object__load(skel->obj); + if (!ASSERT_OK(err, "tailcall_cgrp_storage__load")) + goto out; + + prog_fd = bpf_program__fd(skel->progs.callee_prog); + err = bpf_map_update_elem(prog_array_fd, &key, &prog_fd, BPF_ANY); + ASSERT_OK(err, "update_prog_array"); +out: + tailcall_cgrp_storage__destroy(skel); + tailcall_cgrp_storage_owner__destroy(owner_skel); +} + +static void test_tailcall_cgrp_storage_diff_storage(void) +{ + struct tailcall_cgrp_storage_owner *owner_skel = NULL; + struct tailcall_cgrp_storage *skel = NULL; + int err, prog_array_fd; + + owner_skel = tailcall_cgrp_storage_owner__open_and_load(); + if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load")) + return; + + prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array); + + skel = tailcall_cgrp_storage__open(); + if (!ASSERT_OK_PTR(skel, "tailcall_cgrp_storage__open")) + goto out; + + err = bpf_map__reuse_fd(skel->maps.prog_array, prog_array_fd); + if (!ASSERT_OK(err, "reuse_prog_array")) + goto out; + + err = bpf_object__load(skel->obj); + ASSERT_ERR(err, "tailcall_cgrp_storage__load"); +out: + tailcall_cgrp_storage__destroy(skel); + tailcall_cgrp_storage_owner__destroy(owner_skel); +} + +static void test_tailcall_cgrp_storage_no_storage(void) +{ + struct tailcall_cgrp_storage_owner *owner_skel = NULL; + struct tailcall_cgrp_storage_no_storage *skel = NULL; + int err, prog_array_fd; + + owner_skel = tailcall_cgrp_storage_owner__open_and_load(); + if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load")) + return; + + prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array); + + skel = tailcall_cgrp_storage_no_storage__open(); + if (!ASSERT_OK_PTR(skel, "tailcall_cgrp_storage_no_storage__open")) + goto out; + + err = bpf_map__reuse_fd(skel->maps.prog_array, prog_array_fd); + if (!ASSERT_OK(err, "reuse_prog_array")) + goto out; + + err = bpf_object__load(skel->obj); + ASSERT_ERR(err, "tailcall_cgrp_storage_no_storage__load"); +out: + tailcall_cgrp_storage_no_storage__destroy(skel); + tailcall_cgrp_storage_owner__destroy(owner_skel); +} + +static void test_tailcall_cgrp_storage_no_storage_leaf(void) +{ + struct tailcall_cgrp_storage_owner *owner_skel = NULL; + struct tailcall_cgrp_storage_no_storage *skel = NULL; + int err, key = 0, prog_array_fd, prog_fd; + + owner_skel = tailcall_cgrp_storage_owner__open_and_load(); + if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load")) + return; + + prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array); + + skel = tailcall_cgrp_storage_no_storage__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tailcall_cgrp_storage_no_storage__open_and_load")) + goto out; + + prog_fd = bpf_program__fd(skel->progs.leaf_prog); + err = bpf_map_update_elem(prog_array_fd, &key, &prog_fd, BPF_ANY); + if (!ASSERT_OK(err, "update_prog_array_leaf")) + goto out; + + prog_fd = bpf_program__fd(skel->progs.caller_prog); + err = bpf_map_update_elem(prog_array_fd, &key, &prog_fd, BPF_ANY); + ASSERT_ERR(err, "update_prog_array_bridge"); +out: + tailcall_cgrp_storage_no_storage__destroy(skel); + tailcall_cgrp_storage_owner__destroy(owner_skel); +} + +static void test_tailcall_cgrp_storage_no_storage_bridge(void) +{ + struct tailcall_cgrp_storage_owner *owner_skel = NULL; + struct tailcall_cgrp_storage_no_storage *bridge_skel = NULL; + struct tailcall_cgrp_storage *callee_skel = NULL; + int err, key = 0, prog_array_fd, prog_fd, storage_map_fd; + + owner_skel = tailcall_cgrp_storage_owner__open_and_load(); + if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load")) + return; + + prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array); + storage_map_fd = bpf_map__fd(owner_skel->maps.storage_map); + + callee_skel = tailcall_cgrp_storage__open(); + if (!ASSERT_OK_PTR(callee_skel, "tailcall_cgrp_storage__open")) + goto out; + + bpf_program__set_autoload(callee_skel->progs.caller_prog, false); + + err = bpf_map__reuse_fd(callee_skel->maps.prog_array, prog_array_fd); + if (!ASSERT_OK(err, "reuse_prog_array")) + goto out; + + err = bpf_map__reuse_fd(callee_skel->maps.storage_map, storage_map_fd); + if (!ASSERT_OK(err, "reuse_storage_map")) + goto out; + + err = bpf_object__load(callee_skel->obj); + if (!ASSERT_OK(err, "tailcall_cgrp_storage__load")) + goto out; + + prog_fd = bpf_program__fd(callee_skel->progs.callee_prog); + err = bpf_map_update_elem(prog_array_fd, &key, &prog_fd, BPF_ANY); + if (!ASSERT_OK(err, "update_prog_array")) + goto out; + + bridge_skel = tailcall_cgrp_storage_no_storage__open(); + if (!ASSERT_OK_PTR(bridge_skel, "tailcall_cgrp_storage_no_storage__open")) + goto out; + + err = bpf_map__reuse_fd(bridge_skel->maps.prog_array, prog_array_fd); + if (!ASSERT_OK(err, "reuse_prog_array")) + goto out; + + err = bpf_object__load(bridge_skel->obj); + ASSERT_ERR(err, "tailcall_cgrp_storage_no_storage_bridge__load"); +out: + tailcall_cgrp_storage_no_storage__destroy(bridge_skel); + tailcall_cgrp_storage__destroy(callee_skel); + tailcall_cgrp_storage_owner__destroy(owner_skel); +} + noinline void uprobe_sleepable_trigger(void) { asm volatile (""); @@ -1725,6 +1904,55 @@ out: tailcall_sleepable__destroy(skel); } +static void test_tailcall_callback(void) +{ + RUN_TESTS(tailcall_callback); +} + +static void test_tailcall_bpf2bpf_fexit_links(void) +{ + struct tailcall_bpf2bpf_fexit *skel1 = NULL, *skel2 = NULL; + struct tailcall_bpf2bpf2 *skel_tc; + int err, prog_fd; + + skel_tc = tailcall_bpf2bpf2__open_and_load(); + if (!ASSERT_OK_PTR(skel_tc, "tailcall_bpf2bpf2__open_and_load")) + return; + + skel1 = tailcall_bpf2bpf_fexit__open(); + if (!ASSERT_OK_PTR(skel1, "tailcall_bpf2bpf_fexit__open")) + goto out; + + prog_fd = bpf_program__fd(skel_tc->progs.classifier_0); + err = bpf_program__set_attach_target(skel1->progs.fexit, prog_fd, "subprog_tail"); + if (!ASSERT_OK(err, "bpf_program__set_attach_target")) + goto out; + + err = tailcall_bpf2bpf_fexit__load(skel1); + if (!ASSERT_OK(err, "tailcall_bpf2bpf_fexit__load")) + goto out; + + skel1->links.fexit = bpf_program__attach_trace(skel1->progs.fexit); + if (!ASSERT_OK_PTR(skel1->links.fexit, "bpf_program__attach_trace")) + goto out; + + skel2 = tailcall_bpf2bpf_fexit__open(); + if (!ASSERT_OK_PTR(skel2, "tailcall_bpf2bpf_fexit__open")) + goto out; + + err = bpf_program__set_attach_target(skel2->progs.fexit, prog_fd, "subprog_tail"); + if (!ASSERT_OK(err, "bpf_program__set_attach_target")) + goto out; + + err = tailcall_bpf2bpf_fexit__load(skel2); + ASSERT_OK(err, "tailcall_bpf2bpf_fexit__load"); + +out: + tailcall_bpf2bpf_fexit__destroy(skel1); + tailcall_bpf2bpf_fexit__destroy(skel2); + tailcall_bpf2bpf2__destroy(skel_tc); +} + void test_tailcalls(void) { if (test__start_subtest("tailcall_1")) @@ -1781,4 +2009,17 @@ void test_tailcalls(void) test_tailcall_failure(); if (test__start_subtest("tailcall_sleepable")) test_tailcall_sleepable(); + if (test__start_subtest("tailcall_cgrp_storage")) + test_tailcall_cgrp_storage(); + if (test__start_subtest("tailcall_cgrp_storage_diff_storage")) + test_tailcall_cgrp_storage_diff_storage(); + if (test__start_subtest("tailcall_cgrp_storage_no_storage")) + test_tailcall_cgrp_storage_no_storage(); + if (test__start_subtest("tailcall_cgrp_storage_no_storage_leaf")) + test_tailcall_cgrp_storage_no_storage_leaf(); + if (test__start_subtest("tailcall_cgrp_storage_no_storage_bridge")) + test_tailcall_cgrp_storage_no_storage_bridge(); + test_tailcall_callback(); + if (test__start_subtest("tailcall_bpf2bpf_fexit_links")) + test_tailcall_bpf2bpf_fexit_links(); } diff --git a/tools/testing/selftests/bpf/prog_tests/task_kfunc.c b/tools/testing/selftests/bpf/prog_tests/task_kfunc.c index 83b90335967a..30d403028f98 100644 --- a/tools/testing/selftests/bpf/prog_tests/task_kfunc.c +++ b/tools/testing/selftests/bpf/prog_tests/task_kfunc.c @@ -68,6 +68,36 @@ cleanup: task_kfunc_success__destroy(skel); } +static void run_syscall_success_test(const char *prog_name) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct task_kfunc_success *skel; + struct bpf_program *prog; + int err; + + skel = open_load_task_kfunc_skel(); + if (!ASSERT_OK_PTR(skel, "open_load_skel")) + return; + + if (!ASSERT_OK(skel->bss->err, "pre_run_err")) + goto cleanup; + + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) + goto cleanup; + if (!ASSERT_EQ(opts.retval, 0, "retval")) + goto cleanup; + + ASSERT_OK(skel->bss->err, "post_run_err"); + +cleanup: + task_kfunc_success__destroy(skel); +} + static int run_vpid_test(void *prog_name) { struct task_kfunc_success *skel; @@ -140,17 +170,28 @@ static const char * const success_tests[] = { "test_task_acquire_release_argument", "test_task_acquire_release_current", "test_task_acquire_leave_in_map", - "test_task_xchg_release", "test_task_map_acquire_release", "test_task_current_acquire_release", "test_task_from_pid_arg", "test_task_from_pid_current", "test_task_from_pid_invalid", "task_kfunc_acquire_trusted_walked", + "task_kfunc_acquire_after_spin_unlock_non_sleepable", + "task_kfunc_acquire_after_spin_unlock_explicit_rcu", + "task_kfunc_acquire_after_spin_unlock_preempt_disabled", + "task_kfunc_acquire_after_spin_unlock_irq_disabled", + "task_kfunc_acquire_after_rcu_unlock_preempt_disabled", + "task_kfunc_acquire_after_rcu_unlock_irq_disabled", + "task_kfunc_acquire_after_preempt_enable_explicit_rcu", + "task_kfunc_acquire_after_irq_restore_explicit_rcu", "test_task_kfunc_flavor_relo", "test_task_kfunc_flavor_relo_not_found", }; +static const char * const syscall_success_tests[] = { + "test_task_xchg_release", +}; + static const char * const vpid_success_tests[] = { "test_task_from_vpid_current", "test_task_from_vpid_invalid", @@ -167,6 +208,13 @@ void test_task_kfunc(void) run_success_test(success_tests[i]); } + for (i = 0; i < ARRAY_SIZE(syscall_success_tests); i++) { + if (!test__start_subtest(syscall_success_tests[i])) + continue; + + run_syscall_success_test(syscall_success_tests[i]); + } + for (i = 0; i < ARRAY_SIZE(vpid_success_tests); i++) { if (!test__start_subtest(vpid_success_tests[i])) continue; diff --git a/tools/testing/selftests/bpf/prog_tests/task_local_data.h b/tools/testing/selftests/bpf/prog_tests/task_local_data.h index 8342e2fe5260..8ae4fb2027f7 100644 --- a/tools/testing/selftests/bpf/prog_tests/task_local_data.h +++ b/tools/testing/selftests/bpf/prog_tests/task_local_data.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ #ifndef __TASK_LOCAL_DATA_H #define __TASK_LOCAL_DATA_H @@ -22,14 +22,17 @@ /* * OPTIONS * - * Define the option before including the header + * Define the option before including the header. Using different options in + * different translation units is strongly discouraged. * * TLD_FREE_DATA_ON_THREAD_EXIT - Frees memory on thread exit automatically * * Thread-specific memory for storing TLD is allocated lazily on the first call to * tld_get_data(). The thread that calls it must also call tld_free() on thread exit * to prevent memory leak. Pthread will be included if the option is defined. A pthread - * key will be registered with a destructor that calls tld_free(). + * key will be registered with a destructor that calls tld_free(). Enabled only when + * the option is defined and TLD_DEFINE_KEY/tld_create_key() is called in the same + * translation unit. * * * TLD_DYN_DATA_SIZE - The maximum size of memory allocated for TLDs created dynamically @@ -47,19 +50,16 @@ * TLD_NAME_LEN - The maximum length of the name of a TLD (default: 62) * * Setting TLD_NAME_LEN will affect the maximum number of TLDs a process can store, - * TLD_MAX_DATA_CNT. + * TLD_MAX_DATA_CNT. Must be consistent with task_local_data.bpf.h. * * - * TLD_DATA_USE_ALIGNED_ALLOC - Always use aligned_alloc() instead of malloc() + * TLD_DONT_ROUND_UP_DATA_SIZE - Don't round up memory size allocated for data if + * the memory allocator has low overhead aligned_alloc() implementation. * - * When allocating the memory for storing TLDs, we need to make sure there is a memory - * region of the X bytes within a page. This is due to the limit posed by UPTR: memory - * pinned to the kernel cannot exceed a page nor can it cross the page boundary. The - * library normally calls malloc(2*X) given X bytes of total TLDs, and only uses - * aligned_alloc(PAGE_SIZE, X) when X >= PAGE_SIZE / 2. This is to reduce memory wastage - * as not all memory allocator can use the exact amount of memory requested to fulfill - * aligned_alloc(). For example, some may round the size up to the alignment. Enable the - * option to always use aligned_alloc() if the implementation has low memory overhead. + * For some memory allocators, when calling aligned_alloc(alignment, size), size + * does not need to be an integral multiple of alignment and it can be fulfilled + * without using round_up(size, alignment) bytes of memory. Enable this option to + * reduce memory usage. */ #define TLD_PAGE_SIZE getpagesize() @@ -68,7 +68,7 @@ #define TLD_ROUND_MASK(x, y) ((__typeof__(x))((y) - 1)) #define TLD_ROUND_UP(x, y) ((((x) - 1) | TLD_ROUND_MASK(x, y)) + 1) -#define TLD_READ_ONCE(x) (*(volatile typeof(x) *)&(x)) +#define TLD_ROUND_UP_POWER_OF_TWO(x) (1UL << (sizeof(x) * 8 - __builtin_clzl(x - 1))) #ifndef TLD_DYN_DATA_SIZE #define TLD_DYN_DATA_SIZE 64 @@ -90,7 +90,7 @@ typedef struct { struct tld_metadata { char name[TLD_NAME_LEN]; - _Atomic __u16 size; + _Atomic __u16 size; /* size of tld_data_u->data */ }; struct tld_meta_u { @@ -99,27 +99,34 @@ struct tld_meta_u { struct tld_metadata metadata[]; }; +/* + * The unused field ensures map_val.start > 0. On the BPF side, __tld_fetch_key() + * calculates off by summing map_val.start and tld_key_t.off and treats off == 0 + * as key not cached. + */ struct tld_data_u { - __u64 start; /* offset of tld_data_u->data in a page */ - char data[]; + __u64 unused; + char data[] __attribute__((aligned(8))); }; struct tld_map_value { void *data; struct tld_meta_u *meta; + __u16 start; /* offset of tld_data_u->data in a page */ }; struct tld_meta_u * _Atomic tld_meta_p __attribute__((weak)); __thread struct tld_data_u *tld_data_p __attribute__((weak)); -__thread void *tld_data_alloc_p __attribute__((weak)); #ifdef TLD_FREE_DATA_ON_THREAD_EXIT +bool _Atomic tld_pthread_key_init __attribute__((weak)); pthread_key_t tld_pthread_key __attribute__((weak)); static void tld_free(void); static void __tld_thread_exit_handler(void *unused) { + (void)unused; tld_free(); } #endif @@ -143,20 +150,16 @@ static int __tld_init_meta_p(void) goto out; } -#ifdef TLD_FREE_DATA_ON_THREAD_EXIT - pthread_key_create(&tld_pthread_key, __tld_thread_exit_handler); -#endif out: return err; } static int __tld_init_data_p(int map_fd) { - bool use_aligned_alloc = false; struct tld_map_value map_val; struct tld_data_u *data; - void *data_alloc = NULL; int err, tid_fd = -1; + size_t size, size_pot; tid_fd = syscall(SYS_pidfd_open, sys_gettid(), O_EXCL); if (tid_fd < 0) { @@ -164,47 +167,37 @@ static int __tld_init_data_p(int map_fd) goto out; } -#ifdef TLD_DATA_USE_ALIGNED_ALLOC - use_aligned_alloc = true; -#endif - /* * tld_meta_p->size = TLD_DYN_DATA_SIZE + * total size of TLDs defined via TLD_DEFINE_KEY() */ - data_alloc = (use_aligned_alloc || tld_meta_p->size * 2 >= TLD_PAGE_SIZE) ? - aligned_alloc(TLD_PAGE_SIZE, tld_meta_p->size) : - malloc(tld_meta_p->size * 2); - if (!data_alloc) { + size = tld_meta_p->size + sizeof(struct tld_data_u); + size_pot = TLD_ROUND_UP_POWER_OF_TWO(size); +#ifdef TLD_DONT_ROUND_UP_DATA_SIZE + data = (struct tld_data_u *)aligned_alloc(size_pot, size); +#else + data = (struct tld_data_u *)aligned_alloc(size_pot, size_pot); +#endif + if (!data) { err = -ENOMEM; goto out; } /* * Always pass a page-aligned address to UPTR since the size of tld_map_value::data - * is a page in BTF. If data_alloc spans across two pages, use the page that contains large - * enough memory. + * is a page in BTF. */ - if (TLD_PAGE_SIZE - (~TLD_PAGE_MASK & (intptr_t)data_alloc) >= tld_meta_p->size) { - map_val.data = (void *)(TLD_PAGE_MASK & (intptr_t)data_alloc); - data = data_alloc; - data->start = (~TLD_PAGE_MASK & (intptr_t)data_alloc) + - offsetof(struct tld_data_u, data); - } else { - map_val.data = (void *)(TLD_ROUND_UP((intptr_t)data_alloc, TLD_PAGE_SIZE)); - data = (void *)(TLD_ROUND_UP((intptr_t)data_alloc, TLD_PAGE_SIZE)); - data->start = offsetof(struct tld_data_u, data); - } - map_val.meta = TLD_READ_ONCE(tld_meta_p); + map_val.data = (void *)(TLD_PAGE_MASK & (intptr_t)data); + map_val.start = (~TLD_PAGE_MASK & (intptr_t)data) + sizeof(struct tld_data_u); + map_val.meta = tld_meta_p; err = bpf_map_update_elem(map_fd, &tid_fd, &map_val, 0); if (err) { - free(data_alloc); + free(data); goto out; } tld_data_p = data; - tld_data_alloc_p = data_alloc; #ifdef TLD_FREE_DATA_ON_THREAD_EXIT pthread_setspecific(tld_pthread_key, (void *)1); #endif @@ -217,15 +210,24 @@ out: static tld_key_t __tld_create_key(const char *name, size_t size, bool dyn_data) { int err, i, sz, off = 0; + bool uninit = false; __u16 cnt; - if (!TLD_READ_ONCE(tld_meta_p)) { + if (!tld_meta_p) { err = __tld_init_meta_p(); if (err) - return (tld_key_t){err}; + return (tld_key_t){(__s16)err}; + } + +#ifdef TLD_FREE_DATA_ON_THREAD_EXIT + if (atomic_compare_exchange_strong(&tld_pthread_key_init, &uninit, true)) { + err = pthread_key_create(&tld_pthread_key, __tld_thread_exit_handler); + if (err) + return (tld_key_t){(__s16)err}; } +#endif - for (i = 0; i < TLD_MAX_DATA_CNT; i++) { + for (i = 0; i < (int)TLD_MAX_DATA_CNT; i++) { retry: cnt = atomic_load(&tld_meta_p->cnt); if (i < cnt) { @@ -245,7 +247,8 @@ retry: * TLD_DYN_DATA_SIZE is allocated for tld_create_key() */ if (dyn_data) { - if (off + TLD_ROUND_UP(size, 8) > tld_meta_p->size) + if (off + TLD_ROUND_UP(size, 8) > tld_meta_p->size || + tld_meta_p->size > TLD_PAGE_SIZE - sizeof(struct tld_data_u)) return (tld_key_t){-E2BIG}; } else { if (off + TLD_ROUND_UP(size, 8) > TLD_PAGE_SIZE - sizeof(struct tld_data_u)) @@ -290,7 +293,7 @@ retry: #define TLD_DEFINE_KEY(key, name, size) \ tld_key_t key; \ \ -__attribute__((constructor)) \ +__attribute__((constructor(101))) \ void __tld_define_key_##key(void) \ { \ key = __tld_create_key(name, size, false); \ @@ -350,7 +353,7 @@ static inline int tld_key_err_or_zero(tld_key_t key) __attribute__((unused)) static void *tld_get_data(int map_fd, tld_key_t key) { - if (!TLD_READ_ONCE(tld_meta_p)) + if (!tld_meta_p) return NULL; /* tld_data_p is allocated on the first invocation of tld_get_data() */ @@ -367,14 +370,14 @@ static void *tld_get_data(int map_fd, tld_key_t key) * * Users must call tld_free() on thread exit to prevent memory leak. Alternatively, * define TLD_FREE_DATA_ON_THREAD_EXIT and a thread exit handler will be registered - * to free the memory automatically. + * to free the memory automatically. Calling tld_free() before thread exit is + * undefined behavior, which may lead to null-pointer dereference. */ __attribute__((unused)) static void tld_free(void) { - if (tld_data_alloc_p) { - free(tld_data_alloc_p); - tld_data_alloc_p = NULL; + if (tld_data_p) { + free(tld_data_p); tld_data_p = NULL; } } diff --git a/tools/testing/selftests/bpf/prog_tests/task_local_storage.c b/tools/testing/selftests/bpf/prog_tests/task_local_storage.c index 7bee33797c71..5b2b56cc3a4f 100644 --- a/tools/testing/selftests/bpf/prog_tests/task_local_storage.c +++ b/tools/testing/selftests/bpf/prog_tests/task_local_storage.c @@ -25,24 +25,31 @@ static void test_sys_enter_exit(void) { struct task_local_storage *skel; + pid_t pid = sys_gettid(); int err; skel = task_local_storage__open_and_load(); if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) return; - skel->bss->target_pid = sys_gettid(); - err = task_local_storage__attach(skel); if (!ASSERT_OK(err, "skel_attach")) goto out; + /* Set target_pid after attach so that syscalls made during + * attach are not counted. + */ + skel->bss->target_pid = pid; + sys_gettid(); sys_gettid(); - /* 3x syscalls: 1x attach and 2x gettid */ - ASSERT_EQ(skel->bss->enter_cnt, 3, "enter_cnt"); - ASSERT_EQ(skel->bss->exit_cnt, 3, "exit_cnt"); + skel->bss->target_pid = 0; + + /* 2x gettid syscalls */ + ASSERT_EQ(skel->bss->update_err, 0, "update_err"); + ASSERT_EQ(skel->bss->enter_cnt, 2, "enter_cnt"); + ASSERT_EQ(skel->bss->exit_cnt, 2, "exit_cnt"); ASSERT_EQ(skel->bss->mismatch_cnt, 0, "mismatch_cnt"); out: task_local_storage__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/tc_change_tail_pmtu.c b/tools/testing/selftests/bpf/prog_tests/tc_change_tail_pmtu.c new file mode 100644 index 000000000000..7acdbd5757a9 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tc_change_tail_pmtu.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <netinet/tcp.h> + +#include "test_progs.h" +#include "network_helpers.h" +#include "test_tc_change_tail_pmtu.skel.h" + +#define CLIENT_NS "tc-change-tail-cli-ns" +#define SERVER_NS "tc-change-tail-srv-ns" +#define CLIENT_IP "192.168.1.1" +#define SERVER_IP "192.168.1.2" + +#define TEST_PMTU 1000 +#define TEST_MSS_MAX (TEST_PMTU - 20 - 20) +#define TIMEOUT_MS 3000 +#define XFER_BYTES 8192 + +void test_tc_change_tail_pmtu(void) +{ + LIBBPF_OPTS(bpf_tcx_opts, tcx_opts); + int mss_before = 0, mss_after = 0, ifindex, port; + int srv_fd = -1, srv_conn_fd = -1, cli_fd = -1; + struct test_tc_change_tail_pmtu *skel = NULL; + struct nstoken *nstoken = NULL; + static char buf[XFER_BYTES]; + socklen_t optlen; + ssize_t bytes; + size_t total; + + if (!ASSERT_OK(make_netns(CLIENT_NS), "make client ns")) + return; + if (!ASSERT_OK(make_netns(SERVER_NS), "make server ns")) + goto out_client_ns; + + nstoken = open_netns(CLIENT_NS); + if (!ASSERT_OK_PTR(nstoken, "open client ns")) + goto out; + SYS(out, "ip link add veth1 type veth peer name veth2 netns " SERVER_NS); + SYS(out, "ip -4 addr add " CLIENT_IP "/24 dev veth1"); + SYS(out, "ip link set veth1 up"); + ifindex = if_nametoindex("veth1"); + if (!ASSERT_NEQ(ifindex, 0, "if_nametoindex")) + goto out; + close_netns(nstoken); + nstoken = NULL; + + nstoken = open_netns(SERVER_NS); + if (!ASSERT_OK_PTR(nstoken, "open server ns")) + goto out; + SYS(out, "ip -4 addr add " SERVER_IP "/24 dev veth2"); + SYS(out, "ip link set veth2 up"); + srv_fd = start_server(AF_INET, SOCK_STREAM, SERVER_IP, 0, TIMEOUT_MS); + if (!ASSERT_OK_FD(srv_fd, "start server")) + goto out; + close_netns(nstoken); + nstoken = NULL; + + skel = test_tc_change_tail_pmtu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open and load skeleton")) + goto out; + + port = get_socket_local_port(srv_fd); + if (!ASSERT_GE(port, 0, "get server port")) + goto out; + + skel->bss->server_port = port; + skel->bss->pmtu = TEST_PMTU; + + nstoken = open_netns(CLIENT_NS); + if (!ASSERT_OK_PTR(nstoken, "open client ns")) + goto out; + + skel->links.change_tail_icmp = + bpf_program__attach_tcx(skel->progs.change_tail_icmp, ifindex, + &tcx_opts); + if (!ASSERT_OK_PTR(skel->links.change_tail_icmp, "attach tcx")) + goto out; + + cli_fd = connect_to_fd(srv_fd, TIMEOUT_MS); + if (!ASSERT_OK_FD(cli_fd, "connect to server")) + goto out; + srv_conn_fd = accept(srv_fd, NULL, NULL); + if (!ASSERT_OK_FD(srv_conn_fd, "accept connection")) + goto out; + if (!ASSERT_OK(settimeo(srv_conn_fd, TIMEOUT_MS), "set server timeout")) + goto out; + + optlen = sizeof(mss_before); + if (!ASSERT_OK(getsockopt(cli_fd, IPPROTO_TCP, TCP_MAXSEG, &mss_before, + &optlen), "get mss before")) + goto out; + + bytes = send(cli_fd, buf, sizeof(buf), 0); + if (!ASSERT_EQ(bytes, (ssize_t)sizeof(buf), "send data")) + goto out; + + for (total = 0; total < sizeof(buf); total += bytes) { + bytes = recv(srv_conn_fd, buf, sizeof(buf), 0); + if (bytes <= 0) + break; + } + + ASSERT_EQ(total, sizeof(buf), "receive data"); + ASSERT_OK(skel->data->change_tail_ret, "change tail"); + ASSERT_OK(skel->bss->adjust_room_ret, "adjust room"); + ASSERT_TRUE(skel->bss->icmp_sent, "icmp sent"); + + optlen = sizeof(mss_after); + if (!ASSERT_OK(getsockopt(cli_fd, IPPROTO_TCP, TCP_MAXSEG, &mss_after, + &optlen), "get mss after")) + goto out; + + ASSERT_LT(mss_after, mss_before, "mss reduced"); + ASSERT_LE(mss_after, TEST_MSS_MAX, "mss below pmtu"); +out: + close(srv_conn_fd); + close(cli_fd); + close(srv_fd); + test_tc_change_tail_pmtu__destroy(skel); + close_netns(nstoken); + remove_netns(SERVER_NS); +out_client_ns: + remove_netns(CLIENT_NS); +} diff --git a/tools/testing/selftests/bpf/prog_tests/tc_qevent.c b/tools/testing/selftests/bpf/prog_tests/tc_qevent.c new file mode 100644 index 000000000000..67e1d17567ab --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tc_qevent.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include <network_helpers.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <unistd.h> +#include <string.h> + +#include "test_tc_qevent.skel.h" + +#define NS_TX "tc_qevent_tx" +#define NS_RX "tc_qevent_rx" +#define IP_TX "10.255.0.1" +#define IP_RX "10.255.0.2" +#define PIN_PATH "/sys/fs/bpf/tc_qevent_redirect" + +static void blast_udp(void) +{ + struct sockaddr_in dst = {}; + char buf[1400] = {}; + int fd, i; + + fd = socket(AF_INET, SOCK_DGRAM, 0); + if (!ASSERT_GE(fd, 0, "udp socket")) + return; + + dst.sin_family = AF_INET; + dst.sin_port = htons(12345); + inet_pton(AF_INET, IP_RX, &dst.sin_addr); + + /* + * Push far more than the RED queue can hold. Once qavg crosses qth_min + * every further packet hits the congestion_drop / early_drop qevent. + */ + for (i = 0; i < 50000; i++) + sendto(fd, buf, sizeof(buf), MSG_DONTWAIT, + (struct sockaddr *)&dst, sizeof(dst)); + + close(fd); +} + +static void run_qevent_redirect(struct bpf_program *prog, __u64 *counter) +{ + struct nstoken *tok = NULL; + int err; + + SYS_NOFAIL("ip netns del %s", NS_TX); + SYS_NOFAIL("ip netns del %s", NS_RX); + unlink(PIN_PATH); + + err = bpf_program__pin(prog, PIN_PATH); + if (!ASSERT_OK(err, "pin prog")) + return; + + SYS(unpin, "ip netns add %s", NS_TX); + SYS(del_tx, "ip netns add %s", NS_RX); + SYS(del_rx, "ip -n %s link add veth0 type veth peer name veth1 netns %s", NS_TX, NS_RX); + SYS(del_rx, "ip -n %s addr add %s/24 dev veth0", NS_TX, IP_TX); + SYS(del_rx, "ip -n %s link set veth0 up", NS_TX); + SYS(del_rx, "ip -n %s addr add %s/24 dev veth1", NS_RX, IP_RX); + SYS(del_rx, "ip -n %s link set veth1 up", NS_RX); + + tok = open_netns(NS_TX); + if (!ASSERT_OK_PTR(tok, "open_netns")) + goto del_rx; + + SYS(close_ns, "tc qdisc add dev veth0 root handle 1: htb default 1"); + SYS(close_ns, "tc class add dev veth0 parent 1: classid 1:1 htb rate 1mbit ceil 1mbit"); + + if (system("tc qdisc add dev veth0 parent 1:1 handle 11: red " + "limit 500000 avpkt 1000 probability 1 min 5000 max 6000 " + "burst 6 qevent early_drop block 10 2>/dev/null")) { + test__skip(); + goto close_ns; + } + + if (system("tc filter add block 10 bpf da object-pinned " + PIN_PATH " 2>/dev/null")) { + test__skip(); + goto close_ns; + } + + blast_udp(); + ASSERT_GT(*counter, 0, "qevent classifier ran"); +close_ns: + close_netns(tok); +del_rx: + SYS_NOFAIL("ip netns del %s", NS_RX); +del_tx: + SYS_NOFAIL("ip netns del %s", NS_TX); +unpin: + bpf_program__unpin(prog, PIN_PATH); +} + +void test_tc_qevent(void) +{ + struct test_tc_qevent *skel; + + skel = test_tc_qevent__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + if (test__start_subtest("redirect_verdict")) + run_qevent_redirect(skel->progs.qevent_redirect_verdict, + &skel->bss->verdict_calls); + if (test__start_subtest("redirect_helper")) + run_qevent_redirect(skel->progs.qevent_redirect_helper, + &skel->bss->helper_calls); + + test_tc_qevent__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c index 64fbda082309..af8968b89ad7 100644 --- a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c +++ b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c @@ -192,6 +192,8 @@ static int create_netkit(int mode, char *prim, char *peer) req.n.nlmsg_len += sizeof(struct ifinfomsg); addattr_l(&req.n, sizeof(req), IFLA_IFNAME, peer, strlen(peer)); addattr_nest_end(&req.n, peer_info); + addattr32(&req.n, sizeof(req), IFLA_NETKIT_SCRUB, + NETKIT_SCRUB_NONE); addattr_nest_end(&req.n, data); addattr_nest_end(&req.n, linkinfo); @@ -405,6 +407,24 @@ fail: return -1; } +static struct bpf_link *netns_attach_nk(const char *ns, int ifindex, + struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_netkit_opts, optl); + struct nstoken *nstoken = NULL; + struct bpf_link *link = NULL; + + nstoken = open_netns(ns); + if (!ASSERT_OK_PTR(nstoken, "setns")) + goto cleanup; + + link = bpf_program__attach_netkit(prog, ifindex, &optl); +cleanup: + if (nstoken) + close_netns(nstoken); + return link; +} + static void test_tcp(int family, const char *addr, __u16 port) { int listen_fd = -1, accept_fd = -1, client_fd = -1; @@ -1082,6 +1102,53 @@ done: close_netns(nstoken); } +static void test_tc_redirect_peer_ing(struct netns_setup_result *setup_result) +{ + struct test_tc_peer *skel; + struct nstoken *nstoken; + int err; + + nstoken = open_netns(NS_FWD); + if (!ASSERT_OK_PTR(nstoken, "setns fwd")) + return; + + skel = test_tc_peer__open(); + if (!ASSERT_OK_PTR(skel, "test_tc_peer__open")) + goto done; + + skel->rodata->IFINDEX_SRC = setup_result->ifindex_src_fwd; + skel->rodata->IFINDEX_DST = setup_result->ifindex_dst_fwd; + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc_src_ing, + BPF_NETKIT_PRIMARY), 0, "src_prog_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc_dst_ing, + BPF_NETKIT_PRIMARY), 0, "dst_prog_attach_type"); + + err = test_tc_peer__load(skel); + if (!ASSERT_OK(err, "test_tc_peer__load")) + goto done; + + skel->links.tc_src_ing = netns_attach_nk(NS_SRC, + setup_result->ifindex_src, + skel->progs.tc_src_ing); + if (!ASSERT_OK_PTR(skel->links.tc_src_ing, "attach_src")) + goto done; + skel->links.tc_dst_ing = netns_attach_nk(NS_DST, + setup_result->ifindex_dst, + skel->progs.tc_dst_ing); + if (!ASSERT_OK_PTR(skel->links.tc_dst_ing, "attach_dst")) + goto done; + + if (!ASSERT_OK(set_forwarding(false), "disable forwarding")) + goto done; + + test_connectivity(); + +done: + if (skel) + test_tc_peer__destroy(skel); + close_netns(nstoken); +} + static int tun_open(char *name) { struct ifreq ifr; @@ -1280,6 +1347,7 @@ static void *test_tc_redirect_run_tests(void *arg) RUN_TEST(tc_redirect_peer, MODE_VETH); RUN_TEST(tc_redirect_peer, MODE_NETKIT); + RUN_TEST(tc_redirect_peer_ing, MODE_NETKIT); RUN_TEST(tc_redirect_peer_l3, MODE_VETH); RUN_TEST(tc_redirect_peer_l3, MODE_NETKIT); RUN_TEST(tc_redirect_neigh, MODE_VETH); diff --git a/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c b/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c index 56685fc03c7e..80e6315da2a5 100644 --- a/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c +++ b/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c @@ -507,6 +507,10 @@ static void misc(void) ASSERT_EQ(misc_skel->bss->nr_hwtstamp, 0, "nr_hwtstamp"); + ASSERT_TRUE(misc_skel->bss->nodelay_est_ok, "nodelay_est_ok"); + ASSERT_TRUE(misc_skel->bss->nodelay_hdr_len_reject, "nodelay_hdr_len_reject"); + ASSERT_TRUE(misc_skel->bss->nodelay_write_hdr_reject, "nodelay_write_hdr_reject"); + check_linum: ASSERT_FALSE(check_error_linum(&sk_fds), "check_error_linum"); sk_fds_close(&sk_fds); diff --git a/tools/testing/selftests/bpf/prog_tests/test_bpf_smc.c b/tools/testing/selftests/bpf/prog_tests/test_bpf_smc.c index de22734abc4d..40d38280c091 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_bpf_smc.c +++ b/tools/testing/selftests/bpf/prog_tests/test_bpf_smc.c @@ -131,8 +131,10 @@ static bool get_smc_nl_family_id(void) goto fail; ret = recv(fd, &msg, sizeof(msg), 0); - if (!ASSERT_FALSE(msg.n.nlmsg_type == NLMSG_ERROR || ret < 0 || - !NLMSG_OK(&msg.n, ret), "nl_family response")) + if (msg.n.nlmsg_type == NLMSG_ERROR) + goto fail; + if (!ASSERT_FALSE(ret < 0 || !NLMSG_OK(&msg.n, ret), + "nl_family response")) goto fail; nl = (struct nlattr *)GENLMSG_DATA(&msg); diff --git a/tools/testing/selftests/bpf/prog_tests/test_dst_clear.c b/tools/testing/selftests/bpf/prog_tests/test_dst_clear.c new file mode 100644 index 000000000000..7c35ca6f4539 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_dst_clear.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <sys/types.h> +#include <sys/socket.h> +#include <net/if.h> + +#include "test_progs.h" +#include "network_helpers.h" +#include "test_dst_clear.skel.h" + +#define IPV4_IFACE_ADDR "1.0.0.1" +#define UDP_TEST_PORT 7777 + +void test_ns_dst_clear(void) +{ + LIBBPF_OPTS(bpf_tcx_opts, tcx_opts); + struct test_dst_clear *skel; + struct sockaddr_in addr; + struct bpf_link *link; + socklen_t addrlen; + char buf[128] = {}; + int sockfd, err; + + skel = test_dst_clear__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + return; + + SYS(fail, "ip addr add %s/8 dev lo", IPV4_IFACE_ADDR); + + link = bpf_program__attach_tcx(skel->progs.dst_clear, + if_nametoindex("lo"), &tcx_opts); + if (!ASSERT_OK_PTR(link, "attach_tcx")) + goto fail; + skel->links.dst_clear = link; + + addrlen = sizeof(addr); + err = make_sockaddr(AF_INET, IPV4_IFACE_ADDR, UDP_TEST_PORT, + (void *)&addr, &addrlen); + if (!ASSERT_OK(err, "make_sockaddr")) + goto fail; + sockfd = socket(AF_INET, SOCK_DGRAM, 0); + if (!ASSERT_NEQ(sockfd, -1, "socket")) + goto fail; + err = sendto(sockfd, buf, sizeof(buf), 0, (void *)&addr, addrlen); + close(sockfd); + if (!ASSERT_EQ(err, sizeof(buf), "send")) + goto fail; + + ASSERT_TRUE(skel->bss->had_dst, "had_dst"); + ASSERT_TRUE(skel->bss->dst_cleared, "dst_cleared"); + +fail: + test_dst_clear__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_global_funcs.c b/tools/testing/selftests/bpf/prog_tests/test_global_funcs.c index e905cbaf6b3d..500446808908 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_global_funcs.c +++ b/tools/testing/selftests/bpf/prog_tests/test_global_funcs.c @@ -18,6 +18,7 @@ #include "test_global_func15.skel.h" #include "test_global_func16.skel.h" #include "test_global_func17.skel.h" +#include "test_global_func_deep_stack.skel.h" #include "test_global_func_ctx_args.skel.h" #include "bpf/libbpf_internal.h" @@ -155,6 +156,7 @@ void test_test_global_funcs(void) RUN_TESTS(test_global_func15); RUN_TESTS(test_global_func16); RUN_TESTS(test_global_func17); + RUN_TESTS(test_global_func_deep_stack); RUN_TESTS(test_global_func_ctx_args); if (test__start_subtest("ctx_arg_rewrite")) diff --git a/tools/testing/selftests/bpf/prog_tests/test_lsm.c b/tools/testing/selftests/bpf/prog_tests/test_lsm.c index bdc4fc06bc5a..d7495efd4a56 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_lsm.c +++ b/tools/testing/selftests/bpf/prog_tests/test_lsm.c @@ -5,36 +5,14 @@ */ #include <test_progs.h> -#include <sys/mman.h> #include <sys/wait.h> #include <unistd.h> -#include <malloc.h> -#include <stdlib.h> #include "lsm.skel.h" #include "lsm_tailcall.skel.h" char *CMD_ARGS[] = {"true", NULL}; -#define GET_PAGE_ADDR(ADDR, PAGE_SIZE) \ - (char *)(((unsigned long) (ADDR + PAGE_SIZE)) & ~(PAGE_SIZE-1)) - -int stack_mprotect(void) -{ - void *buf; - long sz; - int ret; - - sz = sysconf(_SC_PAGESIZE); - if (sz < 0) - return sz; - - buf = alloca(sz * 3); - ret = mprotect(GET_PAGE_ADDR(buf, sz), sz, - PROT_READ | PROT_WRITE | PROT_EXEC); - return ret; -} - int exec_cmd(int *monitored_pid) { int child_pid, child_status; diff --git a/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c b/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c new file mode 100644 index 000000000000..d0ba2ca587b0 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> + +#include "map_kptr.skel.h" + +void test_map_uninit_mem_exposure(void) +{ + size_t value_sz, slot_sz, lookup_sz, tail_sz; + int err, key, nr_cpus, cpu, map_fd; + __u8 *value = NULL, *zero = NULL; + struct bpf_program *prog; + struct map_kptr *skel; + + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) + return; + + skel = map_kptr__open(); + if (!ASSERT_OK_PTR(skel, "map_kptr__open")) + return; + + bpf_object__for_each_program(prog, skel->obj) { + err = bpf_program__set_autoload(prog, false); + if (!ASSERT_OK(err, "bpf_program__set_autoload")) + goto out; + } + + err = map_kptr__load(skel); + if (!ASSERT_OK(err, "map_kptr__load")) + goto out; + + value_sz = bpf_map__value_size((skel)->maps.pcpu_array); + slot_sz = roundup(value_sz, 8); + tail_sz = slot_sz - value_sz; + if (!ASSERT_NEQ(tail_sz, 0, "tail_sz")) + goto out; + + lookup_sz = slot_sz * nr_cpus; + map_fd = bpf_map__fd(skel->maps.pcpu_array); + + value = malloc(lookup_sz); + zero = calloc(1, tail_sz); + if (!ASSERT_OK_PTR(value, "malloc value") || !ASSERT_OK_PTR(zero, "calloc zero")) + goto out; + + key = 0; + memset(value, 0x2B, lookup_sz); + err = bpf_map_update_elem(map_fd, &key, value, BPF_ANY); + if (!ASSERT_OK(err, "bpf_map_update_elem")) + goto out; + + memset(value, 0xFF, lookup_sz); + err = bpf_map_lookup_elem(map_fd, &key, value); + if (!ASSERT_OK(err, "bpf_map_lookup_elem")) + goto out; + + for (cpu = 0; cpu < nr_cpus; cpu++) { + __u8 *tail = value + cpu * slot_sz + value_sz; + + if (!ASSERT_MEMEQ(tail, zero, tail_sz, "zeroed tail bytes")) + goto out; + } + +out: + free(zero); + free(value); + map_kptr__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c new file mode 100644 index 000000000000..7f9f54ba3fbe --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> + +#include "struct_ops_arena.skel.h" +#include "struct_ops_arena_attach.skel.h" +#include "struct_ops_arena_fail.skel.h" + +#if defined(__x86_64__) || defined(__aarch64__) +/* + * Attach callbacks with __arena and __arena__nullable arguments and drive + * them through the bpf_testmod_ops3_call_test_arena*() kfuncs. + */ +static void arena_arg(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct struct_ops_arena *skel; + struct bpf_link *link = NULL; + int err; + + skel = struct_ops_arena__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena__open_and_load")) + return; + + link = bpf_map__attach_struct_ops(skel->maps.testmod_arena); + if (!ASSERT_OK_PTR(link, "attach_struct_ops")) + goto out; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.trigger), + &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "trigger_retval"); + +out: + bpf_link__destroy(link); + struct_ops_arena__destroy(skel); +} + +/* + * A program with no arena cannot attach to a member with an __arena + * argument. + */ +static void arena_arg_fail(void) +{ + struct struct_ops_arena_fail *skel; + + skel = struct_ops_arena_fail__open_and_load(); + if (ASSERT_ERR_PTR(skel, "struct_ops_arena_fail__open_and_load")) + return; + + struct_ops_arena_fail__destroy(skel); +} + +static void arena_arg_attach_one(int target_fd, const char *prog_name) +{ + struct struct_ops_arena_attach *skel; + struct bpf_program *prog, *pos; + char log_buf[64 * 1024]; + int err; + + skel = struct_ops_arena_attach__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_attach__open")) + return; + + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, prog_name)) + goto out; + + bpf_object__for_each_program(pos, skel->obj) + bpf_program__set_autoload(pos, pos == prog); + + err = bpf_program__set_attach_target(prog, target_fd, "test_arena_cb"); + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + log_buf[0] = '\0'; + bpf_program__set_log_buf(prog, log_buf, sizeof(log_buf)); + err = struct_ops_arena_attach__load(skel); + + ASSERT_EQ(err, -EOPNOTSUPP, prog_name); + ASSERT_HAS_SUBSTR(log_buf, "Cannot attach to a target with arena context arguments", + "verifier_log"); + +out: + struct_ops_arena_attach__destroy(skel); +} + +static void arena_arg_attach(void) +{ + struct struct_ops_arena *skel; + int target_fd; + + skel = struct_ops_arena__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena__open_and_load")) + return; + + target_fd = bpf_program__fd(skel->progs.test_arena_cb); + arena_arg_attach_one(target_fd, "fentry_test_arena"); + arena_arg_attach_one(target_fd, "fexit_test_arena"); + arena_arg_attach_one(target_fd, "freplace_test_arena"); + + struct_ops_arena__destroy(skel); +} +#endif + +/* + * Serialized because it attaches the singleton bpf_testmod_ops3, which + * test_struct_ops_private_stack also attaches; registering it twice fails + * with -EEXIST. + */ +void serial_test_struct_ops_arena(void) +{ + /* + * Arena struct_ops arguments need JIT support, currently x86-64 and + * arm64 only. Elsewhere verification fails with "JIT does not support + * arena arguments", so the programs cannot even load. + */ +#if defined(__x86_64__) || defined(__aarch64__) + if (test__start_subtest("arena_arg")) + arena_arg(); + if (test__start_subtest("arena_arg_fail")) + arena_arg_fail(); + if (test__start_subtest("arena_arg_attach")) + arena_arg_attach(); +#else + test__skip(); +#endif +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c new file mode 100644 index 000000000000..0f321e889862 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "struct_ops_multi_args.skel.h" + +void test_struct_ops_multi_args(void) +{ + RUN_TESTS(struct_ops_multi_args); +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_task_local_data.c b/tools/testing/selftests/bpf/prog_tests/test_task_local_data.c index 9556ad3d986f..6a5806b36113 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_task_local_data.c +++ b/tools/testing/selftests/bpf/prog_tests/test_task_local_data.c @@ -3,8 +3,14 @@ #include <bpf/btf.h> #include <test_progs.h> +/* + * Only a page is pinned to kernel, so the maximum amount of dynamic data + * allowed is page_size - sizeof(struct tld_data_u) - static TLD fields. + */ +#define TLD_DYN_DATA_SIZE_MAX (getpagesize() - sizeof(struct tld_data_u) - 8) + #define TLD_FREE_DATA_ON_THREAD_EXIT -#define TLD_DYN_DATA_SIZE (getpagesize() - 8) +#define TLD_DYN_DATA_SIZE TLD_DYN_DATA_SIZE_MAX #include "task_local_data.h" struct test_tld_struct { @@ -24,12 +30,12 @@ TLD_DEFINE_KEY(value0_key, "value0", sizeof(int)); * sequentially. Users of task local data library should not touch * library internal. */ -static void reset_tld(void) +static void reset_tld(__u16 dyn_data_size) { - if (TLD_READ_ONCE(tld_meta_p)) { + if (tld_meta_p) { /* Remove TLDs created by tld_create_key() */ tld_meta_p->cnt = 1; - tld_meta_p->size = TLD_DYN_DATA_SIZE; + tld_meta_p->size = dyn_data_size + 8; memset(&tld_meta_p->metadata[1], 0, (TLD_MAX_DATA_CNT - 1) * sizeof(struct tld_metadata)); } @@ -127,7 +133,7 @@ static void test_task_local_data_basic(void) tld_key_t key; int i, err; - reset_tld(); + reset_tld(TLD_DYN_DATA_SIZE_MAX); ASSERT_OK(pthread_mutex_init(&global_mutex, NULL), "pthread_mutex_init"); @@ -147,11 +153,13 @@ static void test_task_local_data_basic(void) /* * Shouldn't be able to store data exceed a page. Create a TLD just big - * enough to exceed a page. TLDs already created are int value0, int - * value1, and struct test_tld_struct value2. + * enough to exceed a page. Data already contains struct tld_data_u, + * value0 and value1 of int type, and value 2 of struct test_tld_struct. */ - key = tld_create_key("value_not_exist", - TLD_PAGE_SIZE - 2 * sizeof(int) - sizeof(struct test_tld_struct) + 1); + key = tld_create_key("value_not_exist", TLD_PAGE_SIZE + 1 - + sizeof(struct tld_data_u) - + TLD_ROUND_UP(sizeof(int), 8) * 2 - + TLD_ROUND_UP(sizeof(struct test_tld_struct), 8)); ASSERT_EQ(tld_key_err_or_zero(key), -E2BIG, "tld_create_key"); key = tld_create_key("value2", sizeof(struct test_tld_struct)); @@ -239,7 +247,7 @@ static void test_task_local_data_race(void) tld_keys[0] = value0_key; for (j = 0; j < 100; j++) { - reset_tld(); + reset_tld(TLD_DYN_DATA_SIZE_MAX); for (i = 0; i < TEST_RACE_THREAD_NUM; i++) { /* @@ -288,10 +296,80 @@ out: test_task_local_data__destroy(skel); } +static void test_task_local_data_dyn_size(__u16 dyn_data_size) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct test_task_local_data *skel; + int max_keys, i, err, fd, *data; + char name[TLD_NAME_LEN]; + tld_key_t key; + + reset_tld(dyn_data_size); + + skel = test_task_local_data__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + tld_keys = calloc(TLD_MAX_DATA_CNT, sizeof(tld_key_t)); + if (!ASSERT_OK_PTR(tld_keys, "calloc tld_keys")) + goto out; + + fd = bpf_map__fd(skel->maps.tld_data_map); + + /* Create as many int-sized TLDs as the dynamic data size allows */ + max_keys = dyn_data_size / TLD_ROUND_UP(sizeof(int), 8); + for (i = 0; i < max_keys; i++) { + snprintf(name, TLD_NAME_LEN, "value_%d", i); + tld_keys[i] = tld_create_key(name, sizeof(int)); + if (!ASSERT_FALSE(tld_key_is_err(tld_keys[i]), "tld_create_key")) + goto out; + + data = tld_get_data(fd, tld_keys[i]); + if (!ASSERT_OK_PTR(data, "tld_get_data")) + goto out; + *data = i; + } + + /* The next key should fail with E2BIG */ + key = tld_create_key("overflow", sizeof(int)); + ASSERT_EQ(tld_key_err_or_zero(key), -E2BIG, "tld_create_key overflow"); + + /* Verify data for value_i do not overlap */ + for (i = 0; i < max_keys; i++) { + data = tld_get_data(fd, tld_keys[i]); + if (!ASSERT_OK_PTR(data, "tld_get_data")) + goto out; + + ASSERT_EQ(*data, i, "tld_get_data value_i"); + } + + /* Verify BPF side can still read the static key */ + data = tld_get_data(fd, value0_key); + if (!ASSERT_OK_PTR(data, "tld_get_data value0")) + goto out; + *data = 0xdeadbeef; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.task_main), &opts); + ASSERT_OK(err, "run task_main"); + ASSERT_EQ(skel->bss->test_value0, 0xdeadbeef, "tld_get_data value0"); + +out: + if (tld_keys) { + free(tld_keys); + tld_keys = NULL; + } + tld_free(); + test_task_local_data__destroy(skel); +} + void test_task_local_data(void) { if (test__start_subtest("task_local_data_basic")) test_task_local_data_basic(); if (test__start_subtest("task_local_data_race")) test_task_local_data_race(); + if (test__start_subtest("task_local_data_dyn_size_small")) + test_task_local_data_dyn_size(64); + if (test__start_subtest("task_local_data_dyn_size_zero")) + test_task_local_data_dyn_size(0); } diff --git a/tools/testing/selftests/bpf/prog_tests/test_tc_tunnel.c b/tools/testing/selftests/bpf/prog_tests/test_tc_tunnel.c index 7fc4d7dd70ef..67ba27d69347 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_tc_tunnel.c +++ b/tools/testing/selftests/bpf/prog_tests/test_tc_tunnel.c @@ -168,7 +168,7 @@ static int check_server_rx_data(struct subtest_cfg *cfg, static struct connection *connect_client_to_server(struct subtest_cfg *cfg) { - struct network_helper_opts opts = {.timeout_ms = 500}; + struct network_helper_opts opts = {.timeout_ms = 1000}; int family = cfg->ipproto == 6 ? AF_INET6 : AF_INET; struct connection *conn = NULL; int client_fd, server_fd; @@ -206,18 +206,13 @@ static void disconnect_client_from_server(struct subtest_cfg *cfg, free(conn); } -static int send_and_test_data(struct subtest_cfg *cfg, bool must_succeed) +static int send_and_test_data(struct subtest_cfg *cfg) { struct connection *conn; int err, res = -1; conn = connect_client_to_server(cfg); - if (!must_succeed && !ASSERT_ERR_PTR(conn, "connection that must fail")) - goto end; - else if (!must_succeed) - return 0; - - if (!ASSERT_OK_PTR(conn, "connection that must succeed")) + if (!ASSERT_OK_PTR(conn, "connect to server")) return -1; err = send(conn->client_fd, tx_buffer, DEFAULT_TEST_DATA_SIZE, 0); @@ -391,7 +386,7 @@ static void run_test(struct subtest_cfg *cfg) goto fail; /* Basic communication must work */ - if (!ASSERT_OK(send_and_test_data(cfg, true), "connect without any encap")) + if (!ASSERT_OK(send_and_test_data(cfg), "connect without any encap")) goto fail; /* Attach encapsulation program to client */ @@ -403,7 +398,7 @@ static void run_test(struct subtest_cfg *cfg) if (!ASSERT_OK(configure_kernel_decapsulation(cfg), "configure kernel decapsulation")) goto fail; - if (!ASSERT_OK(send_and_test_data(cfg, true), + if (!ASSERT_OK(send_and_test_data(cfg), "connect with encap prog and kern decap")) goto fail; } @@ -411,7 +406,7 @@ static void run_test(struct subtest_cfg *cfg) /* Replace kernel decapsulation with BPF decapsulation, test must pass */ if (!ASSERT_OK(configure_ebpf_decapsulation(cfg), "configure ebpf decapsulation")) goto fail; - ASSERT_OK(send_and_test_data(cfg, true), "connect with encap and decap progs"); + ASSERT_OK(send_and_test_data(cfg), "connect with encap and decap progs"); fail: close_netns(nstoken); @@ -443,7 +438,6 @@ static int setup(void) SYS(fail_close_ns_client, "ip link add %s type veth peer name %s", "veth1 mtu 1500 netns " CLIENT_NS " address " MAC_ADDR_VETH1, "veth2 mtu 1500 netns " SERVER_NS " address " MAC_ADDR_VETH2); - SYS(fail_close_ns_client, "ethtool -K veth1 tso off"); SYS(fail_close_ns_client, "ip link set veth1 up"); nstoken_server = open_netns(SERVER_NS); if (!ASSERT_OK_PTR(nstoken_server, "open server ns")) diff --git a/tools/testing/selftests/bpf/prog_tests/test_veristat.c b/tools/testing/selftests/bpf/prog_tests/test_veristat.c index 9aff08ac55c0..11f3de2b66ad 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_veristat.c +++ b/tools/testing/selftests/bpf/prog_tests/test_veristat.c @@ -37,6 +37,14 @@ static struct fixture *init_fixture(void) return fix; } +static void read_output(struct fixture *fix) +{ + ssize_t len = pread(fix->fd, fix->output, fix->sz - 1, 0); + + fix->output[len < 0 ? 0 : len] = 0; + ASSERT_GE(len, 0, "pread"); +} + static void teardown_fixture(struct fixture *fix) { free(fix->output); @@ -74,7 +82,7 @@ static void test_set_global_vars_succeeds(void) " -G \"struct11 [ 7 ] [ 5 ] .struct2[0][1].u.mat[3][0] = 175\" " \ " -vl2 > %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("=0xf000000000000001 ", "var_s64 = 0xf000000000000001"); __CHECK_STR("=0xfedcba9876543210 ", "var_u64 = 0xfedcba9876543210"); __CHECK_STR("=0x80000000 ", "var_s32 = -0x80000000"); @@ -116,7 +124,7 @@ static void test_set_global_vars_from_file_succeeds(void) syncfs(fd); SYS(out, "%s set_global_vars.bpf.o -G \"@%s\" -vl2 > %s", fix->veristat, input_file, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("=0x8000 ", "var_s16 = -32768"); __CHECK_STR("=0xecec ", "var_u16 = 60652"); @@ -134,7 +142,7 @@ static void test_set_global_vars_out_of_range(void) "%s set_global_vars.bpf.o -G \"var_s32 = 2147483648\" -vl2 2> %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("is out of range [-2147483648; 2147483647]", "out of range"); out: @@ -149,7 +157,7 @@ static void test_unsupported_ptr_array_type(void) "%s set_global_vars.bpf.o -G \"ptr_arr[0] = 0\" -vl2 2> %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("Can't set ptr_arr[0]. Only ints and enums are supported", "ptr_arr"); out: @@ -164,7 +172,7 @@ static void test_array_out_of_bounds(void) "%s set_global_vars.bpf.o -G \"arr[99] = 0\" -vl2 2> %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("Array index 99 is out of bounds", "arr[99]"); out: @@ -179,7 +187,7 @@ static void test_array_index_not_found(void) "%s set_global_vars.bpf.o -G \"arr[EG2] = 0\" -vl2 2> %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("Can't resolve enum value EG2", "arr[EG2]"); out: @@ -230,6 +238,97 @@ out: teardown_fixture(fix); } +/* + * Name filter tests below run veristat on veristat_foo.bpf.o and + * veristat_bar.bpf.o, both defining programs 'foo', 'bar' and 'buz'. + * Every entry describes a single (filters, file, prog) combination and + * tells whether that program is expected in the veristat output: + * 'true' if it is, 'false' if it is not and -1 if veristat is expected + * to reject the filter. + */ +#define FILTER_OBJS "veristat_foo.bpf.o veristat_bar.bpf.o" + +static const struct name_filter_case { + const char *filters; + const char *file; + const char *prog; + int included; +} name_filter_cases[] = { + /* no filters, every program is processed */ + { "", "foo", "foo", true }, + { "", "foo", "bar", true }, + { "", "foo", "buz", true }, + { "", "bar", "foo", true }, + { "", "bar", "bar", true }, + { "", "bar", "buz", true }, + /* deny filters */ + { "-f '!*foo*'", "foo", "bar", false }, + { "-f '!*foo*'", "bar", "foo", false }, + { "-f '!*foo*'", "bar", "bar", true }, + { "-f '!*foo*/bar'", "foo", "bar", false }, + { "-f '!*foo*/bar'", "foo", "buz", true }, + { "-f '!*foo*/bar'", "bar", "bar", true }, + { "-f '!*foo*/'", "foo", "bar", false }, + { "-f '!*foo*/'", "bar", "bar", true }, + { "-f '!/bar'", "foo", "bar", false }, + { "-f '!/bar'", "foo", "foo", true }, + { "-f '!/'", "foo", "bar", -1 }, + { "-f '!'", "foo", "bar", -1 }, + /* allow filters */ + { "-f '*foo*'", "foo", "bar", true }, + { "-f '*foo*'", "bar", "foo", true }, + { "-f '*foo*'", "bar", "bar", false }, + { "-f '*foo*/bar'", "foo", "bar", true }, + { "-f '*foo*/bar'", "foo", "buz", false }, + { "-f '*foo*/bar'", "bar", "bar", false }, + { "-f '*foo*/'", "foo", "bar", true }, + { "-f '*foo*/'", "bar", "bar", false }, + { "-f '/bar'", "foo", "bar", true }, + { "-f '/bar'", "foo", "foo", false }, + { "-f '/'", "foo", "bar", -1 }, + { "-f ''", "foo", "bar", -1 }, + /* allow and deny filters combined */ + { "-f '*foo*/' -f '!/bar'", "foo", "foo", true }, + { "-f '*foo*/' -f '!/bar'", "foo", "bar", false }, + { "-f '*foo*/' -f '!/bar'", "bar", "foo", false }, +}; + +static void test_name_filters(void) +{ + struct fixture *fix = init_fixture(); + const struct name_filter_case *t; + char cmd[512], row[64], name[128]; + int i, err; + + for (i = 0; i < ARRAY_SIZE(name_filter_cases); i++) { + t = &name_filter_cases[i]; + /* stderr is merged with stdout in order to catch error messages */ + snprintf(cmd, sizeof(cmd), "%s " FILTER_OBJS " -q -o csv -e file,prog %s > %s 2>&1", + fix->veristat, t->filters, fix->tmpfile); + err = system(cmd); + read_output(fix); + + snprintf(row, sizeof(row), "veristat_%s.bpf.o,%s", t->file, t->prog); + snprintf(name, sizeof(name), "veristat %s: %s", t->filters, row); + switch (t->included) { + case true: + ASSERT_OK(err, name); + ASSERT_HAS_SUBSTR(fix->output, row, name); + break; + case false: + ASSERT_OK(err, name); + ASSERT_FALSE(!!strstr(fix->output, row), name); + break; + case -1: + ASSERT_NEQ(err, 0, name); + ASSERT_HAS_SUBSTR(fix->output, "Invalid filter", name); + break; + } + } + + teardown_fixture(fix); +} + void test_veristat(void) { if (test__start_subtest("set_global_vars_succeeds")) @@ -256,6 +355,8 @@ void test_veristat(void) if (test__start_subtest("test_no_array_index_for_array")) test_no_array_index_for_array(); + if (test__start_subtest("name_filters")) + test_name_filters(); } #undef __CHECK_STR diff --git a/tools/testing/selftests/bpf/prog_tests/test_xdp_veth.c b/tools/testing/selftests/bpf/prog_tests/test_xdp_veth.c index 3e98a1665936..1675b32753a8 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_xdp_veth.c +++ b/tools/testing/selftests/bpf/prog_tests/test_xdp_veth.c @@ -456,7 +456,11 @@ static void xdp_veth_egress(u32 flags) .remote_flags = flags, } }; - const char magic_mac[6] = { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF}; + const unsigned char egress_macs[VETH_PAIRS_COUNT][ETH_ALEN] = { + { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0x01 }, + { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0x02 }, + { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0x03 }, + }; struct xdp_redirect_multi_kern *xdp_redirect_multi_kern; struct bpf_object *bpf_objs[VETH_EGRESS_SKEL_NB]; struct xdp_redirect_map *xdp_redirect_map; @@ -512,7 +516,13 @@ static void xdp_veth_egress(u32 flags) &net_config, prog_cfg, i)) goto destroy_xdp_redirect_map; - err = bpf_map_update_elem(mac_map, &ifindex, magic_mac, 0); + { + __be64 mac = 0; + + memcpy(&mac, egress_macs[i], ETH_ALEN); + err = bpf_map_update_elem(mac_map, &ifindex, &mac, 0); + } + if (!ASSERT_OK(err, "bpf_map_update_elem")) goto destroy_xdp_redirect_map; @@ -531,15 +541,162 @@ static void xdp_veth_egress(u32 flags) for (i = 0; i < 2; i++) { u32 key = i; + __be64 expected = 0; u64 res; err = bpf_map_lookup_elem(res_map, &key, &res); if (!ASSERT_OK(err, "get MAC res")) goto destroy_xdp_redirect_map; - ASSERT_STRNEQ((const char *)&res, magic_mac, ETH_ALEN, "compare mac"); + /* store_mac_1/2 run on the second/third remote veths. */ + memcpy(&expected, egress_macs[i + 1], ETH_ALEN); + ASSERT_EQ(res, expected, "compare mac"); + } + +destroy_xdp_redirect_map: + close_netns(nstoken); + xdp_redirect_map__destroy(xdp_redirect_map); +destroy_xdp_redirect_multi_kern: + xdp_redirect_multi_kern__destroy(xdp_redirect_multi_kern); +destroy_xdp_dummy: + xdp_dummy__destroy(xdp_dummy); + + cleanup_network(&net_config); +} + +static void xdp_veth_egress_last_dst(u32 flags) +{ + struct prog_configuration prog_cfg[VETH_PAIRS_COUNT] = { + { + .local_name = "xdp_redirect_map_all_prog", + .remote_name = "xdp_dummy_prog", + .local_flags = flags, + .remote_flags = flags, + }, + { + .local_name = "xdp_redirect_map_all_prog", + .remote_name = "store_mac_1", + .local_flags = flags, + .remote_flags = flags, + }, + { + .local_name = "xdp_redirect_map_all_prog", + .remote_name = "xdp_dummy_prog", + .local_flags = flags, + .remote_flags = flags, + } + }; + const unsigned char egress_macs[VETH_PAIRS_COUNT][ETH_ALEN] = { + { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0x01 }, + { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0x02 }, + { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0x03 }, + }; + struct xdp_redirect_multi_kern *xdp_redirect_multi_kern; + struct bpf_object *bpf_objs[VETH_EGRESS_SKEL_NB]; + struct xdp_redirect_map *xdp_redirect_map; + struct net_configuration net_config = {}; + int mac_map, egress_map, res_map; + struct nstoken *nstoken = NULL; + struct xdp_dummy *xdp_dummy; + __be64 sentinel_mac = 0; + __be64 last_mac = 0; + __be64 res; + u32 key; + int err; + int i; + + xdp_dummy = xdp_dummy__open_and_load(); + if (!ASSERT_OK_PTR(xdp_dummy, "xdp_dummy__open_and_load")) + return; + + xdp_redirect_multi_kern = xdp_redirect_multi_kern__open_and_load(); + if (!ASSERT_OK_PTR(xdp_redirect_multi_kern, "xdp_redirect_multi_kern__open_and_load")) + goto destroy_xdp_dummy; + + xdp_redirect_map = xdp_redirect_map__open_and_load(); + if (!ASSERT_OK_PTR(xdp_redirect_map, "xdp_redirect_map__open_and_load")) + goto destroy_xdp_redirect_multi_kern; + + if (!ASSERT_OK(create_network(&net_config), "create network")) + goto destroy_xdp_redirect_map; + + mac_map = bpf_map__fd(xdp_redirect_multi_kern->maps.mac_map); + if (!ASSERT_OK_FD(mac_map, "open mac_map")) + goto destroy_xdp_redirect_map; + + egress_map = bpf_map__fd(xdp_redirect_multi_kern->maps.map_egress); + if (!ASSERT_OK_FD(egress_map, "open map_egress")) + goto destroy_xdp_redirect_map; + + bpf_objs[0] = xdp_dummy->obj; + bpf_objs[1] = xdp_redirect_multi_kern->obj; + bpf_objs[2] = xdp_redirect_map->obj; + + nstoken = open_netns(net_config.ns0_name); + if (!ASSERT_OK_PTR(nstoken, "open NS0")) + goto destroy_xdp_redirect_map; + + for (i = 0; i < VETH_PAIRS_COUNT; i++) { + struct bpf_devmap_val devmap_val = {}; + int ifindex = if_nametoindex(net_config.veth_cfg[i].local_veth); + u32 key = i; + + SYS(destroy_xdp_redirect_map, + "ip -n %s neigh add %s lladdr 00:00:00:00:00:01 dev %s", + net_config.veth_cfg[i].namespace, IP_NEIGH, + net_config.veth_cfg[i].remote_veth); + + if (attach_programs_to_veth_pair(bpf_objs, VETH_EGRESS_SKEL_NB, + &net_config, prog_cfg, i)) + goto destroy_xdp_redirect_map; + + { + __be64 mac = 0; + + memcpy(&mac, egress_macs[i], ETH_ALEN); + err = bpf_map_update_elem(mac_map, &ifindex, &mac, 0); + } + + if (!ASSERT_OK(err, "bpf_map_update_elem")) + goto destroy_xdp_redirect_map; + + devmap_val.ifindex = ifindex; + devmap_val.bpf_prog.fd = -1; + + if (i == VETH_PAIRS_COUNT - 1) + devmap_val.bpf_prog.fd = + bpf_program__fd(xdp_redirect_multi_kern->progs.xdp_devmap_prog); + + err = bpf_map_update_elem(egress_map, &key, &devmap_val, 0); + if (!ASSERT_OK(err, "bpf_map_update_elem")) + goto destroy_xdp_redirect_map; } + res_map = bpf_map__fd(xdp_redirect_map->maps.rx_mac); + if (!ASSERT_OK_FD(res_map, "open rx_map")) + goto destroy_xdp_redirect_map; + + memcpy(&sentinel_mac, egress_macs[VETH_PAIRS_COUNT - 1], ETH_ALEN); + memcpy(&last_mac, egress_macs[VETH_PAIRS_COUNT - 1], ETH_ALEN); + + key = 0; + err = bpf_map_update_elem(res_map, &key, &sentinel_mac, 0); + if (!ASSERT_OK(err, "init rx mac")) + goto destroy_xdp_redirect_map; + + SYS_NOFAIL("ip netns exec %s ping %s -i 0.1 -c 4 -W1 > /dev/null ", + net_config.veth_cfg[0].namespace, IP_NEIGH); + + err = bpf_map_lookup_elem(res_map, &key, &res); + if (!ASSERT_OK(err, "get MAC res")) + goto destroy_xdp_redirect_map; + + if (!ASSERT_NEQ(res, sentinel_mac, "rx_mac overwritten by store_mac_1")) + goto destroy_xdp_redirect_map; + + if (!ASSERT_NEQ(res, last_mac, "earlier dst not rewritten by last dst")) + goto destroy_xdp_redirect_map; + destroy_xdp_redirect_map: close_netns(nstoken); xdp_redirect_map__destroy(xdp_redirect_map); @@ -596,4 +753,7 @@ void test_xdp_veth_egress(void) if (test__start_subtest("SKB_MODE/egress")) xdp_veth_egress(XDP_FLAGS_SKB_MODE); + + if (test__start_subtest("SKB_MODE/egress_last_dst")) + xdp_veth_egress_last_dst(XDP_FLAGS_SKB_MODE); } diff --git a/tools/testing/selftests/bpf/prog_tests/test_xsk.c b/tools/testing/selftests/bpf/prog_tests/test_xsk.c index 7950c504ed28..4549358cc8c2 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_xsk.c +++ b/tools/testing/selftests/bpf/prog_tests/test_xsk.c @@ -7,7 +7,6 @@ #include <linux/netdev.h> #include <poll.h> #include <pthread.h> -#include <signal.h> #include <string.h> #include <sys/mman.h> #include <sys/socket.h> @@ -65,11 +64,6 @@ static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr) eth_hdr->h_proto = htons(ETH_P_LOOPBACK); } -static bool is_umem_valid(struct ifobject *ifobj) -{ - return !!ifobj->umem->umem; -} - static u32 mode_to_xdp_flags(enum test_mode mode) { return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE; @@ -213,6 +207,7 @@ static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, for (i = 0; i < MAX_INTERFACES; i++) { struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; + struct xsk_umem_info *umem_real; ifobj->xsk = &ifobj->xsk_arr[0]; ifobj->use_poll = false; @@ -229,24 +224,30 @@ static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, ifobj->tx_on = false; } - memset(ifobj->umem, 0, sizeof(*ifobj->umem)); - ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS; - ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; - + umem_real = ifobj->xsk_arr[0].umem_real; + memset(umem_real, 0, sizeof(*umem_real)); for (j = 0; j < MAX_SOCKETS; j++) { - memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j])); - ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; - ifobj->xsk_arr[j].batch_size = DEFAULT_BATCH_SIZE; + struct xsk_socket_info *xsk = &ifobj->xsk_arr[j]; + + memset(xsk, 0, sizeof(*xsk)); + xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; + if (j == 0) + xsk->umem_real = umem_real; + xsk->umem = umem_real; + xsk->batch_size = DEFAULT_BATCH_SIZE; if (i == 0) - ifobj->xsk_arr[j].pkt_stream = test->tx_pkt_stream_default; + xsk->pkt_stream = test->tx_pkt_stream_default; else - ifobj->xsk_arr[j].pkt_stream = test->rx_pkt_stream_default; + xsk->pkt_stream = test->rx_pkt_stream_default; - memcpy(ifobj->xsk_arr[j].src_mac, g_mac, ETH_ALEN); - memcpy(ifobj->xsk_arr[j].dst_mac, g_mac, ETH_ALEN); - ifobj->xsk_arr[j].src_mac[5] += ((j * 2) + 0); - ifobj->xsk_arr[j].dst_mac[5] += ((j * 2) + 1); + memcpy(xsk->src_mac, g_mac, ETH_ALEN); + memcpy(xsk->dst_mac, g_mac, ETH_ALEN); + xsk->src_mac[5] += ((j * 2) + 0); + xsk->dst_mac[5] += ((j * 2) + 1); } + + ifobj->xsk->umem->num_frames = DEFAULT_UMEM_BUFFERS; + ifobj->xsk->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; } if (ifobj_tx->hw_ring_size_supp) @@ -303,6 +304,18 @@ static void test_spec_reset(struct test_spec *test) __test_spec_init(test, test->ifobj_tx, test->ifobj_rx); } +static void test_spec_set_unaligned(struct test_spec *test) +{ + test->ifobj_tx->xsk->umem->unaligned_mode = true; + test->ifobj_rx->xsk->umem->unaligned_mode = true; +} + +static void test_spec_set_frame_size(struct test_spec *test, u32 size) +{ + test->ifobj_tx->xsk->umem->frame_size = size; + test->ifobj_rx->xsk->umem->frame_size = size; +} + static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx, struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx, struct bpf_map *xskmap_tx) @@ -414,14 +427,14 @@ static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pk } /* Search for the end of the packet in verbatim mode */ - if (!pkt_continues(pkt->options) || !pkt->valid) + if (!pkt_continues(pkt->options)) return nb_frags; next_frag = pkt_stream->current_pkt_nb; pkt++; while (next_frag++ < pkt_stream->nb_pkts) { nb_frags++; - if (!pkt_continues(pkt->options) || !pkt->valid) + if (!pkt_continues(pkt->options)) break; pkt++; } @@ -652,11 +665,11 @@ static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, s if (!frame->valid || !pkt_continues(frame->options)) payload++; } else { - if (frame->valid) + if (frame->valid) { len += frame->len; - if (frame->valid && pkt_continues(frame->options)) - continue; - + if (pkt_continues(frame->options)) + continue; + } pkt->pkt_nb = pkt_nb; pkt->len = len; pkt->valid = frame->valid; @@ -810,11 +823,11 @@ static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 exp { u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum; void *data = xsk_umem__get_data(umem->buffer, addr); + u64 umem_sz = umem_size(umem); addr -= umem->base_addr; - if (addr >= umem->num_frames * umem->frame_size || - addr + len > umem->num_frames * umem->frame_size) { + if (addr >= umem_sz || addr + len > umem_sz) { ksft_print_msg("Frag invalid addr: %llx len: %u\n", (unsigned long long)addr, len); return false; @@ -991,7 +1004,7 @@ static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk) return TEST_FAILURE; if (!ret) { - if (!is_umem_valid(test->ifobj_tx)) + if (test->poll_tmout) return TEST_PASS; ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__); @@ -1130,7 +1143,7 @@ static int receive_pkts(struct test_spec *test) break; res = __receive_pkts(test, xsk); - if (!(res == TEST_PASS || res == TEST_CONTINUE)) + if (res != TEST_CONTINUE) return res; ret = gettimeofday(&tv_now, NULL); @@ -1147,11 +1160,12 @@ static int receive_pkts(struct test_spec *test) return TEST_PASS; } -static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout) +static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, + bool test_timeout) { u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len; + struct xsk_umem_info *umem = ifobject->xsk_arr[0].umem_real; struct pkt_stream *pkt_stream = xsk->pkt_stream; - struct xsk_umem_info *umem = ifobject->umem; bool use_poll = ifobject->use_poll; struct pollfd fds = { }; int ret; @@ -1159,7 +1173,7 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, b buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len); /* pkts_in_flight might be negative if many invalid packets are sent */ if (pkts_in_flight >= (int)((umem_size(umem) - xsk->batch_size * buffer_len) / - buffer_len)) { + buffer_len) && !test_timeout) { ret = kick_tx(xsk); if (ret) return TEST_FAILURE; @@ -1172,7 +1186,7 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, b while (xsk_ring_prod__reserve(&xsk->tx, xsk->batch_size, &idx) < xsk->batch_size) { if (use_poll) { ret = poll(&fds, 1, POLL_TMOUT); - if (timeout) { + if (test_timeout) { if (ret < 0) { ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, errno); @@ -1210,7 +1224,7 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, b while (nb_frags_left--) { struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i); - tx_desc->addr = pkt_get_addr(pkt, ifobject->umem); + tx_desc->addr = pkt_get_addr(pkt, umem); if (pkt_stream->verbatim) { tx_desc->len = pkt->len; tx_desc->options = pkt->options; @@ -1236,10 +1250,9 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, b } } - if (pkt && pkt->valid) { + if (pkt && pkt->valid) valid_pkts++; - valid_frags += nb_frags; - } + valid_frags += nb_frags; } pthread_mutex_lock(&pacing_mutex); @@ -1252,7 +1265,7 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, b if (use_poll) { ret = poll(&fds, 1, POLL_TMOUT); if (ret <= 0) { - if (ret == 0 && timeout) + if (ret == 0 && test_timeout) return TEST_PASS; ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret); @@ -1260,14 +1273,14 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, b } } - if (!timeout) { + if (!test_timeout) { if (complete_pkts(xsk, i)) return TEST_FAILURE; usleep(10); - return TEST_PASS; } + /* Loop completion is driven by send_pkts() stream progress checks. */ return TEST_CONTINUE; } @@ -1303,7 +1316,6 @@ bool all_packets_sent(struct test_spec *test, unsigned long *bitmap) static int send_pkts(struct test_spec *test, struct ifobject *ifobject) { - bool timeout = !is_umem_valid(test->ifobj_rx); DECLARE_BITMAP(bitmap, test->nb_sockets); u32 i, ret; @@ -1318,19 +1330,18 @@ static int send_pkts(struct test_spec *test, struct ifobject *ifobject) __set_bit(i, bitmap); continue; } - ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], timeout); - if (ret == TEST_CONTINUE && !test->fail) - continue; - - if ((ret || test->fail) && !timeout) - return TEST_FAILURE; - - if (ret == TEST_PASS && timeout) + ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], test->poll_tmout); + if (ret != TEST_CONTINUE) return ret; - ret = wait_for_tx_completion(&ifobject->xsk_arr[i]); - if (ret) + if (test->fail) return TEST_FAILURE; + + if (!test->poll_tmout) { + ret = wait_for_tx_completion(&ifobject->xsk_arr[i]); + if (ret) + return TEST_FAILURE; + } } } @@ -1488,14 +1499,25 @@ static int xsk_configure(struct test_spec *test, struct ifobject *ifobject, static int thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject) { - int ret = xsk_configure(test, ifobject, test->ifobj_rx->umem, true); + struct xsk_umem_info *umem_rx, *umem_tx; + int ret; + + if (!test->ifobj_rx || !test->ifobj_rx->xsk_arr[0].umem->umem) { + ksft_print_msg("Error: RX UMEM is not initialized before shared-UMEM TX setup\n"); + return -EINVAL; + } + umem_rx = test->ifobj_rx->xsk_arr[0].umem; + umem_tx = ifobject->xsk_arr[0].umem_real; + memcpy(umem_tx, umem_rx, sizeof(*umem_tx)); + umem_tx->base_addr = 0; + umem_tx->next_buffer = 0; + + ret = xsk_configure(test, ifobject, umem_rx, true); if (ret) return ret; ifobject->xsk = &ifobject->xsk_arr[0]; ifobject->xskmap = test->ifobj_rx->xskmap; - memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info)); - ifobject->umem->base_addr = 0; return 0; } @@ -1548,31 +1570,37 @@ static int xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream static int thread_common_ops(struct test_spec *test, struct ifobject *ifobject) { + struct xsk_umem_info *umem = ifobject->xsk->umem; LIBBPF_OPTS(bpf_xdp_query_opts, opts); int mmap_flags; - u64 umem_sz; + u64 umem_sz, mmap_sz; void *bufs; int ret; u32 i; - umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size; + umem_sz = umem_size(umem); mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; - if (ifobject->umem->unaligned_mode) + if (umem->unaligned_mode) mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB; if (ifobject->shared_umem) umem_sz *= 2; - bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); + mmap_sz = umem->unaligned_mode ? + ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE : umem_sz; + + bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); if (bufs == MAP_FAILED) return -errno; - ret = xsk_configure_umem(ifobject, ifobject->umem, bufs, umem_sz); + umem->mmap_size = mmap_sz; + + ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz); if (ret) return ret; - ret = xsk_configure(test, ifobject, ifobject->umem, false); + ret = xsk_configure(test, ifobject, umem, false); if (ret) return ret; @@ -1581,14 +1609,13 @@ static int thread_common_ops(struct test_spec *test, struct ifobject *ifobject) if (!ifobject->rx_on) return 0; - ret = xsk_populate_fill_ring(ifobject->umem, ifobject->xsk->pkt_stream, + ret = xsk_populate_fill_ring(umem, ifobject->xsk->pkt_stream, ifobject->use_fill_ring); if (ret) return ret; for (i = 0; i < test->nb_sockets; i++) { - ifobject->xsk = &ifobject->xsk_arr[i]; - ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, i); + ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk_arr[i].xsk, i); if (ret) return ret; } @@ -1642,7 +1669,8 @@ void *worker_testapp_validate_rx(void *arg) strerror(-err)); } - pthread_barrier_wait(&barr); + if (test->use_barrier) + pthread_barrier_wait(&barr); /* We leave only now in case of error to avoid getting stuck in the barrier */ if (err) { @@ -1675,19 +1703,10 @@ void *worker_testapp_validate_rx(void *arg) static void testapp_clean_xsk_umem(struct ifobject *ifobj) { - u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size; - - if (ifobj->shared_umem) - umem_sz *= 2; - - umem_sz = ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; - xsk_umem__delete(ifobj->umem->umem); - munmap(ifobj->umem->buffer, umem_sz); -} + struct xsk_umem_info *umem = ifobj->xsk->umem; -static void handler(int signum) -{ - pthread_exit(NULL); + xsk_umem__delete(umem->umem); + munmap(umem->buffer, umem->mmap_size); } static bool xdp_prog_changed_rx(struct test_spec *test) @@ -1794,9 +1813,18 @@ static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *i return TEST_FAILURE; } - if (ifobj2) { + err = xsk_attach_xdp_progs(test, ifobj1, ifobj2); + if (err) { + ksft_print_msg("Error: failed to attach XDP programs: %d (%s)\n", + err, strerror(-err)); + return TEST_FAILURE; + } + test->use_barrier = !!ifobj2; + + if (test->use_barrier) { if (pthread_barrier_init(&barr, NULL, 2)) return TEST_FAILURE; + pkt_stream_reset(ifobj2->xsk->pkt_stream); } @@ -1804,29 +1832,27 @@ static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *i pkt_stream_reset(ifobj1->xsk->pkt_stream); pkts_in_flight = 0; - signal(SIGUSR1, handler); /*Spawn RX thread */ pthread_create(&t0, NULL, ifobj1->func_ptr, test); - if (ifobj2) { + if (test->use_barrier) { pthread_barrier_wait(&barr); if (pthread_barrier_destroy(&barr)) { - pthread_kill(t0, SIGUSR1); + test->use_barrier = false; + pthread_join(t0, NULL); clean_sockets(test, ifobj1); clean_umem(test, ifobj1, NULL); return TEST_FAILURE; } + } + if (ifobj2) { /*Spawn TX thread */ pthread_create(&t1, NULL, ifobj2->func_ptr, test); - pthread_join(t1, NULL); } - if (!ifobj2) - pthread_kill(t0, SIGUSR1); - else - pthread_join(t0, NULL); + pthread_join(t0, NULL); if (test->total_steps == test->current_step || test->fail) { clean_sockets(test, ifobj1); @@ -1845,8 +1871,8 @@ static int testapp_validate_traffic(struct test_spec *test) struct ifobject *ifobj_rx = test->ifobj_rx; struct ifobject *ifobj_tx = test->ifobj_tx; - if ((ifobj_rx->umem->unaligned_mode && !ifobj_rx->unaligned_supp) || - (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) { + if ((ifobj_rx->xsk->umem->unaligned_mode && !ifobj_rx->unaligned_supp) || + (ifobj_tx->xsk->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) { ksft_print_msg("No huge pages present.\n"); return TEST_SKIP; } @@ -1863,8 +1889,6 @@ static int testapp_validate_traffic(struct test_spec *test) } } - if (xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx)) - return TEST_FAILURE; return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx); } @@ -1953,12 +1977,13 @@ int testapp_xdp_prog_cleanup(struct test_spec *test) int testapp_headroom(struct test_spec *test) { - test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE; + test->ifobj_rx->xsk->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE; return testapp_validate_traffic(test); } int testapp_stats_rx_dropped(struct test_spec *test) { + struct xsk_umem_info *umem = test->ifobj_rx->xsk->umem; u32 umem_tr = test->ifobj_tx->umem_tailroom; if (test->mode == TEST_MODE_ZC) { @@ -1968,7 +1993,7 @@ int testapp_stats_rx_dropped(struct test_spec *test) if (pkt_stream_replace_half(test, (MIN_PKT_SIZE * 3) + umem_tr, 0)) return TEST_FAILURE; - test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size - + umem->frame_headroom = umem->frame_size - XDP_PACKET_HEADROOM - (MIN_PKT_SIZE * 2) - umem_tr; if (pkt_stream_receive_half(test)) return TEST_FAILURE; @@ -2025,8 +2050,7 @@ int testapp_stats_fill_empty(struct test_spec *test) int testapp_send_receive_unaligned(struct test_spec *test) { - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; + test_spec_set_unaligned(test); /* Let half of the packets straddle a 4K buffer boundary */ if (pkt_stream_replace_half(test, MIN_PKT_SIZE, -MIN_PKT_SIZE / 2)) return TEST_FAILURE; @@ -2037,8 +2061,7 @@ int testapp_send_receive_unaligned(struct test_spec *test) int testapp_send_receive_unaligned_mb(struct test_spec *test) { test->mtu = MAX_ETH_JUMBO_SIZE; - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; + test_spec_set_unaligned(test); if (pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE)) return TEST_FAILURE; return testapp_validate_traffic(test); @@ -2064,8 +2087,8 @@ int testapp_send_receive_mb(struct test_spec *test) int testapp_invalid_desc_mb(struct test_spec *test) { - struct xsk_umem_info *umem = test->ifobj_tx->umem; - u64 umem_size = umem->num_frames * umem->frame_size; + struct xsk_umem_info *umem = test->ifobj_tx->xsk->umem; + u64 umem_sz = umem_size(umem); struct pkt pkts[] = { /* Valid packet for synch to start with */ {0, MIN_PKT_SIZE, 0, true, 0}, @@ -2075,13 +2098,16 @@ int testapp_invalid_desc_mb(struct test_spec *test) {0, 0, 0, false, 0}, /* Invalid address in the second frame */ {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - {umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {umem_sz * 2, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {0, MIN_PKT_SIZE, 0, false, 0}, /* Invalid len in the middle */ {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {0, MIN_PKT_SIZE, 0, false, 0}, /* Invalid options in the middle */ {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION}, + {0, MIN_PKT_SIZE, 0, false, 0}, /* Transmit 2 frags, receive 3 */ {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD}, {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0}, @@ -2093,8 +2119,8 @@ int testapp_invalid_desc_mb(struct test_spec *test) if (umem->unaligned_mode) { /* Crossing a chunk boundary allowed */ - pkts[12].valid = true; - pkts[13].valid = true; + pkts[15].valid = true; + pkts[16].valid = true; } test->mtu = MAX_ETH_JUMBO_SIZE; @@ -2105,8 +2131,8 @@ int testapp_invalid_desc_mb(struct test_spec *test) int testapp_invalid_desc(struct test_spec *test) { - struct xsk_umem_info *umem = test->ifobj_tx->umem; - u64 umem_size = umem->num_frames * umem->frame_size; + struct xsk_umem_info *umem = test->ifobj_tx->xsk->umem; + u64 umem_sz = umem_size(umem); struct pkt pkts[] = { /* Zero packet address allowed */ {0, MIN_PKT_SIZE, 0, true}, @@ -2117,11 +2143,11 @@ int testapp_invalid_desc(struct test_spec *test) /* Packet too large */ {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false}, /* Up to end of umem allowed */ - {umem_size - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true}, + {umem_sz - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true}, /* After umem ends */ - {umem_size, MIN_PKT_SIZE, 0, false}, + {umem_sz, MIN_PKT_SIZE, 0, false}, /* Straddle the end of umem */ - {umem_size - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, + {umem_sz - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, /* Straddle a 4K boundary */ {0x1000 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, /* Straddle a 2K boundary */ @@ -2139,9 +2165,9 @@ int testapp_invalid_desc(struct test_spec *test) } if (test->ifobj_tx->shared_umem) { - pkts[4].offset += umem_size; - pkts[5].offset += umem_size; - pkts[6].offset += umem_size; + pkts[4].offset += umem_sz; + pkts[5].offset += umem_sz; + pkts[6].offset += umem_sz; } if (pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts))) @@ -2202,16 +2228,33 @@ int testapp_xdp_shared_umem(struct test_spec *test) int testapp_poll_txq_tmout(struct test_spec *test) { + bool shared_umem = test->ifobj_tx->shared_umem; + int ret; + + test->poll_tmout = true; + /* + * POLL_TXQ_FULL exercises TX timeout setup in isolation. + * Keep TX out of shared-UMEM mode here so TX setup does not require + * RX UMEM to be initialized first. + */ + test->ifobj_tx->shared_umem = false; test->ifobj_tx->use_poll = true; /* create invalid frame by set umem frame_size and pkt length equal to 2048 */ - test->ifobj_tx->umem->frame_size = 2048; - if (pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048)) + test->ifobj_tx->xsk->umem->frame_size = 2048; + if (pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048)) { + test->ifobj_tx->shared_umem = shared_umem; return TEST_FAILURE; - return testapp_validate_traffic_single_thread(test, test->ifobj_tx); + } + + ret = testapp_validate_traffic_single_thread(test, test->ifobj_tx); + test->ifobj_tx->shared_umem = shared_umem; + + return ret; } int testapp_poll_rxq_tmout(struct test_spec *test) { + test->poll_tmout = true; test->ifobj_rx->use_poll = true; return testapp_validate_traffic_single_thread(test, test->ifobj_rx); } @@ -2229,7 +2272,7 @@ int testapp_too_many_frags(struct test_spec *test) max_frags += 1; } - pkts = calloc(2 * max_frags + 2, sizeof(struct pkt)); + pkts = calloc(2 * max_frags + 3, sizeof(struct pkt)); if (!pkts) return TEST_FAILURE; @@ -2247,24 +2290,30 @@ int testapp_too_many_frags(struct test_spec *test) } pkts[max_frags].options = 0; - /* An invalid packet with the max amount of frags but signals packet - * continues on the last frag - */ - for (i = max_frags + 1; i < 2 * max_frags + 1; i++) { + /* An invalid packet with the max + 1 amount of frags */ + for (i = max_frags + 1; i < 2 * max_frags + 2; i++) { pkts[i].len = MIN_PKT_SIZE; pkts[i].options = XDP_PKT_CONTD; - pkts[i].valid = false; + pkts[i].valid = true; } + pkts[2 * max_frags + 1].options = 0; /* Valid packet for synch */ - pkts[2 * max_frags + 1].len = MIN_PKT_SIZE; - pkts[2 * max_frags + 1].valid = true; + pkts[2 * max_frags + 2].len = MIN_PKT_SIZE; + pkts[2 * max_frags + 2].valid = true; - if (pkt_stream_generate_custom(test, pkts, 2 * max_frags + 2)) { + if (pkt_stream_generate_custom(test, pkts, 2 * max_frags + 3)) { free(pkts); return TEST_FAILURE; } + /* The generated Tx stream must keep the too-big packet valid so that + * __send_pkts() accounts its descriptors in outstanding_tx. The Rx + * stream, however, must not expect this packet on the wire. + */ + test->ifobj_rx->xsk->pkt_stream->pkts[2].valid = false; + test->ifobj_rx->xsk->pkt_stream->nb_valid_entries--; + ret = testapp_validate_traffic(test); free(pkts); return ret; @@ -2337,8 +2386,7 @@ int testapp_send_receive(struct test_spec *test) int testapp_send_receive_2k_frame(struct test_spec *test) { - test->ifobj_tx->umem->frame_size = 2048; - test->ifobj_rx->umem->frame_size = 2048; + test_spec_set_frame_size(test, 2048); if (pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE)) return TEST_FAILURE; return testapp_validate_traffic(test); @@ -2363,34 +2411,30 @@ int testapp_aligned_inv_desc(struct test_spec *test) int testapp_aligned_inv_desc_2k_frame(struct test_spec *test) { - test->ifobj_tx->umem->frame_size = 2048; - test->ifobj_rx->umem->frame_size = 2048; + test_spec_set_frame_size(test, 2048); return testapp_invalid_desc(test); } int testapp_unaligned_inv_desc(struct test_spec *test) { - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; + test_spec_set_unaligned(test); return testapp_invalid_desc(test); } int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test) { - u64 page_size, umem_size; + u64 page_size, umem_sz; /* Odd frame size so the UMEM doesn't end near a page boundary. */ - test->ifobj_tx->umem->frame_size = 4001; - test->ifobj_rx->umem->frame_size = 4001; - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; + test_spec_set_frame_size(test, 4001); + test_spec_set_unaligned(test); /* This test exists to test descriptors that staddle the end of * the UMEM but not a page. */ page_size = sysconf(_SC_PAGESIZE); - umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size; - assert(umem_size % page_size > MIN_PKT_SIZE); - assert(umem_size % page_size < page_size - MIN_PKT_SIZE); + umem_sz = umem_size(test->ifobj_tx->xsk->umem); + assert(umem_sz % page_size > MIN_PKT_SIZE); + assert(umem_sz % page_size < page_size - MIN_PKT_SIZE); return testapp_invalid_desc(test); } @@ -2402,8 +2446,7 @@ int testapp_aligned_inv_desc_mb(struct test_spec *test) int testapp_unaligned_inv_desc_mb(struct test_spec *test) { - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; + test_spec_set_unaligned(test); return testapp_invalid_desc_mb(test); } @@ -2447,9 +2490,9 @@ int testapp_hw_sw_max_ring_size(struct test_spec *test) test->total_steps = 2; test->ifobj_tx->ring.tx_pending = test->ifobj_tx->ring.tx_max_pending; test->ifobj_tx->ring.rx_pending = test->ifobj_tx->ring.rx_max_pending; - test->ifobj_rx->umem->num_frames = max_descs; - test->ifobj_rx->umem->fill_size = max_descs; - test->ifobj_rx->umem->comp_size = max_descs; + test->ifobj_rx->xsk->umem->num_frames = max_descs; + test->ifobj_rx->xsk->umem->fill_size = max_descs; + test->ifobj_rx->xsk->umem->comp_size = max_descs; test->ifobj_tx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; test->ifobj_rx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; @@ -2590,8 +2633,8 @@ struct ifobject *ifobject_create(void) if (!ifobj->xsk_arr) goto out_xsk_arr; - ifobj->umem = calloc(1, sizeof(*ifobj->umem)); - if (!ifobj->umem) + ifobj->xsk_arr[0].umem_real = calloc(1, sizeof(struct xsk_umem_info)); + if (!ifobj->xsk_arr[0].umem_real) goto out_umem; return ifobj; @@ -2605,7 +2648,9 @@ out_xsk_arr: void ifobject_delete(struct ifobject *ifobj) { - free(ifobj->umem); + if (ifobj->xsk_arr) + free(ifobj->xsk_arr[0].umem_real); + free(ifobj->xsk_arr); free(ifobj); } diff --git a/tools/testing/selftests/bpf/prog_tests/test_xsk.h b/tools/testing/selftests/bpf/prog_tests/test_xsk.h index 1ab8aee4ce56..03753ddc5dcd 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_xsk.h +++ b/tools/testing/selftests/bpf/prog_tests/test_xsk.h @@ -83,6 +83,7 @@ typedef int (*test_func_t)(struct test_spec *test); struct xsk_socket_info { struct xsk_ring_cons rx; struct xsk_ring_prod tx; + struct xsk_umem_info *umem_real; struct xsk_umem_info *umem; struct xsk_socket *xsk; struct pkt_stream *pkt_stream; @@ -102,6 +103,7 @@ struct xsk_umem_info { struct xsk_ring_cons cq; struct xsk_umem *umem; u64 next_buffer; + u64 mmap_size; u32 num_frames; u32 frame_headroom; void *buffer; @@ -123,7 +125,6 @@ struct ifobject { char ifname[MAX_INTERFACE_NAME_CHARS]; struct xsk_socket_info *xsk; struct xsk_socket_info *xsk_arr; - struct xsk_umem_info *umem; thread_func_t func_ptr; validation_func_t validation_func; struct xsk_xdp_progs *xdp_progs; @@ -206,6 +207,8 @@ struct test_spec { bool set_ring; bool adjust_tail; bool adjust_tail_support; + bool poll_tmout; + bool use_barrier; enum test_mode mode; char name[MAX_TEST_NAME_SIZE]; }; diff --git a/tools/testing/selftests/bpf/prog_tests/timer_mim.c b/tools/testing/selftests/bpf/prog_tests/timer_mim.c index c930c7d7105b..fa7bb769ca31 100644 --- a/tools/testing/selftests/bpf/prog_tests/timer_mim.c +++ b/tools/testing/selftests/bpf/prog_tests/timer_mim.c @@ -59,10 +59,32 @@ void serial_test_timer_mim(void) int err; old_print_fn = libbpf_set_print(NULL); - timer_reject_skel = timer_mim_reject__open_and_load(); - libbpf_set_print(old_print_fn); - if (!ASSERT_ERR_PTR(timer_reject_skel, "timer_reject_skel_load")) + timer_reject_skel = timer_mim_reject__open(); + if (!ASSERT_OK_PTR(timer_reject_skel, "timer_reject_skel_open")) + goto cleanup; + bpf_program__set_autoload(timer_reject_skel->progs.test1, true); + err = timer_mim_reject__load(timer_reject_skel); + ASSERT_ERR(err, "timer_reject_skel_load"); + timer_mim_reject__destroy(timer_reject_skel); + + timer_reject_skel = timer_mim_reject__open(); + if (!ASSERT_OK_PTR(timer_reject_skel, "callback_reject_skel_open")) goto cleanup; + bpf_program__set_autoload(timer_reject_skel->progs.callback_map_uid_mismatch, true); + err = timer_mim_reject__load(timer_reject_skel); + ASSERT_ERR(err, "callback_reject_skel_load"); + timer_mim_reject__destroy(timer_reject_skel); + + timer_reject_skel = timer_mim_reject__open(); + if (!ASSERT_OK_PTR(timer_reject_skel, "callback_accept_skel_open")) + goto cleanup; + bpf_program__set_autoload(timer_reject_skel->progs.callback_map_uid_match, true); + err = timer_mim_reject__load(timer_reject_skel); + if (!ASSERT_OK(err, "callback_accept_skel_load")) + goto cleanup; + timer_mim_reject__destroy(timer_reject_skel); + timer_reject_skel = NULL; + libbpf_set_print(old_print_fn); timer_skel = timer_mim__open_and_load(); if (!timer_skel && errno == EOPNOTSUPP) { @@ -75,6 +97,7 @@ void serial_test_timer_mim(void) err = timer_mim(timer_skel); ASSERT_OK(err, "timer_mim"); cleanup: + libbpf_set_print(old_print_fn); timer_mim__destroy(timer_skel); timer_mim_reject__destroy(timer_reject_skel); } diff --git a/tools/testing/selftests/bpf/prog_tests/token.c b/tools/testing/selftests/bpf/prog_tests/token.c index b81dde283052..f2f5d36ae00a 100644 --- a/tools/testing/selftests/bpf/prog_tests/token.c +++ b/tools/testing/selftests/bpf/prog_tests/token.c @@ -1,9 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ #define _GNU_SOURCE -#include <test_progs.h> #include <bpf/btf.h> -#include "cap_helpers.h" #include <fcntl.h> #include <sched.h> #include <signal.h> @@ -15,9 +13,17 @@ #include <sys/stat.h> #include <sys/syscall.h> #include <sys/un.h> + +#include "bpf_util.h" +#include "cap_helpers.h" +#include "sysctl_helpers.h" +#include "test_progs.h" +#include "trace_helpers.h" + #include "priv_map.skel.h" #include "priv_prog.skel.h" #include "dummy_st_ops_success.skel.h" +#include "token_kallsyms.skel.h" #include "token_lsm.skel.h" #include "priv_freplace_prog.skel.h" @@ -1045,6 +1051,58 @@ err_out: return -EINVAL; } +static bool kallsyms_has_bpf_func(struct ksyms *ksyms, const char *func_name) +{ + char name[256]; + int i; + + for (i = 0; i < ksyms->sym_cnt; i++) { + if (sscanf(ksyms->syms[i].name, "bpf_prog_%*[^_]_%255s", name) == 1 && + strcmp(name, func_name) == 0) + return true; + } + return false; +} + +static int userns_obj_priv_prog_kallsyms(int mnt_fd, struct token_lsm *lsm_skel) +{ + const char *func_names[] = { "xdp_main", "token_ksym_subprog" }; + LIBBPF_OPTS(bpf_object_open_opts, opts); + struct token_kallsyms *skel; + struct ksyms *ksyms = NULL; + char buf[256]; + int i, err; + + snprintf(buf, sizeof(buf), "/proc/self/fd/%d", mnt_fd); + opts.bpf_token_path = buf; + skel = token_kallsyms__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "token_kallsyms__open_opts")) + return -EINVAL; + + err = token_kallsyms__load(skel); + if (!ASSERT_OK(err, "token_kallsyms__load")) + goto cleanup; + + ksyms = load_kallsyms_local(); + if (!ASSERT_OK_PTR(ksyms, "load_kallsyms_local")) { + err = -EINVAL; + goto cleanup; + } + + for (i = 0; i < ARRAY_SIZE(func_names); i++) { + if (!ASSERT_TRUE(kallsyms_has_bpf_func(ksyms, func_names[i]), + func_names[i])) { + err = -EINVAL; + break; + } + } + +cleanup: + free_kallsyms_local(ksyms); + token_kallsyms__destroy(skel); + return err; +} + #define bit(n) (1ULL << (n)) static int userns_bpf_token_info(int mnt_fd, struct token_lsm *lsm_skel) @@ -1082,7 +1140,7 @@ cleanup: return err; } -void test_token(void) +void serial_test_token(void) { if (test__start_subtest("map_token")) { struct bpffs_opts opts = { @@ -1194,4 +1252,26 @@ void test_token(void) subtest_userns(&opts, userns_bpf_token_info); } + if (test__start_subtest("obj_priv_prog_kallsyms")) { + char perf_paranoid_orig[32] = {}; + char kptr_restrict_orig[32] = {}; + struct bpffs_opts opts = { + .cmds = bit(BPF_BTF_LOAD) | bit(BPF_PROG_LOAD), + .progs = bit(BPF_PROG_TYPE_XDP), + .attachs = ~0ULL, + }; + + if (sysctl_set_or_fail("/proc/sys/kernel/perf_event_paranoid", perf_paranoid_orig, "0")) + goto cleanup; + if (sysctl_set_or_fail("/proc/sys/kernel/kptr_restrict", kptr_restrict_orig, "0")) + goto cleanup; + + subtest_userns(&opts, userns_obj_priv_prog_kallsyms); + +cleanup: + if (perf_paranoid_orig[0]) + sysctl_set_or_fail("/proc/sys/kernel/perf_event_paranoid", NULL, perf_paranoid_orig); + if (kptr_restrict_orig[0]) + sysctl_set_or_fail("/proc/sys/kernel/kptr_restrict", NULL, kptr_restrict_orig); + } } diff --git a/tools/testing/selftests/bpf/prog_tests/tp_btf_ids.c b/tools/testing/selftests/bpf/prog_tests/tp_btf_ids.c new file mode 100644 index 000000000000..c0e7e11e71b8 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tp_btf_ids.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <bpf/btf.h> + +#define TRACEFS "/sys/kernel/tracing" +#define DEBUGFS_TRACING "/sys/kernel/debug/tracing" +#define EVENT_SUBPATH "events/bpf_testmod/bpf_testmod_test_read/btf_ids" + +struct btf_ids_info { + __u32 obj_id; + __u32 raw_id; + __u32 tp_id; +}; + +static const char *btf_ids_path(char *buf, size_t sz) +{ + if (access(TRACEFS "/trace", F_OK) == 0) + snprintf(buf, sz, "%s/%s", TRACEFS, EVENT_SUBPATH); + else + snprintf(buf, sz, "%s/%s", DEBUGFS_TRACING, EVENT_SUBPATH); + return buf; +} + +static int read_btf_ids(struct btf_ids_info *info) +{ + char path[256], buf[256]; + int fd, n; + + fd = open(btf_ids_path(path, sizeof(path)), O_RDONLY); + if (fd < 0) + return -errno; + + n = read(fd, buf, sizeof(buf) - 1); + close(fd); + if (n <= 0) + return -EIO; + buf[n] = '\0'; + + if (sscanf(buf, + "btf_obj_id: %u\nraw_btf_id: %u\ntp_btf_id: %u\n", + &info->obj_id, &info->raw_id, &info->tp_id) != 3) + return -EINVAL; + return 0; +} + +static const char *param_name(struct btf *btf, const struct btf_param *p) +{ + return btf__name_by_offset(btf, p->name_off); +} + +static const char *member_name(struct btf *btf, const struct btf_member *m) +{ + return btf__name_by_offset(btf, m->name_off); +} + +void test_tp_btf_ids(void) +{ + const struct btf_type *proto_t, *rec_t; + const struct btf_param *params; + const struct btf_member *members; + struct btf_ids_info info; + struct btf *vmlinux_btf, *btf; + const char *name; + int err; + + if (!env.has_testmod) { + test__skip(); + return; + } + + err = read_btf_ids(&info); + if (!ASSERT_OK(err, "read btf_ids")) + return; + + ASSERT_GT(info.obj_id, 0, "obj_id non-zero"); + ASSERT_GT(info.raw_id, 0, "raw_id non-zero"); + ASSERT_GT(info.tp_id, 0, "tp_id non-zero"); + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(vmlinux_btf, "load vmlinux BTF")) + return; + + /* Module BTF is split BTF; load with vmlinux as base. */ + btf = btf__load_from_kernel_by_id_split(info.obj_id, vmlinux_btf); + if (!ASSERT_OK_PTR(btf, "load module BTF")) { + btf__free(vmlinux_btf); + return; + } + + /* + * raw_btf_id should be the FUNC_PROTO of __bpf_trace_<call>: + * void *__data, struct task_struct *task, + * struct bpf_testmod_test_read_ctx *ctx + */ + proto_t = btf__type_by_id(btf, info.raw_id); + if (!ASSERT_OK_PTR(proto_t, "raw type_by_id")) + goto out; + if (!ASSERT_TRUE(btf_is_func_proto(proto_t), "raw is FUNC_PROTO")) + goto out; + if (!ASSERT_EQ(btf_vlen(proto_t), 3, "func_proto arg count")) + goto out; + + params = btf_params(proto_t); + ASSERT_STREQ(param_name(btf, ¶ms[0]), "__data", "arg0 name"); + ASSERT_STREQ(param_name(btf, ¶ms[1]), "task", "arg1 name"); + ASSERT_STREQ(param_name(btf, ¶ms[2]), "ctx", "arg2 name"); + + /* + * tp_btf_id should be STRUCT trace_event_raw_<call> with the + * fields declared by TP_STRUCT__entry plus the common header. + */ + rec_t = btf__type_by_id(btf, info.tp_id); + if (!ASSERT_OK_PTR(rec_t, "tp type_by_id")) + goto out; + if (!ASSERT_TRUE(btf_is_struct(rec_t), "tp is STRUCT")) + goto out; + name = btf__name_by_offset(btf, rec_t->name_off); + ASSERT_STREQ(name, "trace_event_raw_bpf_testmod_test_read", + "tp struct name"); + if (!ASSERT_GE(btf_vlen(rec_t), 5, "tp struct field count")) + goto out; + + members = btf_members(rec_t); + ASSERT_STREQ(member_name(btf, &members[0]), "ent", "field0 name"); + ASSERT_STREQ(member_name(btf, &members[1]), "pid", "field1 name"); + ASSERT_STREQ(member_name(btf, &members[2]), "comm", "field2 name"); + ASSERT_STREQ(member_name(btf, &members[3]), "off", "field3 name"); + ASSERT_STREQ(member_name(btf, &members[4]), "len", "field4 name"); +out: + btf__free(btf); + btf__free(vmlinux_btf); +} diff --git a/tools/testing/selftests/bpf/prog_tests/trace_printk.c b/tools/testing/selftests/bpf/prog_tests/trace_printk.c index e56e88596d64..a5a8104c1ddd 100644 --- a/tools/testing/selftests/bpf/prog_tests/trace_printk.c +++ b/tools/testing/selftests/bpf/prog_tests/trace_printk.c @@ -6,18 +6,21 @@ #include "trace_printk.lskel.h" #define SEARCHMSG "testing,testing" +#define SEARCHMSG_UTF8 "ä¸æ–‡,测试" static void trace_pipe_cb(const char *str, void *data) { if (strstr(str, SEARCHMSG) != NULL) - (*(int *)data)++; + ((int *)data)[0]++; + if (strstr(str, SEARCHMSG_UTF8)) + ((int *)data)[1]++; } void serial_test_trace_printk(void) { struct trace_printk_lskel__bss *bss; struct trace_printk_lskel *skel; - int err = 0, found = 0; + int err = 0, found[2] = {}; skel = trace_printk_lskel__open(); if (!ASSERT_OK_PTR(skel, "trace_printk__open")) @@ -46,11 +49,24 @@ void serial_test_trace_printk(void) if (!ASSERT_GT(bss->trace_printk_ret, 0, "bss->trace_printk_ret")) goto cleanup; - /* verify our search string is in the trace buffer */ - ASSERT_OK(read_trace_pipe_iter(trace_pipe_cb, &found, 1000), - "read_trace_pipe_iter"); + if (!ASSERT_GT(bss->trace_printk_utf8_ran, 0, "bss->trace_printk_utf8_ran")) + goto cleanup; + + if (!ASSERT_GT(bss->trace_printk_utf8_ret, 0, "bss->trace_printk_utf8_ret")) + goto cleanup; + + if (!ASSERT_LT(bss->trace_printk_invalid_spec_ret, 0, + "bss->trace_printk_invalid_spec_ret")) + goto cleanup; + + /* verify our search strings are in the trace buffer */ + ASSERT_OK(read_trace_pipe_iter(trace_pipe_cb, found, 1000), + "read_trace_pipe_iter"); + + if (!ASSERT_EQ(found[0], bss->trace_printk_ran, "found")) + goto cleanup; - if (!ASSERT_EQ(found, bss->trace_printk_ran, "found")) + if (!ASSERT_EQ(found[1], bss->trace_printk_utf8_ran, "found_utf8")) goto cleanup; cleanup: diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_failure.c b/tools/testing/selftests/bpf/prog_tests/tracing_failure.c index f9f9e1cb87bf..eb585918f0d4 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_failure.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_failure.c @@ -76,6 +76,24 @@ static void test_fexit_noreturns(void) "Attaching fexit/fsession/fmod_ret to __noreturn function 'do_exit' is rejected."); } +static void test_fexit_int128_ret(void) +{ + /* + * __int128 is returned in a register pair on x86_64 and arm64, so + * bpf_testmod_test_int128_ret() is BTF-encoded and attachable and the + * verifier can reject its >8 byte return value. Other architectures + * return a __int128 differently (e.g. s390x returns larger values by + * reference, which makes pahole skip BTF encoding of the function), so + * only exercise this on x86_64 and arm64. + */ +#if defined(__x86_64__) || defined(__aarch64__) + test_tracing_fail_prog("fexit_int128_ret", + "with a >8 byte return value is not supported for this attach type"); +#else + test__skip(); +#endif +} + void test_tracing_failure(void) { if (test__start_subtest("bpf_spin_lock")) @@ -86,4 +104,6 @@ void test_tracing_failure(void) test_tracing_deny(); if (test__start_subtest("fexit_noreturns")) test_fexit_noreturns(); + if (test__start_subtest("fexit_int128_ret")) + test_fexit_int128_ret(); } diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c new file mode 100644 index 000000000000..0aa9532a05cf --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c @@ -0,0 +1,1029 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include <bpf/btf.h> +#include <search.h> +#include "bpf/libbpf_internal.h" +#include "tracing_multi.skel.h" +#include "tracing_multi_module.skel.h" +#include "tracing_multi_intersect.skel.h" +#include "tracing_multi_session.skel.h" +#include "tracing_multi_fail.skel.h" +#include "tracing_multi_verifier.skel.h" +#include "tracing_multi_bench.skel.h" +#include "tracing_multi_rollback.skel.h" +#include "trace_helpers.h" + +static __u64 bpf_fentry_test_cookies[] = { + 8, /* bpf_fentry_test1 */ + 9, /* bpf_fentry_test2 */ + 7, /* bpf_fentry_test3 */ + 5, /* bpf_fentry_test4 */ + 4, /* bpf_fentry_test5 */ + 2, /* bpf_fentry_test6 */ + 3, /* bpf_fentry_test7 */ + 1, /* bpf_fentry_test8 */ + 10, /* bpf_fentry_test9 */ + 6, /* bpf_fentry_test10 */ +}; + +static const char * const bpf_fentry_test[] = { + "bpf_fentry_test1", + "bpf_fentry_test2", + "bpf_fentry_test3", + "bpf_fentry_test4", + "bpf_fentry_test5", + "bpf_fentry_test6", + "bpf_fentry_test7", + "bpf_fentry_test8", + "bpf_fentry_test9", + "bpf_fentry_test10", +}; + +static const char * const bpf_testmod_fentry_test[] = { + "bpf_testmod_fentry_test1", + "bpf_testmod_fentry_test2", + "bpf_testmod_fentry_test3", + "bpf_testmod_fentry_test7", + "bpf_testmod_fentry_test11", +}; + +#define FUNCS_CNT (ARRAY_SIZE(bpf_fentry_test)) + +static int get_random_funcs(const char **funcs) +{ + int i, cnt = 0; + + for (i = 0; i < FUNCS_CNT; i++) { + if (rand() % 2) + funcs[cnt++] = bpf_fentry_test[i]; + } + /* we always need at least one.. */ + if (!cnt) + funcs[cnt++] = bpf_fentry_test[rand() % FUNCS_CNT]; + return cnt; +} + +static int compare(const void *ppa, const void *ppb) +{ + const char *pa = *(const char **) ppa; + const char *pb = *(const char **) ppb; + + return strcmp(pa, pb); +} + +static void tdestroy_free_nop(void *ptr) +{ +} + +static __u32 *get_ids(const char * const funcs[], int funcs_cnt, const char *mod) +{ + struct btf *btf, *vmlinux_btf = NULL; + __u32 nr, type_id, cnt = 0; + void *root = NULL; + __u32 *ids = NULL; + int i, err = 0; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "btf__load_vmlinux_btf")) + return NULL; + + if (mod) { + vmlinux_btf = btf; + btf = btf__load_module_btf(mod, vmlinux_btf); + if (!ASSERT_OK_PTR(btf, "btf__load_module_btf")) { + btf__free(vmlinux_btf); + return NULL; + } + } + + ids = calloc(funcs_cnt, sizeof(ids[0])); + if (!ids) + goto out; + + /* + * We sort function names by name and search them + * below for each function. + */ + for (i = 0; i < funcs_cnt; i++) { + if (!tsearch(&funcs[i], &root, compare)) { + ASSERT_FAIL("tsearch failed"); + err = -1; + goto error; + } + } + + nr = btf__type_cnt(btf); + for (type_id = 1; type_id < nr && cnt < funcs_cnt; type_id++) { + const struct btf_type *type; + const char *str, ***val; + unsigned int idx; + + type = btf__type_by_id(btf, type_id); + if (!type) { + err = -1; + break; + } + + if (BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) + continue; + + str = btf__name_by_offset(btf, type->name_off); + if (!str) { + err = -1; + break; + } + + val = tfind(&str, &root, compare); + if (!val) + continue; + + /* + * We keep pointer for each function name so we can get the original + * array index and have the resulting ids array matching the original + * function array. + * + * Doing it this way allow us to easily test the cookies support, + * because each cookie is attached to particular function/id. + */ + idx = *val - funcs; + ids[idx] = type_id; + cnt++; + } + +error: + if (err) { + free(ids); + ids = NULL; + } + +out: + tdestroy(root, tdestroy_free_nop); + btf__free(vmlinux_btf); + btf__free(btf); + return ids; +} + +static void tracing_multi_test_run(struct tracing_multi *skel) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, prog_fd; + + prog_fd = bpf_program__fd(skel->progs.test_fentry); + err = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(err, "test_run"); + + /* extra +1 count for sleepable programs */ + ASSERT_EQ(skel->bss->test_result_fentry, FUNCS_CNT + 1, "test_result_fentry"); + ASSERT_EQ(skel->bss->test_result_fexit, FUNCS_CNT + 1, "test_result_fexit"); +} + +static void test_skel_api(void) +{ + struct tracing_multi *skel; + int err; + + skel = tracing_multi__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi__open_and_load")) + return; + + skel->bss->pid = getpid(); + + err = tracing_multi__attach(skel); + if (!ASSERT_OK(err, "tracing_multi__attach")) + goto cleanup; + + tracing_multi_test_run(skel); + +cleanup: + tracing_multi__destroy(skel); +} + +static void test_link_api_pattern(void) +{ + struct tracing_multi *skel; + + skel = tracing_multi__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi__open_and_load")) + return; + + skel->bss->pid = getpid(); + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + "bpf_fentry_test*", NULL); + if (!ASSERT_OK_PTR(skel->links.test_fentry, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fexit = bpf_program__attach_tracing_multi(skel->progs.test_fexit, + "bpf_fentry_test*", NULL); + if (!ASSERT_OK_PTR(skel->links.test_fexit, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fentry_s = bpf_program__attach_tracing_multi(skel->progs.test_fentry_s, + "bpf_fentry_test1", NULL); + if (!ASSERT_OK_PTR(skel->links.test_fentry_s, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fexit_s = bpf_program__attach_tracing_multi(skel->progs.test_fexit_s, + "bpf_fentry_test1", NULL); + if (!ASSERT_OK_PTR(skel->links.test_fexit_s, "bpf_program__attach_tracing_multi")) + goto cleanup; + + tracing_multi_test_run(skel); + +cleanup: + tracing_multi__destroy(skel); +} + +static void test_link_api_ids(bool test_cookies) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tracing_multi *skel; + size_t cnt = FUNCS_CNT; + __u32 *ids; + + skel = tracing_multi__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi__open_and_load")) + return; + + skel->bss->pid = getpid(); + skel->bss->test_cookies = test_cookies; + + ids = get_ids(bpf_fentry_test, cnt, NULL); + if (!ASSERT_OK_PTR(ids, "get_ids")) + goto cleanup; + + opts.ids = ids; + opts.cnt = cnt; + + if (test_cookies) + opts.cookies = bpf_fentry_test_cookies; + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + NULL, &opts); + if (!ASSERT_OK_PTR(skel->links.test_fentry, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fexit = bpf_program__attach_tracing_multi(skel->progs.test_fexit, + NULL, &opts); + if (!ASSERT_OK_PTR(skel->links.test_fexit, "bpf_program__attach_tracing_multi")) + goto cleanup; + + /* Only bpf_fentry_test1 is allowed for sleepable programs. */ + opts.cnt = 1; + skel->links.test_fentry_s = bpf_program__attach_tracing_multi(skel->progs.test_fentry_s, + NULL, &opts); + if (!ASSERT_OK_PTR(skel->links.test_fentry_s, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fexit_s = bpf_program__attach_tracing_multi(skel->progs.test_fexit_s, + NULL, &opts); + if (!ASSERT_OK_PTR(skel->links.test_fexit_s, "bpf_program__attach_tracing_multi")) + goto cleanup; + + tracing_multi_test_run(skel); + +cleanup: + tracing_multi__destroy(skel); + free(ids); +} + +static void test_module_skel_api(void) +{ + struct tracing_multi_module *skel = NULL; + int err; + + skel = tracing_multi_module__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi__open_and_load")) + return; + + skel->bss->pid = getpid(); + + err = tracing_multi_module__attach(skel); + if (!ASSERT_OK(err, "tracing_multi__attach")) + goto cleanup; + + ASSERT_OK(trigger_module_test_read(1), "trigger_read"); + ASSERT_EQ(skel->bss->test_result_fentry, 5, "test_result_fentry"); + ASSERT_EQ(skel->bss->test_result_fexit, 5, "test_result_fexit"); + +cleanup: + tracing_multi_module__destroy(skel); +} + +static void test_module_link_api_pattern(void) +{ + struct tracing_multi_module *skel = NULL; + + skel = tracing_multi_module__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_module__open_and_load")) + return; + + skel->bss->pid = getpid(); + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + "bpf_testmod:bpf_testmod_fentry_test*", NULL); + if (!ASSERT_OK_PTR(skel->links.test_fentry, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fexit = bpf_program__attach_tracing_multi(skel->progs.test_fexit, + "bpf_testmod:bpf_testmod_fentry_test*", NULL); + if (!ASSERT_OK_PTR(skel->links.test_fexit, "bpf_program__attach_tracing_multi")) + goto cleanup; + + ASSERT_OK(trigger_module_test_read(1), "trigger_read"); + ASSERT_EQ(skel->bss->test_result_fentry, 5, "test_result_fentry"); + ASSERT_EQ(skel->bss->test_result_fexit, 5, "test_result_fexit"); + +cleanup: + tracing_multi_module__destroy(skel); +} + +static void test_module_link_api_ids(void) +{ + size_t cnt = ARRAY_SIZE(bpf_testmod_fentry_test); + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tracing_multi_module *skel = NULL; + __u32 *ids; + + skel = tracing_multi_module__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_module__open_and_load")) + return; + + skel->bss->pid = getpid(); + + ids = get_ids(bpf_testmod_fentry_test, cnt, "bpf_testmod"); + if (!ASSERT_OK_PTR(ids, "get_ids")) + goto cleanup; + + opts.ids = ids; + opts.cnt = cnt; + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + NULL, &opts); + if (!ASSERT_OK_PTR(skel->links.test_fentry, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fexit = bpf_program__attach_tracing_multi(skel->progs.test_fexit, + NULL, &opts); + if (!ASSERT_OK_PTR(skel->links.test_fexit, "bpf_program__attach_tracing_multi")) + goto cleanup; + + ASSERT_OK(trigger_module_test_read(1), "trigger_read"); + ASSERT_EQ(skel->bss->test_result_fentry, 5, "test_result_fentry"); + ASSERT_EQ(skel->bss->test_result_fexit, 5, "test_result_fexit"); + +cleanup: + tracing_multi_module__destroy(skel); + free(ids); +} + +static bool is_set(__u32 mask, __u32 bit) +{ + return (1 << bit) & mask; +} + +static void __test_intersect(__u32 mask, const struct bpf_program *progs[4], __u64 *test_results[4]) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct bpf_link *links[4] = { NULL }; + const char *funcs[FUNCS_CNT]; + __u64 expected[4]; + __u32 *ids, i; + int err, cnt; + + /* + * We have 4 programs in progs and the mask bits pick which + * of them gets attached to randomly chosen functions. + */ + for (i = 0; i < 4; i++) { + if (!is_set(mask, i)) + continue; + + cnt = get_random_funcs(funcs); + ids = get_ids(funcs, cnt, NULL); + if (!ASSERT_OK_PTR(ids, "get_ids")) + goto cleanup; + + opts.ids = ids; + opts.cnt = cnt; + links[i] = bpf_program__attach_tracing_multi(progs[i], NULL, &opts); + free(ids); + + if (!ASSERT_OK_PTR(links[i], "bpf_program__attach_tracing_multi")) + goto cleanup; + + expected[i] = *test_results[i] + cnt; + } + + err = bpf_prog_test_run_opts(bpf_program__fd(progs[0]), &topts); + ASSERT_OK(err, "test_run"); + + for (i = 0; i < 4; i++) { + if (!is_set(mask, i)) + continue; + ASSERT_EQ(*test_results[i], expected[i], "test_results"); + } + +cleanup: + for (i = 0; i < 4; i++) + bpf_link__destroy(links[i]); +} + +static void test_intersect(void) +{ + struct tracing_multi_intersect *skel; + const struct bpf_program *progs[4]; + __u64 *test_results[4]; + __u32 i; + + skel = tracing_multi_intersect__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_intersect__open_and_load")) + return; + + skel->bss->pid = getpid(); + + progs[0] = skel->progs.fentry_1; + progs[1] = skel->progs.fexit_1; + progs[2] = skel->progs.fentry_2; + progs[3] = skel->progs.fexit_2; + + test_results[0] = &skel->bss->test_result_fentry_1; + test_results[1] = &skel->bss->test_result_fexit_1; + test_results[2] = &skel->bss->test_result_fentry_2; + test_results[3] = &skel->bss->test_result_fexit_2; + + for (i = 1; i < 16; i++) + __test_intersect(i, progs, test_results); + + tracing_multi_intersect__destroy(skel); +} + +static void test_fentry_after_multi(void) +{ + static const char * const funcs[] = { + "bpf_fentry_test1", + }; + struct bpf_link *fentry_link = NULL, *multi_link = NULL; + struct tracing_multi_intersect *skel = NULL; + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + LIBBPF_OPTS(bpf_test_run_opts, topts); + __u32 *ids = NULL; + int err; + + skel = tracing_multi_intersect__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_intersect__open_and_load")) + return; + + skel->bss->pid = getpid(); + + ids = get_ids(funcs, ARRAY_SIZE(funcs), NULL); + if (!ASSERT_OK_PTR(ids, "get_ids")) + goto cleanup; + + opts.ids = ids; + opts.cnt = ARRAY_SIZE(funcs); + multi_link = bpf_program__attach_tracing_multi(skel->progs.fentry_1, NULL, &opts); + if (!ASSERT_OK_PTR(multi_link, "attach_multi")) + goto cleanup; + + fentry_link = bpf_program__attach(skel->progs.fentry); + if (!ASSERT_OK_PTR(fentry_link, "attach_fentry")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.fentry_1), &topts); + if (!ASSERT_OK(err, "test_run")) + goto cleanup; + ASSERT_EQ(skel->bss->test_result_fentry_1, 1, "multi_fentry"); + ASSERT_EQ(skel->bss->test_result_fentry, 1, "fentry"); + + err = bpf_link__destroy(fentry_link); + fentry_link = NULL; + if (!ASSERT_OK(err, "destroy_fentry")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.fentry_1), &topts); + if (!ASSERT_OK(err, "test_run_multi")) + goto cleanup; + ASSERT_EQ(skel->bss->test_result_fentry_1, 2, "multi_fentry_only"); + ASSERT_EQ(skel->bss->test_result_fentry, 1, "fentry_detached"); + + err = bpf_link__destroy(multi_link); + multi_link = NULL; + if (!ASSERT_OK(err, "destroy_multi")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.fentry_1), &topts); + if (!ASSERT_OK(err, "test_run_detached")) + goto cleanup; + ASSERT_EQ(skel->bss->test_result_fentry_1, 2, "multi_fentry_detached"); + ASSERT_EQ(skel->bss->test_result_fentry, 1, "fentry_still_detached"); + +cleanup: + bpf_link__destroy(fentry_link); + bpf_link__destroy(multi_link); + free(ids); + tracing_multi_intersect__destroy(skel); +} + +static void test_session(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct tracing_multi_session *skel; + int err, prog_fd; + + skel = tracing_multi_session__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_session__open_and_load")) + return; + + skel->bss->pid = getpid(); + + err = tracing_multi_session__attach(skel); + if (!ASSERT_OK(err, "tracing_multi_session__attach")) + goto cleanup; + + /* execute kernel session */ + prog_fd = bpf_program__fd(skel->progs.test_session_1); + err = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(err, "test_run"); + + /* 10 for test_session_1, 1 for test_fsession_s */ + ASSERT_EQ(skel->bss->test_result_fentry, 11, "test_result_fentry"); + /* extra count (+1 for each fexit execution) for test_result_fexit cookie check/inc */ + ASSERT_EQ(skel->bss->test_result_fexit, 22, "test_result_fexit"); + + skel->bss->test_result_fentry = 0; + skel->bss->test_result_fexit = 0; + + /* execute bpf_testmo.ko session */ + ASSERT_OK(trigger_module_test_read(1), "trigger_read"); + + /* 5 for test_session_2 */ + ASSERT_EQ(skel->bss->test_result_fentry, 5, "test_result_fentry"); + /* extra count (+1 for each fexit execution) for test_result_fexit cookie */ + ASSERT_EQ(skel->bss->test_result_fexit, 10, "test_result_fexit"); + + +cleanup: + tracing_multi_session__destroy(skel); +} + +static void test_attach_api_fails(void) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + static const char * const func[] = { + "bpf_fentry_test2", + }; + struct tracing_multi_fail *skel = NULL; + __u32 ids[2] = {}, *ids2 = NULL; + __u64 cookies[2]; + + skel = tracing_multi_fail__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_fail__open_and_load")) + return; + + /* fail#1 (libbpf) pattern and opts NULL */ + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + NULL, NULL); + if (!ASSERT_EQ(libbpf_get_error(skel->links.test_fentry), -EINVAL, "fail_1")) + goto cleanup; + + /* fail#2 (libbpf) pattern and ids */ + LIBBPF_OPTS_RESET(opts, + .ids = ids, + .cnt = 2, + ); + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + "bpf_fentry_test*", &opts); + if (!ASSERT_EQ(libbpf_get_error(skel->links.test_fentry), -EINVAL, "fail_2")) + goto cleanup; + + /* fail#3 (libbpf) pattern and cookies */ + LIBBPF_OPTS_RESET(opts, + .ids = NULL, + .cnt = 2, + .cookies = cookies, + ); + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + "bpf_fentry_test*", &opts); + if (!ASSERT_EQ(libbpf_get_error(skel->links.test_fentry), -EINVAL, "fail_3")) + goto cleanup; + + /* fail#4 (libbpf) bogus pattern */ + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + "bpf_not_really_a_function*", NULL); + if (!ASSERT_EQ(libbpf_get_error(skel->links.test_fentry), -EINVAL, "fail_4")) + goto cleanup; + + /* fail#5 (kernel) abnormal cnt */ + LIBBPF_OPTS_RESET(opts, + .ids = ids, + .cnt = INT_MAX, + ); + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + NULL, &opts); + if (!ASSERT_EQ(libbpf_get_error(skel->links.test_fentry), -E2BIG, "fail_5")) + goto cleanup; + + /* fail#6 (kernel) attach sleepable program to not-allowed function */ + ids2 = get_ids(func, 1, NULL); + if (!ASSERT_OK_PTR(ids2, "get_ids")) + goto cleanup; + + LIBBPF_OPTS_RESET(opts, + .ids = ids2, + .cnt = 1, + ); + + skel->links.test_fentry_s = bpf_program__attach_tracing_multi(skel->progs.test_fentry_s, + NULL, &opts); + if (!ASSERT_EQ(libbpf_get_error(skel->links.test_fentry_s), -EINVAL, "fail_6")) + goto cleanup; + + /* fail#7 (kernel) attach with duplicate id */ + ids[0] = ids2[0]; + ids[1] = ids2[0]; + + LIBBPF_OPTS_RESET(opts, + .ids = ids, + .cnt = 2, + ); + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + NULL, &opts); + ASSERT_EQ(libbpf_get_error(skel->links.test_fentry), -EINVAL, "fail_7"); + +cleanup: + tracing_multi_fail__destroy(skel); + free(ids2); +} + +void serial_test_tracing_multi_bench_attach(void) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tracing_multi_bench *skel = NULL; + long attach_start_ns, attach_end_ns; + long detach_start_ns, detach_end_ns; + double attach_delta, detach_delta; + struct bpf_link *link = NULL; + size_t i, cap = 0, cnt = 0; + struct ksyms *ksyms = NULL; + void *root = NULL; + void *dups = NULL; + __u32 *ids = NULL; + __u32 nr, type_id; + struct btf *btf; + int err; + +#ifndef __x86_64__ + test__skip(); + return; +#endif + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "btf__load_vmlinux_btf")) + return; + + skel = tracing_multi_bench__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_bench__open_and_load")) + goto cleanup; + + if (!ASSERT_OK(bpf_get_ksyms(&ksyms, true), "get_syms")) + goto cleanup; + + /* Get all ftrace 'safe' symbols.. */ + for (i = 0; i < ksyms->filtered_cnt; i++) { + if (!tsearch(&ksyms->filtered_syms[i], &root, compare)) { + ASSERT_FAIL("tsearch failed"); + goto cleanup; + } + } + + /* + * Collect names that are not unique in kallsyms. The kernel resolves a + * tracing-multi BTF id to an address with kallsyms_lookup_name(), which + * returns the first symbol of that name. For a duplicate name that may + * be a different (non-ftrace-able) instance than the ftrace-able one in + * available_filter_functions, so attaching to it by BTF id fails with + * -ENOENT (e.g. t_start/t_next/t_stop). ksyms->syms is sorted by name, + * so equal names are adjacent. + */ + for (i = 1; i < ksyms->sym_cnt; i++) { + if (strcmp(ksyms->syms[i].name, ksyms->syms[i - 1].name)) + continue; + if (!tsearch(&ksyms->syms[i].name, &dups, compare)) { + ASSERT_FAIL("tsearch failed"); + goto cleanup; + } + } + + /* ..and filter them through BTF and btf_type_is_traceable_func. */ + nr = btf__type_cnt(btf); + for (type_id = 1; type_id < nr; type_id++) { + const struct btf_type *type; + const char *str; + + type = btf__type_by_id(btf, type_id); + if (!type) + break; + + if (BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) + continue; + + str = btf__name_by_offset(btf, type->name_off); + if (!str) + break; + + if (!tfind(&str, &root, compare)) + continue; + + /* Skip names that are not unique in kallsyms, see above. */ + if (tfind(&str, &dups, compare)) + continue; + + if (!btf_type_is_traceable_func(btf, type)) + continue; + + err = libbpf_ensure_mem((void **) &ids, &cap, sizeof(*ids), cnt + 1); + if (err) + goto cleanup; + + ids[cnt++] = type_id; + } + + opts.ids = ids; + opts.cnt = cnt; + + attach_start_ns = get_time_ns(); + link = bpf_program__attach_tracing_multi(skel->progs.bench, NULL, &opts); + attach_end_ns = get_time_ns(); + + if (!ASSERT_OK_PTR(link, "bpf_program__attach_tracing_multi")) + goto cleanup; + + detach_start_ns = get_time_ns(); + bpf_link__destroy(link); + detach_end_ns = get_time_ns(); + + attach_delta = (attach_end_ns - attach_start_ns) / 1000000000.0; + detach_delta = (detach_end_ns - detach_start_ns) / 1000000000.0; + + printf("%s: found %lu functions\n", __func__, cnt); + printf("%s: attached in %7.3lfs\n", __func__, attach_delta); + printf("%s: detached in %7.3lfs\n", __func__, detach_delta); + +cleanup: + tracing_multi_bench__destroy(skel); + tdestroy(root, tdestroy_free_nop); + tdestroy(dups, tdestroy_free_nop); + free_kallsyms_local(ksyms); + free(ids); + btf__free(btf); +} + +static void tracing_multi_rollback_run(struct tracing_multi_rollback *skel) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, prog_fd; + + prog_fd = bpf_program__fd(skel->progs.test_fentry); + err = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(err, "test_run"); + + /* make sure the rollback code did not leave any program attached */ + ASSERT_EQ(skel->bss->test_result_fentry, 0, "test_result_fentry"); + ASSERT_EQ(skel->bss->test_result_fexit, 0, "test_result_fexit"); +} + +static void test_rollback_put(void) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tracing_multi_rollback *skel = NULL; + size_t cnt = FUNCS_CNT; + __u32 *ids = NULL; + int err; + + skel = tracing_multi_rollback__open(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_rollback__open")) + return; + + bpf_program__set_autoload(skel->progs.test_fentry, true); + bpf_program__set_autoload(skel->progs.test_fexit, true); + + err = tracing_multi_rollback__load(skel); + if (!ASSERT_OK(err, "tracing_multi_rollback__load")) + goto cleanup; + + ids = get_ids(bpf_fentry_test, cnt, NULL); + if (!ASSERT_OK_PTR(ids, "get_ids")) + goto cleanup; + + /* + * Mangle last id to trigger rollback, which needs to do put + * on get-ed trampolines. + */ + ids[9] = 0; + + opts.ids = ids; + opts.cnt = cnt; + + skel->bss->pid = getpid(); + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + NULL, &opts); + if (!ASSERT_ERR_PTR(skel->links.test_fentry, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fexit = bpf_program__attach_tracing_multi(skel->progs.test_fexit, + NULL, &opts); + if (!ASSERT_ERR_PTR(skel->links.test_fexit, "bpf_program__attach_tracing_multi")) + goto cleanup; + + /* We don't really attach any program, but let's make sure. */ + tracing_multi_rollback_run(skel); + +cleanup: + tracing_multi_rollback__destroy(skel); + free(ids); +} + +static void fillers_cleanup(struct tracing_multi_rollback **skels, int cnt) +{ + int i; + + for (i = 0; i < cnt; i++) + tracing_multi_rollback__destroy(skels[i]); + + free(skels); +} + +static struct tracing_multi_rollback *extra_load_and_link(void) +{ + struct tracing_multi_rollback *skel; + int err; + + skel = tracing_multi_rollback__open(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_rollback__open")) + goto cleanup; + + bpf_program__set_autoload(skel->progs.extra, true); + + err = tracing_multi_rollback__load(skel); + if (!ASSERT_OK(err, "tracing_multi_rollback__load")) + goto cleanup; + + skel->links.extra = bpf_program__attach_trace(skel->progs.extra); + if (!ASSERT_OK_PTR(skel->links.extra, "bpf_program__attach_trace")) + goto cleanup; + + return skel; + +cleanup: + tracing_multi_rollback__destroy(skel); + return NULL; +} + +static struct tracing_multi_rollback **fillers_load_and_link(int max) +{ + struct tracing_multi_rollback **skels, *skel; + int i, err; + + skels = calloc(max + 1, sizeof(*skels)); + if (!ASSERT_OK_PTR(skels, "calloc")) + return NULL; + + for (i = 0; i < max; i++) { + skel = skels[i] = tracing_multi_rollback__open(); + if (!ASSERT_OK_PTR(skels[i], "tracing_multi_rollback__open")) + goto cleanup; + + bpf_program__set_autoload(skel->progs.filler, true); + + err = tracing_multi_rollback__load(skel); + if (!ASSERT_OK(err, "tracing_multi_rollback__load")) + goto cleanup; + + skel->links.filler = bpf_program__attach_trace(skel->progs.filler); + if (!ASSERT_OK_PTR(skels[i]->links.filler, "bpf_program__attach_trace")) + goto cleanup; + } + + return skels; + +cleanup: + fillers_cleanup(skels, i + 1); + return NULL; +} + +static void test_rollback_unlink(void) +{ + struct tracing_multi_rollback *skel = NULL, *extra; + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tracing_multi_rollback **fillers; + size_t cnt = FUNCS_CNT; + __u32 *ids = NULL; + int err, max; + + max = get_bpf_max_tramp_links(); + if (!ASSERT_GE(max, 1, "bpf_max_tramp_links")) + return; + + /* Attach maximum allowed programs to bpf_fentry_test10 */ + fillers = fillers_load_and_link(max); + if (!ASSERT_OK_PTR(fillers, "fillers_load_and_link")) + return; + + extra = extra_load_and_link(); + if (!ASSERT_OK_PTR(extra, "extra_load_and_link")) + goto cleanup; + + skel = tracing_multi_rollback__open(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_rollback__open")) + goto cleanup; + + bpf_program__set_autoload(skel->progs.test_fentry, true); + bpf_program__set_autoload(skel->progs.test_fexit, true); + + /* + * Attach tracing_multi link on bpf_fentry_test1-10, which will + * fail on bpf_fentry_test10 function, because it already has + * maximum allowed programs attached. + * + * The rollback needs to unlink already link-ed trampolines and + * put all of them. + */ + err = tracing_multi_rollback__load(skel); + if (!ASSERT_OK(err, "tracing_multi_rollback__load")) + goto cleanup; + + ids = get_ids(bpf_fentry_test, cnt, NULL); + if (!ASSERT_OK_PTR(ids, "get_ids")) + goto cleanup; + + opts.ids = ids; + opts.cnt = cnt; + + skel->bss->pid = getpid(); + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + NULL, &opts); + if (!ASSERT_ERR_PTR(skel->links.test_fentry, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fexit = bpf_program__attach_tracing_multi(skel->progs.test_fexit, + NULL, &opts); + if (!ASSERT_ERR_PTR(skel->links.test_fexit, "bpf_program__attach_tracing_multi")) + goto cleanup; + + tracing_multi_rollback_run(skel); + +cleanup: + fillers_cleanup(fillers, max); + tracing_multi_rollback__destroy(extra); + tracing_multi_rollback__destroy(skel); + free(ids); +} + +void serial_test_tracing_multi_attach_rollback(void) +{ + if (test__start_subtest("put")) + test_rollback_put(); + if (test__start_subtest("unlink")) + test_rollback_unlink(); +} + +void test_tracing_multi_test(void) +{ +#ifndef __x86_64__ + test__skip(); + return; +#endif + + if (test__start_subtest("skel_api")) + test_skel_api(); + if (test__start_subtest("link_api_pattern")) + test_link_api_pattern(); + if (test__start_subtest("link_api_ids")) + test_link_api_ids(false); + if (test__start_subtest("module_skel_api")) + test_module_skel_api(); + if (test__start_subtest("module_link_api_pattern")) + test_module_link_api_pattern(); + if (test__start_subtest("module_link_api_ids")) + test_module_link_api_ids(); + if (test__start_subtest("intersect")) + test_intersect(); + if (test__start_subtest("cookies")) + test_link_api_ids(true); + if (test__start_subtest("session")) + test_session(); + if (test__start_subtest("attach_api_fails")) + test_attach_api_fails(); + RUN_TESTS(tracing_multi_verifier); + if (test__start_subtest("fentry_after_multi")) + test_fentry_after_multi(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c index 6f8c0bfb0415..15b95d0235b5 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c @@ -4,6 +4,7 @@ #include <test_progs.h> #include "tracing_struct.skel.h" #include "tracing_struct_many_args.skel.h" +#include "tracing_struct_int128.skel.h" static void test_struct_args(void) { @@ -112,6 +113,39 @@ destroy_skel: tracing_struct_many_args__destroy(skel); } +static void test_int128_args(void) +{ + /* + * __int128 arguments are passed in a register pair on x86_64 and + * arm64, which the trampoline packs into two context slots. Other + * architectures pass a __int128 differently (e.g. s390x passes larger + * arguments by reference), so only exercise this on x86_64 and arm64. + */ +#if defined(__x86_64__) || defined(__aarch64__) + struct tracing_struct_int128 *skel; + int err; + + skel = tracing_struct_int128__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_struct_int128__open_and_load")) + return; + + err = tracing_struct_int128__attach(skel); + if (!ASSERT_OK(err, "tracing_struct_int128__attach")) + goto destroy_skel; + + ASSERT_OK(trigger_module_test_read(256), "trigger_read"); + + ASSERT_EQ(skel->bss->t_b, 2, "t:b"); + ASSERT_EQ(skel->bss->t_c, 3, "t:c"); + ASSERT_EQ(skel->bss->t_ret, 6, "t ret"); + +destroy_skel: + tracing_struct_int128__destroy(skel); +#else + test__skip(); +#endif +} + static void test_union_args(void) { struct tracing_struct *skel; @@ -145,6 +179,8 @@ void test_tracing_struct(void) test_struct_args(); if (test__start_subtest("struct_many_args")) test_struct_many_args(); + if (test__start_subtest("int128_args")) + test_int128_args(); if (test__start_subtest("union_args")) test_union_args(); } diff --git a/tools/testing/selftests/bpf/prog_tests/trampoline_count.c b/tools/testing/selftests/bpf/prog_tests/trampoline_count.c index 6cd7349d4a2b..7321850db75f 100644 --- a/tools/testing/selftests/bpf/prog_tests/trampoline_count.c +++ b/tools/testing/selftests/bpf/prog_tests/trampoline_count.c @@ -30,16 +30,14 @@ static struct bpf_program *load_prog(char *file, char *name, struct inst *inst) return prog; } -/* TODO: use different target function to run in concurrent mode */ -void serial_test_trampoline_count(void) +void test_trampoline_count(void) { char *file = "test_trampoline_count.bpf.o"; char *const progs[] = { "fentry_test", "fmod_ret_test", "fexit_test" }; - int bpf_max_tramp_links, err, i, prog_fd; + int bpf_max_tramp_links, i; struct bpf_program *prog; struct bpf_link *link; struct inst *inst; - LIBBPF_OPTS(bpf_test_run_opts, opts); bpf_max_tramp_links = get_bpf_max_tramp_links(); if (!ASSERT_GE(bpf_max_tramp_links, 1, "bpf_max_tramp_links")) @@ -80,16 +78,7 @@ void serial_test_trampoline_count(void) goto cleanup; /* and finally execute the probe */ - prog_fd = bpf_program__fd(prog); - if (!ASSERT_GE(prog_fd, 0, "bpf_program__fd")) - goto cleanup; - - err = bpf_prog_test_run_opts(prog_fd, &opts); - if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) - goto cleanup; - - ASSERT_EQ(opts.retval & 0xffff, 33, "bpf_modify_return_test.result"); - ASSERT_EQ(opts.retval >> 16, 2, "bpf_modify_return_test.side_effect"); + ASSERT_OK(trigger_module_test_read(256), "trigger_module_test_read"); cleanup: for (; i >= 0; i--) { diff --git a/tools/testing/selftests/bpf/prog_tests/unpriv_bpf_disabled.c b/tools/testing/selftests/bpf/prog_tests/unpriv_bpf_disabled.c index 472f4f9fa95f..64404602b9ab 100644 --- a/tools/testing/selftests/bpf/prog_tests/unpriv_bpf_disabled.c +++ b/tools/testing/selftests/bpf/prog_tests/unpriv_bpf_disabled.c @@ -8,6 +8,7 @@ #include "cap_helpers.h" #include "bpf_util.h" +#include "sysctl_helpers.h" /* Using CAP_LAST_CAP is risky here, since it can get pulled in from * an old /usr/include/linux/capability.h and be < CAP_BPF; as a result @@ -36,26 +37,6 @@ static void process_perfbuf(void *ctx, int cpu, void *data, __u32 len) got_perfbuf_val = *(__u32 *)data; } -static int sysctl_set(const char *sysctl_path, char *old_val, const char *new_val) -{ - int ret = 0; - FILE *fp; - - fp = fopen(sysctl_path, "r+"); - if (!fp) - return -errno; - if (old_val && fscanf(fp, "%s", old_val) <= 0) { - ret = -ENOENT; - } else if (!old_val || strcmp(old_val, new_val) != 0) { - fseek(fp, 0, SEEK_SET); - if (fprintf(fp, "%s", new_val) < 0) - ret = -errno; - } - fclose(fp); - - return ret; -} - static void test_unpriv_bpf_disabled_positive(struct test_unpriv_bpf_disabled *skel, __u32 prog_id, int prog_fd, int perf_fd, char **map_paths, int *map_fds) diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c index 56cbea280fbd..f0baf5738b75 100644 --- a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +++ b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c @@ -2,6 +2,7 @@ #include <unistd.h> #include <pthread.h> +#include <fcntl.h> #include <test_progs.h> #include "uprobe_multi.skel.h" #include "uprobe_multi_bench.skel.h" @@ -536,7 +537,37 @@ static void test_attach_api_fails(void) link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); if (!ASSERT_ERR(link_fd, "link_fd")) goto cleanup; - ASSERT_EQ(link_fd, -EINVAL, "pid_is_wrong"); + if (!ASSERT_EQ(link_fd, -EINVAL, "pid_is_wrong")) + goto cleanup; + + /* wrong path_fd */ + LIBBPF_OPTS_RESET(opts, + .uprobe_multi.path = NULL, + .uprobe_multi.path_fd = -1, + .uprobe_multi.flags = BPF_F_UPROBE_MULTI_PATH_FD, + .uprobe_multi.offsets = (unsigned long *)&offset, + .uprobe_multi.cnt = 1, + ); + + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_ERR(link_fd, "link_fd")) + goto cleanup; + if (!ASSERT_EQ(link_fd, -EBADF, "path_fd_is_wrong")) + goto cleanup; + + /* path and path_fd both set with BPF_F_UPROBE_MULTI_PATH_FD flag */ + LIBBPF_OPTS_RESET(opts, + .uprobe_multi.path = path, + .uprobe_multi.path_fd = 1, + .uprobe_multi.flags = BPF_F_UPROBE_MULTI_PATH_FD, + .uprobe_multi.offsets = (unsigned long *)&offset, + .uprobe_multi.cnt = 1, + ); + + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_ERR(link_fd, "link_fd")) + goto cleanup; + ASSERT_EQ(link_fd, -EINVAL, "path_and_path_fd_together"); cleanup: if (link_fd >= 0) @@ -757,6 +788,65 @@ static void test_link_api(void) __test_link_api(&child); } +static void test_link_api_path_fd(void) +{ + LIBBPF_OPTS(bpf_link_create_opts, opts); + const char *resolve_path = "/proc/self/exe"; + int prog_fd, link_fd = -1, path_fd = -1; + struct uprobe_multi *skel = NULL; + unsigned long *offsets = NULL; + const char *syms[3] = { + "uprobe_multi_func_1", + "uprobe_multi_func_2", + "uprobe_multi_func_3", + }; + int err; + + err = elf_resolve_syms_offsets(resolve_path, ARRAY_SIZE(syms), syms, + &offsets, STT_FUNC); + if (!ASSERT_OK(err, "elf_resolve_syms_offsets")) + return; + + path_fd = open(resolve_path, O_RDONLY); + if (!ASSERT_GE(path_fd, 0, "path_fd")) + goto cleanup; + + opts.uprobe_multi.path_fd = path_fd; + opts.uprobe_multi.offsets = offsets; + opts.uprobe_multi.cnt = ARRAY_SIZE(syms); + opts.uprobe_multi.flags = BPF_F_UPROBE_MULTI_PATH_FD; + + skel = uprobe_multi__open_and_load(); + if (!ASSERT_OK_PTR(skel, "uprobe_multi__open_and_load")) + goto cleanup; + + prog_fd = bpf_program__fd(skel->progs.uprobe); + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_GE(link_fd, 0, "bpf_link_create")) + goto cleanup; + + skel->bss->uprobe_multi_func_1_addr = (__u64)uprobe_multi_func_1; + skel->bss->uprobe_multi_func_2_addr = (__u64)uprobe_multi_func_2; + skel->bss->uprobe_multi_func_3_addr = (__u64)uprobe_multi_func_3; + skel->bss->pid = getpid(); + + uprobe_multi_func_1(); + uprobe_multi_func_2(); + uprobe_multi_func_3(); + + ASSERT_EQ(skel->bss->uprobe_multi_func_1_result, 1, "uprobe_multi_func_1_result"); + ASSERT_EQ(skel->bss->uprobe_multi_func_2_result, 1, "uprobe_multi_func_2_result"); + ASSERT_EQ(skel->bss->uprobe_multi_func_3_result, 1, "uprobe_multi_func_3_result"); + +cleanup: + if (link_fd >= 0) + close(link_fd); + if (path_fd >= 0) + close(path_fd); + uprobe_multi__destroy(skel); + free(offsets); +} + static struct bpf_program * get_program(struct uprobe_multi_consumers *skel, int prog) { @@ -1354,6 +1444,8 @@ void test_uprobe_multi_test(void) test_attach_api_syms(); if (test__start_subtest("link_api")) test_link_api(); + if (test__start_subtest("link_api_path_fd")) + test_link_api_path_fd(); if (test__start_subtest("bench_uprobe")) test_bench_attach_uprobe(); if (test__start_subtest("bench_usdt")) diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c b/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c index 955a37751b52..e193206fc5d2 100644 --- a/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c +++ b/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c @@ -4,6 +4,8 @@ #ifdef __x86_64__ +#define _GNU_SOURCE +#include <sched.h> #include <unistd.h> #include <asm/ptrace.h> #include <linux/compiler.h> @@ -13,11 +15,12 @@ #include <sys/syscall.h> #include <sys/prctl.h> #include <asm/prctl.h> +#include <stdnoreturn.h> #include "uprobe_syscall.skel.h" #include "uprobe_syscall_executed.skel.h" #include "bpf/libbpf_internal.h" -#define USDT_NOP .byte 0x0f, 0x1f, 0x44, 0x00, 0x00 +#define USDT_NOP .byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 #include "usdt.h" #pragma GCC diagnostic ignored "-Wattributes" @@ -26,7 +29,7 @@ __attribute__((aligned(16))) __nocf_check __weak __naked unsigned long uprobe_regs_trigger(void) { asm volatile ( - ".byte 0x0f, 0x1f, 0x44, 0x00, 0x00\n" /* nop5 */ + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ "movq $0xdeadbeef, %rax\n" "ret\n" ); @@ -345,9 +348,9 @@ cleanup: __attribute__((aligned(16))) __nocf_check __weak __naked void uprobe_test(void) { - asm volatile (" \n" - ".byte 0x0f, 0x1f, 0x44, 0x00, 0x00 \n" - "ret \n" + asm volatile ( + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ + "ret\n" ); } @@ -357,6 +360,50 @@ __nocf_check __weak void usdt_test(void) USDT(optimized_uprobe, usdt); } +/* + * Assembly-level red zone clobbering test. Stores known values in the + * red zone (below RSP), executes a nop10 (uprobe site), and checks that + * the values survived. Returns 0 if intact, 1 if clobbered. + * + * The nop5 optimization used CALL (which pushes a return address to + * [rsp-8]), the value at -8(%rsp) was overwritten. The nop10 optimization + * should escape that by moving stackpointer below the redzone before + * doing the CALL. + * + * Align the code at 64 bytes, to make sure nop10 is not on page boundary. + */ +__attribute__((aligned(64))) +__nocf_check __weak __naked unsigned long uprobe_red_zone_test(void) +{ + asm volatile ( + "movabs $0x1111111111111111, %%rax\n" + "movq %%rax, -8(%%rsp)\n" + "movabs $0x2222222222222222, %%rax\n" + "movq %%rax, -16(%%rsp)\n" + "movabs $0x3333333333333333, %%rax\n" + "movq %%rax, -24(%%rsp)\n" + + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10: uprobe site */ + + "movabs $0x1111111111111111, %%rax\n" + "cmpq %%rax, -8(%%rsp)\n" + "jne 1f\n" + "movabs $0x2222222222222222, %%rax\n" + "cmpq %%rax, -16(%%rsp)\n" + "jne 1f\n" + "movabs $0x3333333333333333, %%rax\n" + "cmpq %%rax, -24(%%rsp)\n" + "jne 1f\n" + + "xorl %%eax, %%eax\n" + "retq\n" + "1:\n" + "movl $1, %%eax\n" + "retq\n" + ::: "rax", "memory" + ); +} + static int find_uprobes_trampoline(void *tramp_addr) { void *start, *end; @@ -388,14 +435,15 @@ static int find_uprobes_trampoline(void *tramp_addr) return ret; } -static unsigned char nop5[5] = { 0x0f, 0x1f, 0x44, 0x00, 0x00 }; +static unsigned char nop10[10] = { 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }; +static unsigned char lea_rsp[5] = { 0x48, 0x8d, 0x64, 0x24, 0x80 }; -static void *find_nop5(void *fn) +static void *find_nop10(void *fn) { int i; - for (i = 0; i < 10; i++) { - if (!memcmp(nop5, fn + i, 5)) + for (i = 0; i < 128; i++) { + if (!memcmp(nop10, fn + i, 10)) return fn + i; } return NULL; @@ -403,6 +451,16 @@ static void *find_nop5(void *fn) typedef void (__attribute__((nocf_check)) *trigger_t)(void); +static void check_attach_notrigger(struct uprobe_syscall_executed *skel, + void *addr, int executed) +{ + unsigned char *op = addr; + + /* Make sure bpf program was not executed. */ + ASSERT_EQ(skel->bss->executed, executed, "executed"); + ASSERT_EQ(*op, 0xcc, "int3"); +} + static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigger, void *addr, int executed) { @@ -420,7 +478,8 @@ static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigge ASSERT_EQ(skel->bss->executed, executed, "executed"); /* .. and check the trampoline is as expected. */ - call = (struct __arch_relative_insn *) addr; + ASSERT_OK(memcmp(addr, lea_rsp, 5), "lea_rsp"); + call = (struct __arch_relative_insn *)(addr + 5); tramp = (void *) (call + 1) + call->raddr; ASSERT_EQ(call->op, 0xe8, "call"); ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); @@ -428,21 +487,26 @@ static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigge return tramp; } -static void check_detach(void *addr, void *tramp) +static bool check_detach(void *addr, void *tramp) { + static const unsigned char nop10_prefix[] = { 0x66, 0x2e, 0x0f, 0x1f, 0x84 }; + bool ok = true; + /* [uprobes_trampoline] stays after detach */ - ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); - ASSERT_OK(memcmp(addr, nop5, 5), "nop5"); + ok &= ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); + ok &= ASSERT_OK(memcmp(addr, nop10_prefix, 5), "nop10_prefix"); + return ok; } -static void check(struct uprobe_syscall_executed *skel, struct bpf_link *link, - trigger_t trigger, void *addr, int executed) +static void *check(struct uprobe_syscall_executed *skel, struct bpf_link *link, + trigger_t trigger, void *addr, int executed) { void *tramp; tramp = check_attach(skel, trigger, addr, executed); bpf_link__destroy(link); check_detach(addr, tramp); + return tramp; } static void test_uprobe_legacy(void) @@ -453,6 +517,7 @@ static void test_uprobe_legacy(void) ); struct bpf_link *link; unsigned long offset; + void *tramp; offset = get_uprobe_offset(&uprobe_test); if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) @@ -470,7 +535,30 @@ static void test_uprobe_legacy(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) goto cleanup; - check(skel, link, uprobe_test, uprobe_test, 2); + tramp = check(skel, link, uprobe_test, uprobe_test, 2); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + 0, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) + goto cleanup; + + check_attach_notrigger(skel, uprobe_test, 2); + + bpf_link__destroy(link); + if (!check_detach(uprobe_test, tramp)) + goto cleanup; + + uprobe_test(); + ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + 0, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) + goto cleanup; + + check(skel, link, uprobe_test, uprobe_test, 4); /* uretprobe */ skel->bss->executed = 0; @@ -492,6 +580,7 @@ static void test_uprobe_multi(void) LIBBPF_OPTS(bpf_uprobe_multi_opts, opts); struct bpf_link *link; unsigned long offset; + void *tramp; offset = get_uprobe_offset(&uprobe_test); if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) @@ -512,7 +601,30 @@ static void test_uprobe_multi(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) goto cleanup; - check(skel, link, uprobe_test, uprobe_test, 2); + tramp = check(skel, link, uprobe_test, uprobe_test, 2); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check_attach_notrigger(skel, uprobe_test, 2); + + bpf_link__destroy(link); + if (!check_detach(uprobe_test, tramp)) + goto cleanup; + + uprobe_test(); + ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check(skel, link, uprobe_test, uprobe_test, 4); /* uretprobe.multi */ skel->bss->executed = 0; @@ -536,6 +648,7 @@ static void test_uprobe_session(void) ); struct bpf_link *link; unsigned long offset; + void *tramp; offset = get_uprobe_offset(&uprobe_test); if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) @@ -555,7 +668,30 @@ static void test_uprobe_session(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) goto cleanup; - check(skel, link, uprobe_test, uprobe_test, 4); + tramp = check(skel, link, uprobe_test, uprobe_test, 4); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check_attach_notrigger(skel, uprobe_test, 4); + + bpf_link__destroy(link); + if (!check_detach(uprobe_test, tramp)) + goto cleanup; + + uprobe_test(); + ASSERT_EQ(skel->bss->executed, 4, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check(skel, link, uprobe_test, uprobe_test, 8); cleanup: uprobe_syscall_executed__destroy(skel); @@ -565,11 +701,11 @@ static void test_uprobe_usdt(void) { struct uprobe_syscall_executed *skel; struct bpf_link *link; - void *addr; + void *addr, *tramp; errno = 0; - addr = find_nop5(usdt_test); - if (!ASSERT_OK_PTR(addr, "find_nop5")) + addr = find_nop10(usdt_test); + if (!ASSERT_OK_PTR(addr, "find_nop10")) return; skel = uprobe_syscall_executed__open_and_load(); @@ -584,7 +720,32 @@ static void test_uprobe_usdt(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) goto cleanup; - check(skel, link, usdt_test, addr, 2); + tramp = check(skel, link, usdt_test, addr, 2); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_usdt(skel->progs.test_usdt, + -1 /* all PIDs */, "/proc/self/exe", + "optimized_uprobe", "usdt", NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) + goto cleanup; + + check_attach_notrigger(skel, addr, 2); + + bpf_link__destroy(link); + if (!check_detach(addr, tramp)) + goto cleanup; + + usdt_test(); + ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_usdt(skel->progs.test_usdt, + -1 /* all PIDs */, "/proc/self/exe", + "optimized_uprobe", "usdt", NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) + goto cleanup; + + check(skel, link, usdt_test, addr, 4); cleanup: uprobe_syscall_executed__destroy(skel); @@ -757,12 +918,124 @@ cleanup: #define __NR_uprobe 336 #endif +static void test_uprobe_red_zone(void) +{ + struct uprobe_syscall_executed *skel; + struct bpf_link *link; + void *nop10_addr; + size_t offset; + int i; + + nop10_addr = find_nop10(uprobe_red_zone_test); + if (!ASSERT_NEQ(nop10_addr, NULL, "find_nop10")) + return; + + skel = uprobe_syscall_executed__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + offset = get_uprobe_offset(nop10_addr); + link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + 0, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(link, "attach_uprobe")) + goto cleanup; + + for (i = 0; i < 10; i++) + ASSERT_EQ(uprobe_red_zone_test(), 0, "red_zone_intact"); + + bpf_link__destroy(link); + +cleanup: + uprobe_syscall_executed__destroy(skel); +} + static void test_uprobe_error(void) { long err = syscall(__NR_uprobe); ASSERT_EQ(err, -1, "error"); - ASSERT_EQ(errno, ENXIO, "errno"); + ASSERT_EQ(errno, EPROTO, "errno"); +} + +__attribute__((aligned(16))) +__nocf_check __weak __naked void uprobe_fork_test(void) +{ + asm volatile ( + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ + "ret\n" + ); +} + +static noreturn int child_func(void *arg) +{ + struct uprobe_syscall_executed *skel = arg; + + /* Make sure the child's probe is still there and optimized.. */ + if (memcmp(uprobe_fork_test, lea_rsp, sizeof(lea_rsp))) + _exit(1); + + skel->bss->pid = getpid(); + + /* .. and it executes properly. */ + uprobe_fork_test(); + + if (skel->bss->executed != 3) + _exit(2); + + _exit(0); +} + +static void test_uprobe_fork_optimized(bool clone_vm) +{ + struct uprobe_syscall_executed *skel = NULL; + unsigned long offset; + int pid, status, err; + char stack[65535]; + + offset = get_uprobe_offset(&uprobe_fork_test); + if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) + return; + + skel = uprobe_syscall_executed__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto cleanup; + + skel->links.test_uprobe = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + -1, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(skel->links.test_uprobe, "attach_uprobe")) + goto cleanup; + + skel->bss->pid = getpid(); + + /* Trigger optimization of uprobe in uprobe_fork_test. */ + uprobe_fork_test(); + uprobe_fork_test(); + + /* Make sure it got optimied. */ + if (!ASSERT_OK(memcmp(uprobe_fork_test, lea_rsp, sizeof(lea_rsp)), "optimized")) + goto cleanup; + + if (clone_vm) { + pid = clone(child_func, stack + sizeof(stack), CLONE_VM|SIGCHLD, skel); + if (!ASSERT_GT(pid, 0, "clone")) + goto cleanup; + } else { + pid = fork(); + if (!ASSERT_GE(pid, 0, "fork")) + goto cleanup; + if (pid == 0) + child_func(skel); + } + + /* Wait for the child and verify it exited properly with 0. */ + err = waitpid(pid, &status, 0); + if (ASSERT_EQ(err, pid, "waitpid")) { + ASSERT_EQ(WIFEXITED(status), 1, "child_exited"); + ASSERT_EQ(WEXITSTATUS(status), 0, "child_exit_code"); + } + +cleanup: + uprobe_syscall_executed__destroy(skel); } static void __test_uprobe_syscall(void) @@ -783,6 +1056,12 @@ static void __test_uprobe_syscall(void) test_uprobe_usdt(); if (test__start_subtest("uprobe_race")) test_uprobe_race(); + if (test__start_subtest("uprobe_red_zone")) + test_uprobe_red_zone(); + if (test__start_subtest("uprobe_optimized_fork")) + test_uprobe_fork_optimized(false); + if (test__start_subtest("uprobe_optimized_clone_vm")) + test_uprobe_fork_optimized(true); if (test__start_subtest("uprobe_error")) test_uprobe_error(); if (test__start_subtest("uprobe_regs_equal")) diff --git a/tools/testing/selftests/bpf/prog_tests/usdt.c b/tools/testing/selftests/bpf/prog_tests/usdt.c index f4be5269fa90..8004c9568ffa 100644 --- a/tools/testing/selftests/bpf/prog_tests/usdt.c +++ b/tools/testing/selftests/bpf/prog_tests/usdt.c @@ -247,6 +247,148 @@ cleanup: #undef TRIGGER } +#ifdef __x86_64__ +extern void usdt_1(void); +extern void usdt_2(void); +extern void usdt_red_zone_trigger(void); + +static unsigned char nop1[1] = { 0x90 }; +static unsigned char nop1_nop10_combo[11] = { 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }; + +static void *find_instr(void *fn, unsigned char *instr, size_t cnt) +{ + int i; + + for (i = 0; i < 10; i++) { + if (!memcmp(instr, fn + i, cnt)) + return fn + i; + } + return NULL; +} + +static void subtest_optimized_attach(void) +{ + struct test_usdt *skel; + __u8 *addr_1, *addr_2; + + /* usdt_1 USDT probe has single nop instruction */ + addr_1 = find_instr(usdt_1, nop1_nop10_combo, 11); + if (!ASSERT_NULL(addr_1, "usdt_1_find_nop1_nop10_combo")) + return; + + addr_1 = find_instr(usdt_1, nop1, 1); + if (!ASSERT_OK_PTR(addr_1, "usdt_1_find_nop1")) + return; + + /* usdt_2 USDT probe has nop,nop10 instructions combo */ + addr_2 = find_instr(usdt_2, nop1_nop10_combo, 11); + if (!ASSERT_OK_PTR(addr_2, "usdt_2_find_nop1_nop10_combo")) + return; + + skel = test_usdt__open_and_load(); + if (!ASSERT_OK_PTR(skel, "test_usdt__open_and_load")) + return; + + skel->bss->expected_ip = (unsigned long) addr_1; + + /* + * Attach program on top of usdt_1 which is single nop probe, + * so the probe won't get optimized. + */ + skel->links.usdt_executed = bpf_program__attach_usdt(skel->progs.usdt_executed, + 0 /*self*/, "/proc/self/exe", + "optimized_attach", "usdt_1", NULL); + if (!ASSERT_OK_PTR(skel->links.usdt_executed, "bpf_program__attach_usdt")) + goto cleanup; + + usdt_1(); + usdt_1(); + + /* int3 is on addr_1 address */ + ASSERT_EQ(*addr_1, 0xcc, "int3"); + ASSERT_EQ(skel->bss->executed, 2, "executed"); + + bpf_link__destroy(skel->links.usdt_executed); + + /* we expect the nop10 ip */ + skel->bss->expected_ip = (unsigned long) addr_2 + 1; + + /* + * Attach program on top of usdt_2 which is probe defined on top + * of nop1,nop10 combo, so the probe gets optimized on top of nop10. + */ + skel->links.usdt_executed = bpf_program__attach_usdt(skel->progs.usdt_executed, + 0 /*self*/, "/proc/self/exe", + "optimized_attach", "usdt_2", NULL); + if (!ASSERT_OK_PTR(skel->links.usdt_executed, "bpf_program__attach_usdt")) + goto cleanup; + + usdt_2(); + usdt_2(); + + /* nop stays on addr_2 address */ + ASSERT_EQ(*addr_2, 0x90, "nop"); + + /* + * lea -0x80(%rsp), %rsp + * call ... + */ + static unsigned char expected[] = { 0x48, 0x8d, 0x64, 0x24, 0x80, 0xe8 }; + + ASSERT_MEMEQ(addr_2 + 1, expected, sizeof(expected), "lea_and_call"); + ASSERT_EQ(skel->bss->executed, 4, "executed"); + +cleanup: + test_usdt__destroy(skel); +} + +/* + * Test that USDT arguments survive nop10 optimization in a function where + * the compiler places operands in the red zone. + * + * Signal handlers are prone to having the compiler place USDT argument + * operands in the red zone (below rsp). + * + * The nop5 optimization used CALL (which pushes a return address to + * [rsp-8]), the value at -8(%rsp) was overwritten. The nop10 optimization + * should escape that by moving stackpointer below the redzone before + * doing the CALL. + */ +static void subtest_optimized_red_zone(void) +{ + struct test_usdt *skel; + int i; + + skel = test_usdt__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + skel->bss->expected_arg[0] = 0xDEADBEEF; + skel->bss->expected_arg[1] = 0xCAFEBABE; + skel->bss->expected_arg[2] = 0xFEEDFACE; + skel->bss->expected_pid = getpid(); + + skel->links.usdt_check_arg = bpf_program__attach_usdt( + skel->progs.usdt_check_arg, 0, "/proc/self/exe", + "optimized_attach", "usdt_red_zone", NULL); + if (!ASSERT_OK_PTR(skel->links.usdt_check_arg, "attach_usdt_red_zone")) + goto cleanup; + + for (i = 0; i < 10; i++) + usdt_red_zone_trigger(); + + ASSERT_EQ(skel->bss->arg_total, 10, "arg_total"); + ASSERT_EQ(skel->bss->arg_bad, 0, "arg_bad"); + ASSERT_EQ(skel->bss->arg_last[0], 0xDEADBEEF, "arg_last_1"); + ASSERT_EQ(skel->bss->arg_last[1], 0xCAFEBABE, "arg_last_2"); + ASSERT_EQ(skel->bss->arg_last[2], 0xFEEDFACE, "arg_last_3"); + +cleanup: + test_usdt__destroy(skel); +} + +#endif + unsigned short test_usdt_100_semaphore SEC(".probes"); unsigned short test_usdt_300_semaphore SEC(".probes"); unsigned short test_usdt_400_semaphore SEC(".probes"); @@ -516,6 +658,10 @@ void test_usdt(void) #ifdef __x86_64__ if (test__start_subtest("basic_optimized")) subtest_basic_usdt(true); + if (test__start_subtest("optimized_attach")) + subtest_optimized_attach(); + if (test__start_subtest("optimized_red_zone")) + subtest_optimized_red_zone(); #endif if (test__start_subtest("multispec")) subtest_multispec_usdt(); diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 302286a80154..8b439e194bcc 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -2,7 +2,10 @@ #include <test_progs.h> +#include "arena_kfunc.skel.h" +#include "arena_kfunc_jit.skel.h" #include "cap_helpers.h" +#include "verifier_align.skel.h" #include "verifier_and.skel.h" #include "verifier_arena.skel.h" #include "verifier_arena_large.skel.h" @@ -20,7 +23,9 @@ #include "verifier_bpf_trap.skel.h" #include "verifier_bswap.skel.h" #include "verifier_btf_ctx_access.skel.h" +#include "verifier_btf_flex_array.skel.h" #include "verifier_btf_unreliable_prog.skel.h" +#include "verifier_call_large_imm.skel.h" #include "verifier_cfg.skel.h" #include "verifier_cgroup_inv_retcode.skel.h" #include "verifier_cgroup_skb.skel.h" @@ -36,6 +41,7 @@ #include "verifier_div0.skel.h" #include "verifier_div_mod_bounds.skel.h" #include "verifier_div_overflow.skel.h" +#include "verifier_flow_keys.skel.h" #include "verifier_global_subprogs.skel.h" #include "verifier_global_ptr_args.skel.h" #include "verifier_gotol.skel.h" @@ -48,21 +54,25 @@ #include "verifier_iterating_callbacks.skel.h" #include "verifier_jeq_infer_not_null.skel.h" #include "verifier_jit_convergence.skel.h" +#include "verifier_kfunc_perfmon.skel.h" #include "verifier_ld_ind.skel.h" #include "verifier_ldsx.skel.h" #include "verifier_leak_ptr.skel.h" #include "verifier_linked_scalars.skel.h" #include "verifier_live_stack.skel.h" +#include "verifier_liveness_exp.skel.h" #include "verifier_load_acquire.skel.h" #include "verifier_loops1.skel.h" #include "verifier_lwt.skel.h" #include "verifier_map_in_map.skel.h" +#include "verifier_map_lookup_refine.skel.h" #include "verifier_map_ptr.skel.h" #include "verifier_map_ptr_mixing.skel.h" #include "verifier_map_ret_val.skel.h" #include "verifier_masking.skel.h" #include "verifier_may_goto_1.skel.h" #include "verifier_may_goto_2.skel.h" +#include "verifier_mem_size_reg.skel.h" #include "verifier_meta_access.skel.h" #include "verifier_movsx.skel.h" #include "verifier_mtu.skel.h" @@ -71,9 +81,11 @@ #include "verifier_netfilter_retcode.skel.h" #include "verifier_bpf_fastcall.skel.h" #include "verifier_or_jmp32_k.skel.h" +#include "verifier_percpu_addr.skel.h" #include "verifier_precision.skel.h" #include "verifier_prevent_map_lookup.skel.h" #include "verifier_private_stack.skel.h" +#include "verifier_ptr_to_buf.skel.h" #include "verifier_raw_stack.skel.h" #include "verifier_raw_tp_writable.skel.h" #include "verifier_reg_equal.skel.h" @@ -89,9 +101,13 @@ #include "verifier_sockmap_mutate.skel.h" #include "verifier_spill_fill.skel.h" #include "verifier_spin_lock.skel.h" +#include "verifier_stack_arg.skel.h" +#include "verifier_stack_arg_order.skel.h" #include "verifier_stack_ptr.skel.h" #include "verifier_store_release.skel.h" +#include "verifier_subprog_insn_stats.skel.h" #include "verifier_subprog_precision.skel.h" +#include "verifier_subprog_topo.skel.h" #include "verifier_subreg.skel.h" #include "verifier_tailcall.skel.h" #include "verifier_tailcall_jit.skel.h" @@ -111,9 +127,12 @@ #include "verifier_xdp.skel.h" #include "verifier_xdp_direct_packet_access.skel.h" #include "verifier_bits_iter.skel.h" +#include "verifier_set_retval.skel.h" #include "verifier_lsm.skel.h" #include "verifier_jit_inline.skel.h" #include "irq.skel.h" +#include "verifier_ctx_ptr_param.skel.h" +#include "verifier_zext.skel.h" #define MAX_ENTRIES 11 @@ -149,6 +168,11 @@ static void run_tests_aux(const char *skel_name, #define RUN(skel) run_tests_aux(#skel, skel##__elf_bytes, NULL) +void test_arena_kfunc(void) { RUN_TESTS(arena_kfunc); } + +void test_arena_kfunc_jit(void) { RUN_TESTS(arena_kfunc_jit); } + +void test_verifier_align(void) { RUN(verifier_align); } void test_verifier_and(void) { RUN(verifier_and); } void test_verifier_arena(void) { RUN(verifier_arena); } void test_verifier_arena_large(void) { RUN(verifier_arena_large); } @@ -164,14 +188,16 @@ void test_verifier_bpf_get_stack(void) { RUN(verifier_bpf_get_stack); } void test_verifier_bpf_trap(void) { RUN(verifier_bpf_trap); } void test_verifier_bswap(void) { RUN(verifier_bswap); } void test_verifier_btf_ctx_access(void) { RUN(verifier_btf_ctx_access); } +void test_verifier_btf_flex_array(void) { RUN(verifier_btf_flex_array); } void test_verifier_btf_unreliable_prog(void) { RUN(verifier_btf_unreliable_prog); } +void test_verifier_call_large_imm(void) { RUN(verifier_call_large_imm); } void test_verifier_cfg(void) { RUN(verifier_cfg); } void test_verifier_cgroup_inv_retcode(void) { RUN(verifier_cgroup_inv_retcode); } void test_verifier_cgroup_skb(void) { RUN(verifier_cgroup_skb); } void test_verifier_cgroup_storage(void) { RUN(verifier_cgroup_storage); } void test_verifier_const(void) { RUN(verifier_const); } void test_verifier_const_or(void) { RUN(verifier_const_or); } -void test_verifier_ctx(void) { RUN(verifier_ctx); } +void test_verifier_ctx(void) { RUN_TESTS(verifier_ctx); } void test_verifier_ctx_sk_msg(void) { RUN(verifier_ctx_sk_msg); } void test_verifier_d_path(void) { RUN(verifier_d_path); } void test_verifier_default_trusted_ptr(void) { RUN_TESTS(verifier_default_trusted_ptr); } @@ -180,6 +206,7 @@ void test_verifier_direct_stack_access_wraparound(void) { RUN(verifier_direct_st void test_verifier_div0(void) { RUN(verifier_div0); } void test_verifier_div_mod_bounds(void) { RUN(verifier_div_mod_bounds); } void test_verifier_div_overflow(void) { RUN(verifier_div_overflow); } +void test_verifier_flow_keys(void) { RUN(verifier_flow_keys); } void test_verifier_global_subprogs(void) { RUN(verifier_global_subprogs); } void test_verifier_global_ptr_args(void) { RUN(verifier_global_ptr_args); } void test_verifier_gotol(void) { RUN(verifier_gotol); } @@ -192,21 +219,25 @@ void test_verifier_int_ptr(void) { RUN(verifier_int_ptr); } void test_verifier_iterating_callbacks(void) { RUN(verifier_iterating_callbacks); } void test_verifier_jeq_infer_not_null(void) { RUN(verifier_jeq_infer_not_null); } void test_verifier_jit_convergence(void) { RUN(verifier_jit_convergence); } +void test_verifier_kfunc_perfmon(void) { RUN(verifier_kfunc_perfmon); } void test_verifier_load_acquire(void) { RUN(verifier_load_acquire); } void test_verifier_ld_ind(void) { RUN(verifier_ld_ind); } void test_verifier_ldsx(void) { RUN(verifier_ldsx); } void test_verifier_leak_ptr(void) { RUN(verifier_leak_ptr); } void test_verifier_linked_scalars(void) { RUN(verifier_linked_scalars); } void test_verifier_live_stack(void) { RUN(verifier_live_stack); } +void test_verifier_liveness_exp(void) { RUN(verifier_liveness_exp); } void test_verifier_loops1(void) { RUN(verifier_loops1); } void test_verifier_lwt(void) { RUN(verifier_lwt); } void test_verifier_map_in_map(void) { RUN(verifier_map_in_map); } +void test_verifier_map_lookup_refine(void) { RUN(verifier_map_lookup_refine); } void test_verifier_map_ptr(void) { RUN(verifier_map_ptr); } void test_verifier_map_ptr_mixing(void) { RUN(verifier_map_ptr_mixing); } void test_verifier_map_ret_val(void) { RUN(verifier_map_ret_val); } void test_verifier_masking(void) { RUN(verifier_masking); } void test_verifier_may_goto_1(void) { RUN(verifier_may_goto_1); } void test_verifier_may_goto_2(void) { RUN(verifier_may_goto_2); } +void test_verifier_mem_size_reg(void) { RUN(verifier_mem_size_reg); } void test_verifier_meta_access(void) { RUN(verifier_meta_access); } void test_verifier_movsx(void) { RUN(verifier_movsx); } void test_verifier_mul(void) { RUN(verifier_mul); } @@ -214,9 +245,11 @@ void test_verifier_netfilter_ctx(void) { RUN(verifier_netfilter_ctx); } void test_verifier_netfilter_retcode(void) { RUN(verifier_netfilter_retcode); } void test_verifier_bpf_fastcall(void) { RUN(verifier_bpf_fastcall); } void test_verifier_or_jmp32_k(void) { RUN(verifier_or_jmp32_k); } +void test_verifier_percpu_addr(void) { RUN(verifier_percpu_addr); } void test_verifier_precision(void) { RUN(verifier_precision); } void test_verifier_prevent_map_lookup(void) { RUN(verifier_prevent_map_lookup); } void test_verifier_private_stack(void) { RUN(verifier_private_stack); } +void test_verifier_ptr_to_buf(void) { RUN(verifier_ptr_to_buf); } void test_verifier_raw_stack(void) { RUN(verifier_raw_stack); } void test_verifier_raw_tp_writable(void) { RUN(verifier_raw_tp_writable); } void test_verifier_reg_equal(void) { RUN(verifier_reg_equal); } @@ -232,9 +265,13 @@ void test_verifier_sock_addr(void) { RUN(verifier_sock_addr); } void test_verifier_sockmap_mutate(void) { RUN(verifier_sockmap_mutate); } void test_verifier_spill_fill(void) { RUN(verifier_spill_fill); } void test_verifier_spin_lock(void) { RUN(verifier_spin_lock); } +void test_verifier_stack_arg(void) { RUN(verifier_stack_arg); } +void test_verifier_stack_arg_order(void) { RUN(verifier_stack_arg_order); } void test_verifier_stack_ptr(void) { RUN(verifier_stack_ptr); } void test_verifier_store_release(void) { RUN(verifier_store_release); } +void test_verifier_subprog_insn_stats(void) { RUN(verifier_subprog_insn_stats); } void test_verifier_subprog_precision(void) { RUN(verifier_subprog_precision); } +void test_verifier_subprog_topo(void) { RUN(verifier_subprog_topo); } void test_verifier_subreg(void) { RUN(verifier_subreg); } void test_verifier_tailcall(void) { RUN(verifier_tailcall); } void test_verifier_tailcall_jit(void) { RUN(verifier_tailcall_jit); } @@ -253,10 +290,13 @@ void test_verifier_xadd(void) { RUN(verifier_xadd); } void test_verifier_xdp(void) { RUN(verifier_xdp); } void test_verifier_xdp_direct_packet_access(void) { RUN(verifier_xdp_direct_packet_access); } void test_verifier_bits_iter(void) { RUN(verifier_bits_iter); } +void test_verifier_set_retval(void) { RUN(verifier_set_retval); } void test_verifier_lsm(void) { RUN(verifier_lsm); } void test_irq(void) { RUN(irq); } void test_verifier_mtu(void) { RUN(verifier_mtu); } void test_verifier_jit_inline(void) { RUN(verifier_jit_inline); } +void test_verifier_ctx_ptr_param(void) { RUN(verifier_ctx_ptr_param); } +void test_verifier_zext(void) { RUN_TESTS(verifier_zext); } static int init_test_val_map(struct bpf_object *obj, char *map_name) { diff --git a/tools/testing/selftests/bpf/prog_tests/verifier_log.c b/tools/testing/selftests/bpf/prog_tests/verifier_log.c index aaa2854974c0..4542bb586d72 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier_log.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier_log.c @@ -25,10 +25,10 @@ static bool check_prog_load(int prog_fd, bool expect_err, const char *tag) static struct { /* strategically placed before others to avoid accidental modification by kernel */ - char filler[1024]; - char buf[1024]; + char filler[16384]; + char buf[16384]; /* strategically placed after buf[] to catch more accidental corruptions */ - char reference[1024]; + char reference[16384]; } logs; static const struct bpf_insn *insns; static size_t insn_cnt; @@ -317,6 +317,7 @@ static void verif_btf_log_subtest(bool bad_btf) res = load_btf(&opts, true); ASSERT_EQ(res, -ENOSPC, "half_log_fd"); ASSERT_EQ(strlen(logs.buf), 24, "log_fixed_25"); + strscpy(op_name, "log_fixed", sizeof(op_name)); ASSERT_STRNEQ(logs.buf, logs.reference, 24, op_name); /* validate rolling verifier log logic: try all variations of log buf diff --git a/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c index 4d69d9d55e17..f327feb8e38c 100644 --- a/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c +++ b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c @@ -18,6 +18,7 @@ #include <linux/keyctl.h> #include <sys/xattr.h> #include <linux/fsverity.h> +#include <linux/module_signature.h> #include <test_progs.h> #include "test_verify_pkcs7_sig.skel.h" @@ -33,29 +34,6 @@ #define SHA256_DIGEST_SIZE 32 #endif -/* In stripped ARM and x86-64 modules, ~ is surprisingly rare. */ -#define MODULE_SIG_STRING "~Module signature appended~\n" - -/* - * Module signature information block. - * - * The constituents of the signature section are, in order: - * - * - Signer's name - * - Key identifier - * - Signature data - * - Information block - */ -struct module_signature { - __u8 algo; /* Public-key crypto algorithm [0] */ - __u8 hash; /* Digest algorithm [0] */ - __u8 id_type; /* Key identifier type [PKEY_ID_PKCS7] */ - __u8 signer_len; /* Length of signer's name [0] */ - __u8 key_id_len; /* Length of key identifier [0] */ - __u8 __pad[3]; - __be32 sig_len; /* Length of signature data */ -}; - struct data { __u8 data[MAX_DATA_SIZE]; __u32 data_len; @@ -215,7 +193,7 @@ static int populate_data_item_mod(struct data *data_item) return 0; modlen = st.st_size; - marker_len = sizeof(MODULE_SIG_STRING) - 1; + marker_len = sizeof(MODULE_SIGNATURE_MARKER) - 1; fd = open(mod_path, O_RDONLY); if (fd == -1) @@ -228,7 +206,7 @@ static int populate_data_item_mod(struct data *data_item) if (mod == MAP_FAILED) return -errno; - if (strncmp(mod + modlen - marker_len, MODULE_SIG_STRING, marker_len)) { + if (strncmp(mod + modlen - marker_len, MODULE_SIGNATURE_MARKER, marker_len)) { ret = -EINVAL; goto out; } diff --git a/tools/testing/selftests/bpf/prog_tests/vmlinux.c b/tools/testing/selftests/bpf/prog_tests/vmlinux.c index 6fb2217d940b..b5fdd593910d 100644 --- a/tools/testing/selftests/bpf/prog_tests/vmlinux.c +++ b/tools/testing/selftests/bpf/prog_tests/vmlinux.c @@ -14,21 +14,61 @@ static void nsleep() (void)syscall(__NR_nanosleep, &ts, NULL); } +static const char *hrtimer_func = "hrtimer_start_range_ns"; + +static int setup_hrtimer_progs(struct test_vmlinux *skel) +{ + int err; + + if (libbpf_find_vmlinux_btf_id("hrtimer_start_range_ns_user", BPF_TRACE_FENTRY) > 0) + hrtimer_func = "hrtimer_start_range_ns_user"; + + err = bpf_program__set_attach_target(skel->progs.handle__fentry, 0, hrtimer_func); + if (err) + return err; + + /* + * Bare SEC("kprobe") has no target function, so attach it manually + * later after selecting the hrtimer function to probe. + */ + bpf_program__set_autoattach(skel->progs.handle__kprobe, false); + + return 0; +} + void test_vmlinux(void) { int err; struct test_vmlinux* skel; struct test_vmlinux__bss *bss; + struct bpf_link *kprobe_link = NULL; - skel = test_vmlinux__open_and_load(); - if (!ASSERT_OK_PTR(skel, "test_vmlinux__open_and_load")) + skel = test_vmlinux__open(); + if (!ASSERT_OK_PTR(skel, "test_vmlinux__open")) return; + + err = setup_hrtimer_progs(skel); + if (!ASSERT_OK(err, "setup_hrtimer_progs")) + goto cleanup; + + err = test_vmlinux__load(skel); + if (!ASSERT_OK(err, "test_vmlinux__load")) + goto cleanup; + bss = skel->bss; err = test_vmlinux__attach(skel); if (!ASSERT_OK(err, "test_vmlinux__attach")) goto cleanup; + /* manually attach kprobe with the selected function */ + if (hrtimer_func) { + kprobe_link = bpf_program__attach_kprobe(skel->progs.handle__kprobe, + false /* retprobe */, hrtimer_func); + if (!ASSERT_OK_PTR(kprobe_link, "bpf_program__attach_kprobe")) + goto cleanup; + } + /* trigger everything */ nsleep(); @@ -39,5 +79,6 @@ void test_vmlinux(void) ASSERT_TRUE(bss->fentry_called, "fentry"); cleanup: + bpf_link__destroy(kprobe_link); test_vmlinux__destroy(skel); } diff --git a/tools/testing/selftests/bpf/prog_tests/wakeup_source.c b/tools/testing/selftests/bpf/prog_tests/wakeup_source.c new file mode 100644 index 000000000000..ebfdc03271b9 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/wakeup_source.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright 2026 Google LLC */ + +#include <test_progs.h> +#include <bpf/btf.h> +#include <fcntl.h> +#include "test_wakeup_source.skel.h" +#include "wakeup_source_fail.skel.h" +#include "progs/wakeup_source.h" + +static int lock_ws(const char *name) +{ + int fd; + ssize_t bytes; + + fd = open("/sys/power/wake_lock", O_WRONLY); + if (!ASSERT_OK_FD(fd, "open /sys/power/wake_lock")) + return -1; + + bytes = write(fd, name, strlen(name)); + close(fd); + if (!ASSERT_EQ(bytes, strlen(name), "write to wake_lock")) + return -1; + + return 0; +} + +static void unlock_ws(const char *name) +{ + int fd; + + fd = open("/sys/power/wake_unlock", O_WRONLY); + if (fd < 0) + return; + + write(fd, name, strlen(name)); + close(fd); +} + +struct rb_ctx { + const char *name; + bool found; + long long active_time_ns; + long long total_time_ns; +}; + +static int process_sample(void *ctx, void *data, size_t len) +{ + struct rb_ctx *rb_ctx = ctx; + struct wakeup_event_t *e = data; + + if (strcmp(e->name, rb_ctx->name) == 0) { + rb_ctx->found = true; + rb_ctx->active_time_ns = e->active_time_ns; + rb_ctx->total_time_ns = e->total_time_ns; + } + return 0; +} + +void test_wakeup_source(void) +{ + struct btf *btf; + int id; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "btf_vmlinux")) + return; + + id = btf__find_by_name_kind(btf, "bpf_wakeup_sources_get_head", BTF_KIND_FUNC); + btf__free(btf); + + if (id < 0) { + printf("%s:SKIP:bpf_wakeup_sources_get_head kfunc not found in BTF\n", __func__); + test__skip(); + return; + } + + if (test__start_subtest("iterate_and_verify_times")) { + struct test_wakeup_source *skel; + struct ring_buffer *rb = NULL; + struct rb_ctx rb_ctx = { + .name = "bpf_selftest_ws_times", + .found = false, + }; + int err; + + skel = test_wakeup_source__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + rb = ring_buffer__new(bpf_map__fd(skel->maps.rb), process_sample, &rb_ctx, NULL); + if (!ASSERT_OK_PTR(rb, "ring_buffer__new")) + goto destroy; + + /* Create a temporary wakeup source */ + if (!ASSERT_OK(lock_ws(rb_ctx.name), "lock_ws")) + goto unlock; + + err = bpf_prog_test_run_opts(bpf_program__fd( + skel->progs.iterate_wakeupsources), NULL); + ASSERT_OK(err, "bpf_prog_test_run"); + + ring_buffer__consume(rb); + + ASSERT_TRUE(rb_ctx.found, "found_test_ws_in_rb"); + ASSERT_GT(rb_ctx.active_time_ns, 0, "active_time_gt_0"); + ASSERT_GT(rb_ctx.total_time_ns, 0, "total_time_gt_0"); + +unlock: + unlock_ws(rb_ctx.name); +destroy: + if (rb) + ring_buffer__free(rb); + test_wakeup_source__destroy(skel); + } + + RUN_TESTS(wakeup_source_fail); +} diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_bonding.c b/tools/testing/selftests/bpf/prog_tests/xdp_bonding.c index e8ea26464349..c42488e445c2 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_bonding.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_bonding.c @@ -191,13 +191,18 @@ fail: return -1; } -static void bonding_cleanup(struct skeletons *skeletons) +static void link_cleanup(struct skeletons *skeletons) { - restore_root_netns(); while (skeletons->nlinks) { skeletons->nlinks--; bpf_link__destroy(skeletons->links[skeletons->nlinks]); } +} + +static void bonding_cleanup(struct skeletons *skeletons) +{ + restore_root_netns(); + link_cleanup(skeletons); ASSERT_OK(system("ip link delete bond1"), "delete bond1"); ASSERT_OK(system("ip link delete veth1_1"), "delete veth1_1"); ASSERT_OK(system("ip link delete veth1_2"), "delete veth1_2"); @@ -493,6 +498,90 @@ out: system("ip link del bond_nest2"); } +/* + * Test that XDP redirect via xdp_master_redirect() does not crash when + * the bond master device is not up. When bond is in round-robin mode but + * never opened, rr_tx_counter is NULL. + */ +static void test_xdp_bonding_redirect_no_up(struct skeletons *skeletons) +{ + struct nstoken *nstoken = NULL; + int xdp_pass_fd; + int veth1_ifindex; + int err; + char pkt[ETH_HLEN + 1]; + struct xdp_md ctx_in = {}; + + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, + .data_in = &pkt, + .data_size_in = sizeof(pkt), + .ctx_in = &ctx_in, + .ctx_size_in = sizeof(ctx_in), + .flags = BPF_F_TEST_XDP_LIVE_FRAMES, + .repeat = 1, + .batch_size = 1, + ); + + /* We can't use bonding_setup() because bond will be active */ + SYS(out, "ip netns add ns_rr_no_up"); + nstoken = open_netns("ns_rr_no_up"); + if (!ASSERT_OK_PTR(nstoken, "open ns_rr_no_up")) + goto out; + + /* bond0: active-backup, UP with slave veth0. + * Attaching native XDP to bond0 enables bpf_master_redirect_enabled_key + * globally. + */ + SYS(out, "ip link add bond0 type bond mode active-backup"); + SYS(out, "ip link add veth0 type veth peer name veth0p"); + SYS(out, "ip link set veth0 master bond0"); + SYS(out, "ip link set bond0 up"); + SYS(out, "ip link set veth0p up"); + + /* bond1: round-robin, never UP -> rr_tx_counter stays NULL */ + SYS(out, "ip link add bond1 type bond mode balance-rr"); + SYS(out, "ip link add veth1 type veth peer name veth1p"); + SYS(out, "ip link set veth1 master bond1"); + + veth1_ifindex = if_nametoindex("veth1"); + if (!ASSERT_GT(veth1_ifindex, 0, "veth1_ifindex")) + goto out; + + /* Attach native XDP to bond0 -> enables global redirect key */ + if (xdp_attach(skeletons, skeletons->xdp_tx->progs.xdp_tx, "bond0")) + goto out; + + /* Attach generic XDP (XDP_TX) to veth1. + * When packets arrive at veth1 via netif_receive_skb, do_xdp_generic() + * runs this program. XDP_TX + bond slave triggers xdp_master_redirect(). + */ + err = bpf_xdp_attach(veth1_ifindex, + bpf_program__fd(skeletons->xdp_tx->progs.xdp_tx), + XDP_FLAGS_SKB_MODE, NULL); + if (!ASSERT_OK(err, "attach generic XDP to veth1")) + goto out; + + /* Run BPF_PROG_TEST_RUN with XDP_PASS live frames on veth1. + * XDP_PASS frames become SKBs with skb->dev = veth1, entering + * netif_receive_skb -> do_xdp_generic -> xdp_master_redirect. + * Without the fix, bond_rr_gen_slave_id() dereferences NULL + * rr_tx_counter and crashes. + */ + xdp_pass_fd = bpf_program__fd(skeletons->xdp_dummy->progs.xdp_dummy_prog); + + memset(pkt, 0, sizeof(pkt)); + ctx_in.data_end = sizeof(pkt); + ctx_in.ingress_ifindex = veth1_ifindex; + + err = bpf_prog_test_run_opts(xdp_pass_fd, &opts); + ASSERT_OK(err, "xdp_pass test_run should not crash"); + +out: + link_cleanup(skeletons); + close_netns(nstoken); + SYS_NOFAIL("ip netns del ns_rr_no_up"); +} + static void test_xdp_bonding_features(struct skeletons *skeletons) { LIBBPF_OPTS(bpf_xdp_query_opts, query_opts); @@ -738,6 +827,9 @@ void serial_test_xdp_bonding(void) if (test__start_subtest("xdp_bonding_redirect_multi")) test_xdp_bonding_redirect_multi(&skeletons); + if (test__start_subtest("xdp_bonding_redirect_no_up")) + test_xdp_bonding_redirect_no_up(&skeletons); + out: xdp_dummy__destroy(skeletons.xdp_dummy); xdp_tx__destroy(skeletons.xdp_tx); diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c b/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c index 26159e0499c7..448807676176 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> #include <network_helpers.h> +#include <linux/ipv6.h> +#include <arpa/inet.h> #include "test_xdp_context_test_run.skel.h" #include "test_xdp_meta.skel.h" @@ -8,9 +10,12 @@ #define TX_NAME "veth1" #define TX_NETNS "xdp_context_tx" #define RX_NETNS "xdp_context_rx" +#define RX_MAC "02:00:00:00:00:01" +#define TX_MAC "02:00:00:00:00:02" #define TAP_NAME "tap0" #define DUMMY_NAME "dum0" #define TAP_NETNS "xdp_context_tuntap" +#define LWT_NETNS "xdp_context_lwt" #define TEST_PAYLOAD_LEN 32 static const __u8 test_payload[TEST_PAYLOAD_LEN] = { @@ -187,6 +192,42 @@ static int write_test_packet(int tap_fd) return 0; } +/* Inject Ethernet+IPv6+UDP frame into TAP */ +static int write_test_packet_udp(int tap_fd) +{ + __u8 pkt[sizeof(struct ethhdr) + sizeof(struct ipv6hdr) + + sizeof(struct udphdr) + TEST_PAYLOAD_LEN] = {}; + struct ethhdr *eth = (void *)pkt; + struct ipv6hdr *ip6 = (void *)(eth + 1); + struct udphdr *udp = (void *)(ip6 + 1); + __u8 *payload = (void *)(udp + 1); + const __u8 tap_mac[ETH_ALEN] = { 0x02, 0, 0, 0, 0, 0x01 }; + int n; + + memcpy(eth->h_dest, tap_mac, ETH_ALEN); + eth->h_proto = htons(ETH_P_IPV6); + + ip6->version = 6; + ip6->hop_limit = 64; + ip6->nexthdr = IPPROTO_UDP; + ip6->payload_len = htons(sizeof(*udp) + TEST_PAYLOAD_LEN); + inet_pton(AF_INET6, "fd00::2", &ip6->saddr); + inet_pton(AF_INET6, "fd00:1::1", &ip6->daddr); + + udp->source = htons(42); + udp->dest = htons(42); + udp->len = htons(sizeof(*udp) + TEST_PAYLOAD_LEN); + /* UDP checksum is not validated on the forwarding path. */ + + memcpy(payload, test_payload, TEST_PAYLOAD_LEN); + + n = write(tap_fd, pkt, sizeof(pkt)); + if (!ASSERT_EQ(n, sizeof(pkt), "write frame")) + return -1; + + return 0; +} + static void dump_err_stream(const struct bpf_program *prog) { char buf[512]; @@ -518,3 +559,137 @@ void test_xdp_context_tuntap(void) test_xdp_meta__destroy(skel); } + +/* + * Test topology: + * + * tap0 fd00::1 + * RX: injected IPv6 UDP frame, XDP ingress sets metadata + * fwd: encap route prepends outer header(s) + * TX: TC egress validates metadata + * + * A routable IPv6 UDP frame is written into the tap fd, so it enters the RX + * path where XDP stores metadata. Routing then forwards it back out the same + * tap through an encapsulating route that prepends outer header(s). The TC + * egress program checks that the pushed header did not silently corrupt + * metadata. + */ +#define LWT_PIN_PATH "/sys/fs/bpf/xdp_context_lwt_xmit" + +enum lwt_encap_type { + LWT_ENCAP_BPF, + LWT_ENCAP_MPLS, + LWT_ENCAP_SEG6, + LWT_ENCAP_IOAM6, +}; + +static void test_lwt_encap(struct test_xdp_meta *skel, + enum lwt_encap_type type) +{ + LIBBPF_OPTS(bpf_tc_hook, tc_hook, .attach_point = BPF_TC_EGRESS); + LIBBPF_OPTS(bpf_tc_opts, tc_opts, .handle = 1, .priority = 1); + struct bpf_program *lwt_prog = NULL; + struct netns_obj *ns = NULL; + const char *encap; + bool pinned = false; + int tap_ifindex; + int tap_fd = -1; + int ret; + + skel->bss->test_pass = false; + + switch (type) { + case LWT_ENCAP_BPF: + encap = "encap bpf xmit pinned " LWT_PIN_PATH " via fd00::2"; + lwt_prog = skel->progs.dummy_lwt_xmit; + break; + case LWT_ENCAP_MPLS: + encap = "encap mpls 100 via inet6 fd00::2"; + break; + case LWT_ENCAP_SEG6: + encap = "encap seg6 mode encap segs fd00::2"; + break; + case LWT_ENCAP_IOAM6: + encap = "encap ioam6 mode encap tundst fd00::2 " + "trace prealloc type 0x800000 ns 0 size 4 via fd00::2"; + break; + default: + return; + } + + if (lwt_prog) { + unlink(LWT_PIN_PATH); + ret = bpf_program__pin(lwt_prog, LWT_PIN_PATH); + if (!ASSERT_OK(ret, "pin lwt prog")) + return; + pinned = true; + } + + ns = netns_new(LWT_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto close; + + tap_fd = open_tuntap(TAP_NAME, true); + if (!ASSERT_GE(tap_fd, 0, "open_tuntap")) + goto close; + + SYS(close, "ip link set dev " TAP_NAME " address " RX_MAC); + SYS(close, "sysctl -wq net.ipv6.conf.all.forwarding=1"); + SYS(close, "ip addr add fd00::1/64 dev " TAP_NAME " nodad"); + SYS(close, "ip link set dev " TAP_NAME " up"); + SYS(close, "ip neigh add fd00::2 lladdr " TX_MAC " nud permanent dev " TAP_NAME); + SYS(close, "ip -6 route add fd00:1::/64 %s dev %s", encap, TAP_NAME); + + tap_ifindex = if_nametoindex(TAP_NAME); + if (!ASSERT_GE(tap_ifindex, 0, "if_nametoindex")) + goto close; + + ret = bpf_xdp_attach(tap_ifindex, bpf_program__fd(skel->progs.ing_xdp), + 0, NULL); + if (!ASSERT_GE(ret, 0, "bpf_xdp_attach")) + goto close; + + tc_hook.ifindex = tap_ifindex; + ret = bpf_tc_hook_create(&tc_hook); + if (!ASSERT_OK(ret, "bpf_tc_hook_create")) + goto close; + + tc_opts.prog_fd = bpf_program__fd(skel->progs.tc_is_meta_empty); + ret = bpf_tc_attach(&tc_hook, &tc_opts); + if (!ASSERT_OK(ret, "bpf_tc_attach")) + goto close; + + ret = write_test_packet_udp(tap_fd); + if (!ASSERT_OK(ret, "write_test_packet_udp")) + goto close; + + if (!ASSERT_TRUE(skel->bss->test_pass, "test_pass")) + dump_err_stream(skel->progs.tc_is_meta_empty); + +close: + if (tap_fd >= 0) + close(tap_fd); + netns_free(ns); + if (pinned) + unlink(LWT_PIN_PATH); +} + +void test_xdp_context_lwt_encap(void) +{ + struct test_xdp_meta *skel; + + skel = test_xdp_meta__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open and load skeleton")) + return; + + if (test__start_subtest("bpf_encap")) + test_lwt_encap(skel, LWT_ENCAP_BPF); + if (test__start_subtest("mpls_encap")) + test_lwt_encap(skel, LWT_ENCAP_MPLS); + if (test__start_subtest("seg6_encap")) + test_lwt_encap(skel, LWT_ENCAP_SEG6); + if (test__start_subtest("ioam6_encap")) + test_lwt_encap(skel, LWT_ENCAP_IOAM6); + + test_xdp_meta__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/progs/arena_atomics.c b/tools/testing/selftests/bpf/progs/arena_atomics.c index d1841aac94a2..73bc2b835f3f 100644 --- a/tools/testing/selftests/bpf/progs/arena_atomics.c +++ b/tools/testing/selftests/bpf/progs/arena_atomics.c @@ -5,7 +5,7 @@ #include <bpf/bpf_tracing.h> #include <stdbool.h> #include <stdatomic.h> -#include "bpf_arena_common.h" +#include <bpf_arena_common.h> #include "../../../include/linux/filter.h" #include "bpf_misc.h" @@ -28,8 +28,10 @@ bool skip_all_tests = true; #if defined(ENABLE_ATOMICS_TESTS) && \ defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) + (defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ + defined(__TARGET_ARCH_s390)) bool skip_lacq_srel_tests __attribute((__section__(".data"))) = false; #else bool skip_lacq_srel_tests = true; @@ -315,8 +317,10 @@ int load_acquire(const void *ctx) { #if defined(ENABLE_ATOMICS_TESTS) && \ defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) + (defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ + defined(__TARGET_ARCH_s390)) #define LOAD_ACQUIRE_ARENA(SIZEOP, SIZE, SRC, DST) \ { asm volatile ( \ @@ -367,8 +371,10 @@ int store_release(const void *ctx) { #if defined(ENABLE_ATOMICS_TESTS) && \ defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) + (defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ + defined(__TARGET_ARCH_s390)) #define STORE_RELEASE_ARENA(SIZEOP, DST, VAL) \ { asm volatile ( \ diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc.c b/tools/testing/selftests/bpf/progs/arena_kfunc.c new file mode 100644 index 000000000000..bf0d304e0e59 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_kfunc.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_experimental.h" +#include <bpf_arena_common.h> +#include "../test_kmods/bpf_testmod_kfunc.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + /* page 0 hosts the arena global, page 1 is for allocations */ + __uint(max_entries, 2); +} arena SEC(".maps"); + +/* + * Occupies page 0 so no allocation lands at arena offset 0, which the + * nullable tests below must be able to tell apart from NULL. + */ +u64 __arena arena_pad; + +/* volatile to force the scalar reloads below */ +volatile u64 stash; + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_arg_forms(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + u64 ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + /* PTR_TO_ARENA argument */ + *val = 41; + ret = bpf_kfunc_arena_arg_test((u64 *)val); + if (ret != 41 || *val != 42) + return 2; + + /* the low 32 bits as a scalar */ + stash = (u32)(u64)val; + ret = bpf_kfunc_arena_arg_test((u64 *)stash); + if (ret != 42 || *val != 43) + return 3; + + /* the full user address as a scalar */ + stash = (u64)val; + bpf_addr_space_cast(stash, 1, 0); + ret = bpf_kfunc_arena_arg_test((u64 *)stash); + if (ret != 43 || *val != 44) + return 4; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} + +/* + * Pin the rebase semantics using the capture kfuncs, which return the raw + * argument value: __arena rebases unconditionally, so zero low 32 bits + * arrive as the arena kernel base, while __arena__nullable turns them into + * NULL. + */ +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_arg_rebase(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + u64 base, off; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + base = bpf_kfunc_arena_cap_test(NULL); + if (!base) + return 2; + + /* only the low 32 bits contribute */ + stash = 0xbadc0ffe00000000; + if (bpf_kfunc_arena_cap_test((u64 *)stash) != base) + return 3; + + off = (u32)(u64)val; + if (bpf_kfunc_arena_cap_test((u64 *)val) != base + off) + return 4; + + if (bpf_kfunc_arena_cap_nullable_test(NULL) != 0) + return 5; + + stash = 0xbadc0ffe00000000; + if (bpf_kfunc_arena_cap_nullable_test((u64 *)stash) != 0) + return 6; + + if (bpf_kfunc_arena_cap_nullable_test((u64 *)val) != base + off) + return 7; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_args5(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + val[0] = 1; + val[1] = 2; + val[2] = 4; + val[3] = 8; + val[4] = 16; + + if (bpf_kfunc_arena_args5_test((u64 *)&val[0], (u64 *)&val[1], + (u64 *)&val[2], (u64 *)&val[3], + (u64 *)&val[4]) != 31) + return 2; + if (bpf_kfunc_arena_args5_test((u64 *)&val[0], (u64 *)&val[1], + (u64 *)&val[2], (u64 *)&val[3], NULL) != 15) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_arg_mixed(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + val[0] = 7; + val[1] = 5; + + if (bpf_kfunc_arena_mixed_test((u64 *)&val[0], NULL) != 7) + return 2; + + if (bpf_kfunc_arena_mixed_test((u64 *)&val[0], (u64 *)&val[1]) != 12) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} + +/* kernel-side faults on unpopulated pages recover via the scratch page */ +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_arg_unpopulated(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + stash = (u64)val + PAGE_SIZE; + bpf_kfunc_arena_arg_test((u64 *)stash); + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__failure __msg("arena pointer requires a program with an associated arena") +int arena_arg_no_arena(void *ctx) +{ + bpf_kfunc_arena_arg_test((u64 *)1); + return 0; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__failure __msg("is not a pointer to arena or scalar") +int arena_arg_bad_reg(void *ctx) +{ + u64 buf = 0; + + /* use the arena so the program passes the arena presence check */ + bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + bpf_kfunc_arena_arg_test(&buf); + return 0; +} + +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__failure __msg("arena pointer cannot be a stack argument") +int arena_arg_stack(void *ctx) +{ + bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + bpf_kfunc_arena_stack_arg_test(1, 2, 3, 4, 5, (u64 *)1); + return 0; +} +#else +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__description("arena_arg_stack: not supported, dummy test") +__success +int arena_arg_stack(void *ctx) +{ + return 0; +} +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc_jit.c b/tools/testing/selftests/bpf/progs/arena_kfunc_jit.c new file mode 100644 index 000000000000..b5a01cbc33a7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_kfunc_jit.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +/* + * Verify the JIT-emitted rebase sequences for __arena and __arena__nullable + * kfunc arguments. The capture kfuncs take the argument without + * dereferencing it, so these tests pin only the emitted code. + */ +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_experimental.h" +#include <bpf_arena_common.h> +#include "../test_kmods/bpf_testmod_kfunc.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* volatile to force the scalar reloads below */ +volatile u64 stash; + +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + +SEC("syscall") +__arch_x86_64 +__jited("...") +__jited(" movl %edi, %edi") +__jited(" addq %r12, %rdi") +__jited("...") +__jited(" callq {{.*}}") +__arch_arm64 +__jited("...") +__jited(" add x0, x28, w0, uxtw") +__jited(" {{(bl|mov) .*}}") +__success +int arena_arg_jit_rebase(void *ctx) +{ + stash = (u64)bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + bpf_kfunc_arena_cap_test((u64 *)stash); + return 0; +} + +SEC("syscall") +__arch_x86_64 +__jited("...") +__jited(" movl %edi, %edi") +__jited(" testl %edi, %edi") +__jited(" je L0") +__jited(" addq %r12, %rdi") +__jited("L0: callq {{.*}}") +__arch_arm64 +__jited("...") +__jited(" mov w0, w0") +__jited(" cbz w0, L0") +__jited(" add x0, x28, w0, uxtw") +__jited("L0: {{.*}}") +__success +int arena_arg_jit_nullable(void *ctx) +{ + stash = (u64)bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + bpf_kfunc_arena_cap_nullable_test((u64 *)stash); + return 0; +} + +SEC("syscall") +__arch_x86_64 +__jited("...") +__jited(" movl %edi, %edi") +__jited(" addq %r12, %rdi") +__jited(" movl %esi, %esi") +__jited(" addq %r12, %rsi") +__jited(" movl %edx, %edx") +__jited(" addq %r12, %rdx") +__jited(" movl %ecx, %ecx") +__jited(" addq %r12, %rcx") +__jited(" movl %r8d, %r8d") +__jited(" testl %r8d, %r8d") +__jited(" je L0") +__jited(" addq %r12, %r8") +__jited("L0: callq {{.*}}") +__arch_arm64 +__jited("...") +__jited(" add x0, x28, w0, uxtw") +__jited(" add x1, x28, w1, uxtw") +__jited(" add x2, x28, w2, uxtw") +__jited(" add x3, x28, w3, uxtw") +__jited(" mov w4, w4") +__jited(" cbz w4, L0") +__jited(" add x4, x28, w4, uxtw") +__jited("L0: {{.*}}") +__success +int arena_arg_jit_args5(void *ctx) +{ + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + val[0] = 1; + val[1] = 2; + val[2] = 4; + val[3] = 8; + val[4] = 16; + + bpf_kfunc_arena_args5_test((u64 *)&val[0], (u64 *)&val[1], + (u64 *)&val[2], (u64 *)&val[3], + (u64 *)&val[4]); + return 0; +} + +#endif /* __BPF_FEATURE_ADDR_SPACE_CAST */ + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/arena_mem_usage.c b/tools/testing/selftests/bpf/progs/arena_mem_usage.c new file mode 100644 index 000000000000..455ecd669a5a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_mem_usage.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_arena_common.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1000); /* number of pages */ +#ifdef __TARGET_ARCH_arm64 + __ulong(map_extra, 0x1ull << 32); /* start of mmap() region */ +#else + __ulong(map_extra, 0x1ull << 44); /* start of mmap() region */ +#endif +} arena SEC(".maps"); + +void __arena *ptr; +int alloc_cnt; /* in: pages to allocate */ +long free_byte_off; /* in: byte offset within ptr to start freeing */ +int free_cnt; /* in: pages to free */ + +SEC("syscall") +int alloc(void *ctx) +{ + ptr = bpf_arena_alloc_pages(&arena, NULL, alloc_cnt, NUMA_NO_NODE, 0); + /* Success/failure is checked from user space via skel->bss->ptr. */ + return 0; +} + +SEC("syscall") +int free_pages(void *ctx) +{ + if (!ptr) + return 1; + bpf_arena_free_pages(&arena, (char __arena *)ptr + free_byte_off, free_cnt); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/arena_spin_lock.c b/tools/testing/selftests/bpf/progs/arena_spin_lock.c index 086b57a426cf..92e75ec3844c 100644 --- a/tools/testing/selftests/bpf/progs/arena_spin_lock.c +++ b/tools/testing/selftests/bpf/progs/arena_spin_lock.c @@ -4,7 +4,8 @@ #include <bpf/bpf_tracing.h> #include <bpf/bpf_helpers.h> #include "bpf_misc.h" -#include "bpf_arena_spin_lock.h" +#include <bpf_arena_common.h> +#include <bpf_arena_spin_lock.h> struct { __uint(type, BPF_MAP_TYPE_ARENA); @@ -22,6 +23,13 @@ int cs_count; #if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST) arena_spinlock_t __arena lock; int test_skip = 1; + +/* + * Storage for the queue nodes declared by bpf_arena_spin_lock.h. Each program + * linking the arena spinlock provides exactly one definition; libarena's lives + * in libarena/src/common.bpf.c. + */ +struct arena_qnode __arena __hidden qnodes[_Q_MAX_CPUS][_Q_MAX_NODES]; #else int test_skip = 2; #endif diff --git a/tools/testing/selftests/bpf/progs/bench_bpf_timing.bpf.h b/tools/testing/selftests/bpf/progs/bench_bpf_timing.bpf.h new file mode 100644 index 000000000000..6a1ad75f1fd7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bench_bpf_timing.bpf.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#ifndef __BENCH_BPF_TIMING_BPF_H__ +#define __BENCH_BPF_TIMING_BPF_H__ + +#include <stdbool.h> +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf_may_goto.h> + +#ifndef BENCH_NR_SAMPLES +#define BENCH_NR_SAMPLES 4096 +#endif +#ifndef BENCH_NR_CPUS +#define BENCH_NR_CPUS 256 +#endif +#define BENCH_CPU_MASK (BENCH_NR_CPUS - 1) + +__u64 timing_samples[BENCH_NR_CPUS][BENCH_NR_SAMPLES]; +__u32 timing_idx[BENCH_NR_CPUS]; + +volatile __u32 batch_iters; +volatile __u32 timing_enabled; + +static __always_inline void bench_record_sample(__u64 elapsed_ns) +{ + __u32 cpu, idx; + + if (!timing_enabled) + return; + + cpu = bpf_get_smp_processor_id() & BENCH_CPU_MASK; + idx = timing_idx[cpu]; + + if (idx >= BENCH_NR_SAMPLES) + return; + + timing_samples[cpu][idx] = elapsed_ns; + timing_idx[cpu] = idx + 1; +} + +/* + * @body: expression to time; return value (int) stored in __bench_result. + * @reset: undo body's side-effects so each iteration starts identically. + * May reference __bench_result. Use ({}) for empty reset. + * + * Runs batch_iters timed iterations, then one untimed iteration whose + * return value the macro evaluates to (for validation). + */ +#define BENCH_BPF_LOOP(body, reset) ({ \ + __u64 __bench_start = bpf_ktime_get_ns(); \ + __u32 __bench_i; \ + int __bench_result; \ + \ + for (__bench_i = 0; \ + __bench_i < batch_iters && can_loop; \ + __bench_i++) { \ + __bench_result = (body); \ + reset; \ + } \ + \ + bench_record_sample(bpf_ktime_get_ns() - __bench_start); \ + \ + __bench_result = (body); \ + __bench_result; \ +}) + +#endif /* __BENCH_BPF_TIMING_BPF_H__ */ diff --git a/tools/testing/selftests/bpf/progs/bench_local_storage_create.c b/tools/testing/selftests/bpf/progs/bench_local_storage_create.c index c8ec0d0368e4..25ca6045fea3 100644 --- a/tools/testing/selftests/bpf/progs/bench_local_storage_create.c +++ b/tools/testing/selftests/bpf/progs/bench_local_storage_create.c @@ -8,7 +8,6 @@ long create_errs = 0; long create_cnts = 0; -long kmalloc_cnts = 0; __u32 bench_pid = 0; struct storage { @@ -29,16 +28,6 @@ struct { __type(value, struct storage); } task_storage_map SEC(".maps"); -SEC("raw_tp/kmalloc") -int BPF_PROG(kmalloc, unsigned long call_site, const void *ptr, - size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags, - int node) -{ - __sync_fetch_and_add(&kmalloc_cnts, 1); - - return 0; -} - SEC("tp_btf/sched_process_fork") int BPF_PROG(sched_process_fork, struct task_struct *parent, struct task_struct *child) { diff --git a/tools/testing/selftests/bpf/progs/bpf_cc_cubic.c b/tools/testing/selftests/bpf/progs/bpf_cc_cubic.c index 9af19dfe4e80..bccf677b94b6 100644 --- a/tools/testing/selftests/bpf/progs/bpf_cc_cubic.c +++ b/tools/testing/selftests/bpf/progs/bpf_cc_cubic.c @@ -23,7 +23,7 @@ #define TCP_REORDERING (12) extern void cubictcp_init(struct sock *sk) __ksym; -extern void cubictcp_cwnd_event(struct sock *sk, enum tcp_ca_event event) __ksym; +extern void cubictcp_cwnd_event_tx_start(struct sock *sk) __ksym; extern __u32 cubictcp_recalc_ssthresh(struct sock *sk) __ksym; extern void cubictcp_state(struct sock *sk, __u8 new_state) __ksym; extern __u32 tcp_reno_undo_cwnd(struct sock *sk) __ksym; @@ -108,9 +108,9 @@ void BPF_PROG(bpf_cubic_init, struct sock *sk) } SEC("struct_ops") -void BPF_PROG(bpf_cubic_cwnd_event, struct sock *sk, enum tcp_ca_event event) +void BPF_PROG(bpf_cubic_cwnd_event_tx_start, struct sock *sk) { - cubictcp_cwnd_event(sk, event); + cubictcp_cwnd_event_tx_start(sk); } SEC("struct_ops") @@ -172,7 +172,7 @@ struct tcp_congestion_ops cc_cubic = { .cong_control = (void *)bpf_cubic_cong_control, .set_state = (void *)bpf_cubic_state, .undo_cwnd = (void *)bpf_cubic_undo_cwnd, - .cwnd_event = (void *)bpf_cubic_cwnd_event, + .cwnd_event_tx_start = (void *)bpf_cubic_cwnd_event_tx_start, .pkts_acked = (void *)bpf_cubic_acked, .name = "bpf_cc_cubic", }; diff --git a/tools/testing/selftests/bpf/progs/bpf_cubic.c b/tools/testing/selftests/bpf/progs/bpf_cubic.c index 46fb2b37d3a7..ebd5a1e69f56 100644 --- a/tools/testing/selftests/bpf/progs/bpf_cubic.c +++ b/tools/testing/selftests/bpf/progs/bpf_cubic.c @@ -16,6 +16,7 @@ #include "bpf_tracing_net.h" #include <bpf/bpf_tracing.h> +#include <errno.h> char _license[] SEC("license") = "GPL"; @@ -170,10 +171,18 @@ static void bictcp_hystart_reset(struct sock *sk) ca->sample_cnt = 0; } +bool nodelay_init_reject = false; +bool nodelay_cwnd_event_tx_start_reject = false; + SEC("struct_ops") void BPF_PROG(bpf_cubic_init, struct sock *sk) { struct bpf_bictcp *ca = inet_csk_ca(sk); + int true_val = 1, ret; + + ret = bpf_setsockopt(sk, SOL_TCP, TCP_NODELAY, &true_val, sizeof(true_val)); + if (ret == -EOPNOTSUPP) + nodelay_init_reject = true; bictcp_reset(ca); @@ -185,24 +194,26 @@ void BPF_PROG(bpf_cubic_init, struct sock *sk) } SEC("struct_ops") -void BPF_PROG(bpf_cubic_cwnd_event, struct sock *sk, enum tcp_ca_event event) +void BPF_PROG(bpf_cubic_cwnd_event_tx_start, struct sock *sk) { - if (event == CA_EVENT_TX_START) { - struct bpf_bictcp *ca = inet_csk_ca(sk); - __u32 now = tcp_jiffies32; - __s32 delta; - - delta = now - tcp_sk(sk)->lsndtime; - - /* We were application limited (idle) for a while. - * Shift epoch_start to keep cwnd growth to cubic curve. - */ - if (ca->epoch_start && delta > 0) { - ca->epoch_start += delta; - if (after(ca->epoch_start, now)) - ca->epoch_start = now; - } - return; + struct bpf_bictcp *ca = inet_csk_ca(sk); + __u32 now = tcp_jiffies32; + int true_val = 1, ret; + __s32 delta; + + ret = bpf_setsockopt(sk, SOL_TCP, TCP_NODELAY, &true_val, sizeof(true_val)); + if (ret == -EOPNOTSUPP) + nodelay_cwnd_event_tx_start_reject = true; + + delta = now - tcp_sk(sk)->lsndtime; + + /* We were application limited (idle) for a while. + * Shift epoch_start to keep cwnd growth to cubic curve. + */ + if (ca->epoch_start && delta > 0) { + ca->epoch_start += delta; + if (after(ca->epoch_start, now)) + ca->epoch_start = now; } } @@ -537,7 +548,7 @@ struct tcp_congestion_ops cubic = { .cong_avoid = (void *)bpf_cubic_cong_avoid, .set_state = (void *)bpf_cubic_state, .undo_cwnd = (void *)bpf_cubic_undo_cwnd, - .cwnd_event = (void *)bpf_cubic_cwnd_event, + .cwnd_event_tx_start = (void *)bpf_cubic_cwnd_event_tx_start, .pkts_acked = (void *)bpf_cubic_acked, .name = "bpf_cubic", }; diff --git a/tools/testing/selftests/bpf/progs/bpf_for_bench.c b/tools/testing/selftests/bpf/progs/bpf_for_bench.c new file mode 100644 index 000000000000..f9c723051fc7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_for_bench.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +int nr_loops; +long hits; + +static int outer_loop(__u32 index, void *data) +{ + int i; + + /* + * Empty body: the work being measured is the open-coded numeric iterator itself + * (bpf_iter_num_new/next/destroy behind bpf_for()). + */ + bpf_for(i, 0, nr_loops) + ; + __sync_add_and_fetch(&hits, nr_loops); + return 0; +} + +SEC("fentry/" SYS_PREFIX "sys_getpgid") +int benchmark(void *ctx) +{ + bpf_loop(1000, outer_loop, NULL, 0); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/bpf_gotox.c b/tools/testing/selftests/bpf/progs/bpf_gotox.c index 216c71b94c64..99b3c9c9a01c 100644 --- a/tools/testing/selftests/bpf/progs/bpf_gotox.c +++ b/tools/testing/selftests/bpf/progs/bpf_gotox.c @@ -421,6 +421,36 @@ int use_nonstatic_global_other_sec(void *ctx) return __nonstatic_global(in_user); } +SEC("syscall") +int load_with_nonzero_offset(struct simple_ctx *ctx) +{ + void *jj[] = { &&l1, &&l2, &&l3 }; + + /* + * This makes LLVM to generate a load from the jj map with an offset: + * r1 = 0x0 ll + * r1 = *(u64 *)(r1 + 0x10) + * gotox r1 + */ + if (ctx->x == 2) + goto *jj[ctx->x]; + + ret_user = 1; + return 1; + +l1: + /* never reached, but leave it here to outsmart LLVM */ + ret_user = 0; + return 0; +l2: + /* never reached, but leave it here to outsmart LLVM */ + ret_user = 3; + return 3; +l3: + ret_user = 5; + return 5; +} + #else /* __BPF_FEATURE_GOTOX */ #define SKIP_TEST(TEST_NAME) \ @@ -442,6 +472,7 @@ SKIP_TEST(use_static_global_other_sec); SKIP_TEST(use_nonstatic_global1); SKIP_TEST(use_nonstatic_global2); SKIP_TEST(use_nonstatic_global_other_sec); +SKIP_TEST(load_with_nonzero_offset); #endif /* __BPF_FEATURE_GOTOX */ diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_bpf_rhash_map.c b/tools/testing/selftests/bpf/progs/bpf_iter_bpf_rhash_map.c new file mode 100644 index 000000000000..86f6c0d5eadb --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_iter_bpf_rhash_map.c @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_RHASH); + __uint(map_flags, BPF_F_NO_PREALLOC); + __uint(max_entries, 64); + __type(key, __u32); + __type(value, __u64); +} rhashmap SEC(".maps"); + +__u32 key_sum = 0; +__u64 val_sum = 0; +__u32 elem_count = 0; +__u32 err = 0; + +SEC("iter/bpf_map_elem") +int dump_bpf_rhash_map(struct bpf_iter__bpf_map_elem *ctx) +{ + __u32 *key = ctx->key; + __u64 *val = ctx->value; + + if (!key || !val) + return 0; + + key_sum += *key; + val_sum += *val; + elem_count++; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c b/tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c index d64ba7ddaed5..d7fb561ed4fb 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c @@ -52,7 +52,7 @@ SEC("iter/task_vma") int proc_maps(struct bpf_iter__task_vma *ctx) bpf_d_path(&file->f_path, d_path_buf, D_PATH_BUF_SIZE); BPF_SEQ_PRINTF(seq, "%08llx ", vma->vm_pgoff << 12); - BPF_SEQ_PRINTF(seq, "%02x:%02x %u", MAJOR(dev), MINOR(dev), + BPF_SEQ_PRINTF(seq, "%02x:%02x %llu", MAJOR(dev), MINOR(dev), file->f_inode->i_ino); BPF_SEQ_PRINTF(seq, "\t%s\n", d_path_buf); } else { diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_unix.c b/tools/testing/selftests/bpf/progs/bpf_iter_unix.c index fea275df9e22..a2652c8c3616 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_unix.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_unix.c @@ -7,6 +7,13 @@ char _license[] SEC("license") = "GPL"; +SEC(".maps") struct { + __uint(type, BPF_MAP_TYPE_SOCKMAP); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u64); +} sockmap; + static long sock_i_ino(const struct sock *sk) { const struct socket *sk_socket = sk->sk_socket; @@ -76,5 +83,8 @@ int dump_unix(struct bpf_iter__unix *ctx) BPF_SEQ_PRINTF(seq, "\n"); + /* Test for deadlock. */ + bpf_map_update_elem(&sockmap, &(int){0}, sk, 0); + return 0; } diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index c9bfbe1bafc1..5eacf1b43252 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -103,8 +103,8 @@ * - TEST_DATA_LEN * __retval_unpriv Same, but load program in unprivileged mode. * - * __description Text to be used instead of a program name for display - * and filtering purposes. + * __description Text to be used for display and as an additional filter + * alias, while the original program name stays matchable. * * __log_level Log level to use for the program, numeric value expected. * @@ -130,39 +130,43 @@ * __linear_size Specify the size of the linear area of non-linear skbs, or * 0 for linear skbs. */ -#define __msg(msg) __attribute__((btf_decl_tag("comment:test_expect_msg=" XSTR(__COUNTER__) "=" msg))) -#define __not_msg(msg) __attribute__((btf_decl_tag("comment:test_expect_not_msg=" XSTR(__COUNTER__) "=" msg))) -#define __xlated(msg) __attribute__((btf_decl_tag("comment:test_expect_xlated=" XSTR(__COUNTER__) "=" msg))) -#define __jited(msg) __attribute__((btf_decl_tag("comment:test_jited=" XSTR(__COUNTER__) "=" msg))) -#define __failure __attribute__((btf_decl_tag("comment:test_expect_failure"))) -#define __success __attribute__((btf_decl_tag("comment:test_expect_success"))) -#define __description(desc) __attribute__((btf_decl_tag("comment:test_description=" desc))) -#define __msg_unpriv(msg) __attribute__((btf_decl_tag("comment:test_expect_msg_unpriv=" XSTR(__COUNTER__) "=" msg))) -#define __not_msg_unpriv(msg) __attribute__((btf_decl_tag("comment:test_expect_not_msg_unpriv=" XSTR(__COUNTER__) "=" msg))) -#define __xlated_unpriv(msg) __attribute__((btf_decl_tag("comment:test_expect_xlated_unpriv=" XSTR(__COUNTER__) "=" msg))) -#define __jited_unpriv(msg) __attribute__((btf_decl_tag("comment:test_jited=" XSTR(__COUNTER__) "=" msg))) -#define __failure_unpriv __attribute__((btf_decl_tag("comment:test_expect_failure_unpriv"))) -#define __success_unpriv __attribute__((btf_decl_tag("comment:test_expect_success_unpriv"))) -#define __log_level(lvl) __attribute__((btf_decl_tag("comment:test_log_level="#lvl))) -#define __flag(flag) __attribute__((btf_decl_tag("comment:test_prog_flags="#flag))) -#define __retval(val) __attribute__((btf_decl_tag("comment:test_retval="XSTR(val)))) -#define __retval_unpriv(val) __attribute__((btf_decl_tag("comment:test_retval_unpriv="XSTR(val)))) -#define __auxiliary __attribute__((btf_decl_tag("comment:test_auxiliary"))) -#define __auxiliary_unpriv __attribute__((btf_decl_tag("comment:test_auxiliary_unpriv"))) -#define __btf_path(path) __attribute__((btf_decl_tag("comment:test_btf_path=" path))) -#define __arch(arch) __attribute__((btf_decl_tag("comment:test_arch=" arch))) +#define __test_tag(tag) __attribute__((btf_decl_tag("comment:" XSTR(__COUNTER__) ":" tag))) + +#define __msg(msg) __test_tag("test_expect_msg=" msg) +#define __not_msg(msg) __test_tag("test_expect_not_msg=" msg) +#define __xlated(msg) __test_tag("test_expect_xlated=" msg) +#define __jited(msg) __test_tag("test_jited=" msg) +#define __failure __test_tag("test_expect_failure") +#define __success __test_tag("test_expect_success") +#define __description(desc) __test_tag("test_description=" desc) +#define __msg_unpriv(msg) __test_tag("test_expect_msg_unpriv=" msg) +#define __not_msg_unpriv(msg) __test_tag("test_expect_not_msg_unpriv=" msg) +#define __xlated_unpriv(msg) __test_tag("test_expect_xlated_unpriv=" msg) +#define __jited_unpriv(msg) __test_tag("test_jited_unpriv=" msg) +#define __failure_unpriv __test_tag("test_expect_failure_unpriv") +#define __success_unpriv __test_tag("test_expect_success_unpriv") +#define __log_level(lvl) __test_tag("test_log_level=" #lvl) +#define __flag(flag) __test_tag("test_prog_flags=" #flag) +#define __retval(val) __test_tag("test_retval=" XSTR(val)) +#define __retval_unpriv(val) __test_tag("test_retval_unpriv=" XSTR(val)) +#define __auxiliary __test_tag("test_auxiliary") +#define __auxiliary_unpriv __test_tag("test_auxiliary_unpriv") +#define __btf_path(path) __test_tag("test_btf_path=" path) +#define __btf_func_path(path) __test_tag("test_btf_func_path=" path) +#define __arch(arch) __test_tag("test_arch=" arch) #define __arch_x86_64 __arch("X86_64") #define __arch_arm64 __arch("ARM64") #define __arch_riscv64 __arch("RISCV64") #define __arch_s390x __arch("s390x") -#define __caps_unpriv(caps) __attribute__((btf_decl_tag("comment:test_caps_unpriv=" EXPAND_QUOTE(caps)))) -#define __load_if_JITed() __attribute__((btf_decl_tag("comment:load_mode=jited"))) -#define __load_if_no_JITed() __attribute__((btf_decl_tag("comment:load_mode=no_jited"))) -#define __stderr(msg) __attribute__((btf_decl_tag("comment:test_expect_stderr=" XSTR(__COUNTER__) "=" msg))) -#define __stderr_unpriv(msg) __attribute__((btf_decl_tag("comment:test_expect_stderr_unpriv=" XSTR(__COUNTER__) "=" msg))) -#define __stdout(msg) __attribute__((btf_decl_tag("comment:test_expect_stdout=" XSTR(__COUNTER__) "=" msg))) -#define __stdout_unpriv(msg) __attribute__((btf_decl_tag("comment:test_expect_stdout_unpriv=" XSTR(__COUNTER__) "=" msg))) -#define __linear_size(sz) __attribute__((btf_decl_tag("comment:test_linear_size=" XSTR(sz)))) +#define __arch_loongarch __arch("LOONGARCH") +#define __caps_unpriv(caps) __test_tag("test_caps_unpriv=" EXPAND_QUOTE(caps)) +#define __load_if_JITed() __test_tag("load_mode=jited") +#define __load_if_no_JITed() __test_tag("load_mode=no_jited") +#define __stderr(msg) __test_tag("test_expect_stderr=" msg) +#define __stderr_unpriv(msg) __test_tag("test_expect_stderr_unpriv=" msg) +#define __stdout(msg) __test_tag("test_expect_stdout=" msg) +#define __stdout_unpriv(msg) __test_tag("test_expect_stdout_unpriv=" msg) +#define __linear_size(sz) __test_tag("test_linear_size=" XSTR(sz)) /* Define common capabilities tested using __caps_unpriv */ #define CAP_NET_ADMIN 12 @@ -188,6 +192,10 @@ #define POINTER_VALUE 0xbadcafe #define TEST_DATA_LEN 64 +#ifndef __aligned +#define __aligned(x) __attribute__((aligned(x))) +#endif + #ifndef __used #define __used __attribute__((used)) #endif @@ -256,9 +264,12 @@ #endif #if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) || \ - (defined(__TARGET_ARCH_powerpc)) + (defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ + defined(__TARGET_ARCH_s390) || \ + defined(__TARGET_ARCH_powerpc) || \ + defined(__TARGET_ARCH_loongarch)) #define CAN_USE_LOAD_ACQ_STORE_REL #endif diff --git a/tools/testing/selftests/bpf/progs/bpf_nop_bench.c b/tools/testing/selftests/bpf/progs/bpf_nop_bench.c new file mode 100644 index 000000000000..01ed284c1bb3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_nop_bench.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bench_bpf_timing.bpf.h" + +SEC("syscall") +int bench_nop(void *ctx) +{ + return BENCH_BPF_LOOP(0, ({})); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/bpf_qdisc_dynptr_use_after_invalidate_clone.c b/tools/testing/selftests/bpf/progs/bpf_qdisc_dynptr_use_after_invalidate_clone.c new file mode 100644 index 000000000000..ac626cfa2a98 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_qdisc_dynptr_use_after_invalidate_clone.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include "bpf_experimental.h" +#include "bpf_qdisc_common.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +int proto; + +SEC("struct_ops") +__success +int BPF_PROG(dynptr_use_after_invalidate_clone, struct sk_buff *skb, struct Qdisc *sch, + struct bpf_sk_buff_ptr *to_free) +{ + struct bpf_dynptr ptr, ptr_clone; + struct ethhdr *hdr; + + bpf_dynptr_from_skb((struct __sk_buff *)skb, 0, &ptr); + + bpf_dynptr_clone(&ptr, &ptr_clone); + + hdr = bpf_dynptr_slice(&ptr_clone, 0, NULL, sizeof(*hdr)); + if (!hdr) { + bpf_qdisc_skb_drop(skb, to_free); + return NET_XMIT_DROP; + } + + *(int *)&ptr = 0; + + proto = hdr->h_proto; + + bpf_qdisc_skb_drop(skb, to_free); + + return NET_XMIT_DROP; +} + +SEC("struct_ops") +__auxiliary +struct sk_buff *BPF_PROG(bpf_qdisc_test_dequeue, struct Qdisc *sch) +{ + return NULL; +} + +SEC("struct_ops") +__auxiliary +int BPF_PROG(bpf_qdisc_test_init, struct Qdisc *sch, struct nlattr *opt, + struct netlink_ext_ack *extack) +{ + return 0; +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_reset, struct Qdisc *sch) +{ +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_destroy, struct Qdisc *sch) +{ +} + +SEC(".struct_ops") +struct Qdisc_ops test = { + .enqueue = (void *)dynptr_use_after_invalidate_clone, + .dequeue = (void *)bpf_qdisc_test_dequeue, + .init = (void *)bpf_qdisc_test_init, + .reset = (void *)bpf_qdisc_test_reset, + .destroy = (void *)bpf_qdisc_test_destroy, + .id = "bpf_qdisc_test", +}; diff --git a/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr.c b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr.c new file mode 100644 index 000000000000..1d96f7987a3f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include "bpf_experimental.h" +#include "bpf_qdisc_common.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +int proto; + +SEC("struct_ops") +__failure __msg("Expected an initialized dynptr as R1") +int BPF_PROG(invalid_dynptr, struct sk_buff *skb, struct Qdisc *sch, + struct bpf_sk_buff_ptr *to_free) +{ + struct bpf_dynptr ptr; + struct ethhdr *hdr; + + bpf_dynptr_from_skb((struct __sk_buff *)skb, 0, &ptr); + + bpf_qdisc_skb_drop(skb, to_free); + + hdr = bpf_dynptr_slice(&ptr, 0, NULL, sizeof(*hdr)); + if (!hdr) + return NET_XMIT_DROP; + + proto = hdr->h_proto; + + return NET_XMIT_DROP; +} + +SEC("struct_ops") +__auxiliary +struct sk_buff *BPF_PROG(bpf_qdisc_test_dequeue, struct Qdisc *sch) +{ + return NULL; +} + +SEC("struct_ops") +__auxiliary +int BPF_PROG(bpf_qdisc_test_init, struct Qdisc *sch, struct nlattr *opt, + struct netlink_ext_ack *extack) +{ + return 0; +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_reset, struct Qdisc *sch) +{ +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_destroy, struct Qdisc *sch) +{ +} + +SEC(".struct_ops") +struct Qdisc_ops test = { + .enqueue = (void *)invalid_dynptr, + .dequeue = (void *)bpf_qdisc_test_dequeue, + .init = (void *)bpf_qdisc_test_init, + .reset = (void *)bpf_qdisc_test_reset, + .destroy = (void *)bpf_qdisc_test_destroy, + .id = "bpf_qdisc_test", +}; diff --git a/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_cross_frame.c b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_cross_frame.c new file mode 100644 index 000000000000..2e23b8593af9 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_cross_frame.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include "bpf_experimental.h" +#include "bpf_qdisc_common.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +int proto; + +static __noinline int free_skb(struct sk_buff *skb) +{ + bpf_kfree_skb(skb); + return 0; +} + +SEC("struct_ops") +__failure __msg("invalid mem access 'scalar'") +int BPF_PROG(invalid_dynptr_cross_frame, struct sk_buff *skb, struct Qdisc *sch, + struct bpf_sk_buff_ptr *to_free) +{ + struct bpf_dynptr ptr; + struct ethhdr *hdr; + + bpf_dynptr_from_skb((struct __sk_buff *)skb, 0, &ptr); + + hdr = bpf_dynptr_slice(&ptr, 0, NULL, sizeof(*hdr)); + if (!hdr) + return NET_XMIT_DROP; + + free_skb(skb); + + proto = hdr->h_proto; + + return NET_XMIT_DROP; +} + +SEC("struct_ops") +__auxiliary +struct sk_buff *BPF_PROG(bpf_qdisc_test_dequeue, struct Qdisc *sch) +{ + return NULL; +} + +SEC("struct_ops") +__auxiliary +int BPF_PROG(bpf_qdisc_test_init, struct Qdisc *sch, struct nlattr *opt, + struct netlink_ext_ack *extack) +{ + return 0; +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_reset, struct Qdisc *sch) +{ +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_destroy, struct Qdisc *sch) +{ +} + +SEC(".struct_ops") +struct Qdisc_ops test = { + .enqueue = (void *)invalid_dynptr_cross_frame, + .dequeue = (void *)bpf_qdisc_test_dequeue, + .init = (void *)bpf_qdisc_test_init, + .reset = (void *)bpf_qdisc_test_reset, + .destroy = (void *)bpf_qdisc_test_destroy, + .id = "bpf_qdisc_test", +}; diff --git a/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_slice.c b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_slice.c new file mode 100644 index 000000000000..731216c4e45a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__invalid_dynptr_slice.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include "bpf_experimental.h" +#include "bpf_qdisc_common.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +int proto; + +SEC("struct_ops") +__failure __msg("invalid mem access 'scalar'") +int BPF_PROG(invalid_dynptr_slice, struct sk_buff *skb, struct Qdisc *sch, + struct bpf_sk_buff_ptr *to_free) +{ + struct bpf_dynptr ptr; + struct ethhdr *hdr; + + bpf_dynptr_from_skb((struct __sk_buff *)skb, 0, &ptr); + + hdr = bpf_dynptr_slice(&ptr, 0, NULL, sizeof(*hdr)); + if (!hdr) { + bpf_qdisc_skb_drop(skb, to_free); + return NET_XMIT_DROP; + } + + bpf_qdisc_skb_drop(skb, to_free); + + proto = hdr->h_proto; + + return NET_XMIT_DROP; +} + +SEC("struct_ops") +__auxiliary +struct sk_buff *BPF_PROG(bpf_qdisc_test_dequeue, struct Qdisc *sch) +{ + return NULL; +} + +SEC("struct_ops") +__auxiliary +int BPF_PROG(bpf_qdisc_test_init, struct Qdisc *sch, struct nlattr *opt, + struct netlink_ext_ack *extack) +{ + return 0; +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_reset, struct Qdisc *sch) +{ +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_destroy, struct Qdisc *sch) +{ +} + +SEC(".struct_ops") +struct Qdisc_ops test = { + .enqueue = (void *)invalid_dynptr_slice, + .dequeue = (void *)bpf_qdisc_test_dequeue, + .init = (void *)bpf_qdisc_test_init, + .reset = (void *)bpf_qdisc_test_reset, + .destroy = (void *)bpf_qdisc_test_destroy, + .id = "bpf_qdisc_test", +}; diff --git a/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__untrusted_write.c b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__untrusted_write.c new file mode 100644 index 000000000000..688c2a049ae3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__untrusted_write.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include "bpf_experimental.h" +#include "bpf_qdisc_common.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +SEC("struct_ops") +__failure __msg("only read is supported") +int BPF_PROG(untrusted_write, struct sk_buff *skb, struct Qdisc *sch, + struct bpf_sk_buff_ptr *to_free) +{ + struct Qdisc *next = sch->next_sched; + + /* + * sch is trusted, but the walk of next_sched yields a plain + * PTR_TO_BTF_ID which may fault on a dereference. A store through + * it does not get an exception table entry, there is no probed + * store to rewrite it into, hence it has to be rejected before + * bpf_qdisc_btf_struct_access() gets to allow the write to limit. + */ + next->limit = 1000; + + bpf_qdisc_skb_drop(skb, to_free); + return NET_XMIT_DROP; +} + +SEC("struct_ops") +__auxiliary +struct sk_buff *BPF_PROG(bpf_qdisc_test_dequeue, struct Qdisc *sch) +{ + return NULL; +} + +SEC("struct_ops") +__auxiliary +int BPF_PROG(bpf_qdisc_test_init, struct Qdisc *sch, struct nlattr *opt, + struct netlink_ext_ack *extack) +{ + return 0; +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_reset, struct Qdisc *sch) +{ +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_destroy, struct Qdisc *sch) +{ +} + +SEC(".struct_ops") +struct Qdisc_ops test = { + .enqueue = (void *)untrusted_write, + .dequeue = (void *)bpf_qdisc_test_dequeue, + .init = (void *)bpf_qdisc_test_init, + .reset = (void *)bpf_qdisc_test_reset, + .destroy = (void *)bpf_qdisc_test_destroy, + .id = "bpf_qdisc_test", +}; diff --git a/tools/testing/selftests/bpf/progs/bpf_qdisc_fq.c b/tools/testing/selftests/bpf/progs/bpf_qdisc_fq.c index 1a3233a275c7..8107f5934d2d 100644 --- a/tools/testing/selftests/bpf/progs/bpf_qdisc_fq.c +++ b/tools/testing/selftests/bpf/progs/bpf_qdisc_fq.c @@ -196,18 +196,13 @@ fq_flows_remove_front(struct bpf_list_head *head, struct bpf_spin_lock *lock, static bool fq_flows_is_empty(struct bpf_list_head *head, struct bpf_spin_lock *lock) { - struct bpf_list_node *node; + bool empty; bpf_spin_lock(lock); - node = bpf_list_pop_front(head); - if (node) { - bpf_list_push_front(head, node); - bpf_spin_unlock(lock); - return false; - } + empty = bpf_list_empty(head); bpf_spin_unlock(lock); - return true; + return empty; } /* flow->age is used to denote the state of the flow (not-detached, detached, throttled) diff --git a/tools/testing/selftests/bpf/progs/bpf_smc.c b/tools/testing/selftests/bpf/progs/bpf_smc.c index 70d8b08f5914..6263a45bf006 100644 --- a/tools/testing/selftests/bpf/progs/bpf_smc.c +++ b/tools/testing/selftests/bpf/progs/bpf_smc.c @@ -8,6 +8,10 @@ char _license[] SEC("license") = "GPL"; +#ifndef SMC_HS_CTRL_NAME_MAX +#define SMC_HS_CTRL_NAME_MAX 16 +#endif + enum { BPF_SMC_LISTEN = 10, }; @@ -18,6 +22,20 @@ struct smc_sock___local { bool use_fallback; } __attribute__((preserve_access_index)); +struct smc_hs_ctrl___local { + char name[SMC_HS_CTRL_NAME_MAX]; + int (*syn_option)(struct tcp_sock *); + int (*synack_option)(const struct tcp_sock *, struct inet_request_sock *); +} __attribute__((preserve_access_index)); + +struct netns_smc___local { + struct smc_hs_ctrl___local *hs_ctrl; +} __attribute__((preserve_access_index)); + +struct net___local { + struct netns_smc___local smc; +} __attribute__((preserve_access_index)); + int smc_cnt = 0; int fallback_cnt = 0; @@ -88,8 +106,14 @@ int BPF_PROG(smc_run, int family, int type, int protocol) task = bpf_get_current_task_btf(); /* Prevent from affecting other tests */ - if (!task || !task->nsproxy->net_ns->smc.hs_ctrl) + if (!task) { return protocol; + } else { + struct net___local *net = (struct net___local *)task->nsproxy->net_ns; + + if (!bpf_core_field_exists(struct net___local, smc) || !net->smc.hs_ctrl) + return protocol; + } return IPPROTO_SMC; } @@ -110,7 +134,7 @@ int BPF_PROG(bpf_smc_set_tcp_option, struct tcp_sock *tp) } SEC(".struct_ops") -struct smc_hs_ctrl linkcheck = { +struct smc_hs_ctrl___local linkcheck = { .name = "linkcheck", .syn_option = (void *)bpf_smc_set_tcp_option, .synack_option = (void *)bpf_smc_set_tcp_option_cond, diff --git a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h index d8dacef37c16..593b38f90417 100644 --- a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h +++ b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h @@ -8,6 +8,9 @@ #define AF_INET 2 #define AF_INET6 10 +/* include/linux/net.h */ +#define SOCK_TYPE_MASK 0xf + #define SOL_SOCKET 1 #define SO_REUSEADDR 2 #define SO_SNDBUF 7 @@ -28,10 +31,12 @@ #define __SO_ACCEPTCON (1 << 16) #define IP_TOS 1 +#define IP_TRANSPARENT 19 #define SOL_IPV6 41 #define IPV6_TCLASS 67 #define IPV6_AUTOFLOWLABEL 70 +#define IPV6_TRANSPARENT 75 #define TC_ACT_UNSPEC (-1) #define TC_ACT_OK 0 diff --git a/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c b/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c new file mode 100644 index 000000000000..8d38aafe66a2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "../test_kmods/bpf_testmod_kfunc.h" + +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) + +long subprog_call_mem_kfunc(long a, long b, long c, long d, long e, long size) +{ + char buf[8] = {}; + + return bpf_kfunc_call_stack_arg_mem(a, b, c, d, e, buf, size); +} + +#else + +long subprog_call_mem_kfunc(void) +{ + return 0; +} + +#endif diff --git a/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c new file mode 100644 index 000000000000..99bc115f8380 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) + +int subprog_bad_order_6args(int a, int b, int c, int d, int e, int f) +{ + return a + b + c + d + e + f; +} + +int subprog_call_before_load_6args(int a, int b, int c, int d, int e, int f) +{ + return a + b + c + d + e + f; +} + +int subprog_pruning_call_before_load_6args(int a, int b, int c, int d, int e, int f) +{ + return a + b + c + d + e + f; +} + +void subprog_bad_ptr_7args(long *a, int b, int c, int d, int e, int f, int g) +{ +} + +#else + +int subprog_bad_order_6args(void) +{ + return 0; +} + +int subprog_call_before_load_6args(void) +{ + return 0; +} + +int subprog_pruning_call_before_load_6args(void) +{ + return 0; +} + +void subprog_bad_ptr_7args(void) +{ +} + +#endif diff --git a/tools/testing/selftests/bpf/progs/btf_data.c b/tools/testing/selftests/bpf/progs/btf_data.c index baa525275bde..8082c13490ab 100644 --- a/tools/testing/selftests/bpf/progs/btf_data.c +++ b/tools/testing/selftests/bpf/progs/btf_data.c @@ -48,3 +48,43 @@ int func(struct root_struct *root) { return 0; } + +int kfunc_a(struct root_struct *root) +{ + return 0; +} + +int kfunc_b(struct root_struct *root) +{ + return 0; +} + +struct root_struct *kfunc_c(struct root_struct *a, struct root_struct *b) +{ + return a; +} + +int kfunc_d(struct root_struct *a, struct root_struct *b) +{ + return 0; +} + +int kfunc_e(struct root_struct *a__arena, + struct root_struct *b__arena__nullable, + struct root_struct *c__arena, + struct root_struct *d__arena__nullable, + struct root_struct *e__arena) +{ + return 0; +} + +int kfunc_f(struct root_struct *a, struct root_struct *b__arena, int flags) +{ + return 0; +} + +struct root_struct *kfunc_g(struct root_struct *a__arena, + struct root_struct *b__arena__nullable) +{ + return a__arena; +} diff --git a/tools/testing/selftests/bpf/progs/cgroup_iter_memcg.c b/tools/testing/selftests/bpf/progs/cgroup_iter_memcg.c index 59fb70a3cc50..06a385c9d85b 100644 --- a/tools/testing/selftests/bpf/progs/cgroup_iter_memcg.c +++ b/tools/testing/selftests/bpf/progs/cgroup_iter_memcg.c @@ -26,12 +26,18 @@ int cgroup_memcg_query(struct bpf_iter__cgroup *ctx) bpf_mem_cgroup_flush_stats(memcg); - memcg_query.nr_anon_mapped = bpf_mem_cgroup_page_state(memcg, NR_ANON_MAPPED); - memcg_query.nr_shmem = bpf_mem_cgroup_page_state(memcg, NR_SHMEM); - memcg_query.nr_file_pages = bpf_mem_cgroup_page_state(memcg, NR_FILE_PAGES); - memcg_query.nr_file_mapped = bpf_mem_cgroup_page_state(memcg, NR_FILE_MAPPED); - memcg_query.memcg_kmem = bpf_mem_cgroup_page_state(memcg, MEMCG_KMEM); - memcg_query.pgfault = bpf_mem_cgroup_vm_events(memcg, PGFAULT); + memcg_query.nr_anon_mapped = bpf_mem_cgroup_page_state( + memcg, + bpf_core_enum_value(enum node_stat_item, NR_ANON_MAPPED)); + memcg_query.nr_shmem = bpf_mem_cgroup_page_state( + memcg, bpf_core_enum_value(enum node_stat_item, NR_SHMEM)); + memcg_query.nr_file_pages = bpf_mem_cgroup_page_state( + memcg, bpf_core_enum_value(enum node_stat_item, NR_FILE_PAGES)); + memcg_query.nr_file_mapped = bpf_mem_cgroup_page_state( + memcg, + bpf_core_enum_value(enum node_stat_item, NR_FILE_MAPPED)); + memcg_query.pgfault = bpf_mem_cgroup_vm_events( + memcg, bpf_core_enum_value(enum vm_event_item, PGFAULT)); bpf_put_mem_cgroup(memcg); diff --git a/tools/testing/selftests/bpf/progs/cgroup_storage.c b/tools/testing/selftests/bpf/progs/cgroup_storage.c index db1e4d2d3281..59da1d95e5b9 100644 --- a/tools/testing/selftests/bpf/progs/cgroup_storage.c +++ b/tools/testing/selftests/bpf/progs/cgroup_storage.c @@ -21,4 +21,47 @@ int bpf_prog(struct __sk_buff *skb) return (*counter & 1); } +/* Maps for OOB test */ +struct { + __uint(type, BPF_MAP_TYPE_CGROUP_STORAGE); + __type(key, struct bpf_cgroup_storage_key); + __type(value, __u32); /* 4-byte value - not 8-byte aligned */ +} cgroup_storage_oob SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_LRU_PERCPU_HASH); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u32); /* 4-byte value - same as cgroup storage */ +} lru_map SEC(".maps"); + +SEC("cgroup/sock_create") +int trigger_oob(struct bpf_sock *sk) +{ + __u32 key = 0; + __u32 *cgroup_val; + __u32 value = 0x12345678; + + /* Get cgroup storage value */ + cgroup_val = bpf_get_local_storage(&cgroup_storage_oob, 0); + if (!cgroup_val) + return 0; + + /* Initialize cgroup storage */ + *cgroup_val = value; + + /* This triggers the OOB read: + * bpf_map_update_elem() -> htab_map_update_elem() -> + * pcpu_init_value() -> copy_map_value_long() -> + * bpf_obj_memcpy(..., long_memcpy=true) -> + * bpf_long_memcpy(dst, src, round_up(4, 8)) + * + * The copy size is rounded up to 8 bytes, but cgroup_val + * points to a 4-byte buffer, causing a 4-byte OOB read. + */ + bpf_map_update_elem(&lru_map, &key, cgroup_val, BPF_ANY); + + return 1; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c index 9fe9c4a4e8f6..efe7bcae70f8 100644 --- a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c @@ -29,7 +29,7 @@ static struct __cgrps_kfunc_map_value *insert_lookup_cgrp(struct cgroup *cgrp) } SEC("tp_btf/cgroup_mkdir") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(cgrp_kfunc_acquire_untrusted, struct cgroup *cgrp, const char *path) { struct cgroup *acquired; @@ -48,7 +48,7 @@ int BPF_PROG(cgrp_kfunc_acquire_untrusted, struct cgroup *cgrp, const char *path } SEC("tp_btf/cgroup_mkdir") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(cgrp_kfunc_acquire_no_null_check, struct cgroup *cgrp, const char *path) { struct cgroup *acquired; @@ -64,7 +64,7 @@ int BPF_PROG(cgrp_kfunc_acquire_no_null_check, struct cgroup *cgrp, const char * } SEC("tp_btf/cgroup_mkdir") -__failure __msg("arg#0 pointer type STRUCT cgroup must point") +__failure __msg("R1 is fp expected STRUCT cgroup") int BPF_PROG(cgrp_kfunc_acquire_fp, struct cgroup *cgrp, const char *path) { struct cgroup *acquired, *stack_cgrp = (struct cgroup *)&path; @@ -106,7 +106,7 @@ int BPF_PROG(cgrp_kfunc_acquire_trusted_walked, struct cgroup *cgrp, const char } SEC("tp_btf/cgroup_mkdir") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(cgrp_kfunc_acquire_null, struct cgroup *cgrp, const char *path) { struct cgroup *acquired; @@ -154,7 +154,7 @@ int BPF_PROG(cgrp_kfunc_xchg_unreleased, struct cgroup *cgrp, const char *path) } SEC("tp_btf/cgroup_mkdir") -__failure __msg("must be referenced or trusted") +__failure __msg("release kfunc bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(cgrp_kfunc_rcu_get_release, struct cgroup *cgrp, const char *path) { struct cgroup *kptr; @@ -175,7 +175,7 @@ int BPF_PROG(cgrp_kfunc_rcu_get_release, struct cgroup *cgrp, const char *path) } SEC("tp_btf/cgroup_mkdir") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(cgrp_kfunc_release_untrusted, struct cgroup *cgrp, const char *path) { struct __cgrps_kfunc_map_value *v; @@ -191,7 +191,7 @@ int BPF_PROG(cgrp_kfunc_release_untrusted, struct cgroup *cgrp, const char *path } SEC("tp_btf/cgroup_mkdir") -__failure __msg("arg#0 pointer type STRUCT cgroup must point") +__failure __msg("release kfunc bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(cgrp_kfunc_release_fp, struct cgroup *cgrp, const char *path) { struct cgroup *acquired = (struct cgroup *)&path; @@ -203,7 +203,7 @@ int BPF_PROG(cgrp_kfunc_release_fp, struct cgroup *cgrp, const char *path) } SEC("tp_btf/cgroup_mkdir") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(cgrp_kfunc_release_null, struct cgroup *cgrp, const char *path) { struct __cgrps_kfunc_map_value local, *v; @@ -237,7 +237,7 @@ int BPF_PROG(cgrp_kfunc_release_null, struct cgroup *cgrp, const char *path) } SEC("tp_btf/cgroup_mkdir") -__failure __msg("release kernel function bpf_cgroup_release expects") +__failure __msg("release kfunc bpf_cgroup_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(cgrp_kfunc_release_unacquired, struct cgroup *cgrp, const char *path) { /* Cannot release trusted cgroup pointer which was not acquired. */ diff --git a/tools/testing/selftests/bpf/progs/cgrp_ls_sleepable.c b/tools/testing/selftests/bpf/progs/cgrp_ls_sleepable.c index a2de95f85648..37bd6b03ba01 100644 --- a/tools/testing/selftests/bpf/progs/cgrp_ls_sleepable.c +++ b/tools/testing/selftests/bpf/progs/cgrp_ls_sleepable.c @@ -4,6 +4,7 @@ #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> #include "bpf_misc.h" +#include "err.h" char _license[] SEC("license") = "GPL"; @@ -16,6 +17,7 @@ struct { __s32 target_pid; __u64 cgroup_id; +long update_err; int target_hid; bool is_cgroup1; @@ -123,3 +125,19 @@ int yes_rcu_lock(void *ctx) bpf_rcu_read_unlock(); return 0; } + +SEC("fexit/bpf_local_storage_update") +int BPF_PROG(fexit_update, void *owner, struct bpf_local_storage_map *smap, + void *value, u64 map_flags, bool swap_uptrs, + struct bpf_local_storage_data *ret) +{ + struct task_struct *task = bpf_get_current_task_btf(); + + if (task->pid != target_pid) + return 0; + + if (IS_ERR_VALUE(ret)) + update_err = PTR_ERR(ret); + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/clone_attach_btf_id.c b/tools/testing/selftests/bpf/progs/clone_attach_btf_id.c new file mode 100644 index 000000000000..0ffa3ec3e1a0 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/clone_attach_btf_id.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2025 Meta */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +SEC("fentry/bpf_fentry_test1") +int BPF_PROG(fentry_handler, int a) +{ + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/compute_live_registers.c b/tools/testing/selftests/bpf/progs/compute_live_registers.c index f05e120f3450..d055fc7b3b95 100644 --- a/tools/testing/selftests/bpf/progs/compute_live_registers.c +++ b/tools/testing/selftests/bpf/progs/compute_live_registers.c @@ -3,7 +3,7 @@ #include <linux/bpf.h> #include <bpf/bpf_helpers.h> #include "../../../include/linux/filter.h" -#include "bpf_arena_common.h" +#include <bpf_arena_common.h> #include "bpf_misc.h" struct { diff --git a/tools/testing/selftests/bpf/progs/connect_force_port4.c b/tools/testing/selftests/bpf/progs/connect_force_port4.c index 27a632dd382e..d5be6a559d6a 100644 --- a/tools/testing/selftests/bpf/progs/connect_force_port4.c +++ b/tools/testing/selftests/bpf/progs/connect_force_port4.c @@ -14,6 +14,8 @@ char _license[] SEC("license") = "GPL"; +__u16 port = 0; + struct svc_addr { __be32 addr; __be16 port; @@ -40,7 +42,7 @@ int connect4(struct bpf_sock_addr *ctx) if (bpf_bind(ctx, (struct sockaddr *)&sa, sizeof(sa)) != 0) return 0; - /* Rewire service 1.2.3.4:60000 to backend 127.0.0.1:60123. */ + /* Rewire service 1.2.3.4:60000 to backend 127.0.0.1:port. */ if (ctx->user_port == bpf_htons(60000)) { orig = bpf_sk_storage_get(&service_mapping, ctx->sk, 0, BPF_SK_STORAGE_GET_F_CREATE); @@ -51,7 +53,7 @@ int connect4(struct bpf_sock_addr *ctx) orig->port = ctx->user_port; ctx->user_ip4 = bpf_htonl(0x7f000001); - ctx->user_port = bpf_htons(60123); + ctx->user_port = bpf_htons(port); } return 1; } @@ -63,7 +65,7 @@ int getsockname4(struct bpf_sock_addr *ctx) return 1; /* Expose local server as 1.2.3.4:60000 to client. */ - if (ctx->user_port == bpf_htons(60123)) { + if (ctx->user_port == bpf_htons(port)) { ctx->user_ip4 = bpf_htonl(0x01020304); ctx->user_port = bpf_htons(60000); } @@ -79,7 +81,7 @@ int getpeername4(struct bpf_sock_addr *ctx) return 1; /* Expose service 1.2.3.4:60000 as peer instead of backend. */ - if (ctx->user_port == bpf_htons(60123)) { + if (ctx->user_port == bpf_htons(port)) { orig = bpf_sk_storage_get(&service_mapping, ctx->sk, 0, 0); if (orig) { ctx->user_ip4 = orig->addr; diff --git a/tools/testing/selftests/bpf/progs/connect_force_port6.c b/tools/testing/selftests/bpf/progs/connect_force_port6.c index 19cad93e612f..a1a671b39083 100644 --- a/tools/testing/selftests/bpf/progs/connect_force_port6.c +++ b/tools/testing/selftests/bpf/progs/connect_force_port6.c @@ -13,6 +13,8 @@ char _license[] SEC("license") = "GPL"; +__u16 port = 0; + struct svc_addr { __be32 addr[4]; __be16 port; @@ -39,7 +41,7 @@ int connect6(struct bpf_sock_addr *ctx) if (bpf_bind(ctx, (struct sockaddr *)&sa, sizeof(sa)) != 0) return 0; - /* Rewire service [fc00::1]:60000 to backend [::1]:60124. */ + /* Rewire service [fc00::1]:60000 to backend [::1]:port. */ if (ctx->user_port == bpf_htons(60000)) { orig = bpf_sk_storage_get(&service_mapping, ctx->sk, 0, BPF_SK_STORAGE_GET_F_CREATE); @@ -56,7 +58,7 @@ int connect6(struct bpf_sock_addr *ctx) ctx->user_ip6[1] = 0; ctx->user_ip6[2] = 0; ctx->user_ip6[3] = bpf_htonl(1); - ctx->user_port = bpf_htons(60124); + ctx->user_port = bpf_htons(port); } return 1; } @@ -68,7 +70,7 @@ int getsockname6(struct bpf_sock_addr *ctx) return 1; /* Expose local server as [fc00::1]:60000 to client. */ - if (ctx->user_port == bpf_htons(60124)) { + if (ctx->user_port == bpf_htons(port)) { ctx->user_ip6[0] = bpf_htonl(0xfc000000); ctx->user_ip6[1] = 0; ctx->user_ip6[2] = 0; @@ -87,7 +89,7 @@ int getpeername6(struct bpf_sock_addr *ctx) return 1; /* Expose service [fc00::1]:60000 as peer instead of backend. */ - if (ctx->user_port == bpf_htons(60124)) { + if (ctx->user_port == bpf_htons(port)) { orig = bpf_sk_storage_get(&service_mapping, ctx->sk, 0, 0); if (orig) { ctx->user_ip6[0] = orig->addr[0]; diff --git a/tools/testing/selftests/bpf/progs/cpumask_common.h b/tools/testing/selftests/bpf/progs/cpumask_common.h index 86085b79f5ca..8fe01308d210 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_common.h +++ b/tools/testing/selftests/bpf/progs/cpumask_common.h @@ -61,7 +61,7 @@ u32 bpf_cpumask_any_distribute(const struct cpumask *src) __ksym __weak; u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1, const struct cpumask *src2) __ksym __weak; u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym __weak; -int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t src__sz) __ksym __weak; +int bpf_cpumask_populate(struct bpf_cpumask *cpumask, void *src, size_t src__sz) __ksym __weak; void bpf_rcu_read_lock(void) __ksym __weak; void bpf_rcu_read_unlock(void) __ksym __weak; diff --git a/tools/testing/selftests/bpf/progs/cpumask_failure.c b/tools/testing/selftests/bpf/progs/cpumask_failure.c index 61c32e91e8c3..4628feb53d86 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_failure.c +++ b/tools/testing/selftests/bpf/progs/cpumask_failure.c @@ -45,7 +45,7 @@ int BPF_PROG(test_alloc_no_release, struct task_struct *task, u64 clone_flags) } SEC("tp_btf/task_newtask") -__failure __msg("NULL pointer passed to trusted arg0") +__failure __msg("NULL pointer passed to trusted R1") int BPF_PROG(test_alloc_double_release, struct task_struct *task, u64 clone_flags) { struct bpf_cpumask *cpumask; @@ -73,7 +73,7 @@ int BPF_PROG(test_acquire_wrong_cpumask, struct task_struct *task, u64 clone_fla } SEC("tp_btf/task_newtask") -__failure __msg("bpf_cpumask_set_cpu args#1 expected pointer to STRUCT bpf_cpumask") +__failure __msg("bpf_cpumask_set_cpu R2 expected pointer to STRUCT bpf_cpumask") int BPF_PROG(test_mutate_cpumask, struct task_struct *task, u64 clone_flags) { /* Can't set the CPU of a non-struct bpf_cpumask. */ @@ -107,7 +107,7 @@ int BPF_PROG(test_insert_remove_no_release, struct task_struct *task, u64 clone_ } SEC("tp_btf/task_newtask") -__failure __msg("NULL pointer passed to trusted arg0") +__failure __msg("NULL pointer passed to trusted R1") int BPF_PROG(test_cpumask_null, struct task_struct *task, u64 clone_flags) { /* NULL passed to kfunc. */ @@ -116,9 +116,9 @@ int BPF_PROG(test_cpumask_null, struct task_struct *task, u64 clone_flags) return 0; } -SEC("tp_btf/task_newtask") +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") __failure __msg("R2 must be a rcu pointer") -int BPF_PROG(test_global_mask_out_of_rcu, struct task_struct *task, u64 clone_flags) +int BPF_PROG(test_global_mask_out_of_rcu) { struct bpf_cpumask *local, *prev; @@ -133,6 +133,10 @@ int BPF_PROG(test_global_mask_out_of_rcu, struct task_struct *task, u64 clone_fl return 0; } + /* + * Use a sleepable program so explicit RCU is the only source of RCU + * protection. + */ bpf_rcu_read_lock(); local = global_mask; if (!local) { @@ -151,7 +155,7 @@ int BPF_PROG(test_global_mask_out_of_rcu, struct task_struct *task, u64 clone_fl } SEC("tp_btf/task_newtask") -__failure __msg("NULL pointer passed to trusted arg1") +__failure __msg("NULL pointer passed to trusted R2") int BPF_PROG(test_global_mask_no_null_check, struct task_struct *task, u64 clone_flags) { struct bpf_cpumask *local, *prev; @@ -179,7 +183,7 @@ int BPF_PROG(test_global_mask_no_null_check, struct task_struct *task, u64 clone } SEC("tp_btf/task_newtask") -__failure __msg("Possibly NULL pointer passed to helper arg2") +__failure __msg("Possibly NULL pointer passed to helper R2") int BPF_PROG(test_global_mask_rcu_no_null_check, struct task_struct *task, u64 clone_flags) { struct bpf_cpumask *prev, *curr; @@ -231,7 +235,7 @@ int BPF_PROG(test_populate_invalid_destination, struct task_struct *task, u64 cl u64 bits; int ret; - ret = bpf_cpumask_populate((struct cpumask *)invalid, &bits, sizeof(bits)); + ret = bpf_cpumask_populate(invalid, &bits, sizeof(bits)); if (!ret) err = 2; @@ -252,7 +256,7 @@ int BPF_PROG(test_populate_invalid_source, struct task_struct *task, u64 clone_f return 0; } - ret = bpf_cpumask_populate((struct cpumask *)local, garbage, 8); + ret = bpf_cpumask_populate(local, garbage, 8); if (!ret) err = 2; @@ -260,3 +264,22 @@ int BPF_PROG(test_populate_invalid_source, struct task_struct *task, u64 clone_f return 0; } + +SEC("tp_btf/task_newtask") +__failure __msg("expected pointer to STRUCT bpf_cpumask but R1 has a pointer to STRUCT cpumask") +int BPF_PROG(test_populate_borrowed_destination, struct task_struct *task, u64 clone_flags) +{ + u64 bits; + int ret; + + /* + * task->cpus_ptr is a borrowed, read-only struct cpumask *, not an + * owned struct bpf_cpumask *. The verifier must reject it as a + * writable destination for bpf_cpumask_populate(). + */ + ret = bpf_cpumask_populate((struct bpf_cpumask *)task->cpus_ptr, &bits, sizeof(bits)); + if (!ret) + err = 2; + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/cpumask_success.c b/tools/testing/selftests/bpf/progs/cpumask_success.c index 0e04c31b91c0..36f77b9732d4 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_success.c +++ b/tools/testing/selftests/bpf/progs/cpumask_success.c @@ -785,7 +785,7 @@ int BPF_PROG(test_populate_reject_small_mask, struct task_struct *task, u64 clon return 0; /* The kfunc should prevent this operation */ - ret = bpf_cpumask_populate((struct cpumask *)local, &toofewbits, sizeof(toofewbits)); + ret = bpf_cpumask_populate(local, &toofewbits, sizeof(toofewbits)); if (ret != -EACCES) err = 2; @@ -824,7 +824,7 @@ int BPF_PROG(test_populate_reject_unaligned, struct task_struct *task, u64 clone /* Misalign the source array by a byte. */ src = &((char *)bits)[1]; - ret = bpf_cpumask_populate((struct cpumask *)mask, src, CPUMASK_TEST_MASKLEN); + ret = bpf_cpumask_populate(mask, src, CPUMASK_TEST_MASKLEN); if (ret != -EINVAL) err = 2; @@ -855,7 +855,7 @@ int BPF_PROG(test_populate, struct task_struct *task, u64 clone_flags) } /* Pass the entire bits array, the kfunc will only copy the valid bits. */ - ret = bpf_cpumask_populate((struct cpumask *)mask, bits, CPUMASK_TEST_MASKLEN); + ret = bpf_cpumask_populate(mask, bits, CPUMASK_TEST_MASKLEN); if (ret) { err = 2; goto out; @@ -866,7 +866,7 @@ int BPF_PROG(test_populate, struct task_struct *task, u64 clone_flags) * access NR_CPUS, the upper bound for nr_cpus, so we infer * it from the size of cpumask_t. */ - if (nr_cpus < 0 || nr_cpus >= CPUMASK_TEST_MASKLEN * 8) { + if (nr_cpus < 0 || nr_cpus > CPUMASK_TEST_MASKLEN * 8) { err = 3; goto out; } diff --git a/tools/testing/selftests/bpf/progs/crypto_bench.c b/tools/testing/selftests/bpf/progs/crypto_bench.c index 4ac956b26240..4c0a09aa1e6c 100644 --- a/tools/testing/selftests/bpf/progs/crypto_bench.c +++ b/tools/testing/selftests/bpf/progs/crypto_bench.c @@ -11,10 +11,19 @@ #include "crypto_common.h" const volatile unsigned int len = 16; -char cipher[128] = {}; +/* + * cipher[] and key[] are 8-byte aligned and 'params' is kept off the stack to + * work around an LLVM code generation bug. clang lowers the memcpy() of these + * byte-aligned globals into a per-byte load/store sequence staged on the stack, + * and additionally materializes the on-stack 'struct bpf_crypto_params' twice. + * Both blow the 512-byte BPF stack limit. Aligning the sources lets clang copy + * word-wise, and a global 'params' removes the large object from the stack. + */ +char cipher[128] __attribute__((aligned(8))) = {}; u32 key_len, authsize; char dst[256] = {}; -u8 key[256] = {}; +u8 key[256] __attribute__((aligned(8))) = {}; +static struct bpf_crypto_params params; long hits = 0; int status; @@ -22,11 +31,6 @@ SEC("syscall") int crypto_setup(void *args) { struct bpf_crypto_ctx *cctx; - struct bpf_crypto_params params = { - .type = "skcipher", - .key_len = key_len, - .authsize = authsize, - }; int err = 0; status = 0; @@ -36,6 +40,9 @@ int crypto_setup(void *args) return 0; } + __builtin_memcpy(¶ms.type, "skcipher", sizeof("skcipher")); + params.key_len = key_len; + params.authsize = authsize; __builtin_memcpy(¶ms.algo, cipher, sizeof(cipher)); __builtin_memcpy(¶ms.key, key, sizeof(key)); cctx = bpf_crypto_ctx_create(¶ms, sizeof(params), &err); diff --git a/tools/testing/selftests/bpf/progs/crypto_sanity.c b/tools/testing/selftests/bpf/progs/crypto_sanity.c index dfd8a258f14a..e81f5ac3b1ae 100644 --- a/tools/testing/selftests/bpf/progs/crypto_sanity.c +++ b/tools/testing/selftests/bpf/progs/crypto_sanity.c @@ -10,11 +10,20 @@ #include "bpf_kfuncs.h" #include "crypto_common.h" -unsigned char key[256] = {}; +/* + * key[] and algo[] are 8-byte aligned and 'params' is kept off the stack to + * work around an LLVM code generation bug. clang lowers the memcpy() of these + * byte-aligned globals into a per-byte load/store sequence staged on the stack, + * and additionally materializes the on-stack 'struct bpf_crypto_params' twice. + * Both blow the 512-byte BPF stack limit. Aligning the sources lets clang copy + * word-wise, and a global 'params' removes the large object from the stack. + */ +unsigned char key[256] __attribute__((aligned(8))) = {}; u16 udp_test_port = 7777; u32 authsize, key_len; -char algo[128] = {}; +char algo[128] __attribute__((aligned(8))) = {}; char dst[16] = {}, dst_bad[8] = {}; +static struct bpf_crypto_params params; int status; static int skb_dynptr_validate(struct __sk_buff *skb, struct bpf_dynptr *psrc) @@ -53,11 +62,6 @@ static int skb_dynptr_validate(struct __sk_buff *skb, struct bpf_dynptr *psrc) SEC("syscall") int skb_crypto_setup(void *ctx) { - struct bpf_crypto_params params = { - .type = "skcipher", - .key_len = key_len, - .authsize = authsize, - }; struct bpf_crypto_ctx *cctx; int err; @@ -67,6 +71,9 @@ int skb_crypto_setup(void *ctx) return 0; } + __builtin_memcpy(¶ms.type, "skcipher", sizeof("skcipher")); + params.key_len = key_len; + params.authsize = authsize; __builtin_memcpy(¶ms.algo, algo, sizeof(algo)); __builtin_memcpy(¶ms.key, key, sizeof(key)); diff --git a/tools/testing/selftests/bpf/progs/dynptr_fail.c b/tools/testing/selftests/bpf/progs/dynptr_fail.c index 8f2ae9640886..1cd61d72c166 100644 --- a/tools/testing/selftests/bpf/progs/dynptr_fail.c +++ b/tools/testing/selftests/bpf/progs/dynptr_fail.c @@ -78,7 +78,7 @@ static int get_map_val_dynptr(struct bpf_dynptr *ptr) * bpf_ringbuf_submit/discard_dynptr call */ SEC("?raw_tp") -__failure __msg("Unreleased reference id=2") +__failure __msg("Unreleased reference id=1") int ringbuf_missing_release1(void *ctx) { struct bpf_dynptr ptr = {}; @@ -91,7 +91,7 @@ int ringbuf_missing_release1(void *ctx) } SEC("?raw_tp") -__failure __msg("Unreleased reference id=4") +__failure __msg("Unreleased reference id=3") int ringbuf_missing_release2(void *ctx) { struct bpf_dynptr ptr1, ptr2; @@ -136,7 +136,7 @@ int ringbuf_missing_release_callback(void *ctx) /* Can't call bpf_ringbuf_submit/discard_dynptr on a non-initialized dynptr */ SEC("?raw_tp") -__failure __msg("arg 1 is an unacquired reference") +__failure __msg("Expected an initialized dynptr as R1") int ringbuf_release_uninit_dynptr(void *ctx) { struct bpf_dynptr ptr; @@ -149,7 +149,7 @@ int ringbuf_release_uninit_dynptr(void *ctx) /* A dynptr can't be used after it has been invalidated */ SEC("?raw_tp") -__failure __msg("Expected an initialized dynptr as arg #2") +__failure __msg("Expected an initialized dynptr as R3") int use_after_invalid(void *ctx) { struct bpf_dynptr ptr; @@ -448,7 +448,7 @@ int invalid_helper2(void *ctx) /* A bpf_dynptr is invalidated if it's been written into */ SEC("?raw_tp") -__failure __msg("Expected an initialized dynptr as arg #0") +__failure __msg("Expected an initialized dynptr as R1") int invalid_write1(void *ctx) { struct bpf_dynptr ptr; @@ -650,7 +650,7 @@ int invalid_offset(void *ctx) /* Can't release a dynptr twice */ SEC("?raw_tp") -__failure __msg("arg 1 is an unacquired reference") +__failure __msg("Expected an initialized dynptr as R1") int release_twice(void *ctx) { struct bpf_dynptr ptr; @@ -677,7 +677,7 @@ static int release_twice_callback_fn(__u32 index, void *data) * within a callback function, fails */ SEC("?raw_tp") -__failure __msg("arg 1 is an unacquired reference") +__failure __msg("Expected an initialized dynptr as R1") int release_twice_callback(void *ctx) { struct bpf_dynptr ptr; @@ -705,6 +705,48 @@ int dynptr_from_mem_invalid_api(void *ctx) return 0; } +/* Cannot create dynptr from dynptr data */ +SEC("?raw_tp") +__failure __msg("Unsupported reg type mem for bpf_dynptr_from_mem data") +int dynptr_from_dynptr_data(void *ctx) +{ + struct bpf_dynptr ptr, ptr2; + __u8 *data; + + if (get_map_val_dynptr(&ptr)) + return 0; + + data = bpf_dynptr_data(&ptr, 0, sizeof(__u32)); + if (!data) + return 0; + + /* this should fail */ + bpf_dynptr_from_mem(data, sizeof(__u32), 0, &ptr2); + + return 0; +} + +/* Cannot create dynptr from dynptr slice */ +SEC("?tc") +__failure __msg("Unsupported reg type mem for bpf_dynptr_from_mem data") +int dynptr_from_dynptr_slice(struct __sk_buff *skb) +{ + struct bpf_dynptr ptr, ptr2; + struct ethhdr *hdr; + char buffer[sizeof(*hdr)] = {}; + + bpf_dynptr_from_skb(skb, 0, &ptr); + + hdr = bpf_dynptr_slice_rdwr(&ptr, 0, buffer, sizeof(buffer)); + if (!hdr) + return SK_DROP; + + /* this should fail */ + bpf_dynptr_from_mem(hdr, sizeof(*hdr), 0, &ptr2); + + return SK_PASS; +} + SEC("?tc") __failure __msg("cannot overwrite referenced dynptr") __log_level(2) int dynptr_pruning_overwrite(struct __sk_buff *ctx) @@ -1070,7 +1112,7 @@ int dynptr_overwrite_ref(void *ctx) /* Reject writes to dynptr slot from bpf_dynptr_read */ SEC("?raw_tp") -__failure __msg("potential write to dynptr at off=-16") +__failure __msg("cannot overwrite referenced dynptr") int dynptr_read_into_slot(void *ctx) { union { @@ -1516,7 +1558,7 @@ int BPF_PROG(skb_invalid_ctx_fexit, void *skb) /* Reject writes to dynptr slot for uninit arg */ SEC("?raw_tp") -__failure __msg("potential write to dynptr at off=-16") +__failure __msg("cannot overwrite referenced dynptr") int uninit_write_into_slot(void *ctx) { struct { @@ -1547,7 +1589,8 @@ int xdp_invalid_ctx(void *ctx) __u32 hdr_size = sizeof(struct ethhdr); /* Can't pass in variable-sized len to bpf_dynptr_slice */ SEC("?tc") -__failure __msg("unbounded memory access") +__failure __msg("must be a known constant") +__msg("requires this memory size to be a verifier-known constant") int dynptr_slice_var_len1(struct __sk_buff *skb) { struct bpf_dynptr ptr; @@ -1567,6 +1610,7 @@ int dynptr_slice_var_len1(struct __sk_buff *skb) /* Can't pass in variable-sized len to bpf_dynptr_slice */ SEC("?tc") __failure __msg("must be a known constant") +__msg("requires this memory size to be a verifier-known constant") int dynptr_slice_var_len2(struct __sk_buff *skb) { char buffer[sizeof(struct ethhdr)] = {}; @@ -1593,6 +1637,36 @@ static int callback(__u32 index, void *data) return 0; } +/* A commuted add should preserve the parent id of a dynptr data slice. */ +SEC("?raw_tp") +__failure __msg("invalid mem access 'scalar'") +int dynptr_slice_commuted_invalidate(void *ctx) +{ + struct bpf_dynptr ptr; + __u32 *slice, *derived; + + bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(__u32), 0, &ptr); + + slice = bpf_dynptr_data(&ptr, 0, sizeof(__u32)); + if (!slice) + goto done; + + asm volatile ("%[dst] = 0;" + "%[dst] += %[src];" + "%[src] = 0;" + : [dst]"=&r"(derived), [src]"+r"(slice) + : + : "memory"); + + bpf_ringbuf_discard_dynptr(&ptr, 0); + val = *derived; + return 0; + +done: + bpf_ringbuf_discard_dynptr(&ptr, 0); + return 0; +} + /* If the dynptr is written into in a callback function, its data * slices should be invalidated as well. */ @@ -1642,7 +1716,7 @@ int invalid_slice_rdwr_rdonly(struct __sk_buff *skb) /* bpf_dynptr_adjust can only be called on initialized dynptrs */ SEC("?raw_tp") -__failure __msg("Expected an initialized dynptr as arg #0") +__failure __msg("Expected an initialized dynptr as R1") int dynptr_adjust_invalid(void *ctx) { struct bpf_dynptr ptr = {}; @@ -1655,7 +1729,7 @@ int dynptr_adjust_invalid(void *ctx) /* bpf_dynptr_is_null can only be called on initialized dynptrs */ SEC("?raw_tp") -__failure __msg("Expected an initialized dynptr as arg #0") +__failure __msg("Expected an initialized dynptr as R1") int dynptr_is_null_invalid(void *ctx) { struct bpf_dynptr ptr = {}; @@ -1668,7 +1742,7 @@ int dynptr_is_null_invalid(void *ctx) /* bpf_dynptr_is_rdonly can only be called on initialized dynptrs */ SEC("?raw_tp") -__failure __msg("Expected an initialized dynptr as arg #0") +__failure __msg("Expected an initialized dynptr as R1") int dynptr_is_rdonly_invalid(void *ctx) { struct bpf_dynptr ptr = {}; @@ -1681,7 +1755,7 @@ int dynptr_is_rdonly_invalid(void *ctx) /* bpf_dynptr_size can only be called on initialized dynptrs */ SEC("?raw_tp") -__failure __msg("Expected an initialized dynptr as arg #0") +__failure __msg("Expected an initialized dynptr as R1") int dynptr_size_invalid(void *ctx) { struct bpf_dynptr ptr = {}; @@ -1694,7 +1768,7 @@ int dynptr_size_invalid(void *ctx) /* Only initialized dynptrs can be cloned */ SEC("?raw_tp") -__failure __msg("Expected an initialized dynptr as arg #0") +__failure __msg("Expected an initialized dynptr as R1") int clone_invalid1(void *ctx) { struct bpf_dynptr ptr1 = {}; @@ -1728,7 +1802,7 @@ int clone_invalid2(struct xdp_md *xdp) /* Invalidating a dynptr should invalidate its clones */ SEC("?raw_tp") -__failure __msg("Expected an initialized dynptr as arg #2") +__failure __msg("Expected an initialized dynptr as R3") int clone_invalidate1(void *ctx) { struct bpf_dynptr clone; @@ -1749,7 +1823,7 @@ int clone_invalidate1(void *ctx) /* Invalidating a dynptr should invalidate its parent */ SEC("?raw_tp") -__failure __msg("Expected an initialized dynptr as arg #2") +__failure __msg("Expected an initialized dynptr as R3") int clone_invalidate2(void *ctx) { struct bpf_dynptr ptr; @@ -1770,7 +1844,7 @@ int clone_invalidate2(void *ctx) /* Invalidating a dynptr should invalidate its siblings */ SEC("?raw_tp") -__failure __msg("Expected an initialized dynptr as arg #2") +__failure __msg("Expected an initialized dynptr as R3") int clone_invalidate3(void *ctx) { struct bpf_dynptr ptr; @@ -1981,7 +2055,7 @@ __noinline long global_call_bpf_dynptr(const struct bpf_dynptr *dynptr) } SEC("?raw_tp") -__failure __msg("arg#0 expected pointer to stack or const struct bpf_dynptr") +__failure __msg("R1 expected pointer to stack or const struct bpf_dynptr") int test_dynptr_reg_type(void *ctx) { struct task_struct *current = NULL; @@ -1993,3 +2067,118 @@ int test_dynptr_reg_type(void *ctx) global_call_bpf_dynptr((const struct bpf_dynptr *)current); return 0; } + +/* Overwriting a referenced dynptr is allowed if a clone still holds the ref */ +SEC("?raw_tp") +__success +int dynptr_overwrite_ref_with_clone(void *ctx) +{ + struct bpf_dynptr ptr, clone; + + bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &ptr); + + bpf_dynptr_clone(&ptr, &clone); + + /* Overwrite the original - clone still holds the ref */ + *(volatile __u8 *)&ptr = 0; + + bpf_ringbuf_discard_dynptr(&clone, 0); + + return 0; +} + +/* Overwriting the last referenced dynptr should still be rejected */ +SEC("?raw_tp") +__failure __msg("cannot overwrite referenced dynptr") +int dynptr_overwrite_ref_last_clone(void *ctx) +{ + struct bpf_dynptr ptr, clone; + + bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &ptr); + + bpf_dynptr_clone(&ptr, &clone); + + /* Overwrite the original - clone still holds the ref, OK */ + *(volatile __u8 *)&ptr = 0; + + /* Overwrite the last holder - this should fail */ + *(volatile __u8 *)&clone = 0; + + return 0; +} + +/* Overwriting a clone should be allowed if the original still holds the ref */ +SEC("?raw_tp") +__success +int dynptr_overwrite_clone_with_original(void *ctx) +{ + struct bpf_dynptr ptr, clone; + + bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &ptr); + + bpf_dynptr_clone(&ptr, &clone); + + /* Overwrite the clone - original still holds the ref */ + *(volatile __u8 *)&clone = 0; + + bpf_ringbuf_discard_dynptr(&ptr, 0); + + return 0; +} + +/* Data slices from the destroyed dynptr should be invalidated */ +SEC("?raw_tp") +__failure __msg("invalid mem access 'scalar'") +int dynptr_overwrite_ref_invalidate_slice(void *ctx) +{ + struct bpf_dynptr ptr, clone; + int *data; + + bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr); + + data = bpf_dynptr_data(&ptr, 0, sizeof(val)); + if (!data) + return 0; + + bpf_dynptr_clone(&ptr, &clone); + + /* Overwrite the original - clone holds the ref */ + *(volatile __u8 *)&ptr = 0; + + /* data was from the original dynptr, should be invalid now */ + *data = 123; + + return 0; +} + +/* + * Data slices from a dynptr clone should remain valid after + * overwriting the original dynptr + */ +SEC("?raw_tp") +__success +int dynptr_overwrite_ref_clone_slice_valid(void *ctx) +{ + struct bpf_dynptr ptr, clone; + int *data; + + bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr); + + bpf_dynptr_clone(&ptr, &clone); + + data = bpf_dynptr_data(&clone, 0, sizeof(val)); + if (!data) { + bpf_ringbuf_discard_dynptr(&clone, 0); + return 0; + } + + /* Overwrite the original - clone holds the ref */ + *(volatile __u8 *)&ptr = 0; + + /* data is from the clone, should still be valid */ + *data = 123; + + bpf_ringbuf_discard_dynptr(&clone, 0); + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/dynptr_success.c b/tools/testing/selftests/bpf/progs/dynptr_success.c index e0d672d93adf..b668ebd61fc7 100644 --- a/tools/testing/selftests/bpf/progs/dynptr_success.c +++ b/tools/testing/selftests/bpf/progs/dynptr_success.c @@ -10,6 +10,7 @@ #include "errno.h" #define PAGE_SIZE_64K 65536 +#define TEST_SKB_LINEAR_SIZE (sizeof(struct ethhdr) + sizeof(struct iphdr)) char _license[] SEC("license") = "GPL"; @@ -212,6 +213,25 @@ int test_dynptr_skb_data(struct __sk_buff *skb) } SEC("?tc") +int test_dynptr_skb_slice_non_linear(struct __sk_buff *skb) +{ + struct bpf_dynptr ptr; + void *data; + + if (bpf_dynptr_from_skb(skb, 0, &ptr)) { + err = 1; + return 1; + } + + /* Ensure we cannot read past the end of the buffer. */ + data = bpf_dynptr_slice(&ptr, TEST_SKB_LINEAR_SIZE + 1, NULL, 1); + if (data) + err = 2; + + return 1; +} + +SEC("?tc") int test_dynptr_skb_meta_data(struct __sk_buff *skb) { struct bpf_dynptr meta; @@ -914,7 +934,7 @@ void *user_ptr; char expected_str[384]; __u32 test_len[7] = {0/* placeholder */, 0, 1, 2, 255, 256, 257}; -typedef int (*bpf_read_dynptr_fn_t)(struct bpf_dynptr *dptr, u64 off, +typedef int (*bpf_read_dynptr_fn_t)(const struct bpf_dynptr *dptr, u64 off, u64 size, const void *unsafe_ptr); /* Returns the offset just before the end of the maximum sized xdp fragment. @@ -1106,7 +1126,7 @@ int test_copy_from_user_str_dynptr(void *ctx) return 0; } -static int bpf_copy_data_from_user_task(struct bpf_dynptr *dptr, u64 off, +static int bpf_copy_data_from_user_task(const struct bpf_dynptr *dptr, u64 off, u64 size, const void *unsafe_ptr) { struct task_struct *task = bpf_get_current_task_btf(); @@ -1114,7 +1134,7 @@ static int bpf_copy_data_from_user_task(struct bpf_dynptr *dptr, u64 off, return bpf_copy_from_user_task_dynptr(dptr, off, size, unsafe_ptr, task); } -static int bpf_copy_data_from_user_task_str(struct bpf_dynptr *dptr, u64 off, +static int bpf_copy_data_from_user_task_str(const struct bpf_dynptr *dptr, u64 off, u64 size, const void *unsafe_ptr) { struct task_struct *task = bpf_get_current_task_btf(); diff --git a/tools/testing/selftests/bpf/progs/empty_skb.c b/tools/testing/selftests/bpf/progs/empty_skb.c index 4b0cd6753251..44326f5cc8bb 100644 --- a/tools/testing/selftests/bpf/progs/empty_skb.c +++ b/tools/testing/selftests/bpf/progs/empty_skb.c @@ -35,3 +35,10 @@ int tc_redirect_egress(struct __sk_buff *skb) ret = bpf_clone_redirect(skb, ifindex, 0); return 0; } + +SEC("tc") +int tc_adjust_room(struct __sk_buff *skb) +{ + ret = bpf_skb_adjust_room(skb, 4, BPF_ADJ_ROOM_NET, 0); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/exceptions.c b/tools/testing/selftests/bpf/progs/exceptions.c index f09cd14d8e04..91c81971e58c 100644 --- a/tools/testing/selftests/bpf/progs/exceptions.c +++ b/tools/testing/selftests/bpf/progs/exceptions.c @@ -109,6 +109,20 @@ int exception_tail_call(struct __sk_buff *ctx) { return ret + 8; } +__weak +void throw_11(void) +{ + bpf_throw(11); +} + +SEC("tc") +int exception_throw_from_void_global(struct __sk_buff *ctx) +{ + throw_11(); + + return 0; +} + __noinline int exception_ext_global(struct __sk_buff *ctx) { volatile int ret = 0; @@ -198,6 +212,36 @@ int exception_throw_subprog(struct __sk_buff *ctx) return 0; } +u64 exception_cb_stack_src = 0x1234; + +/* + * The address handed to the helper has to be this callback's own stack + * slot, not one from a frame that is already gone. + */ +__noinline int exception_cb_stack(u64 cookie) +{ + volatile u64 val = 0xdead; + + bpf_probe_read_kernel((void *)&val, sizeof(val), &exception_cb_stack_src); + return val; +} + +/* Throws from a subprogram that has a stack of its own. */ +__noinline static int throwing_subprog_stack(struct __sk_buff *ctx) +{ + volatile u64 pad[4] = {}; + + bpf_throw(pad[0]); + return 0; +} + +SEC("tc") +__exception_cb(exception_cb_stack) +int exception_throw_subprog_stack_cb(struct __sk_buff *ctx) +{ + return throwing_subprog_stack(ctx); +} + __noinline int assert_nz_gfunc(u64 c) { volatile u64 cookie = c; @@ -365,4 +409,118 @@ int exception_bad_assert_range_with(struct __sk_buff *ctx) return 1; } +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) \ + && defined(__BPF_FEATURE_STACK_ARGUMENT) + +const volatile bool has_stack_arg = true; + +long arg1 = 1, arg2 = 2, arg3 = 3, arg4 = 4, arg5 = 5; +long arg6 = 6, arg7 = 7, arg8 = 8, arg9 = 9, arg10 = 10; + +__noinline static long throwing_many_args(long a, long b, long c, long d, + long e, long f, long g, long h, + long i, long j) +{ + bpf_throw(a + b + c + d + e + f + g + h + i + j); + return 0; +} + +__noinline int exception_cb_sa(u64 cookie) +{ + return cookie + 1; +} + +SEC("tc") +__exception_cb(exception_cb_sa) +int exception_throw_stack_arg(struct __sk_buff *ctx) +{ + throwing_many_args(arg1, arg2, arg3, arg4, arg5, + arg6, arg7, arg8, arg9, arg10); + return 0; +} + +__noinline static long no_throw_many_args(long a, long b, long c, long d, + long e, long f, long g, long h, + long i, long j) +{ + return a + b + c + d + e + f + g + h + i + j; +} + +SEC("tc") +__exception_cb(exception_cb_sa) +int exception_throw_after_stack_arg(struct __sk_buff *ctx) +{ + long ret; + + ret = no_throw_many_args(arg1, arg2, arg3, arg4, arg5, + arg6, arg7, arg8, arg9, arg10); + if (ret > 0) + bpf_throw(ret); + return 0; +} + +__noinline static long subprog_throw_sa(long val) +{ + throwing_many_args(val, val + 1, val + 2, val + 3, val + 4, + val + 5, val + 6, val + 7, val + 8, val + 9); + return 0; +} + +SEC("tc") +__exception_cb(exception_cb_sa) +int exception_throw_subprog_stack_arg(struct __sk_buff *ctx) +{ + subprog_throw_sa(arg1); + return 0; +} + +__noinline static long subprog_throw_after_sa(long val) +{ + long ret; + + ret = no_throw_many_args(val, val + 1, val + 2, val + 3, val + 4, + val + 5, val + 6, val + 7, val + 8, val + 9); + if (ret > 0) + bpf_throw(ret); + return 0; +} + +SEC("tc") +__exception_cb(exception_cb_sa) +int exception_throw_subprog_after_stack_arg(struct __sk_buff *ctx) +{ + subprog_throw_after_sa(arg1); + return 0; +} + +#else + +const volatile bool has_stack_arg = false; + +SEC("tc") +int exception_throw_stack_arg(struct __sk_buff *ctx) +{ + return 0; +} + +SEC("tc") +int exception_throw_after_stack_arg(struct __sk_buff *ctx) +{ + return 0; +} + +SEC("tc") +int exception_throw_subprog_stack_arg(struct __sk_buff *ctx) +{ + return 0; +} + +SEC("tc") +int exception_throw_subprog_after_stack_arg(struct __sk_buff *ctx) +{ + return 0; +} + +#endif + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/exceptions_assert.c b/tools/testing/selftests/bpf/progs/exceptions_assert.c index 858af5988a38..e4abf4172fca 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_assert.c +++ b/tools/testing/selftests/bpf/progs/exceptions_assert.c @@ -59,7 +59,7 @@ check_assert(s64, >=, ge_neg, INT_MIN); SEC("?tc") __log_level(2) __failure -__msg(": R0=0 R1=ctx() R2=scalar(smin=0xffffffff80000002,smax=smax32=0x7ffffffd,smin32=0x80000002) R10=fp0") +__msg(": R1=ctx() R2=scalar(smin=0xffffffff80000002,smax=smax32=0x7ffffffd,smin32=0x80000002) R10=fp0") int check_assert_range_s64(struct __sk_buff *ctx) { struct bpf_sock *sk = ctx->sk; @@ -86,7 +86,7 @@ int check_assert_range_u64(struct __sk_buff *ctx) SEC("?tc") __log_level(2) __failure -__msg(": R0=0 R1=ctx() R2=4096 R10=fp0") +__msg(": R1=ctx() R2=4096 R10=fp0") int check_assert_single_range_s64(struct __sk_buff *ctx) { struct bpf_sock *sk = ctx->sk; @@ -114,7 +114,7 @@ int check_assert_single_range_u64(struct __sk_buff *ctx) SEC("?tc") __log_level(2) __failure -__msg(": R1=pkt(off=64,r=64) R2=pkt_end() R6=pkt(r=64) R10=fp0") +__msg(": R6=pkt(r=64) R10=fp0") int check_assert_generic(struct __sk_buff *ctx) { u8 *data_end = (void *)(long)ctx->data_end; diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index 9ea1353488d7..ac44d60e5066 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c @@ -34,11 +34,15 @@ struct { private(A) struct bpf_spin_lock lock; private(A) struct bpf_rb_root rbtree __contains(foo, node); -__noinline void *exception_cb_bad_ret_type(u64 cookie) +__noinline void *exception_cb_bad_ret_type1(u64 cookie) { return NULL; } +__noinline void exception_cb_bad_ret_type2(u64 cookie) +{ +} + __noinline int exception_cb_bad_arg_0(void) { return 0; @@ -55,8 +59,8 @@ __noinline int exception_cb_ok_arg_small(int a) } SEC("?tc") -__exception_cb(exception_cb_bad_ret_type) -__failure __msg("Global function exception_cb_bad_ret_type() doesn't return scalar.") +__exception_cb(exception_cb_bad_ret_type1) +__failure __msg("Global function exception_cb_bad_ret_type1() return value not void or scalar.") int reject_exception_cb_type_1(struct __sk_buff *ctx) { bpf_throw(0); @@ -90,6 +94,15 @@ int reject_exception_cb_type_4(struct __sk_buff *ctx) return 0; } +SEC("?tc") +__exception_cb(exception_cb_bad_ret_type2) +__failure __msg("exception cb cannot return void") +int reject_exception_cb_type_5(struct __sk_buff *ctx) +{ + bpf_throw(0); + return 0; +} + __noinline static int timer_cb(void *map, int *key, struct bpf_timer *timer) { @@ -195,6 +208,28 @@ int reject_with_reference(void *ctx) return 0; } +__noinline int global_subprog_may_throw(struct __sk_buff *ctx) +{ + if (ctx->len) + bpf_throw(0); + return 0; +} + +SEC("?tc") +__failure __msg("Unreleased reference") +int reject_global_subprog_throw_with_reference(struct __sk_buff *ctx) +{ + struct foo *f; + + f = bpf_obj_new(typeof(*f)); + if (!f) + return 0; + if (ctx->protocol) + global_subprog_may_throw(ctx); + bpf_obj_drop(f); + return 0; +} + __noinline static int subprog_ref(struct __sk_buff *ctx) { struct foo *f; @@ -353,6 +388,21 @@ int reject_exception_throw_cb_diff(struct __sk_buff *ctx) return 0; } +__weak +void foo(void) +{ + bpf_throw(1); +} + +SEC("?fentry/bpf_check") +__failure __msg("At program exit the register R1 has smin=1 smax=1 should") +int reject_out_of_range_global_throw(struct __sk_buff *skb) +{ + foo(); + + return 0; +} + __noinline static int always_throws(void) { bpf_throw(0); diff --git a/tools/testing/selftests/bpf/progs/fentry_sleepable.c b/tools/testing/selftests/bpf/progs/fentry_sleepable.c new file mode 100644 index 000000000000..8c0fc691d329 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/fentry_sleepable.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char LICENSE[] SEC("license") = "GPL"; + +void *user_ptr; +int retval; + +SEC("fentry.s") +int BPF_PROG(fentry_xdp) +{ + char buff[64]; + + retval = bpf_copy_from_user(buff, sizeof(buff), user_ptr); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c index 983b7c233382..f4bbf87b82dd 100644 --- a/tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c +++ b/tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c @@ -53,14 +53,23 @@ int BPF_PROG(test_subprog1, struct sk_buff *skb, int ret) * r0 = *(u32 *)(r1 + 0) * w0 <<= 1 * exit - * In such case the verifier falls back to conservative and + * Before llvm23, in such case the verifier falls back to conservative and * tracing program can access arguments and return value as u64 - * instead of accurate types. + * instead of accurate types. With llvm23, the true signature + * int test_pkt_access_subprog2(volatile struct __sk_buff *skb) + * is available in btf. */ +#if __clang_major__ >= 23 +struct args_subprog2 { + __u64 args[1]; + __u64 ret; +}; +#else struct args_subprog2 { __u64 args[5]; __u64 ret; }; +#endif __u64 test_result_subprog2 = 0; SEC("fexit/test_pkt_access_subprog2") int test_subprog2(struct args_subprog2 *ctx) diff --git a/tools/testing/selftests/bpf/progs/fib_lookup.c b/tools/testing/selftests/bpf/progs/fib_lookup.c index 7b5dd2214ff4..36b7218d9ae2 100644 --- a/tools/testing/selftests/bpf/progs/fib_lookup.c +++ b/tools/testing/selftests/bpf/progs/fib_lookup.c @@ -4,7 +4,11 @@ #include <linux/types.h> #include <linux/bpf.h> #include <linux/pkt_cls.h> +#include <linux/if_ether.h> +#include <linux/ip.h> +#include <linux/in.h> #include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> struct bpf_fib_lookup fib_params = {}; int fib_lookup_ret = 0; @@ -19,4 +23,57 @@ int fib_lookup(struct __sk_buff *skb) return TC_ACT_SHOT; } +SEC("xdp") +int fib_lookup_xdp(struct xdp_md *ctx) +{ + fib_lookup_ret = bpf_fib_lookup(ctx, &fib_params, sizeof(fib_params), + lookup_flags); + + return XDP_DROP; +} + +int redirected = 0; +int passed = 0; +int delivered = 0; + +SEC("xdp") +int fib_lookup_redirect(struct xdp_md *ctx) +{ + struct bpf_fib_lookup params = fib_params; + long ret; + + ret = bpf_fib_lookup(ctx, ¶ms, sizeof(params), lookup_flags); + if (ret == BPF_FIB_LKUP_RET_SUCCESS) { + redirected++; + return bpf_redirect(params.ifindex, 0); + } + + passed++; + return XDP_PASS; +} + +SEC("xdp") +int xdp_count(struct xdp_md *ctx) +{ + void *data = (void *)(long)ctx->data; + void *data_end = (void *)(long)ctx->data_end; + struct ethhdr *eth = data; + struct iphdr *iph; + + /* + * count only the test's TCP frames: the netns has live + * link-local traffic (DAD, MLD) that would satisfy a bare + * counter + */ + if ((void *)(eth + 1) > data_end || + eth->h_proto != bpf_htons(ETH_P_IP)) + return XDP_DROP; + iph = (void *)(eth + 1); + if ((void *)(iph + 1) > data_end || iph->protocol != IPPROTO_TCP) + return XDP_DROP; + + delivered++; + return XDP_DROP; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/file_reader.c b/tools/testing/selftests/bpf/progs/file_reader.c index 462712ff3b8a..aa2c05cce2b3 100644 --- a/tools/testing/selftests/bpf/progs/file_reader.c +++ b/tools/testing/selftests/bpf/progs/file_reader.c @@ -50,7 +50,7 @@ int on_open_expect_fault(void *c) goto out; local_err = bpf_dynptr_read(tmp_buf, user_buf_sz, &dynptr, user_buf_sz, 0); - if (local_err == -EFAULT) { /* Expect page fault */ + if (local_err == -EFAULT || local_err == 0) { /* Expect page fault or success */ local_err = 0; run_success = 1; } diff --git a/tools/testing/selftests/bpf/progs/file_reader_fail.c b/tools/testing/selftests/bpf/progs/file_reader_fail.c index 32fe28ed2439..3bb9e2612f8f 100644 --- a/tools/testing/selftests/bpf/progs/file_reader_fail.c +++ b/tools/testing/selftests/bpf/progs/file_reader_fail.c @@ -30,7 +30,7 @@ int on_nanosleep_unreleased_ref(void *ctx) SEC("xdp") __failure -__msg("Expected a dynptr of type file as arg #0") +__msg("Expected a dynptr of type file as R1") int xdp_wrong_dynptr_type(struct xdp_md *xdp) { struct bpf_dynptr dynptr; @@ -42,7 +42,7 @@ int xdp_wrong_dynptr_type(struct xdp_md *xdp) SEC("xdp") __failure -__msg("Expected an initialized dynptr as arg #0") +__msg("Expected an initialized dynptr as R1") int xdp_no_dynptr_type(struct xdp_md *xdp) { struct bpf_dynptr dynptr; @@ -50,3 +50,63 @@ int xdp_no_dynptr_type(struct xdp_md *xdp) bpf_dynptr_file_discard(&dynptr); return 0; } + +SEC("lsm/file_open") +__failure +__msg("Leaking reference id={{[0-9]+}} alloc_insn={{[0-9]+}}. Release it first.") +int use_file_dynptr_after_put_file(void *ctx) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct file *file = bpf_get_task_exe_file(task); + struct bpf_dynptr dynptr; + char buf[64]; + + if (!file) + return 0; + + if (bpf_dynptr_from_file(file, 0, &dynptr)) + goto out; + + /* this should fail - file dynptr should be discarded first to prevent resource leak */ + bpf_put_file(file); + + bpf_dynptr_read(buf, sizeof(buf), &dynptr, 0, 0); + return 0; + +out: + bpf_dynptr_file_discard(&dynptr); + bpf_put_file(file); + return 0; +} + +SEC("lsm/file_open") +__failure +__msg("Leaking reference id={{[0-9]+}} alloc_insn={{[0-9]+}}. Release it first.") +int use_file_dynptr_slice_after_put_file(void *ctx) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct file *file = bpf_get_task_exe_file(task); + struct bpf_dynptr dynptr; + char buf[1]; + const char *data; + + if (!file) + return 0; + + if (bpf_dynptr_from_file(file, 0, &dynptr)) + goto out; + + data = bpf_dynptr_slice(&dynptr, 0, buf, sizeof(buf)); + if (!data) + goto out; + + /* this should fail - file dynptr should be discarded first to prevent resource leak */ + bpf_put_file(file); + + return data[0]; + +out: + bpf_dynptr_file_discard(&dynptr); + bpf_put_file(file); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/freplace_cls_redirect.c b/tools/testing/selftests/bpf/progs/freplace_cls_redirect.c deleted file mode 100644 index 7e94412d47a5..000000000000 --- a/tools/testing/selftests/bpf/progs/freplace_cls_redirect.c +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (c) 2020 Facebook - -#include <linux/stddef.h> -#include <linux/bpf.h> -#include <linux/pkt_cls.h> -#include <bpf/bpf_endian.h> -#include <bpf/bpf_helpers.h> - -struct { - __uint(type, BPF_MAP_TYPE_SOCKMAP); - __type(key, int); - __type(value, int); - __uint(max_entries, 2); -} sock_map SEC(".maps"); - -SEC("freplace/cls_redirect") -int freplace_cls_redirect_test(struct __sk_buff *skb) -{ - int ret = 0; - const int zero = 0; - struct bpf_sock *sk; - - sk = bpf_map_lookup_elem(&sock_map, &zero); - if (!sk) - return TC_ACT_SHOT; - - ret = bpf_map_update_elem(&sock_map, &zero, sk, 0); - bpf_sk_release(sk); - - return ret == 0 ? TC_ACT_OK : TC_ACT_SHOT; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/freplace_int_with_void.c b/tools/testing/selftests/bpf/progs/freplace_int_with_void.c new file mode 100644 index 000000000000..cbb8f8ff2581 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/freplace_int_with_void.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <linux/pkt_cls.h> +#include <bpf/bpf_helpers.h> + +SEC("freplace/global_func2") +void test_freplace_int_with_void(struct __sk_buff *skb) +{ +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/freplace_void.c b/tools/testing/selftests/bpf/progs/freplace_void.c new file mode 100644 index 000000000000..68b114f477fe --- /dev/null +++ b/tools/testing/selftests/bpf/progs/freplace_void.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +SEC("freplace/foo") +void test_freplace_void(struct __sk_buff *skb) +{ +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/get_func_args_fsession_test.c b/tools/testing/selftests/bpf/progs/get_func_args_fsession_test.c new file mode 100644 index 000000000000..bb597f24b659 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/get_func_args_fsession_test.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <errno.h> + +char _license[] SEC("license") = "GPL"; + +__u64 test1_result = 0; + +SEC("fsession/bpf_fentry_test1") +int BPF_PROG(test1) +{ + __u64 cnt = bpf_get_func_arg_cnt(ctx); + __u64 a = 0, z = 0, ret = 0; + __s64 err; + + test1_result = cnt == 1; + + /* valid arguments */ + err = bpf_get_func_arg(ctx, 0, &a); + test1_result &= err == 0 && ((int) a == 1); + + /* not valid argument */ + err = bpf_get_func_arg(ctx, 1, &z); + test1_result &= err == -EINVAL; + + if (bpf_session_is_return(ctx)) { + err = bpf_get_func_ret(ctx, &ret); + test1_result &= err == 0 && ret == 2; + } else { + err = bpf_get_func_ret(ctx, &ret); + test1_result &= err == 0 && ret == 0; + } + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/get_func_args_test.c b/tools/testing/selftests/bpf/progs/get_func_args_test.c index 075a1180ec26..1bf47f64d096 100644 --- a/tools/testing/selftests/bpf/progs/get_func_args_test.c +++ b/tools/testing/selftests/bpf/progs/get_func_args_test.c @@ -165,41 +165,3 @@ int BPF_PROG(tp_test2) return 0; } - -__u64 test7_result = 0; -#if defined(bpf_target_x86) || defined(bpf_target_arm64) || defined(bpf_target_riscv) -SEC("fsession/bpf_fentry_test1") -int BPF_PROG(test7) -{ - __u64 cnt = bpf_get_func_arg_cnt(ctx); - __u64 a = 0, z = 0, ret = 0; - __s64 err; - - test7_result = cnt == 1; - - /* valid arguments */ - err = bpf_get_func_arg(ctx, 0, &a); - test7_result &= err == 0 && ((int) a == 1); - - /* not valid argument */ - err = bpf_get_func_arg(ctx, 1, &z); - test7_result &= err == -EINVAL; - - if (bpf_session_is_return(ctx)) { - err = bpf_get_func_ret(ctx, &ret); - test7_result &= err == 0 && ret == 2; - } else { - err = bpf_get_func_ret(ctx, &ret); - test7_result &= err == 0 && ret == 0; - } - - return 0; -} -#else -SEC("fentry/bpf_fentry_test1") -int BPF_PROG(test7) -{ - test7_result = 1; - return 0; -} -#endif diff --git a/tools/testing/selftests/bpf/progs/get_func_ip_fsession_test.c b/tools/testing/selftests/bpf/progs/get_func_ip_fsession_test.c new file mode 100644 index 000000000000..bbeea0d512e3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/get_func_ip_fsession_test.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +__u64 test1_entry_result = 0; +__u64 test1_exit_result = 0; + +SEC("fsession/bpf_fentry_test1") +int BPF_PROG(test1, int a) +{ + __u64 addr = bpf_get_func_ip(ctx); + + if (bpf_session_is_return(ctx)) + test1_exit_result = (const void *) addr == &bpf_fentry_test1; + else + test1_entry_result = (const void *) addr == &bpf_fentry_test1; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/get_func_ip_test.c b/tools/testing/selftests/bpf/progs/get_func_ip_test.c index 45eaa54d1ac7..2011cacdeb18 100644 --- a/tools/testing/selftests/bpf/progs/get_func_ip_test.c +++ b/tools/testing/selftests/bpf/progs/get_func_ip_test.c @@ -103,26 +103,3 @@ int BPF_URETPROBE(test8, int ret) test8_result = (const void *) addr == (const void *) uprobe_trigger; return 0; } - -__u64 test9_entry_result = 0; -__u64 test9_exit_result = 0; -#if defined(bpf_target_x86) || defined(bpf_target_arm64) || defined(bpf_target_riscv) -SEC("fsession/bpf_fentry_test1") -int BPF_PROG(test9, int a) -{ - __u64 addr = bpf_get_func_ip(ctx); - - if (bpf_session_is_return(ctx)) - test9_exit_result = (const void *) addr == &bpf_fentry_test1; - else - test9_entry_result = (const void *) addr == &bpf_fentry_test1; - return 0; -} -#else -SEC("fentry/bpf_fentry_test1") -int BPF_PROG(test9, int a) -{ - test9_entry_result = test9_exit_result = 1; - return 0; -} -#endif diff --git a/tools/testing/selftests/bpf/progs/get_smp_processor_id.c b/tools/testing/selftests/bpf/progs/get_smp_processor_id.c new file mode 100644 index 000000000000..cf4791a5cf07 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/get_smp_processor_id.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +__u64 cpu_nr_result; + +SEC("raw_tp") +void call_bpf_get_smp_processor_id(void) +{ + register __u64 r0 asm("r0") = -1; + asm volatile ("call %[bpf_get_smp_processor_id];" + : "+r"(r0) + : __imm(bpf_get_smp_processor_id) + : "r1", "r2", "r3", "r4", "r5", "memory"); + cpu_nr_result = r0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/htab_reuse.c b/tools/testing/selftests/bpf/progs/htab_reuse.c index 7f7368cb3095..1c7fa7ee45ee 100644 --- a/tools/testing/selftests/bpf/progs/htab_reuse.c +++ b/tools/testing/selftests/bpf/progs/htab_reuse.c @@ -17,3 +17,19 @@ struct { __type(value, struct htab_val); __uint(map_flags, BPF_F_NO_PREALLOC); } htab SEC(".maps"); + +#define HTAB_NDATA 256 + +struct htab_val_large { + struct bpf_spin_lock lock; + __u32 seq; + __u64 data[HTAB_NDATA]; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 8); + __type(key, unsigned int); + __type(value, struct htab_val_large); + __uint(map_flags, BPF_F_NO_PREALLOC); +} htab_lock_consistency SEC(".maps"); diff --git a/tools/testing/selftests/bpf/progs/htab_update.c b/tools/testing/selftests/bpf/progs/htab_update.c index 195d3b2fba00..62c1b1325ec2 100644 --- a/tools/testing/selftests/bpf/progs/htab_update.c +++ b/tools/testing/selftests/bpf/progs/htab_update.c @@ -22,8 +22,8 @@ struct { int pid = 0; int update_err = 0; -SEC("?fentry/bpf_obj_free_fields") -int bpf_obj_free_fields(void *ctx) +SEC("?fentry/bpf_obj_cancel_fields") +int bpf_obj_cancel_fields(void *ctx) { __u32 key = 0; struct val value = { .payload = 1 }; diff --git a/tools/testing/selftests/bpf/progs/icmp_send.c b/tools/testing/selftests/bpf/progs/icmp_send.c new file mode 100644 index 000000000000..c642ccdf9fd5 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/icmp_send.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> + +/* 127.0.0.1 in host byte order */ +#define SERVER_IP 0x7F000001 +/* ::1 in host byte order (last 32-bit word) */ +#define SERVER_IP6_LO 0x00000001 + +__u16 server_port = 0; +int unreach_type = 0; +int unreach_code = 0; +int kfunc_ret = -1; +int target_pid = -1; + +unsigned int rec_count = 0; +int rec_kfunc_rets[] = { -1, -1 }; + +SEC("cgroup_skb/egress") +int egress(struct __sk_buff *skb) +{ + void *data = (void *)(long)skb->data; + void *data_end = (void *)(long)skb->data_end; + struct iphdr *iph; + struct ipv6hdr *ip6h; + struct tcphdr *tcph; + __u8 version; + + if (data + 1 > data_end) + return SK_PASS; + + version = (*((__u8 *)data)) >> 4; + + if (version == 4) { + iph = data; + if ((void *)(iph + 1) > data_end || + iph->protocol != IPPROTO_TCP || + iph->daddr != bpf_htonl(SERVER_IP)) + return SK_PASS; + + tcph = (void *)iph + iph->ihl * 4; + if ((void *)(tcph + 1) > data_end || + tcph->dest != bpf_htons(server_port)) + return SK_PASS; + + } else if (version == 6) { + ip6h = data; + if ((void *)(ip6h + 1) > data_end || + ip6h->nexthdr != IPPROTO_TCP) + return SK_PASS; + + if (ip6h->daddr.in6_u.u6_addr32[0] != 0 || + ip6h->daddr.in6_u.u6_addr32[1] != 0 || + ip6h->daddr.in6_u.u6_addr32[2] != 0 || + ip6h->daddr.in6_u.u6_addr32[3] != bpf_htonl(SERVER_IP6_LO)) + return SK_PASS; + + tcph = (void *)(ip6h + 1); + if ((void *)(tcph + 1) > data_end || + tcph->dest != bpf_htons(server_port)) + return SK_PASS; + } else { + return SK_PASS; + } + + kfunc_ret = bpf_icmp_send(skb, unreach_type, unreach_code); + + return SK_DROP; +} + +SEC("cgroup_skb/egress") +int recursion(struct __sk_buff *skb) +{ + void *data = (void *)(long)skb->data; + void *data_end = (void *)(long)skb->data_end; + struct icmphdr *icmph; + struct tcphdr *tcph; + struct iphdr *iph; + int ret; + + if ((bpf_get_current_pid_tgid() >> 32) != target_pid) + return SK_PASS; + + iph = data; + if ((void *)(iph + 1) > data_end || iph->version != 4) + return SK_PASS; + + if (iph->daddr != bpf_htonl(SERVER_IP)) + return SK_PASS; + + if (iph->protocol == IPPROTO_TCP) { + tcph = (void *)iph + iph->ihl * 4; + if ((void *)(tcph + 1) > data_end || + tcph->dest != bpf_htons(server_port)) + return SK_PASS; + } else if (iph->protocol == IPPROTO_ICMP) { + icmph = (void *)iph + iph->ihl * 4; + if ((void *)(icmph + 1) > data_end || + icmph->type != unreach_type || icmph->code != unreach_code) + return SK_PASS; + } else { + return SK_PASS; + } + + /* + * This call will provoke a recursion: the ICMP packet generated by the + * kfunc will re-trigger this program since we are in the root cgroup in + * which the kernel ICMP socket belongs. However when re-entering the + * kfunc, it should return EBUSY. + */ + ret = bpf_icmp_send(skb, unreach_type, unreach_code); + rec_kfunc_rets[rec_count & 1] = ret; + __sync_fetch_and_add(&rec_count, 1); + + /* Let the first ICMP error message pass */ + if (iph->protocol == IPPROTO_ICMP) + return SK_PASS; + + return SK_DROP; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/testing/selftests/bpf/progs/irq.c b/tools/testing/selftests/bpf/progs/irq.c index 74d912b22de9..a4a007866a33 100644 --- a/tools/testing/selftests/bpf/progs/irq.c +++ b/tools/testing/selftests/bpf/progs/irq.c @@ -15,7 +15,7 @@ struct bpf_res_spin_lock lockA __hidden SEC(".data.A"); struct bpf_res_spin_lock lockB __hidden SEC(".data.B"); SEC("?tc") -__failure __msg("arg#0 doesn't point to an irq flag on stack") +__failure __msg("R1 doesn't point to an irq flag on stack") int irq_save_bad_arg(struct __sk_buff *ctx) { bpf_local_irq_save(&global_flags); @@ -23,7 +23,7 @@ int irq_save_bad_arg(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("arg#0 doesn't point to an irq flag on stack") +__failure __msg("R1 doesn't point to an irq flag on stack") int irq_restore_bad_arg(struct __sk_buff *ctx) { bpf_local_irq_restore(&global_flags); @@ -490,7 +490,7 @@ int irq_non_sleepable_global_subprog(void *ctx) } SEC("?syscall") -__failure __msg("global functions that may sleep are not allowed in non-sleepable context") +__failure __msg("sleepable global function") int irq_sleepable_helper_global_subprog(void *ctx) { unsigned long flags; @@ -502,7 +502,7 @@ int irq_sleepable_helper_global_subprog(void *ctx) } SEC("?syscall") -__failure __msg("global functions that may sleep are not allowed in non-sleepable context") +__failure __msg("sleepable global function") int irq_sleepable_global_subprog_indirect(void *ctx) { unsigned long flags; diff --git a/tools/testing/selftests/bpf/progs/iter_buf_null_fail.c b/tools/testing/selftests/bpf/progs/iter_buf_null_fail.c new file mode 100644 index 000000000000..3daad40515e6 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/iter_buf_null_fail.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Qi Tang */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +/* Verify that the verifier rejects direct access to nullable PTR_TO_BUF. */ +SEC("iter/bpf_map_elem") +__failure __msg("invalid mem access") +int iter_buf_null_deref(struct bpf_iter__bpf_map_elem *ctx) +{ + /* + * ctx->key is PTR_TO_BUF | PTR_MAYBE_NULL | MEM_RDONLY. + * Direct access without null check must be rejected. + */ + volatile __u32 v = *(__u32 *)ctx->key; + + (void)v; + return 0; +} + +/* Verify that access after a null check is still accepted. */ +SEC("iter/bpf_map_elem") +__success +int iter_buf_null_check_ok(struct bpf_iter__bpf_map_elem *ctx) +{ + __u32 *key = ctx->key; + + if (!key) + return 0; + + volatile __u32 v = *key; + + (void)v; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/iters.c b/tools/testing/selftests/bpf/progs/iters.c index 7f27b517d5d5..c6699159dacd 100644 --- a/tools/testing/selftests/bpf/progs/iters.c +++ b/tools/testing/selftests/bpf/progs/iters.c @@ -88,6 +88,89 @@ int iter_err_unsafe_asm_loop(const void *ctx) return 0; } +/* + * Naked function, so there is no compiler-generated glue and the whole inlined program can be + * matched. Pinned to arches whose JITs zero-extend 32-bit writes implicitly + * (bpf_jit_needs_zext() == false); on arches that need explicit zero-extension the verifier + * interleaves "wN = wN" insns and the fixed shape below would not match. The inlining itself is + * arch independent, so checking it on these arches is sufficient. + * + * bpf_iter_num_new() emits the full range check (distance computation and both the -EINVAL and + * -E2BIG error paths); bpf_iter_num_next() and bpf_iter_num_destroy() are inlined too. + */ +SEC("raw_tp") +__arch_x86_64 +__arch_arm64 +__success +__xlated("r6 = r10") +__xlated("r6 += -8") +__xlated("call unknown") +__xlated("r3 = r0") +__xlated("r3 &= 65535") +__xlated("r1 = r6") +__xlated("r2 = 0") +/* bpf_iter_num_new(&it, 0, <non-const>) with the range check kept */ +__xlated("if w2 s> w3 goto pc+8") +__xlated("w0 = w3") +__xlated("w0 -= w2") +__xlated("if r0 > 0x800000 goto pc+8") +__xlated("w2 += -1") +__xlated("*(u32 *)(r1 +0) = r2") +__xlated("*(u32 *)(r1 +4) = r3") +__xlated("r0 = 0") +__xlated("goto pc+5") +__xlated("*(u64 *)(r1 +0) = 0") +__xlated("r0 = -22") +__xlated("goto pc+2") +__xlated("*(u64 *)(r1 +0) = 0") +__xlated("r0 = -7") +__xlated("r1 = r6") +/* bpf_iter_num_next(&it) */ +__xlated("r0 = *(u32 *)(r1 +0)") +__xlated("w0 += 1") +__xlated("r2 = *(u32 *)(r1 +4)") +__xlated("if w0 s>= w2 goto pc+3") +__xlated("*(u32 *)(r1 +0) = r0") +__xlated("r0 = r1") +__xlated("goto pc+2") +__xlated("*(u64 *)(r1 +0) = 0") +__xlated("r0 = 0") +__xlated("if r0 != 0x0 goto pc-11") +__xlated("r1 = r6") +/* bpf_iter_num_destroy(&it) is inlined to a nop */ +__xlated("goto pc+0") +__xlated("r0 = 0") +__xlated("exit") +int __naked iter_num_new_inlined(void) +{ + asm volatile ( + /* r6 points to struct bpf_iter_num on the stack */ + "r6 = r10;" + "r6 += -8;" + /* non-constant end so the range checks are kept */ + "call %[bpf_get_prandom_u32];" + "r3 = r0;" + "r3 &= 0xffff;" + "r1 = r6;" + "r2 = 0;" + "call %[bpf_iter_num_new];" + "1:" + "r1 = r6;" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r6;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy) + : __clobber_common, "r6" + ); +} + SEC("raw_tp") __success int iter_while_loop(const void *ctx) @@ -1605,7 +1688,7 @@ int iter_subprog_check_stacksafe(const void *ctx) struct bpf_iter_num global_it; SEC("raw_tp") -__failure __msg("arg#0 expected pointer to an iterator on stack") +__failure __msg("R1 expected pointer to an iterator on stack") int iter_new_bad_arg(const void *ctx) { bpf_iter_num_new(&global_it, 0, 1); @@ -1613,7 +1696,7 @@ int iter_new_bad_arg(const void *ctx) } SEC("raw_tp") -__failure __msg("arg#0 expected pointer to an iterator on stack") +__failure __msg("R1 expected pointer to an iterator on stack") int iter_next_bad_arg(const void *ctx) { bpf_iter_num_next(&global_it); @@ -1621,7 +1704,7 @@ int iter_next_bad_arg(const void *ctx) } SEC("raw_tp") -__failure __msg("arg#0 expected pointer to an iterator on stack") +__failure __msg("R1 expected pointer to an iterator on stack") int iter_destroy_bad_arg(const void *ctx) { bpf_iter_num_destroy(&global_it); @@ -1651,7 +1734,7 @@ int clean_live_states(const void *ctx) SEC("?raw_tp") __flag(BPF_F_TEST_STATE_FREQ) -__failure __msg("misaligned stack access off 0+-31+0 size 8") +__failure __msg("misaligned stack access off -31+0 size 8") __naked int absent_mark_in_the_middle_state(void) { /* This is equivalent to C program below. @@ -1726,7 +1809,7 @@ static int noop(void) SEC("?raw_tp") __flag(BPF_F_TEST_STATE_FREQ) -__failure __msg("misaligned stack access off 0+-31+0 size 8") +__failure __msg("misaligned stack access off -31+0 size 8") __naked int absent_mark_in_the_middle_state2(void) { /* This is equivalent to C program below. @@ -1802,7 +1885,7 @@ __naked int absent_mark_in_the_middle_state2(void) SEC("?raw_tp") __flag(BPF_F_TEST_STATE_FREQ) -__failure __msg("misaligned stack access off 0+-31+0 size 8") +__failure __msg("misaligned stack access off -31+0 size 8") __naked int absent_mark_in_the_middle_state3(void) { /* @@ -2066,4 +2149,43 @@ __naked int stack_misc_vs_scalar_in_a_loop(void) ); } +__used +static int loop_cb5(int i, __u64 *ctx) +{ + /* unsafe on a second iteration */ + small_arr[*ctx] = i; + *ctx = 100500; + return 0; +} + +SEC("raw_tp") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("memory access is {{.*}} and is outside of the object of size 64") +__naked void loop_counter_precision_2nd_iter(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "*(u64 *)(r10 - 8) = 0;" + "r1 = 2;" + "if r0 == 42 goto +1;" + "r1 = 1;" + "r2 = loop_cb5 ll;" + "r3 = r10;" + "r3 += -8;" + "r4 = 0;" + /* + * Explore with nr_loops=1 on a first path and nr_loops=2 on a second path. + * Buggy verifier did not propagate r1 precision properly, + * and thus checkpoints created for nr_loops=1 case matched nr_loops=2 case. + */ + "call %[bpf_loop];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_loop), + __imm(bpf_get_prandom_u32) + : __clobber_all + ); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/iters_state_safety.c b/tools/testing/selftests/bpf/progs/iters_state_safety.c index d273b46dfc7c..e5bb9fe6d5e5 100644 --- a/tools/testing/selftests/bpf/progs/iters_state_safety.c +++ b/tools/testing/selftests/bpf/progs/iters_state_safety.c @@ -30,7 +30,7 @@ int force_clang_to_emit_btf_for_externs(void *ctx) SEC("?raw_tp") __success __log_level(2) -__msg("fp-8=iter_num(ref_id=1,state=active,depth=0)") +__msg("fp-8=iter_num(id=1,state=active,depth=0)") int create_and_destroy(void *ctx) { struct bpf_iter_num iter; @@ -52,6 +52,28 @@ int create_and_destroy(void *ctx) return 0; } +/* fp+0 is not a stack slot. bpf_get_spi(0) used to alias spi 0 (fp-8). */ +SEC("?raw_tp") +__failure __msg("cannot pass in iter at an offset=0") +int destroy_fp0_fail(void *ctx) +{ + struct bpf_iter_num iter; + + asm volatile ("r1 = %[iter];" + "r2 = 0;" + "r3 = 1000;" + "call %[bpf_iter_num_new];" + /* r10 is fp+0, one byte above the top of the BPF stack */ + "r1 = r10;" + "call %[bpf_iter_num_destroy];" + : + : __imm_ptr(iter), ITER_HELPERS + : __clobber_common + ); + + return 0; +} + SEC("?raw_tp") __failure __msg("Unreleased reference id=1") int create_and_forget_to_destroy_fail(void *ctx) @@ -73,7 +95,7 @@ int create_and_forget_to_destroy_fail(void *ctx) } SEC("?raw_tp") -__failure __msg("expected an initialized iter_num as arg #0") +__failure __msg("expected an initialized iter_num as R1") int destroy_without_creating_fail(void *ctx) { /* init with zeros to stop verifier complaining about uninit stack */ @@ -91,7 +113,7 @@ int destroy_without_creating_fail(void *ctx) } SEC("?raw_tp") -__failure __msg("expected an initialized iter_num as arg #0") +__failure __msg("expected an initialized iter_num as R1") int compromise_iter_w_direct_write_fail(void *ctx) { struct bpf_iter_num iter; @@ -143,7 +165,7 @@ int compromise_iter_w_direct_write_and_skip_destroy_fail(void *ctx) } SEC("?raw_tp") -__failure __msg("expected an initialized iter_num as arg #0") +__failure __msg("expected an initialized iter_num as R1") int compromise_iter_w_helper_write_fail(void *ctx) { struct bpf_iter_num iter; @@ -196,7 +218,7 @@ int leak_iter_from_subprog_fail(void *ctx) SEC("?raw_tp") __success __log_level(2) -__msg("fp-8=iter_num(ref_id=1,state=active,depth=0)") +__msg("fp-8=iter_num(id=1,state=active,depth=0)") int valid_stack_reuse(void *ctx) { struct bpf_iter_num iter; @@ -230,7 +252,7 @@ int valid_stack_reuse(void *ctx) } SEC("?raw_tp") -__failure __msg("expected uninitialized iter_num as arg #0") +__failure __msg("expected uninitialized iter_num as R1") int double_create_fail(void *ctx) { struct bpf_iter_num iter; @@ -258,7 +280,7 @@ int double_create_fail(void *ctx) } SEC("?raw_tp") -__failure __msg("expected an initialized iter_num as arg #0") +__failure __msg("expected an initialized iter_num as R1") int double_destroy_fail(void *ctx) { struct bpf_iter_num iter; @@ -284,7 +306,7 @@ int double_destroy_fail(void *ctx) } SEC("?raw_tp") -__failure __msg("expected an initialized iter_num as arg #0") +__failure __msg("expected an initialized iter_num as R1") int next_without_new_fail(void *ctx) { struct bpf_iter_num iter; @@ -305,7 +327,7 @@ int next_without_new_fail(void *ctx) } SEC("?raw_tp") -__failure __msg("expected an initialized iter_num as arg #0") +__failure __msg("expected an initialized iter_num as R1") int next_after_destroy_fail(void *ctx) { struct bpf_iter_num iter; diff --git a/tools/testing/selftests/bpf/progs/iters_task_failure.c b/tools/testing/selftests/bpf/progs/iters_task_failure.c index fe3663dedbe1..566a1d3dffea 100644 --- a/tools/testing/selftests/bpf/progs/iters_task_failure.c +++ b/tools/testing/selftests/bpf/progs/iters_task_failure.c @@ -62,6 +62,52 @@ int BPF_PROG(iter_tasks_lock_and_unlock) } SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__failure __msg("expected an RCU CS when using bpf_iter_task_next") +__flag(BPF_F_TEST_STATE_FREQ) +int BPF_PROG(iter_tasks_rcu_state_pruning) +{ + struct bpf_iter_task it; + + asm volatile ( + "call %[bpf_rcu_read_lock];" + "r1 = %[it];" + "r2 = 0;" + "r3 = 0;" /* BPF_TASK_ITER_ALL_PROCS */ + "call %[bpf_iter_task_new];" + + "call %[bpf_get_prandom_u32];" + "if w0 == 0 goto unprotected_%=;" + + /* Keep the outer RCU lock active on the straight-line path. */ + "call %[bpf_rcu_read_lock];" + "call %[bpf_rcu_read_unlock];" + "goto merge_%=;" + + "unprotected_%=:" + /* Create an unprotected gap on the taken path. */ + "call %[bpf_rcu_read_unlock];" + "call %[bpf_rcu_read_lock];" + + "merge_%=: r1 = %[it];" + "call %[bpf_iter_task_next];" + "r1 = %[it];" + "call %[bpf_iter_task_destroy];" + "call %[bpf_rcu_read_unlock];" + : + : __imm_ptr(it), + __imm(bpf_get_prandom_u32), + __imm(bpf_iter_task_new), + __imm(bpf_iter_task_next), + __imm(bpf_iter_task_destroy), + __imm(bpf_rcu_read_lock), + __imm(bpf_rcu_read_unlock) + : __clobber_common + ); + + return 0; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") __failure __msg("expected an RCU CS when using bpf_iter_css_next") int BPF_PROG(iter_css_lock_and_unlock) { diff --git a/tools/testing/selftests/bpf/progs/iters_testmod.c b/tools/testing/selftests/bpf/progs/iters_testmod.c index 5379e9960ffd..76012dbbdb41 100644 --- a/tools/testing/selftests/bpf/progs/iters_testmod.c +++ b/tools/testing/selftests/bpf/progs/iters_testmod.c @@ -29,7 +29,7 @@ out: } SEC("raw_tp/sys_enter") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") int iter_next_trusted_or_null(const void *ctx) { struct task_struct *cur_task = bpf_get_current_task_btf(); @@ -67,7 +67,7 @@ out: } SEC("raw_tp/sys_enter") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") int iter_next_rcu_or_null(const void *ctx) { struct task_struct *cur_task = bpf_get_current_task_btf(); diff --git a/tools/testing/selftests/bpf/progs/iters_testmod_seq.c b/tools/testing/selftests/bpf/progs/iters_testmod_seq.c index 83791348bed5..d00888f6687a 100644 --- a/tools/testing/selftests/bpf/progs/iters_testmod_seq.c +++ b/tools/testing/selftests/bpf/progs/iters_testmod_seq.c @@ -20,8 +20,8 @@ __s64 res_empty; SEC("raw_tp/sys_enter") __success __log_level(2) -__msg("fp-16=iter_testmod_seq(ref_id=1,state=active,depth=0)") -__msg("fp-16=iter_testmod_seq(ref_id=1,state=drained,depth=0)") +__msg("fp-16=iter_testmod_seq(id=1,state=active,depth=0)") +__msg("fp-16=iter_testmod_seq(id=1,state=drained,depth=0)") __msg("call bpf_iter_testmod_seq_destroy") int testmod_seq_empty(const void *ctx) { @@ -38,8 +38,8 @@ __s64 res_full; SEC("raw_tp/sys_enter") __success __log_level(2) -__msg("fp-16=iter_testmod_seq(ref_id=1,state=active,depth=0)") -__msg("fp-16=iter_testmod_seq(ref_id=1,state=drained,depth=0)") +__msg("fp-16=iter_testmod_seq(id=1,state=active,depth=0)") +__msg("fp-16=iter_testmod_seq(id=1,state=drained,depth=0)") __msg("call bpf_iter_testmod_seq_destroy") int testmod_seq_full(const void *ctx) { @@ -58,8 +58,8 @@ static volatile int zero = 0; SEC("raw_tp/sys_enter") __success __log_level(2) -__msg("fp-16=iter_testmod_seq(ref_id=1,state=active,depth=0)") -__msg("fp-16=iter_testmod_seq(ref_id=1,state=drained,depth=0)") +__msg("fp-16=iter_testmod_seq(id=1,state=active,depth=0)") +__msg("fp-16=iter_testmod_seq(id=1,state=drained,depth=0)") __msg("call bpf_iter_testmod_seq_destroy") int testmod_seq_truncated(const void *ctx) { @@ -79,7 +79,7 @@ int testmod_seq_truncated(const void *ctx) SEC("?raw_tp") __failure -__msg("expected an initialized iter_testmod_seq as arg #1") +__msg("expected an initialized iter_testmod_seq as R2") int testmod_seq_getter_before_bad(const void *ctx) { struct bpf_iter_testmod_seq it; @@ -89,7 +89,7 @@ int testmod_seq_getter_before_bad(const void *ctx) SEC("?raw_tp") __failure -__msg("expected an initialized iter_testmod_seq as arg #1") +__msg("expected an initialized iter_testmod_seq as R2") int testmod_seq_getter_after_bad(const void *ctx) { struct bpf_iter_testmod_seq it; diff --git a/tools/testing/selftests/bpf/progs/kfunc_call_fail.c b/tools/testing/selftests/bpf/progs/kfunc_call_fail.c index a1963497f0bf..7e93f7fb1329 100644 --- a/tools/testing/selftests/bpf/progs/kfunc_call_fail.c +++ b/tools/testing/selftests/bpf/progs/kfunc_call_fail.c @@ -4,6 +4,18 @@ #include <bpf/bpf_helpers.h> #include "../test_kmods/bpf_testmod_kfunc.h" +static struct bpf_spin_lock kfunc_call_lock SEC(".data.A"); + +SEC("?tc") +int kfunc_call_test_spin_lock_unsafe(struct __sk_buff *skb) +{ + bpf_spin_lock(&kfunc_call_lock); + bpf_kfunc_trigger_ctx_check(); + bpf_spin_unlock(&kfunc_call_lock); + + return 0; +} + struct syscall_test_args { __u8 data[16]; size_t size; @@ -103,6 +115,66 @@ int kfunc_call_test_get_mem_fail_oob(struct __sk_buff *skb) return ret; } +SEC("?tc") +int kfunc_call_test_get_mem_fail_zero_size(struct __sk_buff *skb) +{ + struct prog_test_ref_kfunc *pt; + unsigned long s = 0; + int *p = NULL; + int ret = 0; + + pt = bpf_kfunc_call_test_acquire(&s); + if (pt) { + /* + * An explicit rdwr_buf_size of 0 gives R0 a zero-sized buffer, + * so any access is out of bounds, hence -EACCES. Previously the + * verifier treated a zero size as "no size argument" and sized + * R0 after the pointed-to return type, wrongly allowing the read. + */ + p = bpf_kfunc_call_test_get_rdwr_mem(pt, 0); + if (p) + ret = p[0]; + else + ret = -1; + + bpf_kfunc_call_test_release(pt); + } + return ret; +} + +SEC("?tc") +int kfunc_call_test_get_mem_fail_oversized(struct __sk_buff *skb) +{ + struct prog_test_ref_kfunc *pt; + unsigned long s = 0; + int *p = NULL; + int ret = 0; + + pt = bpf_kfunc_call_test_acquire(&s); + if (pt) { + /* + * rdwr_buf_size is a const int, so a C literal is narrowed to + * 32 bits before the call. Force the full 64-bit value 2^64 - 192 + * (0xffffffffffffff40, > U32_MAX) into the argument register with + * a 64-bit immediate load. The verifier records r0_size from the + * full register value and must reject it before that value is + * truncated into R0's u32 mem_size. + */ + asm volatile ( + "r1 = %[pt];" + "r2 = %[oversized] ll;" + "call %[get_rdwr_mem];" + "%[p] = r0;" + : [p] "=r"(p) + : [pt] "r"(pt), + [oversized] "i"(0xffffffffffffff40LL), + [get_rdwr_mem] "i"(bpf_kfunc_call_test_get_rdwr_mem) + : "r0", "r1", "r2", "r3", "r4", "r5"); + bpf_kfunc_call_test_release(pt); + } + return ret; +} + int not_const_size = 2 * sizeof(int); SEC("?tc") diff --git a/tools/testing/selftests/bpf/progs/kfunc_call_test.c b/tools/testing/selftests/bpf/progs/kfunc_call_test.c index 8b86113a0126..8e6560c31e78 100644 --- a/tools/testing/selftests/bpf/progs/kfunc_call_test.c +++ b/tools/testing/selftests/bpf/progs/kfunc_call_test.c @@ -2,8 +2,118 @@ /* Copyright (c) 2021 Facebook */ #include <vmlinux.h> #include <bpf/bpf_helpers.h> +#include "bpf_misc.h" #include "../test_kmods/bpf_testmod_kfunc.h" +static struct bpf_spin_lock kfunc_call_lock SEC(".data.A"); + +SEC("tc") +int kfunc_call_test_spin_lock_safe(struct __sk_buff *skb) +{ + bpf_spin_lock(&kfunc_call_lock); + bpf_testmod_test_mod_kfunc(42); + bpf_spin_unlock(&kfunc_call_lock); + + return 0; +} + +SEC("tc") +int kfunc_call_test5(struct __sk_buff *skb) +{ + struct bpf_sock *sk = skb->sk; + int ret; + u32 val32; + u16 val16; + u8 val8; + + if (!sk) + return -1; + + sk = bpf_sk_fullsock(sk); + if (!sk) + return -1; + + /* + * Test with constant values to verify zero-extension. + * ISA-dependent BPF asm: + * With ALU32: w1 = 0xFF; w2 = 0xFFFF; w3 = 0xFFFFffff + * Without ALU32: r1 = 0xFF; r2 = 0xFFFF; r3 = 0xFFFFffff + * Both zero-extend to 64-bit before the kfunc call. + */ + ret = bpf_kfunc_call_test5(0xFF, 0xFFFF, 0xFFFFffffULL); + if (ret) + return ret; + + val32 = bpf_get_prandom_u32(); + val16 = val32 & 0xFFFF; + val8 = val32 & 0xFF; + ret = bpf_kfunc_call_test5(val8, val16, val32); + if (ret) + return ret; + + /* + * Test multiplication with different operand sizes: + * + * val8 * 0xFF: + * - Both operands promote to int (32-bit signed) + * - Result: 32-bit multiplication, truncated to u8, then zero-extended + * + * val16 * 0xFFFF: + * - Both operands promote to int (32-bit signed) + * - Result: 32-bit multiplication, truncated to u16, then zero-extended + * + * val32 * 0xFFFFffffULL: + * - val32 (u32) promotes to unsigned long long (due to ULL suffix) + * - Result: 64-bit unsigned multiplication, truncated to u32, then zero-extended + */ + ret = bpf_kfunc_call_test5(val8 * 0xFF, val16 * 0xFFFF, val32 * 0xFFFFffffULL); + if (ret) + return ret; + + return 0; +} + +/* + * Assembly version testing the multiplication edge case explicitly. + * This ensures consistent testing across different ISA versions. + */ +SEC("tc") +__naked int kfunc_call_test5_asm(void) +{ + asm volatile ( + /* Get a random u32 value */ + "call %[bpf_get_prandom_u32];" + "r6 = r0;" /* Save val32 in r6 */ + + /* Prepare first argument: val8 * 0xFF */ + "r1 = r6;" + "r1 &= 0xFF;" /* val8 = val32 & 0xFF */ + "r7 = 0xFF;" + "r1 *= r7;" /* 64-bit mult: r1 = r1 * r7 */ + + /* Prepare second argument: val16 * 0xFFFF */ + "r2 = r6;" + "r2 &= 0xFFFF;" /* val16 = val32 & 0xFFFF */ + "r7 = 0xFFFF;" + "r2 *= r7;" /* 64-bit mult: r2 = r2 * r7 */ + + /* Prepare third argument: val32 * 0xFFFFffff */ + "r3 = r6;" /* val32 */ + "r7 = 0xFFFFffff;" + "r3 *= r7;" /* 64-bit mult: r3 = r3 * r7 */ + + /* Call kfunc with multiplication results */ + "call bpf_kfunc_call_test5;" + + /* Check return value */ + "if r0 != 0 goto exit_%=;" + "r0 = 0;" + "exit_%=: exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + SEC("tc") int kfunc_call_test4(struct __sk_buff *skb) { diff --git a/tools/testing/selftests/bpf/progs/kfunc_implicit_args_tracing.c b/tools/testing/selftests/bpf/progs/kfunc_implicit_args_tracing.c new file mode 100644 index 000000000000..995f8b8b5b9e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/kfunc_implicit_args_tracing.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <errno.h> + +extern int bpf_kfunc_implicit_arg(int a) __weak __ksym; + +char _license[] SEC("license") = "GPL"; + +/* Shared arg checks; reports arg count and aux, returns 1 on success. */ +static __always_inline __u64 +check_implicit_args(void *ctx, __u64 *arg_cnt, __u64 *aux_arg) +{ + __u64 a = 0, aux = 0, z = 0; + __u64 result; + __s64 err; + + *arg_cnt = bpf_get_func_arg_cnt(ctx); + result = *arg_cnt == 2; + + err = bpf_get_func_arg(ctx, 0, &a); + result &= err == 0 && (int)a == 5; + + err = bpf_get_func_arg(ctx, 1, &aux); + *aux_arg = aux; + result &= err == 0 && aux != 0; + + err = bpf_get_func_arg(ctx, 2, &z); + result &= err == -EINVAL; + + return result; +} + +__u64 fentry_result; +__u64 fentry_arg_cnt; +__u64 fentry_aux_arg; + +SEC("fentry/bpf_kfunc_implicit_arg") +int BPF_PROG(trace_implicit_arg_fentry) +{ + __u64 ret = 0; + __s64 err; + + fentry_result = check_implicit_args(ctx, &fentry_arg_cnt, &fentry_aux_arg); + + err = bpf_get_func_ret(ctx, &ret); + fentry_result &= err == -EOPNOTSUPP; + + return 0; +} + +__u64 fexit_result; +__u64 fexit_arg_cnt; +__u64 fexit_aux_arg; + +SEC("fexit/bpf_kfunc_implicit_arg") +int BPF_PROG(trace_implicit_arg_fexit) +{ + __u64 ret = 0; + __s64 err; + + fexit_result = check_implicit_args(ctx, &fexit_arg_cnt, &fexit_aux_arg); + + err = bpf_get_func_ret(ctx, &ret); + fexit_result &= err == 0 && ret == 5; + + return 0; +} + +SEC("syscall") +int trigger_implicit_arg(void *ctx) +{ + return bpf_kfunc_implicit_arg(5); +} diff --git a/tools/testing/selftests/bpf/progs/kprobe_multi_session.c b/tools/testing/selftests/bpf/progs/kprobe_multi_session.c index bd8b7fb7061e..d52a65b40bbf 100644 --- a/tools/testing/selftests/bpf/progs/kprobe_multi_session.c +++ b/tools/testing/selftests/bpf/progs/kprobe_multi_session.c @@ -76,3 +76,13 @@ int test_kprobe(struct pt_regs *ctx) { return session_check(ctx); } + +/* + * Exact function name (no wildcards) - exercises the fast syms[] path + * in bpf_program__attach_kprobe_multi_opts() which bypasses kallsyms parsing. + */ +SEC("kprobe.session/bpf_fentry_test1") +int test_kprobe_syms(struct pt_regs *ctx) +{ + return session_check(ctx); +} diff --git a/tools/testing/selftests/bpf/progs/kprobe_multi_sleepable.c b/tools/testing/selftests/bpf/progs/kprobe_multi_sleepable.c new file mode 100644 index 000000000000..932e1d9c72e2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/kprobe_multi_sleepable.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +void *user_ptr = 0; + +SEC("kprobe.multi") +int handle_kprobe_multi_sleepable(struct pt_regs *ctx) +{ + int a, err; + + err = bpf_copy_from_user(&a, sizeof(a), user_ptr); + barrier_var(a); + return err; +} + +SEC("fentry/bpf_fentry_test1") +int BPF_PROG(fentry) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/kprobe_write_ctx.c b/tools/testing/selftests/bpf/progs/kprobe_write_ctx.c index f77aef0474d3..adbf52afe490 100644 --- a/tools/testing/selftests/bpf/progs/kprobe_write_ctx.c +++ b/tools/testing/selftests/bpf/progs/kprobe_write_ctx.c @@ -19,4 +19,23 @@ int kprobe_multi_write_ctx(struct pt_regs *ctx) ctx->ax = 0; return 0; } + +SEC("?kprobe") +int kprobe_dummy(struct pt_regs *regs) +{ + return 0; +} + +SEC("?freplace") +int freplace_kprobe(struct pt_regs *regs) +{ + regs->di = 0; + return 0; +} + +SEC("?fentry/bpf_fentry_test1") +int BPF_PROG(fentry) +{ + return 0; +} #endif diff --git a/tools/testing/selftests/bpf/progs/kptr_xchg_inline.c b/tools/testing/selftests/bpf/progs/kptr_xchg_inline.c index 2414ac20b6d5..ca5943166057 100644 --- a/tools/testing/selftests/bpf/progs/kptr_xchg_inline.c +++ b/tools/testing/selftests/bpf/progs/kptr_xchg_inline.c @@ -25,14 +25,14 @@ __naked int kptr_xchg_inline(void) "if r0 == 0 goto 1f;" "r1 = r0;" "r2 = 0;" - "call %[bpf_obj_drop_impl];" + "call %[bpf_obj_drop];" "1:" "r0 = 0;" "exit;" : : __imm_addr(ptr), __imm(bpf_kptr_xchg), - __imm(bpf_obj_drop_impl) + __imm(bpf_obj_drop) : __clobber_all ); } diff --git a/tools/testing/selftests/bpf/progs/ksock_common.h b/tools/testing/selftests/bpf/progs/ksock_common.h new file mode 100644 index 000000000000..01edaeb9fdd4 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/ksock_common.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 Isovalent */ + +#ifndef _KSOCK_COMMON_H +#define _KSOCK_COMMON_H + +#include "errno.h" + +#define SOCK_DGRAM 2 +#define IPPROTO_UDP 17 + +struct bpf_ksock *bpf_ksock_create(const struct bpf_ksock_create_opts *opts, + u32 opts__sz, int *err__uninit) __ksym; +int bpf_ksock_connect(struct bpf_ksock *ks, const union bpf_ksock_addr *addr, + u32 addr__sz) __ksym; +struct bpf_ksock *bpf_ksock_acquire(struct bpf_ksock *ks) __ksym; +void bpf_ksock_release(struct bpf_ksock *ks) __ksym; +int bpf_ksock_send(struct bpf_ksock *ks, const void *data, u32 data__sz) __ksym; +void bpf_rcu_read_lock(void) __ksym; +void bpf_rcu_read_unlock(void) __ksym; + +struct __ksock_ctx_value { + struct bpf_ksock __kptr * ctx; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct __ksock_ctx_value); + __uint(max_entries, 1); +} __ksock_ctx_map SEC(".maps"); + +static inline struct __ksock_ctx_value *ksock_ctx_value_lookup(void) +{ + u32 key = 0; + + return bpf_map_lookup_elem(&__ksock_ctx_map, &key); +} + +static inline struct bpf_ksock *ksock_ctx_get(void) +{ + struct __ksock_ctx_value *v; + struct bpf_ksock *ks = NULL, *tmp; + + v = ksock_ctx_value_lookup(); + if (!v) + return NULL; + + bpf_rcu_read_lock(); + tmp = v->ctx; + if (tmp) + ks = bpf_ksock_acquire(tmp); + bpf_rcu_read_unlock(); + + return ks; +} + +static inline int ksock_ctx_insert(struct bpf_ksock *ctx) +{ + struct __ksock_ctx_value *v; + struct bpf_ksock *old; + + v = ksock_ctx_value_lookup(); + if (!v) { + bpf_ksock_release(ctx); + return -ENOENT; + } + + old = bpf_kptr_xchg(&v->ctx, ctx); + if (old) { + bpf_ksock_release(old); + return -EEXIST; + } + + return 0; +} + +#endif /* _KSOCK_COMMON_H */ diff --git a/tools/testing/selftests/bpf/progs/ksock_lsm.c b/tools/testing/selftests/bpf/progs/ksock_lsm.c new file mode 100644 index 000000000000..9808451098ef --- /dev/null +++ b/tools/testing/selftests/bpf/progs/ksock_lsm.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_endian.h> +#include "bpf_tracing_net.h" +#include "ksock_common.h" + +char send_data[32] = "hello from bpf ksock"; + +__be32 ipv4_remote; +__u16 remote_port; +int target_pid; +int send_ret = -1; + +SEC("syscall") +int ksock_setup(void *ctx) +{ + struct bpf_ksock_create_opts create_opts = {}; + union bpf_ksock_addr addr = {}; + struct bpf_ksock *ks; + int err = 0; + + create_opts.family = AF_INET; + create_opts.type = SOCK_DGRAM; + create_opts.protocol = IPPROTO_UDP; + + ks = bpf_ksock_create(&create_opts, sizeof(create_opts), &err); + if (!ks) + return err; + + addr.sin.sin_family = AF_INET; + addr.sin.sin_port = bpf_htons(remote_port); + addr.sin.sin_addr.s_addr = ipv4_remote; + + err = bpf_ksock_connect(ks, &addr, sizeof(addr)); + if (err) { + bpf_ksock_release(ks); + return err; + } + + err = ksock_ctx_insert(ks); + if (err && err != -EEXIST) + return err; + return 0; +} + +SEC("lsm.s/socket_bind") +int BPF_PROG(ksock_socket_bind, struct socket *sock, struct sockaddr *address, + int addrlen, int ret) +{ + struct bpf_ksock *ks; + u32 pid = bpf_get_current_pid_tgid() >> 32; + + if (ret || pid != target_pid) + return ret; + + ks = ksock_ctx_get(); + if (!ks) { + send_ret = -ENOENT; + return ret; + } + + send_ret = bpf_ksock_send(ks, send_data, sizeof(send_data)); + bpf_ksock_release(ks); + + return ret; +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c b/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c new file mode 100644 index 000000000000..fd2ccfdb5802 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" +#include "ksock_common.h" + +char send_data[11] = "dummy data"; + +SEC("lsm.s/socket_sendmsg") +__description("bpf_ksock_send is rejected from socket_sendmsg LSM hook") +__failure __msg("calling kernel function bpf_ksock_send is not allowed") +int BPF_PROG(ksock_socket_sendmsg, struct socket *sock, struct msghdr *msg, + int size, int ret) +{ + struct __ksock_ctx_value *v; + struct bpf_ksock *ks; + + v = ksock_ctx_value_lookup(); + if (!v) + return ret; + + ks = bpf_kptr_xchg(&v->ctx, NULL); + if (!ks) + return ret; + + bpf_ksock_send(ks, send_data, sizeof(send_data)); + bpf_ksock_release(ks); + + return ret; +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/ksock_wq.c b/tools/testing/selftests/bpf/progs/ksock_wq.c new file mode 100644 index 000000000000..16a1873d132e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/ksock_wq.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include "bpf_experimental.h" +#include "bpf_tracing_net.h" +#include "errno.h" +#include "ksock_common.h" + +struct ksock_wq_value { + struct bpf_wq work; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, struct ksock_wq_value); +} work_map SEC(".maps"); + +int create_err; +u32 callback_done; + +static int ksock_wq_callback(void *map, int *key, void *value) +{ + struct bpf_ksock_create_opts opts = { + .family = AF_INET, + .type = SOCK_DGRAM, + .protocol = IPPROTO_UDP, + }; + struct bpf_ksock *ks; + int err = 0; + + ks = bpf_ksock_create(&opts, sizeof(opts), &err); + if (ks) + bpf_ksock_release(ks); + create_err = err; + __sync_fetch_and_add(&callback_done, 1); + return 0; +} + +SEC("syscall") +int ksock_wq_start(void *ctx) +{ + struct ksock_wq_value *value; + u32 key = 0; + int err; + + value = bpf_map_lookup_elem(&work_map, &key); + if (!value) + return -ENOENT; + err = bpf_wq_init(&value->work, &work_map, 0); + if (err) + return err; + err = bpf_wq_set_callback(&value->work, ksock_wq_callback, 0); + if (err) + return err; + return bpf_wq_start(&value->work, 0); +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/linked_list.c b/tools/testing/selftests/bpf/progs/linked_list.c index 421f40835acd..fa97faa5358b 100644 --- a/tools/testing/selftests/bpf/progs/linked_list.c +++ b/tools/testing/selftests/bpf/progs/linked_list.c @@ -290,6 +290,77 @@ int test_list_in_list(struct bpf_spin_lock *lock, struct bpf_list_head *head) return list_in_list(lock, head, true); } +#define MAX_LIST_CLEAR_NODES 256 + +static __always_inline +int clear_list(struct bpf_spin_lock *lock, struct bpf_list_head *head) +{ + struct bpf_list_node *n; + int i; + + for (i = 0; i < MAX_LIST_CLEAR_NODES; i++) { + bpf_spin_lock(lock); + n = bpf_list_pop_front(head); + bpf_spin_unlock(lock); + if (!n) + return 0; + bpf_obj_drop(container_of(n, struct foo, node2)); + } + return 1; +} + +SEC("syscall") +int clear_map_list(void *ctx) +{ + struct map_value *v; + + v = bpf_map_lookup_elem(&array_map, &(int){0}); + if (!v) + return 1; + return clear_list(&v->lock, &v->head); +} + +SEC("syscall") +int clear_inner_map_list(void *ctx) +{ + struct map_value *v; + void *map; + + map = bpf_map_lookup_elem(&map_of_maps, &(int){0}); + if (!map) + return 1; + v = bpf_map_lookup_elem(map, &(int){0}); + if (!v) + return 1; + return clear_list(&v->lock, &v->head); +} + +SEC("syscall") +int clear_global_list(void *ctx) +{ + return clear_list(&glock, &ghead); +} + +SEC("syscall") +int clear_global_nested_list(void *ctx) +{ + return clear_list(&ghead_nested.inner.lock, &ghead_nested.inner.head); +} + +SEC("syscall") +int clear_global_array_list(void *ctx) +{ + int ret; + + ret = clear_list(&glock_c, &ghead_array[0]); + if (ret) + return ret; + ret = clear_list(&glock_c, &ghead_array[1]); + if (ret) + return ret; + return clear_list(&glock_c, &ghead_array_one[0]); +} + SEC("tc") int map_list_push_pop(void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/linked_list_fail.c b/tools/testing/selftests/bpf/progs/linked_list_fail.c index ddd26d1a083f..031e77a288ee 100644 --- a/tools/testing/selftests/bpf/progs/linked_list_fail.c +++ b/tools/testing/selftests/bpf/progs/linked_list_fail.c @@ -167,6 +167,16 @@ CHECK_OP(push_back); #undef CHECK_OP #undef INIT +struct obj_new_flex_elem { + int lo; + int hi; +}; + +struct obj_new_flex { + int hdr; + struct obj_new_flex_elem cells[]; +}; + SEC("?kprobe/xyz") int map_compat_kprobe(void *ctx) { @@ -231,6 +241,19 @@ int obj_new_no_struct(void *ctx) } SEC("?tc") +int obj_new_flex_array(void *ctx) +{ + struct obj_new_flex *p; + + p = bpf_obj_new_impl(bpf_core_type_id_local(struct obj_new_flex), NULL); + if (!p) + return 0; + p->cells[0].hi = 42; + bpf_obj_drop_impl(p, NULL); + return 0; +} + +SEC("?tc") int obj_drop_non_zero_off(void *ctx) { void *f; diff --git a/tools/testing/selftests/bpf/progs/lpm_trie_bench.c b/tools/testing/selftests/bpf/progs/lpm_trie_bench.c index a0e6ebd5507a..2831cf4445e8 100644 --- a/tools/testing/selftests/bpf/progs/lpm_trie_bench.c +++ b/tools/testing/selftests/bpf/progs/lpm_trie_bench.c @@ -7,7 +7,7 @@ #include <bpf/bpf_helpers.h> #include <bpf/bpf_core_read.h> #include "bpf_misc.h" -#include "bpf_atomic.h" +#include <bpf_atomic.h> #include "progs/lpm_trie.h" #define BPF_OBJ_NAME_LEN 16U diff --git a/tools/testing/selftests/bpf/progs/lru_lock_nmi.c b/tools/testing/selftests/bpf/progs/lru_lock_nmi.c new file mode 100644 index 000000000000..c0692cd54237 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lru_lock_nmi.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, 64); + __type(key, __u32); + __type(value, __u64); +} lru_map SEC(".maps"); + +int hits; + +SEC("perf_event") +int oncpu(void *ctx) +{ + /* + * Key range deliberately wider than max_entries to force LRU + * eviction on every other update. + */ + __u32 key = bpf_get_prandom_u32() % 128; + bool do_update = bpf_get_prandom_u32() & 1; + __u64 val = 1; + + if (do_update) + bpf_map_update_elem(&lru_map, &key, &val, BPF_ANY); + else + bpf_map_delete_elem(&lru_map, &key); + __sync_fetch_and_add(&hits, 1); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/lsm_bdev.c b/tools/testing/selftests/bpf/progs/lsm_bdev.c new file mode 100644 index 000000000000..45554e6db605 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lsm_bdev.c @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Christian Brauner <brauner@kernel.org> */ + +/* + * BPF LSM block device integrity tracker for dm-verity. + * + * Tracks block devices in a hashmap keyed by bd_dev. When dm-verity + * calls security_bdev_setintegrity() during verity_preresume(), the + * setintegrity hook records the roothash and signature-validity data. + * The free hook cleans up when the device goes away. The alloc hook + * counts allocations for test validation. + * + * The sleepable hooks exercise bpf_copy_from_user() to verify that + * the sleepable classification actually permits sleepable helpers. + */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +struct verity_info { + __u8 has_roothash; /* LSM_INT_DMVERITY_ROOTHASH seen */ + __u8 sig_valid; /* LSM_INT_DMVERITY_SIG_VALID value (non-NULL = valid) */ + __u32 setintegrity_cnt; /* total setintegrity calls for this dev */ +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 64); + __type(key, __u32); /* dev_t from bdev->bd_dev */ + __type(value, struct verity_info); +} verity_devices SEC(".maps"); + +/* Global counters exposed to userspace via skeleton bss. */ +int alloc_count; + +char _license[] SEC("license") = "GPL"; + +SEC("lsm.s/bdev_setintegrity") +int BPF_PROG(bdev_setintegrity, struct block_device *bdev, + enum lsm_integrity_type type, const void *value, size_t size) +{ + struct verity_info zero = {}; + struct verity_info *info; + __u32 dev; + char buf; + + /* + * Exercise a sleepable helper to confirm the verifier + * allows it in this sleepable hook. + */ + (void)bpf_copy_from_user(&buf, sizeof(buf), NULL); + + dev = bdev->bd_dev; + + info = bpf_map_lookup_elem(&verity_devices, &dev); + if (!info) { + bpf_map_update_elem(&verity_devices, &dev, &zero, BPF_NOEXIST); + info = bpf_map_lookup_elem(&verity_devices, &dev); + if (!info) + return 0; + } + + if (type == LSM_INT_DMVERITY_ROOTHASH) + info->has_roothash = 1; + else if (type == LSM_INT_DMVERITY_SIG_VALID) + info->sig_valid = (value != NULL); + + __sync_fetch_and_add(&info->setintegrity_cnt, 1); + + return 0; +} + +SEC("lsm/bdev_free_security") +void BPF_PROG(bdev_free_security, struct block_device *bdev) +{ + __u32 dev = bdev->bd_dev; + + bpf_map_delete_elem(&verity_devices, &dev); +} + +SEC("lsm.s/bdev_alloc_security") +int BPF_PROG(bdev_alloc_security, struct block_device *bdev) +{ + char buf; + + /* + * Exercise a sleepable helper to confirm the verifier + * allows it in this sleepable hook. + */ + (void)bpf_copy_from_user(&buf, sizeof(buf), NULL); + + __sync_fetch_and_add(&alloc_count, 1); + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/lsm_cgroup.c b/tools/testing/selftests/bpf/progs/lsm_cgroup.c index d7598538aa2d..3bfa479104be 100644 --- a/tools/testing/selftests/bpf/progs/lsm_cgroup.c +++ b/tools/testing/selftests/bpf/progs/lsm_cgroup.c @@ -35,6 +35,8 @@ int called_socket_bind; int called_socket_bind2; int called_socket_alloc; int called_socket_clone; +int skipcap_retval = -4095; +int socket_retval = -4095; static __always_inline int test_local_storage(void) { @@ -190,3 +192,31 @@ int BPF_PROG(socket_clone, struct sock *newsk, const struct request_sock *req) return 1; } + +SEC("lsm_cgroup/inode_xattr_skipcap") +int BPF_PROG(skipcap_first, const char *name) +{ + return 0; +} + +SEC("lsm_cgroup/inode_xattr_skipcap") +int BPF_PROG(skipcap_second, const char *name) +{ + skipcap_retval = bpf_get_retval(); + bpf_set_retval(0); + return 1; +} + +SEC("lsm_cgroup/socket_create") +int BPF_PROG(socket_first, int family, int type, int protocol, int kern) +{ + return 0; +} + +SEC("lsm_cgroup/socket_create") +int BPF_PROG(socket_second, int family, int type, int protocol, int kern) +{ + socket_retval = bpf_get_retval(); + bpf_set_retval(0); + return 1; +} diff --git a/tools/testing/selftests/bpf/progs/lwt_misc.c b/tools/testing/selftests/bpf/progs/lwt_misc.c new file mode 100644 index 000000000000..b392317088d2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lwt_misc.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +SEC("lwt_xmit") +__success __retval(0) +int test_missing_dst(struct __sk_buff *skb) +{ + struct iphdr iph; + + __builtin_memset(&iph, 0, sizeof(struct iphdr)); + iph.ihl = 5; + iph.version = 4; + + bpf_lwt_push_encap(skb, BPF_LWT_ENCAP_IP, &iph, sizeof(struct iphdr)); + + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/map_kptr.c b/tools/testing/selftests/bpf/progs/map_kptr.c index e708ffbe1f61..0d87c97dac99 100644 --- a/tools/testing/selftests/bpf/progs/map_kptr.c +++ b/tools/testing/selftests/bpf/progs/map_kptr.c @@ -4,6 +4,18 @@ #include <bpf/bpf_helpers.h> #include "../test_kmods/bpf_testmod_kfunc.h" +struct map_uninit_value { + struct prog_test_ref_kfunc __kptr_untrusted *unref_ptr; + __u32 data; +} __attribute__((packed)); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, int); + __type(value, struct map_uninit_value); + __uint(max_entries, 1); +} pcpu_array SEC(".maps"); + struct map_value { struct prog_test_ref_kfunc __kptr_untrusted *unref_ptr; struct prog_test_ref_kfunc __kptr *ref_ptr; @@ -489,8 +501,7 @@ int test_map_kptr_ref3(struct __sk_buff *ctx) int num_of_refs; -SEC("syscall") -int count_ref(void *ctx) +static __always_inline int read_ref_count(void) { struct prog_test_ref_kfunc *p; unsigned long arg = 0; @@ -500,12 +511,96 @@ int count_ref(void *ctx) return 1; num_of_refs = p->cnt.refs.counter; - bpf_kfunc_call_test_release(p); return 0; } SEC("syscall") +int count_ref(void *ctx) +{ + return read_ref_count(); +} + +static __always_inline int stash_ref_ptr(struct map_value *v) +{ + struct prog_test_ref_kfunc *p, *old; + unsigned long arg = 0; + + p = bpf_kfunc_call_test_acquire(&arg); + if (!p) + return 1; + + old = bpf_kptr_xchg(&v->ref_ptr, p); + if (old) { + bpf_kfunc_call_test_release(old); + old = bpf_kptr_xchg(&v->ref_ptr, NULL); + if (old) + bpf_kfunc_call_test_release(old); + return 2; + } + return 0; +} + +static __always_inline int check_refs(int expected) +{ + int ret; + + ret = read_ref_count(); + if (ret) + return ret; + return num_of_refs == expected ? 0 : 3; +} + +SEC("syscall") +int test_array_map_update_kptr(void *ctx) +{ + struct map_value init = {}, *v; + int key = 0, ret; + + v = bpf_map_lookup_elem(&array_map, &key); + if (!v) + return 1; + ret = stash_ref_ptr(v); + if (ret) + return ret; + ret = check_refs(3); + if (ret) + return ret; + ret = bpf_map_update_elem(&array_map, &key, &init, BPF_EXIST); + if (ret) + return 4; + return check_refs(3); +} + +#define DEFINE_HASH_UPDATE_KPTR_TEST(name, map) \ +SEC("syscall") \ +int name(void *ctx) \ +{ \ + struct map_value init = {}, *v; \ + int key = 0, ret; \ + \ + ret = bpf_map_update_elem(&map, &key, &init, BPF_NOEXIST); \ + if (ret) \ + return 1; \ + v = bpf_map_lookup_elem(&map, &key); \ + if (!v) \ + return 2; \ + ret = stash_ref_ptr(v); \ + if (ret) \ + return ret; \ + ret = check_refs(3); \ + if (ret) \ + return ret; \ + ret = bpf_map_update_elem(&map, &key, &init, BPF_EXIST); \ + if (ret) \ + return 4; \ + return check_refs(3); \ +} + +DEFINE_HASH_UPDATE_KPTR_TEST(test_hash_map_update_kptr, hash_map) +DEFINE_HASH_UPDATE_KPTR_TEST(test_hash_malloc_map_update_kptr, hash_malloc_map) + +SEC("syscall") int test_ls_map_kptr_ref1(void *ctx) { struct task_struct *current; diff --git a/tools/testing/selftests/bpf/progs/map_kptr_fail.c b/tools/testing/selftests/bpf/progs/map_kptr_fail.c index 6443b320c732..eee35d203b66 100644 --- a/tools/testing/selftests/bpf/progs/map_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/map_kptr_fail.c @@ -252,7 +252,7 @@ int reject_untrusted_store_to_ref(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("R2 must be referenced") +__failure __msg("release helper bpf_kptr_xchg expects referenced PTR_TO_BTF_ID passed to R2") int reject_untrusted_xchg(struct __sk_buff *ctx) { struct prog_test_ref_kfunc *p; @@ -364,7 +364,7 @@ int kptr_xchg_ref_state(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("Possibly NULL pointer passed to helper arg2") +__failure __msg("Possibly NULL pointer passed to helper R2") int kptr_xchg_possibly_null(struct __sk_buff *ctx) { struct prog_test_ref_kfunc *p; @@ -385,4 +385,65 @@ int kptr_xchg_possibly_null(struct __sk_buff *ctx) return 0; } +SEC("?tc") +/* + * A compiler with BPF_ST folds the constant into a store-immediate, which the + * verifier rejects on a different path (and with a different message) than the + * BPF_STX form. + */ +#ifdef __BPF_FEATURE_ST +__failure __msg("BPF_ST imm must be 0 when storing to kptr at off=8") +#else +__failure __msg("invalid kptr access, R") +#endif +int reject_scalar_store_to_kptr(struct __sk_buff *ctx) +{ + struct map_value *v; + int key = 0; + + v = bpf_map_lookup_elem(&array_map, &key); + if (!v) + return 0; + + *(volatile u64 *)&v->unref_ptr = 0xBADC0DE; + return 0; +} + +SEC("?tc") +__description("reject imprecise scalar store to kptr after state pruning") +__failure __msg("invalid kptr access, R7 type=scalar") +__naked void reject_imprecise_scalar_store_to_kptr(void) +{ + asm volatile ( + "r0 = 0;" + "*(u32 *)(r10 - 4) = r0;" + "r2 = r10;" + "r2 += -4;" + "r1 = %[array_map] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto l2_%=;" + "r6 = r0;" + "r9 = *(u64 *)(r6 + 0);" + "if r9 != 0 goto l0_%=;" + "r7 = 0;" + ".rept 10;" + "r5 = 1;" + ".endr;" + "goto l1_%=;" + "l0_%=:" + "r7 = 0x4141414141414141 ll;" + ".rept 10;" + "r5 = 1;" + ".endr;" + "l1_%=:" + "*(u64 *)(r6 + 8) = r7;" + "l2_%=:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(array_map) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/map_ptr_kern.c b/tools/testing/selftests/bpf/progs/map_ptr_kern.c index efaf622c28dd..373c8d17ea55 100644 --- a/tools/testing/selftests/bpf/progs/map_ptr_kern.c +++ b/tools/testing/selftests/bpf/progs/map_ptr_kern.c @@ -647,8 +647,14 @@ static inline int check_devmap_hash(void) return 1; } +struct bpf_ringbuf { + unsigned long consumer_pos; + unsigned long producer_pos; +} __attribute__((preserve_access_index)); + struct bpf_ringbuf_map { struct bpf_map map; + struct bpf_ringbuf *rb; } __attribute__((preserve_access_index)); struct { @@ -659,9 +665,20 @@ static inline int check_ringbuf(void) { struct bpf_ringbuf_map *ringbuf = (struct bpf_ringbuf_map *)&m_ringbuf; struct bpf_map *map = (struct bpf_map *)&m_ringbuf; + struct bpf_ringbuf *rb; + void *ptr; VERIFY(check(&ringbuf->map, map, 0, 0, page_size)); + ptr = bpf_ringbuf_reserve(&m_ringbuf, 128, 0); + VERIFY(ptr); + + bpf_ringbuf_discard(ptr, 0); + rb = ringbuf->rb; + VERIFY(rb); + VERIFY(rb->consumer_pos == 0); + VERIFY(rb->producer_pos == 128 + BPF_RINGBUF_HDR_SZ); + return 1; } diff --git a/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c b/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c index 3b984b6ae7c0..3e0d4f687aaa 100644 --- a/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c +++ b/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c @@ -3,12 +3,13 @@ #include <vmlinux.h> #include <bpf/bpf_core_read.h> #include "bpf_misc.h" +#include "bpf_kfuncs.h" #include "../test_kmods/bpf_testmod_kfunc.h" SEC("tp_btf/sys_enter") __success __log_level(2) -__msg("r8 = *(u64 *)(r7 +0) ; R7=ptr_nameidata(off={{[0-9]+}}) R8=rdonly_untrusted_mem(sz=0)") +__msg("r8 = *(u64 *)(r7 +0) ; R7=ptr_nameidata(imm={{[0-9]+}}) R8=rdonly_untrusted_mem(sz=0)") __msg("r9 = *(u8 *)(r8 +0) ; R8=rdonly_untrusted_mem(sz=0) R9=scalar") int btf_id_to_ptr_mem(void *ctx) { @@ -137,7 +138,7 @@ int helper_param_not_ok(void *ctx) p = bpf_rdonly_cast(0, 0); /* - * Any helper with ARG_CONST_SIZE_OR_ZERO constraint will do, + * Any helper with ARG_MEM_SIZE_OR_ZERO constraint will do, * the most permissive constraint */ bpf_copy_from_user(p, 0, (void *)42); @@ -164,6 +165,239 @@ int mixed_mem_type(void *ctx) return *p; } +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 4096); +} ringbuf SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, u64); +} array SEC(".maps"); + +char dynptr_data[8]; + +int zero; + +SEC("socket") +__success +__log_level(2) +__msg("r8 = *(u64 *)(r7 +0){{.*}}R7=untrusted_ptr_sock") +__msg("r8 = *(u64 *)(r7 +0){{.*}}R7=ringbuf_mem") +__retval(0) +int mixed_mem_untrusted_btf_id_type(void *ctx) +{ + u64 *p, *q, v; + + p = bpf_ringbuf_reserve(&ringbuf, sizeof(*p), 0); + if (!p) + return 1; + *p = 42; + q = bpf_rdonly_cast(0, bpf_core_type_id_kernel(struct sock)); + /* + * The load below is reached with PTR_TO_MEM | MEM_RINGBUF on one + * path and with PTR_TO_BTF_ID | PTR_UNTRUSTED on the other. The + * merged type has to keep the BPF_PROBE_MEM rewrite, otherwise + * the NULL deref taken at runtime panics the kernel instead of + * returning 0. + */ + asm volatile ( + "r7 = %[p];" + "if %[zero] != 0 goto +1;" + "r7 = %[q];" + "r8 = *(u64 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [q]"r"(q), + [zero]"r"(zero) + : "r7", "r8"); + bpf_ringbuf_discard(p, 0); + return v; +} + +SEC("socket") +__success +__log_level(2) +__msg("r8 = *(u32 *)(r7 +0){{.*}}R7=ptr_nameidata") +__msg("r8 = *(u32 *)(r7 +0){{.*}}R7=ringbuf_mem") +__retval(0) +int mixed_mem_btf_id_type(void *ctx) +{ + struct task_struct *task; + u32 *p, *q; + u64 v; + + p = bpf_ringbuf_reserve(&ringbuf, sizeof(*p), 0); + if (!p) + return 1; + *p = 42; + task = bpf_get_current_task_btf(); + /* + * A plain BTF pointer walk yields a bare PTR_TO_BTF_ID, and + * task->nameidata is NULL unless the task currently is in the + * middle of a path lookup. + */ + q = (u32 *)&task->nameidata->flags; + /* + * Same as above, except that the other path yields a bare + * PTR_TO_BTF_ID. Merging it with PTR_TO_MEM used to drop the + * BPF_PROBE_MEM rewrite the bare PTR_TO_BTF_ID would have + * gotten on its own. + */ + asm volatile ( + "r7 = %[p];" + "if %[zero] != 0 goto +1;" + "r7 = %[q];" + "r8 = *(u32 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [q]"r"(q), + [zero]"r"(zero) + : "r7", "r8"); + bpf_ringbuf_discard(p, 0); + return v; +} + +SEC("socket") +__success +__log_level(2) +__msg("r8 = *(u32 *)(r7 +0){{.*}}R7=ptr_nameidata") +__msg("r8 = *(u32 *)(r7 +0){{.*}}R7=rdonly_mem") +__retval(0) +int mixed_rdonly_mem_btf_id_type(void *ctx) +{ + struct task_struct *task; + struct bpf_dynptr dptr; + char buf[sizeof(u32)]; + u32 *p, *q; + u64 v; + + if (bpf_dynptr_from_mem(dynptr_data, sizeof(dynptr_data), 0, &dptr)) + return 1; + p = bpf_dynptr_slice(&dptr, 0, buf, sizeof(buf)); + if (!p) + return 1; + task = bpf_get_current_task_btf(); + q = (u32 *)&task->nameidata->flags; + /* + * Same as above, except that the PTR_TO_MEM side already carries + * MEM_RDONLY. Merging it with a bare PTR_TO_BTF_ID used to yield + * PTR_TO_MEM | MEM_RDONLY, which is not rewritten either since + * only its PTR_UNTRUSTED variant is. + */ + asm volatile ( + "r7 = %[p];" + "if %[zero] != 0 goto +1;" + "r7 = %[q];" + "r8 = *(u32 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [q]"r"(q), + [zero]"r"(zero) + : "r7", "r8"); + return v; +} + +SEC("socket") +__success +__log_level(2) +__msg("r8 = *(u64 *)(r7 +0){{.*}}R7=ringbuf_mem") +__msg("r8 = *(u64 *)(r7 +0){{.*}}R7=rdonly_untrusted_mem") +__retval(0) +int mixed_mem_mem_type(void *ctx) +{ + u64 *p, *q, v; + + p = bpf_ringbuf_reserve(&ringbuf, sizeof(*p), 0); + if (!p) + return 1; + *p = 42; + q = bpf_rdonly_cast(0, 0); + /* + * Both paths are PTR_TO_MEM based, so they used to not trip the + * type mismatch check and skipped the merge altogether, leaving + * the insn with the PTR_TO_MEM | MEM_RINGBUF recorded first and + * hence without the BPF_PROBE_MEM rewrite the other path needs. + */ + asm volatile ( + "r7 = %[q];" + "if %[zero] == 0 goto +1;" + "r7 = %[p];" + "r8 = *(u64 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [q]"r"(q), + [zero]"r"(zero) + : "r7", "r8"); + bpf_ringbuf_discard(p, 0); + return v; +} + +SEC("socket") +__failure +__msg("same insn cannot be used with different pointers") +int mixed_map_value_mem_type(void *ctx) +{ + u64 *p, *q, v; + u32 key = 0; + + p = bpf_map_lookup_elem(&array, &key); + if (!p) + return 1; + q = bpf_rdonly_cast(0, 0); + /* + * PTR_TO_MAP_VALUE is neither PTR_TO_MEM nor PTR_TO_BTF_ID based, + * so it cannot be merged into a type which keeps the BPF_PROBE_MEM + * rewrite the PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED of the other + * path needs. Both bases were mismatch ok, hence the load used to be + * accepted with the PTR_TO_MAP_VALUE recorded and the NULL deref on + * the second path panicked the kernel. + */ + asm volatile ( + "r7 = %[q];" + "if %[zero] == 0 goto +1;" + "r7 = %[p];" + "r8 = *(u64 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [q]"r"(q), + [zero]"r"(zero) + : "r7", "r8"); + return v; +} + +SEC("socket") +__failure +__msg("same insn cannot be used with different pointers") +int mixed_stack_mem_type(void *ctx) +{ + u64 *p = bpf_rdonly_cast(0, 0); + u64 s = 42, v; + + /* + * Same as above, but for a PTR_TO_STACK on the other path. + */ + asm volatile ( + "r7 = %[p];" + "if %[zero] == 0 goto +1;" + "r7 = %[s];" + "r8 = *(u64 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [s]"r"(&s), + [zero]"r"(zero) + : "r7", "r8"); + return v; +} + __attribute__((__aligned__(8))) u8 global[] = { 0x11, 0x22, 0x33, 0x44, @@ -226,4 +460,21 @@ int null_check(void *ctx) return 0; } +SEC("socket") +__success +__retval(1) +int ldx_is_ok_commuted_addr(void *ctx) +{ + int v, *p, *derived; + + v = 1; + p = bpf_rdonly_cast(&v, 0); + asm volatile ("%[dst] = 0;" + "%[dst] += %[src];" + : [dst]"=&r"(derived) + : [src]"r"(p) + : "memory"); + return *derived; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/modify_return.c b/tools/testing/selftests/bpf/progs/modify_return.c index 3376d4849f58..68fabd2efe8d 100644 --- a/tools/testing/selftests/bpf/progs/modify_return.c +++ b/tools/testing/selftests/bpf/progs/modify_return.c @@ -12,11 +12,14 @@ char _license[] SEC("license") = "GPL"; static int sequence = 0; __s32 input_retval = 0; +__u32 test_pid = 0; __u64 fentry_result = 0; SEC("fentry/bpf_modify_return_test") int BPF_PROG(fentry_test, int a, __u64 b) { + if (bpf_get_current_pid_tgid() >> 32 != test_pid) + return 0; sequence++; fentry_result = (sequence == 1); return 0; @@ -26,6 +29,8 @@ __u64 fmod_ret_result = 0; SEC("fmod_ret/bpf_modify_return_test") int BPF_PROG(fmod_ret_test, int a, int *b, int ret) { + if (bpf_get_current_pid_tgid() >> 32 != test_pid) + return ret; sequence++; /* This is the first fmod_ret program, the ret passed should be 0 */ fmod_ret_result = (sequence == 2 && ret == 0); @@ -36,6 +41,8 @@ __u64 fexit_result = 0; SEC("fexit/bpf_modify_return_test") int BPF_PROG(fexit_test, int a, __u64 b, int ret) { + if (bpf_get_current_pid_tgid() >> 32 != test_pid) + return 0; sequence++; /* If the input_reval is non-zero a successful modification should have * occurred. @@ -55,6 +62,8 @@ SEC("fentry/bpf_modify_return_test2") int BPF_PROG(fentry_test2, int a, int *b, short c, int d, void *e, char f, int g) { + if (bpf_get_current_pid_tgid() >> 32 != test_pid) + return 0; sequence2++; fentry_result2 = (sequence2 == 1); return 0; @@ -65,6 +74,8 @@ SEC("fmod_ret/bpf_modify_return_test2") int BPF_PROG(fmod_ret_test2, int a, int *b, short c, int d, void *e, char f, int g, int ret) { + if (bpf_get_current_pid_tgid() >> 32 != test_pid) + return ret; sequence2++; /* This is the first fmod_ret program, the ret passed should be 0 */ fmod_ret_result2 = (sequence2 == 2 && ret == 0); @@ -76,6 +87,8 @@ SEC("fexit/bpf_modify_return_test2") int BPF_PROG(fexit_test2, int a, int *b, short c, int d, void *e, char f, int g, int ret) { + if (bpf_get_current_pid_tgid() >> 32 != test_pid) + return 0; sequence2++; /* If the input_reval is non-zero a successful modification should have * occurred. diff --git a/tools/testing/selftests/bpf/progs/mptcpify.c b/tools/testing/selftests/bpf/progs/mptcpify.c index cbdc730c3a47..e3f8cb54dbe9 100644 --- a/tools/testing/selftests/bpf/progs/mptcpify.c +++ b/tools/testing/selftests/bpf/progs/mptcpify.c @@ -15,7 +15,7 @@ int BPF_PROG(mptcpify, int family, int type, int protocol) return protocol; if ((family == AF_INET || family == AF_INET6) && - type == SOCK_STREAM && + (type & SOCK_TYPE_MASK) == SOCK_STREAM && (!protocol || protocol == IPPROTO_TCP)) { return IPPROTO_MPTCP; } diff --git a/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c b/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c index f2b8eb2ff76f..3701f4ea58c7 100644 --- a/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c +++ b/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c @@ -33,6 +33,20 @@ struct { __type(value, struct elem); } array SEC(".maps"); +struct kernel_percpu_elem { + struct task_struct __percpu_kptr *task; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct kernel_percpu_elem); +} kernel_percpu_array SEC(".maps"); + +struct task_struct *bpf_task_from_pid(s32 pid) __ksym; +void bpf_task_release(struct task_struct *p) __ksym; + long ret; SEC("?fentry/bpf_fentry_test1") @@ -110,7 +124,7 @@ int BPF_PROG(test_array_map_3) } SEC("?fentry.s/bpf_fentry_test1") -__failure __msg("arg#0 expected for bpf_percpu_obj_drop_impl()") +__failure __msg("R1 expected for bpf_percpu_obj_drop()") int BPF_PROG(test_array_map_4) { struct val_t __percpu_kptr *p; @@ -124,7 +138,7 @@ int BPF_PROG(test_array_map_4) } SEC("?fentry.s/bpf_fentry_test1") -__failure __msg("arg#0 expected for bpf_obj_drop_impl()") +__failure __msg("R1 expected for bpf_obj_drop()") int BPF_PROG(test_array_map_5) { struct val_t *p; @@ -137,6 +151,51 @@ int BPF_PROG(test_array_map_5) return 0; } +SEC("?syscall") +__failure __msg("invalid kptr access, R2 type=trusted_ptr_ expected=ptr_task_struct") +int reject_kernel_ptr_into_percpu_kptr(void *ctx) +{ + struct kernel_percpu_elem *e; + struct task_struct *p, *old; + int index = 0; + + e = bpf_map_lookup_elem(&kernel_percpu_array, &index); + if (!e) + return 0; + + p = bpf_task_from_pid(1); + if (!p) + return 0; + + old = bpf_kptr_xchg(&e->task, p); + if (old) + bpf_task_release(old); + return 0; +} + +SEC("?fentry.s/bpf_fentry_test1") +__failure __msg("invalid kptr access, R2 type=ptr_ expected=ptr_val_t") +int BPF_PROG(reject_plain_alloc_into_percpu_kptr) +{ + struct val_t __percpu_kptr *old; + struct val_t *p; + struct elem *e; + int index = 0; + + e = bpf_map_lookup_elem(&array, &index); + if (!e) + return 0; + + p = bpf_obj_new(struct val_t); + if (!p) + return 0; + + old = bpf_kptr_xchg(&e->pc, p); + if (old) + bpf_percpu_obj_drop(old); + return 0; +} + SEC("?fentry.s/bpf_fentry_test1") __failure __msg("bpf_percpu_obj_new type ID argument must be of a struct of scalars") int BPF_PROG(test_array_map_6) diff --git a/tools/testing/selftests/bpf/progs/preempt_lock.c b/tools/testing/selftests/bpf/progs/preempt_lock.c index 7d04254e61f1..81c459435680 100644 --- a/tools/testing/selftests/bpf/progs/preempt_lock.c +++ b/tools/testing/selftests/bpf/progs/preempt_lock.c @@ -115,6 +115,58 @@ int preempt_sleepable_helper(void *ctx) return 0; } +SEC("?uprobe.s") +__failure __msg("sleepable helper bpf_get_stack#") +int preempt_sleepable_get_stack(struct pt_regs *ctx) +{ + struct bpf_stack_build_id stack; + + bpf_preempt_disable(); + bpf_get_stack(ctx, &stack, sizeof(stack), + BPF_F_USER_STACK | BPF_F_USER_BUILD_ID); + bpf_preempt_enable(); + return 0; +} + +SEC("?uprobe.s") +__failure __msg("sleepable helper bpf_get_task_stack#") +int preempt_sleepable_get_task_stack(void *ctx) +{ + struct bpf_stack_build_id stack; + struct task_struct *task; + + task = bpf_get_current_task_btf(); + bpf_preempt_disable(); + bpf_get_task_stack(task, &stack, sizeof(stack), + BPF_F_USER_STACK | BPF_F_USER_BUILD_ID); + bpf_preempt_enable(); + return 0; +} + +SEC("?uprobe.s") +__success +int sleepable_get_stack(struct pt_regs *ctx) +{ + struct bpf_stack_build_id stack; + + bpf_get_stack(ctx, &stack, sizeof(stack), + BPF_F_USER_STACK | BPF_F_USER_BUILD_ID); + return 0; +} + +SEC("?uprobe.s") +__success +int sleepable_get_task_stack(void *ctx) +{ + struct bpf_stack_build_id stack; + struct task_struct *task; + + task = bpf_get_current_task_btf(); + bpf_get_task_stack(task, &stack, sizeof(stack), + BPF_F_USER_STACK | BPF_F_USER_BUILD_ID); + return 0; +} + SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") __failure __msg("kernel func bpf_copy_from_user_str is sleepable within non-preemptible region") int preempt_sleepable_kfunc(void *ctx) @@ -177,7 +229,7 @@ global_subprog_calling_sleepable_global(int i) } SEC("?syscall") -__failure __msg("global functions that may sleep are not allowed in non-sleepable context") +__failure __msg("sleepable global function") int preempt_global_sleepable_helper_subprog(struct __sk_buff *ctx) { preempt_disable(); @@ -188,7 +240,7 @@ int preempt_global_sleepable_helper_subprog(struct __sk_buff *ctx) } SEC("?syscall") -__failure __msg("global functions that may sleep are not allowed in non-sleepable context") +__failure __msg("sleepable global function") int preempt_global_sleepable_kfunc_subprog(struct __sk_buff *ctx) { preempt_disable(); @@ -199,7 +251,7 @@ int preempt_global_sleepable_kfunc_subprog(struct __sk_buff *ctx) } SEC("?syscall") -__failure __msg("global functions that may sleep are not allowed in non-sleepable context") +__failure __msg("sleepable global function") int preempt_global_sleepable_subprog_indirect(struct __sk_buff *ctx) { preempt_disable(); diff --git a/tools/testing/selftests/bpf/progs/pyperf.h b/tools/testing/selftests/bpf/progs/pyperf.h index 86484f07e1d1..fd93a96e5901 100644 --- a/tools/testing/selftests/bpf/progs/pyperf.h +++ b/tools/testing/selftests/bpf/progs/pyperf.h @@ -85,9 +85,11 @@ static void *get_thread_state(void *tls_base, PidData *pidData) return thread_state; } -static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData, - FrameData *frame, Symbol *symbol) +__weak bool __get_frame_data(long frame_ptr_, PidData *pidData __arg_nonnull, + FrameData *frame __arg_nonnull, Symbol *symbol __arg_nonnull) { + void *frame_ptr = (void *)frame_ptr_; + // read data from PyFrameObject bpf_probe_read_user(&frame->f_back, sizeof(frame->f_back), @@ -119,6 +121,12 @@ static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData, return true; } +static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData, + FrameData *frame, Symbol *symbol) +{ + return __get_frame_data((long)frame_ptr, pidData, frame, symbol); +} + struct { __uint(type, BPF_MAP_TYPE_HASH); __uint(max_entries, 1); diff --git a/tools/testing/selftests/bpf/progs/raw_tp_null_fail.c b/tools/testing/selftests/bpf/progs/raw_tp_null_fail.c index 0d58114a4955..725d73c9ffe1 100644 --- a/tools/testing/selftests/bpf/progs/raw_tp_null_fail.c +++ b/tools/testing/selftests/bpf/progs/raw_tp_null_fail.c @@ -22,3 +22,56 @@ int test_raw_tp_null_sched_pi_setprio_arg_2(void *ctx) { asm volatile("r1 = *(u64 *)(r1 +8); r1 = *(u64 *)(r1 +0);" ::: __clobber_all); return 0; } + +/* Plain raw tracepoint arguments remain scalar values. */ +SEC("raw_tp/signal_generate") +__success +int test_raw_tp_signal_generate_info_scalar(void *ctx) +{ + asm volatile("r1 = *(u64 *)(r1 +8); if r1 != 1 goto +0;" ::: __clobber_all); + return 0; +} + +/* tp_btf programs may inspect the sentinel as a scalar value. */ +SEC("tp_btf/signal_generate") +__success +int test_tp_btf_signal_generate_info_scalar(void *ctx) +{ + asm volatile("r1 = *(u64 *)(r1 +8); if r1 != 1 goto +0;" ::: __clobber_all); + return 0; +} + +/* SEND_SIG_PRIV is non-NULL, so a NULL check cannot make info safe. */ +SEC("tp_btf/signal_generate") +__failure __msg("R1 invalid mem access 'scalar'") +int test_tp_btf_signal_generate_info_no_deref(void *ctx) +{ + asm volatile("r1 = *(u64 *)(r1 +8); if r1 == 0 goto +1; " + "r1 = *(u32 *)(r1 +0);" ::: __clobber_all); + return 0; +} + +SEC("tp_btf/signal_deliver") +__failure __msg("R1 invalid mem access 'scalar'") +int test_tp_btf_signal_deliver_info_no_deref(void *ctx) +{ + asm volatile("r1 = *(u64 *)(r1 +8); r1 = *(u32 *)(r1 +0);" ::: __clobber_all); + return 0; +} + +SEC("tp_btf/sched_process_wait") +__failure __msg("R1 invalid mem access 'trusted_ptr_or_null_'") +int test_raw_tp_null_sched_process_wait_arg_1(void *ctx) +{ + asm volatile("r1 = *(u64 *)(r1 +0); r1 = *(u32 *)(r1 +0);" ::: __clobber_all); + return 0; +} + +SEC("tp_btf/sched_process_wait") +__success +int test_raw_tp_null_sched_process_wait_arg_1_checked(void *ctx) +{ + asm volatile("r1 = *(u64 *)(r1 +0); if r1 == 0 goto +1; " + "r1 = *(u32 *)(r1 +0);" ::: __clobber_all); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/rbtree_fail.c b/tools/testing/selftests/bpf/progs/rbtree_fail.c index 70b7baf9304b..4504608196ab 100644 --- a/tools/testing/selftests/bpf/progs/rbtree_fail.c +++ b/tools/testing/selftests/bpf/progs/rbtree_fail.c @@ -16,6 +16,7 @@ struct node_data { private(A) struct bpf_spin_lock glock; private(A) struct bpf_rb_root groot __contains(node_data, node); private(A) struct bpf_rb_root groot2 __contains(node_data, node); +private(B) struct bpf_res_spin_lock res_glock; static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b) { @@ -134,7 +135,7 @@ unlock_err: } SEC("?tc") -__failure __msg("arg#1 expected pointer to allocated object") +__failure __msg("R2 expected pointer to allocated object") long rbtree_api_add_to_multiple_trees(void *ctx) { struct node_data *n; @@ -153,7 +154,7 @@ long rbtree_api_add_to_multiple_trees(void *ctx) } SEC("?tc") -__failure __msg("Possibly NULL pointer passed to trusted arg1") +__failure __msg("Possibly NULL pointer passed to trusted R2") long rbtree_api_use_unchecked_remove_retval(void *ctx) { struct bpf_rb_node *res; @@ -265,6 +266,53 @@ static bool less__bad_fn_call_first_unlock_after(struct bpf_rb_node *a, const st return node_a->key < node_b->key; } +static bool less__bad_res_spin_unlock(struct bpf_rb_node *a, const struct bpf_rb_node *b) +{ + bpf_res_spin_unlock(&res_glock); + return false; +} + +static __noinline void rbtree_cb_unlock_relock(void) +{ + bpf_spin_unlock(&glock); + bpf_spin_lock(&glock); +} + +static __noinline void rbtree_cb_nested_unlock(void) +{ + rbtree_cb_unlock_relock(); + asm volatile (""); +} + +static bool less__bad_subprog_unlock(struct bpf_rb_node *a, const struct bpf_rb_node *b) +{ + struct node_data *node_a; + struct node_data *node_b; + + node_a = container_of(a, struct node_data, node); + node_b = container_of(b, struct node_data, node); + rbtree_cb_nested_unlock(); + + return node_a->key < node_b->key; +} + +static __noinline void rbtree_cb_noop(void) +{ + asm volatile (""); +} + +static bool less__subprog_allowed(struct bpf_rb_node *a, const struct bpf_rb_node *b) +{ + struct node_data *node_a; + struct node_data *node_b; + + node_a = container_of(a, struct node_data, node); + node_b = container_of(b, struct node_data, node); + rbtree_cb_noop(); + + return node_a->key < node_b->key; +} + static __always_inline long add_with_cb(bool (cb)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) { @@ -281,7 +329,7 @@ long add_with_cb(bool (cb)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) } SEC("?tc") -__failure __msg("arg#1 expected pointer to allocated object") +__failure __msg("R2 expected pointer to allocated object") long rbtree_api_add_bad_cb_bad_fn_call_add(void *ctx) { return add_with_cb(less__bad_fn_call_add); @@ -301,4 +349,40 @@ long rbtree_api_add_bad_cb_bad_fn_call_first_unlock_after(void *ctx) return add_with_cb(less__bad_fn_call_first_unlock_after); } +SEC("?tc") +__failure __msg("can't res_spin_{lock,unlock} in rbtree cb") +long rbtree_api_add_bad_cb_res_spin_unlock(void *ctx) +{ + struct node_data *n; + + n = bpf_obj_new(typeof(*n)); + if (!n) + return 1; + + bpf_spin_lock(&glock); + if (bpf_res_spin_lock(&res_glock)) { + bpf_spin_unlock(&glock); + bpf_obj_drop(n); + return 1; + } + bpf_rbtree_add(&groot, &n->node, less__bad_res_spin_unlock); + bpf_res_spin_unlock(&res_glock); + bpf_spin_unlock(&glock); + return 0; +} + +SEC("?tc") +__failure __msg("can't spin_{lock,unlock} in rbtree cb") +long rbtree_api_add_bad_cb_subprog_unlock(void *ctx) +{ + return add_with_cb(less__bad_subprog_unlock); +} + +SEC("?tc") +__success +long rbtree_api_add_cb_subprog_allowed(void *ctx) +{ + return add_with_cb(less__subprog_allowed); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/rbtree_search_kptr.c b/tools/testing/selftests/bpf/progs/rbtree_search_kptr.c new file mode 100644 index 000000000000..610aae45e2dc --- /dev/null +++ b/tools/testing/selftests/bpf/progs/rbtree_search_kptr.c @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 KylinSoft Corporation. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_experimental.h" + +#define NR_NODES 16 + +struct node_data { + int data; +}; + +struct tree_node { + struct bpf_rb_node node; + u64 key; + struct node_data __kptr * node_data; +}; + +struct tree_node_ref { + struct bpf_refcount ref; + struct bpf_rb_node node; + u64 key; + struct node_data __kptr * node_data; +}; + +#define private(name) SEC(".data." #name) __hidden __aligned(8) + +private(A) struct bpf_rb_root root __contains(tree_node, node); +private(A) struct bpf_spin_lock lock; + +private(B) struct bpf_rb_root root_r __contains(tree_node_ref, node); +private(B) struct bpf_spin_lock lock_r; + +static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b) +{ + struct tree_node *node_a, *node_b; + + node_a = container_of(a, struct tree_node, node); + node_b = container_of(b, struct tree_node, node); + + return node_a->key < node_b->key; +} + +SEC("syscall") +__retval(0) +long rbtree_search_kptr(void *ctx) +{ + struct tree_node *tnode; + struct bpf_rb_node *rb_n; + struct node_data __kptr * node_data; + int lookup_key = NR_NODES / 2; + int lookup_data = NR_NODES / 2; + int i, data, ret = 0; + + for (i = 0; i < NR_NODES && can_loop; i++) { + tnode = bpf_obj_new(typeof(*tnode)); + if (!tnode) + return __LINE__; + + node_data = bpf_obj_new(typeof(*node_data)); + if (!node_data) { + bpf_obj_drop(tnode); + return __LINE__; + } + + tnode->key = i; + node_data->data = i; + + node_data = bpf_kptr_xchg(&tnode->node_data, node_data); + if (node_data) + bpf_obj_drop(node_data); + + bpf_spin_lock(&lock); + bpf_rbtree_add(&root, &tnode->node, less); + bpf_spin_unlock(&lock); + } + + bpf_spin_lock(&lock); + rb_n = bpf_rbtree_root(&root); + while (rb_n && can_loop) { + tnode = container_of(rb_n, struct tree_node, node); + node_data = bpf_kptr_xchg(&tnode->node_data, NULL); + if (!node_data) { + ret = __LINE__; + goto fail; + } + + data = node_data->data; + node_data = bpf_kptr_xchg(&tnode->node_data, node_data); + if (node_data) { + bpf_spin_unlock(&lock); + bpf_obj_drop(node_data); + return __LINE__; + } + + if (lookup_key == tnode->key) { + if (data == lookup_data) + break; + + ret = __LINE__; + goto fail; + } + + if (lookup_key < tnode->key) + rb_n = bpf_rbtree_left(&root, rb_n); + else + rb_n = bpf_rbtree_right(&root, rb_n); + } + bpf_spin_unlock(&lock); + + while (can_loop) { + bpf_spin_lock(&lock); + rb_n = bpf_rbtree_first(&root); + if (!rb_n) { + bpf_spin_unlock(&lock); + return 0; + } + + rb_n = bpf_rbtree_remove(&root, rb_n); + if (!rb_n) { + ret = __LINE__; + goto fail; + } + bpf_spin_unlock(&lock); + + tnode = container_of(rb_n, struct tree_node, node); + + node_data = bpf_kptr_xchg(&tnode->node_data, NULL); + if (node_data) + bpf_obj_drop(node_data); + + bpf_obj_drop(tnode); + } + + return 0; +fail: + bpf_spin_unlock(&lock); + return ret; +} + +static bool less_r(struct bpf_rb_node *a, const struct bpf_rb_node *b) +{ + struct tree_node_ref *node_a, *node_b; + + node_a = container_of(a, struct tree_node_ref, node); + node_b = container_of(b, struct tree_node_ref, node); + + return node_a->key < node_b->key; +} + +SEC("syscall") +__retval(0) +long rbtree_search_kptr_ref(void *ctx) +{ + struct tree_node_ref *tnode_r, *tnode_m; + struct bpf_rb_node *rb_n; + struct node_data __kptr * node_data; + int lookup_key = NR_NODES / 2; + int lookup_data = NR_NODES / 2; + int i, data, ret = 0; + + for (i = 0; i < NR_NODES && can_loop; i++) { + tnode_r = bpf_obj_new(typeof(*tnode_r)); + if (!tnode_r) + return __LINE__; + + node_data = bpf_obj_new(typeof(*node_data)); + if (!node_data) { + bpf_obj_drop(tnode_r); + return __LINE__; + } + + tnode_r->key = i; + node_data->data = i; + + node_data = bpf_kptr_xchg(&tnode_r->node_data, node_data); + if (node_data) + bpf_obj_drop(node_data); + + /* Unused reference */ + tnode_m = bpf_refcount_acquire(tnode_r); + if (!tnode_m) + return __LINE__; + + bpf_spin_lock(&lock_r); + bpf_rbtree_add(&root_r, &tnode_r->node, less_r); + bpf_spin_unlock(&lock_r); + + bpf_obj_drop(tnode_m); + } + + bpf_spin_lock(&lock_r); + rb_n = bpf_rbtree_root(&root_r); + while (rb_n && can_loop) { + tnode_r = container_of(rb_n, struct tree_node_ref, node); + node_data = bpf_kptr_xchg(&tnode_r->node_data, NULL); + if (!node_data) { + ret = __LINE__; + goto fail; + } + + data = node_data->data; + node_data = bpf_kptr_xchg(&tnode_r->node_data, node_data); + if (node_data) { + bpf_spin_unlock(&lock_r); + bpf_obj_drop(node_data); + return __LINE__; + } + + if (lookup_key == tnode_r->key) { + if (data == lookup_data) + break; + + ret = __LINE__; + goto fail; + } + + if (lookup_key < tnode_r->key) + rb_n = bpf_rbtree_left(&root_r, rb_n); + else + rb_n = bpf_rbtree_right(&root_r, rb_n); + } + bpf_spin_unlock(&lock_r); + + while (can_loop) { + bpf_spin_lock(&lock_r); + rb_n = bpf_rbtree_first(&root_r); + if (!rb_n) { + bpf_spin_unlock(&lock_r); + return 0; + } + + rb_n = bpf_rbtree_remove(&root_r, rb_n); + if (!rb_n) { + ret = __LINE__; + goto fail; + } + bpf_spin_unlock(&lock_r); + + tnode_r = container_of(rb_n, struct tree_node_ref, node); + + node_data = bpf_kptr_xchg(&tnode_r->node_data, NULL); + if (node_data) + bpf_obj_drop(node_data); + + bpf_obj_drop(tnode_r); + } + + return 0; +fail: + bpf_spin_unlock(&lock_r); + return ret; +} + +SEC("syscall") +__failure __msg("R1 type=scalar expected=map_value, ptr_, ptr_") +long non_own_ref_kptr_xchg_no_lock(void *ctx) +{ + struct tree_node *tnode; + struct bpf_rb_node *rb_n; + struct node_data __kptr * node_data; + int data; + + bpf_spin_lock(&lock); + rb_n = bpf_rbtree_first(&root); + if (!rb_n) { + bpf_spin_unlock(&lock); + return __LINE__; + } + bpf_spin_unlock(&lock); + + tnode = container_of(rb_n, struct tree_node, node); + node_data = bpf_kptr_xchg(&tnode->node_data, NULL); + if (!node_data) + return __LINE__; + + data = node_data->data; + if (data < 0) + return __LINE__; + + node_data = bpf_kptr_xchg(&tnode->node_data, node_data); + if (node_data) + return __LINE__; + + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/rcu_read_lock.c b/tools/testing/selftests/bpf/progs/rcu_read_lock.c index b4e073168fb1..cdb255addbc3 100644 --- a/tools/testing/selftests/bpf/progs/rcu_read_lock.c +++ b/tools/testing/selftests/bpf/progs/rcu_read_lock.c @@ -549,3 +549,79 @@ int rcu_read_lock_sleepable_global_subprog_indirect(void *ctx) bpf_rcu_read_unlock(); return 0; } + +struct rcu_node_data { + long key; + struct bpf_rb_node node; +}; + +struct rcu_node_stash { + struct rcu_node_data __kptr *node; +}; + +/* + * Necessary so that LLVM emits BTF for rcu_node_data rather than just a + * fwd reference to it, same as in progs/local_kptr_stash.c. + */ +struct rcu_node_data *just_here_because_btf_bug; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct rcu_node_stash); +} node_stash SEC(".maps"); + +long non_own_ref_key; + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +int non_own_ref_untrusted_ld(void *ctx) +{ + struct rcu_node_stash *stash; + struct rcu_node_data *node; + int key = 0; + + stash = bpf_map_lookup_elem(&node_stash, &key); + if (!stash) + return 0; + bpf_rcu_read_lock(); + node = stash->node; + if (!node) { + bpf_rcu_read_unlock(); + return 0; + } + bpf_rcu_read_unlock(); + /* + * The unlock leaves node as PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED, + * and the load below has to get the BPF_PROBE_MEM rewrite for it, + * otherwise a bad address panics the kernel. + */ + non_own_ref_key = node->key; + return 0; +} + +long rcu_untrusted_wq_flags; + +SEC("?tp_btf/tcp_probe") +int BPF_PROG(rcu_untrusted_union_ld, struct sock *sk) +{ + struct socket_wq *wq; + + /* + * sk_wq sits in a two member union, so btf_struct_walk() marks the + * pointer PTR_UNTRUSTED, and the __rcu tag on the member adds MEM_RCU + * on top of it. struct sock is not on the __safe_rcu_or_null allow + * list, hence the two stay combined and the load below has to get the + * BPF_PROBE_MEM rewrite for PTR_TO_BTF_ID | PTR_UNTRUSTED | MEM_RCU, + * otherwise a bad address panics the kernel. + * + * The __rcu tag only reaches BTF on a clang built kernel, that is, one + * with CONFIG_PAHOLE_HAS_BTF_TAG. On a gcc built kernel the walk yields + * a plain untrusted pointer, which is rewritten either way. + */ + wq = sk->sk_wq; + if (!wq) + return 0; + rcu_untrusted_wq_flags = wq->flags; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr.c b/tools/testing/selftests/bpf/progs/refcounted_kptr.c index 1aca85d86aeb..cae00f7b0a24 100644 --- a/tools/testing/selftests/bpf/progs/refcounted_kptr.c +++ b/tools/testing/selftests/bpf/progs/refcounted_kptr.c @@ -23,6 +23,15 @@ struct map_value { struct node_data __kptr *node; }; +struct node_refcount_only { + long key; + struct bpf_refcount refcount; +}; + +struct map_value_refcount_only { + struct node_refcount_only __kptr *node; +}; + struct { __uint(type, BPF_MAP_TYPE_ARRAY); __type(key, int); @@ -30,6 +39,13 @@ struct { __uint(max_entries, 2); } stashed_nodes SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct map_value_refcount_only); + __uint(max_entries, 1); +} stashed_refcount_only SEC(".maps"); + struct node_acquire { long key; long data; @@ -368,6 +384,427 @@ INSERT_STASH_READ(true, "insert_stash_read: remove from tree"); INSERT_STASH_READ(false, "insert_stash_read: don't remove from tree"); SEC("tc") +__description("list_empty_test: list empty before add, non-empty after add") +__success __retval(0) +int list_empty_test(void *ctx) +{ + struct node_data *node_new; + + bpf_spin_lock(&lock); + if (!bpf_list_empty(&head)) { + bpf_spin_unlock(&lock); + return -1; + } + bpf_spin_unlock(&lock); + + node_new = bpf_obj_new(typeof(*node_new)); + if (!node_new) + return -2; + + bpf_spin_lock(&lock); + bpf_list_push_front(&head, &node_new->l); + + if (bpf_list_empty(&head)) { + bpf_spin_unlock(&lock); + return -3; + } + bpf_spin_unlock(&lock); + return 0; +} + +static struct node_data *__add_in_list(struct bpf_list_head *head, + struct bpf_spin_lock *lock) +{ + struct node_data *node_new, *node_ref; + + node_new = bpf_obj_new(typeof(*node_new)); + if (!node_new) + return NULL; + + node_ref = bpf_refcount_acquire(node_new); + + bpf_spin_lock(lock); + bpf_list_push_front(head, &node_new->l); + bpf_spin_unlock(lock); + return node_ref; +} + +SEC("tc") +__description("list_is_edge_test1: is_first on first node, is_last on last node") +__success __retval(0) +int list_is_edge_test1(void *ctx) +{ + struct node_data *node_first, *node_last; + int err = 0; + + node_last = __add_in_list(&head, &lock); + if (!node_last) + return -1; + + node_first = __add_in_list(&head, &lock); + if (!node_first) { + bpf_obj_drop(node_last); + return -2; + } + + bpf_spin_lock(&lock); + if (!bpf_list_is_first(&head, &node_first->l)) { + err = -3; + goto fail; + } + if (!bpf_list_is_last(&head, &node_last->l)) + err = -4; + +fail: + bpf_spin_unlock(&lock); + bpf_obj_drop(node_first); + bpf_obj_drop(node_last); + return err; +} + +SEC("tc") +__description("list_is_edge_test2: accept list_front/list_back return value") +__success __retval(0) +int list_is_edge_test2(void *ctx) +{ + struct bpf_list_node *front, *back; + struct node_data *a, *b; + long err = 0; + + a = __add_in_list(&head, &lock); + if (!a) + return -1; + + b = __add_in_list(&head, &lock); + if (!b) { + bpf_obj_drop(a); + return -2; + } + + bpf_spin_lock(&lock); + front = bpf_list_front(&head); + back = bpf_list_back(&head); + if (!front || !back) { + err = -3; + goto out_unlock; + } + + if (!bpf_list_is_first(&head, front) || bpf_list_is_last(&head, front)) { + err = -4; + goto out_unlock; + } + + if (!bpf_list_is_last(&head, back) || bpf_list_is_first(&head, back)) { + err = -5; + goto out_unlock; + } + +out_unlock: + bpf_spin_unlock(&lock); + bpf_obj_drop(a); + bpf_obj_drop(b); + return err; +} + +SEC("tc") +__description("list_is_edge_test3: single node is both first and last") +__success __retval(0) +int list_is_edge_test3(void *ctx) +{ + struct node_data *tmp; + struct bpf_list_node *node; + long err = 0; + + tmp = __add_in_list(&head, &lock); + if (!tmp) + return -1; + + bpf_spin_lock(&lock); + node = bpf_list_front(&head); + if (!node) { + bpf_spin_unlock(&lock); + bpf_obj_drop(tmp); + return -2; + } + + if (!bpf_list_is_first(&head, node) || !bpf_list_is_last(&head, node)) + err = -3; + bpf_spin_unlock(&lock); + + bpf_obj_drop(tmp); + return err; +} + +SEC("tc") +__description("list_del_test1: del returns removed nodes") +__success __retval(0) +int list_del_test1(void *ctx) +{ + struct node_data *node_first, *node_last; + struct bpf_list_node *bpf_node_first, *bpf_node_last; + int err = 0; + + node_last = __add_in_list(&head, &lock); + if (!node_last) + return -1; + + node_first = __add_in_list(&head, &lock); + if (!node_first) { + bpf_obj_drop(node_last); + return -2; + } + + bpf_spin_lock(&lock); + bpf_node_last = bpf_list_del(&head, &node_last->l); + bpf_node_first = bpf_list_del(&head, &node_first->l); + bpf_spin_unlock(&lock); + + if (bpf_node_first) + bpf_obj_drop(container_of(bpf_node_first, struct node_data, l)); + else + err = -3; + + if (bpf_node_last) + bpf_obj_drop(container_of(bpf_node_last, struct node_data, l)); + else + err = -4; + + bpf_obj_drop(node_first); + bpf_obj_drop(node_last); + return err; +} + +SEC("tc") +__description("list_del_test2: remove an arbitrary node from the list") +__success __retval(0) +int list_del_test2(void *ctx) +{ + struct bpf_rb_node *rb; + struct bpf_list_node *l; + struct node_data *n; + long err; + + err = __insert_in_tree_and_list(&head, &root, &lock); + if (err) + return err; + + bpf_spin_lock(&lock); + rb = bpf_rbtree_first(&root); + if (!rb) { + bpf_spin_unlock(&lock); + return -4; + } + + rb = bpf_rbtree_remove(&root, rb); + if (!rb) { + bpf_spin_unlock(&lock); + return -5; + } + + n = container_of(rb, struct node_data, r); + l = bpf_list_del(&head, &n->l); + bpf_spin_unlock(&lock); + bpf_obj_drop(n); + if (!l) + return -6; + + bpf_obj_drop(container_of(l, struct node_data, l)); + return 0; +} + +SEC("tc") +__description("list_del_test3: list_del accepts list_front return value as node") +__success __retval(0) +int list_del_test3(void *ctx) +{ + struct node_data *tmp; + struct bpf_list_node *bpf_node, *l; + long err = 0; + + tmp = __add_in_list(&head, &lock); + if (!tmp) + return -1; + + bpf_spin_lock(&lock); + bpf_node = bpf_list_front(&head); + if (!bpf_node) { + bpf_spin_unlock(&lock); + err = -2; + goto fail; + } + + l = bpf_list_del(&head, bpf_node); + bpf_spin_unlock(&lock); + if (!l) { + err = -3; + goto fail; + } + + bpf_obj_drop(container_of(l, struct node_data, l)); + bpf_obj_drop(tmp); + return 0; + +fail: + bpf_obj_drop(tmp); + return err; +} + +SEC("tc") +__description("list_add_test1: insert new node after prev") +__success __retval(0) +int list_add_test1(void *ctx) +{ + struct node_data *node_first; + struct node_data *new_node; + long err = 0; + + node_first = __add_in_list(&head, &lock); + if (!node_first) + return -1; + + new_node = bpf_obj_new(typeof(*new_node)); + if (!new_node) { + err = -2; + goto fail; + } + + bpf_spin_lock(&lock); + err = bpf_list_add(&head, &new_node->l, &node_first->l); + bpf_spin_unlock(&lock); + if (err) { + err = -3; + goto fail; + } + +fail: + bpf_obj_drop(node_first); + return err; +} + +SEC("tc") +__description("list_add_test2: list_add accepts list_front return value as prev") +__success __retval(0) +int list_add_test2(void *ctx) +{ + struct node_data *new_node, *tmp; + struct bpf_list_node *bpf_node; + long err = 0; + + tmp = __add_in_list(&head, &lock); + if (!tmp) + return -1; + + new_node = bpf_obj_new(typeof(*new_node)); + if (!new_node) { + err = -2; + goto fail; + } + + bpf_spin_lock(&lock); + bpf_node = bpf_list_front(&head); + if (!bpf_node) { + bpf_spin_unlock(&lock); + bpf_obj_drop(new_node); + err = -3; + goto fail; + } + + err = bpf_list_add(&head, &new_node->l, bpf_node); + bpf_spin_unlock(&lock); + if (err) { + err = -4; + goto fail; + } + +fail: + bpf_obj_drop(tmp); + return err; +} + +struct uninit_head_val { + struct bpf_spin_lock lock; + struct bpf_list_head head __contains(node_data, l); +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct uninit_head_val); + __uint(max_entries, 1); +} uninit_head_map SEC(".maps"); + +SEC("tc") +__description("list_push_back_uninit_head: push_back on 0-initialized list head") +__success __retval(0) +int list_push_back_uninit_head(void *ctx) +{ + struct uninit_head_val *st; + struct node_data *node; + int ret = -1, key = 0; + + st = bpf_map_lookup_elem(&uninit_head_map, &key); + if (!st) + return -1; + + node = bpf_obj_new(typeof(*node)); + if (!node) + return -1; + + bpf_spin_lock(&st->lock); + ret = bpf_list_push_back(&st->head, &node->l); + bpf_spin_unlock(&st->lock); + + return ret; +} + +SEC("?tc") +__failure __msg("bpf_spin_lock at off=32 must be held for bpf_list_head") +long list_del_without_lock_fail(void *ctx) +{ + struct node_data *n; + struct bpf_list_node *l; + + n = bpf_obj_new(typeof(*n)); + if (!n) + return -1; + + /* Error case: delete list node without holding lock */ + l = bpf_list_del(&head, &n->l); + bpf_obj_drop(n); + if (!l) + return -2; + bpf_obj_drop(container_of(l, struct node_data, l)); + + return 0; +} + +SEC("?tc") +__failure __msg("bpf_spin_lock at off=32 must be held for bpf_list_head") +long list_add_without_lock_fail(void *ctx) +{ + struct node_data *n, *prev; + long err; + + n = bpf_obj_new(typeof(*n)); + if (!n) + return -1; + + prev = bpf_obj_new(typeof(*prev)); + if (!prev) { + bpf_obj_drop(n); + return -1; + } + + /* Error case: add list node without holding lock */ + err = bpf_list_add(&head, &n->l, &prev->l); + bpf_obj_drop(prev); + if (err) + return -2; + + return 0; +} + +SEC("tc") __success long rbtree_refcounted_node_ref_escapes(void *ctx) { @@ -411,6 +848,51 @@ long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx) return 0; } +SEC("tc") +__success +long refcount_acquire_owning_input_no_null_check(void *ctx) +{ + struct node_refcount_only *n, *m; + + n = bpf_obj_new(typeof(*n)); + if (!n) + return 1; + + m = bpf_refcount_acquire(n); + bpf_obj_drop(m); + bpf_obj_drop(n); + + return 0; +} + +SEC("?syscall") +__success +long refcount_acquire_rcu_map_kptr_null_checked(void *ctx) +{ + struct map_value_refcount_only *mapval; + struct node_refcount_only *n, *m; + int idx = 0; + + mapval = bpf_map_lookup_elem(&stashed_refcount_only, &idx); + if (!mapval) + return 1; + + bpf_rcu_read_lock(); + n = mapval->node; + if (!n) { + bpf_rcu_read_unlock(); + return 2; + } + m = bpf_refcount_acquire(n); + bpf_rcu_read_unlock(); + + if (!m) + return 3; + bpf_obj_drop(m); + + return 0; +} + static long __stash_map_empty_xchg(struct node_data *n, int idx) { struct map_value *mapval = bpf_map_lookup_elem(&stashed_nodes, &idx); @@ -500,7 +982,7 @@ long rbtree_wrong_owner_remove_fail_a2(void *ctx) return 0; } -SEC("?fentry.s/bpf_testmod_test_read") +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") __success int BPF_PROG(rbtree_sleepable_rcu, struct file *file, struct kobject *kobj, @@ -534,7 +1016,7 @@ err_out: return 0; } -SEC("?fentry.s/bpf_testmod_test_read") +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") __success int BPF_PROG(rbtree_sleepable_rcu_no_explicit_rcu_lock, struct file *file, struct kobject *kobj, @@ -615,13 +1097,31 @@ int percpu_hash_refcount_leak(void *ctx) struct map_value *v; int key = 0; - v = bpf_map_lookup_elem(&percpu_hash, &key); + v = bpf_map_lookup_percpu_elem(&percpu_hash, &key, 0); if (!v) return 0; return __insert_in_list(&head, &lock, &v->node); } +SEC("syscall") +int clear_percpu_hash_kptr(void *ctx) +{ + struct node_data *n; + struct map_value *v; + int key = 0; + + v = bpf_map_lookup_percpu_elem(&percpu_hash, &key, 0); + if (!v) + return 0; + + n = bpf_kptr_xchg(&v->node, NULL); + if (!n) + return 0; + bpf_obj_drop(n); + return probe_read_refcount(); +} + SEC("tc") int check_percpu_hash_refcount(void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c index 836c8ab7b908..338e43822ffe 100644 --- a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c @@ -13,12 +13,56 @@ struct node_acquire { struct bpf_refcount refcount; }; +struct node_refcounted { + long key; + struct bpf_list_node list; + struct bpf_refcount refcount; +}; + +struct node_refcount_only { + long key; + struct bpf_refcount refcount; +}; + +struct map_value_refcount_only { + struct node_refcount_only __kptr *node; +}; + +struct rcu_graph_node { + struct bpf_rb_node node; + long data; +}; + +struct rcu_graph_node *just_here_because_btf_bug; + +struct map_value_rcu_graph { + struct rcu_graph_node __kptr *node; +}; + extern void bpf_rcu_read_lock(void) __ksym; extern void bpf_rcu_read_unlock(void) __ksym; #define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) private(A) struct bpf_spin_lock glock; private(A) struct bpf_rb_root groot __contains(node_acquire, node); +private(B) struct bpf_spin_lock lock; +private(B) struct bpf_list_head head __contains(node_refcounted, list); +private(C) struct bpf_spin_lock graph_lock; +private(C) struct bpf_rb_root graph_root __contains(rcu_graph_node, node); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct map_value_refcount_only); + __uint(max_entries, 1); +} stashed_refcount_only SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct map_value_rcu_graph); + __uint(max_entries, 1); +} stashed_rcu_graph SEC(".maps"); static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b) { @@ -54,7 +98,8 @@ long rbtree_refcounted_node_ref_escapes(void *ctx) } SEC("?tc") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") +__msg("requires a non-NULL value of type (void *)") long refcount_acquire_maybe_null(void *ctx) { struct node_acquire *n, *m; @@ -73,6 +118,128 @@ long refcount_acquire_maybe_null(void *ctx) } SEC("?tc") +__failure __msg("R1 is neither owning or non-owning ref") +__msg("expects a pointer to a BPF-managed refcounted object, but R1 is a context pointer") +long refcount_acquire_non_object(void *ctx) +{ + return bpf_refcount_acquire(ctx) != NULL; +} + +SEC("?syscall") +__failure __msg("Possibly NULL pointer passed to trusted R1") +long refcount_acquire_rcu_map_kptr_unchecked_drop(void *ctx) +{ + struct map_value_refcount_only *mapval; + struct node_refcount_only *tmp, *n, *m; + int idx = 0; + + /* Force Clang to emit complete BTF for struct node_refcount_only. */ + tmp = bpf_obj_new(typeof(*tmp)); + if (!tmp) + return 3; + bpf_obj_drop(tmp); + + mapval = bpf_map_lookup_elem(&stashed_refcount_only, &idx); + if (!mapval) + return 1; + + bpf_rcu_read_lock(); + n = mapval->node; + if (!n) { + bpf_rcu_read_unlock(); + return 2; + } + m = bpf_refcount_acquire(n); + bpf_rcu_read_unlock(); + + bpf_obj_drop(m); + + return 0; +} + +SEC("?syscall") +__failure +__msg("bpf_rbtree_remove can only take non-owning or refcounted " + "bpf_rb_node pointer") +long rbtree_remove_after_rcu_unlock(void *ctx) +{ + struct map_value_rcu_graph *mapval; + struct bpf_rb_node *rb_node; + struct rcu_graph_node *node; + int idx = 0; + + mapval = bpf_map_lookup_elem(&stashed_rcu_graph, &idx); + if (!mapval) + return 0; + + bpf_rcu_read_lock(); + node = mapval->node; + if (!node) { + bpf_rcu_read_unlock(); + return 0; + } + bpf_rcu_read_unlock(); + + bpf_spin_lock(&graph_lock); + rb_node = bpf_rbtree_remove(&graph_root, &node->node); + bpf_spin_unlock(&graph_lock); + if (rb_node) + bpf_obj_drop(container_of(rb_node, struct rcu_graph_node, node)); + + return 0; +} + +SEC("?syscall") +__failure __msg("R1 is neither owning or non-owning ref") +long refcount_acquire_after_rcu_unlock(void *ctx) +{ + struct map_value_refcount_only *mapval; + struct node_refcount_only *node, *ref; + int idx = 0; + + mapval = bpf_map_lookup_elem(&stashed_refcount_only, &idx); + if (!mapval) + return 0; + + bpf_rcu_read_lock(); + node = mapval->node; + if (!node) { + bpf_rcu_read_unlock(); + return 0; + } + bpf_rcu_read_unlock(); + + ref = bpf_refcount_acquire(node); + if (ref) + bpf_obj_drop(ref); + + return 0; +} + +SEC("?syscall") +__failure __msg("invalid mem access 'scalar'") +long graph_kptr_after_spin_unlock(void *ctx) +{ + struct map_value_rcu_graph *mapval; + struct rcu_graph_node *node; + int idx = 0; + + mapval = bpf_map_lookup_elem(&stashed_rcu_graph, &idx); + if (!mapval) + return 0; + + bpf_spin_lock(&graph_lock); + node = mapval->node; + if (!node) { + bpf_spin_unlock(&graph_lock); + return 0; + } + bpf_spin_unlock(&graph_lock); + + return node->data; +} + +SEC("?tc") __failure __msg("Unreleased reference id=3 alloc_insn={{[0-9]+}}") long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx) { @@ -93,7 +260,33 @@ long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx) return 0; } -SEC("?fentry.s/bpf_testmod_test_read") +SEC("?tc") +__failure __msg("dereference of modified ptr_ ptr R1") +long refcount_acquire_list_node_offset(void *ctx) +{ + struct node_refcounted *node, *base, *ref; + struct bpf_list_node *list_node; + + node = bpf_obj_new(typeof(*node)); + if (!node) + return 1; + + bpf_spin_lock(&lock); + bpf_list_push_front(&head, &node->list); + list_node = bpf_list_pop_front(&head); + bpf_spin_unlock(&lock); + if (!list_node) + return 2; + + base = container_of(list_node, struct node_refcounted, list); + ref = bpf_refcount_acquire(list_node); + if (ref) + bpf_obj_drop(ref); + bpf_obj_drop(base); + return 0; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") __failure __msg("function calls are not allowed while holding a lock") int BPF_PROG(rbtree_fail_sleepable_lock_across_rcu, struct file *file, struct kobject *kobj, diff --git a/tools/testing/selftests/bpf/progs/rhash.c b/tools/testing/selftests/bpf/progs/rhash.c new file mode 100644 index 000000000000..aea4de8dc781 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/rhash.c @@ -0,0 +1,360 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <stdbool.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" + +#define ENOENT 2 +#define EEXIST 17 + +char _license[] SEC("license") = "GPL"; + +int err; + +struct elem { + char arr[128]; + int val; +}; + +struct special_elem { + struct task_struct __kptr *task; + int val; +}; + +struct { + __uint(type, BPF_MAP_TYPE_RHASH); + __uint(map_flags, BPF_F_NO_PREALLOC); + __uint(max_entries, 128); + __type(key, int); + __type(value, struct elem); +} rhmap SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_RHASH); + __uint(map_flags, BPF_F_NO_PREALLOC); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct special_elem); +} special_fields SEC(".maps"); + +extern struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; +extern void bpf_task_release(struct task_struct *p) __ksym; + +SEC("syscall") +int test_rhash_lookup_update(void *ctx) +{ + int key = 5; + struct elem empty = {.val = 3, .arr = {0}}; + struct elem *e; + + err = 1; + e = bpf_map_lookup_elem(&rhmap, &key); + if (e) + return 1; + + err = bpf_map_update_elem(&rhmap, &key, &empty, BPF_NOEXIST); + if (err) + return 1; + + e = bpf_map_lookup_elem(&rhmap, &key); + if (!e || e->val != empty.val) { + err = 2; + return 2; + } + + err = 0; + return 0; +} + +SEC("syscall") +int test_rhash_update_delete(void *ctx) +{ + int key = 6; + struct elem empty = {.val = 4, .arr = {0}}; + struct elem *e; + + err = 1; + e = bpf_map_lookup_elem(&rhmap, &key); + if (e) + return 1; + + err = bpf_map_update_elem(&rhmap, &key, &empty, BPF_NOEXIST); + if (err) + return 2; + + err = bpf_map_delete_elem(&rhmap, &key); + if (err) + return 3; + + e = bpf_map_lookup_elem(&rhmap, &key); + if (e) { + err = 4; + return 4; + } + + err = 0; + return 0; +} + +SEC("syscall") +int test_rhash_update_elements(void *ctx) +{ + int key = 0; + struct elem empty = {.val = 4, .arr = {0}}; + struct elem *e; + int i; + + err = 1; + + for (i = 0; i < 128; ++i) { + key = i; + e = bpf_map_lookup_elem(&rhmap, &key); + if (e) + return 1; + + empty.val = key; + err = bpf_map_update_elem(&rhmap, &key, &empty, BPF_NOEXIST); + if (err) + return 2; + + e = bpf_map_lookup_elem(&rhmap, &key); + if (!e || e->val != key) { + err = 4; + return 4; + } + } + + for (i = 0; i < 128; ++i) { + key = i; + err = bpf_map_delete_elem(&rhmap, &key); + if (err) + return 3; + + e = bpf_map_lookup_elem(&rhmap, &key); + if (e) { + err = 5; + return 5; + } + } + + err = 0; + return 0; +} + +SEC("syscall") +int test_rhash_update_exist(void *ctx) +{ + int key = 10; + struct elem val1 = {.val = 100, .arr = {0}}; + struct elem val2 = {.val = 200, .arr = {0}}; + struct elem *e; + int ret; + + err = 1; + + /* BPF_EXIST on non-existent key should fail with -ENOENT */ + ret = bpf_map_update_elem(&rhmap, &key, &val1, BPF_EXIST); + if (ret != -ENOENT) + return 1; + + /* Insert element first */ + ret = bpf_map_update_elem(&rhmap, &key, &val1, BPF_NOEXIST); + if (ret) + return 2; + + /* Verify initial value */ + e = bpf_map_lookup_elem(&rhmap, &key); + if (!e || e->val != 100) + return 3; + + /* BPF_EXIST on existing key should succeed and update value */ + ret = bpf_map_update_elem(&rhmap, &key, &val2, BPF_EXIST); + if (ret) + return 4; + + /* Verify value was updated */ + e = bpf_map_lookup_elem(&rhmap, &key); + if (!e || e->val != 200) + return 5; + + /* Cleanup */ + bpf_map_delete_elem(&rhmap, &key); + err = 0; + return 0; +} + +SEC("syscall") +int test_rhash_update_any(void *ctx) +{ + int key = 11; + struct elem val1 = {.val = 111, .arr = {0}}; + struct elem val2 = {.val = 222, .arr = {0}}; + struct elem *e; + int ret; + + err = 1; + + /* BPF_ANY on non-existent key should insert */ + ret = bpf_map_update_elem(&rhmap, &key, &val1, BPF_ANY); + if (ret) + return 1; + + e = bpf_map_lookup_elem(&rhmap, &key); + if (!e || e->val != 111) + return 2; + + /* BPF_ANY on existing key should update */ + ret = bpf_map_update_elem(&rhmap, &key, &val2, BPF_ANY); + if (ret) + return 3; + + e = bpf_map_lookup_elem(&rhmap, &key); + if (!e || e->val != 222) + return 4; + + /* Cleanup */ + bpf_map_delete_elem(&rhmap, &key); + err = 0; + return 0; +} + +SEC("syscall") +int test_rhash_noexist_duplicate(void *ctx) +{ + int key = 12; + struct elem val = {.val = 600, .arr = {0}}; + int ret; + + err = 1; + + /* Insert element */ + ret = bpf_map_update_elem(&rhmap, &key, &val, BPF_NOEXIST); + if (ret) + return 1; + + /* Try to insert again with BPF_NOEXIST - should fail with -EEXIST */ + ret = bpf_map_update_elem(&rhmap, &key, &val, BPF_NOEXIST); + if (ret != -EEXIST) + return 2; + + /* Cleanup */ + bpf_map_delete_elem(&rhmap, &key); + err = 0; + return 0; +} + +SEC("syscall") +int test_rhash_delete_nonexistent(void *ctx) +{ + int key = 99999; + int ret; + + err = 1; + + /* Delete non-existent key should return -ENOENT */ + ret = bpf_map_delete_elem(&rhmap, &key); + if (ret != -ENOENT) + return 1; + + err = 0; + return 0; +} + +SEC("syscall") +int test_rhash_kptr_update(void *ctx) +{ + struct special_elem val1 = { .val = 1 }; + struct special_elem val2 = { .val = 2 }; + struct task_struct *task, *old; + struct special_elem *elem; + int key = 0; + + err = 1; + if (bpf_map_update_elem(&special_fields, &key, &val1, BPF_NOEXIST)) + return 1; + + err = 2; + elem = bpf_map_lookup_elem(&special_fields, &key); + if (!elem) + return 2; + + err = 3; + task = bpf_task_acquire(bpf_get_current_task_btf()); + if (!task) + return 3; + + err = 4; + old = bpf_kptr_xchg(&elem->task, task); + if (old) { + bpf_task_release(old); + return 4; + } + + err = 5; + if (bpf_map_update_elem(&special_fields, &key, &val2, BPF_EXIST)) + return 5; + + err = 6; + elem = bpf_map_lookup_elem(&special_fields, &key); + if (!elem || elem->val != 2) + return 6; + + err = 7; + old = bpf_kptr_xchg(&elem->task, NULL); + if (!old) + return 7; + bpf_task_release(old); + + err = 8; + if (bpf_map_delete_elem(&special_fields, &key)) + return 8; + + err = 0; + return 0; +} + +SEC("syscall") +int test_rhash_kptr_delete(void *ctx) +{ + struct special_elem val = {}; + struct task_struct *task, *old; + struct special_elem *elem; + int key = 0; + + err = 1; + if (bpf_map_update_elem(&special_fields, &key, &val, BPF_NOEXIST)) + return 1; + + err = 2; + elem = bpf_map_lookup_elem(&special_fields, &key); + if (!elem) + return 2; + + err = 3; + task = bpf_task_acquire(bpf_get_current_task_btf()); + if (!task) + return 3; + + err = 4; + old = bpf_kptr_xchg(&elem->task, task); + if (old) { + bpf_task_release(old); + return 4; + } + + err = 5; + if (bpf_map_delete_elem(&special_fields, &key)) + return 5; + + err = 6; + old = bpf_kptr_xchg(&elem->task, NULL); + if (!old) + return 6; + bpf_task_release(old); + + err = 0; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/rhash_timer.c b/tools/testing/selftests/bpf/progs/rhash_timer.c new file mode 100644 index 000000000000..2e06a463c605 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/rhash_timer.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <errno.h> +#include <bpf/bpf_helpers.h> + +#define CLOCK_MONOTONIC 1 +#define TIMER_NSEC (60ULL * 1000 * 1000 * 1000) + +struct timer_value { + struct bpf_timer timer; + u64 data; +}; + +struct { + __uint(type, BPF_MAP_TYPE_RHASH); + __uint(map_flags, BPF_F_NO_PREALLOC); + __uint(max_entries, 1); + __type(key, u64); + __type(value, struct timer_value); +} timer_map SEC(".maps"); + +u64 armed; +u64 cancelled; +long timer_init_err; +long timer_set_callback_err; +long timer_start_err; +long timer_cancel_err; + +static int timer_cb(void *map, u64 *key, struct timer_value *value) +{ + return 0; +} + +static long arm_timer_cb(struct bpf_map *map, u64 *key, + struct timer_value *value, void *ctx) +{ + u64 key_copy = *key; + long err; + + err = bpf_map_delete_elem(map, &key_copy); + if (err) + return 1; + + err = bpf_timer_init(&value->timer, map, CLOCK_MONOTONIC); + if (err) { + timer_init_err = err; + return 1; + } + + err = bpf_timer_set_callback(&value->timer, timer_cb); + if (err) { + timer_set_callback_err = err; + return 1; + } + + err = bpf_timer_start(&value->timer, TIMER_NSEC, BPF_F_TIMER_CPU_PIN); + if (err) { + timer_start_err = err; + return 1; + } + + __sync_fetch_and_add(&armed, 1); + return 1; +} + +static long cancel_timer_cb(struct bpf_map *map, u64 *key, + struct timer_value *value, void *ctx) +{ + long err; + + err = bpf_timer_cancel(&value->timer); + if (err == -EINVAL) + return 1; + if (err < 0) { + timer_cancel_err = err; + return 1; + } + + __sync_fetch_and_add(&cancelled, 1); + return 1; +} + +SEC("syscall") +int arm_deleted_timer(void *ctx) +{ + bpf_for_each_map_elem(&timer_map, arm_timer_cb, NULL, 0); + return 0; +} + +SEC("syscall") +int cancel_recycled_timer(void *ctx) +{ + bpf_for_each_map_elem(&timer_map, cancel_timer_cb, NULL, 0); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/setget_sockopt.c b/tools/testing/selftests/bpf/progs/setget_sockopt.c index d330b1511979..d96e99b67aeb 100644 --- a/tools/testing/selftests/bpf/progs/setget_sockopt.c +++ b/tools/testing/selftests/bpf/progs/setget_sockopt.c @@ -69,12 +69,14 @@ static const struct sockopt_test sol_tcp_tests[] = { static const struct sockopt_test sol_ip_tests[] = { { .opt = IP_TOS, .new = 0xe1, .expected = 0xe1, .tcp_expected = 0xe0, }, + { .opt = IP_TRANSPARENT, .flip = 1, }, { .opt = 0, }, }; static const struct sockopt_test sol_ipv6_tests[] = { { .opt = IPV6_TCLASS, .new = 0xe1, .expected = 0xe1, .tcp_expected = 0xe0, }, { .opt = IPV6_AUTOFLOWLABEL, .flip = 1, }, + { .opt = IPV6_TRANSPARENT, .flip = 1, }, { .opt = 0, }, }; @@ -387,6 +389,24 @@ int _getsockopt(struct bpf_sockopt *ctx) return 1; } +int v4mapped_v6_ip_tos_enable; +int v4mapped_v6_ip_tos_ret; +int v4mapped_v6_ip_tos_cnt; +int v4mapped_v6_ip_tos_val; + +static void test_v4mapped_v6_ip_tos(struct bpf_sock_ops *skops) +{ + int tos = v4mapped_v6_ip_tos_val; + + if (!v4mapped_v6_ip_tos_enable || skops->op != BPF_SOCK_OPS_TCP_CONNECT_CB) + return; + if (skops->family != AF_INET6) + return; + + v4mapped_v6_ip_tos_cnt++; + v4mapped_v6_ip_tos_ret = bpf_setsockopt(skops, IPPROTO_IP, IP_TOS, &tos, sizeof(tos)); +} + SEC("sockops") int skops_sockopt(struct bpf_sock_ops *skops) { @@ -401,6 +421,11 @@ int skops_sockopt(struct bpf_sock_ops *skops) if (!sk) return 1; + if (v4mapped_v6_ip_tos_enable) { + test_v4mapped_v6_ip_tos(skops); + return 1; + } + switch (skops->op) { case BPF_SOCK_OPS_TCP_LISTEN_CB: nr_listen += !(bpf_test_sockopt(skops, sk) || diff --git a/tools/testing/selftests/bpf/progs/sk_bypass_prot_mem.c b/tools/testing/selftests/bpf/progs/sk_bypass_prot_mem.c index 09a00d11ffcc..bae5283fca6b 100644 --- a/tools/testing/selftests/bpf/progs/sk_bypass_prot_mem.c +++ b/tools/testing/selftests/bpf/progs/sk_bypass_prot_mem.c @@ -5,6 +5,7 @@ #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> #include <errno.h> +#include "err.h" extern int tcp_memory_per_cpu_fw_alloc __ksym; extern int udp_memory_per_cpu_fw_alloc __ksym; @@ -97,6 +98,7 @@ int sock_create(struct bpf_sock *ctx) return 1; err: + set_if_not_errno_or_zero(err, -EFAULT); bpf_set_retval(err); return 0; } diff --git a/tools/testing/selftests/bpf/progs/snprintf_btf_void.c b/tools/testing/selftests/bpf/progs/snprintf_btf_void.c new file mode 100644 index 000000000000..44af80fbbb80 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/snprintf_btf_void.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "btf_ptr.h" +#include <bpf/bpf_helpers.h> + +__u32 type_id; +/* A buffer we own to render the selected type from, kept in bounds. */ +char obj[256]; +char out[64]; +long ret; + +SEC("raw_tp/sys_enter") +int dump_type(void *ctx) +{ + struct btf_ptr ptr = { + .ptr = obj, + .type_id = type_id, + .flags = 0, + }; + + ret = bpf_snprintf_btf(out, sizeof(out), &ptr, sizeof(ptr), 0); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/sock_destroy_prog.c b/tools/testing/selftests/bpf/progs/sock_destroy_prog.c index 9e0bf7a54cec..0a8887543218 100644 --- a/tools/testing/selftests/bpf/progs/sock_destroy_prog.c +++ b/tools/testing/selftests/bpf/progs/sock_destroy_prog.c @@ -7,6 +7,8 @@ #include "bpf_tracing_net.h" __be16 serv_port = 0; +int tw_found = 0; +int tw_destroy_err = 0; int bpf_sock_destroy(struct sock_common *sk) __ksym; @@ -100,6 +102,34 @@ int iter_tcp6_server(struct bpf_iter__tcp *ctx) return 0; } +SEC("iter/tcp") +int iter_tcp6_timewait(struct bpf_iter__tcp *ctx) +{ + struct sock_common *sk_common = ctx->sk_common; + __u64 *val; + int key = 0; + + if (!sk_common) + return 0; + + if (sk_common->skc_family != AF_INET6) + return 0; + + if (!bpf_skc_to_tcp_timewait_sock(sk_common)) + return 0; + + val = bpf_map_lookup_elem(&tcp_conn_sockets, &key); + if (!val) + return 0; + /* The timewait sock inherits the cookie of the closed client sock. */ + if (bpf_get_socket_cookie(sk_common) != *val) + return 0; + + tw_found++; + tw_destroy_err = bpf_sock_destroy(sk_common); + + return 0; +} SEC("iter/udp") int iter_udp6_client(struct bpf_iter__udp *ctx) diff --git a/tools/testing/selftests/bpf/progs/sock_ops_get_sk.c b/tools/testing/selftests/bpf/progs/sock_ops_get_sk.c new file mode 100644 index 000000000000..3a0689f8ce7c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/sock_ops_get_sk.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* + * Test the SOCK_OPS_GET_SK() and SOCK_OPS_GET_FIELD() macros in + * sock_ops_convert_ctx_access() when dst_reg == src_reg. + * + * When dst_reg == src_reg, the macros borrow a temporary register to load + * is_fullsock / is_locked_tcp_sock, because dst_reg holds the ctx pointer + * and cannot be clobbered before ctx->sk / ctx->field is read. If + * is_fullsock == 0 (e.g., TCP_NEW_SYN_RECV with a request_sock), the macro + * must still zero dst_reg so the verifier's PTR_TO_SOCKET_OR_NULL / + * SCALAR_VALUE type is correct at runtime. A missing clear leaves a stale + * ctx pointer in dst_reg that passes NULL checks (GET_SK) or leaks a kernel + * address as a scalar (GET_FIELD). + * + * When dst_reg != src_reg, dst_reg itself is used to load is_fullsock, so + * the JEQ (dst_reg == 0) naturally leaves it zeroed on the !fullsock path. + */ + +int bug_detected; +int null_seen; + +SEC("sockops") +__naked void sock_ops_get_sk_same_reg(void) +{ + asm volatile ( + "r7 = *(u32 *)(r1 + %[is_fullsock_off]);" + "r1 = *(u64 *)(r1 + %[sk_off]);" + "if r7 != 0 goto 2f;" + "if r1 == 0 goto 1f;" + "r1 = %[bug_detected] ll;" + "r2 = 1;" + "*(u32 *)(r1 + 0) = r2;" + "goto 2f;" + "1:" + "r1 = %[null_seen] ll;" + "r2 = 1;" + "*(u32 *)(r1 + 0) = r2;" + "2:" + "r0 = 1;" + "exit;" + : + : __imm_const(is_fullsock_off, offsetof(struct bpf_sock_ops, is_fullsock)), + __imm_const(sk_off, offsetof(struct bpf_sock_ops, sk)), + __imm_addr(bug_detected), + __imm_addr(null_seen) + : __clobber_all); +} + +/* SOCK_OPS_GET_FIELD: same-register, is_locked_tcp_sock == 0 path. */ +int field_bug_detected; +int field_null_seen; + +SEC("sockops") +__naked void sock_ops_get_field_same_reg(void) +{ + asm volatile ( + "r7 = *(u32 *)(r1 + %[is_fullsock_off]);" + "r1 = *(u32 *)(r1 + %[snd_cwnd_off]);" + "if r7 != 0 goto 2f;" + "if r1 == 0 goto 1f;" + "r1 = %[field_bug_detected] ll;" + "r2 = 1;" + "*(u32 *)(r1 + 0) = r2;" + "goto 2f;" + "1:" + "r1 = %[field_null_seen] ll;" + "r2 = 1;" + "*(u32 *)(r1 + 0) = r2;" + "2:" + "r0 = 1;" + "exit;" + : + : __imm_const(is_fullsock_off, offsetof(struct bpf_sock_ops, is_fullsock)), + __imm_const(snd_cwnd_off, offsetof(struct bpf_sock_ops, snd_cwnd)), + __imm_addr(field_bug_detected), + __imm_addr(field_null_seen) + : __clobber_all); +} + +/* SOCK_OPS_GET_SK: different-register, is_fullsock == 0 path. */ +int diff_reg_bug_detected; +int diff_reg_null_seen; + +SEC("sockops") +__naked void sock_ops_get_sk_diff_reg(void) +{ + asm volatile ( + "r7 = r1;" + "r6 = *(u32 *)(r7 + %[is_fullsock_off]);" + "r2 = *(u64 *)(r7 + %[sk_off]);" + "if r6 != 0 goto 2f;" + "if r2 == 0 goto 1f;" + "r1 = %[diff_reg_bug_detected] ll;" + "r3 = 1;" + "*(u32 *)(r1 + 0) = r3;" + "goto 2f;" + "1:" + "r1 = %[diff_reg_null_seen] ll;" + "r3 = 1;" + "*(u32 *)(r1 + 0) = r3;" + "2:" + "r0 = 1;" + "exit;" + : + : __imm_const(is_fullsock_off, offsetof(struct bpf_sock_ops, is_fullsock)), + __imm_const(sk_off, offsetof(struct bpf_sock_ops, sk)), + __imm_addr(diff_reg_bug_detected), + __imm_addr(diff_reg_null_seen) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/sock_read_xattr.c b/tools/testing/selftests/bpf/progs/sock_read_xattr.c new file mode 100644 index 000000000000..c4a8eae8cc3c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/sock_read_xattr.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Christian Brauner */ + +#include <vmlinux.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_core_read.h> +#include "bpf_experimental.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +char value[16]; +int read_ret = -1; +__u32 monitored_pid = 0; + +static __always_inline void read_xattr(struct socket *sock) +{ + struct bpf_dynptr value_ptr; + + bpf_dynptr_from_mem(value, sizeof(value), 0, &value_ptr); + bpf_sock_read_xattr(sock, "user.bpf_test", &value_ptr); +} + +SEC("lsm.s/socket_connect") +__success +int BPF_PROG(trusted_sock_ptr_sleepable, struct socket *sock) +{ + read_xattr(sock); + return 0; +} + +SEC("lsm/socket_connect") +__success +int BPF_PROG(trusted_sock_ptr_non_sleepable, struct socket *sock) +{ + read_xattr(sock); + return 0; +} + +SEC("lsm.s/socket_connect") +__success +int BPF_PROG(read_sock_xattr, struct socket *sock) +{ + struct bpf_dynptr value_ptr; + __u32 pid = bpf_get_current_pid_tgid() >> 32; + + if (pid != monitored_pid) + return 0; + + bpf_dynptr_from_mem(value, sizeof(value), 0, &value_ptr); + read_ret = bpf_sock_read_xattr(sock, "user.bpf_test", &value_ptr); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/sockmap_parse_prog.c b/tools/testing/selftests/bpf/progs/sockmap_parse_prog.c index c9abfe3a11af..56e9aebf05f2 100644 --- a/tools/testing/selftests/bpf/progs/sockmap_parse_prog.c +++ b/tools/testing/selftests/bpf/progs/sockmap_parse_prog.c @@ -5,28 +5,6 @@ SEC("sk_skb1") int bpf_prog1(struct __sk_buff *skb) { - void *data_end = (void *)(long) skb->data_end; - void *data = (void *)(long) skb->data; - __u8 *d = data; - int err; - - if (data + 10 > data_end) { - err = bpf_skb_pull_data(skb, 10); - if (err) - return SK_DROP; - - data_end = (void *)(long)skb->data_end; - data = (void *)(long)skb->data; - if (data + 10 > data_end) - return SK_DROP; - } - - /* This write/read is a bit pointless but tests the verifier and - * strparser handler for read/write pkt data and access into sk - * fields. - */ - d = data; - d[7] = 1; return skb->len; } diff --git a/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c b/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c index 0660f29dca95..3177bc5b733a 100644 --- a/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c +++ b/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c @@ -44,8 +44,18 @@ int bpf_prog2(struct __sk_buff *skb) __sink(lport); __sink(rport); - if (data + 8 > data_end) - return SK_DROP; + if (data + 8 > data_end) { + if (bpf_skb_pull_data(skb, 8)) + return SK_DROP; + + data = (void *)(long)skb->data; + data_end = (void *)(long)skb->data_end; + + if (data + 8 > data_end) + return SK_DROP; + + d = data; + } map = d[0]; sk = d[1]; diff --git a/tools/testing/selftests/bpf/progs/sockopt_sk.c b/tools/testing/selftests/bpf/progs/sockopt_sk.c index cb990a7d3d45..5e0b27e7855c 100644 --- a/tools/testing/selftests/bpf/progs/sockopt_sk.c +++ b/tools/testing/selftests/bpf/progs/sockopt_sk.c @@ -149,6 +149,20 @@ int _setsockopt(struct bpf_sockopt *ctx) if (sk && sk->family == AF_NETLINK) goto out; + if (sk && sk->family == AF_INET && sk->type == SOCK_RAW) { + struct bpf_tcp_sock *tp = bpf_tcp_sock(sk); + + if (tp) { + char saved_syn[60]; + + bpf_getsockopt(sk, SOL_TCP, TCP_SAVED_SYN, + &saved_syn, sizeof(saved_syn)); + goto consumed; + } + + goto out; + } + /* Make sure bpf_get_netns_cookie is callable. */ if (bpf_get_netns_cookie(NULL) == 0) @@ -224,6 +238,8 @@ int _setsockopt(struct bpf_sockopt *ctx) return 0; /* couldn't get sk storage */ storage->val = optval[0]; + +consumed: ctx->optlen = -1; /* BPF has consumed this option, don't call kernel * setsockopt handler. */ diff --git a/tools/testing/selftests/bpf/progs/stack_arg.c b/tools/testing/selftests/bpf/progs/stack_arg.c new file mode 100644 index 000000000000..944e3bb603e7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/stack_arg.c @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <stdbool.h> +#include <bpf/bpf_helpers.h> +#include "bpf_kfuncs.h" + +#define CLOCK_MONOTONIC 1 + +struct timer_elem { + struct bpf_timer timer; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct timer_elem); +} timer_map SEC(".maps"); + +int timer_result; + +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) + +const volatile bool has_stack_arg = true; + +__noinline static int static_func_many_args(int a, int b, int c, int d, + int e, int f, int g, int h, + int i, int j) +{ + return a + b + c + d + e + f + g + h + i + j; +} + +__noinline int global_calls_many_args(int a, int b, int c) +{ + return static_func_many_args(a, b, c, a + 3, a + 4, a + 5, a + 6, + a + 7, a + 8, a + 9); +} + +SEC("tc") +int test_global_many_args(void) +{ + return global_calls_many_args(1, 2, 3); +} + +struct test_data { + long x; + long y; +}; + +/* 1+2+3+4+5+6+7+8+9+10+20 = 75 */ +__noinline static long func_with_ptr_stack_arg(long a, long b, long c, long d, + long e, long f, long g, long h, + long i, struct test_data *p) +{ + return a + b + c + d + e + f + g + h + i + p->x + p->y; +} + +__noinline long global_ptr_stack_arg(long a, long b, long c, long d, long e) +{ + struct test_data data = { .x = 10, .y = 20 }; + + return func_with_ptr_stack_arg(a, b, c, d, e, a + 5, a + 6, a + 7, + a + 8, &data); +} + +SEC("tc") +int test_bpf2bpf_ptr_stack_arg(void) +{ + return global_ptr_stack_arg(1, 2, 3, 4, 5); +} + +/* 1+2+3+4+5+6+7+10+8+20 = 66 */ +__noinline static long func_with_mix_stack_args(long a, long b, long c, long d, + long e, long f, long g, + struct test_data *p, + long h, struct test_data *q) +{ + return a + b + c + d + e + f + g + p->x + h + q->y; +} + +__noinline long global_mix_stack_args(long a, long b, long c, long d, long e) +{ + struct test_data p = { .x = 10 }; + struct test_data q = { .y = 20 }; + + return func_with_mix_stack_args(a, b, c, d, e, e + 1, e + 2, &p, + e + 3, &q); +} + +SEC("tc") +int test_bpf2bpf_mix_stack_args(void) +{ + return global_mix_stack_args(1, 2, 3, 4, 5); +} + +/* + * Nesting test: func_outer calls func_inner, both with struct pointer + * as stack arg. + * + * func_inner: (a+1)+...+(i+1) + p->x + p->y + * = 2+3+4+5+6+7+8+9+10+10+20 = 84 + */ +__noinline static long func_inner_ptr(long a, long b, long c, long d, + long e, long f, long g, long h, + long i, struct test_data *p) +{ + return a + b + c + d + e + f + g + h + i + p->x + p->y; +} + +__noinline static long func_outer_ptr(long a, long b, long c, long d, + long e, long f, long g, long h, + long i, struct test_data *p) +{ + return func_inner_ptr(a + 1, b + 1, c + 1, d + 1, e + 1, + f + 1, g + 1, h + 1, i + 1, p); +} + +__noinline long global_nesting_ptr(long a, long b, long c, long d, long e) +{ + struct test_data data = { .x = 10, .y = 20 }; + + return func_outer_ptr(a, b, c, d, e, a + 5, a + 6, a + 7, a + 8, + &data); +} + +SEC("tc") +int test_bpf2bpf_nesting_stack_arg(void) +{ + return global_nesting_ptr(1, 2, 3, 4, 5); +} + +/* 1+2+3+4+5+6+7+8+9+sizeof(pkt_v4) = 45+54 = 99 */ +__noinline static long func_with_dynptr(long a, long b, long c, long d, + long e, long f, long g, long h, + long i, struct bpf_dynptr *ptr) +{ + return a + b + c + d + e + f + g + h + i + bpf_dynptr_size(ptr); +} + +__noinline long global_dynptr_stack_arg(void *ctx __arg_ctx, long a, long b, + long c, long d) +{ + struct bpf_dynptr ptr; + + bpf_dynptr_from_skb(ctx, 0, &ptr); + return func_with_dynptr(a, b, c, d, d + 1, d + 2, d + 3, d + 4, + d + 5, &ptr); +} + +SEC("tc") +int test_bpf2bpf_dynptr_stack_arg(struct __sk_buff *skb) +{ + return global_dynptr_stack_arg(skb, 1, 2, 3, 4); +} + +/* foo1: a+b+c+d+e+f+g+h+i+j */ +__noinline static int foo1(int a, int b, int c, int d, int e, + int f, int g, int h, int i, int j) +{ + return a + b + c + d + e + f + g + h + i + j; +} + +/* foo2: a+b+c+d+e+f+g+h+i+j+k+l */ +__noinline static int foo2(int a, int b, int c, int d, int e, + int f, int g, int h, int i, int j, + int k, int l) +{ + return a + b + c + d + e + f + g + h + i + j + k + l; +} + +/* global_two_callees calls foo1 (5 stack args) and foo2 (7 stack args). + * The outgoing stack arg area is sized for foo2 (the larger callee). + * Stores for foo1 are a subset of the area used by foo2. + * Result: foo1(1..10) + foo2(1..12) = 55 + 78 = 133 + * + * Pass a-e through so the compiler can't constant-fold the stack args away. + */ +__noinline int global_two_callees(int a, int b, int c, int d, int e) +{ + int ret; + + ret = foo1(a, b, c, d, e, a + 5, a + 6, a + 7, a + 8, a + 9); + ret += foo2(a, b, c, d, e, a + 5, a + 6, a + 7, a + 8, a + 9, + a + 10, a + 11); + return ret; +} + +SEC("tc") +int test_two_callees(void) +{ + return global_two_callees(1, 2, 3, 4, 5); +} + +const volatile int timer_base = 10; + +static int timer_cb_many_args(void *map, int *key, struct bpf_timer *timer) +{ + int v = timer_base; + + timer_result = static_func_many_args(v, v * 2, v * 3, v * 4, v * 5, + v * 6, v * 7, v * 8, v * 9, + v * 10); + return 0; +} + +SEC("tc") +int test_async_cb_many_args(void) +{ + struct timer_elem *elem; + int key = 0; + + elem = bpf_map_lookup_elem(&timer_map, &key); + if (!elem) + return -1; + + bpf_timer_init(&elem->timer, &timer_map, CLOCK_MONOTONIC); + bpf_timer_set_callback(&elem->timer, timer_cb_many_args); + bpf_timer_start(&elem->timer, 1, 0); + return 0; +} + +#else + +const volatile bool has_stack_arg = false; + +SEC("tc") +int test_global_many_args(void) +{ + return 0; +} + +SEC("tc") +int test_bpf2bpf_ptr_stack_arg(void) +{ + return 0; +} + +SEC("tc") +int test_bpf2bpf_mix_stack_args(void) +{ + return 0; +} + +SEC("tc") +int test_bpf2bpf_nesting_stack_arg(void) +{ + return 0; +} + +SEC("tc") +int test_bpf2bpf_dynptr_stack_arg(struct __sk_buff *skb) +{ + return 0; +} + +SEC("tc") +int test_two_callees(void) +{ + return 0; +} + +SEC("tc") +int test_async_cb_many_args(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/stack_arg_fail.c b/tools/testing/selftests/bpf/progs/stack_arg_fail.c new file mode 100644 index 000000000000..ad9d4bfe15dc --- /dev/null +++ b/tools/testing/selftests/bpf/progs/stack_arg_fail.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "../test_kmods/bpf_testmod_kfunc.h" +#include "bpf_misc.h" + +#if defined(__BPF_FEATURE_STACK_ARGUMENT) + +SEC("tc") +__failure __msg("Unrecognized *(R11-8) type STRUCT") +int test_stack_arg_big(struct __sk_buff *skb) +{ + struct prog_test_big_arg s = { .a = 1, .b = 2 }; + + return bpf_kfunc_call_stack_arg_big(1, 2, 3, 4, 5, s); +} + +SEC("socket") +__description("r11 in ALU instruction") +__failure __msg("R11 is invalid") +__naked void r11_alu_reject(void) +{ + asm volatile ( + "r11 += 1;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("r11 store with non-DW size") +__failure __msg("R11 is invalid") +__naked void r11_store_non_dw(void) +{ + asm volatile ( + "*(u32 *)(r11 - 8) = r1;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("r11 store with unaligned offset") +__failure __msg("R11 is invalid") +__naked void r11_store_unaligned(void) +{ + asm volatile ( + "*(u64 *)(r11 - 4) = r1;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("r11 store with positive offset") +__failure __msg("R11 is invalid") +__naked void r11_store_positive_off(void) +{ + asm volatile ( + "*(u64 *)(r11 + 8) = r1;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("r11 load with negative offset") +__failure __msg("R11 is invalid") +__naked void r11_load_negative_off(void) +{ + asm volatile ( + "r0 = *(u64 *)(r11 - 8);" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("r11 load with non-DW size") +__failure __msg("R11 is invalid") +__naked void r11_load_non_dw(void) +{ + asm volatile ( + "r0 = *(u32 *)(r11 + 8);" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("r11 store with zero offset") +__failure __msg("R11 is invalid") +__naked void r11_store_zero_off(void) +{ + asm volatile ( + "*(u64 *)(r11 + 0) = r1;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +#else + +SEC("tc") +__description("stack_arg_fail: not supported, dummy test") +__success +int test_stack_arg_big(struct __sk_buff *skb) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c b/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c new file mode 100644 index 000000000000..345f2da2e361 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_kfuncs.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) + +const volatile bool has_stack_arg = true; + +struct bpf_iter_testmod_seq { + u64 :64; + u64 :64; +}; + +extern int bpf_iter_testmod_seq_new(struct bpf_iter_testmod_seq *it, s64 value, int cnt) __ksym; +extern void bpf_iter_testmod_seq_destroy(struct bpf_iter_testmod_seq *it) __ksym; + +struct timer_map_value { + struct bpf_timer timer; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct timer_map_value); +} kfunc_timer_map SEC(".maps"); + +SEC("tc") +int test_stack_arg_scalar(struct __sk_buff *skb) +{ + return bpf_kfunc_call_stack_arg(1, 2, 3, 4, 5, 6, 7, 8, 9, 10); +} + +SEC("tc") +int test_stack_arg_ptr(struct __sk_buff *skb) +{ + struct prog_test_pass1 p = { .x0 = 10, .x1 = 20 }; + + return bpf_kfunc_call_stack_arg_ptr(1, 2, 3, 4, 5, 6, 7, 8, 9, &p); +} + +SEC("tc") +int test_stack_arg_mix(struct __sk_buff *skb) +{ + struct prog_test_pass1 p = { .x0 = 10 }; + struct prog_test_pass1 q = { .x1 = 20 }; + + return bpf_kfunc_call_stack_arg_mix(1, 2, 3, 4, 5, 6, 7, &p, 8, &q); +} + +/* 1+2+3+4+5+6+7+8+9+sizeof(pkt_v4) = 45+54 = 99 */ +SEC("tc") +int test_stack_arg_dynptr(struct __sk_buff *skb) +{ + struct bpf_dynptr ptr; + + bpf_dynptr_from_skb(skb, 0, &ptr); + return bpf_kfunc_call_stack_arg_dynptr(1, 2, 3, 4, 5, 6, 7, 8, 9, &ptr); +} + +/* 1 + 2 + 3 + 4 + 5 + (1 + 2 + ... + 16) = 15 + 136 = 151 */ +SEC("tc") +int test_stack_arg_mem(struct __sk_buff *skb) +{ + char buf[16] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; + + return bpf_kfunc_call_stack_arg_mem(1, 2, 3, 4, 5, buf, sizeof(buf)); +} + +/* 1+2+3+4+5+6+7+8+9+100 = 145 */ +SEC("tc") +int test_stack_arg_iter(struct __sk_buff *skb) +{ + struct bpf_iter_testmod_seq it; + u64 ret; + + bpf_iter_testmod_seq_new(&it, 100, 10); + ret = bpf_kfunc_call_stack_arg_iter(1, 2, 3, 4, 5, 6, 7, 8, 9, &it); + bpf_iter_testmod_seq_destroy(&it); + return ret; +} + +const char cstr[] = "hello"; + +/* 1+2+3+4+5+6+7+8+9 = 45 */ +SEC("tc") +int test_stack_arg_const_str(struct __sk_buff *skb) +{ + return bpf_kfunc_call_stack_arg_const_str(1, 2, 3, 4, 5, 6, 7, 8, 9, + cstr); +} + +/* 1+2+3+4+5+6+7+8+9 = 45 */ +SEC("tc") +int test_stack_arg_timer(struct __sk_buff *skb) +{ + struct timer_map_value *val; + int key = 0; + + val = bpf_map_lookup_elem(&kfunc_timer_map, &key); + if (!val) + return 0; + return bpf_kfunc_call_stack_arg_timer(1, 2, 3, 4, 5, 6, 7, 8, 9, + &val->timer); +} + +#else + +const volatile bool has_stack_arg = false; + +SEC("tc") +int test_stack_arg_scalar(struct __sk_buff *skb) +{ + return 0; +} + +SEC("tc") +int test_stack_arg_ptr(struct __sk_buff *skb) +{ + return 0; +} + +SEC("tc") +int test_stack_arg_mix(struct __sk_buff *skb) +{ + return 0; +} + +SEC("tc") +int test_stack_arg_dynptr(struct __sk_buff *skb) +{ + return 0; +} + +SEC("tc") +int test_stack_arg_mem(struct __sk_buff *skb) +{ + return 0; +} + +SEC("tc") +int test_stack_arg_iter(struct __sk_buff *skb) +{ + return 0; +} + +SEC("tc") +int test_stack_arg_const_str(struct __sk_buff *skb) +{ + return 0; +} + +SEC("tc") +int test_stack_arg_timer(struct __sk_buff *skb) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/stack_arg_precision.c b/tools/testing/selftests/bpf/progs/stack_arg_precision.c new file mode 100644 index 000000000000..bee2eeec021d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/stack_arg_precision.c @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "../test_kmods/bpf_testmod_kfunc.h" +#include "bpf_misc.h" + +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) + +/* Force kfunc extern BTF generation for inline asm call below. + * Uses its own SEC so it's not included as a .text subprog. + * The '?' prefix sets autoload=false so libbpf won't load it. + */ +SEC("?tc") +int __btf_kfunc_gen(struct __sk_buff *ctx) +{ + char buf[8] = {}; + + return bpf_kfunc_call_stack_arg_mem(0, 0, 0, 0, 0, buf, sizeof(buf)); +} + +/* + * Test precision backtracking across bpf-to-bpf call for kfunc stack arg. + * subprog_call_mem_kfunc receives a size as incoming stack arg (arg6) + * and forwards it as mem__sz (arg7) to bpf_kfunc_call_stack_arg_mem. + */ +__naked __noinline __used +static long subprog_call_mem_kfunc(long a, long b, long c, long d, long e, long size) +{ + asm volatile ( + "r1 = *(u64 *)(r11 + 8);" /* r1 = incoming arg6 (size) */ + "r2 = 0x0807060504030201 ll;" /* r2 = buf contents */ + "*(u64 *)(r10 - 8) = r2;" /* store buf to stack */ + "r2 = r10;" + "r2 += -8;" /* r2 = &buf */ + "*(u64 *)(r11 - 8) = r2;" /* outgoing arg6 = buf */ + "*(u64 *)(r11 - 16) = r1;" /* outgoing arg7 = size */ + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "call %[bpf_kfunc_call_stack_arg_mem];" + "exit;" + : + : __imm(bpf_kfunc_call_stack_arg_mem) + : __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: precision backtracking across bpf2bpf call for kfunc") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__btf_func_path("btf__stack_arg_precision.bpf.o") +__msg("mark_precise: frame1: last_idx 26 first_idx 13 subseq_idx -1") +__msg("mark_precise: frame1: regs= stack= before 25: (b7) r5 = 5") +__msg("mark_precise: frame1: regs= stack= before 24: (b7) r4 = 4") +__msg("mark_precise: frame1: regs= stack= before 23: (b7) r3 = 3") +__msg("mark_precise: frame1: regs= stack= before 22: (b7) r2 = 2") +__msg("mark_precise: frame1: regs= stack= before 21: (b7) r1 = 1") +__msg("mark_precise: frame1: regs= stack= before 20: (7b) *(u64 *)(r11 -16) = r1") +__msg("mark_precise: frame1: regs=r1 stack= before 19: (7b) *(u64 *)(r11 -8) = r2") +__msg("mark_precise: frame1: regs=r1 stack= before 18: (07) r2 += -8") +__msg("mark_precise: frame1: regs=r1 stack= before 17: (bf) r2 = r10") +__msg("mark_precise: frame1: regs=r1 stack= before 16: (7b) *(u64 *)(r10 -8) = r2") +__msg("mark_precise: frame1: regs=r1 stack= before 14: (18) r2 = 0x807060504030201") +__msg("mark_precise: frame1: regs=r1 stack= before 13: (79) r1 = *(u64 *)(r11 +8)") +__msg("mark_precise: frame1: parent state regs= stack=: frame1: R10=fp0") +__msg("mark_precise: frame0: parent state regs= stack=: R10=fp0") +__msg("mark_precise: frame1: last_idx 11 first_idx 11 subseq_idx 13") +__msg("mark_precise: frame1: regs= stack= before 11: (85) call pc+1") +__msg("mark_precise: frame0: parent state regs= stack=: R1=1 R2=2 R3=3 R4=4 R5=5 R10=fp0") +__msg("mark_precise: frame0: last_idx 9 first_idx 7 subseq_idx 11") +__msg("mark_precise: frame0: regs= stack= before 9: (05) goto pc+1") +__msg("mark_precise: frame0: regs= stack= before 8: (7a) *(u64 *)(r11 -8) = 4") +__msg("mark_precise: frame1: last_idx 26 first_idx 13 subseq_idx -1 ") +__msg("mark_precise: frame1: regs= stack= before 25: (b7) r5 = 5") +__msg("mark_precise: frame1: regs= stack= before 24: (b7) r4 = 4") +__msg("mark_precise: frame1: regs= stack= before 23: (b7) r3 = 3") +__msg("mark_precise: frame1: regs= stack= before 22: (b7) r2 = 2") +__msg("mark_precise: frame1: regs= stack= before 21: (b7) r1 = 1") +__msg("mark_precise: frame1: regs= stack= before 20: (7b) *(u64 *)(r11 -16) = r1") +__msg("mark_precise: frame1: regs=r1 stack= before 19: (7b) *(u64 *)(r11 -8) = r2") +__msg("mark_precise: frame1: regs=r1 stack= before 18: (07) r2 += -8") +__msg("mark_precise: frame1: regs=r1 stack= before 17: (bf) r2 = r10") +__msg("mark_precise: frame1: regs=r1 stack= before 16: (7b) *(u64 *)(r10 -8) = r2") +__msg("mark_precise: frame1: regs=r1 stack= before 14: (18) r2 = 0x807060504030201") +__msg("mark_precise: frame1: regs=r1 stack= before 13: (79) r1 = *(u64 *)(r11 +8)") +__msg("mark_precise: frame1: parent state regs= stack=: frame1: R10=fp0") +__msg("mark_precise: frame0: parent state regs= stack=: R10=fp0") +__msg("mark_precise: frame1: last_idx 11 first_idx 11 subseq_idx 13 ") +__msg("mark_precise: frame1: regs= stack= before 11: (85) call pc+1") +__msg("mark_precise: frame0: parent state regs= stack=: R1=1 R2=2 R3=3 R4=4 R5=5 R10=fp0") +__msg("mark_precise: frame0: last_idx 10 first_idx 10 subseq_idx 11 ") +__msg("mark_precise: frame0: regs= stack= before 10: (7a) *(u64 *)(r11 -8) = 6") +__naked void stack_arg_precision_bpf2bpf(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "if r6 < 2 goto l0_%=;" + "*(u64 *)(r11 - 8) = 4;" + "goto l1_%=;" + "l0_%=:" + "*(u64 *)(r11 - 8) = 6;" + "l1_%=:" + "call subprog_call_mem_kfunc;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all + ); +} + +#else + +SEC("socket") +__description("stack_arg_precision: not supported, dummy test") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/stream.c b/tools/testing/selftests/bpf/progs/stream.c index 6f999ba951a3..8e8e1339dc74 100644 --- a/tools/testing/selftests/bpf/progs/stream.c +++ b/tools/testing/selftests/bpf/progs/stream.c @@ -5,7 +5,7 @@ #include <bpf/bpf_helpers.h> #include "bpf_misc.h" #include "bpf_experimental.h" -#include "bpf_arena_common.h" +#include <bpf_arena_common.h> struct arr_elem { struct bpf_res_spin_lock lock; @@ -64,6 +64,8 @@ SEC("syscall") __arch_x86_64 __arch_arm64 __arch_s390x +__arch_riscv64 +__arch_loongarch __success __retval(0) __stderr("ERROR: Timeout detected for may_goto instruction") __stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}") @@ -184,6 +186,151 @@ int stream_arena_read_fault(void *ctx) return 0; } +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +__stderr("ERROR: Arena READ access at unmapped address 0x{{.*}}") +__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}") +__stderr("Call trace:\n" +"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n" +"|[ \t]+[^\n]+\n)*}}") +int stream_arena_load_acquire_fault(void *ctx) +{ + static const struct bpf_insn load_acquire_insn = { + .code = 0xc3, /* BPF_STX | BPF_ATOMIC | BPF_W */ + .dst_reg = 0, /* BPF_REG_0 */ + .src_reg = 1, /* BPF_REG_1 */ + .off = 0x7fff, + .imm = 0x100, /* BPF_LOAD_ACQ */ + }; + struct bpf_arena *ptr = (void *)&arena; + u64 user_vm_start, val; + + /* + * Prevent GCC bounds warning: casting &arena to struct bpf_arena * + * triggers bounds checking since the map definition is smaller than + * struct bpf_arena. barrier_var() makes the pointer opaque to GCC, + * preventing the bounds analysis. + */ + barrier_var(ptr); + user_vm_start = ptr->user_vm_start; + fault_addr = user_vm_start + 0x7fff; + bpf_addr_space_cast(user_vm_start, 0, 1); + asm volatile ( + "r1 = %[user_vm_start];" + "r0 = 1;" + ".8byte %[load_acquire_insn];" /* r0 = load_acquire((u32 *)(r1 + 0x7fff)) */ + "%[val] = r0;" + : [val] "=r" (val) + : [user_vm_start] "r" (user_vm_start), + __imm_insn(load_acquire_insn, load_acquire_insn) + : "r0", "r1" + ); + return val; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +__stderr("ERROR: Arena WRITE access at unmapped address 0x{{.*}}") +__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}") +__stderr("Call trace:\n" +"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n" +"|[ \t]+[^\n]+\n)*}}") +int stream_arena_xchg_fault(void *ctx) +{ + static const struct bpf_insn xchg_insn = { + .code = 0xc3, /* BPF_STX | BPF_ATOMIC | BPF_W */ + .dst_reg = 1, /* BPF_REG_1 */ + .src_reg = 2, /* BPF_REG_2 */ + .off = 0x7fff, + .imm = 0xe1, /* BPF_XCHG */ + }; + struct bpf_arena *ptr = (void *)&arena; + u64 user_vm_start, val; + + /* + * Prevent GCC bounds warning: casting &arena to struct bpf_arena * + * triggers bounds checking since the map definition is smaller than + * struct bpf_arena. barrier_var() makes the pointer opaque to GCC, + * preventing the bounds analysis. + */ + barrier_var(ptr); + user_vm_start = ptr->user_vm_start; + fault_addr = user_vm_start + 0x7fff; + bpf_addr_space_cast(user_vm_start, 0, 1); + /* + * A read-modify-write carrying BPF_FETCH writes to memory, so the fault + * has to be reported as a WRITE from the dst_reg address, but it also + * reads the old value into src_reg, so the exception handler has to + * clear src_reg. Poison it up front, the returned value must be 0. + */ + asm volatile ( + "r1 = %[user_vm_start];" + "r2 = 1;" + ".8byte %[xchg_insn];" /* r2 = xchg((u32 *)(r1 + 0x7fff), r2) */ + "%[val] = r2;" + : [val] "=r" (val) + : [user_vm_start] "r" (user_vm_start), + __imm_insn(xchg_insn, xchg_insn) + : "r1", "r2" + ); + return val; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +__stderr("ERROR: Arena WRITE access at unmapped address 0x{{.*}}") +__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}") +__stderr("Call trace:\n" +"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n" +"|[ \t]+[^\n]+\n)*}}") +int stream_arena_cmpxchg_fault(void *ctx) +{ + static const struct bpf_insn cmpxchg_insn = { + .code = 0xc3, /* BPF_STX | BPF_ATOMIC | BPF_W */ + .dst_reg = 1, /* BPF_REG_1 */ + .src_reg = 2, /* BPF_REG_2 */ + .off = 0x7fff, + .imm = 0xf1, /* BPF_CMPXCHG */ + }; + struct bpf_arena *ptr = (void *)&arena; + u64 user_vm_start, val; + + /* + * Prevent GCC bounds warning: casting &arena to struct bpf_arena * + * triggers bounds checking since the map definition is smaller than + * struct bpf_arena. barrier_var() makes the pointer opaque to GCC, + * preventing the bounds analysis. + */ + barrier_var(ptr); + user_vm_start = ptr->user_vm_start; + fault_addr = user_vm_start + 0x7fff; + bpf_addr_space_cast(user_vm_start, 0, 1); + /* + * Same as the exchange above, except that a BPF_CMPXCHG reads the old + * value into r0 rather than into src_reg, so r0 is the register the + * exception handler has to clear. It doubles as the compare value, but + * the comparison never happens since the access faults first. + */ + asm volatile ( + "r1 = %[user_vm_start];" + "r0 = 1;" + "r2 = 2;" + ".8byte %[cmpxchg_insn];" /* r0 = cmpxchg((u32 *)(r1 + 0x7fff), r0, r2) */ + "%[val] = r0;" + : [val] "=r" (val) + : [user_vm_start] "r" (user_vm_start), + __imm_insn(cmpxchg_insn, cmpxchg_insn) + : "r0", "r1", "r2" + ); + return val; +} + static __noinline void subprog(void) { int __arena *addr = (int __arena *)0xdeadbeef; diff --git a/tools/testing/selftests/bpf/progs/stream_fail.c b/tools/testing/selftests/bpf/progs/stream_fail.c index 8e8249f3521c..21428bb1ee59 100644 --- a/tools/testing/selftests/bpf/progs/stream_fail.c +++ b/tools/testing/selftests/bpf/progs/stream_fail.c @@ -23,7 +23,7 @@ int stream_vprintk_scalar_arg(void *ctx) } SEC("syscall") -__failure __msg("arg#1 doesn't point to a const string") +__failure __msg("R2 doesn't point to a const string") int stream_vprintk_string_arg(void *ctx) { bpf_stream_vprintk(BPF_STDOUT, ctx, NULL, 0); diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena.c b/tools/testing/selftests/bpf/progs/struct_ops_arena.c new file mode 100644 index 000000000000..8aa8639df91f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_experimental.h" +#include <bpf_arena_common.h> +#include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + /* page 0 hosts the arena globals, page 1 is for allocations */ + __uint(max_entries, 2); +} arena SEC(".maps"); + +/* also associates the callbacks with the arena */ +u64 __arena arena_touch; +/* raw value of the last __arena ctx argument, captured by test_arena_cb */ +u64 __arena cb_ptr_val; + +SEC("struct_ops/test_arena") +int test_arena_cb(unsigned long long *ctx) +{ + u64 __arena *ptr = (u64 __arena *)ctx[0]; + + arena_touch++; + cb_ptr_val = ctx[0]; + *ptr += 1; + return 0; +} + +SEC("struct_ops/test_arena_nullable") +int test_arena_nullable_cb(unsigned long long *ctx) +{ + u64 __arena *ptr = (u64 __arena *)ctx[0]; + + arena_touch++; + if (!ptr) + return 0xbee; + *ptr += 1; + return 0; +} + +SEC("struct_ops/test_arena_stack") +int test_arena_stack_cb(unsigned long long *ctx) +{ + u64 __arena *ptr = (u64 __arena *)ctx[8]; + + arena_touch++; + /* pin the slot layout: the leading args fill ctx[0]..ctx[7] */ + if (ctx[0] != 1 || ctx[7] != 8) + return 0xbad; + *ptr += 1; + return 0; +} + +SEC("struct_ops/test_arena_multislot") +int test_arena_multislot_cb(unsigned long long *ctx) +{ + u64 __arena *ptr = (u64 __arena *)ctx[2]; + + arena_touch++; + /* + * The 16-byte struct occupies ctx[0] and ctx[1], so @ptr is argument + * one but slot two. Getting that wrong hands the callback a scalar. + */ + if (ctx[0] != 11 || ctx[1] != 22) + return 0xbad; + *ptr += 1; + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena = { + .test_arena = (void *)test_arena_cb, + .test_arena_nullable = (void *)test_arena_nullable_cb, + .test_arena_stack = (void *)test_arena_stack_cb, + .test_arena_multislot = (void *)test_arena_multislot_cb, +}; + +SEC("syscall") +int trigger(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + int ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + ret = bpf_testmod_ops3_call_test_arena((u64 *)val); + if (ret) + return 2; + if (*val != 42) + return 3; + + /* + * The callback must have seen exactly (u32)(kaddr - kern_vm_start), + * which is the arena offset of val with the upper 32 bits clear. + */ + if (cb_ptr_val != (u32)(u64)val) + return 4; + + ret = bpf_testmod_ops3_call_test_arena_nullable((u64 *)val); + if (ret) + return 5; + if (*val != 43) + return 6; + + /* NULL survives the nullable kfunc and the trampoline as NULL */ + ret = bpf_testmod_ops3_call_test_arena_nullable(NULL); + if (ret != 0xbee) + return 7; + + /* the arena pointer is stack-passed into the trampoline here */ + ret = bpf_testmod_ops3_call_test_arena_stack((u64 *)val); + if (ret) + return 8; + if (*val != 44) + return 9; + + /* a multi-slot arg precedes the arena pointer here */ + ret = bpf_testmod_ops3_call_test_arena_multislot((u64 *)val); + if (ret) + return 10; + if (*val != 45) + return 11; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_attach.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_attach.c new file mode 100644 index 000000000000..081a770307e5 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_attach.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +SEC("fentry") +int BPF_PROG(fentry_test_arena, unsigned long long *st_ops_ctx) +{ + return 0; +} + +SEC("fexit") +int BPF_PROG(fexit_test_arena, unsigned long long *st_ops_ctx, int ret) +{ + return 0; +} + +SEC("freplace") +int freplace_test_arena(unsigned long long *st_ops_ctx) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_fail.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_fail.c new file mode 100644 index 000000000000..1c0ec727d637 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_fail.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* No arena in the program: attaching to test_arena must be rejected. */ +SEC("struct_ops/test_arena") +int test_arena_no_arena(unsigned long long *ctx) +{ + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_fail = { + .test_arena = (void *)test_arena_no_arena, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_multi_args.c b/tools/testing/selftests/bpf/progs/struct_ops_multi_args.c new file mode 100644 index 000000000000..c62be15757f0 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_multi_args.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Varun R Mallya */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "../test_kmods/bpf_testmod.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} prog_array SEC(".maps"); + +SEC("struct_ops/test_refcounted_multi") +__failure __msg("program with __ref argument cannot tail call") +int test_refcounted_multi(unsigned long long *ctx) +{ + /* ctx[2] is used because the refcounted variable is the third argument */ + struct task_struct *refcounted_task = (struct task_struct *)ctx[2]; + + bpf_task_release(refcounted_task); + bpf_tail_call(ctx, &prog_array, 0); + + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops testmod_ref_acquire = { + .test_refcounted_multi = (void *)test_refcounted_multi, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_private_stack.c b/tools/testing/selftests/bpf/progs/struct_ops_private_stack.c index dbe646013811..3cd0c1a55cbd 100644 --- a/tools/testing/selftests/bpf/progs/struct_ops_private_stack.c +++ b/tools/testing/selftests/bpf/progs/struct_ops_private_stack.c @@ -7,12 +7,6 @@ char _license[] SEC("license") = "GPL"; -#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) -bool skip __attribute((__section__(".data"))) = false; -#else -bool skip = true; -#endif - void bpf_testmod_ops3_call_test_2(void) __ksym; int val_i, val_j; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c index 3d89ad7cbe2a..1442728f5604 100644 --- a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c +++ b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c @@ -7,12 +7,6 @@ char _license[] SEC("license") = "GPL"; -#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) -bool skip __attribute((__section__(".data"))) = false; -#else -bool skip = true; -#endif - void bpf_testmod_ops3_call_test_2(void) __ksym; int val_i, val_j; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_recur.c b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_recur.c index b1f6d7e5a8e5..faaa0f8d65a4 100644 --- a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_recur.c +++ b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_recur.c @@ -7,12 +7,6 @@ char _license[] SEC("license") = "GPL"; -#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) -bool skip __attribute((__section__(".data"))) = false; -#else -bool skip = true; -#endif - void bpf_testmod_ops3_call_test_1(void) __ksym; int val_i, val_j; diff --git a/tools/testing/selftests/bpf/progs/tailcall3.c b/tools/testing/selftests/bpf/progs/tailcall3.c index f60bcd7b8d4b..204f19c30a3e 100644 --- a/tools/testing/selftests/bpf/progs/tailcall3.c +++ b/tools/testing/selftests/bpf/progs/tailcall3.c @@ -5,7 +5,7 @@ struct { __uint(type, BPF_MAP_TYPE_PROG_ARRAY); - __uint(max_entries, 1); + __uint(max_entries, 2); __uint(key_size, sizeof(__u32)); __uint(value_size, sizeof(__u32)); } jmp_table SEC(".maps"); @@ -23,6 +23,9 @@ int classifier_0(struct __sk_buff *skb) SEC("tc") int entry(struct __sk_buff *skb) { + /* prog == NULL case */ + bpf_tail_call_static(skb, &jmp_table, 1); + bpf_tail_call_static(skb, &jmp_table, 0); return 0; } diff --git a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf2.c b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf2.c index ce97d141daee..c4fadee5aadc 100644 --- a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf2.c +++ b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf2.c @@ -13,11 +13,14 @@ struct { static __noinline int subprog_tail(struct __sk_buff *skb) { + int ret = 1; + if (load_byte(skb, 0)) bpf_tail_call_static(skb, &jmp_table, 1); else bpf_tail_call_static(skb, &jmp_table, 0); - return 1; + barrier_var(ret); + return ret; } int count = 0; diff --git a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_hierarchy1.c b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_hierarchy1.c index d556b19413d7..1fd07824d88a 100644 --- a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_hierarchy1.c +++ b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_hierarchy1.c @@ -16,20 +16,25 @@ int count = 0; static __noinline int subprog_tail(struct __sk_buff *skb) { + int ret = 0; + bpf_tail_call_static(skb, &jmp_table, 0); - return 0; + barrier_var(ret); + return ret; } SEC("tc") int entry(struct __sk_buff *skb) { - int ret = 1; + int ret = 1, ret1, ret2; clobber_regs_stack(); count++; - subprog_tail(skb); - subprog_tail(skb); + ret1 = subprog_tail(skb); + ret2 = subprog_tail(skb); + __sink(ret1); + __sink(ret2); return ret; } diff --git a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_hierarchy2.c b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_hierarchy2.c index ae94c9c70ab7..6fde0ab92148 100644 --- a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_hierarchy2.c +++ b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_hierarchy2.c @@ -25,8 +25,11 @@ int count1 = 0; static __noinline int subprog_tail0(struct __sk_buff *skb) { + int ret = 0; + bpf_tail_call_static(skb, &jmp_table, 0); - return 0; + barrier_var(ret); + return ret; } __auxiliary @@ -41,16 +44,22 @@ int classifier_0(struct __sk_buff *skb) static __noinline int subprog_tail1(struct __sk_buff *skb) { + int ret = 0; + bpf_tail_call_static(skb, &jmp_table, 1); - return 0; + barrier_var(ret); + return ret; } __auxiliary SEC("tc") int classifier_1(struct __sk_buff *skb) { + int ret; + count1++; - subprog_tail1(skb); + ret = subprog_tail1(skb); + __sink(ret); return 0; } @@ -59,13 +68,14 @@ __retval(33) SEC("tc") int tailcall_bpf2bpf_hierarchy_2(struct __sk_buff *skb) { - int ret = 0; + int ret = 0, ret1, ret2; clobber_regs_stack(); - subprog_tail0(skb); - subprog_tail1(skb); - + ret1 = subprog_tail0(skb); + ret2 = subprog_tail1(skb); + __sink(ret1); + __sink(ret2); __sink(ret); return (count1 << 16) | count0; } diff --git a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_hierarchy3.c b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_hierarchy3.c index 56b6b0099840..0ef9cfb2da8d 100644 --- a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_hierarchy3.c +++ b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_hierarchy3.c @@ -33,17 +33,24 @@ int count = 0; static __noinline int subprog_tail(struct __sk_buff *skb, void *jmp_table) { + int ret = 0; + bpf_tail_call_static(skb, jmp_table, 0); - return 0; + barrier_var(ret); + return ret; } __auxiliary SEC("tc") int classifier_0(struct __sk_buff *skb) { + int ret1, ret2; + count++; - subprog_tail(skb, &jmp_table0); - subprog_tail(skb, &jmp_table1); + ret1 = subprog_tail(skb, &jmp_table0); + ret2 = subprog_tail(skb, &jmp_table1); + __sink(ret1); + __sink(ret2); return count; } diff --git a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_hierarchy_fentry.c b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_hierarchy_fentry.c index 5261395713cd..6db9afee2095 100644 --- a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_hierarchy_fentry.c +++ b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf_hierarchy_fentry.c @@ -18,18 +18,25 @@ int count = 0; static __noinline int subprog_tail(void *ctx) { + int ret = 0; + bpf_tail_call_static(ctx, &jmp_table, 0); - return 0; + barrier_var(ret); + return ret; } SEC("fentry/dummy") int BPF_PROG(fentry, struct sk_buff *skb) { + int ret1, ret2; + clobber_regs_stack(); count++; - subprog_tail(ctx); - subprog_tail(ctx); + ret1 = subprog_tail(ctx); + ret2 = subprog_tail(ctx); + __sink(ret1); + __sink(ret2); return 0; } diff --git a/tools/testing/selftests/bpf/progs/tailcall_callback.c b/tools/testing/selftests/bpf/progs/tailcall_callback.c new file mode 100644 index 000000000000..14fa7a87028e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tailcall_callback.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_test_utils.h" + +int classifier_0(struct __sk_buff *skb); + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __array(values, void (void)); +} jmp_table SEC(".maps") = { + .values = { + [0] = (void *) &classifier_0, + }, +}; + +__auxiliary +SEC("tc") +int classifier_0(struct __sk_buff *skb) +{ + return 0; +} + +static __noinline +int subprog_tail0(struct __sk_buff *skb) +{ + int ret = 0; + + bpf_tail_call_static(skb, &jmp_table, 0); + barrier_var(ret); + return ret; +} + +static __noinline +int callback_loop(int index, void **cb_ctx) +{ + int ret; + + ret = subprog_tail0(*cb_ctx); + barrier_var(ret); + return ret ? 1 : 0; +} + +static __noinline +int callback_tail(int index, void **cb_ctx) +{ + bpf_tail_call_static(*cb_ctx, &jmp_table, 0); + return 0; +} + +static __noinline +int callback_empty(int index, void *data) +{ + return 0; +} + +/* callback involving subprog with tail call is rejected */ +SEC("tc") +__failure __msg("cannot tail call within callback") +int tailcall_callback_1(struct __sk_buff *skb) +{ + clobber_regs_stack(); + + bpf_loop(1, callback_loop, &skb, 0); + return 0; +} + +/* subprogs with tailcall do not affect no-tailcall callback */ +SEC("tc") +__success +__retval(0) +int tailcall_callback_2(struct __sk_buff *skb) +{ + int ret; + + clobber_regs_stack(); + + ret = subprog_tail0(skb); + __sink(ret); + + bpf_loop(1, callback_empty, NULL, 0); + return 0; +} + +/* callback with a direct tail call is rejected without a verifier bug */ +SEC("tc") +__failure __msg("cannot tail call within callback") +int tailcall_callback_3(struct __sk_buff *skb) +{ + bpf_loop(1, callback_tail, &skb, 0); + return 0; +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tailcall_cgrp_storage.c b/tools/testing/selftests/bpf/progs/tailcall_cgrp_storage.c new file mode 100644 index 000000000000..4dd3a0033d75 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tailcall_cgrp_storage.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); + __type(key, struct bpf_cgroup_storage_key); + __type(value, __u64); +} storage_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} prog_array SEC(".maps"); + +SEC("cgroup_skb/egress") +int caller_prog(struct __sk_buff *skb) +{ + __u64 *storage; + + storage = bpf_get_local_storage(&storage_map, 0); + if (storage) + *storage = 1; + + bpf_tail_call(skb, &prog_array, 0); + return 1; +} + +SEC("cgroup_skb/egress") +int callee_prog(struct __sk_buff *skb) +{ + __u64 *storage; + + storage = bpf_get_local_storage(&storage_map, 0); + if (storage) + *storage = 1; + + return 1; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tailcall_cgrp_storage_no_storage.c b/tools/testing/selftests/bpf/progs/tailcall_cgrp_storage_no_storage.c new file mode 100644 index 000000000000..5c69b0af6ff9 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tailcall_cgrp_storage_no_storage.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} prog_array SEC(".maps"); + +SEC("cgroup_skb/egress") +int caller_prog(struct __sk_buff *skb) +{ + bpf_tail_call(skb, &prog_array, 0); + return 1; +} + +SEC("cgroup_skb/egress") +int leaf_prog(struct __sk_buff *skb) +{ + return 1; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tailcall_cgrp_storage_owner.c b/tools/testing/selftests/bpf/progs/tailcall_cgrp_storage_owner.c new file mode 100644 index 000000000000..d7e8ec9855c5 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tailcall_cgrp_storage_owner.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); + __type(key, struct bpf_cgroup_storage_key); + __type(value, __u64); +} storage_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} prog_array SEC(".maps"); + +SEC("cgroup_skb/egress") +int prog_array_owner(struct __sk_buff *skb) +{ + __u64 *storage; + + storage = bpf_get_local_storage(&storage_map, 0); + if (storage) + *storage = 1; + + bpf_tail_call(skb, &prog_array, 0); + return 1; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_common.h b/tools/testing/selftests/bpf/progs/task_kfunc_common.h index e9c4fea7a4bb..a0c599b58c29 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_common.h +++ b/tools/testing/selftests/bpf/progs/task_kfunc_common.h @@ -20,12 +20,26 @@ struct { __uint(max_entries, 1); } __tasks_kfunc_map SEC(".maps"); +struct task_kptr_lock_value { + struct bpf_spin_lock lock; + struct task_struct __kptr * task; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct task_kptr_lock_value); + __uint(max_entries, 1); +} task_kptr_lock_map SEC(".maps"); + struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; void bpf_task_release(struct task_struct *p) __ksym; struct task_struct *bpf_task_from_pid(s32 pid) __ksym; struct task_struct *bpf_task_from_vpid(s32 vpid) __ksym; void bpf_rcu_read_lock(void) __ksym; void bpf_rcu_read_unlock(void) __ksym; +void bpf_local_irq_save(unsigned long *flags) __weak __ksym; +void bpf_local_irq_restore(unsigned long *flags) __weak __ksym; static inline struct __tasks_kfunc_map_value *tasks_kfunc_map_value_lookup(struct task_struct *p) { diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c index 4c07ea193f72..f96b0c13ed1a 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c @@ -5,6 +5,7 @@ #include <bpf/bpf_tracing.h> #include <bpf/bpf_helpers.h> +#include "../bpf_experimental.h" #include "bpf_misc.h" #include "task_kfunc_common.h" @@ -28,7 +29,7 @@ static struct __tasks_kfunc_map_value *insert_lookup_task(struct task_struct *ta } SEC("tp_btf/task_newtask") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(task_kfunc_acquire_untrusted, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired; @@ -49,7 +50,7 @@ int BPF_PROG(task_kfunc_acquire_untrusted, struct task_struct *task, u64 clone_f } SEC("tp_btf/task_newtask") -__failure __msg("arg#0 pointer type STRUCT task_struct must point") +__failure __msg("R1 is fp expected STRUCT task_struct") int BPF_PROG(task_kfunc_acquire_fp, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired, *stack_task = (struct task_struct *)&clone_flags; @@ -100,7 +101,7 @@ int BPF_PROG(task_kfunc_acquire_unsafe_kretprobe_rcu, struct task_struct *task, } SEC("tp_btf/task_newtask") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(task_kfunc_acquire_null, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired; @@ -149,7 +150,7 @@ int BPF_PROG(task_kfunc_xchg_unreleased, struct task_struct *task, u64 clone_fla } SEC("tp_btf/task_newtask") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(task_kfunc_acquire_release_no_null_check, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired; @@ -162,7 +163,7 @@ int BPF_PROG(task_kfunc_acquire_release_no_null_check, struct task_struct *task, } SEC("tp_btf/task_newtask") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(task_kfunc_release_untrusted, struct task_struct *task, u64 clone_flags) { struct __tasks_kfunc_map_value *v; @@ -178,7 +179,7 @@ int BPF_PROG(task_kfunc_release_untrusted, struct task_struct *task, u64 clone_f } SEC("tp_btf/task_newtask") -__failure __msg("arg#0 pointer type STRUCT task_struct must point") +__failure __msg("release kfunc bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(task_kfunc_release_fp, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired = (struct task_struct *)&clone_flags; @@ -190,7 +191,7 @@ int BPF_PROG(task_kfunc_release_fp, struct task_struct *task, u64 clone_flags) } SEC("tp_btf/task_newtask") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(task_kfunc_release_null, struct task_struct *task, u64 clone_flags) { struct __tasks_kfunc_map_value local, *v; @@ -224,7 +225,7 @@ int BPF_PROG(task_kfunc_release_null, struct task_struct *task, u64 clone_flags) } SEC("tp_btf/task_newtask") -__failure __msg("release kernel function bpf_task_release expects") +__failure __msg("release kfunc bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(task_kfunc_release_unacquired, struct task_struct *task, u64 clone_flags) { /* Cannot release trusted task pointer which was not acquired. */ @@ -234,7 +235,46 @@ int BPF_PROG(task_kfunc_release_unacquired, struct task_struct *task, u64 clone_ } SEC("tp_btf/task_newtask") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("bpf_obj_drop cannot be used in tracing programs on types with NMI unsafe fields") +int BPF_PROG(task_kfunc_obj_drop_with_kptr, struct task_struct *task, u64 clone_flags) +{ + struct __tasks_kfunc_map_value *local; + + local = bpf_obj_new(typeof(*local)); + if (!local) + return 0; + + bpf_obj_drop(local); + return 0; +} + +SEC("tp_btf/task_newtask") +__failure __msg("bpf_obj_drop cannot be used in tracing programs on types with NMI unsafe fields") +int BPF_PROG(task_kfunc_obj_drop_nmi_with_kptr, struct task_struct *task, + u64 clone_flags) +{ + struct __tasks_kfunc_map_value *local; + struct task_struct *acquired, *old; + + (void)clone_flags; + + local = bpf_obj_new(typeof(*local)); + if (!local) + return 0; + + acquired = bpf_task_acquire(task); + if (acquired) { + old = bpf_kptr_xchg(&local->task, acquired); + if (old) + bpf_task_release(old); + } + + bpf_obj_drop(local); + return 0; +} + +SEC("tp_btf/task_newtask") +__failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(task_kfunc_from_pid_no_null_check, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired; @@ -248,7 +288,7 @@ int BPF_PROG(task_kfunc_from_pid_no_null_check, struct task_struct *task, u64 cl } SEC("tp_btf/task_newtask") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(task_kfunc_from_vpid_no_null_check, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired; @@ -313,7 +353,7 @@ int BPF_PROG(task_access_comm4, struct task_struct *task, const char *buf, bool } SEC("tp_btf/task_newtask") -__failure __msg("R1 must be referenced or trusted") +__failure __msg("release kfunc bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(task_kfunc_release_in_map, struct task_struct *task, u64 clone_flags) { struct task_struct *local; @@ -338,3 +378,76 @@ int BPF_PROG(task_kfunc_release_in_map, struct task_struct *task, u64 clone_flag return 0; } + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__failure __msg("R1 must be a rcu pointer") +int BPF_PROG(task_kfunc_acquire_after_final_spin_unlock) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_spin_lock(&v->lock); + task = v->task; + bpf_spin_unlock(&v->lock); + if (!task) + return 0; + + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + return 0; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__failure __msg("R1 must be a rcu pointer") +int BPF_PROG(task_kfunc_acquire_after_preempt_enable) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_preempt_disable(); + task = v->task; + bpf_preempt_enable(); + if (!task) + return 0; + + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + return 0; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__failure __msg("R1 must be a rcu pointer") +int BPF_PROG(task_kfunc_acquire_after_irq_restore) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + unsigned long flags; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_local_irq_save(&flags); + task = v->task; + bpf_local_irq_restore(&flags); + if (!task) + return 0; + + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_success.c b/tools/testing/selftests/bpf/progs/task_kfunc_success.c index 5fb4fc19d26a..6545b124dee1 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_success.c +++ b/tools/testing/selftests/bpf/progs/task_kfunc_success.c @@ -6,6 +6,7 @@ #include <bpf/bpf_helpers.h> #include "../bpf_experimental.h" +#include "bpf_misc.h" #include "task_kfunc_common.h" char _license[] SEC("license") = "GPL"; @@ -140,17 +141,17 @@ int BPF_PROG(test_task_acquire_leave_in_map, struct task_struct *task, u64 clone return 0; } -SEC("tp_btf/task_newtask") -int BPF_PROG(test_task_xchg_release, struct task_struct *task, u64 clone_flags) +SEC("syscall") +int test_task_xchg_release(const void *ctx) { - struct task_struct *kptr, *acquired; + struct task_struct *task, *kptr, *acquired; struct __tasks_kfunc_map_value *v, *local; int refcnt, refcnt_after_drop; long status; - if (!is_test_kfunc_task()) - return 0; + (void)ctx; + task = bpf_get_current_task_btf(); status = tasks_kfunc_map_insert(task); if (status) { err = 1; @@ -191,7 +192,7 @@ int BPF_PROG(test_task_xchg_release, struct task_struct *task, u64 clone_flags) return 0; } - /* Stash a copy into local kptr and check if it is released recursively */ + /* Stash a copy into local kptr and check if it is released recursively. */ acquired = bpf_task_acquire(kptr); if (!acquired) { err = 7; @@ -220,7 +221,6 @@ int BPF_PROG(test_task_xchg_release, struct task_struct *task, u64 clone_flags) } bpf_task_release(kptr); - return 0; } @@ -367,6 +367,200 @@ int BPF_PROG(task_kfunc_acquire_trusted_walked, struct task_struct *task, u64 cl return 0; } +SEC("fentry/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_spin_unlock_non_sleepable) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_spin_lock(&v->lock); + task = v->task; + bpf_spin_unlock(&v->lock); + if (!task) + return 0; + + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_spin_unlock_explicit_rcu) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_rcu_read_lock(); + bpf_spin_lock(&v->lock); + task = v->task; + bpf_spin_unlock(&v->lock); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_rcu_read_unlock(); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_spin_unlock_preempt_disabled) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_preempt_disable(); + bpf_spin_lock(&v->lock); + task = v->task; + bpf_spin_unlock(&v->lock); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_preempt_enable(); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_spin_unlock_irq_disabled) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + unsigned long flags; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_local_irq_save(&flags); + bpf_spin_lock(&v->lock); + task = v->task; + bpf_spin_unlock(&v->lock); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_local_irq_restore(&flags); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_rcu_unlock_preempt_disabled) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_preempt_disable(); + bpf_rcu_read_lock(); + task = v->task; + bpf_rcu_read_unlock(); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_preempt_enable(); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_rcu_unlock_irq_disabled) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + unsigned long flags; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_local_irq_save(&flags); + bpf_rcu_read_lock(); + task = v->task; + bpf_rcu_read_unlock(); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_local_irq_restore(&flags); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_preempt_enable_explicit_rcu) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_preempt_disable(); + task = v->task; + bpf_rcu_read_lock(); + bpf_preempt_enable(); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_rcu_read_unlock(); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_irq_restore_explicit_rcu) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + unsigned long flags; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_local_irq_save(&flags); + task = v->task; + bpf_rcu_read_lock(); + bpf_local_irq_restore(&flags); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_rcu_read_unlock(); + return 0; +} + SEC("syscall") int test_task_from_vpid_current(const void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/task_local_data.bpf.h b/tools/testing/selftests/bpf/progs/task_local_data.bpf.h index fed53d63a7e5..a31a399870be 100644 --- a/tools/testing/selftests/bpf/progs/task_local_data.bpf.h +++ b/tools/testing/selftests/bpf/progs/task_local_data.bpf.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ #ifndef __TASK_LOCAL_DATA_BPF_H #define __TASK_LOCAL_DATA_BPF_H @@ -61,6 +61,7 @@ #define TLD_ROUND_UP(x, y) ((((x) - 1) | TLD_ROUND_MASK(x, y)) + 1) #define TLD_MAX_DATA_CNT (__PAGE_SIZE / sizeof(struct tld_metadata) - 1) +#define TLD_DATA_SIZE (__PAGE_SIZE - sizeof(__u64)) #ifndef TLD_NAME_LEN #define TLD_NAME_LEN 62 @@ -86,13 +87,14 @@ struct tld_meta_u { }; struct tld_data_u { - __u64 start; /* offset of tld_data_u->data in a page */ - char data[__PAGE_SIZE - sizeof(__u64)]; + __u64 unused; + char data[__PAGE_SIZE - sizeof(__u64)] __attribute__((aligned(8))); }; struct tld_map_value { struct tld_data_u __uptr *data; struct tld_meta_u __uptr *meta; + __u16 start; /* offset of tld_data_u->data in a page */ }; typedef struct tld_uptr_dummy { @@ -176,7 +178,7 @@ static int __tld_fetch_key(struct tld_object *tld_obj, const char *name, int i_s if (!tld_obj->data_map || !tld_obj->data_map->data || !tld_obj->data_map->meta) return 0; - start = tld_obj->data_map->data->start; + start = tld_obj->data_map->start; cnt = tld_obj->data_map->meta->cnt; metadata = tld_obj->data_map->meta->metadata; @@ -188,6 +190,8 @@ static int __tld_fetch_key(struct tld_object *tld_obj, const char *name, int i_s return start + off; off += TLD_ROUND_UP(metadata[i].size, 8); + if (off > TLD_DATA_SIZE) + break; } return -cnt; diff --git a/tools/testing/selftests/bpf/progs/task_local_storage.c b/tools/testing/selftests/bpf/progs/task_local_storage.c index 80a0a20db88d..34fa3d6451d2 100644 --- a/tools/testing/selftests/bpf/progs/task_local_storage.c +++ b/tools/testing/selftests/bpf/progs/task_local_storage.c @@ -14,12 +14,15 @@ struct { __type(value, long); } enter_id SEC(".maps"); +#include "err.h" + #define MAGIC_VALUE 0xabcd1234 pid_t target_pid = 0; int mismatch_cnt = 0; int enter_cnt = 0; int exit_cnt = 0; +long update_err = 0; SEC("tp_btf/sys_enter") int BPF_PROG(on_enter, struct pt_regs *regs, long id) @@ -62,3 +65,19 @@ int BPF_PROG(on_exit, struct pt_regs *regs, long id) __sync_fetch_and_add(&mismatch_cnt, 1); return 0; } + +SEC("fexit/bpf_local_storage_update") +int BPF_PROG(fexit_update, void *owner, struct bpf_local_storage_map *smap, + void *value, u64 map_flags, bool swap_uptrs, + struct bpf_local_storage_data *ret) +{ + struct task_struct *task = bpf_get_current_task_btf(); + + if (task->pid != target_pid) + return 0; + + if (IS_ERR_VALUE(ret)) + update_err = PTR_ERR(ret); + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/task_work_fail.c b/tools/testing/selftests/bpf/progs/task_work_fail.c index 82e4b8913333..3186e7b4b24e 100644 --- a/tools/testing/selftests/bpf/progs/task_work_fail.c +++ b/tools/testing/selftests/bpf/progs/task_work_fail.c @@ -58,7 +58,7 @@ int mismatch_map(struct pt_regs *args) } SEC("perf_event") -__failure __msg("arg#1 doesn't point to a map value") +__failure __msg("R2 doesn't point to a map value") int no_map_task_work(struct pt_regs *args) { struct task_struct *task; @@ -70,7 +70,7 @@ int no_map_task_work(struct pt_regs *args) } SEC("perf_event") -__failure __msg("Possibly NULL pointer passed to trusted arg1") +__failure __msg("Possibly NULL pointer passed to trusted R2") int task_work_null(struct pt_regs *args) { struct task_struct *task; @@ -81,7 +81,7 @@ int task_work_null(struct pt_regs *args) } SEC("perf_event") -__failure __msg("Possibly NULL pointer passed to trusted arg2") +__failure __msg("Possibly NULL pointer passed to trusted R3") int map_null(struct pt_regs *args) { struct elem *work; diff --git a/tools/testing/selftests/bpf/progs/tcp_ca_kfunc.c b/tools/testing/selftests/bpf/progs/tcp_ca_kfunc.c index f95862f570b7..0a3e9d35bf6f 100644 --- a/tools/testing/selftests/bpf/progs/tcp_ca_kfunc.c +++ b/tools/testing/selftests/bpf/progs/tcp_ca_kfunc.c @@ -8,7 +8,7 @@ extern void bbr_init(struct sock *sk) __ksym; extern void bbr_main(struct sock *sk, u32 ack, int flag, const struct rate_sample *rs) __ksym; extern u32 bbr_sndbuf_expand(struct sock *sk) __ksym; extern u32 bbr_undo_cwnd(struct sock *sk) __ksym; -extern void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event) __ksym; +extern void bbr_cwnd_event_tx_start(struct sock *sk) __ksym; extern u32 bbr_ssthresh(struct sock *sk) __ksym; extern u32 bbr_min_tso_segs(struct sock *sk) __ksym; extern void bbr_set_state(struct sock *sk, u8 new_state) __ksym; @@ -16,6 +16,7 @@ extern void bbr_set_state(struct sock *sk, u8 new_state) __ksym; extern void dctcp_init(struct sock *sk) __ksym; extern void dctcp_update_alpha(struct sock *sk, u32 flags) __ksym; extern void dctcp_cwnd_event(struct sock *sk, enum tcp_ca_event ev) __ksym; +extern void dctcp_cwnd_event_tx_start(struct sock *sk) __ksym; extern u32 dctcp_ssthresh(struct sock *sk) __ksym; extern u32 dctcp_cwnd_undo(struct sock *sk) __ksym; extern void dctcp_state(struct sock *sk, u8 new_state) __ksym; @@ -24,7 +25,7 @@ extern void cubictcp_init(struct sock *sk) __ksym; extern u32 cubictcp_recalc_ssthresh(struct sock *sk) __ksym; extern void cubictcp_cong_avoid(struct sock *sk, u32 ack, u32 acked) __ksym; extern void cubictcp_state(struct sock *sk, u8 new_state) __ksym; -extern void cubictcp_cwnd_event(struct sock *sk, enum tcp_ca_event event) __ksym; +extern void cubictcp_cwnd_event_tx_start(struct sock *sk) __ksym; extern void cubictcp_acked(struct sock *sk, const struct ack_sample *sample) __ksym; SEC("struct_ops") @@ -69,9 +70,15 @@ u32 BPF_PROG(undo_cwnd, struct sock *sk) SEC("struct_ops") void BPF_PROG(cwnd_event, struct sock *sk, enum tcp_ca_event event) { - bbr_cwnd_event(sk, event); dctcp_cwnd_event(sk, event); - cubictcp_cwnd_event(sk, event); +} + +SEC("struct_ops") +void BPF_PROG(cwnd_event_tx_start, struct sock *sk) +{ + bbr_cwnd_event_tx_start(sk); + dctcp_cwnd_event_tx_start(sk); + cubictcp_cwnd_event_tx_start(sk); } SEC("struct_ops") @@ -111,6 +118,7 @@ struct tcp_congestion_ops tcp_ca_kfunc = { .sndbuf_expand = (void *)sndbuf_expand, .undo_cwnd = (void *)undo_cwnd, .cwnd_event = (void *)cwnd_event, + .cwnd_event_tx_start = (void *)cwnd_event_tx_start, .ssthresh = (void *)ssthresh, .min_tso_segs = (void *)min_tso_segs, .set_state = (void *)set_state, diff --git a/tools/testing/selftests/bpf/progs/tcp_ca_untrusted_btf_write.c b/tools/testing/selftests/bpf/progs/tcp_ca_untrusted_btf_write.c new file mode 100644 index 000000000000..eda4697aac80 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tcp_ca_untrusted_btf_write.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "bpf_tracing_net.h" +#include <bpf/bpf_core_read.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +SEC("struct_ops") +void BPF_PROG(untrusted_btf_write_init, struct sock *sk) +{ + struct tcp_sock *tp; + int v = 1; + void *p; + + p = bpf_rdonly_cast(&v, 0); + tp = bpf_rdonly_cast(p, bpf_core_type_id_kernel(struct tcp_sock)); + tp->snd_cwnd = 1; +} + +SEC(".struct_ops") +struct tcp_congestion_ops untrusted_btf_write = { + .init = (void *)untrusted_btf_write_init, + .name = "bpf_ro_btf", +}; diff --git a/tools/testing/selftests/bpf/progs/test_access_variable_array.c b/tools/testing/selftests/bpf/progs/test_access_variable_array.c deleted file mode 100644 index 326b7d1f496a..000000000000 --- a/tools/testing/selftests/bpf/progs/test_access_variable_array.c +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2023 Bytedance */ - -#include "vmlinux.h" -#include <bpf/bpf_helpers.h> -#include <bpf/bpf_tracing.h> - -unsigned long span = 0; - -SEC("fentry/sched_balance_rq") -int BPF_PROG(fentry_fentry, int this_cpu, struct rq *this_rq, - struct sched_domain *sd) -{ - span = sd->span[0]; - - return 0; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_bpf_nf.c b/tools/testing/selftests/bpf/progs/test_bpf_nf.c index 076fbf03a126..eda9b7bbab75 100644 --- a/tools/testing/selftests/bpf/progs/test_bpf_nf.c +++ b/tools/testing/selftests/bpf/progs/test_bpf_nf.c @@ -10,6 +10,8 @@ #define EINVAL 22 #define ENOENT 2 +#define CT_OPTS_ERROR_GUARD 0x12345678 + #define NF_CT_ZONE_DIR_ORIG (1 << IP_CT_DIR_ORIGINAL) #define NF_CT_ZONE_DIR_REPL (1 << IP_CT_DIR_REPLY) @@ -19,6 +21,8 @@ int test_einval_reserved = 0; int test_einval_reserved_new = 0; int test_einval_netns_id = 0; int test_einval_len_opts = 0; +int test_einval_len_opts_small_lookup = 0; +int test_einval_len_opts_small_alloc = 0; int test_eproto_l4proto = 0; int test_enonet_netns_id = 0; int test_enoent_lookup = 0; @@ -124,6 +128,28 @@ nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32, else test_einval_len_opts = opts_def.error; + opts_def.error = CT_OPTS_ERROR_GUARD; + ct = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, + sizeof(opts_def.netns_id)); + if (ct) { + bpf_ct_release(ct); + test_einval_len_opts_small_lookup = -EINVAL; + } else { + test_einval_len_opts_small_lookup = opts_def.error; + } + + opts_def.error = CT_OPTS_ERROR_GUARD; + ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, + sizeof(opts_def.netns_id)); + if (ct) { + ct = bpf_ct_insert_entry(ct); + if (ct) + bpf_ct_release(ct); + test_einval_len_opts_small_alloc = -EINVAL; + } else { + test_einval_len_opts_small_alloc = opts_def.error; + } + opts_def.l4proto = IPPROTO_ICMP; ct = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); @@ -164,8 +190,8 @@ nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32, ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); if (ct) { - __u16 sport = bpf_get_prandom_u32(); - __u16 dport = bpf_get_prandom_u32(); + __u16 sport = bpf_get_prandom_u32() % 65535 + 1; + __u16 dport = bpf_get_prandom_u32() % 65535 + 1; union nf_inet_addr saddr = {}; union nf_inet_addr daddr = {}; struct nf_conn *ct_ins; @@ -267,8 +293,8 @@ nf_ct_opts_new_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple * ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); if (ct) { - __u16 sport = bpf_get_prandom_u32(); - __u16 dport = bpf_get_prandom_u32(); + __u16 sport = bpf_get_prandom_u32() % 65535 + 1; + __u16 dport = bpf_get_prandom_u32() % 65535 + 1; union nf_inet_addr saddr = {}; union nf_inet_addr daddr = {}; struct nf_conn *ct_ins; diff --git a/tools/testing/selftests/bpf/progs/test_bpf_nf_fail.c b/tools/testing/selftests/bpf/progs/test_bpf_nf_fail.c index 2c156cd166af..332cda89caba 100644 --- a/tools/testing/selftests/bpf/progs/test_bpf_nf_fail.c +++ b/tools/testing/selftests/bpf/progs/test_bpf_nf_fail.c @@ -152,7 +152,7 @@ int change_status_after_alloc(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("Possibly NULL pointer passed to trusted arg1") +__failure __msg("Possibly NULL pointer passed to trusted R2") int lookup_null_bpf_tuple(struct __sk_buff *ctx) { struct bpf_ct_opts___local opts = {}; @@ -165,7 +165,7 @@ int lookup_null_bpf_tuple(struct __sk_buff *ctx) } SEC("?tc") -__failure __msg("Possibly NULL pointer passed to trusted arg3") +__failure __msg("Possibly NULL pointer passed to trusted R4") int lookup_null_bpf_opts(struct __sk_buff *ctx) { struct bpf_sock_tuple tup = {}; @@ -178,7 +178,7 @@ int lookup_null_bpf_opts(struct __sk_buff *ctx) } SEC("?xdp") -__failure __msg("Possibly NULL pointer passed to trusted arg1") +__failure __msg("Possibly NULL pointer passed to trusted R2") int xdp_lookup_null_bpf_tuple(struct xdp_md *ctx) { struct bpf_ct_opts___local opts = {}; @@ -191,7 +191,7 @@ int xdp_lookup_null_bpf_tuple(struct xdp_md *ctx) } SEC("?xdp") -__failure __msg("Possibly NULL pointer passed to trusted arg3") +__failure __msg("Possibly NULL pointer passed to trusted R4") int xdp_lookup_null_bpf_opts(struct xdp_md *ctx) { struct bpf_sock_tuple tup = {}; diff --git a/tools/testing/selftests/bpf/progs/test_dst_clear.c b/tools/testing/selftests/bpf/progs/test_dst_clear.c new file mode 100644 index 000000000000..c22a6eeb4798 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_dst_clear.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include "bpf_tracing_net.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> + +#define UDP_TEST_PORT 7777 + +void *bpf_cast_to_kern_ctx(void *) __ksym; + +bool had_dst = false; +bool dst_cleared = false; + +SEC("tc/egress") +int dst_clear(struct __sk_buff *skb) +{ + struct sk_buff *kskb; + struct iphdr iph; + struct udphdr udph; + int err; + + if (skb->protocol != __bpf_constant_htons(ETH_P_IP)) + return TC_ACT_OK; + + if (bpf_skb_load_bytes(skb, ETH_HLEN, &iph, sizeof(iph))) + return TC_ACT_OK; + + if (iph.protocol != IPPROTO_UDP) + return TC_ACT_OK; + + if (bpf_skb_load_bytes(skb, ETH_HLEN + sizeof(iph), &udph, sizeof(udph))) + return TC_ACT_OK; + + if (udph.dest != __bpf_constant_htons(UDP_TEST_PORT)) + return TC_ACT_OK; + + kskb = bpf_cast_to_kern_ctx(skb); + had_dst = (kskb->_skb_refdst != 0); + + /* Same-protocol encap (IPIP): protocol stays IPv4, but the dst + * from the original routing is no longer valid for the outer hdr. + */ + err = bpf_skb_adjust_room(skb, (s32)sizeof(struct iphdr), + BPF_ADJ_ROOM_MAC, + BPF_F_ADJ_ROOM_FIXED_GSO | + BPF_F_ADJ_ROOM_ENCAP_L3_IPV4); + if (err) + return TC_ACT_SHOT; + + dst_cleared = (kskb->_skb_refdst == 0); + + return TC_ACT_SHOT; +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_fill_link_info.c b/tools/testing/selftests/bpf/progs/test_fill_link_info.c index fac33a14f200..c85081538e93 100644 --- a/tools/testing/selftests/bpf/progs/test_fill_link_info.c +++ b/tools/testing/selftests/bpf/progs/test_fill_link_info.c @@ -12,7 +12,7 @@ extern bool CONFIG_PPC64 __kconfig __weak; /* This function is here to have CONFIG_X86_KERNEL_IBT, * CONFIG_PPC_FTRACE_OUT_OF_LINE, CONFIG_KPROBES_ON_FTRACE, - * CONFIG_PPC6 used and added to object BTF. + * CONFIG_PPC64 used and added to object BTF. */ int unused(void) { @@ -58,4 +58,10 @@ int BPF_PROG(umulti_run) return 0; } +SEC("fentry.multi") +int BPF_PROG(tmulti_run) +{ + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_global_func3.c b/tools/testing/selftests/bpf/progs/test_global_func3.c index 142b682d3c2f..b66abb350fb0 100644 --- a/tools/testing/selftests/bpf/progs/test_global_func3.c +++ b/tools/testing/selftests/bpf/progs/test_global_func3.c @@ -5,57 +5,105 @@ #include <bpf/bpf_helpers.h> #include "bpf_misc.h" -__attribute__ ((noinline)) +static __attribute__ ((noinline)) int f1(struct __sk_buff *skb) { return skb->len; } -__attribute__ ((noinline)) +static __attribute__ ((noinline)) int f2(int val, struct __sk_buff *skb) { return f1(skb) + val; } -__attribute__ ((noinline)) +static __attribute__ ((noinline)) int f3(int val, struct __sk_buff *skb, int var) { return f2(var, skb) + val; } -__attribute__ ((noinline)) +static __attribute__ ((noinline)) int f4(struct __sk_buff *skb) { return f3(1, skb, 2); } -__attribute__ ((noinline)) +static __attribute__ ((noinline)) int f5(struct __sk_buff *skb) { return f4(skb); } -__attribute__ ((noinline)) +static __attribute__ ((noinline)) int f6(struct __sk_buff *skb) { return f5(skb); } -__attribute__ ((noinline)) +static __attribute__ ((noinline)) int f7(struct __sk_buff *skb) { return f6(skb); } -__attribute__ ((noinline)) +static __attribute__ ((noinline)) int f8(struct __sk_buff *skb) { return f7(skb); } +static __attribute__ ((noinline)) +int f9(struct __sk_buff *skb) +{ + return f8(skb); +} + +static __attribute__ ((noinline)) +int f10(struct __sk_buff *skb) +{ + return f9(skb); +} + +static __attribute__ ((noinline)) +int f11(struct __sk_buff *skb) +{ + return f10(skb); +} + +static __attribute__ ((noinline)) +int f12(struct __sk_buff *skb) +{ + return f11(skb); +} + +static __attribute__ ((noinline)) +int f13(struct __sk_buff *skb) +{ + return f12(skb); +} + +static __attribute__ ((noinline)) +int f14(struct __sk_buff *skb) +{ + return f13(skb); +} + +static __attribute__ ((noinline)) +int f15(struct __sk_buff *skb) +{ + return f14(skb); +} + +static __attribute__ ((noinline)) +int f16(struct __sk_buff *skb) +{ + return f15(skb); +} + SEC("tc") -__failure __msg("the call stack of 8 frames") +__failure __msg("the call stack of 17 frames") int global_func3(struct __sk_buff *skb) { - return f8(skb); + return f16(skb); } diff --git a/tools/testing/selftests/bpf/progs/test_global_func7.c b/tools/testing/selftests/bpf/progs/test_global_func7.c index f182febfde3c..9e59625c1c92 100644 --- a/tools/testing/selftests/bpf/progs/test_global_func7.c +++ b/tools/testing/selftests/bpf/progs/test_global_func7.c @@ -12,7 +12,7 @@ void foo(struct __sk_buff *skb) } SEC("tc") -__failure __msg("foo() doesn't return scalar") +__success int global_func7(struct __sk_buff *skb) { foo(skb); diff --git a/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c b/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c new file mode 100644 index 000000000000..1b634b543b62 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Meta Platforms, Inc and affiliates. */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* + * Macro tricks to tersely define for long non-recursive call chains. Add + * computation to the functions prevent tail recursion from reducing the + * stack size to 0. + */ + +#define CAT(a, b) a ## b +#define XCAT(a, b) CAT(a, b) + +#define F_0 \ +__attribute__((noinline)) \ +int f0(unsigned long a) \ +{ \ + volatile long b = a + 16; \ + if (a == 0) \ + return 0; \ + return b; \ +} + +#define FN(n, prev) \ +__attribute__((noinline)) \ +int XCAT(f, n)(unsigned long a) \ +{ \ + volatile long b = XCAT(f, prev)(a - 1); \ + if (!b) \ + return 0; \ + return b + 1; \ +} + +/* Call chain 33 levels deep. */ +#define F_1 F_0 FN(1, 0) +#define F_2 F_1 FN(2, 1) +#define F_3 F_2 FN(3, 2) +#define F_4 F_3 FN(4, 3) +#define F_5 F_4 FN(5, 4) +#define F_6 F_5 FN(6, 5) +#define F_7 F_6 FN(7, 6) +#define F_8 F_7 FN(8, 7) +#define F_9 F_8 FN(9, 8) +#define F_10 F_9 FN(10, 9) +#define F_11 F_10 FN(11, 10) +#define F_12 F_11 FN(12, 11) +#define F_13 F_12 FN(13, 12) +#define F_14 F_13 FN(14, 13) +#define F_15 F_14 FN(15, 14) +#define F_16 F_15 FN(16, 15) +#define F_17 F_16 FN(17, 16) +#define F_18 F_17 FN(18, 17) +#define F_19 F_18 FN(19, 18) +#define F_20 F_19 FN(20, 19) +#define F_21 F_20 FN(21, 20) +#define F_22 F_21 FN(22, 21) +#define F_23 F_22 FN(23, 22) +#define F_24 F_23 FN(24, 23) +#define F_25 F_24 FN(25, 24) +#define F_26 F_25 FN(26, 25) +#define F_27 F_26 FN(27, 26) +#define F_28 F_27 FN(28, 27) +#define F_29 F_28 FN(29, 28) +#define F_30 F_29 FN(30, 29) +#define F_31 F_30 FN(31, 30) +#define F_32 F_31 FN(32, 31) + +#define CAT2(a, b) a ## b +#define XCAT2(a, b) CAT2(a, b) + +#define F(n) XCAT2(F_, n) + +F(32) + +/* Ensure that even 32 levels deep, the function verifies. */ +SEC("syscall") +__success +int global_func_deep_stack_success(struct __sk_buff *skb) +{ + return f31(55); +} + +/* + * Check we actually honor stack limits (33 * 16 = 528 > 512 = MAX_STACK_DEPTH). + * The stack depth is 16 because the verifier calls round_up_stack_depth() on + * the size. + */ +SEC("syscall") +__failure __msg("combined stack size of 34 calls") +int global_func_deep_stack_fail(struct __sk_buff *skb) +{ + return f32(123); +} diff --git a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c new file mode 100644 index 000000000000..5dc21b3b4cb5 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* Used for testing map name. */ +int loong SEC(".percpu.looooooooong"); +int data3 SEC(".data.percpu"); +int data2 SEC(".percpu.data"); + +int run; +/* cpu_id as array to verify map value resizing. */ +int cpu_id[1] SEC(".percpu"); +int data SEC(".percpu") = -1; +int nums[7] SEC(".percpu"); +bool set SEC(".percpu") = false; +struct { + char set; + int i; + int nums[7]; +} struct_data SEC(".percpu") = { + .set = 0, + .i = -1, +}; + +SEC("raw_tp/task_rename") +__auxiliary +int update_percpu_data(void *ctx) +{ + struct_data.nums[6] = 0xc0de; + struct_data.set = 1; + struct_data.i = 1; + nums[6] = 0xc0de; + data = 1; + run++; + set = true; + cpu_id[0] = bpf_get_smp_processor_id(); + return 0; +} + +static const char fmt[] SEC(".percpu.fmt") = "data %d\n"; + +SEC("?kprobe") +__failure __msg("R{{[0-9]+}} points to percpu_array map which cannot be used as const string") +int verifier_strncmp(void *ctx) +{ + return bpf_strncmp("test", 5, fmt); +} + +SEC("?kprobe") +__failure __msg("R{{[0-9]+}} points to percpu_array map which cannot be used as const string") +int verifier_snprintf(void *ctx) +{ + u64 args[] = { data }; + char buf[128]; + int len; + + len = bpf_snprintf(buf, sizeof(buf), fmt, args, sizeof(args)); + if (len > 0) + bpf_printk("snprintf: %s\n", buf); + return 0; +} + +volatile const __u32 num_cpus = 0; +volatile const int num_off; +volatile const int elem_sz; +__u32 sum = 0; +bool run_iter = false; + +SEC("iter/bpf_map_elem") +__auxiliary +int dump_percpu_data(struct bpf_iter__bpf_map_elem *ctx) +{ + void *pptr = ctx->value; + int i; + + if (!pptr) + return 0; + + run_iter = true; + + for (i = 0; i < num_cpus; i++) { + sum += *(int *) (pptr + num_off); + pptr += elem_sz; + } + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c b/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c index d249113ed657..bf48fc43c7ab 100644 --- a/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c +++ b/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c @@ -11,12 +11,7 @@ #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> #include "bpf_misc.h" - -extern struct bpf_key *bpf_lookup_system_key(__u64 id) __ksym; -extern void bpf_key_put(struct bpf_key *key) __ksym; -extern int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_ptr, - struct bpf_dynptr *sig_ptr, - struct bpf_key *trusted_keyring) __ksym; +#include "bpf_kfuncs.h" struct { __uint(type, BPF_MAP_TYPE_RINGBUF); @@ -38,14 +33,14 @@ SEC("?lsm.s/bpf") __failure __msg("cannot pass in dynptr at an offset=-8") int BPF_PROG(not_valid_dynptr, int cmd, union bpf_attr *attr, unsigned int size, bool kernel) { - unsigned long val; + unsigned long val = 0; return bpf_verify_pkcs7_signature((struct bpf_dynptr *)&val, (struct bpf_dynptr *)&val, NULL); } SEC("?lsm.s/bpf") -__failure __msg("arg#0 expected pointer to stack or const struct bpf_dynptr") +__failure __msg("R1 expected pointer to stack or const struct bpf_dynptr") int BPF_PROG(not_ptr_to_stack, int cmd, union bpf_attr *attr, unsigned int size, bool kernel) { static struct bpf_dynptr val; diff --git a/tools/testing/selftests/bpf/progs/test_kfunc_param_nullable.c b/tools/testing/selftests/bpf/progs/test_kfunc_param_nullable.c index 967081bbcfe1..ca35b92ea095 100644 --- a/tools/testing/selftests/bpf/progs/test_kfunc_param_nullable.c +++ b/tools/testing/selftests/bpf/progs/test_kfunc_param_nullable.c @@ -29,7 +29,7 @@ int kfunc_dynptr_nullable_test2(struct __sk_buff *skb) } SEC("tc") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") int kfunc_dynptr_nullable_test3(struct __sk_buff *skb) { struct bpf_dynptr data; diff --git a/tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c b/tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c index 7a6620671a83..cbe4284c032f 100644 --- a/tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c +++ b/tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c @@ -13,9 +13,9 @@ int bpf_decoder(unsigned int *sample) if (LIRC_IS_PULSE(*sample)) { unsigned int duration = LIRC_VALUE(*sample); - if (duration & 0x10000) + if (duration & 0x1000) bpf_rc_keydown(sample, 0x40, duration & 0xffff, 0); - if (duration & 0x20000) + if (duration & 0x2000) bpf_rc_pointer_rel(sample, (duration >> 8) & 0xff, duration & 0xff); } diff --git a/tools/testing/selftests/bpf/progs/test_lwt_ip_encap.c b/tools/testing/selftests/bpf/progs/test_lwt_ip_encap.c index d6cb986e7533..4a934fccf8f5 100644 --- a/tools/testing/selftests/bpf/progs/test_lwt_ip_encap.c +++ b/tools/testing/selftests/bpf/progs/test_lwt_ip_encap.c @@ -1,11 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 -#include <stddef.h> +#include "vmlinux.h" #include <string.h> -#include <linux/bpf.h> -#include <linux/ip.h> -#include <linux/ipv6.h> #include <bpf/bpf_helpers.h> #include <bpf/bpf_endian.h> +#include <bpf/bpf_tracing.h> struct grehdr { __be16 flags; @@ -64,13 +62,13 @@ int bpf_lwt_encap_gre6(struct __sk_buff *skb) hdr.ip6hdr.nexthdr = 47; /* IPPROTO_GRE */ hdr.ip6hdr.hop_limit = 0x40; /* fb01::1 */ - hdr.ip6hdr.saddr.s6_addr[0] = 0xfb; - hdr.ip6hdr.saddr.s6_addr[1] = 1; - hdr.ip6hdr.saddr.s6_addr[15] = 1; + hdr.ip6hdr.saddr.in6_u.u6_addr8[0] = 0xfb; + hdr.ip6hdr.saddr.in6_u.u6_addr8[1] = 1; + hdr.ip6hdr.saddr.in6_u.u6_addr8[15] = 1; /* fb10::1 */ - hdr.ip6hdr.daddr.s6_addr[0] = 0xfb; - hdr.ip6hdr.daddr.s6_addr[1] = 0x10; - hdr.ip6hdr.daddr.s6_addr[15] = 1; + hdr.ip6hdr.daddr.in6_u.u6_addr8[0] = 0xfb; + hdr.ip6hdr.daddr.in6_u.u6_addr8[1] = 0x10; + hdr.ip6hdr.daddr.in6_u.u6_addr8[15] = 1; hdr.greh.protocol = skb->protocol; @@ -82,4 +80,141 @@ int bpf_lwt_encap_gre6(struct __sk_buff *skb) return BPF_LWT_REROUTE; } +#define VXLAN_PORT 4789 +#define VXLAN_FLAGS 0x08000000 +#define VXLAN_VNI 1 + +#define ETH_ALEN 6 /* Octets in one ethernet addr */ +#define ETH_P_IP 0x0800 /* Internet Protocol packet */ +#define ETH_P_IPV6 0x86DD /* IPv6 over bluebook */ + +static const __u8 bcast[ETH_ALEN] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +static const __u8 srcmac[ETH_ALEN] = { + 0x02, 0x00, 0x00, 0x00, 0x00, 0x01, +}; + +SEC("encap_vxlan") +int bpf_lwt_encap_vxlan(struct __sk_buff *skb) +{ + struct encap_hdr { + struct iphdr iph; + struct udphdr udph; + struct vxlanhdr vxh; + struct ethhdr eth; + } __attribute__((__packed__)) hdr; + int err; + + memset(&hdr, 0, sizeof(hdr)); + + hdr.iph.ihl = 5; + hdr.iph.version = 4; + hdr.iph.ttl = 0x40; + hdr.iph.protocol = 17; /* IPPROTO_UDP */ + hdr.iph.tot_len = bpf_htons(skb->len + sizeof(hdr)); +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + hdr.iph.saddr = 0x640510ac; /* 172.16.5.100 */ + hdr.iph.daddr = 0x641110ac; /* 172.16.17.100 */ +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + hdr.iph.saddr = 0xac100564; /* 172.16.5.100 */ + hdr.iph.daddr = 0xac101164; /* 172.16.17.100 */ +#else +#error "Fix your compiler's __BYTE_ORDER__?!" +#endif + + hdr.udph.source = bpf_htons(VXLAN_PORT); + hdr.udph.dest = bpf_htons(VXLAN_PORT); + hdr.udph.len = bpf_htons(skb->len + sizeof(hdr.udph) + sizeof(hdr.vxh) + + sizeof(hdr.eth)); + + hdr.vxh.vx_flags = bpf_htonl(VXLAN_FLAGS); + hdr.vxh.vx_vni = bpf_htonl(VXLAN_VNI << 8); + + __builtin_memcpy(hdr.eth.h_dest, bcast, ETH_ALEN); + __builtin_memcpy(hdr.eth.h_source, srcmac, ETH_ALEN); + hdr.eth.h_proto = bpf_htons(ETH_P_IP); + + err = bpf_lwt_push_encap(skb, BPF_LWT_ENCAP_IP, &hdr, sizeof(hdr)); + if (err) + return BPF_DROP; + + return BPF_LWT_REROUTE; +} + +SEC("encap_vxlan6") +int bpf_lwt_encap_vxlan6(struct __sk_buff *skb) +{ + struct encap_hdr { + struct ipv6hdr ip6hdr; + struct udphdr udph; + struct vxlanhdr vxh; + struct ethhdr eth; + } __attribute__((__packed__)) hdr; + int err; + + memset(&hdr, 0, sizeof(hdr)); + + hdr.ip6hdr.version = 6; + hdr.ip6hdr.nexthdr = 17; /* IPPROTO_UDP */ + hdr.ip6hdr.hop_limit = 0x40; + hdr.ip6hdr.payload_len = bpf_htons(skb->len + sizeof(hdr.udph) + sizeof(hdr.vxh) + + sizeof(hdr.eth)); + /* fb05::1 */ + hdr.ip6hdr.saddr.in6_u.u6_addr8[0] = 0xfb; + hdr.ip6hdr.saddr.in6_u.u6_addr8[1] = 0x05; + hdr.ip6hdr.saddr.in6_u.u6_addr8[15] = 1; + /* fb11::1 */ + hdr.ip6hdr.daddr.in6_u.u6_addr8[0] = 0xfb; + hdr.ip6hdr.daddr.in6_u.u6_addr8[1] = 0x11; + hdr.ip6hdr.daddr.in6_u.u6_addr8[15] = 1; + + hdr.udph.source = bpf_htons(VXLAN_PORT); + hdr.udph.dest = bpf_htons(VXLAN_PORT); + hdr.udph.len = bpf_htons(skb->len + sizeof(hdr.udph) + sizeof(hdr.vxh) + + sizeof(hdr.eth)); + + hdr.vxh.vx_flags = bpf_htonl(VXLAN_FLAGS); + hdr.vxh.vx_vni = bpf_htonl(VXLAN_VNI << 8); + + __builtin_memcpy(hdr.eth.h_dest, bcast, ETH_ALEN); + __builtin_memcpy(hdr.eth.h_source, srcmac, ETH_ALEN); + hdr.eth.h_proto = bpf_htons(ETH_P_IPV6); + + err = bpf_lwt_push_encap(skb, BPF_LWT_ENCAP_IP, &hdr, sizeof(hdr)); + if (err) + return BPF_DROP; + + return BPF_LWT_REROUTE; +} + +volatile const int tgt_ip_version; + +__u16 transport_hdr = 0; +__u16 network_hdr = 0; +bool fexit_triggered = false; + +SEC("?fexit/bpf_lwt_push_ip_encap") +int BPF_PROG(fexit_lwt_push_ip_encap, struct sk_buff *skb, void *hdr, u32 len, bool ingress, + int retval) +{ + struct iphdr *iph; + + if (retval || fexit_triggered) + return 0; + + iph = (typeof(iph)) (skb->head + skb->network_header); + if (iph->version != tgt_ip_version) + return 0; + + if ((iph->version == 4 && iph->protocol == 17 /* IPPROTO_UDP */) || + (iph->version == 6 && ((struct ipv6hdr *)iph)->nexthdr == 17 /* IPPROTO_UDP */)) { + fexit_triggered = true; + transport_hdr = skb->transport_header; + network_hdr = skb->network_header; + } + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_misc_tcp_hdr_options.c b/tools/testing/selftests/bpf/progs/test_misc_tcp_hdr_options.c index d487153a839d..ed5a0011b863 100644 --- a/tools/testing/selftests/bpf/progs/test_misc_tcp_hdr_options.c +++ b/tools/testing/selftests/bpf/progs/test_misc_tcp_hdr_options.c @@ -29,6 +29,10 @@ unsigned int nr_syn = 0; unsigned int nr_fin = 0; unsigned int nr_hwtstamp = 0; +bool nodelay_est_ok = false; +bool nodelay_hdr_len_reject = false; +bool nodelay_write_hdr_reject = false; + /* Check the header received from the active side */ static int __check_active_hdr_in(struct bpf_sock_ops *skops, bool check_syn) { @@ -300,7 +304,7 @@ static int handle_passive_estab(struct bpf_sock_ops *skops) SEC("sockops") int misc_estab(struct bpf_sock_ops *skops) { - int true_val = 1; + int true_val = 1, false_val = 0, ret; switch (skops->op) { case BPF_SOCK_OPS_TCP_LISTEN_CB: @@ -316,10 +320,19 @@ int misc_estab(struct bpf_sock_ops *skops) case BPF_SOCK_OPS_PARSE_HDR_OPT_CB: return handle_parse_hdr(skops); case BPF_SOCK_OPS_HDR_OPT_LEN_CB: + ret = bpf_setsockopt(skops, SOL_TCP, TCP_NODELAY, &true_val, sizeof(true_val)); + if (ret == -EOPNOTSUPP) + nodelay_hdr_len_reject = true; return handle_hdr_opt_len(skops); case BPF_SOCK_OPS_WRITE_HDR_OPT_CB: + ret = bpf_setsockopt(skops, SOL_TCP, TCP_NODELAY, &true_val, sizeof(true_val)); + if (ret == -EOPNOTSUPP) + nodelay_write_hdr_reject = true; return handle_write_hdr_opt(skops); case BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB: + ret = bpf_setsockopt(skops, SOL_TCP, TCP_NODELAY, &false_val, sizeof(false_val)); + if (!ret) + nodelay_est_ok = true; return handle_passive_estab(skops); } diff --git a/tools/testing/selftests/bpf/progs/test_module_attach.c b/tools/testing/selftests/bpf/progs/test_module_attach.c index 03d7f89787a1..5609e388fb58 100644 --- a/tools/testing/selftests/bpf/progs/test_module_attach.c +++ b/tools/testing/selftests/bpf/progs/test_module_attach.c @@ -7,23 +7,21 @@ #include <bpf/bpf_core_read.h> #include "../test_kmods/bpf_testmod.h" -__u32 raw_tp_read_sz = 0; +__u32 sz = 0; -SEC("raw_tp/bpf_testmod_test_read") +SEC("?raw_tp/bpf_testmod_test_read") int BPF_PROG(handle_raw_tp, struct task_struct *task, struct bpf_testmod_test_read_ctx *read_ctx) { - raw_tp_read_sz = BPF_CORE_READ(read_ctx, len); + sz = BPF_CORE_READ(read_ctx, len); return 0; } -__u32 raw_tp_bare_write_sz = 0; - -SEC("raw_tp/bpf_testmod_test_write_bare_tp") +SEC("?raw_tp/bpf_testmod_test_write_bare_tp") int BPF_PROG(handle_raw_tp_bare, struct task_struct *task, struct bpf_testmod_test_write_ctx *write_ctx) { - raw_tp_bare_write_sz = BPF_CORE_READ(write_ctx, len); + sz = BPF_CORE_READ(write_ctx, len); return 0; } @@ -31,7 +29,7 @@ int raw_tp_writable_bare_in_val = 0; int raw_tp_writable_bare_early_ret = 0; int raw_tp_writable_bare_out_val = 0; -SEC("raw_tp.w/bpf_testmod_test_writable_bare_tp") +SEC("?raw_tp.w/bpf_testmod_test_writable_bare_tp") int BPF_PROG(handle_raw_tp_writable_bare, struct bpf_testmod_test_writable_ctx *writable) { @@ -41,76 +39,65 @@ int BPF_PROG(handle_raw_tp_writable_bare, return 0; } -__u32 tp_btf_read_sz = 0; - -SEC("tp_btf/bpf_testmod_test_read") +SEC("?tp_btf/bpf_testmod_test_read") int BPF_PROG(handle_tp_btf, struct task_struct *task, struct bpf_testmod_test_read_ctx *read_ctx) { - tp_btf_read_sz = read_ctx->len; + sz = read_ctx->len; return 0; } -__u32 fentry_read_sz = 0; - -SEC("fentry/bpf_testmod_test_read") +SEC("?fentry/bpf_testmod_test_read") int BPF_PROG(handle_fentry, struct file *file, struct kobject *kobj, struct bin_attribute *bin_attr, char *buf, loff_t off, size_t len) { - fentry_read_sz = len; + sz = len; return 0; } -__u32 fentry_manual_read_sz = 0; - -SEC("fentry") +SEC("?fentry") int BPF_PROG(handle_fentry_manual, struct file *file, struct kobject *kobj, struct bin_attribute *bin_attr, char *buf, loff_t off, size_t len) { - fentry_manual_read_sz = len; + sz = len; return 0; } -__u32 fentry_explicit_read_sz = 0; - -SEC("fentry/bpf_testmod:bpf_testmod_test_read") +SEC("?fentry/bpf_testmod:bpf_testmod_test_read") int BPF_PROG(handle_fentry_explicit, struct file *file, struct kobject *kobj, struct bin_attribute *bin_attr, char *buf, loff_t off, size_t len) { - fentry_explicit_read_sz = len; + sz = len; return 0; } -__u32 fentry_explicit_manual_read_sz = 0; - -SEC("fentry") +SEC("?fentry") int BPF_PROG(handle_fentry_explicit_manual, struct file *file, struct kobject *kobj, struct bin_attribute *bin_attr, char *buf, loff_t off, size_t len) { - fentry_explicit_manual_read_sz = len; + sz = len; return 0; } -__u32 fexit_read_sz = 0; -int fexit_ret = 0; +int retval = 0; -SEC("fexit/bpf_testmod_test_read") +SEC("?fexit/bpf_testmod_test_read") int BPF_PROG(handle_fexit, struct file *file, struct kobject *kobj, struct bin_attribute *bin_attr, char *buf, loff_t off, size_t len, int ret) { - fexit_read_sz = len; - fexit_ret = ret; + sz = len; + retval = ret; return 0; } -SEC("fexit/bpf_testmod_return_ptr") +SEC("?fexit/bpf_testmod_return_ptr") int BPF_PROG(handle_fexit_ret, int arg, struct file *ret) { long buf = 0; @@ -122,18 +109,16 @@ int BPF_PROG(handle_fexit_ret, int arg, struct file *ret) return 0; } -__u32 fmod_ret_read_sz = 0; - -SEC("fmod_ret/bpf_testmod_test_read") +SEC("?fmod_ret/bpf_testmod_test_read") int BPF_PROG(handle_fmod_ret, struct file *file, struct kobject *kobj, struct bin_attribute *bin_attr, char *buf, loff_t off, size_t len) { - fmod_ret_read_sz = len; + sz = len; return 0; /* don't override the exit code */ } -SEC("kprobe.multi/bpf_testmod_test_read") +SEC("?kprobe.multi/bpf_testmod_test_read") int BPF_PROG(kprobe_multi) { return 0; diff --git a/tools/testing/selftests/bpf/progs/test_probe_user.c b/tools/testing/selftests/bpf/progs/test_probe_user.c index a8e501af9604..4bc86c7654b1 100644 --- a/tools/testing/selftests/bpf/progs/test_probe_user.c +++ b/tools/testing/selftests/bpf/progs/test_probe_user.c @@ -5,13 +5,22 @@ #include <bpf/bpf_core_read.h> #include "bpf_misc.h" -static struct sockaddr_in old; +struct test_pro_bss { + struct sockaddr_in old; + __u32 test_pid; +}; + +struct test_pro_bss bss; static int handle_sys_connect_common(struct sockaddr_in *uservaddr) { struct sockaddr_in new; + __u32 cur = bpf_get_current_pid_tgid() >> 32; + + if (bss.test_pid && cur != bss.test_pid) + return 0; - bpf_probe_read_user(&old, sizeof(old), uservaddr); + bpf_probe_read_user(&bss.old, sizeof(bss.old), uservaddr); __builtin_memset(&new, 0xab, sizeof(new)); bpf_probe_write_user(uservaddr, &new, sizeof(new)); diff --git a/tools/testing/selftests/bpf/progs/test_ringbuf_map_key.c b/tools/testing/selftests/bpf/progs/test_ringbuf_map_key.c index 21bb7da90ea5..0efafa927a3d 100644 --- a/tools/testing/selftests/bpf/progs/test_ringbuf_map_key.c +++ b/tools/testing/selftests/bpf/progs/test_ringbuf_map_key.c @@ -35,7 +35,7 @@ SEC("fentry/" SYS_PREFIX "sys_getpgid") int test_ringbuf_mem_map_key(void *ctx) { int cur_pid = bpf_get_current_pid_tgid() >> 32; - struct sample *sample, sample_copy; + struct sample *sample; int *lookup_val; if (cur_pid != pid) @@ -55,16 +55,11 @@ int test_ringbuf_mem_map_key(void *ctx) lookup_val = (int *)bpf_map_lookup_elem(&hash_map, sample); __sink(lookup_val); - /* workaround - memcpy is necessary so that verifier doesn't - * complain with: - * verifier internal error: more than one arg with ref_obj_id R3 - * when trying to do bpf_map_update_elem(&hash_map, sample, &sample->seq, BPF_ANY); - * + /* * Since bpf_map_lookup_elem above uses 'sample' as key, test using * sample field as value below */ - __builtin_memcpy(&sample_copy, sample, sizeof(struct sample)); - bpf_map_update_elem(&hash_map, &sample_copy, &sample->seq, BPF_ANY); + bpf_map_update_elem(&hash_map, sample, &sample->seq, BPF_ANY); bpf_ringbuf_submit(sample, 0); return 0; diff --git a/tools/testing/selftests/bpf/progs/test_signed_loader.c b/tools/testing/selftests/bpf/progs/test_signed_loader.c new file mode 100644 index 000000000000..50451a69b99a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_signed_loader.c @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> + +/* + * Minimal, map-less program. Driven through libbpf's gen_loader (gen_hash) + * by prog_tests/signed_loader.c so the generated light-skeleton loader can be + * exercised against good and tampered metadata, which the kernel now verifies + * at load time via the insns||metadata signature. A socket filter needs no + * load-time attach resolution, and having no maps keeps the generated loader's + * ctx trivial (0 maps, 1 prog). + */ +SEC("socket") +int probe(void *ctx) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_signed_loader_data.c b/tools/testing/selftests/bpf/progs/test_signed_loader_data.c new file mode 100644 index 000000000000..43e2074d0042 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_signed_loader_data.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> + +/* + * A single initialized global, so the generated loader has one internal + * (.data) map that it seeds with an initial value while loading. + * prog_tests/signed_loader.c uses this to check that a signed loader + * keeps the attested contents and ignores a ctx-supplied initial_value: + * the host cannot re-seed a signed program's maps through the loader ctx. + */ +__u64 magic = 0x5eed1234abad1deaULL; + +SEC("socket") +int probe(void *ctx) +{ + return (int)magic; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_signed_loader_lsm.c b/tools/testing/selftests/bpf/progs/test_signed_loader_lsm.c new file mode 100644 index 000000000000..575a9b7910c8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_signed_loader_lsm.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +__u32 monitored_tid; + +int sig_keyring_serial; +int sig_keyring_type; +int sig_verdict; +int seen; + +SEC("lsm/bpf_prog_load") +int BPF_PROG(inspect_prog_load, struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token, bool kernel) +{ + __u32 tid = bpf_get_current_pid_tgid() & 0xffffffff; + + if (!monitored_tid || tid != monitored_tid) + return 0; + + seen++; + sig_keyring_serial = prog->aux->sig.keyring_serial; + sig_keyring_type = prog->aux->sig.keyring_type; + sig_verdict = prog->aux->sig.verdict; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/test_signed_loader_map.c b/tools/testing/selftests/bpf/progs/test_signed_loader_map.c new file mode 100644 index 000000000000..4478ce6f1fd9 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_signed_loader_map.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> + +/* + * One explicit array map and no global variables, so the generated loader + * has exactly one map to create (no .rodata/.bss). prog_tests/signed_loader.c + * uses this to check that a signed loader ignores ctx-supplied max_entries: + * the map must keep its attested size (4), not whatever the host puts in + * the loader ctx. + */ +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 4); + __type(key, __u32); + __type(value, __u64); +} amap SEC(".maps"); + +SEC("socket") +int probe(void *ctx) +{ + __u32 key = 0; + __u64 *val = bpf_map_lookup_elem(&amap, &key); + + return val ? (int)*val : 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_sleepable_tracepoints.c b/tools/testing/selftests/bpf/progs/test_sleepable_tracepoints.c new file mode 100644 index 000000000000..254f7fd895d9 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_sleepable_tracepoints.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <asm/unistd.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_core_read.h> +#include <bpf/bpf_helpers.h> + +char _license[] SEC("license") = "GPL"; + +int target_pid; +int prog_triggered; +long err; +char copied_byte; + +static int copy_getcwd_arg(char *ubuf) +{ + err = bpf_copy_from_user(&copied_byte, sizeof(copied_byte), ubuf); + if (err) + return err; + + prog_triggered = 1; + return 0; +} + +SEC("tp_btf.s/sys_enter") +int BPF_PROG(handle_sys_enter_tp_btf, struct pt_regs *regs, long id) +{ + if ((bpf_get_current_pid_tgid() >> 32) != target_pid || + id != __NR_getcwd) + return 0; + + return copy_getcwd_arg((void *)PT_REGS_PARM1_SYSCALL(regs)); +} + +SEC("raw_tp.s/sys_enter") +int BPF_PROG(handle_sys_enter_raw_tp, struct pt_regs *regs, long id) +{ + if ((bpf_get_current_pid_tgid() >> 32) != target_pid || + id != __NR_getcwd) + return 0; + + return copy_getcwd_arg((void *)PT_REGS_PARM1_CORE_SYSCALL(regs)); +} + +SEC("tp.s/syscalls/sys_enter_getcwd") +int handle_sys_enter_tp(struct syscall_trace_enter *args) +{ + if ((bpf_get_current_pid_tgid() >> 32) != target_pid) + return 0; + + return copy_getcwd_arg((void *)args->args[0]); +} + +SEC("tp.s/syscalls/sys_exit_getcwd") +int handle_sys_exit_tp(struct syscall_trace_exit *args) +{ + struct pt_regs *regs; + + if ((bpf_get_current_pid_tgid() >> 32) != target_pid) + return 0; + + regs = (struct pt_regs *)bpf_task_pt_regs(bpf_get_current_task_btf()); + return copy_getcwd_arg((void *)PT_REGS_PARM1_CORE_SYSCALL(regs)); +} + +SEC("raw_tp.s") +int BPF_PROG(handle_raw_tp_bare, struct pt_regs *regs, long id) +{ + return 0; +} + +SEC("tp.s") +int handle_tp_bare(void *ctx) +{ + return 0; +} + +SEC("tracepoint.s/syscalls/sys_enter_getcwd") +int handle_sys_enter_tp_alias(struct syscall_trace_enter *args) +{ + return 0; +} + +SEC("raw_tracepoint.s/sys_enter") +int BPF_PROG(handle_sys_enter_raw_tp_alias, struct pt_regs *regs, long id) +{ + return 0; +} + +SEC("raw_tp.s/sys_enter") +int BPF_PROG(handle_test_run, struct pt_regs *regs, long id) +{ + if ((__u64)regs == 0x1234ULL && (__u64)id == 0x5678ULL) + return (__u64)regs + (__u64)id; + + return 0; +} + +SEC("raw_tp.s/sched_switch") +int BPF_PROG(handle_raw_tp_non_faultable, bool preempt, + struct task_struct *prev, struct task_struct *next) +{ + return 0; +} + +SEC("tp.s/sched/sched_switch") +int handle_tp_non_syscall(void *ctx) +{ + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/test_sleepable_tracepoints_fail.c b/tools/testing/selftests/bpf/progs/test_sleepable_tracepoints_fail.c new file mode 100644 index 000000000000..1a0748a9520b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_sleepable_tracepoints_fail.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +/* Sleepable program on a non-faultable tracepoint should fail to load */ +SEC("tp_btf.s/sched_switch") +__failure __msg("Sleepable program cannot attach to non-faultable tracepoint") +int BPF_PROG(handle_sched_switch, bool preempt, + struct task_struct *prev, struct task_struct *next) +{ + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/test_sockmap_kern.h b/tools/testing/selftests/bpf/progs/test_sockmap_kern.h index f48f85f1bd70..284a2f2e50cf 100644 --- a/tools/testing/selftests/bpf/progs/test_sockmap_kern.h +++ b/tools/testing/selftests/bpf/progs/test_sockmap_kern.h @@ -85,13 +85,6 @@ struct { __type(value, int); } sock_skb_opts SEC(".maps"); -struct { - __uint(type, TEST_MAP_TYPE); - __uint(max_entries, 20); - __uint(key_size, sizeof(int)); - __uint(value_size, sizeof(int)); -} tls_sock_map SEC(".maps"); - SEC("sk_skb/stream_parser") int bpf_prog1(struct __sk_buff *skb) { @@ -135,55 +128,6 @@ int bpf_prog2(struct __sk_buff *skb) } -static inline void bpf_write_pass(struct __sk_buff *skb, int offset) -{ - int err = bpf_skb_pull_data(skb, 6 + offset); - void *data_end; - char *c; - - if (err) - return; - - c = (char *)(long)skb->data; - data_end = (void *)(long)skb->data_end; - - if (c + 5 + offset < data_end) - memcpy(c + offset, "PASS", 4); -} - -SEC("sk_skb/stream_verdict") -int bpf_prog3(struct __sk_buff *skb) -{ - int err, *f, ret = SK_PASS; - const int one = 1; - - f = bpf_map_lookup_elem(&sock_skb_opts, &one); - if (f && *f) { - __u64 flags = 0; - - ret = 0; - flags = *f; - - err = bpf_skb_adjust_room(skb, -13, 0, 0); - if (err) - return SK_DROP; - err = bpf_skb_adjust_room(skb, 4, 0, 0); - if (err) - return SK_DROP; - bpf_write_pass(skb, 0); -#ifdef SOCKMAP - return bpf_sk_redirect_map(skb, &tls_sock_map, ret, flags); -#else - return bpf_sk_redirect_hash(skb, &tls_sock_map, &ret, flags); -#endif - } - err = bpf_skb_adjust_room(skb, 4, 0, 0); - if (err) - return SK_DROP; - bpf_write_pass(skb, 13); - return ret; -} - SEC("sockops") int bpf_sockmap(struct bpf_sock_ops *skops) { diff --git a/tools/testing/selftests/bpf/progs/test_sockmap_ktls.c b/tools/testing/selftests/bpf/progs/test_sockmap_ktls.c deleted file mode 100644 index 83df4919c224..000000000000 --- a/tools/testing/selftests/bpf/progs/test_sockmap_ktls.c +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include <linux/bpf.h> -#include <bpf/bpf_helpers.h> -#include <bpf/bpf_endian.h> - -int cork_byte; -int push_start; -int push_end; -int apply_bytes; -int pop_start; -int pop_end; - -struct { - __uint(type, BPF_MAP_TYPE_SOCKMAP); - __uint(max_entries, 20); - __type(key, int); - __type(value, int); -} sock_map SEC(".maps"); - -SEC("sk_msg") -int prog_sk_policy(struct sk_msg_md *msg) -{ - if (cork_byte > 0) - bpf_msg_cork_bytes(msg, cork_byte); - if (push_start > 0 && push_end > 0) - bpf_msg_push_data(msg, push_start, push_end, 0); - if (pop_start >= 0 && pop_end > 0) - bpf_msg_pop_data(msg, pop_start, pop_end, 0); - - return SK_PASS; -} - -SEC("sk_msg") -int prog_sk_policy_redir(struct sk_msg_md *msg) -{ - int two = 2; - - bpf_msg_apply_bytes(msg, apply_bytes); - return bpf_msg_redirect_map(msg, &sock_map, two, 0); -} diff --git a/tools/testing/selftests/bpf/progs/test_sockmap_msg_pop_data.c b/tools/testing/selftests/bpf/progs/test_sockmap_msg_pop_data.c new file mode 100644 index 000000000000..301e65b95256 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_sockmap_msg_pop_data.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> + +struct { + __uint(type, BPF_MAP_TYPE_SOCKMAP); + __uint(max_entries, 1); + __type(key, int); + __type(value, int); +} sock_map SEC(".maps"); + +#define POP_START 0x48a3 +#define POP_LEN 0xfffffffd + +long pop_data_ret = 1; + +SEC("sk_msg") +int prog_msg_pop_data(struct sk_msg_md *msg) +{ + if (msg->size <= POP_START) + return SK_PASS; + + pop_data_ret = bpf_msg_pop_data(msg, POP_START, POP_LEN, 0); + return SK_PASS; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_sockmap_strp.c b/tools/testing/selftests/bpf/progs/test_sockmap_strp.c index dde3d5bec515..fe88fa6d40bc 100644 --- a/tools/testing/selftests/bpf/progs/test_sockmap_strp.c +++ b/tools/testing/selftests/bpf/progs/test_sockmap_strp.c @@ -50,4 +50,11 @@ int prog_skb_parser_partial(struct __sk_buff *skb) return 10; } +SEC("sk_skb/stream_parser") +int prog_skb_parser_resize(struct __sk_buff *skb) +{ + bpf_skb_change_tail(skb, skb->len, 0); + return skb->len; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_sockmap_update.c b/tools/testing/selftests/bpf/progs/test_sockmap_update.c deleted file mode 100644 index 6d64ea536e3d..000000000000 --- a/tools/testing/selftests/bpf/progs/test_sockmap_update.c +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (c) 2020 Cloudflare -#include "vmlinux.h" -#include <bpf/bpf_helpers.h> - -struct { - __uint(type, BPF_MAP_TYPE_SOCKMAP); - __uint(max_entries, 1); - __type(key, __u32); - __type(value, __u64); -} src SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_SOCKMAP); - __uint(max_entries, 1); - __type(key, __u32); - __type(value, __u64); -} dst_sock_map SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_SOCKHASH); - __uint(max_entries, 1); - __type(key, __u32); - __type(value, __u64); -} dst_sock_hash SEC(".maps"); - -SEC("tc") -int copy_sock_map(void *ctx) -{ - struct bpf_sock *sk; - bool failed = false; - __u32 key = 0; - - sk = bpf_map_lookup_elem(&src, &key); - if (!sk) - return SK_DROP; - - if (bpf_map_update_elem(&dst_sock_map, &key, sk, 0)) - failed = true; - - if (bpf_map_update_elem(&dst_sock_hash, &key, sk, 0)) - failed = true; - - bpf_sk_release(sk); - return failed ? SK_DROP : SK_PASS; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_spin_lock_fail.c b/tools/testing/selftests/bpf/progs/test_spin_lock_fail.c index f678ee6bd7ea..55282f20fa32 100644 --- a/tools/testing/selftests/bpf/progs/test_spin_lock_fail.c +++ b/tools/testing/selftests/bpf/progs/test_spin_lock_fail.c @@ -14,17 +14,18 @@ struct array_map { __type(key, int); __type(value, struct foo); __uint(max_entries, 1); -} array_map SEC(".maps"); +} array_map SEC(".maps"), array_map_b SEC(".maps"); struct { __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); - __uint(max_entries, 1); + __uint(max_entries, 2); __type(key, int); __type(value, int); __array(values, struct array_map); } map_of_maps SEC(".maps") = { .values = { [0] = &array_map, + [1] = &array_map_b, }, }; @@ -314,4 +315,66 @@ int lock_global_sleepable_subprog_indirect(struct __sk_buff *ctx) return ret; } +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 2); + __type(key, int); + __type(value, struct foo); +} callback_array_map SEC(".maps"); + +struct callback_ctx { + struct foo *value; +}; + +static long lock_different_value(struct bpf_map *map, int *key, + struct foo *value, struct callback_ctx *ctx) +{ + bpf_spin_lock(&value->lock); + bpf_spin_unlock(&ctx->value->lock); + return 0; +} + +static long nest_lock_different_value(struct bpf_map *map, int *key, + struct foo *value, void *data) +{ + struct callback_ctx ctx = { .value = value }; + + bpf_for_each_map_elem(&callback_array_map, lock_different_value, &ctx, 0); + return 0; +} + +SEC("?tc") +int callback_value_lock_identity(void *ctx) +{ + bpf_for_each_map_elem(&callback_array_map, nest_lock_different_value, NULL, 0); + return 0; +} + +static long nest_lock_different_inner_value(struct bpf_map *map, int *key, + struct foo *value, void *data) +{ + struct callback_ctx ctx = { .value = value }; + int inner_key = 1; + void *inner_map; + + inner_map = bpf_map_lookup_elem(&map_of_maps, &inner_key); + if (!inner_map) + return 0; + bpf_for_each_map_elem(inner_map, lock_different_value, &ctx, 0); + return 0; +} + +SEC("?tc") +int callback_inner_map_value_lock_identity(void *ctx) +{ + int inner_key = 0; + void *inner_map; + + inner_map = bpf_map_lookup_elem(&map_of_maps, &inner_key); + if (!inner_map) + return 0; + bpf_for_each_map_elem(inner_map, nest_lock_different_inner_value, NULL, 0); + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_tc_change_tail_pmtu.c b/tools/testing/selftests/bpf/progs/test_tc_change_tail_pmtu.c new file mode 100644 index 000000000000..5c4c07545bc9 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_tc_change_tail_pmtu.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <stdbool.h> +#include <stddef.h> + +#include <linux/bpf.h> +#include <linux/icmp.h> +#include <linux/if_ether.h> +#include <linux/in.h> +#include <linux/ip.h> +#include <linux/tcp.h> + +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> + +#define ICMP_SAMPLE_LEN (sizeof(struct iphdr) + 8) +#define ICMP_HDRS_LEN (sizeof(struct iphdr) + sizeof(struct icmphdr)) + +__be16 server_port = 0; +__u16 pmtu = 0; + +long change_tail_ret = 1; +long adjust_room_ret = 0; +bool icmp_sent = false; +bool icmp_err = false; + +static __always_inline __sum16 csum_fold(__wsum csum) +{ + csum = (csum & 0xffff) + (csum >> 16); + csum = (csum & 0xffff) + (csum >> 16); + + return (__sum16)~csum; +} + +SEC("tc/egress") +int change_tail_icmp(struct __sk_buff *skb) +{ + __u8 smac[ETH_ALEN], dmac[ETH_ALEN]; + void *data, *data_end; + struct icmphdr *icmp; + struct ethhdr *eth; + struct tcphdr *tcp; + __be32 saddr, daddr; + struct iphdr *ip; + __wsum csum; + + if (icmp_sent || icmp_err) + return TCX_PASS; + + data = (void *)(long)skb->data; + data_end = (void *)(long)skb->data_end; + + eth = data; + if ((void *)(eth + 1) > data_end) + return TCX_PASS; + if (eth->h_proto != bpf_htons(ETH_P_IP)) + return TCX_PASS; + + ip = (void *)(eth + 1); + if ((void *)(ip + 1) > data_end) + return TCX_PASS; + if (ip->ihl != 5 || ip->protocol != IPPROTO_TCP) + return TCX_PASS; + + tcp = (void *)(ip + 1); + if ((void *)(tcp + 1) > data_end) + return TCX_PASS; + if (tcp->dest != server_port) + return TCX_PASS; + if (bpf_ntohs(ip->tot_len) <= sizeof(*ip) + tcp->doff * 4) + return TCX_PASS; + + __builtin_memcpy(smac, eth->h_source, ETH_ALEN); + __builtin_memcpy(dmac, eth->h_dest, ETH_ALEN); + saddr = ip->saddr; + daddr = ip->daddr; + + change_tail_ret = bpf_skb_change_tail(skb, ETH_HLEN + ICMP_SAMPLE_LEN, 0); + if (change_tail_ret) { + icmp_err = true; + return TCX_PASS; + } + + adjust_room_ret = bpf_skb_adjust_room(skb, ICMP_HDRS_LEN, + BPF_ADJ_ROOM_MAC, + BPF_F_ADJ_ROOM_NO_CSUM_RESET); + if (adjust_room_ret) { + icmp_err = true; + return TCX_DROP; + } + + data = (void *)(long)skb->data; + data_end = (void *)(long)skb->data_end; + + eth = data; + ip = (void *)(eth + 1); + icmp = (void *)(ip + 1); + if ((void *)icmp + sizeof(*icmp) + ICMP_SAMPLE_LEN > data_end) { + icmp_err = true; + return TCX_DROP; + } + + __builtin_memcpy(eth->h_dest, smac, ETH_ALEN); + __builtin_memcpy(eth->h_source, dmac, ETH_ALEN); + + __builtin_memset(icmp, 0, sizeof(*icmp)); + icmp->type = ICMP_DEST_UNREACH; + icmp->code = ICMP_FRAG_NEEDED; + icmp->un.frag.mtu = bpf_htons(pmtu); + + __builtin_memset(ip, 0, sizeof(*ip)); + ip->version = 4; + ip->ihl = 5; + ip->ttl = 64; + ip->protocol = IPPROTO_ICMP; + ip->tot_len = bpf_htons(ICMP_HDRS_LEN + ICMP_SAMPLE_LEN); + ip->saddr = daddr; + ip->daddr = saddr; + + csum = bpf_csum_diff(NULL, 0, (__be32 *)icmp, + sizeof(*icmp) + ICMP_SAMPLE_LEN, 0); + icmp->checksum = csum_fold(csum); + csum = bpf_csum_diff(NULL, 0, (__be32 *)ip, sizeof(*ip), 0); + ip->check = csum_fold(csum); + icmp_sent = true; + return bpf_redirect(skb->ifindex, BPF_F_INGRESS); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_tc_peer.c b/tools/testing/selftests/bpf/progs/test_tc_peer.c index 365eacb5dc34..cfb9ef7f467c 100644 --- a/tools/testing/selftests/bpf/progs/test_tc_peer.c +++ b/tools/testing/selftests/bpf/progs/test_tc_peer.c @@ -35,6 +35,28 @@ int tc_src(struct __sk_buff *skb) } SEC("tc") +int tc_dst_ing(struct __sk_buff *skb) +{ + if (!skb->mark) { + skb->mark = 0x1; + return bpf_redirect_peer(IFINDEX_SRC, BPF_F_EGRESS); + } + + return bpf_redirect(IFINDEX_DST, 0); +} + +SEC("tc") +int tc_src_ing(struct __sk_buff *skb) +{ + if (!skb->mark) { + skb->mark = 0x1; + return bpf_redirect_peer(IFINDEX_DST, BPF_F_EGRESS); + } + + return bpf_redirect(IFINDEX_SRC, 0); +} + +SEC("tc") int tc_dst_l3(struct __sk_buff *skb) { return bpf_redirect(IFINDEX_SRC, 0); diff --git a/tools/testing/selftests/bpf/progs/test_tc_qevent.c b/tools/testing/selftests/bpf/progs/test_tc_qevent.c new file mode 100644 index 000000000000..1529c111f4aa --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_tc_qevent.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> + +int redirect_ifindex = 1; +__u64 verdict_calls = 0; +__u64 helper_calls = 0; + +SEC("tc") +int qevent_redirect_verdict(struct __sk_buff *skb) +{ + __sync_fetch_and_add(&verdict_calls, 1); + return TCX_REDIRECT; +} + +SEC("tc") +int qevent_redirect_helper(struct __sk_buff *skb) +{ + __sync_fetch_and_add(&helper_calls, 1); + return bpf_redirect(redirect_ifindex, 0); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_tc_tunnel.c b/tools/testing/selftests/bpf/progs/test_tc_tunnel.c index 7376df405a6b..853bca962910 100644 --- a/tools/testing/selftests/bpf/progs/test_tc_tunnel.c +++ b/tools/testing/selftests/bpf/progs/test_tc_tunnel.c @@ -6,6 +6,7 @@ #include <bpf/bpf_helpers.h> #include <bpf/bpf_endian.h> +#include <bpf/bpf_core_read.h> #include "bpf_tracing_net.h" #include "bpf_compiler.h" @@ -37,6 +38,22 @@ struct vxlanhdr___local { #define EXTPROTO_VXLAN 0x1 +#define SKB_GSO_UDP_TUNNEL_MASK (SKB_GSO_UDP_TUNNEL | \ + SKB_GSO_UDP_TUNNEL_CSUM) + +#define SKB_GSO_TUNNEL_MASK (SKB_GSO_UDP_TUNNEL_MASK | \ + SKB_GSO_GRE | \ + SKB_GSO_GRE_CSUM | \ + SKB_GSO_IPXIP4 | \ + SKB_GSO_IPXIP6 | \ + SKB_GSO_ESP) + +#define BPF_F_ADJ_ROOM_DECAP_L4_MASK (BPF_F_ADJ_ROOM_DECAP_L4_UDP | \ + BPF_F_ADJ_ROOM_DECAP_L4_GRE) + +#define BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK (BPF_F_ADJ_ROOM_DECAP_IPXIP4 | \ + BPF_F_ADJ_ROOM_DECAP_IPXIP6) + #define VXLAN_FLAGS bpf_htonl(1<<27) #define VNI_ID 1 #define VXLAN_VNI bpf_htonl(VNI_ID << 8) @@ -589,9 +606,12 @@ int __encap_ip6vxlan_eth(struct __sk_buff *skb) return TC_ACT_OK; } -static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) +static int decap_internal(struct __sk_buff *skb, int off, int len, char proto, + __u64 ipxip_flag) { __u64 flags = BPF_F_ADJ_ROOM_FIXED_GSO; + struct sk_buff *kskb; + struct skb_shared_info *shinfo; struct ipv6_opt_hdr ip6_opt_hdr; struct gre_hdr greh; struct udphdr udph; @@ -599,10 +619,12 @@ static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) switch (proto) { case IPPROTO_IPIP: - flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV4; + flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV4 | + ipxip_flag; break; case IPPROTO_IPV6: - flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV6; + flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV6 | + ipxip_flag; break; case NEXTHDR_DEST: if (bpf_skb_load_bytes(skb, off + len, &ip6_opt_hdr, @@ -610,10 +632,12 @@ static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) return TC_ACT_OK; switch (ip6_opt_hdr.nexthdr) { case IPPROTO_IPIP: - flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV4; + flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV4 | + ipxip_flag; break; case IPPROTO_IPV6: - flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV6; + flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV6 | + ipxip_flag; break; default: return TC_ACT_OK; @@ -621,6 +645,11 @@ static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) break; case IPPROTO_GRE: olen += sizeof(struct gre_hdr); + if (!bpf_core_enum_value_exists(enum bpf_adj_room_flags, + BPF_F_ADJ_ROOM_DECAP_L4_GRE)) + return TC_ACT_SHOT; + flags |= BPF_F_ADJ_ROOM_DECAP_L4_GRE; + if (bpf_skb_load_bytes(skb, off + len, &greh, sizeof(greh)) < 0) return TC_ACT_OK; switch (bpf_ntohs(greh.protocol)) { @@ -634,6 +663,10 @@ static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) break; case IPPROTO_UDP: olen += sizeof(struct udphdr); + if (!bpf_core_enum_value_exists(enum bpf_adj_room_flags, + BPF_F_ADJ_ROOM_DECAP_L4_UDP)) + return TC_ACT_SHOT; + flags |= BPF_F_ADJ_ROOM_DECAP_L4_UDP; if (bpf_skb_load_bytes(skb, off + len, &udph, sizeof(udph)) < 0) return TC_ACT_OK; switch (bpf_ntohs(udph.dest)) { @@ -655,6 +688,40 @@ static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) if (bpf_skb_adjust_room(skb, -olen, BPF_ADJ_ROOM_MAC, flags)) return TC_ACT_SHOT; + kskb = bpf_cast_to_kern_ctx(skb); + shinfo = bpf_core_cast(kskb->head + kskb->end, struct skb_shared_info); + if (shinfo->gso_size) { + if ((flags & BPF_F_ADJ_ROOM_DECAP_L4_UDP) && + (shinfo->gso_type & SKB_GSO_UDP_TUNNEL_MASK)) + return TC_ACT_SHOT; + + if ((flags & BPF_F_ADJ_ROOM_DECAP_L4_GRE) && + (shinfo->gso_type & (SKB_GSO_GRE | SKB_GSO_GRE_CSUM))) + return TC_ACT_SHOT; + + if ((flags & BPF_F_ADJ_ROOM_DECAP_IPXIP4) && + (shinfo->gso_type & SKB_GSO_IPXIP4)) + return TC_ACT_SHOT; + + if ((flags & BPF_F_ADJ_ROOM_DECAP_IPXIP6) && + (shinfo->gso_type & SKB_GSO_IPXIP6)) + return TC_ACT_SHOT; + + if (flags & (BPF_F_ADJ_ROOM_DECAP_L4_MASK | + BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK)) { + if ((shinfo->gso_type & SKB_GSO_TUNNEL_MASK) && + !kskb->encapsulation) + return TC_ACT_SHOT; + if (!(shinfo->gso_type & SKB_GSO_TUNNEL_MASK) && + kskb->encapsulation) + return TC_ACT_SHOT; + } + } else if ((flags & (BPF_F_ADJ_ROOM_DECAP_L4_MASK | + BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK)) && + kskb->encapsulation) { + return TC_ACT_SHOT; + } + return TC_ACT_OK; } @@ -662,6 +729,10 @@ static int decap_ipv4(struct __sk_buff *skb) { struct iphdr iph_outer; + if (!bpf_core_enum_value_exists(enum bpf_adj_room_flags, + BPF_F_ADJ_ROOM_DECAP_IPXIP4)) + return TC_ACT_SHOT; + if (bpf_skb_load_bytes(skb, ETH_HLEN, &iph_outer, sizeof(iph_outer)) < 0) return TC_ACT_OK; @@ -670,19 +741,25 @@ static int decap_ipv4(struct __sk_buff *skb) return TC_ACT_OK; return decap_internal(skb, ETH_HLEN, sizeof(iph_outer), - iph_outer.protocol); + iph_outer.protocol, + BPF_F_ADJ_ROOM_DECAP_IPXIP4); } static int decap_ipv6(struct __sk_buff *skb) { struct ipv6hdr iph_outer; + if (!bpf_core_enum_value_exists(enum bpf_adj_room_flags, + BPF_F_ADJ_ROOM_DECAP_IPXIP6)) + return TC_ACT_SHOT; + if (bpf_skb_load_bytes(skb, ETH_HLEN, &iph_outer, sizeof(iph_outer)) < 0) return TC_ACT_OK; return decap_internal(skb, ETH_HLEN, sizeof(iph_outer), - iph_outer.nexthdr); + iph_outer.nexthdr, + BPF_F_ADJ_ROOM_DECAP_IPXIP6); } SEC("tc") diff --git a/tools/testing/selftests/bpf/progs/test_trampoline_count.c b/tools/testing/selftests/bpf/progs/test_trampoline_count.c index 7765720da7d5..02f52806b1b2 100644 --- a/tools/testing/selftests/bpf/progs/test_trampoline_count.c +++ b/tools/testing/selftests/bpf/progs/test_trampoline_count.c @@ -3,20 +3,20 @@ #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> -SEC("fentry/bpf_modify_return_test") -int BPF_PROG(fentry_test, int a, int *b) +SEC("fentry/bpf_testmod_trampoline_count_test") +int BPF_PROG(fentry_test) { return 0; } -SEC("fmod_ret/bpf_modify_return_test") -int BPF_PROG(fmod_ret_test, int a, int *b, int ret) +SEC("fmod_ret/bpf_testmod_trampoline_count_test") +int BPF_PROG(fmod_ret_test, int ret) { return 0; } -SEC("fexit/bpf_modify_return_test") -int BPF_PROG(fexit_test, int a, int *b, int ret) +SEC("fexit/bpf_testmod_trampoline_count_test") +int BPF_PROG(fexit_test, int ret) { return 0; } diff --git a/tools/testing/selftests/bpf/progs/test_tunnel_kern.c b/tools/testing/selftests/bpf/progs/test_tunnel_kern.c index 32127f1cd687..30f1de458669 100644 --- a/tools/testing/selftests/bpf/progs/test_tunnel_kern.c +++ b/tools/testing/selftests/bpf/progs/test_tunnel_kern.c @@ -6,6 +6,7 @@ * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. */ +#define BPF_NO_KFUNC_PROTOTYPES #include "vmlinux.h" #include <bpf/bpf_core_read.h> #include <bpf/bpf_helpers.h> @@ -36,12 +37,10 @@ enum bpf_fou_encap_type___local { FOU_BPF_ENCAP_GUE___local, }; -struct bpf_fou_encap; - int bpf_skb_set_fou_encap(struct __sk_buff *skb_ctx, - struct bpf_fou_encap *encap, int type) __ksym; + struct bpf_fou_encap___local *encap, int type) __ksym; int bpf_skb_get_fou_encap(struct __sk_buff *skb_ctx, - struct bpf_fou_encap *encap) __ksym; + struct bpf_fou_encap___local *encap) __ksym; struct xfrm_state * bpf_xdp_get_xfrm_state(struct xdp_md *ctx, struct bpf_xfrm_state_opts *opts, u32 opts__sz) __ksym; @@ -781,7 +780,7 @@ int ipip_gue_set_tunnel(struct __sk_buff *skb) encap.sport = 0; encap.dport = bpf_htons(5555); - ret = bpf_skb_set_fou_encap(skb, (struct bpf_fou_encap *)&encap, + ret = bpf_skb_set_fou_encap(skb, &encap, bpf_core_enum_value(enum bpf_fou_encap_type___local, FOU_BPF_ENCAP_GUE___local)); if (ret < 0) { @@ -820,7 +819,7 @@ int ipip_fou_set_tunnel(struct __sk_buff *skb) encap.sport = 0; encap.dport = bpf_htons(5555); - ret = bpf_skb_set_fou_encap(skb, (struct bpf_fou_encap *)&encap, + ret = bpf_skb_set_fou_encap(skb, &encap, FOU_BPF_ENCAP_FOU___local); if (ret < 0) { log_err(ret); @@ -843,7 +842,7 @@ int ipip_encap_get_tunnel(struct __sk_buff *skb) return TC_ACT_SHOT; } - ret = bpf_skb_get_fou_encap(skb, (struct bpf_fou_encap *)&encap); + ret = bpf_skb_get_fou_encap(skb, &encap); if (ret < 0) { log_err(ret); return TC_ACT_SHOT; diff --git a/tools/testing/selftests/bpf/progs/test_usdt.c b/tools/testing/selftests/bpf/progs/test_usdt.c index a78c87537b07..0ee78fb050a1 100644 --- a/tools/testing/selftests/bpf/progs/test_usdt.c +++ b/tools/testing/selftests/bpf/progs/test_usdt.c @@ -138,4 +138,41 @@ int usdt_sib(struct pt_regs *ctx) return 0; } +#ifdef __TARGET_ARCH_x86 +int executed; +unsigned long expected_ip; + +SEC("usdt") +int usdt_executed(struct pt_regs *ctx) +{ + if (expected_ip == ctx->ip) + executed++; + return 0; +} + +int arg_total; +int arg_bad; +long arg_last[3]; +long expected_arg[3]; +int expected_pid; + +SEC("usdt") +int BPF_USDT(usdt_check_arg, long arg1, long arg2, long arg3) +{ + if (expected_pid != (bpf_get_current_pid_tgid() >> 32)) + return 0; + + __sync_fetch_and_add(&arg_total, 1); + arg_last[0] = arg1; + arg_last[1] = arg2; + arg_last[2] = arg3; + + if (arg1 != expected_arg[0] || + arg2 != expected_arg[1] || + arg3 != expected_arg[2]) + __sync_fetch_and_add(&arg_bad, 1); + + return 0; +} +#endif char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_vmlinux.c b/tools/testing/selftests/bpf/progs/test_vmlinux.c index 78b23934d9f8..eea556940df6 100644 --- a/tools/testing/selftests/bpf/progs/test_vmlinux.c +++ b/tools/testing/selftests/bpf/progs/test_vmlinux.c @@ -69,7 +69,7 @@ int BPF_PROG(handle__tp_btf, struct pt_regs *regs, long id) return 0; } -SEC("kprobe/hrtimer_start_range_ns") +SEC("kprobe") int BPF_KPROBE(handle__kprobe, struct hrtimer *timer, ktime_t tim, u64 delta_ns, const enum hrtimer_mode mode) { @@ -78,7 +78,7 @@ int BPF_KPROBE(handle__kprobe, struct hrtimer *timer, ktime_t tim, u64 delta_ns, return 0; } -SEC("fentry/hrtimer_start_range_ns") +SEC("fentry") int BPF_PROG(handle__fentry, struct hrtimer *timer, ktime_t tim, u64 delta_ns, const enum hrtimer_mode mode) { diff --git a/tools/testing/selftests/bpf/progs/test_wakeup_source.c b/tools/testing/selftests/bpf/progs/test_wakeup_source.c new file mode 100644 index 000000000000..fd2fb6aebd82 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_wakeup_source.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright 2026 Google LLC */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_core_read.h> +#include "bpf_experimental.h" +#include "bpf_misc.h" +#include "wakeup_source.h" + +#define MAX_LOOP_ITER 1000 +#define RB_SIZE (16384 * 4) + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, RB_SIZE); +} rb SEC(".maps"); + +struct bpf_ws_lock; +struct bpf_ws_lock *bpf_wakeup_sources_read_lock(void) __ksym; +void bpf_wakeup_sources_read_unlock(struct bpf_ws_lock *lock) __ksym; +void *bpf_wakeup_sources_get_head(void) __ksym; + +SEC("syscall") +__success __retval(0) +int iterate_wakeupsources(void *ctx) +{ + struct list_head *head = bpf_wakeup_sources_get_head(); + struct list_head *pos = head; + struct bpf_ws_lock *lock; + int i; + + lock = bpf_wakeup_sources_read_lock(); + if (!lock) + return 0; + + bpf_for(i, 0, MAX_LOOP_ITER) { + if (bpf_core_read(&pos, sizeof(pos), &pos->next) || !pos || pos == head) + break; + + struct wakeup_event_t *e = bpf_ringbuf_reserve(&rb, sizeof(*e), 0); + + if (!e) + break; + + struct wakeup_source *ws = bpf_core_cast( + (void *)pos - bpf_core_field_offset(struct wakeup_source, entry), + struct wakeup_source); + s64 active_time = 0; + bool active = BPF_CORE_READ_BITFIELD(ws, active); + bool autosleep_enable = BPF_CORE_READ_BITFIELD(ws, autosleep_enabled); + s64 last_time = ws->last_time; + s64 max_time = ws->max_time; + s64 prevent_sleep_time = ws->prevent_sleep_time; + s64 total_time = ws->total_time; + + if (active) { + s64 curr_time = bpf_ktime_get_ns(); + s64 prevent_time = ws->start_prevent_time; + + if (curr_time > last_time) + active_time = curr_time - last_time; + + total_time += active_time; + if (active_time > max_time) + max_time = active_time; + if (autosleep_enable && curr_time > prevent_time) + prevent_sleep_time += curr_time - prevent_time; + } + + e->active_count = ws->active_count; + e->active_time_ns = active_time; + e->event_count = ws->event_count; + e->expire_count = ws->expire_count; + e->last_time_ns = last_time; + e->max_time_ns = max_time; + e->prevent_sleep_time_ns = prevent_sleep_time; + e->total_time_ns = total_time; + e->wakeup_count = ws->wakeup_count; + + if (bpf_probe_read_kernel_str( + e->name, WAKEUP_NAME_LEN, ws->name) < 0) + e->name[0] = '\0'; + + bpf_ringbuf_submit(e, 0); + } + + bpf_wakeup_sources_read_unlock(lock); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_xdp_meta.c b/tools/testing/selftests/bpf/progs/test_xdp_meta.c index fa73b17cb999..08b03be0b891 100644 --- a/tools/testing/selftests/bpf/progs/test_xdp_meta.c +++ b/tools/testing/selftests/bpf/progs/test_xdp_meta.c @@ -21,10 +21,6 @@ bool test_pass; -static const __u8 smac_want[ETH_ALEN] = { - 0x12, 0x34, 0xDE, 0xAD, 0xBE, 0xEF, -}; - static const __u8 meta_want[META_SIZE] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, @@ -32,11 +28,6 @@ static const __u8 meta_want[META_SIZE] = { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, }; -static bool check_smac(const struct ethhdr *eth) -{ - return !__builtin_memcmp(eth->h_source, smac_want, ETH_ALEN); -} - static bool check_metadata(const char *file, int line, __u8 *meta_have) { if (!__builtin_memcmp(meta_have, meta_want, META_SIZE)) @@ -280,18 +271,47 @@ fail: return TC_ACT_SHOT; } +/* Test packets carry test metadata pattern as payload. */ +static bool is_test_packet_xdp(struct xdp_md *ctx) +{ + __u8 meta_have[META_SIZE]; + __u32 len; + + len = bpf_xdp_get_buff_len(ctx); + if (len < META_SIZE) + return false; + if (bpf_xdp_load_bytes(ctx, len - META_SIZE, meta_have, META_SIZE)) + return false; + if (__builtin_memcmp(meta_have, meta_want, META_SIZE)) + return false; + + return true; +} + +/* Test packets carry test metadata pattern as payload. */ +static bool is_test_packet_tc(struct __sk_buff *ctx) +{ + __u8 meta_have[META_SIZE]; + + if (ctx->len < META_SIZE) + return false; + if (bpf_skb_load_bytes(ctx, ctx->len - META_SIZE, meta_have, META_SIZE)) + return false; + if (__builtin_memcmp(meta_have, meta_want, META_SIZE)) + return false; + + return true; +} + /* Reserve and clear space for metadata but don't populate it */ SEC("xdp") int ing_xdp_zalloc_meta(struct xdp_md *ctx) { - struct ethhdr *eth = ctx_ptr(ctx, data); __u8 *meta; int ret; /* Drop any non-test packets */ - if (eth + 1 > ctx_ptr(ctx, data_end)) - return XDP_DROP; - if (!check_smac(eth)) + if (!is_test_packet_xdp(ctx)) return XDP_DROP; ret = bpf_xdp_adjust_meta(ctx, -META_SIZE); @@ -310,33 +330,24 @@ int ing_xdp_zalloc_meta(struct xdp_md *ctx) SEC("xdp") int ing_xdp(struct xdp_md *ctx) { - __u8 *data, *data_meta, *data_end, *payload; - struct ethhdr *eth; + __u8 *data, *data_meta; int ret; + /* Drop any non-test packets */ + if (!is_test_packet_xdp(ctx)) + return XDP_DROP; + ret = bpf_xdp_adjust_meta(ctx, -META_SIZE); if (ret < 0) return XDP_DROP; data_meta = ctx_ptr(ctx, data_meta); - data_end = ctx_ptr(ctx, data_end); data = ctx_ptr(ctx, data); - eth = (struct ethhdr *)data; - payload = data + sizeof(struct ethhdr); - - if (payload + META_SIZE > data_end || - data_meta + META_SIZE > data) + if (data_meta + META_SIZE > data) return XDP_DROP; - /* The Linux networking stack may send other packets on the test - * interface that interfere with the test. Just drop them. - * The test packets can be recognized by their source MAC address. - */ - if (!check_smac(eth)) - return XDP_DROP; - - __builtin_memcpy(data_meta, payload, META_SIZE); + __builtin_memcpy(data_meta, meta_want, META_SIZE); return XDP_PASS; } @@ -353,7 +364,7 @@ int clone_data_meta_survives_data_write(struct __sk_buff *ctx) if (eth + 1 > ctx_ptr(ctx, data_end)) goto out; /* Ignore non-test packets */ - if (!check_smac(eth)) + if (!is_test_packet_tc(ctx)) goto out; if (meta_have + META_SIZE > eth) @@ -383,7 +394,7 @@ int clone_data_meta_survives_meta_write(struct __sk_buff *ctx) if (eth + 1 > ctx_ptr(ctx, data_end)) goto out; /* Ignore non-test packets */ - if (!check_smac(eth)) + if (!is_test_packet_tc(ctx)) goto out; if (meta_have + META_SIZE > eth) @@ -416,7 +427,7 @@ int clone_meta_dynptr_survives_data_slice_write(struct __sk_buff *ctx) if (!eth) goto out; /* Ignore non-test packets */ - if (!check_smac(eth)) + if (!is_test_packet_tc(ctx)) goto out; bpf_dynptr_from_skb_meta(ctx, 0, &meta); @@ -436,16 +447,11 @@ out: SEC("tc") int clone_meta_dynptr_survives_meta_slice_write(struct __sk_buff *ctx) { - struct bpf_dynptr data, meta; - const struct ethhdr *eth; + struct bpf_dynptr meta; __u8 *meta_have; - bpf_dynptr_from_skb(ctx, 0, &data); - eth = bpf_dynptr_slice(&data, 0, NULL, sizeof(*eth)); - if (!eth) - goto out; /* Ignore non-test packets */ - if (!check_smac(eth)) + if (!is_test_packet_tc(ctx)) goto out; bpf_dynptr_from_skb_meta(ctx, 0, &meta); @@ -471,15 +477,10 @@ int clone_meta_dynptr_rw_before_data_dynptr_write(struct __sk_buff *ctx) { struct bpf_dynptr data, meta; __u8 meta_have[META_SIZE]; - const struct ethhdr *eth; int err; - bpf_dynptr_from_skb(ctx, 0, &data); - eth = bpf_dynptr_slice(&data, 0, NULL, sizeof(*eth)); - if (!eth) - goto out; /* Ignore non-test packets */ - if (!check_smac(eth)) + if (!is_test_packet_tc(ctx)) goto out; /* Expect read-write metadata before unclone */ @@ -492,6 +493,7 @@ int clone_meta_dynptr_rw_before_data_dynptr_write(struct __sk_buff *ctx) goto out; /* Helper write to payload will unclone the packet */ + bpf_dynptr_from_skb(ctx, 0, &data); bpf_dynptr_write(&data, offsetof(struct ethhdr, h_proto), "x", 1, 0); err = bpf_dynptr_read(meta_have, META_SIZE, &meta, 0, 0); @@ -511,17 +513,12 @@ out: SEC("tc") int clone_meta_dynptr_rw_before_meta_dynptr_write(struct __sk_buff *ctx) { - struct bpf_dynptr data, meta; + struct bpf_dynptr meta; __u8 meta_have[META_SIZE]; - const struct ethhdr *eth; int err; - bpf_dynptr_from_skb(ctx, 0, &data); - eth = bpf_dynptr_slice(&data, 0, NULL, sizeof(*eth)); - if (!eth) - goto out; /* Ignore non-test packets */ - if (!check_smac(eth)) + if (!is_test_packet_tc(ctx)) goto out; /* Expect read-write metadata before unclone */ @@ -545,6 +542,28 @@ out: return TC_ACT_SHOT; } +SEC("lwt_xmit") +int dummy_lwt_xmit(struct __sk_buff *ctx) +{ + if (bpf_skb_change_head(ctx, sizeof(struct ipv6hdr), 0)) + return BPF_DROP; + + return BPF_OK; +} + +SEC("tc") +int tc_is_meta_empty(struct __sk_buff *ctx) +{ + if (!is_test_packet_tc(ctx)) + return TC_ACT_OK; + + if (ctx->data_meta != ctx->data) + return TC_ACT_OK; + + test_pass = true; + return TC_ACT_OK; +} + SEC("tc") int helper_skb_vlan_push_pop(struct __sk_buff *ctx) { diff --git a/tools/testing/selftests/bpf/progs/timer_mim_reject.c b/tools/testing/selftests/bpf/progs/timer_mim_reject.c index dd3f1ed6d6e6..83f31138336b 100644 --- a/tools/testing/selftests/bpf/progs/timer_mim_reject.c +++ b/tools/testing/selftests/bpf/progs/timer_mim_reject.c @@ -43,7 +43,7 @@ static int timer_cb(void *map, int *key, struct hmap_elem *val) return 0; } -SEC("fentry/bpf_fentry_test1") +SEC("?fentry/bpf_fentry_test1") int BPF_PROG(test1, int a) { struct hmap_elem init = {}; @@ -72,3 +72,85 @@ int BPF_PROG(test1, int a) err |= 8; return 0; } + +struct callback_ctx { + void *map; +}; + +static int mismatch_iter_cb(void *map, int *key, struct hmap_elem *val, struct callback_ctx *ctx) +{ + bpf_timer_init(&val->timer, ctx->map, CLOCK_MONOTONIC); + return 0; +} + +static int timer_mismatch_cb(void *map, int *key, struct hmap_elem *val) +{ + struct callback_ctx ctx = { .map = map }; + struct bpf_map *inner_map2; + int array_key2 = ARRAY_KEY2; + + inner_map2 = bpf_map_lookup_elem(&outer_arr, &array_key2); + if (!inner_map2) + return 0; + bpf_for_each_map_elem(inner_map2, mismatch_iter_cb, &ctx, 0); + return 0; +} + +static int match_iter_cb(void *map, int *key, struct hmap_elem *val, struct callback_ctx *ctx) +{ + bpf_timer_init(&val->timer, map, CLOCK_MONOTONIC); + return 0; +} + +static int timer_match_cb(void *map, int *key, struct hmap_elem *val) +{ + struct callback_ctx ctx = {}; + struct bpf_map *inner_map2; + int array_key2 = ARRAY_KEY2; + + inner_map2 = bpf_map_lookup_elem(&outer_arr, &array_key2); + if (!inner_map2) + return 0; + bpf_for_each_map_elem(inner_map2, match_iter_cb, &ctx, 0); + return 0; +} + +SEC("?fentry/bpf_fentry_test1") +int BPF_PROG(callback_map_uid_mismatch, int a) +{ + struct hmap_elem *val; + struct bpf_map *inner_map; + int array_key = ARRAY_KEY; + int hash_key = HASH_KEY; + + inner_map = bpf_map_lookup_elem(&outer_arr, &array_key); + if (!inner_map) + return 0; + val = bpf_map_lookup_elem(inner_map, &hash_key); + if (!val) + return 0; + + bpf_timer_init(&val->timer, inner_map, CLOCK_MONOTONIC); + bpf_timer_set_callback(&val->timer, timer_mismatch_cb); + return 0; +} + +SEC("?fentry/bpf_fentry_test1") +int BPF_PROG(callback_map_uid_match, int a) +{ + struct hmap_elem *val; + struct bpf_map *inner_map; + int array_key = ARRAY_KEY; + int hash_key = HASH_KEY; + + inner_map = bpf_map_lookup_elem(&outer_arr, &array_key); + if (!inner_map) + return 0; + val = bpf_map_lookup_elem(inner_map, &hash_key); + if (!val) + return 0; + + bpf_timer_init(&val->timer, inner_map, CLOCK_MONOTONIC); + bpf_timer_set_callback(&val->timer, timer_match_cb); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/timer_start_deadlock.c b/tools/testing/selftests/bpf/progs/timer_start_deadlock.c index 019518ee18cd..afabd15bdac4 100644 --- a/tools/testing/selftests/bpf/progs/timer_start_deadlock.c +++ b/tools/testing/selftests/bpf/progs/timer_start_deadlock.c @@ -27,13 +27,13 @@ static int timer_cb(void *map, int *key, struct elem *value) return 0; } -SEC("tp_btf/hrtimer_cancel") -int BPF_PROG(tp_hrtimer_cancel, struct hrtimer *hrtimer) +SEC("tp_btf/hrtimer_start") +int BPF_PROG(tp_hrtimer_start, struct hrtimer *hrtimer, enum hrtimer_mode mode, bool was_armed) { struct bpf_timer *timer; int key = 0; - if (!in_timer_start) + if (!in_timer_start || !was_armed) return 0; tp_called = 1; @@ -60,7 +60,7 @@ int start_timer(void *ctx) /* * call hrtimer_start() twice, so that 2nd call does - * remove_hrtimer() and trace_hrtimer_cancel() tracepoint. + * trace_hrtimer_start(was_armed=1) tracepoint. */ in_timer_start = 1; bpf_timer_start(timer, 1000000000, 0); diff --git a/tools/testing/selftests/bpf/progs/token_kallsyms.c b/tools/testing/selftests/bpf/progs/token_kallsyms.c new file mode 100644 index 000000000000..c9f9344f3eb2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/token_kallsyms.c @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> + +char _license[] SEC("license") = "GPL"; + +__weak +int token_ksym_subprog(void) +{ + return 0; +} + +SEC("xdp") +int xdp_main(struct xdp_md *xdp) +{ + return token_ksym_subprog(); +} diff --git a/tools/testing/selftests/bpf/progs/trace_printk.c b/tools/testing/selftests/bpf/progs/trace_printk.c index 6695478c2b25..f4c538ec3ebd 100644 --- a/tools/testing/selftests/bpf/progs/trace_printk.c +++ b/tools/testing/selftests/bpf/progs/trace_printk.c @@ -10,13 +10,23 @@ char _license[] SEC("license") = "GPL"; int trace_printk_ret = 0; int trace_printk_ran = 0; +int trace_printk_invalid_spec_ret = 0; +int trace_printk_utf8_ret = 0; +int trace_printk_utf8_ran = 0; const char fmt[] = "Testing,testing %d\n"; +static const char utf8_fmt[] = "ä¸æ–‡,测试 %d\n"; +/* Non-ASCII bytes after '%' must still be rejected. */ +static const char invalid_spec_fmt[] = "%\x80\n"; SEC("fentry/" SYS_PREFIX "sys_nanosleep") int sys_enter(void *ctx) { trace_printk_ret = bpf_trace_printk(fmt, sizeof(fmt), ++trace_printk_ran); + trace_printk_utf8_ret = bpf_trace_printk(utf8_fmt, sizeof(utf8_fmt), + ++trace_printk_utf8_ran); + trace_printk_invalid_spec_ret = bpf_trace_printk(invalid_spec_fmt, + sizeof(invalid_spec_fmt)); return 0; } diff --git a/tools/testing/selftests/bpf/progs/tracing_failure.c b/tools/testing/selftests/bpf/progs/tracing_failure.c index 65e485c4468c..f7a095767679 100644 --- a/tools/testing/selftests/bpf/progs/tracing_failure.c +++ b/tools/testing/selftests/bpf/progs/tracing_failure.c @@ -30,3 +30,9 @@ int BPF_PROG(fexit_noreturns) { return 0; } + +SEC("?fexit/bpf_testmod_test_int128_ret") +int BPF_PROG(fexit_int128_ret) +{ + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/tracing_multi_attach.c b/tools/testing/selftests/bpf/progs/tracing_multi_attach.c new file mode 100644 index 000000000000..332d0a423a43 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_multi_attach.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +__hidden extern int tracing_multi_arg_check(__u64 *ctx, __u64 *test_result, bool is_return); + +__u64 test_result_fentry = 0; +__u64 test_result_fexit = 0; + +SEC("fentry.multi/bpf_fentry_test*") +int BPF_PROG(test_fentry) +{ + tracing_multi_arg_check(ctx, &test_result_fentry, false); + return 0; +} + +SEC("fexit.multi/bpf_fentry_test*") +int BPF_PROG(test_fexit) +{ + tracing_multi_arg_check(ctx, &test_result_fexit, true); + return 0; +} + +SEC("fentry.multi.s/bpf_fentry_test1") +int BPF_PROG(test_fentry_s) +{ + tracing_multi_arg_check(ctx, &test_result_fentry, false); + return 0; +} + +SEC("fexit.multi.s/bpf_fentry_test1") +int BPF_PROG(test_fexit_s) +{ + tracing_multi_arg_check(ctx, &test_result_fexit, true); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/tracing_multi_attach_module.c b/tools/testing/selftests/bpf/progs/tracing_multi_attach_module.c new file mode 100644 index 000000000000..b3374f2db450 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_multi_attach_module.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +__hidden extern int tracing_multi_arg_check(__u64 *ctx, __u64 *test_result, bool is_return); + +__u64 test_result_fentry = 0; +__u64 test_result_fexit = 0; + +SEC("fentry.multi/bpf_testmod:bpf_testmod_fentry_test*") +int BPF_PROG(test_fentry) +{ + tracing_multi_arg_check(ctx, &test_result_fentry, false); + return 0; +} + +SEC("fexit.multi/bpf_testmod:bpf_testmod_fentry_test*") +int BPF_PROG(test_fexit) +{ + tracing_multi_arg_check(ctx, &test_result_fexit, true); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/tracing_multi_bench.c b/tools/testing/selftests/bpf/progs/tracing_multi_bench.c new file mode 100644 index 000000000000..beae946cb8c4 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_multi_bench.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +SEC("fentry.multi") +int BPF_PROG(bench) +{ + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/tracing_multi_check.c b/tools/testing/selftests/bpf/progs/tracing_multi_check.c new file mode 100644 index 000000000000..b2959ba71179 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_multi_check.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +int pid = 0; +bool test_cookies = false; + +/* bpf_fentry_test1 is exported as kfunc via vmlinux.h */ +extern const void bpf_fentry_test2 __ksym; +extern const void bpf_fentry_test3 __ksym; +extern const void bpf_fentry_test4 __ksym; +extern const void bpf_fentry_test5 __ksym; +extern const void bpf_fentry_test6 __ksym; +extern const void bpf_fentry_test7 __ksym; +extern const void bpf_fentry_test8 __ksym; +extern const void bpf_fentry_test9 __ksym; +extern const void bpf_fentry_test10 __ksym; + +extern const void bpf_testmod_fentry_test1 __ksym; +extern const void bpf_testmod_fentry_test2 __ksym; +extern const void bpf_testmod_fentry_test3 __ksym; +extern const void bpf_testmod_fentry_test7 __ksym; +extern const void bpf_testmod_fentry_test11 __ksym; + +int tracing_multi_arg_check(__u64 *ctx, __u64 *test_result, bool is_return) +{ + void *ip = (void *) bpf_get_func_ip(ctx); + __u64 value = 0, ret = 0, cookie = 0; + long err = 0; + + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 1; + + if (is_return) + err |= bpf_get_func_ret(ctx, &ret); + if (test_cookies) + cookie = bpf_get_attach_cookie(ctx); + + if (ip == &bpf_fentry_test1) { + int a; + + err |= bpf_get_func_arg(ctx, 0, &value); + a = (int) value; + + err |= is_return ? ret != 2 : 0; + err |= test_cookies ? cookie != 8 : 0; + + *test_result += err == 0 && a == 1; + } else if (ip == &bpf_fentry_test2) { + __u64 b; + int a; + + err |= bpf_get_func_arg(ctx, 0, &value); + a = (int) value; + err |= bpf_get_func_arg(ctx, 1, &value); + b = value; + + err |= is_return ? ret != 5 : 0; + err |= test_cookies ? cookie != 9 : 0; + + *test_result += err == 0 && a == 2 && b == 3; + } else if (ip == &bpf_fentry_test3) { + __u64 c; + char a; + int b; + + err |= bpf_get_func_arg(ctx, 0, &value); + a = (char) value; + err |= bpf_get_func_arg(ctx, 1, &value); + b = (int) value; + err |= bpf_get_func_arg(ctx, 2, &value); + c = value; + + err |= is_return ? ret != 15 : 0; + err |= test_cookies ? cookie != 7 : 0; + + *test_result += err == 0 && a == 4 && b == 5 && c == 6; + } else if (ip == &bpf_fentry_test4) { + void *a; + char b; + int c; + __u64 d; + + err |= bpf_get_func_arg(ctx, 0, &value); + a = (void *) value; + err |= bpf_get_func_arg(ctx, 1, &value); + b = (char) value; + err |= bpf_get_func_arg(ctx, 2, &value); + c = (int) value; + err |= bpf_get_func_arg(ctx, 3, &value); + d = value; + + err |= is_return ? ret != 34 : 0; + err |= test_cookies ? cookie != 5 : 0; + + *test_result += err == 0 && a == (void *) 7 && b == 8 && c == 9 && d == 10; + } else if (ip == &bpf_fentry_test5) { + __u64 a; + void *b; + short c; + int d; + __u64 e; + + err |= bpf_get_func_arg(ctx, 0, &value); + a = value; + err |= bpf_get_func_arg(ctx, 1, &value); + b = (void *) value; + err |= bpf_get_func_arg(ctx, 2, &value); + c = (short) value; + err |= bpf_get_func_arg(ctx, 3, &value); + d = (int) value; + err |= bpf_get_func_arg(ctx, 4, &value); + e = value; + + err |= is_return ? ret != 65 : 0; + err |= test_cookies ? cookie != 4 : 0; + + *test_result += err == 0 && a == 11 && b == (void *) 12 && c == 13 && d == 14 && e == 15; + } else if (ip == &bpf_fentry_test6) { + __u64 a; + void *b; + short c; + int d; + void *e; + __u64 f; + + err |= bpf_get_func_arg(ctx, 0, &value); + a = value; + err |= bpf_get_func_arg(ctx, 1, &value); + b = (void *) value; + err |= bpf_get_func_arg(ctx, 2, &value); + c = (short) value; + err |= bpf_get_func_arg(ctx, 3, &value); + d = (int) value; + err |= bpf_get_func_arg(ctx, 4, &value); + e = (void *) value; + err |= bpf_get_func_arg(ctx, 5, &value); + f = value; + + err |= is_return ? ret != 111 : 0; + err |= test_cookies ? cookie != 2 : 0; + + *test_result += err == 0 && a == 16 && b == (void *) 17 && c == 18 && d == 19 && e == (void *) 20 && f == 21; + } else if (ip == &bpf_fentry_test7) { + err |= is_return ? ret != 0 : 0; + err |= test_cookies ? cookie != 3 : 0; + + *test_result += err == 0 ? 1 : 0; + } else if (ip == &bpf_fentry_test8) { + err |= is_return ? ret != 0 : 0; + err |= test_cookies ? cookie != 1 : 0; + + *test_result += err == 0 ? 1 : 0; + } else if (ip == &bpf_fentry_test9) { + err |= is_return ? ret != 0 : 0; + err |= test_cookies ? cookie != 10 : 0; + + *test_result += err == 0 ? 1 : 0; + } else if (ip == &bpf_fentry_test10) { + err |= is_return ? ret != 0 : 0; + err |= test_cookies ? cookie != 6 : 0; + + *test_result += err == 0 ? 1 : 0; + } else if (ip == &bpf_testmod_fentry_test1) { + int a; + + err |= bpf_get_func_arg(ctx, 0, &value); + a = (int) value; + + err |= is_return ? ret != 2 : 0; + + *test_result += err == 0 && a == 1; + } else if (ip == &bpf_testmod_fentry_test2) { + int a; + __u64 b; + + err |= bpf_get_func_arg(ctx, 0, &value); + a = (int) value; + err |= bpf_get_func_arg(ctx, 1, &value); + b = (__u64) value; + + err |= is_return ? ret != 5 : 0; + + *test_result += err == 0 && a == 2 && b == 3; + } else if (ip == &bpf_testmod_fentry_test3) { + char a; + int b; + __u64 c; + + err |= bpf_get_func_arg(ctx, 0, &value); + a = (char) value; + err |= bpf_get_func_arg(ctx, 1, &value); + b = (int) value; + err |= bpf_get_func_arg(ctx, 2, &value); + c = (__u64) value; + + err |= is_return ? ret != 15 : 0; + + *test_result += err == 0 && a == 4 && b == 5 && c == 6; + } else if (ip == &bpf_testmod_fentry_test7) { + err |= is_return ? ret != 133 : 0; + + *test_result += err == 0; + } else if (ip == &bpf_testmod_fentry_test11) { + err |= is_return ? ret != 231 : 0; + + *test_result += err == 0; + } + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/tracing_multi_fail.c b/tools/testing/selftests/bpf/progs/tracing_multi_fail.c new file mode 100644 index 000000000000..7f0375f4213d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_multi_fail.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +SEC("fentry.multi") +int BPF_PROG(test_fentry) +{ + return 0; +} + +SEC("fentry.multi.s") +int BPF_PROG(test_fentry_s) +{ + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/tracing_multi_intersect_attach.c b/tools/testing/selftests/bpf/progs/tracing_multi_intersect_attach.c new file mode 100644 index 000000000000..5b0af8f4c62f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_multi_intersect_attach.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +__hidden extern int tracing_multi_arg_check(__u64 *ctx, __u64 *test_result, bool is_return); + +__u64 test_result_fentry_1 = 0; +__u64 test_result_fentry_2 = 0; +__u64 test_result_fexit_1 = 0; +__u64 test_result_fexit_2 = 0; +__u64 test_result_fentry = 0; + +SEC("fentry/bpf_fentry_test1") +int BPF_PROG(fentry) +{ + tracing_multi_arg_check(ctx, &test_result_fentry, false); + return 0; +} + +SEC("fentry.multi") +int BPF_PROG(fentry_1) +{ + tracing_multi_arg_check(ctx, &test_result_fentry_1, false); + return 0; +} + +SEC("fentry.multi") +int BPF_PROG(fentry_2) +{ + tracing_multi_arg_check(ctx, &test_result_fentry_2, false); + return 0; +} + +SEC("fexit.multi") +int BPF_PROG(fexit_1) +{ + tracing_multi_arg_check(ctx, &test_result_fexit_1, true); + return 0; +} + +SEC("fexit.multi") +int BPF_PROG(fexit_2) +{ + tracing_multi_arg_check(ctx, &test_result_fexit_2, true); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/tracing_multi_rollback.c b/tools/testing/selftests/bpf/progs/tracing_multi_rollback.c new file mode 100644 index 000000000000..a49d1d841f3a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_multi_rollback.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +int pid = 0; + +__u64 test_result_fentry = 0; +__u64 test_result_fexit = 0; + +SEC("?fentry.multi") +int BPF_PROG(test_fentry) +{ + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 0; + + test_result_fentry++; + return 0; +} + +SEC("?fexit.multi") +int BPF_PROG(test_fexit) +{ + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 0; + + test_result_fexit++; + return 0; +} + +SEC("?fentry/bpf_fentry_test1") +int BPF_PROG(extra) +{ + return 0; +} + +SEC("?fentry/bpf_fentry_test10") +int BPF_PROG(filler) +{ + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/tracing_multi_session_attach.c b/tools/testing/selftests/bpf/progs/tracing_multi_session_attach.c new file mode 100644 index 000000000000..7c9a46016ccd --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_multi_session_attach.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +__hidden extern int tracing_multi_arg_check(__u64 *ctx, __u64 *test_result, bool is_return); + +__u64 test_result_fentry = 0; +__u64 test_result_fexit = 0; + +SEC("fsession.multi/bpf_fentry_test*") +int BPF_PROG(test_session_1) +{ + volatile __u64 *cookie = bpf_session_cookie(ctx); + + if (bpf_session_is_return(ctx)) { + if (tracing_multi_arg_check(ctx, &test_result_fexit, true)) + return 0; + /* extra count for test_result_fexit cookie */ + test_result_fexit += *cookie == 0xbeafbeafbeafbeaf; + } else { + if (tracing_multi_arg_check(ctx, &test_result_fentry, false)) + return 0; + *cookie = 0xbeafbeafbeafbeaf; + } + return 0; +} + +SEC("fsession.multi.s/bpf_fentry_test1") +int BPF_PROG(test_fsession_s) +{ + volatile __u64 *cookie = bpf_session_cookie(ctx); + + if (bpf_session_is_return(ctx)) { + if (tracing_multi_arg_check(ctx, &test_result_fexit, true)) + return 0; + /* extra count for test_result_fexit cookie */ + test_result_fexit += *cookie == 0xbeafbeafbeafbeaf; + } else { + if (tracing_multi_arg_check(ctx, &test_result_fentry, false)) + return 0; + *cookie = 0xbeafbeafbeafbeaf; + } + return 0; +} + +SEC("fsession.multi/bpf_testmod:bpf_testmod_fentry_test*") +int BPF_PROG(test_session_2) +{ + volatile __u64 *cookie = bpf_session_cookie(ctx); + + if (bpf_session_is_return(ctx)) { + if (tracing_multi_arg_check(ctx, &test_result_fexit, true)) + return 0; + /* extra count for test_result_fexit cookie */ + test_result_fexit += *cookie == 0xbeafbeafbeafbeaf; + } else { + if (tracing_multi_arg_check(ctx, &test_result_fentry, false)) + return 0; + *cookie = 0xbeafbeafbeafbeaf; + } + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/tracing_multi_verifier.c b/tools/testing/selftests/bpf/progs/tracing_multi_verifier.c new file mode 100644 index 000000000000..7b6ed41bf452 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_multi_verifier.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +SEC("fentry.multi/bpf_fentry_test1") +__failure +__msg("func 'bpf_multi_func' doesn't have 1-th argument") +int BPF_PROG(fentry_direct_access, int a) +{ + return a; +} + +SEC("fexit.multi/bpf_fentry_test3") +__failure +__msg("invalid bpf_context access off=24 size=8") +int BPF_PROG(fexit_direct_access, char a, int b, __u64 c, int ret) +{ + return ret; +} + +SEC("fsession.multi/bpf_fentry_test4") +__failure +__msg("invalid bpf_context access off=16 size=8") +int BPF_PROG(fsession_direct_access, void *a, char b, int c, __u64 d, int ret) +{ + return c; +} diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128.c new file mode 100644 index 000000000000..4638dfec1f38 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_helpers.h> + +long t_b, t_c, t_ret; + +SEC("fexit/bpf_testmod_test_int128_arg") +int test_int128_arg_fexit(unsigned long long *ctx) +{ + t_b = (int)ctx[2]; + t_c = (long)ctx[3]; + t_ret = (long)ctx[4]; + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/trigger_bench.c b/tools/testing/selftests/bpf/progs/trigger_bench.c index 4ea0422d1042..3225b4aee8ff 100644 --- a/tools/testing/selftests/bpf/progs/trigger_bench.c +++ b/tools/testing/selftests/bpf/progs/trigger_bench.c @@ -1,10 +1,11 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2020 Facebook -#include <linux/bpf.h> +#include "vmlinux.h" #include <asm/unistd.h> #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> #include "bpf_misc.h" +#include "bpf/usdt.bpf.h" char _license[] SEC("license") = "GPL"; @@ -180,3 +181,10 @@ int bench_trigger_rawtp(void *ctx) handle(ctx); return 0; } + +SEC("?usdt") +int bench_trigger_usdt(void *ctx) +{ + inc_counter(); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/uninit_stack.c b/tools/testing/selftests/bpf/progs/uninit_stack.c index 046a204c8fc6..5db02323c89c 100644 --- a/tools/testing/selftests/bpf/progs/uninit_stack.c +++ b/tools/testing/selftests/bpf/progs/uninit_stack.c @@ -76,6 +76,7 @@ __naked int helper_uninit_to_misc(void *ctx) * thus showing the stack state, matched by __msg(). \ */ \ call %[dummy]; \ + r1 = *(u64*)(r10 - 104); \ r0 = 0; \ exit; \ " diff --git a/tools/testing/selftests/bpf/progs/user_ringbuf_fail.c b/tools/testing/selftests/bpf/progs/user_ringbuf_fail.c index 54de0389f878..c0d0422b8030 100644 --- a/tools/testing/selftests/bpf/progs/user_ringbuf_fail.c +++ b/tools/testing/selftests/bpf/progs/user_ringbuf_fail.c @@ -146,7 +146,7 @@ try_discard_dynptr(struct bpf_dynptr *dynptr, void *context) * not be able to read past the end of the pointer. */ SEC("?raw_tp") -__failure __msg("cannot release unowned const bpf_dynptr") +__failure __msg("CONST_PTR_TO_DYNPTR cannot be released") int user_ringbuf_callback_discard_dynptr(void *ctx) { bpf_user_ringbuf_drain(&user_ringbuf, try_discard_dynptr, NULL, 0); @@ -166,7 +166,7 @@ try_submit_dynptr(struct bpf_dynptr *dynptr, void *context) * not be able to read past the end of the pointer. */ SEC("?raw_tp") -__failure __msg("cannot release unowned const bpf_dynptr") +__failure __msg("CONST_PTR_TO_DYNPTR cannot be released") int user_ringbuf_callback_submit_dynptr(void *ctx) { bpf_user_ringbuf_drain(&user_ringbuf, try_submit_dynptr, NULL, 0); diff --git a/tools/testing/selftests/bpf/progs/verifier_align.c b/tools/testing/selftests/bpf/progs/verifier_align.c new file mode 100644 index 000000000000..3e52686515ca --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_align.c @@ -0,0 +1,581 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +/* Converted from tools/testing/selftests/bpf/prog_tests/align.c */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* Four tests of known constants. These aren't staggeringly + * interesting since we track exact values now. + */ + +SEC("tc") +__success __log_level(2) +__flag(BPF_F_ANY_ALIGNMENT) +__msg("0: R1=ctx() R10=fp0") +__msg("0: {{.*}} R3=2") +__msg("1: {{.*}} R3=4") +__msg("2: {{.*}} R3=8") +__msg("3: {{.*}} R3=16") +__msg("4: {{.*}} R3=32") +__naked void mov(void) +{ + asm volatile (" \ + r3 = 2; \ + r3 = 4; \ + r3 = 8; \ + r3 = 16; \ + r3 = 32; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("tc") +__success __log_level(2) +__flag(BPF_F_ANY_ALIGNMENT) +__msg("0: R1=ctx() R10=fp0") +__msg("0: {{.*}}R3=1") +__msg("1: {{.*}}R3=2") +__msg("2: {{.*}}R3=4") +__msg("3: {{.*}}R3=8") +__msg("4: {{.*}}R3=16") +__msg("5: {{.*}}R3=1") +__msg("6: {{.*}}R4=32") +__msg("7: {{.*}}R4=16") +__msg("8: {{.*}}R4=8") +__msg("9: {{.*}}R4=4") +__msg("10: {{.*}}R4=2") +__naked void shift(void) +{ + asm volatile (" \ + r3 = 1; \ + r3 <<= 1; \ + r3 <<= 1; \ + r3 <<= 1; \ + r3 <<= 1; \ + r3 >>= 4; \ + r4 = 32; \ + r4 >>= 1; \ + r4 >>= 1; \ + r4 >>= 1; \ + r4 >>= 1; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("tc") +__success __log_level(2) +__flag(BPF_F_ANY_ALIGNMENT) +__msg("0: R1=ctx() R10=fp0") +__msg("0: {{.*}}R3=4") +__msg("1: {{.*}}R3=8") +__msg("2: {{.*}}R3=10") +__msg("3: {{.*}}R4=8") +__msg("4: {{.*}}R4=12") +__msg("5: {{.*}}R4=14") +__naked void addsub(void) +{ + asm volatile (" \ + r3 = 4; \ + r3 += 4; \ + r3 += 2; \ + r4 = 8; \ + r4 += 4; \ + r4 += 2; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("tc") +__success __log_level(2) +__flag(BPF_F_ANY_ALIGNMENT) +__msg("0: R1=ctx() R10=fp0") +__msg("0: {{.*}}R3=7") +__msg("1: {{.*}}R3=7") +__msg("2: {{.*}}R3=14") +__msg("3: {{.*}}R3=56") +__naked void mul(void) +{ + asm volatile (" \ + r3 = 7; \ + r3 *= 1; \ + r3 *= 2; \ + r3 *= 4; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* Tests using unknown values */ + +#define PREP_PKT_POINTERS \ + "r2 = *(u32*)(r1 + %[__sk_buff_data]);" \ + "r3 = *(u32*)(r1 + %[__sk_buff_data_end]);" + +#define __LOAD_UNKNOWN(DST_REG, LBL) \ + "r2 = *(u32*)(r1 + %[__sk_buff_data]);" \ + "r3 = *(u32*)(r1 + %[__sk_buff_data_end]);" \ + "r0 = r2;" \ + "r0 += 8;" \ + "if r3 >= r0 goto " LBL ";" \ + "exit;" \ +LBL ":" \ + DST_REG " = *(u8*)(r2 + 0);" + +#define LOAD_UNKNOWN(DST_REG) __LOAD_UNKNOWN(DST_REG, "l99_%=") + +SEC("tc") +__success __log_level(2) +__flag(BPF_F_ANY_ALIGNMENT) +__msg("6: {{.*}} R2=pkt(r=8)") +__msg("6: {{.*}} R3={{[^)]*}}var_off=(0x0; 0xff)") +__msg("7: {{.*}} R3={{[^)]*}}var_off=(0x0; 0x1fe)") +__msg("8: {{.*}} R3={{[^)]*}}var_off=(0x0; 0x3fc)") +__msg("9: {{.*}} R3={{[^)]*}}var_off=(0x0; 0x7f8)") +__msg("10: {{.*}} R3={{[^)]*}}var_off=(0x0; 0xff0)") +__msg("12: {{.*}} R3=pkt_end()") +__msg("17: {{.*}} R4={{[^)]*}}var_off=(0x0; 0xff)") +__msg("18: {{.*}} R4={{[^)]*}}var_off=(0x0; 0x1fe0)") +__msg("19: {{.*}} R4={{[^)]*}}var_off=(0x0; 0xff0)") +__msg("20: {{.*}} R4={{[^)]*}}var_off=(0x0; 0x7f8)") +__msg("21: {{.*}} R4={{[^)]*}}var_off=(0x0; 0x3fc)") +__msg("22: {{.*}} R4={{[^)]*}}var_off=(0x0; 0x1fe)") +__naked void unknown_shift(void) +{ + asm volatile (" \ + " __LOAD_UNKNOWN("r3", "l99_%=") " \ + r3 <<= 1; \ + r3 <<= 1; \ + r3 <<= 1; \ + r3 <<= 1; \ + " __LOAD_UNKNOWN("r4", "l98_%=") " \ + r4 <<= 5; \ + r4 >>= 1; \ + r4 >>= 1; \ + r4 >>= 1; \ + r4 >>= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + +SEC("tc") +__success __log_level(2) +__flag(BPF_F_ANY_ALIGNMENT) +__msg("6: {{.*}} R3={{[^)]*}}var_off=(0x0; 0xff)") +__msg("7: {{.*}} R4={{[^)]*}}var_off=(0x0; 0xff)") +__msg("8: {{.*}} R4={{[^)]*}}var_off=(0x0; 0xff)") +__msg("9: {{.*}} R4={{[^)]*}}var_off=(0x0; 0xff)") +__msg("10: {{.*}} R4={{[^)]*}}var_off=(0x0; 0x1fe)") +__msg("11: {{.*}} R4={{[^)]*}}var_off=(0x0; 0xff)") +__msg("12: {{.*}} R4={{[^)]*}}var_off=(0x0; 0x3fc)") +__msg("13: {{.*}} R4={{[^)]*}}var_off=(0x0; 0xff)") +__msg("14: {{.*}} R4={{[^)]*}}var_off=(0x0; 0x7f8)") +__msg("15: {{.*}} R4={{[^)]*}}var_off=(0x0; 0xff0)") +__naked void unknown_mul(void) +{ + asm volatile (" \ + " LOAD_UNKNOWN("r3") " \ + r4 = r3; \ + r4 *= 1; \ + r4 = r3; \ + r4 *= 2; \ + r4 = r3; \ + r4 *= 4; \ + r4 = r3; \ + r4 *= 8; \ + r4 *= 2; \ + r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + +SEC("tc") +__success __log_level(2) +__msg("2: {{.*}} R5=pkt(r=0)") +__msg("4: {{.*}} R5=pkt(r=0,imm=14)") +__msg("5: {{.*}} R4=pkt(r=0,imm=14)") +__msg("9: {{.*}} R5=pkt(r=18,imm=14)") +__msg("10: {{.*}} R4={{[^)]*}}var_off=(0x0; 0xff){{.*}} R5=pkt(r=18,imm=14)") +__msg("13: {{.*}} R4={{[^)]*}}var_off=(0x0; 0xffff)") +__msg("14: {{.*}} R4={{[^)]*}}var_off=(0x0; 0xffff)") +__naked void packet_const_offset(void) +{ + asm volatile (" \ + " PREP_PKT_POINTERS " \ + r5 = r2; \ + r0 = 0; \ + /* Skip over ethernet header. */ \ + r5 += 14; \ + r4 = r5; \ + r4 += 4; \ + if r3 >= r4 goto l0_%=; \ + exit; \ +l0_%=: r4 = *(u8*)(r5 + 0); \ + r4 = *(u8*)(r5 + 1); \ + r4 = *(u8*)(r5 + 2); \ + r4 = *(u8*)(r5 + 3); \ + r4 = *(u16*)(r5 + 0); \ + r4 = *(u16*)(r5 + 2); \ + r4 = *(u32*)(r5 + 0); \ + r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + +SEC("tc") +__success __log_level(2) +__flag(BPF_F_ANY_ALIGNMENT) +/* Calculated offset in R6 has unknown value, but known + * alignment of 4. + */ +__msg("6: {{.*}} R2=pkt(r=8)") +__msg("7: {{.*}} R6={{[^)]*}}var_off=(0x0; 0x3fc)") +/* Offset is added to packet pointer R5, resulting in + * known fixed offset, and variable offset from R6. + */ +__msg("11: {{.*}} R5=pkt(id=1,{{[^)]*}},var_off=(0x2; 0x7fc)") +/* At the time the word size load is performed from R5, + * it's total offset is NET_IP_ALIGN + reg->off (0) + + * reg->aux_off (14) which is 16. Then the variable + * offset is considered using reg->aux_off_align which + * is 4 and meets the load's requirements. + */ +__msg("15: {{.*}} R5={{[^)]*}}var_off=(0x2; 0x7fc)") +/* Variable offset is added to R5 packet pointer, + * resulting in auxiliary alignment of 4. To avoid BPF + * verifier's precision backtracking logging + * interfering we also have a no-op R4 = R5 + * instruction to validate R5 state. We also check + * that R4 is what it should be in such case. + */ +__msg("18: {{.*}} R4={{[^)]*}}var_off=(0x0; 0x3fc){{.*}} R5={{[^)]*}}var_off=(0x0; 0x3fc)") +/* Constant offset is added to R5, resulting in + * reg->off of 14. + */ +__msg("19: {{.*}} R5=pkt(id=2,{{[^)]*}}var_off=(0x2; 0x7fc)") +/* At the time the word size load is performed from R5, + * its total fixed offset is NET_IP_ALIGN + reg->off + * (14) which is 16. Then the variable offset is 4-byte + * aligned, so the total offset is 4-byte aligned and + * meets the load's requirements. + */ +__msg("24: {{.*}} R5={{[^)]*}}var_off=(0x2; 0x7fc)") +/* Constant offset is added to R5 packet pointer, + * resulting in reg->off value of 14. + */ +__msg("26: {{.*}} R5=pkt(r=8,imm=14)") +/* Variable offset is added to R5, resulting in a + * variable offset of (4n). See comment for insn #18 + * for R4 = R5 trick. + */ +__msg("28: {{.*}} R4={{[^)]*}}var_off=(0x2; 0x7fc){{.*}} R5={{[^)]*}}var_off=(0x2; 0x7fc)") +/* Constant is added to R5 again, setting reg->off to 18. */ +__msg("29: {{.*}} R5=pkt(id=3,{{[^)]*}}var_off=(0x2; 0x7fc)") +/* And once more we add a variable; resulting {{[^)]*}}var_off + * is still (4n), fixed offset is not changed. + * Also, we create a new reg->id. + */ +__msg("31: {{.*}} R4={{[^)]*}}var_off=(0x2; 0xffc){{.*}} R5={{[^)]*}}var_off=(0x2; 0xffc)") +/* At the time the word size load is performed from R5, + * its total fixed offset is NET_IP_ALIGN + reg->off (18) + * which is 20. Then the variable offset is (4n), so + * the total offset is 4-byte aligned and meets the + * load's requirements. + */ +__msg("35: {{.*}} R5={{[^)]*}}var_off=(0x2; 0xffc)") +__naked void packet_variable_offset(void) +{ + asm volatile (" \ + " LOAD_UNKNOWN("r6") " \ + r6 <<= 2; \ + /* First, add a constant to the R5 packet pointer,\ + * then a variable with a known alignment. \ + */ \ + r5 = r2; \ + r5 += 14; \ + r5 += r6; \ + r4 = r5; \ + r4 += 4; \ + if r3 >= r4 goto l0_%=; \ + exit; \ +l0_%=: r4 = *(u32*)(r5 + 0); \ + /* Now, test in the other direction. Adding first\ + * the variable offset to R5, then the constant.\ + */ \ + r5 = r2; \ + r5 += r6; \ + r4 = r5; \ + r5 += 14; \ + r4 = r5; \ + r4 += 4; \ + if r3 >= r4 goto l1_%=; \ + exit; \ +l1_%=: r4 = *(u32*)(r5 + 0); \ + /* Test multiple accumulations of unknown values\ + * into a packet pointer. \ + */ \ + r5 = r2; \ + r5 += 14; \ + r5 += r6; \ + r4 = r5; \ + r5 += 4; \ + r5 += r6; \ + r4 = r5; \ + r4 += 4; \ + if r3 >= r4 goto l2_%=; \ + exit; \ +l2_%=: r4 = *(u32*)(r5 + 0); \ + r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + +SEC("tc") +__success __log_level(2) +__flag(BPF_F_ANY_ALIGNMENT) +/* Calculated offset in R6 has unknown value, but known + * alignment of 4. + */ +__msg("6: {{.*}} R2=pkt(r=8)") +__msg("7: {{.*}} R6={{[^)]*}}var_off=(0x0; 0x3fc)") +/* Adding 14 makes R6 be (4n+2) */ +__msg("8: {{.*}} R6={{[^)]*}}var_off=(0x2; 0x7fc)") +/* Packet pointer has (4n+2) offset */ +__msg("11: {{.*}} R5={{[^)]*}}var_off=(0x2; 0x7fc)") +__msg("12: {{.*}} R4={{[^)]*}}var_off=(0x2; 0x7fc)") +/* At the time the word size load is performed from R5, + * its total fixed offset is NET_IP_ALIGN + reg->off (0) + * which is 2. Then the variable offset is (4n+2), so + * the total offset is 4-byte aligned and meets the + * load's requirements. + */ +__msg("15: {{.*}} R5={{[^)]*}}var_off=(0x2; 0x7fc)") +/* Newly read value in R6 was shifted left by 2, so has + * known alignment of 4. + */ +__msg("17: {{.*}} R6={{[^)]*}}var_off=(0x0; 0x3fc)") +/* Added (4n) to packet pointer's (4n+2) {{[^)]*}}var_off, giving + * another (4n+2). + */ +__msg("19: {{.*}} R5={{[^)]*}}var_off=(0x2; 0xffc)") +__msg("20: {{.*}} R4={{[^)]*}}var_off=(0x2; 0xffc)") +/* At the time the word size load is performed from R5, + * its total fixed offset is NET_IP_ALIGN + reg->off (0) + * which is 2. Then the variable offset is (4n+2), so + * the total offset is 4-byte aligned and meets the + * load's requirements. + */ +__msg("23: {{.*}} R5={{[^)]*}}var_off=(0x2; 0xffc)") +__naked void packet_variable_offset_2(void) +{ + asm volatile (" \ + /* Create an unknown offset, (4n+2)-aligned */ \ + " LOAD_UNKNOWN("r6") " \ + r6 <<= 2; \ + r6 += 14; \ + /* Add it to the packet pointer */ \ + r5 = r2; \ + r5 += r6; \ + /* Check bounds and perform a read */ \ + r4 = r5; \ + r4 += 4; \ + if r3 >= r4 goto l0_%=; \ + exit; \ +l0_%=: r6 = *(u32*)(r5 + 0); \ + /* Make a (4n) offset from the value we just read */\ + r6 &= 0xff; \ + r6 <<= 2; \ + /* Add it to the packet pointer */ \ + r5 += r6; \ + /* Check bounds and perform a read */ \ + r4 = r5; \ + r4 += 4; \ + if r3 >= r4 goto l1_%=; \ + exit; \ +l1_%=: r6 = *(u32*)(r5 + 0); \ + r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + +SEC("tc") +__failure __log_level(2) +__msg("3: {{.*}} R5=pkt_end()") +/* (ptr - ptr) << 2 == unknown, (4n) */ +__msg("5: {{.*}} R5={{[^)]*}}var_off=(0x0; 0xfffffffffffffffc)") +/* (4n) + 14 == (4n+2). We blow our bounds, because + * the add could overflow. + */ +__msg("6: {{.*}} R5={{[^)]*}}var_off=(0x2; 0xfffffffffffffffc)") +/* Checked s>=0 */ +__msg("9: {{.*}} R5={{[^)]*}}var_off=(0x2; 0x7ffffffffffffffc)") +/* packet pointer + nonnegative (4n+2) */ +__msg("11: {{.*}} R4={{[^)]*}}var_off=(0x2; 0x7ffffffffffffffc){{.*}} R6={{[^)]*}}var_off=(0x2; 0x7ffffffffffffffc)") +__msg("12: (07) r4 += 4") +/* packet smax bound overflow */ +__msg("pkt pointer offset -9223372036854775808 is not allowed") +__naked void dubious_pointer_arithmetic(void) +{ + asm volatile (" \ + " PREP_PKT_POINTERS " \ + r0 = 0; \ + /* (ptr - ptr) << 2 */ \ + r5 = r3; \ + r5 -= r2; \ + r5 <<= 2; \ + /* We have a (4n) value. Let's make a packet offset\ + * out of it. First add 14, to make it a (4n+2)\ + */ \ + r5 += 14; \ + /* Then make sure it's nonnegative */ \ + if r5 s>= 0 goto l0_%=; \ + exit; \ +l0_%=: /* Add it to packet pointer */ \ + r6 = r2; \ + r6 += r5; \ + /* Check bounds and perform a read */ \ + r4 = r6; \ + r4 += 4; \ + if r3 >= r4 goto l1_%=; \ + exit; \ +l1_%=: r4 = *(u32*)(r6 + 0); \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + +SEC("tc") +__success __log_level(2) +__flag(BPF_F_ANY_ALIGNMENT) +/* Calculated offset in R6 has unknown value, but known + * alignment of 4. + */ +__msg("6: {{.*}} R2=pkt(r=8)") +__msg("8: {{.*}} R6={{[^)]*}}var_off=(0x0; 0x3fc)") +/* Adding 14 makes R6 be (4n+2) */ +__msg("9: {{.*}} R6={{[^)]*}}var_off=(0x2; 0x7fc)") +/* New unknown value in R7 is (4n) */ +__msg("10: {{.*}} R7={{[^)]*}}var_off=(0x0; 0x3fc)") +/* Subtracting it from R6 blows our unsigned bounds */ +__msg("11: {{.*}} R6={{[^)]*}}var_off=(0x2; 0xfffffffffffffffc)") +/* Checked s>= 0 */ +__msg("14: {{.*}} R6={{[^)]*}}var_off=(0x2; 0x7fc)") +/* At the time the word size load is performed from R5, + * its total fixed offset is NET_IP_ALIGN + reg->off (0) + * which is 2. Then the variable offset is (4n+2), so + * the total offset is 4-byte aligned and meets the + * load's requirements. + */ +__msg("20: {{.*}} R5={{[^)]*}}var_off=(0x2; 0x7fc)") +__naked void variable_subtraction(void) +{ + asm volatile (" \ + /* Create an unknown offset, (4n+2)-aligned */ \ + " LOAD_UNKNOWN("r6") " \ + r7 = r6; \ + r6 <<= 2; \ + r6 += 14; \ + /* Create another unknown, (4n)-aligned, and subtract\ + * it from the first one \ + */ \ + r7 <<= 2; \ + r6 -= r7; \ + /* Bounds-check the result */ \ + if r6 s>= 0 goto l0_%=; \ + exit; \ +l0_%=: /* Add it to the packet pointer */ \ + r5 = r2; \ + r5 += r6; \ + /* Check bounds and perform a read */ \ + r4 = r5; \ + r4 += 4; \ + if r3 >= r4 goto l1_%=; \ + exit; \ +l1_%=: r6 = *(u32*)(r5 + 0); \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + +SEC("tc") +__success __log_level(2) +__flag(BPF_F_ANY_ALIGNMENT) +/* Calculated offset in R6 has unknown value, but known + * alignment of 4. + */ +__msg("6: {{.*}} R2=pkt(r=8)") +__msg("9: {{.*}} R6={{[^)]*}}var_off=(0x0; 0x3c)") +/* Adding 14 makes R6 be (4n+2) */ +__msg("10: {{.*}} R6={{[^)]*}}var_off=(0x2; 0x7c)") +/* Subtracting from packet pointer overflows ubounds */ +__msg("13: R5={{[^)]*}}var_off=(0xffffffffffffff82; 0x7c)") +/* New unknown value in R7 is (4n), >= 76 */ +__msg("14: {{.*}} R7={{[^)]*}}var_off=(0x0; 0x7fc)") +/* Adding it to packet pointer gives nice bounds again */ +__msg("16: {{.*}} R5={{[^)]*}}var_off=(0x2; 0x7fc)") +/* At the time the word size load is performed from R5, + * its total fixed offset is NET_IP_ALIGN + reg->off (0) + * which is 2. Then the variable offset is (4n+2), so + * the total offset is 4-byte aligned and meets the + * load's requirements. + */ +__msg("20: {{.*}} R5={{[^)]*}}var_off=(0x2; 0x7fc)") +__naked void pointer_variable_subtraction(void) +{ + asm volatile (" \ + /* Create an unknown offset, (4n+2)-aligned and bounded\ + * to [14,74] \ + */ \ + " LOAD_UNKNOWN("r6") " \ + r7 = r6; \ + r6 &= 0xf; \ + r6 <<= 2; \ + r6 += 14; \ + /* Subtract it from the packet pointer */ \ + r5 = r2; \ + r5 -= r6; \ + /* Create another unknown, (4n)-aligned and >= 74.\ + * That in fact means >= 76, since 74 mod 4 == 2\ + */ \ + r7 <<= 2; \ + r7 += 76; \ + /* Add it to the packet pointer */ \ + r5 += r7; \ + /* Check bounds and perform a read */ \ + r4 = r5; \ + r4 += 4; \ + if r3 >= r4 goto l0_%=; \ + exit; \ +l0_%=: r6 = *(u32*)(r5 + 0); \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index c4b8daac4388..3e33766547c0 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c @@ -8,19 +8,21 @@ #include <bpf/bpf_tracing.h> #include "bpf_misc.h" #include "bpf_experimental.h" -#include "bpf_arena_common.h" +#include <bpf_arena_common.h> #define private(name) SEC(".bss." #name) __hidden __attribute__((aligned(8))) +#ifdef __TARGET_ARCH_arm64 +#define ARENA_VM_START ((1ull << 32) | (~0u - __PAGE_SIZE * 2 + 1)) +#else +#define ARENA_VM_START ((1ull << 44) | (~0u - __PAGE_SIZE * 2 + 1)) +#endif + struct { __uint(type, BPF_MAP_TYPE_ARENA); __uint(map_flags, BPF_F_MMAPABLE); __uint(max_entries, 2); /* arena of two pages close to 32-bit boundary*/ -#ifdef __TARGET_ARCH_arm64 - __ulong(map_extra, (1ull << 32) | (~0u - __PAGE_SIZE * 2 + 1)); /* start of mmap() region */ -#else - __ulong(map_extra, (1ull << 44) | (~0u - __PAGE_SIZE * 2 + 1)); /* start of mmap() region */ -#endif + __ulong(map_extra, ARENA_VM_START); /* start of mmap() region */ } arena SEC(".maps"); SEC("socket") @@ -93,6 +95,34 @@ int basic_alloc1(void *ctx) return 0; } +SEC("syscall") +__success __retval(0) +int free_scalar_below_arena(void *ctx) +{ + void __arena *page1, *page2, *page3; + __u64 bad_addr = ARENA_VM_START - __PAGE_SIZE; + + page1 = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!page1) + return 1; + + page2 = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!page2) + return 2; + + page3 = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (page3) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)bad_addr, 1); + + page3 = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (page3) + return 4; + + return 0; +} + SEC("socket") __success __retval(0) int basic_alloc2_nosleep(void *ctx) @@ -477,4 +507,280 @@ int arena_kfuncs_under_bpf_lock(void *ctx) return 0; } + +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + +/* + * Test that scalar += PTR_TO_ARENA correctly upgrades the + * destination register to a PTR_TO_ARENA. + */ +SEC("syscall") +__success __retval(0) +int scalar_add_arena_ptr(void *ctx) +{ + int __arena *scalar, *arena_ptr; + + volatile char __arena *base = arena_base(&arena); + + asm volatile ( + "%[arena_ptr] = 8192;" + "%[arena_ptr] = addr_space_cast(%[arena_ptr], 0x0, 0x1);" + "%[scalar] = 12;" + "%[scalar] += %[arena_ptr];" + : [scalar] "=r"(scalar), + [arena_ptr] "=&r"(arena_ptr) + : "r"(base) + : + ); + return 0; +} + +/* + * Tests that PTR_TO_ARENA + PTR_TO_ARENA is allowed. + */ +SEC("syscall") +__success __retval(0) +int arena_ptr_add_arena_ptr(void *ctx) +{ + int __arena *arena_ptr2, *arena_ptr1; + + /* Needed for the verifier to link the arena to the subprog. */ + volatile char __arena *base = arena_base(&arena); + + asm volatile ( + "%[arena_ptr1] = 8192;" + "%[arena_ptr1] = addr_space_cast(%[arena_ptr1], 0x0, 0x1);" + "%[arena_ptr2] = 4096;" + "%[arena_ptr2] = addr_space_cast(%[arena_ptr2], 0x0, 0x1);" + "%[arena_ptr2] += %[arena_ptr1];" + : [arena_ptr2] "=r"(arena_ptr2), + [arena_ptr1] "=&r"(arena_ptr1) + : "r"(base) + : + ); + return 0; +} + +SEC("syscall") +__failure __msg("same insn cannot be used with and without arena pointer") +int mixed_arena_scalar_alu64_scalar_first(void *ctx) +{ + volatile register __u64 reg asm("r3"); + __u32 pick_arena = bpf_get_prandom_u32(); + + reg = 1ULL << 32; + + if (pick_arena) { + asm volatile ( + "r9 = %[arena] ll;" + "%[reg] = 0;" + "%[reg] = addr_space_cast(%[reg], 0x0, 0x1);" + : [reg] "=r"(reg) + : __imm_addr(arena) + : "r9" + ); + } + + reg += 1; + + return 0; +} + +SEC("syscall") +__failure __msg("same insn cannot be used with and without arena pointer") +int mixed_arena_scalar_alu64_arena_first(void *ctx) +{ + volatile register __u64 reg asm("r3"); + __u32 pick_scalar = bpf_get_prandom_u32(); + + asm volatile ( + "r9 = %[arena] ll;" + "%[reg] = 0;" + "%[reg] = addr_space_cast(%[reg], 0x0, 0x1);" + : [reg] "=r"(reg) + : __imm_addr(arena) + : "r9" + ); + + if (pick_scalar) + reg = 1ULL << 32; + + reg += 1; + + return 0; +} + +SEC("syscall") +__success __retval(0) +int scalar_xor_arena_ptr(void *ctx) +{ + int __arena *scalar, *arena_ptr; + + volatile char __arena *base = arena_base(&arena); + + asm volatile ( + "%[arena_ptr] = 8192;" + "%[arena_ptr] = addr_space_cast(%[arena_ptr], 0x0, 0x1);" + "%[scalar] = 12;" + "%[scalar] ^= %[arena_ptr];" + : [scalar] "=r"(scalar), + [arena_ptr] "=&r"(arena_ptr) + : "r"(base) + : + ); + return 0; +} + +/* + * Tests that PTR_TO_ARENA and non-arena pointers can be added. + */ +SEC("syscall") +__success __retval(0) +int arena_ptr_add_to_non_arena_ptr(void *ctx) +{ + register int __arena *arena_ptr asm("r3"); + register void *dst asm("r4"); + + volatile char __arena *base = arena_base(&arena); + + asm volatile ( + "%[arena_ptr] = 8192;" + "%[arena_ptr] = addr_space_cast(%[arena_ptr], 0x0, 0x1);" + "%[dst] = %[ctx];" + "%[dst] += %[arena_ptr];" + : [arena_ptr] "=&r"(arena_ptr), + [dst] "=&r"(dst) + : [ctx] "r"(ctx), "r"(base) + : + ); + + (void)ctx; + + return 0; +} + +SEC("syscall") +__success __retval(0) +int non_arena_ptr_add_to_arena_ptr(void *ctx) +{ + register int __arena *arena_ptr asm("r3"); + register void *src asm("r4"); + + volatile char __arena *base = arena_base(&arena); + + asm volatile ( + "%[arena_ptr] = 8192;" + "%[arena_ptr] = addr_space_cast(%[arena_ptr], 0x0, 0x1);" + "%[src] = %[ctx];" + "%[arena_ptr] += %[src];" + : [arena_ptr] "=&r"(arena_ptr), + [src] "=&r"(src) + : [ctx] "r"(ctx), "r"(base) + : + ); + + (void)ctx; + + return 0; +} + +SEC("socket") +__description("arena and stack atomic at the same instruction") +__failure __msg("same insn cannot be used with different pointers") +__arch_x86_64 +__load_if_JITed() +__naked void mixed_arena_stack_atomic(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r6 = r10; \ + r6 += -8; \ + r9 = 0; \ + *(u64 *)(r6 + 0) = r9; \ + r7 = 8192; \ + r7 = addr_space_cast(r7, 0, 1); \ + call %[bpf_get_prandom_u32]; \ + if w0 != 0 goto 1f; \ + r8 = r6; \ + goto 2f; \ +1: r8 = r7; \ +2: r9 = 1; \ + lock *(u64 *)(r8 + 0) += r9; \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(arena), + __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +#endif /* defined(__BPF_FEATURE_ADDR_SPACE_CAST) */ + +static __noinline +u32 __arena *check_arena_arg_nonglobal(u32 __arena *arg) +{ + volatile u32 val = *arg; + + *arg = val + 1; + + return arg; +} + +__weak +u32 __arena *check_arena_arg_global(u32 __arena *arg) +{ + volatile u32 val = *arg; + + *arg = val + 1; + + return arg; +} + +__weak +u32 volatile __arena *check_arena_arg_quals1(u32 volatile __arena *arg1, u32 __arena volatile *arg2) +{ + *arg1 = *arg1 + 1; + *arg2 = *arg1 + 1; + + return arg2; +} + +__weak +u32 __arena volatile *check_arena_arg_quals2(u32 volatile __arena *arg1, u32 __arena volatile *arg2) +{ + *arg1 = *arg1 + 1; + *arg2 = *arg2 + 1; + + return arg2; +} + +SEC("syscall") +__success __retval(0) +int check_arena_arg_ret(void *ctx) +{ + u32 __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + u32 __arena *arg = page; + u32 __arena volatile *arg1; + u32 __arena volatile *ret1; + u32 volatile __arena *arg2; + u32 volatile __arena *ret2; + + if (!arg) + return 1; + + /* Make sure we use {arg, ret}{1, 2}. */ + + arg = check_arena_arg_nonglobal(page); + arg = check_arena_arg_global(arg); + + arg1 = arg2 = page; + ret1 = check_arena_arg_quals1(arg1, arg2); + ret2 = check_arena_arg_quals2(arg1, arg2); + + if (!(*ret1 ||*ret2)) + return -EINVAL; + + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_globals1.c b/tools/testing/selftests/bpf/progs/verifier_arena_globals1.c index 83182ddbfb95..45d364b0bc85 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena_globals1.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena_globals1.c @@ -6,7 +6,7 @@ #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> #include "bpf_experimental.h" -#include "bpf_arena_common.h" +#include <bpf_arena_common.h> #include "bpf_misc.h" #define ARENA_PAGES (1UL<< (32 - __builtin_ffs(__PAGE_SIZE) + 1)) diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_globals2.c b/tools/testing/selftests/bpf/progs/verifier_arena_globals2.c index e6bd7b61f9f1..b51594dbc005 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena_globals2.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena_globals2.c @@ -7,7 +7,7 @@ #include <bpf/bpf_tracing.h> #include "bpf_misc.h" #include "bpf_experimental.h" -#include "bpf_arena_common.h" +#include <bpf_arena_common.h> #define ARENA_PAGES (32) diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_large.c b/tools/testing/selftests/bpf/progs/verifier_arena_large.c index 5f7e7afee169..6ab8730d4878 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena_large.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena_large.c @@ -7,7 +7,7 @@ #include <bpf/bpf_tracing.h> #include "bpf_misc.h" #include "bpf_experimental.h" -#include "bpf_arena_common.h" +#include <bpf_arena_common.h> #define ARENA_SIZE (1ull << 32) diff --git a/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c b/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c index 39aff82549c9..1b653bfb63eb 100644 --- a/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c +++ b/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c @@ -9,6 +9,11 @@ char _license[] SEC("license") = "GPL"; +struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; +void bpf_task_release(struct task_struct *p) __ksym; +void bpf_rcu_read_lock(void) __ksym; +void bpf_rcu_read_unlock(void) __ksym; + /* Timer tests */ struct timer_elem { @@ -31,7 +36,7 @@ static int timer_cb(void *map, int *key, struct bpf_timer *timer) } SEC("fentry/bpf_fentry_test1") -__failure __msg("helper call might sleep in a non-sleepable prog") +__failure __msg("sleepable helper bpf_copy_from_user#{{[0-9]+}} in non-sleepable prog") int timer_non_sleepable_prog(void *ctx) { struct timer_elem *val; @@ -47,7 +52,7 @@ int timer_non_sleepable_prog(void *ctx) } SEC("lsm.s/file_open") -__failure __msg("helper call might sleep in a non-sleepable prog") +__failure __msg("sleepable helper bpf_copy_from_user#{{[0-9]+}} in non-sleepable prog") int timer_sleepable_prog(void *ctx) { struct timer_elem *val; @@ -62,10 +67,109 @@ int timer_sleepable_prog(void *ctx) return 0; } +static int timer_sys_bpf_cb(void *map, int *key, struct bpf_timer *timer) +{ + __u64 attr = 0; + + bpf_sys_bpf(BPF_MAP_FREEZE, &attr, sizeof(attr)); + return 0; +} + +SEC("syscall") +__failure __msg("sleepable helper bpf_sys_bpf#{{[0-9]+}} in non-sleepable prog") +int timer_sys_bpf_prog(void *ctx) +{ + struct timer_elem *val; + int key = 0; + + val = bpf_map_lookup_elem(&timer_map, &key); + if (!val) + return 0; + + bpf_timer_init(&val->t, &timer_map, 0); + bpf_timer_set_callback(&val->t, timer_sys_bpf_cb); + return 0; +} + +static int timer_sys_close_cb(void *map, int *key, struct bpf_timer *timer) +{ + bpf_sys_close(0); + return 0; +} + +SEC("syscall") +__failure __msg("sleepable helper bpf_sys_close#{{[0-9]+}} in non-sleepable prog") +int timer_sys_close_prog(void *ctx) +{ + struct timer_elem *val; + int key = 0; + + val = bpf_map_lookup_elem(&timer_map, &key); + if (!val) + return 0; + + bpf_timer_init(&val->t, &timer_map, 0); + bpf_timer_set_callback(&val->t, timer_sys_close_cb); + return 0; +} + +static int timer_btf_find_cb(void *map, int *key, struct bpf_timer *timer) +{ + char name[] = "task_struct"; + + bpf_btf_find_by_name_kind(name, sizeof(name), BTF_KIND_STRUCT, 0); + return 0; +} + +SEC("syscall") +__failure __msg("sleepable helper bpf_btf_find_by_name_kind#{{[0-9]+}} in non-sleepable prog") +int timer_btf_find_prog(void *ctx) +{ + struct timer_elem *val; + int key = 0; + + val = bpf_map_lookup_elem(&timer_map, &key); + if (!val) + return 0; + + bpf_timer_init(&val->t, &timer_map, 0); + bpf_timer_set_callback(&val->t, timer_btf_find_cb); + return 0; +} + +SEC("syscall") +__success +int syscall_sys_bpf_prog(void *ctx) +{ + __u64 attr = 0; + + bpf_sys_bpf(BPF_MAP_FREEZE, &attr, sizeof(attr)); + return 0; +} + +SEC("syscall") +__success +int syscall_sys_close_prog(void *ctx) +{ + bpf_sys_close(0); + return 0; +} + +SEC("syscall") +__success +int syscall_btf_find_prog(void *ctx) +{ + char name[] = "task_struct"; + + bpf_btf_find_by_name_kind(name, sizeof(name), BTF_KIND_STRUCT, 0); + return 0; +} + /* Workqueue tests */ struct wq_elem { struct bpf_wq w; + struct task_struct __kptr *task; }; struct { @@ -119,6 +223,106 @@ int wq_sleepable_prog(void *ctx) return 0; } +__noinline int wq_global_acquire(void) +{ + struct task_struct *task, *acquired; + struct wq_elem *val; + int key = 0; + + val = bpf_map_lookup_elem(&wq_map, &key); + if (!val) + return 0; + + task = val->task; + if (!task) + return 0; + + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + return 0; +} + +static int wq_global_rcu_cb(void *map, int *key, void *value) +{ + wq_global_acquire(); + return 0; +} + +SEC("fentry/bpf_fentry_test1") +__failure __msg("R1 must be a rcu pointer") +int wq_global_rcu_prog(void *ctx) +{ + struct wq_elem *val; + int key = 0; + + val = bpf_map_lookup_elem(&wq_map, &key); + if (!val) + return 0; + + bpf_wq_init(&val->w, &wq_map, 0); + bpf_wq_set_callback(&val->w, wq_global_rcu_cb, 0); + return 0; +} + +static int wq_global_rcu_lock_cb(void *map, int *key, void *value) +{ + bpf_rcu_read_lock(); + wq_global_acquire(); + bpf_rcu_read_unlock(); + return 0; +} + +SEC("fentry/bpf_fentry_test1") +__success +int wq_global_rcu_lock_prog(void *ctx) +{ + struct wq_elem *val; + int key = 0; + + /* Verify the same global subprog in non-sleepable and protected contexts. */ + wq_global_acquire(); + + val = bpf_map_lookup_elem(&wq_map, &key); + if (!val) + return 0; + + bpf_wq_init(&val->w, &wq_map, 0); + bpf_wq_set_callback(&val->w, wq_global_rcu_lock_cb, 0); + return 0; +} + +__weak __noinline int wq_global_no_rcu(void) +{ + return 0; +} + +static int wq_global_no_rcu_cb(void *map, int *key, void *value) +{ + wq_global_no_rcu(); + return 0; +} + +SEC("fentry/bpf_fentry_test1") +__success __log_level(4) +__msg("subprog {{[0-9]+}} (wq_global_no_rcu) global insns_self 4 insns_total 4 stack 0") +int wq_global_no_rcu_prog(void *ctx) +{ + struct wq_elem *val; + int key = 0; + + /* Verify the same global in non-sleepable and unprotected contexts. */ + wq_global_no_rcu(); + + val = bpf_map_lookup_elem(&wq_map, &key); + if (!val) + return 0; + + bpf_wq_init(&val->w, &wq_map, 0); + bpf_wq_set_callback(&val->w, wq_global_no_rcu_cb, 0); + return 0; +} + /* Task work tests */ struct task_work_elem { diff --git a/tools/testing/selftests/bpf/progs/verifier_basic_stack.c b/tools/testing/selftests/bpf/progs/verifier_basic_stack.c index fb62e09f2114..0eb495ce85c1 100644 --- a/tools/testing/selftests/bpf/progs/verifier_basic_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_basic_stack.c @@ -27,7 +27,8 @@ __naked void stack_out_of_bounds(void) SEC("socket") __description("uninitialized stack1") -__success __log_level(4) __msg("stack depth 8") +__success __log_level(4) +__msg("subprog 0 (uninitialized_stack1) main {{.*}} stack 8") __failure_unpriv __msg_unpriv("invalid read from stack") __naked void uninitialized_stack1(void) { @@ -45,7 +46,8 @@ __naked void uninitialized_stack1(void) SEC("socket") __description("uninitialized stack2") -__success __log_level(4) __msg("stack depth 8") +__success __log_level(4) +__msg("subprog 0 (uninitialized_stack2) main insns_self {{[0-9]+}} insns_total {{[0-9]+}} stack 8") __failure_unpriv __msg_unpriv("invalid read from stack") __naked void uninitialized_stack2(void) { @@ -97,4 +99,45 @@ __naked void misaligned_read_from_stack(void) " ::: __clobber_all); } +SEC("socket") +__description("stack pointer arithmetic preserves frame number") +__failure __msg("R7 invalid mem access 'scalar'") +__naked void stack_ptr_arith_preserves_frameno(void) +{ + asm volatile ("\ + r3 = 0; \ + *(u64 *)(r10 - 8) = r3; \ + r1 = %[map_hash_8b] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + if r0 != 0 goto +2; \ + r0 = 0; \ + exit; \ + r1 = r0; \ + r2 = 0; \ + r3 = 0; \ + call stack_ptr_arith_preserves_frameno_subprog;\ + r0 = 0; \ + exit; \ + ": + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b) + : __clobber_all); +} + +static __used __naked void stack_ptr_arith_preserves_frameno_subprog(void) +{ + asm volatile ("\ + *(u64 *)(r10 - 8) = r1; \ + r6 = -8; \ + r6 += r10; \ + *(u64 *)(r6 + 0) = r2; \ + r7 = *(u64 *)(r10 - 8); \ + *(u64 *)(r7 + 0) = r3; \ + r0 = 0; \ + exit; \ + "::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_bits_iter.c b/tools/testing/selftests/bpf/progs/verifier_bits_iter.c index 8bcddadfc4da..dd97f2027505 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bits_iter.c +++ b/tools/testing/selftests/bpf/progs/verifier_bits_iter.c @@ -32,7 +32,7 @@ int BPF_PROG(no_destroy, struct bpf_iter_meta *meta, struct cgroup *cgrp) SEC("iter/cgroup") __description("uninitialized iter in ->next()") -__failure __msg("expected an initialized iter_bits as arg #0") +__failure __msg("expected an initialized iter_bits as R1") int BPF_PROG(next_uninit, struct bpf_iter_meta *meta, struct cgroup *cgrp) { struct bpf_iter_bits it = {}; @@ -43,7 +43,7 @@ int BPF_PROG(next_uninit, struct bpf_iter_meta *meta, struct cgroup *cgrp) SEC("iter/cgroup") __description("uninitialized iter in ->destroy()") -__failure __msg("expected an initialized iter_bits as arg #0") +__failure __msg("expected an initialized iter_bits as R1") int BPF_PROG(destroy_uninit, struct bpf_iter_meta *meta, struct cgroup *cgrp) { struct bpf_iter_bits it = {}; diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c index 79a328276805..df8d5309657e 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c @@ -202,7 +202,7 @@ l0_%=: /* exit */ \ SEC("tc") __description("bounds check based on reg_off + var_off + insn_off. test1") -__failure __msg("value_size=8 off=1073741825") +__failure __msg("map_value pointer offset 1073741822 is not allowed") __naked void var_off_insn_off_test1(void) { asm volatile (" \ @@ -1066,7 +1066,6 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("bound check with JMP_JSLT for crossing 64-bit signed boundary") __success __retval(0) -__flag(BPF_F_TEST_REG_INVARIANTS) __naked void crossing_64_bit_signed_boundary_2(void) { asm volatile (" \ @@ -1148,7 +1147,6 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("bound check with JMP32_JSLT for crossing 32-bit signed boundary") __success __retval(0) -__flag(BPF_F_TEST_REG_INVARIANTS) __naked void crossing_32_bit_signed_boundary_2(void) { asm volatile (" \ @@ -1197,7 +1195,7 @@ l0_%=: r1 = r6; \ r3 += -8; \ r5 = 0; \ /* The 4th argument of bpf_skb_store_bytes is defined as \ - * ARG_CONST_SIZE, so 0 is not allowed. The 'r4 != 0' \ + * ARG_MEM_SIZE, so 0 is not allowed. The 'r4 != 0' \ * is providing us this exclusion of zero from initial \ * [0, 7] range. \ */ \ @@ -1241,7 +1239,8 @@ l0_%=: r0 = 0; \ SEC("tc") __description("multiply mixed sign bounds. test 1") __success __log_level(2) -__msg("r6 *= r7 {{.*}}; R6=scalar(smin=umin=0x1bc16d5cd4927ee1,smax=umax=0x1bc16d674ec80000,smax32=0x7ffffeff,umax32=0xfffffeff,var_off=(0x1bc16d4000000000; 0x3ffffffeff))") +__msg("r6 *= r7 {{.*}}; R6=scalar(smin=umin=0x1bc16d5cd4927ee1,smax=umax=0x1bc16d674ec80000,smax32=0x7ffffeff,var_off=(0x1bc16d4000000000; 0x3ffffffeff))") +/* cnum can't represent both [0, 0xffff_feff] and [0x8000_0000, 0x7fff_feff], so it picks one */ __naked void mult_mixed0_sign(void) { asm volatile ( @@ -1536,7 +1535,7 @@ __naked void sub32_partial_overflow(void) SEC("socket") __description("dead branch on jset, does not result in invariants violation error") __success __log_level(2) -__retval(0) __flag(BPF_F_TEST_REG_INVARIANTS) +__retval(0) __naked void jset_range_analysis(void) { asm volatile (" \ @@ -1572,7 +1571,7 @@ l0_%=: r0 = 0; \ */ SEC("socket") __description("bounds deduction cross sign boundary, negative overlap") -__success __log_level(2) __flag(BPF_F_TEST_REG_INVARIANTS) +__success __log_level(2) __msg("7: (1f) r0 -= r6 {{.*}} R0=scalar(smin=smin32=-655,smax=smax32=-146,umin=0xfffffffffffffd71,umax=0xffffffffffffff6e,umin32=0xfffffd71,umax32=0xffffff6e,var_off=(0xfffffffffffffc00; 0x3ff))") __retval(0) __naked void bounds_deduct_negative_overlap(void) @@ -1616,7 +1615,7 @@ l0_%=: r0 = 0; \ */ SEC("socket") __description("bounds deduction cross sign boundary, positive overlap") -__success __log_level(2) __flag(BPF_F_TEST_REG_INVARIANTS) +__success __log_level(2) __msg("3: (2d) if r0 > r1 {{.*}} R0=scalar(smin=smin32=0,smax=umax=smax32=umax32=127,var_off=(0x0; 0x7f))") __retval(0) __naked void bounds_deduct_positive_overlap(void) @@ -1649,8 +1648,9 @@ l0_%=: r0 = 0; \ */ SEC("socket") __description("bounds deduction cross sign boundary, two overlaps") -__failure __flag(BPF_F_TEST_REG_INVARIANTS) -__msg("3: (2d) if r0 > r1 {{.*}} R0=scalar(smin=smin32=-128,smax=smax32=127,umax=0xffffffffffffff80)") +__failure +__msg("3: (2d) if r0 > r1 {{.*}} R0=scalar(smin=smin32=-128,smax=smax32=127)") +/* smin=-128 includes point 0xffffffffffffff80 */ __msg("frame pointer is read only") __naked void bounds_deduct_two_overlaps(void) { @@ -1713,7 +1713,7 @@ SEC("socket") __description("conditional jump on same register, branch taken") __not_msg("20: (b7) r0 = 1 {{.*}} R0=1") __success __log_level(2) -__retval(0) __flag(BPF_F_TEST_REG_INVARIANTS) +__retval(0) __naked void condition_jump_on_same_register(void *ctx) { asm volatile(" \ @@ -1748,7 +1748,7 @@ SEC("socket") __description("jset on same register, constant value branch taken") __not_msg("7: (b7) r0 = 1 {{.*}} R0=1") __success __log_level(2) -__retval(0) __flag(BPF_F_TEST_REG_INVARIANTS) +__retval(0) __naked void jset_on_same_register_1(void *ctx) { asm volatile(" \ @@ -1770,7 +1770,7 @@ SEC("socket") __description("jset on same register, scalar value branch taken") __not_msg("12: (b7) r0 = 1 {{.*}} R0=1") __success __log_level(2) -__retval(0) __flag(BPF_F_TEST_REG_INVARIANTS) +__retval(0) __naked void jset_on_same_register_2(void *ctx) { asm volatile(" \ @@ -1800,7 +1800,6 @@ __description("jset on same register, scalar value unknown branch 1") __msg("3: (b7) r0 = 0 {{.*}} R0=0") __msg("5: (b7) r0 = 1 {{.*}} R0=1") __success __log_level(2) -__flag(BPF_F_TEST_REG_INVARIANTS) __naked void jset_on_same_register_3(void *ctx) { asm volatile(" \ @@ -1822,7 +1821,6 @@ __description("jset on same register, scalar value unknown branch 2") __msg("4: (b7) r0 = 0 {{.*}} R0=0") __msg("6: (b7) r0 = 1 {{.*}} R0=1") __success __log_level(2) -__flag(BPF_F_TEST_REG_INVARIANTS) __naked void jset_on_same_register_4(void *ctx) { asm volatile(" \ @@ -1845,7 +1843,6 @@ __description("jset on same register, scalar value unknown branch 3") __msg("4: (b7) r0 = 0 {{.*}} R0=0") __msg("6: (b7) r0 = 1 {{.*}} R0=1") __success __log_level(2) -__flag(BPF_F_TEST_REG_INVARIANTS) __naked void jset_on_same_register_5(void *ctx) { asm volatile(" \ @@ -1877,7 +1874,6 @@ SEC("socket") __description("bounds refinement with single-value tnum on umax") __msg("3: (15) if r0 == 0xe0 {{.*}} R0=240") __success __log_level(2) -__flag(BPF_F_TEST_REG_INVARIANTS) __naked void bounds_refinement_tnum_umax(void *ctx) { asm volatile(" \ @@ -1896,26 +1892,25 @@ __naked void bounds_refinement_tnum_umax(void *ctx) /* This test covers the bounds deduction when the u64 range and the tnum * overlap only at umin. After instruction 3, the ranges look as follows: * - * 0 umin=0xe00 umax=0xeff U64_MAX + * 0 umin=0xe1 umax=0xf0 U64_MAX * | [xxxxxxxxxxxxxx] | * |----------------------------|------------------------------| * | x x | tnum values * - * The verifier can therefore deduce that the R0=0xe0=224. + * The verifier can therefore deduce that the R0=0xe1=225. */ SEC("socket") __description("bounds refinement with single-value tnum on umin") -__msg("3: (15) if r0 == 0xf0 {{.*}} R0=224") +__msg("3: (15) if r0 == 0xf1 {{.*}} R0=225") __success __log_level(2) -__flag(BPF_F_TEST_REG_INVARIANTS) __naked void bounds_refinement_tnum_umin(void *ctx) { asm volatile(" \ call %[bpf_get_prandom_u32]; \ - r0 |= 0xe0; \ - r0 &= 0xf0; \ - if r0 == 0xf0 goto +2; \ - if r0 == 0xe0 goto +1; \ + r0 |= 0xe1; \ + r0 &= 0xf1; \ + if r0 == 0xf1 goto +2; \ + if r0 == 0xe1 goto +1; \ r10 = 0; \ exit; \ " : @@ -2002,7 +1997,6 @@ __naked void bounds_refinement_multiple_overlaps(void *ctx) SEC("socket") __success -__flag(BPF_F_TEST_REG_INVARIANTS) __naked void signed_unsigned_intersection32_case1(void *ctx) { asm volatile(" \ @@ -2020,7 +2014,6 @@ __naked void signed_unsigned_intersection32_case1(void *ctx) SEC("socket") __success -__flag(BPF_F_TEST_REG_INVARIANTS) __naked void signed_unsigned_intersection32_case2(void *ctx) { asm volatile(" \ @@ -2037,6 +2030,41 @@ __naked void signed_unsigned_intersection32_case2(void *ctx) : __clobber_all); } +/* + * After instruction 3, the u64 and s64 ranges look as follows: + * 0 umin=2 umax=0xff..ff00..03 U64_MAX + * | [xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx] | + * |----------------------------|------------------------------| + * |xx] [xxxxxxxxxxxxxxxxxxxxxxxxxxxx| + * 0 smax=2 smin=0x800..02 -1 + * + * The two ranges can't be refined because they overlap in two places. Once we + * add an upper-bound to u64 at instruction 4, the refinement can happen. This + * test validates that this refinement does happen and is not overwritten by + * the less-precise 32bits ranges. + */ +SEC("socket") +__description("bounds refinement: 64bits ranges not overwritten by 32bits ranges") +__msg("3: (65) if r0 s> 0x2 {{.*}} R0=scalar(smin=0x8000000000000002,smax=2,smin32=umin32=2,smax32=umax32=3,var_off{{.*}}))") +/* Can't represent both [S64_MIN+2, 2] and [2, U64_MAX - U32_MAX + 2] at the same time, picks shorter interval */ +__msg("4: (25) if r0 > 0x13 {{.*}} R0=2") +__success __log_level(2) +__naked void refinement_32bounds_not_overwriting_64bounds(void *ctx) +{ + asm volatile(" \ + call %[bpf_get_prandom_u32]; \ + if w0 < 2 goto +5; \ + if w0 > 3 goto +4; \ + if r0 s> 2 goto +3; \ + if r0 > 19 goto +2; \ + if r0 == 2 goto +1; \ + r10 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + SEC("socket") __description("maybe_fork_scalars: OR with constant rejects OOB") __failure __msg("invalid access to map value") @@ -2131,4 +2159,180 @@ l0_%=: r0 = 0; \ : __clobber_all); } +/* + * Last jump can be detected as always taken because the intersection of R5 and + * R7 32bit tnums produces a constant that isn't within R7's s32 bounds. + */ +SEC("socket") +__description("dead branch: tnums give impossible constant if equal") +__success +__naked void tnums_equal_impossible_constant(void *ctx) +{ + asm volatile(" \ + call %[bpf_get_prandom_u32]; \ + r5 = r0; \ + /* Set r5's var_off32 to (0; 0xfffffffc) */ \ + r5 &= 0xfffffffffffffffc; \ + r7 = r0; \ + /* Set r7's var_off32 to (0x0; 0x1) */ \ + r7 &= 0x1; \ + /* Now, s32=[-43; -42], var_off32=(0xffffffd4; 0x3) */ \ + r7 += -43; \ + /* On fallthrough, var_off32=-44, not in s32 */ \ + if w5 != w7 goto +1; \ + r10 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * 32-bit range starts before 64-bit range low bits in each 2^32 block. + * + * N*2^32 (N+1)*2^32 (N+2)*2^32 (N+3)*2^32 + * ||----|=====|--|----------||----|=====|-------------||--|-|=====|-------------|| + * |< b >| | |< b >| | |< b >| + * | | | | + * |<---------------+- a -+---------------->| + * | | + * |< t >| refined r0 range + * + * a = u64 [0x1'00000008, 0x3'00000001] + * b = u32 [2, 5] + * t = u64 [0x2'00000002, 0x2'00000005] + */ +SEC("socket") +__success +__flag(BPF_F_TEST_REG_INVARIANTS) +__naked void deduce64_from_32_before_block_start(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = 0x100000008 ll; \ + if r0 < r1 goto 2f; \ + r1 = 0x300000001 ll; \ + if r0 > r1 goto 2f; /* u64: [0x1'00000008, 0x3'00000001] */ \ + if w0 < 2 goto 2f; \ + if w0 > 5 goto 2f; /* u32: [2, 5] */ \ + r2 = 0x200000002 ll; \ + r3 = 0x200000005 ll; \ + if r0 >= r2 goto 1f; /* should be always true */ \ + r10 = 0; /* dead code */ \ +1: if r0 <= r3 goto 2f; /* should be always true */ \ + r10 = 0; /* dead code */ \ +2: exit; \ + " + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * 32-bit range crossing U32_MAX / 0 boundary. + * + * N*2^32 (N+1)*2^32 (N+2)*2^32 (N+3)*2^32 + * ||===|---------|------|===||===|----------------|===||===|---------|------|===|| + * |b >| | |< b||b >| |< b||b >| | |< b| + * | | | | + * |<-----+----------------- a --------------+-------->| + * | | + * |<---------------- t ------------->| refined r0 range + * + * a = u64 [0x1'00000006, 0x2'FFFFFFEF] + * b = s32 [-16, 5] (u32 wrapping [0xFFFFFFF0, 0x00000005]) + * t = u64 [0x1'FFFFFFF0, 0x2'00000005] + */ +SEC("socket") +__success +__flag(BPF_F_TEST_REG_INVARIANTS) +__naked void deduce64_from_32_wrapping_32bit(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = 0x100000006 ll; \ + if r0 < r1 goto 2f; \ + r1 = 0x2ffffffef ll; \ + if r0 > r1 goto 2f; /* u64: [0x1'00000006, 0x2'FFFFFFEF] */ \ + if w0 s< -16 goto 2f; \ + if w0 s> 5 goto 2f; /* s32: [-16, 5] */ \ + r1 = 0x1fffffff0 ll; \ + r2 = 0x200000005 ll; \ + if r0 >= r1 goto 1f; /* should be always true */ \ + r10 = 0; /* dead code */ \ +1: if r0 <= r2 goto 2f; /* should be always true */ \ + r10 = 0; /* dead code */ \ +2: exit; \ + " + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Unprivileged variable pointer arithmetic on a PTR_TO_MAP_VALUE whose + * offset collapses to a constant. The Spectre-v1 speculative path snapshots + * the pointer while its r32 has just been blanked but its offset not yet + * synced; the following register move used to trip reg_bounds_sanity_check() + * ("const subreg tnum out of sync with range bounds"). With + * BPF_F_TEST_REG_INVARIANTS that violation turns into a load failure, so the + * unprivileged program must still load. + */ +SEC("socket") +__success __success_unpriv +__flag(BPF_F_TEST_REG_INVARIANTS) +__naked void spec_ptr_alu_const_offset(void) +{ + asm volatile (" \ + call %[bpf_ktime_get_ns]; \ + *(u64*)(r10 - 16) = r0; \ + r1 = 0; \ + *(u64*)(r10 - 8) = r1; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map_hash_8b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r1 = *(u64*)(r10 - 16); \ + r2 = 0x40000000; \ + if r1 > r2 goto l0_%=; \ + if r1 s> 1 goto l0_%=; /* r1 in [0, 1] */ \ + r0 += r1; /* ptr += bounded scalar */ \ + r9 = r0; /* used to trip the warning */ \ + *(u8*)(r0 + 0) = r1; \ +l0_%=: r0 = 0; \ + exit; \ + " + : + : __imm(bpf_ktime_get_ns), + __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b) + : __clobber_all); +} + +/* Check that range_within() compares cnum ranges, not min/max projections. */ +SEC("socket") +__failure __msg("div by zero") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void range_within_cnum_cross_both_boundaries(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = 0x80000020; \ + if r0 > r1 goto 1f; \ + r0 += 0x7FFFFFF0; /* PATH 1 */ \ + goto 2f; \ +1: call %[bpf_get_prandom_u32]; /* PATH 2 */ \ + if r0 < 0x100 goto 3f; \ + if r0 > 0x200 goto 3f; \ +2: /* PATH 1: r0 ∈ [0x7FFFFFF0, U32_MAX] ∪ [0, 0x10] */ \ + /* PATH 2: r0 ∈ [0x100, 0x200] */ \ + if r0 != 0x100 goto 3f; /* True only on PATH 2 */ \ + r0 /= 0; \ +3: exit; \ + " + :: __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b), + __imm(bpf_get_prandom_u32) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c index fb4fa465d67c..a73b837553fb 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +++ b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c @@ -10,7 +10,8 @@ SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 8") +__log_level(4) +__msg("subprog 0 (simple) main insns_self {{[0-9]+}} insns_total {{[0-9]+}} stack 8") __xlated("4: r5 = 5") __xlated("5: r0 = ") __xlated("6: r0 = &(void __percpu *)(r0)") @@ -96,7 +97,8 @@ __naked void canary_zero_spills(void) SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 16") +__log_level(4) +__msg("subprog 0 (wrong_reg_in_pattern1) main {{.*}} stack 16") __xlated("1: *(u64 *)(r10 -16) = r1") __xlated("...") __xlated("3: r0 = &(void __percpu *)(r0)") @@ -598,7 +600,8 @@ __naked static void subprogs_use_independent_offsets_aux(void) SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 8") +__log_level(4) +__msg("subprog 0 (helper_call_does_not_prevent_bpf_fastcall) main {{.*}} stack 8") __xlated("2: r0 = &(void __percpu *)(r0)") __success __naked void helper_call_does_not_prevent_bpf_fastcall(void) @@ -618,9 +621,120 @@ __naked void helper_call_does_not_prevent_bpf_fastcall(void) : __clobber_all); } +/* A jump to the first spill executes the whole pattern, rewrite is safe. */ +SEC("raw_tp") +__arch_x86_64 +__log_level(4) +__msg("subprog 0 (jump_to_first_spill) main {{.*}} stack 0") +__xlated("2: if r0 == 0x2a goto pc+0") +__xlated("3: r0 = ") +__xlated("4: r0 = &(void __percpu *)(r0)") +__success +__naked void jump_to_first_spill(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1;" + "if r0 == 42 goto l0_%=;" +"l0_%=:" + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_get_smp_processor_id) + : __clobber_all); +} + +/* A jump to the call skips the spill, the pattern must be kept. */ SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 24") +__log_level(4) +__msg("subprog 0 (jump_to_call) main {{.*}} stack 8") +__xlated("2: if r0 == 0x2a goto pc+1") +__xlated("3: *(u64 *)(r10 -8) = r1") +__xlated("...") +__xlated("7: r1 = *(u64 *)(r10 -8)") +__success +__naked void jump_to_call(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1;" + "if r0 == 42 goto l0_%=;" + "*(u64 *)(r10 - 8) = r1;" +"l0_%=:" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_get_smp_processor_id) + : __clobber_all); +} + +/* A jump to the fill skips the spill, the pattern must be kept. */ +SEC("raw_tp") +__arch_x86_64 +__log_level(4) +__msg("subprog 0 (jump_to_fill) main {{.*}} stack 8") +__xlated("2: if r0 == 0x2a goto pc+4") +__xlated("3: *(u64 *)(r10 -8) = r1") +__xlated("...") +__xlated("7: r1 = *(u64 *)(r10 -8)") +__success +__naked void jump_to_fill(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1;" + "if r0 == 42 goto l0_%=;" + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" +"l0_%=:" + "r1 = *(u64 *)(r10 - 8);" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_get_smp_processor_id) + : __clobber_all); +} + +/* Same as above, but the fill is entered by an unconditional jump. */ +SEC("raw_tp") +__arch_x86_64 +__log_level(4) +__msg("subprog 0 (unconditional_jump_to_fill) main {{.*}} stack 8") +__xlated("3: *(u64 *)(r10 -8) = r1") +__xlated("...") +__xlated("7: r1 = *(u64 *)(r10 -8)") +__xlated("8: exit") +__xlated("9: goto pc-3") +__success +__naked void unconditional_jump_to_fill(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1;" + "if r0 == 42 goto l1_%=;" + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" +"l0_%=:" + "r1 = *(u64 *)(r10 - 8);" + "exit;" +"l1_%=:" + "goto l0_%=;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_get_smp_processor_id) + : __clobber_all); +} + +SEC("raw_tp") +__arch_x86_64 +__log_level(4) +__msg("subprog 0 (may_goto_interaction_x86_64) main {{.*}} stack 24") /* may_goto counter at -24 */ __xlated("0: *(u64 *)(r10 -24) =") /* may_goto timestamp at -16 */ @@ -630,13 +744,13 @@ __xlated("...") __xlated("4: r0 = &(void __percpu *)(r0)") __xlated("...") /* may_goto expansion starts */ -__xlated("6: r11 = *(u64 *)(r10 -24)") -__xlated("7: if r11 == 0x0 goto pc+6") -__xlated("8: r11 -= 1") -__xlated("9: if r11 != 0x0 goto pc+2") -__xlated("10: r11 = -24") +__xlated("6: r12 = *(u64 *)(r10 -24)") +__xlated("7: if r12 == 0x0 goto pc+6") +__xlated("8: r12 -= 1") +__xlated("9: if r12 != 0x0 goto pc+2") +__xlated("10: r12 = -24") __xlated("11: call unknown") -__xlated("12: *(u64 *)(r10 -24) = r11") +__xlated("12: *(u64 *)(r10 -24) = r12") /* may_goto expansion ends */ __xlated("13: *(u64 *)(r10 -8) = r1") __xlated("14: exit") @@ -660,7 +774,10 @@ __naked void may_goto_interaction_x86_64(void) SEC("raw_tp") __arch_arm64 -__log_level(4) __msg("stack depth 24") +__arch_riscv64 +__arch_loongarch +__log_level(4) +__msg("subprog 0 (may_goto_interaction) main {{.*}} stack 24") /* may_goto counter at -24 */ __xlated("0: *(u64 *)(r10 -24) =") /* may_goto timestamp at -16 */ @@ -668,18 +785,18 @@ __xlated("1: *(u64 *)(r10 -16) =") __xlated("2: r1 = 1") __xlated("3: call bpf_get_smp_processor_id") /* may_goto expansion starts */ -__xlated("4: r11 = *(u64 *)(r10 -24)") -__xlated("5: if r11 == 0x0 goto pc+6") -__xlated("6: r11 -= 1") -__xlated("7: if r11 != 0x0 goto pc+2") -__xlated("8: r11 = -24") +__xlated("4: r12 = *(u64 *)(r10 -24)") +__xlated("5: if r12 == 0x0 goto pc+6") +__xlated("6: r12 -= 1") +__xlated("7: if r12 != 0x0 goto pc+2") +__xlated("8: r12 = -24") __xlated("9: call unknown") -__xlated("10: *(u64 *)(r10 -24) = r11") +__xlated("10: *(u64 *)(r10 -24) = r12") /* may_goto expansion ends */ __xlated("11: *(u64 *)(r10 -8) = r1") __xlated("12: exit") __success -__naked void may_goto_interaction_arm64(void) +__naked void may_goto_interaction(void) { asm volatile ( "r1 = 1;" @@ -707,7 +824,9 @@ __naked static void dummy_loop_callback(void) SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 32+0") +__log_level(4) +__msg("subprog 0 (bpf_loop_interaction1) main {{.*}} stack 32") +__msg("subprog 1 (dummy_loop_callback) static {{.*}} stack 0") __xlated("2: r1 = 1") __xlated("3: r0 =") __xlated("4: r0 = &(void __percpu *)(r0)") @@ -755,7 +874,9 @@ __naked int bpf_loop_interaction1(void) SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 40+0") +__log_level(4) +__msg("subprog 0 (bpf_loop_interaction2) main {{.*}} stack 40") +__msg("subprog 1 (dummy_loop_callback) static {{.*}} stack 0") /* call bpf_get_smp_processor_id */ __xlated("2: r1 = 42") __xlated("3: r0 =") @@ -800,7 +921,9 @@ __naked int bpf_loop_interaction2(void) SEC("raw_tp") __arch_x86_64 __log_level(4) -__msg("stack depth 512+0") +__msg("stack depth max 512") +__msg("subprog 0 (cumulative_stack_depth) main {{.*}} stack 512") +__msg("subprog 1 (cumulative_stack_depth_subprog) static {{.*}} stack 0") /* just to print xlated version when debugging */ __xlated("r0 = &(void __percpu *)(r0)") __success diff --git a/tools/testing/selftests/bpf/progs/verifier_btf_flex_array.c b/tools/testing/selftests/bpf/progs/verifier_btf_flex_array.c new file mode 100644 index 000000000000..59b84261f622 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_btf_flex_array.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +#include "bpf_experimental.h" +#include "bpf_misc.h" + +struct test_empty_event {}; + +struct test_flex_batch { + int nr; + struct test_empty_event events[]; +}; + +struct map_value { + struct test_flex_batch __kptr *batch; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct map_value); + __uint(max_entries, 1); +} batches SEC(".maps"); + +SEC("syscall") +__description("btf walk into flexible array of zero-sized elements") +__failure __msg("access beyond struct test_flex_batch at off 4 size 1") +int stash_and_peek(void *ctx) +{ + struct test_flex_batch *b, *old; + struct map_value *v; + int key = 0; + + v = bpf_map_lookup_elem(&batches, &key); + if (!v) + return 0; + + b = bpf_obj_new(struct test_flex_batch); + if (!b) + return 0; + b->nr = 1; + + old = bpf_kptr_xchg(&v->batch, b); + if (old) + bpf_obj_drop(old); + + b = v->batch; + if (!b) + return 0; + + return b->nr + *(char *)&b->events[0]; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_call_large_imm.c b/tools/testing/selftests/bpf/progs/verifier_call_large_imm.c new file mode 100644 index 000000000000..7998df07f6a6 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_call_large_imm.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +int call_happened = 0; + +/* + * 32765 is the exact minimum number of padding instructions needed to + * trigger the verifier failure, because: + * 1. Counting the wrapper instructions around the padding block (one + * "r0=0" and two "exit" instructions), the actual jump distance + * evaluates to N + 3. + * 2. To overflow the s16 max bound (32767), we need N + 3 > 32767. + * Thus, N = 32765 is the exact minimum padding size required. + */ +static __attribute__((noinline)) void padding_subprog(void) +{ + asm volatile ( + "r0 = 0;" + ".rept 32765;" + "r0 += 0;" + ".endr;" + ::: __clobber_all); +} + +static __attribute__((noinline)) int target_subprog(void) +{ + /* Use volatile variable here to prevent optimization. */ + volatile int magic_ret = 3; + return magic_ret; +} + +SEC("syscall") +__success __retval(3) +int call_large_imm_test(void *ctx) +{ + /* + * Landing pad to handle call error on kernel without the fix, + * preventing kernel panic. + */ + asm volatile ( + "r0 = 0;" + ".rept 32768;" + "r0 += 0;" + ".endr;" + ::: __clobber_all); + + /* + * The call_happened variable is 1 only when the call insn wrongly + * go back to the landing pad above. + */ + if (call_happened == 1) { + /* Use volatile variable here to prevent optimization. */ + volatile int flag = -1; + return flag; + } + + call_happened = 1; + + padding_subprog(); + + return target_subprog(); +} + +char LICENSE[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_cfg.c b/tools/testing/selftests/bpf/progs/verifier_cfg.c index c1f55e1d80a4..3c3bb03e8217 100644 --- a/tools/testing/selftests/bpf/progs/verifier_cfg.c +++ b/tools/testing/selftests/bpf/progs/verifier_cfg.c @@ -3,6 +3,7 @@ #include <linux/bpf.h> #include <bpf/bpf_helpers.h> +#include "../../../include/linux/filter.h" #include "bpf_misc.h" SEC("socket") @@ -56,6 +57,19 @@ __naked void out_of_range_jump2(void) } SEC("socket") +__description("invalid DW LDSX instruction in diagnostics") +__failure __msg("BUG_ldx_99") +__log_level(2) +__naked void invalid_dw_ldsx(void) +{ + asm volatile (" \ + .8byte %[ldsx_dw]; \ +" : + : __imm_insn(ldsx_dw, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | BPF_DW, BPF_REG_0, BPF_REG_0, 0, 0)) + : __clobber_all); +} + +SEC("socket") __description("loop (back-edge)") __failure __msg("unreachable insn 1") __msg_unpriv("back-edge") diff --git a/tools/testing/selftests/bpf/progs/verifier_cgroup_storage.c b/tools/testing/selftests/bpf/progs/verifier_cgroup_storage.c index 9a13f5c11ac7..884080a5bffc 100644 --- a/tools/testing/selftests/bpf/progs/verifier_cgroup_storage.c +++ b/tools/testing/selftests/bpf/progs/verifier_cgroup_storage.c @@ -305,4 +305,33 @@ __naked void cpu_cgroup_storage_access_6(void) : __clobber_all); } +/* + * Verification takes two paths: with r2 being scalar zero on path (1) + * and with r2 being some other scalar on path (2). + * Check that the verifier does not use checkpoints created + * on path (1) to prune path (2). + */ +SEC("cgroup/skb") +__failure +__flag(BPF_F_TEST_STATE_FREQ) +__msg("get_local_storage() doesn't support non-zero flags") +__naked void non_zero_flags_on_a_pruned_path(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + /* r2 is 0 on the path explored first, 1 on the other */\ + r2 = 1; \ + if r0 == 0 goto 1f; \ + r2 = 0; \ +1: r1 = %[cgroup_storage] ll; \ + call %[bpf_get_local_storage]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32), + __imm(bpf_get_local_storage), + __imm_addr(cgroup_storage) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_ctx.c b/tools/testing/selftests/bpf/progs/verifier_ctx.c index 5ebf7d9bcc55..7856dad3d1f3 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ctx.c +++ b/tools/testing/selftests/bpf/progs/verifier_ctx.c @@ -4,6 +4,10 @@ #include "vmlinux.h" #include <bpf/bpf_helpers.h> #include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +static const char ctx_strncmp_target[] = "ctx"; +static const char ctx_snprintf_fmt[] = ""; SEC("tc") __description("context stores via BPF_ATOMIC") @@ -69,7 +73,6 @@ __naked void ctx_pointer_to_helper_1(void) SEC("socket") __description("pass modified ctx pointer to helper, 2") __failure __msg("negative offset ctx ptr R1 off=-612 disallowed") -__failure_unpriv __msg_unpriv("negative offset ctx ptr R1 off=-612 disallowed") __naked void ctx_pointer_to_helper_2(void) { asm volatile (" \ @@ -292,4 +295,568 @@ padding_access("cgroup/post_bind4", bpf_sock, dst_port, 2); __failure __msg("invalid bpf_context access") padding_access("sk_reuseport", sk_reuseport_md, hash, 4); +SEC("?syscall") +__description("syscall: write to ctx with fixed offset") +__success +int syscall_ctx_fixed_off_write(void *ctx) +{ + char *p = ctx; + + *(__u32 *)p = 0; + *(__u32 *)(p + 4) = 0; + return 0; +} + +SEC("?syscall") +__description("syscall: read ctx with fixed offset") +__success +int syscall_ctx_fixed_off_read(void *ctx) +{ + char *p = ctx; + volatile __u32 val; + + val = *(__u32 *)(p + 4); + (void)val; + return 0; +} + +SEC("?syscall") +__description("syscall: unaligned read ctx with fixed offset") +__success +int syscall_ctx_unaligned_fixed_off_read(void *ctx) +{ + char *p = ctx; + volatile __u32 val; + + val = *(__u32 *)(p + 2); + (void)val; + return 0; +} + +SEC("?syscall") +__description("syscall: unaligned write ctx with fixed offset") +__success +int syscall_ctx_unaligned_fixed_off_write(void *ctx) +{ + char *p = ctx; + + *(__u32 *)(p + 2) = 0; + return 0; +} + +SEC("?syscall") +__description("syscall: read ctx with variable offset") +__success +int syscall_ctx_var_off_read(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + volatile __u32 val; + + off &= 0xfc; + p += off; + val = *(__u32 *)p; + (void)val; + return 0; +} + +SEC("?syscall") +__description("syscall: write ctx with variable offset") +__success +int syscall_ctx_var_off_write(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + + off &= 0xfc; + p += off; + *(__u32 *)p = 0; + return 0; +} + +SEC("?syscall") +__description("syscall: unaligned read ctx with variable offset") +__success +int syscall_ctx_unaligned_var_off_read(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + volatile __u32 val; + + off &= 0xfc; + off += 2; + p += off; + val = *(__u32 *)p; + (void)val; + return 0; +} + +SEC("?syscall") +__description("syscall: unaligned write ctx with variable offset") +__success +int syscall_ctx_unaligned_var_off_write(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + + off &= 0xfc; + off += 2; + p += off; + *(__u32 *)p = 0; + return 0; +} + +SEC("?syscall") +__description("syscall: reject ctx access past U16_MAX with fixed offset") +__failure __msg("outside of the allowed memory range") +int syscall_ctx_u16_max_fixed_off(void *ctx) +{ + char *p = ctx; + volatile __u32 val; + + p += 65535; + val = *(__u32 *)p; + (void)val; + return 0; +} + +SEC("?syscall") +__description("syscall: reject ctx access past U16_MAX with variable offset") +__failure __msg("outside of the allowed memory range") +int syscall_ctx_u16_max_var_off(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + volatile __u32 val; + + off &= 0xffff; + off += 1; + p += off; + val = *(__u32 *)p; + (void)val; + return 0; +} + +SEC("?syscall") +__description("syscall: reject negative variable offset ctx access") +__failure __msg("min value is negative") +int syscall_ctx_neg_var_off(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + + off &= 4; + p -= off; + return *(__u32 *)p; +} + +SEC("?syscall") +__description("syscall: reject unbounded variable offset ctx access") +__failure __msg("unbounded memory access") +int syscall_ctx_unbounded_var_off(void *ctx) +{ + __u64 off = (__u32)bpf_get_prandom_u32(); + char *p = ctx; + + off <<= 2; + p += off; + return *(__u32 *)p; +} + +SEC("?syscall") +__description("syscall: helper read ctx with fixed offset") +__success +int syscall_ctx_helper_fixed_off_read(void *ctx) +{ + char *p = ctx; + + p += 4; + return bpf_strncmp(p, 4, ctx_strncmp_target); +} + +SEC("?syscall") +__description("syscall: helper write ctx with fixed offset") +__success +int syscall_ctx_helper_fixed_off_write(void *ctx) +{ + char *p = ctx; + + p += 4; + return bpf_probe_read_kernel(p, 4, 0); +} + +SEC("?syscall") +__description("syscall: helper unaligned read ctx with fixed offset") +__success +int syscall_ctx_helper_unaligned_fixed_off_read(void *ctx) +{ + char *p = ctx; + + p += 2; + return bpf_strncmp(p, 4, ctx_strncmp_target); +} + +SEC("?syscall") +__description("syscall: helper unaligned write ctx with fixed offset") +__success +int syscall_ctx_helper_unaligned_fixed_off_write(void *ctx) +{ + char *p = ctx; + + p += 2; + return bpf_probe_read_kernel(p, 4, 0); +} + +SEC("?syscall") +__description("syscall: helper read ctx with variable offset") +__success +int syscall_ctx_helper_var_off_read(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + + off &= 0xfc; + p += off; + return bpf_strncmp(p, 4, ctx_strncmp_target); +} + +SEC("?syscall") +__description("syscall: helper write ctx with variable offset") +__success +int syscall_ctx_helper_var_off_write(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + + off &= 0xfc; + p += off; + return bpf_probe_read_kernel(p, 4, 0); +} + +SEC("?syscall") +__description("syscall: helper unaligned read ctx with variable offset") +__success +int syscall_ctx_helper_unaligned_var_off_read(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + + off &= 0xfc; + off += 2; + p += off; + return bpf_strncmp(p, 4, ctx_strncmp_target); +} + +SEC("?syscall") +__description("syscall: helper unaligned write ctx with variable offset") +__success +int syscall_ctx_helper_unaligned_var_off_write(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + + off &= 0xfc; + off += 2; + p += off; + return bpf_probe_read_kernel(p, 4, 0); +} + +SEC("?syscall") +__description("syscall: reject helper read ctx past U16_MAX with fixed offset") +__failure __msg("outside of the allowed memory range") +int syscall_ctx_helper_u16_max_fixed_off_read(void *ctx) +{ + char *p = ctx; + + p += 65535; + return bpf_strncmp(p, 4, ctx_strncmp_target); +} + +SEC("?syscall") +__description("syscall: reject helper write ctx past U16_MAX with fixed offset") +__failure __msg("outside of the allowed memory range") +int syscall_ctx_helper_u16_max_fixed_off_write(void *ctx) +{ + char *p = ctx; + + p += 65535; + return bpf_probe_read_kernel(p, 4, 0); +} + +SEC("?syscall") +__description("syscall: reject helper read ctx past U16_MAX with variable offset") +__failure __msg("outside of the allowed memory range") +int syscall_ctx_helper_u16_max_var_off_read(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + + off &= 0xffff; + off += 1; + p += off; + return bpf_strncmp(p, 4, ctx_strncmp_target); +} + +SEC("?syscall") +__description("syscall: reject helper write ctx past U16_MAX with variable offset") +__failure __msg("outside of the allowed memory range") +int syscall_ctx_helper_u16_max_var_off_write(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + + off &= 0xffff; + off += 1; + p += off; + return bpf_probe_read_kernel(p, 4, 0); +} + +SEC("?syscall") +__description("syscall: helper read zero-sized ctx access") +__success +int syscall_ctx_helper_zero_sized_read(void *ctx) +{ + return bpf_snprintf(0, 0, ctx_snprintf_fmt, ctx, 0); +} + +SEC("?syscall") +__description("syscall: helper write zero-sized ctx access") +__success +int syscall_ctx_helper_zero_sized_write(void *ctx) +{ + return bpf_probe_read_kernel(ctx, 0, 0); +} + +SEC("?syscall") +__description("syscall: kfunc access ctx with fixed offset") +__success +int syscall_ctx_kfunc_fixed_off(void *ctx) +{ + char *p = ctx; + + p += 4; + bpf_kfunc_call_test_mem_len_pass1(p, 4); + return 0; +} + +SEC("?syscall") +__description("syscall: kfunc access ctx with variable offset") +__success +int syscall_ctx_kfunc_var_off(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + + off &= 0xfc; + p += off; + bpf_kfunc_call_test_mem_len_pass1(p, 4); + return 0; +} + +SEC("?syscall") +__description("syscall: kfunc unaligned access ctx with fixed offset") +__success +int syscall_ctx_kfunc_unaligned_fixed_off(void *ctx) +{ + char *p = ctx; + + p += 2; + bpf_kfunc_call_test_mem_len_pass1(p, 4); + return 0; +} + +SEC("?syscall") +__description("syscall: kfunc unaligned access ctx with variable offset") +__success +int syscall_ctx_kfunc_unaligned_var_off(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + + off &= 0xfc; + off += 2; + p += off; + bpf_kfunc_call_test_mem_len_pass1(p, 4); + return 0; +} + +SEC("?syscall") +__description("syscall: reject kfunc ctx access past U16_MAX with fixed offset") +__failure __msg("outside of the allowed memory range") +int syscall_ctx_kfunc_u16_max_fixed_off(void *ctx) +{ + char *p = ctx; + + p += 65535; + bpf_kfunc_call_test_mem_len_pass1(p, 4); + return 0; +} + +SEC("?syscall") +__description("syscall: reject kfunc ctx access past U16_MAX with variable offset") +__failure __msg("outside of the allowed memory range") +int syscall_ctx_kfunc_u16_max_var_off(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + + off &= 0xffff; + off += 1; + p += off; + bpf_kfunc_call_test_mem_len_pass1(p, 4); + return 0; +} + +SEC("?syscall") +__description("syscall: kfunc access zero-sized ctx") +__success +int syscall_ctx_kfunc_zero_sized(void *ctx) +{ + bpf_kfunc_call_test_mem_len_pass1(ctx, 0); + return 0; +} + +/* + * For non-syscall program types without convert_ctx_access, direct ctx + * dereference is still allowed after adding a fixed offset, while variable + * and negative direct accesses reject. + * + * Passing ctx as a helper or kfunc memory argument is only permitted for + * syscall programs, so the helper and kfunc cases below validate rejection + * for non-syscall ctx pointers at fixed, variable, and zero-sized accesses. + */ +#define no_rewrite_ctx_access(type, name, off, load_t) \ + SEC("?" type) \ + __description(type ": read ctx at fixed offset") \ + __success \ + int no_rewrite_##name##_fixed(void *ctx) \ + { \ + char *p = ctx; \ + volatile load_t val; \ + \ + val = *(load_t *)(p + off); \ + (void)val; \ + return 0; \ + } \ + SEC("?" type) \ + __description(type ": reject variable offset ctx access") \ + __failure __msg("variable ctx access var_off=") \ + int no_rewrite_##name##_var(void *ctx) \ + { \ + __u64 off_var = bpf_get_prandom_u32(); \ + char *p = ctx; \ + \ + off_var &= 4; \ + p += off_var; \ + return *(load_t *)p; \ + } \ + SEC("?" type) \ + __description(type ": reject negative offset ctx access") \ + __failure __msg("invalid bpf_context access") \ + int no_rewrite_##name##_neg(void *ctx) \ + { \ + char *p = ctx; \ + \ + p -= 612; \ + return *(load_t *)p; \ + } \ + SEC("?" type) \ + __description(type ": reject helper read ctx at fixed offset") \ + __failure __msg("dereference of modified ctx ptr") \ + int no_rewrite_##name##_helper_read_fixed(void *ctx) \ + { \ + char *p = ctx; \ + \ + p += off; \ + return bpf_strncmp(p, 4, ctx_strncmp_target); \ + } \ + SEC("?" type) \ + __description(type ": reject helper write ctx at fixed offset") \ + __failure __msg("dereference of modified ctx ptr") \ + int no_rewrite_##name##_helper_write_fixed(void *ctx) \ + { \ + char *p = ctx; \ + \ + p += off; \ + return bpf_probe_read_kernel(p, 4, 0); \ + } \ + SEC("?" type) \ + __description(type ": reject helper read ctx with variable offset") \ + __failure __msg("variable ctx access var_off=") \ + int no_rewrite_##name##_helper_read_var(void *ctx) \ + { \ + __u64 off_var = bpf_get_prandom_u32(); \ + char *p = ctx; \ + \ + off_var &= 4; \ + p += off_var; \ + return bpf_strncmp(p, 4, ctx_strncmp_target); \ + } \ + SEC("?" type) \ + __description(type ": reject helper write ctx with variable offset") \ + __failure __msg("variable ctx access var_off=") \ + int no_rewrite_##name##_helper_write_var(void *ctx) \ + { \ + __u64 off_var = bpf_get_prandom_u32(); \ + char *p = ctx; \ + \ + off_var &= 4; \ + p += off_var; \ + return bpf_probe_read_kernel(p, 4, 0); \ + } \ + SEC("?" type) \ + __description(type ": reject helper read zero-sized ctx access") \ + __failure __msg("R4 type=ctx expected=fp") \ + int no_rewrite_##name##_helper_read_zero(void *ctx) \ + { \ + return bpf_snprintf(0, 0, ctx_snprintf_fmt, ctx, 0); \ + } \ + SEC("?" type) \ + __description(type ": reject helper write zero-sized ctx access") \ + __failure __msg("R1 type=ctx expected=fp") \ + int no_rewrite_##name##_helper_write_zero(void *ctx) \ + { \ + return bpf_probe_read_kernel(ctx, 0, 0); \ + } \ + SEC("?" type) \ + __description(type ": reject kfunc ctx at fixed offset") \ + __failure __msg("dereference of modified ctx ptr") \ + int no_rewrite_##name##_kfunc_fixed(void *ctx) \ + { \ + char *p = ctx; \ + \ + p += off; \ + bpf_kfunc_call_test_mem_len_pass1(p, 4); \ + return 0; \ + } \ + SEC("?" type) \ + __description(type ": reject kfunc ctx with variable offset") \ + __failure __msg("variable ctx access var_off=") \ + int no_rewrite_##name##_kfunc_var(void *ctx) \ + { \ + __u64 off_var = bpf_get_prandom_u32(); \ + char *p = ctx; \ + \ + off_var &= 4; \ + p += off_var; \ + bpf_kfunc_call_test_mem_len_pass1(p, 4); \ + return 0; \ + } \ + SEC("?" type) \ + __description(type ": reject kfunc zero-sized ctx access") \ + __failure __msg("R1 type=ctx expected=fp") \ + int no_rewrite_##name##_kfunc_zero(void *ctx) \ + { \ + bpf_kfunc_call_test_mem_len_pass1(ctx, 0); \ + return 0; \ + } + +no_rewrite_ctx_access("kprobe", kprobe, 8, u64); +no_rewrite_ctx_access("tracepoint", tp, 8, u64); +no_rewrite_ctx_access("raw_tp", raw_tp, 8, u64); +no_rewrite_ctx_access("raw_tracepoint.w", raw_tp_w, 8, u64); +no_rewrite_ctx_access("fentry/bpf_modify_return_test", fentry, 8, u64); +no_rewrite_ctx_access("cgroup/dev", cgroup_dev, 4, u32); +no_rewrite_ctx_access("netfilter", netfilter, offsetof(struct bpf_nf_ctx, skb), u64); + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_ctx_ptr_param.c b/tools/testing/selftests/bpf/progs/verifier_ctx_ptr_param.c new file mode 100644 index 000000000000..d5cc8fc01fe6 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_ctx_ptr_param.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Verifier tests for single- and multi-level pointer parameter handling + * Copyright (c) 2026 CrowdStrike, Inc. + */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" + +SEC("fentry/bpf_fentry_test_ppvoid") +__description("fentry/void**: void ** inferred as scalar") +__success __retval(0) +__log_level(2) +__msg("R1=ctx() R2=scalar()") +__naked void fentry_ppvoid_as_scalar(void) +{ + asm volatile (" \ + r2 = *(u64 *)(r1 + 0); \ + r0 = 0; \ + exit; \ + " ::: __clobber_all); +} + +SEC("fentry/bpf_fentry_test_pppvoid") +__description("fentry/void***: void *** inferred as scalar") +__success __retval(0) +__log_level(2) +__msg("R1=ctx() R2=scalar()") +__naked void fentry_pppvoid_as_scalar(void) +{ + asm volatile (" \ + r2 = *(u64 *)(r1 + 0); \ + r0 = 0; \ + exit; \ + " ::: __clobber_all); +} + +SEC("fentry/bpf_fentry_test_ppfile") +__description("fentry/struct file**: struct file ** inferred as scalar") +__success __retval(0) +__log_level(2) +__msg("R1=ctx() R2=scalar()") +__naked void fentry_ppfile_as_scalar(void) +{ + asm volatile (" \ + r2 = *(u64 *)(r1 + 0); \ + r0 = 0; \ + exit; \ + " ::: __clobber_all); +} + +SEC("fexit/bpf_fexit_test_ret_ppfile") +__description("fexit/return struct file**: returned struct file ** inferred as scalar") +__success __retval(0) +__log_level(2) +__msg("R1=ctx() R2=scalar()") +__naked void fexit_ppfile_as_scalar(void) +{ + asm volatile (" \ + r2 = *(u64 *)(r1 + 0); \ + r0 = 0; \ + exit; \ + " ::: __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c index 911caa8fd1b7..915a9707298b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c @@ -412,7 +412,7 @@ l0_%=: r0 = 0; \ SEC("tc") __description("direct packet access: test17 (pruning, alignment)") -__failure __msg("misaligned packet access off 2+0+15+-4 size 4") +__failure __msg("misaligned packet access off 2+15+-4 size 4") __flag(BPF_F_STRICT_ALIGNMENT) __naked void packet_access_test17_pruning_alignment(void) { @@ -569,7 +569,7 @@ l0_%=: r0 = 0; \ SEC("tc") __description("direct packet access: test23 (x += pkt_ptr, 4)") -__failure __msg("invalid access to packet, off=0 size=8, R5(id=3,off=0,r=0)") +__failure __msg("invalid access to packet, off=31 size=8, R5(id=3,off=31,r=0)") __flag(BPF_F_ANY_ALIGNMENT) __naked void test23_x_pkt_ptr_4(void) { @@ -859,4 +859,65 @@ l0_%=: r0 = 1; \ : __clobber_all); } +SEC("tc") +__description("direct packet access: pkt_range cleared after sub with known scalar") +__failure __msg("invalid access to packet") +__naked void pkt_range_clear_after_sub(void) +{ + asm volatile (" \ + r9 = *(u32*)(r1 + %[__sk_buff_data]); \ + r8 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r9 += 256; \ + if r9 >= r8 goto l0_%=; \ + r0 = 0; \ + exit; \ +l0_%=: /* r9 has AT_PKT_END (pkt + 256 >= pkt_end) */ \ + r9 -= 256; \ + /* \ + * AT_PKT_END must not survive the arithmetic. \ + * is_pkt_ptr_branch_taken must validate both \ + * branches when visiting the next condition. \ + */ \ + if r9 < r8 goto l1_%=; \ + r0 = 0; \ + exit; \ +l1_%=: r0 = *(u8*)(r9 + 0); \ + r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + +SEC("tc") +__description("direct packet access: pkt_range cleared after add with known scalar") +__failure __msg("invalid access to packet") +__naked void pkt_range_clear_after_add(void) +{ + asm volatile (" \ + r9 = *(u32*)(r1 + %[__sk_buff_data]); \ + r8 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r9 += 256; \ + if r9 >= r8 goto l0_%=; \ + r0 = 0; \ + exit; \ +l0_%=: /* r9 has AT_PKT_END (pkt + 256 >= pkt_end) */ \ + r9 += -256; \ + /* \ + * Same as sub, but goes through BPF_ADD path. \ + * AT_PKT_END must not survive the arithmetic. \ + */ \ + if r9 < r8 goto l1_%=; \ + r0 = 0; \ + exit; \ +l1_%=: r0 = *(u8*)(r9 + 0); \ + r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_div_mod_bounds.c b/tools/testing/selftests/bpf/progs/verifier_div_mod_bounds.c index 4672af0b3268..e814a054d69a 100644 --- a/tools/testing/selftests/bpf/progs/verifier_div_mod_bounds.c +++ b/tools/testing/selftests/bpf/progs/verifier_div_mod_bounds.c @@ -36,7 +36,7 @@ l0_%=: r0 = *(u64 *)(r1 + 0); \ SEC("socket") __description("UDIV32, zero divisor") __success __retval(0) __log_level(2) -__msg("w1 /= w2 {{.*}}; R1=0 R2=0") +__msg("w1 /= w2 {{.*}}; R1=0") __naked void udiv32_zero_divisor(void) { asm volatile (" \ @@ -81,7 +81,7 @@ l0_%=: r0 = *(u64 *)(r1 + 0); \ SEC("socket") __description("UDIV64, zero divisor") __success __retval(0) __log_level(2) -__msg("r1 /= r2 {{.*}}; R1=0 R2=0") +__msg("r1 /= r2 {{.*}}; R1=0") __naked void udiv64_zero_divisor(void) { asm volatile (" \ @@ -242,7 +242,7 @@ l1_%=: r0 = *(u64 *)(r1 + 0); \ SEC("socket") __description("SDIV32, zero divisor") __success __retval(0) __log_level(2) -__msg("w1 s/= w2 {{.*}}; R1=0 R2=0") +__msg("w1 s/= w2 {{.*}}; R1=0") __naked void sdiv32_zero_divisor(void) { asm volatile (" \ @@ -275,6 +275,7 @@ __naked void sdiv32_overflow_1(void) w2 += 10; \ if w1 s> w2 goto l0_%=; \ w1 s/= -1; \ + r2 = r1; \ l0_%=: r0 = 0; \ exit; \ " : @@ -443,7 +444,7 @@ l1_%=: r0 = *(u64 *)(r1 + 0); \ SEC("socket") __description("SDIV64, zero divisor") __success __retval(0) __log_level(2) -__msg("r1 s/= r2 {{.*}}; R1=0 R2=0") +__msg("r1 s/= r2 {{.*}}; R1=0") __naked void sdiv64_zero_divisor(void) { asm volatile (" \ @@ -476,6 +477,7 @@ __naked void sdiv64_overflow_1(void) r2 += 10; \ if r1 s> r2 goto l0_%=; \ r1 s/= -1; \ + r2 = r1; \ l0_%=: r0 = 0; \ exit; \ " : @@ -553,7 +555,7 @@ l0_%=: r0 = *(u64 *)(r1 + 0); \ SEC("socket") __description("UMOD32, zero divisor") __success __retval(0) __log_level(2) -__msg("w1 %= w2 {{.*}}; R1=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=9,var_off=(0x1; 0x8)) R2=0") +__msg("w1 %= w2 {{.*}}; R1=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=9,var_off=(0x1; 0x8))") __naked void umod32_zero_divisor(void) { asm volatile (" \ @@ -624,7 +626,7 @@ l0_%=: r0 = *(u64 *)(r1 + 0); \ SEC("socket") __description("UMOD64, zero divisor") __success __retval(0) __log_level(2) -__msg("r1 %= r2 {{.*}}; R1=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=9,var_off=(0x1; 0x8)) R2=0") +__msg("r1 %= r2 {{.*}}; R1=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=9,var_off=(0x1; 0x8))") __naked void umod64_zero_divisor(void) { asm volatile (" \ @@ -833,7 +835,7 @@ l1_%=: r0 = *(u64 *)(r1 + 0); \ SEC("socket") __description("SMOD32, zero divisor") __success __retval(0) __log_level(2) -__msg("w1 s%= w2 {{.*}}; R1=scalar(smin=0,smax=umax=0xffffffff,smin32=-8,smax32=10,var_off=(0x0; 0xffffffff)) R2=0") +__msg("w1 s%= w2 {{.*}}; R1=scalar(smin=0,smax=umax=0xffffffff,smin32=-8,smax32=10,var_off=(0x0; 0xffffffff))") __naked void smod32_zero_divisor(void) { asm volatile (" \ @@ -1084,7 +1086,7 @@ l1_%=: r0 = *(u64 *)(r1 + 0); \ SEC("socket") __description("SMOD64, zero divisor") __success __retval(0) __log_level(2) -__msg("r1 s%= r2 {{.*}}; R1=scalar(smin=smin32=-8,smax=smax32=10) R2=0") +__msg("r1 s%= r2 {{.*}}; R1=scalar(smin=smin32=-8,smax=smax32=10)") __naked void smod64_zero_divisor(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_flow_keys.c b/tools/testing/selftests/bpf/progs/verifier_flow_keys.c new file mode 100644 index 000000000000..d780a36a6e9a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_flow_keys.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Bounds checks for PTR_TO_FLOW_KEYS pointer arithmetic. */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* sizeof(struct bpf_flow_keys) is well under 4096, so +0x1000 is OOB. */ + +SEC("flow_dissector") +__description("flow_keys: in-bounds constant pointer arithmetic accepted") +__success +__naked void flow_keys_const_inbounds(void) +{ + asm volatile (" \ + r1 = *(u64 *)(r1 + %[flow_keys]); \ + r1 += 8; \ + r0 = *(u64 *)(r1 + 0); \ + r0 = 0; \ + exit; \ +" : + : __imm_const(flow_keys, offsetof(struct __sk_buff, flow_keys)) + : __clobber_all); +} + +SEC("flow_dissector") +__description("flow_keys: OOB via constant pointer arithmetic rejected") +__failure __msg("invalid access to flow keys off=4096 size=8") +__naked void flow_keys_const_oob_read(void) +{ + asm volatile (" \ + r1 = *(u64 *)(r1 + %[flow_keys]); \ + r1 += 4096; \ + r0 = *(u64 *)(r1 + 0); \ + r0 = 0; \ + exit; \ +" : + : __imm_const(flow_keys, offsetof(struct __sk_buff, flow_keys)) + : __clobber_all); +} + +SEC("flow_dissector") +__description("flow_keys: OOB write via constant pointer arithmetic rejected") +__failure __msg("invalid access to flow keys off=4096 size=8") +__naked void flow_keys_const_oob_write(void) +{ + asm volatile (" \ + r1 = *(u64 *)(r1 + %[flow_keys]); \ + r1 += 4096; \ + r2 = 0; \ + *(u64 *)(r1 + 0) = r2; \ + r0 = 0; \ + exit; \ +" : + : __imm_const(flow_keys, offsetof(struct __sk_buff, flow_keys)) + : __clobber_all); +} + +/* Equivalent OOB expressed directly in insn->off; this form was always + * rejected and is kept to show both forms now share one diagnostic. + */ +SEC("flow_dissector") +__description("flow_keys: OOB via insn->off rejected") +__failure __msg("invalid access to flow keys off=4096 size=8") +__naked void flow_keys_insn_off_oob(void) +{ + asm volatile (" \ + r1 = *(u64 *)(r1 + %[flow_keys]); \ + r0 = *(u64 *)(r1 + 4096); \ + r0 = 0; \ + exit; \ +" : + : __imm_const(flow_keys, offsetof(struct __sk_buff, flow_keys)) + : __clobber_all); +} + +SEC("flow_dissector") +__description("flow_keys: variable pointer arithmetic rejected") +__failure __msg("R1 pointer arithmetic on flow_keys prohibited") +__naked void flow_keys_var_read(void) +{ + asm volatile (" \ + r6 = r1; \ + call %[bpf_get_prandom_u32]; \ + r0 &= 0xFFFF; \ + r1 = *(u64 *)(r6 + %[flow_keys]); \ + r1 += r0; \ + r0 = *(u64 *)(r1 + 0); \ + r0 = 0; \ + exit; \ +" : + : __imm_const(flow_keys, offsetof(struct __sk_buff, flow_keys)), + __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c index e7dae0cf9c17..dcc2dd46751a 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c @@ -56,6 +56,30 @@ int trusted_task_arg_nullable(void *ctx) return res; } +/* + * Check that the verifier does not use checkpoints created + * on path with r1 == 0 to prune path with r1 != 0. + */ +SEC("?tp_btf/task_newtask") +__failure +__flag(BPF_F_TEST_STATE_FREQ) +__msg("R1 type=scalar expected=ptr_, trusted_ptr_, rcu_ptr_") +__naked int null_btf_id_arg_global_subprog(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 42;" + "if r0 > 42 goto 1f;" + "r1 = 0;" + "1:" + "call subprog_trusted_task_nullable;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + __weak int subprog_trusted_task_nonnull(struct task_struct *task __arg_trusted) { return task->pid + task->tgid; @@ -153,7 +177,7 @@ __weak int subprog_trusted_destroy(struct task_struct *task __arg_trusted) SEC("?tp_btf/task_newtask") __failure __log_level(2) -__msg("release kernel function bpf_task_release expects refcounted PTR_TO_BTF_ID") +__msg("release kfunc bpf_task_release expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(trusted_destroy_fail, struct task_struct *task, u64 clone_flags) { return subprog_trusted_destroy(task); @@ -287,6 +311,25 @@ int trusted_to_untrusted_mem(void *ctx) return subprog_void_untrusted(bpf_get_current_task_btf()); } +__weak int subprog_write_mem_arg(int *p) +{ + if (!p) + return 0; + + *p = 42; + return 0; +} + +SEC("?tp_btf/task_newtask") +__failure +__msg("only read is supported") +int trusted_btf_field_to_writable_mem(void *ctx) +{ + struct task_struct *task = bpf_get_current_task_btf(); + + return subprog_write_mem_arg(&task->prio); +} + SEC("tp_btf/sys_enter") __success int anything_to_untrusted_mem(void *ctx) @@ -307,4 +350,57 @@ int anything_to_untrusted_mem(void *ctx) return 0; } +struct pkt_arg { + __u64 x; + __u8 pad[56]; +}; + +__weak int subprog_pkt_ptr_no_change(struct pkt_arg *p) +{ + if (!p) + return 0; + + return p->x; +} + +SEC("?tc") +__success +int pkt_ptr_to_global_mem_arg_no_change(struct __sk_buff *skb) +{ + void *data = (void *)(long)skb->data; + void *data_end = (void *)(long)skb->data_end; + struct pkt_arg *p = data; + + if ((void *)(p + 1) > data_end) + return 0; + + return subprog_pkt_ptr_no_change(p); +} + +__weak int subprog_pkt_ptr_changes_data(struct __sk_buff *skb __arg_ctx, + struct pkt_arg *p) +{ + if (!p) + return 0; + + bpf_skb_pull_data(skb, 0); + return p->x; +} + +SEC("?tc") +__failure __log_level(2) +__msg("R2 is a packet pointer, but func#{{[0-9]+}} may change packet data") +__msg("Caller passes invalid args into func#{{[0-9]+}} ('subprog_pkt_ptr_changes_data')") +int pkt_ptr_to_global_mem_arg_changes_data(struct __sk_buff *skb) +{ + void *data = (void *)(long)skb->data; + void *data_end = (void *)(long)skb->data_end; + struct pkt_arg *p = data; + + if ((void *)(p + 1) > data_end) + return 0; + + return subprog_pkt_ptr_changes_data(skb, p); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c index 20904cd2baa2..966f49348787 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c @@ -46,12 +46,21 @@ __noinline long global_dead(void) } SEC("?raw_tp") -__success __log_level(2) +__success __log_level(6) /* main prog is validated completely first */ __msg("('global_calls_good_only') is global and assumed valid.") /* eventually global_good() is transitively validated as well */ __msg("Validating global_good() func") __msg("('global_good') is safe for any args that match its prototype") +__msg("subprog 0 (chained_global_func_calls_success) main insns_self 7 insns_total 7 stack") +__msg("subprog {{[0-9]+}} (global_calls_good_only) global insns_self 2 insns_total 2 stack") +#if defined(__BPF_CPU_VERSION__) && __BPF_CPU_VERSION__ >= 4 +__msg("subprog {{[0-9]+}} (global_good) global insns_self 3 insns_total 3 stack") +__msg("processed 12 insns") +#else +__msg("subprog {{[0-9]+}} (global_good) global insns_self 5 insns_total 5 stack") +__msg("processed 14 insns") +#endif int chained_global_func_calls_success(void) { int sum = 0; @@ -134,7 +143,6 @@ __noinline __weak int subprog_user_anon_mem(user_struct_t *t) SEC("?tracepoint") __failure __log_level(2) -__msg("invalid bpf_context access") __msg("Caller passes invalid args into func#1 ('subprog_user_anon_mem')") int anon_user_mem_invalid(void *ctx) { @@ -152,6 +160,23 @@ int anon_user_mem_valid(void *ctx) return subprog_user_anon_mem(&t); } +__noinline __weak int subprog_user_anon_mem_huge(int (*p)[0x3fffffff]) +{ + return p ? (*p)[1] : 0; +} + +SEC("?tracepoint") +__failure __log_level(2) +__msg("R1 memory size 4294967292 is too large") +int anon_user_mem_huge_size_invalid(void *ctx) +{ + int (*p)[0x3fffffff]; + int tiny = 42; + + p = (void *)&tiny; + return subprog_user_anon_mem_huge(p) + tiny; +} + __noinline __weak int subprog_nonnull_ptr_good(int *p1 __arg_nonnull, int *p2 __arg_nonnull) { return (*p1) * (*p2); /* good, no need for NULL checks */ @@ -168,6 +193,16 @@ int arg_tag_nonnull_ptr_good(void *ctx) return subprog_nonnull_ptr_good(&x, &y); } +SEC("?raw_tp") +__failure __log_level(2) +__msg("R1 is expected to be non-NULL") +int arg_tag_nonnull_ptr_null_bad(void *ctx) +{ + int y = 74; + + return subprog_nonnull_ptr_good(NULL, &y); +} + /* this global subprog can be now called from many types of entry progs, each * with different context type */ @@ -358,6 +393,100 @@ int arg_tag_ctx_syscall(void *ctx) return tracing_subprog_void(ctx) + tracing_subprog_u64(ctx) + tp_whatever(ctx); } +__weak int syscall_array_bpf_for(void *ctx __arg_ctx) +{ + int *arr = ctx; + int i; + + bpf_for(i, 0, 100) + arr[i] *= i; + + return 0; +} + +SEC("?syscall") +__success __log_level(2) +int arg_tag_ctx_syscall_bpf_for(void *ctx) +{ + return syscall_array_bpf_for(ctx); +} + +SEC("syscall") +__auxiliary +int syscall_tailcall_target(void *ctx) +{ + return syscall_array_bpf_for(ctx); +} + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __array(values, int (void *)); +} syscall_prog_array SEC(".maps") = { + .values = { + [0] = (void *)&syscall_tailcall_target, + }, +}; + +SEC("?syscall") +__success __log_level(2) +int arg_tag_ctx_syscall_tailcall(void *ctx) +{ + bpf_tail_call(ctx, &syscall_prog_array, 0); + return 0; +} + +SEC("?syscall") +__failure __log_level(2) +__msg("dereference of modified ctx ptr R1 off=8 disallowed") +int arg_tag_ctx_syscall_tailcall_fixed_off_bad(void *ctx) +{ + char *p = ctx; + + p += 8; + bpf_tail_call(p, &syscall_prog_array, 0); + return 0; +} + +SEC("?syscall") +__failure __log_level(2) +__msg("variable ctx access var_off=(0x0; 0x4) disallowed") +int arg_tag_ctx_syscall_tailcall_var_off_bad(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + + off &= 4; + p += off; + bpf_tail_call(p, &syscall_prog_array, 0); + return 0; +} + +SEC("?syscall") +__failure __log_level(2) +__msg("dereference of modified ctx ptr R1 off=8 disallowed") +int arg_tag_ctx_syscall_fixed_off_bad(void *ctx) +{ + char *p = ctx; + + p += 8; + return subprog_ctx_tag(p); +} + +SEC("?syscall") +__failure __log_level(2) +__msg("variable ctx access var_off=(0x0; 0x4) disallowed") +int arg_tag_ctx_syscall_var_off_bad(void *ctx) +{ + __u64 off = bpf_get_prandom_u32(); + char *p = ctx; + + off &= 4; + p += off; + return subprog_ctx_tag(p); +} + __weak int subprog_dynptr(struct bpf_dynptr *dptr) { long *d, t, buf[1] = {}; @@ -388,4 +517,23 @@ int arg_tag_dynptr(struct xdp_md *ctx) return subprog_dynptr(&dptr); } +__weak +void foo(void) +{ +} + +SEC("?tc") +__failure __msg("R0 !read_ok") +int return_from_void_global(struct __sk_buff *skb) +{ + foo(); + + asm volatile( + "r1 = r0;" + ::: + ); + + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_gotox.c b/tools/testing/selftests/bpf/progs/verifier_gotox.c index 607dad058ca1..0e27c2c79c57 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotox.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotox.c @@ -6,7 +6,7 @@ #include "bpf_misc.h" #include "../../../include/linux/filter.h" -#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) +#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_powerpc) #define DEFINE_SIMPLE_JUMP_TABLE_PROG(NAME, SRC_REG, OFF, IMM, OUTCOME) \ \ @@ -47,6 +47,54 @@ DEFINE_SIMPLE_JUMP_TABLE_PROG(reserved_field_src_reg, BPF_REG_1, 0, 0, __fa DEFINE_SIMPLE_JUMP_TABLE_PROG(reserved_field_non_zero_off, BPF_REG_0, 1, 0, __failure __msg("BPF_JA|BPF_X uses reserved fields")) DEFINE_SIMPLE_JUMP_TABLE_PROG(reserved_field_non_zero_imm, BPF_REG_0, 0, 1, __failure __msg("BPF_JA|BPF_X uses reserved fields")) +#define DEFINE_TERMINAL_GOTOX_PROG(NAME, BASE) \ + __naked void NAME(void) \ + { \ + asm volatile (" \ + .pushsection .jumptables,\"\",@progbits; \ +jt0_%=: \ + .quad ret0_%= - " BASE "; \ + .size jt0_%=, 8; \ + .global jt0_%=; \ + .popsection; \ + \ + r0 = jt0_%= ll; \ + r0 = *(u64 *)(r0 + 0); \ + goto end_%=; \ +ret0_%=: \ + r0 = 0; \ + exit; \ +end_%=: \ + .8byte %[gotox_r0]; \ +" : \ + : __imm_insn(gotox_r0, BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, \ + BPF_REG_0, 0, 0, 0)) \ + : __clobber_all); \ + } + +SEC("socket") +__success __retval(0) +DEFINE_TERMINAL_GOTOX_PROG(jump_table_terminal_gotox, "socket") + +static __noinline __used +DEFINE_TERMINAL_GOTOX_PROG(terminal_gotox_subprog1, ".text") + +static __noinline __used int terminal_gotox_subprog2(void) +{ + return 0; +} + +SEC("socket") +__success __retval(0) +__naked void jump_table_terminal_gotox_subprog(void) +{ + asm volatile (" \ + call terminal_gotox_subprog1; \ + call terminal_gotox_subprog2; \ + exit; \ +" ::: __clobber_all); +} + /* * Gotox is forbidden when there is no jump table loaded * which points to the sub-function where the gotox is used @@ -131,7 +179,7 @@ DEFINE_INVALID_SIZE_PROG(u16, __failure __msg("Invalid read of 2 bytes from insn DEFINE_INVALID_SIZE_PROG(u8, __failure __msg("Invalid read of 1 bytes from insn_array")) SEC("socket") -__failure __msg("misaligned value access off 0+1+0 size 8") +__failure __msg("misaligned value access off 1+0 size 8") __naked void jump_table_misaligned_access(void) { asm volatile (" \ @@ -187,7 +235,7 @@ jt0_%=: \ } SEC("socket") -__failure __msg("invalid access to map value, value_size=16 off=-24 size=8") +__failure __msg("R0 min value is negative") __naked void jump_table_invalid_mem_acceess_neg(void) { asm volatile (" \ @@ -384,6 +432,31 @@ jt0_%=: \ : __clobber_all); } -#endif /* __TARGET_ARCH_x86 || __TARGET_ARCH_arm64 */ +/* check valid spill/fill, ptr to insn */ +SEC("socket") +__success +__naked void spill_fill_ptr_to_insn(void) +{ + asm volatile ( + ".pushsection .jumptables,\"\",@progbits;" + "jt0_%=:" + ".quad ret0_%= - socket;" + ".size jt0_%=, 8;" + ".global jt0_%=;" + ".popsection;" + "r0 = jt0_%= ll;" + "r0 = *(u64 *)(r0 + 0);" + "*(u64 *)(r10 - 8) = r0;" + "r0 = *(u64 *)(r10 - 8);" + ".8byte %[gotox_r0];" + "ret0_%=:" + "r0 = 0;" + "exit;" + : + : __imm_insn(gotox_r0, BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_0, 0, 0, 0)) + : __clobber_all); +} + +#endif /* __TARGET_ARCH_x86 || __TARGET_ARCH_arm64 || __TARGET_ARCH_powerpc*/ char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c index f2c54e4d89eb..343fc08d9747 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c @@ -85,7 +85,7 @@ __naked void stack_bitwise_and_zero_included(void) r2 += -64; \ r4 = 0; \ /* Call bpf_ringbuf_output(), it is one of a few helper functions with\ - * ARG_CONST_SIZE_OR_ZERO parameter allowed in unpriv mode.\ + * ARG_MEM_SIZE_OR_ZERO parameter allowed in unpriv mode.\ * For unpriv this should signal an error, because memory at &fp[-64] is\ * not initialized. \ */ \ @@ -278,7 +278,7 @@ __naked void stack_jmp_no_min_check(void) r2 += -64; \ r4 = 0; \ /* Call bpf_ringbuf_output(), it is one of a few helper functions with\ - * ARG_CONST_SIZE_OR_ZERO parameter allowed in unpriv mode.\ + * ARG_MEM_SIZE_OR_ZERO parameter allowed in unpriv mode.\ * For unpriv this should signal an error, because memory at &fp[-64] is\ * not initialized. \ */ \ @@ -778,7 +778,7 @@ __naked void variable_memory_8_bytes_leak(void) r3 += 1; \ r4 = 0; \ /* Call bpf_ringbuf_output(), it is one of a few helper functions with\ - * ARG_CONST_SIZE_OR_ZERO parameter allowed in unpriv mode.\ + * ARG_MEM_SIZE_OR_ZERO parameter allowed in unpriv mode.\ * For unpriv this should signal an error, because memory region [1, 64]\ * at &fp[-64] is not fully initialized. \ */ \ diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_helper_packet_access.c index 74f5f9cd153d..71cee3f58324 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_packet_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_packet_access.c @@ -360,7 +360,7 @@ l0_%=: r0 = 0; \ SEC("tc") __description("helper access to packet: test15, cls helper fail sub") -__failure __msg("invalid access to packet") +__failure __msg("R1 min value is negative") __naked void test15_cls_helper_fail_sub(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_value_access.c b/tools/testing/selftests/bpf/progs/verifier_helper_value_access.c index 886498b5e6f3..c6603a118fdc 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_value_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_value_access.c @@ -91,7 +91,7 @@ l0_%=: exit; \ /* Call a function taking a pointer and a size which doesn't allow the size to * be zero (i.e. bpf_trace_printk() declares the second argument to be - * ARG_CONST_SIZE, not ARG_CONST_SIZE_OR_ZERO). We attempt to pass zero for the + * ARG_MEM_SIZE, not ARG_MEM_SIZE_OR_ZERO). We attempt to pass zero for the * size and expect to fail. */ SEC("tracepoint") @@ -1100,7 +1100,7 @@ l0_%=: exit; \ SEC("tracepoint") __description("map helper access to adjusted map (via const imm): out-of-bound 2") -__failure __msg("invalid access to map value, value_size=16 off=-4 size=8") +__failure __msg("R2 min value is negative") __naked void imm_out_of_bound_2(void) { asm volatile (" \ @@ -1176,7 +1176,7 @@ l0_%=: exit; \ SEC("tracepoint") __description("map helper access to adjusted map (via const reg): out-of-bound 2") -__failure __msg("invalid access to map value, value_size=16 off=-4 size=8") +__failure __msg("R2 min value is negative") __naked void reg_out_of_bound_2(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_int_ptr.c b/tools/testing/selftests/bpf/progs/verifier_int_ptr.c index 59e34d558654..6627f44faf4b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_int_ptr.c +++ b/tools/testing/selftests/bpf/progs/verifier_int_ptr.c @@ -65,7 +65,7 @@ __naked void ptr_to_long_half_uninitialized(void) SEC("cgroup/sysctl") __description("arg pointer to long misaligned") -__failure __msg("misaligned stack access off 0+-20+0 size 8") +__failure __msg("misaligned stack access off -20+0 size 8") __naked void arg_ptr_to_long_misaligned(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c b/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c index 75dd922e4e9f..1fbcc5228306 100644 --- a/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c +++ b/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c @@ -169,6 +169,23 @@ static int iter_limit_cb(__u32 idx, struct num_context *ctx) } SEC("?raw_tp") +__failure __msg("R1 type=ctx expected=scalar") +__naked void bpf_loop_reject_pointer(void) +{ + asm volatile ( + "r2 = %[iter_limit_cb];" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "exit;" + : + : __imm_ptr(iter_limit_cb), + __imm(bpf_loop) + : __clobber_common + ); +} + +SEC("?raw_tp") __success int bpf_loop_iter_limit_ok(void *unused) { diff --git a/tools/testing/selftests/bpf/progs/verifier_jeq_infer_not_null.c b/tools/testing/selftests/bpf/progs/verifier_jeq_infer_not_null.c index bf16b00502f2..3c789c565b18 100644 --- a/tools/testing/selftests/bpf/progs/verifier_jeq_infer_not_null.c +++ b/tools/testing/selftests/bpf/progs/verifier_jeq_infer_not_null.c @@ -3,7 +3,9 @@ #include <linux/bpf.h> #include <bpf/bpf_helpers.h> +#include <stdbool.h> #include "bpf_misc.h" +#include "bpf_kfuncs.h" struct { __uint(type, BPF_MAP_TYPE_XSKMAP); @@ -12,6 +14,13 @@ struct { __type(value, int); } map_xskmap SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1); + __type(key, int); + __type(value, int); +} map_hash SEC(".maps"); + /* This is equivalent to the following program: * * r6 = skb->sk; @@ -210,4 +219,242 @@ l0_%=: /* return 0; */ \ : __clobber_all); } +/* Verified that we can detect the pointer as non_null when comparing with + * register with value 0. JEQ test case. + */ +SEC("xdp") +__success __log_level(2) +/* to make sure the branch is not falsely predicted*/ +__msg("r0 = *(u32 *)(r0 +0)") +__msg("from 7 to 9") +__naked void jeq_reg_reg_null_check(void) +{ + asm volatile (" \ + *(u32*)(r10 - 8) = 0; \ + r1 = %[map_xskmap] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + r1 = 0; \ + if r0 == r1 goto 1f; \ + r0 = *(u32*)(r0 +0); \ +1: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_xskmap) + : __clobber_all); +} + +/* Same as above but for JNE. + */ +SEC("xdp") +__success __log_level(2) +/* to make sure the branch is not falsely predicted*/ +__msg("r0 = *(u32 *)(r0 +0)") +__msg("from 7 to 9") +__naked void jne_reg_reg_null_check(void) +{ + asm volatile (" \ + *(u32*)(r10 - 8) = 0; \ + r1 = %[map_xskmap] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + r1 = 0; \ + if r0 != r1 goto 1f; \ + goto 2f; \ +1: r0 = *(u32*)(r0 +0); \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_xskmap) + : __clobber_all); +} + +/* + * A comparison between PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED and + * PTR_TO_MAP_VALUE_OR_NULL should not infer that map pointer is not null. + * A bug in check_cond_jmp_op() made such inference possible. + */ +SEC("raw_tp") +__failure +__msg("error: invalid dereference of R0 (a nullable map value pointer)") +__msg(">>> 11 | (61) r0 = *(u32 *)(r0 +0)") +__naked void untrusted_mem_does_not_infer_map_value_non_null(void) +{ + asm volatile (" \ + /* r6 = bpf_rdonly_cast(0, 0); */ \ + r1 = 0; \ + r2 = 0; \ + call %[bpf_rdonly_cast]; \ + r6 = r0; \ + /* r0 = bpf_map_lookup_elem(map_hash, &key); */ \ + *(u64 *)(r10 - 8) = 0; \ + r1 = %[map_hash] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + /* \ + * buggy verifier assumed that r6 can't be null \ + * and marked r0 non-null as well. \ + */ \ + if r6 != r0 goto 1f; \ + r0 = *(u32 *)(r0 + 0); \ +1: r0 = 0; \ + exit; \ +" : + : __imm(bpf_rdonly_cast), + __imm(bpf_map_lookup_elem), + __imm_addr(map_hash) + : __clobber_all); +} + +/* + * A pointer with an offset that is not bounded from above may be null at + * runtime, hence it is not a witness for the pointer it is compared with. + */ +SEC("socket") +__failure +__msg("error: invalid dereference of R7 (a nullable map value pointer)") +__naked void unbounded_offset_does_not_infer_map_value_non_null(void) +{ + asm volatile (" \ + /* r6 = bpf_map_lookup_elem(map_hash, &0); */ \ + *(u64 *)(r10 - 8) = 0; \ + r1 = %[map_hash] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 1f; \ + r6 = r0; \ + /* r7 = bpf_map_lookup_elem(map_hash, &1); */ \ + *(u64 *)(r10 - 8) = 1; \ + r1 = %[map_hash] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + r7 = r0; \ + /* pointer - pointer is an unknown scalar */ \ + r8 = r7; \ + r8 -= r6; \ + /* r8 is in [0, S64_MAX] */ \ + r8 <<= 1; \ + r8 >>= 1; \ + /* r6 may wrap to zero at runtime */ \ + r6 += r8; \ + if r7 != r6 goto 1f; \ + r0 = *(u8 *)(r7 + 0); \ +1: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash) + : __clobber_all); +} + +/* Same, but the offset is bounded, so the inference is still done. */ +SEC("socket") +__success +__naked void bounded_offset_infers_map_value_non_null(void) +{ + asm volatile (" \ + /* r6 = bpf_map_lookup_elem(map_hash, &0); */ \ + *(u64 *)(r10 - 8) = 0; \ + r1 = %[map_hash] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 1f; \ + r6 = r0; \ + /* r7 = bpf_map_lookup_elem(map_hash, &1); */ \ + *(u64 *)(r10 - 8) = 1; \ + r1 = %[map_hash] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + r7 = r0; \ + /* pointer - pointer is an unknown scalar */ \ + r8 = r7; \ + r8 -= r6; \ + /* r8 is in [0, 3] */ \ + r8 &= 3; \ + r6 += r8; \ + if r7 != r6 goto 1f; \ + r0 = *(u8 *)(r7 + 0); \ +1: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash) + : __clobber_all); +} + +/* + * The low 32 bits of a map value pointer may be zero, hence a 32-bit + * compare with zero cannot be predicted from the pointer being non-NULL + * and both successors of such a jump have to be verified. + */ +SEC("socket") +__failure __msg("invalid access to map value, value_size=4 off=32 size=4") +__naked void jmp32_ptr_vs_zero_jne(void) +{ + asm volatile (" \ + /* r0 = bpf_map_lookup_elem(map_hash, &key); */ \ + *(u64 *)(r10 - 8) = 0; \ + r1 = %[map_hash] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 1f; \ + if w0 != 0 goto 1f; \ + r0 = *(u32 *)(r0 + 32); \ +1: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash) + : __clobber_all); +} + +/* + * The below program is explored in two paths: r6 == 0 and r6 == 1. + * On the first path comparison "if r0 == r6 goto 2f" should mark r6 as precise, + * otherwise unsafe path with r6 == 1 would be incorrectly pruned. + */ +SEC("socket") +__failure +__flag(BPF_F_TEST_STATE_FREQ) +__msg("error: invalid dereference of R0 (a nullable map value pointer)") +__naked void imprecise_zero_does_not_infer_map_value_non_null(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + /* r6 is 0 on the path explored first, 1 on the other */\ + r6 = 1; \ + if r0 == 0 goto 1f; \ + r6 = 0; \ + /* r0 = bpf_map_lookup_elem(map_hash, &0); */ \ +1: *(u64 *)(r10 - 8) = 0; \ + r1 = %[map_hash] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == r6 goto 2f; \ + r0 = *(u8 *)(r0 + 0); \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32), + __imm(bpf_map_lookup_elem), + __imm_addr(map_hash) + : __clobber_all); +} + +void kfunc_root(void) +{ + bpf_rdonly_cast(0, 0); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_jit_inline.c b/tools/testing/selftests/bpf/progs/verifier_jit_inline.c index 4ea254063646..02e562f56f9d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_jit_inline.c +++ b/tools/testing/selftests/bpf/progs/verifier_jit_inline.c @@ -9,7 +9,11 @@ __success __retval(0) __arch_x86_64 __jited(" addq %gs:{{.*}}, %rax") __arch_arm64 -__jited(" mrs x7, SP_EL0") +__jited(" mrs x8, SP_EL0") +__arch_riscv64 +__jited(" mv a5, tp") +__arch_loongarch +__jited(" move $a5, $tp") int inline_bpf_get_current_task(void) { bpf_get_current_task(); @@ -17,4 +21,15 @@ int inline_bpf_get_current_task(void) return 0; } +SEC("fentry/bpf_fentry_test2") +__success __retval(0) +__arch_loongarch +__jited(" ld.wu $a5, $tp, 16") +int inline_bpf_get_smp_processor_id(void) +{ + bpf_get_smp_processor_id(); + + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_kfunc_perfmon.c b/tools/testing/selftests/bpf/progs/verifier_kfunc_perfmon.c new file mode 100644 index 000000000000..76c39ef30e96 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_kfunc_perfmon.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +void *user_ptr; +char dynptr_buf[8]; +u64 kaddr; + +extern struct kmem_cache *bpf_get_kmem_cache(u64 addr) __ksym; + +SEC("socket") +__success +__caps_unpriv(CAP_BPF) +__failure_unpriv +__msg_unpriv("bpf_rdonly_cast is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") +int rdonly_cast_noperfmon(void *ctx) +{ + char *p = bpf_rdonly_cast(0, 0); + + return p[0x7fff]; +} + +SEC("socket") +__success +__caps_unpriv(CAP_BPF) +__failure_unpriv +__msg_unpriv("bpf_probe_read_kernel_dynptr is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") +int probe_read_kernel_dynptr_noperfmon(void *ctx) +{ + struct bpf_dynptr dptr; + + bpf_dynptr_from_mem(dynptr_buf, sizeof(dynptr_buf), 0, &dptr); + bpf_probe_read_kernel_dynptr(&dptr, 0, sizeof(dynptr_buf), user_ptr); + return 0; +} + +SEC("socket") +__success +__caps_unpriv(CAP_BPF) +__failure_unpriv +__msg_unpriv("bpf_stream_vprintk is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") +int stream_vprintk_noperfmon(void *ctx) +{ + bpf_stream_printk(BPF_STDOUT, "%pB", (void *)kaddr); + return 0; +} + +SEC("socket") +__success +__caps_unpriv(CAP_BPF) +__failure_unpriv +__msg_unpriv("bpf_get_kmem_cache is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") +int get_kmem_cache_noperfmon(void *ctx) +{ + return !!bpf_get_kmem_cache(kaddr); +} + +__weak int subprog_untrusted_read(void *p __arg_untrusted) +{ + return *(char *)p; +} + +SEC("socket") +__success +__caps_unpriv(CAP_BPF) +__failure_unpriv +__msg_unpriv("rdonly_untrusted_mem access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") +int arg_untrusted_read_noperfmon(void *ctx) +{ + return subprog_untrusted_read(0); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_ld_ind.c b/tools/testing/selftests/bpf/progs/verifier_ld_ind.c index c925ba9a2e74..32989f981fb6 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ld_ind.c +++ b/tools/testing/selftests/bpf/progs/verifier_ld_ind.c @@ -107,4 +107,242 @@ __naked void ind_check_calling_conv_r7(void) : __clobber_all); } +/* + * ld_{abs,ind} subprog that always sets r0=1 on the success path. + * bpf_gen_ld_abs() emits a hidden exit with r0=0 when the load helper + * fails. The verifier must model this failure return so that callers + * account for r0=0 as a possible return value. + */ +__naked __noinline __used +static int ldabs_subprog(void) +{ + asm volatile ( + "r6 = r1;" + ".8byte %[ld_abs];" + "r0 = 1;" + "exit;" + : + : __imm_insn(ld_abs, BPF_LD_ABS(BPF_W, 0)) + : __clobber_all); +} + +__naked __noinline __used +static int ldind_subprog(void) +{ + asm volatile ( + "r6 = r1;" + "r7 = 0;" + ".8byte %[ld_ind];" + "r0 = 1;" + "exit;" + : + : __imm_insn(ld_ind, BPF_LD_IND(BPF_W, BPF_REG_7, 0)) + : __clobber_all); +} + +SEC("socket") +__description("ld_abs: subprog early exit on ld_abs failure") +__failure __msg("R9 !read_ok") +__naked void ld_abs_subprog_early_exit(void) +{ + asm volatile ( + "call ldabs_subprog;" + "if r0 != 0 goto l_exit_%=;" + "r0 = r9;" + "l_exit_%=:" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("ld_ind: subprog early exit on ld_ind failure") +__failure __msg("R9 !read_ok") +__naked void ld_ind_subprog_early_exit(void) +{ + asm volatile ( + "call ldind_subprog;" + "if r0 != 0 goto l_exit_%=;" + "r0 = r9;" + "l_exit_%=:" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("ld_abs: subprog with both paths safe") +__success +__naked void ld_abs_subprog_both_paths_safe(void) +{ + asm volatile ( + "call ldabs_subprog;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("ld_ind: subprog with both paths safe") +__success +__naked void ld_ind_subprog_both_paths_safe(void) +{ + asm volatile ( + "call ldind_subprog;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +__naked __noinline __used +static int ld_abs_callback(void) +{ + asm volatile ( + "r6 = *(u64 *)(r2 + 0);" + ".8byte %[ld_abs];" + "r0 = 0;" + "exit;" + : + : __imm_insn(ld_abs, BPF_LD_ABS(BPF_W, 0)) + : __clobber_all); +} + +SEC("socket") +__description("ld_abs: reject in callback") +__failure __msg("cannot use BPF_LD_[ABS|IND] within callback") +int ld_abs_callback_reject(struct __sk_buff *skb) +{ + bpf_loop(1, ld_abs_callback, &skb, 0); + return 0; +} + +__naked __noinline __used +static int ld_ind_callback_subprog(void) +{ + asm volatile ( + "r6 = r1;" + "r7 = 0;" + ".8byte %[ld_ind];" + "r0 = 0;" + "exit;" + : + : __imm_insn(ld_ind, BPF_LD_IND(BPF_W, BPF_REG_7, 0)) + : __clobber_all); +} + +__naked __noinline __used +static int ld_ind_callback(void) +{ + asm volatile ( + "r1 = *(u64 *)(r2 + 0);" + "call ld_ind_callback_subprog;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("ld_ind: reject in callback subprog") +__failure __msg("cannot use BPF_LD_[ABS|IND] within callback") +int ld_ind_callback_subprog_reject(struct __sk_buff *skb) +{ + bpf_loop(1, ld_ind_callback, &skb, 0); + return 0; +} + +static __noinline int ld_ind_global_static(struct __sk_buff *skb) +{ + asm volatile ( + "r6 = %[skb];" + "r7 = 0;" + ".8byte %[ld_ind];" + : + : [skb] "r"(skb), + __imm_insn(ld_ind, BPF_LD_IND(BPF_W, BPF_REG_7, 0)) + : __clobber_common, "r6", "r7"); + return skb->mark; +} + +__noinline int ld_ind_global(struct __sk_buff *skb) +{ + return ld_ind_global_static(skb); +} + +static int ld_ind_global_callback(__u32 index, struct __sk_buff **ctx) +{ + ld_ind_global(*ctx); + return 0; +} + +SEC("socket") +__description("ld_ind: reject in callback global subprog") +__failure __msg("cannot use BPF_LD_[ABS|IND] within callback") +int ld_ind_global_callback_reject(struct __sk_buff *skb) +{ + bpf_loop(1, ld_ind_global_callback, &skb, 0); + return 0; +} + +SEC("socket") +__description("ld_ind: allow in non-callback global subprog") +__success +int ld_ind_global_subprog_ok(struct __sk_buff *skb) +{ + return ld_ind_global(skb); +} + +/* + * ld_{abs,ind} in subprogs require scalar (int) return type in BTF. + * A test with void return must be rejected. + */ +__naked __noinline __used +static void ldabs_void_subprog(void) +{ + asm volatile ( + "r6 = r1;" + ".8byte %[ld_abs];" + "r0 = 1;" + "exit;" + : + : __imm_insn(ld_abs, BPF_LD_ABS(BPF_W, 0)) + : __clobber_all); +} + +SEC("socket") +__description("ld_abs: reject void return subprog") +__failure __msg("LD_ABS is only allowed in functions that return 'int'") +__naked void ld_abs_void_subprog_reject(void) +{ + asm volatile ( + "call ldabs_void_subprog;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +__naked __noinline __used +static void ldind_void_subprog(void) +{ + asm volatile ( + "r6 = r1;" + "r7 = 0;" + ".8byte %[ld_ind];" + "r0 = 1;" + "exit;" + : + : __imm_insn(ld_ind, BPF_LD_IND(BPF_W, BPF_REG_7, 0)) + : __clobber_all); +} + +SEC("socket") +__description("ld_ind: reject void return subprog") +__failure __msg("LD_ABS is only allowed in functions that return 'int'") +__naked void ld_ind_void_subprog_reject(void) +{ + asm volatile ( + "call ldind_void_subprog;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_ldsx.c b/tools/testing/selftests/bpf/progs/verifier_ldsx.c index c8494b682c31..41340877dc9d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ldsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_ldsx.c @@ -3,7 +3,7 @@ #include <linux/bpf.h> #include <bpf/bpf_helpers.h> #include "bpf_misc.h" -#include "bpf_arena_common.h" +#include <bpf_arena_common.h> #if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ @@ -274,11 +274,11 @@ __jited("movslq 0x10(%rdi,%r12), %r15") __jited("movswq 0x18(%rdi,%r12), %r15") __jited("movsbq 0x20(%rdi,%r12), %r15") __arch_arm64 -__jited("add x11, x7, x28") +__jited("add x11, x8, x28") __jited("ldrsw x21, [x11, #0x10]") -__jited("add x11, x7, x28") +__jited("add x11, x8, x28") __jited("ldrsh x21, [x11, #0x18]") -__jited("add x11, x7, x28") +__jited("add x11, x8, x28") __jited("ldrsb x21, [x11, #0x20]") __jited("add x11, x0, x28") __jited("ldrsw x22, [x11, #0x10]") diff --git a/tools/testing/selftests/bpf/progs/verifier_linked_scalars.c b/tools/testing/selftests/bpf/progs/verifier_linked_scalars.c index f4f8a055af8a..d571fbfc86a3 100644 --- a/tools/testing/selftests/bpf/progs/verifier_linked_scalars.c +++ b/tools/testing/selftests/bpf/progs/verifier_linked_scalars.c @@ -535,4 +535,179 @@ int spurious_precision_marks(void *ctx) return 0; } +/* + * Test that r += r (self-add, src_reg == dst_reg) clears the scalar ID + * so that sync_linked_regs() does not propagate an incorrect delta. + */ +SEC("socket") +__failure +__msg("div by zero") +__naked void scalars_self_add_clears_id(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; /* r6 unknown, id A */ \ + r7 = r6; /* r7 linked to r6, id A */ \ + call %[bpf_get_prandom_u32]; \ + r8 = r0; /* r8 unknown, id B */ \ + r9 = r8; /* r9 linked to r8, id B */ \ + if r7 != 1 goto l_exit_%=; \ + /* r7 == 1; sync propagates: r6 = 1 (known, id A) */ \ + r6 += r6; /* r6 = 2; should clear id */ \ + if r7 == r9 goto l_exit_%=; \ + /* Bug: r6 synced to r7(1)+delta(2)=3; Fix: r6 = 2 */ \ + if r6 == 3 goto l_exit_%=; \ + r0 /= 0; \ +l_exit_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Same as above but with alu32 such that w6 += w6 also clears id. */ +SEC("socket") +__failure +__msg("div by zero") +__naked void scalars_self_add_alu32_clears_id(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w6 = w0; \ + w7 = w6; \ + call %[bpf_get_prandom_u32]; \ + w8 = w0; \ + w9 = w8; \ + if w7 != 1 goto l_exit_%=; \ + w6 += w6; \ + if w7 == w9 goto l_exit_%=; \ + if w6 == 3 goto l_exit_%=; \ + r0 /= 0; \ +l_exit_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Test that stale delta from a cleared BPF_ADD_CONST does not leak + * through assign_scalar_id_before_mov() into a new id, causing + * sync_linked_regs() to compute an incorrect offset. + */ +SEC("socket") +__failure +__msg("div by zero") +__naked void scalars_stale_delta_from_cleared_id(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; /* r6 unknown, gets id A */ \ + r6 += 5; /* id A|ADD_CONST, delta 5 */ \ + r6 ^= 0; /* id cleared; delta stays 5 */ \ + r8 = r6; /* new id B, stale delta 5 */ \ + r8 += 3; /* id B|ADD_CONST, delta 3 */ \ + r9 = r6; /* id B, stale delta 5 */ \ + if r9 != 10 goto l_exit_%=; \ + /* Bug: r8 = 10+(3-5) = 8; Fix: r8 = 10+(3-0) = 13 */ \ + if r8 == 8 goto l_exit_%=; \ + r0 /= 0; \ +l_exit_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Same as above but with alu32. */ +SEC("socket") +__failure +__msg("div by zero") +__naked void scalars_stale_delta_from_cleared_id_alu32(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w6 = w0; \ + w6 += 5; \ + w6 ^= 0; \ + w8 = w6; \ + w8 += 3; \ + w9 = w6; \ + if w9 != 10 goto l_exit_%=; \ + if w8 == 8 goto l_exit_%=; \ + r0 /= 0; \ +l_exit_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Test that regsafe() verifies base_id consistency for BPF_ADD_CONST + * linked scalars during state pruning. + * + * The false branch (explored first) links R3 to R2 via ADD_CONST. + * The true branch (runtime path) links R3 to R4 (unrelated base_id). + * At the merge point, pruning must fail because the linkage topology + * differs. + */ +SEC("socket") +__description("linked scalars: add_const base_id must be consistent for pruning") +__failure __msg("invalid variable-offset") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void add_const_base_id_pruning(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64*)(r10 - 16) = r1; \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + r6 &= 1; \ + if r6 >= 1 goto l_true_%=; \ + \ + /* False branch (explored first, old state) */ \ + call %[bpf_get_prandom_u32]; \ + r2 = r0; \ + r2 &= 0xff; /* R2 = scalar(id=A) [0,255] */ \ + r3 = r2; /* R3 linked to R2 (id=A) */ \ + r3 += 10; /* R3 id=A|ADD_CONST, delta=10 */\ + r6 = 0; \ + goto l_merge_%=; \ + \ +l_true_%=: \ + /* True branch (runtime path, cur state) */ \ + call %[bpf_get_prandom_u32]; \ + r2 = r0; \ + r2 &= 0xff; /* R2 = scalar [0,255], id=0 */ \ + r4 = r0; \ + r4 &= 0xff; /* R4 = scalar [0,255], id=0 */ \ + r3 = r4; /* R3 linked to R4 (new id=C) */\ + r3 += 10; /* R3 id=C|ADD_CONST, delta=10 */\ + r6 = 0; \ + \ +l_merge_%=: \ + /* At merge, old R3 linked to R2, cur R3 linked to R4. */\ + /* Pruning must fail: base_ids A vs C inconsistent. */ \ + if r2 >= 6 goto l_exit_%=; \ + /* sync_linked_regs: R2<6 => R3<16 in old state. */ \ + /* Without fix: R3 in [10,15] from incorrect pruning. */\ + /* With fix: R3 in [10,265], not synced from R2. */ \ + r3 -= 10; /* [0,5] vs [0,255] */ \ + r9 = r10; \ + r9 += -16; \ + r9 += r3; /* fp-16+[0,5] vs fp-16+[0,255] */\ + *(u8*)(r9 + 0) = r6; /* within 16B vs past fp */ \ +l_exit_%=: \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index 2de105057bbc..401152b2b64f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -3,8 +3,10 @@ #include <linux/bpf.h> #include <bpf/bpf_helpers.h> +#include "../../../include/linux/filter.h" #include "bpf_misc.h" +char _license[] SEC("license") = "GPL"; struct { __uint(type, BPF_MAP_TYPE_HASH); __uint(max_entries, 1); @@ -12,14 +14,20 @@ struct { __type(value, long long); } map SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u64); +} array_map_8b SEC(".maps"); + +const char snprintf_u64_fmt[] = "%llu"; + SEC("socket") __log_level(2) -__msg("(0) frame 0 insn 2 +written -8") -__msg("(0) frame 0 insn 1 +live -24") -__msg("(0) frame 0 insn 1 +written -8") -__msg("(0) frame 0 insn 0 +live -8,-24") -__msg("(0) frame 0 insn 0 +written -8") -__msg("(0) live stack update done in 2 iterations") +__msg("0: (79) r1 = *(u64 *)(r10 -8) ; use: fp0-8") +__msg("1: (79) r2 = *(u64 *)(r10 -24) ; use: fp0-24") +__msg("2: (7b) *(u64 *)(r10 -8) = r1 ; def: fp0-8") __naked void simple_read_simple_write(void) { asm volatile ( @@ -33,12 +41,8 @@ __naked void simple_read_simple_write(void) SEC("socket") __log_level(2) -__msg("(0) frame 0 insn 1 +live -8") -__not_msg("(0) frame 0 insn 1 +written") -__msg("(0) live stack update done in 2 iterations") -__msg("(0) frame 0 insn 1 +live -16") -__msg("(0) frame 0 insn 1 +written -32") -__msg("(0) live stack update done in 2 iterations") +__msg("2: (79) r0 = *(u64 *)(r10 -8) ; use: fp0-8") +__msg("6: (79) r0 = *(u64 *)(r10 -16) ; use: fp0-16") __naked void read_write_join(void) { asm volatile ( @@ -58,13 +62,9 @@ __naked void read_write_join(void) SEC("socket") __log_level(2) -__msg("2: (25) if r0 > 0x2a goto pc+1") -__msg("7: (95) exit") -__msg("(0) frame 0 insn 2 +written -16") -__msg("(0) live stack update done in 2 iterations") -__msg("7: (95) exit") -__not_msg("(0) frame 0 insn 2") -__msg("(0) live stack update done in 1 iterations") +__msg("stack use/def subprog#0 must_write_not_same_slot (d0,cs0):") +__msg("6: (7b) *(u64 *)(r2 +0) = r0{{$}}") +__msg("Live regs before insn:") __naked void must_write_not_same_slot(void) { asm volatile ( @@ -83,10 +83,8 @@ __naked void must_write_not_same_slot(void) SEC("socket") __log_level(2) -__msg("(0) frame 0 insn 0 +written -8,-16") -__msg("(0) live stack update done in 2 iterations") -__msg("(0) frame 0 insn 0 +written -8") -__msg("(0) live stack update done in 2 iterations") +__msg("0: (7a) *(u64 *)(r10 -8) = 0 ; def: fp0-8") +__msg("5: (85) call bpf_map_lookup_elem#1 ; use: fp0-8h") __naked void must_write_not_same_type(void) { asm volatile ( @@ -110,10 +108,11 @@ __naked void must_write_not_same_type(void) SEC("socket") __log_level(2) -__msg("(2,4) frame 0 insn 4 +written -8") -__msg("(2,4) live stack update done in 2 iterations") -__msg("(0) frame 0 insn 2 +written -8") -__msg("(0) live stack update done in 2 iterations") +/* Callee writes fp[0]-8: stack_use at call site has slots 0,1 live */ +__msg("stack use/def subprog#0 caller_stack_write (d0,cs0):") +__msg("2: (85) call pc+1{{$}}") +__msg("stack use/def subprog#1 write_first_param (d1,cs2):") +__msg("4: (7a) *(u64 *)(r1 +0) = 7 ; def: fp0-8") __naked void caller_stack_write(void) { asm volatile ( @@ -135,23 +134,15 @@ static __used __naked void write_first_param(void) SEC("socket") __log_level(2) -/* caller_stack_read() function */ -__msg("2: .12345.... (85) call pc+4") -__msg("5: .12345.... (85) call pc+1") -__msg("6: 0......... (95) exit") -/* read_first_param() function */ -__msg("7: .1........ (79) r0 = *(u64 *)(r1 +0)") -__msg("8: 0......... (95) exit") -/* update for callsite at (2) */ -__msg("(2,7) frame 0 insn 7 +live -8") -__msg("(2,7) live stack update done in 2 iterations") -__msg("(0) frame 0 insn 2 +live -8") -__msg("(0) live stack update done in 2 iterations") -/* update for callsite at (5) */ -__msg("(5,7) frame 0 insn 7 +live -16") -__msg("(5,7) live stack update done in 2 iterations") -__msg("(0) frame 0 insn 5 +live -16") -__msg("(0) live stack update done in 2 iterations") +__msg("stack use/def subprog#0 caller_stack_read (d0,cs0):") +__msg("2: (85) call pc+{{.*}} ; use: fp0-8{{$}}") +__msg("5: (85) call pc+{{.*}} ; use: fp0-16{{$}}") +__msg("stack use/def subprog#1 read_first_param (d1,cs2):") +__msg("7: (79) r0 = *(u64 *)(r1 +0) ; use: fp0-8{{$}}") +__msg("8: (95) exit") +__msg("stack use/def subprog#1 read_first_param (d1,cs5):") +__msg("7: (79) r0 = *(u64 *)(r1 +0) ; use: fp0-16{{$}}") +__msg("8: (95) exit") __naked void caller_stack_read(void) { asm volatile ( @@ -174,20 +165,48 @@ static __used __naked void read_first_param(void) } SEC("socket") +__success +__naked void arg_track_join_convergence(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "call arg_track_join_convergence_subprog;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void arg_track_join_convergence_subprog(void) +{ + asm volatile ( + "if r1 == 0 goto 1f;" + "r0 = r1;" + "goto 2f;" +"1:" + "r0 = r2;" +"2:" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") __flag(BPF_F_TEST_STATE_FREQ) __log_level(2) -/* read_first_param2() function */ -__msg(" 9: .1........ (79) r0 = *(u64 *)(r1 +0)") -__msg("10: .......... (b7) r0 = 0") -__msg("11: 0......... (05) goto pc+0") -__msg("12: 0......... (95) exit") +/* fp0-8 consumed at insn 9, dead by insn 11. stack_def at insn 4 kills slots 0,1. */ +__msg("4: (7b) *(u64 *)(r10 -8) = r0 ; def: fp0-8") +/* stack_use at call site: callee reads fp0-8, slots 0,1 live */ +__msg("7: (85) call pc+{{.*}} ; use: fp0-8") +/* read_first_param2: no caller stack live inside callee after first read */ +__msg("9: (79) r0 = *(u64 *)(r1 +0) ; use: fp0-8") +__msg("10: (b7) r0 = 0{{$}}") +__msg("11: (05) goto pc+0{{$}}") +__msg("12: (95) exit") /* - * The purpose of the test is to check that checkpoint in - * read_first_param2() stops path traversal. This will only happen if - * verifier understands that fp[0]-8 at insn (12) is not alive. + * Checkpoint at goto +0 fires because fp0-8 is dead → state pruning. */ __msg("12: safe") -__msg("processed 20 insns") __naked void caller_stack_pruning(void) { asm volatile ( @@ -342,3 +361,2482 @@ static __used __naked unsigned long write_tail_call(void) __imm_addr(map_array) : __clobber_all); } + +/* Test precise subprog stack access analysis. + * Caller passes fp-32 (SPI 3) to callee that only accesses arg+0 and arg+8 + * (SPIs 3 and 2). Slots 0 and 1 should NOT be live at the call site. + * + * Insn layout: + * 0: *(u64*)(r10 - 8) = 0 write SPI 0 + * 1: *(u64*)(r10 - 16) = 0 write SPI 1 + * 2: *(u64*)(r10 - 24) = 0 write SPI 2 + * 3: *(u64*)(r10 - 32) = 0 write SPI 3 + * 4: r1 = r10 + * 5: r1 += -32 + * 6: call precise_read_two passes fp-32 (SPI 3) + * 7: r0 = 0 + * 8: exit + * + * At insn 6 only SPIs 2,3 should be live (slots 4-7, 0xf0). + * SPIs 0,1 are written but never read → dead. + */ +SEC("socket") +__log_level(2) +__msg("6: (85) call pc+{{.*}} ; use: fp0-24 fp0-32{{$}}") +__naked void subprog_precise_stack_access(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 24) = 0;" + "*(u64 *)(r10 - 32) = 0;" + "r1 = r10;" + "r1 += -32;" + "call precise_read_two;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Callee reads only at arg+0 (SPI 3) and arg+8 (SPI 2) */ +static __used __naked void precise_read_two(void) +{ + asm volatile ( + "r0 = *(u64 *)(r1 + 0);" + "r2 = *(u64 *)(r1 + 8);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Test that multi-level subprog calls (callee passes arg-derived ptr + * to another BPF subprog) are analyzed precisely. + * + * Caller passes fp-32 (SPI 3). The callee forwards it to inner_callee. + * inner_callee only reads at offset 0 from the pointer. + * The analysis recurses into forward_to_inner -> inner_callee and + * determines only SPI 3 is accessed (slots 6-7, 0xc0), not all of SPIs 0-3. + * + * Insn layout: + * 0: *(u64*)(r10 - 8) = 0 write SPI 0 + * 1: *(u64*)(r10 - 16) = 0 write SPI 1 + * 2: *(u64*)(r10 - 24) = 0 write SPI 2 + * 3: *(u64*)(r10 - 32) = 0 write SPI 3 + * 4: r1 = r10 + * 5: r1 += -32 + * 6: call forward_to_inner passes fp-32 (SPI 3) + * 7: r0 = 0 + * 8: exit + */ +SEC("socket") +__log_level(2) +__msg("6: (85) call pc+{{.*}} ; use: fp0-32{{$}}") +__naked void subprog_multilevel_conservative(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 24) = 0;" + "*(u64 *)(r10 - 32) = 0;" + "r1 = r10;" + "r1 += -32;" + "call forward_to_inner;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Forwards arg to another subprog */ +static __used __naked void forward_to_inner(void) +{ + asm volatile ( + "call inner_callee;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void inner_callee(void) +{ + asm volatile ( + "r0 = *(u64 *)(r1 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Test multi-frame precision loss: callee consumes caller stack early, + * but static liveness keeps it live at pruning points inside callee. + * + * Caller stores map_ptr or scalar(42) at fp-8, then calls + * consume_and_call_inner. The callee reads fp0-8 at entry (consuming + * the slot), then calls do_nothing2. After do_nothing2 returns (a + * pruning point), fp-8 should be dead -- the read already happened. + * But because the call instruction's stack_use includes SPI 0, the + * static live_stack_before at insn 7 is 0x1, keeping fp-8 live inside + * the callee and preventing state pruning between the two paths. + * + * Insn layout: + * 0: call bpf_get_prandom_u32 + * 1: if r0 == 42 goto pc+2 -> insn 4 + * 2: r0 = map ll (ldimm64 part1) + * 3: (ldimm64 part2) + * 4: *(u64)(r10 - 8) = r0 fp-8 = map_ptr OR scalar(42) + * 5: r1 = r10 + * 6: r1 += -8 + * 7: call consume_and_call_inner + * 8: r0 = 0 + * 9: exit + * + * At insn 7, live_stack_before = 0x3 (slots 0-1 live due to stack_use). + * At insn 8, live_stack_before = 0x0 (SPI 0 dead, caller doesn't need it). + */ +SEC("socket") +__flag(BPF_F_TEST_STATE_FREQ) +__log_level(2) +__success +__msg(" 7: (85) call pc+{{.*}} ; use: fp0-8") +__msg(" 8: {{.*}} (b7)") +__naked void callee_consumed_caller_stack(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "if r0 == 42 goto 1f;" + "r0 = %[map] ll;" +"1:" + "*(u64 *)(r10 - 8) = r0;" + "r1 = r10;" + "r1 += -8;" + "call consume_and_call_inner;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_addr(map) + : __clobber_all); +} + +static __used __naked void consume_and_call_inner(void) +{ + asm volatile ( + "r0 = *(u64 *)(r1 + 0);" /* read fp[0]-8 into caller-saved r0 */ + "call do_nothing2;" /* inner call clobbers r0 */ + "r0 = 0;" + "goto +0;" /* checkpoint */ + "r0 = 0;" + "goto +0;" /* checkpoint */ + "r0 = 0;" + "goto +0;" /* checkpoint */ + "r0 = 0;" + "goto +0;" /* checkpoint */ + "exit;" + ::: __clobber_all); +} + +static __used __naked void do_nothing2(void) +{ + asm volatile ( + "r0 = 0;" + "r0 = 0;" + "r0 = 0;" + "r0 = 0;" + "r0 = 0;" + "r0 = 0;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * Reproducer for unsound pruning when clean_verifier_state() promotes + * live STACK_ZERO bytes to STACK_MISC. + * + * Program shape: + * - Build key at fp-4: + * - path A keeps key byte as STACK_ZERO; + * - path B writes unknown byte making it STACK_MISC. + * - Branches merge at a prune point before map_lookup. + * - map_lookup on ARRAY map is value-sensitive to constant zero key: + * - path A: const key 0 => PTR_TO_MAP_VALUE (non-NULL); + * - path B: non-const key => PTR_TO_MAP_VALUE_OR_NULL. + * - Dereference lookup result without null check. + * + * Note this behavior won't trigger at fp-8, since the verifier will + * track 32-bit scalar spill differently as spilled_ptr. + * + * Correct verifier behavior: reject (path B unsafe). + * With blanket STACK_ZERO->STACK_MISC promotion on live slots, cached path A + * state can be generalized and incorrectly prune path B, making program load. + */ +SEC("socket") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("R0 invalid mem access 'map_value_or_null'") +__naked void stack_zero_to_misc_unsound_array_lookup(void) +{ + asm volatile ( + /* key at fp-4: all bytes STACK_ZERO */ + "*(u32 *)(r10 - 4) = 0;" + "call %[bpf_get_prandom_u32];" + /* fall-through (path A) explored first */ + "if r0 != 0 goto l_nonconst%=;" + /* path A: keep key constant zero */ + "goto l_lookup%=;" +"l_nonconst%=:" + /* path B: key byte turns to STACK_MISC, key no longer const */ + "*(u8 *)(r10 - 4) = r0;" +"l_lookup%=:" + /* value-sensitive lookup */ + "r2 = r10;" + "r2 += -4;" + "r1 = %[array_map_8b] ll;" + "call %[bpf_map_lookup_elem];" + /* unsafe when lookup result is map_value_or_null */ + "r0 = *(u64 *)(r0 + 0);" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_map_lookup_elem), + __imm_addr(array_map_8b) + : __clobber_all); +} + +/* + * Subprog variant of stack_zero_to_misc_unsound_array_lookup. + * + * Check unsound pruning when a callee modifies the caller's + * stack through a pointer argument. + * + * Program shape: + * main: + * *(u32)(fp - 4) = 0 key = 0 (all bytes STACK_ZERO) + * r1 = fp - 4 + * call maybe_clobber_key may overwrite key[0] with scalar + * <-- prune point: two states meet here --> + * r2 = fp - 4 + * r1 = array_map_8b + * call bpf_map_lookup_elem value-sensitive on const-zero key + * r0 = *(u64)(r0 + 0) deref without null check + * exit + * + * maybe_clobber_key(r1): + * r6 = r1 save &key + * call bpf_get_prandom_u32 + * if r0 == 0 goto skip path A: key stays STACK_ZERO + * *(u8)(r6 + 0) = r0 path B: key[0] becomes STACK_MISC + * skip: + * r0 = 0 + * exit + * + * Path A: const-zero key => array lookup => PTR_TO_MAP_VALUE => deref OK. + * Path B: non-const key => array lookup => PTR_TO_MAP_VALUE_OR_NULL => UNSAFE. + * + * If the cleaner collapses STACK_ZERO -> STACK_MISC for the live key + * slot, path A's cached state matches path B, pruning the unsafe path. + * + * Correct verifier behaviour: reject. + */ +SEC("socket") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("R0 invalid mem access 'map_value_or_null'") +__naked void subprog_stack_zero_to_misc_unsound(void) +{ + asm volatile ( + /* key at fp-4: all bytes STACK_ZERO */ + "*(u32 *)(r10 - 4) = 0;" + /* subprog may clobber key[0] with a scalar byte */ + "r1 = r10;" + "r1 += -4;" + "call maybe_clobber_key;" + /* value-sensitive array lookup */ + "r2 = r10;" + "r2 += -4;" + "r1 = %[array_map_8b] ll;" + "call %[bpf_map_lookup_elem];" + /* unsafe when result is map_value_or_null (path B) */ + "r0 = *(u64 *)(r0 + 0);" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(array_map_8b) + : __clobber_all); +} + +static __used __naked void maybe_clobber_key(void) +{ + asm volatile ( + "r6 = r1;" + "call %[bpf_get_prandom_u32];" + /* path A (r0==0): key stays STACK_ZERO, explored first */ + "if r0 == 0 goto 1f;" + /* path B (r0!=0): overwrite key[0] with scalar */ + "*(u8 *)(r6 + 0) = r0;" + "1:" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Demonstrate that subprog arg spill/reload breaks arg tracking, + * inflating caller stack liveness and preventing state pruning. + * + * modifier2(fp-24) has two paths: one writes a scalar to *(r1+8) + * = caller fp-16, the other leaves it as zero. After modifier2 + * returns, fp-16 is never read again — it is dead. + * + * spill_reload_reader2(fp-24) only reads caller fp-8 via + * *(r1+16), but it spills r1 across a helper call. This + * breaks compute_subprog_arg_access(): the reload from callee + * stack cannot be connected back to arg1, so arg1 access goes + * "all (conservative)". At the call site (r1 = fp-24, slot 5) + * apply_callee_stack_access() marks slots 0..5 as stack_use — + * pulling fp-16 (slots 2-3) into live_stack_before even though + * the reader never touches it. + * + * Result: at modifier2's return point two states with different + * fp-16 values cannot be pruned. + * + * With correct (or old dynamic) liveness fp-16 is dead at that + * point and the states prune → "6: safe" appears in the log. + */ +SEC("socket") +__flag(BPF_F_TEST_STATE_FREQ) +__log_level(2) +__success +__msg("6: safe") +__naked void spill_reload_inflates_stack_liveness(void) +{ + asm volatile ( + /* struct at fp-24: { ctx; ptr; tail; } */ + "*(u64 *)(r10 - 24) = r1;" /* fp-24 = ctx */ + "*(u64 *)(r10 - 16) = r1;" /* fp-16 = ctx (STACK_SPILL ptr) */ + "*(u64 *)(r10 - 8) = 0;" /* fp-8 = tail */ + /* modifier2 writes different values to fp-16 on two paths */ + "r1 = r10;" + "r1 += -24;" + "call modifier2;" + /* insn 6: prune point — two states with different fp-16 + * path A: fp-16 = STACK_MISC (scalar overwrote pointer) + * path B: fp-16 = STACK_SPILL (original ctx pointer) + * STACK_MISC does NOT subsume STACK_SPILL(ptr), + * so pruning fails unless fp-16 is cleaned (dead). + */ + "r1 = r10;" + "r1 += -24;" + "call spill_reload_reader2;" /* reads fp-8 via *(r1+16) */ + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Two paths: one writes a scalar to *(r1+8) = caller fp-16, + * the other leaves it unchanged. Both return 0 via separate + * exits to prevent pruning inside the subprog at the merge. + */ +static __used __naked void modifier2(void) +{ + asm volatile ( + "r6 = r1;" + "call %[bpf_get_prandom_u32];" + "if r0 == 0 goto 1f;" + "*(u64 *)(r6 + 8) = r0;" /* fp-16 = random */ + "r0 = 0;" + "exit;" /* path A exit */ + "1:" + "r0 = 0;" + "exit;" /* path B exit */ + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Receives r1 = caller fp-24. Only reads *(r1+16) = fp-8. + * Spills r1 across a helper call → arg tracking goes conservative → + * slots 0..5 all appear used instead of just slot 1 (fp-8). + */ +static __used __naked void spill_reload_reader2(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = r1;" /* spill arg1 */ + "call %[bpf_get_prandom_u32];" /* clobbers r1-r5 */ + "r1 = *(u64 *)(r10 - 8);" /* reload arg1 */ + "r0 = *(u64 *)(r1 + 16);" /* read caller fp-8 */ + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* BTF FUNC records are not generated for kfuncs referenced + * from inline assembly. These records are necessary for + * libbpf to link the program. The function below is a hack + * to ensure that BTF FUNC records are generated. + */ +void __kfunc_btf_root(void) +{ + bpf_iter_num_new(0, 0, 0); + bpf_iter_num_next(0); + bpf_iter_num_destroy(0); +} + +/* Test that open-coded iterator kfunc arguments get precise stack + * liveness tracking. struct bpf_iter_num is 8 bytes (1 SPI). + * + * Insn layout: + * 0: *(u64*)(r10 - 8) = 0 write SPI 0 (dead) + * 1: *(u64*)(r10 - 16) = 0 write SPI 1 (dead) + * 2: r1 = r10 + * 3: r1 += -24 iter state at fp-24 (SPI 2) + * 4: r2 = 0 + * 5: r3 = 10 + * 6: call bpf_iter_num_new defines SPI 2 (KF_ITER_NEW) → 0x0 + * 7-8: r1 = fp-24 + * 9: call bpf_iter_num_next uses SPI 2 → 0x30 + * 10: if r0 == 0 goto 2f + * 11: goto 1b + * 12-13: r1 = fp-24 + * 14: call bpf_iter_num_destroy uses SPI 2 → 0x30 + * 15: r0 = 0 + * 16: exit + * + * At insn 6, SPI 2 is defined (KF_ITER_NEW initializes, doesn't read), + * so it kills liveness from successors. live_stack_before = 0x0. + * At insns 9 and 14, SPI 2 is used (iter_next/destroy read the state), + * so live_stack_before = 0x30. + */ +SEC("socket") +__success __log_level(2) +__msg(" 6: (85) call bpf_iter_num_new{{.*}} ; def: fp0-24{{$}}") +__msg(" 9: (85) call bpf_iter_num_next{{.*}} ; use: fp0-24{{$}}") +__msg("14: (85) call bpf_iter_num_destroy{{.*}} ; use: fp0-24{{$}}") +__naked void kfunc_iter_stack_liveness(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" /* SPI 0 - dead */ + "*(u64 *)(r10 - 16) = 0;" /* SPI 1 - dead */ + "r1 = r10;" + "r1 += -24;" + "r2 = 0;" + "r3 = 10;" + "call %[bpf_iter_num_new];" +"1:" + "r1 = r10;" + "r1 += -24;" + "call %[bpf_iter_num_next];" + "if r0 == 0 goto 2f;" + "goto 1b;" +"2:" + "r1 = r10;" + "r1 += -24;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + :: __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy) + : __clobber_all); +} + +/* + * Test for soundness bug in static stack liveness analysis. + * + * The static pre-pass tracks FP-derived register offsets to determine + * which stack slots are accessed. When a PTR_TO_STACK is spilled to + * the stack and later reloaded, the reload (BPF_LDX) kills FP-derived + * tracking, making subsequent accesses through the reloaded pointer + * invisible to the static analysis. + * + * This causes the analysis to incorrectly mark SPI 0 as dead at the + * merge point. clean_verifier_state() zeros it in the cached state, + * and stacksafe() accepts the new state against STACK_INVALID, + * enabling incorrect pruning. + * + * Path A (verified first): stores PTR_TO_MAP_VALUE in SPI 0 + * Path B (verified second): stores scalar 42 in SPI 0 + * After merge: reads SPI 0 through spilled/reloaded PTR_TO_STACK + * and dereferences the result as a pointer. + * + * Correct behavior: reject (path B dereferences a scalar) + * Bug behavior: accept (path B is incorrectly pruned) + */ +SEC("socket") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("R0 invalid mem access 'scalar'") +__naked void spill_ptr_liveness_type_confusion(void) +{ + asm volatile ( + /* Map lookup to get PTR_TO_MAP_VALUE */ + "r1 = %[map] ll;" + "*(u32 *)(r10 - 32) = 0;" + "r2 = r10;" + "r2 += -32;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto l_exit%=;" + /* r6 = PTR_TO_MAP_VALUE (callee-saved) */ + "r6 = r0;" + /* Branch: fall-through (path A) verified first */ + "call %[bpf_get_prandom_u32];" + "if r0 != 0 goto l_scalar%=;" + /* Path A: store map value ptr at SPI 0 */ + "*(u64 *)(r10 - 8) = r6;" + "goto l_merge%=;" +"l_scalar%=:" + /* Path B: store scalar at SPI 0 */ + "r1 = 42;" + "*(u64 *)(r10 - 8) = r1;" +"l_merge%=:" + /* + * Spill PTR_TO_STACK{off=-8} to SPI 1, then reload. + * Reload kills FP-derived tracking, hiding the + * subsequent SPI 0 access from the static analysis. + */ + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 16) = r1;" + "goto +0;" /* checkpoint */ + "goto +0;" /* checkpoint */ + "goto +0;" /* checkpoint */ + "r1 = *(u64 *)(r10 - 16);" + /* Read SPI 0 through reloaded pointer */ + "r0 = *(u64 *)(r1 + 0);" + /* Dereference: safe for map value (path A), + * unsafe for scalar (path B). + */ + "r0 = *(u64 *)(r0 + 0);" + "exit;" +"l_exit%=:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm(bpf_get_prandom_u32), + __imm_addr(map) + : __clobber_all); +} + +/* === Tests for 4-byte stack slot liveness granularity === */ + +/* Test that a 4-byte aligned write is stack_def and kills liveness. + * + * 0: *(u64 *)(r10 - 8) = 0 def slots 0,1 (full SPI 0) + * 1: *(u32 *)(r10 - 8) = 0 def slot 1 (4-byte write kills slot 1) + * 2: r0 = *(u64 *)(r10 - 8) use slots 0,1 + * 3: r0 = 0 + * 4: exit + * + * At insn 1, the 4-byte write defines slot 1. Slot 0 still flows + * backward from insn 2's read: live_stack_before = 0x1. + */ +SEC("socket") +__log_level(2) +__msg("1: (62) *(u32 *)(r10 -8) = 0 ; def: fp0-8h") +__naked void four_byte_write_kills_slot(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "*(u32 *)(r10 - 8) = 0;" + "r0 = *(u64 *)(r10 - 8);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Test that a write to the upper half of an SPI is dead when only + * the lower half is read. This was impossible at SPI granularity + * where any read of the SPI kept the entire SPI live. + * + * 0: *(u32 *)(r10 - 8) = 0 def slot 1 (DEAD: never read) + * 1: *(u32 *)(r10 - 4) = 0 def slot 0 + * 2: r0 = *(u32 *)(r10 - 4) use slot 0 only + * 3: r0 = 0 + * 4: exit + * + * At insn 0, nothing is live (0x0). Previously at SPI granularity, + * the read at insn 2 would mark the full SPI 0 as live and the + * 4-byte writes wouldn't count as def, so insn 0 would have had + * SPI 0 live (0x1). + */ +SEC("socket") +__log_level(2) +__msg("0: (62) *(u32 *)(r10 -8) = 0 ; def: fp0-8h") +__msg("2: (61) r0 = *(u32 *)(r10 -4) ; use: fp0-4h") +__naked void dead_half_spi_write(void) +{ + asm volatile ( + "*(u32 *)(r10 - 8) = 0;" + "*(u32 *)(r10 - 4) = 0;" + "r0 = *(u32 *)(r10 - 4);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Test that a 4-byte read from the upper half of SPI 0 makes only + * slot 1 live (0x2), not the full SPI (0x3). + * + * 0: *(u64 *)(r10 - 8) = 0 def slots 0,1 + * 1: r0 = *(u32 *)(r10 - 8) use slot 1 only (upper half) + * 2: r0 = 0 + * 3: exit + * + * At insn 1, live_stack_before = 0x2 (slot 1 only). + */ +SEC("socket") +__log_level(2) +__msg("1: (61) r0 = *(u32 *)(r10 -8) ; use: fp0-8h") +__naked void four_byte_read_upper_half(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r0 = *(u32 *)(r10 - 8);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Test that a 2-byte write does NOT count as stack_def. + * Sub-4-byte writes don't fully cover a 4-byte slot, + * so liveness passes through. + * + * 0: *(u64 *)(r10 - 8) = 0 def slots 0,1 + * 1: *(u16 *)(r10 - 4) = 0 NOT stack_def (2 < 4 bytes) + * 2: r0 = *(u32 *)(r10 - 4) use slot 0 + * 3: r0 = 0 + * 4: exit + * + * At insn 1, slot 0 still live (0x1) because 2-byte write + * didn't kill it. + */ +SEC("socket") +__log_level(2) +__msg("0: (7a) *(u64 *)(r10 -8) = 0 ; def: fp0-8") +__msg("1: (6a) *(u16 *)(r10 -4) = 0{{$}}") +__msg("2: (61) r0 = *(u32 *)(r10 -4) ; use: fp0-4h") +__naked void two_byte_write_no_kill(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "*(u16 *)(r10 - 4) = 0;" + "r0 = *(u32 *)(r10 - 4);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Test that a 1-byte write does NOT count as stack_def. + * + * 0: *(u64 *)(r10 - 8) = 0 def slots 0,1 + * 1: *(u8 *)(r10 - 4) = 0 NOT stack_def (1 < 4 bytes) + * 2: r0 = *(u32 *)(r10 - 4) use slot 0 + * 3: r0 = 0 + * 4: exit + * + * At insn 1, slot 0 still live (0x1). + */ +SEC("socket") +__log_level(2) +__msg("0: (7a) *(u64 *)(r10 -8) = 0 ; def: fp0-8") +__msg("1: (72) *(u8 *)(r10 -4) = 0") +__msg("2: (61) r0 = *(u32 *)(r10 -4) ; use: fp0-4h") +__naked void one_byte_write_no_kill(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "*(u8 *)(r10 - 4) = 0;" + "r0 = *(u32 *)(r10 - 4);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Test stack access beyond fp-256 exercising the second bitmask word. + * fp-264 is SPI 32, slots 64-65, which are bits 0-1 of live_stack[1]. + * + * 0: *(u64 *)(r10 - 264) = 0 def slots 64,65 + * 1: r0 = *(u64 *)(r10 - 264) use slots 64,65 + * 2: r0 = 0 + * 3: exit + * + * At insn 1, live_stack high word has bits 0,1 set: 0x3:0x0. + */ +SEC("socket") +__log_level(2) +__msg("1: (79) r0 = *(u64 *)(r10 -264) ; use: fp0-264") +__naked void high_stack_second_bitmask_word(void) +{ + asm volatile ( + "*(u64 *)(r10 - 264) = 0;" + "r0 = *(u64 *)(r10 - 264);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Test that two separate 4-byte writes to each half of an SPI + * together kill liveness for the full SPI. + * + * 0: *(u32 *)(r10 - 8) = 0 def slot 1 (upper half) + * 1: *(u32 *)(r10 - 4) = 0 def slot 0 (lower half) + * 2: r0 = *(u64 *)(r10 - 8) use slots 0,1 + * 3: r0 = 0 + * 4: exit + * + * At insn 0: live_stack_before = 0x0 (both slots killed by insns 0,1). + * At insn 1: live_stack_before = 0x2 (slot 1 still live, slot 0 killed here). + */ +SEC("socket") +__log_level(2) +__msg("0: (62) *(u32 *)(r10 -8) = 0 ; def: fp0-8h") +__msg("1: (62) *(u32 *)(r10 -4) = 0 ; def: fp0-4h") +__naked void two_four_byte_writes_kill_full_spi(void) +{ + asm volatile ( + "*(u32 *)(r10 - 8) = 0;" + "*(u32 *)(r10 - 4) = 0;" + "r0 = *(u64 *)(r10 - 8);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Test that 4-byte writes on both branches kill a slot at the + * join point. Previously at SPI granularity, a 4-byte write was + * not stack_def, so liveness would flow backward through the + * branch that only had a 4-byte write. + * + * 0: call bpf_get_prandom_u32 + * 1: if r0 != 0 goto 1f + * 2: *(u64 *)(r10 - 8) = 0 path A: def slots 0,1 + * 3: goto 2f + * 1:4: *(u32 *)(r10 - 4) = 0 path B: def slot 0 + * 2:5: r0 = *(u32 *)(r10 - 4) use slot 0 + * 6: r0 = 0 + * 7: exit + * + * Both paths define slot 0 before the read. At insn 1 (branch), + * live_stack_before = 0x0 because slot 0 is killed on both paths. + */ +SEC("socket") +__log_level(2) +__msg("1: (55) if r0 != 0x0 goto pc+2") +__msg("2: (7a) *(u64 *)(r10 -8) = 0 ; def: fp0-8") +__msg("3: (05) goto pc+1") +__msg("4: (62) *(u32 *)(r10 -4) = 0 ; def: fp0-4h") +__msg("5: (61) r0 = *(u32 *)(r10 -4) ; use: fp0-4h") +__naked void both_branches_kill_slot(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "if r0 != 0 goto 1f;" + "*(u64 *)(r10 - 8) = 0;" + "goto 2f;" +"1:" + "*(u32 *)(r10 - 4) = 0;" +"2:" + "r0 = *(u32 *)(r10 - 4);" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Soundness: cleaning the dead upper half of an SPI must not + * affect the live lower half's type information for pruning. + * + * Both halves of SPI 0 are written separately. Only the lower + * half (slot 0) is used as a 4-byte map key. The upper half + * (slot 1) is dead and cleaned to STACK_INVALID. + * + * Path A: key stays 0 (STACK_ZERO) → non-null array lookup + * Path B: key byte turns STACK_MISC → may-null array lookup + * Deref without null check: safe for A, unsafe for B. + * + * If half-SPI cleaning incorrectly corrupted the live half's + * type info, path A's cached state could generalize and unsoundly + * prune path B. + * + * Expected: reject (path B unsafe). + */ +SEC("socket") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("R0 invalid mem access 'map_value_or_null'") +__naked void half_spi_clean_preserves_stack_zero(void) +{ + asm volatile ( + "*(u32 *)(r10 - 4) = 0;" /* slot 0: STACK_ZERO */ + "*(u32 *)(r10 - 8) = 0;" /* slot 1: STACK_ZERO (dead) */ + "call %[bpf_get_prandom_u32];" + "if r0 != 0 goto l_nonconst%=;" + "goto l_lookup%=;" +"l_nonconst%=:" + "*(u8 *)(r10 - 4) = r0;" /* slot 0: STACK_MISC */ +"l_lookup%=:" + "r2 = r10;" + "r2 += -4;" + "r1 = %[array_map_8b] ll;" + "call %[bpf_map_lookup_elem];" + "r0 = *(u64 *)(r0 + 0);" /* unsafe if null */ + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_map_lookup_elem), + __imm_addr(array_map_8b) + : __clobber_all); +} + +/* + * Model of scx_lavd's pick_idle_cpu_at_cpdom iat block: + * conditional block with helper call and temporary stack spill, + * spill dead after merge. + * + * Path A (fall-through): spill r6 to fp-8 across helper call + * Path B (branch taken): skip the block entirely + * At merge (insn 6): fp-8 is dead (never read after merge) + * + * Static liveness marks fp-8 dead at merge. clean_verifier_state() + * converts path A's STACK_SPILL to STACK_INVALID. Path B has + * STACK_INVALID. stacksafe() matches -> path B pruned -> "6: safe". + */ +SEC("socket") +__flag(BPF_F_TEST_STATE_FREQ) +__success +__log_level(2) +__msg("6: safe") +__naked void dead_spill_at_merge_enables_pruning(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r6 = 7;" + "if r0 != 0 goto l_skip%=;" + /* conditional block: spill, call, reload */ + "*(u64 *)(r10 - 8) = r6;" + "call %[bpf_get_prandom_u32];" + "r6 = *(u64 *)(r10 - 8);" +"l_skip%=:" + /* fp-8 dead. Path B pruned here -> "6: safe" */ + "r0 = r6;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * FP-offset tracking loses precision on second ADD, killing all liveness. + * + * fp_off_insn_xfer() handles "FP itself + negative imm" precisely + * (e.g. r6 = r10; r6 += -24 -> slot 5). But any subsequent ADD/SUB + * on a register that already has non-zero spis falls through to + * spis_set_all(), because the code only handles the FP-itself case. + * + * A write through this imprecise register enters the non-zero-spis + * branch of set_indirect_stack_access(), which OR's the all-ones + * mask into stack_def. The backward liveness equation + * + * stack_in = (stack_out & ~stack_def) | stack_use + * + * sees ~ALL = 0, killing ALL slot liveness at that instruction. + * + * At the merge pruning point, live_stack_before is empty. + * clean_verifier_state() marks fp-8 as STACK_INVALID. + * stacksafe() skips STACK_INVALID (line "continue"), so pruning + * succeeds regardless of the current state's fp-8 value. + * Path B is pruned, its null deref is never explored. + * + * Correct behavior: reject (path B dereferences NULL). + * Bug behavior: accept (path B pruned away). + */ +SEC("socket") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("R1 invalid mem access 'scalar'") +__naked void fp_add_loses_precision_kills_liveness(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "if r0 != 0 goto l_pathB%=;" + + /* Path A (fall-through, explored first): fp-8 = 0 */ + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "goto l_merge%=;" + +"l_pathB%=:" + /* Path B (explored second): fp-8 = 42 */ + "r1 = 42;" + "*(u64 *)(r10 - 8) = r1;" + +"l_merge%=:" + /* + * Create imprecise FP-derived register. + * r6 = r10 - 24 gets precise slot 5. + * r6 += 8 hits the else branch (spis non-zero, delta > 0) + * and sets spis to ALL. r6 is actually r10-16. + */ + "r6 = r10;" + "r6 += -24;" + "r6 += 8;" + + /* + * Write through imprecise r6. Actually writes to fp-16 + * (does NOT touch fp-8), but liveness marks ALL slots + * as stack_def, killing fp-8's liveness. + */ + "r7 = 0;" + "*(u64 *)(r6 + 0) = r7;" + + /* Read fp-8: liveness says dead, but value is needed. */ + "r2 = *(u64 *)(r10 - 8);" + "if r2 == 42 goto l_danger%=;" + + /* r2 != 42 (path A: r2 == 0): safe exit */ + "r0 = 0;" + "exit;" + +"l_danger%=:" + /* Only reachable from path B (r2 == 42): null deref */ + "r1 = 0;" + "r0 = *(u64 *)(r1 + 0);" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("R1 invalid mem access 'scalar'") +__naked void fp_spill_loses_precision_kills_liveness(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "if r0 != 0 goto l_pathB%=;" + + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "goto l_merge%=;" + +"l_pathB%=:" + "r1 = 42;" + "*(u64 *)(r10 - 8) = r1;" + +"l_merge%=:" + "r6 = r10;" + "r6 += -64;" + "*(u64 *)(r10 - 160) = r6;" + "r6 = *(u64 *)(r10 - 160);" + + "r7 = 0;" + "*(u64 *)(r6 + 0) = r7;" + + "r2 = *(u64 *)(r10 - 8);" + "if r2 == 42 goto l_danger%=;" + + "r0 = *(u64 *)(r10 - 56);" + "exit;" + +"l_danger%=:" + "r1 = 0;" + "r0 = *(u64 *)(r1 + 0);" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* === Tests for frame-based AT_FP tracking === */ + +/* + * Test 1: conditional_stx_in_subprog + * Subprog conditionally writes caller's slot. + * Verify slot stays live (backward pass handles conditional def via CFG). + * + * Main writes fp-8=42, calls cond_writer(fp-8), reads fp-8. + * cond_writer only writes on one path → parent_def only on that path. + * The backward parent_live correctly keeps fp-8 live at entry + * (conditional write doesn't kill liveness at the join). + */ +SEC("socket") +__log_level(2) +/* fp-8 live at call (callee conditionally writes → slot not killed) */ +__msg("1: (7b) *(u64 *)(r10 -8) = r1 ; def: fp0-8") +__msg("4: (85) call pc+2{{$}}") +__msg("5: (79) r0 = *(u64 *)(r10 -8) ; use: fp0-8") +__naked void conditional_stx_in_subprog(void) +{ + asm volatile ( + "r1 = 42;" + "*(u64 *)(r10 - 8) = r1;" + "r1 = r10;" + "r1 += -8;" + "call cond_writer;" + "r0 = *(u64 *)(r10 - 8);" + "exit;" + ::: __clobber_all); +} + +/* Conditionally writes to *(r1+0) */ +static __used __naked void cond_writer(void) +{ + asm volatile ( + "r6 = r1;" + "call %[bpf_get_prandom_u32];" + "if r0 == 0 goto 1f;" + "*(u64 *)(r6 + 0) = r0;" + "1:" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__log_level(2) +__msg("4: (85) call pc+{{.*}} ; use: fp0-16") +__msg("7: (85) call pc+{{.*}} ; use: fp0-32") +__naked void multiple_callsites_different_offsets(void) +{ + asm volatile ( + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 32) = 0;" + "r1 = r10;" + "r1 += -16;" + "call read_first_param;" + "r1 = r10;" + "r1 += -32;" + "call read_first_param;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * Test 3: nested_fp_passthrough + * main→A→B, main's FP forwarded to B. B accesses main's stack. + * Verify liveness propagates through. + * + * Main passes fp-32 to outer_forwarder, which passes it to inner_reader. + * inner_reader reads at arg+0 (= main's fp-32). + * parent_live propagates transitively: inner→outer→main. + */ +SEC("socket") +__log_level(2) +/* At call to outer_forwarder: main's fp-32 (slots 6,7) should be live */ +__msg("6: (85) call pc+{{.*}} ; use: fp0-32") +__naked void nested_fp_passthrough(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "*(u64 *)(r10 - 16) = 0;" + "*(u64 *)(r10 - 24) = 0;" + "*(u64 *)(r10 - 32) = 0;" + "r1 = r10;" + "r1 += -32;" + "call outer_forwarder;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Forwards arg to inner_reader */ +static __used __naked void outer_forwarder(void) +{ + asm volatile ( + "call inner_reader;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void inner_reader(void) +{ + asm volatile ( + "r0 = *(u64 *)(r1 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * Test 4: callee_must_write_before_read + * Callee unconditionally writes parent slot before reading. + * Verify slot is NOT live at call site (parent_def kills it). + */ +SEC("socket") +__log_level(2) +/* fp-8 NOT live at call: callee writes before reading (parent_def kills it) */ +__msg("2: .12345.... (85) call pc+") +__naked void callee_must_write_before_read(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "call write_then_read;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Unconditionally writes *(r1+0), then reads it back */ +static __used __naked void write_then_read(void) +{ + asm volatile ( + "r6 = r1;" + "r7 = 99;" + "*(u64 *)(r6 + 0) = r7;" + "r0 = *(u64 *)(r6 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * Test 5: return_site_liveness_bleeding + * Main calls subprog twice. Slot used after one call but not the other. + * Context-insensitive: slot conservatively live at both. + * + * After first call: read fp-8. + * After second call: don't read fp-8. + * Since parent_live is per-subprog (not per call-site), + * fp-8 is live at both call sites. + */ +SEC("socket") +__log_level(2) +/* Both calls have fp-8 live due to context-insensitive parent_live */ +__msg("3: (85) call pc+{{.*}} ; use: fp0-8") +__msg("7: (85) call pc+{{.*}} ; use: fp0-8") +__naked void return_site_liveness_bleeding(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r1 = r10;" + "r1 += -8;" + "call read_first_param;" + "r0 = *(u64 *)(r10 - 8);" + "r1 = r10;" + "r1 += -8;" + "call read_first_param;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__log_level(2) +__msg("9: (85) call bpf_loop#181 ; use: fp0-16") +__naked void callback_conditional_read_beyond_ctx(void) +{ + asm volatile ( + "r1 = 42;" + "*(u64 *)(r10 - 8) = r1;" + "*(u64 *)(r10 - 16) = r1;" + "r1 = 2;" + "r2 = cb_cond_read ll;" + "r3 = r10;" + "r3 += -8;" + "r4 = 0;" + "call %[bpf_loop];" + "r0 = 0;" + "exit;" + :: __imm(bpf_loop) + : __clobber_all); +} + +/* Callback conditionally reads *(ctx - 8) = caller fp-16 */ +static __used __naked void cb_cond_read(void) +{ + asm volatile ( + "r6 = r2;" + "call %[bpf_get_prandom_u32];" + "if r0 == 0 goto 1f;" + "r0 = *(u64 *)(r6 - 8);" + "1:" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__log_level(2) +__msg("14: (7b) *(u64 *)(r6 -8) = r7 ; def: fp0-16") +__msg("15: (79) r0 = *(u64 *)(r6 -8) ; use: fp0-16") +__naked void callback_write_before_read_kills(void) +{ + asm volatile ( + "r1 = 42;" + "*(u64 *)(r10 - 8) = r1;" + "*(u64 *)(r10 - 16) = r1;" + "r1 = 2;" + "r2 = cb_write_read ll;" + "r3 = r10;" + "r3 += -8;" + "r4 = 0;" + "call %[bpf_loop];" + "r0 = 0;" + "exit;" + :: __imm(bpf_loop) + : __clobber_all); +} + +/* Callback unconditionally writes *(ctx-8), then reads it back. + * The write (parent_def) kills liveness before entry. + */ +static __used __naked void cb_write_read(void) +{ + asm volatile ( + "r6 = r2;" + "r7 = 99;" + "*(u64 *)(r6 - 8) = r7;" + "r0 = *(u64 *)(r6 - 8);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * bpf_loop callback conditionally writes fp-16 then unconditionally + * reads it. The conditional write does NOT kill liveness + */ +SEC("socket") +__log_level(2) +__msg("9: (85) call bpf_loop#181 ; use: fp0-16") +__naked void callback_conditional_write_preserves(void) +{ + asm volatile ( + "r1 = 42;" + "*(u64 *)(r10 - 8) = r1;" + "*(u64 *)(r10 - 16) = r1;" + "r1 = 2;" + "r2 = cb_cond_write_read ll;" + "r3 = r10;" + "r3 += -8;" + "r4 = 0;" + "call %[bpf_loop];" + "r0 = 0;" + "exit;" + :: __imm(bpf_loop) + : __clobber_all); +} + +static __used __naked void cb_cond_write_read(void) +{ + asm volatile ( + "r6 = r2;" + "call %[bpf_get_prandom_u32];" + "if r0 == 0 goto 1f;" + "*(u64 *)(r6 - 8) = r0;" + "1:" + "r0 = *(u64 *)(r6 - 8);" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Two bpf_loop calls with the same callback but different ctx pointers. + * + * First call: ctx=fp-8, second call: ctx=fp-24. + */ +SEC("socket") +__log_level(2) +__msg(" 8: (85) call bpf_loop{{.*}} ; use: fp0-8") +__msg("15: (85) call bpf_loop{{.*}} ; use: fp0-24") +__naked void callback_two_calls_different_ctx(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "*(u64 *)(r10 - 24) = 0;" + "r1 = 1;" + "r2 = cb_read_ctx ll;" + "r3 = r10;" + "r3 += -8;" + "r4 = 0;" + "call %[bpf_loop];" + "r1 = 1;" + "r2 = cb_read_ctx ll;" + "r3 = r10;" + "r3 += -24;" + "r4 = 0;" + "call %[bpf_loop];" + "r0 = 0;" + "exit;" + :: __imm(bpf_loop) + : __clobber_all); +} + +/* Callback reads at ctx+0 unconditionally */ +static __used __naked void cb_read_ctx(void) +{ + asm volatile ( + "r0 = *(u64 *)(r2 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * Reproducer for unsound pruning in refined_caller_live_stack(). + * + * Three-level call chain: main → mid_fwd → grandchild_deref. + * Main passes &fp-8 to mid_fwd, which forwards R1 to grandchild_deref. + * grandchild_deref reads main's fp-8 through the forwarded pointer + * and dereferences the result. + * + * refined_caller_live_stack() has a callee_offset++ when mid_fwd + * (frame 1) is mid-call. This drops the transitive parent_live + * contribution at mid_fwd's call instruction — the only place + * where grandchild_deref's read of main's fp-8 is recorded. + * As a result, main's fp-8 is cleaned to STACK_INVALID at the + * pruning point inside grandchild_deref, and path B is + * incorrectly pruned against path A. + * + * Path A: main stores PTR_TO_MAP_VALUE at fp-8 + * Path B: main stores scalar 42 at fp-8 + * + * Correct behavior: reject (path B dereferences scalar) + * Bug behavior: accept (path B pruned against cleaned path A) + */ +SEC("socket") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("R0 invalid mem access 'scalar'") +__naked void transitive_parent_stack_read_unsound(void) +{ + asm volatile ( + /* Map lookup to get PTR_TO_MAP_VALUE */ + "r1 = %[map] ll;" + "*(u32 *)(r10 - 32) = 0;" + "r2 = r10;" + "r2 += -32;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto l_exit%=;" + "r6 = r0;" + /* Branch: path A (fall-through) explored first */ + "call %[bpf_get_prandom_u32];" + "if r0 != 0 goto l_scalar%=;" + /* Path A: fp-8 = PTR_TO_MAP_VALUE */ + "*(u64 *)(r10 - 8) = r6;" + "goto l_merge%=;" +"l_scalar%=:" + /* Path B: fp-8 = scalar 42 */ + "r1 = 42;" + "*(u64 *)(r10 - 8) = r1;" +"l_merge%=:" + /* Pass &fp-8 to mid_fwd → grandchild_deref */ + "r1 = r10;" + "r1 += -8;" + "call mid_fwd;" + "r0 = 0;" + "exit;" +"l_exit%=:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm(bpf_get_prandom_u32), + __imm_addr(map) + : __clobber_all); +} + +/* Forwards R1 (ptr to main's fp-8) to grandchild_deref */ +static __used __naked void mid_fwd(void) +{ + asm volatile ( + "call grandchild_deref;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Reads main's fp-8 through forwarded pointer, dereferences result */ +static __used __naked void grandchild_deref(void) +{ + asm volatile ( + "goto +0;" /* checkpoint */ + "goto +0;" /* checkpoint */ + /* read main's fp-8: map_ptr (path A) or scalar (path B) */ + "r0 = *(u64 *)(r1 + 0);" + /* dereference: safe for map_ptr, unsafe for scalar */ + "r0 = *(u64 *)(r0 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__log_level(2) +__success +__msg("14: (79) r1 = *(u64 *)(r10 -8) // r6=fp0-8 r7=fp1-16 fp-8=fp1-16 fp-16=fp0-8") +__msg("15: (79) r0 = *(u64 *)(r1 +0) // r1=fp1-16 r6=fp0-8 r7=fp1-16 fp-8=fp1-16 fp-16=fp0-8") +__msg("stack use/def subprog#1 mid_two_fp_threshold (d1,cs2):") +__msg("14: (79) r1 = *(u64 *)(r10 -8) ; use: fp1-8") +__msg("15: (79) r0 = *(u64 *)(r1 +0) ; use: fp1-16") +__naked void two_fp_clear_stack_threshold(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "call mid_two_fp_threshold;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void mid_two_fp_threshold(void) +{ + asm volatile ( + "r6 = r1;" + "r7 = r10;" + "r7 += -16;" + "*(u64 *)(r10 - 8) = r7;" + "*(u64 *)(r10 - 16) = r6;" + "r1 = r10;" + "r1 += -8;" + "r2 = r6;" + "call inner_nop_fptest;" + "r1 = *(u64 *)(r10 - 8);" + "r0 = *(u64 *)(r1 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void inner_nop_fptest(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__log_level(2) +__success +__msg("13: (79) r1 = *(u64 *)(r10 -8) // r6=fp0-8 r7=fp1-16 fp-8=fp1-16 fp-16=fp0-8") +__msg("14: (79) r0 = *(u64 *)(r1 +0) // r1=fp1-16 r6=fp0-8 r7=fp1-16 fp-8=fp1-16 fp-16=fp0-8") +__msg("stack use/def subprog#1 mid_one_fp_threshold (d1,cs2):") +__msg("13: (79) r1 = *(u64 *)(r10 -8) ; use: fp1-8") +__msg("14: (79) r0 = *(u64 *)(r1 +0) ; use: fp1-16") +__naked void one_fp_clear_stack_threshold(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "call mid_one_fp_threshold;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void mid_one_fp_threshold(void) +{ + asm volatile ( + "r6 = r1;" + "r7 = r10;" + "r7 += -16;" + "*(u64 *)(r10 - 8) = r7;" + "*(u64 *)(r10 - 16) = r6;" + "r1 = r10;" + "r1 += -8;" + "call inner_nop_fptest;" + "r1 = *(u64 *)(r10 - 8);" + "r0 = *(u64 *)(r1 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * Reproducer for unsound pruning when a subprog forwards a parent + * stack pointer (AT_PARENT) to a helper with a memory argument. + * + * set_call_stack_access_at() previously only tracked AT_CURRENT args, + * skipping AT_PARENT entirely. This meant helper reads through parent + * stack pointers did not set parent_use, letting the slot appear dead + * at pruning checkpoints inside the subprog. + * + * Program shape: + * main: + * *(u32)(fp-4) = 0 key = STACK_ZERO (const 0) + * call bpf_get_prandom_u32 + * if r0 != 0 goto clobber path A (fall-through) first + * goto merge + * clobber: + * *(u8)(fp-4) = r0 path B: key[0] = STACK_MISC + * merge: + * r1 = fp - 4 + * call fwd_parent_key_to_helper + * r0 = 0 + * exit + * + * fwd_parent_key_to_helper(r1 = &caller_fp-4): + * goto +0 checkpoint + * r2 = r1 R2 = AT_PARENT ptr to caller fp-4 + * r1 = array_map_8b ll R1 = array map + * call bpf_map_lookup_elem reads key_size(4) from parent fp-4 + * r0 = *(u64 *)(r0 + 0) deref without null check + * r0 = 0 + * exit + * + * Path A: STACK_ZERO key = const 0 -> array lookup -> PTR_TO_MAP_VALUE + * (non-NULL for in-bounds const key) -> deref OK. + * Path B: STACK_MISC key = unknown -> array lookup -> + * PTR_TO_MAP_VALUE_OR_NULL -> deref UNSAFE. + * + * Bug: AT_PARENT R2 arg to bpf_map_lookup_elem skipped -> parent_use + * not set -> fp-4 cleaned at checkpoint -> STACK_ZERO collapses + * to STACK_INVALID -> path B pruned -> deref never checked. + * + * Correct verifier behavior: reject (path B deref of map_value_or_null). + */ +SEC("socket") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("R0 invalid mem access 'map_value_or_null'") +__naked void helper_parent_stack_read_unsound(void) +{ + asm volatile ( + /* key at fp-4: all bytes STACK_ZERO */ + "*(u32 *)(r10 - 4) = 0;" + "call %[bpf_get_prandom_u32];" + /* fall-through (path A) explored first */ + "if r0 != 0 goto l_clobber%=;" + /* path A: key stays constant zero */ + "goto l_merge%=;" +"l_clobber%=:" + /* path B: key[0] becomes STACK_MISC, key no longer const */ + "*(u8 *)(r10 - 4) = r0;" +"l_merge%=:" + "r1 = r10;" + "r1 += -4;" + "call fwd_parent_key_to_helper;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Subprog forwards parent stack pointer to bpf_map_lookup_elem as key + * on an array map, then dereferences the result without a null check. + * R1 = &parent_fp-4 (AT_PARENT in this frame). + * + * The helper reads key_size(4) bytes from parent stack. The deref of + * R0 reads the map value, NOT parent stack, so record_insn_mem_accesses + * does not set parent_use for it. The ONLY parent stack access is + * through the helper's R2 arg. + */ +static __used __naked void fwd_parent_key_to_helper(void) +{ + asm volatile ( + "goto +0;" /* checkpoint */ + "r2 = r1;" /* R2 = parent ptr (AT_PARENT) */ + "r1 = %[array_map_8b] ll;" /* R1 = array map */ + "call %[bpf_map_lookup_elem];" /* reads 4 bytes from parent fp-4 */ + /* deref without null check: safe for PTR_TO_MAP_VALUE, + * unsafe for PTR_TO_MAP_VALUE_OR_NULL + */ + "r0 = *(u64 *)(r0 + 0);" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(array_map_8b) + : __clobber_all); +} + +/* + * Regression for keeping later helper args after a whole-stack fallback + * on an earlier local arg. The first bpf_snprintf() arg is a local + * frame-derived pointer with offset-imprecise tracking (`fp1 ?`), which + * conservatively marks the whole local stack live. The fourth arg still + * forwards &parent_fp-8 and must contribute nonlocal_use[0]=0:3. + */ +SEC("socket") +__log_level(2) +__success +__msg("call bpf_snprintf{{.*}} ; use: fp1-8..-512 fp0-8") +__naked void helper_arg_fallback_keeps_scanning(void) +{ + asm volatile ( + "r1 = 42;" + "*(u64 *)(r10 - 8) = r1;" + "r1 = r10;" + "r1 += -8;" + "call helper_snprintf_parent_after_local_fallback;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void helper_snprintf_parent_after_local_fallback(void) +{ + asm volatile ( + "r6 = r1;" /* save &parent_fp-8 */ + "call %[bpf_get_prandom_u32];" + "r0 &= 8;" + "r1 = r10;" + "r1 += -16;" + "r1 += r0;" /* local fp, offset-imprecise */ + "r2 = 8;" + "r3 = %[snprintf_u64_fmt] ll;" + "r4 = r6;" /* later arg: parent fp-8 */ + "r5 = 8;" + "call %[bpf_snprintf];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_snprintf), + __imm_addr(snprintf_u64_fmt) + : __clobber_all); +} + +/* + * Test that propagate_callee_ancestor() correctly chains ancestor + * liveness across sequential calls within a single frame. + * + * main → mid_seq_touch → {nop_callee, deref_ancestor} + * + * mid_seq_touch receives two pointers: R1 = &main_fp-8 (forwarded to + * deref_ancestor) and R2 = &main_fp-16 (read directly by mid_seq_touch). + * The direct read of fp-16 forces ensure_anc_arrays() to allocate + * ancestor_live[0] for mid_seq_touch, so refined_caller_live_stack() + * uses the refined path (not the conservative fallback). + * + * mid_seq_touch calls nop_callee first (no-op, creates a pruning point), + * then calls deref_ancestor which reads main's fp-8 and dereferences it. + * + * propagate_callee_ancestor() propagates deref_ancestor's entry + * ancestor_live[0] into mid_seq_touch's anc_use[0] at the call-to-deref + * instruction. mid_seq_touch's backward pass flows this backward so + * ancestor_live[0] includes fp-8 at the pruning point between the calls. + * + * Without propagation, mid_seq_touch's ancestor_live[0] only has fp-16 + * (from the direct read) — fp-8 is missing. refined_caller_live_stack() + * Term 1 says fp-8 is dead, the verifier cleans it, and path B + * (scalar 42) is incorrectly pruned against path A (MAP_VALUE). + * + * Path A: main stores PTR_TO_MAP_VALUE at fp-8 → deref succeeds + * Path B: main stores scalar 42 at fp-8 → deref must fail + * + * Correct: reject (path B dereferences scalar) + */ +SEC("socket") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("R0 invalid mem access 'scalar'") +__naked void propagate_callee_ancestor_chain(void) +{ + asm volatile ( + /* Map lookup to get PTR_TO_MAP_VALUE */ + "r1 = %[map] ll;" + "*(u32 *)(r10 - 32) = 0;" + "r2 = r10;" + "r2 += -32;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto l_exit%=;" + "r6 = r0;" + /* Branch: path A (fall-through) explored first */ + "call %[bpf_get_prandom_u32];" + "if r0 != 0 goto l_scalar%=;" + /* Path A: fp-8 = PTR_TO_MAP_VALUE */ + "*(u64 *)(r10 - 8) = r6;" + "goto l_merge%=;" +"l_scalar%=:" + /* Path B: fp-8 = scalar 42 */ + "r1 = 42;" + "*(u64 *)(r10 - 8) = r1;" +"l_merge%=:" + /* fp-16 = dummy value (mid_seq_touch reads it directly) */ + "r1 = 99;" + "*(u64 *)(r10 - 16) = r1;" + /* R1 = &fp-8 (for deref_ancestor), R2 = &fp-16 (for mid_seq_touch) */ + "r1 = r10;" + "r1 += -8;" + "r2 = r10;" + "r2 += -16;" + "call mid_seq_touch;" + "r0 = 0;" + "exit;" +"l_exit%=:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm(bpf_get_prandom_u32), + __imm_addr(map) + : __clobber_all); +} + +/* + * R1 = &main_fp-8 (forwarded to deref_ancestor) + * R2 = &main_fp-16 (read directly here → allocates ancestor_live[0]) + * + * Reads main's fp-16 to force ancestor_live[0] allocation, then + * calls nop_callee (pruning point), then deref_ancestor. + */ +static __used __naked void mid_seq_touch(void) +{ + asm volatile ( + "r6 = r1;" /* save &main_fp-8 in callee-saved */ + "r0 = *(u64 *)(r2 + 0);" /* read main's fp-16: triggers anc_use[0] */ + "call nop_callee;" /* no-op, creates pruning point after */ + "r1 = r6;" /* restore ptr to &main_fp-8 */ + "call deref_ancestor;" /* reads main's fp-8, dereferences */ + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void nop_callee(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Reads main's fp-8 through forwarded pointer, dereferences result */ +static __used __naked void deref_ancestor(void) +{ + asm volatile ( + "r0 = *(u64 *)(r1 + 0);" /* read main's fp-8 */ + "r0 = *(u64 *)(r0 + 0);" /* deref: safe for map_ptr, unsafe for scalar */ + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * Test: callee loads an fp-derived pointer from caller's stack, then + * reads through it to access another caller stack slot. + * + * main stores PTR_TO_MAP_VALUE at fp-24, stores &fp-24 (an fp-derived + * pointer) at fp-8, passes &fp-8 through mid_fwd_spilled_ptr to + * load_ptr_deref_grandchild. The leaf loads the pointer from main's + * fp-8, then reads main's fp-24 through the loaded pointer. + * + * fill_from_stack() in arg_track_xfer() only handles local-frame + * FP-derived loads (src_is_local_fp check requires frame == depth). + * When a callee loads from a parent-frame pointer (frame < depth), + * the loaded value gets ARG_NONE instead of being recognized as + * fp-derived. Subsequent reads through that loaded pointer are + * invisible to liveness — nonlocal_use is never set for fp-24. + * + * clean_live_states() cleans the current state at every prune point. + * Because liveness misses fp-24, refined_caller_live_stack() tells + * __clean_func_state() that fp-24 is dead, which destroys the + * PTR_TO_MAP_VALUE spill before the grandchild can read it. + * The grandchild then reads STACK_INVALID → scalar, and the deref + * is rejected with "R0 invalid mem access 'scalar'" — even though + * fp-24 is genuinely live and holds a valid map pointer. + * + * This is a false positive: a valid program incorrectly rejected. + */ +SEC("socket") +__flag(BPF_F_TEST_STATE_FREQ) +__success +__naked void spilled_fp_cross_frame_deref(void) +{ + asm volatile ( + /* Map lookup to get PTR_TO_MAP_VALUE */ + "r1 = %[map] ll;" + "*(u32 *)(r10 - 32) = 0;" + "r2 = r10;" + "r2 += -32;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto l_exit%=;" + /* fp-24 = PTR_TO_MAP_VALUE */ + "*(u64 *)(r10 - 24) = r0;" + /* Store pointer to fp-24 at fp-8 */ + "r1 = r10;" + "r1 += -24;" + "*(u64 *)(r10 - 8) = r1;" + /* R1 = &fp-8: pointer to the spilled ptr */ + "r1 = r10;" + "r1 += -8;" + "call mid_fwd_spilled_ptr;" + "r0 = 0;" + "exit;" +"l_exit%=:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +/* Forwards R1 (ptr to main's fp-8, which holds &main_fp-24) to leaf */ +static __used __naked void mid_fwd_spilled_ptr(void) +{ + asm volatile ( + "call load_ptr_deref_grandchild;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * R1 = &main_fp-8 (where main stored ptr to fp-24) + * Loads the ptr from main's fp-8, reads main's fp-24 through it, + * then dereferences the result. + */ +static __used __naked void load_ptr_deref_grandchild(void) +{ + asm volatile ( + /* Load ptr from main's fp-8 → r2 = &main_fp-24 */ + "r2 = *(u64 *)(r1 + 0);" + /* Read main's fp-24 through loaded ptr */ + "r0 = *(u64 *)(r2 + 0);" + /* Dereference: safe for map_ptr */ + "r0 = *(u64 *)(r0 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * Exercise merge_nonlocal_live(). + * + * merge_shared_mid is analyzed twice (once from each wrapper), so the + * callsite within merge_shared_mid that calls merge_leaf_read gets its + * nonlocal_live info merged twice via merge_nonlocal_live(). + */ +SEC("socket") +__log_level(2) +__success +__msg("14: (85) call pc+2 r1: fp0-16") +__msg("17: (79) r0 = *(u64 *)(r1 +0) // r1=fp0-16") +__msg("14: (85) call pc+2 r1: fp0-8") +__msg("17: (79) r0 = *(u64 *)(r1 +0) // r1=fp0-8") +__msg("5: (85) call pc+{{.*}} ; use: fp0-8 fp0-16") +__naked void test_merge_nonlocal_live(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "*(u64 *)(r10 - 16) = r1;" + "r1 = r10;" + "r1 += -8;" + "call merge_wrapper_a;" + "r1 = r10;" + "r1 += -16;" + "call merge_wrapper_b;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void merge_wrapper_a(void) +{ + asm volatile ( + "call merge_shared_mid;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void merge_wrapper_b(void) +{ + asm volatile ( + "call merge_shared_mid;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void merge_shared_mid(void) +{ + asm volatile ( + "call merge_leaf_read;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void merge_leaf_read(void) +{ + asm volatile ( + "r0 = *(u64 *)(r1 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* Same bpf_loop instruction calls different callbacks depending on branch. */ +SEC("socket") +__log_level(2) +__success +__msg("call bpf_loop#181 ; use: fp2-8..-512 fp1-8..-512 fp0-8..-512") +__naked void bpf_loop_two_callbacks(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "*(u64 *)(r10 - 16) = r1;" + "r1 = r10;" + "r1 += -8;" + "call dyn_wrapper_a;" + "r1 = r10;" + "r1 += -16;" + "call dyn_wrapper_b;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void dyn_wrapper_a(void) +{ + asm volatile ( + "call mid_dynamic_cb;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void dyn_wrapper_b(void) +{ + asm volatile ( + "call mid_dynamic_cb;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void mid_dynamic_cb(void) +{ + asm volatile ( + "r6 = r1;" + "call %[bpf_get_prandom_u32];" + "if r0 == 0 goto 1f;" + "r2 = dyn_cb_a ll;" + "goto 2f;" + "1:" + "r2 = dyn_cb_b ll;" + "2:" + "r1 = 1;" + "r3 = r6;" /* ctx = fp-derived ptr from parent */ + "r4 = 0;" + "call %[bpf_loop];" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32), + __imm(bpf_loop) + : __clobber_all); +} + +/* Callback A/B: read parent stack through ctx */ +static __used __naked void dyn_cb_a(void) +{ + asm volatile ( + "r0 = *(u64 *)(r2 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void dyn_cb_b(void) +{ + asm volatile ( + "r0 = *(u64 *)(r2 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * Path A: r0 = map_lookup result (non-FP, ARG_NONE for stack tracking) + * Path B: r0 = fp-8 (FP-derived, frame=0, off=-8) + * At the join: r0 is not guaranteed to be a frame pointer. + */ +SEC("socket") +__log_level(2) +__msg("10: (79) r0 = *(u64 *)(r10 -8) // r0=fp0-8|fp0+0") +__naked void stack_or_non_stack_write(void) +{ + asm volatile ( + /* initial write to fp-8 */ + "*(u64 *)(r10 - 8) = 0;" + /* map lookup to get a non-FP pointer */ + "r2 = r10;" + "r2 += -4;" + "r1 = %[map] ll;" + "call %[bpf_map_lookup_elem];" + /* r0 = map_value (ARG_NONE) */ + "if r0 != 0 goto 1f;" + /* path B: r0 = fp-8 */ + "r0 = r10;" + "r0 += -8;" +"1:" + /* join: the write is not a def for fp[0]-8 */ + "*(u64 *)(r0 + 0) = 7;" + /* read fp-8: should be non-poisoned */ + "r0 = *(u64 *)(r10 - 8);" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(map) + : __clobber_all); +} + +SEC("socket") +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("subprog#2 write_first_read_second:") +__msg("17: (7a) *(u64 *)(r1 +0) = 42{{$}}") +__msg("18: (79) r0 = *(u64 *)(r2 +0) // r1=fp0-8 r2=fp0-16{{$}}") +__msg("stack use/def subprog#2 write_first_read_second (d2,cs15):") +__msg("17: (7a) *(u64 *)(r1 +0) = 42{{$}}") +__msg("18: (79) r0 = *(u64 *)(r2 +0) ; use: fp0-8 fp0-16") +__naked void shared_instance_must_write_overwrite(void) +{ + asm volatile ( + "r1 = 1;" + "*(u64 *)(r10 - 8) = r1;" + "*(u64 *)(r10 - 16) = r1;" + /* Call 1: write_first_read_second(&fp[-8], &fp[-16]) */ + "r1 = r10;" + "r1 += -8;" + "r2 = r10;" + "r2 += -16;" + "call forwarding_rw;" + /* Call 2: write_first_read_second(&fp[-16], &fp[-8]) */ + "r1 = r10;" + "r1 += -16;" + "r2 = r10;" + "r2 += -8;" + "call forwarding_rw;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void forwarding_rw(void) +{ + asm volatile ( + "call write_first_read_second;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void write_first_read_second(void) +{ + asm volatile ( + "*(u64 *)(r1 + 0) = 42;" + "r0 = *(u64 *)(r2 + 0);" + "exit;" + ::: __clobber_all); +} + +/* + * Shared must_write when (callsite, depth) instance is reused. + * Main calls fwd_to_stale_wr at two sites. fwd_to_stale_wr calls + * stale_wr_leaf at a single internal callsite. Both calls share + * stale_wr_leaf's (callsite, depth) instance. + * + * Call 1: stale_wr_leaf(map_value, fp-8) writes map, reads fp-8. + * Call 2: stale_wr_leaf(fp-8, fp-8) writes fp-8, reads fp-8. + * + * The analysis can't presume that stale_wr_leaf() always writes fp-8, + * it must conservatively join must_write masks computed for both calls. + */ +SEC("socket") +__success +__naked void stale_must_write_cross_callsite(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + /* Call 1: map_value write, fp-8 read (processed second in PO) */ + "*(u32 *)(r10 - 16) = 0;" + "r1 = %[map] ll;" + "r2 = r10;" + "r2 += -16;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 1f;" + "r1 = r0;" + "r2 = r10;" + "r2 += -8;" + "call fwd_to_stale_wr;" + /* Call 2: fp-8 write, fp-8 read (processed first in PO) */ + "r1 = r10;" + "r1 += -8;" + "r2 = r1;" + "call fwd_to_stale_wr;" +"1:" + "r0 = 0;" + "exit;" + :: __imm_addr(map), + __imm(bpf_map_lookup_elem) + : __clobber_all); +} + +static __used __naked void fwd_to_stale_wr(void) +{ + asm volatile ( + "call stale_wr_leaf;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void stale_wr_leaf(void) +{ + asm volatile ( + "*(u64 *)(r1 + 0) = 42;" + "r0 = *(u64 *)(r2 + 0);" + "exit;" + ::: __clobber_all); +} + +#ifdef CAN_USE_LOAD_ACQ_STORE_REL + +SEC("socket") +__log_level(2) +__success +__msg("*(u64 *)(r0 +0) = 42 ; def: fp0-16") +__naked void load_acquire_dont_clear_dst(void) +{ + asm volatile ( + "r0 = r10;" + "r0 += -16;" + "*(u64 *)(r0 + 0) = r0;" /* fp[-16] == &fp[-16] */ + ".8byte %[load_acquire_insn];" /* load_acquire is a special case for BPF_STX, */ + "r0 = *(u64 *)(r10 - 16);" /* it shouldn't clear tracking info for */ + "*(u64 *)(r0 + 0) = 42;" /* dst register, r0 in this case. */ + "r0 = 0;" + "exit;" + : + : __imm_insn(load_acquire_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_0, 0)) + : __clobber_all); +} + +#endif /* CAN_USE_LOAD_ACQ_STORE_REL */ + +SEC("socket") +__success +__naked void imprecise_fill_loses_cross_frame(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r1 = r10;" + "r1 += -8;" + "call imprecise_fill_cross_frame;" + "exit;" + ::: __clobber_all); +} + +static __used __naked void imprecise_fill_cross_frame(void) +{ + asm volatile ( + /* spill &caller_fp-8 to callee's fp-8 */ + "*(u64 *)(r10 - 8) = r1;" + /* imprecise FP pointer in r1 */ + "r1 = r10;" + "r2 = -8;" + "r1 += r2;" + /* load from imprecise offset. fill_from_stack returns + * ARG_IMPRECISE{mask=BIT(1)}, losing frame 0 + */ + "r1 = *(u64 *)(r1 + 0);" + /* read caller's fp-8 through loaded pointer, should mark fp0-8 live */ + "r0 = *(u64 *)(r1 + 0);" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Test that spill_to_stack with multi-offset dst (sz=8) joins instead + * of overwriting. r1 has offsets [-8, -16]. Both slots hold FP-derived + * pointers. Writing through r1 should join *val with existing values, + * not destroy them. + * + * fp-8 = &fp-24 + * fp-16 = &fp-32 + * r1 = fp-8 or fp-16 (two offsets from branch) + * *(u64 *)(r1 + 0) = &fp-24 -- writes to one slot, other untouched + * r0 = *(u64 *)(r10 - 16) -- fill from fp-16 + * r0 = *(u64 *)(r0 + 0) -- deref: should produce use + */ +SEC("socket") +__log_level(2) +__success +__msg("20: (79) r0 = *(u64 *)(r10 -16)") +__msg("21: (79) r0 = *(u64 *)(r0 +0) ; use: fp0-24 fp0-32") +__naked void spill_join_with_multi_off(void) +{ + asm volatile ( + /* fp-8 = &fp-24, fp-16 = &fp-32 (different pointers) */ + "*(u64 *)(r10 - 24) = 0;" + "*(u64 *)(r10 - 32) = 0;" + "r1 = r10;" + "r1 += -24;" + "*(u64 *)(r10 - 8) = r1;" + "r1 = r10;" + "r1 += -32;" + "*(u64 *)(r10 - 16) = r1;" + /* create r1 with two candidate offsets: fp-8 or fp-16 */ + "call %[bpf_get_prandom_u32];" + "if r0 == 0 goto 1f;" + "r1 = r10;" + "r1 += -8;" + "goto 2f;" +"1:" + "r1 = r10;" + "r1 += -16;" +"2:" + /* write &fp-24 through multi-offset r1: hits one slot, other untouched */ + "r2 = r10;" + "r2 += -24;" + "*(u64 *)(r1 + 0) = r2;" + /* read back *fp-8 and *fp-16 */ + "r0 = *(u64 *)(r10 - 8);" + "r0 = *(u64 *)(r0 + 0);" + "r0 = *(u64 *)(r10 - 16);" + "r0 = *(u64 *)(r0 + 0);" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Test that spill_to_stack with imprecise dst (off_cnt == 0, sz=8) + * joins instead of overwriting. Use "r2 = -8; r1 += r2" to make + * arg tracking lose offset precision while the main verifier keeps + * r1 as PTR_TO_STACK with fixed offset. Both slots hold FP-derived + * pointers. Writing through r1 should join *val with existing + * values, not destroy them. + * + * fp-8 = &fp-24 + * fp-16 = &fp-32 + * r1 = fp-8 (imprecise to arg tracking) + * *(u64 *)(r1 + 0) = &fp-24 -- since r1 is imprecise, this adds &fp-24 + * to the set of possible values for all slots, + * hence the values at fp-16 become [fp-24, fp-32] + * r0 = *(u64 *)(r10 - 16) + * r0 = *(u64 *)(r0 + 0) -- deref: should produce use of fp-24 or fp-32 + */ +SEC("socket") +__log_level(2) +__success +__msg("15: (79) r0 = *(u64 *)(r0 +0) ; use: fp0-24 fp0-32") +__naked void spill_join_with_imprecise_off(void) +{ + asm volatile ( + "*(u64 *)(r10 - 24) = 0;" + "*(u64 *)(r10 - 32) = 0;" + "r1 = r10;" + "r1 += -24;" + "*(u64 *)(r10 - 8) = r1;" + "r1 = r10;" + "r1 += -32;" + "*(u64 *)(r10 - 16) = r1;" + /* r1 = fp-8 but arg tracking sees off_cnt == 0 */ + "r1 = r10;" + "r2 = -8;" + "r1 += r2;" + /* write through imprecise r1 */ + "r3 = r10;" + "r3 += -24;" + "*(u64 *)(r1 + 0) = r3;" + /* read back fp-16: at_stack should still track &fp-32 */ + "r0 = *(u64 *)(r10 - 16);" + /* deref: should produce use for fp-32 */ + "r0 = *(u64 *)(r0 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * Same as spill_join_with_multi_off but the write is BPF_ST (store + * immediate) instead of BPF_STX. BPF_ST goes through + * clear_stack_for_all_offs() rather than spill_to_stack(), and that + * path also needs to join instead of overwriting. + * + * fp-8 = &fp-24 + * fp-16 = &fp-32 + * r1 = fp-8 or fp-16 (two offsets from branch) + * *(u64 *)(r1 + 0) = 0 -- BPF_ST with immediate + * r0 = *(u64 *)(r10 - 16) -- fill from fp-16 + * r0 = *(u64 *)(r0 + 0) -- deref: should produce use + */ +SEC("socket") +__log_level(2) +__failure +__msg("15: (7a) *(u64 *)(r1 +0) = 0 fp-8: fp0-24 -> fp0-24|fp0+0 fp-16: fp0-32 -> fp0-32|fp0+0") +__msg("17: (79) r0 = *(u64 *)(r0 +0) ; use: fp0-32") +__naked void st_imm_join_with_multi_off(void) +{ + asm volatile ( + "*(u64 *)(r10 - 24) = 0;" + "*(u64 *)(r10 - 32) = 0;" + "r1 = r10;" + "r1 += -24;" + "*(u64 *)(r10 - 8) = r1;" + "r1 = r10;" + "r1 += -32;" + "*(u64 *)(r10 - 16) = r1;" + /* create r1 with two candidate offsets: fp-8 or fp-16 */ + "call %[bpf_get_prandom_u32];" + "if r0 == 0 goto 1f;" + "r1 = r10;" + "r1 += -8;" + "goto 2f;" +"1:" + "r1 = r10;" + "r1 += -16;" +"2:" + /* BPF_ST: store immediate through multi-offset r1 */ + "*(u64 *)(r1 + 0) = 0;" + /* read back fp-16 and deref */ + "r0 = *(u64 *)(r10 - 16);" + "r0 = *(u64 *)(r0 + 0);" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * Check that BPF_ST with a known offset fully overwrites stack slot + * from the arg tracking point of view. + */ +SEC("socket") +__log_level(2) +__success +__msg("5: (7a) *(u64 *)(r1 +0) = 0 fp-8: fp0-16 -> _{{$}}") +__naked void st_imm_join_with_single_off(void) +{ + asm volatile ( + "r2 = r10;" + "r2 += -16;" + "*(u64 *)(r10 - 8) = r2;" + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r1 + 0) = 0;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * Same as spill_join_with_imprecise_off but the write is BPF_ST. + * Use "r2 = -8; r1 += r2" to make arg tracking lose offset + * precision while the main verifier keeps r1 as fixed-offset. + * + * fp-8 = &fp-24 + * fp-16 = &fp-32 + * r1 = fp-8 (imprecise to arg tracking) + * *(u64 *)(r1 + 0) = 0 -- BPF_ST with immediate + * r0 = *(u64 *)(r10 - 16) -- fill from fp-16 + * r0 = *(u64 *)(r0 + 0) -- deref: should produce use + */ +SEC("socket") +__log_level(2) +__success +__msg("13: (79) r0 = *(u64 *)(r0 +0) ; use: fp0-32") +__naked void st_imm_join_with_imprecise_off(void) +{ + asm volatile ( + "*(u64 *)(r10 - 24) = 0;" + "*(u64 *)(r10 - 32) = 0;" + "r1 = r10;" + "r1 += -24;" + "*(u64 *)(r10 - 8) = r1;" + "r1 = r10;" + "r1 += -32;" + "*(u64 *)(r10 - 16) = r1;" + /* r1 = fp-8 but arg tracking sees off_cnt == 0 */ + "r1 = r10;" + "r2 = -8;" + "r1 += r2;" + /* store immediate through imprecise r1 */ + "*(u64 *)(r1 + 0) = 0;" + /* read back fp-16 */ + "r0 = *(u64 *)(r10 - 16);" + /* deref: should produce use */ + "r0 = *(u64 *)(r0 + 0);" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * Test that spilling through an ARG_IMPRECISE pointer joins with + * existing at_stack values. Subprog receives r1 = fp0-24 and + * r2 = map_value, creates an ARG_IMPRECISE pointer by joining caller + * and callee FP on two branches. + * + * Setup: callee spills &fp1-16 to fp1-8 (precise, tracked). + * Then writes map_value through ARG_IMPRECISE r1 — on path A + * this hits fp1-8, on path B it hits caller stack. + * Since spill_to_stack is skipped for ARG_IMPRECISE dst, + * fp1-8 tracking isn't joined with none. + * + * Expected after the imprecise write: + * - arg tracking should show fp1-8 = fp1-16|fp1+0 (joined with none) + * - read from fp1-8 and deref should produce use for fp1-16 + * - write through it should NOT produce def for fp1-16 + */ +SEC("socket") +__log_level(2) +__success +__msg("26: (79) r0 = *(u64 *)(r10 -8) // r1=IMP3 r6=fp0-24 r7=fp1-16 fp-8=fp1-16|fp1+0") +__naked void imprecise_dst_spill_join(void) +{ + asm volatile ( + "*(u64 *)(r10 - 24) = 0;" + /* map lookup for a valid non-FP pointer */ + "*(u32 *)(r10 - 32) = 0;" + "r1 = %[map] ll;" + "r2 = r10;" + "r2 += -32;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 1f;" + /* r1 = &caller_fp-24, r2 = map_value */ + "r1 = r10;" + "r1 += -24;" + "r2 = r0;" + "call imprecise_dst_spill_join_sub;" +"1:" + "r0 = 0;" + "exit;" + :: __imm_addr(map), + __imm(bpf_map_lookup_elem) + : __clobber_all); +} + +static __used __naked void imprecise_dst_spill_join_sub(void) +{ + asm volatile ( + /* r6 = &caller_fp-24 (frame=0), r8 = map_value */ + "r6 = r1;" + "r8 = r2;" + /* spill &fp1-16 to fp1-8: at_stack[0] = fp1-16 */ + "*(u64 *)(r10 - 16) = 0;" + "r7 = r10;" + "r7 += -16;" + "*(u64 *)(r10 - 8) = r7;" + /* branch to create ARG_IMPRECISE pointer */ + "call %[bpf_get_prandom_u32];" + /* path B: r1 = caller fp-24 (frame=0) */ + "r1 = r6;" + "if r0 == 0 goto 1f;" + /* path A: r1 = callee fp-8 (frame=1) */ + "r1 = r10;" + "r1 += -8;" +"1:" + /* r1 = ARG_IMPRECISE{mask=BIT(0)|BIT(1)}. + * Write map_value (non-FP) through r1. On path A this overwrites fp1-8. + * Should join at_stack[0] with none: fp1-16|fp1+0. + */ + "*(u64 *)(r1 + 0) = r8;" + /* read fp1-8: should be fp1-16|fp1+0 (joined) */ + "r0 = *(u64 *)(r10 - 8);" + "*(u64 *)(r0 + 0) = 42;" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all); +} diff --git a/tools/testing/selftests/bpf/progs/verifier_liveness_exp.c b/tools/testing/selftests/bpf/progs/verifier_liveness_exp.c new file mode 100644 index 000000000000..72646fa2745e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_liveness_exp.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* + * Exponential complexity in analyze_subprog() liveness analysis. + * + * analyze_subprog() recurses into each call site that passes FP-derived + * arguments, creating a unique func_instance per (callsite, depth). + * There is no memoization for callees reached with equivalent entry args. + * Even if memoization were added, it can be defeated by passing a distinct + * FP offset at each call site. arg_track keys on (frame, off[]), so + * r1=fp-8, r1=fp-16, ... r1=fp-400 produce 50 unique cache keys per level. + * + * This test chains 8 subprograms (within the MAX_CALL_FRAMES limit). Each + * intermediate function calls the next one 50 times, each time with a + * different FP-relative offset in r1. + * + * Without complexity limits in analyze_subprog() the resulting 50^7 ~ 7.8 * 10^11 + * recursive analyze_subprog() calls will cause a CPU soft lockup or OOM. + * + * The BPF program itself is ~1200 instructions and perfectly valid. + */ + +char _license[] SEC("license") = "GPL"; + +/* Call fn with r1 = r10 + off (a unique FP-derived arg per call site) */ +#define C(fn, off) "r1 = r10;" \ + "r1 += -" #off ";" \ + "call " #fn ";" + +/* 50 calls, each with a distinct FP offset: -8, -16, ... -400 */ +#define CALLS_50(fn) \ + C(fn, 8) C(fn, 16) C(fn, 24) C(fn, 32) C(fn, 40) \ + C(fn, 48) C(fn, 56) C(fn, 64) C(fn, 72) C(fn, 80) \ + C(fn, 88) C(fn, 96) C(fn, 104) C(fn, 112) C(fn, 120) \ + C(fn, 128) C(fn, 136) C(fn, 144) C(fn, 152) C(fn, 160) \ + C(fn, 168) C(fn, 176) C(fn, 184) C(fn, 192) C(fn, 200) \ + C(fn, 208) C(fn, 216) C(fn, 224) C(fn, 232) C(fn, 240) \ + C(fn, 248) C(fn, 256) C(fn, 264) C(fn, 272) C(fn, 280) \ + C(fn, 288) C(fn, 296) C(fn, 304) C(fn, 312) C(fn, 320) \ + C(fn, 328) C(fn, 336) C(fn, 344) C(fn, 352) C(fn, 360) \ + C(fn, 368) C(fn, 376) C(fn, 384) C(fn, 392) C(fn, 400) + +/* Leaf: depth 7, no further calls */ +__naked __noinline __used +static unsigned long exp_sub7(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* depth 6 -> calls exp_sub7 x50 with distinct offsets */ +__naked __noinline __used +static unsigned long exp_sub6(void) +{ + asm volatile ( + CALLS_50(exp_sub7) + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* depth 5 -> calls exp_sub6 x50 */ +__naked __noinline __used +static unsigned long exp_sub5(void) +{ + asm volatile ( + CALLS_50(exp_sub6) + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* depth 4 -> calls exp_sub5 x50 */ +__naked __noinline __used +static unsigned long exp_sub4(void) +{ + asm volatile ( + CALLS_50(exp_sub5) + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* depth 3 -> calls exp_sub4 x50 */ +__naked __noinline __used +static unsigned long exp_sub3(void) +{ + asm volatile ( + CALLS_50(exp_sub4) + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* depth 2 -> calls exp_sub3 x50 */ +__naked __noinline __used +static unsigned long exp_sub2(void) +{ + asm volatile ( + CALLS_50(exp_sub3) + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* depth 1 -> calls exp_sub2 x50 */ +__naked __noinline __used +static unsigned long exp_sub1(void) +{ + asm volatile ( + CALLS_50(exp_sub2) + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * Entry: depth 0. Calls exp_sub1 50 times, each with a distinct + * FP offset in r1. Every call site produces a unique arg_track, + * defeating any memoization keyed on entry args. + */ +SEC("?raw_tp") +__failure __log_level(2) +__msg("liveness analysis exceeded complexity limit") +__naked int liveness_exponential_complexity(void) +{ + asm volatile ( + CALLS_50(exp_sub1) + "r0 = 0;" + "exit;" + ::: __clobber_all); +} diff --git a/tools/testing/selftests/bpf/progs/verifier_load_acquire.c b/tools/testing/selftests/bpf/progs/verifier_load_acquire.c index 74f4f19c10b8..d17026d7480d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_load_acquire.c +++ b/tools/testing/selftests/bpf/progs/verifier_load_acquire.c @@ -3,6 +3,7 @@ #include <linux/bpf.h> #include <bpf/bpf_helpers.h> +#include <bpf/bpf_core_read.h> #include "../../../include/linux/filter.h" #include "bpf_misc.h" @@ -148,6 +149,22 @@ __naked void load_acquire_from_ctx_pointer(void) : __clobber_all); } +SEC("socket") +__description("load-acquire from ctx pointer, same dst and src register") +__failure __failure_unpriv __msg("BPF_ATOMIC loads from R6 ctx is not allowed") +__naked void load_acquire_ctx_same_dst_src(void) +{ + asm volatile ( + "r6 = r1;" + ".8byte %[load_acquire_insn];" // w6 = load_acquire((u32 *)(r6 + 0)); + "r0 = 0;" + "exit;" + : + : __imm_insn(load_acquire_insn, + BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_6, BPF_REG_6, 0)) + : __clobber_all); +} + SEC("xdp") __description("load-acquire from pkt pointer") __failure __msg("BPF_ATOMIC loads from R2 pkt is not allowed") @@ -206,6 +223,33 @@ __naked void load_acquire_from_sock_pointer(void) } SEC("socket") +__description("load-acquire from rdonly_untrusted_mem pointer") +__failure __msg("BPF_ATOMIC loads from R{{[0-9]+}} rdonly_untrusted_mem is not allowed") +int load_acquire_from_rdonly_untrusted_mem(void *ctx) +{ + __u64 val = 0; + void *p; + + /* + * bpf_rdonly_cast(x, 0) yields PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED. + * A regular BPF_LDX from it is rewritten to BPF_PROBE_MEM, but a + * load-acquire is not, so it must be rejected, otherwise the JIT emits + * a plain load with no exception table entry and a fault would crash + * the kernel. + */ + p = bpf_rdonly_cast(&val, 0); + asm volatile ( + "r1 = %[p];" + ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r1 + 0)); + : + : [p] "r" (p), + __imm_insn(load_acquire_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0)) + : "r0", "r1"); + return 0; +} + +SEC("socket") __description("load-acquire with invalid register R15") __failure __failure_unpriv __msg("R15 is invalid") __naked void load_acquire_with_invalid_reg(void) diff --git a/tools/testing/selftests/bpf/progs/verifier_loops1.c b/tools/testing/selftests/bpf/progs/verifier_loops1.c index fbdde80e7b90..48a966cda199 100644 --- a/tools/testing/selftests/bpf/progs/verifier_loops1.c +++ b/tools/testing/selftests/bpf/progs/verifier_loops1.c @@ -138,8 +138,7 @@ l0_%=: exit; \ SEC("tracepoint") __description("bounded recursion") __failure -/* verifier limitation in detecting max stack depth */ -__msg("the call stack of 8 frames is too deep !") +__msg("recursive call from") __naked void bounded_recursion(void) { asm volatile (" \ @@ -304,4 +303,40 @@ __naked void maybe_exit_scc_bug1(void) ::: __clobber_all); } +/* + * The loop reads zero from the caller's stack on its first iteration and + * one from the callee's stack on its second iteration. At the loop header, + * only the frame number of the pointer in r1 changes. + */ +static __naked __noinline __used +void loop_stack_frames_reg(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 1;" +"1:" + "r0 = *(u64 *)(r1 + 0);" + "if r0 != 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "goto 1b;" +"2:" + "exit;" + ::: __clobber_all); +} + +SEC("xdp") +__description("bounded loop changing stack frame in a register") +__success __retval(1) +__flag(BPF_F_TEST_STATE_FREQ) +__naked void bounded_loop_stack_frames_reg(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r1 = r10;" + "r1 += -8;" + "call loop_stack_frames_reg;" + "exit;" + ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_lsm.c b/tools/testing/selftests/bpf/progs/verifier_lsm.c index 38e8e9176862..c724bf389f5c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_lsm.c +++ b/tools/testing/selftests/bpf/progs/verifier_lsm.c @@ -188,4 +188,28 @@ int BPF_PROG(null_check, struct file *file) return 0; } +SEC("lsm_cgroup/file_open") +__description("sleepable lsm_cgroup program is rejected") +__failure __msg("Program of this type cannot be sleepable") +__flag(BPF_F_SLEEPABLE) +int BPF_PROG(sleepable_lsm_cgroup) +{ + return 0; +} + +SEC("lsm/file_mprotect") +__description("lsm retval load must reset stale register bounds") +__failure __msg("div by zero") +__naked int retval_load_resets_bounds(void *ctx) +{ + asm volatile ( + "r6 = 0;" + "r6 = *(u64 *)(r1 + 24);" + "if r6 == 0 goto +1;" + "r6 /= 0;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_map_in_map.c b/tools/testing/selftests/bpf/progs/verifier_map_in_map.c index 16b761e510f0..d3be69a9a755 100644 --- a/tools/testing/selftests/bpf/progs/verifier_map_in_map.c +++ b/tools/testing/selftests/bpf/progs/verifier_map_in_map.c @@ -18,6 +18,20 @@ struct { }); } map_in_map SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __uint(max_entries, 1); + __type(key, int); + __type(value, int); + __array(values, struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(map_flags, BPF_F_INNER_MAP); + __uint(max_entries, 8); + __type(key, int); + __type(value, long); + }); +} map_in_map_dyn SEC(".maps"); + SEC("socket") __description("map in map access") __success __success_unpriv __retval(0) @@ -45,6 +59,32 @@ l0_%=: r0 = 0; \ : __clobber_all); } +SEC("socket") +__description("map in map dynamic inner array lookup is nullable") +__failure __msg("invalid mem access 'map_value_or_null'") +__naked void map_in_map_dynamic_inner_array_lookup_is_nullable(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u32*)(r10 - 4) = r1; \ + r2 = r10; \ + r2 += -4; \ + r1 = %[map_in_map_dyn] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + *(u32*)(r10 - 8) = 4; \ + r2 = r10; \ + r2 += -8; \ + r1 = r0; \ + call %[bpf_map_lookup_elem]; \ + r0 = *(u64 *)(r0 + 0); \ +l0_%=: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_in_map_dyn) + : __clobber_all); +} + SEC("xdp") __description("map in map state pruning") __success __msg("processed 15 insns") @@ -114,7 +154,8 @@ l0_%=: r0 = 0; \ SEC("socket") __description("forgot null checking on the inner map pointer") -__failure __msg("R1 type=map_value_or_null expected=map_ptr") +__failure __msg("R1 type=map_ptr_or_null expected=map_ptr") +__msg("map_ptr_or_null, but this argument accepts map_ptr") __failure_unpriv __naked void on_the_inner_map_pointer(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c b/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c new file mode 100644 index 000000000000..c01abf54923d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +struct inner_map { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, int); +} inner_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __uint(max_entries, 1); + __type(key, int); + __array(values, struct inner_map); +} outer_map SEC(".maps") = { + .values = { [0] = &inner_map }, +}; + +SEC("?tc") +__failure __msg("type=map_ptr_or_null expected=fp") +int mapofmaps_value_as_kfunc_mem_buf(struct __sk_buff *skb) +{ + struct bpf_dynptr dptr; + __u32 key = 0; + void *inner; + char *p; + + inner = bpf_map_lookup_elem(&outer_map, &key); + /* intentionally NOT NULL-checked: type is map_ptr_or_null */ + + bpf_dynptr_from_skb(skb, 0, &dptr); + /* arg3 is mem+size */ + p = bpf_dynptr_slice(&dptr, 0, inner, 4); + if (p) + return p[0]; + return 0; +} + +SEC("?tc") +__failure __msg("type=map_ptr_or_null expected=fp") +int mapofmaps_value_as_helper_mem_buf(struct __sk_buff *skb) +{ + __u32 key = 0; + void *inner; + + inner = bpf_map_lookup_elem(&outer_map, &key); + /* intentionally NOT NULL-checked: type is map_ptr_or_null */ + + /* arg1 is mem+size */ + return bpf_csum_diff(inner, 4, NULL, 0, 0) + skb->len; +} + +SEC("?tc") +__failure __msg("type=map_ptr_or_null expected=fp") +int mapofmaps_value_as_helper_fixed_mem(struct __sk_buff *skb) +{ + char th[sizeof(struct tcphdr)] = {}; + __u32 key = 0; + void *inner; + + inner = bpf_map_lookup_elem(&outer_map, &key); + /* intentionally NOT NULL-checked: type is map_ptr_or_null */ + + /* arg1 is fixed-sized mem */ + return bpf_tcp_raw_check_syncookie_ipv4(inner, (void *)th); +} diff --git a/tools/testing/selftests/bpf/progs/verifier_map_ptr.c b/tools/testing/selftests/bpf/progs/verifier_map_ptr.c index e2767d27d8aa..e0a65835c861 100644 --- a/tools/testing/selftests/bpf/progs/verifier_map_ptr.c +++ b/tools/testing/selftests/bpf/progs/verifier_map_ptr.c @@ -70,13 +70,17 @@ __naked void bpf_map_ptr_write_rejected(void) : __clobber_all); } -/* The first element of struct bpf_map is a SHA256 hash of 32 bytes, accessing - * into this array is valid. The opts field is now at offset 33. +/* + * struct bpf_map starts with the SHA256 hash sha[32] at offset 0 (a readable + * byte array), followed by the ops pointer at offset 32 and the inner_map_meta + * pointer at offset 40. Reading a u32 at offset 41 reaches into the middle of + * the inner_map_meta pointer, i.e. a partial pointer access, which is + * rejected. */ SEC("socket") __description("bpf_map_ptr: read non-existent field rejected") __failure -__msg("cannot access ptr member ops with moff 32 in struct bpf_map with off 33 size 4") +__msg("cannot access ptr member inner_map_meta with moff 40 in struct bpf_map with off 41 size 4") __failure_unpriv __msg_unpriv("access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") __flag(BPF_F_ANY_ALIGNMENT) @@ -85,7 +89,32 @@ __naked void read_non_existent_field_rejected(void) asm volatile (" \ r6 = 0; \ r1 = %[map_array_48b] ll; \ - r6 = *(u32*)(r1 + 33); \ + r6 = *(u32*)(r1 + 41); \ + r0 = 1; \ + exit; \ +" : + : __imm_addr(map_array_48b) + : __clobber_all); +} + +/* + * The sha byte array spans offsets 0..31 (mend 32). Reading a u32 at offset + * 30 starts inside sha but extends past its end, which the verifier rejects + * as an out-of-bounds scalar access. + */ +SEC("socket") +__description("bpf_map_ptr: read beyond sha field rejected") +__failure +__msg("access beyond the end of member sha (mend:32) in struct bpf_map with off 30 size 4") +__failure_unpriv +__msg_unpriv("access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") +__flag(BPF_F_ANY_ALIGNMENT) +__naked void read_beyond_sha_field_rejected(void) +{ + asm volatile (" \ + r6 = 0; \ + r1 = %[map_array_48b] ll; \ + r6 = *(u32*)(r1 + 30); \ r0 = 1; \ exit; \ " : @@ -103,7 +132,7 @@ __naked void ptr_read_ops_field_accepted(void) asm volatile (" \ r6 = 0; \ r1 = %[map_array_48b] ll; \ - r6 = *(u64*)(r1 + 0); \ + r6 = *(u64*)(r1 + 32); \ r0 = 1; \ exit; \ " : diff --git a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c index 6d1edaef9213..db7e30da234f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c +++ b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c @@ -11,6 +11,8 @@ __description("may_goto 0") __arch_x86_64 __arch_s390x __arch_arm64 +__arch_riscv64 +__arch_loongarch __xlated("0: r0 = 1") __xlated("1: exit") __success @@ -31,6 +33,8 @@ __description("batch 2 of may_goto 0") __arch_x86_64 __arch_s390x __arch_arm64 +__arch_riscv64 +__arch_loongarch __xlated("0: r0 = 1") __xlated("1: exit") __success @@ -53,6 +57,8 @@ __description("may_goto batch with offsets 2/1/0") __arch_x86_64 __arch_s390x __arch_arm64 +__arch_riscv64 +__arch_loongarch __xlated("0: r0 = 1") __xlated("1: exit") __success @@ -79,15 +85,17 @@ __description("may_goto batch with offsets 2/0") __arch_x86_64 __arch_s390x __arch_arm64 +__arch_riscv64 +__arch_loongarch __xlated("0: *(u64 *)(r10 -16) = 65535") __xlated("1: *(u64 *)(r10 -8) = 0") -__xlated("2: r11 = *(u64 *)(r10 -16)") -__xlated("3: if r11 == 0x0 goto pc+6") -__xlated("4: r11 -= 1") -__xlated("5: if r11 != 0x0 goto pc+2") -__xlated("6: r11 = -16") +__xlated("2: r12 = *(u64 *)(r10 -16)") +__xlated("3: if r12 == 0x0 goto pc+6") +__xlated("4: r12 -= 1") +__xlated("5: if r12 != 0x0 goto pc+2") +__xlated("6: r12 = -16") __xlated("7: call unknown") -__xlated("8: *(u64 *)(r10 -16) = r11") +__xlated("8: *(u64 *)(r10 -16) = r12") __xlated("9: r0 = 1") __xlated("10: r0 = 2") __xlated("11: exit") @@ -106,4 +114,63 @@ __naked void may_goto_batch_2(void) : __clobber_all); } +/* + * Use bpf_get_prandom_u32() to prevent DCE from removing the checks. + * retval: 0=all ok, 1-6=R0-R5 clobbered. + */ +SEC("syscall") +__description("timed may_goto preserves R0-R5") +__arch_x86_64 +__arch_s390x +__arch_arm64 +__arch_riscv64 +__arch_loongarch +__success +__retval(0) +__naked void timed_may_goto_preserves_regs(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r0 = 0x1111;" + "r0 += r6;" + "r1 = 0x2222;" + "r1 += r6;" + "r2 = 0x3333;" + "r2 += r6;" + "r3 = 0x4444;" + "r3 += r6;" + "r4 = 0x5555;" + "r4 += r6;" + "r5 = 0x6666;" + "r5 += r6;" + ".8byte %[may_goto];" + ".8byte %[loop];" + "r0 -= r6;" + "r1 -= r6;" + "r2 -= r6;" + "r3 -= r6;" + "r4 -= r6;" + "r5 -= r6;" + "if r0 != 0x1111 goto 1f;" + "if r1 != 0x2222 goto 2f;" + "if r2 != 0x3333 goto 3f;" + "if r3 != 0x4444 goto 4f;" + "if r4 != 0x5555 goto 5f;" + "if r5 != 0x6666 goto 6f;" + "r0 = 0;" + "exit;" + "1: r0 = 1; exit;" + "2: r0 = 2; exit;" + "3: r0 = 3; exit;" + "4: r0 = 4; exit;" + "5: r0 = 5; exit;" + "6: r0 = 6; exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 1, 0)), + __imm_insn(loop, BPF_RAW_INSN(BPF_JMP | BPF_JA, 0, 0, -2, 0)) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_mem_size_reg.c b/tools/testing/selftests/bpf/progs/verifier_mem_size_reg.c new file mode 100644 index 000000000000..7e24706a764e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_mem_size_reg.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +/* + * The __szk size of a kfunc memory/size pair must be marked precise even when + * the nullable buffer is passed as NULL. + */ +SEC("?tc") +__success __log_level(2) +__msg("mark_precise: frame0: regs=r4 stack= before") +int dynptr_slice_null_buf_size_precise(struct __sk_buff *skb) +{ + struct bpf_dynptr dptr; + char *p; + + bpf_dynptr_from_skb(skb, 0, &dptr); + + p = bpf_dynptr_slice(&dptr, 0, NULL, 8); + if (p) + return p[0]; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/verifier_meta_access.c b/tools/testing/selftests/bpf/progs/verifier_meta_access.c index d81722fb5f19..62235f032ffe 100644 --- a/tools/testing/selftests/bpf/progs/verifier_meta_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_meta_access.c @@ -27,7 +27,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("meta access, test2") -__failure __msg("invalid access to packet, off=-8") +__failure __msg("R0 min value is negative") __naked void meta_access_test2(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/progs/verifier_netfilter_ctx.c b/tools/testing/selftests/bpf/progs/verifier_netfilter_ctx.c index e2cbc5bda65e..b5d7f567d0d4 100644 --- a/tools/testing/selftests/bpf/progs/verifier_netfilter_ctx.c +++ b/tools/testing/selftests/bpf/progs/verifier_netfilter_ctx.c @@ -113,4 +113,82 @@ int with_valid_ctx_access_test6(struct bpf_nf_ctx *ctx) return th->dest == bpf_htons(22) ? NF_ACCEPT : NF_DROP; } +SEC("netfilter") +__description("netfilter test prog with skb write access") +__failure __msg("only read is supported") +int skb_len_write(struct bpf_nf_ctx *ctx) +{ + ctx->skb->len = 1; + return 1; +} + +SEC("netfilter") +__description("netfilter test prog with skb data write access") +__failure __msg("cannot write into rdonly_untrusted_mem") +int skb_data_write(struct bpf_nf_ctx *ctx) +{ + ctx->skb->data[0] = 0; + return 1; +} + +SEC("netfilter") +__description("netfilter test prog with bpf_dynptr_write") +__success __failure_unpriv +__retval(0) +int with_dynptr_write(struct bpf_nf_ctx *ctx) +{ + struct __sk_buff *skb = (struct __sk_buff *)ctx->skb; + struct bpf_dynptr ptr; + u8 buffer[1] = {}; + + if (bpf_dynptr_from_skb(skb, 0, &ptr)) + return 1; + + if (bpf_dynptr_write(&ptr, 0, buffer, sizeof(buffer), 0)) + return 0; /* must always fail */ + + return 1; +} + +SEC("netfilter") +__description("netfilter test prog with bpf_dynptr_slice_rdwr") +__failure __msg("the prog does not allow writes to packet data") +int with_dynptr_rdwr(struct bpf_nf_ctx *ctx) +{ + struct __sk_buff *skb = (struct __sk_buff *)ctx->skb; + u8 buffer_iph[20] = {}; + struct bpf_dynptr ptr; + struct iphdr *iph; + + if (bpf_dynptr_from_skb(skb, 0, &ptr)) + return 1; + + iph = bpf_dynptr_slice_rdwr(&ptr, 0, buffer_iph, sizeof(buffer_iph)); + if (!iph) + return 0; + + return 1; +} + +SEC("netfilter") +__description("netfilter test prog with bpf_dynptr_slice + write") +__failure __msg("cannot write into rdonly_mem") +int with_dynptr_store(struct bpf_nf_ctx *ctx) +{ + struct __sk_buff *skb = (struct __sk_buff *)ctx->skb; + u8 buffer_iph[20] = {}; + struct bpf_dynptr ptr; + struct iphdr *iph; + + if (bpf_dynptr_from_skb(skb, 0, &ptr)) + return 1; + + iph = bpf_dynptr_slice(&ptr, 0, buffer_iph, sizeof(buffer_iph)); + if (!iph) + return 0; + iph->protocol = 42; + + return 1; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c b/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c new file mode 100644 index 000000000000..967f4e6e3a49 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +#if defined(__TARGET_ARCH_x86) + +int percpu_data SEC(".percpu"); + +/* + * An ld_imm64 of a per-CPU map value is followed by a mov_percpu_addr that + * reuses the same register, so check that the add resolves into the register + * the address was loaded into, for every register. + */ +SEC("raw_tp") +__description("per-CPU address resolution") +__success +__arch_x86_64 +__jited(" movabsq $0x{{.*}}, %rax") +__jited(" addq %gs:{{.*}}, %rax") +__jited(" movabsq $0x{{.*}}, %rdi") +__jited(" addq %gs:{{.*}}, %rdi") +__jited(" movabsq $0x{{.*}}, %rsi") +__jited(" addq %gs:{{.*}}, %rsi") +__jited(" movabsq $0x{{.*}}, %rdx") +__jited(" addq %gs:{{.*}}, %rdx") +__jited(" movabsq $0x{{.*}}, %rcx") +__jited(" addq %gs:{{.*}}, %rcx") +__jited(" movabsq $0x{{.*}}, %r8") +__jited(" addq %gs:{{.*}}, %r8") +__jited(" movabsq $0x{{.*}}, %rbx") +__jited(" addq %gs:{{.*}}, %rbx") +__jited(" movabsq $0x{{.*}}, %r13") +__jited(" addq %gs:{{.*}}, %r13") +__jited(" movabsq $0x{{.*}}, %r14") +__jited(" addq %gs:{{.*}}, %r14") +__jited(" movabsq $0x{{.*}}, %r15") +__jited(" addq %gs:{{.*}}, %r15") +__naked void percpu_addr(void) +{ + asm volatile (" \ + r0 = %[percpu_data] ll; \ + r1 = %[percpu_data] ll; \ + r2 = %[percpu_data] ll; \ + r3 = %[percpu_data] ll; \ + r4 = %[percpu_data] ll; \ + r5 = %[percpu_data] ll; \ + r6 = %[percpu_data] ll; \ + r7 = %[percpu_data] ll; \ + r8 = %[percpu_data] ll; \ + r9 = %[percpu_data] ll; \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(percpu_data) + : __clobber_all); +} + +#else + +SEC("raw_tp") +__description("percpu addr dummy") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_precision.c b/tools/testing/selftests/bpf/progs/verifier_precision.c index 4794903aec8e..f4459561bf39 100644 --- a/tools/testing/selftests/bpf/progs/verifier_precision.c +++ b/tools/testing/selftests/bpf/progs/verifier_precision.c @@ -2,8 +2,10 @@ /* Copyright (C) 2023 SUSE LLC */ #include <linux/bpf.h> #include <bpf/bpf_helpers.h> +#include <stdbool.h> #include "../../../include/linux/filter.h" #include "bpf_misc.h" +#include "bpf_kfuncs.h" struct { __uint(type, BPF_MAP_TYPE_ARRAY); @@ -75,8 +77,8 @@ __naked int bpf_end_to_be(void) #if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ - defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390)) && \ - __clang_major__ >= 18 + defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ + defined(__TARGET_ARCH_loongarch)) && __clang_major__ >= 18 SEC("?raw_tp") __success __log_level(2) @@ -642,4 +644,102 @@ __naked int bpf_atomic_cmpxchg_32bit_precision(void) : __clobber_all); } +/* + * Verification takes two paths: with r1 being scalar zero on path (1) + * and with r1 being some other scalar on path (2). + * Check that the verifier does not use checkpoints created + * on path (1) to prune path (2). + */ +SEC("?tc") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("R1 type=scalar expected=fp") +__naked int null_mem_arg_zero_size(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 42;" + "if r0 > 42 goto 1f;" + "r1 = 0;" + "1:" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + /* + * ARG_PTR_TO_MEM | PTR_MAYBE_NULL parameter can be NULL, + * but can't be some other scalar value. + */ + "call %[bpf_csum_diff];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_csum_diff) + : __clobber_all); +} + +__weak int subprog_mem_arg(int *p) +{ + if (p) + return *p; + return 0; +} + +/* + * Verification takes two paths: with r1 being scalar zero on path (1) + * and with r1 being some other scalar on path (2). + * Check that the verifier does not use checkpoints created + * on path (1) to prune path (2). + */ +SEC("?raw_tp") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("R1 type=scalar expected=fp") +__naked int null_mem_arg_global_subprog(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 42;" + "if r0 > 42 goto 1f;" + "r1 = 0;" + "1:" + "call subprog_mem_arg;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Same as above, check that path with r3 == 0 does not prune the path with r3 != 0 */ +SEC("?tc") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("R3 type=scalar expected=fp") +int null_kfunc_arg_dynptr_slice(struct __sk_buff *skb) +{ + struct bpf_dynptr ptr; + + bpf_dynptr_from_skb(skb, 0, &ptr); + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r3 = 42;" + "if r0 > 42 goto 1f;" + "r3 = 0;" + "1:" + "r1 = %[ptr];" + "r2 = 0;" + "r4 = 8;" + "call %[bpf_dynptr_slice];" + : + : __imm_ptr(ptr), + __imm(bpf_get_prandom_u32), + __imm(bpf_dynptr_slice) + : __clobber_common); + return 0; +} + +void __kfunc_btf_root(void) +{ + bpf_dynptr_slice(0, 0, 0, 0); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_private_stack.c b/tools/testing/selftests/bpf/progs/verifier_private_stack.c index 1ecd34ebde19..ea0a7e73331d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_private_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_private_stack.c @@ -86,6 +86,9 @@ __naked static void cumulative_stack_depth_subprog(void) SEC("kprobe") __description("Private stack, subtree > MAX_BPF_STACK") __success +__log_level(4) __msg("stack depth max 512") +__msg("subprog 0 (private_stack_nested_1) main {{.*}} stack 512") +__msg("subprog 1 (cumulative_stack_depth_subprog) static {{.*}} stack 32") __arch_x86_64 /* private stack fp for the main prog */ __jited(" movabsq $0x{{.*}}, %r9") @@ -93,6 +96,7 @@ __jited(" addq %gs:{{.*}}, %r9") __jited(" movl $0x2a, %edi") __jited(" movq %rdi, -0x200(%r9)") __jited(" pushq %r9") +__jited("...") __jited(" callq 0x{{.*}}") __jited(" popq %r9") __jited(" xorl %eax, %eax") @@ -152,11 +156,13 @@ __jited(" endbr64") __jited(" movabsq $0x{{.*}}, %r9") __jited(" addq %gs:{{.*}}, %r9") __jited(" pushq %r9") +__jited("...") __jited(" callq") __jited(" popq %r9") __jited(" movl $0x2a, %edi") __jited(" movq %rdi, -0x200(%r9)") __jited(" pushq %r9") +__jited("...") __jited(" callq") __jited(" popq %r9") __arch_arm64 @@ -170,12 +176,12 @@ __jited(" mrs x10, TPIDR_EL{{[0-1]}}") __jited(" add x27, x27, x10") __jited(" add x25, x27, {{.*}}") __jited(" bl 0x{{.*}}") -__jited(" add x7, x0, #0x0") +__jited(" mov x8, x0") __jited(" mov x0, #0x2a") __jited(" str x0, [x27]") __jited(" bl 0x{{.*}}") -__jited(" add x7, x0, #0x0") -__jited(" mov x7, #0x0") +__jited(" mov x8, x0") +__jited(" mov x8, #0x0") __jited(" ldp x25, x27, [sp], {{.*}}") __naked void private_stack_callback(void) { @@ -198,6 +204,7 @@ __description("Private stack, exception in main prog") __success __retval(0) __arch_x86_64 __jited(" pushq %r9") +__jited("...") __jited(" callq") __jited(" popq %r9") __arch_arm64 @@ -220,7 +227,7 @@ __jited(" mov x0, #0x2a") __jited(" str x0, [x27]") __jited(" mov x0, #0x0") __jited(" bl 0x{{.*}}") -__jited(" add x7, x0, #0x0") +__jited(" mov x8, x0") __jited(" ldp x27, x28, [sp], #0x10") int private_stack_exception_main_prog(void) { @@ -245,6 +252,7 @@ __success __retval(0) __arch_x86_64 __jited(" movq %rdi, -0x200(%r9)") __jited(" pushq %r9") +__jited("...") __jited(" callq") __jited(" popq %r9") __arch_arm64 @@ -258,7 +266,7 @@ __jited(" add x25, x27, {{.*}}") __jited(" mov x0, #0x2a") __jited(" str x0, [x27]") __jited(" bl 0x{{.*}}") -__jited(" add x7, x0, #0x0") +__jited(" mov x8, x0") __jited(" ldp x27, x28, [sp], #0x10") int private_stack_exception_sub_prog(void) { @@ -324,6 +332,12 @@ int private_stack_async_callback_1(void) SEC("fentry/bpf_fentry_test9") __description("Private stack, async callback, potential nesting") __success __retval(0) +__load_if_JITed() +__log_level(4) __msg("stack depth max 272") +__msg("subprog 0 (private_stack_async_callback_2) main {{.*}} stack 8") +__msg("subprog 1 (timer_cb1) static {{.*}} stack 0") +__msg("subprog 2 (subprog1) static {{.*}} stack 256") +__msg("subprog 3 (subprog2) static {{.*}} stack 0") __arch_x86_64 __jited(" subq $0x100, %rsp") __arch_arm64 @@ -344,6 +358,21 @@ int private_stack_async_callback_2(void) return 0; } +SEC("fentry/bpf_fentry_test9") +__description("private stack, max stack depth is private stack") +__success +__log_level(4) __msg("stack depth max 256") +__msg("subprog 0 (private_stack_max_depth) main {{.*}} stack 8") +__msg("subprog 1 (subprog1) static insns_self {{[0-9]+}} insns_total {{[0-9]+}} stack 256") +__msg("subprog 2 (subprog2) static insns_self {{[0-9]+}} insns_total {{[0-9]+}} stack 0") +int private_stack_max_depth(void) +{ + int x = 0; + + subprog1(&x); + return 0; +} + #else SEC("kprobe") diff --git a/tools/testing/selftests/bpf/progs/verifier_ptr_to_buf.c b/tools/testing/selftests/bpf/progs/verifier_ptr_to_buf.c new file mode 100644 index 000000000000..12cf24db46a0 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_ptr_to_buf.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +SEC("iter/bpf_map_elem") +__description("PTR_TO_BUF: reject negative const offset") +__failure +__msg("invalid negative rdwr buffer offset") +__naked void ptr_to_buf_reject_negative_const_offset(void) +{ + asm volatile ("r0 = 0; \ + r2 = *(u64 *)(r1 + %[value_off]); \ + if r2 == 0 goto l0_%=; \ + r2 += -8; \ + r0 = *(u64 *)(r2 + 0); \ +l0_%=: \ + exit; \ + " + : + : __imm_const(value_off, + offsetof(struct bpf_iter__bpf_map_elem, value)) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_raw_tp_writable.c b/tools/testing/selftests/bpf/progs/verifier_raw_tp_writable.c index 14a0172e2141..4055a6443bc2 100644 --- a/tools/testing/selftests/bpf/progs/verifier_raw_tp_writable.c +++ b/tools/testing/selftests/bpf/progs/verifier_raw_tp_writable.c @@ -47,4 +47,20 @@ l0_%=: /* shift the buffer pointer to a variable location */\ : __clobber_all); } +SEC("raw_tracepoint.w") +__description("raw_tracepoint_writable: reject negative const offset") +__failure +__msg("invalid negative tracepoint buffer offset") +__naked void tracepoint_writable_reject_negative_const_offset(void) +{ + asm volatile (" \ + r6 = *(u64 *)(r1 + 0); \ + r6 += -8; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" : + : + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c b/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c index 910365201f68..199ad18f8eb5 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c +++ b/tools/testing/selftests/bpf/progs/verifier_ref_tracking.c @@ -263,7 +263,7 @@ l0_%=: r0 = 0; \ SEC("lsm.s/bpf") __description("reference tracking: release user key reference without check") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void user_key_reference_without_check(void) { asm volatile (" \ @@ -282,7 +282,7 @@ __naked void user_key_reference_without_check(void) SEC("lsm.s/bpf") __description("reference tracking: release system key reference without check") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void system_key_reference_without_check(void) { asm volatile (" \ @@ -300,7 +300,7 @@ __naked void system_key_reference_without_check(void) SEC("lsm.s/bpf") __description("reference tracking: release with NULL key pointer") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") __naked void release_with_null_key_pointer(void) { asm volatile (" \ @@ -1288,7 +1288,7 @@ l1_%=: r1 = r6; \ SEC("tc") __description("reference tracking: bpf_sk_release(listen_sk)") -__failure __msg("R1 must be referenced when passed to release function") +__failure __msg("release helper bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") __naked void bpf_sk_release_listen_sk(void) { asm volatile ( diff --git a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c index 58c7704d61cd..256547048cc4 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c +++ b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c @@ -4,6 +4,13 @@ #include <bpf/bpf_helpers.h> #include "bpf_misc.h" +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1); + __type(key, long long); + __type(value, long long); +} map_hash_8b SEC(".maps"); + /* Check that precision marks propagate through scalar IDs. * Registers r{0,1,2} have the same scalar ID. * Range information is propagated for scalars sharing same ID. @@ -264,13 +271,13 @@ void precision_many_frames__bar(void) */ SEC("socket") __success __log_level(2) -__msg("11: (0f) r2 += r1") +__msg("12: (0f) r2 += r1") /* foo frame */ -__msg("frame1: regs=r1 stack= before 10: (bf) r2 = r10") -__msg("frame1: regs=r1 stack= before 9: (25) if r1 > 0x7 goto pc+0") -__msg("frame1: regs=r1 stack=-8,-16 before 8: (7b) *(u64 *)(r10 -16) = r1") -__msg("frame1: regs=r1 stack=-8 before 7: (7b) *(u64 *)(r10 -8) = r1") -__msg("frame1: regs=r1 stack= before 4: (85) call pc+2") +__msg("frame1: regs=r1 stack= before 11: (bf) r2 = r10") +__msg("frame1: regs=r1 stack= before 10: (25) if r1 > 0x7 goto pc+0") +__msg("frame1: regs=r1 stack=-8,-16 before 9: (7b) *(u64 *)(r10 -16) = r1") +__msg("frame1: regs=r1 stack=-8 before 8: (7b) *(u64 *)(r10 -8) = r1") +__msg("frame1: regs=r1 stack= before 4: (85) call pc+3") /* main frame */ __msg("frame0: regs=r1 stack=-8 before 3: (7b) *(u64 *)(r10 -8) = r1") __msg("frame0: regs=r1 stack= before 2: (bf) r1 = r0") @@ -286,6 +293,7 @@ __naked void precision_stack(void) "r1 = r0;" "*(u64*)(r10 - 8) = r1;" "call precision_stack__foo;" + "r0 = *(u64*)(r10 - 8);" "r0 = 0;" "exit;" : @@ -309,6 +317,8 @@ void precision_stack__foo(void) */ "r2 = r10;" "r2 += r1;" + "r0 = *(u64*)(r10 - 8);" + "r0 = *(u64*)(r10 - 16);" "exit" ::: __clobber_all); } @@ -369,37 +379,37 @@ __naked void precision_two_ids(void) SEC("socket") __success __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) -/* check that r0 and r6 have different IDs after 'if', - * collect_linked_regs() can't tie more than 6 registers for a single insn. +/* + * check that r5 is unlinked after 'if', collect_linked_regs() can't tie + * more than 5 registers for a single insn and the register compared by + * the jump is not exempt from that. */ -__msg("8: (25) if r0 > 0x7 goto pc+0 ; R0=scalar(id=1") -__msg("14: (bf) r6 = r6 ; R6=scalar(id=2") -/* check that r{0-5} are marked precise after 'if' */ -__msg("frame0: regs=r0 stack= before 8: (25) if r0 > 0x7 goto pc+0") -__msg("frame0: parent state regs=r0,r1,r2,r3,r4,r5 stack=:") +__msg("7: (25) if r5 > 0x7 goto pc+0 ; R5=scalar(smin=") +__msg("12: (bf) r5 = r5 ; R5=scalar(id=2") +/* check that r{0-4} are marked precise after 'if' */ +__msg("frame0: regs=r0 stack= before 7: (25) if r5 > 0x7 goto pc+0") +__msg("frame0: parent state regs=r0,r1,r2,r3,r4 stack=:") __naked void linked_regs_too_many_regs(void) { asm volatile ( /* r0 = random number up to 0xff */ "call %[bpf_ktime_get_ns];" "r0 &= 0xff;" - /* tie r{0-6} IDs */ + /* tie r{0-5} IDs */ "r1 = r0;" "r2 = r0;" "r3 = r0;" "r4 = r0;" "r5 = r0;" - "r6 = r0;" - /* propagate range for r{0-6} */ - "if r0 > 7 goto +0;" - /* keep r{1-5} live */ + /* r{0-4} fill the record, r5 does not fit and is unlinked */ + "if r5 > 7 goto +0;" + /* keep r{1-4} live */ "r1 = r1;" "r2 = r2;" "r3 = r3;" "r4 = r4;" + /* make r5 appear in the log */ "r5 = r5;" - /* make r6 appear in the log */ - "r6 = r6;" /* force r0 to be precise, * this would cause r{0-4} to be precise because of shared IDs */ @@ -592,10 +602,10 @@ __naked void check_ids_in_regsafe_2(void) */ SEC("socket") __success __log_level(2) -__msg("11: (1d) if r3 == r4 goto pc+0") +__msg("14: (1d) if r3 == r4 goto pc+0") __msg("frame 0: propagating r3,r4") -__msg("11: safe") -__msg("processed 15 insns") +__msg("14: safe") +__msg("processed 18 insns") __flag(BPF_F_TEST_STATE_FREQ) __naked void no_scalar_id_for_const(void) { @@ -605,6 +615,7 @@ __naked void no_scalar_id_for_const(void) "if r0 > 7 goto l0_%=;" /* possibly generate same scalar ids for r3 and r4 */ "r1 = 0;" + "r1 ^= r1;" /* prevent bpf_prune_dead_branches from folding the branch */ "r1 = r1;" "r3 = r1;" "r4 = r1;" @@ -612,7 +623,9 @@ __naked void no_scalar_id_for_const(void) "l0_%=:" /* possibly generate different scalar ids for r3 and r4 */ "r1 = 0;" + "r1 ^= r1;" "r2 = 0;" + "r2 ^= r2;" "r3 = r1;" "r4 = r2;" "l1_%=:" @@ -628,10 +641,10 @@ __naked void no_scalar_id_for_const(void) /* Same as no_scalar_id_for_const() but for 32-bit values */ SEC("socket") __success __log_level(2) -__msg("11: (1e) if w3 == w4 goto pc+0") +__msg("14: (1e) if w3 == w4 goto pc+0") __msg("frame 0: propagating r3,r4") -__msg("11: safe") -__msg("processed 15 insns") +__msg("14: safe") +__msg("processed 18 insns") __flag(BPF_F_TEST_STATE_FREQ) __naked void no_scalar_id_for_const32(void) { @@ -641,6 +654,7 @@ __naked void no_scalar_id_for_const32(void) "if r0 > 7 goto l0_%=;" /* possibly generate same scalar ids for r3 and r4 */ "w1 = 0;" + "w1 ^= w1;" /* prevent bpf_prune_dead_branches from folding the branch */ "w1 = w1;" "w3 = w1;" "w4 = w1;" @@ -648,11 +662,13 @@ __naked void no_scalar_id_for_const32(void) "l0_%=:" /* possibly generate different scalar ids for r3 and r4 */ "w1 = 0;" + "w1 ^= w1;" "w2 = 0;" + "w2 ^= w2;" "w3 = w1;" "w4 = w2;" "l1_%=:" - /* predictable jump, marks r1 and r2 precise */ + /* predictable jump, marks r3 and r4 precise */ "if w3 == w4 goto +0;" "r0 = 0;" "exit;" @@ -796,9 +812,9 @@ __success __log_level(2) /* The exit instruction should be reachable from two states, * use two matches and "processed .. insns" to ensure this. */ -__msg("15: (95) exit") -__msg("15: (95) exit") -__msg("processed 20 insns") +__msg("16: (95) exit") +__msg("16: (95) exit") +__msg("processed 22 insns") __flag(BPF_F_TEST_STATE_FREQ) __naked void two_old_ids_one_cur_id(void) { @@ -829,6 +845,11 @@ __naked void two_old_ids_one_cur_id(void) "r2 = r10;" "r2 += r6;" "r2 += r7;" + /* + * keep r8 and r9 live, otherwise r6->id and r7->id + * will become singular and reset to zero before if r6 > r7 + */ + "r9 += r8;" "exit;" : : __imm(bpf_ktime_get_ns) @@ -902,4 +923,53 @@ __naked void linked_regs_and_subreg_def(void) : __clobber_all); } +/* + * A scalar is spilled to the stack and then filled twice: once via a + * sign-extending load (BPF_MEMSX) into r4 and once via a zero-extending + * load (BPF_MEM) into r5. coerce_reg_to_size_sx() gives r4 a different + * value than the spilled/zero-extended siblings, so r4 must not keep the + * shared scalar id. Otherwise the later 'if r5 == 0x80000000' refines r4 + * through sync_linked_regs() to a known 0x80000000, while at runtime r4 + * is the sign-extended 0xffffffff80000000. The test turns that discrepancy + * into an out-of-bounds map value access (r4 >> 63 is believed 0 but is 1 + * at runtime), which must be rejected. + */ +SEC("socket") +__failure __msg("R0 max value is outside of the allowed memory range") +__naked void ldsx_fill_scalar_id_not_shared(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64*)(r10 - 8) = r1; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map_hash_8b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + /* r7 = unknown u32, keep only bit 31 */ \ + r7 = *(u32*)(r0 + 0); \ + r2 = 0x80000000 ll; \ + r7 &= r2; \ + /* link r6 and r7 via a fresh scalar id */ \ + r6 = r7; \ + /* spill r7 (u32) to the stack */ \ + *(u32*)(r10 - 8) = r7; \ + /* sign-extending fill: must drop the id */ \ + r4 = *(s32*)(r10 - 8); \ + /* zero-extending fill: keeps the id */ \ + r5 = *(u32*)(r10 - 8); \ + /* r5 becomes known 0x80000000 on fall-through */\ + if r5 != r2 goto l0_%=; \ + /* verifier believes r4 == 0 here, runtime is 1 */\ + r4 >>= 63; \ + r0 += r4; \ + r0 = *(u8*)(r0 + 7); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_sdiv.c b/tools/testing/selftests/bpf/progs/verifier_sdiv.c index fd59d57e8e37..95f3239ce228 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sdiv.c +++ b/tools/testing/selftests/bpf/progs/verifier_sdiv.c @@ -778,10 +778,10 @@ __arch_x86_64 __xlated("0: r2 = 0x8000000000000000") __xlated("2: r3 = -1") __xlated("3: r4 = r2") -__xlated("4: r11 = r3") -__xlated("5: r11 += 1") -__xlated("6: if r11 > 0x1 goto pc+4") -__xlated("7: if r11 == 0x0 goto pc+1") +__xlated("4: r12 = r3") +__xlated("5: r12 += 1") +__xlated("6: if r12 > 0x1 goto pc+4") +__xlated("7: if r12 == 0x0 goto pc+1") __xlated("8: r2 = 0") __xlated("9: r2 = -r2") __xlated("10: goto pc+1") @@ -812,10 +812,10 @@ __success __retval(-5) __arch_x86_64 __xlated("0: r2 = 5") __xlated("1: r3 = -1") -__xlated("2: r11 = r3") -__xlated("3: r11 += 1") -__xlated("4: if r11 > 0x1 goto pc+4") -__xlated("5: if r11 == 0x0 goto pc+1") +__xlated("2: r12 = r3") +__xlated("3: r12 += 1") +__xlated("4: if r12 > 0x1 goto pc+4") +__xlated("5: if r12 == 0x0 goto pc+1") __xlated("6: r2 = 0") __xlated("7: r2 = -r2") __xlated("8: goto pc+1") @@ -890,10 +890,10 @@ __arch_x86_64 __xlated("0: w2 = -2147483648") __xlated("1: w3 = -1") __xlated("2: w4 = w2") -__xlated("3: r11 = r3") -__xlated("4: w11 += 1") -__xlated("5: if w11 > 0x1 goto pc+4") -__xlated("6: if w11 == 0x0 goto pc+1") +__xlated("3: r12 = r3") +__xlated("4: w12 += 1") +__xlated("5: if w12 > 0x1 goto pc+4") +__xlated("6: if w12 == 0x0 goto pc+1") __xlated("7: w2 = 0") __xlated("8: w2 = -w2") __xlated("9: goto pc+1") @@ -925,10 +925,10 @@ __arch_x86_64 __xlated("0: w2 = -5") __xlated("1: w3 = -1") __xlated("2: w4 = w2") -__xlated("3: r11 = r3") -__xlated("4: w11 += 1") -__xlated("5: if w11 > 0x1 goto pc+4") -__xlated("6: if w11 == 0x0 goto pc+1") +__xlated("3: r12 = r3") +__xlated("4: w12 += 1") +__xlated("5: if w12 > 0x1 goto pc+4") +__xlated("6: if w12 == 0x0 goto pc+1") __xlated("7: w2 = 0") __xlated("8: w2 = -w2") __xlated("9: goto pc+1") @@ -1004,10 +1004,10 @@ __arch_x86_64 __xlated("0: r2 = 0x8000000000000000") __xlated("2: r3 = -1") __xlated("3: r4 = r2") -__xlated("4: r11 = r3") -__xlated("5: r11 += 1") -__xlated("6: if r11 > 0x1 goto pc+3") -__xlated("7: if r11 == 0x1 goto pc+3") +__xlated("4: r12 = r3") +__xlated("5: r12 += 1") +__xlated("6: if r12 > 0x1 goto pc+3") +__xlated("7: if r12 == 0x1 goto pc+3") __xlated("8: w2 = 0") __xlated("9: goto pc+1") __xlated("10: r2 s%= r3") @@ -1034,10 +1034,10 @@ __arch_x86_64 __xlated("0: r2 = 5") __xlated("1: r3 = -1") __xlated("2: r4 = r2") -__xlated("3: r11 = r3") -__xlated("4: r11 += 1") -__xlated("5: if r11 > 0x1 goto pc+3") -__xlated("6: if r11 == 0x1 goto pc+3") +__xlated("3: r12 = r3") +__xlated("4: r12 += 1") +__xlated("5: if r12 > 0x1 goto pc+3") +__xlated("6: if r12 == 0x1 goto pc+3") __xlated("7: w2 = 0") __xlated("8: goto pc+1") __xlated("9: r2 s%= r3") @@ -1108,10 +1108,10 @@ __arch_x86_64 __xlated("0: w2 = -2147483648") __xlated("1: w3 = -1") __xlated("2: w4 = w2") -__xlated("3: r11 = r3") -__xlated("4: w11 += 1") -__xlated("5: if w11 > 0x1 goto pc+3") -__xlated("6: if w11 == 0x1 goto pc+4") +__xlated("3: r12 = r3") +__xlated("4: w12 += 1") +__xlated("5: if w12 > 0x1 goto pc+3") +__xlated("6: if w12 == 0x1 goto pc+4") __xlated("7: w2 = 0") __xlated("8: goto pc+1") __xlated("9: w2 s%= w3") @@ -1140,10 +1140,10 @@ __arch_x86_64 __xlated("0: w2 = -5") __xlated("1: w3 = -1") __xlated("2: w4 = w2") -__xlated("3: r11 = r3") -__xlated("4: w11 += 1") -__xlated("5: if w11 > 0x1 goto pc+3") -__xlated("6: if w11 == 0x1 goto pc+4") +__xlated("3: r12 = r3") +__xlated("4: w12 += 1") +__xlated("5: if w12 > 0x1 goto pc+3") +__xlated("6: if w12 == 0x1 goto pc+4") __xlated("7: w2 = 0") __xlated("8: goto pc+1") __xlated("9: w2 s%= w3") diff --git a/tools/testing/selftests/bpf/progs/verifier_set_retval.c b/tools/testing/selftests/bpf/progs/verifier_set_retval.c new file mode 100644 index 000000000000..1415cd15cede --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_set_retval.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" + +SEC("lsm_cgroup/socket_create") +__description("lsm_cgroup bpf_set_retval success") +__success +int BPF_PROG(lsm_cgroup_set_retval_zero_valid, int family, int type, int protocol, int kern) +{ + bpf_set_retval(0); + return 0; +} + +SEC("lsm_cgroup/socket_create") +__description("lsm_cgroup bpf_set_retval valid errno") +__success +int BPF_PROG(lsm_cgroup_set_retval_negative_valid, int family, int type, int protocol, int kern) +{ + bpf_set_retval(-12); + return 0; +} + +SEC("lsm_cgroup/socket_create") +__description("lsm_cgroup bpf_set_retval invalid negative value") +__failure __msg("should have been in [-4095, 0]") +int BPF_PROG(lsm_cgroup_set_retval_negative_invalid, int family, int type, int protocol, int kern) +{ + bpf_set_retval(-4096); + return 0; +} + +SEC("lsm_cgroup/socket_create") +__description("lsm_cgroup bpf_set_retval invalid positive value") +__failure __msg("should have been in [-4095, 0]") +int BPF_PROG(lsm_cgroup_set_retval_positive_invalid, int family, int type, int protocol, int kern) +{ + bpf_set_retval(1); + return 0; +} + +SEC("cgroup/dev") +__description("cgroup_device bpf_set_retval success") +__success +int cgroup_dev_set_retval_0(struct bpf_cgroup_dev_ctx *ctx) +{ + bpf_set_retval(0); + return 1; +} + +SEC("cgroup/dev") +__description("cgroup_device bpf_set_retval valid errno") +__success +int cgroup_dev_set_retval_neg_maxerrno(struct bpf_cgroup_dev_ctx *ctx) +{ + bpf_set_retval(-4095); + return 1; +} + +SEC("cgroup/dev") +__description("cgroup_device bpf_set_retval invalid positive value") +__failure __msg("should have been in [-4095, 0]") +int cgroup_dev_set_retval_1(struct bpf_cgroup_dev_ctx *ctx) +{ + bpf_set_retval(1); + return 1; +} + +SEC("cgroup/dev") +__description("cgroup_device bpf_set_retval invalid negative value") +__failure __msg("should have been in [-4095, 0]") +int cgroup_dev_set_retval_neg_4096(struct bpf_cgroup_dev_ctx *ctx) +{ + bpf_set_retval(-4096); + return 1; +} + +SEC("cgroup/dev") +__description("bpf_set_retval bounds check survives state pruning") +__failure __msg("should have been in [-4095, 0]") +__naked int cgroup_dev_set_retval_pruning_bypass(struct bpf_cgroup_dev_ctx *ctx) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "if r0 != 0 goto 1f;" + "r0 = r0;" + "r0 = r0;" + "r0 = r0;" + "r0 = r0;" + "goto 2f;" + "1:" + "call %[bpf_get_prandom_u32];" + "2:" + "r1 = r0;" + "call %[bpf_set_retval];" + "r0 = 1;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_set_retval) + : __clobber_common + ); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_sock.c b/tools/testing/selftests/bpf/progs/verifier_sock.c index a2132c72d3b8..bf9f6fb6582c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sock.c +++ b/tools/testing/selftests/bpf/progs/verifier_sock.c @@ -88,6 +88,44 @@ l0_%=: r0 = *(u32*)(r1 + %[bpf_sock_family]); \ : __clobber_all); } +SEC("socket") +__description("skb->sk: sk->rx_queue_mapping [no sign extension]") +__success __success_unpriv __retval(0) +__naked void sk_rx_queue_mapping_no_sign_ext(void) +{ + asm volatile (" \ + r1 = *(u64*)(r1 + %[__sk_buff_sk]); \ + if r1 != 0 goto l0_%=; \ + r0 = 0xdead; \ + exit; \ +l0_%=: r0 = *(u32*)(r1 + %[bpf_sock_rx_queue_mapping]); \ + r0 >>= 32; \ + exit; \ +" : + : __imm_const(__sk_buff_sk, offsetof(struct __sk_buff, sk)), + __imm_const(bpf_sock_rx_queue_mapping, offsetof(struct bpf_sock, rx_queue_mapping)) + : __clobber_all); +} + +SEC("socket") +__description("skb->sk: sk->rx_queue_mapping [narrow load mask]") +__success __success_unpriv __retval(0) +__naked void sk_rx_queue_mapping_narrow_load_mask(void) +{ + asm volatile (" \ + r1 = *(u64*)(r1 + %[__sk_buff_sk]); \ + if r1 != 0 goto l0_%=; \ + r0 = 0xdead; \ + exit; \ +l0_%=: r0 = *(u16*)(r1 + %[bpf_sock_rx_queue_mapping]); \ + r0 >>= 16; \ + exit; \ +" : + : __imm_const(__sk_buff_sk, offsetof(struct __sk_buff, sk)), + __imm_const(bpf_sock_rx_queue_mapping, offsetof(struct bpf_sock, rx_queue_mapping)) + : __clobber_all); +} + SEC("cgroup/skb") __description("skb->sk: sk->type [fullsock field]") __failure __msg("invalid sock_common access") @@ -603,7 +641,7 @@ l2_%=: r0 = *(u32*)(r0 + %[bpf_tcp_sock_snd_cwnd]); \ SEC("tc") __description("bpf_sk_release(skb->sk)") -__failure __msg("R1 must be referenced when passed to release function") +__failure __msg("release helper bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") __naked void bpf_sk_release_skb_sk(void) { asm volatile (" \ @@ -620,7 +658,7 @@ l0_%=: r0 = 0; \ SEC("tc") __description("bpf_sk_release(bpf_sk_fullsock(skb->sk))") -__failure __msg("R1 must be referenced when passed to release function") +__failure __msg("release helper bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") __naked void bpf_sk_fullsock_skb_sk(void) { asm volatile (" \ @@ -644,7 +682,7 @@ l1_%=: r1 = r0; \ SEC("tc") __description("bpf_sk_release(bpf_tcp_sock(skb->sk))") -__failure __msg("R1 must be referenced when passed to release function") +__failure __msg("release helper bpf_sk_release expects referenced PTR_TO_BTF_ID passed to R1") __naked void bpf_tcp_sock_skb_sk(void) { asm volatile (" \ @@ -1120,8 +1158,11 @@ int tail_call(struct __sk_buff *sk) static __noinline int static_tail_call(struct __sk_buff *sk) { + int ret = 0; + bpf_tail_call_static(sk, &jmp_table, 0); - return 0; + barrier_var(ret); + return ret; } /* Tail calls in sub-programs invalidate packet pointers. */ @@ -1144,10 +1185,12 @@ __failure __msg("invalid mem access") int invalidate_pkt_pointers_by_static_tail_call(struct __sk_buff *sk) { int *p = (void *)(long)sk->data; + int ret; if ((void *)(p + 1) > (void *)(long)sk->data_end) return TCX_DROP; - static_tail_call(sk); + ret = static_tail_call(sk); + __sink(ret); *p = 42; /* this is unsafe */ return TCX_PASS; } diff --git a/tools/testing/selftests/bpf/progs/verifier_sockmap_mutate.c b/tools/testing/selftests/bpf/progs/verifier_sockmap_mutate.c index fe4b123187b8..20332a731d4e 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sockmap_mutate.c +++ b/tools/testing/selftests/bpf/progs/verifier_sockmap_mutate.c @@ -74,7 +74,7 @@ static __always_inline void test_sockmap_lookup_and_mutate(void) } SEC("action") -__success +__failure __msg("cannot update sockmap in this context") int test_sched_act(struct __sk_buff *skb) { test_sockmap_mutate(skb->sk); @@ -82,7 +82,7 @@ int test_sched_act(struct __sk_buff *skb) } SEC("classifier") -__success +__failure __msg("cannot update sockmap in this context") int test_sched_cls(struct __sk_buff *skb) { test_sockmap_mutate(skb->sk); @@ -90,7 +90,7 @@ int test_sched_cls(struct __sk_buff *skb) } SEC("flow_dissector") -__success +__failure __msg("cannot update sockmap in this context") int test_flow_dissector_delete(struct __sk_buff *skb __always_unused) { test_sockmap_delete(); @@ -98,7 +98,7 @@ int test_flow_dissector_delete(struct __sk_buff *skb __always_unused) } SEC("flow_dissector") -__failure __msg("program of this type cannot use helper bpf_sk_release") +__failure __msg("cannot update sockmap in this context") int test_flow_dissector_update(struct __sk_buff *skb __always_unused) { test_sockmap_lookup_and_update(); /* no access to skb->sk */ @@ -146,7 +146,7 @@ int test_sk_reuseport(struct sk_reuseport_md *ctx) } SEC("socket") -__success +__failure __msg("cannot update sockmap in this context") int test_socket_filter(struct __sk_buff *skb) { test_sockmap_mutate(skb->sk); @@ -179,7 +179,7 @@ int test_sockops_update_dedicated(struct bpf_sock_ops *ctx) } SEC("xdp") -__success +__failure __msg("cannot update sockmap in this context") int test_xdp(struct xdp_md *ctx __always_unused) { test_sockmap_lookup_and_mutate(); diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c index 7a13dbd794b2..39a1766dae3f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c +++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c @@ -634,6 +634,32 @@ __naked void partial_stack_load_preserves_partial_zeros(void) : __clobber_common); } +SEC("raw_tp") +__log_level(2) +__success +__msg("mark_precise: frame0: regs= stack=-8") +__msg("R2=0") +__naked void stack_load_preserves_mixed_zero_and_zero_spill(void) +{ + asm volatile ( + /* fp-8 has scalar const-zero spill bytes and STACK_ZERO bytes. */ + ".8byte %[fp4_st_zero];" /* LLVM-18+: *(u32 *)(r10 -4) = 0; */ + "r0 = 0;" + "*(u32 *)(r10 -8) = r0;" + + "r1 = %[single_byte_buf];" + "r2 = *(u64 *)(r10 -8);" + "r1 += r2;" + "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + + "r0 = 0;" + "exit;" + : + : __imm_ptr(single_byte_buf), + __imm_insn(fp4_st_zero, BPF_ST_MEM(BPF_W, BPF_REG_FP, -4, 0)) + : __clobber_common); +} + char two_byte_buf[2] SEC(".data.two_byte_buf"); SEC("raw_tp") @@ -650,13 +676,13 @@ __msg("mark_precise: frame0: last_idx 9 first_idx 7 subseq_idx -1") __msg("mark_precise: frame0: regs=r2 stack= before 8: (79) r2 = *(u64 *)(r10 -8)") __msg("mark_precise: frame0: regs= stack=-8 before 7: (bf) r1 = r6") /* note, fp-8 is precise, fp-16 is not yet precise, we'll get there */ -__msg("mark_precise: frame0: parent state regs= stack=-8: R0=1 R1=ctx() R6=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8=P1 fp-16=1") +__msg("mark_precise: frame0: parent state regs= stack=-8: R6=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8=P1 fp-16=1") __msg("mark_precise: frame0: last_idx 6 first_idx 3 subseq_idx 7") __msg("mark_precise: frame0: regs= stack=-8 before 6: (05) goto pc+0") __msg("mark_precise: frame0: regs= stack=-8 before 5: (7b) *(u64 *)(r10 -16) = r0") __msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r0 = 1") __msg("mark_precise: frame0: regs= stack=-8 before 3: (7a) *(u64 *)(r10 -8) = 1") -__msg("10: R1=map_value(map=.data.two_byte_,ks=4,vs=2,off=1) R2=1") +__msg("10: R1=map_value(map=.data.two_byte_,ks=4,vs=2,imm=1) R2=1") /* validate load from fp-16, which was initialized using BPF_STX_MEM */ __msg("12: (79) r2 = *(u64 *)(r10 -16) ; R2=1 R10=fp0 fp-16=1") __msg("13: (0f) r1 += r2") @@ -668,12 +694,12 @@ __msg("mark_precise: frame0: regs= stack=-16 before 9: (0f) r1 += r2") __msg("mark_precise: frame0: regs= stack=-16 before 8: (79) r2 = *(u64 *)(r10 -8)") __msg("mark_precise: frame0: regs= stack=-16 before 7: (bf) r1 = r6") /* now both fp-8 and fp-16 are precise, very good */ -__msg("mark_precise: frame0: parent state regs= stack=-16: R0=1 R1=ctx() R6=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8=P1 fp-16=P1") +__msg("mark_precise: frame0: parent state regs= stack=-16: R6=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8=P1 fp-16=P1") __msg("mark_precise: frame0: last_idx 6 first_idx 3 subseq_idx 7") __msg("mark_precise: frame0: regs= stack=-16 before 6: (05) goto pc+0") __msg("mark_precise: frame0: regs= stack=-16 before 5: (7b) *(u64 *)(r10 -16) = r0") __msg("mark_precise: frame0: regs=r0 stack= before 4: (b7) r0 = 1") -__msg("14: R1=map_value(map=.data.two_byte_,ks=4,vs=2,off=1) R2=1") +__msg("14: R1=map_value(map=.data.two_byte_,ks=4,vs=2,imm=1) R2=1") __naked void stack_load_preserves_const_precision(void) { asm volatile ( @@ -726,13 +752,13 @@ __msg("9: (0f) r1 += r2") __msg("mark_precise: frame0: last_idx 9 first_idx 7 subseq_idx -1") __msg("mark_precise: frame0: regs=r2 stack= before 8: (61) r2 = *(u32 *)(r10 -8)") __msg("mark_precise: frame0: regs= stack=-8 before 7: (bf) r1 = r6") -__msg("mark_precise: frame0: parent state regs= stack=-8: R0=1 R1=ctx() R6=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8=????P1 fp-16=????1") +__msg("mark_precise: frame0: parent state regs= stack=-8: R6=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8=????P1 fp-16=????1") __msg("mark_precise: frame0: last_idx 6 first_idx 3 subseq_idx 7") __msg("mark_precise: frame0: regs= stack=-8 before 6: (05) goto pc+0") __msg("mark_precise: frame0: regs= stack=-8 before 5: (63) *(u32 *)(r10 -16) = r0") __msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r0 = 1") __msg("mark_precise: frame0: regs= stack=-8 before 3: (62) *(u32 *)(r10 -8) = 1") -__msg("10: R1=map_value(map=.data.two_byte_,ks=4,vs=2,off=1) R2=1") +__msg("10: R1=map_value(map=.data.two_byte_,ks=4,vs=2,imm=1) R2=1") /* validate load from fp-16, which was initialized using BPF_STX_MEM */ __msg("12: (61) r2 = *(u32 *)(r10 -16) ; R2=1 R10=fp0 fp-16=????1") __msg("13: (0f) r1 += r2") @@ -743,12 +769,12 @@ __msg("mark_precise: frame0: regs= stack=-16 before 10: (73) *(u8 *)(r1 +0) = r2 __msg("mark_precise: frame0: regs= stack=-16 before 9: (0f) r1 += r2") __msg("mark_precise: frame0: regs= stack=-16 before 8: (61) r2 = *(u32 *)(r10 -8)") __msg("mark_precise: frame0: regs= stack=-16 before 7: (bf) r1 = r6") -__msg("mark_precise: frame0: parent state regs= stack=-16: R0=1 R1=ctx() R6=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8=????P1 fp-16=????P1") +__msg("mark_precise: frame0: parent state regs= stack=-16: R6=map_value(map=.data.two_byte_,ks=4,vs=2) R10=fp0 fp-8=????P1 fp-16=????P1") __msg("mark_precise: frame0: last_idx 6 first_idx 3 subseq_idx 7") __msg("mark_precise: frame0: regs= stack=-16 before 6: (05) goto pc+0") __msg("mark_precise: frame0: regs= stack=-16 before 5: (63) *(u32 *)(r10 -16) = r0") __msg("mark_precise: frame0: regs=r0 stack= before 4: (b7) r0 = 1") -__msg("14: R1=map_value(map=.data.two_byte_,ks=4,vs=2,off=1) R2=1") +__msg("14: R1=map_value(map=.data.two_byte_,ks=4,vs=2,imm=1) R2=1") __naked void stack_load_preserves_const_precision_subreg(void) { asm volatile ( @@ -780,6 +806,8 @@ __naked void stack_load_preserves_const_precision_subreg(void) "r1 += r2;" "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + "r2 = *(u64 *)(r10 -8);" /* keep slots alive */ + "r2 = *(u64 *)(r10 -16);" "r0 = 0;" "exit;" : @@ -1279,4 +1307,182 @@ __naked void stack_noperfmon_spill_32bit_onto_64bit_slot(void) : __clobber_all); } +/* + * stacksafe(): check if 32-bit scalar spill in old state is considered + * equivalent to STACK_MISC in cur state. + * 32-bit scalar spill creates slot[0-3] = STACK_MISC, slot[4-7] = STACK_SPILL. + * Without 32-bit spill support in stacksafe(), the STACK_SPILL vs STACK_MISC + * mismatch at slot[4] causes pruning to fail. + */ +SEC("socket") +__success __log_level(2) +__msg("8: (79) r1 = *(u64 *)(r10 -8)") +__msg("8: safe") +__msg("processed 11 insns") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void old_imprecise_scalar32_vs_cur_stack_misc(void) +{ + asm volatile( + /* get a random value for branching */ + "call %[bpf_ktime_get_ns];" + "if r0 == 0 goto 1f;" + /* conjure 32-bit scalar spill at fp-8 */ + "r0 = 42;" + "*(u32*)(r10 - 8) = r0;" + "goto 2f;" +"1:" + /* conjure STACK_MISC at fp-8 */ + "call %[bpf_ktime_get_ns];" + "*(u16*)(r10 - 8) = r0;" + "*(u16*)(r10 - 6) = r0;" +"2:" + /* read fp-8, should be considered safe on second visit */ + "r1 = *(u64*)(r10 - 8);" + "exit;" + : + : __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("raw_tp") +__success +__naked void var_off_write_over_scalar_spill(void) +{ + asm volatile ( + /* Get an unknown value bounded to {0, 4} */ + "call %[bpf_ktime_get_ns];" + "r6 = r0;" + "r6 &= 4;" + + /* Spill a scalar to fp-16 */ + "r7 = 0xdeadbeef00000000 ll;" + "*(u64 *)(r10 - 16) = r7;" + + /* + * Variable-offset 4-byte write covering [fp-12, fp-4). + * This touches stype[3..0] of the spill slot at fp-16 but + * leaves stype[7..4] as STACK_SPILL. check_stack_write_var_off() + * must scrub the entire slot when setting spilled_ptr to NOT_INIT, + * otherwise a subsequent sub-register fill sees a non-scalar + * spilled_ptr and is rejected. + */ + "r8 = r10;" + "r8 += r6;" + "r8 += -12;" + "r9 = 0;" + "*(u32 *)(r8 + 0) = r9;" + + /* + * 4-byte read from fp-16. Without the fix this fails with + * "invalid size of register fill" because is_spilled_reg() + * sees STACK_SPILL while spilled_ptr.type == NOT_INIT. + */ + "r0 = *(u32 *)(r10 - 16);" + "r0 = 0;" + "exit;" + : + : __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +SEC("socket") +__description("partial fill from cleaned pointer spill") +__failure +__log_level(2) +__msg("1: (05) goto pc+0") +__msg("2: (61) r0 = *(u32 *)(r10 -4)") +__msg("invalid size of register fill") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void partial_fill_from_cleaned_pointer_spill(void) +{ + /* Spill R1(ctx), then force a checkpoint and half-slot cleanup. */ + asm volatile ("*(u64 *)(r10 - 8) = r1;" + "goto +0;" + "r0 = *(u32 *)(r10 - 4);" + "exit;" + ::: __clobber_all); +} + +SEC("raw_tp") +__failure +__msg("access may be outside object bounds") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void imprecise_scalar_spill_half_dead(void) +{ + asm volatile ( + /* + * Fork two paths: the one explored first spills an imprecise zero, + * the one explored second, an imprecise non-zero scalar. + */ + "call %[bpf_get_prandom_u32];" + "if r0 > 42 goto 1f;" + "r6 = 0;" + "goto 2f;" +"1:" + /* causes out of bounds access on a second path. */ + "r6 = 100500;" +"2:" + /* Force a checkpoint before the spill. */ + "goto +0;" + "*(u64 *)(r10 - 8) = r6;" + /* + * Force stack cleanup, only the low half of the spill is alive, + * so the dead high half is degraded to raw stack bytes. + * Buggy verifier converted it to STACK_ZERO w/o proper precision propagation. + */ + "goto +0;" + "r7 = *(u32 *)(r10 - 4);" + /* Use r7 as an offset into a one-byte buffer. */ + "r1 = %[single_byte_buf] ll;" + "r1 += r7;" + "r0 = *(u8 *)(r1 + 0);" + "exit;" +: +: __imm(bpf_get_prandom_u32), + __imm_addr(single_byte_buf) +: __clobber_all); +} + +/* check valid spill/fill, ptr to tp buffer */ +SEC("raw_tracepoint.w") +__success +__naked void spill_fill_ptr_to_tp_buffer(void) +{ + asm volatile ( + "r6 = *(u64*)(r1 + 0);" /* r6 is the writable tracepoint buffer */ + "*(u64*)(r10 - 8) = r6;" + "r7 = *(u64*)(r10 - 8);" + "r0 = 0;" + "*(u64*)(r7 + 0) = r0;" /* should be able to write through the buffer */ + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +__noinline int spill_fill_dynptr_subprog(struct bpf_dynptr *dptr) +{ + long *p; + + asm volatile ("*(u64 *)(r10 - 8) = %[dptr];" /* spill the CONST_PTR_TO_DYNPTR argument */ + "%[dptr] = *(u64 *)(r10 - 8);" + : [dptr] "+r"(dptr) :: "memory"); + p = bpf_dynptr_data(dptr, 0, sizeof(*p)); + if (!p) + return 0; + return 0; +} + +static char dptr_mem_buf[16]; + +/* check valid spill/fill, const ptr to dynptr */ +SEC("socket") +__success +int spill_fill_const_ptr_to_dynptr(void) +{ + struct bpf_dynptr ptr; + + bpf_dynptr_from_mem(dptr_mem_buf, sizeof(dptr_mem_buf), 0, &ptr); + return spill_fill_dynptr_subprog(&ptr); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c new file mode 100644 index 000000000000..7e0ce5db28a0 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c @@ -0,0 +1,447 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1); + __type(key, long long); + __type(value, long long); +} map_hash_8b SEC(".maps"); + +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) + +__noinline __used +static int subprog_6args(int a, int b, int c, int d, int e, int f) +{ + return a + b + c + d + e + f; +} + +__noinline __used +static int subprog_7args(int a, int b, int c, int d, int e, int f, int g) +{ + return a + b + c + d + e + f + g; +} + +__noinline __used +static long subprog_deref_arg6(long a, long b, long c, long d, long e, long *f) +{ + return *f; +} + +SEC("tc") +__description("stack_arg: subprog with 6 args") +__success __retval(21) +__naked void stack_arg_6args(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 8) = 6;" + "call subprog_6args;" + "exit;" + ::: __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: two subprogs with >5 args") +__success __retval(90) +__naked void stack_arg_two_subprogs(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 8) = 10;" + "call subprog_6args;" + "r6 = r0;" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 16) = 30;" + "*(u64 *)(r11 - 8) = 20;" + "call subprog_7args;" + "r0 += r6;" + "exit;" + ::: __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: read from uninitialized stack arg slot") +__failure +__msg("invalid read from stack arg off 8 depth 0") +__naked void stack_arg_read_uninitialized(void) +{ + asm volatile ( + "r0 = *(u64 *)(r11 + 8);" + "r0 = 0;" + "exit;" + ::: __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: gap at offset -8, only wrote -16") +__failure +__msg("callee expects 7 args, stack arg1 is not initialized") +__naked void stack_arg_gap_at_minus8(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 16) = 30;" + "call subprog_7args;" + "exit;" + ::: __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: pruning with different stack arg types") +__failure __log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("arg JOIN insn 9 -> 10 r1: fp0-8 + _ => fp0-8|fp0+0") +__msg("arg JOIN insn 9 -> 10 sa0: fp0-8 + _ => fp0-8|fp0+0") +__msg("R{{[0-9]}} invalid mem access 'scalar'") +__naked void stack_arg_pruning_type_mismatch(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + /* local = 0 on program stack */ + "r7 = 0;" + "*(u64 *)(r10 - 8) = r7;" + /* Branch based on random value */ + "if r6 s> 3 goto l0_%=;" + /* Path 1: store stack pointer to outgoing arg6 */ + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r11 - 8) = r1;" + "goto l1_%=;" + "l0_%=:" + /* Path 2: store scalar to outgoing arg6 */ + "*(u64 *)(r11 - 8) = 42;" + "l1_%=:" + /* Call subprog that dereferences arg6 */ + "r1 = r6;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call subprog_deref_arg6;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: release_reference invalidates stack arg slot") +__failure +__msg("callee expects 6 args, stack arg1 is not initialized") +__naked void stack_arg_release_ref(void) +{ + asm volatile ( + "r6 = r1;" + /* struct bpf_sock_tuple tuple = {} */ + "r2 = 0;" + "*(u32 *)(r10 - 8) = r2;" + "*(u64 *)(r10 - 16) = r2;" + "*(u64 *)(r10 - 24) = r2;" + "*(u64 *)(r10 - 32) = r2;" + "*(u64 *)(r10 - 40) = r2;" + "*(u64 *)(r10 - 48) = r2;" + /* sk = bpf_sk_lookup_tcp(ctx, &tuple, sizeof(tuple), 0, 0) */ + "r1 = r6;" + "r2 = r10;" + "r2 += -48;" + "r3 = %[sizeof_bpf_sock_tuple];" + "r4 = 0;" + "r5 = 0;" + "call %[bpf_sk_lookup_tcp];" + /* r0 = sk (PTR_TO_SOCK_OR_NULL) */ + "if r0 == 0 goto l0_%=;" + /* Store sock ref to outgoing arg6 slot */ + "*(u64 *)(r11 - 8) = r0;" + /* Release the reference — invalidates the stack arg slot */ + "r1 = r0;" + "call %[bpf_sk_release];" + /* Call subprog that dereferences arg6 — should fail */ + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "call subprog_deref_arg6;" + "l0_%=:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_sk_lookup_tcp), + __imm(bpf_sk_release), + __imm_const(sizeof_bpf_sock_tuple, sizeof(struct bpf_sock_tuple)) + : __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: pkt pointer in stack arg slot invalidated after pull_data") +__failure +__msg("callee expects 6 args, stack arg1 is not initialized") +__naked void stack_arg_stale_pkt_ptr(void) +{ + asm volatile ( + "r6 = r1;" + "r7 = *(u32 *)(r6 + %[__sk_buff_data]);" + "r8 = *(u32 *)(r6 + %[__sk_buff_data_end]);" + /* check pkt has at least 1 byte */ + "r0 = r7;" + "r0 += 8;" + "if r0 > r8 goto l0_%=;" + /* Store valid pkt pointer to outgoing arg6 slot */ + "*(u64 *)(r11 - 8) = r7;" + /* bpf_skb_pull_data invalidates all pkt pointers */ + "r1 = r6;" + "r2 = 0;" + "call %[bpf_skb_pull_data];" + /* Call subprog that dereferences arg6 — should fail */ + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "call subprog_deref_arg6;" + "l0_%=:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_skb_pull_data), + __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: null propagation rejects deref on null branch") +__failure +__msg("R{{[0-9]}} invalid mem access 'scalar'") +__naked void stack_arg_null_propagation_fail(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + /* r0 = bpf_map_lookup_elem(&map_hash_8b, &key) */ + "r2 = r10;" + "r2 += -8;" + "r1 = %[map_hash_8b] ll;" + "call %[bpf_map_lookup_elem];" + /* Store PTR_TO_MAP_VALUE_OR_NULL to outgoing arg6 slot */ + "*(u64 *)(r11 - 8) = r0;" + /* null check on r0 */ + "if r0 != 0 goto l0_%=;" + /* + * On null branch, outgoing slot is SCALAR(0). + * Call subprog that dereferences arg6 — should fail. + */ + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call subprog_deref_arg6;" + "l0_%=:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b) + : __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: missing store on one branch") +__failure +__msg("callee expects 7 args, stack arg1 is not initialized") +__naked void stack_arg_missing_store_one_branch(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + /* Write arg7 (r11-16) before branch */ + "*(u64 *)(r11 - 16) = 20;" + "if r0 > 0 goto l0_%=;" + /* Path 1: write arg6 and call */ + "*(u64 *)(r11 - 8) = 10;" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "call subprog_7args;" + "goto l1_%=;" + "l0_%=:" + /* Path 2: missing arg6 store, call should fail */ + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "call subprog_7args;" + "l1_%=:" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: share a store for both branches") +__success __retval(0) +__naked void stack_arg_shared_store(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + /* Write arg7 (r11-16) before branch */ + "*(u64 *)(r11 - 16) = 20;" + "if r0 > 0 goto l0_%=;" + /* Path 1: write arg6 and call */ + "*(u64 *)(r11 - 8) = 10;" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "call subprog_7args;" + "goto l1_%=;" + "l0_%=:" + /* Path 2: also write arg6 and call */ + "*(u64 *)(r11 - 8) = 30;" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "call subprog_7args;" + "l1_%=:" + "r0 = 0;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: write beyond max outgoing depth") +__failure +__msg("stack arg write offset -80 exceeds max 7 stack args") +__naked void stack_arg_write_beyond_max(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + /* Write to offset -80, way beyond any callee's needs */ + "*(u64 *)(r11 - 80) = 99;" + "*(u64 *)(r11 - 16) = 20;" + "*(u64 *)(r11 - 8) = 10;" + "call subprog_7args;" + "r0 = 0;" + "exit;" + ::: __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: write unused stack arg slot") +__failure +__msg("func#0 writes 5 stack arg slots, but calls only require 2") +__naked void stack_arg_write_unused_slot(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + /* Write to offset -40, unused for the callee */ + "*(u64 *)(r11 - 40) = 99;" + "*(u64 *)(r11 - 16) = 20;" + "*(u64 *)(r11 - 8) = 10;" + "call subprog_7args;" + "r0 = 0;" + "exit;" + ::: __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: sequential calls reuse slots") +__failure +__msg("callee expects 7 args, stack arg1 is not initialized") +__naked void stack_arg_sequential_calls(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 8) = 6;" + "*(u64 *)(r11 - 16) = 7;" + "call subprog_7args;" + "r6 = r0;" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "call subprog_7args;" + "r0 += r6;" + "exit;" + ::: __clobber_all + ); +} + +#else + +SEC("socket") +__description("stack_arg is not supported by compiler or jit, use a dummy test") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c new file mode 100644 index 000000000000..c9fe4857da3f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) + +__noinline __used __naked +static int subprog_bad_order_6args(int a, int b, int c, int d, int e, int f) +{ + asm volatile ( + "*(u64 *)(r11 - 8) = r1;" + "r0 = *(u64 *)(r11 + 8);" + "exit;" + ::: __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: r11 load after r11 store") +__failure +__msg("r11 load must be before any r11 store or call insn") +__btf_func_path("btf__verifier_stack_arg_order.bpf.o") +__naked void stack_arg_load_after_store(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 8) = 6;" + "call subprog_bad_order_6args;" + "exit;" + ::: __clobber_all + ); +} + +__noinline __used __naked +static int subprog_call_before_load_6args(int a, int b, int c, int d, int e, + int f) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r0 = *(u64 *)(r11 + 8);" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: r11 load after a call") +__failure +__msg("r11 load must be before any r11 store or call insn") +__btf_func_path("btf__verifier_stack_arg_order.bpf.o") +__naked void stack_arg_load_after_call(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 8) = 6;" + "call subprog_call_before_load_6args;" + "exit;" + ::: __clobber_all + ); +} + +__noinline __used __naked +static int subprog_pruning_call_before_load_6args(int a, int b, int c, int d, + int e, int f) +{ + asm volatile ( + "if r1 s> 0 goto l0_%=;" + "goto l1_%=;" + "l0_%=:" + "call %[bpf_get_prandom_u32];" + "l1_%=:" + "r0 = *(u64 *)(r11 + 8);" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: pruning keeps r11 load ordering") +__failure +__flag(BPF_F_TEST_STATE_FREQ) +__msg("r11 load must be before any r11 store or call insn") +__btf_func_path("btf__verifier_stack_arg_order.bpf.o") +__naked void stack_arg_pruning_load_after_call(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = r0;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 8) = 6;" + "call subprog_pruning_call_before_load_6args;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all + ); +} + +/* + * "bad_ptr": the first arg is 'long *', which is not a recognized pointer + * type for static subprogs (not ctx, dynptr, or tagged). btf_prepare_func_args() + * sets arg_cnt = 7 / stack_arg_cnt = 2, then fails with -EINVAL. The subprog + * is marked unreliable but the call still proceeds for static subprogs. + */ +__noinline __used __naked +static void subprog_bad_ptr_7args(long *a, int b, int c, int d, int e, int f, int g) +{ + asm volatile ( + "r0 = *(u64 *)(r11 + 8);" + "r1 = *(u64 *)(r11 + 16);" + "exit;" + ::: __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: read without caller write") +__failure +__msg("callee expects 7 args, stack arg1 is not initialized") +__btf_func_path("btf__verifier_stack_arg_order.bpf.o") +__naked void stack_arg_read_without_write_1(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call subprog_bad_ptr_7args;" + "exit;" + ::: __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: read with not-initialized caller write") +__failure +__msg("R0 !read_ok") +__btf_func_path("btf__verifier_stack_arg_order.bpf.o") +__naked void stack_arg_read_without_write_2(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "*(u64 *)(r11 - 8) = 0;" + "*(u64 *)(r11 - 16) = 0;" + "call subprog_bad_ptr_7args;" + "call subprog_bad_ptr_7args;" + "exit;" + ::: __clobber_all + ); +} + +#else + +SEC("socket") +__description("stack_arg order is not supported by compiler or jit, use a dummy test") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c index 24aabc6083fd..8e8cf8232255 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c @@ -37,7 +37,7 @@ __naked void ptr_to_stack_store_load(void) SEC("socket") __description("PTR_TO_STACK store/load - bad alignment on off") -__failure __msg("misaligned stack access off 0+-8+2 size 8") +__failure __msg("misaligned stack access off -8+2 size 8") __failure_unpriv __naked void load_bad_alignment_on_off(void) { @@ -53,7 +53,7 @@ __naked void load_bad_alignment_on_off(void) SEC("socket") __description("PTR_TO_STACK store/load - bad alignment on reg") -__failure __msg("misaligned stack access off 0+-10+8 size 8") +__failure __msg("misaligned stack access off -10+8 size 8") __failure_unpriv __naked void load_bad_alignment_on_reg(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_subprog_insn_stats.c b/tools/testing/selftests/bpf/progs/verifier_subprog_insn_stats.c new file mode 100644 index 000000000000..8f6082fdb5c8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_subprog_insn_stats.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +struct timer_value { + struct bpf_timer timer; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct timer_value); +} timer_map SEC(".maps"); + +SEC("?raw_tp") +__success __log_level(4) +__msg("subprog 0 (stats_main_only) main insns_self 2 insns_total 2 stack 0") +__msg("processed 2 insns") +__naked int stats_main_only(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +__naked __noinline __used +static int stats_chain_leaf(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +__naked __noinline __used +static int stats_chain_parent(void) +{ + asm volatile ( + "call stats_chain_leaf;" + "exit;" + ); +} + +SEC("?raw_tp") +__success __log_level(4) +/* + * self: 2 + 2 + 2 = 6 + * totals: leaf 2, parent 2 + 2 = 4, main 2 + 4 = 6 + */ +__msg("subprog 0 (stats_static_chain) main insns_self 2 insns_total 6 stack 0") +__msg("subprog {{[0-9]+}} (stats_chain_parent) static insns_self 2 insns_total 4 stack 0") +__msg("subprog {{[0-9]+}} (stats_chain_leaf) static insns_self 2 insns_total 2 stack 0") +__msg("processed 6 insns") +__naked int stats_static_chain(void) +{ + asm volatile ( + "call stats_chain_parent;" + "exit;" + ); +} + +__naked __noinline __used +static int stats_shared_leaf(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +__naked __noinline __used +int stats_global_root(void) +{ + asm volatile ( + "call stats_shared_leaf;" + "exit;" + ); +} + +SEC("?raw_tp") +__success __log_level(4) +/* + * stats_shared_leaf is explored once under each independent root. + * self: main 3 + leaf 4 + global 2 = 9 + * root totals: main 5 + global 4 = 9 + */ +__msg("subprog 0 (stats_shared_roots) main insns_self 3 insns_total 5 stack 0") +__msg("subprog {{[0-9]+}} (stats_shared_leaf) static insns_self 4 insns_total 4 stack 0") +__msg("subprog {{[0-9]+}} (stats_global_root) global insns_self 2 insns_total 4 stack 0") +__msg("processed 9 insns") +__naked int stats_shared_roots(void) +{ + asm volatile ( + "call stats_shared_leaf;" + "call stats_global_root;" + "exit;" + ); +} + +__noinline __used +static int stats_async_leaf(void *map, __u32 *key, struct bpf_timer *timer) +{ + return 0; +} + +__noinline __used +static __u64 stats_async_schedule(struct bpf_map *map, __u32 *key, + struct timer_value *value, void *ctx) +{ + asm volatile ( + "r1 = %[timer];" + "r2 = %[stats_async_leaf];" + "call %[bpf_timer_set_callback];" + : + : [timer] "r" (value), + __imm_ptr(stats_async_leaf), + __imm(bpf_timer_set_callback) + : __clobber_common + ); + return 0; +} + +SEC("?raw_tp") +__success __log_level(4) +/* + * self: 9 + 7 + 2 = 18 + * totals: leaf 2, scheduler 7, main root 18 + */ +__msg("subprog 0 (stats_async_direct) main insns_self 9 insns_total 18 stack 0") +__msg("subprog {{[0-9]+}} (stats_async_schedule) static insns_self 7 insns_total 7 stack 0") +__msg("subprog {{[0-9]+}} (stats_async_leaf) static insns_self 2 insns_total 2 stack 0") +__msg("processed 18 insns") +__naked int stats_async_direct(void) +{ + asm volatile ( + "r1 = %[timer_map] ll;" + "r2 = %[stats_async_schedule];" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_for_each_map_elem];" + "r0 = 0;" + "exit;" + : + : __imm_addr(timer_map), + __imm_ptr(stats_async_schedule), + __imm(bpf_for_each_map_elem) + : __clobber_common + ); +} + +__noinline __used +static int stats_async_nested_leaf(void *map, __u32 *key, struct bpf_timer *timer) +{ + return 0; +} + +__noinline __used +static int stats_async_outer(void *map, __u32 *key, struct bpf_timer *timer) +{ + asm volatile ( + "r1 = %[timer];" + "r2 = %[stats_async_nested_leaf];" + "call %[bpf_timer_set_callback];" + : + : [timer] "r" (timer), + __imm_ptr(stats_async_nested_leaf), + __imm(bpf_timer_set_callback) + : __clobber_common + ); + return 0; +} + +__noinline __used +static __u64 stats_async_nested_schedule(struct bpf_map *map, __u32 *key, + struct timer_value *value, void *ctx) +{ + asm volatile ( + "r1 = %[timer];" + "r2 = %[stats_async_outer];" + "call %[bpf_timer_set_callback];" + : + : [timer] "r" (value), + __imm_ptr(stats_async_outer), + __imm(bpf_timer_set_callback) + : __clobber_common + ); + return 0; +} + +SEC("?raw_tp") +__success __log_level(4) +/* + * self: 9 + 7 + 7 + 2 = 25 + * totals: leaf 2, outer 7, scheduler 7, main root 25 + */ +__msg("subprog 0 (stats_async_nested) main insns_self 9 insns_total 25 stack 0") +__msg("subprog {{[0-9]+}} (stats_async_nested_schedule) static insns_self 7 insns_total 7 stack 0") +__msg("subprog {{[0-9]+}} (stats_async_outer) static insns_self 7 insns_total 7 stack 0") +__msg("subprog {{[0-9]+}} (stats_async_nested_leaf) static insns_self 2 insns_total 2 stack 0") +__msg("processed 25 insns") +__naked int stats_async_nested(void) +{ + asm volatile ( + "r1 = %[timer_map] ll;" + "r2 = %[stats_async_nested_schedule];" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_for_each_map_elem];" + "r0 = 0;" + "exit;" + : + : __imm_addr(timer_map), + __imm_ptr(stats_async_nested_schedule), + __imm(bpf_for_each_map_elem) + : __clobber_common + ); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c index 61886ed554de..dc0c7034c04f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c +++ b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c @@ -282,14 +282,14 @@ __msg("mark_precise: frame0: regs=r0,r6 stack= before 10: (bf) r6 = r0") __msg("mark_precise: frame0: regs=r0 stack= before 9: (85) call bpf_loop") /* State entering callback body popped from states stack */ __msg("from 9 to 17: frame1:") -__msg("17: frame1: R1=scalar() R2=0 R10=fp0 cb") +__msg("17: frame1: R10=fp0 cb") __msg("17: (b7) r0 = 0") __msg("18: (95) exit") __msg("returning from callee:") __msg("to caller at 9:") -__msg("frame 0: propagating r1,r4") +__msg("frame 0: propagating r1,r3,r4") __msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1") -__msg("mark_precise: frame0: regs=r1,r4 stack= before 18: (95) exit") +__msg("mark_precise: frame0: regs=r1,r3,r4 stack= before 18: (95) exit") __msg("from 18 to 9: safe") __naked int callback_result_precise(void) { @@ -411,7 +411,7 @@ __msg("mark_precise: frame0: regs=r6 stack= before 5: (b7) r1 = 1") __msg("mark_precise: frame0: regs=r6 stack= before 4: (b7) r6 = 3") /* State entering callback body popped from states stack */ __msg("from 9 to 15: frame1:") -__msg("15: frame1: R1=scalar() R2=0 R10=fp0 cb") +__msg("15: frame1: R10=fp0 cb") __msg("15: (b7) r0 = 0") __msg("16: (95) exit") __msg("returning from callee:") @@ -419,9 +419,9 @@ __msg("to caller at 9:") /* r1, r4 are always precise for bpf_loop(), * r6 was marked before backtracking to callback body. */ -__msg("frame 0: propagating r1,r4,r6") +__msg("frame 0: propagating r1,r3,r4,r6") __msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1") -__msg("mark_precise: frame0: regs=r1,r4,r6 stack= before 16: (95) exit") +__msg("mark_precise: frame0: regs=r1,r3,r4,r6 stack= before 16: (95) exit") __msg("mark_precise: frame1: regs= stack= before 15: (b7) r0 = 0") __msg("mark_precise: frame1: regs= stack= before 9: (85) call bpf_loop") __msg("mark_precise: frame0: parent state regs= stack=:") @@ -567,7 +567,7 @@ __msg("mark_precise: frame0: regs= stack=-8 before 5: (7b) *(u64 *)(r10 -8) = r6 __msg("mark_precise: frame0: regs=r6 stack= before 4: (b7) r6 = 3") /* State entering callback body popped from states stack */ __msg("from 10 to 17: frame1:") -__msg("17: frame1: R1=scalar() R2=0 R10=fp0 cb") +__msg("17: frame1: R10=fp0 cb") __msg("17: (b7) r0 = 0") __msg("18: (95) exit") __msg("returning from callee:") @@ -575,9 +575,9 @@ __msg("to caller at 10:") /* r1, r4 are always precise for bpf_loop(), * fp-8 was marked before backtracking to callback body. */ -__msg("frame 0: propagating r1,r4,fp-8") +__msg("frame 0: propagating r1,r3,r4,fp-8") __msg("mark_precise: frame0: last_idx 10 first_idx 10 subseq_idx -1") -__msg("mark_precise: frame0: regs=r1,r4 stack=-8 before 18: (95) exit") +__msg("mark_precise: frame0: regs=r1,r3,r4 stack=-8 before 18: (95) exit") __msg("mark_precise: frame1: regs= stack= before 17: (b7) r0 = 0") __msg("mark_precise: frame1: regs= stack= before 10: (85) call bpf_loop#181") __msg("mark_precise: frame0: parent state regs= stack=:") @@ -681,7 +681,7 @@ __msg("mark_precise: frame0: last_idx 10 first_idx 7 subseq_idx -1") __msg("mark_precise: frame0: regs=r7 stack= before 9: (bf) r1 = r8") __msg("mark_precise: frame0: regs=r7 stack= before 8: (27) r7 *= 4") __msg("mark_precise: frame0: regs=r7 stack= before 7: (79) r7 = *(u64 *)(r10 -8)") -__msg("mark_precise: frame0: parent state regs= stack=-8: R0=2 R6=1 R8=map_value(map=.data.vals,ks=4,vs=16) R10=fp0 fp-8=P1") +__msg("mark_precise: frame0: parent state regs= stack=-8: R8=map_value(map=.data.vals,ks=4,vs=16) R10=fp0 fp-8=P1") __msg("mark_precise: frame0: last_idx 18 first_idx 0 subseq_idx 7") __msg("mark_precise: frame0: regs= stack=-8 before 18: (95) exit") __msg("mark_precise: frame1: regs= stack= before 17: (0f) r0 += r2") @@ -846,4 +846,55 @@ __naked int subprog_result_tail_call(void) ); } +__naked __noinline __used +static int ld_abs_subprog(void) +{ + asm volatile ( + "r6 = r1;" + "r7 = r1;" + ".8byte %[ld_abs];" + "exit;" + : + : __imm_insn(ld_abs, BPF_LD_ABS(BPF_W, 0)) + : __clobber_all); +} + +/* + * Buggy verifier did not properly backtrack early subprogram exit + * modelled for BPF_LD | BPF_ABS instruction, causing a segfault. + */ +SEC("socket") +__success +__log_level(2) +/* early exit path */ +__msg("3: (0f) r1 += r7") +__msg("mark_precise: frame0: regs=r7 stack= before 2: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r7 stack= before 9: (20) r0 = *(u32 *)skb[0]") +__msg("mark_precise: frame1: regs= stack= before 8: (bf) r7 = r1") +__msg("mark_precise: frame1: regs= stack= before 7: (bf) r6 = r1") +__msg("mark_precise: frame1: regs= stack= before 1: (85) call pc+5") +__msg("mark_precise: frame0: regs=r7 stack= before 0: (b7) r7 = -8") +/* fallthrough path */ +__msg("3: (0f) r1 += r7") +__msg("mark_precise: frame0: regs=r7 stack= before 2: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r7 stack= before 10: (95) exit") +__msg("mark_precise: frame1: regs= stack= before 9: (20) r0 = *(u32 *)skb[0]") +__msg("mark_precise: frame1: regs= stack= before 8: (bf) r7 = r1") +__msg("mark_precise: frame1: regs= stack= before 7: (bf) r6 = r1") +__msg("mark_precise: frame1: regs= stack= before 1: (85) call pc+5") +__msg("mark_precise: frame0: regs=r7 stack= before 0: (b7) r7 = -8") +__naked int ld_abs_backtrack_both_paths(void) +{ + asm volatile ( + "r7 = -8;" + "call ld_abs_subprog;" + "r1 = r10;" + "r1 += r7;" /* mark r7 as precise */ + "*(u64 *)(r1 + 0) = 0;" + "r0 = 0;" + "exit;" + ::: __clobber_all + ); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_subprog_topo.c b/tools/testing/selftests/bpf/progs/verifier_subprog_topo.c new file mode 100644 index 000000000000..e2b9d14bbc3d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_subprog_topo.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* linear chain main -> A -> B */ +__naked __noinline __used +static unsigned long linear_b(void) +{ + asm volatile ( + "r0 = 42;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long linear_a(void) +{ + asm volatile ( + "call linear_b;" + "exit;" + ); +} + +SEC("?raw_tp") +__success __log_level(2) +__msg("topo_order[0] = linear_b") +__msg("topo_order[1] = linear_a") +__msg("topo_order[2] = topo_linear") +__naked int topo_linear(void) +{ + asm volatile ( + "call linear_a;" + "exit;" + ); +} + +/* diamond main -> A, main -> B, A -> C, B -> C */ +__naked __noinline __used +static unsigned long diamond_c(void) +{ + asm volatile ( + "r0 = 1;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long diamond_b(void) +{ + asm volatile ( + "call diamond_c;" + "exit;" + ); +} + +__naked __noinline __used +static unsigned long diamond_a(void) +{ + asm volatile ( + "call diamond_c;" + "exit;" + ); +} + +SEC("?raw_tp") +__success __log_level(2) +__msg("topo_order[0] = diamond_c") +__msg("topo_order[3] = topo_diamond") +__naked int topo_diamond(void) +{ + asm volatile ( + "call diamond_a;" + "call diamond_b;" + "exit;" + ); +} + +/* main -> global_a (global) -> static_leaf (static, leaf) */ +__naked __noinline __used +static unsigned long static_leaf(void) +{ + asm volatile ( + "r0 = 7;" + "exit;" + ); +} + +__noinline __used +int global_a(int x) +{ + return static_leaf(); +} + +SEC("?raw_tp") +__success __log_level(2) +__msg("topo_order[0] = static_leaf") +__msg("topo_order[1] = global_a") +__msg("topo_order[2] = topo_mixed") +__naked int topo_mixed(void) +{ + asm volatile ( + "r1 = 0;" + "call global_a;" + "exit;" + ); +} + +/* + * shared static callee from global and main: + * main -> shared_leaf (static) + * main -> global_b (global) -> shared_leaf (static) + */ +__naked __noinline __used +static unsigned long shared_leaf(void) +{ + asm volatile ( + "r0 = 99;" + "exit;" + ); +} + +__noinline __used +int global_b(int x) +{ + return shared_leaf(); +} + +SEC("?raw_tp") +__success __log_level(2) +__msg("topo_order[0] = shared_leaf") +__msg("topo_order[1] = global_b") +__msg("topo_order[2] = topo_shared") +__naked int topo_shared(void) +{ + asm volatile ( + "call shared_leaf;" + "r1 = 0;" + "call global_b;" + "exit;" + ); +} + +/* duplicate calls to the same subprog */ +__naked __noinline __used +static unsigned long dup_leaf(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +SEC("?raw_tp") +__success __log_level(2) +__msg("topo_order[0] = dup_leaf") +__msg("topo_order[1] = topo_dup_calls") +__naked int topo_dup_calls(void) +{ + asm volatile ( + "call dup_leaf;" + "call dup_leaf;" + "exit;" + ); +} + +/* main calls bpf_loop() with loop_cb as the callback */ +static int loop_cb(int idx, void *ctx) +{ + return 0; +} + +SEC("?raw_tp") +__success __log_level(2) +__msg("topo_order[0] = loop_cb") +__msg("topo_order[1] = topo_loop_cb") +int topo_loop_cb(void) +{ + bpf_loop(1, loop_cb, NULL, 0); + return 0; +} + +/* + * bpf_loop callback calling another subprog + * main -> bpf_loop(callback=loop_cb2) -> loop_cb2 -> loop_cb2_leaf + */ +__naked __noinline __used +static unsigned long loop_cb2_leaf(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +static int loop_cb2(int idx, void *ctx) +{ + return loop_cb2_leaf(); +} + +SEC("?raw_tp") +__success __log_level(2) +__msg("topo_order[0] = loop_cb2_leaf") +__msg("topo_order[1] = loop_cb2") +__msg("topo_order[2] = topo_loop_cb_chain") +int topo_loop_cb_chain(void) +{ + bpf_loop(1, loop_cb2, NULL, 0); + return 0; +} + +/* no calls (single subprog) */ +SEC("?raw_tp") +__success __log_level(2) +__msg("topo_order[0] = topo_no_calls") +__naked int topo_no_calls(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_subreg.c b/tools/testing/selftests/bpf/progs/verifier_subreg.c index be328100ba53..73b5b0cf6706 100644 --- a/tools/testing/selftests/bpf/progs/verifier_subreg.c +++ b/tools/testing/selftests/bpf/progs/verifier_subreg.c @@ -558,7 +558,8 @@ __description("arsh32 imm sign negative extend check") __success __retval(0) __log_level(2) __msg("3: (17) r6 -= 4095 ; R6=scalar(smin=smin32=-4095,smax=smax32=0)") -__msg("4: (67) r6 <<= 32 ; R6=scalar(smin=0xfffff00100000000,smax=smax32=umax32=0,umax=0xffffffff00000000,smin32=0,var_off=(0x0; 0xffffffff00000000))") +__msg("4: (67) r6 <<= 32 ; R6=scalar(smin=0xfffff00100000000,smax=smax32=umax32=0,smin32=0,var_off=(0x0; 0xffffffff00000000))") +/* represents shorter of signed / unsigned 64-bit ranges */ __msg("5: (c7) r6 s>>= 32 ; R6=scalar(smin=smin32=-4095,smax=smax32=0)") __naked void arsh32_imm_sign_extend_negative_check(void) { @@ -581,7 +582,8 @@ __description("arsh32 imm sign extend check") __success __retval(0) __log_level(2) __msg("3: (17) r6 -= 2047 ; R6=scalar(smin=smin32=-2047,smax=smax32=2048)") -__msg("4: (67) r6 <<= 32 ; R6=scalar(smin=0xfffff80100000000,smax=0x80000000000,umax=0xffffffff00000000,smin32=0,smax32=umax32=0,var_off=(0x0; 0xffffffff00000000))") +__msg("4: (67) r6 <<= 32 ; R6=scalar(smin=0xfffff80100000000,smax=0x80000000000,smin32=0,smax32=umax32=0,var_off=(0x0; 0xffffffff00000000))") +/* represents shorter of signed / unsigned 64-bit ranges */ __msg("5: (c7) r6 s>>= 32 ; R6=scalar(smin=smin32=-2047,smax=smax32=2048)") __naked void arsh32_imm_sign_extend_check(void) { @@ -823,4 +825,169 @@ __naked void arsh_63_or(void) : __clobber_all); } +SEC("socket") +__success __retval(42) +__naked void arsh32_imm1_value(void) +{ + asm volatile (" \ + r0 = 42; \ + r1 = -2147483648; \ + w1 s>>= 1; /* r1 = 0xC0000000 */ \ + r2 = 0xC0000000 ll; \ + if r1 == r2 goto l0_%=; \ + r0 /= 0; /* unreachable */ \ +l0_%=: exit; \ +" : + : + : __clobber_all); +} + +SEC("socket") +__success __retval(1) +__naked void lsh32_reg0_zero_extend_check(void) +{ + asm volatile (" \ + r6 = 1; \ + call %[bpf_get_prandom_u32]; \ + r1 = 0x1000000000 ll; \ + r0 |= r1; \ + w1 = 0; \ + w0 <<= w1; /* reg shift by 0 */ \ + r0 >>= 32; /* must be 0 */ \ + if r0 == 0 goto l0_%=; \ + r6 /= 0; /* unreachable */ \ +l0_%=: r0 = r6; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__success __retval(1) +__naked void rsh32_reg0_zero_extend_check(void) +{ + asm volatile (" \ + r6 = 1; \ + call %[bpf_get_prandom_u32]; \ + r1 = 0x1000000000 ll; \ + r0 |= r1; \ + w1 = 0; \ + w0 >>= w1; /* reg rsh by 0 */ \ + r0 >>= 32; /* must be 0 */ \ + if r0 == 0 goto l0_%=; \ + r6 /= 0; /* unreachable */ \ +l0_%=: r0 = r6; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__success __retval(1) +__naked void arsh32_reg0_zero_extend_check(void) +{ + asm volatile (" \ + r6 = 1; \ + call %[bpf_get_prandom_u32]; \ + r1 = 0x1000000000 ll; \ + r0 |= r1; \ + w1 = 0; \ + w0 s>>= w1; /* reg arsh by 0 */ \ + r0 >>= 32; /* must be 0 */ \ + if r0 == 0 goto l0_%=; \ + r6 /= 0; /* unreachable */ \ +l0_%=: r0 = r6; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__success __retval(42) +__naked void lsh32_imm31_value(void) +{ + asm volatile (" \ + r0 = 42; \ + r1 = 1; \ + w1 <<= 31; /* r1 = 0x80000000 */ \ + r2 = 0x80000000 ll; \ + if r1 == r2 goto l0_%=; \ + r0 /= 0; /* unreachable */ \ +l0_%=: exit; \ +" : + : + : __clobber_all); +} + +SEC("socket") +__success __retval(42) +__naked void rsh32_imm31_value(void) +{ + asm volatile (" \ + r0 = 42; \ + r1 = -2147483648; /* 0x80000000 */ \ + w1 >>= 31; /* r1 = 1 */ \ + if r1 == 1 goto l0_%=; \ + r0 /= 0; /* unreachable */ \ +l0_%=: exit; \ +" : + : + : __clobber_all); +} + +SEC("socket") +__success __retval(42) +__naked void arsh32_imm31_value(void) +{ + asm volatile (" \ + r0 = 42; \ + r1 = -2147483648; /* 0x80000000 */ \ + w1 s>>= 31; /* r1 = 0xFFFFFFFF */ \ + r2 = 0xFFFFFFFF ll; \ + if r1 == r2 goto l0_%=; \ + r0 /= 0; /* unreachable */ \ +l0_%=: exit; \ +" : + : + : __clobber_all); +} + +SEC("socket") +__success __retval(1) +__naked void lsh32_unknown_precise_bounds(void) +{ + asm volatile (" \ + r6 = 1; \ + call %[bpf_get_prandom_u32]; \ + w0 &= 3; /* u32: [0, 3] */ \ + w0 <<= 1; /* u32: [0, 6] */ \ + if w0 < 7 goto l0_%=; \ + r6 /= 0; /* unreachable */ \ +l0_%=: r0 = r6; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__success __retval(1) +__naked void rsh32_unknown_bounds(void) +{ + asm volatile (" \ + r6 = 1; \ + call %[bpf_get_prandom_u32]; \ + w0 >>= 28; /* u32: [0, 15] */ \ + if w0 < 16 goto l0_%=; \ + r6 /= 0; /* unreachable */ \ +l0_%=: r0 = r6; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_tailcall_jit.c b/tools/testing/selftests/bpf/progs/verifier_tailcall_jit.c index 8d60c634a114..48fa34d2959f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_tailcall_jit.c +++ b/tools/testing/selftests/bpf/progs/verifier_tailcall_jit.c @@ -56,6 +56,7 @@ __jited("L1: pushq %rax") /* rbp[-16] = rax */ * (cause original rax might be clobbered by this point) */ __jited(" movq -0x10(%rbp), %rax") +__jited("...") __jited(" callq 0x{{.*}}") /* call to sub() */ __jited(" xorl %eax, %eax") __jited(" leave") diff --git a/tools/testing/selftests/bpf/progs/verifier_uninit.c b/tools/testing/selftests/bpf/progs/verifier_uninit.c index 7718cd7d19ce..691018a46049 100644 --- a/tools/testing/selftests/bpf/progs/verifier_uninit.c +++ b/tools/testing/selftests/bpf/progs/verifier_uninit.c @@ -9,6 +9,7 @@ SEC("socket") __description("read uninitialized register") __failure __msg("R2 !read_ok") +__msg("R2 has never been initialized on this path") __failure_unpriv __naked void read_uninitialized_register(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_unpriv.c b/tools/testing/selftests/bpf/progs/verifier_unpriv.c index 8ee1243e62a8..3069e70fbcbd 100644 --- a/tools/testing/selftests/bpf/progs/verifier_unpriv.c +++ b/tools/testing/selftests/bpf/progs/verifier_unpriv.c @@ -6,6 +6,8 @@ #include "../../../include/linux/filter.h" #include "bpf_misc.h" +extern const int bpf_prog_active __ksym; + #define BPF_SK_LOOKUP(func) \ /* struct bpf_sock_tuple tuple = {} */ \ "r2 = 0;" \ @@ -78,6 +80,41 @@ __naked void dummy_prog_loop1_socket(void) } SEC("socket") +__description("unpriv: pseudo btf id log masks address") +__success_unpriv +__msg_unpriv("0: (18) r1 = 0x0") +__not_msg_unpriv("0: (18) r1 = 0x{{[1-9a-f][0-9a-f]*}}") +__retval_unpriv(0) +__log_level(2) +__naked void pseudo_btf_id_log_masks_address(void) +{ + asm volatile ("r1 = %[bpf_prog_active] ll;" + "r0 = 0;" + "exit;" + : + : __imm_addr(bpf_prog_active) + : __clobber_all); +} + +static int pseudo_func_callback(__u32 index, void *ctx) +{ + return 0; +} + +SEC("socket") +__description("unpriv: pseudo function policy diagnostic") +__success __failure_unpriv +__msg_unpriv("loading/calling other bpf or kernel functions") +__not_msg_unpriv("BPF-to-BPF function call") +__msg_unpriv("policy check failed for BPF function reference") +__msg_unpriv("avoid BPF function references in unprivileged") +int unpriv_pseudo_func_policy(void *ctx) +{ + bpf_loop(1, pseudo_func_callback, NULL, 0); + return 0; +} + +SEC("socket") __description("unpriv: return pointer") __success __failure_unpriv __msg_unpriv("R0 leaks addr") __retval(POINTER_VALUE) @@ -584,7 +621,7 @@ __naked void alu32_mov_u32_const(void) { asm volatile (" \ w7 = 0; \ - w7 &= 1; \ + w7 ^= w7; \ w0 = w7; \ if r0 == 0 goto l0_%=; \ r0 = *(u64*)(r7 + 0); \ @@ -894,7 +931,9 @@ __naked void unpriv_spectre_v1_and_v4_simple(void) { asm volatile (" \ r8 = 0; \ + r8 ^= r8; \ r9 = 0; \ + r9 ^= r9; \ r0 = r10; \ r1 = 0; \ r2 = r10; \ @@ -932,7 +971,9 @@ __naked void unpriv_ldimm64_spectre_v1_and_v4_simple(void) { asm volatile (" \ r8 = 0; \ + r8 ^= r8; \ r9 = 0; \ + r9 ^= r9; \ r0 = r10; \ r1 = 0; \ r2 = r10; \ @@ -972,4 +1013,26 @@ l0_%=: exit; \ : __clobber_all); } +SEC("socket") +__description("unpriv: Spectre v4 stack write slot index") +__success __success_unpriv +__retval(0) +#ifdef SPEC_V4 +__xlated_unpriv("r0 = 0") +__xlated_unpriv("*(u32 *)(r10 -4) = r0") +__xlated_unpriv("nospec") +__xlated_unpriv("*(u32 *)(r10 -8) = r0") +__xlated_unpriv("nospec") +__xlated_unpriv("exit") +#endif +__naked void stack_write_nospec_slot_index(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u32 *)(r10 - 4) = r0; \ + *(u32 *)(r10 - 8) = r0; \ + exit; \ +" ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_value_ptr_arith.c b/tools/testing/selftests/bpf/progs/verifier_value_ptr_arith.c index af7938ce56cb..b3b701b44550 100644 --- a/tools/testing/selftests/bpf/progs/verifier_value_ptr_arith.c +++ b/tools/testing/selftests/bpf/progs/verifier_value_ptr_arith.c @@ -346,7 +346,7 @@ l2_%=: r0 = 1; \ SEC("socket") __description("map access: value_ptr -= known scalar from different maps") __success __failure_unpriv -__msg_unpriv("R0 min value is outside of the allowed memory range") +__msg_unpriv("R0 min value is negative") __retval(1) __naked void known_scalar_from_different_maps(void) { @@ -683,9 +683,7 @@ l0_%=: r0 = 1; \ SEC("socket") __description("map access: value_ptr -= known scalar, lower oob arith, test 1") -__failure __msg("R0 min value is outside of the allowed memory range") -__failure_unpriv -__msg_unpriv("R0 pointer arithmetic of map value goes out of range") +__failure __msg("R0 min value is negative") __naked void lower_oob_arith_test_1(void) { asm volatile (" \ @@ -840,7 +838,7 @@ l0_%=: r0 = 1; \ SEC("socket") __description("map access: value_ptr += known scalar, 3") -__failure __msg("invalid access to map value") +__failure __msg("R0 min value is negative") __failure_unpriv __naked void value_ptr_known_scalar_3(void) { @@ -1207,7 +1205,7 @@ l0_%=: r0 = 1; \ SEC("socket") __description("map access: value_ptr -= known scalar") -__failure __msg("R0 min value is outside of the allowed memory range") +__failure __msg("R0 min value is negative") __failure_unpriv __naked void access_value_ptr_known_scalar(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_var_off.c b/tools/testing/selftests/bpf/progs/verifier_var_off.c index f345466bca68..a63e33675091 100644 --- a/tools/testing/selftests/bpf/progs/verifier_var_off.c +++ b/tools/testing/selftests/bpf/progs/verifier_var_off.c @@ -60,6 +60,116 @@ __naked void stack_read_priv_vs_unpriv(void) } SEC("cgroup/skb") +__description("variable-offset stack read preserves spilled zero") +__success +__log_level(2) +__msg("mark_precise: frame0: regs= stack=-8") +__msg("R3=0") +__retval(0) +__naked void stack_read_var_off_preserves_spilled_zero(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64*)(r10 - 8) = r0; \ + r2 = *(u32*)(r1 + 0); \ + r2 &= 7; \ + r2 -= 8; \ + r2 += r10; \ + r3 = *(u8*)(r2 + 0); \ + r1 = r10; \ + r1 += -1; \ + r1 += r3; \ + *(u8*)(r1 + 0) = r3; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("cgroup/skb") +__description("variable-offset stack read preserves spilled zero across slots") +__success +__log_level(2) +__msg("mark_precise: frame0: regs= stack=-8,-16") +__msg("R3=0") +__retval(0) +__naked void stack_read_var_off_preserves_spilled_zero_across_slots(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64*)(r10 - 8) = r0; \ + *(u64*)(r10 - 16) = r0; \ + r2 = *(u32*)(r1 + 0); \ + r2 &= 15; \ + r2 -= 16; \ + r2 += r10; \ + r3 = *(u8*)(r2 + 0); \ + r1 = r10; \ + r1 += -1; \ + r1 += r3; \ + *(u8*)(r1 + 0) = r3; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("cgroup/skb") +__description("variable-offset stack read preserves partial spilled zero") +__success +__log_level(2) +__msg("mark_precise: frame0: regs= stack=-8") +__msg("R3=0") +__retval(0) +__naked void stack_read_var_off_preserves_partial_spilled_zero(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u8*)(r10 - 9) = r0; \ + *(u8*)(r10 - 10) = r0; \ + *(u8*)(r10 - 11) = r0; \ + *(u8*)(r10 - 12) = r0; \ + *(u8*)(r10 - 13) = r0; \ + *(u8*)(r10 - 14) = r0; \ + *(u8*)(r10 - 15) = r0; \ + *(u32*)(r10 - 8) = r0; \ + r2 = *(u32*)(r1 + 0); \ + r2 &= 15; \ + if r2 > 10 goto l0_%=; \ + r2 -= 15; \ + r2 += r10; \ + r3 = *(u8*)(r2 + 0); \ + r1 = r10; \ + r1 += -1; \ + r1 += r3; \ + *(u8*)(r1 + 0) = r3; \ +l0_%=: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("cgroup/skb") +__description("variable-offset stack read partial spill with misc data") +__failure +__msg("invalid variable-offset write to stack R1") +__naked void stack_read_var_off_partial_spill_with_misc_data(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u32*)(r10 - 8) = r0; \ + r2 = *(u32*)(r1 + 0); \ + r2 &= 7; \ + r2 -= 8; \ + r2 += r10; \ + r3 = *(u8*)(r2 + 0); \ + r1 = r10; \ + r1 += -1; \ + r1 += r3; \ + *(u8*)(r1 + 0) = 0; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("cgroup/skb") __description("variable-offset stack read, uninitialized") __success __failure_unpriv __msg_unpriv("R2 variable stack access prohibited for !root") @@ -88,7 +198,8 @@ __success /* Check that the maximum stack depth is correctly maintained according to the * maximum possible variable offset. */ -__log_level(4) __msg("stack depth 16") +__log_level(4) +__msg("subprog 0 (stack_write_priv_vs_unpriv) main {{.*}} stack 16") __failure_unpriv /* Variable stack access is rejected for unprivileged. */ @@ -128,7 +239,8 @@ __success /* Check that the maximum stack depth is correctly maintained according to the * maximum possible variable offset. */ -__log_level(4) __msg("stack depth 16") +__log_level(4) +__msg("subprog 0 (stack_write_followed_by_read) main {{.*}} stack 16") __failure_unpriv __msg_unpriv("R2 variable stack access prohibited for !root") __retval(0) diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c index 4b392c6c8fc4..8f0c45421f89 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c @@ -13,7 +13,7 @@ static char buf[PATH_MAX]; SEC("lsm.s/file_open") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(get_task_exe_file_kfunc_null) { struct file *acquired; @@ -28,7 +28,7 @@ int BPF_PROG(get_task_exe_file_kfunc_null) } SEC("lsm.s/inode_getxattr") -__failure __msg("arg#0 pointer type STRUCT task_struct must point to scalar, or struct with scalar") +__failure __msg("R1 is fp expected STRUCT task_struct") int BPF_PROG(get_task_exe_file_kfunc_fp) { u64 x; @@ -80,7 +80,7 @@ int BPF_PROG(get_task_exe_file_kfunc_unreleased) } SEC("lsm.s/file_open") -__failure __msg("release kernel function bpf_put_file expects") +__failure __msg("release kfunc bpf_put_file expects referenced PTR_TO_BTF_ID passed to R1") int BPF_PROG(put_file_kfunc_unacquired, struct file *file) { /* Can't release an unacquired pointer. */ @@ -89,7 +89,7 @@ int BPF_PROG(put_file_kfunc_unacquired, struct file *file) } SEC("lsm.s/file_open") -__failure __msg("Possibly NULL pointer passed to trusted arg0") +__failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(path_d_path_kfunc_null) { /* Can't pass NULL value to bpf_path_d_path() kfunc. */ @@ -98,7 +98,7 @@ int BPF_PROG(path_d_path_kfunc_null) } SEC("lsm.s/task_alloc") -__failure __msg("R1 must be referenced or trusted") +__failure __msg("dereference of modified untrusted_ptr_") int BPF_PROG(path_d_path_kfunc_untrusted_from_argument, struct task_struct *task) { struct path *root; @@ -112,7 +112,7 @@ int BPF_PROG(path_d_path_kfunc_untrusted_from_argument, struct task_struct *task } SEC("lsm.s/file_open") -__failure __msg("R1 must be referenced or trusted") +__failure __msg("dereference of modified untrusted_ptr_") int BPF_PROG(path_d_path_kfunc_untrusted_from_current) { struct path *pwd; @@ -128,7 +128,7 @@ int BPF_PROG(path_d_path_kfunc_untrusted_from_current) } SEC("lsm.s/file_open") -__failure __msg("kernel function bpf_path_d_path args#0 expected pointer to STRUCT path but R1 has a pointer to STRUCT file") +__failure __msg("kernel function bpf_path_d_path R1 expected pointer to STRUCT path but R1 has a pointer to STRUCT file") int BPF_PROG(path_d_path_kfunc_type_mismatch, struct file *file) { bpf_path_d_path((struct path *)&file->f_task_work, buf, sizeof(buf)); diff --git a/tools/testing/selftests/bpf/progs/verifier_xdp_direct_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_xdp_direct_packet_access.c index df2dfd1b15d1..9866bc154194 100644 --- a/tools/testing/selftests/bpf/progs/verifier_xdp_direct_packet_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_xdp_direct_packet_access.c @@ -69,7 +69,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_data' > pkt_end, bad access 1") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void pkt_end_bad_access_1_1(void) { @@ -131,7 +131,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_data' > pkt_end, corner case -1, bad access") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void corner_case_1_bad_access_1(void) { @@ -173,7 +173,7 @@ l1_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_end > pkt_data', corner case -1, bad access") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void corner_case_1_bad_access_2(void) { @@ -279,7 +279,7 @@ l1_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_data' < pkt_end, corner case -1, bad access") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void corner_case_1_bad_access_3(void) { @@ -384,7 +384,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_end < pkt_data', bad access 1") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void pkt_data_bad_access_1_1(void) { @@ -446,7 +446,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_end < pkt_data', corner case -1, bad access") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void corner_case_1_bad_access_4(void) { @@ -487,7 +487,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_data' >= pkt_end, corner case -1, bad access") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void corner_case_1_bad_access_5(void) { @@ -590,7 +590,7 @@ l1_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_end >= pkt_data', bad access 1") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void pkt_data_bad_access_1_2(void) { @@ -654,7 +654,7 @@ l1_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_end >= pkt_data', corner case -1, bad access") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void corner_case_1_bad_access_6(void) { @@ -697,7 +697,7 @@ l1_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_data' <= pkt_end, bad access 1") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void pkt_end_bad_access_1_2(void) { @@ -761,7 +761,7 @@ l1_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_data' <= pkt_end, corner case -1, bad access") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void corner_case_1_bad_access_7(void) { @@ -803,7 +803,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_end <= pkt_data', corner case -1, bad access") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void corner_case_1_bad_access_8(void) { @@ -905,7 +905,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_meta' > pkt_data, bad access 1") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void pkt_data_bad_access_1_3(void) { @@ -926,7 +926,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_meta' > pkt_data, bad access 2") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void pkt_data_bad_access_2_5(void) { @@ -967,7 +967,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_meta' > pkt_data, corner case -1, bad access") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void corner_case_1_bad_access_9(void) { @@ -1009,7 +1009,7 @@ l1_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_data > pkt_meta', corner case -1, bad access") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void corner_case_1_bad_access_10(void) { @@ -1031,7 +1031,7 @@ l1_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_data > pkt_meta', bad access 2") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void pkt_meta_bad_access_2_1(void) { @@ -1115,7 +1115,7 @@ l1_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_meta' < pkt_data, corner case -1, bad access") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void corner_case_1_bad_access_11(void) { @@ -1137,7 +1137,7 @@ l1_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_meta' < pkt_data, bad access 2") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void pkt_data_bad_access_2_6(void) { @@ -1220,7 +1220,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_data < pkt_meta', bad access 1") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void pkt_meta_bad_access_1_1(void) { @@ -1241,7 +1241,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_data < pkt_meta', bad access 2") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void pkt_meta_bad_access_2_2(void) { @@ -1282,7 +1282,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_data < pkt_meta', corner case -1, bad access") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void corner_case_1_bad_access_12(void) { @@ -1323,7 +1323,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_meta' >= pkt_data, corner case -1, bad access") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void corner_case_1_bad_access_13(void) { @@ -1344,7 +1344,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_meta' >= pkt_data, bad access 2") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void pkt_data_bad_access_2_7(void) { @@ -1426,7 +1426,7 @@ l1_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_data >= pkt_meta', bad access 1") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void pkt_meta_bad_access_1_2(void) { @@ -1448,7 +1448,7 @@ l1_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_data >= pkt_meta', bad access 2") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void pkt_meta_bad_access_2_3(void) { @@ -1490,7 +1490,7 @@ l1_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_data >= pkt_meta', corner case -1, bad access") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void corner_case_1_bad_access_14(void) { @@ -1533,7 +1533,7 @@ l1_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_meta' <= pkt_data, bad access 1") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void pkt_data_bad_access_1_4(void) { @@ -1555,7 +1555,7 @@ l1_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_meta' <= pkt_data, bad access 2") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void pkt_data_bad_access_2_8(void) { @@ -1597,7 +1597,7 @@ l1_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_meta' <= pkt_data, corner case -1, bad access") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void corner_case_1_bad_access_15(void) { @@ -1639,7 +1639,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_data <= pkt_meta', corner case -1, bad access") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void corner_case_1_bad_access_16(void) { @@ -1660,7 +1660,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("XDP pkt read, pkt_data <= pkt_meta', bad access 2") -__failure __msg("R1 offset is outside of the packet") +__failure __msg("R1 {{min|max}} value is outside of the allowed memory range") __flag(BPF_F_ANY_ALIGNMENT) __naked void pkt_meta_bad_access_2_4(void) { @@ -1719,4 +1719,39 @@ l0_%=: r0 = 0; \ : __clobber_all); } +SEC("xdp") +__description("XDP pkt regsafe preserves packet pointer class displacement") +__failure __msg("R2 min value is outside of the allowed memory range") +__flag(BPF_F_ANY_ALIGNMENT) __flag(BPF_F_TEST_STATE_FREQ) +__naked void pkt_regsafe_class_displacement(void) +{ + asm volatile (" \ + r8 = *(u32 *)(r1 + %[xdp_md_data_end]); \ + r9 = *(u32 *)(r1 + %[xdp_md_data]); \ + r4 = *(u32 *)(r1 + %[xdp_md_rx_queue_index]); \ + r4 &= 15; \ + r0 = *(u32 *)(r1 + %[xdp_md_ingress_ifindex]); \ + if r0 != 0 goto l0_%=; \ + r2 = r9; \ + r2 += r4; \ + r3 = r2; \ + r3 += 8; \ + goto l1_%=; \ +l0_%=: r4 &= 3; \ + r4 += 8; \ + r2 = r9; \ + r2 += r4; \ + r3 = r2; \ +l1_%=: if r3 > r8 goto l2_%=; \ + r0 = *(u64 *)(r2 + 0); \ +l2_%=: r0 = 0; \ + exit; \ +" : + : __imm_const(xdp_md_data, offsetof(struct xdp_md, data)), + __imm_const(xdp_md_data_end, offsetof(struct xdp_md, data_end)), + __imm_const(xdp_md_rx_queue_index, offsetof(struct xdp_md, rx_queue_index)), + __imm_const(xdp_md_ingress_ifindex, offsetof(struct xdp_md, ingress_ifindex)) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_zext.c b/tools/testing/selftests/bpf/progs/verifier_zext.c new file mode 100644 index 000000000000..572017fe28fb --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_zext.c @@ -0,0 +1,418 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "../../../include/linux/filter.h" +#include <bpf_arena_common.h> +#include <bpf/bpf_core_read.h> +#include "bpf_misc.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE | BPF_F_NO_USER_CONV); + __uint(max_entries, 1); +} arena SEC(".maps"); + +extern long bpf_kfunc_call_test4(signed char a, short b, int c, long d) __ksym; + +/* to retain debug info for BTF generation */ +void __kfunc_btf_root(void) +{ + bpf_kfunc_call_test4(0, 0, 0, 0); + bpf_arena_alloc_pages(0, 0, 0, 0, 0); + bpf_rdonly_cast(0, 0); +} + +SEC("socket") +__flag(BPF_F_TEST_STATE_FREQ) +__flag(BPF_F_TEST_RND_HI32) +__success __retval(0) +__naked void zext_lost_across_checkpoint(void) +{ + asm volatile (" \ + call %[bpf_ktime_get_ns]; \ + r8 = r0; \ + r6 = 0xdeadbeefcafebabe ll; /* inject some value for r6's upper half */ \ + if r8 != 0 goto 1f; /* fall-through cached first, branch pruned */ \ + r6 = 32; /* full 64-bit def */ \ + goto 2f; \ +1: w6 = 32; /* 32-bit def, zext mark lost */ \ +2: r0 = r6; /* buggy verifier believed upper 32 bits are 0 */ \ + /* and thus did not zero extended w6 = 32. */ \ + r0 >>= 32; \ + exit; \ +" : + : __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +/* 32-bit ALU result read as 64-bit -> zext */ +SEC("socket") +__success __log_level(2) +__msg("w1 = w0{{ +}}; zext") +__naked void zext_alu32_hi_used(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w1 = w0; \ + r0 = r1; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* 32-bit ALU result read only as 32-bit -> no zext */ +SEC("socket") +__success __log_level(2) +__not_msg("; zext") +__naked void no_zext_alu32_hi_unused(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w1 = w0; /* MOV */ \ + w2 = w1; \ + w2 += w1; /* ALU32, BPF_X */ \ + w2 += 1; /* ALU32, BPF_K */ \ + w2 = w2; /* keep w2 alive for previous instruction */ \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* 64-bit definition is never zero extended */ +SEC("socket") +__success __log_level(2) +__not_msg("r1 = r0{{.*}}; zext") +__naked void no_zext_mov64(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = r0; \ + r0 = r1; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Narrow load result read as 64-bit -> zext */ +SEC("socket") +__success __log_level(2) +__msg("r1 = *(u32 *)(r10 -8){{ +}}; zext") +__naked void zext_narrow_load_hi_used(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64 *)(r10 - 8) = r0; \ + r1 = *(u32 *)(r10 - 8); \ + r0 = r1; \ + exit; \ +" ::: __clobber_all); +} + +/* 32-bit atomic fetch result read as 64-bit -> zext */ +SEC("socket") +__success __log_level(2) +__msg("r1 = atomic_fetch_add((u32 *)(r10 -8), r1){{ +}}; zext") +__naked void zext_atomic_fetch32_hi_used(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64 *)(r10 - 8) = r1; \ + w1 = 1; \ + .8byte %[fetch_add32]; \ + r0 = r1; \ + exit; \ +" : + : __imm_insn(fetch_add32, + BPF_ATOMIC_OP(BPF_W, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_1, -8)) + : __clobber_all); +} + +/* 32-bit atomic cmpxchg result (r0) read as 64-bit -> zext */ +SEC("socket") +__success __log_level(2) +__msg("r0 = atomic_cmpxchg((u32 *)(r10 -8), r0, r1){{ +}}; zext") +__naked void zext_cmpxchg32_hi_used(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64 *)(r10 - 8) = r1; \ + w0 = 0; \ + w1 = 1; \ + .8byte %[cmpxchg32]; \ + r2 = r0; \ + r0 = r2; \ + exit; \ +" : + : __imm_insn(cmpxchg32, + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, BPF_REG_10, BPF_REG_1, -8)) + : __clobber_all); +} + +/* 32-bit def before a branch, upper half used on one branch -> zext */ +SEC("socket") +__success __log_level(2) +__msg("w6 = 32{{ +}}; zext") +__naked void zext_cfg_hi_used_one_branch(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w6 = 32; \ + if r0 == 0 goto 1f; \ + r0 = r6; \ + exit; \ +1: r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* r1's upper half is dead, so 'w1 = 1' must NOT be marked for zero extension. */ +SEC("socket") +__success __log_level(2) +__not_msg("w1 = 1{{.*}}; zext") +__naked void no_zext_other_reg_hi_used(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + r6 <<= 32; \ + w1 = 1; \ + r0 = r6; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* LD_ABS defines r0; when r0 is read as 64-bit it must be zero extended */ +SEC("socket") +__success __log_level(2) +__msg("r0 = *(u8 *)skb[0]{{.*}}; zext") +__naked void zext_ld_abs_hi_used(void) +{ + asm volatile (" \ + r6 = r1; \ + r0 = *(u8 *)skb[0]; \ + r7 = r0; \ + r0 = r7; \ + exit; \ +" ::: __clobber_all); +} + +/* Helper parameters are read as 64-bit (call_use_mask() fallback) */ +SEC("socket") +__success __log_level(2) +__msg("w2 = 1{{ +}}; zext") +__naked void helper_param_read_as_64bit(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -8; \ + w2 = 1; \ + call %[bpf_trace_printk]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_trace_printk) + : __clobber_all); +} + +static __used __naked int subprog_reads_arg_as_64bit(void) +{ + asm volatile (" \ + r0 = r1; \ + exit; \ +" ::: __clobber_all); +} + +/* subprogram parameters are conservatively read as 64-bit */ +SEC("socket") +__success __log_level(2) +__msg("w1 = w0{{ +}}; zext") +__naked void subprog_param_read_as_64bit(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w1 = w0; \ + call subprog_reads_arg_as_64bit; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* kfunc parameters are zero extended */ +SEC("tc") +__success __log_level(2) +__msg("w1 = 1{{ +}}; zext") +__msg("w2 = 1{{ +}}; zext") +__msg("w3 = 1{{ +}}; zext") +__msg("w4 = 1{{ +}}; zext") +__naked void kfunc_param_read_per_btf(void) +{ + asm volatile (" \ + w1 = 1; \ + w2 = 1; \ + w3 = 1; \ + w4 = 1; \ + call bpf_kfunc_call_test4; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__success __log_level(2) +__not_msg("; zext") +__naked void alu32_and_32bit_conditional(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w1 = w0; \ + if w1 > 42 goto 1f; /* BPF_K */ \ + w2 = 28; \ + if w2 > w1 goto 1f; /* BPF_X */ \ + r0 = 0; \ +1: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__success __log_level(2) +__msg("w1 = w0{{ +}}; zext") +__naked void alu32_and_64bit_conditional(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w1 = w0; \ + if r1 > 42 goto 1f; /* BPF_K */ \ + r2 = 28; \ + if r2 > r1 goto 1f; /* BPF_X */ \ + r0 = 0; \ +1: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__success __log_level(2) +__not_msg("; zext") +__naked void alu64_and_conditionals(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = r0; \ + if w1 > 42 goto 1f; /* BPF_K */ \ + if r1 > 42 goto 1f; /* BPF_K */ \ + r2 = 28; \ + if w2 > w1 goto 1f; /* BPF_X */ \ + if r2 > r1 goto 1f; /* BPF_X */ \ + r0 = 0; \ +1: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__arch_s390x +__xlated("7: w1 = w0") +__xlated("8: w1 = w1") +__xlated("9: w1 += 8") +__xlated("10: w1 = w1") +__xlated("11: w2 = w1") +__xlated("12: w2 = w2") +__xlated("13: *(u64 *)(r1 +0) = r2") +__naked void arena_ptr(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = 0; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r1 = addr_space_cast(r0, 0, 1); /* needs zext */ \ + r1 += 8; /* needs zext */ \ + r2 = addr_space_cast(r1, 1, 0); /* needs zext because of BPF_F_NO_USER_CONV */ \ + *(u64 *)(r1 +0) = r2; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_arena_alloc_pages), + __imm_addr(arena) + : __clobber_all); +} + +/* + * Result of a 32-bit cmpxchg is always explicitly zero extended. + * Check that this holds for arenas (BPF_PROBE_ATOMIC instruction flavor). + */ +SEC("socket") +__success +__xlated("probe r0 = atomic_cmpxchg((u32 *)(r1 +0), r0, r2)") +__xlated("w0 = w0") +__naked void zext_arena_cmpxchg32(void) +{ + asm volatile (" \ + r9 = %[arena] ll; /* associate the arena with the program */ \ + r1 = 0; \ + r1 = addr_space_cast(r1, 0, 1); \ + r0 = 0; \ + r2 = 0; \ + .8byte %[cmpxchg32]; \ + r0 >>= 32; /* make the upper half live */ \ + exit; \ +" : + : __imm_addr(arena), + __imm_insn(cmpxchg32, + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, BPF_REG_1, BPF_REG_2, 0)) + : __clobber_all); +} + +#endif + +/* Check if probe mem loads keep their zero extension. */ +SEC("socket") +__success __log_level(2) +__arch_s390x +__xlated("3: r1 = *(u64 *)(r0 +0)") +__xlated("4: r2 = *(u32 *)(r0 +0)") +__xlated("5: w2 = w2") +__xlated("6: r3 = *(u16 *)(r0 +0)") +__xlated("7: w3 = w3") +__xlated("8: r4 = *(u8 *)(r0 +0)") +__xlated("9: w4 = w4") +__naked void probe_mem(void) +{ + asm volatile (" \ + r1 = 0; \ + r2 = 0; \ + call %[bpf_rdonly_cast]; \ + r1 = *(u64 *)(r0 + 0); /* BPF_PROBE_MEM */ \ + r2 = *(u32 *)(r0 + 0); /* BPF_PROBE_MEM */ \ + r3 = *(u16 *)(r0 + 0); /* BPF_PROBE_MEM */ \ + r4 = *(u8 *)(r0 + 0); /* BPF_PROBE_MEM */ \ + r0 = r1; /* make the registers used */ \ + r0 += r2; \ + r0 += r3; \ + r0 += r4; \ +1: exit; \ +" : + : __imm(bpf_rdonly_cast) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/veristat_bar.c b/tools/testing/selftests/bpf/progs/veristat_bar.c new file mode 100644 index 000000000000..83d2a2a1dfc9 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/veristat_bar.c @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include "veristat_foo.c" diff --git a/tools/testing/selftests/bpf/progs/veristat_foo.c b/tools/testing/selftests/bpf/progs/veristat_foo.c new file mode 100644 index 000000000000..bd24b97664b4 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/veristat_foo.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +/* + * Programs below exist only to exercise veristat's -f name filters, + * their bodies are irrelevant, only the names matter. + * This file is also included by veristat_bar.c, so that the same set of + * program names is available in two differently named object files. + */ + +SEC("socket") +int foo(void *ctx) +{ + return 0; +} + +SEC("socket") +int bar(void *ctx) +{ + return 0; +} + +SEC("socket") +int buz(void *ctx) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/wakeup_source.h b/tools/testing/selftests/bpf/progs/wakeup_source.h new file mode 100644 index 000000000000..cd74de92c82f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/wakeup_source.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright 2026 Google LLC */ + +#ifndef __WAKEUP_SOURCE_H__ +#define __WAKEUP_SOURCE_H__ + +#define WAKEUP_NAME_LEN 128 + +struct wakeup_event_t { + unsigned long active_count; + long long active_time_ns; + unsigned long event_count; + unsigned long expire_count; + long long last_time_ns; + long long max_time_ns; + long long prevent_sleep_time_ns; + long long total_time_ns; + unsigned long wakeup_count; + char name[WAKEUP_NAME_LEN]; +}; + +#endif /* __WAKEUP_SOURCE_H__ */ diff --git a/tools/testing/selftests/bpf/progs/wakeup_source_fail.c b/tools/testing/selftests/bpf/progs/wakeup_source_fail.c new file mode 100644 index 000000000000..d4d0f1610853 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/wakeup_source_fail.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright 2026 Google LLC */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +struct bpf_ws_lock; + +struct bpf_ws_lock *bpf_wakeup_sources_read_lock(void) __ksym; +void bpf_wakeup_sources_read_unlock(struct bpf_ws_lock *lock) __ksym; +void *bpf_wakeup_sources_get_head(void) __ksym; + +SEC("syscall") +__failure __msg("BPF_EXIT instruction in main prog would lead to reference leak") +int wakeup_source_lock_no_unlock(void *ctx) +{ + struct bpf_ws_lock *lock; + + lock = bpf_wakeup_sources_read_lock(); + if (!lock) + return 0; + + return 0; +} + +SEC("syscall") +__failure __msg("access beyond struct") +int wakeup_source_access_lock_fields(void *ctx) +{ + struct bpf_ws_lock *lock; + int val; + + lock = bpf_wakeup_sources_read_lock(); + if (!lock) + return 0; + + val = *(int *)lock; + + bpf_wakeup_sources_read_unlock(lock); + return val; +} + +SEC("syscall") +__failure __msg("release kfunc bpf_wakeup_sources_read_unlock expects referenced PTR_TO_BTF_ID passed to R1") +int wakeup_source_unlock_no_lock(void *ctx) +{ + struct bpf_ws_lock *lock = (void *)0x1; + + bpf_wakeup_sources_read_unlock(lock); + + return 0; +} + +SEC("syscall") +__failure __msg("Possibly NULL pointer passed to trusted") +int wakeup_source_unlock_null(void *ctx) +{ + bpf_wakeup_sources_read_unlock(NULL); + + return 0; +} + +SEC("syscall") +__failure __msg("R0 invalid mem access 'scalar'") +int wakeup_source_unsafe_dereference(void *ctx) +{ + struct list_head *head = bpf_wakeup_sources_get_head(); + + if (head->next) + return 1; + + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/wq_failures.c b/tools/testing/selftests/bpf/progs/wq_failures.c index 3767f5595bbc..32dc8827e128 100644 --- a/tools/testing/selftests/bpf/progs/wq_failures.c +++ b/tools/testing/selftests/bpf/progs/wq_failures.c @@ -98,7 +98,7 @@ __failure * is a correct bpf_wq pointer. */ __msg(": (85) call bpf_wq_set_callback#") /* anchor message */ -__msg("arg#0 doesn't point to a map value") +__msg("R1 doesn't point to a map value") long test_wrong_wq_pointer(void *ctx) { int key = 0; diff --git a/tools/testing/selftests/bpf/progs/xdp_dummy.c b/tools/testing/selftests/bpf/progs/xdp_dummy.c index d988b2e0cee8..5f1e0771021d 100644 --- a/tools/testing/selftests/bpf/progs/xdp_dummy.c +++ b/tools/testing/selftests/bpf/progs/xdp_dummy.c @@ -10,4 +10,10 @@ int xdp_dummy_prog(struct xdp_md *ctx) return XDP_PASS; } +SEC("xdp") +int __x64_sys_nop(struct xdp_md *ctx) +{ + return XDP_PASS; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/xdp_flowtable.c b/tools/testing/selftests/bpf/progs/xdp_flowtable.c index 7fdc7b23ee74..e67daa02749d 100644 --- a/tools/testing/selftests/bpf/progs/xdp_flowtable.c +++ b/tools/testing/selftests/bpf/progs/xdp_flowtable.c @@ -15,7 +15,10 @@ struct bpf_flowtable_opts___local { s32 error; }; -struct flow_offload_tuple_rhash * +struct flow_offload_tuple_rhash___local { +}; + +struct flow_offload_tuple_rhash___local * bpf_xdp_flow_lookup(struct xdp_md *, struct bpf_fib_lookup *, struct bpf_flowtable_opts___local *, u32) __ksym; @@ -67,7 +70,7 @@ int xdp_flowtable_do_lookup(struct xdp_md *ctx) { void *data_end = (void *)(long)ctx->data_end; struct bpf_flowtable_opts___local opts = {}; - struct flow_offload_tuple_rhash *tuplehash; + struct flow_offload_tuple_rhash___local *tuplehash; struct bpf_fib_lookup tuple = { .ifindex = ctx->ingress_ifindex, }; diff --git a/tools/testing/selftests/bpf/progs/xdp_lb_bench.c b/tools/testing/selftests/bpf/progs/xdp_lb_bench.c new file mode 100644 index 000000000000..13777b3dcac8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/xdp_lb_bench.c @@ -0,0 +1,647 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <stddef.h> +#include <stdbool.h> +#include <linux/bpf.h> +#include <linux/if_ether.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/in.h> +#include <linux/tcp.h> +#include <linux/udp.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> +#include "bpf_compiler.h" +#include "xdp_lb_bench_common.h" +#include "bench_bpf_timing.bpf.h" + +#ifndef IPPROTO_FRAGMENT +#define IPPROTO_FRAGMENT 44 +#endif + +/* jhash helpers */ + +static inline __u32 rol32(__u32 word, unsigned int shift) +{ + return (word << shift) | (word >> ((-shift) & 31)); +} + +#define __jhash_mix(a, b, c) \ +{ \ + a -= c; a ^= rol32(c, 4); c += b; \ + b -= a; b ^= rol32(a, 6); a += c; \ + c -= b; c ^= rol32(b, 8); b += a; \ + a -= c; a ^= rol32(c, 16); c += b; \ + b -= a; b ^= rol32(a, 19); a += c; \ + c -= b; c ^= rol32(b, 4); b += a; \ +} + +#define __jhash_final(a, b, c) \ +{ \ + c ^= b; c -= rol32(b, 14); \ + a ^= c; a -= rol32(c, 11); \ + b ^= a; b -= rol32(a, 25); \ + c ^= b; c -= rol32(b, 16); \ + a ^= c; a -= rol32(c, 4); \ + b ^= a; b -= rol32(a, 14); \ + c ^= b; c -= rol32(b, 24); \ +} + +#define JHASH_INITVAL 0xdeadbeef + +static inline __u32 __jhash_nwords(__u32 a, __u32 b, __u32 c, __u32 initval) +{ + a += initval; + b += initval; + c += initval; + __jhash_final(a, b, c); + return c; +} + +static inline __u32 jhash_2words(__u32 a, __u32 b, __u32 initval) +{ + return __jhash_nwords(a, b, 0, initval + JHASH_INITVAL + (2 << 2)); +} + +static inline __u32 jhash2_4words(const __u32 *k, __u32 initval) +{ + __u32 a, b, c; + + a = b = c = JHASH_INITVAL + (4 << 2) + initval; + + a += k[0]; b += k[1]; c += k[2]; + __jhash_mix(a, b, c); + + a += k[3]; + __jhash_final(a, b, c); + + return c; +} + +static __always_inline void ipv4_csum(struct iphdr *iph) +{ + __u16 *next_iph = (__u16 *)iph; + __u32 csum = 0; + int i; + + __pragma_loop_unroll_full + for (i = 0; i < (int)(sizeof(*iph) >> 1); i++) + csum += *next_iph++; + + csum = (csum & 0xffff) + (csum >> 16); + csum = (csum & 0xffff) + (csum >> 16); + iph->check = ~csum; +} + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 64); + __type(key, struct vip_definition); + __type(value, struct vip_meta); +} vip_map SEC(".maps"); + +struct lru_inner_map { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __type(key, struct flow_key); + __type(value, struct real_pos_lru); + __uint(max_entries, DEFAULT_LRU_SIZE); +} lru_inner SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __type(key, __u32); + __type(value, __u32); + __uint(max_entries, BENCH_NR_CPUS); + __array(values, struct lru_inner_map); +} lru_mapping SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, CH_RINGS_SIZE); + __type(key, __u32); + __type(value, __u32); +} ch_rings SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, MAX_REALS); + __type(key, __u32); + __type(value, struct real_definition); +} reals SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, STATS_SIZE); + __type(key, __u32); + __type(value, struct lb_stats); +} stats SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, MAX_REALS); + __type(key, __u32); + __type(value, struct lb_stats); +} reals_stats SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct ctl_value); +} ctl_array SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct vip_definition); +} vip_miss_stats SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, MAX_REALS); + __type(key, __u32); + __type(value, __u32); +} lru_miss_stats SEC(".maps"); + +volatile __u32 flow_mask; +volatile __u32 cold_lru; +__u32 batch_gen; + +/* + * old_eth MUST be read BEFORE writing the outer header because + * bpf_xdp_adjust_head makes them overlap. + */ +static __always_inline int encap_v4(struct xdp_md *xdp, __be32 saddr, __be32 daddr, + __u16 payload_len, const __u8 *dst_mac) +{ + struct ethhdr *new_eth, *old_eth; + void *data, *data_end; + struct iphdr *iph; + + if (bpf_xdp_adjust_head(xdp, -(int)sizeof(struct iphdr))) + return -1; + + data = (void *)(long)xdp->data; + data_end = (void *)(long)xdp->data_end; + + new_eth = data; + iph = data + sizeof(struct ethhdr); + old_eth = data + sizeof(struct iphdr); + + if (new_eth + 1 > data_end || old_eth + 1 > data_end || iph + 1 > data_end) + return -1; + + __builtin_memcpy(new_eth->h_source, old_eth->h_dest, sizeof(new_eth->h_source)); + __builtin_memcpy(new_eth->h_dest, dst_mac, sizeof(new_eth->h_dest)); + new_eth->h_proto = bpf_htons(ETH_P_IP); + + __builtin_memset(iph, 0, sizeof(*iph)); + iph->version = 4; + iph->ihl = sizeof(*iph) >> 2; + iph->protocol = IPPROTO_IPIP; + iph->tot_len = bpf_htons(payload_len + sizeof(*iph)); + iph->ttl = 64; + iph->saddr = saddr; + iph->daddr = daddr; + ipv4_csum(iph); + + return 0; +} + +static __always_inline int encap_v6(struct xdp_md *xdp, const __be32 saddr[4], + const __be32 daddr[4], __u8 nexthdr, __u16 payload_len, + const __u8 *dst_mac) +{ + struct ethhdr *new_eth, *old_eth; + void *data, *data_end; + struct ipv6hdr *ip6h; + + if (bpf_xdp_adjust_head(xdp, -(int)sizeof(struct ipv6hdr))) + return -1; + + data = (void *)(long)xdp->data; + data_end = (void *)(long)xdp->data_end; + + new_eth = data; + ip6h = data + sizeof(struct ethhdr); + old_eth = data + sizeof(struct ipv6hdr); + + if (new_eth + 1 > data_end || old_eth + 1 > data_end || ip6h + 1 > data_end) + return -1; + + __builtin_memcpy(new_eth->h_source, old_eth->h_dest, sizeof(new_eth->h_source)); + __builtin_memcpy(new_eth->h_dest, dst_mac, sizeof(new_eth->h_dest)); + new_eth->h_proto = bpf_htons(ETH_P_IPV6); + + __builtin_memset(ip6h, 0, sizeof(*ip6h)); + ip6h->version = 6; + ip6h->nexthdr = nexthdr; + ip6h->payload_len = bpf_htons(payload_len); + ip6h->hop_limit = 64; + __builtin_memcpy(&ip6h->saddr, saddr, sizeof(ip6h->saddr)); + __builtin_memcpy(&ip6h->daddr, daddr, sizeof(ip6h->daddr)); + + return 0; +} + +static __always_inline void update_stats(void *map, __u32 key, __u16 bytes) +{ + struct lb_stats *st = bpf_map_lookup_elem(map, &key); + + if (st) { + st->v1 += 1; + st->v2 += bytes; + } +} + +static __always_inline void count_action(int action) +{ + struct lb_stats *st; + __u32 key; + + if (action == XDP_TX) + key = STATS_XDP_TX; + else if (action == XDP_PASS) + key = STATS_XDP_PASS; + else + key = STATS_XDP_DROP; + + st = bpf_map_lookup_elem(&stats, &key); + if (st) + st->v1 += 1; +} + +static __always_inline bool is_under_flood(void) +{ + __u32 key = STATS_NEW_CONN; + struct lb_stats *conn_st = bpf_map_lookup_elem(&stats, &key); + __u64 cur_time; + + if (!conn_st) + return true; + + cur_time = bpf_ktime_get_ns(); + if ((cur_time - conn_st->v2) > ONE_SEC) { + conn_st->v1 = 1; + conn_st->v2 = cur_time; + } else { + conn_st->v1 += 1; + if (conn_st->v1 > MAX_CONN_RATE) + return true; + } + return false; +} + +static __always_inline struct real_definition *connection_table_lookup(void *lru_map, + struct flow_key *flow, + __u32 *out_pos) +{ + struct real_pos_lru *dst_lru; + struct real_definition *real; + __u32 key; + + dst_lru = bpf_map_lookup_elem(lru_map, flow); + if (!dst_lru) + return NULL; + + /* UDP connections use atime-based timeout instead of FIN/RST */ + if (flow->proto == IPPROTO_UDP) { + __u64 cur_time = bpf_ktime_get_ns(); + + if (cur_time - dst_lru->atime > LRU_UDP_TIMEOUT) + return NULL; + dst_lru->atime = cur_time; + } + + key = dst_lru->pos; + *out_pos = key; + real = bpf_map_lookup_elem(&reals, &key); + return real; +} + +static __always_inline bool get_packet_dst(struct real_definition **real, struct flow_key *flow, + struct vip_meta *vip_info, bool is_v6, void *lru_map, + bool is_rst, __u32 *out_pos) +{ + bool under_flood; + __u32 hash, ch_key; + __u32 *ch_val; + __u32 real_pos; + + under_flood = is_under_flood(); + + if (is_v6) { + __u32 src_hash = jhash2_4words((__u32 *)flow->srcv6, MAX_VIPS); + + hash = jhash_2words(src_hash, flow->ports, CH_RING_SIZE); + } else { + hash = jhash_2words(flow->src, flow->ports, CH_RING_SIZE); + } + + ch_key = CH_RING_SIZE * vip_info->vip_num + hash % CH_RING_SIZE; + ch_val = bpf_map_lookup_elem(&ch_rings, &ch_key); + if (!ch_val) + return false; + real_pos = *ch_val; + + *real = bpf_map_lookup_elem(&reals, &real_pos); + if (!(*real)) + return false; + + if (!(vip_info->flags & F_LRU_BYPASS) && !under_flood && !is_rst) { + struct real_pos_lru new_lru = { .pos = real_pos }; + + if (flow->proto == IPPROTO_UDP) + new_lru.atime = bpf_ktime_get_ns(); + bpf_map_update_elem(lru_map, flow, &new_lru, BPF_ANY); + } + + *out_pos = real_pos; + return true; +} + +static __always_inline void update_vip_lru_miss_stats(struct vip_definition *vip, bool is_v6, + __u32 real_idx) +{ + struct vip_definition *miss_vip; + __u32 key = 0; + __u32 *cnt; + + miss_vip = bpf_map_lookup_elem(&vip_miss_stats, &key); + if (!miss_vip) + return; + + if (is_v6) { + if (miss_vip->vipv6[0] != vip->vipv6[0] || miss_vip->vipv6[1] != vip->vipv6[1] || + miss_vip->vipv6[2] != vip->vipv6[2] || miss_vip->vipv6[3] != vip->vipv6[3]) + return; + } else { + if (miss_vip->vip != vip->vip) + return; + } + + if (miss_vip->port != vip->port || miss_vip->proto != vip->proto) + return; + + cnt = bpf_map_lookup_elem(&lru_miss_stats, &real_idx); + if (cnt) + *cnt += 1; +} + +static __noinline int process_packet(struct xdp_md *xdp) +{ + void *data = (void *)(long)xdp->data; + void *data_end = (void *)(long)xdp->data_end; + struct ethhdr *eth = data; + struct real_definition *dst = NULL; + struct vip_definition vip_def = {}; + struct ctl_value *cval; + struct flow_key flow = {}; + struct vip_meta *vip_info; + struct lb_stats *data_stats; + struct udphdr *uh; + __be32 tnl_src[4]; + void *lru_map; + void *l4; + __u16 payload_len; + __u32 real_pos = 0, cpu_num, key; + __u8 proto; + int action = XDP_DROP; + bool is_v6, is_syn = false, is_rst = false; + + if (eth + 1 > data_end) + goto out; + + if (eth->h_proto == bpf_htons(ETH_P_IPV6)) { + is_v6 = true; + } else if (eth->h_proto == bpf_htons(ETH_P_IP)) { + is_v6 = false; + } else { + action = XDP_PASS; + goto out; + } + + if (is_v6) { + struct ipv6hdr *ip6h = (void *)(eth + 1); + + if (ip6h + 1 > data_end) + goto out; + if (ip6h->nexthdr == IPPROTO_FRAGMENT) + goto out; + + payload_len = sizeof(struct ipv6hdr) + bpf_ntohs(ip6h->payload_len); + proto = ip6h->nexthdr; + + __builtin_memcpy(flow.srcv6, &ip6h->saddr, sizeof(flow.srcv6)); + __builtin_memcpy(flow.dstv6, &ip6h->daddr, sizeof(flow.dstv6)); + __builtin_memcpy(vip_def.vipv6, &ip6h->daddr, sizeof(vip_def.vipv6)); + l4 = (void *)(ip6h + 1); + } else { + struct iphdr *iph = (void *)(eth + 1); + + if (iph + 1 > data_end) + goto out; + if (iph->ihl != 5) + goto out; + if (iph->frag_off & bpf_htons(PCKT_FRAGMENTED)) + goto out; + + payload_len = bpf_ntohs(iph->tot_len); + proto = iph->protocol; + + flow.src = iph->saddr; + flow.dst = iph->daddr; + vip_def.vip = iph->daddr; + l4 = (void *)(iph + 1); + } + + /* TCP and UDP share the same port layout at offset 0 */ + if (proto != IPPROTO_TCP && proto != IPPROTO_UDP) { + action = XDP_PASS; + goto out; + } + + uh = l4; + if ((void *)(uh + 1) > data_end) + goto out; + flow.port16[0] = uh->source; + flow.port16[1] = uh->dest; + + if (proto == IPPROTO_TCP) { + struct tcphdr *th = l4; + + if ((void *)(th + 1) > data_end) + goto out; + is_syn = th->syn; + is_rst = th->rst; + } + + flow.proto = proto; + vip_def.port = flow.port16[1]; + vip_def.proto = proto; + + vip_info = bpf_map_lookup_elem(&vip_map, &vip_def); + if (!vip_info) { + action = XDP_PASS; + goto out; + } + + key = STATS_LRU; + data_stats = bpf_map_lookup_elem(&stats, &key); + if (!data_stats) + goto out; + data_stats->v1 += 1; + + cpu_num = bpf_get_smp_processor_id(); + lru_map = bpf_map_lookup_elem(&lru_mapping, &cpu_num); + if (!lru_map) + goto out; + + if (!(vip_info->flags & F_LRU_BYPASS) && !is_syn) + dst = connection_table_lookup(lru_map, &flow, &real_pos); + + if (!dst) { + if (flow.proto == IPPROTO_TCP) { + struct lb_stats *miss_st; + + key = STATS_LRU_MISS; + miss_st = bpf_map_lookup_elem(&stats, &key); + if (miss_st) + miss_st->v1 += 1; + } + + if (!get_packet_dst(&dst, &flow, vip_info, is_v6, lru_map, is_rst, &real_pos)) + goto out; + + update_vip_lru_miss_stats(&vip_def, is_v6, real_pos); + data_stats->v2 += 1; + } + + key = 0; + cval = bpf_map_lookup_elem(&ctl_array, &key); + if (!cval) + goto out; + + update_stats(&stats, vip_info->vip_num, payload_len); + update_stats(&reals_stats, real_pos, payload_len); + + if (is_v6) { + create_encap_ipv6_src(flow.port16[0], flow.srcv6[0], tnl_src); + if (encap_v6(xdp, tnl_src, dst->dstv6, IPPROTO_IPV6, payload_len, cval->mac)) + goto out; + } else if (dst->flags & F_IPV6) { + create_encap_ipv6_src(flow.port16[0], flow.src, tnl_src); + if (encap_v6(xdp, tnl_src, dst->dstv6, IPPROTO_IPIP, payload_len, cval->mac)) + goto out; + } else { + if (encap_v4(xdp, create_encap_ipv4_src(flow.port16[0], flow.src), dst->dst, + payload_len, cval->mac)) + goto out; + } + + action = XDP_TX; + +out: + count_action(action); + return action; +} + +static __always_inline int strip_encap(struct xdp_md *xdp, const struct ethhdr *saved_eth) +{ + void *data = (void *)(long)xdp->data; + void *data_end = (void *)(long)xdp->data_end; + struct ethhdr *eth = data; + int hdr_sz; + + if (eth + 1 > data_end) + return -1; + + hdr_sz = (eth->h_proto == bpf_htons(ETH_P_IPV6)) ? (int)sizeof(struct ipv6hdr) + : (int)sizeof(struct iphdr); + + if (bpf_xdp_adjust_head(xdp, hdr_sz)) + return -1; + + data = (void *)(long)xdp->data; + data_end = (void *)(long)xdp->data_end; + eth = data; + + if (eth + 1 > data_end) + return -1; + + __builtin_memcpy(eth, saved_eth, sizeof(*saved_eth)); + return 0; +} + +static __always_inline void randomize_src(struct xdp_md *xdp, int saddr_off, __u32 *rand_state) +{ + void *data = (void *)(long)xdp->data; + void *data_end = (void *)(long)xdp->data_end; + __u32 *saddr = data + saddr_off; + + *rand_state ^= *rand_state << 13; + *rand_state ^= *rand_state >> 17; + *rand_state ^= *rand_state << 5; + + if ((void *)(saddr + 1) <= data_end) + *saddr = *rand_state & flow_mask; +} + +SEC("xdp") +int xdp_lb_bench(struct xdp_md *xdp) +{ + void *data = (void *)(long)xdp->data; + void *data_end = (void *)(long)xdp->data_end; + struct ethhdr *eth = data; + struct ethhdr saved_eth; + __u32 rand_state = 0; + __u32 batch_hash = 0; + int saddr_off = 0; + bool is_v6; + + if (eth + 1 > data_end) + return XDP_DROP; + + __builtin_memcpy(&saved_eth, eth, sizeof(saved_eth)); + + is_v6 = (saved_eth.h_proto == bpf_htons(ETH_P_IPV6)); + + saddr_off = sizeof(struct ethhdr) + (is_v6 ? offsetof(struct ipv6hdr, saddr) : + offsetof(struct iphdr, saddr)); + + if (flow_mask) + rand_state = bpf_get_prandom_u32() | 1; + + if (cold_lru) { + __u32 *saddr = data + saddr_off; + + batch_gen++; + batch_hash = (batch_gen + bpf_get_smp_processor_id()) * KNUTH_HASH_MULT; + if ((void *)(saddr + 1) <= data_end) + *saddr ^= batch_hash; + } + + return BENCH_BPF_LOOP( + process_packet(xdp), + ({ + if (__bench_result == XDP_TX) { + if (strip_encap(xdp, &saved_eth)) + return XDP_DROP; + if (rand_state) + randomize_src(xdp, saddr_off, &rand_state); + } + if (cold_lru) { + void *d = (void *)(long)xdp->data; + void *de = (void *)(long)xdp->data_end; + __u32 *__sa = d + saddr_off; + + if ((void *)(__sa + 1) <= de) + *__sa ^= batch_hash; + } + }) + ); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/xdping_kern.c b/tools/testing/selftests/bpf/progs/xdping_kern.c deleted file mode 100644 index 44e2b0ef23ae..000000000000 --- a/tools/testing/selftests/bpf/progs/xdping_kern.c +++ /dev/null @@ -1,183 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. */ - -#define KBUILD_MODNAME "foo" -#include <stddef.h> -#include <string.h> -#include <linux/bpf.h> -#include <linux/icmp.h> -#include <linux/in.h> -#include <linux/if_ether.h> -#include <linux/if_packet.h> -#include <linux/if_vlan.h> -#include <linux/ip.h> - -#include <bpf/bpf_helpers.h> -#include <bpf/bpf_endian.h> - -#include "bpf_compiler.h" -#include "xdping.h" - -struct { - __uint(type, BPF_MAP_TYPE_HASH); - __uint(max_entries, 256); - __type(key, __u32); - __type(value, struct pinginfo); -} ping_map SEC(".maps"); - -static __always_inline void swap_src_dst_mac(void *data) -{ - unsigned short *p = data; - unsigned short dst[3]; - - dst[0] = p[0]; - dst[1] = p[1]; - dst[2] = p[2]; - p[0] = p[3]; - p[1] = p[4]; - p[2] = p[5]; - p[3] = dst[0]; - p[4] = dst[1]; - p[5] = dst[2]; -} - -static __always_inline __u16 csum_fold_helper(__wsum sum) -{ - sum = (sum & 0xffff) + (sum >> 16); - return ~((sum & 0xffff) + (sum >> 16)); -} - -static __always_inline __u16 ipv4_csum(void *data_start, int data_size) -{ - __wsum sum; - - sum = bpf_csum_diff(0, 0, data_start, data_size, 0); - return csum_fold_helper(sum); -} - -#define ICMP_ECHO_LEN 64 - -static __always_inline int icmp_check(struct xdp_md *ctx, int type) -{ - void *data_end = (void *)(long)ctx->data_end; - void *data = (void *)(long)ctx->data; - struct ethhdr *eth = data; - struct icmphdr *icmph; - struct iphdr *iph; - - if (data + sizeof(*eth) + sizeof(*iph) + ICMP_ECHO_LEN > data_end) - return XDP_PASS; - - if (eth->h_proto != bpf_htons(ETH_P_IP)) - return XDP_PASS; - - iph = data + sizeof(*eth); - - if (iph->protocol != IPPROTO_ICMP) - return XDP_PASS; - - if (bpf_ntohs(iph->tot_len) - sizeof(*iph) != ICMP_ECHO_LEN) - return XDP_PASS; - - icmph = data + sizeof(*eth) + sizeof(*iph); - - if (icmph->type != type) - return XDP_PASS; - - return XDP_TX; -} - -SEC("xdp") -int xdping_client(struct xdp_md *ctx) -{ - void *data = (void *)(long)ctx->data; - struct pinginfo *pinginfo = NULL; - struct ethhdr *eth = data; - struct icmphdr *icmph; - struct iphdr *iph; - __u64 recvtime; - __be32 raddr; - __be16 seq; - int ret; - __u8 i; - - ret = icmp_check(ctx, ICMP_ECHOREPLY); - - if (ret != XDP_TX) - return ret; - - iph = data + sizeof(*eth); - icmph = data + sizeof(*eth) + sizeof(*iph); - raddr = iph->saddr; - - /* Record time reply received. */ - recvtime = bpf_ktime_get_ns(); - pinginfo = bpf_map_lookup_elem(&ping_map, &raddr); - if (!pinginfo || pinginfo->seq != icmph->un.echo.sequence) - return XDP_PASS; - - if (pinginfo->start) { - __pragma_loop_unroll_full - for (i = 0; i < XDPING_MAX_COUNT; i++) { - if (pinginfo->times[i] == 0) - break; - } - /* verifier is fussy here... */ - if (i < XDPING_MAX_COUNT) { - pinginfo->times[i] = recvtime - - pinginfo->start; - pinginfo->start = 0; - i++; - } - /* No more space for values? */ - if (i == pinginfo->count || i == XDPING_MAX_COUNT) - return XDP_PASS; - } - - /* Now convert reply back into echo request. */ - swap_src_dst_mac(data); - iph->saddr = iph->daddr; - iph->daddr = raddr; - icmph->type = ICMP_ECHO; - seq = bpf_htons(bpf_ntohs(icmph->un.echo.sequence) + 1); - icmph->un.echo.sequence = seq; - icmph->checksum = 0; - icmph->checksum = ipv4_csum(icmph, ICMP_ECHO_LEN); - - pinginfo->seq = seq; - pinginfo->start = bpf_ktime_get_ns(); - - return XDP_TX; -} - -SEC("xdp") -int xdping_server(struct xdp_md *ctx) -{ - void *data = (void *)(long)ctx->data; - struct ethhdr *eth = data; - struct icmphdr *icmph; - struct iphdr *iph; - __be32 raddr; - int ret; - - ret = icmp_check(ctx, ICMP_ECHO); - - if (ret != XDP_TX) - return ret; - - iph = data + sizeof(*eth); - icmph = data + sizeof(*eth) + sizeof(*iph); - raddr = iph->saddr; - - /* Now convert request into echo reply. */ - swap_src_dst_mac(data); - iph->saddr = iph->daddr; - iph->daddr = raddr; - icmph->type = ICMP_ECHOREPLY; - icmph->checksum = 0; - icmph->checksum = ipv4_csum(icmph, ICMP_ECHO_LEN); - - return XDP_TX; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/sysctl_helpers.c b/tools/testing/selftests/bpf/sysctl_helpers.c new file mode 100644 index 000000000000..e2bd824f12d5 --- /dev/null +++ b/tools/testing/selftests/bpf/sysctl_helpers.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <stdio.h> +#include <errno.h> +#include <string.h> + +#include "sysctl_helpers.h" +#include "test_progs.h" + +int sysctl_set(const char *sysctl_path, char *old_val, const char *new_val) +{ + int ret = 0; + FILE *fp; + + fp = fopen(sysctl_path, "r+"); + if (!fp) + return -errno; + if (old_val && fscanf(fp, "%s", old_val) <= 0) { + ret = -ENOENT; + } else if (!old_val || strcmp(old_val, new_val) != 0) { + fseek(fp, 0, SEEK_SET); + if (fprintf(fp, "%s", new_val) < 0) + ret = -errno; + } + fclose(fp); + + return ret; +} + +int sysctl_set_or_fail(const char *sysctl_path, char *old_val, const char *new_val) +{ + int err; + + err = sysctl_set(sysctl_path, old_val, new_val); + if (err) + PRINT_FAIL("failed to set %s to %s: %s\n", sysctl_path, new_val, strerror(-err)); + return err; +} diff --git a/tools/testing/selftests/bpf/sysctl_helpers.h b/tools/testing/selftests/bpf/sysctl_helpers.h new file mode 100644 index 000000000000..35e37bfe1b3b --- /dev/null +++ b/tools/testing/selftests/bpf/sysctl_helpers.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SYSCTL_HELPERS_H +#define __SYSCTL_HELPERS_H + +int sysctl_set(const char *sysctl_path, char *old_val, const char *new_val); +int sysctl_set_or_fail(const char *sysctl_path, char *old_val, const char *new_val); + +#endif diff --git a/tools/testing/selftests/bpf/test_bpftool.py b/tools/testing/selftests/bpf/test_bpftool.py deleted file mode 100644 index 1c2408ee1f5d..000000000000 --- a/tools/testing/selftests/bpf/test_bpftool.py +++ /dev/null @@ -1,174 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# Copyright (c) 2020 SUSE LLC. - -import collections -import functools -import json -import os -import socket -import subprocess -import unittest - - -# Add the source tree of bpftool and /usr/local/sbin to PATH -cur_dir = os.path.dirname(os.path.realpath(__file__)) -bpftool_dir = os.path.abspath(os.path.join(cur_dir, "..", "..", "..", "..", - "tools", "bpf", "bpftool")) -os.environ["PATH"] = bpftool_dir + ":/usr/local/sbin:" + os.environ["PATH"] - - -class IfaceNotFoundError(Exception): - pass - - -class UnprivilegedUserError(Exception): - pass - - -def _bpftool(args, json=True): - _args = ["bpftool"] - if json: - _args.append("-j") - _args.extend(args) - - return subprocess.check_output(_args) - - -def bpftool(args): - return _bpftool(args, json=False).decode("utf-8") - - -def bpftool_json(args): - res = _bpftool(args) - return json.loads(res) - - -def get_default_iface(): - for iface in socket.if_nameindex(): - if iface[1] != "lo": - return iface[1] - raise IfaceNotFoundError("Could not find any network interface to probe") - - -def default_iface(f): - @functools.wraps(f) - def wrapper(*args, **kwargs): - iface = get_default_iface() - return f(*args, iface, **kwargs) - return wrapper - -DMESG_EMITTING_HELPERS = [ - "bpf_probe_write_user", - "bpf_trace_printk", - "bpf_trace_vprintk", - ] - -class TestBpftool(unittest.TestCase): - @classmethod - def setUpClass(cls): - if os.getuid() != 0: - raise UnprivilegedUserError( - "This test suite needs root privileges") - - @default_iface - def test_feature_dev_json(self, iface): - unexpected_helpers = DMESG_EMITTING_HELPERS - expected_keys = [ - "syscall_config", - "program_types", - "map_types", - "helpers", - "misc", - ] - - res = bpftool_json(["feature", "probe", "dev", iface]) - # Check if the result has all expected keys. - self.assertCountEqual(res.keys(), expected_keys) - # Check if unexpected helpers are not included in helpers probes - # result. - for helpers in res["helpers"].values(): - for unexpected_helper in unexpected_helpers: - self.assertNotIn(unexpected_helper, helpers) - - def test_feature_kernel(self): - test_cases = [ - bpftool_json(["feature", "probe", "kernel"]), - bpftool_json(["feature", "probe"]), - bpftool_json(["feature"]), - ] - unexpected_helpers = DMESG_EMITTING_HELPERS - expected_keys = [ - "syscall_config", - "system_config", - "program_types", - "map_types", - "helpers", - "misc", - ] - - for tc in test_cases: - # Check if the result has all expected keys. - self.assertCountEqual(tc.keys(), expected_keys) - # Check if unexpected helpers are not included in helpers probes - # result. - for helpers in tc["helpers"].values(): - for unexpected_helper in unexpected_helpers: - self.assertNotIn(unexpected_helper, helpers) - - def test_feature_kernel_full(self): - test_cases = [ - bpftool_json(["feature", "probe", "kernel", "full"]), - bpftool_json(["feature", "probe", "full"]), - ] - expected_helpers = DMESG_EMITTING_HELPERS - - for tc in test_cases: - # Check if expected helpers are included at least once in any - # helpers list for any program type. Unfortunately we cannot assume - # that they will be included in all program types or a specific - # subset of programs. It depends on the kernel version and - # configuration. - found_helpers = False - - for helpers in tc["helpers"].values(): - if all(expected_helper in helpers - for expected_helper in expected_helpers): - found_helpers = True - break - - self.assertTrue(found_helpers) - - def test_feature_kernel_full_vs_not_full(self): - full_res = bpftool_json(["feature", "probe", "full"]) - not_full_res = bpftool_json(["feature", "probe"]) - not_full_set = set() - full_set = set() - - for helpers in full_res["helpers"].values(): - for helper in helpers: - full_set.add(helper) - - for helpers in not_full_res["helpers"].values(): - for helper in helpers: - not_full_set.add(helper) - - self.assertCountEqual(full_set - not_full_set, - set(DMESG_EMITTING_HELPERS)) - self.assertCountEqual(not_full_set - full_set, set()) - - def test_feature_macros(self): - expected_patterns = [ - r"/\*\*\* System call availability \*\*\*/", - r"#define HAVE_BPF_SYSCALL", - r"/\*\*\* eBPF program types \*\*\*/", - r"#define HAVE.*PROG_TYPE", - r"/\*\*\* eBPF map types \*\*\*/", - r"#define HAVE.*MAP_TYPE", - r"/\*\*\* eBPF helper functions \*\*\*/", - r"#define HAVE.*HELPER", - r"/\*\*\* eBPF misc features \*\*\*/", - ] - - res = bpftool(["feature", "probe", "macros"]) - for pattern in expected_patterns: - self.assertRegex(res, pattern) diff --git a/tools/testing/selftests/bpf/test_bpftool.sh b/tools/testing/selftests/bpf/test_bpftool.sh deleted file mode 100755 index 718f59692ccb..000000000000 --- a/tools/testing/selftests/bpf/test_bpftool.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: GPL-2.0 -# Copyright (c) 2020 SUSE LLC. - -# 'make -C tools/testing/selftests/bpf install' will install to SCRIPT_DIR -SCRIPT_DIR=$(dirname $(realpath $0)) - -# 'make -C tools/testing/selftests/bpf' will install to BPFTOOL_INSTALL_PATH -BPFTOOL_INSTALL_PATH="$SCRIPT_DIR"/tools/sbin -export PATH=$SCRIPT_DIR:$BPFTOOL_INSTALL_PATH:$PATH -python3 -m unittest -v test_bpftool.TestBpftool diff --git a/tools/testing/selftests/bpf/test_kmods/Makefile b/tools/testing/selftests/bpf/test_kmods/Makefile index 63c4d3f6a12f..031c7454ce65 100644 --- a/tools/testing/selftests/bpf/test_kmods/Makefile +++ b/tools/testing/selftests/bpf/test_kmods/Makefile @@ -1,5 +1,16 @@ TEST_KMOD_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) -KDIR ?= $(abspath $(TEST_KMOD_DIR)/../../../../..) +SRCTREE_KDIR := $(abspath $(TEST_KMOD_DIR)/../../../../..) +# Honor O=/KBUILD_OUTPUT only if they point at a prepared kernel build +# directory (one containing Module.symvers); otherwise treat the value as a +# selftests-only output directory and fall back to in-tree or distro headers. +# The parent bpf/Makefile resolves O=/KBUILD_OUTPUT to absolute paths before +# invoking this sub-make so relative paths still anchor to the user's +# invocation directory. +KMOD_O := $(or $(O),$(KBUILD_OUTPUT)) +KMOD_O_VALID := $(if $(KMOD_O),$(if $(wildcard $(KMOD_O)/Module.symvers),$(KMOD_O))) +KDIR ?= $(if $(KMOD_O_VALID),$(SRCTREE_KDIR), \ + $(if $(wildcard $(SRCTREE_KDIR)/Module.symvers),$(SRCTREE_KDIR), \ + /lib/modules/$(shell uname -r)/build)) ifeq ($(V),1) Q = @@ -14,8 +25,21 @@ $(foreach m,$(MODULES),$(eval obj-m += $(m:.ko=.o))) CFLAGS_bpf_testmod.o = -I$(src) +# When BPF_STRICT_BUILD != 0, a missing KDIR is fatal (the default). +# When permissive, skip silently. +PERMISSIVE := $(filter 0,$(BPF_STRICT_BUILD)) + all: - $(Q)$(MAKE) -C $(KDIR) M=$(TEST_KMOD_DIR) modules +ifeq ($(PERMISSIVE),) + $(Q)$(MAKE) -C $(KDIR) $(if $(KMOD_O_VALID),O=$(KMOD_O_VALID) KBUILD_OUTPUT=$(KMOD_O_VALID),KBUILD_OUTPUT=) \ + M=$(TEST_KMOD_DIR) modules +else ifneq ("$(wildcard $(KDIR))", "") + $(Q)$(MAKE) -C $(KDIR) $(if $(KMOD_O_VALID),O=$(KMOD_O_VALID) KBUILD_OUTPUT=$(KMOD_O_VALID),KBUILD_OUTPUT=) \ + M=$(TEST_KMOD_DIR) modules +endif clean: - $(Q)$(MAKE) -C $(KDIR) M=$(TEST_KMOD_DIR) clean +ifneq ("$(wildcard $(KDIR))", "") + $(Q)$(MAKE) -C $(KDIR) $(if $(KMOD_O_VALID),O=$(KMOD_O_VALID) KBUILD_OUTPUT=$(KMOD_O_VALID),KBUILD_OUTPUT=) \ + M=$(TEST_KMOD_DIR) clean +endif diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index e62c6b78657f..9366a3c578f1 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -161,6 +161,33 @@ bpf_testmod_test_arg_ptr_to_struct(struct bpf_testmod_struct_arg_1 *a) { return bpf_testmod_test_struct_arg_result; } +#ifdef __SIZEOF_INT128__ +noinline __int128 +bpf_testmod_test_int128_ret(int a) +{ + bpf_testmod_test_struct_arg_result = a; + return (__int128)a; +} + +/* + * The __int128 'a' is the first argument on purpose. On arm64 a 16-byte + * argument must start in an even-numbered register pair, so placing it + * after a single-register scalar would leave a padding register (x1) + * unused. pahole maps parameters to registers positionally and would then + * see the following argument in an "unexpected" register and skip BTF + * encoding of the whole function, making it unattachable. Keeping the + * __int128 first (x0:x1) avoids the padding while still exercising the + * trampoline packing of a 128-bit argument together with the trailing + * int and long arguments. + */ +noinline long +bpf_testmod_test_int128_arg(__int128 a, int b, long c) +{ + bpf_testmod_test_struct_arg_result = (long)a + b + c; + return bpf_testmod_test_struct_arg_result; +} +#endif + __weak noinline void bpf_testmod_looooooooooooooooooooooooooooooong_name(void) { } @@ -210,6 +237,44 @@ __bpf_kfunc void bpf_kfunc_common_test(void) { } +__bpf_kfunc u64 bpf_kfunc_arena_arg_test(u64 *val__arena) +{ + u64 old; + + old = *val__arena; + *val__arena = old + 1; + return old; +} + +__bpf_kfunc u64 bpf_kfunc_arena_cap_test(u64 *val__arena) +{ + return (u64)val__arena; +} + +__bpf_kfunc u64 bpf_kfunc_arena_cap_nullable_test(u64 *val__arena__nullable) +{ + return (u64)val__arena__nullable; +} + +__bpf_kfunc u64 bpf_kfunc_arena_args5_test(u64 *a__arena, u64 *b__arena, + u64 *c__arena, u64 *d__arena, + u64 *e__arena__nullable) +{ + return *a__arena + *b__arena + *c__arena + *d__arena + + (e__arena__nullable ? *e__arena__nullable : 0); +} + +__bpf_kfunc u64 bpf_kfunc_arena_stack_arg_test(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 *f__arena) +{ + return a + b + c + d + e + *f__arena; +} + +__bpf_kfunc u64 bpf_kfunc_arena_mixed_test(u64 *a__arena, u64 *b__arena__nullable) +{ + return *a__arena + (b__arena__nullable ? *b__arena__nullable : 0); +} + __bpf_kfunc void bpf_kfunc_dynptr_test(struct bpf_dynptr *ptr, struct bpf_dynptr *ptr__nullable) { @@ -320,9 +385,36 @@ static int bpf_testmod_test_4(void) return 0; } +static int bpf_testmod_ops3__test_arena(u64 *ptr__arena) +{ + return 0; +} + +static int bpf_testmod_ops3__test_arena_nullable(u64 *ptr__arena__nullable) +{ + return 0; +} + +static int bpf_testmod_ops3__test_arena_stack(u64 a, u64 b, u64 c, u64 d, + u64 e, u64 f, u64 g, u64 h, + u64 *ptr__arena) +{ + return 0; +} + +static int bpf_testmod_ops3__test_arena_multislot(struct bpf_testmod_arena_pair p, + u64 *ptr__arena) +{ + return 0; +} + static struct bpf_testmod_ops3 __bpf_testmod_ops3 = { .test_1 = bpf_testmod_test_3, .test_2 = bpf_testmod_test_4, + .test_arena = bpf_testmod_ops3__test_arena, + .test_arena_nullable = bpf_testmod_ops3__test_arena_nullable, + .test_arena_stack = bpf_testmod_ops3__test_arena_stack, + .test_arena_multislot = bpf_testmod_ops3__test_arena_multislot, }; static void bpf_testmod_test_struct_ops3(void) @@ -341,6 +433,28 @@ __bpf_kfunc void bpf_testmod_ops3_call_test_2(void) st_ops3->test_2(); } +__bpf_kfunc int bpf_testmod_ops3_call_test_arena(u64 *ptr__arena) +{ + return st_ops3->test_arena(ptr__arena); +} + +__bpf_kfunc int bpf_testmod_ops3_call_test_arena_nullable(u64 *ptr__arena__nullable) +{ + return st_ops3->test_arena_nullable(ptr__arena__nullable); +} + +__bpf_kfunc int bpf_testmod_ops3_call_test_arena_stack(u64 *ptr__arena) +{ + return st_ops3->test_arena_stack(1, 2, 3, 4, 5, 6, 7, 8, ptr__arena); +} + +__bpf_kfunc int bpf_testmod_ops3_call_test_arena_multislot(u64 *ptr__arena) +{ + struct bpf_testmod_arena_pair p = { .a = 11, .b = 22 }; + + return st_ops3->test_arena_multislot(p, ptr__arena); +} + struct bpf_testmod_btf_type_tag_1 { int a; }; @@ -470,6 +584,11 @@ noinline void bpf_testmod_stacktrace_test_1(void) int bpf_testmod_fentry_ok; +noinline int bpf_testmod_trampoline_count_test(void) +{ + return 0; +} + noinline ssize_t bpf_testmod_test_read(struct file *file, struct kobject *kobj, const struct bin_attribute *bin_attr, @@ -509,6 +628,11 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, (void)bpf_testmod_test_arg_ptr_to_struct(&struct_arg1_2); +#ifdef __SIZEOF_INT128__ + (void)bpf_testmod_test_int128_ret(i); + (void)bpf_testmod_test_int128_arg((__int128)1, 2, 3); +#endif + (void)trace_bpf_testmod_test_raw_tp_null_tp(NULL); bpf_testmod_test_struct_ops3(); @@ -548,6 +672,8 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, 21, 22, 23, 24, 25, 26) != 231) goto out; + bpf_testmod_trampoline_count_test(); + bpf_testmod_stacktrace_test_1(); bpf_testmod_fentry_ok = 1; @@ -716,6 +842,13 @@ BTF_ID_FLAGS(func, bpf_iter_testmod_seq_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_testmod_seq_destroy, KF_ITER_DESTROY) BTF_ID_FLAGS(func, bpf_iter_testmod_seq_value) BTF_ID_FLAGS(func, bpf_kfunc_common_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_arg_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_cap_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_cap_nullable_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_args5_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_stack_arg_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_mixed_test) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_pass1) BTF_ID_FLAGS(func, bpf_kfunc_dynptr_test) BTF_ID_FLAGS(func, bpf_kfunc_nested_acquire_nonzero_offset_test, KF_ACQUIRE) BTF_ID_FLAGS(func, bpf_kfunc_nested_acquire_zero_offset_test, KF_ACQUIRE) @@ -730,6 +863,10 @@ BTF_ID_FLAGS(func, bpf_testmod_ctx_create, KF_ACQUIRE | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_testmod_ctx_release, KF_RELEASE) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_1) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_2) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_nullable) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_stack) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_multislot) BTF_ID_FLAGS(func, bpf_kfunc_get_default_trusted_ptr_test); BTF_ID_FLAGS(func, bpf_kfunc_put_default_trusted_ptr_test); BTF_KFUNCS_END(bpf_testmod_common_kfunc_ids) @@ -760,12 +897,133 @@ __bpf_kfunc struct sock *bpf_kfunc_call_test3(struct sock *sk) __bpf_kfunc long noinline bpf_kfunc_call_test4(signed char a, short b, int c, long d) { - /* Provoke the compiler to assume that the caller has sign-extended a, + /* + * Make val as volatile to avoid compiler optimizations. + * Verify that negative signed values remain negative after + * sign-extension (JIT must sign-extend, not zero-extend). + */ + volatile long val; + + /* val will be positive, if JIT does zero-extension instead of sign-extension */ + val = a; + if (val >= 0) + return 1; + + val = b; + if (val >= 0) + return 2; + + val = c; + if (val >= 0) + return 3; + + /* + * Provoke the compiler to assume that the caller has sign-extended a, * b and c on platforms where this is required (e.g. s390x). */ return (long)a + (long)b + (long)c + d; } +__bpf_kfunc int bpf_kfunc_call_test5(u8 a, u16 b, u32 c) +{ + /* + * Make val as volatile to avoid compiler optimizations on the below checks + * In C, assigning u8/u16/u32 to long performs zero-extension. + */ + volatile long val = a; + + /* Check zero-extension */ + if (val != (unsigned long)a) + return 1; + /* Check no sign-extension */ + if (val < 0) + return 2; + + val = b; + if (val != (unsigned long)b) + return 3; + if (val < 0) + return 4; + + val = c; + if (val != (unsigned long)c) + return 5; + if (val < 0) + return 6; + + return 0; +} + +__bpf_kfunc u64 bpf_kfunc_call_stack_arg(u64 a, u64 b, u64 c, u64 d, + u64 e, u64 f, u64 g, u64 h, + u64 i, u64 j) +{ + return a + b + c + d + e + f + g + h + i + j; +} + +__bpf_kfunc u64 bpf_kfunc_call_stack_arg_ptr(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 f, u64 g, u64 h, u64 i, + struct prog_test_pass1 *p) +{ + return a + b + c + d + e + f + g + h + i + p->x0 + p->x1; +} + +__bpf_kfunc u64 bpf_kfunc_call_stack_arg_mix(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 f, u64 g, + struct prog_test_pass1 *p, u64 h, + struct prog_test_pass1 *q) +{ + return a + b + c + d + e + f + g + p->x0 + h + q->x1; +} + +__bpf_kfunc u64 bpf_kfunc_call_stack_arg_dynptr(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 f, u64 g, u64 h, u64 i, + struct bpf_dynptr *ptr) +{ + const struct bpf_dynptr_kern *kern_ptr = (void *)ptr; + + return a + b + c + d + e + f + g + h + i + (kern_ptr->size & 0xFFFFFF); +} + +__bpf_kfunc u64 bpf_kfunc_call_stack_arg_mem(u64 a, u64 b, u64 c, u64 d, u64 e, + void *mem, int mem__sz) +{ + const unsigned char *p = mem; + u64 sum = a + b + c + d + e; + int i; + + for (i = 0; i < mem__sz; i++) + sum += p[i]; + return sum; +} + +__bpf_kfunc u64 bpf_kfunc_call_stack_arg_iter(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 f, u64 g, u64 h, u64 i, + struct bpf_iter_testmod_seq *it__iter) +{ + return a + b + c + d + e + f + g + h + i + it__iter->value; +} + +__bpf_kfunc u64 bpf_kfunc_call_stack_arg_const_str(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 f, u64 g, u64 h, u64 i, + const char *str__str) +{ + return a + b + c + d + e + f + g + h + i; +} + +__bpf_kfunc u64 bpf_kfunc_call_stack_arg_timer(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 f, u64 g, u64 h, u64 i, + struct bpf_timer *timer) +{ + return a + b + c + d + e + f + g + h + i; +} + +__bpf_kfunc u64 bpf_kfunc_call_stack_arg_big(u64 a, u64 b, u64 c, u64 d, u64 e, + struct prog_test_big_arg s) +{ + return a + b + c + d + e + s.a + s.b; +} + static struct prog_test_ref_kfunc prog_test_struct = { .a = 42, .b = 108, @@ -1223,12 +1481,21 @@ __bpf_kfunc void bpf_kfunc_trigger_ctx_check(void) } BTF_KFUNCS_START(bpf_testmod_check_kfunc_ids) -BTF_ID_FLAGS(func, bpf_testmod_test_mod_kfunc) +BTF_ID_FLAGS(func, bpf_testmod_test_mod_kfunc, KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_kfunc_call_test1) BTF_ID_FLAGS(func, bpf_kfunc_call_test2) BTF_ID_FLAGS(func, bpf_kfunc_call_test3) BTF_ID_FLAGS(func, bpf_kfunc_call_test4) -BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_pass1) +BTF_ID_FLAGS(func, bpf_kfunc_call_test5) +BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg) +BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_ptr) +BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_mix) +BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_dynptr) +BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_mem) +BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_iter) +BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_const_str) +BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_timer) +BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_big) BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail1) BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail2) BTF_ID_FLAGS(func, bpf_kfunc_call_test_acquire, KF_ACQUIRE | KF_RET_NULL) @@ -1359,6 +1626,12 @@ static int bpf_testmod_ops__test_refcounted(int dummy, return 0; } +static int bpf_testmod_ops__test_refcounted_multi(int dummy, struct task_struct *task__nullable, + struct task_struct *task__ref) +{ + return 0; +} + static struct task_struct * bpf_testmod_ops__test_return_ref_kptr(int dummy, struct task_struct *task__ref, struct cgroup *cgrp) @@ -1371,6 +1644,7 @@ static struct bpf_testmod_ops __bpf_testmod_ops = { .test_2 = bpf_testmod_test_2, .test_maybe_null = bpf_testmod_ops__test_maybe_null, .test_refcounted = bpf_testmod_ops__test_refcounted, + .test_refcounted_multi = bpf_testmod_ops__test_refcounted_multi, .test_return_ref_kptr = bpf_testmod_ops__test_return_ref_kptr, }; @@ -1843,6 +2117,16 @@ struct bpf_struct_ops testmod_multi_st_ops = { extern int bpf_fentry_test1(int a); +BTF_KFUNCS_START(bpf_testmod_trampoline_count_ids) +BTF_ID_FLAGS(func, bpf_testmod_trampoline_count_test) +BTF_KFUNCS_END(bpf_testmod_trampoline_count_ids) + +static const struct +btf_kfunc_id_set bpf_testmod_trampoline_count_fmodret_set = { + .owner = THIS_MODULE, + .set = &bpf_testmod_trampoline_count_ids, +}; + static int bpf_testmod_init(void) { const struct btf_id_dtor_kfunc bpf_testmod_dtors[] = { @@ -1859,6 +2143,7 @@ static int bpf_testmod_init(void) ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_testmod_kfunc_set); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &bpf_testmod_kfunc_set); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_testmod_kfunc_set); + ret = ret ?: register_btf_fmodret_id_set(&bpf_testmod_trampoline_count_fmodret_set); ret = ret ?: register_bpf_struct_ops(&bpf_bpf_testmod_ops, bpf_testmod_ops); ret = ret ?: register_bpf_struct_ops(&bpf_testmod_ops2, bpf_testmod_ops2); ret = ret ?: register_bpf_struct_ops(&bpf_testmod_ops3, bpf_testmod_ops3); diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h index f6e492f9d042..210b919290cc 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h @@ -39,6 +39,9 @@ struct bpf_testmod_ops { int (*unsupported_ops)(void); /* Used to test ref_acquired arguments. */ int (*test_refcounted)(int dummy, struct task_struct *task); + /* Used to test checking of __ref arguments when it not the first argument. */ + int (*test_refcounted_multi)(int dummy, struct task_struct *task, + struct task_struct *task2); /* Used to test returning referenced kptr. */ struct task_struct *(*test_return_ref_kptr)(int dummy, struct task_struct *task, struct cgroup *cgrp); @@ -100,9 +103,23 @@ struct bpf_testmod_ops2 { int (*test_1)(void); }; +/* 16 bytes, so it takes two argument slots when passed by value */ +struct bpf_testmod_arena_pair { + u64 a; + u64 b; +}; + struct bpf_testmod_ops3 { int (*test_1)(void); int (*test_2)(void); + /* Used to test arena pointer arguments. */ + int (*test_arena)(u64 *ptr); + int (*test_arena_nullable)(u64 *ptr); + /* enough leading args to force @ptr onto the stack on x86 and arm64 */ + int (*test_arena_stack)(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, + u64 g, u64 h, u64 *ptr); + /* a multi-slot leading arg, so @ptr is not at the slot its arg index suggests */ + int (*test_arena_multislot)(struct bpf_testmod_arena_pair p, u64 *ptr); }; struct st_ops_args { diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index b393bf771131..7d81070eefe7 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -26,6 +26,8 @@ struct prog_test_ref_kfunc { }; #endif +struct bpf_iter_testmod_seq; + struct prog_test_pass1 { int x0; struct { @@ -48,6 +50,11 @@ struct prog_test_pass2 { } x; }; +struct prog_test_big_arg { + __u64 a; + __u64 b; +}; + struct prog_test_fail1 { void *p; int x; @@ -91,6 +98,15 @@ void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p) __ksym; void bpf_kfunc_call_test_ref(struct prog_test_ref_kfunc *p) __ksym; void bpf_kfunc_call_test_mem_len_pass1(void *mem, int len) __ksym; +__u64 bpf_kfunc_arena_arg_test(__u64 *val__arena) __ksym; +__u64 bpf_kfunc_arena_cap_test(__u64 *val__arena) __ksym; +__u64 bpf_kfunc_arena_cap_nullable_test(__u64 *val__arena__nullable) __ksym; +__u64 bpf_kfunc_arena_args5_test(__u64 *a__arena, __u64 *b__arena, + __u64 *c__arena, __u64 *d__arena, + __u64 *e__arena__nullable) __ksym; +__u64 bpf_kfunc_arena_stack_arg_test(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 *f__arena) __ksym; +__u64 bpf_kfunc_arena_mixed_test(__u64 *a__arena, __u64 *b__arena__nullable) __ksym; int *bpf_kfunc_call_test_get_rdwr_mem(struct prog_test_ref_kfunc *p, const int rdwr_buf_size) __ksym; int *bpf_kfunc_call_test_get_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) __ksym; int *bpf_kfunc_call_test_acq_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) __ksym; @@ -104,12 +120,43 @@ u32 bpf_kfunc_call_test_static_unused_arg(u32 arg, u32 unused) __ksym; #endif void bpf_testmod_test_mod_kfunc(int i) __ksym; +int bpf_testmod_ops3_call_test_arena(__u64 *ptr__arena) __ksym; +int bpf_testmod_ops3_call_test_arena_nullable(__u64 *ptr__arena__nullable) __ksym; +int bpf_testmod_ops3_call_test_arena_stack(__u64 *ptr__arena) __ksym; +int bpf_testmod_ops3_call_test_arena_multislot(__u64 *ptr__arena) __ksym; __u64 bpf_kfunc_call_test1(struct sock *sk, __u32 a, __u64 b, __u32 c, __u64 d) __ksym; int bpf_kfunc_call_test2(struct sock *sk, __u32 a, __u32 b) __ksym; struct sock *bpf_kfunc_call_test3(struct sock *sk) __ksym; long bpf_kfunc_call_test4(signed char a, short b, int c, long d) __ksym; +int bpf_kfunc_call_test5(__u8 a, __u16 b, __u32 c) __ksym; +__u64 bpf_kfunc_call_stack_arg(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, __u64 f, __u64 g, __u64 h, + __u64 i, __u64 j) __ksym; +__u64 bpf_kfunc_call_stack_arg_ptr(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 f, __u64 g, __u64 h, __u64 i, + struct prog_test_pass1 *p) __ksym; +__u64 bpf_kfunc_call_stack_arg_mix(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 f, __u64 g, + struct prog_test_pass1 *p, __u64 h, + struct prog_test_pass1 *q) __ksym; +__u64 bpf_kfunc_call_stack_arg_dynptr(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 f, __u64 g, __u64 h, __u64 i, + struct bpf_dynptr *ptr) __ksym; +__u64 bpf_kfunc_call_stack_arg_mem(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + void *mem, int mem__sz) __ksym; +__u64 bpf_kfunc_call_stack_arg_iter(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 f, __u64 g, __u64 h, __u64 i, + struct bpf_iter_testmod_seq *it__iter) __ksym; +__u64 bpf_kfunc_call_stack_arg_const_str(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 f, __u64 g, __u64 h, __u64 i, + const char *str__str) __ksym; +__u64 bpf_kfunc_call_stack_arg_timer(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 f, __u64 g, __u64 h, __u64 i, + struct bpf_timer *timer) __ksym; +__u64 bpf_kfunc_call_stack_arg_big(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + struct prog_test_big_arg s) __ksym; void bpf_kfunc_call_test_pass_ctx(struct __sk_buff *skb) __ksym; void bpf_kfunc_call_test_pass1(struct prog_test_pass1 *p) __ksym; diff --git a/tools/testing/selftests/bpf/test_lirc_mode2_user.c b/tools/testing/selftests/bpf/test_lirc_mode2_user.c index 88e4aeab21b7..cd191da20d14 100644 --- a/tools/testing/selftests/bpf/test_lirc_mode2_user.c +++ b/tools/testing/selftests/bpf/test_lirc_mode2_user.c @@ -50,8 +50,8 @@ int main(int argc, char **argv) { struct bpf_object *obj; int ret, lircfd, progfd, inputfd; - int testir1 = 0x1dead; - int testir2 = 0x20101; + int testir1 = 0x1ead; + int testir2 = 0x2101; u32 prog_ids[10], prog_flags[10], prog_cnt; if (argc != 3) { @@ -125,7 +125,7 @@ int main(int argc, char **argv) } if (event.type == EV_MSC && event.code == MSC_SCAN && - event.value == 0xdead) { + event.value == 0x1ead) { break; } } diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 338c035c3688..07807757b518 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ #include <linux/capability.h> +#include <linux/err.h> #include <stdlib.h> #include <test_progs.h> #include <bpf/btf.h> @@ -11,39 +12,15 @@ #include "cap_helpers.h" #include "jit_disasm_helpers.h" -#define str_has_pfx(str, pfx) \ - (strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0) +static inline const char *str_has_pfx(const char *str, const char *pfx) +{ + size_t len = strlen(pfx); + + return strncmp(str, pfx, len) == 0 ? str + len : NULL; +} #define TEST_LOADER_LOG_BUF_SZ 2097152 -#define TEST_TAG_EXPECT_FAILURE "comment:test_expect_failure" -#define TEST_TAG_EXPECT_SUCCESS "comment:test_expect_success" -#define TEST_TAG_EXPECT_MSG_PFX "comment:test_expect_msg=" -#define TEST_TAG_EXPECT_NOT_MSG_PFX "comment:test_expect_not_msg=" -#define TEST_TAG_EXPECT_XLATED_PFX "comment:test_expect_xlated=" -#define TEST_TAG_EXPECT_FAILURE_UNPRIV "comment:test_expect_failure_unpriv" -#define TEST_TAG_EXPECT_SUCCESS_UNPRIV "comment:test_expect_success_unpriv" -#define TEST_TAG_EXPECT_MSG_PFX_UNPRIV "comment:test_expect_msg_unpriv=" -#define TEST_TAG_EXPECT_NOT_MSG_PFX_UNPRIV "comment:test_expect_not_msg_unpriv=" -#define TEST_TAG_EXPECT_XLATED_PFX_UNPRIV "comment:test_expect_xlated_unpriv=" -#define TEST_TAG_LOG_LEVEL_PFX "comment:test_log_level=" -#define TEST_TAG_PROG_FLAGS_PFX "comment:test_prog_flags=" -#define TEST_TAG_DESCRIPTION_PFX "comment:test_description=" -#define TEST_TAG_RETVAL_PFX "comment:test_retval=" -#define TEST_TAG_RETVAL_PFX_UNPRIV "comment:test_retval_unpriv=" -#define TEST_TAG_AUXILIARY "comment:test_auxiliary" -#define TEST_TAG_AUXILIARY_UNPRIV "comment:test_auxiliary_unpriv" -#define TEST_BTF_PATH "comment:test_btf_path=" -#define TEST_TAG_ARCH "comment:test_arch=" -#define TEST_TAG_JITED_PFX "comment:test_jited=" -#define TEST_TAG_JITED_PFX_UNPRIV "comment:test_jited_unpriv=" -#define TEST_TAG_CAPS_UNPRIV "comment:test_caps_unpriv=" -#define TEST_TAG_LOAD_MODE_PFX "comment:load_mode=" -#define TEST_TAG_EXPECT_STDERR_PFX "comment:test_expect_stderr=" -#define TEST_TAG_EXPECT_STDERR_PFX_UNPRIV "comment:test_expect_stderr_unpriv=" -#define TEST_TAG_EXPECT_STDOUT_PFX "comment:test_expect_stdout=" -#define TEST_TAG_EXPECT_STDOUT_PFX_UNPRIV "comment:test_expect_stdout_unpriv=" -#define TEST_TAG_LINEAR_SIZE "comment:test_linear_size=" /* Warning: duplicated in bpf_misc.h */ #define POINTER_VALUE 0xbadcafe @@ -69,6 +46,7 @@ enum load_mode { struct test_subspec { char *name; + char *description; bool expect_failure; struct expected_msgs expect_msgs; struct expected_msgs expect_xlated; @@ -85,6 +63,7 @@ struct test_spec { struct test_subspec priv; struct test_subspec unpriv; const char *btf_custom_path; + const char *btf_custom_func_path; int log_level; int prog_flags; int mode_mask; @@ -115,7 +94,7 @@ void test_loader_fini(struct test_loader *tester) free(tester->log_buf); } -static void free_msgs(struct expected_msgs *msgs) +void free_msgs(struct expected_msgs *msgs) { int i; @@ -142,9 +121,13 @@ static void free_test_spec(struct test_spec *spec) free_msgs(&spec->priv.stdout); free(spec->priv.name); + free(spec->priv.description); free(spec->unpriv.name); + free(spec->unpriv.description); spec->priv.name = NULL; + spec->priv.description = NULL; spec->unpriv.name = NULL; + spec->unpriv.description = NULL; } /* Compiles regular expression matching pattern. @@ -161,21 +144,21 @@ static void free_test_spec(struct test_spec *spec) static int compile_regex(const char *pattern, regex_t *regex) { char err_buf[256], buf[256] = {}, *ptr, *buf_end; - const char *original_pattern = pattern; + const char *original_pattern = pattern, *next; bool in_regex = false; int err; buf_end = buf + sizeof(buf); ptr = buf; while (*pattern && ptr < buf_end - 2) { - if (!in_regex && str_has_pfx(pattern, "{{")) { + if (!in_regex && (next = str_has_pfx(pattern, "{{"))) { in_regex = true; - pattern += 2; + pattern = next; continue; } - if (in_regex && str_has_pfx(pattern, "}}")) { + if (in_regex && (next = str_has_pfx(pattern, "}}"))) { in_regex = false; - pattern += 2; + pattern = next; continue; } if (in_regex) { @@ -343,33 +326,49 @@ static void update_flags(int *flags, int flag, bool clear) *flags |= flag; } -/* Matches a string of form '<pfx>[^=]=.*' and returns it's suffix. - * Used to parse btf_decl_tag values. - * Such values require unique prefix because compiler does not add - * same __attribute__((btf_decl_tag(...))) twice. - * Test suite uses two-component tags for such cases: - * - * <pfx> __COUNTER__ '=' - * - * For example, two consecutive __msg tags '__msg("foo") __msg("foo")' - * would be encoded as: - * - * [18] DECL_TAG 'comment:test_expect_msg=0=foo' type_id=15 component_idx=-1 - * [19] DECL_TAG 'comment:test_expect_msg=1=foo' type_id=15 component_idx=-1 - * - * And the purpose of this function is to extract 'foo' from the above. - */ -static const char *skip_dynamic_pfx(const char *s, const char *pfx) +static const char *skip_decl_tag_pfx(const char *s) { - const char *msg; + int n = 0; - if (strncmp(s, pfx, strlen(pfx)) != 0) - return NULL; - msg = s + strlen(pfx); - msg = strchr(msg, '='); - if (!msg) + if (sscanf(s, "comment:%*d:%n", &n) < 0 || !n) return NULL; - return msg + 1; + return s + n; +} + +static int compare_decl_tags(const void *a, const void *b) +{ + return strverscmp(*(const char **)a, *(const char **)b); +} + +/* + * Compilers don't guarantee order in which BTF attributes would be generated, + * while order is important for test tags like __msg. + * Each test tag has the following prefix: "comment:" __COUNTER__, + * when sorted using strverscmp this gives same order as in the original C code. + */ +static const char **collect_decl_tags(struct btf *btf, int id, int *cnt) +{ + const char **tmp, **tags = NULL; + const struct btf_type *t; + int i; + + *cnt = 0; + for (i = 1; i < btf__type_cnt(btf); i++) { + t = btf__type_by_id(btf, i); + if (!btf_is_decl_tag(t) || t->type != id || btf_decl_tag(t)->component_idx != -1) + continue; + tmp = realloc(tags, (*cnt + 1) * sizeof(*tags)); + if (!tmp) { + free(tags); + return ERR_PTR(-ENOMEM); + } + tags = tmp; + tags[(*cnt)++] = btf__str_by_offset(btf, t->name_off); + } + + if (*cnt) + qsort(tags, *cnt, sizeof(*tags), compare_decl_tags); + return tags; } enum arch { @@ -378,6 +377,7 @@ enum arch { ARCH_ARM64 = 0x4, ARCH_RISCV64 = 0x8, ARCH_S390X = 0x10, + ARCH_LOONGARCH = 0x20, }; static int get_current_arch(void) @@ -390,6 +390,8 @@ static int get_current_arch(void) return ARCH_RISCV64; #elif defined(__s390x__) return ARCH_S390X; +#elif defined(__loongarch__) + return ARCH_LOONGARCH; #endif return ARCH_UNKNOWN; } @@ -415,7 +417,9 @@ static int parse_test_spec(struct test_loader *tester, bool stdout_on_next_line = true; bool unpriv_stdout_on_next_line = true; bool collect_jit = false; - int func_id, i, err = 0; + const char **tags = NULL; + int func_id, i, nr_tags; + int err = 0; u32 arch_mask = 0; u32 load_mask = 0; struct btf *btf; @@ -438,63 +442,61 @@ static int parse_test_spec(struct test_loader *tester, return -EINVAL; } - for (i = 1; i < btf__type_cnt(btf); i++) { + tags = collect_decl_tags(btf, func_id, &nr_tags); + if (IS_ERR(tags)) + return PTR_ERR(tags); + + for (i = 0; i < nr_tags; i++) { const char *s, *val, *msg; - const struct btf_type *t; bool clear; int flags; - t = btf__type_by_id(btf, i); - if (!btf_is_decl_tag(t)) - continue; - - if (t->type != func_id || btf_decl_tag(t)->component_idx != -1) + s = skip_decl_tag_pfx(tags[i]); + if (!s) continue; - - s = btf__str_by_offset(btf, t->name_off); - if (str_has_pfx(s, TEST_TAG_DESCRIPTION_PFX)) { - description = s + sizeof(TEST_TAG_DESCRIPTION_PFX) - 1; - } else if (strcmp(s, TEST_TAG_EXPECT_FAILURE) == 0) { + if ((val = str_has_pfx(s, "test_description="))) { + description = val; + } else if (strcmp(s, "test_expect_failure") == 0) { spec->priv.expect_failure = true; spec->mode_mask |= PRIV; - } else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS) == 0) { + } else if (strcmp(s, "test_expect_success") == 0) { spec->priv.expect_failure = false; spec->mode_mask |= PRIV; - } else if (strcmp(s, TEST_TAG_EXPECT_FAILURE_UNPRIV) == 0) { + } else if (strcmp(s, "test_expect_failure_unpriv") == 0) { spec->unpriv.expect_failure = true; spec->mode_mask |= UNPRIV; has_unpriv_result = true; - } else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS_UNPRIV) == 0) { + } else if (strcmp(s, "test_expect_success_unpriv") == 0) { spec->unpriv.expect_failure = false; spec->mode_mask |= UNPRIV; has_unpriv_result = true; - } else if (strcmp(s, TEST_TAG_AUXILIARY) == 0) { + } else if (strcmp(s, "test_auxiliary") == 0) { spec->auxiliary = true; spec->mode_mask |= PRIV; - } else if (strcmp(s, TEST_TAG_AUXILIARY_UNPRIV) == 0) { + } else if (strcmp(s, "test_auxiliary_unpriv") == 0) { spec->auxiliary = true; spec->mode_mask |= UNPRIV; - } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX))) { + } else if ((msg = str_has_pfx(s, "test_expect_msg="))) { err = push_msg(msg, false, &spec->priv.expect_msgs); if (err) goto cleanup; spec->mode_mask |= PRIV; - } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_NOT_MSG_PFX))) { + } else if ((msg = str_has_pfx(s, "test_expect_not_msg="))) { err = push_msg(msg, true, &spec->priv.expect_msgs); if (err) goto cleanup; spec->mode_mask |= PRIV; - } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX_UNPRIV))) { + } else if ((msg = str_has_pfx(s, "test_expect_msg_unpriv="))) { err = push_msg(msg, false, &spec->unpriv.expect_msgs); if (err) goto cleanup; spec->mode_mask |= UNPRIV; - } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_NOT_MSG_PFX_UNPRIV))) { + } else if ((msg = str_has_pfx(s, "test_expect_not_msg_unpriv="))) { err = push_msg(msg, true, &spec->unpriv.expect_msgs); if (err) goto cleanup; spec->mode_mask |= UNPRIV; - } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX))) { + } else if ((msg = str_has_pfx(s, "test_jited="))) { if (arch_mask == 0) { PRINT_FAIL("__jited used before __arch_*"); goto cleanup; @@ -506,7 +508,7 @@ static int parse_test_spec(struct test_loader *tester, goto cleanup; spec->mode_mask |= PRIV; } - } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX_UNPRIV))) { + } else if ((msg = str_has_pfx(s, "test_jited_unpriv="))) { if (arch_mask == 0) { PRINT_FAIL("__unpriv_jited used before __arch_*"); goto cleanup; @@ -518,41 +520,36 @@ static int parse_test_spec(struct test_loader *tester, goto cleanup; spec->mode_mask |= UNPRIV; } - } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX))) { + } else if ((msg = str_has_pfx(s, "test_expect_xlated="))) { err = push_disasm_msg(msg, &xlated_on_next_line, &spec->priv.expect_xlated); if (err) goto cleanup; spec->mode_mask |= PRIV; - } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX_UNPRIV))) { + } else if ((msg = str_has_pfx(s, "test_expect_xlated_unpriv="))) { err = push_disasm_msg(msg, &unpriv_xlated_on_next_line, &spec->unpriv.expect_xlated); if (err) goto cleanup; spec->mode_mask |= UNPRIV; - } else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX)) { - val = s + sizeof(TEST_TAG_RETVAL_PFX) - 1; + } else if ((val = str_has_pfx(s, "test_retval="))) { err = parse_retval(val, &spec->priv.retval, "__retval"); if (err) goto cleanup; spec->priv.execute = true; spec->mode_mask |= PRIV; - } else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX_UNPRIV)) { - val = s + sizeof(TEST_TAG_RETVAL_PFX_UNPRIV) - 1; + } else if ((val = str_has_pfx(s, "test_retval_unpriv="))) { err = parse_retval(val, &spec->unpriv.retval, "__retval_unpriv"); if (err) goto cleanup; spec->mode_mask |= UNPRIV; spec->unpriv.execute = true; has_unpriv_retval = true; - } else if (str_has_pfx(s, TEST_TAG_LOG_LEVEL_PFX)) { - val = s + sizeof(TEST_TAG_LOG_LEVEL_PFX) - 1; + } else if ((val = str_has_pfx(s, "test_log_level="))) { err = parse_int(val, &spec->log_level, "test log level"); if (err) goto cleanup; - } else if (str_has_pfx(s, TEST_TAG_PROG_FLAGS_PFX)) { - val = s + sizeof(TEST_TAG_PROG_FLAGS_PFX) - 1; - + } else if ((val = str_has_pfx(s, "test_prog_flags="))) { clear = val[0] == '!'; if (clear) val++; @@ -577,8 +574,7 @@ static int parse_test_spec(struct test_loader *tester, goto cleanup; update_flags(&spec->prog_flags, flags, clear); } - } else if (str_has_pfx(s, TEST_TAG_ARCH)) { - val = s + sizeof(TEST_TAG_ARCH) - 1; + } else if ((val = str_has_pfx(s, "test_arch="))) { if (strcmp(val, "X86_64") == 0) { arch = ARCH_X86_64; } else if (strcmp(val, "ARM64") == 0) { @@ -587,6 +583,8 @@ static int parse_test_spec(struct test_loader *tester, arch = ARCH_RISCV64; } else if (strcmp(val, "s390x") == 0) { arch = ARCH_S390X; + } else if (strcmp(val, "LOONGARCH") == 0) { + arch = ARCH_LOONGARCH; } else { PRINT_FAIL("bad arch spec: '%s'\n", val); err = -EINVAL; @@ -596,16 +594,16 @@ static int parse_test_spec(struct test_loader *tester, collect_jit = get_current_arch() == arch; unpriv_jit_on_next_line = true; jit_on_next_line = true; - } else if (str_has_pfx(s, TEST_BTF_PATH)) { - spec->btf_custom_path = s + sizeof(TEST_BTF_PATH) - 1; - } else if (str_has_pfx(s, TEST_TAG_CAPS_UNPRIV)) { - val = s + sizeof(TEST_TAG_CAPS_UNPRIV) - 1; + } else if ((val = str_has_pfx(s, "test_btf_path="))) { + spec->btf_custom_path = val; + } else if ((val = str_has_pfx(s, "test_btf_func_path="))) { + spec->btf_custom_func_path = val; + } else if ((val = str_has_pfx(s, "test_caps_unpriv="))) { err = parse_caps(val, &spec->unpriv.caps, "test caps"); if (err) goto cleanup; spec->mode_mask |= UNPRIV; - } else if (str_has_pfx(s, TEST_TAG_LOAD_MODE_PFX)) { - val = s + sizeof(TEST_TAG_LOAD_MODE_PFX) - 1; + } else if ((val = str_has_pfx(s, "load_mode="))) { if (strcmp(val, "jited") == 0) { load_mask = JITED; } else if (strcmp(val, "no_jited") == 0) { @@ -615,32 +613,31 @@ static int parse_test_spec(struct test_loader *tester, err = -EINVAL; goto cleanup; } - } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_STDERR_PFX))) { + } else if ((msg = str_has_pfx(s, "test_expect_stderr="))) { err = push_disasm_msg(msg, &stderr_on_next_line, &spec->priv.stderr); if (err) goto cleanup; - } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_STDERR_PFX_UNPRIV))) { + } else if ((msg = str_has_pfx(s, "test_expect_stderr_unpriv="))) { err = push_disasm_msg(msg, &unpriv_stderr_on_next_line, &spec->unpriv.stderr); if (err) goto cleanup; - } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_STDOUT_PFX))) { + } else if ((msg = str_has_pfx(s, "test_expect_stdout="))) { err = push_disasm_msg(msg, &stdout_on_next_line, &spec->priv.stdout); if (err) goto cleanup; - } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_STDOUT_PFX_UNPRIV))) { + } else if ((msg = str_has_pfx(s, "test_expect_stdout_unpriv="))) { err = push_disasm_msg(msg, &unpriv_stdout_on_next_line, &spec->unpriv.stdout); if (err) goto cleanup; - } else if (str_has_pfx(s, TEST_TAG_LINEAR_SIZE)) { + } else if ((val = str_has_pfx(s, "test_linear_size="))) { switch (bpf_program__type(prog)) { case BPF_PROG_TYPE_SCHED_ACT: case BPF_PROG_TYPE_SCHED_CLS: case BPF_PROG_TYPE_CGROUP_SKB: - val = s + sizeof(TEST_TAG_LINEAR_SIZE) - 1; err = parse_int(val, &spec->linear_sz, "test linear size"); if (err) goto cleanup; @@ -659,33 +656,56 @@ static int parse_test_spec(struct test_loader *tester, if (spec->mode_mask == 0) spec->mode_mask = PRIV; - if (!description) - description = spec->prog_name; - if (spec->mode_mask & PRIV) { - spec->priv.name = strdup(description); + spec->priv.name = strdup(spec->prog_name); if (!spec->priv.name) { PRINT_FAIL("failed to allocate memory for priv.name\n"); err = -ENOMEM; goto cleanup; } + + if (description) { + spec->priv.description = strdup(description); + if (!spec->priv.description) { + PRINT_FAIL("failed to allocate memory for priv.description\n"); + err = -ENOMEM; + goto cleanup; + } + } } if (spec->mode_mask & UNPRIV) { - int descr_len = strlen(description); + int name_len = strlen(spec->prog_name); const char *suffix = " @unpriv"; + int suffix_len = strlen(suffix); char *name; - name = malloc(descr_len + strlen(suffix) + 1); + name = malloc(name_len + suffix_len + 1); if (!name) { PRINT_FAIL("failed to allocate memory for unpriv.name\n"); err = -ENOMEM; goto cleanup; } - strcpy(name, description); - strcpy(&name[descr_len], suffix); + strcpy(name, spec->prog_name); + strcpy(&name[name_len], suffix); spec->unpriv.name = name; + + if (description) { + int descr_len = strlen(description); + char *descr; + + descr = malloc(descr_len + suffix_len + 1); + if (!descr) { + PRINT_FAIL("failed to allocate memory for unpriv.description\n"); + err = -ENOMEM; + goto cleanup; + } + + strcpy(descr, description); + strcpy(&descr[descr_len], suffix); + spec->unpriv.description = descr; + } } if (spec->mode_mask & (PRIV | UNPRIV)) { @@ -711,9 +731,11 @@ static int parse_test_spec(struct test_loader *tester, spec->valid = true; + free(tags); return 0; cleanup: + free(tags); free_test_spec(spec); return err; } @@ -775,6 +797,43 @@ static void emit_stderr(const char *stderr, bool force) fprintf(stdout, "STDERR:\n=============\n%s=============\n", stderr); } +static void verify_stderr(int prog_fd, struct expected_msgs *msgs) +{ + LIBBPF_OPTS(bpf_prog_stream_read_opts, ropts); + char *buf; + int ret; + + if (!msgs->cnt) + return; + + buf = malloc(TEST_LOADER_LOG_BUF_SZ); + if (!ASSERT_NEQ(buf, NULL, "malloc")) + return; + + ret = bpf_prog_stream_read(prog_fd, 2, buf, TEST_LOADER_LOG_BUF_SZ - 1, + &ropts); + if (ret > 0) { + buf[ret] = '\0'; + emit_stderr(buf, false); + validate_msgs(buf, msgs, emit_stderr); + } else { + ASSERT_GT(ret, 0, "stderr stream read"); + } + + free(buf); +} + +void verify_test_stderr(struct bpf_object *obj, struct bpf_program *prog) +{ + struct test_spec spec = {}; + + if (parse_test_spec(NULL, obj, prog, &spec)) + return; + + verify_stderr(bpf_program__fd(prog), &spec.priv.stderr); + free_test_spec(&spec); +} + static void emit_stdout(const char *bpf_stdout, bool force) { if (!force && env.verbosity == VERBOSE_NONE) @@ -1124,6 +1183,123 @@ static int get_stream(int stream_id, int prog_fd, char *text, size_t text_sz) return ret; } +/* + * Fix up the program's BTF using BTF from a separate file. + * + * For __naked subprogs, clang drops parameter names from BTF. Find FUNC + * entries with anonymous parameters and replace their FUNC_PROTO with the + * properly-named version from the custom file. + */ +static int fixup_btf_from_path(struct bpf_object *obj, const char *path) +{ + struct btf *prog_btf, *custom_btf; + __u32 i, j, cnt, custom_cnt; + int err = 0; + + prog_btf = bpf_object__btf(obj); + if (!prog_btf) + return 0; + + custom_btf = btf__parse(path, NULL); + if (!ASSERT_OK_PTR(custom_btf, "parse_custom_btf")) + return -EINVAL; + + cnt = btf__type_cnt(prog_btf); + custom_cnt = btf__type_cnt(custom_btf); + + /* Fix up FUNC entries with anonymous params. + * Save all data from prog_btf BEFORE calling btf__add_*, + * since those calls may reallocate the BTF data buffer + * and invalidate any pointers obtained from btf__type_by_id. + */ + for (i = 1; i < cnt; i++) { + const struct btf_type *t = btf__type_by_id(prog_btf, i); + const struct btf_type *fp, *custom_t, *custom_fp; + const struct btf_param *params, *custom_params; + __u32 ret_type_id, vlen; + __u32 *prog_param_types = NULL; + const char *name; + int new_proto_id; + + if (!btf_is_func(t)) + continue; + + fp = btf__type_by_id(prog_btf, t->type); + if (!fp || !btf_is_func_proto(fp) || btf_vlen(fp) == 0) + continue; + + /* Check if any param is anonymous */ + params = btf_params(fp); + if (params[0].name_off != 0) + continue; + + /* Find matching FUNC by name in custom BTF */ + name = btf__name_by_offset(prog_btf, t->name_off); + if (!name) + continue; + + for (j = 1; j < custom_cnt; j++) { + const char *cname; + + custom_t = btf__type_by_id(custom_btf, j); + if (!btf_is_func(custom_t)) + continue; + cname = btf__name_by_offset(custom_btf, custom_t->name_off); + if (cname && strcmp(name, cname) == 0) + break; + } + if (j >= custom_cnt) + continue; + + custom_fp = btf__type_by_id(custom_btf, custom_t->type); + if (!custom_fp || !btf_is_func_proto(custom_fp)) + continue; + + vlen = btf_vlen(fp); + if (vlen != btf_vlen(custom_fp)) + continue; + + /* Save data before btf__add_* calls invalidate pointers */ + ret_type_id = fp->type; + prog_param_types = malloc(vlen * sizeof(*prog_param_types)); + if (!prog_param_types) { + err = -ENOMEM; + break; + } + for (j = 0; j < vlen; j++) + prog_param_types[j] = params[j].type; + + /* Add a new FUNC_PROTO: param names from custom, types from prog */ + new_proto_id = btf__add_func_proto(prog_btf, ret_type_id); + if (new_proto_id < 0) { + err = new_proto_id; + free(prog_param_types); + break; + } + + custom_params = btf_params(custom_fp); + for (j = 0; j < vlen; j++) { + const char *pname; + + pname = btf__name_by_offset(custom_btf, custom_params[j].name_off); + err = btf__add_func_param(prog_btf, pname ?: "", prog_param_types[j]); + if (err) + break; + } + free(prog_param_types); + if (err) + break; + + /* Update the FUNC to point to the new FUNC_PROTO (re-fetch + * since btf__add_* may have reallocated the data buffer). + */ + ((struct btf_type *)btf__type_by_id(prog_btf, i))->type = new_proto_id; + } + + btf__free(custom_btf); + return err; +} + /* this function is forced noinline and has short generic name to look better * in test_progs output (in case of a failure) */ @@ -1148,7 +1324,7 @@ void run_subtest(struct test_loader *tester, int links_cnt = 0; bool should_load; - if (!test__start_subtest(subspec->name)) + if (!test__start_subtest_with_desc(subspec->name, subspec->description)) return; if ((get_current_arch() & spec->arch_mask) == 0) { @@ -1180,13 +1356,27 @@ void run_subtest(struct test_loader *tester, } } - /* Implicitly reset to NULL if next test case doesn't specify */ + /* Implicitly reset to NULL if next test case doesn't specify. + * btf_custom_func_path also serves as btf_custom_path for kfunc resolution. + */ open_opts->btf_custom_path = spec->btf_custom_path; + if (!open_opts->btf_custom_path) + open_opts->btf_custom_path = spec->btf_custom_func_path; tobj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, open_opts); if (!ASSERT_OK_PTR(tobj, "obj_open_mem")) /* shouldn't happen */ goto subtest_cleanup; + /* Fix up __naked subprog BTF using a separate file with named params */ + if (spec->btf_custom_func_path) { + err = fixup_btf_from_path(tobj, spec->btf_custom_func_path); + if (err) { + PRINT_FAIL("failed to fixup BTF from %s: %d\n", + spec->btf_custom_func_path, err); + goto tobj_cleanup; + } + } + i = 0; bpf_object__for_each_program(tprog_iter, tobj) { spec_iter = &specs[i++]; @@ -1300,17 +1490,7 @@ void run_subtest(struct test_loader *tester, goto tobj_cleanup; } - if (subspec->stderr.cnt) { - err = get_stream(2, bpf_program__fd(tprog), - tester->log_buf, tester->log_buf_sz); - if (err <= 0) { - PRINT_FAIL("Unexpected retval from get_stream(): %d, errno = %d\n", - err, errno); - goto tobj_cleanup; - } - emit_stderr(tester->log_buf, false /*force*/); - validate_msgs(tester->log_buf, &subspec->stderr, emit_stderr); - } + verify_stderr(bpf_program__fd(tprog), &subspec->stderr); if (subspec->stdout.cnt) { err = get_stream(1, bpf_program__fd(tprog), diff --git a/tools/testing/selftests/bpf/test_maps.c b/tools/testing/selftests/bpf/test_maps.c index ccc5acd55ff9..6a2641ee7897 100644 --- a/tools/testing/selftests/bpf/test_maps.c +++ b/tools/testing/selftests/bpf/test_maps.c @@ -260,6 +260,16 @@ static void test_hashmap_percpu(unsigned int task, void *data) close(fd); } +#define MAP_RETRIES 20 + +static bool can_retry(int err) +{ + return (err == EAGAIN || err == EBUSY || + ((err == ENOMEM || err == E2BIG) && + map_opts.map_flags == BPF_F_NO_PREALLOC)); +} + + #define VALUE_SIZE 3 static int helper_fill_hashmap(int max_entries) { @@ -274,10 +284,11 @@ static int helper_fill_hashmap(int max_entries) for (i = 0; i < max_entries; i++) { key = i; value[0] = key; - ret = bpf_map_update_elem(fd, &key, value, BPF_NOEXIST); + ret = map_update_retriable(fd, &key, value, BPF_NOEXIST, + MAP_RETRIES, can_retry); CHECK(ret != 0, "can't update hashmap", - "err: %s\n", strerror(ret)); + "err: %s\n", strerror(-ret)); } return fd; @@ -748,16 +759,15 @@ static void test_sockmap(unsigned int tasks, void *data) goto out_sockmap; } - /* Test update with unsupported UDP socket */ + /* Test update with unsupported unbound UDP socket */ udp = socket(AF_INET, SOCK_DGRAM, 0); - i = 0; - err = bpf_map_update_elem(fd, &i, &udp, BPF_ANY); - if (err) { - printf("Failed socket update SOCK_DGRAM '%i:%i'\n", - i, udp); + CHECK(udp < 0, "socket(AF_INET, SOCK_DGRAM)", "errno:%d\n", errno); + err = bpf_map_update_elem(fd, &(int){0}, &udp, BPF_ANY); + close(udp); + if (!err) { + printf("Unexpectedly succeeded unbound UDP update '0:%i'\n", udp); goto out_sockmap; } - close(udp); /* Test update without programs */ for (i = 0; i < 6; i++) { @@ -1392,17 +1402,9 @@ static void test_map_stress(void) #define DO_UPDATE 1 #define DO_DELETE 0 -#define MAP_RETRIES 20 #define MAX_DELAY_US 50000 #define MIN_DELAY_RANGE_US 5000 -static bool can_retry(int err) -{ - return (err == EAGAIN || err == EBUSY || - ((err == ENOMEM || err == E2BIG) && - map_opts.map_flags == BPF_F_NO_PREALLOC)); -} - int map_update_retriable(int map_fd, const void *key, const void *value, int flags, int attempts, retry_for_error_fn need_retry) { diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 0929f4a7bda4..46eb201b96a3 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -165,6 +165,8 @@ struct prog_test_def { void (*run_test)(void); void (*run_serial_test)(void); bool should_run; + bool not_built; + bool selected; bool need_cgroup_cleanup; bool should_tmon; }; @@ -308,16 +310,34 @@ static bool match_subtest(struct test_filter_set *filter, return false; } +static bool match_subtest_desc(struct test_filter_set *filter, + const char *test_name, + const char *subtest_name, + const char *subtest_desc) +{ + if (match_subtest(filter, test_name, subtest_name)) + return true; + + if (!subtest_desc || !subtest_desc[0] || + strcmp(subtest_name, subtest_desc) == 0) + return false; + + return match_subtest(filter, test_name, subtest_desc); +} + static bool should_run_subtest(struct test_selector *sel, struct test_selector *subtest_sel, int subtest_num, const char *test_name, - const char *subtest_name) + const char *subtest_name, + const char *subtest_desc) { - if (match_subtest(&sel->blacklist, test_name, subtest_name)) + if (match_subtest_desc(&sel->blacklist, test_name, + subtest_name, subtest_desc)) return false; - if (match_subtest(&sel->whitelist, test_name, subtest_name)) + if (match_subtest_desc(&sel->whitelist, test_name, + subtest_name, subtest_desc)) return true; if (!sel->whitelist.cnt && !subtest_sel->num_set) @@ -354,6 +374,8 @@ static void print_test_result(const struct prog_test_def *test, const struct tes fprintf(env.stdout_saved, "#%-*d %s:", TEST_NUM_WIDTH, test->test_num, test->test_name); if (test_state->error_cnt) fprintf(env.stdout_saved, "FAIL"); + else if (test->not_built) + fprintf(env.stdout_saved, "SKIP (not built)"); else if (!skipped_cnt) fprintf(env.stdout_saved, "OK"); else if (skipped_cnt == subtests_cnt || !subtests_cnt) @@ -402,10 +424,12 @@ static void jsonw_write_log_message(json_writer_t *w, char *log_buf, size_t log_ } } +/* @quiet elides the human readable output, the JSON report is unaffected */ static void dump_test_log(const struct prog_test_def *test, const struct test_state *test_state, bool skip_ok_subtests, bool par_exec_result, + bool quiet, json_writer_t *w) { bool test_failed = test_state->error_cnt > 0; @@ -427,7 +451,7 @@ static void dump_test_log(const struct prog_test_def *test, if (verbose() && !par_exec_result) return; - if (test_state->log_cnt && print_test) + if (test_state->log_cnt && print_test && !quiet) print_test_log(test_state->log_buf, test_state->log_cnt); if (w && print_test) { @@ -449,15 +473,16 @@ static void dump_test_log(const struct prog_test_def *test, if ((skip_ok_subtests && !subtest_failed) || subtest_filtered) continue; - if (subtest_state->log_cnt && print_subtest) { + if (subtest_state->log_cnt && print_subtest && !quiet) { print_test_log(subtest_state->log_buf, subtest_state->log_cnt); } - print_subtest_name(test->test_num, i + 1, - test->test_name, subtest_state->name, - test_result(subtest_state->error_cnt, - subtest_state->skipped)); + if (!quiet) + print_subtest_name(test->test_num, i + 1, + test->test_name, subtest_state->name, + test_result(subtest_state->error_cnt, + subtest_state->skipped)); if (w && print_subtest) { jsonw_start_object(w); @@ -474,7 +499,8 @@ static void dump_test_log(const struct prog_test_def *test, jsonw_end_object(w); } - print_test_result(test, test_state); + if (!quiet) + print_test_result(test, test_state); } /* A bunch of tests set custom affinity per-thread and/or per-process. Reset @@ -544,24 +570,26 @@ void test__end_subtest(void) env.subtest_state = NULL; } -bool test__start_subtest(const char *subtest_name) +bool test__start_subtest_with_desc(const char *subtest_name, const char *subtest_desc) { struct prog_test_def *test = env.test; struct test_state *state = env.test_state; struct subtest_state *subtest_state; + const char *subtest_display_name; size_t sub_state_size = sizeof(*subtest_state); + void *tmp; if (env.subtest_state) test__end_subtest(); state->subtest_num++; - state->subtest_states = - realloc(state->subtest_states, - state->subtest_num * sub_state_size); - if (!state->subtest_states) { + tmp = realloc(state->subtest_states, state->subtest_num * sub_state_size); + if (!tmp) { + state->subtest_num--; fprintf(stderr, "Not enough memory to allocate subtest result\n"); return false; } + state->subtest_states = tmp; subtest_state = &state->subtest_states[state->subtest_num - 1]; @@ -574,7 +602,9 @@ bool test__start_subtest(const char *subtest_name) return false; } - subtest_state->name = strdup(subtest_name); + subtest_display_name = subtest_desc ? subtest_desc : subtest_name; + + subtest_state->name = strdup(subtest_display_name); if (!subtest_state->name) { fprintf(env.stderr_saved, "Subtest #%d: failed to copy subtest name!\n", @@ -586,14 +616,15 @@ bool test__start_subtest(const char *subtest_name) &env.subtest_selector, state->subtest_num, test->test_name, - subtest_name)) { + subtest_name, + subtest_desc)) { subtest_state->filtered = true; return false; } - subtest_state->should_tmon = match_subtest(&env.tmon_selector.whitelist, - test->test_name, - subtest_name); + subtest_state->should_tmon = match_subtest_desc(&env.tmon_selector.whitelist, + test->test_name, subtest_name, + subtest_desc); env.subtest_state = subtest_state; stdio_hijack_init(&subtest_state->log_buf, &subtest_state->log_cnt); @@ -602,6 +633,11 @@ bool test__start_subtest(const char *subtest_name) return true; } +bool test__start_subtest(const char *subtest_name) +{ + return test__start_subtest_with_desc(subtest_name, NULL); +} + void test__force_log(void) { env.test_state->force_log = true; @@ -699,11 +735,14 @@ int compare_map_keys(int map1_fd, int map2_fd) int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len) { __u32 key, next_key, *cur_key_p, *next_key_p; - char *val_buf1, *val_buf2; - int i, err = 0; + char *val_buf1 = NULL, *val_buf2 = NULL; + int i, err = -ENOMEM; val_buf1 = malloc(stack_trace_len); val_buf2 = malloc(stack_trace_len); + if (!val_buf1 || !val_buf2) + goto out; + err = 0; cur_key_p = NULL; next_key_p = &key; while (bpf_map_get_next_key(smap_fd, cur_key_p, next_key_p) == 0) { @@ -864,6 +903,7 @@ enum ARG_KEYS { ARG_JSON_SUMMARY = 'J', ARG_TRAFFIC_MONITOR = 'm', ARG_WATCHDOG_TIMEOUT = 'w', + ARG_NO_ERROR_SUMMARY = -2, }; static const struct argp_option opts[] = { @@ -896,6 +936,8 @@ static const struct argp_option opts[] = { #endif { "watchdog-timeout", ARG_WATCHDOG_TIMEOUT, "SECONDS", 0, "Kill the process if tests are not making progress for specified number of seconds." }, + { "no-error-summary", ARG_NO_ERROR_SUMMARY, NULL, 0, + "Do not re-print the aggregated error logs of failed tests at the end of the run." }, {}, }; @@ -1097,6 +1139,9 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) case ARG_DEBUG: env->debug = true; break; + case ARG_NO_ERROR_SUMMARY: + env->error_summary = false; + break; case ARG_JSON_SUMMARY: env->json = fopen(arg, "w"); if (env->json == NULL) { @@ -1230,7 +1275,7 @@ int get_bpf_max_tramp_links_from(struct btf *btf) const struct btf_type *t; __u32 i, type_cnt; const char *name; - __u16 j, vlen; + __u32 j, vlen; for (i = 1, type_cnt = btf__type_cnt(btf); i < type_cnt; i++) { t = btf__type_by_id(btf, i); @@ -1269,7 +1314,7 @@ static void dump_crash_log(void) if (env.test) { env.test_state->error_cnt++; - dump_test_log(env.test, env.test_state, true, false, NULL); + dump_test_log(env.test, env.test_state, true, false, false, NULL); } } @@ -1427,7 +1472,7 @@ static void run_one_test(int test_num) free(stop_libbpf_log_capture()); - dump_test_log(test, state, false, false, NULL); + dump_test_log(test, state, false, false, false, NULL); } struct dispatch_data { @@ -1482,13 +1527,15 @@ static int dispatch_thread_send_subtests(int sock_fd, struct test_state *state) struct subtest_state *subtest_state; int subtest_num = state->subtest_num; - state->subtest_states = malloc(subtest_num * sizeof(*subtest_state)); + state->subtest_states = calloc(subtest_num, sizeof(*subtest_state)); + if (!state->subtest_states) { + state->subtest_num = 0; + return -ENOMEM; + } for (int i = 0; i < subtest_num; i++) { subtest_state = &state->subtest_states[i]; - memset(subtest_state, 0, sizeof(*subtest_state)); - if (read_prog_test_msg(sock_fd, &msg, MSG_SUBTEST_DONE)) return 1; @@ -1586,7 +1633,7 @@ static void *dispatch_thread(void *ctx) } while (false); pthread_mutex_lock(&stdout_output_lock); - dump_test_log(test, state, false, true, NULL); + dump_test_log(test, state, false, true, false, NULL); pthread_mutex_unlock(&stdout_output_lock); } /* while (true) */ error: @@ -1609,11 +1656,12 @@ done: static void calculate_summary_and_print_errors(struct test_env *env) { - int i; - int succ_cnt = 0, fail_cnt = 0, sub_succ_cnt = 0, skip_cnt = 0; + int i, j; + int succ_cnt = 0, fail_cnt = 0, sub_succ_cnt = 0, sub_fail_cnt = 0, skip_cnt = 0; json_writer_t *w = NULL; for (i = 0; i < prog_test_cnt; i++) { + struct prog_test_def *test = &prog_test_defs[i]; struct test_state *state = &test_states[i]; if (!state->tested) @@ -1622,10 +1670,14 @@ static void calculate_summary_and_print_errors(struct test_env *env) sub_succ_cnt += state->sub_succ_cnt; skip_cnt += state->skip_cnt; - if (state->error_cnt) + if (state->error_cnt) { fail_cnt++; - else + for (j = 0; j < state->subtest_num; j++) + if (state->subtest_states[j].error_cnt) + sub_fail_cnt++; + } else if (!test->not_built) { succ_cnt++; + } } if (env->json) { @@ -1640,6 +1692,7 @@ static void calculate_summary_and_print_errors(struct test_env *env) jsonw_uint_field(w, "success_subtest", sub_succ_cnt); jsonw_uint_field(w, "skipped", skip_cnt); jsonw_uint_field(w, "failed", fail_cnt); + jsonw_uint_field(w, "failed_subtest", sub_fail_cnt); jsonw_name(w, "results"); jsonw_start_array(w); } @@ -1648,9 +1701,14 @@ static void calculate_summary_and_print_errors(struct test_env *env) * We only print error logs summary when there are failed tests and * verbose mode is not enabled. Otherwise, results may be inconsistent. * + * --no-error-summary elides the human readable dump. The walk still + * happens when a JSON report was requested, so the JSON output keeps + * its per-test results; with no JSON report there is nothing left to + * do and the whole loop is skipped. */ - if (!verbose() && fail_cnt) { - printf("\nAll error logs:\n"); + if (!verbose() && fail_cnt && (env->error_summary || w)) { + if (env->error_summary) + printf("\nAll error logs:\n"); /* print error logs again */ for (i = 0; i < prog_test_cnt; i++) { @@ -1660,7 +1718,8 @@ static void calculate_summary_and_print_errors(struct test_env *env) if (!state->tested || !state->error_cnt) continue; - dump_test_log(test, state, true, true, w); + dump_test_log(test, state, true, true, + !env->error_summary, w); } } @@ -1673,8 +1732,13 @@ static void calculate_summary_and_print_errors(struct test_env *env) if (env->json) fclose(env->json); - printf("Summary: %d/%d PASSED, %d SKIPPED, %d FAILED\n", - succ_cnt, sub_succ_cnt, skip_cnt, fail_cnt); + if (env->not_built_cnt) + printf("Summary: %d/%d PASSED, %d SKIPPED (%d not built), %d/%d FAILED\n", + succ_cnt, sub_succ_cnt, skip_cnt, env->not_built_cnt, + fail_cnt, sub_fail_cnt); + else + printf("Summary: %d/%d PASSED, %d SKIPPED, %d/%d FAILED\n", + succ_cnt, sub_succ_cnt, skip_cnt, fail_cnt, sub_fail_cnt); env->succ_cnt = succ_cnt; env->sub_succ_cnt = sub_succ_cnt; @@ -1704,7 +1768,7 @@ static void server_main(void) data[i].worker_id = i; data[i].sock_fd = env.worker_socks[i]; rc = pthread_create(&dispatcher_threads[i], NULL, dispatch_thread, &data[i]); - if (rc < 0) { + if (rc) { perror("Failed to launch dispatcher thread"); exit(EXIT_ERR_SETUP_INFRA); } @@ -1745,6 +1809,19 @@ static void server_main(void) run_one_test(i); } + /* mark not-built tests as skipped */ + for (int i = 0; i < prog_test_cnt; i++) { + struct prog_test_def *test = &prog_test_defs[i]; + struct test_state *state = &test_states[i]; + + if (test->not_built && test->selected) { + state->tested = true; + state->skip_cnt = 1; + env.not_built_cnt++; + print_test_result(test, state); + } + } + /* generate summary */ fflush(stderr); fflush(stdout); @@ -1836,7 +1913,6 @@ static int worker_main_send_subtests(int sock, struct test_state *state) worker_main_send_log(sock, subtest_state->log_buf, subtest_state->log_cnt); free_subtest_state(subtest_state); - free(subtest_state->name); } out: @@ -1973,6 +2049,7 @@ int main(int argc, char **argv) env.secs_till_notify = 10; env.secs_till_kill = 120; + env.error_summary = true; err = argp_parse(&argp, argc, argv, 0, NULL, &env); if (err) return err; @@ -2019,15 +2096,20 @@ int main(int argc, char **argv) struct prog_test_def *test = &prog_test_defs[i]; test->test_num = i + 1; - test->should_run = should_run(&env.test_selector, - test->test_num, test->test_name); + test->selected = should_run(&env.test_selector, + test->test_num, test->test_name); + test->should_run = test->selected; - if ((test->run_test == NULL && test->run_serial_test == NULL) || - (test->run_test != NULL && test->run_serial_test != NULL)) { + if (test->run_test && test->run_serial_test) { fprintf(stderr, "Test %d:%s must have either test_%s() or serial_test_%sl() defined.\n", test->test_num, test->test_name, test->test_name, test->test_name); exit(EXIT_ERR_SETUP_INFRA); } + if (!test->run_test && !test->run_serial_test) { + test->not_built = true; + test->should_run = false; + continue; + } if (test->should_run) test->should_tmon = should_tmon(&env.tmon_selector, test->test_name); } @@ -2079,9 +2161,18 @@ int main(int argc, char **argv) for (i = 0; i < prog_test_cnt; i++) { struct prog_test_def *test = &prog_test_defs[i]; + struct test_state *state = &test_states[i]; - if (!test->should_run) + if (!test->should_run) { + if (test->not_built && test->selected && + !env.get_test_cnt && !env.list_test_names) { + state->tested = true; + state->skip_cnt = 1; + env.not_built_cnt++; + print_test_result(test, state); + } continue; + } if (env.get_test_cnt) { env.succ_cnt++; diff --git a/tools/testing/selftests/bpf/test_progs.h b/tools/testing/selftests/bpf/test_progs.h index eebfc18cdcd2..ea493c477fbd 100644 --- a/tools/testing/selftests/bpf/test_progs.h +++ b/tools/testing/selftests/bpf/test_progs.h @@ -105,6 +105,7 @@ struct test_env { struct test_selector tmon_selector; bool verifier_stats; bool debug; + bool error_summary; enum verbosity verbosity; bool jit_enabled; @@ -123,8 +124,9 @@ struct test_env { int succ_cnt; /* successful tests */ int sub_succ_cnt; /* successful sub-tests */ - int fail_cnt; /* total failed tests + sub-tests */ + int fail_cnt; /* failed tests */ int skip_cnt; /* skipped tests */ + int not_built_cnt; /* tests not built */ int saved_netns_fd; int workers; /* number of worker process */ @@ -181,6 +183,7 @@ struct msg { extern struct test_env env; void test__force_log(void); +bool test__start_subtest_with_desc(const char *name, const char *description); bool test__start_subtest(const char *name); void test__end_subtest(void); void test__skip(void); @@ -562,5 +565,7 @@ struct expected_msgs { void validate_msgs(const char *log_buf, struct expected_msgs *msgs, void (*emit_fn)(const char *buf, bool force)); +void free_msgs(struct expected_msgs *msgs); +void verify_test_stderr(struct bpf_object *obj, struct bpf_program *prog); #endif /* __TEST_PROGS_H */ diff --git a/tools/testing/selftests/bpf/test_sockmap.c b/tools/testing/selftests/bpf/test_sockmap.c index 76568db7a664..aaf2050e8845 100644 --- a/tools/testing/selftests/bpf/test_sockmap.c +++ b/tools/testing/selftests/bpf/test_sockmap.c @@ -26,7 +26,6 @@ #include <linux/sock_diag.h> #include <linux/bpf.h> #include <linux/if_link.h> -#include <linux/tls.h> #include <assert.h> #include <libgen.h> @@ -41,13 +40,6 @@ int running; static void running_handler(int a); -#ifndef TCP_ULP -# define TCP_ULP 31 -#endif -#ifndef SOL_TLS -# define SOL_TLS 282 -#endif - /* randomly selected ports for testing on lo */ #define S1_PORT 10000 #define S2_PORT 10001 @@ -63,10 +55,10 @@ int s1, s2, c1, c2, p1, p2; int test_cnt; int passed; int failed; -int map_fd[9]; -struct bpf_map *maps[9]; -struct bpf_program *progs[9]; -struct bpf_link *links[9]; +int map_fd[8]; +struct bpf_map *maps[8]; +struct bpf_program *progs[8]; +struct bpf_link *links[8]; int txmsg_pass; int txmsg_redir; @@ -81,10 +73,6 @@ int txmsg_start_pop; int txmsg_pop; int txmsg_ingress; int txmsg_redir_skb; -int txmsg_ktls_skb; -int txmsg_ktls_skb_drop; -int txmsg_ktls_skb_redir; -int ktls; int peek_flag; int skb_use_parser; int txmsg_omit_skb_parser; @@ -115,7 +103,6 @@ static const struct option long_options[] = { {"txmsg_pop", required_argument, NULL, 'x'}, {"txmsg_ingress", no_argument, &txmsg_ingress, 1 }, {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 }, - {"ktls", no_argument, &ktls, 1 }, {"peek", no_argument, &peek_flag, 1 }, {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1}, {"whitelist", required_argument, NULL, 'n' }, @@ -183,7 +170,6 @@ static void test_reset(void) txmsg_pass = txmsg_drop = txmsg_redir = 0; txmsg_apply = txmsg_cork = 0; txmsg_ingress = txmsg_redir_skb = 0; - txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0; txmsg_omit_skb_parser = 0; skb_use_parser = 0; } @@ -238,71 +224,6 @@ static void usage(char *argv[]) printf("\n"); } -char *sock_to_string(int s) -{ - if (s == c1) - return "client1"; - else if (s == c2) - return "client2"; - else if (s == s1) - return "server1"; - else if (s == s2) - return "server2"; - else if (s == p1) - return "peer1"; - else if (s == p2) - return "peer2"; - else - return "unknown"; -} - -static int sockmap_init_ktls(int verbose, int s) -{ - struct tls12_crypto_info_aes_gcm_128 tls_tx = { - .info = { - .version = TLS_1_2_VERSION, - .cipher_type = TLS_CIPHER_AES_GCM_128, - }, - }; - struct tls12_crypto_info_aes_gcm_128 tls_rx = { - .info = { - .version = TLS_1_2_VERSION, - .cipher_type = TLS_CIPHER_AES_GCM_128, - }, - }; - int so_buf = 6553500; - int err; - - err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls")); - if (err) { - fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err); - return -EINVAL; - } - err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx)); - if (err) { - fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err); - return -EINVAL; - } - err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx)); - if (err) { - fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err); - return -EINVAL; - } - err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf)); - if (err) { - fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err); - return -EINVAL; - } - err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf)); - if (err) { - fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err); - return -EINVAL; - } - - if (verbose) - fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s)); - return 0; -} static int sockmap_init_sockets(int verbose) { int i, err, one = 1; @@ -514,7 +435,8 @@ static int msg_alloc_iov(struct msghdr *msg, return 0; unwind_iov: for (i--; i >= 0 ; i--) - free(msg->msg_iov[i].iov_base); + free(iov[i].iov_base); + free(iov); return -ENOMEM; } @@ -557,19 +479,6 @@ static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz, for (i = 0, j = 0; i < msg->msg_iovlen && size; i++, j = 0) { unsigned char *d = msg->msg_iov[i].iov_base; - /* Special case test for skb ingress + ktls */ - if (i == 0 && txmsg_ktls_skb) { - if (msg->msg_iov[i].iov_len < 4) - return -EDATAINTEGRITY; - if (memcmp(d, "PASS", 4) != 0) { - fprintf(stderr, - "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n", - i, 0, d[0], d[1], d[2], d[3]); - return -EDATAINTEGRITY; - } - j = 4; /* advance index past PASS header */ - } - for (; j < msg->msg_iov[i].iov_len && size; j++) { if (push > 0 && check_cnt == verify_push_start + verify_push_len - push) { @@ -849,21 +758,6 @@ static int sendmsg_test(struct sockmap_options *opt) else rx_fd = p2; - if (ktls) { - /* Redirecting into non-TLS socket which sends into a TLS - * socket is not a valid test. So in this case lets not - * enable kTLS but still run the test. - */ - if (!txmsg_redir || txmsg_ingress) { - err = sockmap_init_ktls(opt->verbose, rx_fd); - if (err) - return err; - } - err = sockmap_init_ktls(opt->verbose, c1); - if (err) - return err; - } - if (opt->tx_wait_mem) { struct timeval timeout; int rxtx_buf_len = 1024; @@ -882,7 +776,7 @@ static int sendmsg_test(struct sockmap_options *opt) rxpid = fork(); if (rxpid == 0) { - if (opt->drop_expected || txmsg_ktls_skb_drop) + if (opt->drop_expected) _exit(0); if (!iov_buf) /* zero bytes sent case */ @@ -1073,28 +967,8 @@ static int run_options(struct sockmap_options *options, int cg_fd, int test) return -1; } - /* Attach programs to TLS sockmap */ - if (txmsg_ktls_skb) { - if (!txmsg_omit_skb_parser) { - links[2] = bpf_program__attach_sockmap(progs[0], map_fd[8]); - if (!links[2]) { - fprintf(stderr, - "ERROR: bpf_program__attach_sockmap (TLS sockmap %i->%i): (%s)\n", - bpf_program__fd(progs[0]), map_fd[8], strerror(errno)); - return -1; - } - } - - links[3] = bpf_program__attach_sockmap(progs[2], map_fd[8]); - if (!links[3]) { - fprintf(stderr, "ERROR: bpf_program__attach_sockmap (TLS sockmap): (%s)\n", - strerror(errno)); - return -1; - } - } - /* Attach to cgroups */ - err = bpf_prog_attach(bpf_program__fd(progs[3]), cg_fd, BPF_CGROUP_SOCK_OPS, 0); + err = bpf_prog_attach(bpf_program__fd(progs[2]), cg_fd, BPF_CGROUP_SOCK_OPS, 0); if (err) { fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n", err, strerror(errno)); @@ -1110,15 +984,15 @@ run: /* Attach txmsg program to sockmap */ if (txmsg_pass) - tx_prog = progs[4]; + tx_prog = progs[3]; else if (txmsg_redir) - tx_prog = progs[5]; + tx_prog = progs[4]; else if (txmsg_apply) - tx_prog = progs[6]; + tx_prog = progs[5]; else if (txmsg_cork) - tx_prog = progs[7]; + tx_prog = progs[6]; else if (txmsg_drop) - tx_prog = progs[8]; + tx_prog = progs[7]; else tx_prog = NULL; @@ -1291,34 +1165,6 @@ run: } } - if (txmsg_ktls_skb) { - int ingress = BPF_F_INGRESS; - - i = 0; - err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY); - if (err) { - fprintf(stderr, - "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", - err, strerror(errno)); - } - - if (txmsg_ktls_skb_redir) { - i = 1; - err = bpf_map_update_elem(map_fd[7], - &i, &ingress, BPF_ANY); - if (err) { - fprintf(stderr, - "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", - err, strerror(errno)); - } - } - - if (txmsg_ktls_skb_drop) { - i = 1; - err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY); - } - } - if (txmsg_redir_skb) { int skb_fd = (test == SENDMSG || test == SENDPAGE) ? p2 : p1; @@ -1373,7 +1219,7 @@ run: fprintf(stderr, "unknown test\n"); out: /* Detach and zero all the maps */ - bpf_prog_detach2(bpf_program__fd(progs[3]), cg_fd, BPF_CGROUP_SOCK_OPS); + bpf_prog_detach2(bpf_program__fd(progs[2]), cg_fd, BPF_CGROUP_SOCK_OPS); for (i = 0; i < ARRAY_SIZE(links); i++) { if (links[i]) @@ -1457,10 +1303,6 @@ static void test_options(char *options) append_str(options, "ingress,", OPTSTRING); if (txmsg_redir_skb) append_str(options, "redir_skb,", OPTSTRING); - if (txmsg_ktls_skb) - append_str(options, "ktls_skb,", OPTSTRING); - if (ktls) - append_str(options, "ktls,", OPTSTRING); if (peek_flag) append_str(options, "peek,", OPTSTRING); } @@ -1602,57 +1444,6 @@ static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt) test_send(opt, cgrp); } -static void test_txmsg_skb(int cgrp, struct sockmap_options *opt) -{ - bool data = opt->data_test; - int k = ktls; - - opt->data_test = true; - ktls = 1; - - txmsg_pass = txmsg_drop = 0; - txmsg_ingress = txmsg_redir = 0; - txmsg_ktls_skb = 1; - txmsg_pass = 1; - - /* Using data verification so ensure iov layout is - * expected from test receiver side. e.g. has enough - * bytes to write test code. - */ - opt->iov_length = 100; - opt->iov_count = 1; - opt->rate = 1; - test_exec(cgrp, opt); - - txmsg_ktls_skb_drop = 1; - test_exec(cgrp, opt); - - txmsg_ktls_skb_drop = 0; - txmsg_ktls_skb_redir = 1; - test_exec(cgrp, opt); - txmsg_ktls_skb_redir = 0; - - /* Tests that omit skb_parser */ - txmsg_omit_skb_parser = 1; - ktls = 0; - txmsg_ktls_skb = 0; - test_exec(cgrp, opt); - - txmsg_ktls_skb_drop = 1; - test_exec(cgrp, opt); - txmsg_ktls_skb_drop = 0; - - txmsg_ktls_skb_redir = 1; - test_exec(cgrp, opt); - - ktls = 1; - test_exec(cgrp, opt); - txmsg_omit_skb_parser = 0; - - opt->data_test = data; - ktls = k; -} - /* Test cork with hung data. This tests poor usage patterns where * cork can leave data on the ring if user program is buggy and * doesn't flush them somehow. They do take some time however @@ -1908,8 +1699,6 @@ static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt) { txmsg_pass = 1; skb_use_parser = 512; - if (ktls == 1) - skb_use_parser = 570; opt->iov_length = 256; opt->iov_count = 1; opt->rate = 2; @@ -1918,8 +1707,6 @@ static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt) static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt) { - if (ktls == 1) - return; skb_use_parser = 10; opt->iov_length = 20; opt->iov_count = 1; @@ -1938,7 +1725,6 @@ char *map_names[] = { "sock_bytes", "sock_redir_flags", "sock_skb_opts", - "tls_sock_map", }; static int populate_progs(char *bpf_file) @@ -1988,7 +1774,6 @@ struct _test test[] = { {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem}, {"txmsg test drop", test_txmsg_drop}, {"txmsg test ingress redirect", test_txmsg_ingress_redir}, - {"txmsg test skb", test_txmsg_skb}, {"txmsg test apply", test_txmsg_apply}, {"txmsg test cork", test_txmsg_cork}, {"txmsg test hanging corks", test_txmsg_cork_hangs}, @@ -2085,20 +1870,10 @@ static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt) __test_selftests(cg_fd, opt); } -static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt) -{ - opt->map = BPF_SOCKHASH_FILENAME; - opt->prepend = "ktls"; - ktls = 1; - __test_selftests(cg_fd, opt); - ktls = 0; -} - static int test_selftest(int cg_fd, struct sockmap_options *opt) { test_selftests_sockmap(cg_fd, opt); test_selftests_sockhash(cg_fd, opt); - test_selftests_ktls(cg_fd, opt); test_print_results(); return 0; } diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index a8ae03c57bba..bffb7360434c 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -1560,7 +1560,7 @@ static void do_test_single(struct bpf_test *test, bool unpriv, opts.expected_attach_type = test->expected_attach_type; if (expected_ret == VERBOSE_ACCEPT) - opts.log_level = 2; + opts.log_level = 2 | 4; else if (verbose) opts.log_level = verif_log_level | 4; /* force stats */ else diff --git a/tools/testing/selftests/bpf/test_xdping.sh b/tools/testing/selftests/bpf/test_xdping.sh deleted file mode 100755 index c3d82e0a7378..000000000000 --- a/tools/testing/selftests/bpf/test_xdping.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: GPL-2.0 - -# xdping tests -# Here we setup and teardown configuration required to run -# xdping, exercising its options. -# -# Setup is similar to test_tunnel tests but without the tunnel. -# -# Topology: -# --------- -# root namespace | tc_ns0 namespace -# | -# ---------- | ---------- -# | veth1 | --------- | veth0 | -# ---------- peer ---------- -# -# Device Configuration -# -------------------- -# Root namespace with BPF -# Device names and addresses: -# veth1 IP: 10.1.1.200 -# xdp added to veth1, xdpings originate from here. -# -# Namespace tc_ns0 with BPF -# Device names and addresses: -# veth0 IPv4: 10.1.1.100 -# For some tests xdping run in server mode here. -# - -readonly TARGET_IP="10.1.1.100" -readonly TARGET_NS="xdp_ns0" - -readonly LOCAL_IP="10.1.1.200" - -setup() -{ - ip netns add $TARGET_NS - ip link add veth0 type veth peer name veth1 - ip link set veth0 netns $TARGET_NS - ip netns exec $TARGET_NS ip addr add ${TARGET_IP}/24 dev veth0 - ip addr add ${LOCAL_IP}/24 dev veth1 - ip netns exec $TARGET_NS ip link set veth0 up - ip link set veth1 up -} - -cleanup() -{ - set +e - ip netns delete $TARGET_NS 2>/dev/null - ip link del veth1 2>/dev/null - if [[ $server_pid -ne 0 ]]; then - kill -TERM $server_pid - fi -} - -test() -{ - client_args="$1" - server_args="$2" - - echo "Test client args '$client_args'; server args '$server_args'" - - server_pid=0 - if [[ -n "$server_args" ]]; then - ip netns exec $TARGET_NS ./xdping $server_args & - server_pid=$! - sleep 10 - fi - ./xdping $client_args $TARGET_IP - - if [[ $server_pid -ne 0 ]]; then - kill -TERM $server_pid - server_pid=0 - fi - - echo "Test client args '$client_args'; server args '$server_args': PASS" -} - -set -e - -server_pid=0 - -trap cleanup EXIT - -setup - -for server_args in "" "-I veth0 -s -S" ; do - # client in skb mode - client_args="-I veth1 -S" - test "$client_args" "$server_args" - - # client with count of 10 RTT measurements. - client_args="-I veth1 -S -c 10" - test "$client_args" "$server_args" -done - -# Test drv mode -test "-I veth1 -N" "-I veth0 -s -N" -test "-I veth1 -N -c 10" "-I veth0 -s -N" - -echo "OK. All tests passed" -exit 0 diff --git a/tools/testing/selftests/bpf/testing_helpers.c b/tools/testing/selftests/bpf/testing_helpers.c index 66af0d13751a..c970e7793dfc 100644 --- a/tools/testing/selftests/bpf/testing_helpers.c +++ b/tools/testing/selftests/bpf/testing_helpers.c @@ -5,6 +5,8 @@ #include <stdlib.h> #include <string.h> #include <errno.h> +#include <sys/mman.h> +#include <alloca.h> #include <bpf/bpf.h> #include <bpf/libbpf.h> #include "disasm.h" @@ -368,7 +370,7 @@ int delete_module(const char *name, int flags) return syscall(__NR_delete_module, name, flags); } -int unload_module(const char *name, bool verbose) +int try_unload_module(const char *name, int retries, bool verbose) { int ret, cnt = 0; @@ -379,7 +381,7 @@ int unload_module(const char *name, bool verbose) ret = delete_module(name, 0); if (!ret || errno != EAGAIN) break; - if (++cnt > 10000) { + if (++cnt > retries) { fprintf(stdout, "Unload of %s timed out\n", name); break; } @@ -400,6 +402,11 @@ int unload_module(const char *name, bool verbose) return 0; } +int unload_module(const char *name, bool verbose) +{ + return try_unload_module(name, 10000, verbose); +} + static int __load_module(const char *path, const char *param_values, bool verbose) { int fd; @@ -511,3 +518,19 @@ bool is_jit_enabled(void) return enabled; } + +int stack_mprotect(void) +{ + void *buf; + long sz; + int ret; + + sz = sysconf(_SC_PAGESIZE); + if (sz < 0) + return sz; + + buf = alloca(sz * 3); + ret = mprotect((void *)(((unsigned long)(buf + sz)) & ~(sz - 1)), sz, + PROT_READ | PROT_WRITE | PROT_EXEC); + return ret; +} diff --git a/tools/testing/selftests/bpf/testing_helpers.h b/tools/testing/selftests/bpf/testing_helpers.h index eb20d3772218..2edc6fb7fc52 100644 --- a/tools/testing/selftests/bpf/testing_helpers.h +++ b/tools/testing/selftests/bpf/testing_helpers.h @@ -40,6 +40,7 @@ int finit_module(int fd, const char *param_values, int flags); int delete_module(const char *name, int flags); int load_module(const char *path, bool verbose); int load_module_params(const char *path, const char *param_values, bool verbose); +int try_unload_module(const char *name, int retries, bool verbose); int unload_module(const char *name, bool verbose); static inline __u64 get_time_ns(void) @@ -58,5 +59,6 @@ struct bpf_insn; int get_xlated_program(int fd_prog, struct bpf_insn **buf, __u32 *cnt); int testing_prog_flags(void); bool is_jit_enabled(void); +int stack_mprotect(void); #endif /* __TESTING_HELPERS_H */ diff --git a/tools/testing/selftests/bpf/trace_helpers.c b/tools/testing/selftests/bpf/trace_helpers.c index 0e63daf83ed5..679008b310d9 100644 --- a/tools/testing/selftests/bpf/trace_helpers.c +++ b/tools/testing/selftests/bpf/trace_helpers.c @@ -546,9 +546,10 @@ static const char * const trace_blacklist[] = { "__rcu_read_lock", "__rcu_read_unlock", "bpf_get_numa_node_id", + "___migrate_enable", }; -static bool skip_entry(char *name) +bool is_unsafe_function(const char *name) { int i; @@ -651,7 +652,7 @@ int bpf_get_ksyms(struct ksyms **ksymsp, bool kernel) free(name); if (sscanf(buf, "%ms$*[^\n]\n", &name) != 1) continue; - if (skip_entry(name)) + if (is_unsafe_function(name)) continue; ks = search_kallsyms_custom_local(ksyms, name, search_kallsyms_compare); @@ -728,7 +729,7 @@ int bpf_get_addrs(unsigned long **addrsp, size_t *cntp, bool kernel) free(name); if (sscanf(buf, "%p %ms$*[^\n]\n", &addr, &name) != 2) continue; - if (skip_entry(name)) + if (is_unsafe_function(name)) continue; if (cnt == max_cnt) { diff --git a/tools/testing/selftests/bpf/trace_helpers.h b/tools/testing/selftests/bpf/trace_helpers.h index d5bf1433675d..01c8ecc45627 100644 --- a/tools/testing/selftests/bpf/trace_helpers.h +++ b/tools/testing/selftests/bpf/trace_helpers.h @@ -63,4 +63,5 @@ int read_build_id(const char *path, char *build_id, size_t size); int bpf_get_ksyms(struct ksyms **ksymsp, bool kernel); int bpf_get_addrs(unsigned long **addrsp, size_t *cntp, bool kernel); +bool is_unsafe_function(const char *name); #endif diff --git a/tools/testing/selftests/bpf/uprobe_multi.c b/tools/testing/selftests/bpf/uprobe_multi.c index dd38dc68f635..0af330b6c364 100644 --- a/tools/testing/selftests/bpf/uprobe_multi.c +++ b/tools/testing/selftests/bpf/uprobe_multi.c @@ -100,6 +100,9 @@ int __attribute__((weak)) trigger_uprobe(bool build_id_resident) int page_sz = sysconf(_SC_PAGESIZE); void *addr; + unsigned char vec[1]; + int poll = 0; + /* page-align build ID start */ addr = (void *)((uintptr_t)&build_id_start & ~(page_sz - 1)); @@ -108,9 +111,19 @@ int __attribute__((weak)) trigger_uprobe(bool build_id_resident) * do MADV_POPULATE_READ, and then MADV_PAGEOUT, if necessary */ madvise(addr, page_sz, MADV_POPULATE_READ); - if (!build_id_resident) - madvise(addr, page_sz, MADV_PAGEOUT); - + if (!build_id_resident) { + do { + madvise(addr, page_sz, MADV_PAGEOUT); + /* check if page has been evicted */ + mincore(addr, page_sz, vec); + if (!(vec[0] & 1)) + break; + /* if page is still resident re-attempt MADV_POPULATE_READ/MADV_PAGEOUT */ + madvise(addr, page_sz, MADV_POPULATE_READ); + poll++; + usleep(100); + } while (poll < 500); + } (void)uprobe(); return 0; @@ -131,6 +144,8 @@ int main(int argc, char **argv) return trigger_uprobe(true /* page-in build ID */); error: - fprintf(stderr, "usage: %s <bench|usdt>\n", argv[0]); + fprintf(stderr, + "usage: %s <bench|usdt|uprobe-paged-out|uprobe-paged-in>\n", + argv[0]); return -1; } diff --git a/tools/testing/selftests/bpf/uprobe_multi.ld b/tools/testing/selftests/bpf/uprobe_multi.ld index a2e94828bc8c..2063714b2899 100644 --- a/tools/testing/selftests/bpf/uprobe_multi.ld +++ b/tools/testing/selftests/bpf/uprobe_multi.ld @@ -1,8 +1,8 @@ SECTIONS { - . = ALIGN(4096); + . = ALIGN(65536); .note.gnu.build-id : { *(.note.gnu.build-id) } - . = ALIGN(4096); + . = ALIGN(65536); } INSERT AFTER .text; diff --git a/tools/testing/selftests/bpf/usdt.h b/tools/testing/selftests/bpf/usdt.h index 549d1f774810..75687f50f4e2 100644 --- a/tools/testing/selftests/bpf/usdt.h +++ b/tools/testing/selftests/bpf/usdt.h @@ -312,6 +312,8 @@ struct usdt_sema { volatile unsigned short active; }; #ifndef USDT_NOP #if defined(__ia64__) || defined(__s390__) || defined(__s390x__) #define USDT_NOP nop 0 +#elif defined(__x86_64__) +#define USDT_NOP .byte 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 /* nop, nop10 */ #else #define USDT_NOP nop #endif diff --git a/tools/testing/selftests/bpf/usdt_1.c b/tools/testing/selftests/bpf/usdt_1.c new file mode 100644 index 000000000000..4f06e8bcf58b --- /dev/null +++ b/tools/testing/selftests/bpf/usdt_1.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 + +#if defined(__x86_64__) + +/* + * Include usdt.h with defined USDT_NOP macro to use single + * nop instruction. + */ +#define USDT_NOP .byte 0x90 +#include "usdt.h" + +__attribute__((aligned(16))) +void usdt_1(void) +{ + USDT(optimized_attach, usdt_1); +} + +#endif diff --git a/tools/testing/selftests/bpf/usdt_2.c b/tools/testing/selftests/bpf/usdt_2.c new file mode 100644 index 000000000000..5e38f8605b02 --- /dev/null +++ b/tools/testing/selftests/bpf/usdt_2.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 + +#if defined(__x86_64__) + +/* + * Include usdt.h with default nop,nop10 instructions combo. + */ +#include "usdt.h" + +__attribute__((aligned(16))) +void usdt_2(void) +{ + USDT(optimized_attach, usdt_2); +} + +static volatile unsigned long usdt_red_zone_arg1 = 0xDEADBEEF; +static volatile unsigned long usdt_red_zone_arg2 = 0xCAFEBABE; +static volatile unsigned long usdt_red_zone_arg3 = 0xFEEDFACE; + +void __attribute__((noinline)) usdt_red_zone_trigger(void) +{ + unsigned long a1 = usdt_red_zone_arg1; + unsigned long a2 = usdt_red_zone_arg2; + unsigned long a3 = usdt_red_zone_arg3; + + USDT(optimized_attach, usdt_red_zone, a1, a2, a3); +} + +#endif diff --git a/tools/testing/selftests/bpf/verifier/calls.c b/tools/testing/selftests/bpf/verifier/calls.c index 9ca83dce100d..eb6e3baef412 100644 --- a/tools/testing/selftests/bpf/verifier/calls.c +++ b/tools/testing/selftests/bpf/verifier/calls.c @@ -31,7 +31,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "arg#0 pointer type STRUCT prog_test_fail1 must point to scalar", + .errstr = "R1 is fp expected STRUCT prog_test_fail1", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail1", 2 }, }, @@ -46,7 +46,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "max struct nesting depth exceeded\narg#0 pointer type STRUCT prog_test_fail2", + .errstr = "max struct nesting depth exceeded\nR1 is fp expected STRUCT prog_test_fail2", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail2", 2 }, }, @@ -61,7 +61,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "arg#0 pointer type STRUCT prog_test_fail3 must point to scalar", + .errstr = "R1 is fp expected STRUCT prog_test_fail3", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail3", 2 }, }, @@ -76,7 +76,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "arg#0 expected pointer to ctx, but got fp", + .errstr = "R1 expected pointer to ctx, but got fp", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_pass_ctx", 2 }, }, @@ -91,7 +91,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "arg#0 pointer type UNKNOWN must point to scalar", + .errstr = "R1 pointer type UNKNOWN must point to scalar", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_mem_len_fail1", 2 }, }, @@ -109,7 +109,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "Possibly NULL pointer passed to trusted arg0", + .errstr = "Possibly NULL pointer passed to trusted R1", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_acquire", 3 }, { "bpf_kfunc_call_test_release", 5 }, @@ -152,7 +152,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "kernel function bpf_kfunc_call_memb1_release args#0 expected pointer", + .errstr = "kernel function bpf_kfunc_call_memb1_release R1 expected pointer", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_memb_acquire", 1 }, { "bpf_kfunc_call_memb1_release", 5 }, @@ -220,7 +220,7 @@ }, .result_unpriv = REJECT, .result = REJECT, - .errstr = "variable trusted_ptr_ access var_off=(0x0; 0x7) disallowed", + .errstr = "R1 must have zero offset when passed to release func or trusted arg to kfunc", }, { "calls: invalid kfunc call: referenced arg needs refcounted PTR_TO_BTF_ID", @@ -455,7 +455,7 @@ BPF_EXIT_INSN(), }, .prog_type = BPF_PROG_TYPE_TRACEPOINT, - .errstr = "the call stack of 9 frames is too deep", + .errstr = "recursive call", .result = REJECT, }, { @@ -812,7 +812,7 @@ BPF_EXIT_INSN(), }, .prog_type = BPF_PROG_TYPE_TRACEPOINT, - .errstr = "the call stack of 9 frames is too deep", + .errstr = "recursive call", .result = REJECT, }, { @@ -824,7 +824,7 @@ BPF_EXIT_INSN(), }, .prog_type = BPF_PROG_TYPE_TRACEPOINT, - .errstr = "the call stack of 9 frames is too deep", + .errstr = "recursive call", .result = REJECT, }, { @@ -1091,7 +1091,17 @@ /* stack_main=32, stack_A=256, stack_B=64 * and max(main+A, main+A+B) < 512 */ - .result = ACCEPT, + .result = VERBOSE_ACCEPT, + .errstr = "stack depth max 352\t" + "subprog 0 (<unknown>) main insns_self \t" + " insns_total \t" + " stack 32\t" + "subprog 1 (<unknown>) static insns_self \t" + " insns_total \t" + " stack 256\t" + "subprog 2 (<unknown>) static insns_self \t" + " insns_total \t" + " stack 64", }, { "calls: stack depth check using three frames. test2", @@ -1219,6 +1229,30 @@ BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call H */ BPF_EXIT_INSN(), /* H */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call I */ + BPF_EXIT_INSN(), + /* I */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call J */ + BPF_EXIT_INSN(), + /* J */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call K */ + BPF_EXIT_INSN(), + /* K */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call L */ + BPF_EXIT_INSN(), + /* L */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call M */ + BPF_EXIT_INSN(), + /* M */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call N */ + BPF_EXIT_INSN(), + /* N */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call O */ + BPF_EXIT_INSN(), + /* O */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call P */ + BPF_EXIT_INSN(), + /* P */ BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }, @@ -1257,6 +1291,30 @@ BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call H */ BPF_EXIT_INSN(), /* H */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call I */ + BPF_EXIT_INSN(), + /* I */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call J */ + BPF_EXIT_INSN(), + /* J */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call K */ + BPF_EXIT_INSN(), + /* K */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call L */ + BPF_EXIT_INSN(), + /* L */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call M */ + BPF_EXIT_INSN(), + /* M */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call N */ + BPF_EXIT_INSN(), + /* N */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call O */ + BPF_EXIT_INSN(), + /* O */ + BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call P */ + BPF_EXIT_INSN(), + /* P */ BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }, @@ -1521,6 +1579,7 @@ .errstr = "R0 invalid mem access 'scalar'", .result_unpriv = REJECT, .errstr_unpriv = "invalid read from stack R7 off=-16 size=8", + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "calls: two calls that receive map_value via arg=ptr_stack_of_caller. test1", @@ -2409,27 +2468,3 @@ .errstr_unpriv = "", .prog_type = BPF_PROG_TYPE_CGROUP_SKB, }, -{ - "calls: several args with ref_obj_id", - .insns = { - /* Reserve at least sizeof(struct iphdr) bytes in the ring buffer. - * With a smaller size, the verifier would reject the call to - * bpf_tcp_raw_gen_syncookie_ipv4 before we can reach the - * ref_obj_id error. - */ - BPF_MOV64_IMM(BPF_REG_2, 20), - BPF_MOV64_IMM(BPF_REG_3, 0), - BPF_LD_MAP_FD(BPF_REG_1, 0), - BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_ringbuf_reserve), - /* if r0 == 0 goto <exit> */ - BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3), - BPF_MOV64_REG(BPF_REG_1, BPF_REG_0), - BPF_MOV64_REG(BPF_REG_2, BPF_REG_0), - BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_tcp_raw_gen_syncookie_ipv4), - BPF_EXIT_INSN(), - }, - .fixup_map_ringbuf = { 2 }, - .result = REJECT, - .errstr = "more than one arg with ref_obj_id", - .prog_type = BPF_PROG_TYPE_SCHED_CLS, -}, diff --git a/tools/testing/selftests/bpf/verifier/junk_insn.c b/tools/testing/selftests/bpf/verifier/junk_insn.c index 89d690f1992a..735d3b9510cf 100644 --- a/tools/testing/selftests/bpf/verifier/junk_insn.c +++ b/tools/testing/selftests/bpf/verifier/junk_insn.c @@ -10,7 +10,7 @@ { "junk insn2", .insns = { - BPF_RAW_INSN(1, 0, 0, 0, 0), + BPF_RAW_INSN(BPF_LDX | BPF_MEM | BPF_W, 0, 0, 0, 1), BPF_EXIT_INSN(), }, .errstr = "BPF_LDX uses reserved fields", @@ -28,7 +28,7 @@ { "junk insn4", .insns = { - BPF_RAW_INSN(-1, -1, -1, -1, -1), + BPF_RAW_INSN(-1, 0, 0, -1, -1), BPF_EXIT_INSN(), }, .errstr = "unknown opcode ff", @@ -37,7 +37,7 @@ { "junk insn5", .insns = { - BPF_RAW_INSN(0x7f, -1, -1, -1, -1), + BPF_RAW_INSN(0x7f, 0, 0, -1, -1), BPF_EXIT_INSN(), }, .errstr = "BPF_ALU uses reserved fields", diff --git a/tools/testing/selftests/bpf/verifier/pseudo_func.c b/tools/testing/selftests/bpf/verifier/pseudo_func.c new file mode 100644 index 000000000000..63c5c67d51de --- /dev/null +++ b/tools/testing/selftests/bpf/verifier/pseudo_func.c @@ -0,0 +1,45 @@ +/* + * Buggy verifier accepted the program below while not patching BPF_PSEUDO_FUNC + * load instruction to contain a real address. Which resulted in a function call + * to a bogus address. + */ +{ + "BPF_PSEUDO_FUNC reference to the main program", + .insns = { + /* r6 = bpf_map_lookup_elem(&timer_map, &(int){0}); */ + BPF_ST_MEM(BPF_W, BPF_REG_10, -4, 0), + BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), + BPF_LD_MAP_FD(BPF_REG_1, 0), + BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem), + BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 10), + BPF_MOV64_REG(BPF_REG_6, BPF_REG_0), + /* bpf_timer_init(r6, &timer_map, 0); */ + BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), + BPF_LD_MAP_FD(BPF_REG_2, 0), + BPF_MOV64_IMM(BPF_REG_3, 0), + BPF_EMIT_CALL(BPF_FUNC_timer_init), + /* bpf_timer_set_callback(r6, <insn #0>); */ + BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), + BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, BPF_REG_2, BPF_PSEUDO_FUNC, 0, -15), + BPF_RAW_INSN(0, 0, 0, 0, 0), + BPF_EMIT_CALL(BPF_FUNC_timer_set_callback), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_TRACEPOINT, + .fixup_map_timer = { 3, 9 }, + .result = REJECT, + .errstr = "callback function cannot be the main program", + .func_info = { { 0, 4 /* main_prog */ } }, + .func_info_cnt = 1, + .btf_strings = "\0int\0ctx\0main_prog", + .btf_types = { + /* 1: int */ BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), + /* 2: void* */ BTF_PTR_ENC(0), + /* 3: int __(void *) */ BTF_FUNC_PROTO_ENC(1, 1), + BTF_FUNC_PROTO_ARG_ENC(5, 2), + /* 4: main_prog */ BTF_FUNC_ENC(9, 3), + BTF_END_RAW + } +}, diff --git a/tools/testing/selftests/bpf/verifier/sleepable.c b/tools/testing/selftests/bpf/verifier/sleepable.c index 1f0d2bdc673f..6dabc5522945 100644 --- a/tools/testing/selftests/bpf/verifier/sleepable.c +++ b/tools/testing/selftests/bpf/verifier/sleepable.c @@ -76,7 +76,20 @@ .runs = -1, }, { - "sleepable raw tracepoint reject", + "sleepable raw tracepoint accept", + .insns = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_TRACING, + .expected_attach_type = BPF_TRACE_RAW_TP, + .kfunc = "sys_enter", + .result = ACCEPT, + .flags = BPF_F_SLEEPABLE, + .runs = -1, +}, +{ + "sleepable raw tracepoint reject non-faultable", .insns = { BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), @@ -85,7 +98,7 @@ .expected_attach_type = BPF_TRACE_RAW_TP, .kfunc = "sched_switch", .result = REJECT, - .errstr = "Only fentry/fexit/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepable", + .errstr = "Sleepable program cannot attach to non-faultable tracepoint", .flags = BPF_F_SLEEPABLE, .runs = -1, }, diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c index 75f85e0362f5..e70741c6b9b7 100644 --- a/tools/testing/selftests/bpf/veristat.c +++ b/tools/testing/selftests/bpf/veristat.c @@ -48,6 +48,7 @@ enum stat_id { SIZE, JITED_SIZE, STACK, + MAX_STACK, PROG_TYPE, ATTACH_TYPE, MEMORY_PEAK, @@ -513,6 +514,40 @@ cleanup: return err == 0; } +/* Exact filter match */ +static bool name_filter_matches(struct filter *f, const char *filename, const char *prog_name) +{ + if (f->any_glob) + return glob_matches(filename, f->any_glob) || + (prog_name && glob_matches(prog_name, f->any_glob)); + if (f->file_glob && f->prog_glob) + return prog_name && + glob_matches(filename, f->file_glob) && + glob_matches(prog_name, f->prog_glob); + if (f->file_glob) + return glob_matches(filename, f->file_glob); + if (f->prog_glob) + return prog_name && glob_matches(prog_name, f->prog_glob); + return false; +} + +/* Check if the filter does not outright reject the file name */ +static bool name_filter_may_match(struct filter *f, const char *filename) +{ + if (f->file_glob) + return glob_matches(filename, f->file_glob); + /* + * If we don't know program name yet, any_glob filter + * has to assume that current BPF object file might be + * relevant; we'll check again later on after opening + * BPF object file, at which point program name will + * be known finally. + */ + if (f->any_glob || f->prog_glob) + return true; + return false; +} + static bool should_process_file_prog(const char *filename, const char *prog_name) { struct filter *f; @@ -520,16 +555,7 @@ static bool should_process_file_prog(const char *filename, const char *prog_name for (i = 0; i < env.deny_filter_cnt; i++) { f = &env.deny_filters[i]; - if (f->kind != FILTER_NAME) - continue; - - if (f->any_glob && glob_matches(filename, f->any_glob)) - return false; - if (f->any_glob && prog_name && glob_matches(prog_name, f->any_glob)) - return false; - if (f->file_glob && glob_matches(filename, f->file_glob)) - return false; - if (f->prog_glob && prog_name && glob_matches(prog_name, f->prog_glob)) + if (f->kind == FILTER_NAME && name_filter_matches(f, filename, prog_name)) return false; } @@ -539,24 +565,15 @@ static bool should_process_file_prog(const char *filename, const char *prog_name continue; allow_cnt++; - if (f->any_glob) { - if (glob_matches(filename, f->any_glob)) - return true; - /* If we don't know program name yet, any_glob filter - * has to assume that current BPF object file might be - * relevant; we'll check again later on after opening - * BPF object file, at which point program name will - * be known finally. - */ - if (!prog_name || glob_matches(prog_name, f->any_glob)) - return true; - } else { - if (f->file_glob && !glob_matches(filename, f->file_glob)) - continue; - if (f->prog_glob && prog_name && !glob_matches(prog_name, f->prog_glob)) - continue; + if (prog_name && name_filter_matches(f, filename, prog_name)) + return true; + /* + * If there is no prog_name and the file name is not blocked by + * the filter, allow to open the file. Afterwards there would be + * a second refining query with prog_name set. + */ + if (!prog_name && name_filter_may_match(f, filename)) return true; - } } /* if there are no file/prog name allow filters, allow all progs, @@ -702,6 +719,12 @@ static int append_filter(struct filter **filters, int *cnt, const char *str) } } + if ((!f->any_glob && !f->file_glob && !f->prog_glob) || + (f->any_glob && strcmp(f->any_glob, "") == 0)) { + fprintf(stderr, "Invalid filter: '%s'\n", str); + return -EINVAL; + } + *cnt += 1; return 0; } @@ -789,13 +812,13 @@ cleanup: } static const struct stat_specs default_csv_output_spec = { - .spec_cnt = 15, + .spec_cnt = 16, .ids = { FILE_NAME, PROG_NAME, VERDICT, DURATION, TOTAL_INSNS, TOTAL_STATES, PEAK_STATES, MAX_STATES_PER_INSN, MARK_READ_MAX_LEN, SIZE, JITED_SIZE, PROG_TYPE, ATTACH_TYPE, - STACK, MEMORY_PEAK, + STACK, MAX_STACK, MEMORY_PEAK, }, }; @@ -834,6 +857,7 @@ static struct stat_def { [SIZE] = { "Program size", {"prog_size"}, }, [JITED_SIZE] = { "Jited size", {"prog_size_jited"}, }, [STACK] = {"Stack depth", {"stack_depth", "stack"}, }, + [MAX_STACK] = {"Max stack depth", {"max_stack_depth"}, }, [PROG_TYPE] = { "Program type", {"prog_type"}, }, [ATTACH_TYPE] = { "Attach type", {"attach_type", }, }, [MEMORY_PEAK] = { "Peak memory (MiB)", {"mem_peak", }, }, @@ -991,13 +1015,15 @@ static void free_verif_stats(struct verif_stats *stats, size_t stat_cnt) static char verif_log_buf[64 * 1024]; -#define MAX_PARSED_LOG_LINES 100 +/* Keep room for all 256 subprogram records and trailing statistics. */ +#define MAX_PARSED_LOG_LINES 300 static int parse_verif_log(char * const buf, size_t buf_sz, struct verif_stats *s) { const char *cur; - int pos, lines, sub_stack, cnt = 0; - char *state = NULL, *token, stack[512]; + long sub_stack; + int pos, lines, cnt = 0; + char *state = NULL, *token, stack[512] = {}; buf[buf_sz - 1] = '\0'; @@ -1023,11 +1049,24 @@ static int parse_verif_log(char * const buf, size_t buf_sz, struct verif_stats * &s->stats[MARK_READ_MAX_LEN])) continue; - if (1 == sscanf(cur, "stack depth %511s", stack)) + /* + * New kernels emit one "subprog <id> (<name>) <kind>" record + * per subprogram with the stack depth at the end, while old + * kernels emit a single "stack depth <a+...+n> max <max>" + * line. Match both formats so veristat works against either + * kernel. + */ + if (sscanf(cur, "stack depth max %ld", &s->stats[MAX_STACK]) == 1) + continue; + if (sscanf(cur, "subprog %*d %*s %*s insns_self %*d insns_total %*d stack %ld", &sub_stack) == 1) { + s->stats[STACK] += sub_stack; + continue; + } + if (2 == sscanf(cur, "stack depth %511s max %ld", stack, &s->stats[MAX_STACK])) continue; } while ((token = strtok_r(cnt++ ? NULL : stack, "+", &state))) { - if (sscanf(token, "%d", &sub_stack) == 0) + if (sscanf(token, "%ld", &sub_stack) == 0) break; s->stats[STACK] += sub_stack; } @@ -1236,7 +1275,7 @@ static void mask_unrelated_struct_ops_progs(struct bpf_object *obj, } } -static void fixup_obj(struct bpf_object *obj, struct bpf_program *prog, const char *filename) +static void fixup_obj_maps(struct bpf_object *obj) { struct bpf_map *map; @@ -1246,20 +1285,51 @@ static void fixup_obj(struct bpf_object *obj, struct bpf_program *prog, const ch /* fix up map size, if necessary */ switch (bpf_map__type(map)) { + /* + * if the verifier doesn't use max_entries + * then set to 1 to avoid -ENOMEM + */ + case BPF_MAP_TYPE_HASH: + case BPF_MAP_TYPE_PERCPU_HASH: + case BPF_MAP_TYPE_LRU_HASH: + case BPF_MAP_TYPE_LRU_PERCPU_HASH: + case BPF_MAP_TYPE_SOCKHASH: + case BPF_MAP_TYPE_DEVMAP_HASH: + case BPF_MAP_TYPE_QUEUE: + case BPF_MAP_TYPE_STACK: + case BPF_MAP_TYPE_BLOOM_FILTER: + case BPF_MAP_TYPE_STACK_TRACE: + bpf_map__set_max_entries(map, 1); + break; + + /* ringbufs must be page-aligned */ + case BPF_MAP_TYPE_RINGBUF: + case BPF_MAP_TYPE_USER_RINGBUF: + bpf_map__set_max_entries(map, sysconf(_SC_PAGESIZE)); + break; + case BPF_MAP_TYPE_SK_STORAGE: case BPF_MAP_TYPE_TASK_STORAGE: case BPF_MAP_TYPE_INODE_STORAGE: case BPF_MAP_TYPE_CGROUP_STORAGE: case BPF_MAP_TYPE_CGRP_STORAGE: - break; case BPF_MAP_TYPE_STRUCT_OPS: - mask_unrelated_struct_ops_progs(obj, map, prog); break; default: if (bpf_map__max_entries(map) == 0) bpf_map__set_max_entries(map, 1); } } +} + +static void fixup_obj(struct bpf_object *obj, struct bpf_program *prog, const char *filename) +{ + struct bpf_map *map; + + bpf_object__for_each_map(map, obj) { + if (bpf_map__type(map) == BPF_MAP_TYPE_STRUCT_OPS) + mask_unrelated_struct_ops_progs(obj, map, prog); + } /* SEC(freplace) programs can't be loaded with veristat as is, * but we can try guessing their target program's expected type by @@ -1608,6 +1678,7 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf const char *base_filename = basename(strdupa(filename)); const char *prog_name = bpf_program__name(prog); long mem_peak_a, mem_peak_b, mem_peak = -1; + LIBBPF_OPTS(bpf_prog_load_opts, opts); char *buf; int buf_sz, log_level; struct verif_stats *stats; @@ -1647,9 +1718,6 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf } verif_log_buf[0] = '\0'; - bpf_program__set_log_buf(prog, buf, buf_sz); - bpf_program__set_log_level(prog, log_level); - /* increase chances of successful BPF object loading */ fixup_obj(obj, prog, base_filename); @@ -1658,15 +1726,22 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf if (env.force_reg_invariants) bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_REG_INVARIANTS); - err = bpf_object__prepare(obj); - if (!err) { - cgroup_err = reset_stat_cgroup(); - mem_peak_a = cgroup_memory_peak(); - err = bpf_object__load(obj); - mem_peak_b = cgroup_memory_peak(); - if (!cgroup_err && mem_peak_a >= 0 && mem_peak_b >= 0) - mem_peak = mem_peak_b - mem_peak_a; + opts.log_buf = buf; + opts.log_size = buf_sz; + opts.log_level = log_level; + + cgroup_err = reset_stat_cgroup(); + mem_peak_a = cgroup_memory_peak(); + fd = bpf_program__clone(prog, &opts); + if (fd < 0) { + err = fd; + if (env.verbose) + fprintf(stderr, "Failed to load program %s %d\n", prog_name, err); } + mem_peak_b = cgroup_memory_peak(); + if (!cgroup_err && mem_peak_a >= 0 && mem_peak_b >= 0) + mem_peak = mem_peak_b - mem_peak_a; + env.progs_processed++; stats->file_name = strdup(base_filename); @@ -1678,7 +1753,6 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf stats->stats[MEMORY_PEAK] = mem_peak < 0 ? -1 : mem_peak / (1024 * 1024); memset(&info, 0, info_len); - fd = bpf_program__fd(prog); if (fd > 0 && bpf_prog_get_info_by_fd(fd, &info, &info_len) == 0) { stats->stats[JITED_SIZE] = info.jited_prog_len; if (env.dump_mode & DUMP_JITED) @@ -1699,7 +1773,8 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf if (verif_log_buf != buf) free(buf); - + if (fd > 0) + close(fd); return 0; } @@ -2182,8 +2257,8 @@ static int set_global_vars(struct bpf_object *obj, struct var_preset *presets, i static int process_obj(const char *filename) { const char *base_filename = basename(strdupa(filename)); - struct bpf_object *obj = NULL, *tobj; - struct bpf_program *prog, *tprog, *lprog; + struct bpf_object *obj = NULL; + struct bpf_program *prog; libbpf_print_fn_t old_libbpf_print_fn; LIBBPF_OPTS(bpf_object_open_opts, opts); int err = 0, prog_cnt = 0; @@ -2222,51 +2297,24 @@ static int process_obj(const char *filename) env.files_processed++; bpf_object__for_each_program(prog, obj) { + bpf_program__set_autoload(prog, true); prog_cnt++; } - if (prog_cnt == 1) { - prog = bpf_object__next_program(obj, NULL); - bpf_program__set_autoload(prog, true); - err = set_global_vars(obj, env.presets, env.npresets); - if (err) { - fprintf(stderr, "Failed to set global variables %d\n", err); - goto cleanup; - } - process_prog(filename, obj, prog); + fixup_obj_maps(obj); + + err = set_global_vars(obj, env.presets, env.npresets); + if (err) { + fprintf(stderr, "Failed to set global variables %d\n", err); goto cleanup; } - bpf_object__for_each_program(prog, obj) { - const char *prog_name = bpf_program__name(prog); - - tobj = bpf_object__open_file(filename, &opts); - if (!tobj) { - err = -errno; - fprintf(stderr, "Failed to open '%s': %d\n", filename, err); - goto cleanup; - } - - err = set_global_vars(tobj, env.presets, env.npresets); - if (err) { - fprintf(stderr, "Failed to set global variables %d\n", err); - goto cleanup; - } - - lprog = NULL; - bpf_object__for_each_program(tprog, tobj) { - const char *tprog_name = bpf_program__name(tprog); - - if (strcmp(prog_name, tprog_name) == 0) { - bpf_program__set_autoload(tprog, true); - lprog = tprog; - } else { - bpf_program__set_autoload(tprog, false); - } - } + err = bpf_object__prepare(obj); + if (err && env.verbose) /* run process_prog() anyway to output per program failures */ + fprintf(stderr, "Failed to prepare BPF object for loading %d\n", err); - process_prog(filename, tobj, lprog); - bpf_object__close(tobj); + bpf_object__for_each_program(prog, obj) { + process_prog(filename, obj, prog); } cleanup: @@ -2292,6 +2340,7 @@ static int cmp_stat(const struct verif_stats *s1, const struct verif_stats *s2, case SIZE: case JITED_SIZE: case STACK: + case MAX_STACK: case VERDICT: case DURATION: case TOTAL_INSNS: @@ -2526,6 +2575,7 @@ static void prepare_value(const struct verif_stats *s, enum stat_id id, case MAX_STATES_PER_INSN: case MARK_READ_MAX_LEN: case STACK: + case MAX_STACK: case SIZE: case JITED_SIZE: case MEMORY_PEAK: @@ -2616,7 +2666,8 @@ static int parse_stat_value(const char *str, enum stat_id id, struct verif_stats case SIZE: case JITED_SIZE: case MEMORY_PEAK: - case STACK: { + case STACK: + case MAX_STACK: { long val; int err, n; @@ -3264,17 +3315,14 @@ static int handle_verif_mode(void) create_stat_cgroup(); for (i = 0; i < env.filename_cnt; i++) { err = process_obj(env.filenames[i]); - if (err) { + if (err) fprintf(stderr, "Failed to process '%s': %d\n", env.filenames[i], err); - goto out; - } } qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats); output_prog_stats(); -out: destroy_stat_cgroup(); return err; } diff --git a/tools/testing/selftests/bpf/vmtest.sh b/tools/testing/selftests/bpf/vmtest.sh index 2f869daf8a06..6a3d026d76bd 100755 --- a/tools/testing/selftests/bpf/vmtest.sh +++ b/tools/testing/selftests/bpf/vmtest.sh @@ -382,7 +382,7 @@ main() local exit_command="poweroff -f" local debug_shell="no" - while getopts ':hskl:id:j:' opt; do + while getopts ':hsl:id:j:' opt; do case ${opt} in l) LOCAL_ROOTFS_IMAGE="$OPTARG" @@ -428,8 +428,17 @@ main() if [[ $# -eq 0 && "${debug_shell}" == "no" ]]; then echo "No command specified, will run ${DEFAULT_COMMAND} in the vm" - else - command="$@" + elif [[ $# -gt 0 ]]; then + # Quote each argument so the command survives into the guest: the + # host expands ${command} into the generated init script, which + # the guest bash then parses as shell source. Without the %q + # escapes an argument with a space or a glob (e.g. -a 'verifier_*') + # is re-split and expanded against /root/bpf there. + # + # Skip this when there is no command: printf '%q ' would still + # apply the format once and emit '', which is not the empty + # command that -s (debug shell) expects. + command=$(printf '%q ' "$@") fi local kconfig_file="${OUTPUT_DIR}/latest.config" diff --git a/tools/testing/selftests/bpf/xdp_lb_bench_common.h b/tools/testing/selftests/bpf/xdp_lb_bench_common.h new file mode 100644 index 000000000000..aed20a963701 --- /dev/null +++ b/tools/testing/selftests/bpf/xdp_lb_bench_common.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#ifndef XDP_LB_BENCH_COMMON_H +#define XDP_LB_BENCH_COMMON_H + +#define F_IPV6 (1 << 0) +#define F_LRU_BYPASS (1 << 1) + +#define CH_RING_SIZE 65537 /* per-VIP consistent hash ring slots */ +#define MAX_VIPS 16 +#define CH_RINGS_SIZE (MAX_VIPS * CH_RING_SIZE) +#define MAX_REALS 512 +#define DEFAULT_LRU_SIZE 100000 /* connection tracking cache size */ +#define ONE_SEC 1000000000U /* 1 sec in nanosec */ +#define MAX_CONN_RATE 100000000 /* high enough to never trigger in bench */ +#define LRU_UDP_TIMEOUT 30000000000ULL /* 30 sec in nanosec */ +#define PCKT_FRAGMENTED 0x3FFF +#define KNUTH_HASH_MULT 2654435761U +#define IPIP_V4_PREFIX 4268 /* 172.16/12 in network order */ +#define IPIP_V6_PREFIX1 1 /* 0100::/64 (RFC 6666 discard) */ +#define IPIP_V6_PREFIX2 0 +#define IPIP_V6_PREFIX3 0 + +/* Stats indices (0..MAX_VIPS-1 are per-VIP packet/byte counters) */ +#define STATS_LRU (MAX_VIPS + 0) /* v1: total VIP packets, v2: LRU misses */ +#define STATS_XDP_TX (MAX_VIPS + 1) +#define STATS_XDP_PASS (MAX_VIPS + 2) +#define STATS_XDP_DROP (MAX_VIPS + 3) +#define STATS_NEW_CONN (MAX_VIPS + 4) /* v1: conn count, v2: last reset ts */ +#define STATS_LRU_MISS (MAX_VIPS + 5) /* v1: TCP LRU misses */ +#define STATS_SIZE (MAX_VIPS + 6) + +#ifdef __BPF__ +#define lb_htons(x) bpf_htons(x) +#define LB_INLINE static __always_inline +#else +#define lb_htons(x) htons(x) +#define LB_INLINE static inline +#endif + +LB_INLINE __be32 create_encap_ipv4_src(__u16 port, __be32 src) +{ + __u32 ip_suffix = lb_htons(port); + + ip_suffix <<= 16; + ip_suffix ^= src; + return (0xFFFF0000 & ip_suffix) | IPIP_V4_PREFIX; +} + +LB_INLINE void create_encap_ipv6_src(__u16 port, __be32 src, __be32 *saddr) +{ + saddr[0] = IPIP_V6_PREFIX1; + saddr[1] = IPIP_V6_PREFIX2; + saddr[2] = IPIP_V6_PREFIX3; + saddr[3] = src ^ port; +} + +struct flow_key { + union { + __be32 src; + __be32 srcv6[4]; + }; + union { + __be32 dst; + __be32 dstv6[4]; + }; + union { + __u32 ports; + __u16 port16[2]; + }; + __u8 proto; + __u8 pad[3]; +}; + +struct vip_definition { + union { + __be32 vip; + __be32 vipv6[4]; + }; + __u16 port; + __u8 proto; + __u8 pad; +}; + +struct vip_meta { + __u32 flags; + __u32 vip_num; +}; + +struct real_pos_lru { + __u32 pos; + __u64 atime; +}; + +struct real_definition { + __be32 dst; + __be32 dstv6[4]; + __u8 flags; +}; + +struct lb_stats { + __u64 v1; + __u64 v2; +}; + +struct ctl_value { + __u8 mac[6]; + __u8 pad[2]; +}; + +#endif /* XDP_LB_BENCH_COMMON_H */ diff --git a/tools/testing/selftests/bpf/xdping.c b/tools/testing/selftests/bpf/xdping.c deleted file mode 100644 index 9ed8c796645d..000000000000 --- a/tools/testing/selftests/bpf/xdping.c +++ /dev/null @@ -1,254 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. */ - -#include <linux/bpf.h> -#include <linux/if_link.h> -#include <arpa/inet.h> -#include <assert.h> -#include <errno.h> -#include <signal.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> -#include <libgen.h> -#include <net/if.h> -#include <sys/types.h> -#include <sys/socket.h> -#include <netdb.h> - -#include "bpf/bpf.h" -#include "bpf/libbpf.h" - -#include "xdping.h" -#include "testing_helpers.h" - -static int ifindex; -static __u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; - -static void cleanup(int sig) -{ - bpf_xdp_detach(ifindex, xdp_flags, NULL); - if (sig) - exit(1); -} - -static int get_stats(int fd, __u16 count, __u32 raddr) -{ - struct pinginfo pinginfo = { 0 }; - char inaddrbuf[INET_ADDRSTRLEN]; - struct in_addr inaddr; - __u16 i; - - inaddr.s_addr = raddr; - - printf("\nXDP RTT data:\n"); - - if (bpf_map_lookup_elem(fd, &raddr, &pinginfo)) { - perror("bpf_map_lookup elem"); - return 1; - } - - for (i = 0; i < count; i++) { - if (pinginfo.times[i] == 0) - break; - - printf("64 bytes from %s: icmp_seq=%d ttl=64 time=%#.5f ms\n", - inet_ntop(AF_INET, &inaddr, inaddrbuf, - sizeof(inaddrbuf)), - count + i + 1, - (double)pinginfo.times[i]/1000000); - } - - if (i < count) { - fprintf(stderr, "Expected %d samples, got %d.\n", count, i); - return 1; - } - - bpf_map_delete_elem(fd, &raddr); - - return 0; -} - -static void show_usage(const char *prog) -{ - fprintf(stderr, - "usage: %s [OPTS] -I interface destination\n\n" - "OPTS:\n" - " -c count Stop after sending count requests\n" - " (default %d, max %d)\n" - " -I interface interface name\n" - " -N Run in driver mode\n" - " -s Server mode\n" - " -S Run in skb mode\n", - prog, XDPING_DEFAULT_COUNT, XDPING_MAX_COUNT); -} - -int main(int argc, char **argv) -{ - __u32 mode_flags = XDP_FLAGS_DRV_MODE | XDP_FLAGS_SKB_MODE; - struct addrinfo *a, hints = { .ai_family = AF_INET }; - __u16 count = XDPING_DEFAULT_COUNT; - struct pinginfo pinginfo = { 0 }; - const char *optstr = "c:I:NsS"; - struct bpf_program *main_prog; - int prog_fd = -1, map_fd = -1; - struct sockaddr_in rin; - struct bpf_object *obj; - struct bpf_map *map; - char *ifname = NULL; - char filename[256]; - int opt, ret = 1; - __u32 raddr = 0; - int server = 0; - char cmd[256]; - - while ((opt = getopt(argc, argv, optstr)) != -1) { - switch (opt) { - case 'c': - count = atoi(optarg); - if (count < 1 || count > XDPING_MAX_COUNT) { - fprintf(stderr, - "min count is 1, max count is %d\n", - XDPING_MAX_COUNT); - return 1; - } - break; - case 'I': - ifname = optarg; - ifindex = if_nametoindex(ifname); - if (!ifindex) { - fprintf(stderr, "Could not get interface %s\n", - ifname); - return 1; - } - break; - case 'N': - xdp_flags |= XDP_FLAGS_DRV_MODE; - break; - case 's': - /* use server program */ - server = 1; - break; - case 'S': - xdp_flags |= XDP_FLAGS_SKB_MODE; - break; - default: - show_usage(basename(argv[0])); - return 1; - } - } - - if (!ifname) { - show_usage(basename(argv[0])); - return 1; - } - if (!server && optind == argc) { - show_usage(basename(argv[0])); - return 1; - } - - if ((xdp_flags & mode_flags) == mode_flags) { - fprintf(stderr, "-N or -S can be specified, not both.\n"); - show_usage(basename(argv[0])); - return 1; - } - - if (!server) { - /* Only supports IPv4; see hints initialization above. */ - if (getaddrinfo(argv[optind], NULL, &hints, &a) || !a) { - fprintf(stderr, "Could not resolve %s\n", argv[optind]); - return 1; - } - memcpy(&rin, a->ai_addr, sizeof(rin)); - raddr = rin.sin_addr.s_addr; - freeaddrinfo(a); - } - - /* Use libbpf 1.0 API mode */ - libbpf_set_strict_mode(LIBBPF_STRICT_ALL); - - snprintf(filename, sizeof(filename), "%s_kern.bpf.o", argv[0]); - - if (bpf_prog_test_load(filename, BPF_PROG_TYPE_XDP, &obj, &prog_fd)) { - fprintf(stderr, "load of %s failed\n", filename); - return 1; - } - - main_prog = bpf_object__find_program_by_name(obj, - server ? "xdping_server" : "xdping_client"); - if (main_prog) - prog_fd = bpf_program__fd(main_prog); - if (!main_prog || prog_fd < 0) { - fprintf(stderr, "could not find xdping program"); - return 1; - } - - map = bpf_object__next_map(obj, NULL); - if (map) - map_fd = bpf_map__fd(map); - if (!map || map_fd < 0) { - fprintf(stderr, "Could not find ping map"); - goto done; - } - - signal(SIGINT, cleanup); - signal(SIGTERM, cleanup); - - printf("Setting up XDP for %s, please wait...\n", ifname); - - printf("XDP setup disrupts network connectivity, hit Ctrl+C to quit\n"); - - if (bpf_xdp_attach(ifindex, prog_fd, xdp_flags, NULL) < 0) { - fprintf(stderr, "Link set xdp fd failed for %s\n", ifname); - goto done; - } - - if (server) { - close(prog_fd); - close(map_fd); - printf("Running server on %s; press Ctrl+C to exit...\n", - ifname); - do { } while (1); - } - - /* Start xdping-ing from last regular ping reply, e.g. for a count - * of 10 ICMP requests, we start xdping-ing using reply with seq number - * 10. The reason the last "real" ping RTT is much higher is that - * the ping program sees the ICMP reply associated with the last - * XDP-generated packet, so ping doesn't get a reply until XDP is done. - */ - pinginfo.seq = htons(count); - pinginfo.count = count; - - if (bpf_map_update_elem(map_fd, &raddr, &pinginfo, BPF_ANY)) { - fprintf(stderr, "could not communicate with BPF map: %s\n", - strerror(errno)); - cleanup(0); - goto done; - } - - /* We need to wait for XDP setup to complete. */ - sleep(10); - - snprintf(cmd, sizeof(cmd), "ping -c %d -I %s %s", - count, ifname, argv[optind]); - - printf("\nNormal ping RTT data\n"); - printf("[Ignore final RTT; it is distorted by XDP using the reply]\n"); - - ret = system(cmd); - - if (!ret) - ret = get_stats(map_fd, count, raddr); - - cleanup(0); - -done: - if (prog_fd > 0) - close(prog_fd); - if (map_fd > 0) - close(map_fd); - - return ret; -} diff --git a/tools/testing/selftests/bpf/xdping.h b/tools/testing/selftests/bpf/xdping.h deleted file mode 100644 index afc578df77be..000000000000 --- a/tools/testing/selftests/bpf/xdping.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. */ - -#define XDPING_MAX_COUNT 10 -#define XDPING_DEFAULT_COUNT 4 - -struct pinginfo { - __u64 start; - __be16 seq; - __u16 count; - __u32 pad; - __u64 times[XDPING_MAX_COUNT]; -}; |