diff options
Diffstat (limited to 'scripts')
228 files changed, 9537 insertions, 1119 deletions
diff --git a/scripts/Makefile b/scripts/Makefile index 0941e5ce7b57..3434a82a119f 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -35,6 +35,7 @@ HOSTCFLAGS_sorttable.o = -I$(srctree)/tools/include HOSTLDLIBS_sorttable = -lpthread HOSTCFLAGS_asn1_compiler.o = -I$(srctree)/include HOSTCFLAGS_sign-file.o = $(shell $(HOSTPKG_CONFIG) --cflags libcrypto 2> /dev/null) +HOSTCFLAGS_sign-file.o += -I$(srctree)/tools/include/uapi/ HOSTLDLIBS_sign-file = $(shell $(HOSTPKG_CONFIG) --libs libcrypto 2> /dev/null || echo -lcrypto) ifdef CONFIG_UNWINDER_ORC diff --git a/scripts/Makefile.asm-headers b/scripts/Makefile.asm-headers index 8a4856e74180..f1c3d287c14b 100644 --- a/scripts/Makefile.asm-headers +++ b/scripts/Makefile.asm-headers @@ -48,14 +48,13 @@ syscall-y := $(addprefix $(obj)/, $(syscall-y)) generated-y := $(addprefix $(obj)/, $(generated-y)) # Remove stale wrappers when the corresponding files are removed from generic-y -old-headers := $(wildcard $(obj)/*.h) +old-headers := $(shell test -d $(obj) && find $(obj) -name '*.h' ! -name '.*') unwanted := $(filter-out $(generic-y) $(generated-y) $(syscall-y),$(old-headers)) -quiet_cmd_wrap = WRAP $@ - cmd_wrap = echo "\#include <asm-generic/$*.h>" > $@ +filechk_wrap = echo "\#include <asm-generic/$*.h>" quiet_cmd_remove = REMOVE $(unwanted) - cmd_remove = rm -f $(unwanted) + cmd_remove = rm -f $(unwanted); find $(obj) -type d -empty -delete quiet_cmd_syshdr = SYSHDR $@ cmd_syshdr = $(CONFIG_SHELL) $(syshdr) \ @@ -74,8 +73,8 @@ all: $(generic-y) $(syscall-y) $(if $(unwanted),$(call cmd,remove)) @: -$(obj)/%.h: $(srctree)/$(generic)/%.h - $(call cmd,wrap) +$(obj)/%.h: $(srctree)/$(generic)/%.h FORCE + $(call filechk,wrap) $(obj)/unistd_%.h: $(syscalltbl) $(syshdr) FORCE $(call if_changed,syshdr) diff --git a/scripts/Makefile.autofdo b/scripts/Makefile.autofdo index 1caf2457e585..1442043da139 100644 --- a/scripts/Makefile.autofdo +++ b/scripts/Makefile.autofdo @@ -3,14 +3,18 @@ # Enable available and selected Clang AutoFDO features. CFLAGS_AUTOFDO_CLANG := -fdebug-info-for-profiling -mllvm -enable-fs-discriminator=true -mllvm -improved-fs-discriminator=true +RUSTFLAGS_AUTOFDO_CLANG := $(if $(call rustc-min-version,109800),-Zdebuginfo-for-profiling,-Zdebug-info-for-profiling) -Cllvm-args=-enable-fs-discriminator=true -Cllvm-args=-improved-fs-discriminator=true ifndef CONFIG_DEBUG_INFO CFLAGS_AUTOFDO_CLANG += -gmlt + RUSTFLAGS_AUTOFDO_CLANG += -Cdebuginfo=line-tables-only endif ifdef CLANG_AUTOFDO_PROFILE CFLAGS_AUTOFDO_CLANG += -fprofile-sample-use=$(CLANG_AUTOFDO_PROFILE) -ffunction-sections CFLAGS_AUTOFDO_CLANG += -fsplit-machine-functions + RUSTFLAGS_AUTOFDO_CLANG += -Zprofile-sample-use=$(CLANG_AUTOFDO_PROFILE) -Zfunction-sections=y + RUSTFLAGS_AUTOFDO_CLANG += -Cllvm-args=-split-machine-functions endif ifdef CONFIG_LTO_CLANG_THIN @@ -21,4 +25,4 @@ ifdef CONFIG_LTO_CLANG_THIN KBUILD_LDFLAGS += -plugin-opt=-split-machine-functions endif -export CFLAGS_AUTOFDO_CLANG +export CFLAGS_AUTOFDO_CLANG RUSTFLAGS_AUTOFDO_CLANG diff --git a/scripts/Makefile.btf b/scripts/Makefile.btf index 562a04b40e06..a1812985a61a 100644 --- a/scripts/Makefile.btf +++ b/scripts/Makefile.btf @@ -14,9 +14,9 @@ pahole-flags-$(call test-ge, $(pahole-ver), 125) += --skip_encoding_btf_inconsis else # Switch to using --btf_features for v1.26 and later. -pahole-flags-$(call test-ge, $(pahole-ver), 126) = -j$(JOBS) --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func,decl_tag_kfuncs +pahole-flags-$(call test-ge, $(pahole-ver), 126) = -j$(JOBS) --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func -pahole-flags-$(call test-ge, $(pahole-ver), 130) += --btf_features=attributes +pahole-flags-$(call test-ge, $(pahole-ver), 131) += --btf_features=layout endif diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 3652b85be545..4349108e75e1 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -28,6 +28,7 @@ ldflags-y := subdir-asflags-y := subdir-ccflags-y := +subdir-rustflags-y := # Read auto.conf if it exists, otherwise ignore -include $(objtree)/include/config/auto.conf @@ -159,10 +160,10 @@ targets += $(targets-for-builtin) $(targets-for-modules) # Linus' kernel sanity checking tool ifeq ($(KBUILD_CHECKSRC),1) - quiet_cmd_checksrc = CHECK $< + quiet_cmd_checksrc = CHECK $(patsubst $(srctree)/%,%,$<) cmd_checksrc = $(CHECK) $(CHECKFLAGS) $(c_flags) $< else ifeq ($(KBUILD_CHECKSRC),2) - quiet_cmd_force_checksrc = CHECK $< + quiet_cmd_force_checksrc = CHECK $(patsubst $(srctree)/%,%,$<) cmd_force_checksrc = $(CHECK) $(CHECKFLAGS) $(c_flags) $< endif @@ -310,18 +311,14 @@ $(obj)/%.lst: $(obj)/%.c FORCE # The features in this list are the ones allowed for non-`rust/` code. # -# - Stable since Rust 1.79.0: `feature(slice_ptr_len)`. -# - Stable since Rust 1.81.0: `feature(lint_reasons)`. -# - Stable since Rust 1.82.0: `feature(asm_const)`, -# `feature(offset_of_nested)`, `feature(raw_ref_op)`. -# - Stable since Rust 1.84.0: `feature(strict_provenance)`. # - Stable since Rust 1.87.0: `feature(asm_goto)`. +# - Stable since Rust 1.89.0: `feature(generic_arg_infer)`. # - Expected to become stable: `feature(arbitrary_self_types)`. # - To be determined: `feature(used_with_arg)`. # # Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on # the unstable features in use. -rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons,offset_of_nested,raw_ref_op,slice_ptr_len,strict_provenance,used_with_arg +rust_allowed_features := arbitrary_self_types,asm_goto,generic_arg_infer,used_with_arg # `--out-dir` is required to avoid temporaries being created by `rustc` in the # current working directory, which may be not accessible in the out-of-tree @@ -333,8 +330,8 @@ rust_common_cmd = \ -Zcrate-attr=no_std \ -Zcrate-attr='feature($(rust_allowed_features))' \ -Zunstable-options --extern pin_init --extern kernel \ + --extern zerocopy --extern zerocopy_derive \ --crate-type rlib -L $(objtree)/rust/ \ - --crate-name $(basename $(notdir $@)) \ --sysroot=/dev/null \ --out-dir $(dir $@) --emit=dep-info=$(depfile) @@ -347,7 +344,13 @@ rust_common_cmd = \ # would not match each other. quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ - cmd_rustc_o_rs = $(rust_common_cmd) --emit=obj=$@ $< $(cmd_objtool) + cmd_rustc_o_rs = $(rust_common_cmd) --emit=$(if $(CONFIG_RUST_INLINE_HELPERS),llvm-bc=$(patsubst %.o,%.bc,$@),obj=$@) $< \ + $(if $(CONFIG_RUST_INLINE_HELPERS),;$(LLVM_LINK) --internalize --suppress-warnings $(patsubst %.o,%.bc,$@) \ + $(objtree)/rust/helpers/helpers$(if $(part-of-module),_module).bc -o $(patsubst %.o,%.m.bc,$@); \ + $(CC) $(CLANG_FLAGS) $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS)) \ + $(CC_FLAGS_RUST_INLINE_HELPERS) -Wno-override-module -c $(patsubst %.o,%.m.bc,$@) -o $@ \ + $(cmd_ld_single)) \ + $(cmd_objtool) define rule_rustc_o_rs $(call cmd_and_fixdep,rustc_o_rs) diff --git a/scripts/Makefile.compiler b/scripts/Makefile.compiler index ef91910de265..06bbe29c846c 100644 --- a/scripts/Makefile.compiler +++ b/scripts/Makefile.compiler @@ -80,7 +80,7 @@ ld-option = $(call try-run, $(LD) $(KBUILD_LDFLAGS) $(1) -v,$(1),$(2),$(3)) # TODO: remove RUSTC_BOOTSTRAP=1 when we raise the minimum GNU Make version to 4.4 __rustc-option = $(call try-run,\ echo '$(pound)![allow(missing_docs)]$(pound)![feature(no_core)]$(pound)![no_core]' | RUSTC_BOOTSTRAP=1\ - $(1) --sysroot=/dev/null $(filter-out --sysroot=/dev/null --target=%,$(2)) $(3)\ + $(1) --sysroot=/dev/null $(KBUILD_RUSTFLAGS_OPTION_CHKS) $(filter-out --sysroot=/dev/null --target=%target.json,$(2)) $(3)\ --crate-type=rlib --out-dir=$(TMPOUT) --emit=obj=- - >/dev/null,$(3),$(4)) # rustc-option diff --git a/scripts/Makefile.kasan b/scripts/Makefile.kasan index 0ba2aac3b8dc..91504e81247a 100644 --- a/scripts/Makefile.kasan +++ b/scripts/Makefile.kasan @@ -71,8 +71,6 @@ ifdef CONFIG_KASAN_SW_TAGS CFLAGS_KASAN := -fsanitize=kernel-hwaddress -# This sets flags that will enable SW_TAGS KASAN once enabled in Rust. These -# will not work today, and is guarded against in dependencies for CONFIG_RUST. RUSTFLAGS_KASAN := -Zsanitizer=kernel-hwaddress \ -Zsanitizer-recover=kernel-hwaddress diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 0718e39cedda..0a4fdd8bd975 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -123,6 +123,9 @@ ifeq ($(CONFIG_AUTOFDO_CLANG),y) _c_flags += $(if $(patsubst n%,, \ $(AUTOFDO_PROFILE_$(target-stem).o)$(AUTOFDO_PROFILE)$(is-kernel-object)), \ $(CFLAGS_AUTOFDO_CLANG)) +_rust_flags += $(if $(patsubst n%,, \ + $(AUTOFDO_PROFILE_$(target-stem).o)$(AUTOFDO_PROFILE)$(is-kernel-object)), \ + $(RUSTFLAGS_AUTOFDO_CLANG)) endif # @@ -187,7 +190,11 @@ objtool-args-$(CONFIG_HAVE_JUMP_LABEL_HACK) += --hacks=jump_label objtool-args-$(CONFIG_HAVE_NOINSTR_HACK) += --hacks=noinstr objtool-args-$(CONFIG_MITIGATION_CALL_DEPTH_TRACKING) += --hacks=skylake objtool-args-$(CONFIG_X86_KERNEL_IBT) += --ibt -objtool-args-$(CONFIG_FINEIBT) += --cfi +objtool-args-$(CONFIG_CALL_PADDING) += --prefix=$(CONFIG_FUNCTION_PADDING_BYTES) +ifdef CONFIG_CALL_PADDING +objtool-args-$(CONFIG_CFI) += --cfi +objtool-args-$(CONFIG_FINEIBT) += --fineibt +endif objtool-args-$(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL) += --mcount ifdef CONFIG_FTRACE_MCOUNT_USE_OBJTOOL objtool-args-$(CONFIG_HAVE_OBJTOOL_NOP_MCOUNT) += --mnop @@ -200,7 +207,6 @@ objtool-args-$(CONFIG_STACK_VALIDATION) += --stackval objtool-args-$(CONFIG_HAVE_STATIC_CALL_INLINE) += --static-call objtool-args-$(CONFIG_HAVE_UACCESS_VALIDATION) += --uaccess objtool-args-$(or $(CONFIG_GCOV_KERNEL),$(CONFIG_KCOV)) += --no-unreachable -objtool-args-$(CONFIG_PREFIX_SYMBOLS) += --prefix=$(CONFIG_FUNCTION_PADDING_BYTES) objtool-args-$(CONFIG_OBJTOOL_WERROR) += --werror objtool-args = $(objtool-args-y) \ @@ -249,6 +255,13 @@ ifdef CONFIG_LTO_CLANG cmd_ld_single = $(if $(objtool-enabled)$(is-single-obj-m), ; $(LD) $(ld_flags) -r -o $(tmp-target) $@; mv $(tmp-target) $@) endif +ifdef CONFIG_LTO_CLANG_THIN_DIST +# Save the _c_flags, sliently. +quiet_cmd_save_c_flags = + saved_c_flags = $(_c_flags) $(modkern_cflags) + cmd_save_c_flags = printf '\n%s\n' 'saved_c_flags_$@ := $(call escsq,$(saved_c_flags))' >> $(dot-target).cmd +endif + quiet_cmd_cc_o_c = CC $(quiet_modtag) $@ cmd_cc_o_c = $(CC) $(c_flags) -c -o $@ $< \ $(cmd_ld_single) \ @@ -256,6 +269,7 @@ quiet_cmd_cc_o_c = CC $(quiet_modtag) $@ define rule_cc_o_c $(call cmd_and_fixdep,cc_o_c) + $(call cmd,save_c_flags) $(call cmd,checksrc) $(call cmd,checkdoc) $(call cmd,gen_objtooldep) diff --git a/scripts/Makefile.modfinal b/scripts/Makefile.modfinal index adcbcde16a07..01a37ec872b9 100644 --- a/scripts/Makefile.modfinal +++ b/scripts/Makefile.modfinal @@ -46,17 +46,9 @@ quiet_cmd_btf_ko = BTF [M] $@ $(CONFIG_SHELL) $(srctree)/scripts/gen-btf.sh --btf_base $(objtree)/vmlinux $@; \ fi; -# Same as newer-prereqs, but allows to exclude specified extra dependencies -newer_prereqs_except = $(filter-out $(PHONY) $(1),$?) - -# Same as if_changed, but allows to exclude specified extra dependencies -if_changed_except = $(if $(call newer_prereqs_except,$(2))$(cmd-check), \ - $(cmd); \ - printf '%s\n' 'savedcmd_$@ := $(make-cmd)' > $(dot-target).cmd, @:) - # Re-generate module BTFs if either module's .ko or vmlinux changed %.ko: %.o %.mod.o .module-common.o $(objtree)/scripts/module.lds $(and $(CONFIG_DEBUG_INFO_BTF_MODULES),$(KBUILD_BUILTIN),$(objtree)/vmlinux) FORCE - +$(call if_changed_except,ld_ko_o,$(objtree)/vmlinux) + +$(call if_changed,ld_ko_o) ifdef CONFIG_DEBUG_INFO_BTF_MODULES +$(if $(newer-prereqs),$(call cmd,btf_ko)) endif diff --git a/scripts/Makefile.package b/scripts/Makefile.package index 0ec946f9b905..6d36786ba31c 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -195,7 +195,7 @@ tar%-pkg: linux-$(KERNELRELEASE)-$(ARCH).tar.% FORCE .tmp_modules_cpio: FORCE $(Q)$(MAKE) -f $(srctree)/Makefile $(Q)rm -rf $@ - $(Q)$(MAKE) -f $(srctree)/Makefile INSTALL_MOD_PATH=$@ modules_install + $(Q)$(MAKE) -f $(srctree)/Makefile INSTALL_MOD_PATH=$@/$(INSTALL_MOD_PATH) modules_install quiet_cmd_cpio = CPIO $@ cmd_cpio = $(CONFIG_SHELL) $(srctree)/usr/gen_initramfs.sh -o $@ $< @@ -264,6 +264,7 @@ help: @echo ' tarxz-pkg - Build the kernel as a xz compressed tarball' @echo ' tarzst-pkg - Build the kernel as a zstd compressed tarball' @echo ' modules-cpio-pkg - Build the kernel modules as cpio archive' + @echo ' (uses INSTALL_MOD_PATH inside the archive)' @echo ' perf-tar-src-pkg - Build the perf source tarball with no compression' @echo ' perf-targz-src-pkg - Build the perf source tarball with gzip compression' @echo ' perf-tarbz2-src-pkg - Build the perf source tarball with bz2 compression' diff --git a/scripts/Makefile.thinlto b/scripts/Makefile.thinlto new file mode 100644 index 000000000000..bb83f13f3cd6 --- /dev/null +++ b/scripts/Makefile.thinlto @@ -0,0 +1,40 @@ +PHONY := __default +__default: + +include include/config/auto.conf +include $(srctree)/scripts/Kbuild.include +include $(srctree)/scripts/Makefile.lib + +native-objs := $(patsubst %.o,%.thinlto-native.o,$(call read-file, vmlinux.thinlto-index)) + +__default: $(native-objs) + +# Generate .thinlto-native.o (obj) from .o (bitcode) and .thinlto.bc (summary) files +# --------------------------------------------------------------------------- +quiet_cmd_cc_o_bc = CC $(quiet_modtag) $@ + be_flags = $(shell sed -n '/saved_c_flags_/s/.*:= //p' \ + $(dir $(<)).$(notdir $(<)).cmd) + cmd_cc_o_bc = \ + $(CC) $(be_flags) -x ir -fno-lto -Wno-unused-command-line-argument \ + -fthinlto-index=$(word 2, $^) -c -o $@ $< + +targets += $(native-objs) +$(native-objs): %.thinlto-native.o: %.o %.o.thinlto.bc FORCE + $(call if_changed,cc_o_bc) + +# Add FORCE to the prerequisites of a target to force it to be always rebuilt. +# --------------------------------------------------------------------------- + +PHONY += FORCE +FORCE: + +# Read all saved command lines and dependencies for the $(targets) we +# may be building above, using $(if_changed{,_dep}). As an +# optimization, we don't need to read them if the target does not +# exist, we will rebuild anyway in that case. + +existing-targets := $(wildcard $(sort $(targets))) + +-include $(foreach f, $(existing-targets),$(dir $(f)).$(notdir $(f)).cmd) + +.PHONY: $(PHONY) diff --git a/scripts/Makefile.vdsoinst b/scripts/Makefile.vdsoinst index ac85f9a4a569..d9f7243217bc 100644 --- a/scripts/Makefile.vdsoinst +++ b/scripts/Makefile.vdsoinst @@ -19,9 +19,10 @@ __default: $$(dest) $$(dest): $(1) FORCE $$(call cmd,install) -# Some architectures create .build-id symlinks -ifneq ($(filter arm s390 sparc x86, $(SRCARCH)),) -link := $(install-dir)/.build-id/$$(shell $(READELF) -n $(1) | sed -n 's@^.*Build ID: \(..\)\(.*\)@\1/\2@p').debug +build-id-file := $$(shell $(READELF) -n $(1) 2>/dev/null | sed -n 's@^.*Build ID: \(..\)\(.*\)@\1/\2@p') + +ifneq ($$(build-id-file),) +link := $(install-dir)/.build-id/$$(build-id-file).debug __default: $$(link) $$(link): $$(dest) FORCE diff --git a/scripts/Makefile.vmlinux_a b/scripts/Makefile.vmlinux_a new file mode 100644 index 000000000000..395e29998d7d --- /dev/null +++ b/scripts/Makefile.vmlinux_a @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: GPL-2.0-only + +PHONY := __default +__default: vmlinux.a + +include include/config/auto.conf +include $(srctree)/scripts/Kbuild.include +include $(srctree)/scripts/Makefile.lib + +# Link of built-in-fixup.a +# --------------------------------------------------------------------------- + +quiet_cmd_ar_builtin_fixup = AR $@ + cmd_ar_builtin_fixup = \ + rm -f $@; \ + $(AR) cDPrST $@ $(KBUILD_VMLINUX_OBJS); \ + $(AR) mPi $$($(AR) t $@ | sed -n 1p) $@ $$($(AR) t $@ | grep -F -f $(srctree)/scripts/head-object-list.txt) + +targets += built-in-fixup.a +built-in-fixup.a: $(KBUILD_VMLINUX_OBJS) scripts/head-object-list.txt FORCE + $(call if_changed,ar_builtin_fixup) + +ifdef CONFIG_LTO_CLANG_THIN_DIST + +quiet_cmd_builtin.order = GEN $@ + cmd_builtin.order = $(AR) t $< > $@ + +targets += builtin.order +builtin.order: built-in-fixup.a FORCE + $(call if_changed,builtin.order) + +quiet_cmd_ld_thinlto_index = LD $@ + cmd_ld_thinlto_index = \ + $(LD) $(KBUILD_LDFLAGS) -r --thinlto-index-only=$@ @$< + +targets += vmlinux.thinlto-index +vmlinux.thinlto-index: builtin.order FORCE + $(call if_changed,ld_thinlto_index) + +quiet_cmd_ar_vmlinux.a = GEN $@ + cmd_ar_vmlinux.a = \ + rm -f $@; \ + while read -r obj; do \ + if grep -Fqx $${obj} $(word 2, $^); then \ + echo $${obj%.o}.thinlto-native.o; \ + else \ + echo $${obj}; \ + fi; \ + done < $< | xargs $(AR) cDPrS --thin $@ + +targets += vmlinux.a +vmlinux.a: builtin.order vmlinux.thinlto-index FORCE + $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.thinlto + $(call if_changed,ar_vmlinux.a) + +else + +# vmlinux.a +# --------------------------------------------------------------------------- + +targets += vmlinux.a +vmlinux.a: built-in-fixup.a FORCE + $(call if_changed,copy) + +endif + +# Add FORCE to the prerequisites of a target to force it to be always rebuilt. +# --------------------------------------------------------------------------- + +PHONY += FORCE +FORCE: + +# Read all saved command lines and dependencies for the $(targets) we +# may be building above, using $(if_changed{,_dep}). As an +# optimization, we don't need to read them if the target does not +# exist, we will rebuild anyway in that case. + +existing-targets := $(wildcard $(sort $(targets))) + +-include $(foreach f,$(existing-targets),$(dir $(f)).$(notdir $(f)).cmd) + +.PHONY: $(PHONY) diff --git a/scripts/Makefile.vmlinux_o b/scripts/Makefile.vmlinux_o index 527352c222ff..24a3a4fd271c 100644 --- a/scripts/Makefile.vmlinux_o +++ b/scripts/Makefile.vmlinux_o @@ -47,6 +47,9 @@ endif vmlinux-objtool-args-$(CONFIG_NOINSTR_VALIDATION) += --noinstr \ $(if $(or $(CONFIG_MITIGATION_UNRET_ENTRY),$(CONFIG_MITIGATION_SRSO)), --unret) +# Only used for builds initiated by klp-build +vmlinux-objtool-args-$(if $(KLP_SYMIDS),y) += --klp-symids + objtool-args = $(vmlinux-objtool-args-y) --link # Link of vmlinux.o used for section mismatch analysis diff --git a/scripts/Makefile.warn b/scripts/Makefile.warn index 5567da6c7dfe..35af7d6c6d18 100644 --- a/scripts/Makefile.warn +++ b/scripts/Makefile.warn @@ -28,11 +28,6 @@ endif KBUILD_CFLAGS-$(CONFIG_CC_NO_ARRAY_BOUNDS) += -Wno-array-bounds ifdef CONFIG_CC_IS_CLANG -# The kernel builds with '-std=gnu11' and '-fms-extensions' so use of GNU and -# Microsoft extensions is acceptable. -KBUILD_CFLAGS += -Wno-gnu -KBUILD_CFLAGS += -Wno-microsoft-anon-tag - # Clang checks for overflow/truncation with '%p', while GCC does not: # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111219 KBUILD_CFLAGS += $(call cc-option, -Wno-format-overflow-non-kprintf) @@ -140,16 +135,6 @@ KBUILD_CFLAGS += $(call cc-option, -Wno-stringop-truncation) KBUILD_CFLAGS += -Wno-override-init # alias for -Wno-initializer-overrides in clang ifdef CONFIG_CC_IS_CLANG -# Clang before clang-16 would warn on default argument promotions. -ifneq ($(call clang-min-version, 160000),y) -# Disable -Wformat -KBUILD_CFLAGS += -Wno-format -# Then re-enable flags that were part of the -Wformat group that aren't -# problematic. -KBUILD_CFLAGS += -Wformat-extra-args -Wformat-invalid-specifier -KBUILD_CFLAGS += -Wformat-zero-length -Wnonnull -KBUILD_CFLAGS += -Wformat-insufficient-args -endif KBUILD_CFLAGS += -Wno-pointer-to-enum-cast KBUILD_CFLAGS += -Wno-tautological-constant-out-of-range-compare KBUILD_CFLAGS += -Wno-unaligned-access diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c index cdd5da7e009b..54063d980442 100644 --- a/scripts/basic/fixdep.c +++ b/scripts/basic/fixdep.c @@ -15,13 +15,13 @@ * gcc produces a very nice and correct list of dependencies which * tells make when to remake a file. * - * To use this list as-is however has the drawback that virtually + * However, to use this list as-is has the drawback that virtually * every file in the kernel includes autoconf.h. * * If the user re-runs make *config, autoconf.h will be * regenerated. make notices that and will rebuild every file which * includes autoconf.h, i.e. basically all files. This is extremely - * annoying if the user just changed CONFIG_HIS_DRIVER from n to m. + * annoying if the user just changed CONFIG_USER_DRIVER from n to m. * * So we play the same trick that "mkdep" played before. We replace * the dependency on autoconf.h by a dependency on every config @@ -33,9 +33,9 @@ * which then let make pick up the changes and the files that use * the config symbols are rebuilt. * - * So if the user changes his CONFIG_HIS_DRIVER option, only the objects - * which depend on "include/config/HIS_DRIVER" will be rebuilt, - * so most likely only his driver ;-) + * So if the user changes their CONFIG_USER_DRIVER option, only the objects + * which depend on "include/config/USER_DRIVER" will be rebuilt, + * so most likely only the user's driver ;-) * * The idea above dates, by the way, back to Michael E Chastain, AFAIK. * diff --git a/scripts/bloat-o-meter b/scripts/bloat-o-meter index db5dd18dc2d5..5868a8b11b0f 100755 --- a/scripts/bloat-o-meter +++ b/scripts/bloat-o-meter @@ -18,8 +18,8 @@ group.add_argument('-c', help='categorize output based on symbol type', action=' group.add_argument('-d', help='Show delta of Data Section', action='store_true') group.add_argument('-t', help='Show delta of text Section', action='store_true') parser.add_argument('-p', dest='prefix', help='Arch prefix for the tool being used. Useful in cross build scenarios') -parser.add_argument('file1', help='First file to compare') -parser.add_argument('file2', help='Second file to compare') +parser.add_argument('file_old', help='First file to compare') +parser.add_argument('file_new', help='Second file to compare') args = parser.parse_args() @@ -43,6 +43,7 @@ def getsizes(file, format): if name.startswith("__se_compat_sys"): continue if name.startswith("__addressable_"): continue if name.startswith("__noinstr_text_start"): continue + if name.startswith("_sdata"): continue if name == "linux_banner": continue if name == "vermagic": continue # statics and some other optimizations adds random .NUMBER @@ -86,7 +87,7 @@ def calc(oldfile, newfile, format): def print_result(symboltype, symbolformat): grow, shrink, add, remove, up, down, delta, old, new, otot, ntot = \ - calc(args.file1, args.file2, symbolformat) + calc(args.file_old, args.file_new, symbolformat) print("add/remove: %s/%s grow/shrink: %s/%s up/down: %s/%s (%s)" % \ (add, remove, grow, shrink, up, -down, up-down)) diff --git a/scripts/check-uapi.sh b/scripts/check-uapi.sh index 955581735cb3..c8beec58871c 100755 --- a/scripts/check-uapi.sh +++ b/scripts/check-uapi.sh @@ -33,9 +33,10 @@ Options: -v Verbose operation (print more information about each header being checked). Environmental args: - ABIDIFF Custom path to abidiff binary - CC C compiler (default is "gcc") - ARCH Target architecture for the UAPI check (default is host arch) + ABIDIFF Custom path to abidiff binary + CROSS_COMPILE Toolchain prefix for compiler + CC C compiler (default is "\${CROSS_COMPILE}gcc") + ARCH Target architecture for the UAPI check (default is host arch) Exit codes: $SUCCESS) Success @@ -178,8 +179,11 @@ do_compile() { local -r inc_dir="$1" local -r header="$2" local -r out="$3" - printf "int main(void) { return 0; }\n" | \ - "$CC" -c \ + printf "int f(void) { return 0; }\n" | \ + "$CC" \ + -shared \ + -nostdlib \ + -fPIC \ -o "$out" \ -x c \ -O0 \ @@ -187,6 +191,7 @@ do_compile() { -fno-eliminate-unused-debug-types \ -g \ "-I${inc_dir}" \ + "-Iusr/dummy-include" \ -include "$header" \ - } @@ -195,7 +200,7 @@ do_compile() { run_make_headers_install() { local -r ref="$1" local -r install_dir="$(get_header_tree "$ref")" - make -j "$MAX_THREADS" ARCH="$ARCH" INSTALL_HDR_PATH="$install_dir" \ + make -j "$MAX_THREADS" CROSS_COMPILE="${CROSS_COMPILE}" ARCH="$ARCH" INSTALL_HDR_PATH="$install_dir" \ headers_install > /dev/null } @@ -404,7 +409,7 @@ min_version_is_satisfied() { # Make sure we have the tools we need and the arguments make sense check_deps() { ABIDIFF="${ABIDIFF:-abidiff}" - CC="${CC:-gcc}" + CC="${CC:-${CROSS_COMPILE}gcc}" ARCH="${ARCH:-$(uname -m)}" if [ "$ARCH" = "x86_64" ]; then ARCH="x86" diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index e56374662ff7..8a7787d228a6 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -57,8 +57,12 @@ my %ignore_type = (); my @ignore = (); my $help = 0; my $configuration_file = ".checkpatch.conf"; +my $def_configuration_dirs_help = '.:$HOME:.scripts'; +(my $def_configuration_dirs = $def_configuration_dirs_help) =~ s/\$(\w+)/$ENV{$1}/g; +my $env_config_dir = 'CHECKPATCH_CONFIG_DIR'; my $max_line_length = 100; my $ignore_perl_version = 0; +my $spdx_cxx_comments = 0; my $minimum_perl_version = 5.10.0; my $min_conf_desc_length = 4; my $spelling_file = "$D/spelling.txt"; @@ -135,6 +139,10 @@ Options: file. It's your fault if there's no backup or git --ignore-perl-version override checking of perl version. expect runtime errors. + --spdx-cxx-comments don't force C comments (/* */) for SPDX license + (required by old toolchains), allow also C++ + comments (//). + NOTE: it should *not* be used for Linux mainline. --codespell Use the codespell dictionary for spelling/typos (default:$codespellfile) --codespellfile Use this codespell dictionary @@ -146,6 +154,11 @@ Options: -h, --help, --version display this help and exit When FILE is - read standard input. + +CONFIGURATION FILE +Default configuration options can be stored in $configuration_file, +search path: '$def_configuration_dirs_help' or in a directory specified by +\$$env_config_dir environment variable (fallback to the default search path). EOM exit($exitcode); @@ -237,7 +250,7 @@ sub list_types { exit($exitcode); } -my $conf = which_conf($configuration_file); +my $conf = which_conf($configuration_file, $env_config_dir, $def_configuration_dirs); if (-f $conf) { my @conf_args; open(my $conffile, '<', "$conf") @@ -339,6 +352,7 @@ GetOptions( 'fix!' => \$fix, 'fix-inplace!' => \$fix_inplace, 'ignore-perl-version!' => \$ignore_perl_version, + 'spdx-cxx-comments!' => \$spdx_cxx_comments, 'debug=s' => \%debug, 'test-only=s' => \$tst_only, 'codespell!' => \$codespell, @@ -641,6 +655,7 @@ our $signature_tags = qr{(?xi: Reviewed-by:| Reported-by:| Suggested-by:| + Assisted-by:| To:| Cc: )}; @@ -864,8 +879,6 @@ our %deprecated_apis = ( "DEFINE_IDR" => "DEFINE_XARRAY", "idr_init" => "xa_init", "idr_init_base" => "xa_init_flags", - "rcu_read_lock_trace" => "rcu_read_lock_tasks_trace", - "rcu_read_unlock_trace" => "rcu_read_unlock_tasks_trace", ); #Create a search pattern for all these strings to speed up a loop below @@ -1530,9 +1543,15 @@ sub which { } sub which_conf { - my ($conf) = @_; + my ($conf, $env_key, $paths) = @_; + my $env_dir = $ENV{$env_key}; + + if (defined($env_dir) && $env_dir ne "") { + return "$env_dir/$conf" if (-e "$env_dir/$conf"); + warn "$P: Can't find a readable $conf in '$env_dir', falling back to default search paths\n"; + } - foreach my $path (split(/:/, ".:$ENV{HOME}:.scripts")) { + foreach my $path (split(/:/, $paths)) { if (-e "$path/$conf") { return "$path/$conf"; } @@ -2928,7 +2947,7 @@ sub process { } $checklicenseline = 1; - if ($realfile !~ /^MAINTAINERS/) { + if ($realfile !~ /^(MAINTAINERS|dev\/null)/) { my $last_binding_patch = $is_binding_patch; $is_binding_patch = () = $realfile =~ m@^(?:Documentation/devicetree/|include/dt-bindings/)@; @@ -3105,6 +3124,16 @@ sub process { } } + # Assisted-by uses a free-form value (e.g. "LLM"), not an + # email address, so skip the email format checks below. + if ($sign_off =~ /^Assisted-by:/i) { + if ($email =~ /^\s*$/) { + WARN("BAD_SIGN_OFF", + "Assisted-by requires a value\n" . $herecurr); + } + next; + } + my ($email_name, $name_comment, $email_address, $comment) = parse_email($email); my $suggested_email = format_email(($email_name, $name_comment, $email_address, $comment)); if ($suggested_email eq "") { @@ -3223,10 +3252,10 @@ sub process { if ($sign_off =~ /^reported(?:|-and-tested)-by:$/i) { if (!defined $lines[$linenr]) { WARN("BAD_REPORTED_BY_LINK", - "Reported-by: should be immediately followed by Closes: with a URL to the report\n" . $herecurr . "\n"); - } elsif ($rawlines[$linenr] !~ /^closes:\s*/i) { + "Reported-by: should be immediately followed by Closes: or Link: with a URL to the report\n" . $herecurr . "\n"); + } elsif ($rawlines[$linenr] !~ /^(closes|link):\s*/i) { WARN("BAD_REPORTED_BY_LINK", - "Reported-by: should be immediately followed by Closes: with a URL to the report\n" . $herecurr . $rawlines[$linenr] . "\n"); + "Reported-by: should be immediately followed by Closes: or Link: with a URL to the report\n" . $herecurr . $rawlines[$linenr] . "\n"); } } } @@ -3753,6 +3782,12 @@ sub process { my $vp_file = $dt_path . "vendor-prefixes.yaml"; foreach my $compat (@compats) { + # Skip ID-based PCI and USB compatible patterns. + # DT validation will check them properly. + next if $compat =~ /^pciclass,/; + next if $compat =~ /^pci[a-f0-9]{2,4},/; + next if $compat =~ /^usb(if)?[a-f0-9]{1,4},/; + my $compat2 = $compat; $compat2 =~ s/\,[a-zA-Z0-9]*\-/\,<\.\*>\-/; my $compat3 = $compat; @@ -3791,26 +3826,33 @@ sub process { $checklicenseline = 2; } elsif ($rawline =~ /^\+/) { my $comment = ""; - if ($realfile =~ /\.(h|s|S)$/) { - $comment = '/*'; - } elsif ($realfile =~ /\.(c|rs|dts|dtsi)$/) { + if ($realfile =~ /\.(c|rs|dts|dtsi)$/) { $comment = '//'; } elsif (($checklicenseline == 2) || $realfile =~ /\.(sh|pl|py|awk|tc|yaml)$/) { $comment = '#'; } elsif ($realfile =~ /\.rst$/) { $comment = '..'; } + my $pattern = qr{\Q$comment\E}; + if ($realfile =~ /\.(h|s|S)$/) { + $comment = '/*'; + $pattern = qr{/\*}; + if ($spdx_cxx_comments) { + $comment = '// or /*'; + $pattern = qr{//|/\*}; + } + } # check SPDX comment style for .[chsS] files if ($realfile =~ /\.[chsS]$/ && $rawline =~ /SPDX-License-Identifier:/ && - $rawline !~ m@^\+\s*\Q$comment\E\s*@) { + $rawline !~ m@^\+\s*$pattern\s*@) { WARN("SPDX_LICENSE_TAG", "Improper SPDX comment style for '$realfile', please use '$comment' instead\n" . $herecurr); } if ($comment !~ /^$/ && - $rawline !~ m@^\+\Q$comment\E SPDX-License-Identifier: @) { + $rawline !~ m@^\+$pattern SPDX-License-Identifier: @) { WARN("SPDX_LICENSE_TAG", "Missing or malformed SPDX-License-Identifier tag in line $checklicenseline\n" . $herecurr); } elsif ($rawline =~ /(SPDX-License-Identifier: .*)/) { @@ -3856,6 +3898,14 @@ sub process { "Misplaced SPDX-License-Identifier tag - use line $checklicenseline instead\n" . $herecurr); } +# check for disallowed SPDX file tags + if ($rawline =~ /\bSPDX-.*:/ && + $rawline !~ /\bSPDX-License-Identifier:/ && + $rawline !~ /\bSPDX-FileCopyrightText:/) { + WARN("SPDX_LICENSE_TAG", + "Disallowed SPDX tag\n" . $herecurr); + } + # line length limit (with some exclusions) # # There are a few types of lines that may extend beyond $max_line_length: @@ -4105,6 +4155,7 @@ sub process { $line =~ /^\+[a-z_]*init/ || $line =~ /^\+\s*(?:static\s+)?[A-Z_]*ATTR/ || $line =~ /^\+\s*DECLARE/ || + $line =~ /^\+\s*NOKPROBE_SYMBOL/ || $line =~ /^\+\s*builtin_[\w_]*driver/ || $line =~ /^\+\s*__setup/)) { if (CHK("LINE_SPACING", @@ -4138,7 +4189,7 @@ sub process { $pl =~ s/\b(?:$Attribute|$Sparse)\b//g; if (($pl =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ || # function pointer declarations - $pl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident\s*\)\s*[=,;:\[\(]/ || + $pl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident(?:\s*\[\s*(?:$Ident|$Constant)?\s*\])?\s*\)\s*[=,;:\[\(]/ || # foo bar; where foo is some local typedef or #define $pl =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ || # known declaration macros @@ -4152,7 +4203,7 @@ sub process { # looks like a declaration !($sl =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ || # function pointer declarations - $sl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident\s*\)\s*[=,;:\[\(]/ || + $sl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident(?:\s*\[\s*(?:$Ident|$Constant)?\s*\])?\s*\)\s*[=,;:\[\(]/ || # foo bar; where foo is some local typedef or #define $sl =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ || # known declaration macros @@ -7502,10 +7553,10 @@ sub process { } # check for various structs that are normally const (ops, kgdb, device_tree) -# and avoid what seem like struct definitions 'struct foo {' +# and avoid what seem like struct definitions 'struct foo {' or forward declarations 'struct foo;' if (defined($const_structs) && $line !~ /\bconst\b/ && - $line =~ /\bstruct\s+($const_structs)\b(?!\s*\{)/) { + $line =~ /\bstruct\s+($const_structs)\b(?!\s*[\{;])/) { WARN("CONST_STRUCT", "struct $1 should normally be const\n" . $herecurr); } @@ -7578,12 +7629,15 @@ sub process { # Complain about RCU Tasks Trace used outside of BPF (and of course, RCU). our $rcu_trace_funcs = qr{(?x: + rcu_read_lock_tasks_trace | rcu_read_lock_trace | rcu_read_lock_trace_held | rcu_read_unlock_trace | + rcu_read_unlock_tasks_trace | call_rcu_tasks_trace | synchronize_rcu_tasks_trace | rcu_barrier_tasks_trace | + rcu_tasks_trace_expedite_current | rcu_request_urgent_qs_task )}; our $rcu_trace_paths = qr{(?x: diff --git a/scripts/checksyscalls.sh b/scripts/checksyscalls.sh index 1e5d2eeb726d..e2970421c1ff 100755 --- a/scripts/checksyscalls.sh +++ b/scripts/checksyscalls.sh @@ -10,6 +10,10 @@ # checksyscalls.sh gcc gcc-options # +set -e + +reference_table="$(dirname $0)/../arch/x86/entry/syscalls/syscall_32.tbl" + ignore_list() { cat << EOF #include <asm/types.h> @@ -269,5 +273,10 @@ syscall_list() { done } -(ignore_list && syscall_list $(dirname $0)/../arch/x86/entry/syscalls/syscall_32.tbl) | \ +(ignore_list && syscall_list ${reference_table}) | \ $* -Wno-error -Wno-unused-macros -E -x c - > /dev/null + +# For fixdep +if [ -n "${DEPFILE}" ]; then + echo "${0}: ${0} ${reference_table}" >> "${DEPFILE}" +fi diff --git a/scripts/clang-tools/gen_compile_commands.py b/scripts/clang-tools/gen_compile_commands.py index 96e6e46ad1a7..8d14b81efd73 100755 --- a/scripts/clang-tools/gen_compile_commands.py +++ b/scripts/clang-tools/gen_compile_commands.py @@ -201,6 +201,8 @@ def main(): # Modules are listed in modules.order. if os.path.isdir(path): cmdfiles = cmdfiles_in_dir(path) + elif os.path.basename(path) == 'libgcc.a': + cmdfiles = [] elif path.endswith('.a'): cmdfiles = cmdfiles_for_a(path, ar) elif path.endswith('modules.order'): diff --git a/scripts/clang-tools/run-clang-tools.py b/scripts/clang-tools/run-clang-tools.py index f31ffd09e1ea..e78be82aa693 100755 --- a/scripts/clang-tools/run-clang-tools.py +++ b/scripts/clang-tools/run-clang-tools.py @@ -79,14 +79,15 @@ def run_analysis(entry): def main(): - try: - args = parse_arguments() + args = parse_arguments() + + # Read JSON data into the datastore variable + with open(args.path) as f: + datastore = json.load(f) - lock = multiprocessing.Lock() - pool = multiprocessing.Pool(initializer=init, initargs=(lock, args)) - # Read JSON data into the datastore variable - with open(args.path, "r") as f: - datastore = json.load(f) + lock = multiprocessing.Lock() + try: + with multiprocessing.Pool(initializer=init, initargs=(lock, args)) as pool: pool.map(run_analysis, datastore) except BrokenPipeError: # Python flushes standard streams on exit; redirect remaining output diff --git a/scripts/coccicheck b/scripts/coccicheck index 8dd766009de1..a0a0e0f72bc7 100755 --- a/scripts/coccicheck +++ b/scripts/coccicheck @@ -11,7 +11,7 @@ DIR="$(dirname $(readlink -f $0))/.." SPATCH="`which ${SPATCH:=spatch}`" if [ ! -x "$SPATCH" ]; then - echo 'spatch is part of the Coccinelle project and is available at http://coccinelle.lip6.fr/' + echo 'spatch is part of the Coccinelle project and is available at https://coccinelle.gitlabpages.inria.fr/website' exit 1 fi @@ -229,7 +229,7 @@ coccinelle () { echo " in $FILE." echo '' echo ' More information about semantic patching is available at' - echo ' http://coccinelle.lip6.fr/' + echo ' https://coccinelle.gitlabpages.inria.fr/website' echo '' if [ "`sed -ne 's|^//#||p' $COCCI`" ] ; then diff --git a/scripts/coccinelle/api/alloc/alloc_cast.cocci b/scripts/coccinelle/api/alloc/alloc_cast.cocci index f6f0ccdb6409..a4b83cc8ea01 100644 --- a/scripts/coccinelle/api/alloc/alloc_cast.cocci +++ b/scripts/coccinelle/api/alloc/alloc_cast.cocci @@ -20,6 +20,24 @@ virtual patch virtual org virtual report +@m1@ +identifier i; +expression e; +type T; +position p1; +@@ + +#define i (T@p1 *)e + +@m2@ +identifier i; +expression e; +type T; +position p2; +@@ + +#define i(...) (T@p2 *)e + @initialize:python@ @@ import re @@ -28,15 +46,15 @@ m = re.compile(pattern) @r1 depends on context || patch@ type T; +position p != {m1.p1,m2.p2}; @@ - (T *) + (T@p *) \(kmalloc\|kzalloc\|kcalloc\|kmem_cache_alloc\|kmem_cache_zalloc\| kmem_cache_alloc_node\|kmalloc_node\|kzalloc_node\|vmalloc\|vzalloc\| dma_alloc_coherent\|devm_kmalloc\|devm_kzalloc\| - kvmalloc\|kvzalloc\|kvmalloc_node\|kvzalloc_node\|pci_alloc_consistent\| - pci_zalloc_consistent\|kmem_alloc\|kmem_zalloc\|kmem_zone_alloc\| - kmem_zone_zalloc\|vmalloc_node\|vzalloc_node\)(...) + kvmalloc\|kvzalloc\|kvmalloc_node\|kvzalloc_node\| + vmalloc_node\|vzalloc_node\)(...) //---------------------------------------------------------- // For context mode @@ -57,9 +75,8 @@ type r1.T; \(kmalloc\|kzalloc\|kcalloc\|kmem_cache_alloc\|kmem_cache_zalloc\| kmem_cache_alloc_node\|kmalloc_node\|kzalloc_node\|vmalloc\|vzalloc\| dma_alloc_coherent\|devm_kmalloc\|devm_kzalloc\| - kvmalloc\|kvzalloc\|kvmalloc_node\|kvzalloc_node\|pci_alloc_consistent\| - pci_zalloc_consistent\|kmem_alloc\|kmem_zalloc\|kmem_zone_alloc\| - kmem_zone_zalloc\|vmalloc_node\|vzalloc_node\)(...) + kvmalloc\|kvzalloc\|kvmalloc_node\|kvzalloc_node\| + vmalloc_node\|vzalloc_node\)(...) //---------------------------------------------------------- // For patch mode @@ -80,9 +97,8 @@ type r1.T; \(kmalloc\|kzalloc\|kcalloc\|kmem_cache_alloc\|kmem_cache_zalloc\| kmem_cache_alloc_node\|kmalloc_node\|kzalloc_node\|vmalloc\|vzalloc\| dma_alloc_coherent\|devm_kmalloc\|devm_kzalloc\| - kvmalloc\|kvzalloc\|kvmalloc_node\|kvzalloc_node\|pci_alloc_consistent\| - pci_zalloc_consistent\|kmem_alloc\|kmem_zalloc\|kmem_zone_alloc\| - kmem_zone_zalloc\|vmalloc_node\|vzalloc_node\)(...) + kvmalloc\|kvzalloc\|kvmalloc_node\|kvzalloc_node\| + vmalloc_node\|vzalloc_node\)(...) //---------------------------------------------------------- // For org and report mode @@ -90,16 +106,15 @@ type r1.T; @r2 depends on org || report@ type T; -position p; +position p != {m1.p1,m2.p2}; @@ (T@p *) \(kmalloc\|kzalloc\|kcalloc\|kmem_cache_alloc\|kmem_cache_zalloc\| kmem_cache_alloc_node\|kmalloc_node\|kzalloc_node\|vmalloc\|vzalloc\| dma_alloc_coherent\|devm_kmalloc\|devm_kzalloc\| - kvmalloc\|kvzalloc\|kvmalloc_node\|kvzalloc_node\|pci_alloc_consistent\| - pci_zalloc_consistent\|kmem_alloc\|kmem_zalloc\|kmem_zone_alloc\| - kmem_zone_zalloc\|vmalloc_node\|vzalloc_node\)(...) + kvmalloc\|kvzalloc\|kvmalloc_node\|kvzalloc_node\| + vmalloc_node\|vzalloc_node\)(...) @script:python depends on org@ p << r2.p; diff --git a/scripts/coccinelle/api/alloc/pool_zalloc-simple.cocci b/scripts/coccinelle/api/alloc/pool_zalloc-simple.cocci index 9c61a23b34db..07af8c7433d1 100644 --- a/scripts/coccinelle/api/alloc/pool_zalloc-simple.cocci +++ b/scripts/coccinelle/api/alloc/pool_zalloc-simple.cocci @@ -5,7 +5,7 @@ // Copyright: (C) 2015 Intel Corp. // Options: --no-includes --include-headers // -// Keywords: dma_pool_zalloc, pci_pool_zalloc +// Keywords: dma_pool_zalloc // virtual context @@ -22,7 +22,7 @@ expression x; statement S; @@ -* x = \(dma_pool_alloc\|pci_pool_alloc\)(...); +* x = dma_pool_alloc(...); if ((x==NULL) || ...) S * memset(x,0, ...); @@ -41,17 +41,6 @@ statement S; if ((x==NULL) || ...) S - memset(x,0,...); -@depends on patch@ -expression x; -expression a,b,c; -statement S; -@@ - -- x = pci_pool_alloc(a,b,c); -+ x = pci_pool_zalloc(a,b,c); - if ((x==NULL) || ...) S -- memset(x,0,...); - //---------------------------------------------------------- // For org and report mode //---------------------------------------------------------- @@ -63,7 +52,7 @@ statement S; position p; @@ - x = @p\(dma_pool_alloc\|pci_pool_alloc\)(a,b,c); + x =@p dma_pool_alloc(a,b,c); if ((x==NULL) || ...) S memset(x,0, ...); diff --git a/scripts/coccinelle/api/alloc/zalloc-simple.cocci b/scripts/coccinelle/api/alloc/zalloc-simple.cocci index d66c45356691..ae1d39e71e31 100644 --- a/scripts/coccinelle/api/alloc/zalloc-simple.cocci +++ b/scripts/coccinelle/api/alloc/zalloc-simple.cocci @@ -35,7 +35,7 @@ statement S; @@ * x = (T)\(kmalloc(E1, ...)\|vmalloc(E1)\|dma_alloc_coherent(...,E1,...)\| - kmalloc_node(E1, ...)\|kmem_cache_alloc(...)\|kmem_alloc(E1, ...)\| + kmalloc_node(E1, ...)\|kmem_cache_alloc(...)\| devm_kmalloc(...,E1,...)\|kvmalloc(E1, ...)\|kvmalloc_node(E1,...)\); if ((x==NULL) || ...) S * memset((T2)x,0,E1); @@ -88,15 +88,6 @@ statement S; - x = (T)kmem_cache_alloc(E3,E4); + x = (T)kmem_cache_zalloc(E3,E4); | -- x = kmem_alloc(E1,E2); -+ x = kmem_zalloc(E1,E2); -| -- x = (T *)kmem_alloc(E1,E2); -+ x = kmem_zalloc(E1,E2); -| -- x = (T)kmem_alloc(E1,E2); -+ x = (T)kmem_zalloc(E1,E2); -| - x = devm_kmalloc(E2,E1,E3); + x = devm_kzalloc(E2,E1,E3); | @@ -291,36 +282,6 @@ msg="WARNING: kmem_cache_zalloc should be used for %s, instead of kmem_cache_all coccilib.report.print_report(p[0], msg) //----------------------------------------------------------------- -@r5 depends on org || report@ -type T, T2; -expression x; -expression E1,E2; -statement S; -position p; -@@ - - x = (T)kmem_alloc@p(E1,E2); - if ((x==NULL) || ...) S - memset((T2)x,0,E1); - -@script:python depends on org@ -p << r5.p; -x << r5.x; -@@ - -msg="%s" % (x) -msg_safe=msg.replace("[","@(").replace("]",")") -coccilib.org.print_todo(p[0], msg_safe) - -@script:python depends on report@ -p << r5.p; -x << r5.x; -@@ - -msg="WARNING: kmem_zalloc should be used for %s, instead of kmem_alloc/memset" % (x) -coccilib.report.print_report(p[0], msg) - -//----------------------------------------------------------------- @r6 depends on org || report@ type T, T2; expression x; diff --git a/scripts/coccinelle/api/atomic_as_refcounter.cocci b/scripts/coccinelle/api/atomic_as_refcounter.cocci index bbe5b2932933..af82b9e094aa 100644 --- a/scripts/coccinelle/api/atomic_as_refcounter.cocci +++ b/scripts/coccinelle/api/atomic_as_refcounter.cocci @@ -27,8 +27,6 @@ identifier fname6 =~ ".*call_rcu.*"; | atomic_dec_and_lock@p1(&(a)->x, ...) | - atomic_long_dec_and_lock@p1(&(a)->x, ...) -| atomic_long_dec_and_test@p1(&(a)->x) | atomic64_dec_and_test@p1(&(a)->x) @@ -70,8 +68,6 @@ identifier fname =~ ".*free.*"; | atomic_dec_and_lock@p1(&(a)->x, ...) | - atomic_long_dec_and_lock@p1(&(a)->x, ...) -| atomic_long_dec_and_test@p1(&(a)->x) | atomic64_dec_and_test@p1(&(a)->x) diff --git a/scripts/coccinelle/api/kfree_mismatch.cocci b/scripts/coccinelle/api/kfree_mismatch.cocci index d46a9b3eb7b3..bc90d0c071ac 100644 --- a/scripts/coccinelle/api/kfree_mismatch.cocci +++ b/scripts/coccinelle/api/kfree_mismatch.cocci @@ -29,7 +29,7 @@ position kok, vok; } else { ... E = \(vmalloc\|vzalloc\|vmalloc_user\|vmalloc_node\| - vzalloc_node\|vmalloc_exec\|vmalloc_32\| + vzalloc_node\|vmalloc_32\| vmalloc_32_user\|__vmalloc\|__vmalloc_node_range\| __vmalloc_node\)(...)@vok ... @@ -42,7 +42,7 @@ position kok, vok; if (E == NULL) { ... E = \(vmalloc\|vzalloc\|vmalloc_user\|vmalloc_node\| - vzalloc_node\|vmalloc_exec\|vmalloc_32\| + vzalloc_node\|vmalloc_32\| vmalloc_32_user\|__vmalloc\|__vmalloc_node_range\| __vmalloc_node\)(...)@vok ... @@ -68,7 +68,7 @@ position f != free.fok; * E = \(kmalloc\|kzalloc\|krealloc\|kcalloc\|kmalloc_node\| * kzalloc_node\|kmalloc_array\|kmalloc_array_node\| * kcalloc_node\)(...)@a - ... when != if (...) { ... E = \(vmalloc\|vzalloc\|vmalloc_user\|vmalloc_node\|vzalloc_node\|vmalloc_exec\|vmalloc_32\|vmalloc_32_user\|__vmalloc\|__vmalloc_node_range\|__vmalloc_node\)(...); ... } + ... when != if (...) { ... E = \(vmalloc\|vzalloc\|vmalloc_user\|vmalloc_node\|vzalloc_node\|vmalloc_32\|vmalloc_32_user\|__vmalloc\|__vmalloc_node_range\|__vmalloc_node\)(...); ... } when != is_vmalloc_addr(E) when any * \(vfree\|vfree_atomic\|kvfree\)(E)@f @@ -82,7 +82,7 @@ position f != free.fok; E = \(kmalloc\|kzalloc\|krealloc\|kcalloc\|kmalloc_node\| kzalloc_node\|kmalloc_array\|kmalloc_array_node\| kcalloc_node\)(...)@a - ... when != if (...) { ... E = \(vmalloc\|vzalloc\|vmalloc_user\|vmalloc_node\|vzalloc_node\|vmalloc_exec\|vmalloc_32\|vmalloc_32_user\|__vmalloc\|__vmalloc_node_range\|__vmalloc_node\)(...); ... } + ... when != if (...) { ... E = \(vmalloc\|vzalloc\|vmalloc_user\|vmalloc_node\|vzalloc_node\|vmalloc_32\|vmalloc_32_user\|__vmalloc\|__vmalloc_node_range\|__vmalloc_node\)(...); ... } when != is_vmalloc_addr(E) when any - \(vfree\|vfree_atomic\|kvfree\)(E)@f @@ -95,7 +95,7 @@ position f != free.fok; @@ * E = \(vmalloc\|vzalloc\|vmalloc_user\|vmalloc_node\|vzalloc_node\| -* vmalloc_exec\|vmalloc_32\|vmalloc_32_user\|__vmalloc\| +* vmalloc_32\|vmalloc_32_user\|__vmalloc\| * __vmalloc_node_range\|__vmalloc_node\)(...)@a ... when != is_vmalloc_addr(E) when any @@ -108,7 +108,7 @@ position f != free.fok; @@ E = \(vmalloc\|vzalloc\|vmalloc_user\|vmalloc_node\|vzalloc_node\| - vmalloc_exec\|vmalloc_32\|vmalloc_32_user\|__vmalloc\| + vmalloc_32\|vmalloc_32_user\|__vmalloc\| __vmalloc_node_range\|__vmalloc_node\)(...)@a ... when != is_vmalloc_addr(E) when any diff --git a/scripts/coccinelle/api/kmalloc_objs.cocci b/scripts/coccinelle/api/kmalloc_objs.cocci index db12b7be7247..e9a415b7b6f4 100644 --- a/scripts/coccinelle/api/kmalloc_objs.cocci +++ b/scripts/coccinelle/api/kmalloc_objs.cocci @@ -122,3 +122,14 @@ fresh identifier ALLOC_OBJS = script:python(ALLOC_ARRAY) { alloc_array(ALLOC_ARR - ALLOC(struct_size_t(TYPE, FLEX, COUNT), GFP) + ALLOC_FLEX(TYPE, FLEX, COUNT, GFP) ) + +@drop_gfp_kernel depends on patch && !(file in "tools") && !(file in "samples")@ +identifier ALLOC = {kmalloc_obj,kmalloc_objs,kmalloc_flex, + kzalloc_obj,kzalloc_objs,kzalloc_flex, + kvmalloc_obj,kvmalloc_objs,kvmalloc_flex, + kvzalloc_obj,kvzalloc_objs,kvzalloc_flex}; +@@ + + ALLOC(... +- , GFP_KERNEL + ) diff --git a/scripts/coccinelle/free/devm_free.cocci b/scripts/coccinelle/free/devm_free.cocci index 0880729badbc..947d7e685655 100644 --- a/scripts/coccinelle/free/devm_free.cocci +++ b/scripts/coccinelle/free/devm_free.cocci @@ -26,7 +26,8 @@ virtual report virtual context @r depends on context || org || report@ -expression x; +type T; +T x; @@ ( @@ -56,19 +57,27 @@ expression x; ) @safe depends on context || org || report exists@ -expression x; +r.T x; position p; @@ ( x = kmalloc(...) | + x = kmalloc_obj(...) +| + x = kmalloc_objs(...) +| x = kvasprintf(...) | x = kasprintf(...) | x = kzalloc(...) | + x = kzalloc_obj(...) +| + x = kzalloc_objs(...) +| x = kmalloc_array(...) | x = kcalloc(...) @@ -105,7 +114,7 @@ position p; ) @pb@ -expression r.x; +r.T r.x; position p != safe.p; @@ diff --git a/scripts/coccinelle/free/ifnulldev_put.cocci b/scripts/coccinelle/free/ifnulldev_put.cocci index 2bd2e8fae485..1430d68ff268 100644 --- a/scripts/coccinelle/free/ifnulldev_put.cocci +++ b/scripts/coccinelle/free/ifnulldev_put.cocci @@ -23,13 +23,13 @@ expression E; | dev_put(E); | - dev_put_track(E, ...); + netdev_put(E, ...); | __dev_hold(E); | dev_hold(E); | - dev_hold_track(E, ...); + netdev_hold(E, ...); ) @r depends on context || report || org @ @@ -38,18 +38,18 @@ position p; @@ * if (E != NULL) -* \(__dev_put@p\|dev_put@p\|dev_put_track@p\|__dev_hold@p\|dev_hold@p\| -* dev_hold_track@p\)(E, ...); +* \(__dev_put@p\|dev_put@p\|netdev_put@p\|__dev_hold@p\|dev_hold@p\| +* netdev_hold@p\)(E, ...); @script:python depends on org@ p << r.p; @@ -cocci.print_main("NULL check before dev_{put, hold} functions is not needed", p) +cocci.print_main("NULL check before (net)dev_{put, hold} functions is not needed", p) @script:python depends on report@ p << r.p; @@ -msg = "WARNING: NULL check before dev_{put, hold} functions is not needed." +msg = "WARNING: NULL check before (net)dev_{put, hold} functions is not needed." coccilib.report.print_report(p[0], msg) diff --git a/scripts/coccinelle/free/pci_free_consistent.cocci b/scripts/coccinelle/free/pci_free_consistent.cocci deleted file mode 100644 index e062b9ba09ff..000000000000 --- a/scripts/coccinelle/free/pci_free_consistent.cocci +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/// Find missing pci_free_consistent for every pci_alloc_consistent. -/// -// Confidence: Moderate -// Copyright: (C) 2013 Petr Strnad. -// URL: https://coccinelle.gitlabpages.inria.fr/website -// Keywords: pci_free_consistent, pci_alloc_consistent -// Options: --no-includes --include-headers - -virtual report -virtual org - -@search@ -local idexpression id; -expression x,y,z,e; -position p1,p2; -type T; -@@ - -id = pci_alloc_consistent@p1(x,y,&z) -... when != e = id -if (id == NULL || ...) { ... return ...; } -... when != pci_free_consistent(x,y,id,z) - when != if (id) { ... pci_free_consistent(x,y,id,z) ... } - when != if (y) { ... pci_free_consistent(x,y,id,z) ... } - when != e = (T)id - when exists -( -return 0; -| -return 1; -| -return id; -| -return@p2 ...; -) - -@script:python depends on report@ -p1 << search.p1; -p2 << search.p2; -@@ - -msg = "ERROR: missing pci_free_consistent; pci_alloc_consistent on line %s and return without freeing on line %s" % (p1[0].line,p2[0].line) -coccilib.report.print_report(p2[0],msg) - -@script:python depends on org@ -p1 << search.p1; -p2 << search.p2; -@@ - -msg = "ERROR: missing pci_free_consistent; pci_alloc_consistent on line %s and return without freeing on line %s" % (p1[0].line,p2[0].line) -cocci.print_main(msg,p1) -cocci.print_secs("",p2) diff --git a/scripts/coccinelle/hid/ff_race.cocci b/scripts/coccinelle/hid/ff_race.cocci new file mode 100644 index 000000000000..479f5d1e3184 --- /dev/null +++ b/scripts/coccinelle/hid/ff_race.cocci @@ -0,0 +1,34 @@ +/// Detect HID drivers that initialize force-feedback after hid_hw_start() +/// when HID_CONNECT_HIDINPUT is used. This is a lifecycle violation as +/// the input device is already registered. +// +// Confidence: High +// Copyright: (C) 2026 Gemini. GPLv2. + +virtual report + +@r@ +identifier probe_fn; +expression hdev, flags; +position p1, p2; +@@ + +probe_fn(struct hid_device *hdev, ...) { + <... + hid_hw_start@p1(hdev, flags) + ... + \(input_ff_create\|input_ff_create_memless\)@p2(...) + ...> +} + +@script:python depends on report@ +p1 << r.p1; +p2 << r.p2; +flags << r.flags; +@@ + +# Check if flags include HID_CONNECT_HIDINPUT (0x01) or HID_CONNECT_DEFAULT (0x0f) +# Note: HID_CONNECT_DEFAULT is 0x0f, HID_CONNECT_HIDINPUT is 0x01 +if "HID_CONNECT_HIDINPUT" in flags or "HID_CONNECT_DEFAULT" in flags: + msg = "WARNING: force-feedback initialized after hid_hw_start() with HID_CONNECT_HIDINPUT. Input device is already registered at this point. Use .input_configured() instead." + coccilib.report.print_report(p2[0], msg) diff --git a/scripts/coccinelle/locks/double_lock.cocci b/scripts/coccinelle/locks/double_lock.cocci index 619cfc714409..381060849a7b 100644 --- a/scripts/coccinelle/locks/double_lock.cocci +++ b/scripts/coccinelle/locks/double_lock.cocci @@ -38,7 +38,20 @@ write_lock@p1 write_trylock@p1 ) (E1@p,...); -@balanced@ +@r_candidate exists@ +expression x <= locked.E1; +expression locked.E1; +expression E2; +identifier lock; +position locked.p,p1,p2; +@@ + +lock@p1 (E1@p,...); +... when != E1 + when != \(x = E2\|&x\) +lock@p2 (E1,...); + +@balanced depends on r_candidate@ position p1 != locked.p1; position locked.p; identifier lock,unlock; diff --git a/scripts/coccinelle/locks/mini_lock.cocci b/scripts/coccinelle/locks/mini_lock.cocci index 71065d8a5d54..c65241c895ff 100644 --- a/scripts/coccinelle/locks/mini_lock.cocci +++ b/scripts/coccinelle/locks/mini_lock.cocci @@ -53,11 +53,31 @@ spin_lock_irq@p1 spin_lock_irqsave@p1 ) (E1@p,...); -@looped@ +@err_candidate exists@ +expression E1; +position prelocked.p; +position up != prelocked.p1; +position rc; +identifier lock,unlock; +@@ + +lock(E1@p,...); +... when != E1 + when any +if (...) { + ... when != E1 + return@rc ...; +} +... when != E1 + when any +unlock@up(E1,...); + +@looped exists@ +position err_candidate.rc; position r; @@ -for(...;...;...) { <+... return@r ...; ...+> } +for(...;...;...) { <+... return@rc@r ...; ...+> } @err exists@ expression E1; diff --git a/scripts/coccinelle/misc/minmax.cocci b/scripts/coccinelle/misc/minmax.cocci index ca4830ae3042..b83d4a01e47a 100644 --- a/scripts/coccinelle/misc/minmax.cocci +++ b/scripts/coccinelle/misc/minmax.cocci @@ -17,7 +17,21 @@ virtual org virtual context virtual patch -@rmax depends on !patch@ +@max_candidate disable not_int1, not_int2, neg_if_exp@ +expression E1, E2, E3, E4; +binary operator cmp = {>, >=}; +@@ + + E1 cmp E2 ? E3 : E4 + +@min_candidate disable not_int1, not_int2, neg_if_exp@ +expression E1, E2, E3, E4; +binary operator cmp = {<, <=}; +@@ + + E1 cmp E2 ? E3 : E4 + +@rmax depends on !patch && max_candidate disable not_int1, not_int2, neg_if_exp@ identifier func; expression x, y; binary operator cmp = {>, >=}; @@ -27,11 +41,23 @@ position p; func(...) { <... -* ((x) cmp@p (y) ? (x) : (y)) +* (x) cmp@p (y) ? (x) : (y) ...> } -@rmaxif depends on !patch@ +@maxif_candidate disable not_int1, not_int2, neg_if@ +expression x, y; +expression max_val; +binary operator cmp = {>, >=}; +@@ + +if ((x) cmp (y)) { + max_val = (x); +} else { + max_val = (y); +} + +@rmaxif depends on !patch && maxif_candidate disable not_int1, not_int2, neg_if@ identifier func; expression x, y; expression max_val; @@ -51,7 +77,7 @@ func(...) } // Ignore errcode returns. -@errcode@ +@errcode depends on min_candidate disable not_int1, not_int2, neg_if_exp@ position p; identifier func; expression x; @@ -65,7 +91,7 @@ func(...) ...> } -@rmin depends on !patch@ +@rmin depends on !patch && min_candidate disable not_int1, not_int2, neg_if_exp@ identifier func; expression x, y; binary operator cmp = {<, <=}; @@ -75,11 +101,23 @@ position p != errcode.p; func(...) { <... -* ((x) cmp@p (y) ? (x) : (y)) +* (x) cmp@p (y) ? (x) : (y) ...> } -@rminif depends on !patch@ +@minif_candidate disable not_int1, not_int2, neg_if@ +expression x, y; +expression min_val; +binary operator cmp = {<, <=}; +@@ + +if ((x) cmp (y)) { + min_val = (x); +} else { + min_val = (y); +} + +@rminif depends on !patch && minif_candidate disable not_int1, not_int2, neg_if@ identifier func; expression x, y; expression min_val; @@ -98,7 +136,7 @@ func(...) ...> } -@pmax depends on patch@ +@pmax depends on patch && max_candidate disable not_int1, not_int2, neg_if_exp@ identifier func; expression x, y; binary operator cmp = {>=, >}; @@ -112,7 +150,7 @@ func(...) ...> } -@pmaxif depends on patch@ +@pmaxif depends on patch && maxif_candidate disable not_int1, not_int2, neg_if@ identifier func; expression x, y; expression max_val; @@ -131,7 +169,7 @@ func(...) ...> } -@pmin depends on patch@ +@pmin depends on patch && min_candidate disable not_int1, not_int2, neg_if_exp@ identifier func; expression x, y; binary operator cmp = {<=, <}; @@ -146,7 +184,7 @@ func(...) ...> } -@pminif depends on patch@ +@pminif depends on patch && minif_candidate disable not_int1, not_int2, neg_if@ identifier func; expression x, y; expression min_val; diff --git a/scripts/coccinelle/misc/struct_size.cocci b/scripts/coccinelle/misc/struct_size.cocci index 9b02c37438e4..406884d68d43 100644 --- a/scripts/coccinelle/misc/struct_size.cocci +++ b/scripts/coccinelle/misc/struct_size.cocci @@ -29,9 +29,7 @@ f expression E1, E2; identifier m; @@ -( -* (sizeof(*E1) + (E2 * sizeof(*E1->m))) -) +* sizeof(*E1) + (E2 * sizeof(*E1->m)) //---------------------------------------------------------- // For patch mode @@ -55,9 +53,7 @@ expression E1, E2; identifier m; position p; @@ -( - (sizeof(*E1)@p + (E2 * sizeof(*E1->m))) -) + sizeof(*E1)@p + (E2 * sizeof(*E1->m)) @script:python depends on org@ p << r.p; diff --git a/scripts/config b/scripts/config index ea475c07de28..32428ea909c2 100755 --- a/scripts/config +++ b/scripts/config @@ -38,7 +38,7 @@ commands: options: --file config-file .config file to change (default .config) - --keep-case|-k Keep next symbols' case (dont' upper-case it) + --keep-case|-k Keep next symbols' case (don't upper-case it) $myname doesn't check the validity of the .config file. This is done at next make time. @@ -71,48 +71,45 @@ txt_append() { local anchor="$1" local insert="$2" local infile="$3" - local tmpfile="$infile.swp" # sed append cmd: 'a\' + newline + text + newline cmd="$(printf "a\\%b$insert" "\n")" - sed -e "/$anchor/$cmd" "$infile" >"$tmpfile" - # replace original file with the edited one - mv "$tmpfile" "$infile" + # We don't really need a backup file, but in-place editing with backup + # skipped is not portable due to different implementations parsing + # arguments in incompatible manners. + # Create a backup file anyway to ensure portability. The file will be + # deleted on exit. + sed -E -i.swp -e "/$anchor/$cmd" "$infile" + SED_EDITED=1 } txt_subst() { local before="$1" local after="$2" local infile="$3" - local tmpfile="$infile.swp" - sed -e "s$SED_DELIM$before$SED_DELIM$after$SED_DELIM" "$infile" >"$tmpfile" - # replace original file with the edited one - mv "$tmpfile" "$infile" + sed -E -i.swp -e "s$SED_DELIM$before$SED_DELIM$after$SED_DELIM" "$infile" + SED_EDITED=1 } txt_delete() { local text="$1" local infile="$2" - local tmpfile="$infile.swp" - sed -e "/$text/d" "$infile" >"$tmpfile" - # replace original file with the edited one - mv "$tmpfile" "$infile" + sed -E -i.swp -e "/$text/d" "$infile" + SED_EDITED=1 } set_var() { local name=$1 new=$2 before=$3 - name_re="^($name=|# $name is not set)" + name_re="^($name=.*|# $name is not set)" before_re="^($before=|# $before is not set)" if test -n "$before" && grep -Eq "$before_re" "$FN"; then - txt_append "^$before=" "$new" "$FN" - txt_append "^# $before is not set" "$new" "$FN" + txt_append "$before_re" "$new" "$FN" elif grep -Eq "$name_re" "$FN"; then - txt_subst "^$name=.*" "$new" "$FN" - txt_subst "^# $name is not set" "$new" "$FN" + txt_subst "$name_re" "$new" "$FN" else echo "$new" >>"$FN" fi @@ -121,10 +118,17 @@ set_var() { undef_var() { local name=$1 - txt_delete "^$name=" "$FN" - txt_delete "^# $name is not set" "$FN" + txt_delete "^($name=|# $name is not set)" "$FN" } +SED_EDITED=0 +on_exit() { + if [ "$SED_EDITED" -ge 1 ]; then + rm -f "$FN.swp" + fi +} +trap on_exit EXIT + FN=.config CMDS=() while [[ $# -gt 0 ]]; do diff --git a/scripts/context-analysis-suppression.txt b/scripts/context-analysis-suppression.txt index fd8951d06706..1c51b6153f08 100644 --- a/scripts/context-analysis-suppression.txt +++ b/scripts/context-analysis-suppression.txt @@ -24,6 +24,7 @@ src:*include/linux/mutex*.h=emit src:*include/linux/rcupdate.h=emit src:*include/linux/refcount.h=emit src:*include/linux/rhashtable.h=emit +src:*include/linux/rtmutex*.h=emit src:*include/linux/rwlock*.h=emit src:*include/linux/rwsem.h=emit src:*include/linux/sched*=emit diff --git a/scripts/crypto/gen-aead-testvecs.py b/scripts/crypto/gen-aead-testvecs.py new file mode 100755 index 000000000000..d77d646f75b5 --- /dev/null +++ b/scripts/crypto/gen-aead-testvecs.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Script that generates known-good data used in the AEAD tests. +# +# Requires that python-cryptography be installed. +# +# Copyright 2026 Google LLC + +import hashlib +import sys +import cryptography.hazmat.primitives.ciphers.aead + + +# Deterministically generate 'length' random bytes. +def rand_bytes(length): + seed = length + out = [] + for _ in range(length): + seed = (seed * 25214903917 + 11) % 2**48 + out.append((seed >> 16) % 256) + return bytes(out) + + +# Deterministically generate many different AEAD inputs using exactly the same +# method that the test uses; encrypt them using an independent implementation of +# the algorithm; compute the checksum of all the resulting (ciphertext, authtag) +# pairs concatenated to each other; and print the checksum as a C struct. +def gen_monte_carlo_checksum(alg): + blake2s = hashlib.blake2s() + for data_len in range(1025): + ad_len = data_len % 293 + pt = rand_bytes(data_len) + ad = rand_bytes(ad_len) + if alg == "aes-ccm": + key_len = [16, 24, 32][data_len % 3] + key = rand_bytes(key_len) + nonce = rand_bytes([7, 8, 9, 10, 11, 12, 13][data_len % 7]) + tag_len = [4, 6, 8, 10, 12, 14, 16][data_len % 7] + ccm = cryptography.hazmat.primitives.ciphers.aead.AESCCM( + key, tag_length=tag_len + ) + ct_and_tag = ccm.encrypt(nonce, pt, ad) + elif alg == "aes-gcm": + key_len = [16, 24, 32][data_len % 3] + key = rand_bytes(key_len) + nonce = rand_bytes(12) + tag_len = [4, 8, 12, 13, 14, 15, 16][data_len % 7] + gcm = cryptography.hazmat.primitives.ciphers.aead.AESGCM(key) + # python-cryptography supports only 16-byte GCM tags. However, in + # GCM, shorter tags are simply truncated. Do that below. + ct_and_tag = gcm.encrypt(nonce, pt, ad)[: data_len + tag_len] + + blake2s.update(ct_and_tag) + + name = f"{alg.replace('-', '_')}_monte_carlo_checksum" + value = blake2s.digest() + print(f"static const u8 {name}[BLAKE2S_HASH_SIZE] = {{") + for i in range(0, len(value), 11): + line = "\t" + "".join(f"0x{b:02x}, " for b in value[i : i + 11]) + print(f"{line.rstrip()}") + print("};") + + +if len(sys.argv) != 2 or sys.argv[1] not in ("aes-ccm", "aes-gcm"): + sys.stderr.write("Usage: gen-aead-testvecs.py [aes-ccm|aes-gcm]\n") + sys.exit(1) + +gen_monte_carlo_checksum(sys.argv[1]) diff --git a/scripts/crypto/gen-fips-testvecs.py b/scripts/crypto/gen-fips-testvecs.py index db873f88619a..b8c8a78cb8a8 100755 --- a/scripts/crypto/gen-fips-testvecs.py +++ b/scripts/crypto/gen-fips-testvecs.py @@ -1,36 +1,148 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-or-later # -# Script that generates lib/crypto/fips.h +# Script that generates lib/crypto/fips-aes.h and lib/crypto/fips-sha.h +# +# Requires that python-cryptography be installed. # # Copyright 2025 Google LLC +import cryptography.hazmat.primitives.ciphers +import cryptography.hazmat.primitives.ciphers.aead +import cryptography.hazmat.primitives.cmac import hashlib import hmac -fips_test_data = b"fips test data\0\0" -fips_test_key = b"fips test key\0\0\0" -def print_static_u8_array_definition(name, value): - print('') - print(f'static const u8 {name}[] __initconst __maybe_unused = {{') +def print_static_u8_array_definition(file, name, value): + print("", file=file) + print(f"static const u8 {name}[] __initconst __maybe_unused = {{", file=file) for i in range(0, len(value), 8): - line = '\t' + ''.join(f'0x{b:02x}, ' for b in value[i:i+8]) - print(f'{line.rstrip()}') - print('};') + line = "\t" + "".join(f"0x{b:02x}, " for b in value[i : i + 8]) + print(f"{line.rstrip()}", file=file) + print("};", file=file) + + +def print_header(file): + print("/* SPDX-License-Identifier: GPL-2.0-or-later */", file=file) + print("/* This file was generated by: gen-fips-testvecs.py */", file=file) + print("/* clang-format off */", file=file) + print("", file=file) + print("#include <linux/fips.h>", file=file) + + +def gen_aes_test_data(file): + fips_test_data = b"fips test data\0\0" + fips_test_ad = b"fips test ad\0\0\0\0" + fips_test_iv = b"fips test iv\0\0\0\0" + fips_test_key = b"fips test key\0\0\0" + fips_test_xts_key = b"key1" + (b"\0" * 12) + b"key2" + (b"\0" * 12) + + print_header(file) + print_static_u8_array_definition(file, "fips_test_data", fips_test_data) + print_static_u8_array_definition(file, "fips_test_ad", fips_test_ad) + print_static_u8_array_definition(file, "fips_test_iv", fips_test_iv) + print_static_u8_array_definition(file, "fips_test_key", fips_test_key) + print_static_u8_array_definition(file, "fips_test_xts_key", fips_test_xts_key) + + aes = cryptography.hazmat.primitives.ciphers.algorithms.AES(fips_test_key) + + # AES-CMAC + aes_cmac = cryptography.hazmat.primitives.cmac.CMAC(aes) + aes_cmac.update(fips_test_data) + print_static_u8_array_definition( + file, "fips_test_aes_cmac_value", aes_cmac.finalize() + ) + + # AES-ECB + cipher = cryptography.hazmat.primitives.ciphers.Cipher( + aes, cryptography.hazmat.primitives.ciphers.modes.ECB() + ) + encryptor = cipher.encryptor() + ctext = encryptor.update(fips_test_data) + encryptor.finalize() + print_static_u8_array_definition(file, "fips_test_aes_ecb_ctext", ctext) + + # AES-CBC + cipher = cryptography.hazmat.primitives.ciphers.Cipher( + aes, cryptography.hazmat.primitives.ciphers.modes.CBC(fips_test_iv) + ) + encryptor = cipher.encryptor() + ctext = encryptor.update(fips_test_data) + encryptor.finalize() + print_static_u8_array_definition(file, "fips_test_aes_cbc_ctext", ctext) + + # AES-CBC-CTS + cipher = cryptography.hazmat.primitives.ciphers.Cipher( + aes, cryptography.hazmat.primitives.ciphers.modes.CBC(fips_test_iv) + ) + encryptor = cipher.encryptor() + ctext = encryptor.update(fips_test_data * 2) + encryptor.finalize() + ctext = ctext[16:32] + ctext[0:16] + print_static_u8_array_definition(file, "fips_test_aes_cbc_cts_ctext", ctext) + + # AES-CTR + cipher = cryptography.hazmat.primitives.ciphers.Cipher( + aes, cryptography.hazmat.primitives.ciphers.modes.CTR(fips_test_iv) + ) + encryptor = cipher.encryptor() + ctext = encryptor.update(fips_test_data) + encryptor.finalize() + print_static_u8_array_definition(file, "fips_test_aes_ctr_ctext", ctext) + + # AES-XTS + cipher = cryptography.hazmat.primitives.ciphers.Cipher( + cryptography.hazmat.primitives.ciphers.algorithms.AES(fips_test_xts_key), + cryptography.hazmat.primitives.ciphers.modes.XTS(fips_test_iv), + ) + encryptor = cipher.encryptor() + ctext = encryptor.update(fips_test_data) + encryptor.finalize() + print_static_u8_array_definition(file, "fips_test_aes_xts_ctext", ctext) + + # AES-GCM + cipher = cryptography.hazmat.primitives.ciphers.aead.AESGCM(fips_test_key) + ct_and_tag = cipher.encrypt( + nonce=fips_test_iv[:12], data=fips_test_data, associated_data=fips_test_ad + ) + print_static_u8_array_definition( + file, "fips_test_aes_gcm_ctext_and_tag", ct_and_tag + ) + + # AES-CCM + cipher = cryptography.hazmat.primitives.ciphers.aead.AESCCM( + fips_test_key, tag_length=16 + ) + ct_and_tag = cipher.encrypt( + nonce=fips_test_iv[:13], data=fips_test_data, associated_data=fips_test_ad + ) + print_static_u8_array_definition( + file, "fips_test_aes_ccm_ctext_and_tag", ct_and_tag + ) + + +def gen_sha_test_data(file): + fips_test_data = b"fips test data\0\0" + fips_test_key = b"fips test key\0\0\0" + + print_header(file) + print_static_u8_array_definition(file, "fips_test_data", fips_test_data) + print_static_u8_array_definition(file, "fips_test_key", fips_test_key) + + for alg in "sha1", "sha256", "sha512": + ctx = hmac.new(fips_test_key, digestmod=alg) + ctx.update(fips_test_data) + print_static_u8_array_definition( + file, f"fips_test_hmac_{alg}_value", ctx.digest() + ) -print('/* SPDX-License-Identifier: GPL-2.0-or-later */') -print(f'/* This file was generated by: gen-fips-testvecs.py */') -print() -print('#include <linux/fips.h>') + print_static_u8_array_definition( + file, "fips_test_sha3_256_value", hashlib.sha3_256(fips_test_data).digest() + ) -print_static_u8_array_definition("fips_test_data", fips_test_data) -print_static_u8_array_definition("fips_test_key", fips_test_key) -for alg in 'sha1', 'sha256', 'sha512': - ctx = hmac.new(fips_test_key, digestmod=alg) - ctx.update(fips_test_data) - print_static_u8_array_definition(f'fips_test_hmac_{alg}_value', ctx.digest()) +filename = "lib/crypto/fips-aes.h" +with open(filename, "w") as file: + print(f"Generating {filename}") + gen_aes_test_data(file) -print_static_u8_array_definition(f'fips_test_sha3_256_value', - hashlib.sha3_256(fips_test_data).digest()) +filename = "lib/crypto/fips-sha.h" +with open(filename, "w") as file: + print(f"Generating {filename}") + gen_sha_test_data(file) diff --git a/scripts/crypto/gen-hash-testvecs.py b/scripts/crypto/gen-hash-testvecs.py index 8eeb650fcada..f356f87e1c77 100755 --- a/scripts/crypto/gen-hash-testvecs.py +++ b/scripts/crypto/gen-hash-testvecs.py @@ -3,8 +3,12 @@ # # Script that generates test vectors for the given hash function. # +# Requires that python-cryptography be installed. +# # Copyright 2025 Google LLC +import cryptography.hazmat.primitives.ciphers +import cryptography.hazmat.primitives.cmac import hashlib import hmac import sys @@ -24,6 +28,20 @@ def rand_bytes(length): out.append((seed >> 16) % 256) return bytes(out) +AES_256_KEY_SIZE = 32 + +# AES-CMAC. Just wraps the implementation from python-cryptography. +class AesCmac: + def __init__(self, key): + aes = cryptography.hazmat.primitives.ciphers.algorithms.AES(key) + self.cmac = cryptography.hazmat.primitives.cmac.CMAC(aes) + + def update(self, data): + self.cmac.update(data) + + def digest(self): + return self.cmac.finalize() + POLY1305_KEY_SIZE = 32 # A straightforward, unoptimized implementation of Poly1305. @@ -50,6 +68,52 @@ class Poly1305: m = (self.h + self.s) % 2**128 return m.to_bytes(16, byteorder='little') +GHASH_POLY = sum((1 << i) for i in [128, 7, 2, 1, 0]) +GHASH_BLOCK_SIZE = 16 + +# A straightforward, unoptimized implementation of GHASH. +class Ghash: + + @staticmethod + def reflect_bits_in_bytes(v): + res = 0 + for offs in range(0, 128, 8): + for bit in range(8): + if (v & (1 << (offs + bit))) != 0: + res ^= 1 << (offs + 7 - bit) + return res + + @staticmethod + def bytes_to_poly(data): + return Ghash.reflect_bits_in_bytes(int.from_bytes(data, byteorder='little')) + + @staticmethod + def poly_to_bytes(poly): + return Ghash.reflect_bits_in_bytes(poly).to_bytes(16, byteorder='little') + + def __init__(self, key): + assert len(key) == 16 + self.h = Ghash.bytes_to_poly(key) + self.acc = 0 + + # Note: this supports partial blocks only at the end. + def update(self, data): + for i in range(0, len(data), 16): + # acc += block + self.acc ^= Ghash.bytes_to_poly(data[i:i+16]) + # acc = (acc * h) mod GHASH_POLY + product = 0 + for j in range(127, -1, -1): + if (self.h & (1 << j)) != 0: + product ^= self.acc << j + if (product & (1 << (128 + j))) != 0: + product ^= GHASH_POLY << j + self.acc = product + return self + + def digest(self): + return Ghash.poly_to_bytes(self.acc) + POLYVAL_POLY = sum((1 << i) for i in [128, 127, 126, 121, 0]) POLYVAL_BLOCK_SIZE = 16 @@ -80,9 +144,14 @@ class Polyval: return self.acc.to_bytes(16, byteorder='little') def hash_init(alg): + # The keyed hash functions are assigned a fixed random key here, to present + # them as unkeyed hash functions. This allows all the test cases for + # unkeyed hash functions to work on them. + if alg == 'aes-cmac': + return AesCmac(rand_bytes(AES_256_KEY_SIZE)) + if alg == 'ghash': + return Ghash(rand_bytes(GHASH_BLOCK_SIZE)) if alg == 'poly1305': - # Use a fixed random key here, to present Poly1305 as an unkeyed hash. - # This allows all the test cases for unkeyed hashes to work on Poly1305. return Poly1305(rand_bytes(POLY1305_KEY_SIZE)) if alg == 'polyval': return Polyval(rand_bytes(POLYVAL_BLOCK_SIZE)) @@ -116,6 +185,8 @@ def print_c_struct_u8_array_field(name, value): print('\t\t},') def alg_digest_size_const(alg): + if alg == 'aes-cmac': + return 'AES_BLOCK_SIZE' if alg.startswith('blake2'): return f'{alg.upper()}_HASH_SIZE' return f"{alg.upper().replace('-', '_')}_DIGEST_SIZE" @@ -234,6 +305,15 @@ def gen_additional_poly1305_testvecs(): 'poly1305_allones_macofmacs[POLY1305_DIGEST_SIZE]', Poly1305(key).update(data).digest()) +def gen_additional_ghash_testvecs(): + key = b'\xff' * GHASH_BLOCK_SIZE + hashes = b'' + for data_len in range(0, 4097, 16): + hashes += Ghash(key).update(b'\xff' * data_len).digest() + print_static_u8_array_definition( + 'ghash_allones_hashofhashes[GHASH_DIGEST_SIZE]', + Ghash(key).update(hashes).digest()) + def gen_additional_polyval_testvecs(): key = b'\xff' * POLYVAL_BLOCK_SIZE hashes = b'' @@ -245,16 +325,22 @@ def gen_additional_polyval_testvecs(): if len(sys.argv) != 2: sys.stderr.write('Usage: gen-hash-testvecs.py ALGORITHM\n') - sys.stderr.write('ALGORITHM may be any supported by Python hashlib; or poly1305, polyval, or sha3.\n') + sys.stderr.write('ALGORITHM may be any supported by Python hashlib;\n') + sys.stderr.write(' or aes-cmac, ghash, nh, poly1305, polyval, or sha3.\n') sys.stderr.write('Example: gen-hash-testvecs.py sha512\n') sys.exit(1) alg = sys.argv[1] print('/* SPDX-License-Identifier: GPL-2.0-or-later */') print(f'/* This file was generated by: {sys.argv[0]} {" ".join(sys.argv[1:])} */') -if alg.startswith('blake2'): +if alg == 'aes-cmac': + gen_unkeyed_testvecs(alg) +elif alg.startswith('blake2'): gen_unkeyed_testvecs(alg) gen_additional_blake2_testvecs(alg) +elif alg == 'ghash': + gen_unkeyed_testvecs(alg) + gen_additional_ghash_testvecs() elif alg == 'nh': gen_nh_testvecs() elif alg == 'poly1305': @@ -270,6 +356,9 @@ elif alg == 'sha3': print() print('/* SHAKE test vectors */') gen_additional_sha3_testvecs() +elif alg == 'sm3': + gen_unkeyed_testvecs(alg) + # Kernel doesn't implement HMAC-SM3 library functions yet. else: gen_unkeyed_testvecs(alg) gen_hmac_testvecs(alg) diff --git a/scripts/decode_stacktrace.sh b/scripts/decode_stacktrace.sh index 8d01b741de62..39d60d477bf3 100755 --- a/scripts/decode_stacktrace.sh +++ b/scripts/decode_stacktrace.sh @@ -5,9 +5,11 @@ usage() { echo "Usage:" - echo " $0 -r <release>" - echo " $0 [<vmlinux> [<base_path>|auto [<modules_path>]]]" + echo " $0 [-R] -r <release>" + echo " $0 [-R] [<vmlinux> [<base_path>|auto [<modules_path>]]]" echo " $0 -h" + echo "Options:" + echo " -R: decode return address instead of caller address." } # Try to find a Rust demangler @@ -33,11 +35,17 @@ fi READELF=${UTIL_PREFIX}readelf${UTIL_SUFFIX} ADDR2LINE=${UTIL_PREFIX}addr2line${UTIL_SUFFIX} NM=${UTIL_PREFIX}nm${UTIL_SUFFIX} +decode_retaddr=false if [[ $1 == "-h" ]] ; then usage exit 0 -elif [[ $1 == "-r" ]] ; then +elif [[ $1 == "-R" ]] ; then + decode_retaddr=true + shift 1 +fi + +if [[ $1 == "-r" ]] ; then vmlinux="" basepath="auto" modpath="" @@ -176,13 +184,23 @@ parse_symbol() { # Let's start doing the math to get the exact address into the # symbol. First, strip out the symbol total length. local expr=${symbol%/*} + # Also parse the offset from symbol. + local offset=${expr#*+} + offset=$((offset)) # Now, replace the symbol name with the base address we found # before. expr=${expr/$name/0x$base_addr} # Evaluate it to find the actual address - expr=$((expr)) + # The stack trace shows the return address, which is the next + # instruction after the actual call, so as long as it's in the same + # symbol, subtract one from that to point the call instruction. + if [[ $decode_retaddr == false && $offset != 0 ]]; then + expr=$((expr-1)) + else + expr=$((expr)) + fi local address=$(printf "%x\n" "$expr") # Pass it to addr2line to get filename and line number diff --git a/scripts/decodecode b/scripts/decodecode index 6364218b2178..01d25dc110de 100755 --- a/scripts/decodecode +++ b/scripts/decodecode @@ -12,7 +12,6 @@ faultlinenum=1 cleanup() { rm -f $T $T.s $T.o $T.oo $T.aa $T.dis - exit 1 } die() { @@ -49,7 +48,7 @@ done if [ -z "$code" ]; then rm $T - exit + die "Code line not found" fi echo $code diff --git a/scripts/dtc/checks.c b/scripts/dtc/checks.c index 45d0213f3bf3..946c1429e0f1 100644 --- a/scripts/dtc/checks.c +++ b/scripts/dtc/checks.c @@ -324,7 +324,7 @@ ERROR(node_name_chars, check_node_name_chars, NODECHARS); static void check_node_name_chars_strict(struct check *c, struct dt_info *dti, struct node *node) { - int n = strspn(node->name, c->data); + size_t n = strspn(node->name, c->data); if (n < node->basenamelen) FAIL(c, dti, node, "Character '%c' not recommended in node name", diff --git a/scripts/dtc/dt-check-style b/scripts/dtc/dt-check-style new file mode 100755 index 000000000000..96deffc0d8a7 --- /dev/null +++ b/scripts/dtc/dt-check-style @@ -0,0 +1,1205 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# +# Check DTS coding style on YAML binding examples and on +# .dts/.dtsi/.dtso source files. Enforces rules from +# Documentation/devicetree/bindings/dts-coding-style.rst. +# +# Two modes: +# --mode=relaxed (default) +# Only rules that produce zero warnings on the current tree. +# Suitable for dt_binding_check. +# --mode=strict +# All rules. Required for new submissions. +# +# Two input types (auto-detected by file extension): +# *.yaml -- DT binding; check each example block +# *.dts/*.dtsi/*.dtso -- DTS source; whole file is one block +# +# Rules are declared in a registry (see RULES below); each rule is +# tagged with the lowest mode that runs it. Promoting a rule from +# 'strict' to 'relaxed' is a one-line change. + +import argparse +import re +import sys +from enum import Enum, auto + +import ruamel.yaml + + +# --------------------------------------------------------------------------- +# Line classification +# --------------------------------------------------------------------------- + +class LineType(Enum): + BLANK = auto() + COMMENT = auto() # // ... or /* ... */ on one line + COMMENT_START = auto() # /* without closing */ + COMMENT_BODY = auto() # inside a multi-line comment + COMMENT_END = auto() # closing */ + PREPROCESSOR = auto() # #include / #define / #ifdef / ... + NODE_OPEN = auto() # something { (with optional label/name/addr) + NODE_CLOSE = auto() # }; + PROPERTY = auto() # name = value; or name; + CONTINUATION = auto() # continuation of a multi-line property + + +re_cpp_directive = re.compile( + r'^#\s*(include|define|undef|ifdef|ifndef|if|else|elif|endif|' + r'pragma|error|warning)\b') + +re_dtc_directive = re.compile( + r'^/(dts-v1|include)/') + +# label: name@addr { -- label and addr optional; name can be "/" +# Per the DT spec a node name may start with a digit (e.g. 1wire@...). +# The address part is captured loosely (any non-space, non-brace run) so +# malformed addresses (e.g. memory@0x1000) still reach +# check_unit_address_format() instead of silently bypassing the check. +re_node_header = re.compile( + r'^(?:([a-zA-Z_][a-zA-Z0-9_]*):\s*)?' + r'([a-zA-Z0-9][a-zA-Z0-9,._+-]*|/)' + r'(?:@([^\s{]+))?' + r'\s*\{$') + +re_ref_node = re.compile( + r'^&([a-zA-Z_][a-zA-Z0-9_]*)\s*\{$') + + +def is_preprocessor(stripped): + """Tell C preprocessor directives apart from DTS '#'-prefixed props.""" + if re_cpp_directive.match(stripped) is not None: + return True + if re_dtc_directive.match(stripped) is not None: + return True + return False + + +class DtsLine: + __slots__ = ('lineno', 'raw', 'linetype', 'indent_str', 'stripped', + 'prop_name', 'continuations', + 'node_name', 'node_addr', 'label', 'ref_name', 'depth', + 'closures') + + def __init__(self, lineno, raw, linetype, depth, indent_str, stripped): + self.lineno = lineno # 1-based within the block + self.raw = raw + self.linetype = linetype + self.indent_str = indent_str # leading whitespace as-is + self.depth = depth + self.stripped = stripped + self.prop_name = None + self.continuations = [] + self.node_name = None + self.node_addr = None + self.label = None + self.ref_name = None + self.closures = 1 # count of '}' on a NODE_CLOSE line + + +def _split_code(text): + """Return (code, opens_block) for a leading-stripped line: the + code portion with // and /* */ comments removed (string literals + kept verbatim), and whether a /* */ block comment is left open. + The code portion is right-stripped so the endswith() checks in + classify_lines see code only, not a trailing comment or blanks.""" + out = [] + i = 0 + n = len(text) + while i < n: + c = text[i] + if c == '"': + j = i + 1 + while j < n: + if text[j] == '\\': + j += 2 + continue + if text[j] == '"': + j += 1 + break + j += 1 + out.append(text[i:j]) + i = j + continue + if c == '/' and i + 1 < n and text[i + 1] == '/': + break + if c == '/' and i + 1 < n and text[i + 1] == '*': + end = text.find('*/', i + 2) + if end < 0: + return (''.join(out).rstrip(), True) + i = end + 2 + continue + out.append(c) + i += 1 + return (''.join(out).rstrip(), False) + + +re_only_closures = re.compile(r'(?:\}\s*;?\s*)+$') + + +def classify_lines(text): + """Return a list of DtsLine. Tracks { } depth and groups + continuation lines onto their leading PROPERTY line.""" + out = [] + in_block_comment = False + in_cpp_macro = False + prev_complete = True + depth = 0 + + # Split preserving the indent string verbatim + re_lead = re.compile(r'^([ \t]*)(.*)$') + + for i, raw in enumerate(text.split('\n'), start=1): + m = re_lead.match(raw) + indent_str = m.group(1) + stripped = m.group(2) + + # Continuation of a multi-line C preprocessor directive: the + # previous PREPROCESSOR line ended with a '\\' line splice, so + # this line is part of the same macro. Treat it as + # PREPROCESSOR until the splice chain ends (no trailing '\\' + # or a blank line). + if in_cpp_macro: + dl = DtsLine(i, raw, LineType.PREPROCESSOR, + depth, indent_str, stripped) + out.append(dl) + in_cpp_macro = (bool(stripped) and + stripped.rstrip().endswith('\\')) + continue + + if not stripped: + dl = DtsLine(i, raw, LineType.BLANK, depth, '', '') + out.append(dl) + continue + + if in_block_comment: + ltype = (LineType.COMMENT_END if '*/' in stripped + else LineType.COMMENT_BODY) + if ltype == LineType.COMMENT_END: + in_block_comment = False + dl = DtsLine(i, raw, ltype, depth, indent_str, stripped) + out.append(dl) + continue + + if (stripped.startswith('#') or stripped.startswith('/')) and is_preprocessor(stripped): + dl = DtsLine(i, raw, LineType.PREPROCESSOR, depth, + indent_str, stripped) + out.append(dl) + prev_complete = True + in_cpp_macro = stripped.rstrip().endswith('\\') + continue + + # Strip comments first so all later structural checks see code + # only. An unclosed /* sets in_block_comment for the next line. + code, opens_block = _split_code(stripped) + if opens_block: + in_block_comment = True + + # Pure-comment line: nothing left after stripping. Classify as + # COMMENT_START (carries to next line) or COMMENT, and skip the + # structural classification entirely. + if not code: + ltype = LineType.COMMENT_START if opens_block else LineType.COMMENT + dl = DtsLine(i, raw, ltype, depth, indent_str, stripped) + out.append(dl) + continue + + if not prev_complete: + dl = DtsLine(i, raw, LineType.CONTINUATION, depth, indent_str, code) + out.append(dl) + prev_complete = (code.endswith(';') or + code.endswith('{') or + code.endswith('};')) + continue + + # NODE_CLOSE: the canonical form is "}" or "};" alone. A line + # that is nothing but closures (e.g. "}; };") is still treated + # as NODE_CLOSE for depth tracking, but the multi-closure case + # is flagged separately by check_node_close_alone via + # dl.closures. + if re_only_closures.match(code): + closures = code.count('}') + depth = max(depth - closures, 0) + dl = DtsLine(i, raw, LineType.NODE_CLOSE, depth, indent_str, code) + dl.closures = closures + out.append(dl) + prev_complete = True + continue + + if code.endswith('{'): + dl = DtsLine(i, raw, LineType.NODE_OPEN, depth, indent_str, code) + parse_node_header(dl) + out.append(dl) + depth += 1 + prev_complete = True + continue + + # Property (or first line of a multi-line property). + dl = DtsLine(i, raw, LineType.PROPERTY, depth, indent_str, code) + parse_property_name(dl) + out.append(dl) + prev_complete = code.endswith(';') + + # Group continuation lines onto their leading PROPERTY. + last_prop = None + grouped = [] + for dl in out: + if dl.linetype == LineType.CONTINUATION and last_prop is not None: + last_prop.continuations.append(dl) + continue + if dl.linetype == LineType.PROPERTY: + last_prop = dl + elif dl.linetype != LineType.BLANK and \ + dl.linetype not in (LineType.COMMENT, LineType.COMMENT_BODY, + LineType.COMMENT_END, + LineType.COMMENT_START): + last_prop = None + grouped.append(dl) + return grouped + + +def parse_node_header(dl): + m = re_node_header.match(dl.stripped) + if m: + dl.label = m.group(1) + dl.node_name = m.group(2) + dl.node_addr = m.group(3) + return + m = re_ref_node.match(dl.stripped) + if m: + dl.ref_name = m.group(1) + + +def parse_property_name(dl): + m = re.match(r'^([a-zA-Z0-9#][a-zA-Z0-9,._+#-]*)\s*[=;]', dl.stripped) + if m: + dl.prop_name = m.group(1) + + +def collect_labels_and_refs(text): + """Return (defined_labels, referenced_labels) found anywhere outside + /* */ comments and string literals. Labels named fake_intc* (injected + by dt-extract-example) are skipped.""" + # Strip block comments first so labels inside them don't count + stripped = re.sub(r'/\*.*?\*/', '', text, flags=re.DOTALL) + # Strip line comments + stripped = re.sub(r'//[^\n]*', '', stripped) + # Strip string literals so words inside quotes (e.g. "Error: foo") + # are not picked up as label definitions or &-references. + stripped = re.sub(r'"(?:[^"\\]|\\.)*"', '""', stripped) + defined = set() + referenced = set() + # A label precedes a node header; the next non-space token may start + # with a letter (foo, &ref), a digit (1wire), or '/' (root node). + for m in re.finditer( + r'(?:^|[\s{])([a-zA-Z_][a-zA-Z0-9_]*):\s*[a-zA-Z0-9/&]', + stripped): + name = m.group(1) + if not name.startswith('fake_intc'): + defined.add(name) + for m in re.finditer(r'&([a-zA-Z_][a-zA-Z0-9_]*)', stripped): + referenced.add(m.group(1)) + return defined, referenced + + +# --------------------------------------------------------------------------- +# Rule registry +# --------------------------------------------------------------------------- + +class Ctx: + """Context passed to each rule check. Carries the parsed lines, + raw text, mode and kind.""" + + def __init__(self, lines, text, mode, kind): + self.lines = lines + self.text = text + self.mode = mode # 'relaxed' or 'strict' + if kind in DTS_FAMILY: + self.file_type = 'dts' + else: + self.file_type = 'yaml' + + +class Rule: + __slots__ = ('name', 'mode', 'description', 'check', 'applies_to') + + def __init__(self, name, mode, description, check, + applies_to=('yaml', 'dts', 'dtsi', 'dtso')): + self.name = name + self.mode = mode # 'relaxed' or 'strict' + self.description = description + self.check = check + self.applies_to = applies_to # input types this rule covers + + +# --- individual rule check functions -------------------------------------- + +def check_trailing_whitespace(ctx): + for dl in ctx.lines: + if dl.raw != dl.raw.rstrip(): + yield (dl.lineno, 'trailing whitespace') + + +def check_tab_in_yaml_example(ctx): + """Reject literal tabs in DTS lines when input is YAML. + + For YAML examples, indent and content must use spaces. Tabs inside + a #define value are tolerated (those are CPP macros, not DTS). + For .dts files, this rule does not apply -- tabs are required. + """ + if ctx.file_type != 'yaml': + return + for dl in ctx.lines: + if dl.linetype == LineType.PREPROCESSOR: + continue + if dl.linetype == LineType.BLANK: + continue + if '\t' in dl.raw: + yield (dl.lineno, 'tab character not allowed in DTS example') + + +def check_mixed_indent_chars(ctx): + """Indent must be all-tabs, except for aligning indentation (comments + or continued lines).""" + for dl in ctx.lines: + if not dl.indent_str: + continue + if dl.linetype == LineType.PREPROCESSOR: + continue + if re.search(r' \t', dl.indent_str): + yield (dl.lineno, 'mixed tabs and spaces in indent') + if dl.indent_str.count(' ') > 7: + yield (dl.lineno, 'too many space characters in indent (more than 7)') + for cont in dl.continuations: + if not cont.indent_str: + continue + if cont.linetype == LineType.PREPROCESSOR: + continue + if re.search(r' \t', cont.indent_str): + yield (cont.lineno, 'mixed tabs and spaces in indent') + + +def detect_indent_unit(ctx): + """Find the indent unit used at depth 1 in this block. + + Returns tuple of string (one of: ' ' (2 spaces), ' ' (4 spaces), + '\\t' (tab), or None if depth-1 is empty or ambiguous) and line number when + detection was made).""" + for dl in ctx.lines: + if dl.depth != 1: + continue + if dl.linetype in (LineType.BLANK, LineType.PREPROCESSOR): + continue + if dl.linetype in (LineType.COMMENT_BODY, LineType.COMMENT_END): + continue + if not dl.indent_str: + continue + if dl.indent_str == '\t': + return ('\t', dl.lineno) + if dl.indent_str == ' ': + return (' ', dl.lineno) + if dl.indent_str == ' ': + return (' ', dl.lineno) + # Anything else at depth 1 is non-canonical; flag elsewhere. + return (dl.indent_str, dl.lineno) + return (None, None) + + +def check_indent_unit_relaxed(ctx): + """YAML examples: 2 or 4 spaces. Never tabs or other widths.""" + (unit, lineno) = detect_indent_unit(ctx) + if unit is None: + return + if unit not in (' ', ' '): + yield (lineno, 'indent unit must be 2 or 4 spaces, got %r' % unit) + + +def check_indent_unit_dts(ctx): + """DTS files: 1 tab per level. Always required.""" + (unit, lineno) = detect_indent_unit(ctx) + if unit is None: + return + if unit != '\t': + yield (lineno, 'indent unit must be 1 tab in DTS, got %r' % unit) + + +def check_indent_unit_strict(ctx): + """YAML: must be exactly 4 spaces. DTS: 1 tab (same as relaxed).""" + (unit, lineno) = detect_indent_unit(ctx) + if unit is None: + return + if ctx.file_type == 'yaml': + if unit != ' ': + yield (lineno, 'indent unit must be 4 spaces in strict mode, ' + 'got %r' % unit) + + +def check_indent_consistent(ctx): + """All indented lines must be a multiple of the detected unit.""" + (unit, lineno) = detect_indent_unit(ctx) + if unit is None: + return + if ctx.file_type == 'yaml': + if unit not in (' ', ' '): + return # let check_indent_unit_* report this + else: + if unit != '\t': + return + + for dl in ctx.lines: + if dl.linetype in (LineType.BLANK, LineType.PREPROCESSOR): + continue + if dl.linetype == LineType.CONTINUATION: + continue # continuations align to <, not to indent unit + if dl.linetype in (LineType.COMMENT_BODY, LineType.COMMENT_END): + continue + if not dl.indent_str: + continue + # The indent must be 'unit' repeated dl.depth times, exactly. + # NODE_CLOSE lines have depth equal to the post-decrement value, + # which matches the indent expected. + expected = unit * dl.depth + if dl.indent_str != expected: + yield (dl.lineno, + 'indent mismatch (expected depth %d * %r)' % + (dl.depth, unit)) + + +def check_blank_lines(ctx): + """No two consecutive blank lines, no leading/trailing blank lines + in any node body.""" + lines = ctx.lines + # Consecutive blanks + for i in range(1, len(lines)): + if lines[i].linetype == LineType.BLANK and \ + lines[i - 1].linetype == LineType.BLANK: + yield (lines[i].lineno, 'consecutive blank lines') + # Blank right after { or right before } + for i, dl in enumerate(lines): + if dl.linetype != LineType.BLANK: + continue + prev = lines[i - 1] if i > 0 else None + nxt = lines[i + 1] if i + 1 < len(lines) else None + if prev is not None and prev.linetype == LineType.NODE_OPEN: + yield (dl.lineno, 'blank line at start of node body') + if nxt is not None and nxt.linetype == LineType.NODE_CLOSE: + yield (dl.lineno, 'blank line at end of node body') + + +def _walk_bodies(lines): + """Yield lists of immediate-child NODE_OPEN lines for each node body + in the input. Skips ref-nodes (&label) since those don't have an + intrinsic ordering.""" + body_stack = [[]] + for dl in lines: + if dl.linetype == LineType.NODE_OPEN: + body_stack[-1].append(dl) + body_stack.append([]) + continue + if dl.linetype == LineType.NODE_CLOSE: + if len(body_stack) <= 1: + # Unbalanced; ignore to avoid crashing on malformed input + continue + yield body_stack.pop() + continue + while body_stack: + yield body_stack.pop() + + +def _natural_sort_key(s): + """Split a string into a tuple of (kind, value) pairs that compares + numeric runs as ints, so 'foo10' sorts after 'foo2'.""" + parts = [] + for part in re.split(r'(\d+)', s): + if part.isdigit(): + parts.append((0, int(part))) + else: + parts.append((1, part)) + return tuple(parts) + + +def check_child_address_order(ctx): + """Addressed siblings (foo@N) must appear in ascending address + order within their parent node body.""" + for children in _walk_bodies(ctx.lines): + addressed = [] + for c in children: + if c.node_addr is None: + continue + try: + parts = tuple(int(p, 16) for p in c.node_addr.split(',')) + except ValueError: + continue + addressed.append((parts, c)) + for i in range(1, len(addressed)): + if addressed[i][0] < addressed[i - 1][0]: + dl = addressed[i][1] + yield (dl.lineno, + 'child node @%s out of address order' % + dl.node_addr) + + +def check_child_name_order(ctx): + """Unaddressed siblings must appear in natural-sort order by node + name within their parent node body. Addressed children are scoped + by check_child_address_order; reference nodes (&label { ... }) and + the root node are skipped.""" + for children in _walk_bodies(ctx.lines): + unaddressed = [] + for c in children: + if c.node_addr is not None: + continue + if c.node_name in (None, '/'): + continue + if c.ref_name is not None: + continue + unaddressed.append((_natural_sort_key(c.node_name), c)) + for i in range(1, len(unaddressed)): + if unaddressed[i][0] < unaddressed[i - 1][0]: + dl = unaddressed[i][1] + yield (dl.lineno, + 'child node %r out of name order' % dl.node_name) + + +def _property_bucket(name): + """Return the canonical bucket index for a property: + 0 device_type + 1 compatible + 2 reg / reg-names + 3 ranges + 4 standard properties (no vendor comma in #-stripped name) + 5 vendor-specific properties + 6 status + Plus a sub-key inside the bucket for fixed slots (device_type, compatible, + reg, reg-names, ranges, status). 'standard' and 'vendor' return None for + the sub-key, signalling that the within-bucket key is computed by + the pairing rules.""" + stripped = name.lstrip('#') + if name == 'device_type': + return (0, 0) + if name == 'compatible': + return (1, 0) + if name == 'reg': + return (2, 0) + if name == 'reg-names': + return (2, 1) + if name == 'ranges': + return (3, 0) + if name == 'status': + return (6, 0) + return (5 if ',' in stripped else 4, None) + + +# Declarative pairing rules: each is a callable +# (name, all_names) -> anchor_name_or_None +# If a rule returns an anchor, the property sorts immediately after the +# anchor. Rules are tried in order; the first match wins. If none +# matches, the within-bucket key falls back to natural sort by the +# #-stripped name. + +def _pair_pinctrl_names(name, all_names): + """pinctrl-names follows the highest pinctrl-N in the same node.""" + if name != 'pinctrl-names': + return None + cands = [n for n in all_names if re.match(r'^pinctrl-\d+$', n)] + if not cands: + return None + return max(cands, key=_natural_sort_key) + + +def _pair_x_names(name, all_names): + """Generic <x>-names follows its owning property. The owner is + usually plural (clocks/clock-names, dmas/dma-names, + resets/reset-names) but occasionally singular (reg/reg-names is + handled by the fixed slot above; this rule catches anything else).""" + if not name.endswith('-names'): + return None + base = name[:-len('-names')] + # Try plural and singular forms. + if (base + 's') in all_names: + return base + 's' + if base in all_names: + return base + return None + + +PAIRING_RULES = (_pair_pinctrl_names, _pair_x_names) + + +def _property_sort_key(name, all_names): + """Sort key for a property among its node-body siblings. + + Format: (bucket, within_key, tiebreak). 'within_key' for + standard/vendor buckets follows pairing rules: a property paired + with anchor X sorts as if it were X with a higher tiebreak.""" + bucket, fixed_sub = _property_bucket(name) + if fixed_sub is not None: + return (bucket, (), fixed_sub) + + for rule in PAIRING_RULES: + anchor = rule(name, all_names) + if anchor is not None: + return (bucket, _natural_sort_key(anchor.lstrip('#')), 1) + + return (bucket, _natural_sort_key(name.lstrip('#')), 0) + + +def check_property_order(ctx): + """Properties within a node body must appear in canonical order: + compatible, reg(/reg-names), ranges, then the standard group, then + the vendor-specific group, then status. Inside the standard and + vendor groups, pairing rules apply (e.g. <x>-names follows <x>); + everything else falls back to natural sort by the #-stripped name.""" + lines = ctx.lines + for i, dl in enumerate(lines): + if dl.linetype != LineType.NODE_OPEN: + continue + body_depth = dl.depth + 1 + props = [] + for j in range(i + 1, len(lines)): + d = lines[j] + if d.linetype == LineType.NODE_CLOSE and \ + d.depth == body_depth - 1: + break + if d.linetype == LineType.PROPERTY and d.depth == body_depth \ + and d.prop_name is not None: + props.append(d) + if len(props) < 2: + continue + all_names = [p.prop_name for p in props] + keyed = [(p, _property_sort_key(p.prop_name, all_names)) + for p in props] + for k in range(1, len(keyed)): + if keyed[k][1] < keyed[k - 1][1]: + p = keyed[k][0] + prev = keyed[k - 1][0] + yield (p.lineno, + 'property %r out of canonical order ' + '(should sort before %r)' % + (p.prop_name, prev.prop_name)) + + +def _strip_strings_and_comments(text): + """Remove string literals and /* */ + // comments from a single + line, replacing them with empty strings. Used so syntactic checks + (whitespace, hex case, etc.) don't false-positive on contents of + quoted strings or comments. An unclosed /* on the line is treated + as a comment running to end of line.""" + text = re.sub(r'"(?:[^"\\]|\\.)*"', '""', text) + text = re.sub(r'/\*.*?\*/', '', text) + text = re.sub(r'/\*.*$', '', text) + text = re.sub(r'//.*$', '', text) + return text + + +def check_required_blank_lines(ctx): + """A blank line must precede each child node and the 'status' + property within a node body, except when these are the first + substantive item in the body.""" + lines = ctx.lines + for i, open_dl in enumerate(lines): + if open_dl.linetype != LineType.NODE_OPEN: + continue + body_depth = open_dl.depth + 1 + prev_substantive = None + between_blanks = 0 + depth_inside = 0 + for j in range(i + 1, len(lines)): + d = lines[j] + if d.linetype == LineType.NODE_CLOSE and \ + d.depth == body_depth - 1 and depth_inside == 0: + break + # Track depth inside nested children so we only look at + # immediate-body items. + if d.linetype == LineType.NODE_OPEN and \ + d.depth >= body_depth and depth_inside > 0: + depth_inside += 1 + continue + if d.linetype == LineType.NODE_CLOSE and depth_inside > 0: + depth_inside -= 1 + continue + if depth_inside > 0: + continue + if d.linetype == LineType.BLANK: + if prev_substantive is not None: + between_blanks += 1 + continue + if d.linetype in (LineType.COMMENT, LineType.COMMENT_START, + LineType.COMMENT_BODY, LineType.COMMENT_END, + LineType.PREPROCESSOR): + continue + if d.linetype == LineType.CONTINUATION: + continue + + needs_blank = False + if d.linetype == LineType.NODE_OPEN: + needs_blank = True + depth_inside = 1 # entered the child body + elif d.linetype == LineType.PROPERTY and d.prop_name == 'status': + needs_blank = True + + if needs_blank and prev_substantive is not None and \ + between_blanks == 0: + if d.linetype == LineType.NODE_OPEN: + yield (d.lineno, + 'child node must be preceded by a blank line') + else: + yield (d.lineno, + '"status" must be preceded by a blank line') + + prev_substantive = d + between_blanks = 0 + + +def check_hex_case(ctx): + """Hex literals (0xN) must use lowercase digits and prefix.""" + for dl in ctx.lines: + if dl.linetype in (LineType.BLANK, LineType.COMMENT, + LineType.COMMENT_START, LineType.COMMENT_BODY, + LineType.COMMENT_END, LineType.PREPROCESSOR): + continue + text = _strip_strings_and_comments(dl.raw) + for m in re.finditer(r'\b0[xX][0-9a-fA-F]+\b', text): + lit = m.group(0) + if any(c.isupper() for c in lit[2:]) or lit[1] == 'X': + yield (dl.lineno, + 'hex literal %r must be lowercase' % lit) + + +def check_unit_address_format(ctx): + """Unit addresses must be lowercase hex without leading zeros and + without a '0x' prefix. For multi-cell addresses (comma-separated), + each part is checked independently. A single '0' is permitted + (canonical zero).""" + for dl in ctx.lines: + if dl.linetype != LineType.NODE_OPEN: + continue + if dl.node_addr is None: + continue + addr = dl.node_addr + for part in addr.split(','): + if part[:2] in ('0x', '0X'): + yield (dl.lineno, + 'unit address %r must not have a "0x" prefix' % + addr) + break + if not re.match(r'^[0-9a-fA-F]+$', part): + yield (dl.lineno, + 'unit address %r is not valid hex' % addr) + break + if any(c in 'ABCDEF' for c in part): + yield (dl.lineno, + 'unit address %r must be lowercase hex' % addr) + break + if len(part) > 1 and part.startswith('0'): + yield (dl.lineno, + 'unit address %r has leading zeros' % addr) + break + + +def check_value_whitespace(ctx): + """A <...> cell list must have no whitespace directly after '<' + or directly before '>'. Continuation lines are joined onto the + property so a <...> split across lines is checked too; a '<' or + '>' at a line break is glued straight to the neighbouring value, + so the break itself is not counted as padding. Outside strings + and comments only.""" + for dl in ctx.lines: + if dl.linetype != LineType.PROPERTY: + continue + segs = [_strip_strings_and_comments(dl.raw).strip()] + for cont in dl.continuations: + segs.append(_strip_strings_and_comments(cont.stripped).strip()) + text = '' + for s in segs: + if not s: + continue + if not text or text.endswith('<') or s.startswith('>'): + text += s + else: + text += ' ' + s + for m in re.finditer(r'<([^<>]*)>', text): + content = m.group(1) + if content and content != content.strip(): + yield (dl.lineno, 'extra whitespace inside <...>') + break + + +def check_node_close_alone(ctx): + """The closing '};' of a node must be on its own line. The + classifier accepts a canonical "}" or "};" as NODE_CLOSE; a line + that is all closures (e.g. "}; };") is still NODE_CLOSE for depth + tracking but is flagged here via dl.closures. Any other line that + still contains '};' (in code, not in strings or comments) is + mixing a node close with something else.""" + for dl in ctx.lines: + if dl.linetype == LineType.NODE_CLOSE: + if dl.closures > 1: + yield (dl.lineno, + 'closing brace must be on its own line') + continue + if dl.linetype in (LineType.BLANK, LineType.COMMENT, + LineType.COMMENT_START, LineType.COMMENT_BODY, + LineType.COMMENT_END, LineType.PREPROCESSOR): + continue + text = _strip_strings_and_comments(dl.raw) + if '};' in text: + yield (dl.lineno, + 'closing brace must be on its own line') + + +def _display_col(text): + """Visual column width of text, with tabs expanded to the next + 8-column stop, matching how printf and most editors render a + line and the kernel-wide line length convention.""" + col = 0 + for ch in text: + if ch == '\t': + col = (col // 8 + 1) * 8 + else: + col += 1 + return col + + +def check_line_length(ctx): + """Lines must not exceed 80 columns; tabs count as 8 (see + _display_col).""" + for dl in ctx.lines: + if dl.linetype == LineType.BLANK: + continue + cols = _display_col(dl.raw) + if cols > 80: + yield (dl.lineno, + 'line exceeds 80 columns (%d)' % cols) + + +def check_continuation_alignment(ctx): + """A multi-line property's continuation lines must align their + first non-whitespace character to the display column of the first + '<' or '"' after the '=' in the leading line. Display columns are + used so tab-indented .dts files (where a continuation aligns with + tabs plus spaces) are compared correctly.""" + for dl in ctx.lines: + if dl.linetype != LineType.PROPERTY: + continue + if not dl.continuations: + continue + eq = dl.raw.find('=') + if eq < 0: + continue + # First '<' or '"' after '=' + rest = dl.raw[eq + 1:] + m = re.search(r'[<"]', rest) + if not m: + continue + target_col = _display_col(dl.raw[:eq + 1 + m.start()]) + for cont in dl.continuations: + if _display_col(cont.indent_str) != target_col: + yield (cont.lineno, + 'continuation should align to column %d ' + '(under "<" or \\")' % (target_col + 1)) + + +def check_unclosed_block_comment(ctx): + """Every /* must have a matching */ in the same block. Catches both + a comment opened on its own line (COMMENT_START) and a tail comment + opened on a PROPERTY or other code line (where in_block_comment is + set by _split_code so the next line becomes COMMENT_BODY without a + preceding COMMENT_START).""" + open_lineno = None + for dl in ctx.lines: + if dl.linetype == LineType.COMMENT_START: + open_lineno = dl.lineno + elif dl.linetype == LineType.COMMENT_END: + open_lineno = None + elif dl.linetype == LineType.COMMENT_BODY and open_lineno is None: + # Block was opened by a /* tail on a code line; report at + # the first orphan body line since the originating line is + # already classified as something else. + open_lineno = dl.lineno + if open_lineno is not None: + yield (open_lineno, 'unclosed /* block comment') + + +def check_unused_labels(ctx): + """Labels defined but never referenced are clutter.""" + defined, referenced = collect_labels_and_refs(ctx.text) + for label in sorted(defined - referenced): + # Find the line where this label is defined for line-number + # reporting. + m = re.search(r'(?m)^.*\b' + re.escape(label) + r'\s*:', ctx.text) + lineno = ctx.text[:m.start()].count('\n') + 1 if m else 1 + yield (lineno, 'label %r defined but never &-referenced' % label) + + +# --- registry -------------------------------------------------------------- + +RULES = [ + # 'relaxed' is the default; rules in this group must produce zero + # output on a clean kernel tree (post the small prep-cleanup + # commit at the head of this series). + Rule('trailing-whitespace', 'relaxed', + 'no trailing whitespace on any line', + check_trailing_whitespace), + Rule('tab-in-yaml', 'relaxed', + 'YAML (also DTS examples) may not contain tab characters', + check_tab_in_yaml_example, applies_to=('yaml',)), + Rule('mixed-indent-chars', 'relaxed', + 'indent must not mix tabs and spaces', + check_mixed_indent_chars, applies_to=('dts', 'dtsi', 'dtso')), + Rule('unclosed-block-comment', 'relaxed', + 'every /* block comment must close with */', + check_unclosed_block_comment), + + # DTS files always use tabs; this is not negotiable per kernel + # coding style (.dts files are real source). Relaxed mode. + Rule('indent-unit-dts', 'relaxed', + 'DTS files: 1 tab per nesting level', + check_indent_unit_dts, + applies_to=('dts', 'dtsi', 'dtso')), + + # 'strict' rules are opt-in (e.g. for new submissions via + # checkpatch.pl in a follow-up series). They flag many existing + # files and can be promoted to relaxed once those are cleaned up. + Rule('indent-unit', 'strict', + 'YAML: 2 or 4 spaces per level', + check_indent_unit_relaxed, applies_to=('yaml',)), + Rule('indent-unit-strict', 'strict', + 'YAML: must be 4 spaces per level', + check_indent_unit_strict, applies_to=('yaml',)), + Rule('indent-consistent', 'strict', + 'every line indented at depth * unit', + check_indent_consistent), + Rule('blank-lines', 'strict', + 'no consecutive blanks; no blanks at node body edges', + check_blank_lines), + Rule('child-address-order', 'strict', + 'addressed siblings must be in ascending address order', + check_child_address_order), + Rule('child-name-order', 'strict', + 'unaddressed siblings must be in natural-sort name order', + check_child_name_order), + Rule('property-order', 'strict', + 'canonical bucket + pairing + natural-sort order of properties', + check_property_order), + Rule('required-blank-lines', 'strict', + 'blank line before child nodes and before "status"', + check_required_blank_lines), + Rule('hex-case', 'strict', + 'hex literals must be lowercase', + check_hex_case), + Rule('unit-address-format', 'strict', + 'unit addresses must be lowercase hex without leading zeros', + check_unit_address_format), + Rule('value-whitespace', 'strict', + 'no whitespace directly inside <...> brackets', + check_value_whitespace), + Rule('node-close-alone', 'strict', + 'closing brace must be on its own line', + check_node_close_alone), + Rule('line-length', 'strict', + 'lines must not exceed 80 columns', + check_line_length), + Rule('continuation-alignment', 'strict', + 'multi-line property continuations align under "<" or "\\""', + check_continuation_alignment), + Rule('unused-labels', 'strict', + 'every label must be &-referenced in the same example/file ' + '(skipped for .dtsi/.dtso since labels there are exported)', + check_unused_labels, applies_to=('yaml', 'dts')), +] + + +def select_rules(mode, input_kind): + """Return rules that apply to the given mode and input type.""" + rank = {'relaxed': 0, 'strict': 1} + out = [] + for r in RULES: + if rank[r.mode] > rank[mode]: + continue + if input_kind not in r.applies_to: + continue + out.append(r) + return out + + +# --------------------------------------------------------------------------- +# Block runner +# --------------------------------------------------------------------------- + +def check_block(text, mode, input_type): + """Run all selected rules on a single block of DTS text. Returns a + list of (lineno, rule_name, message) tuples.""" + lines = classify_lines(text) + ctx = Ctx(lines, text, mode, input_type) + rules = select_rules(mode, input_type) + findings = [] + for r in rules: + for lineno, msg in r.check(ctx): + findings.append((lineno, r.name, msg)) + findings.sort(key=lambda t: (t[0], t[1])) + return findings + + +# --------------------------------------------------------------------------- +# Input drivers (YAML examples vs raw DTS) +# --------------------------------------------------------------------------- + +def _yaml_loader(): + return ruamel.yaml.YAML() + + +def iter_yaml_examples(filepath): + """Yield (example_text, base_lineno_in_file, example_index) tuples.""" + yaml = _yaml_loader() + try: + with open(filepath, encoding='utf-8') as f: + data = yaml.load(f) + except Exception as e: + print('%s: error loading YAML: %s' % (filepath, e), + file=sys.stderr) + return + if not isinstance(data, dict) or 'examples' not in data: + return + examples = data['examples'] + if not hasattr(examples, '__iter__'): + return + for i, ex in enumerate(examples): + if not isinstance(ex, str): + continue + try: + base = examples.lc.item(i)[0] + 2 + except Exception: + base = 1 + yield (str(ex), base, i) + + +def iter_dts_file(filepath): + """Treat the whole file as a single block.""" + try: + with open(filepath, encoding='utf-8') as f: + text = f.read() + except Exception as e: + print('%s: error reading: %s' % (filepath, e), file=sys.stderr) + return + yield (text, 1, None) + + +# --------------------------------------------------------------------------- +# Top-level processing +# --------------------------------------------------------------------------- + +def input_kind(filepath): + p = filepath.lower() + if p.endswith('.yaml') or p.endswith('.yml'): + return 'yaml' + if p.endswith('.dts'): + return 'dts' + if p.endswith('.dtsi'): + return 'dtsi' + if p.endswith('.dtso'): + return 'dtso' + return None + + +# All input types that use tab indentation and follow DTS coding style. +DTS_FAMILY = ('dts', 'dtsi', 'dtso') + + +def collect_findings(filepath, mode): + """Return a (lines, count) pair for filepath. lines is a list of + formatted output strings; count is the number of findings.""" + kind = input_kind(filepath) + if kind == 'yaml': + iterator = iter_yaml_examples(filepath) + elif kind in DTS_FAMILY: + iterator = iter_dts_file(filepath) + else: + return (['%s: unknown file type, skipping' % filepath], 0) + + out = [] + for text, base, idx in iterator: + for lineno, rule, msg in check_block(text, mode, kind): + abs_line = base + lineno - 1 + ex_tag = '' if idx is None else ' example %d' % idx + out.append('%s:%d:%s [%s] %s' % + (filepath, abs_line, ex_tag, rule, msg)) + return (out, len(out)) + + +# Worker entry point for ProcessPoolExecutor.map(). Top-level so it is +# picklable on every platform. +def _worker(args): + filepath, mode = args + return collect_findings(filepath, mode) + + +def main(): + import os + ap = argparse.ArgumentParser( + description='Check DTS coding style on YAML examples and ' + '.dts/.dtsi/.dtso files.', + fromfile_prefix_chars='@') + ap.add_argument('--mode', choices=('relaxed', 'strict'), + default='relaxed', + help='which rule set to apply (default: relaxed)') + ap.add_argument('-j', '--jobs', type=int, default=0, + metavar='N', + help='run N workers in parallel (default: respect ' + 'the make jobserver via $PARALLELISM, otherwise ' + 'os.cpu_count(); use 1 to disable multiprocessing)') + ap.add_argument('--list-rules', action='store_true', + help='print all rules with their mode and exit') + ap.add_argument('files', nargs='*', metavar='file', + help='YAML binding files or .dts/.dtsi/.dtso files; ' + 'use @argfile to read paths from a file') + args = ap.parse_args() + + if args.list_rules: + for r in RULES: + applies = ','.join(r.applies_to) + print('%-22s %-7s [%s] %s' % + (r.name, r.mode, applies, r.description)) + return 0 + + if not args.files: + ap.error('no input files') + + if args.jobs > 0: + jobs = args.jobs + else: + # When invoked under scripts/jobserver-exec, $PARALLELISM + # holds the slot count make has reserved for us; this lets + # `make -j N dt_binding_check` constrain our worker pool to N. + try: + jobs = int(os.environ['PARALLELISM']) + except (KeyError, ValueError): + jobs = os.cpu_count() or 1 + # Single-process path: keep import surface small for tests and + # easy debugging. + if jobs == 1 or len(args.files) == 1: + total = 0 + for f in args.files: + lines, n = collect_findings(f, args.mode) + for line in lines: + print(line, file=sys.stderr) + total += n + return 1 if total else 0 + + # Multi-process path. ex.map preserves input order so output is + # deterministic across runs. + from concurrent.futures import ProcessPoolExecutor + total = 0 + work = [(f, args.mode) for f in args.files] + chunk = max(1, len(work) // (jobs * 8)) if work else 1 + with ProcessPoolExecutor(max_workers=jobs) as ex: + for lines, n in ex.map(_worker, work, chunksize=chunk): + for line in lines: + print(line, file=sys.stderr) + total += n + return 1 if total else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/dtc/dt-style-selftest/bad/dts-child-name-order.dts b/scripts/dtc/dt-style-selftest/bad/dts-child-name-order.dts new file mode 100644 index 000000000000..54be9f07b680 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/dts-child-name-order.dts @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + pmu { + compatible = "example,pmu"; + + /* Include labels to be sure they do not affect sorting */ + foo: foo { + label = "foo"; + }; + + label_bar: bar { + label = "bar"; + }; + }; + + memory@a0000000 { + device_type = "memory"; + reg = <0x0 0xa0000000 0x0 0x0>; + }; + + pmu-2 { + compatible = "example,pmu"; + + /* Just reference labels to avoid strict warnings */ + example,foo = <&foo>, <&label_bar>; + }; +}; diff --git a/scripts/dtc/dt-style-selftest/bad/dts-extend-node-child-name-order.dts b/scripts/dtc/dt-style-selftest/bad/dts-extend-node-child-name-order.dts new file mode 100644 index 000000000000..296c58a545d5 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/dts-extend-node-child-name-order.dts @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + pmu { + compatible = "example,pmu"; + + /* Just reference labels to avoid strict warnings */ + example,foo = <&foo>, <&label_bar>; + }; +}; + +&pmu { + /* Include labels to be sure they do not affect sorting */ + foo: foo { + label = "foo"; + }; + + label_bar: bar { + label = "bar"; + }; +}; diff --git a/scripts/dtc/dt-style-selftest/bad/dts-extend-node-digit-node-order.dts b/scripts/dtc/dt-style-selftest/bad/dts-extend-node-digit-node-order.dts new file mode 100644 index 000000000000..813900e3393c --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/dts-extend-node-digit-node-order.dts @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + soc: soc@0 { + compatible = "simple-bus"; + ranges = <0 0 0 0xc0000000>; + + #address-cells = <1>; + #size-cells = <1>; + }; +}; + +&soc { + serial@20000 { + compatible = "example,serial"; + reg = <0x20000 0x1000>; + }; + + interrupt-controller@10000 { + compatible = "example,intc"; + reg = <0x10000 0x1000>; + interrupts = <1 2 3>; + }; + + serial@30000 { + compatible = "example,serial"; + reg = <0x30000 0x1000>; + }; +}; diff --git a/scripts/dtc/dt-style-selftest/bad/dts-mixed-indent.dts b/scripts/dtc/dt-style-selftest/bad/dts-mixed-indent.dts new file mode 100644 index 000000000000..cd3de04ec5a9 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/dts-mixed-indent.dts @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +/* Test fixture: a .dts using wrong indent. */ + +/dts-v1/; + +/ { + compatible = "example,test-board"; + #address-cells = <1>; + #size-cells = <1>; + + soc@0 { + compatible = "simple-bus"; + ranges = <0 0 0 0xc0000000>; + + clocks = <1>, + <2>, + <3>, + <4>; + resets = <5>; + }; +}; diff --git a/scripts/dtc/dt-style-selftest/bad/dts-property-order.dts b/scripts/dtc/dt-style-selftest/bad/dts-property-order.dts new file mode 100644 index 000000000000..f31abb6ceae4 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/dts-property-order.dts @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +/* + * Test fixture: Incorrect property order + */ + +/dts-v1/; + +/ { + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu@0 { + reg = <0x0 0x0>; + compatible = "arm,cortex-a57"; + device_type = "cpu"; + enable-method = "psci"; + }; + }; + + pmu { + compatible = "example,pmu"; + + status = "disabled"; + dma-coherent; + }; + + soc@0 { + ranges = <0 0 0 0xc0000000>; + compatible = "simple-bus"; + + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller@10000 { + reg = <0x10000 0x1000>; + interrupts = <1 2 3>, + <4 5 6>, + <7 8 9>; + compatible = "example,intc"; + }; + }; +}; diff --git a/scripts/dtc/dt-style-selftest/bad/dts-spaces.dts b/scripts/dtc/dt-style-selftest/bad/dts-spaces.dts new file mode 100644 index 000000000000..9dad22adce51 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/dts-spaces.dts @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +/* + * Test fixture: a .dts using space indent (must use tabs). + */ + +/dts-v1/; + +/ { + compatible = "example,test-board"; + #address-cells = <1>; + #size-cells = <1>; +}; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-child-addr-order.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-child-addr-order.yaml new file mode 100644 index 000000000000..3df56e69a1ff --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-child-addr-order.yaml @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-child-order.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with addressed children out of order + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-child-order + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + bus@10000 { + compatible = "simple-bus"; + reg = <0x10000 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + + device@200 { + compatible = "example,test-child-order"; + reg = <0x200 0x10>; + }; + + device@100 { + compatible = "example,test-child-order"; + reg = <0x100 0x10>; + }; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-child-name-order.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-child-name-order.yaml new file mode 100644 index 000000000000..35d85e5573c2 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-child-name-order.yaml @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-child-name-order.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with unaddressed children out of name order + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-child-name-order + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + bus@10000 { + compatible = "simple-bus"; + reg = <0x10000 0x1000>; + + foo { + label = "foo"; + }; + + bar { + label = "bar"; + }; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-cont-align.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-cont-align.yaml new file mode 100644 index 000000000000..92778540b056 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-cont-align.yaml @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-cont-align.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with mis-aligned multi-line property + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-cont-align + reg: + maxItems: 2 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + foo@1000 { + compatible = "example,test-cont-align"; + reg = <0x1000 0x100>, + <0x2000 0x100>; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-digit-node-order.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-digit-node-order.yaml new file mode 100644 index 000000000000..44a9d25e5ba0 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-digit-node-order.yaml @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-digit-node-order.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with digit-leading nodes out of address order + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-digit-node-order + +required: + - compatible + +additionalProperties: false + +examples: + - | + bus@0 { + compatible = "simple-bus"; + #address-cells = <1>; + #size-cells = <1>; + + 3d-engine@20 { + compatible = "example,3d-engine"; + reg = <0x20 0x4>; + }; + + 1wire@10 { + compatible = "example,1wire"; + reg = <0x10 0x4>; + }; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-hex-case.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-hex-case.yaml new file mode 100644 index 000000000000..b26d1bf58de9 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-hex-case.yaml @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-hex-case.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with uppercase hex literals + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-hex-case + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + foo@1000 { + compatible = "example,test-hex-case"; + reg = <0xABCD 0x100>; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-indent-strict.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-indent-strict.yaml new file mode 100644 index 000000000000..bee4cf118d73 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-indent-strict.yaml @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-indent-strict.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture using 2-space indent (rejected by strict mode) + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-indent-strict + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + device@1000 { + compatible = "example,test-indent-strict"; + reg = <0x1000 0x100>; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-label-in-string.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-label-in-string.yaml new file mode 100644 index 000000000000..ba512869b702 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-label-in-string.yaml @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-label-in-string.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture where a label is only "referenced" inside a string + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-label-in-string + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + foo: device@1000 { + compatible = "example,test-label-in-string"; + reg = <0x1000 0x100>; + info = "see &foo for details"; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-line-length.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-line-length.yaml new file mode 100644 index 000000000000..64427bf1c385 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-line-length.yaml @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-line-length.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture exceeding 80 columns + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-line-length + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + foo@1000 { + compatible = "example,test-line-length-this-is-a-very-long-name-indeed-yeah"; + reg = <0x1000 0x100>; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-mixed-indent.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-mixed-indent.yaml new file mode 100644 index 000000000000..5401d1a423a1 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-mixed-indent.yaml @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-mixed-indent.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture mixing tabs and spaces in indent + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-mixed + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + device@1000 { + compatible = "example,test-mixed"; + reg = <0x1000 0x100>; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-multi-close.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-multi-close.yaml new file mode 100644 index 000000000000..4d9fa27b50a2 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-multi-close.yaml @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-multi-close.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with two closing braces on one line + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-multi-close + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + bus@10000 { + compatible = "simple-bus"; + reg = <0x10000 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + + device@100 { + compatible = "example,test-multi-close"; + reg = <0x100 0x10>; + }; }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-node-close.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-node-close.yaml new file mode 100644 index 000000000000..e107659fd9e8 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-node-close.yaml @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-node-close.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with closing brace not on its own line + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-node-close + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + bus@10000 { + compatible = "simple-bus"; + reg = <0x10000 0x1000>; + + empty {}; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-prop-order-device-type.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-prop-order-device-type.yaml new file mode 100644 index 000000000000..e2c69e9ff452 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-prop-order-device-type.yaml @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-prop-order.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with device_type + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-prop-order-device-type + reg: + maxItems: 1 + device_type: true + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + device@1000 { + compatible = "example,test-prop-order-device-type"; + device_type = "cpu"; + reg = <0x1000 0x100>; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-prop-order.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-prop-order.yaml new file mode 100644 index 000000000000..75582a3d2f6e --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-prop-order.yaml @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-prop-order.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with reg before compatible + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-prop-order + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + device@1000 { + reg = <0x1000 0x100>; + compatible = "example,test-prop-order"; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-prop-pairing.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-prop-pairing.yaml new file mode 100644 index 000000000000..767ab21c39f3 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-prop-pairing.yaml @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-prop-pairing.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture exercising <x>-names and pinctrl-names pairing + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-prop-pairing + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + foo@1000 { + compatible = "example,test-prop-pairing"; + reg = <0x1000 0x100>; + clock-names = "bus"; + clocks = <&clk 0>; + pinctrl-names = "default"; + pinctrl-0 = <&p0>; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-required-blank.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-required-blank.yaml new file mode 100644 index 000000000000..8bb53240cffa --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-required-blank.yaml @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-required-blank.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture missing required blank lines + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-required-blank + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + bus@10000 { + compatible = "simple-bus"; + reg = <0x10000 0x1000>; + status = "okay"; + child@100 { + reg = <0x100>; + }; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-tab.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-tab.yaml new file mode 100644 index 000000000000..487d07ff8cb6 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-tab.yaml @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-tab.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with a tab in a DTS line + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-tab + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + device@1000 { + compatible = "example,test-tab"; + reg = <0x1000 0x100>; /* registers */ + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-trailing-comment.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-trailing-comment.yaml new file mode 100644 index 000000000000..2368ada8106f --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-trailing-comment.yaml @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-trailing-comment.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with properties out of order behind trailing comments + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-trailing-comment + +required: + - compatible + +additionalProperties: false + +examples: + - | + foo@0 { /* the device node */ + reg = <0x0 0x4>; /* registers */ + compatible = "example,test-trailing-comment"; // misplaced + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-trailing-ws.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-trailing-ws.yaml new file mode 100644 index 000000000000..5c4b4bd833c5 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-trailing-ws.yaml @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-trailing.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with trailing whitespace + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-trailing + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + device@1000 { + compatible = "example,test-trailing"; + reg = <0x1000 0x100>; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-unclosed-comment.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-unclosed-comment.yaml new file mode 100644 index 000000000000..63c1c08712a5 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-unclosed-comment.yaml @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-unclosed-comment.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with an unclosed /* block comment + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-unclosed-comment + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + /* this comment never closes + device@1000 { + compatible = "example,test-unclosed-comment"; + reg = <0x1000 0x100>; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-unit-addr-prefix.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-unit-addr-prefix.yaml new file mode 100644 index 000000000000..9b3fe508c5fd --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-unit-addr-prefix.yaml @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-unit-addr-prefix.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with 0x-prefixed unit address + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-unit-addr-prefix + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + memory@0x1000 { + compatible = "example,test-unit-addr-prefix"; + reg = <0x1000 0x100>; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-unit-addr.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-unit-addr.yaml new file mode 100644 index 000000000000..93705cd45410 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-unit-addr.yaml @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-unit-addr.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with malformed unit address + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-unit-addr + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + foo@01000 { + compatible = "example,test-unit-addr"; + reg = <0x1000 0x100>; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-unused-label.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-unused-label.yaml new file mode 100644 index 000000000000..28d7176cbf08 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-unused-label.yaml @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-unused-label.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with an unused label + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-unused-label + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + dev: device@1000 { + compatible = "example,test-unused-label"; + reg = <0x1000 0x100>; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-value-ws-multiline.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-value-ws-multiline.yaml new file mode 100644 index 000000000000..504bf0931c27 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-value-ws-multiline.yaml @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-value-ws-multiline.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with extra whitespace in a multi-line cell array + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-value-ws-multiline + +required: + - compatible + +additionalProperties: false + +examples: + - | + foo@0 { + compatible = "example,test-value-ws-multiline"; + reg = < 0x0 0x4 + 0x8 0xc>; + }; diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-value-ws.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-value-ws.yaml new file mode 100644 index 000000000000..342ab9f399f1 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/bad/yaml-value-ws.yaml @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-bad-value-ws.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture with extra whitespace inside <...> + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-value-ws + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + foo@1000 { + compatible = "example,test-value-ws"; + reg = < 0x1000 0x100 >; + }; diff --git a/scripts/dtc/dt-style-selftest/expected/dts-child-name-order.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-child-name-order.dts.txt new file mode 100644 index 000000000000..e2eea0862102 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/dts-child-name-order.dts.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/dts-child-name-order.dts:16: [child-name-order] child node 'bar' out of name order diff --git a/scripts/dtc/dt-style-selftest/expected/dts-extend-node-child-name-order.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-extend-node-child-name-order.dts.txt new file mode 100644 index 000000000000..b6e74e6f923e --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/dts-extend-node-child-name-order.dts.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/dts-extend-node-child-name-order.dts:22: [child-name-order] child node 'bar' out of name order diff --git a/scripts/dtc/dt-style-selftest/expected/dts-extend-node-digit-node-order.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-extend-node-digit-node-order.dts.txt new file mode 100644 index 000000000000..bf252e4d9815 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/dts-extend-node-digit-node-order.dts.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/dts-extend-node-digit-node-order.dts:23: [child-address-order] child node @10000 out of address order diff --git a/scripts/dtc/dt-style-selftest/expected/dts-mixed-indent.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-mixed-indent.dts.txt new file mode 100644 index 000000000000..93146cfb51c7 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/dts-mixed-indent.dts.txt @@ -0,0 +1,9 @@ +# mode=strict +bad/dts-mixed-indent.dts:11: [indent-consistent] indent mismatch (expected depth 1 * '\t') +bad/dts-mixed-indent.dts:11: [mixed-indent-chars] too many space characters in indent (more than 7) +bad/dts-mixed-indent.dts:12: [indent-consistent] indent mismatch (expected depth 2 * '\t') +bad/dts-mixed-indent.dts:12: [mixed-indent-chars] mixed tabs and spaces in indent +bad/dts-mixed-indent.dts:13: [indent-consistent] indent mismatch (expected depth 2 * '\t') +bad/dts-mixed-indent.dts:13: [mixed-indent-chars] mixed tabs and spaces in indent +bad/dts-mixed-indent.dts:16: [mixed-indent-chars] mixed tabs and spaces in indent +bad/dts-mixed-indent.dts:19: [indent-consistent] indent mismatch (expected depth 2 * '\t') diff --git a/scripts/dtc/dt-style-selftest/expected/dts-property-order.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-property-order.dts.txt new file mode 100644 index 000000000000..4bc21328625f --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/dts-property-order.dts.txt @@ -0,0 +1,6 @@ +# mode=strict +bad/dts-property-order.dts:15: [property-order] property 'compatible' out of canonical order (should sort before 'reg') +bad/dts-property-order.dts:16: [property-order] property 'device_type' out of canonical order (should sort before 'compatible') +bad/dts-property-order.dts:25: [property-order] property 'dma-coherent' out of canonical order (should sort before 'status') +bad/dts-property-order.dts:30: [property-order] property 'compatible' out of canonical order (should sort before 'ranges') +bad/dts-property-order.dts:40: [property-order] property 'compatible' out of canonical order (should sort before 'interrupts') diff --git a/scripts/dtc/dt-style-selftest/expected/dts-spaces.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-spaces.dts.txt new file mode 100644 index 000000000000..5afdb101dcee --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/dts-spaces.dts.txt @@ -0,0 +1,2 @@ +# mode=relaxed +bad/dts-spaces.dts:9: [indent-unit-dts] indent unit must be 1 tab in DTS, got ' ' diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-child-addr-order.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-child-addr-order.yaml.txt new file mode 100644 index 000000000000..f0db79a0018b --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-child-addr-order.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-child-addr-order.yaml:37: example 0 [child-address-order] child node @100 out of address order diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-child-name-order.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-child-name-order.yaml.txt new file mode 100644 index 000000000000..bb434b126191 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-child-name-order.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-child-name-order.yaml:34: example 0 [child-name-order] child node 'bar' out of name order diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-cont-align.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-cont-align.yaml.txt new file mode 100644 index 000000000000..b5576dd0f6b1 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-cont-align.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-cont-align.yaml:29: example 0 [continuation-alignment] continuation should align to column 11 (under "<" or \") diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-digit-node-order.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-digit-node-order.yaml.txt new file mode 100644 index 000000000000..6de275e2dcb5 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-digit-node-order.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-digit-node-order.yaml:33: example 0 [child-address-order] child node @10 out of address order diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-hex-case.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-hex-case.yaml.txt new file mode 100644 index 000000000000..6600f7cd1ba5 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-hex-case.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-hex-case.yaml:28: example 0 [hex-case] hex literal '0xABCD' must be lowercase diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-indent-strict.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-indent-strict.yaml.txt new file mode 100644 index 000000000000..c4f31deea4ca --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-indent-strict.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-indent-strict.yaml:27: example 0 [indent-unit-strict] indent unit must be 4 spaces in strict mode, got ' ' diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-label-in-string.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-label-in-string.yaml.txt new file mode 100644 index 000000000000..05da06f81364 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-label-in-string.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-label-in-string.yaml:26: example 0 [unused-labels] label 'foo' defined but never &-referenced diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-line-length.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-line-length.yaml.txt new file mode 100644 index 000000000000..89b36360caa4 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-line-length.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-line-length.yaml:27: example 0 [line-length] line exceeds 80 columns (81) diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-mixed-indent.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-mixed-indent.yaml.txt new file mode 100644 index 000000000000..bc3fc3cf00cc --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-mixed-indent.yaml.txt @@ -0,0 +1,2 @@ +# mode=relaxed +bad/yaml-mixed-indent.yaml:27: example 0 [tab-in-yaml] tab character not allowed in DTS example diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-multi-close.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-multi-close.yaml.txt new file mode 100644 index 000000000000..637d0f8ea103 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-multi-close.yaml.txt @@ -0,0 +1,3 @@ +# mode=strict +bad/yaml-multi-close.yaml:35: example 0 [indent-consistent] indent mismatch (expected depth 0 * ' ') +bad/yaml-multi-close.yaml:35: example 0 [node-close-alone] closing brace must be on its own line diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-node-close.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-node-close.yaml.txt new file mode 100644 index 000000000000..ee894747b5b9 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-node-close.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-node-close.yaml:30: example 0 [node-close-alone] closing brace must be on its own line diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-prop-order-device-type.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-prop-order-device-type.yaml.txt new file mode 100644 index 000000000000..9350e2b80f75 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-prop-order-device-type.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-prop-order-device-type.yaml:29: example 0 [property-order] property 'device_type' out of canonical order (should sort before 'compatible') diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-prop-order.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-prop-order.yaml.txt new file mode 100644 index 000000000000..578df7209170 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-prop-order.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-prop-order.yaml:28: example 0 [property-order] property 'compatible' out of canonical order (should sort before 'reg') diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-prop-pairing.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-prop-pairing.yaml.txt new file mode 100644 index 000000000000..e6e21349a939 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-prop-pairing.yaml.txt @@ -0,0 +1,3 @@ +# mode=strict +bad/yaml-prop-pairing.yaml:30: example 0 [property-order] property 'clocks' out of canonical order (should sort before 'clock-names') +bad/yaml-prop-pairing.yaml:32: example 0 [property-order] property 'pinctrl-0' out of canonical order (should sort before 'pinctrl-names') diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-required-blank.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-required-blank.yaml.txt new file mode 100644 index 000000000000..04ea0bacdcb9 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-required-blank.yaml.txt @@ -0,0 +1,3 @@ +# mode=strict +bad/yaml-required-blank.yaml:29: example 0 [required-blank-lines] "status" must be preceded by a blank line +bad/yaml-required-blank.yaml:30: example 0 [required-blank-lines] child node must be preceded by a blank line diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-tab.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-tab.yaml.txt new file mode 100644 index 000000000000..0bc67ae48dd8 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-tab.yaml.txt @@ -0,0 +1,2 @@ +# mode=relaxed +bad/yaml-tab.yaml:28: example 0 [tab-in-yaml] tab character not allowed in DTS example diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-trailing-comment.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-trailing-comment.yaml.txt new file mode 100644 index 000000000000..69dbb1d03239 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-trailing-comment.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-trailing-comment.yaml:25: example 0 [property-order] property 'compatible' out of canonical order (should sort before 'reg') diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-trailing-ws.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-trailing-ws.yaml.txt new file mode 100644 index 000000000000..cfdbc8476c73 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-trailing-ws.yaml.txt @@ -0,0 +1,2 @@ +# mode=relaxed +bad/yaml-trailing-ws.yaml:27: example 0 [trailing-whitespace] trailing whitespace diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-unclosed-comment.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-unclosed-comment.yaml.txt new file mode 100644 index 000000000000..9a30ee7145e6 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-unclosed-comment.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-unclosed-comment.yaml:26: example 0 [unclosed-block-comment] unclosed /* block comment diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-unit-addr-prefix.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-unit-addr-prefix.yaml.txt new file mode 100644 index 000000000000..8dec6c1176b5 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-unit-addr-prefix.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-unit-addr-prefix.yaml:26: example 0 [unit-address-format] unit address '0x1000' must not have a "0x" prefix diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-unit-addr.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-unit-addr.yaml.txt new file mode 100644 index 000000000000..b52f0ef20bee --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-unit-addr.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-unit-addr.yaml:26: example 0 [unit-address-format] unit address '01000' has leading zeros diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-unused-label.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-unused-label.yaml.txt new file mode 100644 index 000000000000..4f00202f0902 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-unused-label.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-unused-label.yaml:26: example 0 [unused-labels] label 'dev' defined but never &-referenced diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-value-ws-multiline.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-value-ws-multiline.yaml.txt new file mode 100644 index 000000000000..3df55b1762d0 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-value-ws-multiline.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-value-ws-multiline.yaml:25: example 0 [value-whitespace] extra whitespace inside <...> diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-value-ws.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-value-ws.yaml.txt new file mode 100644 index 000000000000..cbb5f88fe85f --- /dev/null +++ b/scripts/dtc/dt-style-selftest/expected/yaml-value-ws.yaml.txt @@ -0,0 +1,2 @@ +# mode=strict +bad/yaml-value-ws.yaml:28: example 0 [value-whitespace] extra whitespace inside <...> diff --git a/scripts/dtc/dt-style-selftest/good/dts-child-name-order.dts b/scripts/dtc/dt-style-selftest/good/dts-child-name-order.dts new file mode 100644 index 000000000000..af0adf5abf70 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/good/dts-child-name-order.dts @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + memory@a0000000 { + device_type = "memory"; + reg = <0x0 0xa0000000 0x0 0x0>; + }; + + pmu { + compatible = "example,pmu"; + + /* Include labels to be sure they do not affect sorting */ + label_bar: bar { + label = "bar"; + }; + + foo: foo { + label = "foo"; + }; + }; + + pmu-2 { + compatible = "example,pmu"; + + /* Just reference labels to avoid strict warnings */ + example,foo = <&foo>, <&label_bar>; + }; +}; diff --git a/scripts/dtc/dt-style-selftest/good/dts-cont-align.dts b/scripts/dtc/dt-style-selftest/good/dts-cont-align.dts new file mode 100644 index 000000000000..36fb4eefcd83 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/good/dts-cont-align.dts @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +/* + * Test fixture: tab-indented .dts with a tab-and-space aligned + * multi-line property. Continuation lines mix tabs for indent and + * spaces for alignment by design; that must not be flagged. + */ + +/dts-v1/; + +/ { + compatible = "example,test-board"; + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller@10000 { + compatible = "example,intc"; + reg = <0x10000 0x1000>; + interrupts = <1 2 3>, + <4 5 6>, + <7 8 9>; + pinmux = < + 0x01 + 0x02 + >; + }; +}; diff --git a/scripts/dtc/dt-style-selftest/good/dts-digit-node-order.dts b/scripts/dtc/dt-style-selftest/good/dts-digit-node-order.dts new file mode 100644 index 000000000000..cdf3f91ebe01 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/good/dts-digit-node-order.dts @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + /* TODO: uncomment when child-address-order is fixed for top-level */ + /* + memory@a0000000 { + device_type = "memory"; + reg = <0x0 0xa0000000 0x0 0x0>; + }; + */ + + pmu { + compatible = "example,pmu"; + }; + + soc@0 { + compatible = "simple-bus"; + ranges = <0 0 0 0xc0000000>; + + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller@10000 { + compatible = "example,intc"; + reg = <0x10000 0x1000>; + interrupts = <1 2 3>; + }; + + serial@20000 { + compatible = "example,serial"; + reg = <0x20000 0x1000>; + }; + + serial@30000 { + compatible = "example,serial"; + reg = <0x30000 0x1000>; + }; + }; +}; diff --git a/scripts/dtc/dt-style-selftest/good/dts-dtc.dts b/scripts/dtc/dt-style-selftest/good/dts-dtc.dts new file mode 100644 index 000000000000..a3207196214c --- /dev/null +++ b/scripts/dtc/dt-style-selftest/good/dts-dtc.dts @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +/* + * Test fixture: dtc directive + */ + +/dts-v1/; + +/include/ "soc.dtsi" +/include/"soc-other.dtsi" + +/ { + compatible = "example,test-board"; + #address-cells = <1>; + #size-cells = <1>; + + leds { + led-0 { + compatible = "example,led"; + }; + }; +}; diff --git a/scripts/dtc/dt-style-selftest/good/dts-extend-node-child-name-order.dts b/scripts/dtc/dt-style-selftest/good/dts-extend-node-child-name-order.dts new file mode 100644 index 000000000000..97d468051f25 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/good/dts-extend-node-child-name-order.dts @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + pmu { + compatible = "example,pmu"; + + /* Just reference labels to avoid strict warnings */ + example,foo = <&foo>, <&label_bar>; + }; +}; + +&pmu { + /* Include labels to be sure they do not affect sorting */ + label_bar: bar { + label = "bar"; + }; + + foo: foo { + label = "foo"; + }; +}; diff --git a/scripts/dtc/dt-style-selftest/good/dts-extend-node-digit-node-order.dts b/scripts/dtc/dt-style-selftest/good/dts-extend-node-digit-node-order.dts new file mode 100644 index 000000000000..faf49962faf2 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/good/dts-extend-node-digit-node-order.dts @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + soc: soc@0 { + compatible = "simple-bus"; + ranges = <0 0 0 0xc0000000>; + + #address-cells = <1>; + #size-cells = <1>; + }; +}; + +&soc { + interrupt-controller@10000 { + compatible = "example,intc"; + reg = <0x10000 0x1000>; + interrupts = <1 2 3>; + }; + + serial@20000 { + compatible = "example,serial"; + reg = <0x20000 0x1000>; + }; + + serial@30000 { + compatible = "example,serial"; + reg = <0x30000 0x1000>; + }; +}; diff --git a/scripts/dtc/dt-style-selftest/good/dts-preprocessor.dts b/scripts/dtc/dt-style-selftest/good/dts-preprocessor.dts new file mode 100644 index 000000000000..bd6d9dddec73 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/good/dts-preprocessor.dts @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +/* + * Test fixture: preprocessor directive + */ + +/dts-v1/; + +#include "soc.dtsi" +#include<dt-bindings/gpio/gpio.h> + +/ { + compatible = "example,test-board"; + #address-cells = <1>; + #size-cells = <1>; + + leds { + led-0 { + compatible = "example,led"; + }; + }; +}; diff --git a/scripts/dtc/dt-style-selftest/good/dts-property-order.dts b/scripts/dtc/dt-style-selftest/good/dts-property-order.dts new file mode 100644 index 000000000000..0e183e3459cd --- /dev/null +++ b/scripts/dtc/dt-style-selftest/good/dts-property-order.dts @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +/* + * Test fixture: Incorrect property order + */ + +/dts-v1/; + +/ { + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x0 0x0>; + enable-method = "psci"; + }; + }; + + pmu { + compatible = "example,pmu"; + dma-coherent; + + status = "disabled"; + }; + + soc@0 { + compatible = "simple-bus"; + ranges = <0 0 0 0xc0000000>; + + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller@10000 { + compatible = "example,intc"; + reg = <0x10000 0x1000>; + interrupts = <1 2 3>; + }; + }; +}; diff --git a/scripts/dtc/dt-style-selftest/good/dts-tab.dts b/scripts/dtc/dt-style-selftest/good/dts-tab.dts new file mode 100644 index 000000000000..ab7b5d1242ba --- /dev/null +++ b/scripts/dtc/dt-style-selftest/good/dts-tab.dts @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +/* + * Test fixture: a properly formatted .dts using one-tab indent. + */ + +/dts-v1/; + +/ { + compatible = "example,test-board"; + #address-cells = <1>; + #size-cells = <1>; + + bus@10000 { + compatible = "simple-bus"; + reg = <0x10000 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + + device@100 { + compatible = "example,test"; + reg = <0x100 0x10>; + }; + + device@200 { + compatible = "example,test"; + reg = <0x200 0x10>; + }; + }; +}; diff --git a/scripts/dtc/dt-style-selftest/good/soc.dtsi b/scripts/dtc/dt-style-selftest/good/soc.dtsi new file mode 100644 index 000000000000..deec1a973c92 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/good/soc.dtsi @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x0 0x0>; + enable-method = "psci"; + }; + }; + + pmu { + compatible = "example,pmu"; + }; + + soc@0 { + compatible = "simple-bus"; + ranges = <0 0 0 0xc0000000>; + + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller@10000 { + compatible = "example,intc"; + reg = <0x10000 0x1000>; + interrupts = <1 2 3>, + <4 5 6>, + <7 8 9>; + }; + + serial@20000 { + compatible = "example,serial"; + reg = <0x20000 0x1000>; + }; + + serial@30000 { + compatible = "example,serial"; + reg = <0x30000 0x1000>; + }; + }; +}; diff --git a/scripts/dtc/dt-style-selftest/good/yaml-4space.yaml b/scripts/dtc/dt-style-selftest/good/yaml-4space.yaml new file mode 100644 index 000000000000..1502f803c24c --- /dev/null +++ b/scripts/dtc/dt-style-selftest/good/yaml-4space.yaml @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-good-4space.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture for dt-check-style + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-4space + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + bus@10000 { + compatible = "simple-bus"; + reg = <0x10000 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + + device@100 { + compatible = "example,test-4space"; + reg = <0x100 0x10>; + }; + + device@200 { + compatible = "example,test-4space"; + reg = <0x200 0x10>; + }; + }; diff --git a/scripts/dtc/dt-style-selftest/good/yaml-tricky-parsing.yaml b/scripts/dtc/dt-style-selftest/good/yaml-tricky-parsing.yaml new file mode 100644 index 000000000000..a836d5f36b93 --- /dev/null +++ b/scripts/dtc/dt-style-selftest/good/yaml-tricky-parsing.yaml @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/test-good-tricky-parsing.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Test fixture exercising parser corner cases + +description: | + Covers patterns that previously broke the classifier: + - "/* ... */ code;" with code following the closing of a one-line + block comment must still parse the code. + - a label on a node whose name starts with a digit (1wire@10). + - a multi-line C preprocessor macro using backslash continuations + must not be parsed as DTS lines. + - a /* comment that opens and closes on the same code line must + not leave the parser in block-comment state. + +maintainers: + - Test User <test@example.com> + +properties: + compatible: + const: example,test-tricky-parsing + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + #define MY_REG(a, b) \ + ((a) << 16 | \ + (b) << 0) + + one_wire: 1wire@10 { + compatible = "example,test-tricky-parsing"; + reg = <MY_REG(0x10, 0)>; + /* inline-closed */ status = "okay"; + }; + + other: device@20 { + compatible = "example,test-tricky-parsing"; + reg = <0x20 0x10>; + }; + + &one_wire { + status = "okay"; + }; + + &other { + status = "okay"; + }; diff --git a/scripts/dtc/dt-style-selftest/run.sh b/scripts/dtc/dt-style-selftest/run.sh new file mode 100755 index 000000000000..8117dd9be90a --- /dev/null +++ b/scripts/dtc/dt-style-selftest/run.sh @@ -0,0 +1,71 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +# +# Run dt-check-style against fixtures under good/ and bad/. +# good/ files must produce no output and exit 0 in both modes. +# bad/ files must produce the expected output (in expected/<name>.txt) +# and exit 1. +# +# The mode used for a bad fixture is whichever produces a violation: +# trailing-whitespace and tab fixtures use the default (relaxed), +# the rest use --mode=strict. The expected output files name the +# mode in their first line. + +set -u + +here=$(cd "$(dirname "$0")" && pwd) +tool="$here/../dt-check-style" +fail=0 + +run() { + file=$1 + mode=$2 + "$tool" --mode="$mode" "$file" 2>&1 +} + +# good/ -- must exit 0 and produce no output in both modes +for f in "$here"/good/*; do + [ -e "$f" ] || continue + for mode in relaxed strict; do + out=$(run "$f" "$mode") + rc=$? + if [ -n "$out" ] || [ "$rc" -ne 0 ]; then + echo "FAIL good/$mode: $(basename "$f") (exit $rc, want 0):" + echo "$out" | sed 's/^/ /' + fail=$((fail + 1)) + fi + done +done + +# bad/ -- must match expected/<name>.txt +for f in "$here"/bad/*; do + [ -e "$f" ] || continue + name=$(basename "$f") + expected="$here/expected/$name.txt" + if [ ! -f "$expected" ]; then + echo "FAIL bad: missing $expected" + fail=$((fail + 1)) + continue + fi + mode=$(head -1 "$expected" | sed 's/^# mode=//') + body=$(tail -n +2 "$expected") + out=$(run "$f" "$mode") + rc=$? + # Strip the directory prefix so expected files are portable. + out=$(printf '%s\n' "$out" | sed "s|$here/bad/|bad/|g") + if [ "$out" != "$body" ] || [ "$rc" -ne 1 ]; then + echo "FAIL bad/$mode: $name (exit $rc, want 1):" + bf=$(mktemp) + printf '%s\n' "$body" > "$bf" + printf '%s\n' "$out" | diff -u "$bf" - | sed 's/^/ /' + rm -f "$bf" + fail=$((fail + 1)) + fi +done + +if [ "$fail" -eq 0 ]; then + echo "PASS" + exit 0 +fi +echo "FAILED ($fail)" +exit 1 diff --git a/scripts/dtc/dtc-lexer.l b/scripts/dtc/dtc-lexer.l index 15d585c80798..1b129b118b0f 100644 --- a/scripts/dtc/dtc-lexer.l +++ b/scripts/dtc/dtc-lexer.l @@ -39,8 +39,6 @@ extern bool treesource_error; #define DPRINT(fmt, ...) do { } while (0) #endif -static int dts_version = 1; - #define BEGIN_DEFAULT() DPRINT("<V1>\n"); \ BEGIN(V1); \ @@ -101,7 +99,6 @@ static void PRINTF(1, 2) lexical_error(const char *fmt, ...); <*>"/dts-v1/" { DPRINT("Keyword: /dts-v1/\n"); - dts_version = 1; BEGIN_DEFAULT(); return DT_V1; } diff --git a/scripts/dtc/dtc.h b/scripts/dtc/dtc.h index 7231200e5d02..473552ebf017 100644 --- a/scripts/dtc/dtc.h +++ b/scripts/dtc/dtc.h @@ -227,7 +227,7 @@ struct node { struct node *next_sibling; char *fullpath; - int basenamelen; + size_t basenamelen; cell_t phandle; int addr_cells, size_cells; diff --git a/scripts/dtc/libfdt/fdt.c b/scripts/dtc/libfdt/fdt.c index 95f644c31f94..56d4dcb2dc92 100644 --- a/scripts/dtc/libfdt/fdt.c +++ b/scripts/dtc/libfdt/fdt.c @@ -110,6 +110,14 @@ int fdt_check_header(const void *fdt) || (fdt_totalsize(fdt) > INT_MAX)) return -FDT_ERR_TRUNCATED; + /* memrsv block must be 8 byte aligned */ + if (fdt_off_mem_rsvmap(fdt) % sizeof(uint64_t)) + return -FDT_ERR_ALIGNMENT; + + /* Structure block must be 4 byte aligned */ + if (fdt_off_dt_struct(fdt) % FDT_TAGSIZE) + return -FDT_ERR_ALIGNMENT; + /* Bounds check memrsv block */ if (!check_off_(hdrsize, fdt_totalsize(fdt), fdt_off_mem_rsvmap(fdt))) diff --git a/scripts/dtc/libfdt/fdt_rw.c b/scripts/dtc/libfdt/fdt_rw.c index 7475cafce071..90ea14e944cc 100644 --- a/scripts/dtc/libfdt/fdt_rw.c +++ b/scripts/dtc/libfdt/fdt_rw.c @@ -22,6 +22,12 @@ static int fdt_blocks_misordered_(const void *fdt, (fdt_off_dt_strings(fdt) + fdt_size_dt_strings(fdt))); } +static void fdt_downgrade_version(void *fdt) +{ + if (!can_assume(LATEST) && fdt_version(fdt) > FDT_LAST_SUPPORTED_VERSION) + fdt_set_version(fdt, FDT_LAST_SUPPORTED_VERSION); +} + static int fdt_rw_probe_(void *fdt) { if (can_assume(VALID_DTB)) @@ -33,9 +39,8 @@ static int fdt_rw_probe_(void *fdt) if (fdt_blocks_misordered_(fdt, sizeof(struct fdt_reserve_entry), fdt_size_dt_struct(fdt))) return -FDT_ERR_BADLAYOUT; - if (!can_assume(LATEST) && fdt_version(fdt) > 17) - fdt_set_version(fdt, 17); + fdt_downgrade_version(fdt); return 0; } diff --git a/scripts/dtc/version_gen.h b/scripts/dtc/version_gen.h index 5730bf457b33..122e684e76a1 100644 --- a/scripts/dtc/version_gen.h +++ b/scripts/dtc/version_gen.h @@ -1 +1 @@ -#define DTC_VERSION "DTC 1.7.2-ga26ef640" +#define DTC_VERSION "DTC 1.7.2-g53373d13" diff --git a/scripts/gcc-plugins/gcc-common.h b/scripts/gcc-plugins/gcc-common.h index 8f1b3500f8e2..abb1964c44d4 100644 --- a/scripts/gcc-plugins/gcc-common.h +++ b/scripts/gcc-plugins/gcc-common.h @@ -309,7 +309,9 @@ typedef const gimple *const_gimple_ptr; #define gimple gimple_ptr #define const_gimple const_gimple_ptr #undef CONST_CAST_GIMPLE -#define CONST_CAST_GIMPLE(X) CONST_CAST(gimple, (X)) +#define CONST_CAST_GIMPLE(X) const_cast<gimple>((X)) +#undef CONST_CAST_TREE +#define CONST_CAST_TREE(X) const_cast<tree>((X)) /* gimple related */ static inline gimple gimple_build_assign_with_ops(enum tree_code subcode, tree lhs, tree op1, tree op2 MEM_STAT_DECL) diff --git a/scripts/gdb/linux/interrupts.py b/scripts/gdb/linux/interrupts.py index f4f715a8f0e3..a68ae91b4531 100644 --- a/scripts/gdb/linux/interrupts.py +++ b/scripts/gdb/linux/interrupts.py @@ -20,7 +20,7 @@ def irq_desc_is_chained(desc): def irqd_is_level(desc): return desc['irq_data']['common']['state_use_accessors'] & constants.LX_IRQD_LEVEL -def show_irq_desc(prec, irq): +def show_irq_desc(prec, chip_width, irq): text = "" desc = mapletree.mtree_load(gdb.parse_and_eval("&sparse_irqs"), irq) @@ -48,7 +48,7 @@ def show_irq_desc(prec, irq): count = cpus.per_cpu(desc['kstat_irqs'], cpu)['cnt'] else: count = 0 - text += "%10u" % (count) + text += "%10u " % (count) name = "None" if desc['irq_data']['chip']: @@ -58,7 +58,7 @@ def show_irq_desc(prec, irq): else: name = "-" - text += " %8s" % (name) + text += " %-*s" % (chip_width, name) if desc['irq_data']['domain']: text += " %*lu" % (prec, desc['irq_data']['hwirq']) @@ -97,64 +97,29 @@ def show_irq_err_count(prec): text += "%*s: %10u\n" % (prec, "ERR", cnt['counter']) return text -def x86_show_irqstat(prec, pfx, field, desc): - irq_stat = gdb.parse_and_eval("&irq_stat") +def x86_show_irqstat(prec, pfx, idx, desc): + irq_stat = gdb.parse_and_eval("&irq_stat.counts[%d]" %idx) text = "%*s: " % (prec, pfx) for cpu in cpus.each_online_cpu(): stat = cpus.per_cpu(irq_stat, cpu) - text += "%10u " % (stat[field]) - text += " %s\n" % (desc) - return text - -def x86_show_mce(prec, var, pfx, desc): - pvar = gdb.parse_and_eval(var) - text = "%*s: " % (prec, pfx) - for cpu in cpus.each_online_cpu(): - text += "%10u " % (cpus.per_cpu(pvar, cpu).dereference()) - text += " %s\n" % (desc) + text += "%10u " % (stat.dereference()) + text += desc return text def x86_show_interupts(prec): - text = x86_show_irqstat(prec, "NMI", '__nmi_count', 'Non-maskable interrupts') - - if constants.LX_CONFIG_X86_LOCAL_APIC: - text += x86_show_irqstat(prec, "LOC", 'apic_timer_irqs', "Local timer interrupts") - text += x86_show_irqstat(prec, "SPU", 'irq_spurious_count', "Spurious interrupts") - text += x86_show_irqstat(prec, "PMI", 'apic_perf_irqs', "Performance monitoring interrupts") - text += x86_show_irqstat(prec, "IWI", 'apic_irq_work_irqs', "IRQ work interrupts") - text += x86_show_irqstat(prec, "RTR", 'icr_read_retry_count', "APIC ICR read retries") - if utils.gdb_eval_or_none("x86_platform_ipi_callback") is not None: - text += x86_show_irqstat(prec, "PLT", 'x86_platform_ipis', "Platform interrupts") - - if constants.LX_CONFIG_SMP: - text += x86_show_irqstat(prec, "RES", 'irq_resched_count', "Rescheduling interrupts") - text += x86_show_irqstat(prec, "CAL", 'irq_call_count', "Function call interrupts") - text += x86_show_irqstat(prec, "TLB", 'irq_tlb_count', "TLB shootdowns") - - if constants.LX_CONFIG_X86_THERMAL_VECTOR: - text += x86_show_irqstat(prec, "TRM", 'irq_thermal_count', "Thermal events interrupts") - - if constants.LX_CONFIG_X86_MCE_THRESHOLD: - text += x86_show_irqstat(prec, "THR", 'irq_threshold_count', "Threshold APIC interrupts") - - if constants.LX_CONFIG_X86_MCE_AMD: - text += x86_show_irqstat(prec, "DFR", 'irq_deferred_error_count', "Deferred Error APIC interrupts") + info_type = gdb.lookup_type('struct irq_stat_info') + info = gdb.parse_and_eval('irq_stat_info') + bitmap = gdb.parse_and_eval('irq_stat_count_show') + bitsperlong = 8 * int(bitmap.type.target().sizeof) - if constants.LX_CONFIG_X86_MCE: - text += x86_show_mce(prec, "&mce_exception_count", "MCE", "Machine check exceptions") - text += x86_show_mce(prec, "&mce_poll_count", "MCP", "Machine check polls") - - text += show_irq_err_count(prec) - - if constants.LX_CONFIG_X86_IO_APIC: - cnt = utils.gdb_eval_or_none("irq_mis_count") - if cnt is not None: - text += "%*s: %10u\n" % (prec, "MIS", cnt['counter']) - - if constants.LX_CONFIG_KVM: - text += x86_show_irqstat(prec, "PIN", 'kvm_posted_intr_ipis', 'Posted-interrupt notification event') - text += x86_show_irqstat(prec, "NPI", 'kvm_posted_intr_nested_ipis', 'Nested posted-interrupt event') - text += x86_show_irqstat(prec, "PIW", 'kvm_posted_intr_wakeup_ipis', 'Posted-interrupt wakeup event') + text = "" + for idx in range(int(info.type.sizeof / info_type.sizeof)): + show = bitmap[int(idx / bitsperlong)] + if not show & 1 << int(idx % bitsperlong): + continue + pfx = info[idx]['symbol'].string() + desc = info[idx]['text'].string() + text += x86_show_irqstat(prec, pfx, idx, desc) return text @@ -166,23 +131,19 @@ def arm_common_show_interrupts(prec): if nr_ipi is None or ipi_desc is None or ipi_types is None: return text - if prec >= 4: - sep = " " - else: - sep = "" - for ipi in range(nr_ipi): - text += "%*s%u:%s" % (prec - 1, "IPI", ipi, sep) + text += "%*s%u: " % (prec - 1, "IPI", ipi) desc = ipi_desc[ipi].cast(irq_desc_type.get_type().pointer()) if desc == 0: continue for cpu in cpus.each_online_cpu(): - text += "%10u" % (cpus.per_cpu(desc['kstat_irqs'], cpu)['cnt']) - text += " %s" % (ipi_types[ipi].string()) + text += "%10u " % (cpus.per_cpu(desc['kstat_irqs'], cpu)['cnt']) + text += "%s" % (ipi_types[ipi].string()) text += "\n" return text def aarch64_show_interrupts(prec): + # Does not work for ARM64 as "ipi_desc" is not available there text = arm_common_show_interrupts(prec) text += "%*s: %10lu\n" % (prec, "ERR", gdb.parse_and_eval("irq_err_count")) return text @@ -209,12 +170,19 @@ class LxInterruptList(gdb.Command): super(LxInterruptList, self).__init__("lx-interruptlist", gdb.COMMAND_DATA) def invoke(self, arg, from_tty): - nr_irqs = gdb.parse_and_eval("nr_irqs") - prec = 3 - j = 1000 - while prec < 10 and j <= nr_irqs: - prec += 1 - j *= 10 + nr_irqs = gdb.parse_and_eval("total_nr_irqs") + constr = utils.gdb_eval_or_none('irq_proc_constraints') + + if constr: + prec = int(constr['num_prec']) + chip_width = int(constr['chip_width']) + else: + prec = 4 + j = 10000 + while prec < 10 and j <= nr_irqs: + prec += 1 + j *= 10 + chip_width = 8 gdb.write("%*s" % (prec + 8, "")) for cpu in cpus.each_online_cpu(): @@ -225,7 +193,7 @@ class LxInterruptList(gdb.Command): raise gdb.GdbError("Unable to find the sparse IRQ tree, is CONFIG_SPARSE_IRQ enabled?") for irq in range(nr_irqs): - gdb.write(show_irq_desc(prec, irq)) + gdb.write(show_irq_desc(prec, chip_width, irq)) gdb.write(arch_show_interrupts(prec)) diff --git a/scripts/gdb/linux/mm.py b/scripts/gdb/linux/mm.py index d78908f6664d..dffadccbb01d 100644 --- a/scripts/gdb/linux/mm.py +++ b/scripts/gdb/linux/mm.py @@ -40,11 +40,11 @@ class x86_page_ops(): self.PAGE_OFFSET = int(gdb.parse_and_eval("page_offset_base")) self.VMEMMAP_START = int(gdb.parse_and_eval("vmemmap_base")) - self.PHYS_BASE = int(gdb.parse_and_eval("phys_base")) + self.PHYS_BASE = int(gdb.parse_and_eval("(unsigned long) phys_base")) self.START_KERNEL_map = 0xffffffff80000000 - self.KERNEL_START = gdb.parse_and_eval("_text") - self.KERNEL_END = gdb.parse_and_eval("_end") + self.KERNEL_START = gdb.parse_and_eval("(unsigned long) &_text") + self.KERNEL_END = gdb.parse_and_eval("(unsigned long) &_end") self.VMALLOC_START = int(gdb.parse_and_eval("vmalloc_base")) if self.VMALLOC_START == 0xffffc90000000000: diff --git a/scripts/gdb/linux/page_owner.py b/scripts/gdb/linux/page_owner.py index 8e713a09cfe7..eeabaeed438b 100644 --- a/scripts/gdb/linux/page_owner.py +++ b/scripts/gdb/linux/page_owner.py @@ -34,6 +34,7 @@ class DumpPageOwner(gdb.Command): max_pfn = None p_ops = None migrate_reason_names = None + mr_never = None def __init__(self): super(DumpPageOwner, self).__init__("lx-dump-page-owner", gdb.COMMAND_SUPPORT) @@ -65,6 +66,7 @@ class DumpPageOwner(gdb.Command): self.max_pfn = int(gdb.parse_and_eval("max_pfn")) self.page_ext_size = int(gdb.parse_and_eval("page_ext_size")) self.migrate_reason_names = gdb.parse_and_eval('migrate_reason_names') + self.mr_never = int(gdb.parse_and_eval('MR_NEVER')) def page_ext_invalid(self, page_ext): if page_ext == gdb.Value(0): @@ -138,7 +140,7 @@ class DumpPageOwner(gdb.Command): else: gdb.write('page last free stack trace:\n') stackdepot.stack_depot_print(page_owner["free_handle"]) - if page_owner['last_migrate_reason'] != -1: + if page_owner['last_migrate_reason'] != self.mr_never: gdb.write('page has been migrated, last migrate reason: %s\n' % self.migrate_reason_names[page_owner['last_migrate_reason']]) def read_page_owner(self): diff --git a/scripts/gdb/linux/slab.py b/scripts/gdb/linux/slab.py index 0e2d93867fe2..ddde25aeca8d 100644 --- a/scripts/gdb/linux/slab.py +++ b/scripts/gdb/linux/slab.py @@ -196,7 +196,7 @@ def slabtrace(alloc, cache_name): if target_cache['flags'] & SLAB_STORE_USER: for i in range(0, nr_node_ids): - cache_node = target_cache['node'][i] + cache_node = target_cache['per_node']['node'][i] if cache_node['nr_slabs']['counter'] == 0: continue process_slab(loc_track, cache_node['partial'], alloc, target_cache) @@ -300,7 +300,7 @@ def slabinfo(): nr_free = 0 nr_slabs = 0 for i in range(0, nr_node_ids): - cache_node = cache['node'][i] + cache_node = cache['per_node']['node'][i] try: nr_slabs += cache_node['nr_slabs']['counter'] nr_objs = int(cache_node['total_objects']['counter']) diff --git a/scripts/gdb/linux/symbols.py b/scripts/gdb/linux/symbols.py index d4308b726183..943ff1228b48 100644 --- a/scripts/gdb/linux/symbols.py +++ b/scripts/gdb/linux/symbols.py @@ -298,7 +298,7 @@ are loaded as well.""" if p == "-bpf": monitor_bpf = True else: - p.append(os.path.abspath(os.path.expanduser(p))) + self.module_paths.append(os.path.abspath(os.path.expanduser(p))) self.module_paths.append(os.getcwd()) if self.breakpoint is not None: diff --git a/scripts/gdb/linux/timerlist.py b/scripts/gdb/linux/timerlist.py index ccc24d30de80..744b032e4d38 100644 --- a/scripts/gdb/linux/timerlist.py +++ b/scripts/gdb/linux/timerlist.py @@ -20,7 +20,7 @@ def ktime_get(): We can't read the hardware timer itself to add any nanoseconds that need to be added since we last stored the time in the timekeeper. But this is probably good enough for debug purposes.""" - tk_core = gdb.parse_and_eval("&tk_core") + tk_core = gdb.parse_and_eval("&timekeeper_data[TIMEKEEPER_CORE]") return tk_core['timekeeper']['tkr_mono']['base'] @@ -90,14 +90,10 @@ def print_cpu(hrtimer_bases, cpu, max_clock_bases): text += f" .{'nohz':15s}: {int(bool(ts['flags'] & TS_FLAG_NOHZ))}\n" text += f" .{'last_tick':15s}: {ts['last_tick']}\n" text += f" .{'tick_stopped':15s}: {int(bool(ts['flags'] & TS_FLAG_STOPPED))}\n" - text += f" .{'idle_jiffies':15s}: {ts['idle_jiffies']}\n" text += f" .{'idle_calls':15s}: {ts['idle_calls']}\n" text += f" .{'idle_sleeps':15s}: {ts['idle_sleeps']}\n" text += f" .{'idle_entrytime':15s}: {ts['idle_entrytime']} nsecs\n" text += f" .{'idle_waketime':15s}: {ts['idle_waketime']} nsecs\n" - text += f" .{'idle_exittime':15s}: {ts['idle_exittime']} nsecs\n" - text += f" .{'idle_sleeptime':15s}: {ts['idle_sleeptime']} nsecs\n" - text += f" .{'iowait_sleeptime':15s}: {ts['iowait_sleeptime']} nsecs\n" text += f" .{'last_jiffies':15s}: {ts['last_jiffies']}\n" text += f" .{'next_timer':15s}: {ts['next_timer']}\n" text += f" .{'idle_expires':15s}: {ts['idle_expires']} nsecs\n" diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index f9b545104f21..dc1219736f77 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -10,8 +10,15 @@ import os import pathlib import subprocess import sys +from typing import Dict, Iterable, List, Literal, Optional, TypedDict -def args_crates_cfgs(cfgs): +def invoke_rustc(args: List[str]) -> str: + return subprocess.check_output( + [os.environ["RUSTC"]] + args, + stdin=subprocess.DEVNULL, + ).decode('utf-8').strip() + +def args_crates_cfgs(cfgs: List[str]) -> Dict[str, List[str]]: crates_cfgs = {} for cfg in cfgs: crate, vals = cfg.split("=", 1) @@ -19,54 +26,165 @@ def args_crates_cfgs(cfgs): return crates_cfgs -def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs, core_edition): +def args_crates_envs(envs: List[str]) -> Dict[str, Dict[str, str]]: + crates_envs = {} + for env in envs: + crate, vals = env.split("=", 1) + crates_envs[crate] = dict(v.split("=", 1) for v in vals.split()) + + return crates_envs + +class Dependency(TypedDict): + crate: int + name: str + + +class Source(TypedDict): + include_dirs: List[str] + exclude_dirs: List[str] + + +class Crate(TypedDict): + display_name: str + root_module: str + is_workspace_member: bool + deps: List[Dependency] + cfg: List[str] + edition: str + env: Dict[str, str] + + +class ProcMacroCrate(Crate): + is_proc_macro: Literal[True] + proc_macro_dylib_path: str # `pathlib.Path` is not JSON serializable. + + +class CrateWithGenerated(Crate): + source: Source + + +def generate_crates( + srctree: pathlib.Path, + objtree: pathlib.Path, + sysroot_src: pathlib.Path, + external_src: Optional[pathlib.Path], + cfgs: List[str], + envs: List[str], + core_edition: str, +) -> List[Crate]: # Generate the configuration list. - cfg = [] + generated_cfg = [] with open(objtree / "include" / "generated" / "rustc_cfg") as fd: for line in fd: line = line.replace("--cfg=", "") line = line.replace("\n", "") - cfg.append(line) + generated_cfg.append(line) - # Now fill the crates list -- dependencies need to come first. - # - # Avoid O(n^2) iterations by keeping a map of indexes. - crates = [] - crates_indexes = {} + # Now fill the crates list. + crates: List[Crate] = [] crates_cfgs = args_crates_cfgs(cfgs) + crates_envs = args_crates_envs(envs) + + def get_crate_name(path: pathlib.Path) -> str: + return invoke_rustc(["--print", "crate-name", str(path)]) - def append_crate(display_name, root_module, deps, cfg=[], is_workspace_member=True, is_proc_macro=False, edition="2021"): - crate = { + def build_crate( + display_name: str, + root_module: pathlib.Path, + deps: List[Dependency], + *, + cfg: Optional[List[str]], + is_workspace_member: Optional[bool], + edition: Optional[str], + ) -> Crate: + cfg = cfg if cfg is not None else crates_cfgs.get(display_name, []) + is_workspace_member = ( + is_workspace_member if is_workspace_member is not None else True + ) + edition = edition if edition is not None else "2021" + crate_env = { + "RUST_MODFILE": "This is only for rust-analyzer", + **crates_envs.get(display_name, {}), + } + return { "display_name": display_name, "root_module": str(root_module), "is_workspace_member": is_workspace_member, - "is_proc_macro": is_proc_macro, - "deps": [{"crate": crates_indexes[dep], "name": dep} for dep in deps], + "deps": deps, "cfg": cfg, "edition": edition, - "env": { - "RUST_MODFILE": "This is only for rust-analyzer" - } + "env": crate_env, } - if is_proc_macro: - proc_macro_dylib_name = subprocess.check_output( - [os.environ["RUSTC"], "--print", "file-names", "--crate-name", display_name, "--crate-type", "proc-macro", "-"], - stdin=subprocess.DEVNULL, - ).decode('utf-8').strip() - crate["proc_macro_dylib_path"] = f"{objtree}/rust/{proc_macro_dylib_name}" - crates_indexes[display_name] = len(crates) + + def append_proc_macro_crate( + display_name: str, + root_module: pathlib.Path, + deps: List[Dependency], + *, + cfg: Optional[List[str]] = None, + is_workspace_member: Optional[bool] = None, + edition: Optional[str] = None, + ) -> Dependency: + crate = build_crate( + display_name, + root_module, + deps, + cfg=cfg, + is_workspace_member=is_workspace_member, + edition=edition, + ) + proc_macro_dylib_name = invoke_rustc([ + "--print", + "file-names", + "--crate-name", + display_name, + "--crate-type", + "proc-macro", + "-", + ]) + proc_macro_crate: ProcMacroCrate = { + **crate, + "is_proc_macro": True, + "proc_macro_dylib_path": str(objtree / "rust" / proc_macro_dylib_name), + } + return register_crate(proc_macro_crate) + + def register_crate(crate: Crate) -> Dependency: + index = len(crates) crates.append(crate) + return {"crate": index, "name": crate["display_name"]} + + def append_crate( + display_name: str, + root_module: pathlib.Path, + deps: List[Dependency], + *, + cfg: Optional[List[str]] = None, + is_workspace_member: Optional[bool] = None, + edition: Optional[str] = None, + ) -> Dependency: + return register_crate( + build_crate( + display_name, + root_module, + deps, + cfg=cfg, + is_workspace_member=is_workspace_member, + edition=edition, + ) + ) def append_sysroot_crate( - display_name, - deps, - cfg=[], - ): - append_crate( + display_name: str, + deps: List[Dependency], + *, + cfg: Optional[List[str]] = None, + ) -> Dependency: + return append_crate( display_name, sysroot_src / display_name / "src" / "lib.rs", deps, - cfg, + cfg=cfg, is_workspace_member=False, # Miguel Ojeda writes: # @@ -98,139 +216,181 @@ def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs, core_edit # NB: sysroot crates reexport items from one another so setting up our transitive dependencies # here is important for ensuring that rust-analyzer can resolve symbols. The sources of truth # for this dependency graph are `(sysroot_src / crate / "Cargo.toml" for crate in crates)`. - append_sysroot_crate("core", [], cfg=crates_cfgs.get("core", [])) - append_sysroot_crate("alloc", ["core"]) - append_sysroot_crate("std", ["alloc", "core"]) - append_sysroot_crate("proc_macro", ["core", "std"]) + core = append_sysroot_crate("core", []) + alloc = append_sysroot_crate("alloc", [core]) + std = append_sysroot_crate("std", [alloc, core]) + proc_macro = append_sysroot_crate("proc_macro", [core, std]) - append_crate( + compiler_builtins = append_crate( "compiler_builtins", srctree / "rust" / "compiler_builtins.rs", - ["core"], + [core], ) - append_crate( + proc_macro2 = append_crate( "proc_macro2", srctree / "rust" / "proc-macro2" / "lib.rs", - ["core", "alloc", "std", "proc_macro"], - cfg=crates_cfgs["proc_macro2"], + [core, alloc, std, proc_macro], ) - append_crate( + quote = append_crate( "quote", srctree / "rust" / "quote" / "lib.rs", - ["core", "alloc", "std", "proc_macro", "proc_macro2"], - cfg=crates_cfgs["quote"], + [core, alloc, std, proc_macro, proc_macro2], edition="2018", ) - append_crate( + syn = append_crate( "syn", srctree / "rust" / "syn" / "lib.rs", - ["std", "proc_macro", "proc_macro2", "quote"], - cfg=crates_cfgs["syn"], + [std, proc_macro, proc_macro2, quote], ) - append_crate( + macros = append_proc_macro_crate( "macros", srctree / "rust" / "macros" / "lib.rs", - ["std", "proc_macro", "proc_macro2", "quote", "syn"], - is_proc_macro=True, + [std, proc_macro, proc_macro2, quote, syn], ) - append_crate( + zerocopy_derive = append_proc_macro_crate( + "zerocopy_derive", + srctree / "rust" / "zerocopy-derive" / "lib.rs", + [std, proc_macro, proc_macro2, quote, syn], + ) + + build_error = append_crate( "build_error", srctree / "rust" / "build_error.rs", - ["core", "compiler_builtins"], + [core, compiler_builtins], ) - append_crate( + pin_init_internal = append_proc_macro_crate( "pin_init_internal", srctree / "rust" / "pin-init" / "internal" / "src" / "lib.rs", - ["std", "proc_macro", "proc_macro2", "quote", "syn"], - cfg=["kernel"], - is_proc_macro=True, + [std, proc_macro, proc_macro2, quote, syn], ) - append_crate( + pin_init = append_crate( "pin_init", srctree / "rust" / "pin-init" / "src" / "lib.rs", - ["core", "compiler_builtins", "pin_init_internal", "macros"], - cfg=["kernel"], + [core, compiler_builtins, pin_init_internal, macros], ) - append_crate( + ffi = append_crate( "ffi", srctree / "rust" / "ffi.rs", - ["core", "compiler_builtins"], + [core, compiler_builtins], + ) + + zerocopy = append_crate( + "zerocopy", + srctree / "rust" / "zerocopy" / "src" / "lib.rs", + [core, compiler_builtins], ) def append_crate_with_generated( - display_name, - deps, - ): - append_crate( + display_name: str, + deps: List[Dependency], + ) -> Dependency: + crate = build_crate( display_name, srctree / "rust"/ display_name / "lib.rs", deps, - cfg=cfg, + cfg=generated_cfg, + is_workspace_member=True, + edition=None, ) - crates[-1]["env"]["OBJTREE"] = str(objtree.resolve(True)) - crates[-1]["source"] = { - "include_dirs": [ - str(srctree / "rust" / display_name), - str(objtree / "rust") - ], - "exclude_dirs": [], + crate["env"]["OBJTREE"] = str(objtree.resolve(True)) + crate_with_generated: CrateWithGenerated = { + **crate, + "source": { + "include_dirs": [ + str(srctree / "rust" / display_name), + str(objtree / "rust"), + ], + "exclude_dirs": [], + }, } + return register_crate(crate_with_generated) - append_crate_with_generated("bindings", ["core", "ffi", "pin_init"]) - append_crate_with_generated("uapi", ["core", "ffi", "pin_init"]) - append_crate_with_generated("kernel", ["core", "macros", "build_error", "pin_init", "ffi", "bindings", "uapi"]) + bindings = append_crate_with_generated("bindings", [core, ffi, pin_init]) + uapi = append_crate_with_generated("uapi", [core, ffi, pin_init]) + kernel = append_crate_with_generated( + "kernel", [core, macros, build_error, pin_init, ffi, bindings, uapi, zerocopy, zerocopy_derive] + ) - def is_root_crate(build_file, target): + scripts = srctree / "scripts" + makefile = (scripts / "Makefile").read_text() + for path in scripts.glob("*.rs"): + name = path.stem + if f"{name}-rust" not in makefile: + continue + append_crate( + name, + path, + [std], + ) + + def is_root_crate(build_file: pathlib.Path, target: str) -> bool: try: - return f"{target}.o" in open(build_file).read() + contents = build_file.read_text() except FileNotFoundError: return False + return f"{target}.o" in contents # Then, the rest outside of `rust/`. # # We explicitly mention the top-level folders we want to cover. - extra_dirs = map(lambda dir: srctree / dir, ("samples", "drivers")) + extra_dirs: Iterable[pathlib.Path] = ( + srctree / dir for dir in ("samples", "drivers") + ) if external_src is not None: extra_dirs = [external_src] for folder in extra_dirs: for path in folder.rglob("*.rs"): logging.info("Checking %s", path) - name = path.name.replace(".rs", "") + file_name = path.stem # Skip those that are not crate roots. - if not is_root_crate(path.parent / "Makefile", name) and \ - not is_root_crate(path.parent / "Kbuild", name): + if not is_root_crate(path.parent / "Makefile", file_name) and \ + not is_root_crate(path.parent / "Kbuild", file_name): continue - logging.info("Adding %s", name) + crate_name = get_crate_name(path) + logging.info("Adding %s", crate_name) append_crate( - name, + crate_name, path, - ["core", "kernel", "pin_init"], - cfg=cfg, + [core, kernel, pin_init, zerocopy, zerocopy_derive], + cfg=generated_cfg, ) return crates -def main(): +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('--verbose', '-v', action='store_true') parser.add_argument('--cfgs', action='append', default=[]) + parser.add_argument('--envs', action='append', default=[]) parser.add_argument("core_edition") parser.add_argument("srctree", type=pathlib.Path) parser.add_argument("objtree", type=pathlib.Path) parser.add_argument("sysroot", type=pathlib.Path) parser.add_argument("sysroot_src", type=pathlib.Path) parser.add_argument("exttree", type=pathlib.Path, nargs="?") - args = parser.parse_args() + + class Args(argparse.Namespace): + verbose: bool + cfgs: List[str] + envs: List[str] + srctree: pathlib.Path + objtree: pathlib.Path + sysroot: pathlib.Path + sysroot_src: pathlib.Path + exttree: Optional[pathlib.Path] + core_edition: str + + args = parser.parse_args(namespace=Args()) logging.basicConfig( format="[%(asctime)s] [%(levelname)s] %(message)s", @@ -238,7 +398,7 @@ def main(): ) rust_project = { - "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree, args.cfgs, args.core_edition), + "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree, args.cfgs, args.envs, args.core_edition), "sysroot": str(args.sysroot), } diff --git a/scripts/generate_rust_target.rs b/scripts/generate_rust_target.rs index 38b3416bb979..3bf296581a88 100644 --- a/scripts/generate_rust_target.rs +++ b/scripts/generate_rust_target.rs @@ -196,7 +196,9 @@ fn main() { } } else if cfg.has("X86_64") { ts.push("arch", "x86_64"); - if cfg.rustc_version_atleast(1, 86, 0) { + if cfg.rustc_version_atleast(1, 98, 0) { + ts.push("rustc-abi", "softfloat"); + } else if cfg.rustc_version_atleast(1, 86, 0) { ts.push("rustc-abi", "x86-softfloat"); } ts.push( @@ -236,7 +238,9 @@ fn main() { panic!("32-bit x86 only works under UML"); } ts.push("arch", "x86"); - if cfg.rustc_version_atleast(1, 86, 0) { + if cfg.rustc_version_atleast(1, 98, 0) { + ts.push("rustc-abi", "softfloat"); + } else if cfg.rustc_version_atleast(1, 86, 0) { ts.push("rustc-abi", "x86-softfloat"); } ts.push( @@ -256,6 +260,8 @@ fn main() { } } else if cfg.has("LOONGARCH") { panic!("loongarch uses the builtin rustc loongarch64-unknown-none-softfloat target"); + } else if cfg.has("S390") { + panic!("s390 uses the builtin rustc s390x-unknown-none-softfloat target"); } else { panic!("Unsupported architecture"); } diff --git a/scripts/get_maintainer.pl b/scripts/get_maintainer.pl index 4414194bedcf..16b80a700d4a 100755 --- a/scripts/get_maintainer.pl +++ b/scripts/get_maintainer.pl @@ -21,6 +21,7 @@ use Cwd; use File::Find; use File::Spec::Functions; use open qw(:std :encoding(UTF-8)); +use JSON::PP; my $cur_path = fastgetcwd() . '/'; my $lk_path = "./"; @@ -68,6 +69,7 @@ my $pattern_depth = 0; my $self_test = undef; my $version = 0; my $help = 0; +my $json = 0; my $find_maintainer_files = 0; my $maintainer_path; my $vcs_used = 0; @@ -285,6 +287,7 @@ if (!GetOptions( 'find-maintainer-files' => \$find_maintainer_files, 'mpath|maintainer-path=s' => \$maintainer_path, 'self-test:s' => \$self_test, + 'json!' => \$json, 'v|version' => \$version, 'h|help|usage' => \$help, )) { @@ -375,8 +378,10 @@ sub read_maintainer_file { ##Filename pattern matching if ($type eq "F" || $type eq "X") { $value =~ s@\.@\\\.@g; ##Convert . to \. + $value =~ s/\*\*/\x00/g; ##Convert ** to placeholder $value =~ s/\*/\.\*/g; ##Convert * to .* $value =~ s/\?/\./g; ##Convert ? to . + $value =~ s/\x00/(?:.*)/g; ##Convert placeholder to (?:.*) ##if pattern is a directory and it lacks a trailing slash, add one if ((-d $value)) { $value =~ s@([^/])$@$1/@; @@ -648,39 +653,48 @@ my %deduplicate_name_hash = (); my %deduplicate_address_hash = (); my @maintainers = get_maintainers(); -if (@maintainers) { - @maintainers = merge_email(@maintainers); - output(@maintainers); -} - -if ($scm) { - @scm = uniq(@scm); - output(@scm); -} - -if ($output_substatus) { - @substatus = uniq(@substatus); - output(@substatus); -} - -if ($status) { - @status = uniq(@status); - output(@status); -} -if ($subsystem) { - @subsystem = uniq(@subsystem); - output(@subsystem); -} +@maintainers = merge_email(@maintainers) if (@maintainers); +@scm = uniq(@scm) if ($scm); +@substatus = uniq(@substatus) if ($output_substatus); +@status = uniq(@status) if ($status); +@subsystem = uniq(@subsystem) if ($subsystem); +@web = uniq(@web) if ($web); +@bug = uniq(@bug) if ($bug); + +if ($json) { + my @json_maintainers; + for my $m (@maintainers) { + my ($addr, $role); + if ($output_roles && $m =~ /^(.*?)\s+\((.+)\)\s*$/) { + $addr = $1; + $role = $2; + } else { + $addr = $m; + } + my ($name, $email_addr) = parse_email($addr); + my %entry = (name => $name, email => $email_addr); + $entry{role} = $role if (defined $role && $role ne ''); + push(@json_maintainers, \%entry); + } -if ($web) { - @web = uniq(@web); - output(@web); -} + my %result = (maintainers => \@json_maintainers); + $result{scm} = \@scm if ($scm); + $result{status} = \@status if ($status); + $result{subsystem} = \@subsystem if ($subsystem); + $result{web} = \@web if ($web); + $result{bug} = \@bug if ($bug); -if ($bug) { - @bug = uniq(@bug); - output(@bug); + my $json_encoder = JSON::PP->new->canonical->utf8; + print($json_encoder->encode(\%result) . "\n"); +} else { + output(@maintainers) if (@maintainers); + output(@scm) if ($scm); + output(@substatus) if ($output_substatus); + output(@status) if ($status); + output(@subsystem) if ($subsystem); + output(@web) if ($web); + output(@bug) if ($bug); } exit($exit); @@ -746,8 +760,10 @@ sub self_test { if (($type eq "F" || $type eq "X") && ($self_test eq "" || $self_test =~ /\bpatterns\b/)) { $value =~ s@\.@\\\.@g; ##Convert . to \. + $value =~ s/\*\*/\x00/g; ##Convert ** to placeholder $value =~ s/\*/\.\*/g; ##Convert * to .* $value =~ s/\?/\./g; ##Convert ? to . + $value =~ s/\x00/(?:.*)/g; ##Convert placeholder to (?:.*) ##if pattern is a directory and it lacks a trailing slash, add one if ((-d $value)) { $value =~ s@([^/])$@$1/@; @@ -921,7 +937,7 @@ sub get_maintainers { my $value_pd = ($value =~ tr@/@@); my $file_pd = ($file =~ tr@/@@); $value_pd++ if (substr($value,-1,1) ne "/"); - $value_pd = -1 if ($value =~ /^\.\*/); + $value_pd = -1 if ($value =~ /^(\.\*|\(\?:\.\*\))/); if ($value_pd >= $file_pd && range_is_maintained($start, $end) && range_has_maintainer($start, $end)) { @@ -955,6 +971,7 @@ sub get_maintainers { $line =~ s/([^\\])\.([^\*])/$1\?$2/g; $line =~ s/([^\\])\.$/$1\?/g; ##Convert . back to ? $line =~ s/\\\./\./g; ##Convert \. to . + $line =~ s/\(\?:\.\*\)/\*\*/g; ##Convert (?:.*) to ** $line =~ s/\.\*/\*/g; ##Convert .* to * } my $count = $line =~ s/^([A-Z]):/$1:\t/g; @@ -1048,7 +1065,7 @@ sub file_match_pattern { if ($file =~ m@^$pattern@) { my $s1 = ($file =~ tr@/@@); my $s2 = ($pattern =~ tr@/@@); - if ($s1 == $s2) { + if ($s1 == $s2 || $pattern =~ /\(\?:/) { return 1; } } @@ -1099,6 +1116,7 @@ Output type options: --separator [, ] => separator for multiple entries on 1 line using --separator also sets --nomultiline if --separator is not [, ] --multiline => print 1 entry per line + --json => output results as JSON Other options: --pattern-depth => Number of pattern directory traversals (default: 0 (all)) diff --git a/scripts/headers_install.sh b/scripts/headers_install.sh index 9c15e748761c..2f1d1767ca26 100755 --- a/scripts/headers_install.sh +++ b/scripts/headers_install.sh @@ -36,6 +36,7 @@ sed -E -e ' s/(^|[^a-zA-Z0-9])__packed([^a-zA-Z0-9_]|$)/\1__attribute__((packed))\2/g s/(^|[[:space:](])(inline|asm|volatile)([[:space:](]|$)/\1__\2__\3/g s@#(ifndef|define|endif[[:space:]]*/[*])[[:space:]]*_UAPI@#\1 @ + s/__ASSEMBLY__/__ASSEMBLER__/g ' $INFILE > $TMPFILE || exit 1 scripts/unifdef -U__KERNEL__ -D__EXPORTED_HEADERS__ $TMPFILE > $OUTFILE diff --git a/scripts/jobserver-exec b/scripts/jobserver-exec index 758e947a6fb9..21b319e6c9a5 100755 --- a/scripts/jobserver-exec +++ b/scripts/jobserver-exec @@ -28,8 +28,8 @@ def main(): sys.exit("usage: " + name +" command [args ...]\n" + __doc__) with JobserverExec() as jobserver: - jobserver.run(sys.argv[1:]) + return jobserver.run(sys.argv[1:]) if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c index 37d5c095ad22..494852ade6d8 100644 --- a/scripts/kallsyms.c +++ b/scripts/kallsyms.c @@ -398,11 +398,13 @@ static void write_src(void) strcpy((char *)table[i]->sym, buf); printf("\t/* %s */\n", table[i]->sym); } + printf(".size kallsyms_names, . - kallsyms_names\n"); printf("\n"); output_label("kallsyms_markers"); for (i = 0; i < markers_cnt; i++) printf("\t.long\t%u\n", markers[i]); + printf(".size kallsyms_markers, . - kallsyms_markers\n"); printf("\n"); free(markers); @@ -415,6 +417,7 @@ static void write_src(void) printf("\t.asciz\t\"%s\"\n", buf); off += strlen(buf) + 1; } + printf(".size kallsyms_token_table, . - kallsyms_token_table\n"); printf("\n"); output_label("kallsyms_token_index"); @@ -441,6 +444,7 @@ static void write_src(void) (unsigned int)table[i]->addr, table[i]->sym); } } + printf(".size kallsyms_offsets, . - kallsyms_offsets\n"); printf("\n"); sort_symbols_by_name(); diff --git a/scripts/kconfig/conf.c b/scripts/kconfig/conf.c index a7b44cd8ae14..fe8ba09b0039 100644 --- a/scripts/kconfig/conf.c +++ b/scripts/kconfig/conf.c @@ -297,9 +297,7 @@ static int conf_askvalue(struct symbol *sym, const char *def) line[1] = 0; if (!sym_is_changeable(sym)) { - printf("%s\n", def); - line[0] = '\n'; - line[1] = 0; + printf("%s\n", def ?: ""); return 0; } @@ -307,7 +305,7 @@ static int conf_askvalue(struct symbol *sym, const char *def) case oldconfig: case syncconfig: if (sym_has_value(sym)) { - printf("%s\n", def); + printf("%s\n", def ?: ""); return 0; } /* fall through */ @@ -350,6 +348,23 @@ static int conf_string(struct menu *menu) } if (def && sym_set_string_value(sym, def)) return 0; + + /* + * A new int or hex symbol whose default fails validation + * cannot be set from an empty answer. When standard input is + * exhausted, as it is for a non-interactive oldconfig or + * syncconfig, re-asking would loop forever and grow the output + * until it exhausts memory. Stop with an error that names the + * symbol instead. String symbols accept any text, and bool and + * tristate symbols (conf_sym()) and choices (conf_choice()) + * accept the default on an empty line, so they are unaffected. + */ + if (feof(stdin)) { + fprintf(stderr, + "\nerror: no value for new symbol '%s' at end of input\n", + sym->name); + exit(1); + } } } diff --git a/scripts/kconfig/confdata.c b/scripts/kconfig/confdata.c index 9599a0408862..4234a51d16fd 100644 --- a/scripts/kconfig/confdata.c +++ b/scripts/kconfig/confdata.c @@ -206,6 +206,78 @@ static void conf_message(const char *fmt, ...) va_end(ap); } +static void conf_changed_input_warning(const char *s) +{ + fputs(s, stderr); +} + +static bool conf_warn_changed_input_enabled(void) +{ + const char *env = getenv("KCONFIG_WARN_CHANGED_INPUT"); + + return env && *env; +} + +static const char *sym_get_user_value_string(struct symbol *sym) +{ + switch (sym->type) { + case S_BOOLEAN: + case S_TRISTATE: + switch (sym->def[S_DEF_USER].tri) { + case yes: + return "y"; + case mod: + return "m"; + default: + return "n"; + } + default: + return sym->def[S_DEF_USER].val ?: ""; + } +} + +static bool sym_user_value_changed(struct symbol *sym) +{ + if (!sym_has_value(sym) || sym->type == S_UNKNOWN) + return false; + + switch (sym->type) { + case S_BOOLEAN: + case S_TRISTATE: + return sym->def[S_DEF_USER].tri != sym_get_tristate_value(sym); + default: + return strcmp(sym_get_user_value_string(sym), + sym_get_string_value(sym)); + } +} + +static void conf_clear_written_flags(void) +{ + struct symbol *sym; + + for_all_symbols(sym) + sym->flags &= ~SYMBOL_WRITTEN; +} + +static void conf_append_changed_input_warning(struct gstr *gs, + struct symbol *sym, + bool *changed_input_found) +{ + if (!sym_user_value_changed(sym)) + return; + + if (!*changed_input_found) { + str_printf(gs, + "warning: user-provided values changed by Kconfig:\n"); + *changed_input_found = true; + } + + str_printf(gs, " %s%s: %s -> %s\n", + CONFIG_, sym->name, + sym_get_user_value_string(sym), + sym_get_string_value(sym)); +} + const char *conf_get_configname(void) { char *name = getenv("KCONFIG_CONFIG"); @@ -759,11 +831,15 @@ int conf_write_defconfig(const char *filename) { struct symbol *sym; struct menu *menu; + struct gstr gs; FILE *out; + bool warn_changed_input = conf_warn_changed_input_enabled(); + bool changed_input_found = false; out = fopen(filename, "w"); if (!out) return 1; + gs = str_new(); sym_clear_all_valid(); @@ -772,10 +848,14 @@ int conf_write_defconfig(const char *filename) sym = menu->sym; - if (!sym || sym_is_choice(sym)) + if (!sym || sym_is_choice(sym) || sym->flags & SYMBOL_WRITTEN) continue; sym_calc_value(sym); + if (warn_changed_input) + conf_append_changed_input_warning(&gs, sym, + &changed_input_found); + sym->flags |= SYMBOL_WRITTEN; if (!(sym->flags & SYMBOL_WRITE)) continue; sym->flags &= ~SYMBOL_WRITE; @@ -798,6 +878,13 @@ int conf_write_defconfig(const char *filename) print_symbol_for_dotconfig(out, sym); } fclose(out); + + conf_clear_written_flags(); + + if (changed_input_found) + conf_changed_input_warning(str_get(&gs)); + + str_free(&gs); return 0; } @@ -809,7 +896,10 @@ int conf_write(const char *name) const char *str; char tmpname[PATH_MAX + 1], oldname[PATH_MAX + 1]; char *env; + struct gstr gs; bool need_newline = false; + bool warn_changed_input = conf_warn_changed_input_enabled(); + bool changed_input_found = false; if (!name) name = conf_get_configname(); @@ -838,6 +928,7 @@ int conf_write(const char *name) } if (!out) return 1; + gs = str_new(); conf_write_heading(out, &comment_style_pound); @@ -859,13 +950,16 @@ int conf_write(const char *name) } else if (!sym_is_choice(sym) && !(sym->flags & SYMBOL_WRITTEN)) { sym_calc_value(sym); + if (warn_changed_input) + conf_append_changed_input_warning(&gs, sym, + &changed_input_found); + sym->flags |= SYMBOL_WRITTEN; if (!(sym->flags & SYMBOL_WRITE)) goto next; if (need_newline) { fprintf(out, "\n"); need_newline = false; } - sym->flags |= SYMBOL_WRITTEN; print_symbol_for_dotconfig(out, sym); } @@ -892,8 +986,12 @@ end_check: } fclose(out); - for_all_symbols(sym) - sym->flags &= ~SYMBOL_WRITTEN; + conf_clear_written_flags(); + + if (changed_input_found) + conf_changed_input_warning(str_get(&gs)); + + str_free(&gs); if (*tmpname) { if (is_same(name, tmpname)) { diff --git a/scripts/kconfig/expr.c b/scripts/kconfig/expr.c index 16f92c4a775a..2b91d16bf14f 100644 --- a/scripts/kconfig/expr.c +++ b/scripts/kconfig/expr.c @@ -738,6 +738,39 @@ bool expr_contains_symbol(struct expr *dep, struct symbol *sym) return false; } +/* + * Check if the expression references 'sym' in a way that is satisfiable + * with 'sym' disabled, e.g.'sym!=y'. + * + * Expects that expr_transform() was already called on 'expr'. + */ +bool expr_contains_symbol_negated(struct expr *dep, struct symbol *sym) +{ + if (!dep) + return false; + + switch (dep->type) { + case E_AND: + case E_OR: + return expr_contains_symbol_negated(dep->left.expr, sym) || + expr_contains_symbol_negated(dep->right.expr, sym); + case E_NOT: + return dep->left.expr->type == E_SYMBOL && + dep->left.expr->left.sym == sym; + case E_EQUAL: + /* sym=n */ + return dep->left.sym == sym && dep->right.sym == &symbol_no; + case E_UNEQUAL: + /* sym!=y, sym!=m */ + return dep->left.sym == sym && + (dep->right.sym == &symbol_yes || + dep->right.sym == &symbol_mod); + default: + break; + } + return false; +} + bool expr_depends_symbol(struct expr *dep, struct symbol *sym) { if (!dep) diff --git a/scripts/kconfig/expr.h b/scripts/kconfig/expr.h index 5f900d18dae0..b580f9fa0f29 100644 --- a/scripts/kconfig/expr.h +++ b/scripts/kconfig/expr.h @@ -38,7 +38,7 @@ union expr_data { * struct expr - expression * * @node: link node for the hash table - * @type: expressoin type + * @type: expression type * @val: calculated tristate value * @val_is_valid: indicate whether the value is valid * @left: left node @@ -160,7 +160,7 @@ struct symbol { #define SYMBOL_MAXLENGTH 256 -/* A property represent the config options that can be associated +/* A property represents the config options that can be associated * with a config "symbol". * Sample: * config FOO @@ -307,6 +307,7 @@ tristate expr_calc_value(struct expr *e); struct expr *expr_eliminate_dups(struct expr *e); struct expr *expr_transform(struct expr *e); bool expr_contains_symbol(struct expr *dep, struct symbol *sym); +bool expr_contains_symbol_negated(struct expr *dep, struct symbol *sym); bool expr_depends_symbol(struct expr *dep, struct symbol *sym); struct expr *expr_trans_compare(struct expr *e, enum expr_type type, struct symbol *sym); diff --git a/scripts/kconfig/kconfig-sym-check.pl b/scripts/kconfig/kconfig-sym-check.pl new file mode 100755 index 000000000000..daa5285fdefc --- /dev/null +++ b/scripts/kconfig/kconfig-sym-check.pl @@ -0,0 +1,132 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 + +use warnings; +use strict; + +my $srctree = shift @ARGV; +unless (defined $srctree) { + $srctree = `git rev-parse --show-toplevel 2>/dev/null`; + chomp $srctree; + my $msg = "Usage: $0 <srctree> [excludes file]\n"; + $msg .= "Please provide <srctree>."; + $msg .= " Is it '$srctree'?" if $srctree; + $msg .= "\n"; + die $msg; +} +my $kconfig_sym_check_excludes = defined $ARGV[0] ? $ARGV[0] : undef; + +sub indent_depth { + my ($ws) = @_; + my $col = 0; + for my $c (split //, $ws) { + $col = $c eq "\t" ? int($col / 8) * 8 + 8 : $col + 1; + } + return $col; +} + +my @files = `git -C \Q$srctree\E ls-files '*Kconfig*' 2>/dev/null`; +if (@files) { + chomp @files; + @files = map { "$srctree/$_" } @files; +} else { + @files = `find \Q$srctree\E -name '*Kconfig*'`; + chomp @files; +} + +@files = grep { !m{/scripts/kconfig/tests/} } @files; + +my %configs = (); +my %refs = (); + +foreach my $file (@files) { + open F, $file or die "Cannot open $file: $!"; + + my $help = 0; + my $help_level; + my $level; + + while (<F>) { + chomp; + + while (/\\\s*$/) { + s/\\\s*$/ /; + my $cont = <F> // last; + chomp $cont; + $_ .= $cont; + } + + next if /^\s*$/; + next if /^\s*#/; + + /^(\s*)/; + $level = indent_depth($1); + + if ($help && $level < $help_level) { + $help = 0; + } + + next if ($help); + + if (/^\s*(help|\-\-\-help\-\-\-)$/) { + $help = 1; + my $next; + while (defined($next = <F>)) { + last unless $next =~ /^\s*(?:#.*)?$/; + } + last unless defined $next; + $next =~ /^(\s*)/; + if (indent_depth($1) >= $level) { + $help_level = indent_depth($1); + } else { + $help = 0; + } + $_ = $next; + redo; + } + + if (/^\s*(config|menuconfig)\s+([a-zA-Z0-9_]+)\s*(#.*)?$/) { + $configs{$2}++; + next; + } + + if (/^\s*(default|def_bool|def_tristate|select|depends\s+on|imply|visible\s+if|range|if|bool|tristate|int|hex|string|prompt)\s+(.+)\s*$/) { + my $s = $2; + $s =~ s/"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'//g; + $s =~ s/#.*//; + $s =~ s/\$\((?:[^()]*|\((?:[^()]*|\([^()]*\))*\))*\)//g; + $s =~ s/%%[^%]*%%//g; + my @syms = split /[^a-zA-Z0-9_]+/, $s; + map { + $refs{$_}++ if (/[a-zA-Z]/ && $_ ne "if" && $_ ne "y" && $_ ne "n" && $_ ne "m" && !/^0[xX][0-9a-fA-F]+$/); + } @syms + } + } + + close F; +} + +my %known_syms = (); +if (defined $kconfig_sym_check_excludes) { + my $file = $kconfig_sym_check_excludes; + open(F, "<", $file) or die "Cannot open $file: $!"; + while (<F>) { + chomp; + next if /^\s*$/; + next if /^\s*#/; + $known_syms{$1}++ if (/^\s*([a-zA-Z0-9_]+)\s*(#.*)?$/); + } +} + +my $ret = 0; +foreach my $k (sort keys %refs) { + next if (exists $configs{$k} || exists $known_syms{$k}); + + print "$k"; + print " - warning: '$k' is probably not what you want; Kconfig tristate literals are always lowercase ('n', 'y', 'm')" if ($k eq "N" || $k eq "Y" || $k eq "M"); + print "\n"; + + $ret = 1; +} + +exit $ret; diff --git a/scripts/kconfig/lexer.l b/scripts/kconfig/lexer.l index 6d2c92c6095d..a6155422b4a6 100644 --- a/scripts/kconfig/lexer.l +++ b/scripts/kconfig/lexer.l @@ -402,7 +402,7 @@ void zconf_initscan(const char *name) exit(1); } - cur_filename = file_lookup(name); + cur_filename = file_lookup(name, NULL, 0); yylineno = 1; } @@ -443,7 +443,7 @@ void zconf_nextfile(const char *name) } yylineno = 1; - cur_filename = file_lookup(name); + cur_filename = file_lookup(name, cur_filename, cur_lineno); } static void zconf_endfile(void) diff --git a/scripts/kconfig/lkc.h b/scripts/kconfig/lkc.h index 798985961215..7e6f6ca299cf 100644 --- a/scripts/kconfig/lkc.h +++ b/scripts/kconfig/lkc.h @@ -51,7 +51,8 @@ static inline void xfwrite(const void *str, size_t len, size_t count, FILE *out) } /* util.c */ -const char *file_lookup(const char *name); +const char *file_lookup(const char *name, + const char *parent_name, int parent_lineno); /* lexer.l */ int yylex(void); diff --git a/scripts/kconfig/menu.c b/scripts/kconfig/menu.c index b2d8d4e11e07..9c079e92a9ed 100644 --- a/scripts/kconfig/menu.c +++ b/scripts/kconfig/menu.c @@ -428,9 +428,19 @@ static void _menu_finalize(struct menu *parent, bool inside_choice) if (!expr_contains_symbol(dep, sym)) /* No dependency, quit */ break; + /* + * Note that it's actually possible to depend on both + * 'SYM!=y' and 'SYM=y', so we need to first check if + * it's a positive dependency before checking if it's + * a negative dependency. See example: + * 'SFC && MTD && !(SFC=y && MTD=m)' + */ if (expr_depends_symbol(dep, sym)) /* Absolute dependency, put in submenu */ goto next; + if (expr_contains_symbol_negated(dep, sym)) + /* Negative dependency, quit */ + break; /* * Also consider it a dependency on sym if our diff --git a/scripts/kconfig/merge_config.sh b/scripts/kconfig/merge_config.sh index 735e1de450c6..ec242e03f509 100755 --- a/scripts/kconfig/merge_config.sh +++ b/scripts/kconfig/merge_config.sh @@ -122,7 +122,7 @@ fi MERGE_LIST=$* -TMP_FILE=$(mktemp ./.tmp.config.XXXXXXXXXX) +TMP_FILE=$(mktemp --tmpdir="$OUTPUT" .tmp.config.XXXXXXXXXX) echo "Using $INITFILE as base" @@ -151,6 +151,7 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do if ! "$AWK" -v prefix="$CONFIG_PREFIX" \ -v warnoverride="$WARNOVERRIDE" \ -v strict="$STRICT" \ + -v outfile="$TMP_FILE.new" \ -v builtin="$BUILTIN" \ -v warnredun="$WARNREDUN" ' BEGIN { @@ -195,7 +196,7 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do # First pass: read merge file, store all lines and index FILENAME == ARGV[1] { - mergefile = FILENAME + mergefile = FILENAME merge_lines[FNR] = $0 merge_total = FNR cfg = get_cfg($0) @@ -212,17 +213,17 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do # Not a config or not in merge file - keep it if (cfg == "" || !(cfg in merge_cfg)) { - print $0 >> ARGV[3] + print $0 >> outfile next } - prev_val = $0 + prev_val = $0 new_val = merge_cfg[cfg] # BUILTIN: do not demote y to m if (builtin == "true" && new_val ~ /=m$/ && prev_val ~ /=y$/) { warn_builtin(cfg, prev_val, new_val) - print $0 >> ARGV[3] + print $0 >> outfile skip_merge[merge_cfg_line[cfg]] = 1 next } @@ -235,7 +236,7 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do # "=n" is the same as "is not set" if (prev_val ~ /=n$/ && new_val ~ / is not set$/) { - print $0 >> ARGV[3] + print $0 >> outfile next } @@ -246,25 +247,20 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do } } - # output file, skip all lines - FILENAME == ARGV[3] { - nextfile - } - END { # Newline in case base file lacks trailing newline - print "" >> ARGV[3] + print "" >> outfile # Append merge file, skipping lines marked for builtin preservation for (i = 1; i <= merge_total; i++) { if (!(i in skip_merge)) { - print merge_lines[i] >> ARGV[3] + print merge_lines[i] >> outfile } } if (strict_violated) { exit 1 } }' \ - "$ORIG_MERGE_FILE" "$TMP_FILE" "$TMP_FILE.new"; then + "$ORIG_MERGE_FILE" "$TMP_FILE"; then # awk exited non-zero, strict mode was violated STRICT_MODE_VIOLATED=true fi @@ -381,7 +377,7 @@ END { STRICT_MODE_VIOLATED=true fi -if [ "$STRICT" == "true" ] && [ "$STRICT_MODE_VIOLATED" == "true" ]; then +if [ "$STRICT" = "true" ] && [ "$STRICT_MODE_VIOLATED" = "true" ]; then echo "Requested and effective config differ" exit 1 fi diff --git a/scripts/kconfig/parser.y b/scripts/kconfig/parser.y index 6d1bbee38f5d..5fb6f07b6ad2 100644 --- a/scripts/kconfig/parser.y +++ b/scripts/kconfig/parser.y @@ -159,14 +159,8 @@ config_stmt: config_entry_start config_option_list yynerrs++; } - /* - * If the same symbol appears twice in a choice block, the list - * node would be added twice, leading to a broken linked list. - * list_empty() ensures that this symbol has not yet added. - */ - if (list_empty(¤t_entry->sym->choice_link)) - list_add_tail(¤t_entry->sym->choice_link, - ¤t_choice->choice_members); + list_add_tail(¤t_entry->sym->choice_link, + ¤t_choice->choice_members); } printd(DEBUG_PARSE, "%s:%d:endconfig\n", cur_filename, cur_lineno); @@ -546,11 +540,10 @@ static int choice_check_sanity(const struct menu *menu) ret = -1; } - if (prop->menu != menu && prop->type == P_PROMPT && - prop->menu->parent != menu->parent) { + if (prop->menu != menu && prop->type == P_PROMPT) { fprintf(stderr, "%s:%d: error: %s", prop->filename, prop->lineno, - "choice value has a prompt outside its choice group\n"); + "choice value must not have a prompt in another entry\n"); ret = -1; } } diff --git a/scripts/kconfig/tests/conftest.py b/scripts/kconfig/tests/conftest.py index d94b79e012c0..66f95e4ed58c 100644 --- a/scripts/kconfig/tests/conftest.py +++ b/scripts/kconfig/tests/conftest.py @@ -37,7 +37,8 @@ class Conf: # runners def _run_conf(self, mode, dot_config=None, out_file='.config', - interactive=False, in_keys=None, extra_env={}): + interactive=False, in_keys=None, extra_env={}, + silent=False): """Run text-based Kconfig executable and save the result. mode: input mode option (--oldaskconfig, --defconfig=<file> etc.) @@ -48,7 +49,10 @@ class Conf: extra_env: additional environments returncode: exit status of the Kconfig executable """ - command = [CONF_PATH, mode, 'Kconfig'] + command = [CONF_PATH] + if silent: + command.append('-s') + command += [mode, 'Kconfig'] # Override 'srctree' environment to make the test as the top directory extra_env['srctree'] = self._test_dir diff --git a/scripts/kconfig/tests/err_repeated_inc/Kconfig b/scripts/kconfig/tests/err_repeated_inc/Kconfig new file mode 100644 index 000000000000..09a88fd29cb5 --- /dev/null +++ b/scripts/kconfig/tests/err_repeated_inc/Kconfig @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only + +source "Kconfig.inc1" diff --git a/scripts/kconfig/tests/err_repeated_inc/Kconfig.inc1 b/scripts/kconfig/tests/err_repeated_inc/Kconfig.inc1 new file mode 100644 index 000000000000..495dc38314a1 --- /dev/null +++ b/scripts/kconfig/tests/err_repeated_inc/Kconfig.inc1 @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only + +source "Kconfig.inc2" +source "Kconfig.inc3" diff --git a/scripts/kconfig/tests/err_repeated_inc/Kconfig.inc2 b/scripts/kconfig/tests/err_repeated_inc/Kconfig.inc2 new file mode 100644 index 000000000000..2b630eec2e99 --- /dev/null +++ b/scripts/kconfig/tests/err_repeated_inc/Kconfig.inc2 @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only + +source "Kconfig.inc3" diff --git a/scripts/kconfig/tests/err_repeated_inc/Kconfig.inc3 b/scripts/kconfig/tests/err_repeated_inc/Kconfig.inc3 new file mode 100644 index 000000000000..a4e40e534e6a --- /dev/null +++ b/scripts/kconfig/tests/err_repeated_inc/Kconfig.inc3 @@ -0,0 +1 @@ +# SPDX-License-Identifier: GPL-2.0-only diff --git a/scripts/kconfig/tests/err_repeated_inc/__init__.py b/scripts/kconfig/tests/err_repeated_inc/__init__.py new file mode 100644 index 000000000000..129d740a874b --- /dev/null +++ b/scripts/kconfig/tests/err_repeated_inc/__init__.py @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 +""" +Detect repeated inclusion error. + +If repeated inclusion is detected, it should fail with error message. +""" + +def test(conf): + assert conf.oldaskconfig() != 0 + assert conf.stderr_contains('expected_stderr') diff --git a/scripts/kconfig/tests/err_repeated_inc/expected_stderr b/scripts/kconfig/tests/err_repeated_inc/expected_stderr new file mode 100644 index 000000000000..53071430ea7d --- /dev/null +++ b/scripts/kconfig/tests/err_repeated_inc/expected_stderr @@ -0,0 +1,2 @@ +Kconfig.inc1:4: error: repeated inclusion of Kconfig.inc3 +Kconfig.inc2:3: note: location of first inclusion of Kconfig.inc3 diff --git a/scripts/kconfig/tests/no_write_if_dep_unmet/__init__.py b/scripts/kconfig/tests/no_write_if_dep_unmet/__init__.py index ffd469d1f226..791ed659c76b 100644 --- a/scripts/kconfig/tests/no_write_if_dep_unmet/__init__.py +++ b/scripts/kconfig/tests/no_write_if_dep_unmet/__init__.py @@ -8,7 +8,7 @@ for symbols with unmet dependency. This was not working correctly for choice values because choice needs a bit different symbol computation. -This checks that no unneeded "# COFIG_... is not set" is contained in +This checks that no unneeded "# CONFIG_... is not set" is contained in the .config file. Related Linux commit: cb67ab2cd2b8abd9650292c986c79901e3073a59 diff --git a/scripts/kconfig/tests/warn_changed_input/Kconfig b/scripts/kconfig/tests/warn_changed_input/Kconfig new file mode 100644 index 000000000000..69845e2f3fb3 --- /dev/null +++ b/scripts/kconfig/tests/warn_changed_input/Kconfig @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: GPL-2.0 + +config DEP + bool "DEP" + help + Test dependency symbol for Kconfig warning coverage. + This is used by the warn_changed_input selftest. + It intentionally stays unset in the input fragment. + The test checks how dependent user input is adjusted. + +config A + bool "A" + depends on DEP + help + Test bool symbol for changed-input diagnostics. + The input fragment requests this symbol as built-in. + The unmet dependency on DEP forces the final value to n. + The warning should report that downgrade. + +config NUM + int "NUM" + range 10 20 + help + Test integer symbol for changed-input diagnostics. + The input fragment requests a value outside the allowed range. + Kconfig resolves it to the constrained in-range value. + The warning should report that adjustment. + +config DUP + bool "DUP" + depends on DEP + help + Test duplicate-definition handling for changed-input diagnostics. + The input fragment requests this symbol as built-in. + The duplicate definition below must not produce a duplicate warning. + This keeps the warning output stable for repeated menu entries. + +config DUP + bool + depends on DEP diff --git a/scripts/kconfig/tests/warn_changed_input/__init__.py b/scripts/kconfig/tests/warn_changed_input/__init__.py new file mode 100644 index 000000000000..4c3bca6af846 --- /dev/null +++ b/scripts/kconfig/tests/warn_changed_input/__init__.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: GPL-2.0 +""" +Test optional warnings for user-provided values changed by Kconfig. + +Warnings should stay disabled by default, and should only appear when +KCONFIG_WARN_CHANGED_INPUT is enabled. +""" + + +def test(conf): + assert conf.olddefconfig('config') == 0 + assert 'user-provided values changed by Kconfig' not in conf.stderr + + assert conf._run_conf('--olddefconfig', dot_config='config', + extra_env={ + 'KCONFIG_WARN_CHANGED_INPUT': '1', + }) == 0 + assert conf.stderr_contains('expected_stderr') + assert conf.config_matches('expected_config') + + assert conf._run_conf('--olddefconfig', dot_config='config', + extra_env={ + 'KCONFIG_WARN_CHANGED_INPUT': '1', + }, silent=True) == 0 + assert conf.stderr_contains('expected_stderr') + + assert conf._run_conf('--savedefconfig=defconfig', dot_config='config', + out_file='defconfig', + extra_env={ + 'KCONFIG_WARN_CHANGED_INPUT': '1', + }) == 0 + assert conf.stderr_contains('expected_stderr') + assert conf.config_matches('expected_defconfig') diff --git a/scripts/kconfig/tests/warn_changed_input/config b/scripts/kconfig/tests/warn_changed_input/config new file mode 100644 index 000000000000..dbe93ff26408 --- /dev/null +++ b/scripts/kconfig/tests/warn_changed_input/config @@ -0,0 +1,3 @@ +CONFIG_A=y +CONFIG_NUM=30 +CONFIG_DUP=y diff --git a/scripts/kconfig/tests/warn_changed_input/expected_config b/scripts/kconfig/tests/warn_changed_input/expected_config new file mode 100644 index 000000000000..fe8bbec66c53 --- /dev/null +++ b/scripts/kconfig/tests/warn_changed_input/expected_config @@ -0,0 +1,6 @@ +# +# Automatically generated file; DO NOT EDIT. +# Main menu +# +# CONFIG_DEP is not set +CONFIG_NUM=20 diff --git a/scripts/kconfig/tests/warn_changed_input/expected_defconfig b/scripts/kconfig/tests/warn_changed_input/expected_defconfig new file mode 100644 index 000000000000..af9e34851d2a --- /dev/null +++ b/scripts/kconfig/tests/warn_changed_input/expected_defconfig @@ -0,0 +1 @@ +CONFIG_NUM=20 diff --git a/scripts/kconfig/tests/warn_changed_input/expected_stderr b/scripts/kconfig/tests/warn_changed_input/expected_stderr new file mode 100644 index 000000000000..9ec8446b4ac2 --- /dev/null +++ b/scripts/kconfig/tests/warn_changed_input/expected_stderr @@ -0,0 +1,4 @@ +warning: user-provided values changed by Kconfig: + CONFIG_A: y -> n + CONFIG_NUM: 30 -> 20 + CONFIG_DUP: y -> n diff --git a/scripts/kconfig/util.c b/scripts/kconfig/util.c index 5cdcee144b58..0809aa061b6a 100644 --- a/scripts/kconfig/util.c +++ b/scripts/kconfig/util.c @@ -18,25 +18,50 @@ static HASHTABLE_DEFINE(file_hashtable, 1U << 11); struct file { struct hlist_node node; + struct { + const char *name; + int lineno; + } parent; char name[]; }; +static void die_duplicated_include(struct file *file, + const char *parent, int lineno) +{ + fprintf(stderr, + "%s:%d: error: repeated inclusion of %s\n" + "%s:%d: note: location of first inclusion of %s\n", + parent, lineno, file->name, + file->parent.name, file->parent.lineno, file->name); + exit(1); +} + /* file already present in list? If not add it */ -const char *file_lookup(const char *name) +const char *file_lookup(const char *name, + const char *parent_name, int parent_lineno) { + const char *parent = NULL; struct file *file; size_t len; int hash = hash_str(name); + if (parent_name) + parent = file_lookup(parent_name, NULL, 0); + hash_for_each_possible(file_hashtable, file, node, hash) - if (!strcmp(name, file->name)) - return file->name; + if (!strcmp(name, file->name)) { + if (!parent_name) + return file->name; + die_duplicated_include(file, parent, parent_lineno); + } len = strlen(name); file = xmalloc(sizeof(*file) + len + 1); memset(file, 0, sizeof(*file)); memcpy(file->name, name, len); file->name[len] = '\0'; + file->parent.name = parent; + file->parent.lineno = parent_lineno; hash_add(file_hashtable, &file->node, hash); diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh index f99e196abeea..ab0b8125c8cb 100755 --- a/scripts/link-vmlinux.sh +++ b/scripts/link-vmlinux.sh @@ -38,7 +38,7 @@ is_enabled() { } # Nice output in kbuild format -# Will be supressed by "make -s" +# Will be suppressed by "make -s" info() { printf " %-7s %s\n" "${1}" "${2}" @@ -106,6 +106,18 @@ vmlinux_link() ${kallsymso} ${btf_vmlinux_bin_o} ${arch_vmlinux_o} ${ldlibs} } +# Check if kallsymso_prev and kallsymso differ +# If symbol sizes within ${kallsymso} change, any symbols within vmlinux are +# likely to shift, invalidating ${kallsymso}. +# Since file size can remain unchanged even if symbol sizes change, compare the +# actual symbols instead of relying on file size only. +kallsymso_changed() +{ + ${NM} -n "${kallsymso_prev}" > "${kallsymso_prev}.sym" + ${NM} -n "${kallsymso}" > "${kallsymso}.sym" + ! cmp -s "${kallsymso_prev}.sym" "${kallsymso}.sym" +} + # Create ${2}.o file with all symbols from the ${1} object file kallsyms() { @@ -126,6 +138,7 @@ kallsyms() ${CC} ${NOSTDINC_FLAGS} ${LINUXINCLUDE} ${KBUILD_CPPFLAGS} \ ${KBUILD_AFLAGS} ${KBUILD_AFLAGS_KERNEL} -c -o "${2}.o" "${2}.S" + kallsymso_prev="${kallsymso:-}" kallsymso=${2}.o } @@ -255,7 +268,12 @@ if is_enabled CONFIG_KALLSYMS; then sysmap_and_kallsyms .tmp_vmlinux2 size2=$(${CONFIG_SHELL} "${srctree}/scripts/file-size.sh" ${kallsymso}) - if [ $size1 -ne $size2 ] || [ -n "${KALLSYMS_EXTRA_PASS}" ]; then + # Due to alignment, file size of the kallsymso object file might remain + # unchanged even if individual symbols within change size. Changed + # symbol sizes can still shift other symbols, though. Therefore, don't + # rely on file size alone. + if [ $size1 -ne $size2 ] || kallsymso_changed || \ + [ -n "${KALLSYMS_EXTRA_PASS}" ]; then vmlinux_link .tmp_vmlinux3 sysmap_and_kallsyms .tmp_vmlinux3 fi diff --git a/scripts/livepatch/Makefile b/scripts/livepatch/Makefile new file mode 100644 index 000000000000..17b590213740 --- /dev/null +++ b/scripts/livepatch/Makefile @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: GPL-2.0 +# Standalone Makefile for developer tooling (not part of kbuild). + +SHELLCHECK := $(shell which shellcheck 2> /dev/null) + +SRCS := \ + klp-build + +.DEFAULT_GOAL := help +.PHONY: help +help: + @echo " check - Run shellcheck on $(SRCS)" + @echo " help - Show this help message" + +.PHONY: check +check: +ifndef SHELLCHECK + $(error shellcheck is not installed. Please install it to run checks) +endif + @$(SHELLCHECK) $(SHELLCHECK_OPTIONS) $(SRCS) diff --git a/scripts/livepatch/init.c b/scripts/livepatch/init.c index f14d8c8fb35f..16aff8f736eb 100644 --- a/scripts/livepatch/init.c +++ b/scripts/livepatch/init.c @@ -51,7 +51,7 @@ static int __init livepatch_mod_init(void) if (!funcs) { ret = -ENOMEM; for (int j = 0; j < i; j++) - kfree(objs[i].funcs); + kfree(objs[j].funcs); goto err_free_objs; } diff --git a/scripts/livepatch/klp-build b/scripts/livepatch/klp-build index 7b82c7503c2b..b52a8489d9f6 100755 --- a/scripts/livepatch/klp-build +++ b/scripts/livepatch/klp-build @@ -3,7 +3,7 @@ # # Build a livepatch module -# shellcheck disable=SC1090,SC2155 +# shellcheck disable=SC1090,SC2155,SC2164 if (( BASH_VERSINFO[0] < 4 || \ (BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] < 4) )); then @@ -11,21 +11,19 @@ if (( BASH_VERSINFO[0] < 4 || \ exit 1 fi -set -o errexit set -o errtrace set -o pipefail set -o nounset # Allow doing 'cmd | mapfile -t array' instead of 'mapfile -t array < <(cmd)'. -# This helps keep execution in pipes so pipefail+errexit can catch errors. +# This helps keep execution in pipes so pipefail+ERR trap can catch errors. shopt -s lastpipe -unset DEBUG_CLONE DIFF_CHECKSUM SKIP_CLEANUP XTRACE +unset DEBUG_CLONE DIFF_CHECKSUM SKIP_CLEANUP VERBOSE XTRACE REPLACE=1 SHORT_CIRCUIT=0 JOBS="$(getconf _NPROCESSORS_ONLN)" -VERBOSE="-s" shopt -o xtrace | grep -q 'on' && XTRACE=1 # Avoid removing the previous $TMP_DIR until args have been fully processed. @@ -35,16 +33,16 @@ SCRIPT="$(basename "$0")" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" FIX_PATCH_LINES="$SCRIPT_DIR/fix-patch-lines" -SRC="$(pwd)" -OBJ="$(pwd)" +OBJTOOL="$PWD/tools/objtool/objtool" +CONFIG="$PWD/.config" +TMP_DIR="$PWD/klp-tmp" -CONFIG="$OBJ/.config" -TMP_DIR="$OBJ/klp-tmp" - -ORIG_DIR="$TMP_DIR/orig" -PATCHED_DIR="$TMP_DIR/patched" -DIFF_DIR="$TMP_DIR/diff" -KMOD_DIR="$TMP_DIR/kmod" +ORIG_DIR="$TMP_DIR/1-orig" +PATCHED_DIR="$TMP_DIR/2-patched" +ORIG_CSUM_DIR="$TMP_DIR/3-checksum-orig" +PATCHED_CSUM_DIR="$TMP_DIR/3-checksum-patched" +DIFF_DIR="$TMP_DIR/4-diff" +KMOD_DIR="$TMP_DIR/5-kmod" STASH_DIR="$TMP_DIR/stash" TIMESTAMP="$TMP_DIR/timestamp" @@ -52,20 +50,37 @@ PATCH_TMP_DIR="$TMP_DIR/tmp" KLP_DIFF_LOG="$DIFF_DIR/diff.log" +# Terminal output colors +read -r COLOR_RESET COLOR_BOLD COLOR_ERROR COLOR_WARN <<< "" +if [[ -t 1 && -t 2 ]]; then + COLOR_RESET="\033[0m" + COLOR_BOLD="\033[1m" + COLOR_ERROR="\033[0;31m" + COLOR_WARN="\033[0;33m" +fi + grep0() { + # shellcheck disable=SC2317 command grep "$@" || true } +# Because pipefail is enabled, the grep0 helper should be used instead of +# grep, otherwise a failed match can propagate to an error. +grep() { + echo "error: $SCRIPT: use grep0 or 'command grep' instead of bare grep" >&2 + exit 1 +} + status() { - echo "$*" + echo -e "${COLOR_BOLD}$*${COLOR_RESET}" } warn() { - echo "error: $SCRIPT: $*" >&2 + echo -e "${COLOR_WARN}warning${COLOR_RESET}: $SCRIPT: $*" >&2 } die() { - warn "$@" + echo -e "${COLOR_ERROR}error${COLOR_RESET}: $SCRIPT: $*" >&2 exit 1 } @@ -73,7 +88,7 @@ declare -a STASHED_FILES stash_file() { local file="$1" - local rel_file="${file#"$SRC"/}" + local rel_file="${file#"$PWD"/}" [[ ! -e "$file" ]] && die "no file to stash: $file" @@ -87,7 +102,7 @@ restore_files() { local file for file in "${STASHED_FILES[@]}"; do - mv -f "$STASH_DIR/$file" "$SRC/$file" || warn "can't restore file: $file" + mv -f "$STASH_DIR/$file" "$PWD/$file" || warn "can't restore file: $file" done STASHED_FILES=() @@ -95,14 +110,14 @@ restore_files() { cleanup() { set +o nounset - revert_patches "--recount" + revert_patches restore_files [[ "$KEEP_TMP" -eq 0 ]] && rm -rf "$TMP_DIR" return 0 } trap_err() { - warn "line ${BASH_LINENO[0]}: '$BASH_COMMAND'" + die "line ${BASH_LINENO[0]}: '$BASH_COMMAND'" } trap cleanup EXIT INT TERM HUP @@ -123,10 +138,11 @@ Options: Advanced Options: -d, --debug Show symbol/reloc cloning decisions -S, --short-circuit=STEP Start at build step (requires prior --keep-tmp) - 1|orig Build original kernel (default) - 2|patched Build patched kernel - 3|diff Diff objects - 4|kmod Build patch module + 1|orig Build original kernel (default) + 2|patched Build patched kernel + 3|checksum Generate checksums + 4|diff Diff objects + 5|kmod Build patch module -T, --keep-tmp Preserve tmp dir on exit EOF @@ -141,6 +157,7 @@ process_args() { local short local long local args + local patch short="hfj:o:vdS:T" long="help,show-first-changed,jobs:,output:,no-replace,verbose,debug,short-circuit:,keep-tmp" @@ -177,7 +194,7 @@ process_args() { shift ;; -v | --verbose) - VERBOSE="V=1" + VERBOSE=1 shift ;; -d | --debug) @@ -189,10 +206,11 @@ process_args() { [[ ! -d "$TMP_DIR" ]] && die "--short-circuit requires preserved klp-tmp dir" keep_tmp=1 case "$2" in - 1 | orig) SHORT_CIRCUIT=1; ;; - 2 | patched) SHORT_CIRCUIT=2; ;; - 3 | diff) SHORT_CIRCUIT=3; ;; - 4 | mod) SHORT_CIRCUIT=4; ;; + 1 | orig) SHORT_CIRCUIT=1; ;; + 2 | patched) SHORT_CIRCUIT=2; ;; + 3 | checksum) SHORT_CIRCUIT=3; ;; + 4 | diff) SHORT_CIRCUIT=4; ;; + 5 | kmod) SHORT_CIRCUIT=5; ;; *) die "invalid short-circuit step '$2'" ;; esac shift 2 @@ -212,13 +230,17 @@ process_args() { esac done - if [[ $# -eq 0 ]]; then + if [[ $# -eq 0 ]] && (( SHORT_CIRCUIT <= 2 )); then usage exit 1 fi KEEP_TMP="$keep_tmp" PATCHES=("$@") + + for patch in "${PATCHES[@]}"; do + [[ -f "$patch" ]] || die "$patch doesn't exist" + done } # temporarily disable xtrace for especially verbose code @@ -249,10 +271,16 @@ validate_config() { [[ -v CONFIG_GCC_PLUGIN_RANDSTRUCT ]] && \ die "kernel option 'CONFIG_GCC_PLUGIN_RANDSTRUCT' not supported" + [[ -v CONFIG_LD_DEAD_CODE_DATA_ELIMINATION ]] && \ + die "kernel option 'CONFIG_LD_DEAD_CODE_DATA_ELIMINATION' not supported" + [[ -v CONFIG_AS_IS_LLVM ]] && \ [[ "$CONFIG_AS_VERSION" -lt 200000 ]] && \ die "Clang assembler version < 20 not supported" + [[ -x "$OBJTOOL" ]] && "$OBJTOOL" klp 2>&1 | command grep -q "not implemented" && \ + die "objtool not built with KLP support; install xxhash-devel/libxxhash-dev (version >= 0.8) and recompile" + return 0 } @@ -282,41 +310,49 @@ set_module_name() { } # Hardcode the value printed by the localversion script to prevent patch -# application from appending it with '+' due to a dirty git working tree. +# application from appending it with '+' due to a dirty working tree. set_kernelversion() { - local file="$SRC/scripts/setlocalversion" + local file="$PWD/scripts/setlocalversion" local kernelrelease stash_file "$file" - kernelrelease="$(cd "$SRC" && make syncconfig &>/dev/null && make -s kernelrelease)" + if [[ -n "$(make -s listnewconfig 2>/dev/null)" ]]; then + die ".config mismatch, check your .config or run 'make olddefconfig'" + fi + make syncconfig &>/dev/null || die "make syncconfig failed" + + kernelrelease="$(make -s kernelrelease)" [[ -z "$kernelrelease" ]] && die "failed to get kernel version" sed -i "2i echo $kernelrelease; exit 0" scripts/setlocalversion } -get_patch_files() { +get_patch_input_files() { local patch="$1" - grep0 -E '^(--- |\+\+\+ )' "$patch" \ + grep0 -E '^--- ' "$patch" \ + | grep0 -v -e '/dev/null' -e '1969-12-31' -e '1970-01-01' \ | gawk '{print $2}' \ | sed 's|^[^/]*/||' \ | sort -u } -# Make sure git re-stats the changed files -git_refresh() { +get_patch_output_files() { local patch="$1" - local files=() - [[ ! -e "$SRC/.git" ]] && return + grep0 -E '^\+\+\+ ' "$patch" \ + | grep0 -v -e '/dev/null' -e '1969-12-31' -e '1970-01-01' \ + | gawk '{print $2}' \ + | sed 's|^[^/]*/||' \ + | sort -u +} - get_patch_files "$patch" | mapfile -t files +get_patch_files() { + local patch="$1" - ( - cd "$SRC" - git update-index -q --refresh -- "${files[@]}" - ) + { get_patch_input_files "$patch"; get_patch_output_files "$patch"; } \ + | sort -u } check_unsupported_patches() { @@ -329,8 +365,8 @@ check_unsupported_patches() { for file in "${files[@]}"; do case "$file" in - lib/*|*.S) - die "unsupported patch to $file" + lib/*|*/vdso/*|*/realmode/rm/*|*.S) + die "${patch}: unsupported patch to $file" ;; esac done @@ -341,34 +377,30 @@ apply_patch() { local patch="$1" shift local extra_args=("$@") + local drift_regex="with fuzz|offset [0-9]+ line" + local output + local status [[ ! -f "$patch" ]] && die "$patch doesn't exist" - - ( - cd "$SRC" - - # The sed strips the version signature from 'git format-patch', - # otherwise 'git apply --recount' warns. - sed -n '/^-- /q;p' "$patch" | - git apply "${extra_args[@]}" - ) + status=0 + output=$(patch -p1 --dry-run --no-backup-if-mismatch -r /dev/null "${extra_args[@]}" < "$patch" 2>&1) || status=$? + if [[ "$status" -ne 0 ]]; then + echo "$output" >&2 + die "$patch did not apply" + elif [[ "$output" =~ $drift_regex ]]; then + [[ -v VERBOSE ]] && echo "$output" >&2 + warn "${patch} applied with fuzz" + fi APPLIED_PATCHES+=("$patch") + patch -p1 --no-backup-if-mismatch -r /dev/null "${extra_args[@]}" --silent < "$patch" } revert_patch() { local patch="$1" - shift - local extra_args=("$@") local tmp=() - ( - cd "$SRC" - - sed -n '/^-- /q;p' "$patch" | - git apply --reverse "${extra_args[@]}" - ) - git_refresh "$patch" + patch -p1 -R --force --no-backup-if-mismatch -r /dev/null &> /dev/null < "$patch" || true for p in "${APPLIED_PATCHES[@]}"; do [[ "$p" == "$patch" ]] && continue @@ -379,19 +411,19 @@ revert_patch() { } apply_patches() { + local extra_args=("$@") local patch for patch in "${PATCHES[@]}"; do - apply_patch "$patch" + apply_patch "$patch" "${extra_args[@]}" done } revert_patches() { - local extra_args=("$@") local patches=("${APPLIED_PATCHES[@]}") for (( i=${#patches[@]}-1 ; i>=0 ; i-- )) ; do - revert_patch "${patches[$i]}" "${extra_args[@]}" + revert_patch "${patches[$i]}" done APPLIED_PATCHES=() @@ -406,8 +438,21 @@ validate_patches() { do_init() { # We're not yet smart enough to handle anything other than in-tree # builds in pwd. - [[ ! "$SRC" -ef "$SCRIPT_DIR/../.." ]] && die "please run from the kernel root directory" - [[ ! "$OBJ" -ef "$SCRIPT_DIR/../.." ]] && die "please run from the kernel root directory" + [[ ! "$PWD" -ef "$SCRIPT_DIR/../.." ]] && die "please run from the kernel root directory" + + if (( SHORT_CIRCUIT >= 2 )); then + [[ -f "$ORIG_DIR/.complete" ]] || die "-S $SHORT_CIRCUIT requires completed $ORIG_DIR" + fi + if (( SHORT_CIRCUIT >= 3 )); then + [[ -f "$PATCHED_DIR/.complete" ]] || die "-S $SHORT_CIRCUIT requires completed $PATCHED_DIR" + fi + if (( SHORT_CIRCUIT >= 4 )); then + [[ -f "$ORIG_CSUM_DIR/.complete" ]] || die "-S $SHORT_CIRCUIT requires completed $ORIG_CSUM_DIR" + [[ -f "$PATCHED_CSUM_DIR/.complete" ]] || die "-S $SHORT_CIRCUIT requires completed $PATCHED_CSUM_DIR" + fi + if (( SHORT_CIRCUIT >= 5 )); then + [[ -f "$DIFF_DIR/.complete" ]] || die "-S $SHORT_CIRCUIT requires completed $DIFF_DIR" + fi (( SHORT_CIRCUIT <= 1 )) && rm -rf "$TMP_DIR" mkdir -p "$TMP_DIR" @@ -415,6 +460,7 @@ do_init() { APPLIED_PATCHES=() [[ -x "$FIX_PATCH_LINES" ]] || die "can't find fix-patch-lines" + command -v recountdiff &>/dev/null || die "recountdiff not found (install patchutils)" validate_config set_module_name @@ -425,25 +471,27 @@ do_init() { refresh_patch() { local patch="$1" local tmpdir="$PATCH_TMP_DIR" - local files=() + local input_files=() + local output_files=() rm -rf "$tmpdir" mkdir -p "$tmpdir/a" mkdir -p "$tmpdir/b" # Get all source files affected by the patch - get_patch_files "$patch" | mapfile -t files + get_patch_input_files "$patch" | mapfile -t input_files + get_patch_output_files "$patch" | mapfile -t output_files # Copy orig source files to 'a' - ( cd "$SRC" && echo "${files[@]}" | xargs cp --parents --target-directory="$tmpdir/a" ) + echo "${input_files[@]}" | xargs cp --parents --target-directory="$tmpdir/a" # Copy patched source files to 'b' - apply_patch "$patch" --recount - ( cd "$SRC" && echo "${files[@]}" | xargs cp --parents --target-directory="$tmpdir/b" ) - revert_patch "$patch" --recount + apply_patch "$patch" "--silent" + echo "${output_files[@]}" | xargs cp --parents --target-directory="$tmpdir/b" + revert_patch "$patch" # Diff 'a' and 'b' to make a clean patch - ( cd "$tmpdir" && git diff --no-index --no-prefix a b > "$patch" ) || true + ( cd "$tmpdir" && diff -Nupr a b > "$patch" ) || true } # Copy the patches to a temporary directory, fix their lines so as not to @@ -466,8 +514,7 @@ fix_patches() { cp -f "$old_patch" "$tmp_patch" refresh_patch "$tmp_patch" - "$FIX_PATCH_LINES" "$tmp_patch" > "$new_patch" - refresh_patch "$new_patch" + "$FIX_PATCH_LINES" "$tmp_patch" | recountdiff > "$new_patch" PATCHES[i]="$new_patch" @@ -484,19 +531,14 @@ clean_kernel() { cmd+=("-j$JOBS") cmd+=("clean") - ( - cd "$SRC" - "${cmd[@]}" - ) + "${cmd[@]}" } build_kernel() { + local build="$1" local log="$TMP_DIR/build.log" - local objtool_args=() local cmd=() - objtool_args=("--checksum") - cmd=("make") # When a patch to a kernel module references a newly created unexported @@ -516,29 +558,33 @@ build_kernel() { # cmd+=("KBUILD_MODPOST_WARN=1") - cmd+=("$VERBOSE") + cmd+=("KLP_SYMIDS=1") + + if [[ -v VERBOSE ]]; then + cmd+=("V=1") + else + cmd+=("-s") + fi cmd+=("-j$JOBS") cmd+=("KCFLAGS=-ffunction-sections -fdata-sections") - cmd+=("OBJTOOL_ARGS=${objtool_args[*]}") cmd+=("vmlinux") cmd+=("modules") - ( - cd "$SRC" - "${cmd[@]}" \ - 1> >(tee -a "$log") \ - 2> >(tee -a "$log" | grep0 -v "modpost.*undefined!" >&2) - ) + "${cmd[@]}" \ + 1> >(tee -a "$log") \ + 2> >(tee -a "$log" | grep0 -v "modpost.*undefined!" >&2) \ + || die "$build kernel build failed" } find_objects() { local opts=("$@") # Find root-level vmlinux.o and non-root-level .ko files, - # excluding klp-tmp/ and .git/ - find "$OBJ" \( -path "$TMP_DIR" -o -path "$OBJ/.git" -o -regex "$OBJ/[^/][^/]*\.ko" \) -prune -o \ + # excluding klp-tmp/ and hidden directories. + find "$PWD" -mindepth 1 \ + \( -path "$TMP_DIR" -o -name ".*" -o -regex "$PWD/[^/][^/]*\.ko" \) -prune -o \ -type f "${opts[@]}" \ - \( -name "*.ko" -o -path "$OBJ/vmlinux.o" \) \ + \( -name "*.ko" -o -path "$PWD/vmlinux.o" \) \ -printf '%P\n' } @@ -551,11 +597,10 @@ copy_orig_objects() { find_objects | mapfile -t files - xtrace_save "copying orig objects" + xtrace_save "copying original objects" for _file in "${files[@]}"; do local rel_file="${_file/.ko/.o}" - local file="$OBJ/$rel_file" - local file_dir="$(dirname "$file")" + local file="$PWD/$rel_file" local orig_file="$ORIG_DIR/$rel_file" local orig_dir="$(dirname "$orig_file")" @@ -566,8 +611,11 @@ copy_orig_objects() { done xtrace_restore + cp -f "$PWD/vmlinux" "$ORIG_DIR" || die "missing vmlinux" + mv -f "$TMP_DIR/build.log" "$ORIG_DIR" touch "$TIMESTAMP" + touch "$ORIG_DIR/.complete" } # Copy all changed objects to $PATCHED_DIR @@ -588,7 +636,7 @@ copy_patched_objects() { xtrace_save "copying changed objects" for _file in "${files[@]}"; do local rel_file="${_file/.ko/.o}" - local file="$OBJ/$rel_file" + local file="$PWD/$rel_file" local orig_file="$ORIG_DIR/$rel_file" local patched_file="$PATCHED_DIR/$rel_file" local patched_dir="$(dirname "$patched_file")" @@ -606,6 +654,38 @@ copy_patched_objects() { (( found == 0 )) && die "no changes detected" mv -f "$TMP_DIR/build.log" "$PATCHED_DIR" + touch "$PATCHED_DIR/.complete" +} + +# Copy .o files to a separate directory and run "objtool klp checksum" on each +# copy. The checksums are written to a .discard.sym_checksum section. +# +# If match_dir is given, only process files which also exist there. +generate_checksums() { + local src_dir="$1" + local dest_dir="$2" + local match_dir="${3:-}" + local files=() + local file + + rm -rf "$dest_dir" + mkdir -p "$dest_dir" + + find "$src_dir" -type f -name "*.o" | mapfile -t files + for file in "${files[@]}"; do + local rel="${file#"$src_dir"/}" + local dest="$dest_dir/$rel" + + [[ -n "$match_dir" && ! -f "$match_dir/$rel" ]] && continue + + mkdir -p "$(dirname "$dest")" + cp -f "$file" "$dest" + "$OBJTOOL" klp checksum "$dest" + done + + [[ -f "$src_dir/vmlinux" ]] && cp -f "$src_dir/vmlinux" "$dest_dir" + + touch "$dest_dir/.complete" } # Diff changed objects, writing output object to $DIFF_DIR @@ -617,23 +697,23 @@ diff_objects() { rm -rf "$DIFF_DIR" mkdir -p "$DIFF_DIR" - find "$PATCHED_DIR" -type f -name "*.o" | mapfile -t files + find "$PATCHED_CSUM_DIR" -type f -name "*.o" | mapfile -t files [[ ${#files[@]} -eq 0 ]] && die "no changes detected" [[ -v DEBUG_CLONE ]] && opts=("--debug") # Diff all changed objects for file in "${files[@]}"; do - local rel_file="${file#"$PATCHED_DIR"/}" + local rel_file="${file#"$PATCHED_CSUM_DIR"/}" local orig_file="$rel_file" - local patched_file="$PATCHED_DIR/$rel_file" + local patched_file="$PATCHED_CSUM_DIR/$rel_file" local out_file="$DIFF_DIR/$rel_file" local filter=() local cmd=() mkdir -p "$(dirname "$out_file")" - cmd=("$SRC/tools/objtool/objtool") + cmd=("$OBJTOOL") cmd+=("klp") cmd+=("diff") (( ${#opts[@]} > 0 )) && cmd+=("${opts[@]}") @@ -650,18 +730,21 @@ diff_objects() { fi ( - cd "$ORIG_DIR" + cd "$ORIG_CSUM_DIR" + [[ -v VERBOSE ]] && echo "cd $ORIG_CSUM_DIR && ${cmd[*]}" "${cmd[@]}" \ 1> >(tee -a "$log") \ 2> >(tee -a "$log" | "${filter[@]}" >&2) || \ die "objtool klp diff failed" ) done + + touch "$DIFF_DIR/.complete" } -# For each changed object, run objtool with --debug-checksum to get the -# per-instruction checksums, and then diff those to find the first changed -# instruction for each function. +# For each changed object, run "objtool klp checksum" with --debug-checksum to +# get the per-instruction checksums, and then diff those to find the first +# changed instruction for each function. diff_checksums() { local orig_log="$ORIG_DIR/checksum.log" local patched_log="$PATCHED_DIR/checksum.log" @@ -685,9 +768,8 @@ diff_checksums() { fi done - cmd=("$SRC/tools/objtool/objtool") - cmd+=("--checksum") - cmd+=("--link") + cmd=("$OBJTOOL") + cmd+=("klp" "checksum") cmd+=("--dry-run") for file in "${!funcs[@]}"; do @@ -696,21 +778,37 @@ diff_checksums() { ( cd "$ORIG_DIR" "${cmd[@]}" "$opt" "$file" &> "$orig_log" || \ - ( cat "$orig_log" >&2; die "objtool --debug-checksum failed" ) + ( cat "$orig_log" >&2; die "objtool klp checksum failed" ) cd "$PATCHED_DIR" "${cmd[@]}" "$opt" "$file" &> "$patched_log" || \ - ( cat "$patched_log" >&2; die "objtool --debug-checksum failed" ) + ( cat "$patched_log" >&2; die "objtool klp checksum failed" ) ) for func in ${funcs[$file]}; do - diff <( grep0 -E "^DEBUG: .*checksum: $func " "$orig_log" | sed "s|$ORIG_DIR/||") \ - <( grep0 -E "^DEBUG: .*checksum: $func " "$patched_log" | sed "s|$PATCHED_DIR/||") \ - | gawk '/^< DEBUG: / { - gsub(/:/, "") - printf "%s: %s: %s\n", $3, $5, $6 - exit - }' || true + local -a orig patched + paste <(grep0 -E "^DEBUG: .*checksum: $func " "$orig_log") \ + <(grep0 -E "^DEBUG: .*checksum: $func " "$patched_log") | + while IFS= read -r line; do + read -ra orig <<< "${line%%$'\t'*}" + read -ra patched <<< "${line#*$'\t'}" + + if [[ ${#patched[@]} -eq 0 ]]; then + printf "%s: %s: %s (removed)\n" "${orig[1]%:}" "${orig[3]}" "${orig[-2]}" + break + elif [[ ${#orig[@]} -eq 0 ]]; then + printf "%s: %s: %s (added)\n" "${patched[1]%:}" "${patched[3]}" "${patched[-2]}" + break + fi + + [[ "${orig[-1]}" == "${patched[-1]}" ]] && continue + + printf "%s: %s: %s" "${orig[1]%:}" "${orig[3]}" "${orig[-2]}" + [[ "${orig[-2]}" != "${patched[-2]}" ]] && \ + printf " (patched: %s)" "${patched[-2]}" + printf "\n" + break + done || true done done } @@ -727,7 +825,7 @@ build_patch_module() { rm -rf "$KMOD_DIR" mkdir -p "$KMOD_DIR" - cp -f "$SRC/scripts/livepatch/init.c" "$KMOD_DIR" + cp -f "$SCRIPT_DIR/init.c" "$KMOD_DIR" echo "obj-m := $NAME.o" > "$makefile" echo -n "$NAME-y := init.o" >> "$makefile" @@ -762,19 +860,20 @@ build_patch_module() { [[ $REPLACE -eq 0 ]] && cflags+=("-DKLP_NO_REPLACE") cmd=("make") - cmd+=("$VERBOSE") + if [[ -v VERBOSE ]]; then + cmd+=("V=1") + else + cmd+=("-s") + fi cmd+=("-j$JOBS") cmd+=("--directory=.") cmd+=("M=$KMOD_DIR") cmd+=("KCFLAGS=${cflags[*]}") # Build a "normal" kernel module with init.c and the diffed objects - ( - cd "$SRC" - "${cmd[@]}" \ - 1> >(tee -a "$log") \ - 2> >(tee -a "$log" >&2) - ) + "${cmd[@]}" \ + 1> >(tee -a "$log") \ + 2> >(tee -a "$log" >&2) kmod_file="$KMOD_DIR/$NAME.ko" @@ -785,7 +884,7 @@ build_patch_module() { objcopy --remove-section=.BTF "$kmod_file" # Fix (and work around) linker wreckage for klp syms / relocs - "$SRC/tools/objtool/objtool" klp post-link "$kmod_file" || die "objtool klp post-link failed" + "$OBJTOOL" klp post-link "$kmod_file" || die "objtool klp post-link failed" cp -f "$kmod_file" "$OUTFILE" } @@ -796,12 +895,15 @@ build_patch_module() { process_args "$@" do_init -if (( SHORT_CIRCUIT <= 1 )); then +if (( SHORT_CIRCUIT <= 2 )); then status "Validating patch(es)" validate_patches +fi + +if (( SHORT_CIRCUIT <= 1 )); then status "Building original kernel" clean_kernel - build_kernel + build_kernel "original" status "Copying original object files" copy_orig_objects fi @@ -809,15 +911,22 @@ fi if (( SHORT_CIRCUIT <= 2 )); then status "Fixing patch(es)" fix_patches - apply_patches + apply_patches "--silent" status "Building patched kernel" - build_kernel + build_kernel "patched" revert_patches status "Copying patched object files" copy_patched_objects fi if (( SHORT_CIRCUIT <= 3 )); then + status "Generating original checksums" + generate_checksums "$ORIG_DIR" "$ORIG_CSUM_DIR" "$PATCHED_DIR" + status "Generating patched checksums" + generate_checksums "$PATCHED_DIR" "$PATCHED_CSUM_DIR" +fi + +if (( SHORT_CIRCUIT <= 4 )); then status "Diffing objects" diff_objects if [[ -v DIFF_CHECKSUM ]]; then @@ -826,7 +935,7 @@ if (( SHORT_CIRCUIT <= 3 )); then fi fi -if (( SHORT_CIRCUIT <= 4 )); then +if (( SHORT_CIRCUIT <= 5 )); then status "Building patch module: $OUTFILE" build_patch_module fi diff --git a/scripts/min-tool-version.sh b/scripts/min-tool-version.sh index 99b5575c1ef7..99bfdcaa3396 100755 --- a/scripts/min-tool-version.sh +++ b/scripts/min-tool-version.sh @@ -27,14 +27,20 @@ llvm) if [ "$SRCARCH" = loongarch ]; then echo 18.0.0 else - echo 15.0.0 + echo 17.0.1 fi ;; rustc) - echo 1.78.0 + if [ "$SRCARCH" = "s390" ]; then + echo 1.96.0 + elif [ "$ARCH" = powerpc ]; then + echo 1.95.0 + else + echo 1.85.0 + fi ;; bindgen) - echo 0.65.1 + echo 0.71.1 ;; *) echo "$1: unknown tool" >&2 diff --git a/scripts/mksysmap b/scripts/mksysmap index c4531eacde20..856b26ba2ac0 100755 --- a/scripts/mksysmap +++ b/scripts/mksysmap @@ -35,7 +35,7 @@ / __efistub_/d # arm64 local symbols in PIE namespace -/ __pi_\\$/d +/ __pi_\$/d / __pi_\.L/d # arm64 local symbols in non-VHE KVM namespace @@ -83,7 +83,7 @@ / _SDA2_BASE_$/d # MODULE_INFO() -/ __UNIQUE_ID_modinfo[0-9]*$/d +/ __UNIQUE_ID_modinfo_[0-9]*$/d # --------------------------------------------------------------------------- # Ignored patterns diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c index 4e99393a35f1..8d36c74dec2d 100644 --- a/scripts/mod/file2alias.c +++ b/scripts/mod/file2alias.c @@ -651,7 +651,26 @@ static void do_vio_entry(struct module *mod, void *symval) module_alias_printf(mod, true, "%s", alias); } -static void do_input(char *alias, +static void __attribute__((format(printf, 3, 4))) +alias_append(char *alias, size_t size, const char *fmt, ...) +{ + size_t len = strlen(alias); + va_list args; + int n; + + if (len >= size) + fatal("alias buffer (%zu) overflow before append\n", size); + + va_start(args, fmt); + n = vsnprintf(alias + len, size - len, fmt, args); + va_end(args); + + if (n < 0 || (size_t)n >= size - len) + fatal("alias buffer (%zu) overflow on append (need %d, have %zu)\n", + size, n, size - len); +} + +static void do_input(char *alias, size_t size, kernel_ulong_t *arr, unsigned int min, unsigned int max) { unsigned int i; @@ -659,13 +678,14 @@ static void do_input(char *alias, for (i = min; i <= max; i++) if (get_unaligned_native(arr + i / BITS_PER_LONG) & (1ULL << (i % BITS_PER_LONG))) - sprintf(alias + strlen(alias), "%X,*", i); + alias_append(alias, size, "%X,*", i); } /* input:b0v0p0e0-eXkXrXaXmXlXsXfXwX where X is comma-separated %02X. */ static void do_input_entry(struct module *mod, void *symval) { char alias[256] = {}; + const size_t sizeof_alias = sizeof(alias); DEF_FIELD(symval, input_device_id, flags); DEF_FIELD(symval, input_device_id, bustype); @@ -687,35 +707,35 @@ static void do_input_entry(struct module *mod, void *symval) ADD(alias, "p", flags & INPUT_DEVICE_ID_MATCH_PRODUCT, product); ADD(alias, "e", flags & INPUT_DEVICE_ID_MATCH_VERSION, version); - sprintf(alias + strlen(alias), "-e*"); + alias_append(alias, sizeof_alias, "-e*"); if (flags & INPUT_DEVICE_ID_MATCH_EVBIT) - do_input(alias, *evbit, 0, INPUT_DEVICE_ID_EV_MAX); - sprintf(alias + strlen(alias), "k*"); + do_input(alias, sizeof_alias, *evbit, 0, INPUT_DEVICE_ID_EV_MAX); + alias_append(alias, sizeof_alias, "k*"); if (flags & INPUT_DEVICE_ID_MATCH_KEYBIT) - do_input(alias, *keybit, + do_input(alias, sizeof_alias, *keybit, INPUT_DEVICE_ID_KEY_MIN_INTERESTING, INPUT_DEVICE_ID_KEY_MAX); - sprintf(alias + strlen(alias), "r*"); + alias_append(alias, sizeof_alias, "r*"); if (flags & INPUT_DEVICE_ID_MATCH_RELBIT) - do_input(alias, *relbit, 0, INPUT_DEVICE_ID_REL_MAX); - sprintf(alias + strlen(alias), "a*"); + do_input(alias, sizeof_alias, *relbit, 0, INPUT_DEVICE_ID_REL_MAX); + alias_append(alias, sizeof_alias, "a*"); if (flags & INPUT_DEVICE_ID_MATCH_ABSBIT) - do_input(alias, *absbit, 0, INPUT_DEVICE_ID_ABS_MAX); - sprintf(alias + strlen(alias), "m*"); + do_input(alias, sizeof_alias, *absbit, 0, INPUT_DEVICE_ID_ABS_MAX); + alias_append(alias, sizeof_alias, "m*"); if (flags & INPUT_DEVICE_ID_MATCH_MSCIT) - do_input(alias, *mscbit, 0, INPUT_DEVICE_ID_MSC_MAX); - sprintf(alias + strlen(alias), "l*"); + do_input(alias, sizeof_alias, *mscbit, 0, INPUT_DEVICE_ID_MSC_MAX); + alias_append(alias, sizeof_alias, "l*"); if (flags & INPUT_DEVICE_ID_MATCH_LEDBIT) - do_input(alias, *ledbit, 0, INPUT_DEVICE_ID_LED_MAX); - sprintf(alias + strlen(alias), "s*"); + do_input(alias, sizeof_alias, *ledbit, 0, INPUT_DEVICE_ID_LED_MAX); + alias_append(alias, sizeof_alias, "s*"); if (flags & INPUT_DEVICE_ID_MATCH_SNDBIT) - do_input(alias, *sndbit, 0, INPUT_DEVICE_ID_SND_MAX); - sprintf(alias + strlen(alias), "f*"); + do_input(alias, sizeof_alias, *sndbit, 0, INPUT_DEVICE_ID_SND_MAX); + alias_append(alias, sizeof_alias, "f*"); if (flags & INPUT_DEVICE_ID_MATCH_FFBIT) - do_input(alias, *ffbit, 0, INPUT_DEVICE_ID_FF_MAX); - sprintf(alias + strlen(alias), "w*"); + do_input(alias, sizeof_alias, *ffbit, 0, INPUT_DEVICE_ID_FF_MAX); + alias_append(alias, sizeof_alias, "w*"); if (flags & INPUT_DEVICE_ID_MATCH_SWBIT) - do_input(alias, *swbit, 0, INPUT_DEVICE_ID_SW_MAX); + do_input(alias, sizeof_alias, *swbit, 0, INPUT_DEVICE_ID_SW_MAX); module_alias_printf(mod, false, "input:%s", alias); } @@ -895,12 +915,16 @@ static const struct dmifield { { NULL, DMI_NONE } }; -static void dmi_ascii_filter(char *d, const char *s) +static void dmi_ascii_filter(char *d, size_t avail, const char *s) { /* Filter out characters we don't want to see in the modalias string */ for (; *s; s++) - if (*s > ' ' && *s < 127 && *s != ':') + if (*s > ' ' && *s < 127 && *s != ':') { + if (avail <= 1) + fatal("%s: alias buffer overflow\n", __func__); *(d++) = *s; + avail--; + } *d = 0; } @@ -909,6 +933,8 @@ static void dmi_ascii_filter(char *d, const char *s) static void do_dmi_entry(struct module *mod, void *symval) { char alias[256] = {}; + const size_t sizeof_alias = sizeof(alias); + size_t len; int i, j; DEF_FIELD_ADDR(symval, dmi_system_id, matches); @@ -916,11 +942,12 @@ static void do_dmi_entry(struct module *mod, void *symval) for (j = 0; j < 4; j++) { if ((*matches)[j].slot && (*matches)[j].slot == dmi_fields[i].field) { - sprintf(alias + strlen(alias), ":%s*", - dmi_fields[i].prefix); - dmi_ascii_filter(alias + strlen(alias), + alias_append(alias, sizeof_alias, ":%s*", + dmi_fields[i].prefix); + len = strlen(alias); + dmi_ascii_filter(alias + len, sizeof_alias - len, (*matches)[j].substr); - strcat(alias, "*"); + alias_append(alias, sizeof_alias, "*"); } } } @@ -1253,6 +1280,8 @@ static void do_tee_entry(struct module *mod, void *symval) static void do_wmi_entry(struct module *mod, void *symval) { DEF_FIELD_ADDR(symval, wmi_device_id, guid_string); + char result[sizeof(*guid_string)]; + int i; if (strlen(*guid_string) != UUID_STRING_LEN) { warn("Invalid WMI device id 'wmi:%s' in '%s'\n", @@ -1260,7 +1289,31 @@ static void do_wmi_entry(struct module *mod, void *symval) return; } - module_alias_printf(mod, false, WMI_MODULE_PREFIX "%s", *guid_string); + for (i = 0; i < UUID_STRING_LEN; i++) { + char value = (*guid_string)[i]; + bool valid = false; + + if (i == 8 || i == 13 || i == 18 || i == 23) { + if (value == '-') + valid = true; + } else { + if (isxdigit(value)) + valid = true; + } + + if (!valid) { + warn("Invalid character %c inside WMI GUID string '%s' in '%s'\n", + value, *guid_string, mod->name); + return; + } + + /* Some GUIDs from BMOF definitions contain lowercase characters */ + result[i] = toupper(value); + } + + result[i] = '\0'; + + module_alias_printf(mod, false, WMI_MODULE_PREFIX "%s", result); } /* Looks like: mhi:S */ diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 0c25b5ad497b..75374c64b8cc 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -56,7 +56,7 @@ static bool allow_missing_ns_imports; static bool error_occurred; -static bool extra_warn; +static bool extra_warn __attribute__((unused)); bool target_is_big_endian; bool host_is_big_endian; @@ -74,7 +74,7 @@ static unsigned int nr_unresolved; #define MODULE_NAME_LEN (64 - sizeof(Elf_Addr)) -void modpost_log(bool is_error, const char *fmt, ...) +void modpost_log(bool is_error, struct module *mod, const char *fmt, ...) { va_list arglist; @@ -87,11 +87,17 @@ void modpost_log(bool is_error, const char *fmt, ...) fprintf(stderr, "modpost: "); + if (mod) + fprintf(stderr, "%s%s: ", mod->name, mod->is_vmlinux ? "" : ".ko"); + va_start(arglist, fmt); vfprintf(stderr, fmt, arglist); va_end(arglist); } +#define mod_warn(mod, fmt, args...) modpost_log(false, mod, fmt, ##args) +#define mod_error(mod, fmt, args...) modpost_log(true, mod, fmt, ##args) + static inline bool strends(const char *str, const char *postfix) { if (strlen(str) < strlen(postfix)) @@ -244,6 +250,11 @@ static struct symbol *alloc_symbol(const char *name) return s; } +static uint8_t get_symbol_flags(const struct symbol *sym) +{ + return sym->is_gpl_only ? KSYM_FLAG_GPL_ONLY : 0; +} + /* For the hash of exported symbols */ static void hash_add_symbol(struct symbol *sym) { @@ -354,9 +365,8 @@ static struct symbol *sym_add_exported(const char *name, struct module *mod, struct symbol *s = find_symbol(name); if (s && (!external_module || s->module->is_vmlinux || s->module == mod)) { - error("%s: '%s' exported twice. Previous export was in %s%s\n", - mod->name, name, s->module->name, - s->module->is_vmlinux ? "" : ".ko"); + mod_error(mod, "symbol '%s' exported twice. Previous export was in %s%s\n", + name, s->module->name, s->module->is_vmlinux ? "" : ".ko"); } s = alloc_symbol(name); @@ -627,7 +637,7 @@ static void handle_symbol(struct module *mod, struct elf_info *info, if (strstarts(symname, "__gnu_lto_")) { /* Should warn here, but modpost runs before the linker */ } else - warn("\"%s\" [%s] is COMMON symbol\n", symname, mod->name); + mod_warn(mod, "'%s' is COMMON symbol\n", symname); break; case SHN_UNDEF: /* undefined symbol */ @@ -760,6 +770,9 @@ static const char *const section_white_list[] = ".gnu.lto*", ".discard.*", ".llvm.call-graph-profile", /* call graph */ + "__llvm_covfun", + "__llvm_covmap", + ".klp.symid", /* objtool --klp-symids */ NULL }; @@ -768,7 +781,7 @@ static const char *const section_white_list[] = * The cause of this is often a section specified in assembler * without "ax" / "aw". */ -static void check_section(const char *modname, struct elf_info *elf, +static void check_section(struct module *mod, struct elf_info *elf, Elf_Shdr *sechdr) { const char *sec = sech_name(elf, sechdr); @@ -776,11 +789,11 @@ static void check_section(const char *modname, struct elf_info *elf, if (sechdr->sh_type == SHT_PROGBITS && !(sechdr->sh_flags & SHF_ALLOC) && !match(sec, section_white_list)) { - warn("%s (%s): unexpected non-allocatable section.\n" - "Did you forget to use \"ax\"/\"aw\" in a .S file?\n" - "Note that for example <linux/init.h> contains\n" - "section definitions for use in .S files.\n\n", - modname, sec); + mod_warn(mod, "unexpected non-allocatable section '%s'.\n" + "Did you forget to use \"ax\"/\"aw\" in a .S file?\n" + "Note that for example <linux/init.h> contains\n" + "section definitions for use in .S files.\n\n", + sec); } } @@ -962,7 +975,7 @@ static int secref_whitelist(const char *fromsec, const char *fromsym, /* symbols in data sections that may refer to any init/exit sections */ if (match(fromsec, PATTERNS(DATA_SECTIONS)) && match(tosec, PATTERNS(ALL_INIT_SECTIONS, ALL_EXIT_SECTIONS)) && - match(fromsym, PATTERNS("*_ops", "*_console"))) + match(fromsym, PATTERNS("*_ops", "*_ops.llvm.*", "*_console"))) return 0; /* Check for pattern 3 */ @@ -1014,7 +1027,7 @@ static bool is_executable_section(struct elf_info *elf, unsigned int secndx) return (elf->sechdrs[secndx].sh_flags & SHF_EXECINSTR) != 0; } -static void default_mismatch_handler(const char *modname, struct elf_info *elf, +static void default_mismatch_handler(struct module *mod, struct elf_info *elf, const struct sectioncheck* const mismatch, Elf_Sym *tsym, unsigned int fsecndx, const char *fromsec, Elf_Addr faddr, @@ -1044,10 +1057,10 @@ static void default_mismatch_handler(const char *modname, struct elf_info *elf, * The format for the reference source: <symbol_name>+<offset> or <address> * The format for the reference destination: <symbol_name> or <address> */ - warn("%s: section mismatch in reference: %s%s0x%x (section: %s) -> %s (section: %s)\n", - modname, fromsym, fromsym[0] ? "+" : "", - (unsigned int)(faddr - (fromsym[0] ? from->st_value : 0)), - fromsec, tosym[0] ? tosym : taddr_str, tosec); + mod_warn(mod, "section mismatch in reference: %s%s0x%x (section: %s) -> %s (section: %s)\n", + fromsym, fromsym[0] ? "+" : "", + (unsigned int)(faddr - (fromsym[0] ? from->st_value : 0)), + fromsec, tosym[0] ? tosym : taddr_str, tosec); if (mismatch->mismatch == EXTABLE_TO_NON_TEXT) { if (match(tosec, mismatch->bad_tosec)) @@ -1056,7 +1069,7 @@ static void default_mismatch_handler(const char *modname, struct elf_info *elf, "Something is seriously wrong and should be fixed.\n" "You might get more information about where this is\n" "coming from by using scripts/check_extable.sh %s\n", - fromsec, (long)faddr, tosec, modname); + fromsec, (long)faddr, tosec, mod->name); else if (is_executable_section(elf, get_secindex(elf, tsym))) warn("The relocation at %s+0x%lx references\n" "section \"%s\" which is not in the list of\n" @@ -1086,22 +1099,22 @@ static void check_export_symbol(struct module *mod, struct elf_info *elf, label_name = sym_name(elf, label); if (!strstarts(label_name, prefix)) { - error("%s: .export_symbol section contains strange symbol '%s'\n", - mod->name, label_name); + mod_error(mod, ".export_symbol section contains strange symbol '%s'\n", + label_name); return; } if (ELF_ST_BIND(sym->st_info) != STB_GLOBAL && ELF_ST_BIND(sym->st_info) != STB_WEAK) { - error("%s: local symbol '%s' was exported\n", mod->name, - label_name + strlen(prefix)); + mod_error(mod, "local symbol '%s' was exported\n", + label_name + strlen(prefix)); return; } name = sym_name(elf, sym); if (strcmp(label_name + strlen(prefix), name)) { - error("%s: .export_symbol section references '%s', but it does not seem to be an export symbol\n", - mod->name, name); + mod_error(mod, ".export_symbol section references '%s', but it does not seem to be an export symbol\n", + name); return; } @@ -1111,8 +1124,8 @@ static void check_export_symbol(struct module *mod, struct elf_info *elf, } else if (!strcmp(data, "")) { is_gpl = false; } else { - error("%s: unknown license '%s' was specified for '%s'\n", - mod->name, data, name); + mod_error(mod, "unknown license '%s' was specified for '%s'\n", + data, name); return; } @@ -1135,11 +1148,11 @@ static void check_export_symbol(struct module *mod, struct elf_info *elf, s->is_func = true; if (match(secname, PATTERNS(ALL_INIT_SECTIONS))) - warn("%s: %s: EXPORT_SYMBOL used for init symbol. Remove __init or EXPORT_SYMBOL.\n", - mod->name, name); + mod_warn(mod, "EXPORT_SYMBOL used for init symbol '%s'. Remove __init or EXPORT_SYMBOL.\n", + name); else if (match(secname, PATTERNS(ALL_EXIT_SECTIONS))) - warn("%s: %s: EXPORT_SYMBOL used for exit symbol. Remove __exit or EXPORT_SYMBOL.\n", - mod->name, name); + mod_warn(mod, "EXPORT_SYMBOL used for exit symbol '%s'. Remove __exit or EXPORT_SYMBOL.\n", + name); } static void check_section_mismatch(struct module *mod, struct elf_info *elf, @@ -1159,7 +1172,7 @@ static void check_section_mismatch(struct module *mod, struct elf_info *elf, if (!mismatch) return; - default_mismatch_handler(mod->name, elf, mismatch, sym, + default_mismatch_handler(mod, elf, mismatch, sym, fsecndx, fromsec, faddr, tosec, taddr); } @@ -1436,7 +1449,7 @@ static void check_sec_ref(struct module *mod, struct elf_info *elf) for (i = 0; i < elf->num_sections; i++) { Elf_Shdr *sechdr = &elf->sechdrs[i]; - check_section(mod->name, elf, sechdr); + check_section(mod, elf, sechdr); /* We want to process only relocation sections and not .init */ if (sechdr->sh_type == SHT_REL || sechdr->sh_type == SHT_RELA) { /* section to which the relocation applies */ @@ -1482,13 +1495,22 @@ static void extract_crcs_for_object(const char *object, struct module *mod) char cmd_file[PATH_MAX]; char *buf, *p; const char *base; - int dirlen, ret; + int dirlen, baselen_without_suffix, ret; base = get_basename(object); dirlen = base - object; - ret = snprintf(cmd_file, sizeof(cmd_file), "%.*s.%s.cmd", - dirlen, object, base); + baselen_without_suffix = strlen(object) - dirlen - strlen(".o"); + + /* + * When CONFIG_LTO_CLANG_THIN_DIST=y, the ELF is *.thinlto-native.o + * but the symbol CRCs are recorded in *.o.cmd file. + */ + if (strends(object, ".thinlto-native.o")) + baselen_without_suffix -= strlen(".thinlto-native"); + + ret = snprintf(cmd_file, sizeof(cmd_file), "%.*s.%.*s.o.cmd", + dirlen, object, baselen_without_suffix, base); if (ret >= sizeof(cmd_file)) { error("%s: too long path was truncated\n", cmd_file); return; @@ -1575,14 +1597,14 @@ static void read_symbols(const char *modname) struct elf_info info = { }; Elf_Sym *sym; - if (!parse_elf(&info, modname)) - return; - if (!strends(modname, ".o")) { error("%s: filename must be suffixed with .o\n", modname); return; } + if (!parse_elf(&info, modname)) + return; + /* strip trailing .o */ mod = new_module(modname, strlen(modname) - strlen(".o")); @@ -1597,7 +1619,7 @@ static void read_symbols(const char *modname) if (!mod->is_vmlinux) { license = get_modinfo(&info, "license"); if (!license) - error("missing MODULE_LICENSE() in %s\n", modname); + mod_error(mod, "missing MODULE_LICENSE()\n"); while (license) { if (!license_is_gpl_compatible(license)) { mod->is_gpl_compatible = false; @@ -1610,14 +1632,14 @@ static void read_symbols(const char *modname) namespace; namespace = get_next_modinfo(&info, "import_ns", namespace)) { if (strstarts(namespace, MODULE_NS_PREFIX)) - error("%s: explicitly importing namespace \"%s\" is not allowed.\n", - mod->name, namespace); + mod_error(mod, "explicitly importing namespace '%s' is not allowed.\n", + namespace); add_namespace(&mod->imported_namespaces, namespace); } if (!get_modinfo(&info, "description")) - warn("missing MODULE_DESCRIPTION() in %s\n", modname); + mod_warn(mod, "missing MODULE_DESCRIPTION()\n"); } for (sym = info.symtab_start; sym < info.symtab_stop; sym++) { @@ -1684,8 +1706,17 @@ void __attribute__((format(printf, 2, 3))) buf_printf(struct buffer *buf, va_start(ap, fmt); len = vsnprintf(tmp, SZ, fmt, ap); - buf_write(buf, tmp, len); va_end(ap); + + if (len < 0) { + perror("vsnprintf failed"); + exit(1); + } + if (len >= SZ) + fatal("buf_printf output truncated for string %s: %d bytes needed, %d available\n", + tmp, len + 1, SZ); + + buf_write(buf, tmp, len); } void buf_write(struct buffer *buf, const char *s, int len) @@ -1747,14 +1778,13 @@ static void check_exports(struct module *mod) exp = find_symbol(s->name); if (!exp) { if (!s->weak && nr_unresolved++ < MAX_UNRESOLVED_REPORTS) - modpost_log(!warn_unresolved, - "\"%s\" [%s.ko] undefined!\n", - s->name, mod->name); + modpost_log(!warn_unresolved, mod, + "symbol '%s' undefined!\n", + s->name); continue; } if (exp->module == mod) { - error("\"%s\" [%s.ko] was exported without definition\n", - s->name, mod->name); + mod_error(mod, "symbol '%s' was exported without definition\n", s->name); continue; } @@ -1767,15 +1797,15 @@ static void check_exports(struct module *mod) if (!verify_module_namespace(exp->namespace, basename) && !contains_namespace(&mod->imported_namespaces, exp->namespace)) { - modpost_log(!allow_missing_ns_imports, - "module %s uses symbol %s from namespace %s, but does not import it.\n", - basename, exp->name, exp->namespace); + modpost_log(!allow_missing_ns_imports, mod, + "module uses symbol '%s' from namespace '%s', but does not import it.\n", + exp->name, exp->namespace); add_namespace(&mod->missing_namespaces, exp->namespace); } if (!mod->is_gpl_compatible && exp->is_gpl_only) - error("GPL-incompatible module %s.ko uses GPL-only symbol '%s'\n", - basename, exp->name); + mod_error(mod, "GPL-incompatible module uses GPL-only symbol '%s'\n", + exp->name); } } @@ -1825,7 +1855,7 @@ static void check_modname_len(struct module *mod) mod_name = get_basename(mod->name); if (strlen(mod_name) >= MODULE_NAME_LEN) - error("module name is too long [%s.ko]\n", mod->name); + mod_error(mod, "module name is too long\n"); } /** @@ -1871,9 +1901,12 @@ static void add_exported_symbols(struct buffer *buf, struct module *mod) if (trim_unused_exports && !sym->used) continue; - buf_printf(buf, "KSYMTAB_%s(%s, \"%s\", \"%s\");\n", + buf_printf(buf, "KSYMTAB_%s(%s, \"%s\");\n", sym->is_func ? "FUNC" : "DATA", sym->name, - sym->is_gpl_only ? "_gpl" : "", sym->namespace); + sym->namespace); + + buf_printf(buf, "SYMBOL_FLAGS(%s, 0x%02x);\n", + sym->name, get_symbol_flags(sym)); } if (!modversions) @@ -1886,13 +1919,12 @@ static void add_exported_symbols(struct buffer *buf, struct module *mod) continue; if (!sym->crc_valid) - warn("EXPORT symbol \"%s\" [%s%s] version generation failed, symbol will not be versioned.\n" - "Is \"%s\" prototyped in <asm/asm-prototypes.h>?\n", - sym->name, mod->name, mod->is_vmlinux ? "" : ".ko", - sym->name); + mod_warn(mod, "EXPORT symbol '%s' version generation failed, symbol will not be versioned.\n" + "Is '%s' prototyped in <asm/asm-prototypes.h>?\n", + sym->name, sym->name); - buf_printf(buf, "SYMBOL_CRC(%s, 0x%08x, \"%s\");\n", - sym->name, sym->crc, sym->is_gpl_only ? "_gpl" : ""); + buf_printf(buf, "SYMBOL_CRC(%s, 0x%08x);\n", + sym->name, sym->crc); } } @@ -1913,8 +1945,7 @@ static void add_extended_versions(struct buffer *b, struct module *mod) if (!s->module) continue; if (!s->crc_valid) { - warn("\"%s\" [%s.ko] has no CRC!\n", - s->name, mod->name); + mod_warn(mod, "symbol '%s' has no CRC!\n", s->name); continue; } buf_printf(b, "\t0x%08x,\n", s->crc); @@ -1957,8 +1988,7 @@ static void add_versions(struct buffer *b, struct module *mod) if (!s->module) continue; if (!s->crc_valid) { - warn("\"%s\" [%s.ko] has no CRC!\n", - s->name, mod->name); + mod_warn(mod, "symbol '%s' has no CRC!\n", s->name); continue; } if (strlen(s->name) >= MODULE_NAME_LEN) { @@ -1966,8 +1996,7 @@ static void add_versions(struct buffer *b, struct module *mod) /* this symbol will only be in the extended info */ continue; } else { - error("too long symbol \"%s\" [%s.ko]\n", - s->name, mod->name); + mod_error(mod, "too long symbol '%s'\n", s->name); break; } } diff --git a/scripts/mod/modpost.h b/scripts/mod/modpost.h index 2aecb8f25c87..d5f6d82837d5 100644 --- a/scripts/mod/modpost.h +++ b/scripts/mod/modpost.h @@ -223,8 +223,8 @@ char *read_text_file(const char *filename); char *get_line(char **stringp); void *sym_get_data(const struct elf_info *info, const Elf_Sym *sym); -void __attribute__((format(printf, 2, 3))) -modpost_log(bool is_error, const char *fmt, ...); +void __attribute__((format(printf, 3, 4))) +modpost_log(bool is_error, struct module *mod, const char *fmt, ...); /* * warn - show the given message, then let modpost continue running, still @@ -239,6 +239,6 @@ modpost_log(bool is_error, const char *fmt, ...); * fatal - show the given message, and bail out immediately. This should be * used when there is no point to continue running modpost. */ -#define warn(fmt, args...) modpost_log(false, fmt, ##args) -#define error(fmt, args...) modpost_log(true, fmt, ##args) +#define warn(fmt, args...) modpost_log(false, NULL, fmt, ##args) +#define error(fmt, args...) modpost_log(true, NULL, fmt, ##args) #define fatal(fmt, args...) do { error(fmt, ##args); exit(1); } while (1) diff --git a/scripts/module.lds.S b/scripts/module.lds.S index 054ef99e8288..d0f200428957 100644 --- a/scripts/module.lds.S +++ b/scripts/module.lds.S @@ -3,11 +3,6 @@ * Archs are free to supply their own linker scripts. ld will * combine them automatically. */ -#ifdef CONFIG_UNWIND_TABLES -#define DISCARD_EH_FRAME -#else -#define DISCARD_EH_FRAME *(.eh_frame) -#endif #include <asm-generic/codetag.lds.h> @@ -20,9 +15,8 @@ SECTIONS { } __ksymtab 0 : ALIGN(8) { *(SORT(___ksymtab+*)) } - __ksymtab_gpl 0 : ALIGN(8) { *(SORT(___ksymtab_gpl+*)) } __kcrctab 0 : ALIGN(4) { *(SORT(___kcrctab+*)) } - __kcrctab_gpl 0 : ALIGN(4) { *(SORT(___kcrctab_gpl+*)) } + __kflagstab 0 : ALIGN(1) { *(SORT(___kflagstab+*)) } .ctors 0 : ALIGN(8) { *(SORT(.ctors.*)) *(.ctors) } .init_array 0 : ALIGN(8) { *(SORT(.init_array.*)) *(.init_array) } @@ -32,30 +26,32 @@ SECTIONS { __jump_table 0 : ALIGN(8) { KEEP(*(__jump_table)) } __ex_table 0 : ALIGN(4) { KEEP(*(__ex_table)) } - __patchable_function_entries : { *(__patchable_function_entries) } + __patchable_function_entries 0 : { *(__patchable_function_entries) } .init.klp_funcs 0 : ALIGN(8) { KEEP(*(.init.klp_funcs)) } .init.klp_objects 0 : ALIGN(8) { KEEP(*(.init.klp_objects)) } #ifdef CONFIG_ARCH_USES_CFI_TRAPS - __kcfi_traps : { KEEP(*(.kcfi_traps)) } + __kcfi_traps 0 : { KEEP(*(.kcfi_traps)) } #endif - .text : { +#ifndef CONFIG_ARCH_WANTS_MODULES_TEXT_SECTIONS + .text 0 : { *(.text .text.[0-9a-zA-Z_]*) } +#endif - .bss : { + .bss 0 : { *(.bss .bss.[0-9a-zA-Z_]*) *(.bss..L*) } - .data : { + .data 0 : { *(.data .data.[0-9a-zA-Z_]*) *(.data..L*) } - .rodata : { + .rodata 0 : { *(.rodata .rodata.[0-9a-zA-Z_]*) *(.rodata..L*) } diff --git a/scripts/nsdeps b/scripts/nsdeps index a3372166ac01..d4b5ebcc2016 100644 --- a/scripts/nsdeps +++ b/scripts/nsdeps @@ -8,7 +8,7 @@ SPATCH_REQ_VERSION="1.0.4" DIR="$(dirname $(readlink -f $0))/.." SPATCH="`which ${SPATCH:=spatch}`" if [ ! -x "$SPATCH" ]; then - echo 'spatch is part of the Coccinelle project and is available at http://coccinelle.lip6.fr/' + echo 'spatch is part of the Coccinelle project and is available at https://coccinelle.gitlabpages.inria.fr/website' exit 1 fi diff --git a/scripts/package/PKGBUILD b/scripts/package/PKGBUILD index 452374d63c24..66e4b6a37783 100644 --- a/scripts/package/PKGBUILD +++ b/scripts/package/PKGBUILD @@ -10,7 +10,7 @@ for pkg in $_extrapackages; do pkgname+=("${pkgbase}-${pkg}") done -pkgver="${KERNELRELEASE//-/_}" +pkgver="$(echo "${KERNELRELEASE}" | sed 's/-\(rc[0-9]\+\)/\1/;s/-/_/g')" # The PKGBUILD is evaluated multiple times. # Running scripts/build-version from here would introduce inconsistencies. pkgrel="${KBUILD_REVISION}" @@ -121,6 +121,9 @@ _package-debug(){ install -Dt "${debugdir}" -m644 vmlinux mkdir -p "${builddir}" ln -sr "${debugdir}/vmlinux" "${builddir}/vmlinux" + + echo "Installing unstripped vDSO(s)..." + ${MAKE} INSTALL_MOD_PATH="${pkgdir}/usr" vdso_install } for _p in "${pkgname[@]}"; do diff --git a/scripts/package/builddeb b/scripts/package/builddeb index 3627ca227e5a..ba1defc61652 100755 --- a/scripts/package/builddeb +++ b/scripts/package/builddeb @@ -139,7 +139,13 @@ install_kernel_headers () { pdir=debian/$1 version=${1#linux-headers-} - CC="${DEB_HOST_GNU_TYPE}-gcc" "${srctree}/scripts/package/install-extmod-build" "${pdir}/usr/src/linux-headers-${version}" + # Override $CC only for cross-compiles, to not unnecessarily rebuild + # scripts/ including plugins, which may lead to a full kernel rebuild. + if [ -n "${CROSS_COMPILE}" ]; then + CC="${DEB_HOST_GNU_TYPE}-gcc" "${srctree}/scripts/package/install-extmod-build" "${pdir}/usr/src/linux-headers-${version}" + else + "${srctree}/scripts/package/install-extmod-build" "${pdir}/usr/src/linux-headers-${version}" + fi mkdir -p $pdir/lib/modules/$version/ ln -s /usr/src/linux-headers-$version $pdir/lib/modules/$version/build diff --git a/scripts/package/kernel.spec b/scripts/package/kernel.spec index b3c956205af0..46e80970f723 100644 --- a/scripts/package/kernel.spec +++ b/scripts/package/kernel.spec @@ -6,7 +6,7 @@ Name: kernel Summary: The Linux Kernel Version: %(echo %{KERNELRELEASE} | sed -e 's/-/_/g') -Release: %{pkg_release} +Release: %{pkg_release}%{?dist} License: GPL Group: System Environment/Kernel Vendor: The Linux Community @@ -67,7 +67,18 @@ This package provides debug information for the kernel image and modules from th %undefine _unique_debug_srcs %undefine _debugsource_packages %undefine _debuginfo_subpackages + +# Preserve .BTF and .BTF.base sections in kernel modules during debuginfo +# stripping. find-debuginfo.sh uses eu-strip which removes non-allocated ELF +# sections like .BTF by default. .BTF.base is required for BTF distillation +# support; without it, module BTF validation fails. +%global with_keep_section %(%{__find_debuginfo} --help 2>&1 | grep -c keep-section) +%if %{with_keep_section} +%global _find_debuginfo_opts -r --keep-section .BTF --keep-section .BTF.base +%else %global _find_debuginfo_opts -r +%endif + %global _missing_build_ids_terminate_build 1 %global _no_recompute_build_ids 1 %{debug_package} diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index d2323de0692c..c30983562a2f 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -121,14 +121,8 @@ fi # Check that the Rust bindings generator is suitable. # # Non-stable and distributions' versions may have a version suffix, e.g. `-dev`. -# -# The dummy parameter `workaround-for-0.69.0` is required to support 0.69.0 -# (https://github.com/rust-lang/rust-bindgen/pull/2678) and 0.71.0 -# (https://github.com/rust-lang/rust-bindgen/pull/3040). It can be removed when -# the minimum version is upgraded past the latter (0.69.1 and 0.71.1 both fixed -# the issue). rust_bindings_generator_output=$( \ - LC_ALL=C "$BINDGEN" --version workaround-for-0.69.0 2>/dev/null + LC_ALL=C "$BINDGEN" --version 2>/dev/null ) || rust_bindings_generator_code=$? if [ -n "$rust_bindings_generator_code" ]; then echo >&2 "***" @@ -163,19 +157,6 @@ if [ "$rust_bindings_generator_cversion" -lt "$rust_bindings_generator_min_cvers echo >&2 "***" exit 1 fi -if [ "$rust_bindings_generator_cversion" -eq 6600 ] || - [ "$rust_bindings_generator_cversion" -eq 6601 ]; then - # Distributions may have patched the issue (e.g. Debian did). - if ! "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_0_66.h >/dev/null; then - echo >&2 "***" - echo >&2 "*** Rust bindings generator '$BINDGEN' versions 0.66.0 and 0.66.1 may not" - echo >&2 "*** work due to a bug (https://github.com/rust-lang/rust-bindgen/pull/2567)," - echo >&2 "*** unless patched (like Debian's)." - echo >&2 "*** Your version: $rust_bindings_generator_version" - echo >&2 "***" - warning=1 - fi -fi # Check that the `libclang` used by the Rust bindings generator is suitable. # @@ -227,14 +208,13 @@ if [ "$bindgen_libclang_cversion" -lt "$bindgen_libclang_min_cversion" ]; then exit 1 fi -if [ "$bindgen_libclang_cversion" -ge 1900100 ] && - [ "$rust_bindings_generator_cversion" -lt 6905 ]; then - # Distributions may have patched the issue (e.g. Debian did). - if ! "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_libclang_concat.h | grep -q foofoo; then +if [ "$bindgen_libclang_cversion" -ge 2200000 ] && + [ "$rust_bindings_generator_cversion" -lt 7201 ]; then + # Distributions may have patched the issue. + if ! "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_libclang_22.h | grep -q 'pub foo'; then echo >&2 "***" - echo >&2 "*** Rust bindings generator '$BINDGEN' < 0.69.5 together with libclang >= 19.1" - echo >&2 "*** may not work due to a bug (https://github.com/rust-lang/rust-bindgen/pull/2824)," - echo >&2 "*** unless patched (like Debian's)." + echo >&2 "*** Rust bindings generator '$BINDGEN' < 0.72.1 together with libclang >= 22" + echo >&2 "*** may not work due to a bug (https://github.com/rust-lang/rust-bindgen/pull/3278)." echo >&2 "*** Your bindgen version: $rust_bindings_generator_version" echo >&2 "*** Your libclang version: $bindgen_libclang_version" echo >&2 "***" diff --git a/scripts/rust_is_available_bindgen_0_66.h b/scripts/rust_is_available_bindgen_libclang_22.h index c0431293421c..6b33544c14a8 100644 --- a/scripts/rust_is_available_bindgen_0_66.h +++ b/scripts/rust_is_available_bindgen_libclang_22.h @@ -1,2 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#define A "\0" +struct S; +struct S { + int foo; +}; diff --git a/scripts/rust_is_available_bindgen_libclang_concat.h b/scripts/rust_is_available_bindgen_libclang_concat.h deleted file mode 100644 index efc6e98d0f1d..000000000000 --- a/scripts/rust_is_available_bindgen_libclang_concat.h +++ /dev/null @@ -1,3 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#define F(x) int x##x -F(foo); diff --git a/scripts/rust_is_available_test.py b/scripts/rust_is_available_test.py index 4fcc319dea84..22bdff980c35 100755 --- a/scripts/rust_is_available_test.py +++ b/scripts/rust_is_available_test.py @@ -54,37 +54,30 @@ else: """) @classmethod - def generate_bindgen(cls, version_stdout, libclang_stderr, version_0_66_patched=False, libclang_concat_patched=False): + def generate_bindgen(cls, version_stdout, libclang_stderr, libclang_22_patched=False): if libclang_stderr is None: libclang_case = f"raise SystemExit({cls.bindgen_default_bindgen_libclang_failure_exit_code})" else: libclang_case = f"print({repr(libclang_stderr)}, file=sys.stderr)" - if version_0_66_patched: - version_0_66_case = "pass" + if libclang_22_patched: + libclang_22_case = "print('pub foo: ::std::os::raw::c_int,')" else: - version_0_66_case = "raise SystemExit(1)" - - if libclang_concat_patched: - libclang_concat_case = "print('pub static mut foofoo: ::std::os::raw::c_int;')" - else: - libclang_concat_case = "pass" + libclang_22_case = "pass" return cls.generate_executable(f"""#!/usr/bin/env python3 import sys if "rust_is_available_bindgen_libclang.h" in " ".join(sys.argv): {libclang_case} -elif "rust_is_available_bindgen_0_66.h" in " ".join(sys.argv): - {version_0_66_case} -elif "rust_is_available_bindgen_libclang_concat.h" in " ".join(sys.argv): - {libclang_concat_case} +elif "rust_is_available_bindgen_libclang_22.h" in " ".join(sys.argv): + {libclang_22_case} else: print({repr(version_stdout)}) """) @classmethod - def generate_bindgen_version(cls, stdout, version_0_66_patched=False): - return cls.generate_bindgen(stdout, cls.bindgen_default_bindgen_libclang_stderr, version_0_66_patched) + def generate_bindgen_version(cls, stdout): + return cls.generate_bindgen(stdout, cls.bindgen_default_bindgen_libclang_stderr) @classmethod def generate_bindgen_libclang_failure(cls): @@ -191,7 +184,13 @@ else: def test_rustc_nonexecutable(self): result = self.run_script(self.Expected.FAILURE, { "RUSTC": self.nonexecutable }) - self.assertIn(f"Running '{self.nonexecutable}' to check the Rust compiler version failed with", result.stderr) + self.assertTrue( + # `dash`. + f"Running '{self.nonexecutable}' to check the Rust compiler version failed with" in result.stderr or + # `bash`. + f"Rust compiler '{self.nonexecutable}' could not be found." in result.stderr, + f"Unexpected `stderr`:\n{result.stderr}" + ) def test_rustc_unexpected_binary(self): result = self.run_script(self.Expected.FAILURE, { "RUSTC": self.unexpected_binary }) @@ -219,7 +218,13 @@ else: def test_bindgen_nonexecutable(self): result = self.run_script(self.Expected.FAILURE, { "BINDGEN": self.nonexecutable }) - self.assertIn(f"Running '{self.nonexecutable}' to check the Rust bindings generator version failed with", result.stderr) + self.assertTrue( + # `dash`. + f"Running '{self.nonexecutable}' to check the Rust bindings generator version failed with" in result.stderr or + # `bash`. + f"Rust bindings generator '{self.nonexecutable}' could not be found." in result.stderr, + f"Unexpected `stderr`:\n{result.stderr}" + ) def test_bindgen_unexpected_binary(self): result = self.run_script(self.Expected.FAILURE, { "BINDGEN": self.unexpected_binary }) @@ -245,19 +250,6 @@ else: result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) self.assertIn(f"Rust bindings generator '{bindgen}' is too old.", result.stderr) - def test_bindgen_bad_version_0_66_0_and_0_66_1(self): - for version in ("0.66.0", "0.66.1"): - with self.subTest(version=version): - bindgen = self.generate_bindgen_version(f"bindgen {version}") - result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "BINDGEN": bindgen }) - self.assertIn(f"Rust bindings generator '{bindgen}' versions 0.66.0 and 0.66.1 may not", result.stderr) - - def test_bindgen_bad_version_0_66_0_and_0_66_1_patched(self): - for version in ("0.66.0", "0.66.1"): - with self.subTest(version=version): - bindgen = self.generate_bindgen_version(f"bindgen {version}", True) - result = self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen }) - def test_bindgen_libclang_failure(self): bindgen = self.generate_bindgen_libclang_failure() result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) @@ -275,19 +267,15 @@ else: result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) self.assertIn(f"libclang (used by the Rust bindings generator '{bindgen}') is too old.", result.stderr) - def test_bindgen_bad_libclang_concat(self): + def test_bindgen_bad_libclang_22(self): for (bindgen_version, libclang_version, expected_not_patched) in ( - ("0.69.4", "18.0.0", self.Expected.SUCCESS), - ("0.69.4", "19.1.0", self.Expected.SUCCESS_WITH_WARNINGS), - ("0.69.4", "19.2.0", self.Expected.SUCCESS_WITH_WARNINGS), + ("0.71.1", "21.1.0", self.Expected.SUCCESS), + ("0.71.1", "22.0.0", self.Expected.SUCCESS_WITH_WARNINGS), + ("0.71.1", "22.1.0", self.Expected.SUCCESS_WITH_WARNINGS), - ("0.69.5", "18.0.0", self.Expected.SUCCESS), - ("0.69.5", "19.1.0", self.Expected.SUCCESS), - ("0.69.5", "19.2.0", self.Expected.SUCCESS), + ("0.72.0", "22.0.0", self.Expected.SUCCESS_WITH_WARNINGS), - ("0.70.0", "18.0.0", self.Expected.SUCCESS), - ("0.70.0", "19.1.0", self.Expected.SUCCESS), - ("0.70.0", "19.2.0", self.Expected.SUCCESS), + ("0.72.1", "22.0.0", self.Expected.SUCCESS), ): with self.subTest(bindgen_version=bindgen_version, libclang_version=libclang_version): cc = self.generate_clang(f"clang version {libclang_version}") @@ -295,9 +283,9 @@ else: bindgen = self.generate_bindgen(f"bindgen {bindgen_version}", libclang_stderr) result = self.run_script(expected_not_patched, { "BINDGEN": bindgen, "CC": cc }) if expected_not_patched == self.Expected.SUCCESS_WITH_WARNINGS: - self.assertIn(f"Rust bindings generator '{bindgen}' < 0.69.5 together with libclang >= 19.1", result.stderr) + self.assertIn(f"Rust bindings generator '{bindgen}' < 0.72.1 together with libclang >= 22", result.stderr) - bindgen = self.generate_bindgen(f"bindgen {bindgen_version}", libclang_stderr, libclang_concat_patched=True) + bindgen = self.generate_bindgen(f"bindgen {bindgen_version}", libclang_stderr, libclang_22_patched=True) result = self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen, "CC": cc }) def test_clang_matches_bindgen_libclang_different_bindgen(self): diff --git a/scripts/rustdoc_test_builder.rs b/scripts/rustdoc_test_builder.rs index f7540bcf595a..df864437cef7 100644 --- a/scripts/rustdoc_test_builder.rs +++ b/scripts/rustdoc_test_builder.rs @@ -28,7 +28,7 @@ fn main() { // // ``` // fn main() { #[allow(non_snake_case)] fn _doctest_main_rust_kernel_file_rs_28_0() { - // fn main() { #[allow(non_snake_case)] fn _doctest_main_rust_kernel_file_rs_37_0() -> Result<(), impl ::core::fmt::Debug> { + // fn main() { #[allow(non_snake_case)] fn _doctest_main_rust_kernel_file_rs_37_0() -> Result<(), impl core::fmt::Debug> { // ``` // // It should be unlikely that doctest code matches such lines (when code is formatted properly). @@ -47,12 +47,16 @@ fn main() { }) .expect("No test function found in `rustdoc`'s output."); - // Qualify `Result` to avoid the collision with our own `Result` coming from the prelude. + // Replicate `rustdoc` 1.87+ behaviour [1] by fully qualifying `Result` to avoid the collision + // with our own `Result` coming from the prelude. + // + // [1]: https://github.com/rust-lang/rust/pull/137807 + // + // TODO: Remove this when MSRV is bumped above 1.87. let body = body.replace( - &format!("{rustdoc_function_name}() -> Result<(), impl ::core::fmt::Debug> {{"), - &format!( - "{rustdoc_function_name}() -> ::core::result::Result<(), impl ::core::fmt::Debug> {{" - ), + &format!("{rustdoc_function_name}() -> Result<(), impl core::fmt::Debug> {{"), + // This intentionally does not use absolute paths to match `rustdoc` 1.87 behaviour. + &format!("{rustdoc_function_name}() -> core::result::Result<(), impl core::fmt::Debug> {{"), ); // For tests that get generated with `Result`, like above, `rustdoc` generates an `unwrap()` on diff --git a/scripts/rustdoc_test_gen.rs b/scripts/rustdoc_test_gen.rs index d61a77219a8c..d087c0d9fcb3 100644 --- a/scripts/rustdoc_test_gen.rs +++ b/scripts/rustdoc_test_gen.rs @@ -31,8 +31,15 @@ use std::{ fs, fs::File, - io::{BufWriter, Read, Write}, - path::{Path, PathBuf}, + io::{ + BufWriter, + Read, + Write, // + }, + path::{ + Path, + PathBuf, // + }, // }; /// Find the real path to the original file based on the `file` portion of the test name. @@ -232,6 +239,24 @@ pub extern "C" fn {kunit_name}(__kunit_test: *mut ::kernel::bindings::kunit) {{ const __LOG_PREFIX: &[u8] = b"rust_doctests_kernel\0"; +/// Dummy module type for doctest context. +struct LocalModule; + +use kernel::{{ + str::CStr, + ModuleMetadata, + ThisModule, // +}}; +use core::ptr::null_mut; + +impl ModuleMetadata for LocalModule {{ + const NAME: &'static CStr = c"rust_doctests_kernel"; + const THIS_MODULE: ThisModule = {{ + // SAFETY: `try_module_get`/`module_put` handle null module pointers gracefully. + unsafe {{ ThisModule::from_ptr(null_mut()) }} + }}; +}} + {rust_tests} "# ) diff --git a/scripts/sbom/sbom.py b/scripts/sbom/sbom.py new file mode 100644 index 000000000000..764175b9c893 --- /dev/null +++ b/scripts/sbom/sbom.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +""" +Compute software bill of materials in SPDX format describing a kernel build. +""" + +import json +import logging +import os +import sys +import time +import uuid +import sbom.sbom_logging as sbom_logging +from sbom.config import get_config +from sbom.path_utils import is_relative_to +from sbom.spdx import JsonLdSpdxDocument, SpdxIdGenerator +from sbom.spdx.core import CreationInfo, SpdxDocument +from sbom.spdx_graph import SpdxIdGeneratorCollection, build_spdx_graphs +from sbom.cmd_graph import CmdGraph + + +def _exit_with_summary(write_output_on_error: bool = False) -> None: + warning_summary = sbom_logging.summarize_warnings() + error_summary = sbom_logging.summarize_errors() + if warning_summary: + logging.warning(warning_summary) + if error_summary: + logging.error(error_summary) + if not write_output_on_error: + logging.info( + "Use --write-output-on-error to generate output documents even when errors occur. " + "Note that in this case the generated documents may be incomplete." + ) + sys.exit(1) + + +def main(): + # Read config + config = get_config() + + # Configure logging + logging.basicConfig( + level=logging.DEBUG if config.debug else logging.INFO, + format="[%(levelname)s] %(message)s", + ) + + # Build cmd graph + logging.debug("Start building cmd graph") + start_time = time.time() + cmd_graph = CmdGraph.create(config.root_paths, config) + logging.debug(f"Built cmd graph in {time.time() - start_time} seconds") + + # Save used files document + if config.generate_used_files: + if config.src_tree == config.obj_tree: + logging.info( + f"Extracting all files from the cmd graph to {config.used_files_file_name} " + "instead of only source files because source files cannot be " + "reliably classified when the source and object trees are identical.", + ) + used_files = [os.path.relpath(node.absolute_path, config.src_tree) for node in cmd_graph] + logging.debug(f"Found {len(used_files)} files in cmd graph.") + else: + used_files = [ + os.path.relpath(node.absolute_path, config.src_tree) + for node in cmd_graph + if is_relative_to(node.absolute_path, config.src_tree) + and not is_relative_to(node.absolute_path, config.obj_tree) + ] + logging.debug(f"Found {len(used_files)} source files in cmd graph") + if not sbom_logging.has_errors() or config.write_output_on_error: + used_files_path = os.path.join(config.output_directory, config.used_files_file_name) + with open(used_files_path, "w", encoding="utf-8") as f: + f.write("\n".join(str(file_path) for file_path in used_files)) + logging.debug(f"Successfully saved {used_files_path}") + + if config.generate_spdx is False: + _exit_with_summary(config.write_output_on_error) + return + + # Build SPDX Documents + logging.debug("Start generating SPDX graph based on cmd graph") + start_time = time.time() + + # The real uuid will be generated based on the content of the SPDX graphs + # to ensure that the same SPDX document is always assigned the same uuid. + PLACEHOLDER_UUID = "00000000-0000-0000-0000-000000000000" + spdx_id_base_namespace = f"{config.spdxId_prefix}{PLACEHOLDER_UUID}/" + spdx_id_generators = SpdxIdGeneratorCollection( + base=SpdxIdGenerator(prefix="p", namespace=spdx_id_base_namespace), + source=SpdxIdGenerator(prefix="s", namespace=f"{spdx_id_base_namespace}source/"), + build=SpdxIdGenerator(prefix="b", namespace=f"{spdx_id_base_namespace}build/"), + output=SpdxIdGenerator(prefix="o", namespace=f"{spdx_id_base_namespace}output/"), + ) + + spdx_graphs = build_spdx_graphs( + cmd_graph, + spdx_id_generators, + config, + ) + spdx_id_uuid = uuid.uuid5( + uuid.NAMESPACE_URL, + "".join( + json.dumps(element.to_dict()) for spdx_graph in spdx_graphs.values() for element in spdx_graph.to_list() + ), + ) + logging.debug(f"Generated SPDX graph in {time.time() - start_time} seconds") + + if not sbom_logging.has_errors() or config.write_output_on_error: + for kernel_sbom_kind, spdx_graph in spdx_graphs.items(): + spdx_graph_objects = spdx_graph.to_list() + # Add warning and error summary to creation info comment + creation_info = next(element for element in spdx_graph_objects if isinstance(element, CreationInfo)) + creation_info.comment = "\n".join([ + sbom_logging.summarize_warnings(), + sbom_logging.summarize_errors(), + ]).strip() + # Replace Placeholder uuid with real uuid for spdxIds + spdx_document = next(element for element in spdx_graph_objects if isinstance(element, SpdxDocument)) + for namespaceMap in spdx_document.namespaceMap: + namespaceMap.namespace = namespaceMap.namespace.replace(PLACEHOLDER_UUID, str(spdx_id_uuid)) + # Serialize SPDX graph to JSON-LD + spdx_doc = JsonLdSpdxDocument(graph=spdx_graph_objects) + save_path = os.path.join(config.output_directory, config.spdx_file_names[kernel_sbom_kind]) + spdx_doc.save(save_path, config.prettify_json) + logging.debug(f"Successfully saved {save_path}") + + _exit_with_summary(config.write_output_on_error) + + +# Call main method +if __name__ == "__main__": + main() diff --git a/scripts/sbom/sbom/__init__.py b/scripts/sbom/sbom/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/scripts/sbom/sbom/__init__.py diff --git a/scripts/sbom/sbom/cmd_graph/__init__.py b/scripts/sbom/sbom/cmd_graph/__init__.py new file mode 100644 index 000000000000..9d661a5c3d93 --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from .cmd_graph import CmdGraph +from .cmd_graph_node import CmdGraphNode, CmdGraphNodeConfig + +__all__ = ["CmdGraph", "CmdGraphNode", "CmdGraphNodeConfig"] diff --git a/scripts/sbom/sbom/cmd_graph/cmd_file.py b/scripts/sbom/sbom/cmd_graph/cmd_file.py new file mode 100644 index 000000000000..dcd63e284a38 --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/cmd_file.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import os +import re +from dataclasses import dataclass, field +from sbom.cmd_graph.deps_parser import parse_cmd_file_deps +from sbom.cmd_graph.savedcmd_parser import parse_inputs_from_commands +import sbom.sbom_logging as sbom_logging +from sbom.path_utils import PathStr + +SAVEDCMD_PATTERN = re.compile(r"^(saved)?cmd_.*?:=\s*(?P<full_command>.+)$") +SOURCE_PATTERN = re.compile(r"^source.*?:=\s*(?P<source_file>.+)$") + + +@dataclass +class CmdFile: + cmd_file_path: PathStr + savedcmd: str + source: PathStr | None = None + deps: list[str] = field(default_factory=list) + make_rules: list[str] = field(default_factory=list) + + @classmethod + def create(cls, cmd_file_path: PathStr) -> "CmdFile | None": + """ + Parses a .cmd file. + .cmd files are assumed to have one of the following structures: + 1. Full Cmd File + (saved)?cmd_<output> := <command> + source_<output> := <main_input> + deps_<output> := \ + <dependencies> + <output> := $(deps_<output>) + $(deps_<output>): + + 2. Command Only Cmd File + (saved)?cmd_<output> := <command> + + 3. Single Dependency Cmd File + (saved)?cmd_<output> := <command> + <output> : <dependency> + + Args: + cmd_file_path (Path): absolute Path to a .cmd file + + Returns: + cmd_file (CmdFile): Parsed cmd file. + """ + with open(cmd_file_path, "rt", encoding="utf-8") as f: + lines = [line.strip() for line in f.readlines() if line.strip() != "" and not line.startswith("#")] + + # savedcmd + match = SAVEDCMD_PATTERN.match(lines[0] if lines else "") + if match is None: + sbom_logging.error( + "Skip parsing '{cmd_file_path}' because no 'savedcmd_' command was found.", cmd_file_path=cmd_file_path + ) + return None + savedcmd = match.group("full_command") + + # Command Only Cmd File + if len(lines) == 1: + return CmdFile(cmd_file_path, savedcmd) + + # Single Dependency Cmd File + if len(lines) == 2: + parts = lines[1].split(":", 1) + if len(parts) != 2: + sbom_logging.error( + "Skip parsing '{cmd_file_path}'. Expected dependency line '<output>: <dependency>' but got {second_line}", cmd_file_path=cmd_file_path, second_line=lines[1] + ) + return None + dep = parts[1].strip() + return CmdFile(cmd_file_path, savedcmd, deps=[dep]) + + # Full Cmd File + # source + line1 = SOURCE_PATTERN.match(lines[1]) + if line1 is None: + sbom_logging.error( + "Skip parsing '{cmd_file_path}' because no 'source_' entry was found.", cmd_file_path=cmd_file_path + ) + return CmdFile(cmd_file_path, savedcmd) + source = line1.group("source_file") + + # deps + deps: list[str] = [] + i = 3 # lines[2] includes the variable assignment but no actual dependency, so we need to start at lines[3]. + while i < len(lines): + if not lines[i].endswith("\\"): + break + deps.append(lines[i][:-1].strip()) + i += 1 + + # make_rules + make_rules = lines[i:] + + return CmdFile(cmd_file_path, savedcmd, source, deps, make_rules) + + def get_dependencies( + self: "CmdFile", target_path: PathStr, obj_tree: PathStr, fail_on_unknown_build_command: bool + ) -> list[PathStr]: + """ + Parses all dependencies required to build a target file from its cmd file. + + Args: + target_path: path to the target file relative to `obj_tree`. + obj_tree: absolute path to the object tree. + fail_on_unknown_build_command: Whether to fail if an unknown build command is encountered. + + Returns: + list[PathStr]: dependency file paths relative to `obj_tree`. + """ + input_files: list[PathStr] = [ + str(p) for p in parse_inputs_from_commands(self.savedcmd, fail_on_unknown_build_command) + ] + if self.deps: + input_files += [str(p) for p in parse_cmd_file_deps(self.deps)] + input_files = _expand_resolve_files(input_files, obj_tree) + + cmd_file_dependencies: list[PathStr] = [] + for input_file in input_files: + # input files are either absolute or relative to the object tree + if os.path.isabs(input_file): + input_file = os.path.relpath(input_file, obj_tree) + if input_file == target_path: + # Skip target file to prevent cycles. This is necessary because some multi stage commands first create an output and then pass it as input to the next command, e.g., objcopy. + continue + cmd_file_dependencies.append(input_file) + unique_cmd_file_dependencies = list(dict.fromkeys(cmd_file_dependencies)) + return unique_cmd_file_dependencies + + +def _expand_resolve_files(input_files: list[PathStr], obj_tree: PathStr) -> list[PathStr]: + """ + Expands resolve files which may reference additional files via '@' notation. + + Args: + input_files (list[PathStr]): List of file paths relative to the object tree, where paths starting with '@' refer to files + containing further file paths, each on a separate line. + obj_tree: Absolute path to the root of the object tree. + + Returns: + list[PathStr]: Flattened list of all input file paths, with any nested '@' file references resolved recursively. + """ + expanded_input_files: list[PathStr] = [] + for input_file in input_files: + if not input_file.startswith("@"): + expanded_input_files.append(input_file) + continue + resolve_file_path = os.path.join(obj_tree, input_file.removeprefix("@")) + if not os.path.exists(resolve_file_path): + sbom_logging.error( + "Skip resolving '{resolve_file_path}' because the response file does not exist.", + resolve_file_path=resolve_file_path, + ) + continue + with open(resolve_file_path, "rt", encoding="utf-8") as f: + resolve_file_content = [line_stripped for line in f.readlines() if (line_stripped := line.strip())] + expanded_input_files += _expand_resolve_files(resolve_file_content, obj_tree) + return expanded_input_files diff --git a/scripts/sbom/sbom/cmd_graph/cmd_graph.py b/scripts/sbom/sbom/cmd_graph/cmd_graph.py new file mode 100644 index 000000000000..2f57965237f4 --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/cmd_graph.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from collections import deque +from dataclasses import dataclass, field +from typing import Iterator + +from sbom.cmd_graph.cmd_graph_node import CmdGraphNode, CmdGraphNodeConfig +from sbom.path_utils import PathStr + + +@dataclass +class CmdGraph: + """Directed acyclic graph of build dependencies primarily inferred from .cmd files produced during kernel builds""" + + roots: list[CmdGraphNode] = field(default_factory=list) + + @classmethod + def create(cls, root_paths: list[PathStr], config: CmdGraphNodeConfig) -> "CmdGraph": + """ + Recursively builds a dependency graph starting from `root_paths`. + Dependencies are mainly discovered by parsing the `.cmd` files. + + Args: + root_paths (list[PathStr]): List of paths to root outputs relative to obj_tree + config (CmdGraphNodeConfig): Configuration options + + Returns: + CmdGraph: A graph of all build dependencies for the given root files. + """ + node_cache: dict[PathStr, CmdGraphNode] = {} + root_nodes = [CmdGraphNode.create(root_path, config, node_cache) for root_path in root_paths] + return CmdGraph(root_nodes) + + def __iter__(self) -> Iterator[CmdGraphNode]: + """Traverse the graph in breadth-first order, yielding each unique node.""" + visited: set[PathStr] = set() + node_stack: deque[CmdGraphNode] = deque(self.roots) + while len(node_stack) > 0: + node = node_stack.popleft() + if node.absolute_path in visited: + continue + + visited.add(node.absolute_path) + node_stack.extend(node.children) + yield node diff --git a/scripts/sbom/sbom/cmd_graph/cmd_graph_node.py b/scripts/sbom/sbom/cmd_graph/cmd_graph_node.py new file mode 100644 index 000000000000..61f3a8140cea --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/cmd_graph_node.py @@ -0,0 +1,142 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass, field +from itertools import chain +import logging +import os +from typing import Iterator, Protocol + +from sbom import sbom_logging +from sbom.cmd_graph.cmd_file import CmdFile +from sbom.cmd_graph.hardcoded_dependencies import get_hardcoded_dependencies +from sbom.cmd_graph.incbin_parser import parse_incbin_statements +from sbom.path_utils import PathStr, has_link, is_relative_to + + +@dataclass +class IncbinDependency: + node: "CmdGraphNode" + full_statement: str + + +class CmdGraphNodeConfig(Protocol): + obj_tree: PathStr + src_tree: PathStr + fail_on_unknown_build_command: bool + + +@dataclass +class CmdGraphNode: + """A node in the cmd graph representing a single file and its dependencies.""" + + absolute_path: PathStr + """Absolute path to the file this node represents.""" + + cmd_file: CmdFile | None = None + """Parsed .cmd file describing how the file at absolute_path was built, or None if not available.""" + + cmd_file_dependencies: list["CmdGraphNode"] = field(default_factory=list) + incbin_dependencies: list[IncbinDependency] = field(default_factory=list) + hardcoded_dependencies: list["CmdGraphNode"] = field(default_factory=list) + + @property + def children(self) -> Iterator["CmdGraphNode"]: + seen: set[PathStr] = set() + for node in chain( + self.cmd_file_dependencies, + (dep.node for dep in self.incbin_dependencies), + self.hardcoded_dependencies, + ): + if node.absolute_path not in seen: + seen.add(node.absolute_path) + yield node + + @classmethod + def create( + cls, + target_path: PathStr, + config: CmdGraphNodeConfig, + cache: dict[PathStr, "CmdGraphNode"] | None = None, + depth: int = 0, + ) -> "CmdGraphNode": + """ + Recursively builds a dependency graph starting from `target_path`. + Dependencies are mainly discovered by parsing the `.<target_path.name>.cmd` file. + + Args: + target_path: Path to the target file relative to obj_tree. + config: Config options + cache: Tracks processed nodes to prevent cycles. + depth: Internal parameter to track the current recursion depth. + + Returns: + CmdGraphNode: cmd graph node representing the target file + """ + if cache is None: + cache = {} + + target_path_absolute = ( + os.path.realpath(p) + if has_link(p:=os.path.join(config.obj_tree, target_path)) + else os.path.normpath(p) + ) + + if target_path_absolute in cache: + return cache[target_path_absolute] + + if depth == 0: + logging.debug(f"Build node: {target_path}") + + cmd_file_path = _to_cmd_path(target_path_absolute) + cmd_file = CmdFile.create(cmd_file_path) if os.path.exists(cmd_file_path) else None + node = CmdGraphNode(target_path_absolute, cmd_file) + cache[target_path_absolute] = node + + if not os.path.exists(target_path_absolute): + error_or_warning = ( + sbom_logging.error + if is_relative_to(target_path_absolute, config.obj_tree) + or is_relative_to(target_path_absolute, config.src_tree) + else sbom_logging.warning + ) + error_or_warning( + "Skip parsing '{target_path_absolute}' because file does not exist", + target_path_absolute=target_path_absolute, + ) + return node + + # Search for dependencies to add to the graph as child nodes. Child paths are always relative to the output tree. + def _build_child_node(child_path: PathStr) -> "CmdGraphNode": + return CmdGraphNode.create(child_path, config, cache, depth + 1) + + node.hardcoded_dependencies = [ + _build_child_node(hardcoded_dependency_path) + for hardcoded_dependency_path in get_hardcoded_dependencies( + target_path_absolute, config.obj_tree, config.src_tree + ) + ] + + if cmd_file is not None: + node.cmd_file_dependencies = [ + _build_child_node(cmd_file_dependency_path) + for cmd_file_dependency_path in cmd_file.get_dependencies( + target_path, config.obj_tree, config.fail_on_unknown_build_command + ) + ] + + if node.absolute_path.endswith(".S"): + node.incbin_dependencies = [ + IncbinDependency( + node=_build_child_node(incbin_statement.path), + full_statement=incbin_statement.full_statement, + ) + for incbin_statement in parse_incbin_statements(node.absolute_path) + ] + + return node + + +def _to_cmd_path(path: PathStr) -> PathStr: + name = os.path.basename(path) + return path.removesuffix(name) + f".{name}.cmd" diff --git a/scripts/sbom/sbom/cmd_graph/deps_parser.py b/scripts/sbom/sbom/cmd_graph/deps_parser.py new file mode 100644 index 000000000000..6a2d92f0778c --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/deps_parser.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import re +import sbom.sbom_logging as sbom_logging +from sbom.path_utils import PathStr + +# Match dependencies on config files +# Example match: "$(wildcard include/config/CONFIG_SOMETHING)" +CONFIG_PATTERN = re.compile(r"\$\(wildcard (include/config/[^)]+)\)") + +# Match dependencies on the objtool binary +# Example match: "$(wildcard ./tools/objtool/objtool)" +OBJTOOL_PATTERN = re.compile(r"\$\(wildcard \./tools/objtool/objtool\)") + +# Match any Makefile wildcard reference +# Example match: "$(wildcard path/to/file)" +WILDCARD_PATTERN = re.compile(r"\$\(wildcard (?P<path>[^)]+)\)") + +# Match ordinary paths: +# - ^(\/)?: Optionally starts with a '/' +# - (([\w\-\.,+~=@ ]*)\/)*: Zero or more directory levels +# - [\w\-\.,+~=@ ]+$: Path component (file or directory) +# Example matches: "/foo/bar.c", "dir1/dir2/file.txt", "plainfile" +VALID_PATH_PATTERN = re.compile(r"^(\/)?(([\w\-\.,+~=@ ]*)\/)*[\w\-\.,+~=@ ]+$") + + +def parse_cmd_file_deps(deps: list[str]) -> list[PathStr]: + """ + Parse dependency strings of a .cmd file and return valid input file paths. + + Args: + deps: List of dependency strings as found in `.cmd` files. + + Returns: + input_files: List of input file paths + """ + input_files: list[PathStr] = [] + for dep in deps: + dep = dep.strip() + match dep: + case _ if CONFIG_PATTERN.match(dep) or OBJTOOL_PATTERN.match(dep): + # config paths like include/config/<CONFIG_NAME> should not be included in the graph + continue + case _ if match := WILDCARD_PATTERN.match(dep): + path = match.group("path") + input_files.append(path) + case _ if VALID_PATH_PATTERN.match(dep): + input_files.append(dep) + case _: + sbom_logging.error("Skip parsing dependency {dep} because of unrecognized format", dep=dep) + return input_files diff --git a/scripts/sbom/sbom/cmd_graph/hardcoded_dependencies.py b/scripts/sbom/sbom/cmd_graph/hardcoded_dependencies.py new file mode 100644 index 000000000000..2eb04d30f4e6 --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/hardcoded_dependencies.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import os +from typing import Callable +import sbom.sbom_logging as sbom_logging +from sbom.path_utils import PathStr, is_relative_to +from sbom.environment import Environment + +HARDCODED_DEPENDENCIES: dict[str, list[str]] = { + # defined in linux/Kbuild + "include/generated/rq-offsets.h": ["kernel/sched/rq-offsets.s"], + "kernel/sched/rq-offsets.s": ["include/generated/asm-offsets.h"], + "include/generated/bounds.h": ["kernel/bounds.s"], + "include/generated/asm-offsets.h": ["arch/{arch}/kernel/asm-offsets.s"], +} +""" +Maps file paths to the list of dependencies required to build them +which are not tracked by the .cmd dependency mechanism. +Paths are relative to either the source tree or the object tree. +""" + +def get_hardcoded_dependencies(path: PathStr, obj_tree: PathStr, src_tree: PathStr) -> list[PathStr]: + """ + Some files in the kernel build process are not tracked by the .cmd dependency mechanism. + Parsing these dependencies programmatically is too complex for the scope of this project. + Therefore, this function provides manually defined dependencies to be added to the build graph. + + Args: + path: absolute path to a file within the src tree or object tree. + obj_tree: absolute Path to the base directory of the object tree. + src_tree: absolute Path to the `linux` source directory. + + Returns: + list[PathStr]: A list of dependency file paths (relative to the object tree) required to build the file at the given path. + """ + if is_relative_to(path, obj_tree): + path = os.path.relpath(path, obj_tree) + elif is_relative_to(path, src_tree): + path = os.path.relpath(path, src_tree) + + if path not in HARDCODED_DEPENDENCIES: + return [] + + template_variables: dict[str, Callable[[], str | None]] = { + "arch": lambda: _get_arch(path), + } + + dependencies: list[PathStr] = [] + for dependency_template in HARDCODED_DEPENDENCIES[path]: + dependency = _evaluate_template(dependency_template, template_variables) + if dependency is None: + continue + if os.path.exists(os.path.join(obj_tree, dependency)): + dependencies.append(dependency) + elif os.path.exists(dependency_absolute := os.path.join(src_tree, dependency)): + dependencies.append(os.path.relpath(dependency_absolute, obj_tree)) + else: + sbom_logging.error( + "Skip hardcoded dependency '{dependency}' for '{path}' because the dependency lies neither in the src tree nor the object tree.", + dependency=dependency, + path=path, + ) + + return dependencies + + +def _evaluate_template(template: str, variables: dict[str, Callable[[], str | None]]) -> str | None: + for key, value_function in variables.items(): + template_key = "{" + key + "}" + if template_key in template: + value = value_function() + if value is None: + return None + template = template.replace(template_key, value) + return template + + +def _get_arch(path: PathStr): + srcarch = Environment.SRCARCH() + if srcarch is None: + sbom_logging.error( + "Skipped architecture specific hardcoded dependency for '{path}' because the SRCARCH environment variable was not set.", + path=path, + ) + return None + return srcarch diff --git a/scripts/sbom/sbom/cmd_graph/incbin_parser.py b/scripts/sbom/sbom/cmd_graph/incbin_parser.py new file mode 100644 index 000000000000..ca289c2b8888 --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/incbin_parser.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass +import re + +from sbom.path_utils import PathStr + +INCBIN_PATTERN = re.compile(r'\s*\.incbin\s+"(?P<path>[^"]+)"') +"""Regex pattern for matching `.incbin "<path>"` statements.""" + + +@dataclass +class IncbinStatement: + """A parsed `.incbin "<path>"` directive.""" + + path: PathStr + """path to the file referenced by the `.incbin` directive.""" + + full_statement: str + """Full `.incbin "<path>"` statement as it originally appeared in the file.""" + + +def parse_incbin_statements(absolute_path: PathStr) -> list[IncbinStatement]: + """ + Parses `.incbin` directives from an `.S` assembly file. + + Args: + absolute_path: Absolute path to the `.S` assembly file. + + Returns: + list[IncbinStatement]: Parsed `.incbin` statements. + """ + with open(absolute_path, "rt", encoding="utf-8") as f: + content = f.read() + return [ + IncbinStatement( + path=match.group("path"), + full_statement=match.group(0).strip(), + ) + for match in INCBIN_PATTERN.finditer(content) + ] diff --git a/scripts/sbom/sbom/cmd_graph/savedcmd_parser/__init__.py b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/__init__.py new file mode 100644 index 000000000000..d13876af4dfd --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from sbom.cmd_graph.savedcmd_parser.savedcmd_parser import parse_inputs_from_commands + +__all__ = ["parse_inputs_from_commands"] diff --git a/scripts/sbom/sbom/cmd_graph/savedcmd_parser/command_parser_registry.py b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/command_parser_registry.py new file mode 100644 index 000000000000..a48040b2c13c --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/command_parser_registry.py @@ -0,0 +1,516 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import re +import shlex +from typing import Callable, Iterator + +import sbom.sbom_logging as sbom_logging +from sbom.environment import Environment +from sbom.cmd_graph.savedcmd_parser.command_splitter import IfBlock, split_commands +from sbom.cmd_graph.savedcmd_parser.tokenizer import ( + CmdParsingError, + Option, + Positional, + tokenize_single_command, + tokenize_single_command_positionals_only, +) +from sbom.path_utils import PathStr + +CommandParser = Callable[[str], list[PathStr]] +CommandParserRegistryEntry = tuple[re.Pattern[str], CommandParser] + + +def _parse_dd_command(command: str) -> list[PathStr]: + match = re.match(r"dd.*?if=(\S+)", command) + if match: + return [match.group(1)] + return [] + + +def _parse_cat_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ["cat", input1, input2, ...] + return [p for p in positionals[1:]] + + +def _parse_compound_command(command: str) -> list[PathStr]: + compound_command_parsers: list[CommandParserRegistryEntry] = [ + (re.compile(r"dd\b"), _parse_dd_command), + (re.compile(r"cat.*?\|"), lambda c: _parse_cat_command(c.split("|")[0])), + (re.compile(r"cat\b[^|>]*$"), _parse_cat_command), + (re.compile(r"echo\b"), _parse_noop), + (re.compile(r"\S+="), _parse_noop), + (re.compile(r"printf\b"), _parse_noop), + (re.compile(r"sed\b"), _parse_sed_command), + ( + re.compile(r"(.*/)scripts/bin2c\s*<"), + lambda c: [input] if (input := c.split("<")[1].split(">")[0].strip()) != "/dev/null" else [], + ), + (re.compile(r"^:$"), _parse_noop), + ] + + match = re.match(r"\s*[\(\{](.*)[\)\}]\s*>", command, re.DOTALL) + if match is None: + raise CmdParsingError("No inner commands found for compound command") + input_files: list[PathStr] = [] + inner_commands = split_commands(match.group(1)) + for inner_command in inner_commands: + if isinstance(inner_command, IfBlock): + sbom_logging.error( + "Skip parsing inner command {inner_command} of compound command because IfBlock is not supported", + inner_command=inner_command, + ) + continue + + parser = next((parser for pattern, parser in compound_command_parsers if pattern.match(inner_command)), None) + if parser is None: + sbom_logging.error( + "Skip parsing inner command {inner_command} of compound command because no matching parser was found", + inner_command=inner_command, + ) + continue + try: + input_files += parser(inner_command) + except (CmdParsingError, IndexError) as e: + sbom_logging.error( + "Skip parsing inner command {inner_command} of compound command because of command parsing error: {error_message}", + inner_command=inner_command, + error_message=str(e), + ) + return input_files + + +def _parse_objcopy_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command, flag_options=["-S", "-w"]) + positionals = [part.value for part in command_parts if isinstance(part, Positional)] + # expect positionals to be ['objcopy', input_file] or ['objcopy', input_file, output_file] + return [positionals[1]] + + +def _parse_link_vmlinux_command(command: str) -> list[PathStr]: + """ + For simplicity we do not parse the `scripts/link-vmlinux.sh` script. + Instead the `vmlinux.a` dependency is just hardcoded for now. + """ + return ["vmlinux.a"] + + +def _parse_cp_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ["cp", input1, ..., destination] + return positionals[1:-1] + + +def _parse_noop(command: str) -> list[PathStr]: + """ + No-op parser for commands with no input files (e.g., 'rm', 'true'). + Returns an empty list. + """ + return [] + + +def _parse_ar_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ['ar', flags, output, input1, input2, ...] + flags = positionals[1] + if "r" not in flags: + # 'r' option indicates that new files are added to the archive. + # If this option is missing we won't find any relevant input files. + return [] + return positionals[3:] + + +def _parse_ar_piped_xargs_command(command: str) -> list[PathStr]: + printf_command, _ = command.split("|", 1) + positionals = tokenize_single_command_positionals_only(printf_command.strip()) + # expect positionals to be ['printf', '{prefix_path}%s ', input1, input2, ...] + prefix_path = positionals[1].removesuffix("%s ") + return [f"{prefix_path}{filename}" for filename in positionals[2:]] + + +def _parse_gcc_or_clang_command(command: str) -> list[PathStr]: + parts = shlex.split(command) + # compile mode: expect last positional argument ending in a source file extension to be the input file + for part in reversed(parts): + if not part.startswith("-") and any(part.endswith(suffix) for suffix in [".c", ".S", ".dts"]): + return [part] + + # linking mode: expect all .o files to be the inputs + return [p for p in parts if p.endswith(".o")] + + +def _parse_rustc_command(command: str) -> list[PathStr]: + parts = shlex.split(command) + # expect last positional argument ending in `.rs` to be the input file + for part in reversed(parts): + if not part.startswith("-") and part.endswith(".rs"): + return [part] + raise CmdParsingError("Could not find .rs input source file") + + +def _parse_rustdoc_command(command: str) -> list[PathStr]: + parts = shlex.split(command) + # expect last positional argument ending in `.rs` to be the input file + for part in reversed(parts): + if not part.startswith("-") and part.endswith(".rs"): + return [part] + raise CmdParsingError("Could not find .rs input source file") + + +def _parse_syscallhdr_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command.strip(), flag_options=["--emit-nr"]) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be ["sh", path/to/syscallhdr.sh, input, output] + return [positionals[2]] + + +def _parse_syscalltbl_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command.strip()) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be ["sh", path/to/syscalltbl.sh, input, output] + return [positionals[2]] + + +def _parse_mkcapflags_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ["sh", path/to/mkcapflags.sh, output, input1, input2] + return [positionals[3], positionals[4]] + + +def _parse_orc_hash_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ["sh", path/to/orc_hash.sh, '<', input, '>', output] + return [positionals[3]] + + +def _parse_xen_hypercalls_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ["sh", path/to/xen-hypercalls.sh, output, input1, input2, ...] + return positionals[3:] + + +def _parse_gen_initramfs_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be ["sh", path/to/gen_initramfs.sh, input1, input2, ...] + return positionals[2:] + + +def _parse_vdso2c_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ['vdso2c', raw_input, stripped_input, output] + return [positionals[1], positionals[2]] + + +def _parse_vdsomunge_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ['vdsomunge', input, output] + return [positionals[1]] + + +def _parse_ld_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command( + command=command.strip(), + flag_options=[ + "-shared", + "--no-undefined", + "--eh-frame-hdr", + "-Bsymbolic", + "-r", + "--no-ld-generated-unwind-info", + "--no-dynamic-linker", + "-pie", + "--no-dynamic-linker--whole-archive", + "--whole-archive", + "--no-whole-archive", + "--start-group", + "--end-group", + ], + ) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be ["ld", input1, input2, ...] + return positionals[1:] + + +def _parse_sed_command(command: str) -> list[PathStr]: + command_parts = shlex.split(command) + # expect command parts to be ["sed", *, input] + input = command_parts[-1] + if input == "/dev/null": + return [] + return [input] + + +def _parse_awk(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command) + options = [p for p in command_parts if isinstance(p, Option)] + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + has_script_file = any(p.name == "-f" for p in options) + # With -f option: expect ["awk", input1, input2, ...] + # Without -f option: expect ["awk", inline_program, input1, input2, ...] + return positionals[1:] if has_script_file else positionals[2:] + + +def _parse_nm_piped_command(command: str) -> list[PathStr]: + nm_command, _ = command.split("|", 1) + command_parts = tokenize_single_command( + command=nm_command.strip(), + flag_options=["-p", "--defined-only"], + ) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be ["nm", input1, input2, ...] + return [p for p in positionals[1:]] + + +def _parse_pnm_to_logo_command(command: str) -> list[PathStr]: + command_parts = shlex.split(command) + # expect command parts to be ["pnmtologo", <options>, input] + return [command_parts[-1]] + + +def _parse_relacheck(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ["relacheck", input, log_reference] + return [positionals[1]] + + +def _parse_gen_hyprel_command(command: str) -> list[PathStr]: + gen_hyprel_command, _ = command.split(">", 1) + command_parts = shlex.split(gen_hyprel_command) + # expect command_parts to be ["gen-hyprel", input] + return [command_parts[1]] + + +def _parse_perl_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command.strip()) + # expect positionals to be ["perl", input] + return [positionals[1]] + + +def _parse_strip_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command, flag_options=["--strip-debug"]) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be ["strip", input1, input2, ...] + return positionals[1:] + + +def _parse_mkpiggy_command(command: str) -> list[PathStr]: + mkpiggy_command, _ = command.split(">", 1) + positionals = tokenize_single_command_positionals_only(mkpiggy_command) + # expect positionals to be ["mkpiggy", input] + return [positionals[1]] + + +def _parse_relocs_command(command: str) -> list[PathStr]: + if ">" not in command: + # Only consider relocs commands that redirect output to a file. + # If there's no redirection, we assume it produces no output file and therefore has no input we care about. + return [] + relocs_command, _ = command.split(">", 1) + command_parts = shlex.split(relocs_command) + # expect command_parts to be ["relocs", options, input] + return [command_parts[-1]] + + +def _parse_mk_elfconfig_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ["mk_elfconfig", "<", input, ">", output] + return [positionals[2]] + + +def _parse_flex_command(command: str) -> list[PathStr]: + parts = shlex.split(command) + # expect last positional argument ending in `.l` to be the input file + for part in reversed(parts): + if not part.startswith("-") and part.endswith(".l"): + return [part] + raise CmdParsingError("Could not find .l input source file in command") + + +def _parse_bison_command(command: str) -> list[PathStr]: + parts = shlex.split(command) + # expect last positional argument ending in `.y` to be the input file + for part in reversed(parts): + if not part.startswith("-") and part.endswith(".y"): + return [part] + raise CmdParsingError("Could not find input .y input source file in command") + + +def _parse_tools_build_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ["tools/build", "input1", "input2", "input3", "output"] + return positionals[1:-1] + + +def _parse_extract_cert_command(command: str) -> list[PathStr]: + command_parts = shlex.split(command) + # expect command parts to be [path/to/extract-cert, input, output] + input = command_parts[1] + if not input: + return [] + return [input] + + +def _parse_dtc_command(command: str) -> list[PathStr]: + wno_flags = [command_part for command_part in shlex.split(command) if command_part.startswith("-Wno-")] + command_parts = tokenize_single_command(command, flag_options=wno_flags) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be [path/to/dtc, input] + return [positionals[1]] + + +def _parse_bindgen_command(command: str) -> list[PathStr]: + command_parts = shlex.split(command) + header_file_input_paths = [part for part in command_parts if part.endswith(".h")] + return header_file_input_paths + + +def _parse_gen_header(command: str) -> list[PathStr]: + command_parts = shlex.split(command) + # expect command parts to be ["python3", path/to/gen_headers.py, ..., "--xml", input] + i = next((i for i, token in enumerate(command_parts) if token == "--xml"), None) + if i is None: + raise CmdParsingError(f"Expected --xml input file in gen_headers command but got {command}") + return [command_parts[i + 1]] + +def _parse_mkuboot_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command) + # mkuboot.sh passes all args to mkimage; -d specifies the data/input image file + for part in command_parts: + if isinstance(part, Option) and part.name == "-d" and part.value is not None: + return [part.value] + raise CmdParsingError("Could not find -d (data file) option in mkuboot.sh command") + + +def _parse_syscallnr_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command.strip()) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be ["sh", path/to/syscallnr.sh, input, output] + return [positionals[2]] + + +def _parse_gen_kernel_hwcaps_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command.strip(), flag_options=["-e"]) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be ["sh", path/to/gen-kernel-hwcaps.sh, input] + return [positionals[2]] + + +class CommandParserRegistry: + """ + Registry mapping command patterns to their input-file parsers. + """ + + def __init__(self, entries: list[CommandParserRegistryEntry]) -> None: + self._entries = entries + + def __iter__(self) -> Iterator[CommandParserRegistryEntry]: + return iter(self._entries) + + @staticmethod + def create() -> "CommandParserRegistry": + def env_or_default_pattern(env_value: str | None, default_pattern: str) -> str: + if env_value is None or not env_value.strip(): + return default_pattern + return rf"(?:{re.escape(env_value.strip())}|{default_pattern})" + + cc_pattern = env_or_default_pattern(Environment.CC(), r"([^\s]+-)?(gcc|clang)") + ld_pattern = env_or_default_pattern(Environment.LD(), r"([^\s]+-)?ld") + ar_pattern = env_or_default_pattern(Environment.AR(), r"([^\s]+-)?ar") + nm_pattern = env_or_default_pattern(Environment.NM(), r"([^\s]+-)?nm") + objcopy_pattern = env_or_default_pattern(Environment.OBJCOPY(), r"([^\s]+-)?objcopy") + strip_pattern = env_or_default_pattern(Environment.STRIP(), r"([^\s]+-)?strip") + + entries: list[CommandParserRegistryEntry] = [ + # Compound commands + (re.compile(r"\(.*?\)\s*>", re.DOTALL), _parse_compound_command), + (re.compile(r"\{.*?\}\s*>", re.DOTALL), _parse_compound_command), + # Standard Unix utilities and system tools + (re.compile(r"^rm\b"), _parse_noop), + (re.compile(r"^mkdir\b"), _parse_noop), + (re.compile(r"^touch\b"), _parse_noop), + (re.compile(r"^cp\b"), _parse_cp_command), + (re.compile(r"^truncate\b"), _parse_noop), + (re.compile(r"^cat\b.*?[\|>]"), lambda c: _parse_cat_command(c.split("|")[0].split(">")[0])), + (re.compile(r"^echo[^|]*$"), _parse_noop), + (re.compile(r"^sed.*?>"), lambda c: _parse_sed_command(c.split(">")[0])), + (re.compile(r"^sed\b"), _parse_noop), + (re.compile(r"^awk.*?<.*?>"), lambda c: [c.split("<")[1].split(">")[0]]), + (re.compile(r"^awk.*?>"), lambda c: _parse_awk(c.split(">")[0])), + (re.compile(r"^(/bin/)?true\b"), _parse_noop), + (re.compile(r"^(/bin/)?false\b"), _parse_noop), + (re.compile(r"^openssl\s+req.*?-new.*?-keyout"), _parse_noop), + # Compilers and code generators + # (C/LLVM toolchain, Rust, Flex/Bison, Bindgen, Perl, etc.) + ( + re.compile(rf"^{cc_pattern}\b"), + lambda command: _parse_gcc_or_clang_command(re.sub(rf"^{cc_pattern}\b", "gcc", command, count=1)), + ), + ( + re.compile(rf"^{ld_pattern}\b"), + lambda command: _parse_ld_command(re.sub(rf"^{ld_pattern}\b", "ld", command, count=1)), + ), + ( + re.compile(rf"^printf\b.*\| xargs {ar_pattern}\b"), + lambda command: _parse_ar_piped_xargs_command( + re.sub(rf"xargs {ar_pattern}\b", "xargs ar", command, count=1) + ), + ), + ( + re.compile(rf"^{ar_pattern}\b"), + lambda command: _parse_ar_command(re.sub(rf"^{ar_pattern}\b", "ar", command, count=1)), + ), + ( + re.compile(rf"^{nm_pattern}\b.*?\|"), + lambda command: _parse_nm_piped_command(re.sub(rf"^{nm_pattern}\b", "nm", command, count=1)), + ), + ( + re.compile(rf"^{objcopy_pattern}\b"), + lambda command: _parse_objcopy_command(re.sub(rf"^{objcopy_pattern}\b", "objcopy", command, count=1)), + ), + ( + re.compile(rf"^{strip_pattern}\b"), + lambda command: _parse_strip_command(re.sub(rf"^{strip_pattern}\b", "strip", command, count=1)), + ), + (re.compile(r".*?rustc\b"), _parse_rustc_command), + (re.compile(r".*?rustdoc\b"), _parse_rustdoc_command), + (re.compile(r"^flex\b"), _parse_flex_command), + (re.compile(r"^bison\b"), _parse_bison_command), + (re.compile(r"^bindgen\b"), _parse_bindgen_command), + (re.compile(r"^perl\b"), _parse_perl_command), + # Kernel-specific build scripts and tools + (re.compile(r"^(.*/)?link-vmlinux\.sh\b"), _parse_link_vmlinux_command), + (re.compile(r"sh (.*/)?syscallhdr\.sh\b"), _parse_syscallhdr_command), + (re.compile(r"sh (.*/)?syscalltbl\.sh\b"), _parse_syscalltbl_command), + (re.compile(r"sh (.*/)?mkcapflags\.sh\b"), _parse_mkcapflags_command), + (re.compile(r"sh (.*/)?orc_hash\.sh\b"), _parse_orc_hash_command), + (re.compile(r"sh (.*/)?xen-hypercalls\.sh\b"), _parse_xen_hypercalls_command), + (re.compile(r"sh (.*/)?gen_initramfs\.sh\b"), _parse_gen_initramfs_command), + (re.compile(r"sh (.*/)?checkundef\.sh\b"), _parse_noop), + (re.compile(r"(bash|sh) (.*/)?mkuboot\.sh\b"), _parse_mkuboot_command), + (re.compile(r"sh (.*/)?syscallnr\.sh\b"), _parse_syscallnr_command), + (re.compile(r"(/bin/)?sh (.*/)?gen-kernel-hwcaps\.sh\b"), lambda c: _parse_gen_kernel_hwcaps_command(c.split(">")[0])), + (re.compile(r"(.*/)?vdso2c\b"), _parse_vdso2c_command), + (re.compile(r"(.*/)?vdsomunge\b"), _parse_vdsomunge_command), + (re.compile(r"^(.*/)?mkpiggy.*?>"), _parse_mkpiggy_command), + (re.compile(r"^(.*/)?relocs\b"), _parse_relocs_command), + (re.compile(r"^(.*/)?mk_elfconfig.*?<.*?>"), _parse_mk_elfconfig_command), + (re.compile(r"^(.*/)?tools/build\b"), _parse_tools_build_command), + (re.compile(r"^(.*/)?certs/extract-cert"), _parse_extract_cert_command), + (re.compile(r"^(.*/)?scripts/dtc/dtc\b"), _parse_dtc_command), + (re.compile(r"^(.*/)?pnmtologo\b"), _parse_pnm_to_logo_command), + (re.compile(r"^(.*/)?kernel/pi/relacheck"), _parse_relacheck), + (re.compile(r"^(.*/)?gen-hyprel\b"), _parse_gen_hyprel_command), + (re.compile(r"^drivers/gpu/drm/radeon/mkregtable"), lambda c: [c.split(" ")[1]]), + (re.compile(r"(.*/)?genheaders\b"), _parse_noop), + (re.compile(r"^(.*/)?mkcpustr\s+>"), _parse_noop), + (re.compile(r"^(.*/)polgen\b"), _parse_noop), + (re.compile(r"make -f .*/arch/x86/Makefile\.postlink"), _parse_noop), + (re.compile(r"^(.*/)?raid6/mktables\s+>"), _parse_noop), + (re.compile(r"^(.*/)?objtool\b"), _parse_noop), + (re.compile(r"^(.*/)?module/gen_test_kallsyms.sh"), _parse_noop), + (re.compile(r"^(.*/)?gen_header.py"), _parse_gen_header), + (re.compile(r"^(.*/)?scripts/rustdoc_test_gen"), _parse_noop), + ] + return CommandParserRegistry(entries) diff --git a/scripts/sbom/sbom/cmd_graph/savedcmd_parser/command_splitter.py b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/command_splitter.py new file mode 100644 index 000000000000..4749f4bd669e --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/command_splitter.py @@ -0,0 +1,128 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import re +from dataclasses import dataclass + + +# If Block pattern to match a simple, single-level if-then-fi block. Nested If blocks are not supported. +IF_BLOCK_PATTERN = re.compile( + r""" + ^if(.*?);\s* # Match 'if <condition>;' (non-greedy) + then(.*?);\s* # Match 'then <body>;' (non-greedy) + fi\b # Match 'fi' + """, + re.VERBOSE, +) + + +@dataclass +class IfBlock: + condition: str + then_statement: str + + +def _unwrap_outer_parentheses(s: str) -> str: + s = s.strip() + if not (s.startswith("(") and s.endswith(")")): + return s + + count = 0 + for i, char in enumerate(s): + if char == "(": + count += 1 + elif char == ")": + count -= 1 + # If count is 0 before the end, outer parentheses don't match + if count == 0 and i != len(s) - 1: + return s + + # outer parentheses do match, unwrap once + return _unwrap_outer_parentheses(s[1:-1]) + + +def _find_first_top_level_command_separator( + commands: str, separators: list[str] = [";", "&&"] +) -> tuple[int | None, int | None]: + def is_escaped(index: int) -> bool: + preceding = commands[:index] + return (len(preceding) - len(preceding.rstrip("\\"))) % 2 == 1 + + in_single_quote = False + in_double_quote = False + in_curly_braces = 0 + in_braces = 0 + for i, char in enumerate(commands): + if char == "'" and not in_double_quote and not is_escaped(i): + # Toggle single quote state (unless inside double quotes or escaped) + in_single_quote = not in_single_quote + elif char == '"' and not in_single_quote and not is_escaped(i): + # Toggle double quote state (unless inside single quotes or escaped) + in_double_quote = not in_double_quote + + if in_single_quote or in_double_quote: + continue + + # Toggle braces state + if char == "{": + in_curly_braces += 1 + if char == "}": + in_curly_braces -= 1 + + if char == "(": + in_braces += 1 + if char == ")": + in_braces -= 1 + + if in_curly_braces > 0 or in_braces > 0: + continue + + # return found separator position and separator length + for separator in separators: + if commands[i : i + len(separator)] == separator: + return i, len(separator) + + return None, None + + +def split_commands(commands: str) -> list[str | IfBlock]: + """ + Splits a string of command-line commands into individual parts. + + This function handles: + - Top-level command separators (e.g., `;` and `&&`) to split multiple commands. + - Conditional if-blocks, returning them as `IfBlock` instances. + - Preserves the order of commands and trims whitespace. + + Args: + commands (str): The raw command string. + + Returns: + list[str | IfBlock]: A list of single commands or `IfBlock` objects. + """ + single_commands: list[str | IfBlock] = [] + remaining_commands = _unwrap_outer_parentheses(commands) + while len(remaining_commands) > 0: + remaining_commands = remaining_commands.strip() + + # if block + matched_if = IF_BLOCK_PATTERN.match(remaining_commands) + if matched_if: + condition, then_statement = matched_if.groups() + single_commands.append(IfBlock(condition.strip(), then_statement.strip())) + full_matched = matched_if.group(0) + remaining_commands = remaining_commands.removeprefix(full_matched).lstrip("; \n") + continue + + # command until next separator + separator_position, separator_length = _find_first_top_level_command_separator(remaining_commands) + if separator_position is not None and separator_length is not None: + single_commands.append(remaining_commands[:separator_position].strip()) + remaining_commands = remaining_commands[separator_position + separator_length :].strip() + continue + + # single last command + single_commands.append(remaining_commands) + break + + return single_commands diff --git a/scripts/sbom/sbom/cmd_graph/savedcmd_parser/savedcmd_parser.py b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/savedcmd_parser.py new file mode 100644 index 000000000000..d2ca842a7849 --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/savedcmd_parser.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import sbom.sbom_logging as sbom_logging +from sbom.cmd_graph.savedcmd_parser.command_splitter import IfBlock, split_commands +from sbom.cmd_graph.savedcmd_parser.command_parser_registry import CommandParserRegistry +from sbom.cmd_graph.savedcmd_parser.tokenizer import CmdParsingError +from sbom.path_utils import PathStr + +DEFAULT_COMMAND_PARSER_REGISTRY = CommandParserRegistry.create() + + +def parse_inputs_from_commands( + commands: str, + fail_on_unknown_build_command: bool, + registry: CommandParserRegistry | None = None, +) -> list[PathStr]: + """ + Extract input files referenced in a set of command-line commands. + + Args: + commands (str): Command line expression to parse. + fail_on_unknown_build_command (bool): Whether to fail if an unknown build command is encountered. If False, errors are logged as warnings. + registry (CommandParserRegistry | None): Registry of single command parsers. + + Returns: + list[PathStr]: List of input file paths required by the commands. + """ + + def log_error_or_warning(message: str, /, **kwargs: str) -> None: + if fail_on_unknown_build_command: + sbom_logging.error(message, **kwargs) + else: + sbom_logging.warning(message, **kwargs) + + if registry is None: + registry = DEFAULT_COMMAND_PARSER_REGISTRY + + input_files: list[PathStr] = [] + for single_command in split_commands(commands): + if isinstance(single_command, IfBlock): + inputs = parse_inputs_from_commands(single_command.then_statement, fail_on_unknown_build_command, registry) + if inputs: + log_error_or_warning( + "Skipped parsing command {then_statement} because input files in IfBlock 'then' statement are not supported", + then_statement=single_command.then_statement, + ) + continue + + matched_parser = next((parser for pattern, parser in registry if pattern.match(single_command)), None) + if matched_parser is None: + log_error_or_warning( + "Skipped parsing command {single_command} because no matching parser was found", + single_command=single_command, + ) + continue + try: + inputs = matched_parser(single_command) + input_files.extend(inputs) + except (CmdParsingError, IndexError, ValueError) as e: + log_error_or_warning( + "Skipped parsing command {single_command} because of command parsing error: {error_message}", + single_command=single_command, + error_message=str(e), + ) + + return [input.strip().rstrip("/") for input in input_files] diff --git a/scripts/sbom/sbom/cmd_graph/savedcmd_parser/tokenizer.py b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/tokenizer.py new file mode 100644 index 000000000000..1bf081f40be7 --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/tokenizer.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import re +import shlex +from dataclasses import dataclass +from typing import Union + + +class CmdParsingError(Exception): + pass + + +@dataclass +class Option: + name: str + value: str | None = None + + +@dataclass +class Positional: + value: str + + +_SUBCOMMAND_PATTERN = re.compile(r"\$\$\(([^()]*)\)") +"""Pattern to match $$(...) blocks""" + + +def tokenize_single_command(command: str, flag_options: list[str] | None = None) -> list[Union[Option, Positional]]: + """ + Parse a shell command into a list of Options and Positionals. + - Positional: the command and any positional arguments. + - Options: handles flags and options with values provided as space-separated, or equals-sign + (e.g., '--opt val', '--opt=val', '--flag'). + + Args: + command: Command line string. + flag_options: Options that are flags without values (e.g., '--verbose'). + + Returns: + List of `Option` and `Positional` objects in command order. + """ + + # Wrap all $$(...) blocks in double quotes to prevent shlex from splitting them. + command_with_protected_subcommands = _SUBCOMMAND_PATTERN.sub(lambda m: f'"$$({m.group(1)})"', command) + tokens = shlex.split(command_with_protected_subcommands) + + parsed: list[Option | Positional] = [] + i = 0 + while i < len(tokens): + token = tokens[i] + + # Positional + if not token.startswith("-"): + parsed.append(Positional(token)) + i += 1 + continue + + # Option without value (--flag) + if (token.startswith("-") and i + 1 < len(tokens) and tokens[i + 1].startswith("-")) or ( + flag_options and token in flag_options + ): + parsed.append(Option(name=token)) + i += 1 + continue + + # Option with equals sign (--opt=val) + if "=" in token: + name, value = token.split("=", 1) + parsed.append(Option(name=name, value=value)) + i += 1 + continue + + # Option with space-separated value (--opt val) + if i + 1 < len(tokens) and not tokens[i + 1].startswith("-"): + parsed.append(Option(name=token, value=tokens[i + 1])) + i += 2 + continue + + raise CmdParsingError(f"Unrecognized token: {token} in command {command}") + + return parsed + + +def tokenize_single_command_positionals_only(command: str) -> list[str]: + command_parts = tokenize_single_command(command) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + if len(positionals) != len(command_parts): + raise CmdParsingError( + f"Invalid command format: expected positional arguments only but got options in command {command}." + ) + return positionals diff --git a/scripts/sbom/sbom/config.py b/scripts/sbom/sbom/config.py new file mode 100644 index 000000000000..6811f782943e --- /dev/null +++ b/scripts/sbom/sbom/config.py @@ -0,0 +1,320 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum +import os +from typing import Any +from sbom.path_utils import PathStr + + +class KernelSpdxDocumentKind(Enum): + SOURCE = "source" + BUILD = "build" + OUTPUT = "output" + + +@dataclass +class KernelSbomConfig: + src_tree: PathStr + """Absolute path to the Linux kernel source directory.""" + + obj_tree: PathStr + """Absolute path to the build output directory.""" + + root_paths: list[PathStr] + """List of paths to root outputs (relative to obj_tree) to base the SBOM on.""" + + generate_spdx: bool + """Whether to generate SPDX SBOM documents. If False, no SPDX files are created.""" + + spdx_file_names: dict[KernelSpdxDocumentKind, str] + """If `generate_spdx` is True, defines the file names for each SPDX SBOM kind + (source, build, output) to store on disk.""" + + generate_used_files: bool + """Whether to generate a flat list of all source files used in the build. + If False, no used-files document is created.""" + + used_files_file_name: str + """If `generate_used_files` is True, specifies the file name for the used-files document.""" + + output_directory: PathStr + """Path to the directory where the generated output documents will be saved.""" + + debug: bool + """Whether to enable debug logging.""" + + fail_on_unknown_build_command: bool + """Whether to fail if an unknown build command is encountered in a .cmd file.""" + + write_output_on_error: bool + """Whether to write output documents even if errors occur.""" + + created: datetime + """Datetime to use for the SPDX created property of the CreationInfo element.""" + + spdxId_prefix: str + """Prefix to use for all SPDX element IDs.""" + + build_type: str + """SPDX buildType property to use for all Build elements.""" + + build_id: str | None + """SPDX buildId property to use for all Build elements.""" + + package_license: str + """License expression applied to all SPDX Packages.""" + + package_version: str | None + """Version string applied to all SPDX Packages.""" + + package_copyright_text: str | None + """Copyright text applied to all SPDX Packages.""" + + prettify_json: bool + """Whether to pretty-print generated SPDX JSON documents.""" + + +def _parse_cli_arguments(parser: argparse.ArgumentParser) -> dict[str, Any]: + """ + Parse command-line arguments using argparse. + + Returns: + Dictionary of parsed arguments. + """ + parser.add_argument( + "--src-tree", + default="../linux", + help="Path to the kernel source tree (default: ../linux)", + ) + parser.add_argument( + "--obj-tree", + default="../linux/kernel_build", + help="Path to the build output directory (default: ../linux/kernel_build)", + ) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "--roots", + nargs="+", + help="Space-separated list of paths relative to obj-tree for which the SBOM will be created.\n" + "Cannot be used together with --roots-file.", + ) + group.add_argument( + "--roots-file", + help="Path to a file containing the root paths (one per line). Cannot be used together with --roots.", + ) + parser.add_argument( + "--generate-spdx", + action="store_true", + default=False, + help=( + "Whether to create sbom-source.spdx.json, sbom-build.spdx.json and " + "sbom-output.spdx.json documents (default: False)" + ), + ) + parser.add_argument( + "--generate-used-files", + action="store_true", + default=False, + help=( + "Whether to create the sbom.used-files.txt file, a flat list of all " + "source files used for the kernel build.\n" + "If src-tree and obj-tree are equal it is not possible to reliably " + "classify source files.\n" + "In this case sbom.used-files.txt will contain all files used for the " + "kernel build including all build artifacts. (default: False)" + ), + ) + parser.add_argument( + "--output-directory", + default=".", + help="Path to the directory where the generated output documents will be stored (default: .)", + ) + parser.add_argument( + "--debug", + action="store_true", + default=False, + help="Enable debug logs (default: False)", + ) + + # Error handling settings + parser.add_argument( + "--do-not-fail-on-unknown-build-command", + action="store_true", + default=False, + help=( + "Whether to fail if an unknown build command is encountered in a .cmd file.\n" + "If set to True, errors are logged as warnings instead. (default: False)" + ), + ) + parser.add_argument( + "--write-output-on-error", + action="store_true", + default=False, + help=( + "Write output documents even if errors occur. The resulting documents " + "may be incomplete.\n" + "A summary of warnings and errors can be found in the 'comment' property " + "of the CreationInfo element. (default: False)" + ), + ) + + # SPDX specific options + spdx_group = parser.add_argument_group("SPDX options", "Options for customizing SPDX document generation") + spdx_group.add_argument( + "--spdxId-prefix", + default="urn:spdx.dev:", + help="The prefix to use for all spdxId properties. (default: urn:spdx.dev:)", + ) + spdx_group.add_argument( + "--build-type", + default="urn:spdx.dev:Kbuild", + help="The SPDX buildType property to use for all Build elements. (default: urn:spdx.dev:Kbuild)", + ) + spdx_group.add_argument( + "--build-id", + default=None, + help="The SPDX buildId property to use for all Build elements.\n" + "If not provided the spdxId of the high level Build element is used as the buildId. (default: None)", + ) + spdx_group.add_argument( + "--package-license", + default="NOASSERTION", + help=( + "The SPDX licenseExpression property to use for the LicenseExpression " + "linked to all SPDX Package elements. (default: NOASSERTION)" + ), + ) + spdx_group.add_argument( + "--package-version", + default=None, + help="The SPDX packageVersion property to use for all SPDX Package elements. (default: None)", + ) + spdx_group.add_argument( + "--package-copyright-text", + default=None, + help=( + "The SPDX copyrightText property to use for all SPDX Package elements.\n" + "If not specified, and if a COPYING file exists in the source tree,\n" + "the package-copyright-text is set to the content of this file. " + "(default: None)" + ), + ) + spdx_group.add_argument( + "--prettify-json", + action="store_true", + default=False, + help="Whether to pretty print the generated spdx.json documents (default: False)", + ) + + args = vars(parser.parse_args()) + return args + + +def get_config() -> KernelSbomConfig: + """ + Parse command-line arguments and construct the configuration object. + + Returns: + KernelSbomConfig: Configuration object with all settings for SBOM generation. + """ + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, + description="Generate SPDX SBOM documents for kernel builds", + ) + args = _parse_cli_arguments(parser) + + # Extract and validate cli arguments + src_tree = os.path.realpath(args["src_tree"]) + obj_tree = os.path.realpath(args["obj_tree"]) + root_paths = [] + if args["roots_file"]: + with open(args["roots_file"], "rt", encoding="utf-8") as f: + root_paths = [root.strip() for root in f.readlines()] + if len(root_paths) == 0: + parser.error("--roots-file must contain at least one path") + else: + root_paths = args["roots"] + _validate_path_arguments(parser, src_tree, obj_tree, root_paths) + + generate_spdx = args["generate_spdx"] + generate_used_files = args["generate_used_files"] + output_directory = os.path.realpath(args["output_directory"]) + debug = args["debug"] + + fail_on_unknown_build_command = not args["do_not_fail_on_unknown_build_command"] + write_output_on_error = args["write_output_on_error"] + + created = datetime.fromtimestamp( + max([os.path.getmtime(os.path.join(obj_tree, root_path)) for root_path in root_paths]), + tz=timezone.utc, + ) + spdxId_prefix = args["spdxId_prefix"] + build_type = args["build_type"] + build_id = args["build_id"] + package_license = args["package_license"] + package_version = args["package_version"] if args["package_version"] is not None else None + package_copyright_text: str | None = None + if args["package_copyright_text"] is not None: + package_copyright_text = args["package_copyright_text"] + elif os.path.isfile(copying_path := os.path.join(src_tree, "COPYING")): + with open(copying_path, "r", encoding="utf-8") as f: + package_copyright_text = f.read() + prettify_json = args["prettify_json"] + + # Hardcoded config + spdx_file_names = { + KernelSpdxDocumentKind.SOURCE: "sbom-source.spdx.json", + KernelSpdxDocumentKind.BUILD: "sbom-build.spdx.json", + KernelSpdxDocumentKind.OUTPUT: "sbom-output.spdx.json", + } + used_files_file_name = "sbom.used-files.txt" + + return KernelSbomConfig( + src_tree=src_tree, + obj_tree=obj_tree, + root_paths=root_paths, + generate_spdx=generate_spdx, + spdx_file_names=spdx_file_names, + generate_used_files=generate_used_files, + used_files_file_name=used_files_file_name, + output_directory=output_directory, + debug=debug, + fail_on_unknown_build_command=fail_on_unknown_build_command, + write_output_on_error=write_output_on_error, + created=created, + spdxId_prefix=spdxId_prefix, + build_type=build_type, + build_id=build_id, + package_license=package_license, + package_version=package_version, + package_copyright_text=package_copyright_text, + prettify_json=prettify_json, + ) + + +def _validate_path_arguments( + parser: argparse.ArgumentParser, + src_tree: PathStr, + obj_tree: PathStr, + root_paths: list[PathStr], +) -> None: + """ + Validate that the provided paths exist. + + Args: + parser: The argument parser, used to emit well-formatted error messages. + src_tree: Absolute path to the source tree. + obj_tree: Absolute path to the object tree. + root_paths: List of root paths relative to obj_tree. + """ + if not os.path.exists(src_tree): + parser.error(f"--src-tree {src_tree} does not exist") + if not os.path.exists(obj_tree): + parser.error(f"--obj-tree {obj_tree} does not exist") + for root_path in root_paths: + if not os.path.isfile(root_path_absolute := os.path.join(obj_tree, root_path)): + parser.error(f"path to root artifact {root_path_absolute} is not a file") diff --git a/scripts/sbom/sbom/environment.py b/scripts/sbom/sbom/environment.py new file mode 100644 index 000000000000..4304066fe974 --- /dev/null +++ b/scripts/sbom/sbom/environment.py @@ -0,0 +1,192 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import os + +KERNEL_BUILD_VARIABLES_ALLOWLIST = [ + "AFLAGS_KERNEL", + "AFLAGS_MODULE", + "AR", + "ARCH", + "ARCH_CORE", + "ARCH_DRIVERS", + "ARCH_LIB", + "AWK", + "BASH", + "BINDGEN", + "BITS", + "CC", + "CC_FLAGS_FPU", + "CC_FLAGS_NO_FPU", + "CFLAGS_GCOV", + "CFLAGS_KERNEL", + "CFLAGS_MODULE", + "CHECK", + "CHECKFLAGS", + "CLIPPY_CONF_DIR", + "CONFIG_SHELL", + "CPP", + "CROSS_COMPILE", + "CURDIR", + "GNUMAKEFLAGS", + "HOSTCC", + "HOSTCXX", + "HOSTPKG_CONFIG", + "HOSTRUSTC", + "INSTALLKERNEL", + "INSTALL_DTBS_PATH", + "INSTALL_HDR_PATH", + "INSTALL_PATH", + "KBUILD_AFLAGS", + "KBUILD_AFLAGS_KERNEL", + "KBUILD_AFLAGS_MODULE", + "KBUILD_BUILTIN", + "KBUILD_CFLAGS", + "KBUILD_CFLAGS_KERNEL", + "KBUILD_CFLAGS_MODULE", + "KBUILD_CHECKSRC", + "KBUILD_CLIPPY", + "KBUILD_CPPFLAGS", + "KBUILD_EXTMOD", + "KBUILD_EXTRA_WARN", + "KBUILD_HOSTCFLAGS", + "KBUILD_HOSTCXXFLAGS", + "KBUILD_HOSTLDFLAGS", + "KBUILD_HOSTLDLIBS", + "KBUILD_HOSTRUSTFLAGS", + "KBUILD_IMAGE", + "KBUILD_LDFLAGS", + "KBUILD_LDFLAGS_MODULE", + "KBUILD_LDS", + "KBUILD_MODULES", + "KBUILD_PROCMACROLDFLAGS", + "KBUILD_RUSTFLAGS", + "KBUILD_RUSTFLAGS_KERNEL", + "KBUILD_RUSTFLAGS_MODULE", + "KBUILD_USERCFLAGS", + "KBUILD_USERLDFLAGS", + "KBUILD_VERBOSE", + "KBUILD_VMLINUX_LIBS", + "KBZIP2", + "KCONFIG_CONFIG", + "KERNELDOC", + "KERNELRELEASE", + "KERNELVERSION", + "KGZIP", + "KLZOP", + "LC_COLLATE", + "LC_NUMERIC", + "LD", + "LDFLAGS_MODULE", + "LEX", + "LINUXINCLUDE", + "LZ4", + "LZMA", + "MAKE", + "MAKEFILES", + "MAKEFILE_LIST", + "MAKEFLAGS", + "MAKELEVEL", + "MAKEOVERRIDES", + "MAKE_COMMAND", + "MAKE_HOST", + "MAKE_TERMERR", + "MAKE_TERMOUT", + "MAKE_VERSION", + "MFLAGS", + "MODLIB", + "NM", + "NOSTDINC_FLAGS", + "O", + "OBJCOPY", + "OBJCOPYFLAGS", + "OBJDUMP", + "PAHOLE", + "PATCHLEVEL", + "PERL", + "PYTHON3", + "Q", + "RCS_FIND_IGNORE", + "READELF", + "REALMODE_CFLAGS", + "RESOLVE_BTFIDS", + "RETHUNK_CFLAGS", + "RETHUNK_RUSTFLAGS", + "RETPOLINE_CFLAGS", + "RETPOLINE_RUSTFLAGS", + "RETPOLINE_VDSO_CFLAGS", + "RUSTC", + "RUSTC_BOOTSTRAP", + "RUSTC_OR_CLIPPY", + "RUSTC_OR_CLIPPY_QUIET", + "RUSTDOC", + "RUSTFLAGS_KERNEL", + "RUSTFLAGS_MODULE", + "RUSTFMT", + "SRCARCH", + "STRIP", + "SUBLEVEL", + "SUFFIXES", + "TAR", + "UTS_MACHINE", + "VERSION", + "VPATH", + "XZ", + "YACC", + "ZSTD", + "building_out_of_srctree", + "cross_compiling", + "objtree", + "quiet", + "rust_common_flags", + "srcroot", + "srctree", + "sub_make_done", + "subdir", +] + + +class Environment: + """ + Read-only accessor for kernel build environment variables. + """ + + @classmethod + def KERNEL_BUILD_VARIABLES(cls) -> dict[str, str]: + return { + name: value.strip() + for name in KERNEL_BUILD_VARIABLES_ALLOWLIST + if (value := os.getenv(name)) is not None and value.strip() + } + + @classmethod + def ARCH(cls) -> str | None: + return os.getenv("ARCH") + + @classmethod + def SRCARCH(cls) -> str | None: + return os.getenv("SRCARCH") + + @classmethod + def CC(cls) -> str | None: + return os.getenv("CC") + + @classmethod + def LD(cls) -> str | None: + return os.getenv("LD") + + @classmethod + def AR(cls) -> str | None: + return os.getenv("AR") + + @classmethod + def NM(cls) -> str | None: + return os.getenv("NM") + + @classmethod + def OBJCOPY(cls) -> str | None: + return os.getenv("OBJCOPY") + + @classmethod + def STRIP(cls) -> str | None: + return os.getenv("STRIP") diff --git a/scripts/sbom/sbom/path_utils.py b/scripts/sbom/sbom/path_utils.py new file mode 100644 index 000000000000..29820046dc88 --- /dev/null +++ b/scripts/sbom/sbom/path_utils.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import os +from functools import lru_cache + +PathStr = str +"""Filesystem path represented as a plain string for better performance than pathlib.Path.""" + + +def is_relative_to(path: PathStr, base: PathStr) -> bool: + return os.path.commonpath([path, base]) == base + +@lru_cache(maxsize=None) +def has_link(path: PathStr) -> bool: + """Returns True if path or any of its ancestor directories is a symlink. Results are cached to avoid duplicate lstat syscalls.""" + if os.path.islink(path): + return True + parent = os.path.dirname(path) + if parent == path: + return False + return has_link(parent) diff --git a/scripts/sbom/sbom/sbom_logging.py b/scripts/sbom/sbom/sbom_logging.py new file mode 100644 index 000000000000..fbc53cc77ef4 --- /dev/null +++ b/scripts/sbom/sbom/sbom_logging.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import logging +import inspect +from typing import Literal + + +MessageTemplate = str + + +class MessageLogger: + """Logger that suppresses repeated messages and stores a summary of all logged messages.""" + + _messages: dict[MessageTemplate, list[str]] + _message_counts: dict[MessageTemplate, int] + _repeated_logs_limit: int + """Maximum number of repeated messages of the same type to log before suppressing further output.""" + + def __init__(self, level: Literal["error", "warning"], repeated_logs_limit: int = 3) -> None: + self._level = level + self._messages = {} + self._message_counts = {} + self._repeated_logs_limit = repeated_logs_limit + + def log(self, template: MessageTemplate, /, **kwargs: str) -> None: + """Log a message based on a template and optional variables. Example: `log("Missing {path}", path=str(p))`.""" + message = template + for key, value in kwargs.items(): + message = message.replace("{" + key + "}", value) + if template not in self._messages: + self._messages[template] = [] + self._message_counts[template] = 0 + self._message_counts[template] += 1 + if self._message_counts[template] <= self._repeated_logs_limit: + if self._level == "error": + logging.error(message) + elif self._level == "warning": + logging.warning(message) + self._messages[template].append(message) + + def get_summary(self) -> str: + if len(self._messages) == 0: + return "" + summary: list[str] = [f"Summarize {self._level}s:"] + for template, messages in self._messages.items(): + for message in messages: + summary.append(message) + n_suppressed_messages = self._message_counts[template] - self._repeated_logs_limit + if n_suppressed_messages > 0: + instances = "instance" if n_suppressed_messages == 1 else "instances" + summary.append(f"... (Found {n_suppressed_messages} more {instances} of this {self._level})") + return "\n".join(summary) + + def has_messages(self) -> bool: + return len(self._message_counts) > 0 + + +_warning_logger: MessageLogger +_error_logger: MessageLogger + + +def warning(msg_template: MessageTemplate, /, **kwargs: str) -> None: + _warning_logger.log(msg_template, **kwargs) + + +def error(msg_template: MessageTemplate, /, **kwargs: str) -> None: + frame = inspect.currentframe() + caller_frame = frame.f_back if frame else None + info = inspect.getframeinfo(caller_frame) if caller_frame else None + if info: + msg_template = f'File "{info.filename}", line {info.lineno}, in {info.function}\n{msg_template}' + _error_logger.log(msg_template, **kwargs) + + +def summarize_warnings() -> str: + return _warning_logger.get_summary() + + +def summarize_errors() -> str: + return _error_logger.get_summary() + + +def has_errors() -> bool: + return _error_logger.has_messages() + + +def init() -> None: + global _warning_logger, _error_logger + _warning_logger = MessageLogger("warning") + _error_logger = MessageLogger("error") + + +init() diff --git a/scripts/sbom/sbom/spdx/__init__.py b/scripts/sbom/sbom/spdx/__init__.py new file mode 100644 index 000000000000..4097b59f8f17 --- /dev/null +++ b/scripts/sbom/sbom/spdx/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from .spdxId import SpdxId, SpdxIdGenerator +from .serialization import JsonLdSpdxDocument + +__all__ = ["JsonLdSpdxDocument", "SpdxId", "SpdxIdGenerator"] diff --git a/scripts/sbom/sbom/spdx/build.py b/scripts/sbom/sbom/spdx/build.py new file mode 100644 index 000000000000..a39ec9c09b16 --- /dev/null +++ b/scripts/sbom/sbom/spdx/build.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass, field +from sbom.spdx.core import DictionaryEntry, Element, Hash + + +@dataclass(kw_only=True) +class Build(Element): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Build/Classes/Build/""" + + type: str = field(init=False, default="build_Build") + build_buildType: str + build_buildId: str + build_environment: list[DictionaryEntry] = field(default_factory=list) + build_configSourceUri: list[str] = field(default_factory=list) + build_configSourceDigest: list[Hash] = field(default_factory=list) diff --git a/scripts/sbom/sbom/spdx/core.py b/scripts/sbom/sbom/spdx/core.py new file mode 100644 index 000000000000..7eb376a1cd88 --- /dev/null +++ b/scripts/sbom/sbom/spdx/core.py @@ -0,0 +1,170 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass, field + +from typing import Any, Literal +from sbom.spdx.spdxId import SpdxId + +SPDX_SPEC_VERSION = "3.0.1" + +ExternalIdentifierType = Literal["email", "gitoid", "urlScheme"] +HashAlgorithm = Literal["sha256", "sha512"] +ProfileIdentifierType = Literal["core", "software", "build", "lite", "simpleLicensing"] +RelationshipType = Literal[ + "contains", + "generates", + "hasDeclaredLicense", + "hasInput", + "hasOutput", + "ancestorOf", + "hasDistributionArtifact", + "dependsOn", +] +RelationshipCompleteness = Literal["complete", "incomplete", "noAssertion"] + + +@dataclass +class SpdxObject: + def to_dict(self) -> dict[str, Any]: + def _to_dict(v: Any): + return v.to_dict() if hasattr(v, "to_dict") else v + + d: dict[str, Any] = {} + for field_name in self.__dataclass_fields__: + value = getattr(self, field_name) + if value is None or value == [] or value == "": + continue + + if isinstance(value, Element): + d[field_name] = value.spdxId + elif isinstance(value, list) and len(value) > 0 and isinstance(value[0], Element): # type: ignore + value: list[Element] = value + d[field_name] = [v.spdxId for v in value] + else: + d[field_name] = [_to_dict(v) for v in value] if isinstance(value, list) else _to_dict(value) # type: ignore + return d + + +@dataclass(kw_only=True) +class IntegrityMethod(SpdxObject): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/IntegrityMethod/""" + + +@dataclass(kw_only=True) +class Hash(IntegrityMethod): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/Hash/""" + + type: str = field(init=False, default="Hash") + hashValue: str + algorithm: HashAlgorithm + + +@dataclass(kw_only=True) +class Element(SpdxObject): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/Element/""" + + type: str = field(init=False, default="Element") + spdxId: SpdxId + creationInfo: str = "_:creationinfo" + name: str | None = None + verifiedUsing: list[Hash] = field(default_factory=list) + comment: str | None = None + + +@dataclass(kw_only=True) +class ExternalMap(SpdxObject): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/ExternalMap/""" + + type: str = field(init=False, default="ExternalMap") + externalSpdxId: SpdxId + + +@dataclass(kw_only=True) +class NamespaceMap(SpdxObject): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/NamespaceMap/""" + + type: str = field(init=False, default="NamespaceMap") + prefix: str + namespace: str + + +@dataclass(kw_only=True) +class ElementCollection(Element): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/ElementCollection/""" + + type: str = field(init=False, default="ElementCollection") + element: list[Element] = field(default_factory=list) + rootElement: list[Element] = field(default_factory=list) + profileConformance: list[ProfileIdentifierType] = field(default_factory=list) + + +@dataclass(kw_only=True) +class SpdxDocument(ElementCollection): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/SpdxDocument/""" + + type: str = field(init=False, default="SpdxDocument") + import_: list[ExternalMap] = field(default_factory=list) + namespaceMap: list[NamespaceMap] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return {("import" if k == "import_" else k): v for k, v in super().to_dict().items()} + + +@dataclass(kw_only=True) +class Agent(Element): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/Agent/""" + + type: str = field(init=False, default="Agent") + + +@dataclass(kw_only=True) +class SoftwareAgent(Agent): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/SoftwareAgent/""" + + type: str = field(init=False, default="SoftwareAgent") + + +@dataclass(kw_only=True) +class CreationInfo(SpdxObject): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/CreationInfo/""" + + type: str = field(init=False, default="CreationInfo") + id: SpdxId = "_:creationinfo" + specVersion: str = SPDX_SPEC_VERSION + createdBy: list[Agent] + created: str + comment: str | None = None + + def to_dict(self) -> dict[str, Any]: + return {("@id" if k == "id" else k): v for k, v in super().to_dict().items()} + + +@dataclass(kw_only=True) +class Relationship(Element): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/Relationship/""" + + type: str = field(init=False, default="Relationship") + relationshipType: RelationshipType + from_: Element # underscore because 'from' is a reserved keyword + to: list[Element] + completeness: RelationshipCompleteness | None = None + + def to_dict(self) -> dict[str, Any]: + return {("from" if k == "from_" else k): v for k, v in super().to_dict().items()} + + +@dataclass(kw_only=True) +class Artifact(Element): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/Artifact/""" + + type: str = field(init=False, default="Artifact") + + +@dataclass(kw_only=True) +class DictionaryEntry(SpdxObject): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/DictionaryEntry/""" + + type: str = field(init=False, default="DictionaryEntry") + key: str + value: str diff --git a/scripts/sbom/sbom/spdx/serialization.py b/scripts/sbom/sbom/spdx/serialization.py new file mode 100644 index 000000000000..b4df7d368d46 --- /dev/null +++ b/scripts/sbom/sbom/spdx/serialization.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import json +from typing import Any +from sbom.path_utils import PathStr +from sbom.spdx.core import SPDX_SPEC_VERSION, SpdxDocument, SpdxObject + + +class JsonLdSpdxDocument: + """Represents an SPDX document in JSON-LD format for serialization.""" + + graph: list[SpdxObject] + + def __init__(self, graph: list[SpdxObject]) -> None: + """ + Initialize a JSON-LD SPDX document from a graph of SPDX objects. + The graph must contain a single SpdxDocument element. + + Args: + graph: List of SPDX objects representing the complete SPDX document. + """ + self.graph = graph + + @property + def context(self) -> list[str | dict[str, str]]: + spdx_document = next(element for element in self.graph if isinstance(element, SpdxDocument)) + return [ + f"https://spdx.org/rdf/{SPDX_SPEC_VERSION}/spdx-context.jsonld", + {ns.prefix: ns.namespace for ns in spdx_document.namespaceMap}, + ] + + def to_dict(self) -> dict[str, Any]: + """ + Convert the SPDX document to a dictionary representation suitable for JSON serialization. + + Returns: + Dictionary with @context and @graph keys following JSON-LD format. + """ + def _item_to_dict(item: SpdxObject) -> dict: + d = item.to_dict() + if isinstance(item, SpdxDocument): + d.pop("namespaceMap", None) + return d + return { + "@context": self.context, + "@graph": [_item_to_dict(item) for item in self.graph], + } + + def save(self, path: PathStr, prettify: bool) -> None: + """ + Save the SPDX document to a JSON file. + + Args: + path: File path where the document will be saved. + prettify: Whether to pretty-print the JSON with indentation. + """ + with open(path, "w", encoding="utf-8") as f: + if prettify: + json.dump(self.to_dict(), f, indent=2) + else: + json.dump(self.to_dict(), f, separators=(",", ":")) diff --git a/scripts/sbom/sbom/spdx/simplelicensing.py b/scripts/sbom/sbom/spdx/simplelicensing.py new file mode 100644 index 000000000000..750ddd24ad89 --- /dev/null +++ b/scripts/sbom/sbom/spdx/simplelicensing.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass, field +from sbom.spdx.core import Element + + +@dataclass(kw_only=True) +class AnyLicenseInfo(Element): + """https://spdx.github.io/spdx-spec/v3.0.1/model/SimpleLicensing/Classes/AnyLicenseInfo/""" + + type: str = field(init=False, default="simplelicensing_AnyLicenseInfo") + + +@dataclass(kw_only=True) +class LicenseExpression(AnyLicenseInfo): + """https://spdx.github.io/spdx-spec/v3.0.1/model/SimpleLicensing/Classes/LicenseExpression/""" + + type: str = field(init=False, default="simplelicensing_LicenseExpression") + simplelicensing_licenseExpression: str diff --git a/scripts/sbom/sbom/spdx/software.py b/scripts/sbom/sbom/spdx/software.py new file mode 100644 index 000000000000..2f46de7c3167 --- /dev/null +++ b/scripts/sbom/sbom/spdx/software.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass, field +from typing import Literal +from sbom.spdx.core import Artifact, ElementCollection, IntegrityMethod + + +SbomType = Literal["source", "build"] +FileKindType = Literal["file", "directory"] +SoftwarePurpose = Literal[ + "source", + "archive", + "library", + "file", + "data", + "configuration", + "executable", + "module", + "application", + "documentation", + "other", +] +ContentIdentifierType = Literal["gitoid", "swhid"] + + +@dataclass(kw_only=True) +class Sbom(ElementCollection): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Software/Classes/Sbom/""" + + type: str = field(init=False, default="software_Sbom") + software_sbomType: list[SbomType] = field(default_factory=list) + + +@dataclass(kw_only=True) +class ContentIdentifier(IntegrityMethod): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Software/Classes/ContentIdentifier/""" + + type: str = field(init=False, default="software_ContentIdentifier") + software_contentIdentifierType: ContentIdentifierType + software_contentIdentifierValue: str + + +@dataclass(kw_only=True) +class SoftwareArtifact(Artifact): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Software/Classes/SoftwareArtifact/""" + + type: str = field(init=False, default="software_Artifact") + software_primaryPurpose: SoftwarePurpose | None = None + software_copyrightText: str | None = None + software_contentIdentifier: list[ContentIdentifier] = field(default_factory=list) + + +@dataclass(kw_only=True) +class Package(SoftwareArtifact): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Software/Classes/Package/""" + + type: str = field(init=False, default="software_Package") + name: str # type: ignore + software_packageVersion: str | None = None + + +@dataclass(kw_only=True) +class File(SoftwareArtifact): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Software/Classes/File/""" + + type: str = field(init=False, default="software_File") + name: str # type: ignore + software_fileKind: FileKindType | None = None diff --git a/scripts/sbom/sbom/spdx/spdxId.py b/scripts/sbom/sbom/spdx/spdxId.py new file mode 100644 index 000000000000..589e85c5f706 --- /dev/null +++ b/scripts/sbom/sbom/spdx/spdxId.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from itertools import count +from typing import Iterator + +SpdxId = str + + +class SpdxIdGenerator: + _namespace: str + _prefix: str | None = None + _counter: Iterator[int] + + def __init__(self, namespace: str, prefix: str | None = None) -> None: + """ + Initialize the SPDX ID generator with a namespace. + + Args: + namespace: The full namespace to use for generated IDs. + prefix: Optional. If provided, generated IDs will use this prefix instead of the full namespace. + """ + self._namespace = namespace + self._prefix = prefix + self._counter = count(0) + + def generate(self) -> SpdxId: + return f"{f'{self._prefix}:' if self._prefix else self._namespace}{next(self._counter)}" + + @property + def prefix(self) -> str | None: + return self._prefix + + @property + def namespace(self) -> str: + return self._namespace diff --git a/scripts/sbom/sbom/spdx_graph/__init__.py b/scripts/sbom/sbom/spdx_graph/__init__.py new file mode 100644 index 000000000000..3557b1d51bf9 --- /dev/null +++ b/scripts/sbom/sbom/spdx_graph/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from .build_spdx_graphs import build_spdx_graphs +from .spdx_graph_model import SpdxIdGeneratorCollection + +__all__ = ["build_spdx_graphs", "SpdxIdGeneratorCollection"] diff --git a/scripts/sbom/sbom/spdx_graph/build_spdx_graphs.py b/scripts/sbom/sbom/spdx_graph/build_spdx_graphs.py new file mode 100644 index 000000000000..ee24e9eaf603 --- /dev/null +++ b/scripts/sbom/sbom/spdx_graph/build_spdx_graphs.py @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from datetime import datetime +from typing import Protocol + +import logging +from sbom.config import KernelSpdxDocumentKind +from sbom.cmd_graph import CmdGraph +from sbom.path_utils import PathStr +from sbom.spdx_graph.kernel_file import KernelFileCollection +from sbom.spdx_graph.spdx_graph_model import SpdxGraph, SpdxIdGeneratorCollection +from sbom.spdx_graph.shared_spdx_elements import SharedSpdxElements +from sbom.spdx_graph.spdx_source_graph import SpdxSourceGraph +from sbom.spdx_graph.spdx_build_graph import SpdxBuildGraph +from sbom.spdx_graph.spdx_output_graph import SpdxOutputGraph + + +class SpdxGraphConfig(Protocol): + obj_tree: PathStr + src_tree: PathStr + created: datetime + build_type: str + build_id: str | None + package_license: str + package_version: str | None + package_copyright_text: str | None + + +def build_spdx_graphs( + cmd_graph: CmdGraph, + spdx_id_generators: SpdxIdGeneratorCollection, + config: SpdxGraphConfig, +) -> dict[KernelSpdxDocumentKind, SpdxGraph]: + """ + Builds SPDX graphs (output, source, and build) based on a cmd dependency graph. + If the source and object trees are identical, no dedicated source graph can be created. + In that case the source files are added to the build graph instead. + + Args: + cmd_graph: The dependency graph of a kernel build. + spdx_id_generators: Collection of SPDX ID generators. + config: Configuration options. + + Returns: + Dictionary of SPDX graphs + """ + shared_elements = SharedSpdxElements.create(spdx_id_generators.base, config.created) + kernel_files = KernelFileCollection.create(cmd_graph, config.obj_tree, config.src_tree, spdx_id_generators) + output_graph = SpdxOutputGraph.create( + root_files=list(kernel_files.output.values()), + shared_elements=shared_elements, + spdx_id_generators=spdx_id_generators, + config=config, + ) + spdx_graphs: dict[KernelSpdxDocumentKind, SpdxGraph] = { + KernelSpdxDocumentKind.OUTPUT: output_graph, + } + + if len(kernel_files.source) > 0: + spdx_graphs[KernelSpdxDocumentKind.SOURCE] = SpdxSourceGraph.create( + source_files=list(kernel_files.source.values()), + external_files=list(kernel_files.external.values()), + shared_elements=shared_elements, + spdx_id_generators=spdx_id_generators, + ) + else: + logging.info( + "Skipped creating a dedicated source SBOM because source files cannot be " + "reliably classified when the source and object trees are identical. " + "Added source files to the build SBOM instead." + ) + + build_graph = SpdxBuildGraph.create( + cmd_graph, + kernel_files, + shared_elements, + output_graph.high_level_build_element, + spdx_id_generators, + ) + spdx_graphs[KernelSpdxDocumentKind.BUILD] = build_graph + + return spdx_graphs diff --git a/scripts/sbom/sbom/spdx_graph/kernel_file.py b/scripts/sbom/sbom/spdx_graph/kernel_file.py new file mode 100644 index 000000000000..505f25f66ebb --- /dev/null +++ b/scripts/sbom/sbom/spdx_graph/kernel_file.py @@ -0,0 +1,315 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass +from enum import Enum +import hashlib +import os +import re +from sbom.cmd_graph import CmdGraph +from sbom.path_utils import PathStr, is_relative_to +from sbom.spdx import SpdxId, SpdxIdGenerator +from sbom.spdx.core import Hash +from sbom.spdx.software import ContentIdentifier, File, SoftwarePurpose +import sbom.sbom_logging as sbom_logging +from sbom.spdx_graph.spdx_graph_model import SpdxIdGeneratorCollection + + +class KernelFileLocation(Enum): + """Represents the location of a file relative to the source/object trees.""" + + SOURCE_TREE = "source_tree" + """File is located in the source tree.""" + OBJ_TREE = "obj_tree" + """File is located in the object tree.""" + EXTERNAL = "external" + """File is located outside both source and object trees.""" + BOTH = "both" + """File is located in a folder that is both source and object tree.""" + + +@dataclass +class KernelFile: + """kernel-specific metadata used to generate an SPDX File element.""" + + absolute_path: PathStr + """Absolute path of the file.""" + file_location: KernelFileLocation + """Location of the file relative to the source/object trees.""" + name: str + """Name of the file element. Should be relative to the source tree if + file_location equals SOURCE_TREE and relative to the object tree if + file_location equals OBJ_TREE. If file_location equals EXTERNAL, the + absolute path is used.""" + license_identifier: str | None + """SPDX license ID if file_location equals SOURCE_TREE or BOTH; otherwise None.""" + spdx_id_generator: SpdxIdGenerator + """Generator for the SPDX ID of the file element.""" + + _spdx_file_element: File | None = None + + @classmethod + def create( + cls, + absolute_path: PathStr, + obj_tree: PathStr, + src_tree: PathStr, + spdx_id_generators: SpdxIdGeneratorCollection, + is_output: bool, + ) -> "KernelFile": + is_in_obj_tree = is_relative_to(absolute_path, obj_tree) + is_in_src_tree = is_relative_to(absolute_path, src_tree) + + # file element name should be relative to output or src tree if possible + if not is_in_src_tree and not is_in_obj_tree: + file_element_name = str(absolute_path) + file_location = KernelFileLocation.EXTERNAL + spdx_id_generator = spdx_id_generators.source if src_tree != obj_tree else spdx_id_generators.build + elif is_in_src_tree and src_tree == obj_tree: + file_element_name = os.path.relpath(absolute_path, obj_tree) + file_location = KernelFileLocation.BOTH + spdx_id_generator = spdx_id_generators.output if is_output else spdx_id_generators.build + elif is_in_obj_tree: + file_element_name = os.path.relpath(absolute_path, obj_tree) + file_location = KernelFileLocation.OBJ_TREE + spdx_id_generator = spdx_id_generators.output if is_output else spdx_id_generators.build + else: + file_element_name = os.path.relpath(absolute_path, src_tree) + file_location = KernelFileLocation.SOURCE_TREE + spdx_id_generator = spdx_id_generators.source + + # parse spdx license identifier + license_identifier = ( + _parse_spdx_license_identifier(absolute_path) + if file_location == KernelFileLocation.SOURCE_TREE or file_location == KernelFileLocation.BOTH + else None + ) + + return KernelFile( + absolute_path, + file_location, + file_element_name, + license_identifier, + spdx_id_generator, + ) + + @property + def spdx_file_element(self) -> File: + if self._spdx_file_element is None: + self._spdx_file_element = _build_file_element( + self.absolute_path, + self.name, + self.spdx_id_generator.generate(), + self.file_location, + ) + return self._spdx_file_element + + +@dataclass +class KernelFileCollection: + """Collection of kernel files.""" + + source: dict[PathStr, KernelFile] + build: dict[PathStr, KernelFile] + output: dict[PathStr, KernelFile] + external: dict[PathStr, KernelFile] + + @classmethod + def create( + cls, + cmd_graph: CmdGraph, + obj_tree: PathStr, + src_tree: PathStr, + spdx_id_generators: SpdxIdGeneratorCollection, + ) -> "KernelFileCollection": + source: dict[PathStr, KernelFile] = {} + build: dict[PathStr, KernelFile] = {} + output: dict[PathStr, KernelFile] = {} + external: dict[PathStr, KernelFile] = {} + root_node_paths = {node.absolute_path for node in cmd_graph.roots} + for node in cmd_graph: + is_root = node.absolute_path in root_node_paths + kernel_file = KernelFile.create( + node.absolute_path, + obj_tree, + src_tree, + spdx_id_generators, + is_root, + ) + if is_root: + output[kernel_file.absolute_path] = kernel_file + elif kernel_file.file_location == KernelFileLocation.SOURCE_TREE: + source[kernel_file.absolute_path] = kernel_file + elif kernel_file.file_location == KernelFileLocation.EXTERNAL: + external[kernel_file.absolute_path] = kernel_file + else: + build[kernel_file.absolute_path] = kernel_file + + return KernelFileCollection(source, build, output, external) + + def to_dict(self) -> dict[PathStr, KernelFile]: + return {**self.source, **self.build, **self.output, **self.external} + + +def _build_file_element(absolute_path: PathStr, name: str, spdx_id: SpdxId, file_location: KernelFileLocation) -> File: + verifiedUsing: list[Hash] = [] + content_identifier: list[ContentIdentifier] = [] + if os.path.isfile(absolute_path): + verifiedUsing = [Hash(algorithm="sha256", hashValue=_sha256(absolute_path))] + content_identifier = [ + ContentIdentifier( + software_contentIdentifierType="gitoid", + software_contentIdentifierValue=_git_blob_oid(absolute_path), + ) + ] + elif file_location == KernelFileLocation.EXTERNAL: + sbom_logging.warning( + "Cannot compute hash for {absolute_path} because file does not exist.", + absolute_path=absolute_path, + ) + else: + sbom_logging.error( + "Cannot compute hash for {absolute_path} because file does not exist.", + absolute_path=absolute_path, + ) + + # primary purpose + primary_purpose = _get_primary_purpose(absolute_path) + + return File( + spdxId=spdx_id, + name=name, + verifiedUsing=verifiedUsing, + software_primaryPurpose=primary_purpose, + software_contentIdentifier=content_identifier, + ) + + +def _sha256(file_path: PathStr, chunk_size: int = 1 << 20) -> str: + """Compute the SHA-256 hex digest of a file, reading it in chunks of chunk_size bytes.""" + h = hashlib.sha256() + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + h.update(chunk) + return h.hexdigest() + + +def _git_blob_oid(file_path: str, chunk_size: int = 1 << 20) -> str: + """Compute the Git blob object ID (SHA-1 hex) for a file, like `git hash-object`, reading it in chunks of chunk_size bytes.""" + h = hashlib.sha1() + h.update(f"blob {os.path.getsize(file_path)}\0".encode()) + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + h.update(chunk) + return h.hexdigest() + + +# REUSE-IgnoreStart +SPDX_LICENSE_IDENTIFIER_PATTERN = re.compile( + r"SPDX-License-Identifier:" # literal tag + r"\s*" # optional whitespace after colon + r"(?P<id>.*?)" # license expression (non-greedy, stops before terminator) + r"(?:\s*" # optional whitespace before terminator (not captured) + r"(-->|\*/|$))", # terminator: XML "-->", C-style "*/", or end of line + re.MULTILINE, # match end of each line, not just end of string +) +# REUSE-IgnoreEnd + + +def _parse_spdx_license_identifier(absolute_path: str, max_bytes: int = 512) -> str | None: + """ + Extracts the SPDX-License-Identifier from the beginning of a source file. + + Args: + absolute_path: Path to the source file. + max_bytes: Maximum number of bytes to scan for the license identifier. + + Returns: + The license identifier string (e.g., 'GPL-2.0-only') if found, otherwise None. + """ + try: + with open(absolute_path, "r", encoding="utf-8") as f: + match = SPDX_LICENSE_IDENTIFIER_PATTERN.search(f.read(max_bytes)) + if match: + return match.group("id") + except (UnicodeDecodeError, OSError): + return None + return None + + +def _get_primary_purpose(absolute_path: PathStr) -> SoftwarePurpose | None: + def ends_with(suffixes: list[str]) -> bool: + return any(absolute_path.endswith(suffix) for suffix in suffixes) + + def includes_path_segments(path_segments: list[str]) -> bool: + return any(segment in absolute_path for segment in path_segments) + + # Source code + if ends_with([".c", ".h", ".S", ".s", ".rs", ".pl", "gen_smb1_mapping", "gen_smb2_mapping"]): + return "source" + + # Libraries + if ends_with([".a", ".so", ".so.raw", ".rlib"]): + return "library" + + # Archives + if ends_with([".xz", ".cpio", ".gz", ".tar", ".zip", "piggy_data"]): + return "archive" + + # Applications + if ends_with(["bzImage", "Image", ".efi"]): + return "application" + + # Executables / machine code + if ends_with([".bin", ".elf", "vmlinux", "vmlinux.unstripped", "vmlinuz", "bpfilter_umh"]): + return "executable" + + # Kernel modules + if ends_with([".ko"]): + return "module" + + # Data files + if ends_with( + [ + ".tbl", + ".relocs", + ".rmeta", + ".in", + ".dbg", + ".x509", + ".pbm", + ".ppm", + ".dtb", + ".uc", + ".inc", + ".dts", + ".dtsi", + ".dtbo", + ".xml", + ".ro", + "initramfs_inc_data", + "default_cpio_list", + "x509_certificate_list", + "utf8data.c_shipped", + "blacklist_hash_list", + "x509_revocation_list", + "cpucaps", + "sysreg", + "mach-types", + ] + ) or includes_path_segments(["drivers/gpu/drm/radeon/reg_srcs/"]): + return "data" + + # Configuration files + if ends_with([".pem", ".key", ".conf", ".config", ".cfg", ".bconf"]): + return "configuration" + + # Documentation + if ends_with([".md"]): + return "documentation" + + # Other / miscellaneous + if ends_with([".o", ".tmp"]): + return "other" + + sbom_logging.warning("Could not infer primary purpose for {absolute_path}", absolute_path=absolute_path) diff --git a/scripts/sbom/sbom/spdx_graph/shared_spdx_elements.py b/scripts/sbom/sbom/spdx_graph/shared_spdx_elements.py new file mode 100644 index 000000000000..115e8778a467 --- /dev/null +++ b/scripts/sbom/sbom/spdx_graph/shared_spdx_elements.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass +from datetime import datetime, timezone +from sbom.spdx.core import CreationInfo, SoftwareAgent +from sbom.spdx.spdxId import SpdxIdGenerator + + +@dataclass(frozen=True) +class SharedSpdxElements: + agent: SoftwareAgent + creation_info: CreationInfo + + @classmethod + def create(cls, spdx_id_generator: SpdxIdGenerator, created: datetime) -> "SharedSpdxElements": + """ + Creates shared SPDX elements used across multiple documents. + + Args: + spdx_id_generator: Generator for creating SPDX IDs. + created: SPDX 'created' property used for the creation info. + + Returns: + SharedSpdxElements with agent and creation info. + """ + agent = SoftwareAgent( + spdxId=spdx_id_generator.generate(), + name="KernelSbom", + ) + creation_info = CreationInfo(createdBy=[agent], created=created.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")) + return SharedSpdxElements(agent=agent, creation_info=creation_info) diff --git a/scripts/sbom/sbom/spdx_graph/spdx_build_graph.py b/scripts/sbom/sbom/spdx_graph/spdx_build_graph.py new file mode 100644 index 000000000000..4d738bc3b3e2 --- /dev/null +++ b/scripts/sbom/sbom/spdx_graph/spdx_build_graph.py @@ -0,0 +1,318 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass +from typing import Mapping +from sbom.cmd_graph import CmdGraph +from sbom.path_utils import PathStr +from sbom.spdx import SpdxIdGenerator +from sbom.spdx.build import Build +from sbom.spdx.core import ExternalMap, NamespaceMap, Relationship, SpdxDocument +from sbom.spdx.software import File, Sbom +from sbom.spdx_graph.kernel_file import KernelFileCollection +from sbom.spdx_graph.shared_spdx_elements import SharedSpdxElements +from sbom.spdx_graph.spdx_graph_model import SpdxGraph, SpdxIdGeneratorCollection +from sbom.spdx_graph.spdx_source_graph import source_file_license_elements + + +@dataclass +class SpdxBuildGraph(SpdxGraph): + """SPDX graph representing build dependencies connecting source files and + distributable output files""" + + @classmethod + def create( + cls, + cmd_graph: CmdGraph, + kernel_files: KernelFileCollection, + shared_elements: SharedSpdxElements, + high_level_build_element: Build, + spdx_id_generators: SpdxIdGeneratorCollection, + ) -> "SpdxBuildGraph": + if len(kernel_files.source) > 0: + return _create_spdx_build_graph( + cmd_graph, + kernel_files, + shared_elements, + high_level_build_element, + spdx_id_generators, + ) + else: + return _create_spdx_build_graph_with_mixed_sources( + cmd_graph, + kernel_files, + shared_elements, + high_level_build_element, + spdx_id_generators, + ) + + +def _create_spdx_build_graph( + cmd_graph: CmdGraph, + kernel_files: KernelFileCollection, + shared_elements: SharedSpdxElements, + high_level_build_element: Build, + spdx_id_generators: SpdxIdGeneratorCollection, +) -> SpdxBuildGraph: + """ + Creates an SPDX build graph where source and output files are referenced + from external documents. + + Args: + cmd_graph: The dependency graph of a kernel build. + kernel_files: Collection of categorized kernel files involved in the build. + shared_elements: SPDX elements shared across multiple documents. + high_level_build_element: The high-level Build element referenced by the build graph. + spdx_id_generators: Collection of generators for SPDX element IDs. + + Returns: + SpdxBuildGraph: The SPDX build graph connecting source files and distributable output files. + """ + # SpdxDocument + build_spdx_document = SpdxDocument( + spdxId=spdx_id_generators.build.generate(), + profileConformance=["core", "software", "build"], + namespaceMap=[ + NamespaceMap(prefix=generator.prefix, namespace=generator.namespace) + for generator in [ + spdx_id_generators.build, + spdx_id_generators.source, + spdx_id_generators.output, + spdx_id_generators.base, + ] + if generator.prefix is not None + ], + ) + + # Sbom + build_sbom = Sbom( + spdxId=spdx_id_generators.build.generate(), + software_sbomType=["build"], + ) + + # Src and object tree elements + obj_tree_element = File( + spdxId=spdx_id_generators.build.generate(), + name="$(obj_tree)", + software_fileKind="directory", + ) + obj_tree_contains_relationship = Relationship( + spdxId=spdx_id_generators.build.generate(), + relationshipType="contains", + from_=obj_tree_element, + to=[], + ) + + # File elements + build_file_elements = [file.spdx_file_element for file in kernel_files.build.values()] + file_relationships = _file_relationships( + cmd_graph=cmd_graph, + file_elements={key: file.spdx_file_element for key, file in kernel_files.to_dict().items()}, + high_level_build_element=high_level_build_element, + spdx_id_generator=spdx_id_generators.build, + ) + + # Update relationships + build_spdx_document.rootElement = [build_sbom] + + build_spdx_document.import_ = [ + *( + ExternalMap(externalSpdxId=file.spdx_file_element.spdxId) + for file in (*kernel_files.source.values(), *kernel_files.external.values()) + ), + ExternalMap(externalSpdxId=high_level_build_element.spdxId), + *(ExternalMap(externalSpdxId=file.spdx_file_element.spdxId) for file in kernel_files.output.values()), + ] + + build_sbom.rootElement = [obj_tree_element] + build_sbom.element = [ + obj_tree_element, + obj_tree_contains_relationship, + *build_file_elements, + *file_relationships, + ] + + obj_tree_contains_relationship.to = [ + *build_file_elements, + *(file.spdx_file_element for file in kernel_files.output.values()), + ] + + # create Spdx graphs + build_graph = SpdxBuildGraph( + build_spdx_document, + shared_elements.agent, + shared_elements.creation_info, + build_sbom, + ) + return build_graph + + +def _create_spdx_build_graph_with_mixed_sources( + cmd_graph: CmdGraph, + kernel_files: KernelFileCollection, + shared_elements: SharedSpdxElements, + high_level_build_element: Build, + spdx_id_generators: SpdxIdGeneratorCollection, +) -> SpdxBuildGraph: + """ + Creates an SPDX build graph where only output files are referenced from + an external document. Source files are included directly in the build graph. + + Args: + cmd_graph: The dependency graph of a kernel build. + kernel_files: Collection of categorized kernel files involved in the build. + shared_elements: SPDX elements shared across multiple documents. + high_level_build_element: The high-level Build element referenced by the build graph. + spdx_id_generators: Collection of generators for SPDX element IDs. + + Returns: + SpdxBuildGraph: The SPDX build graph connecting source files and distributable output files. + """ + # SpdxDocument + build_spdx_document = SpdxDocument( + spdxId=spdx_id_generators.build.generate(), + profileConformance=["core", "software", "build"], + namespaceMap=[ + NamespaceMap(prefix=generator.prefix, namespace=generator.namespace) + for generator in [ + spdx_id_generators.build, + spdx_id_generators.output, + spdx_id_generators.base, + ] + if generator.prefix is not None + ], + ) + + # Sbom + build_sbom = Sbom( + spdxId=spdx_id_generators.build.generate(), + software_sbomType=["build"], + ) + + # File elements + build_file_elements = [file.spdx_file_element for file in kernel_files.build.values()] + external_file_elements = [file.spdx_file_element for file in kernel_files.external.values()] + file_relationships = _file_relationships( + cmd_graph=cmd_graph, + file_elements={key: file.spdx_file_element for key, file in kernel_files.to_dict().items()}, + high_level_build_element=high_level_build_element, + spdx_id_generator=spdx_id_generators.build, + ) + + # Source file license elements + source_file_license_identifiers, source_file_license_relationships = source_file_license_elements( + list(kernel_files.build.values()), spdx_id_generators.build + ) + + # Update relationships + build_spdx_document.rootElement = [build_sbom] + root_file_elements = [file.spdx_file_element for file in kernel_files.output.values()] + build_spdx_document.import_ = [ + ExternalMap(externalSpdxId=high_level_build_element.spdxId), + *(ExternalMap(externalSpdxId=file.spdxId) for file in root_file_elements), + ] + + build_sbom.rootElement = [*root_file_elements] + build_sbom.element = [ + *build_file_elements, + *external_file_elements, + *source_file_license_identifiers, + *source_file_license_relationships, + *file_relationships, + ] + + build_graph = SpdxBuildGraph( + build_spdx_document, + shared_elements.agent, + shared_elements.creation_info, + build_sbom, + ) + return build_graph + + +def _file_relationships( + cmd_graph: CmdGraph, + file_elements: Mapping[PathStr, File], + high_level_build_element: Build, + spdx_id_generator: SpdxIdGenerator, +) -> list[Build | Relationship]: + """ + Construct SPDX Build and Relationship elements representing dependency + relationships in the cmd graph. + + Args: + cmd_graph: The dependency graph of a kernel build. + file_elements: Mapping of filesystem paths (PathStr) to their + corresponding SPDX File elements. + high_level_build_element: The SPDX Build element representing the overall build process/root. + spdx_id_generator: Generator for unique SPDX IDs. + + Returns: + list[Build | Relationship]: List of SPDX Build and Relationship elements + """ + high_level_build_ancestorOf_relationship = Relationship( + spdxId=spdx_id_generator.generate(), + relationshipType="ancestorOf", + from_=high_level_build_element, + completeness="complete", + to=[], + ) + + # Create a relationship between each node (output file) + # and its children (input files) + build_and_relationship_elements: list[Build | Relationship] = [high_level_build_ancestorOf_relationship] + for node in cmd_graph: + # .cmd file dependencies + if node.cmd_file is not None: + build_element = Build( + spdxId=spdx_id_generator.generate(), + build_buildType=high_level_build_element.build_buildType, + build_buildId=high_level_build_element.build_buildId, + comment=node.cmd_file.savedcmd, + ) + build_and_relationship_elements.append(build_element) + + if node.cmd_file_dependencies: + hasInput_relationship = Relationship( + spdxId=spdx_id_generator.generate(), + relationshipType="hasInput", + from_=build_element, + to=[file_elements[dep.absolute_path] for dep in node.cmd_file_dependencies], + ) + build_and_relationship_elements.append(hasInput_relationship) + + hasOutput_relationship = Relationship( + spdxId=spdx_id_generator.generate(), + relationshipType="hasOutput", + from_=build_element, + to=[file_elements[node.absolute_path]], + ) + build_and_relationship_elements.append(hasOutput_relationship) + + high_level_build_ancestorOf_relationship.to.append(build_element) + + # incbin dependencies + if len(node.incbin_dependencies) > 0: + incbin_dependsOn_relationship = Relationship( + spdxId=spdx_id_generator.generate(), + relationshipType="dependsOn", + comment="\n".join([incbin_dependency.full_statement for incbin_dependency in node.incbin_dependencies]), + from_=file_elements[node.absolute_path], + to=[ + file_elements[incbin_dependency.node.absolute_path] + for incbin_dependency in node.incbin_dependencies + ], + ) + build_and_relationship_elements.append(incbin_dependsOn_relationship) + + # hardcoded dependencies + if len(node.hardcoded_dependencies) > 0: + hardcoded_dependency_relationship = Relationship( + spdxId=spdx_id_generator.generate(), + relationshipType="dependsOn", + from_=file_elements[node.absolute_path], + to=[file_elements[n.absolute_path] for n in node.hardcoded_dependencies], + ) + build_and_relationship_elements.append(hardcoded_dependency_relationship) + + return build_and_relationship_elements diff --git a/scripts/sbom/sbom/spdx_graph/spdx_graph_model.py b/scripts/sbom/sbom/spdx_graph/spdx_graph_model.py new file mode 100644 index 000000000000..682194d4362a --- /dev/null +++ b/scripts/sbom/sbom/spdx_graph/spdx_graph_model.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass +from sbom.spdx.core import CreationInfo, SoftwareAgent, SpdxDocument, SpdxObject +from sbom.spdx.software import Sbom +from sbom.spdx.spdxId import SpdxIdGenerator + + +@dataclass +class SpdxGraph: + """Represents the complete graph of a single SPDX document.""" + + spdx_document: SpdxDocument + agent: SoftwareAgent + creation_info: CreationInfo + sbom: Sbom + + def to_list(self) -> list[SpdxObject]: + return [ + self.spdx_document, + self.agent, + self.creation_info, + self.sbom, + *self.sbom.element, + ] + + +@dataclass +class SpdxIdGeneratorCollection: + """Holds SPDX ID generators for different document types to ensure globally unique SPDX IDs.""" + + base: SpdxIdGenerator + source: SpdxIdGenerator + build: SpdxIdGenerator + output: SpdxIdGenerator diff --git a/scripts/sbom/sbom/spdx_graph/spdx_output_graph.py b/scripts/sbom/sbom/spdx_graph/spdx_output_graph.py new file mode 100644 index 000000000000..ff9b2c31fb04 --- /dev/null +++ b/scripts/sbom/sbom/spdx_graph/spdx_output_graph.py @@ -0,0 +1,187 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass +import os +from typing import Protocol +from sbom.environment import Environment +from sbom.path_utils import PathStr +from sbom.spdx.build import Build +from sbom.spdx.core import DictionaryEntry, NamespaceMap, Relationship, SpdxDocument +from sbom.spdx.simplelicensing import LicenseExpression +from sbom.spdx.software import File, Package, Sbom +from sbom.spdx.spdxId import SpdxIdGenerator +from sbom.spdx_graph.kernel_file import KernelFile +from sbom.spdx_graph.shared_spdx_elements import SharedSpdxElements +from sbom.spdx_graph.spdx_graph_model import SpdxGraph, SpdxIdGeneratorCollection + + +class SpdxOutputGraphConfig(Protocol): + obj_tree: PathStr + src_tree: PathStr + build_type: str + build_id: str | None + package_license: str + package_version: str | None + package_copyright_text: str | None + + +@dataclass +class SpdxOutputGraph(SpdxGraph): + """SPDX graph representing distributable output files""" + + high_level_build_element: Build + + @classmethod + def create( + cls, + root_files: list[KernelFile], + shared_elements: SharedSpdxElements, + spdx_id_generators: SpdxIdGeneratorCollection, + config: SpdxOutputGraphConfig, + ) -> "SpdxOutputGraph": + """ + Args: + root_files: List of distributable output files which act as roots + of the dependency graph. + shared_elements: Shared SPDX elements used across multiple documents. + spdx_id_generators: Collection of SPDX ID generators. + config: Configuration options. + + Returns: + SpdxOutputGraph: The SPDX output graph. + """ + # SpdxDocument + spdx_document = SpdxDocument( + spdxId=spdx_id_generators.output.generate(), + profileConformance=["core", "software", "build", "simpleLicensing"], + namespaceMap=[ + NamespaceMap(prefix=generator.prefix, namespace=generator.namespace) + for generator in [spdx_id_generators.output, spdx_id_generators.base] + if generator.prefix is not None + ], + ) + + # Sbom + sbom = Sbom( + spdxId=spdx_id_generators.output.generate(), + software_sbomType=["build"], + ) + + # High-level Build elements + config_source_element = KernelFile.create( + absolute_path=os.path.join(config.obj_tree, ".config"), + obj_tree=config.obj_tree, + src_tree=config.src_tree, + spdx_id_generators=spdx_id_generators, + is_output=True, + ).spdx_file_element + high_level_build_element, high_level_build_element_hasOutput_relationship = _high_level_build_elements( + config.build_type, + config.build_id, + config_source_element, + spdx_id_generators.output, + ) + + # Root file elements + root_file_elements: list[File] = [file.spdx_file_element for file in root_files] + + # Package elements + package_elements = [ + Package( + spdxId=spdx_id_generators.output.generate(), + name=_get_package_name(file.name), + software_packageVersion=config.package_version, + software_copyrightText=config.package_copyright_text, + comment=f"Architecture={arch}" if (arch := Environment.ARCH() or Environment.SRCARCH()) else None, + software_primaryPurpose=file.software_primaryPurpose, + ) + for file in root_file_elements + ] + package_hasDistributionArtifact_file_relationships = [ + Relationship( + spdxId=spdx_id_generators.output.generate(), + relationshipType="hasDistributionArtifact", + from_=package, + to=[file], + ) + for package, file in zip(package_elements, root_file_elements) + ] + package_license_expression = LicenseExpression( + spdxId=spdx_id_generators.output.generate(), + simplelicensing_licenseExpression=config.package_license, + ) + package_hasDeclaredLicense_relationships = [ + Relationship( + spdxId=spdx_id_generators.output.generate(), + relationshipType="hasDeclaredLicense", + from_=package, + to=[package_license_expression], + ) + for package in package_elements + ] + + # Update relationships + spdx_document.rootElement = [sbom] + + sbom.rootElement = [*package_elements] + sbom.element = [ + config_source_element, + high_level_build_element, + high_level_build_element_hasOutput_relationship, + *root_file_elements, + *package_elements, + *package_hasDistributionArtifact_file_relationships, + package_license_expression, + *package_hasDeclaredLicense_relationships, + ] + + high_level_build_element_hasOutput_relationship.to = [*root_file_elements] + + output_graph = SpdxOutputGraph( + spdx_document, + shared_elements.agent, + shared_elements.creation_info, + sbom, + high_level_build_element, + ) + return output_graph + + +def _get_package_name(filename: str) -> str: + """ + Generates a SPDX package name from a filename. + Kernel images (bzImage, Image) get a descriptive name, others use the basename of the file. + """ + KERNEL_FILENAMES = ["bzImage", "Image"] + basename = os.path.basename(filename) + return f"Linux Kernel ({basename})" if basename in KERNEL_FILENAMES else basename + + +def _high_level_build_elements( + build_type: str, + build_id: str | None, + config_source_element: File, + spdx_id_generator: SpdxIdGenerator, +) -> tuple[Build, Relationship]: + build_spdxId = spdx_id_generator.generate() + high_level_build_element = Build( + spdxId=build_spdxId, + build_buildType=build_type, + build_buildId=build_id if build_id is not None else build_spdxId, + build_environment=[ + DictionaryEntry(key=key, value=value) + for key, value in Environment.KERNEL_BUILD_VARIABLES().items() + if value + ], + build_configSourceUri=[config_source_element.spdxId], + build_configSourceDigest=config_source_element.verifiedUsing, + ) + + high_level_build_element_hasOutput_relationship = Relationship( + spdxId=spdx_id_generator.generate(), + relationshipType="hasOutput", + from_=high_level_build_element, + to=[], + ) + return high_level_build_element, high_level_build_element_hasOutput_relationship diff --git a/scripts/sbom/sbom/spdx_graph/spdx_source_graph.py b/scripts/sbom/sbom/spdx_graph/spdx_source_graph.py new file mode 100644 index 000000000000..90880212dedd --- /dev/null +++ b/scripts/sbom/sbom/spdx_graph/spdx_source_graph.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass +from sbom.spdx import SpdxIdGenerator +from sbom.spdx.core import Element, NamespaceMap, Relationship, SpdxDocument +from sbom.spdx.simplelicensing import LicenseExpression +from sbom.spdx.software import File, Sbom +from sbom.spdx_graph.kernel_file import KernelFile +from sbom.spdx_graph.shared_spdx_elements import SharedSpdxElements +from sbom.spdx_graph.spdx_graph_model import SpdxGraph, SpdxIdGeneratorCollection + + +@dataclass +class SpdxSourceGraph(SpdxGraph): + """SPDX graph representing source files""" + + @classmethod + def create( + cls, + source_files: list[KernelFile], + external_files: list[KernelFile], + shared_elements: SharedSpdxElements, + spdx_id_generators: SpdxIdGeneratorCollection, + ) -> "SpdxSourceGraph": + """ + Args: + source_files: List of files within the kernel source tree. + external_files: Files outside both source and object trees. + shared_elements: Shared SPDX elements used across multiple documents. + spdx_id_generators: Collection of SPDX ID generators. + + Returns: + SpdxSourceGraph: The SPDX source graph. + """ + # SpdxDocument + source_spdx_document = SpdxDocument( + spdxId=spdx_id_generators.source.generate(), + profileConformance=["core", "software", "simpleLicensing"], + namespaceMap=[ + NamespaceMap(prefix=generator.prefix, namespace=generator.namespace) + for generator in [spdx_id_generators.source, spdx_id_generators.base] + if generator.prefix is not None + ], + ) + + # Sbom + source_sbom = Sbom( + spdxId=spdx_id_generators.source.generate(), + software_sbomType=["source"], + ) + + # Src Tree Elements + src_tree_element = File( + spdxId=spdx_id_generators.source.generate(), + name="$(src_tree)", + software_fileKind="directory", + ) + src_tree_contains_relationship = Relationship( + spdxId=spdx_id_generators.source.generate(), + relationshipType="contains", + from_=src_tree_element, + to=[], + ) + + # Source file elements + source_file_elements: list[Element] = [file.spdx_file_element for file in source_files] + external_file_elements: list[Element] = [file.spdx_file_element for file in external_files] + + # Source file license elements + source_file_license_identifiers, source_file_license_relationships = source_file_license_elements( + source_files, spdx_id_generators.source + ) + + # Update relationships + source_spdx_document.rootElement = [source_sbom] + source_sbom.rootElement = [src_tree_element] + source_sbom.element = [ + src_tree_element, + src_tree_contains_relationship, + *source_file_elements, + *external_file_elements, + *source_file_license_identifiers, + *source_file_license_relationships, + ] + src_tree_contains_relationship.to = source_file_elements + + source_graph = SpdxSourceGraph( + source_spdx_document, + shared_elements.agent, + shared_elements.creation_info, + source_sbom, + ) + return source_graph + + +def source_file_license_elements( + source_files: list[KernelFile], spdx_id_generator: SpdxIdGenerator +) -> tuple[list[LicenseExpression], list[Relationship]]: + """ + Creates SPDX license expressions and links them to the given source files + via hasDeclaredLicense relationships. + + Args: + source_files: List of files within the kernel source tree. + spdx_id_generator: Generator for unique SPDX IDs. + + Returns: + Tuple of (license expressions, hasDeclaredLicense relationships). + """ + license_expressions: dict[str, LicenseExpression] = {} + for file in source_files: + if file.license_identifier is None or file.license_identifier in license_expressions: + continue + license_expressions[file.license_identifier] = LicenseExpression( + spdxId=spdx_id_generator.generate(), + simplelicensing_licenseExpression=file.license_identifier, + ) + + source_file_license_relationships = [ + Relationship( + spdxId=spdx_id_generator.generate(), + relationshipType="hasDeclaredLicense", + from_=file.spdx_file_element, + to=[license_expressions[file.license_identifier]], + ) + for file in source_files + if file.license_identifier is not None + ] + return ([*license_expressions.values()], source_file_license_relationships) diff --git a/scripts/sbom/tests/__init__.py b/scripts/sbom/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/scripts/sbom/tests/__init__.py diff --git a/scripts/sbom/tests/cmd_graph/__init__.py b/scripts/sbom/tests/cmd_graph/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/scripts/sbom/tests/cmd_graph/__init__.py diff --git a/scripts/sbom/tests/cmd_graph/test_savedcmd_parser.py b/scripts/sbom/tests/cmd_graph/test_savedcmd_parser.py new file mode 100644 index 000000000000..d7776f072e03 --- /dev/null +++ b/scripts/sbom/tests/cmd_graph/test_savedcmd_parser.py @@ -0,0 +1,468 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import os +import unittest +from unittest.mock import patch + +from sbom.cmd_graph.savedcmd_parser import parse_inputs_from_commands +from sbom.cmd_graph.savedcmd_parser.command_parser_registry import CommandParserRegistry +import sbom.sbom_logging as sbom_logging + + +class TestSavedCmdParser(unittest.TestCase): + def _assert_parsing(self, cmd: str, expected: str, registry: CommandParserRegistry | None = None) -> None: + sbom_logging.init() + parsed = parse_inputs_from_commands(cmd, fail_on_unknown_build_command=False, registry=registry) + target = [] if expected == "" else expected.split(" ") + self.assertEqual(parsed, target) + errors = sbom_logging._error_logger._message_counts # type: ignore + self.assertEqual(errors, {}) + + # Error handling tests + def test_malformed_shell_quoting(self): + command = 'gcc "unterminated' + with patch.object(sbom_logging, "warning") as warning: + parsed = parse_inputs_from_commands(command, fail_on_unknown_build_command=False) + + self.assertEqual(parsed, []) + warning.assert_called_once_with( + "Skipped parsing command {single_command} because of command parsing error: {error_message}", + single_command=command, + error_message="No closing quotation", + ) + + def test_missing_positional_argument(self): + command = "objcopy" + with patch.object(sbom_logging, "warning") as warning: + parsed = parse_inputs_from_commands(command, fail_on_unknown_build_command=False) + + self.assertEqual(parsed, []) + warning.assert_called_once_with( + "Skipped parsing command {single_command} because of command parsing error: {error_message}", + single_command=command, + error_message="list index out of range", + ) + + # Compound command tests + def test_dd_cat(self): + cmd = "(dd if=arch/x86/boot/setup.bin bs=4k conv=sync status=none; cat arch/x86/boot/vmlinux.bin) >arch/x86/boot/bzImage" + expected = "arch/x86/boot/setup.bin arch/x86/boot/vmlinux.bin" + self._assert_parsing(cmd, expected) + + def test_manual_file_creation(self): + cmd = """{ symbase=__dtbo_overlay_bad_unresolved; echo '$(pound)include <asm-generic/vmlinux.lds.h>'; echo '.section .rodata,"a"'; echo '.balign STRUCT_ALIGNMENT'; echo ".global $${symbase}_begin"; echo "$${symbase}_begin:"; echo '.incbin "drivers/of/unittest-data/overlay_bad_unresolved.dtbo" '; echo ".global $${symbase}_end"; echo "$${symbase}_end:"; echo '.balign STRUCT_ALIGNMENT'; } > drivers/of/unittest-data/overlay_bad_unresolved.dtbo.S""" + expected = "" + self._assert_parsing(cmd, expected) + + def test_cat_xz_wrap(self): + cmd = "{ cat arch/x86/boot/compressed/vmlinux.bin | sh ../scripts/xz_wrap.sh; printf \\130\\064\\024\\000; } > arch/x86/boot/compressed/vmlinux.bin.xz" + expected = "arch/x86/boot/compressed/vmlinux.bin" + self._assert_parsing(cmd, expected) + + def test_printf_sed(self): + cmd = r"""{ printf 'static char tomoyo_builtin_profile[] __initdata =\n'; sed -e 's/\\/\\\\/g' -e 's/\"/\\"/g' -e 's/\(.*\)/\t"\1\\n"/' -- /dev/null; printf '\t"";\n'; printf 'static char tomoyo_builtin_exception_policy[] __initdata =\n'; sed -e 's/\\/\\\\/g' -e 's/\"/\\"/g' -e 's/\(.*\)/\t"\1\\n"/' -- ../security/tomoyo/policy/exception_policy.conf.default; printf '\t"";\n'; printf 'static char tomoyo_builtin_domain_policy[] __initdata =\n'; sed -e 's/\\/\\\\/g' -e 's/\"/\\"/g' -e 's/\(.*\)/\t"\1\\n"/' -- /dev/null; printf '\t"";\n'; printf 'static char tomoyo_builtin_manager[] __initdata =\n'; sed -e 's/\\/\\\\/g' -e 's/\"/\\"/g' -e 's/\(.*\)/\t"\1\\n"/' -- /dev/null; printf '\t"";\n'; printf 'static char tomoyo_builtin_stat[] __initdata =\n'; sed -e 's/\\/\\\\/g' -e 's/\"/\\"/g' -e 's/\(.*\)/\t"\1\\n"/' -- /dev/null; printf '\t"";\n'; } > security/tomoyo/builtin-policy.h""" + expected = "../security/tomoyo/policy/exception_policy.conf.default" + self._assert_parsing(cmd, expected) + + def test_bin2c_echo(self): + cmd = """(echo "static char tomoyo_builtin_profile[] __initdata ="; ./scripts/bin2c </dev/null; echo ";"; echo "static char tomoyo_builtin_exception_policy[] __initdata ="; ./scripts/bin2c <../security/tomoyo/policy/exception_policy.conf.default; echo ";"; echo "static char tomoyo_builtin_domain_policy[] __initdata ="; ./scripts/bin2c </dev/null; echo ";"; echo "static char tomoyo_builtin_manager[] __initdata ="; ./scripts/bin2c </dev/null; echo ";"; echo "static char tomoyo_builtin_stat[] __initdata ="; ./scripts/bin2c </dev/null; echo ";") >security/tomoyo/builtin-policy.h""" + expected = "../security/tomoyo/policy/exception_policy.conf.default" + self._assert_parsing(cmd, expected) + + def test_cat_colon(self): + cmd = "{ cat init/modules.order; cat usr/modules.order; cat arch/x86/modules.order; cat arch/x86/boot/startup/modules.order; cat kernel/modules.order; cat certs/modules.order; cat mm/modules.order; cat fs/modules.order; cat ipc/modules.order; cat security/modules.order; cat crypto/modules.order; cat block/modules.order; cat io_uring/modules.order; cat lib/modules.order; cat arch/x86/lib/modules.order; cat drivers/modules.order; cat sound/modules.order; cat samples/modules.order; cat net/modules.order; cat virt/modules.order; cat arch/x86/pci/modules.order; cat arch/x86/power/modules.order; cat arch/x86/video/modules.order; :; } > modules.order" + expected = "init/modules.order usr/modules.order arch/x86/modules.order arch/x86/boot/startup/modules.order kernel/modules.order certs/modules.order mm/modules.order fs/modules.order ipc/modules.order security/modules.order crypto/modules.order block/modules.order io_uring/modules.order lib/modules.order arch/x86/lib/modules.order drivers/modules.order sound/modules.order samples/modules.order net/modules.order virt/modules.order arch/x86/pci/modules.order arch/x86/power/modules.order arch/x86/video/modules.order" + self._assert_parsing(cmd, expected) + + def test_cat_zstd(self): + cmd = "{ cat arch/x86/boot/compressed/vmlinux.bin arch/x86/boot/compressed/vmlinux.relocs | zstd -22 --ultra; printf \\340\\362\\066\\003; } > arch/x86/boot/compressed/vmlinux.bin.zst" + expected = "arch/x86/boot/compressed/vmlinux.bin arch/x86/boot/compressed/vmlinux.relocs" + self._assert_parsing(cmd, expected) + + # cat command tests + def test_cat_redirect(self): + cmd = "cat ../fs/unicode/utf8data.c_shipped > fs/unicode/utf8data.c" + expected = "../fs/unicode/utf8data.c_shipped" + self._assert_parsing(cmd, expected) + + def test_cat_piped(self): + cmd = "cat arch/x86/boot/compressed/vmlinux.bin arch/x86/boot/compressed/vmlinux.relocs | gzip -n -f -9 > arch/x86/boot/compressed/vmlinux.bin.gz" + expected = "arch/x86/boot/compressed/vmlinux.bin arch/x86/boot/compressed/vmlinux.relocs" + self._assert_parsing(cmd, expected) + + # sed command tests + def test_sed(self): + cmd = "sed -n 's/.*define *BLIST_\\([A-Z0-9_]*\\) *.*/BLIST_FLAG_NAME(\\1),/p' ../include/scsi/scsi_devinfo.h > drivers/scsi/scsi_devinfo_tbl.c" + expected = "../include/scsi/scsi_devinfo.h" + self._assert_parsing(cmd, expected) + + # awk command tests + def test_awk(self): + cmd = "awk -f ../arch/arm64/tools/gen-cpucaps.awk ../arch/arm64/tools/cpucaps > arch/arm64/include/generated/asm/cpucap-defs.h" + expected = "../arch/arm64/tools/cpucaps" + self._assert_parsing(cmd, expected) + + def test_awk_with_input_redirection(self): + cmd = "awk -v N=1 -f ../lib/raid6/unroll.awk < ../lib/raid6/int.uc > lib/raid6/int1.c" + expected = "../lib/raid6/int.uc" + self._assert_parsing(cmd, expected) + + # openssl command tests + def test_openssl(self): + cmd = "openssl req -new -nodes -utf8 -sha256 -days 36500 -batch -x509 -config certs/x509.genkey -outform PEM -out certs/signing_key.pem -keyout certs/signing_key.pem 2>&1" + expected = "" + self._assert_parsing(cmd, expected) + + # gcc/clang command tests + def test_gcc(self): + cmd = ( + "gcc -Wp,-MMD,arch/x86/pci/.i386.o.d -nostdinc -I../arch/x86/include -I./arch/x86/include/generated -I../include -I./include -I../arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I../include/uapi -I./include/generated/uapi -include ../include/linux/compiler-version.h -include ../include/linux/kconfig.h -include ../include/linux/compiler_types.h -D__KERNEL__ -fmacro-prefix-map=../= -Werror -std=gnu11 -fshort-wchar -funsigned-char -fno-common -fno-PIE -fno-strict-aliasing -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx -fcf-protection=branch -fno-jump-tables -m64 -falign-jumps=1 -falign-loops=1 -mno-80387 -mno-fp-ret-in-387 -mpreferred-stack-boundary=3 -mskip-rax-setup -march=x86-64 -mtune=generic -mno-red-zone -mcmodel=kernel -mstack-protector-guard-reg=gs -mstack-protector-guard-symbol=__ref_stack_chk_guard -Wno-sign-compare -fno-asynchronous-unwind-tables -mindirect-branch=thunk-extern -mindirect-branch-register -mindirect-branch-cs-prefix -mfunction-return=thunk-extern -fno-jump-tables -fpatchable-function-entry=16,16 -fno-delete-null-pointer-checks -O2 -fno-allow-store-data-races -fstack-protector-strong -fomit-frame-pointer -fno-stack-clash-protection -falign-functions=16 -fno-strict-overflow -fno-stack-check -fconserve-stack -fno-builtin-wcslen -Wall -Wextra -Wundef -Werror=implicit-function-declaration -Werror=implicit-int -Werror=return-type -Werror=strict-prototypes -Wno-format-security -Wno-trigraphs -Wno-frame-address -Wno-address-of-packed-member -Wmissing-declarations -Wmissing-prototypes -Wframe-larger-than=2048 -Wno-main -Wvla-larger-than=1 -Wno-pointer-sign -Wcast-function-type -Wno-array-bounds -Wno-stringop-overflow -Wno-alloc-size-larger-than -Wimplicit-fallthrough=5 -Werror=date-time -Werror=incompatible-pointer-types -Werror=designated-init -Wenum-conversion -Wunused -Wno-unused-but-set-variable -Wno-unused-const-variable -Wno-packed-not-aligned -Wno-format-overflow -Wno-format-truncation -Wno-stringop-truncation -Wno-override-init -Wno-missing-field-initializers -Wno-type-limits -Wno-shift-negative-value -Wno-maybe-uninitialized -Wno-sign-compare -Wno-unused-parameter -I../arch/x86/pci -Iarch/x86/pci -DKBUILD_MODFILE=" + "arch/x86/pci/i386" + " -DKBUILD_BASENAME=" + "i386" + " -DKBUILD_MODNAME=" + "i386" + " -D__KBUILD_MODNAME=kmod_i386 -c -o arch/x86/pci/i386.o ../arch/x86/pci/i386.c " + ) + expected = "../arch/x86/pci/i386.c" + self._assert_parsing(cmd, expected) + + def test_gcc_linking(self): + cmd = "gcc -o arch/x86/tools/relocs arch/x86/tools/relocs_32.o arch/x86/tools/relocs_64.o arch/x86/tools/relocs_common.o" + expected = "arch/x86/tools/relocs_32.o arch/x86/tools/relocs_64.o arch/x86/tools/relocs_common.o" + self._assert_parsing(cmd, expected) + + def test_gcc_without_compile_flag(self): + cmd = "gcc -Wp,-MMD,arch/x86/boot/compressed/.mkpiggy.d -Wall -Wmissing-prototypes -Wstrict-prototypes -O2 -fomit-frame-pointer -std=gnu11 -I ../scripts/include -I../tools/include -I arch/x86/boot/compressed -o arch/x86/boot/compressed/mkpiggy ../arch/x86/boot/compressed/mkpiggy.c" + expected = "../arch/x86/boot/compressed/mkpiggy.c" + self._assert_parsing(cmd, expected) + + def test_gcc_with_env_override(self): + with patch.dict(os.environ, {"CC": "ccache gcc"}): + registry = CommandParserRegistry.create() + cmd = "gcc -o arch/x86/tools/relocs arch/x86/tools/relocs_32.o arch/x86/tools/relocs_64.o arch/x86/tools/relocs_common.o" + expected = "arch/x86/tools/relocs_32.o arch/x86/tools/relocs_64.o arch/x86/tools/relocs_common.o" + self._assert_parsing(cmd, expected, registry) + self._assert_parsing(f"ccache {cmd}", expected, registry) + + def test_gcc_dts_preprocessing(self): + cmd = "gcc -E -Wp,-MMD,drivers/of/.empty_root.dtb.d.pre.tmp -nostdinc -I ../scripts/dtc/include-prefixes -undef -D__DTS__ -x assembler-with-cpp -o drivers/of/.empty_root.dtb.dts.tmp ../drivers/of/empty_root.dts" + expected = "../drivers/of/empty_root.dts" + self._assert_parsing(cmd, expected) + + def test_clang(self): + cmd = """clang -Wp,-MMD,arch/x86/entry/.entry_64_compat.o.d -nostdinc -I../arch/x86/include -I./arch/x86/include/generated -I../include -I./include -I../arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I../include/uapi -I./include/generated/uapi -include ../include/linux/compiler-version.h -include ../include/linux/kconfig.h -D__KERNEL__ --target=x86_64-linux-gnu -fintegrated-as -Werror=unknown-warning-option -Werror=ignored-optimization-argument -Werror=option-ignored -Werror=unused-command-line-argument -fmacro-prefix-map=../= -Werror -D__ASSEMBLY__ -fno-PIE -m64 -I../arch/x86/entry -Iarch/x86/entry -DKBUILD_MODFILE='"arch/x86/entry/entry_64_compat"' -DKBUILD_MODNAME='"entry_64_compat"' -D__KBUILD_MODNAME=kmod_entry_64_compat -c -o arch/x86/entry/entry_64_compat.o ../arch/x86/entry/entry_64_compat.S""" + expected = "../arch/x86/entry/entry_64_compat.S" + self._assert_parsing(cmd, expected) + + # ld command tests + def test_ld(self): + cmd = r'ld -o arch/x86/entry/vdso/vdso64.so.dbg -shared --hash-style=both --build-id=sha1 --no-undefined --eh-frame-hdr -Bsymbolic -z noexecstack -m elf_x86_64 -soname linux-vdso.so.1 -z max-page-size=4096 -T arch/x86/entry/vdso/vdso.lds arch/x86/entry/vdso/vdso-note.o arch/x86/entry/vdso/vclock_gettime.o arch/x86/entry/vdso/vgetcpu.o arch/x86/entry/vdso/vgetrandom.o arch/x86/entry/vdso/vgetrandom-chacha.o; if readelf -rW arch/x86/entry/vdso/vdso64.so.dbg | grep -v _NONE | grep -q " R_\w*_"; then (echo >&2 "arch/x86/entry/vdso/vdso64.so.dbg: dynamic relocations are not supported"; rm -f arch/x86/entry/vdso/vdso64.so.dbg; /bin/false); fi' + expected = "arch/x86/entry/vdso/vdso-note.o arch/x86/entry/vdso/vclock_gettime.o arch/x86/entry/vdso/vgetcpu.o arch/x86/entry/vdso/vgetrandom.o arch/x86/entry/vdso/vgetrandom-chacha.o" + self._assert_parsing(cmd, expected) + + def test_ld_with_env_override(self): + with patch.dict(os.environ, {"LD": "some-tool ld"}): + registry = CommandParserRegistry.create() + cmd = r'ld -o arch/x86/entry/vdso/vdso64.so.dbg -shared --hash-style=both --build-id=sha1 --no-undefined --eh-frame-hdr -Bsymbolic -z noexecstack -m elf_x86_64 -soname linux-vdso.so.1 -z max-page-size=4096 -T arch/x86/entry/vdso/vdso.lds arch/x86/entry/vdso/vdso-note.o arch/x86/entry/vdso/vclock_gettime.o arch/x86/entry/vdso/vgetcpu.o arch/x86/entry/vdso/vgetrandom.o arch/x86/entry/vdso/vgetrandom-chacha.o; if readelf -rW arch/x86/entry/vdso/vdso64.so.dbg | grep -v _NONE | grep -q " R_\w*_"; then (echo >&2 "arch/x86/entry/vdso/vdso64.so.dbg: dynamic relocations are not supported"; rm -f arch/x86/entry/vdso/vdso64.so.dbg; /bin/false); fi' + expected = "arch/x86/entry/vdso/vdso-note.o arch/x86/entry/vdso/vclock_gettime.o arch/x86/entry/vdso/vgetcpu.o arch/x86/entry/vdso/vgetrandom.o arch/x86/entry/vdso/vgetrandom-chacha.o" + self._assert_parsing(cmd, expected, registry) + self._assert_parsing(f"some-tool {cmd}", expected, registry) + + def test_ld_whole_archive(self): + cmd = "ld -m elf_x86_64 -z noexecstack -r -o vmlinux.o --whole-archive vmlinux.a --no-whole-archive --start-group --end-group" + expected = "vmlinux.a" + self._assert_parsing(cmd, expected) + + def test_ld_with_at_symbol(self): + cmd = "ld.lld -m elf_x86_64 -z noexecstack -r -o fs/efivarfs/efivarfs.o @fs/efivarfs/efivarfs.mod ; ./tools/objtool/objtool --hacks=jump_label --hacks=noinstr --hacks=skylake --ibt --orc --retpoline --rethunk --static-call --uaccess --prefix=16 --link --module fs/efivarfs/efivarfs.o" + expected = "@fs/efivarfs/efivarfs.mod" + self._assert_parsing(cmd, expected) + + def test_ld_if_objdump(self): + cmd = """ld -o arch/x86/entry/vdso/vdso64.so.dbg -shared --hash-style=both --build-id=sha1 --eh-frame-hdr -Bsymbolic -z noexecstack -m elf_x86_64 -soname linux-vdso.so.1 --no-undefined -z max-page-size=4096 -T arch/x86/entry/vdso/vdso.lds arch/x86/entry/vdso/vdso-note.o arch/x86/entry/vdso/vclock_gettime.o arch/x86/entry/vdso/vgetcpu.o arch/x86/entry/vdso/vsgx.o && sh ./arch/x86/entry/vdso/checkundef.sh 'nm' 'arch/x86/entry/vdso/vdso64.so.dbg'; if objdump -R arch/x86/entry/vdso/vdso64.so.dbg | grep -E -h "R_X86_64_JUMP_SLOT|R_X86_64_GLOB_DAT|R_X86_64_RELATIVE| R_386_GLOB_DAT|R_386_JMP_SLOT|R_386_RELATIVE"; then (echo >&2 "arch/x86/entry/vdso/vdso64.so.dbg: dynamic relocations are not supported"; rm -f arch/x86/entry/vdso/vdso64.so.dbg; /bin/false); fi""" + expected = "arch/x86/entry/vdso/vdso-note.o arch/x86/entry/vdso/vclock_gettime.o arch/x86/entry/vdso/vgetcpu.o arch/x86/entry/vdso/vsgx.o" + self._assert_parsing(cmd, expected) + + # printf | xargs ar command tests + def test_ar_printf(self): + cmd = 'rm -f built-in.a; printf "./%s " init/built-in.a usr/built-in.a arch/x86/built-in.a arch/x86/boot/startup/built-in.a kernel/built-in.a certs/built-in.a mm/built-in.a fs/built-in.a ipc/built-in.a security/built-in.a crypto/built-in.a block/built-in.a io_uring/built-in.a lib/built-in.a arch/x86/lib/built-in.a drivers/built-in.a sound/built-in.a net/built-in.a virt/built-in.a arch/x86/pci/built-in.a arch/x86/power/built-in.a arch/x86/video/built-in.a | xargs ar cDPrST built-in.a' + expected = "./init/built-in.a ./usr/built-in.a ./arch/x86/built-in.a ./arch/x86/boot/startup/built-in.a ./kernel/built-in.a ./certs/built-in.a ./mm/built-in.a ./fs/built-in.a ./ipc/built-in.a ./security/built-in.a ./crypto/built-in.a ./block/built-in.a ./io_uring/built-in.a ./lib/built-in.a ./arch/x86/lib/built-in.a ./drivers/built-in.a ./sound/built-in.a ./net/built-in.a ./virt/built-in.a ./arch/x86/pci/built-in.a ./arch/x86/power/built-in.a ./arch/x86/video/built-in.a" + self._assert_parsing(cmd, expected) + + def test_ar_printf_nested(self): + cmd = 'rm -f arch/x86/pci/built-in.a; printf "arch/x86/pci/%s " i386.o init.o mmconfig_64.o direct.o mmconfig-shared.o fixup.o acpi.o legacy.o irq.o common.o early.o bus_numa.o amd_bus.o | xargs ar cDPrST arch/x86/pci/built-in.a' + expected = "arch/x86/pci/i386.o arch/x86/pci/init.o arch/x86/pci/mmconfig_64.o arch/x86/pci/direct.o arch/x86/pci/mmconfig-shared.o arch/x86/pci/fixup.o arch/x86/pci/acpi.o arch/x86/pci/legacy.o arch/x86/pci/irq.o arch/x86/pci/common.o arch/x86/pci/early.o arch/x86/pci/bus_numa.o arch/x86/pci/amd_bus.o" + self._assert_parsing(cmd, expected) + + # ar command tests + def test_ar_reordering(self): + cmd = "rm -f vmlinux.a; ar cDPrST vmlinux.a built-in.a lib/lib.a arch/x86/lib/lib.a; ar mPiT $$(ar t vmlinux.a | sed -n 1p) vmlinux.a $$(ar t vmlinux.a | grep -F -f ../scripts/head-object-list.txt)" + expected = "built-in.a lib/lib.a arch/x86/lib/lib.a" + self._assert_parsing(cmd, expected) + + def test_ar_default(self): + cmd = "rm -f lib/lib.a; ar cDPrsT lib/lib.a lib/argv_split.o lib/bug.o lib/buildid.o lib/clz_tab.o lib/cmdline.o lib/cpumask.o lib/ctype.o lib/dec_and_lock.o lib/decompress.o lib/decompress_bunzip2.o lib/decompress_inflate.o lib/decompress_unlz4.o lib/decompress_unlzma.o lib/decompress_unlzo.o lib/decompress_unxz.o lib/decompress_unzstd.o lib/dump_stack.o lib/earlycpio.o lib/extable.o lib/flex_proportions.o lib/idr.o lib/iomem_copy.o lib/irq_regs.o lib/is_single_threaded.o lib/klist.o lib/kobject.o lib/kobject_uevent.o lib/logic_pio.o lib/maple_tree.o lib/memcat_p.o lib/nmi_backtrace.o lib/objpool.o lib/plist.o lib/radix-tree.o lib/ratelimit.o lib/rbtree.o lib/seq_buf.o lib/siphash.o lib/string.o lib/sys_info.o lib/timerqueue.o lib/union_find.o lib/vsprintf.o lib/win_minmax.o lib/xarray.o" + expected = "lib/argv_split.o lib/bug.o lib/buildid.o lib/clz_tab.o lib/cmdline.o lib/cpumask.o lib/ctype.o lib/dec_and_lock.o lib/decompress.o lib/decompress_bunzip2.o lib/decompress_inflate.o lib/decompress_unlz4.o lib/decompress_unlzma.o lib/decompress_unlzo.o lib/decompress_unxz.o lib/decompress_unzstd.o lib/dump_stack.o lib/earlycpio.o lib/extable.o lib/flex_proportions.o lib/idr.o lib/iomem_copy.o lib/irq_regs.o lib/is_single_threaded.o lib/klist.o lib/kobject.o lib/kobject_uevent.o lib/logic_pio.o lib/maple_tree.o lib/memcat_p.o lib/nmi_backtrace.o lib/objpool.o lib/plist.o lib/radix-tree.o lib/ratelimit.o lib/rbtree.o lib/seq_buf.o lib/siphash.o lib/string.o lib/sys_info.o lib/timerqueue.o lib/union_find.o lib/vsprintf.o lib/win_minmax.o lib/xarray.o" + self._assert_parsing(cmd, expected) + + def test_ar_llvm(self): + cmd = "llvm-ar mPiT $$(llvm-ar t vmlinux.a | sed -n 1p) vmlinux.a $$(llvm-ar t vmlinux.a | grep -F -f ../scripts/head-object-list.txt)" + expected = "" + self._assert_parsing(cmd, expected) + + # nm command tests + def test_nm(self): + cmd = """llvm-nm -p --defined-only rust/core.o | awk '$$2~/(T|R|D|B)/ && $$3!~/__(pfx|cfi|odr_asan)/ { printf "EXPORT_SYMBOL_RUST_GPL(%s);\n",$$3 }' > rust/exports_core_generated.h""" + expected = "rust/core.o" + self._assert_parsing(cmd, expected) + + def test_nm_vmlinux(self): + cmd = r"nm vmlinux | sed -n -e 's/^\([0-9a-fA-F]*\) [ABbCDGRSTtVW] \(_text\|__start_rodata\|__bss_start\|_end\)$/#define VO_\2 _AC(0x\1,UL)/p' > arch/x86/boot/voffset.h" + expected = "vmlinux" + self._assert_parsing(cmd, expected) + + # objcopy command tests + def test_objcopy(self): + cmd = "objcopy --remove-section='.rel*' --remove-section=!'.rel*.dyn' vmlinux.unstripped vmlinux" + expected = "vmlinux.unstripped" + self._assert_parsing(cmd, expected) + + def test_objcopy_llvm(self): + cmd = "llvm-objcopy --remove-section='.rel*' --remove-section=!'.rel*.dyn' vmlinux.unstripped vmlinux" + expected = "vmlinux.unstripped" + self._assert_parsing(cmd, expected) + + # strip command tests + def test_strip(self): + cmd = "strip --strip-debug -o drivers/firmware/efi/libstub/mem.stub.o drivers/firmware/efi/libstub/mem.o" + expected = "drivers/firmware/efi/libstub/mem.o" + self._assert_parsing(cmd, expected) + + # cp command tests + def test_cp_truncate(self): + cmd = "cp arch/arm64/boot/Image arch/arm64/boot/vmlinux.bin; truncate -s $$(hexdump -s16 -n4 -e '\"%u\"' arch/arm64/boot/Image) arch/arm64/boot/vmlinux.bin" + expected = "arch/arm64/boot/Image" + self._assert_parsing(cmd, expected) + + # rustc command tests + def test_rustc(self): + cmd = """OBJTREE=/workspace/linux/kernel_build rustc -Zbinary_dep_depinfo=y -Astable_features -Dnon_ascii_idents -Dunsafe_op_in_unsafe_fn -Wmissing_docs -Wrust_2018_idioms -Wclippy::all -Wclippy::as_ptr_cast_mut -Wclippy::as_underscore -Wclippy::cast_lossless -Wclippy::ignored_unit_patterns -Wclippy::mut_mut -Wclippy::needless_bitwise_bool -Aclippy::needless_lifetimes -Wclippy::no_mangle_with_rust_abi -Wclippy::ptr_as_ptr -Wclippy::ptr_cast_constness -Wclippy::ref_as_ptr -Wclippy::undocumented_unsafe_blocks -Wclippy::unnecessary_safety_comment -Wclippy::unnecessary_safety_doc -Wrustdoc::missing_crate_level_docs -Wrustdoc::unescaped_backticks -Cpanic=abort -Cembed-bitcode=n -Clto=n -Cforce-unwind-tables=n -Ccodegen-units=1 -Csymbol-mangling-version=v0 -Crelocation-model=static -Zfunction-sections=n -Wclippy::float_arithmetic --target=./scripts/target.json -Ctarget-feature=-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-avx,-avx2 -Zcf-protection=branch -Zno-jump-tables -Ctarget-cpu=x86-64 -Ztune-cpu=generic -Cno-redzone=y -Ccode-model=kernel -Zfunction-return=thunk-extern -Zpatchable-function-entry=16,16 -Copt-level=2 -Cdebug-assertions=n -Coverflow-checks=y -Dwarnings @./include/generated/rustc_cfg --edition=2021 --cfg no_fp_fmt_parse --emit=dep-info=rust/.core.o.d --emit=obj=rust/core.o --emit=metadata=rust/libcore.rmeta --crate-type rlib -L./rust --crate-name core /usr/lib/rust-1.84/lib/rustlib/src/rust/library/core/src/lib.rs --sysroot=/dev/null ;llvm-objcopy --redefine-sym __addsf3=__rust__addsf3 --redefine-sym __eqsf2=__rust__eqsf2 --redefine-sym __extendsfdf2=__rust__extendsfdf2 --redefine-sym __gesf2=__rust__gesf2 --redefine-sym __lesf2=__rust__lesf2 --redefine-sym __ltsf2=__rust__ltsf2 --redefine-sym __mulsf3=__rust__mulsf3 --redefine-sym __nesf2=__rust__nesf2 --redefine-sym __truncdfsf2=__rust__truncdfsf2 --redefine-sym __unordsf2=__rust__unordsf2 --redefine-sym __adddf3=__rust__adddf3 --redefine-sym __eqdf2=__rust__eqdf2 --redefine-sym __ledf2=__rust__ledf2 --redefine-sym __ltdf2=__rust__ltdf2 --redefine-sym __muldf3=__rust__muldf3 --redefine-sym __unorddf2=__rust__unorddf2 --redefine-sym __muloti4=__rust__muloti4 --redefine-sym __multi3=__rust__multi3 --redefine-sym __udivmodti4=__rust__udivmodti4 --redefine-sym __udivti3=__rust__udivti3 --redefine-sym __umodti3=__rust__umodti3 rust/core.o""" + expected = "/usr/lib/rust-1.84/lib/rustlib/src/rust/library/core/src/lib.rs rust/core.o" + self._assert_parsing(cmd, expected) + + # rustdoc command tests + def test_rustdoc(self): + cmd = """OBJTREE=/workspace/linux/kernel_build rustdoc --test --edition=2021 -Zbinary_dep_depinfo=y -Astable_features -Dnon_ascii_idents -Dunsafe_op_in_unsafe_fn -Wmissing_docs -Wrust_2018_idioms -Wunreachable_pub -Wclippy::all -Wclippy::as_ptr_cast_mut -Wclippy::as_underscore -Wclippy::cast_lossless -Wclippy::ignored_unit_patterns -Wclippy::mut_mut -Wclippy::needless_bitwise_bool -Aclippy::needless_lifetimes -Wclippy::no_mangle_with_rust_abi -Wclippy::ptr_as_ptr -Wclippy::ptr_cast_constness -Wclippy::ref_as_ptr -Wclippy::undocumented_unsafe_blocks -Wclippy::unnecessary_safety_comment -Wclippy::unnecessary_safety_doc -Wrustdoc::missing_crate_level_docs -Wrustdoc::unescaped_backticks -Cpanic=abort -Cembed-bitcode=n -Clto=n -Cforce-unwind-tables=n -Ccodegen-units=1 -Csymbol-mangling-version=v0 -Crelocation-model=static -Zfunction-sections=n -Wclippy::float_arithmetic --target=aarch64-unknown-none -Ctarget-feature="-neon" -Cforce-unwind-tables=n -Zbranch-protection=pac-ret -Copt-level=2 -Cdebug-assertions=y -Coverflow-checks=y -Dwarnings -Cforce-frame-pointers=y -Zsanitizer=kernel-address -Zsanitizer-recover=kernel-address -Cllvm-args=-asan-mapping-offset=0xdfff800000000000 -Cpasses=sancov-module -Cllvm-args=-sanitizer-coverage-level=3 -Cllvm-args=-sanitizer-coverage-trace-pc -Cllvm-args=-sanitizer-coverage-trace-compares @./include/generated/rustc_cfg -L./rust --extern ffi --extern pin_init --extern kernel --extern build_error --extern macros --extern bindings --extern uapi --no-run --crate-name kernel -Zunstable-options --sysroot=/dev/null --test-builder ./scripts/rustdoc_test_builder ../rust/kernel/lib.rs >/dev/null""" + expected = "../rust/kernel/lib.rs" + self._assert_parsing(cmd, expected) + + def test_rustdoc_test_gen(self): + cmd = "./scripts/rustdoc_test_gen" + expected = "" + self._assert_parsing(cmd, expected) + + # flex command tests + def test_flex(self): + cmd = "flex -oscripts/kconfig/lexer.lex.c -L ../scripts/kconfig/lexer.l" + expected = "../scripts/kconfig/lexer.l" + self._assert_parsing(cmd, expected) + + # bison command tests + def test_bison(self): + cmd = "bison -o scripts/kconfig/parser.tab.c --defines=scripts/kconfig/parser.tab.h -t -l ../scripts/kconfig/parser.y" + expected = "../scripts/kconfig/parser.y" + self._assert_parsing(cmd, expected) + + # bindgen command tests + def test_bindgen(self): + cmd = ( + "bindgen ../rust/bindings/bindings_helper.h " + "--blocklist-type __kernel_s?size_t --blocklist-type __kernel_ptrdiff_t " + "--opaque-type xregs_state --opaque-type desc_struct --no-doc-comments " + "--rust-target 1.68 --use-core --with-derive-default -o rust/bindings/bindings_generated.rs " + "-- -Wp,-MMD,rust/bindings/.bindings_generated.rs.d -nostdinc -I../arch/x86/include " + "-include ../include/linux/compiler-version.h -D__KERNEL__ -fintegrated-as -fno-builtin -DMODULE; " + "sed -Ei 's/pub const RUST_CONST_HELPER_([a-zA-Z0-9_]*)/pub const \\1/g' rust/bindings/bindings_generated.rs" + ) + expected = "../rust/bindings/bindings_helper.h ../include/linux/compiler-version.h" + self._assert_parsing(cmd, expected) + + # perl command tests + def test_perl(self): + cmd = "perl ../lib/crypto/x86/poly1305-x86_64-cryptogams.pl > lib/crypto/x86/poly1305-x86_64-cryptogams.S" + expected = "../lib/crypto/x86/poly1305-x86_64-cryptogams.pl" + self._assert_parsing(cmd, expected) + + # link-vmlinux.sh command tests + def test_link_vmlinux(self): + cmd = '../scripts/link-vmlinux.sh "ld" "-m elf_x86_64 -z noexecstack" "-z max-page-size=0x200000 --build-id=sha1 --orphan-handling=error --emit-relocs --discard-none" "vmlinux.unstripped"; true' + expected = "vmlinux.a" + self._assert_parsing(cmd, expected) + + def test_link_vmlinux_postlink(self): + cmd = '../scripts/link-vmlinux.sh "ld" "-m elf_x86_64 -z noexecstack --no-warn-rwx-segments" "--emit-relocs --discard-none -z max-page-size=0x200000 --build-id=sha1 -X --orphan-handling=error"; make -f ../arch/x86/Makefile.postlink vmlinux' + expected = "vmlinux.a" + self._assert_parsing(cmd, expected) + + # syscallhdr.sh command tests + def test_syscallhdr(self): + cmd = "sh ../scripts/syscallhdr.sh --abis common,64 --emit-nr ../arch/x86/entry/syscalls/syscall_64.tbl arch/x86/include/generated/uapi/asm/unistd_64.h" + expected = "../arch/x86/entry/syscalls/syscall_64.tbl" + self._assert_parsing(cmd, expected) + + # syscalltbl.sh command tests + def test_syscalltbl(self): + cmd = "sh ../scripts/syscalltbl.sh --abis common,64 ../arch/x86/entry/syscalls/syscall_64.tbl arch/x86/include/generated/asm/syscalls_64.h" + expected = "../arch/x86/entry/syscalls/syscall_64.tbl" + self._assert_parsing(cmd, expected) + + # mkcapflags.sh command tests + def test_mkcapflags(self): + cmd = "sh ../arch/x86/kernel/cpu/mkcapflags.sh arch/x86/kernel/cpu/capflags.c ../arch/x86/kernel/cpu/../../include/asm/cpufeatures.h ../arch/x86/kernel/cpu/../../include/asm/vmxfeatures.h ../arch/x86/kernel/cpu/mkcapflags.sh FORCE" + expected = "../arch/x86/kernel/cpu/../../include/asm/cpufeatures.h ../arch/x86/kernel/cpu/../../include/asm/vmxfeatures.h" + self._assert_parsing(cmd, expected) + + # orc_hash.sh command tests + def test_orc_hash(self): + cmd = "mkdir -p arch/x86/include/generated/asm/; sh ../scripts/orc_hash.sh < ../arch/x86/include/asm/orc_types.h > arch/x86/include/generated/asm/orc_hash.h" + expected = "../arch/x86/include/asm/orc_types.h" + self._assert_parsing(cmd, expected) + + # xen-hypercalls.sh command tests + def test_xen_hypercalls(self): + cmd = "sh '../scripts/xen-hypercalls.sh' arch/x86/include/generated/asm/xen-hypercalls.h ../include/xen/interface/xen-mca.h ../include/xen/interface/xen.h ../include/xen/interface/xenpmu.h" + expected = "../include/xen/interface/xen-mca.h ../include/xen/interface/xen.h ../include/xen/interface/xenpmu.h" + self._assert_parsing(cmd, expected) + + # gen_initramfs.sh command tests + def test_gen_initramfs(self): + cmd = "sh ../usr/gen_initramfs.sh -o usr/initramfs_data.cpio -l usr/.initramfs_data.cpio.d ../usr/default_cpio_list" + expected = "../usr/default_cpio_list" + self._assert_parsing(cmd, expected) + + # mkuboot.sh command tests + def test_mkuboot(self): + cmd = "bash ../scripts/mkuboot.sh -A arm -O linux -C none -T kernel -a 0x8000 -e 0x8000 -n 'Linux-6.15.0' -d arch/arm/boot/zImage arch/arm/boot/uImage" + expected = "arch/arm/boot/zImage" + self._assert_parsing(cmd, expected) + + # syscallnr.sh command tests + def test_syscallnr(self): + cmd = "sh ../arch/arm/tools/syscallnr.sh ../arch/arm/tools/syscall.tbl arch/arm/include/generated/asm/unistd-nr.h" + expected = "../arch/arm/tools/syscall.tbl" + self._assert_parsing(cmd, expected) + + # gen-kernel-hwcaps.sh command tests + def test_gen_kernel_hwcaps(self): + cmd = "/bin/sh -e ../arch/arm64/tools/gen-kernel-hwcaps.sh ../arch/arm64/include/uapi/asm/hwcap.h > arch/arm64/include/generated/asm/kernel-hwcap.h" + expected = "../arch/arm64/include/uapi/asm/hwcap.h" + self._assert_parsing(cmd, expected) + + # vdso2c command tests + def test_vdso2c(self): + cmd = "arch/x86/entry/vdso/vdso2c arch/x86/entry/vdso/vdso64.so.dbg arch/x86/entry/vdso/vdso64.so arch/x86/entry/vdso/vdso-image-64.c" + expected = "arch/x86/entry/vdso/vdso64.so.dbg arch/x86/entry/vdso/vdso64.so" + self._assert_parsing(cmd, expected) + + # vdsomunge command tests + def test_vdsomunge(self): + cmd = "arch/arm64/kernel/vdso32/../../../arm/vdso/vdsomunge arch/arm64/kernel/vdso32/vdso.so.raw arch/arm64/kernel/vdso32/vdso32.so.dbg" + expected = "arch/arm64/kernel/vdso32/vdso.so.raw" + self._assert_parsing(cmd, expected) + + # mkpiggy command tests + def test_mkpiggy(self): + cmd = "arch/x86/boot/compressed/mkpiggy arch/x86/boot/compressed/vmlinux.bin.gz > arch/x86/boot/compressed/piggy.S" + expected = "arch/x86/boot/compressed/vmlinux.bin.gz" + self._assert_parsing(cmd, expected) + + # relocs command tests + def test_relocs(self): + cmd = "arch/x86/tools/relocs vmlinux.unstripped > arch/x86/boot/compressed/vmlinux.relocs;arch/x86/tools/relocs --abs-relocs vmlinux.unstripped" + expected = "vmlinux.unstripped" + self._assert_parsing(cmd, expected) + + def test_relocs_with_realmode(self): + cmd = ( + "arch/x86/tools/relocs --realmode arch/x86/realmode/rm/realmode.elf > arch/x86/realmode/rm/realmode.relocs" + ) + expected = "arch/x86/realmode/rm/realmode.elf" + self._assert_parsing(cmd, expected) + + # mk_elfconfig command tests + def test_mk_elfconfig(self): + cmd = "scripts/mod/mk_elfconfig < scripts/mod/empty.o > scripts/mod/elfconfig.h" + expected = "scripts/mod/empty.o" + self._assert_parsing(cmd, expected) + + # tools/build command tests + def test_build(self): + cmd = "arch/x86/boot/tools/build arch/x86/boot/setup.bin arch/x86/boot/vmlinux.bin arch/x86/boot/zoffset.h arch/x86/boot/bzImage" + expected = "arch/x86/boot/setup.bin arch/x86/boot/vmlinux.bin arch/x86/boot/zoffset.h" + self._assert_parsing(cmd, expected) + + # extract-cert command tests + def test_extract_cert(self): + cmd = 'certs/extract-cert "" certs/signing_key.x509' + expected = "" + self._assert_parsing(cmd, expected) + + # dtc command tests + def test_dtc_cat(self): + cmd = "./scripts/dtc/dtc -o drivers/of/empty_root.dtb -b 0 -i../drivers/of/ -i../scripts/dtc/include-prefixes -Wno-unique_unit_address -Wno-unit_address_vs_reg -Wno-avoid_unnecessary_addr_size -Wno-alias_paths -Wno-graph_child_address -Wno-simple_bus_reg -d drivers/of/.empty_root.dtb.d.dtc.tmp drivers/of/.empty_root.dtb.dts.tmp ; cat drivers/of/.empty_root.dtb.d.pre.tmp drivers/of/.empty_root.dtb.d.dtc.tmp > drivers/of/.empty_root.dtb.d" + expected = "drivers/of/.empty_root.dtb.dts.tmp drivers/of/.empty_root.dtb.d.pre.tmp drivers/of/.empty_root.dtb.d.dtc.tmp" + self._assert_parsing(cmd, expected) + + # pnmtologo command tests + def test_pnmtologo(self): + cmd = "drivers/video/logo/pnmtologo -t clut224 -n logo_linux_clut224 -o drivers/video/logo/logo_linux_clut224.c ../drivers/video/logo/logo_linux_clut224.ppm" + expected = "../drivers/video/logo/logo_linux_clut224.ppm" + self._assert_parsing(cmd, expected) + + # relacheck command tests + def test_relacheck(self): + cmd = "arch/arm64/kernel/pi/relacheck arch/arm64/kernel/pi/idreg-override.pi.o arch/arm64/kernel/pi/idreg-override.o" + expected = "arch/arm64/kernel/pi/idreg-override.pi.o" + self._assert_parsing(cmd, expected) + + # gen-hyprel command tests + def test_gen_hyprel(self): + cmd = "arch/arm64/kvm/hyp/nvhe/gen-hyprel arch/arm64/kvm/hyp/nvhe/kvm_nvhe.tmp.o > arch/arm64/kvm/hyp/nvhe/hyp-reloc.S" + expected = "arch/arm64/kvm/hyp/nvhe/kvm_nvhe.tmp.o" + self._assert_parsing(cmd, expected) + + # mkregtable command tests + def test_mkregtable(self): + cmd = "drivers/gpu/drm/radeon/mkregtable ../drivers/gpu/drm/radeon/reg_srcs/r100 > drivers/gpu/drm/radeon/r100_reg_safe.h" + expected = "../drivers/gpu/drm/radeon/reg_srcs/r100" + self._assert_parsing(cmd, expected) + + # genheaders command tests + def test_genheaders(self): + cmd = "security/selinux/genheaders security/selinux/flask.h security/selinux/av_permissions.h" + expected = "" + self._assert_parsing(cmd, expected) + + # mkcpustr command tests + def test_mkcpustr(self): + cmd = "arch/x86/boot/mkcpustr > arch/x86/boot/cpustr.h" + expected = "" + self._assert_parsing(cmd, expected) + + # polgen command tests + def test_polgen(self): + cmd = "scripts/ipe/polgen/polgen security/ipe/boot_policy.c" + expected = "" + self._assert_parsing(cmd, expected) + + # gen_header.py command tests + def test_gen_header(self): + cmd = "mkdir -p drivers/gpu/drm/msm/generated && python3 ../drivers/gpu/drm/msm/registers/gen_header.py --no-validate --rnn ../drivers/gpu/drm/msm/registers --xml ../drivers/gpu/drm/msm/registers/adreno/a2xx.xml c-defines > drivers/gpu/drm/msm/generated/a2xx.xml.h" + expected = "../drivers/gpu/drm/msm/registers/adreno/a2xx.xml" + self._assert_parsing(cmd, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/sbom/tests/spdx_graph/__init__.py b/scripts/sbom/tests/spdx_graph/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/scripts/sbom/tests/spdx_graph/__init__.py diff --git a/scripts/sbom/tests/spdx_graph/test_kernel_file.py b/scripts/sbom/tests/spdx_graph/test_kernel_file.py new file mode 100644 index 000000000000..35a63a768ba2 --- /dev/null +++ b/scripts/sbom/tests/spdx_graph/test_kernel_file.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import unittest +from pathlib import Path +import tempfile +from sbom.spdx_graph.kernel_file import _parse_spdx_license_identifier # type: ignore + + +class TestKernelFile(unittest.TestCase): + def setUp(self): + self.tmpdir = tempfile.TemporaryDirectory() + self.src_tree = Path(self.tmpdir.name) + + def tearDown(self): + self.tmpdir.cleanup() + + def test_parse_spdx_license_identifier(self): + # REUSE-IgnoreStart + test_cases: list[tuple[str, str | None]] = [ + ("/* SPDX-License-Identifier: MIT*/", "MIT"), + ("// SPDX-License-Identifier: GPL-2.0-only", "GPL-2.0-only"), + ("# SPDX-License-Identifier: GPL-2.0-only", "GPL-2.0-only"), + ("#!/bin/bash\n# SPDX-License-Identifier: GPL-2.0-only", "GPL-2.0-only"), + ("/* SPDX-License-Identifier: GPL-2.0-or-later OR MIT */", "GPL-2.0-or-later OR MIT"), + ("/* SPDX-License-Identifier: Apache-2.0 */\n extra text", "Apache-2.0"), + ("<!-- SPDX-License-Identifier: GPL-2.0 -->", "GPL-2.0"), + ("int main() { return 0; }", None), + ] + # REUSE-IgnoreEnd + + for i, (file_content, expected_identifier) in enumerate(test_cases): + file_path = self.src_tree / f"file_{i}.c" + file_path.write_text(file_content) + self.assertEqual(_parse_spdx_license_identifier(str(file_path)), expected_identifier) diff --git a/scripts/sign-file.c b/scripts/sign-file.c index 73fbefd2e540..86b010ac1514 100644 --- a/scripts/sign-file.c +++ b/scripts/sign-file.c @@ -40,19 +40,7 @@ #endif #include "ssl-common.h" -struct module_signature { - uint8_t algo; /* Public-key crypto algorithm [0] */ - uint8_t hash; /* Digest algorithm [0] */ - uint8_t id_type; /* Key identifier type [PKEY_ID_PKCS7] */ - uint8_t signer_len; /* Length of signer's name [0] */ - uint8_t key_id_len; /* Length of key identifier [0] */ - uint8_t __pad[3]; - uint32_t sig_len; /* Length of signature data */ -}; - -#define PKEY_ID_PKCS7 2 - -static char magic_number[] = "~Module signature appended~\n"; +#include <linux/module_signature.h> static __attribute__((noreturn)) void format(void) @@ -197,7 +185,7 @@ static X509 *read_x509(const char *x509_name) int main(int argc, char **argv) { - struct module_signature sig_info = { .id_type = PKEY_ID_PKCS7 }; + struct module_signature sig_info = { .id_type = MODULE_SIGNATURE_TYPE_PKCS7 }; char *hash_algo = NULL; char *private_key_name = NULL, *raw_sig_name = NULL; char *x509_name, *module_name, *dest_name; @@ -357,7 +345,8 @@ int main(int argc, char **argv) sig_size = BIO_number_written(bd) - module_size; sig_info.sig_len = htonl(sig_size); ERR(BIO_write(bd, &sig_info, sizeof(sig_info)) < 0, "%s", dest_name); - ERR(BIO_write(bd, magic_number, sizeof(magic_number) - 1) < 0, "%s", dest_name); + ERR(BIO_write(bd, MODULE_SIGNATURE_MARKER, sizeof(MODULE_SIGNATURE_MARKER) - 1) < 0, + "%s", dest_name); ERR(BIO_free(bd) != 1, "%s", dest_name); diff --git a/scripts/sorttable.c b/scripts/sorttable.c index e8ed11c680c6..d7b50581c732 100644 --- a/scripts/sorttable.c +++ b/scripts/sorttable.c @@ -116,7 +116,7 @@ static inline void *get_index(void *start, int entsize, int index) } static int extable_ent_size; -static int long_size; +static int long_size __maybe_unused; #define ERRSTR_MAXSZ 256 @@ -891,17 +891,22 @@ static int do_file(char const *const fname, void *addr) table_sort_t custom_sort = NULL; switch (elf_map_machine(ehdr)) { - case EM_AARCH64: #ifdef MCOUNT_SORT_ENABLED + case EM_AARCH64: + /* arm64 also needs RELA-based weak-function fixups. */ sort_reloc = true; rela_type = 0x403; - /* arm64 uses patchable function entry placing before function */ + /* fallthrough */ + case EM_RISCV: + /* arm64 and RISC-V place patchable entries before the function. */ before_func = 8; +#else + case EM_AARCH64: + case EM_RISCV: #endif /* fallthrough */ case EM_386: case EM_LOONGARCH: - case EM_RISCV: case EM_S390: case EM_X86_64: custom_sort = sort_relative_table_with_data; diff --git a/scripts/spelling.txt b/scripts/spelling.txt index 1e89b92c2f9a..3372873cd7bb 100644 --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -57,8 +57,8 @@ acknowledgement||acknowledgment ackowledge||acknowledge ackowledged||acknowledged acording||according -activete||activate actived||activated +activete||activate actualy||actually actvie||active acumulating||accumulating @@ -66,12 +66,12 @@ acumulative||accumulative acumulator||accumulator acutally||actually adapater||adapter +adddress||address adderted||asserted addional||additional additionaly||additionally additonal||additional addres||address -adddress||address addreses||addresses addresss||address addrress||address @@ -95,9 +95,9 @@ alegorical||allegorical algined||aligned algorith||algorithm algorithmical||algorithmically +algorithmn||algorithm algoritm||algorithm algoritms||algorithms -algorithmn||algorithm algorrithm||algorithm algorritm||algorithm aligment||alignment @@ -128,20 +128,20 @@ amount of times||number of times amout||amount amplifer||amplifier amplifyer||amplifier -an union||a union -an user||a user -an userspace||a userspace -an one||a one analysator||analyzer ang||and anniversery||anniversary annoucement||announcement anomolies||anomalies anomoly||anomaly +an one||a one anonynous||anonymous +an union||a union +an user||a user +an userspace||a userspace anway||anyway -aplication||application apeared||appeared +aplication||application appearence||appearance applicaion||application appliction||application @@ -155,8 +155,8 @@ approriately||appropriately apropriate||appropriate aquainted||acquainted aquired||acquired -aquisition||acquisition aquires||acquires +aquisition||acquisition arbitary||arbitrary architechture||architecture archtecture||architecture @@ -176,6 +176,7 @@ assgined||assigned assiged||assigned assigment||assignment assigments||assignments +assinged||assigned assistent||assistant assocaited||associated assocated||associated @@ -189,30 +190,30 @@ assum||assume assumtpion||assumption asume||assume asuming||assuming -asycronous||asynchronous asychronous||asynchronous +asycronous||asynchronous +asymetric||asymmetric +asymmeric||asymmetric asynchnous||asynchronous asynchrnous||asynchronous -asynchronus||asynchronous asynchromous||asynchronous -asymetric||asymmetric -asymmeric||asymmetric +asynchronus||asynchronous +atempt||attempt atleast||at least atomatically||automatically atomicly||atomically -atempt||attempt atrributes||attributes attachement||attachment attatch||attach attched||attached attemp||attempt -attemps||attempts attemping||attempting +attemps||attempts attepmpt||attempt attnetion||attention attruibutes||attributes -authentification||authentication authenicated||authenticated +authentification||authentication automaticaly||automatically automaticly||automatically automatize||automate @@ -257,6 +258,7 @@ begining||beginning beter||better betweeen||between bianries||binaries +binded||bound bitmast||bitmask bitwiedh||bitwidth boardcast||broadcast @@ -287,19 +289,19 @@ calucate||calculate calulate||calculate cancelation||cancellation cancle||cancel -cant||can't -cant'||can't -canot||cannot -cann't||can't cannnot||cannot +cann't||can't +canot||cannot +cant'||can't +cant||can't capabiity||capability capabilites||capabilities capabilties||capabilities capabilty||capability capabitilies||capabilities capablity||capability -capatibilities||capabilities capapbilities||capabilities +capatibilities||capabilities captuer||capture caputure||capture carefuly||carefully @@ -307,9 +309,9 @@ cariage||carriage casued||caused catagory||category cehck||check +chache||cache challange||challenge challanges||challenges -chache||cache chanell||channel changable||changeable chanined||chained @@ -347,6 +349,7 @@ colescing||coalescing collapsable||collapsible colorfull||colorful comand||command +comaptible||compatible comit||commit commerical||commercial comming||coming @@ -357,10 +360,6 @@ committ||commit commmand||command commnunication||communication commoditiy||commodity -comsume||consume -comsumer||consumer -comsuming||consuming -comaptible||compatible compability||compatibility compaibility||compatibility comparsion||comparison @@ -376,22 +375,25 @@ compleatly||completely completition||completion completly||completely complient||compliant -componnents||components compoment||component +componnents||components comppatible||compatible compres||compress compresion||compression compresser||compressor comression||compression +comsume||consume comsumed||consumed +comsumer||consumer +comsuming||consuming comunicate||communicate comunication||communication conbination||combination concurent||concurrent conditionaly||conditionally conditon||condition -condtion||condition condtional||conditional +condtion||condition conected||connected conector||connector configed||configured @@ -428,13 +430,13 @@ continous||continuous continously||continuously continueing||continuing contiuous||continuous -contraints||constraints -contruct||construct contol||control contoller||controller +contraints||constraints controled||controlled controler||controller controll||control +contruct||construct contruction||construction contry||country conuntry||country @@ -465,10 +467,9 @@ debouce||debounce decendant||descendant decendants||descendants decompres||decompress -decsribed||described decrese||decrease decription||description -detault||default +decsribed||described dectected||detected defailt||default deferal||deferral @@ -482,9 +483,9 @@ defintion||definition defintions||definitions defualt||default defult||default -deintializing||deinitializing -deintialize||deinitialize deintialized||deinitialized +deintialize||deinitialize +deintializing||deinitializing deivce||device delared||declared delare||declare @@ -494,8 +495,8 @@ delemiter||delimiter deley||delay delibrately||deliberately delievered||delivered -demodualtor||demodulator demension||dimension +demodualtor||demodulator dependancies||dependencies dependancy||dependency dependant||dependent @@ -505,15 +506,15 @@ depreacte||deprecate desactivate||deactivate desciptor||descriptor desciptors||descriptors -descritpor||descriptor descripto||descriptor descripton||description descrition||description +descritpor||descriptor descritptor||descriptor desctiptor||descriptor +desination||destination desriptor||descriptor desriptors||descriptors -desination||destination destionation||destination destoried||destroyed destory||destroy @@ -521,6 +522,7 @@ destoryed||destroyed destorys||destroys destroied||destroyed detabase||database +detault||default deteced||detected detecion||detection detectt||detect @@ -535,55 +537,54 @@ deveolpment||development devided||divided deviece||device devision||division -diable||disable diabled||disabled +diable||disable dicline||decline +diconnected||disconnected dictionnary||dictionary didnt||didn't diferent||different -differrence||difference -diffrent||different differenciate||differentiate +differrence||difference diffreential||differential +diffrent||different diffrentiate||differentiate difinition||definition digial||digital dimention||dimension dimesions||dimensions -diconnected||disconnected -disabed||disabled -disasembler||disassembler -disble||disable -disgest||digest -disired||desired -dispalying||displaying -dissable||disable -dissapeared||disappeared diplay||display -directon||direction direcly||directly +directon||direction direectly||directly diregard||disregard -disassocation||disassociation -disassocative||disassociative +disabed||disabled disapear||disappear disapeared||disappeared disappared||disappeared -disbale||disable +disasembler||disassembler +disassocation||disassociation +disassocative||disassociative disbaled||disabled -disble||disable +disbale||disable disbled||disabled +disble||disable +disble||disable disconnet||disconnect discontinous||discontinuous +disgest||digest disharge||discharge +disired||desired disnabled||disabled +dispalying||displaying dispertion||dispersion +dissable||disable +dissapeared||disappeared dissapears||disappears dissconect||disconnect distiction||distinction divisable||divisible divsiors||divisors -dsiabled||disabled docuentation||documentation documantation||documentation documentaion||documentation @@ -598,6 +599,7 @@ downlads||downloads droped||dropped droput||dropout druing||during +dsiabled||disabled dyanmic||dynamic dynmaic||dynamic eanable||enable @@ -621,20 +623,20 @@ enble||enable enchanced||enhanced encorporating||incorporating encrupted||encrypted -encrypiton||encryption encryped||encrypted +encrypiton||encryption encryptio||encryption endianess||endianness -enpoint||endpoint enhaced||enhanced enlightnment||enlightenment +enocded||encoded +enought||enough +enpoint||endpoint enqueing||enqueuing +enterily||entirely entires||entries entites||entities entrys||entries -enocded||encoded -enought||enough -enterily||entirely enviroiment||environment enviroment||environment environement||environment @@ -653,8 +655,9 @@ evalute||evaluate evalutes||evaluates evalution||evaluation evaulated||evaluated -excecutable||executable +exaclty||exactly excceed||exceed +excecutable||executable exceded||exceeded exceds||exceeds exceeed||exceed @@ -668,41 +671,41 @@ exeuction||execution existance||existence existant||existent exixt||exist -exsits||exists exlcude||exclude exlcuding||excluding exlcusive||exclusive -exlusive||exclusive exlicitly||explicitly +exlusive||exclusive exmaple||example expecially||especially experies||expires explicite||explicit -explicity||explicitly explicitely||explicitly -explict||explicit +explicity||explicitly explictely||explicitly +explict||explicit explictly||explicitly expresion||expression exprienced||experienced exprimental||experimental -extened||extended +exsits||exists exteneded||extended +extened||extended extensability||extensibility -extention||extension extenstion||extension +extention||extension extracter||extractor faied||failed faield||failed -faild||failed failded||failed +faild||failed failer||failure -faill||fail failied||failed +faill||fail faillure||failure +failng||failing failue||failure failuer||failure -failng||failing faireness||fairness falied||failed faliure||failure @@ -717,15 +720,15 @@ fetcing||fetching fileystem||filesystem fimrware||firmware fimware||firmware +finanize||finalize +findn||find +finilizes||finalizes +finsih||finish firmare||firmware firmaware||firmware firtly||firstly firware||firmware firwmare||firmware -finanize||finalize -findn||find -finilizes||finalizes -finsih||finish fliter||filter flusing||flushing folloing||following @@ -742,9 +745,9 @@ forwared||forwarded frambuffer||framebuffer framming||framing framwork||framework +frequancy||frequency frequence||frequency frequncy||frequency -frequancy||frequency frome||from fronend||frontend fucntion||function @@ -766,9 +769,9 @@ gatable||gateable gateing||gating gauage||gauge gaurenteed||guaranteed -generiously||generously genereate||generate genereted||generated +generiously||generously genric||generic gerenal||general geting||getting @@ -790,18 +793,17 @@ hanlde||handle hanled||handled happend||happened hardare||hardware -harware||hardware hardward||hardware +harware||hardware havind||having +hearbeat||heartbeat heigth||height +heirachy||hierarchy heirarchically||hierarchically heirarchy||hierarchy -heirachy||hierarchy helpfull||helpful -hearbeat||heartbeat heterogenous||heterogeneous hexdecimal||hexadecimal -hybernate||hibernate hiearchy||hierarchy hierachy||hierarchy hierarchie||hierarchy @@ -809,14 +811,14 @@ homogenous||homogeneous horizental||horizontal howver||however hsould||should +hybernate||hibernate hypervior||hypervisor hypter||hyper idel||idle identidier||identifier iligal||illegal -illigal||illegal illgal||illegal -iomaped||iomapped +illigal||illegal imblance||imbalance immeadiately||immediately immedaite||immediate @@ -831,13 +833,14 @@ implemantation||implementation implemenation||implementation implementaiton||implementation implementated||implemented -implemention||implementation implementd||implemented +implemention||implementation implemetation||implementation implemntation||implementation implentation||implementation implmentation||implementation implmenting||implementing +inavlid||invalid incative||inactive incomming||incoming incompaitiblity||incompatibility @@ -869,9 +872,9 @@ infromation||information ingore||ignore inheritence||inheritance inital||initial -initalized||initialized initalised||initialized initalise||initialize +initalized||initialized initalize||initialize initation||initiation initators||initiators @@ -879,20 +882,20 @@ initialiazation||initialization initializationg||initialization initializiation||initialization initializtion||initialization -initialze||initialize initialzed||initialized +initialze||initialize initialzing||initializing initilization||initialization +initilized||initialized initilize||initialize initliaze||initialize -initilized||initialized inofficial||unofficial inrerface||interface insititute||institute instace||instance instal||install -instanciate||instantiate instanciated||instantiated +instanciate||instantiate instuments||instruments insufficent||insufficient intead||instead @@ -911,16 +914,16 @@ intergrated||integrated intermittant||intermittent internel||internal interoprability||interoperability -interuupt||interrupt -interupt||interrupt -interupts||interrupts -interurpt||interrupt interrface||interface interrrupt||interrupt interrup||interrupt interrups||interrupts interruptted||interrupted interupted||interrupted +interupt||interrupt +interupts||interrupts +interurpt||interrupt +interuupt||interrupt intiailized||initialized intial||initial intialisation||initialisation @@ -934,18 +937,18 @@ intrerrupt||interrupt intrrupt||interrupt intterrupt||interrupt intuative||intuitive -inavlid||invalid invaid||invalid invaild||invalid invailid||invalid -invald||invalid invalde||invalid +invald||invalid invalide||invalid invalidiate||invalidate invalud||invalid invididual||individual invokation||invocation invokations||invocations +iomaped||iomapped ireelevant||irrelevant irrelevent||irrelevant isnt||isn't @@ -991,11 +994,11 @@ losted||lost maangement||management machinary||machinery maibox||mailbox +mailformed||malformed maintainance||maintenance maintainence||maintenance maintan||maintain makeing||making -mailformed||malformed malplaced||misplaced malplace||misplace managable||manageable @@ -1005,21 +1008,22 @@ mangement||management manger||manager manoeuvering||maneuvering manufaucturing||manufacturing -mappping||mapping maping||mapping +mappping||mapping matchs||matches mathimatical||mathematical mathimatic||mathematic mathimatics||mathematics -maxmium||maximum maximium||maximum maxium||maximum +maxmium||maximum mechamism||mechanism mechanim||mechanism meetign||meeting memeory||memory memmber||member memoery||memory +memomry||memory memroy||memory ment||meant mergable||mergeable @@ -1036,19 +1040,19 @@ migrateable||migratable miliseconds||milliseconds millenium||millennium milliseonds||milliseconds -minimim||minimum -minium||minimum minimam||minimum +minimim||minimum minimun||minimum +minium||minimum miniumum||minimum minumum||minimum misalinged||misaligned miscelleneous||miscellaneous misformed||malformed -mispelled||misspelled -mispelt||misspelt mising||missing mismactch||mismatch +mispelled||misspelled +mispelt||misspelt missign||missing missmanaged||mismanaged missmatch||mismatch @@ -1061,18 +1065,17 @@ modifer||modifier modul||module modulues||modules momery||memory -memomry||memory monitring||monitoring monochorome||monochrome monochromo||monochrome monocrome||monochrome mopdule||module mroe||more -mulitplied||multiplied muliple||multiple -multipler||multiplier +mulitplied||multiplied multidimensionnal||multidimensional multipe||multiple +multipler||multiplier multple||multiple mumber||number muticast||multicast @@ -1094,6 +1097,7 @@ nerver||never nescessary||necessary nessessary||necessary none existent||non-existent +notfify||notify noticable||noticeable notication||notification notications||notifications @@ -1101,7 +1105,6 @@ notifcations||notifications notifed||notified notifer||notifier notity||notify -notfify||notify nubmer||number numebr||number numer||number @@ -1119,10 +1122,10 @@ occurence||occurrence occure||occurred occuring||occurring ocurrence||occurrence -offser||offset offet||offset offlaod||offload offloded||offloaded +offser||offset offseting||offsetting oflload||offload omited||omitted @@ -1141,25 +1144,25 @@ optionnal||optional optmizations||optimizations orientatied||orientated orientied||oriented -orignal||original originial||original +orignal||original orphanded||orphaned otherise||otherwise ouput||output oustanding||outstanding +oveflow||overflow overaall||overall +overflw||overflow overhread||overhead +overide||override overlaping||overlapping -oveflow||overflow -overflw||overflow overlfow||overflow -overide||override overrided||overridden overriden||overridden overrrun||overrun overun||overrun -overwritting||overwriting overwriten||overwritten +overwritting||overwriting pacakge||package pachage||package packacge||package @@ -1169,11 +1172,11 @@ packtes||packets pakage||package paket||packet pallette||palette -paln||plan palne||plane +paln||plan paramameters||parameters -paramaters||parameters paramater||parameter +paramaters||parameters paramenters||parameters parametes||parameters parametised||parametrised @@ -1241,8 +1244,6 @@ prefered||preferred prefferably||preferably prefitler||prefilter preform||perform -previleged||privileged -previlege||privilege premption||preemption prepaired||prepared prepate||prepare @@ -1250,6 +1251,8 @@ preperation||preparation preprare||prepare pressre||pressure presuambly||presumably +previleged||privileged +previlege||privilege previosuly||previously previsously||previously primative||primitive @@ -1258,17 +1261,17 @@ priorty||priority priting||printing privilaged||privileged privilage||privilege -priviledge||privilege priviledged||privileged +priviledge||privilege priviledges||privileges privleges||privileges -probaly||probably probabalistic||probabilistic +probaly||probably procceed||proceed proccesors||processors procesed||processed -proces||process procesing||processing +proces||process processessing||processing processess||processes processpr||processor @@ -1288,6 +1291,7 @@ progresss||progress prohibitted||prohibited prohibitting||prohibiting promiscous||promiscuous +promixity||proximity promps||prompts pronnounced||pronounced prononciation||pronunciation @@ -1296,15 +1300,14 @@ pronunce||pronounce propery||property propigate||propagate propigation||propagation -propogation||propagation propogate||propagate +propogation||propagation prosess||process protable||portable protcol||protocol protecion||protection protedcted||protected protocoll||protocol -promixity||proximity psudo||pseudo psuedo||pseudo psychadelic||psychedelic @@ -1333,8 +1336,8 @@ recieves||receives recieving||receiving recogniced||recognised recognizeable||recognizable -recompte||recompute recommanded||recommended +recompte||recompute recyle||recycle redect||reject redircet||redirect @@ -1344,8 +1347,8 @@ reename||rename refcounf||refcount refence||reference refered||referred -referencce||reference referenace||reference +referencce||reference refererence||reference refering||referring refernces||references @@ -1353,8 +1356,8 @@ refernnce||reference refrence||reference regiser||register registed||registered -registerd||registered registeration||registration +registerd||registered registeresd||registered registerred||registered registes||registers @@ -1371,8 +1374,8 @@ reloade||reload remoote||remote remore||remote removeable||removable -repective||respective repectively||respectively +repective||respective replacable||replaceable replacments||replacements replys||replies @@ -1389,8 +1392,8 @@ requieres||requires requirment||requirement requred||required requried||required -requst||request requsted||requested +requst||request reregisteration||reregistration reseting||resetting reseved||reserved @@ -1412,11 +1415,11 @@ retransmited||retransmitted retreived||retrieved retreive||retrieve retreiving||retrieving -retrive||retrieve retrived||retrieved +retrive||retrieve retrun||return -retun||return retuned||returned +retun||return reudce||reduce reuest||request reuqest||request @@ -1464,9 +1467,9 @@ seperate||separate seperatly||separately seperator||separator sepperate||separate -seqeunce||sequence -seqeuncer||sequencer seqeuencer||sequencer +seqeuncer||sequencer +seqeunce||sequence sequece||sequence sequemce||sequence sequencial||sequential @@ -1505,8 +1508,8 @@ soley||solely soluation||solution souce||source speach||speech -specfic||specific specfication||specification +specfic||specific specfield||specified speciefied||specified specifc||specific @@ -1515,8 +1518,8 @@ specificatin||specification specificaton||specification specificed||specified specifing||specifying -specifiy||specify specifiying||specifying +specifiy||specify speficied||specified speicify||specify speling||spelling @@ -1543,23 +1546,23 @@ stoppped||stopped straming||streaming struc||struct structres||structures -stuct||struct strucuture||structure +stuct||struct stucture||structure sturcture||structure subdirectoires||subdirectories suble||subtle -substract||subtract submited||submitted submition||submission +substract||subtract succeded||succeeded -suceed||succeed -succesfuly||successfully succesfully||successfully succesful||successful +succesfuly||successfully successed||succeeded successfull||successful successfuly||successfully +suceed||succeed sucessfully||successfully sucessful||successful sucess||success @@ -1568,9 +1571,9 @@ superseeded||superseded suplied||supplied suported||supported suport||support -supportet||supported suppored||supported supporing||supporting +supportet||supported supportin||supporting suppoted||supported suppported||supported @@ -1581,27 +1584,27 @@ surpressed||suppressed surpresses||suppresses susbsystem||subsystem suspeneded||suspended -suspsend||suspend suspicously||suspiciously +suspsend||suspend swaping||swapping switchs||switches -swith||switch swithable||switchable -swithc||switch swithced||switched swithcing||switching +swithc||switch swithed||switched swithing||switching +swith||switch swtich||switch +sychronization||synchronization +sychronously||synchronously syfs||sysfs symetric||symmetric synax||syntax synchonized||synchronized -sychronization||synchronization -sychronously||synchronously synchronuously||synchronously -syncronize||synchronize syncronized||synchronized +syncronize||synchronize syncronizing||synchronizing syncronus||synchronous syste||system @@ -1610,16 +1613,17 @@ sythesis||synthesis tagert||target taht||that tained||tainted -tarffic||traffic +tansition||transition tansmit||transmit +tarffic||traffic targetted||targeted targetting||targeting taskelt||tasklet teh||the temeprature||temperature temorary||temporary -temproarily||temporarily temperture||temperature +temproarily||temporarily theads||threads therfore||therefore thier||their @@ -1629,23 +1633,20 @@ threshhold||threshold thresold||threshold throtting||throttling throught||through -tansition||transition -trackling||tracking -troughput||throughput -trys||tries thses||these -tiggers||triggers tiggered||triggered tiggerring||triggering -tipically||typically +tiggers||triggers timeing||timing timming||timing timout||timeout +tipically||typically tmis||this tolarance||tolerance toogle||toggle torerable||tolerable torlence||tolerance +trackling||tracking traget||target traking||tracking tramsmitted||transmitted @@ -1669,20 +1670,20 @@ trasfer||transfer trasmission||transmission trasmitter||transmitter treshold||threshold -trigged||triggered -triggerd||triggered trigerred||triggered trigerring||triggering +trigged||triggered +triggerd||triggered +troughput||throughput trun||turn +trys||tries tunning||tuning ture||true tyep||type udpate||update -updtes||updates uesd||used -unknwon||unknown uknown||unknown -usccess||success +unamed||unnamed uncommited||uncommitted uncompatible||incompatible uncomressed||uncompressed @@ -1691,6 +1692,7 @@ undeflow||underflow undelying||underlying underun||underrun unecessary||unnecessary +uneeded||unneeded unexecpted||unexpected unexepected||unexpected unexpcted||unexpected @@ -1699,26 +1701,24 @@ unexpeted||unexpected unexpexted||unexpected unfortunatelly||unfortunately unifiy||unify -uniterrupted||uninterrupted uninterruptable||uninterruptible unintialized||uninitialized +uniterrupted||uninterrupted unitialized||uninitialized unkmown||unknown unknonw||unknown unknouwn||unknown unknow||unknown +unknwon||unknown unkown||unknown -unamed||unnamed -uneeded||unneeded -unneded||unneeded +unmached||unmatched unneccecary||unnecessary unneccesary||unnecessary unneccessary||unnecessary unnecesary||unnecessary +unneded||unneeded unneedingly||unnecessarily unnsupported||unsupported -unuspported||unsupported -unmached||unmatched unprecise||imprecise unpriviledged||unprivileged unpriviliged||unprivileged @@ -1726,18 +1726,21 @@ unregester||unregister unresgister||unregister unrgesiter||unregister unsinged||unsigned -unstabel||unstable -unsolicted||unsolicited unsolicitied||unsolicited +unsolicted||unsolicited +unstabel||unstable unsuccessfull||unsuccessful unsuported||unsupported untill||until ununsed||unused unuseful||useless +unuspported||unsupported unvalid||invalid upate||update +updtes||updates upsupported||unsupported upto||up to +usccess||success useable||usable usefule||useful usefull||useful @@ -1759,14 +1762,14 @@ variantions||variations varible||variable varient||variant vaule||value -verbse||verbose veify||verify +verbse||verbose verfication||verification veriosn||version -versoin||version verisons||versions verison||version veritical||vertical +versoin||version verson||version vicefersa||vice-versa virtal||virtual @@ -1779,13 +1782,13 @@ wakeus||wakeups was't||wasn't wathdog||watchdog wating||waiting -wiat||wait wether||whether whataver||whatever whcih||which whenver||whenever wheter||whether whe||when +wiat||wait wierd||weird wihout||without wiil||will diff --git a/scripts/syscall.tbl b/scripts/syscall.tbl index 7a42b32b6577..0ab531605120 100644 --- a/scripts/syscall.tbl +++ b/scripts/syscall.tbl @@ -412,3 +412,4 @@ 469 common file_setattr sys_file_setattr 470 common listns sys_listns 471 common rseq_slice_yield sys_rseq_slice_yield +472 common fchroot sys_fchroot diff --git a/scripts/tags.sh b/scripts/tags.sh index 243373683f98..41e38df96984 100755 --- a/scripts/tags.sh +++ b/scripts/tags.sh @@ -46,13 +46,31 @@ elif [ "${ALLSOURCE_ARCHS}" = "all" ]; then ALLSOURCE_ARCHS=$(find ${tree}arch/ -mindepth 1 -maxdepth 1 -type d -printf '%f ') fi +setup_name_pattern() +{ + pattern=() + for ext; do + if [ ${#pattern[@]} -gt 0 ]; then + pattern+=("-o" "-name" "$ext") + else + pattern+=("(" "-name" "$ext") + fi + done + if [ ${#pattern[@]} -gt 0 ]; then + pattern+=(")") + fi +} + # find sources in arch/$1 find_arch_sources() { for i in $archincludedir; do local prune="$prune ( -path $i ) -prune -o" done - find ${tree}arch/$1 $ignore $prune -name "$2" -not -type l -print; + local src=${tree}arch/$1 + shift + setup_name_pattern "$@" + find $src $ignore $prune "${pattern[@]}" -not -type l -print; } # find sources in arch/$1/include @@ -61,14 +79,17 @@ find_arch_include_sources() local include=$(find ${tree}arch/$1/ -name include -type d -print); if [ -n "$include" ]; then archincludedir="$archincludedir $include" - find $include $ignore -name "$2" -not -type l -print; + shift + setup_name_pattern "$@" + find $include $ignore "${pattern[@]}" -not -type l -print; fi } # find sources in include/ find_include_sources() { - find ${tree}include $ignore -name config -prune -o -name "$1" \ + setup_name_pattern "$@" + find ${tree}include $ignore -name config -prune -o "${pattern[@]}" \ -not -type l -print; } @@ -76,23 +97,24 @@ find_include_sources() # we could benefit from a list of dirs to search in here find_other_sources() { + setup_name_pattern "$@" find ${tree}* $ignore \ \( -path ${tree}include -o -path ${tree}arch -o -name '.tmp_*' \) -prune -o \ - -name "$1" -not -type l -print; + "${pattern[@]}" -not -type l -print; } all_sources() { - find_arch_include_sources ${SRCARCH} '*.[chS]' + find_arch_include_sources ${SRCARCH} '*.[chS]' '*.rs' if [ -n "$archinclude" ]; then - find_arch_include_sources $archinclude '*.[chS]' + find_arch_include_sources $archinclude '*.[chS]' '*.rs' fi - find_include_sources '*.[chS]' + find_include_sources '*.[chS]' '*.rs' for arch in $ALLSOURCE_ARCHS do - find_arch_sources $arch '*.[chS]' + find_arch_sources $arch '*.[chS]' '*.rs' done - find_other_sources '*.[chS]' + find_other_sources '*.[chS]' '*.rs' } all_compiled_sources() @@ -100,7 +122,7 @@ all_compiled_sources() { echo include/generated/autoconf.h find $ignore -name "*.cmd" -exec \ - grep -Poh '(?<=^ )\S+|(?<== )\S+[^\\](?=$)' {} \+ | + grep -Poh '(?<=^ )\S+\.([chS]|rs)(?=\s)|(?<== )\S+\.(?1)(?=$)' {} \+ | awk '!a[$0]++' } | xargs realpath -esq $([ -z "$KBUILD_ABS_SRCTREE" ] && echo --relative-to=.) | sort -u diff --git a/scripts/timer_migration_tree.py b/scripts/timer_migration_tree.py new file mode 100755 index 000000000000..faac9de854bd --- /dev/null +++ b/scripts/timer_migration_tree.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 + +""" +Draw the timer migration tree. + +1) Boot with trace_event==tmigr_connect_cpu_parent,tmigr_connect_child_parent +2) ./timer_migration_tree.py < /sys/kernel/tracing/trace +""" + +import re, sys +from ete3 import Tree + +class Node: + def __init__(self, group): + self.group = group + self.children = [] + self.parent = None + self.num_children = 0 + self.groupmask = 0 + self.lvl = -1 + + def set_groupmask(self, groupmask): + self.groupmask = groupmask + + def set_parent(self, parent): + self.parent = parent + + def add_child(self, child): + self.children.append(child) + + def set_lvl(self, lvl): + self.lvl = lvl + + def set_numa(self, numa): + self.numa = numa + + def set_num_children(self, num_children): + self.num_children = num_children + + def __repr__(self): + if self.parent: + parent_grp = self.parent.group + else: + parent_grp = "-" + return "Group: %s mask: %s parent: %s lvl: %d numa: %d num_children: %d" % (self.group, self.groupmask, parent_grp, self.lvl, self.numa, self.num_children) + +hierarchies = { } + +def get_hierarchy(capacity): + if capacity not in hierarchies: + hierarchies[capacity] = {} + return hierarchies[capacity] + +def get_node(capacity, group): + hier = get_hierarchy(capacity) + if group in hier: + return hier[group] + else: + n = Node(group) + hier[group] = n + return n + +def tmigr_connect_cpu_parent(ts, line): + s = re.search("tmigr_connect_cpu_parent: cpu=([0-9]+) groupmask=([0-9a-zA-Z]+) parent=([0-9a-zA-Z]+) lvl=([0-9]+) numa=([-]?[0-9]+) capacity=([-]?[0-9]+) num_children=([0-9]+)", line) + if s is None: + return False + (cpu, groupmask, parent, lvl, numa, capacity, num_children) = (int(s.group(1)), s.group(2), s.group(3), int(s.group(4)), int(s.group(5)), int(s.group(6)), int(s.group(7))) + n = get_node(capacity, cpu) + p = get_node(capacity, parent) + n.set_parent(p) + n.set_groupmask(groupmask) + n.set_lvl(-1) + p.set_lvl(lvl) + p.set_numa(numa) + n.set_numa(numa) + p.set_num_children(num_children) + p.add_child(n) + +def tmigr_connect_child_parent(ts, line): + s = re.search("tmigr_connect_child_parent: group=([0-9a-zA-Z]+) groupmask=([0-9a-zA-Z]+) parent=([0-9a-zA-Z]+) lvl=([0-9]+) numa=([-]?[0-9]+) capacity=([-]?[0-9]+) num_children=([0-9]+)", line) + if s is None: + return False + (group, groupmask, parent, lvl, numa, capacity, num_children) = (s.group(1), s.group(2), s.group(3), int(s.group(4)), int(s.group(5)), int(s.group(6)), int(s.group(7))) + n = get_node(capacity, group) + p = get_node(capacity, parent) + n.set_parent(p) + n.set_groupmask(groupmask) + p.set_lvl(lvl) + p.set_numa(numa) + p.set_num_children(num_children) + p.add_child(n) + +def populate(enode, node): + enode = enode.add_child(name = node.group) + enode.add_feature("groupmask", "m:%s" % node.groupmask) + enode.add_feature("lvl", "lvl:%d" % node.lvl) + enode.add_feature("numa", "node %d" % node.numa) + enode.add_feature("num_children", "c=%d" % node.num_children) + for child in node.children: + populate(enode, child) + +if __name__ == "__main__": + for line in sys.stdin: + s = re.search("([0-9]+[.][0-9]{6}): (.+?)$", line, re.S) + if s is not None: + if tmigr_connect_cpu_parent(float(s.group(1)), s.group(2)): + continue + if tmigr_connect_child_parent(float(s.group(1)), s.group(2)): + continue + + for cap in hierarchies: + h = hierarchies[cap] + print("Tree for capacity %d" % cap) + for k in h: + n = h[k] + while n.parent != None: + n = n.parent + root = Tree() + populate(root, n) + print(root.get_ascii(show_internal=True, attributes=["name", "numa", "lvl"])) + break diff --git a/scripts/update-intel-ucode-defs.py b/scripts/update-intel-ucode-defs.py new file mode 100755 index 000000000000..9d6cc2c6075f --- /dev/null +++ b/scripts/update-intel-ucode-defs.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +import argparse +import re +import shutil +import subprocess +import sys +import os + +script = os.path.relpath(__file__) + +DESCRIPTION = f""" +For Intel CPUs, update the microcode revisions that determine +X86_BUG_OLD_MICROCODE. + +This script is intended to be run in response to releases of the +official Intel microcode GitHub repository: +https://github.com/intel/Intel-Linux-Processor-Microcode-Data-Files.git + +It takes the Intel microcode files as input and uses iucode-tool to +extract the revision information. It prints the output in the format +expected by intel-ucode-defs.h. + +Usage: + ./{script} /path/to/microcode/files > /path/to/intel-ucode-defs.h + +Typically, someone at Intel would see a new public release, wait for at +least three months to ensure the update is stable, run this script to +refresh the intel-ucode-defs.h file, and send a patch upstream to update +the mainline and stable versions. + +Any exception to this process should be supported with an appropriate +justification. +""" + +SIG_RE = re.compile(r'sig (0x[0-9a-fA-F]+)') +PFM_RE = re.compile(r'pf_mask (0x[0-9a-fA-F]+)') +REV_RE = re.compile(r'rev (0x[0-9a-fA-F]+)') + +# Functions to extract family, model, and stepping +def bits(val, bottom, top): + mask = (1 << (top + 1 - bottom)) - 1 + return (val >> bottom) & mask + +def family(sig): + if bits(sig, 8, 11) == 0xf: + return bits(sig, 8, 11) + bits(sig, 20, 27) + return bits(sig, 8, 11) + +def model(sig): + return bits(sig, 4, 7) | (bits(sig, 16, 19) << 4) + +def step(sig): + return bits(sig, 0, 3) + +class Ucode: + def __init__(self, sig, pfm, rev): + self.family = family(sig) + self.model = model(sig) + self.steppings = 1 << step(sig) + self.platforms = pfm + self.rev = rev + + self.key = (self.family, self.model, self.steppings, self.platforms) + + def __eq__(self, other): + return self.key == other.key + + def __hash__(self): + return hash(self.key) + + def __str__(self): + return "{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x%x, .model = 0x%02x, .steppings = 0x%04x, .platform_mask = 0x%02x, .driver_data = 0x%x }," % \ + (self.family, self.model, self.steppings, self.platforms, self.rev) + +def main(): + parser = argparse.ArgumentParser(description=DESCRIPTION, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('ucode_files', nargs='+', help='Path(s) to the microcode files') + + args = parser.parse_args() + + # Process the microcode files using iucode-tool + iucode_tool = shutil.which("iucode-tool") or shutil.which("iucode_tool") + if iucode_tool is None: + print("Error: iucode-tool not found, please install it", file=sys.stderr) + sys.exit(1) + + cmd = [iucode_tool, '--list-all'] + args.ucode_files + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print("Error: iucode-tool ran into an error, exiting", file=sys.stderr) + if result.stderr: + print(result.stderr, file=sys.stderr, end='') + sys.exit(1) + + ucodes = set() + + # Parse the output of iucode-tool + for line in result.stdout.splitlines(): + sig_match = SIG_RE.search(line) + pfm_match = PFM_RE.search(line) + rev_match = REV_RE.search(line) + + if not (sig_match and pfm_match and rev_match): + continue + + sig = int(sig_match.group(1), 16) + pfm = int(pfm_match.group(1), 16) + rev = int(rev_match.group(1), 16) + debug_rev = bits(rev, 31, 31) + if debug_rev != 0: + print("Error: Debug ucode file found, exiting", file=sys.stderr) + sys.exit(1) + + ucodes.add(Ucode(sig, pfm, rev)) + + if not ucodes: + print("Error: No valid microcode files found, exiting", file=sys.stderr) + sys.exit(1) + + # Sort and print the microcode entries + print("/* SPDX-License-Identifier: GPL-2.0 */") + print("/* Auto-generated by scripts/update-intel-ucode-defs.py */") + for u in sorted(ucodes, key=lambda x: x.key): + print(u) + +if __name__ == "__main__": + main() diff --git a/scripts/ver_linux b/scripts/ver_linux index d6f2362d3792..00bdaf30d590 100755 --- a/scripts/ver_linux +++ b/scripts/ver_linux @@ -7,7 +7,7 @@ BEGIN { usage = "If some fields are empty or look unusual you may have an old version.\n" - usage = usage "Compare to the current minimal requirements in Documentation/Changes.\n" + usage = usage "Compare to the current minimal requirements in Documentation/process/changes.rst\n" print usage system("uname -a") @@ -17,37 +17,58 @@ BEGIN { libc = "libc[.]so[.][0-9]+$" libcpp = "(libg|stdc)[+]+[.]so([.][0-9]+)+$" + printversion("bash", version("bash --version")) + printversion("bc", version("bc --version")) + printversion("bindgen", version("bindgen --version")) + printversion("binutils", version("ld -v")) + printversion("bison", version("bison --version")) + printversion("btrfs-progs", version("btrfs --version")) + printversion("Clang", version("clang --version")) + printversion("Console-tools", version("loadkeys -V")) + printversion("Dynamic linker (ldd)", version("ldd --version")) + printversion("e2fsprogs", version("e2fsck -V")) + printversion("flex", version("flex --version")) + printversion("gdb", version("gdb -version")) + printversion("GNU awk", version("gawk --version")) printversion("GNU C", version("gcc -dumpversion")) - printversion("GNU Make", version("make --version")) - printversion("Binutils", version("ld -v")) - printversion("Util-linux", version("mount --version")) - printversion("Mount", version("mount --version")) - printversion("Module-init-tools", version("depmod -V")) - printversion("E2fsprogs", version("tune2fs")) - printversion("Jfsutils", version("fsck.jfs -V")) - printversion("Xfsprogs", version("xfs_db -V")) - printversion("Pcmciautils", version("pccardctl -V")) - printversion("Pcmcia-cs", version("cardmgr -V")) - printversion("Quota-tools", version("quota -V")) - printversion("PPP", version("pppd --version")) + printversion("GNU make", version("make --version")) + printversion("GNU tar", version("tar --version")) + printversion("GRUB2", version("grub2-install --version")) + printversion("GRUB", version("grub-install --version")) + printversion("gtags", version("gtags --version")) + printversion("iptables", version("iptables -V")) printversion("Isdn4k-utils", version("isdnctrl")) - printversion("Nfs-utils", version("showmount --version")) - printversion("Bison", version("bison --version")) - printversion("Flex", version("flex --version")) + printversion("jfsutils", version("fsck.jfs -V")) + printversion("Kbd", version("loadkeys -V")) + printversion("kmod", version("kmod -V")) while ("ldconfig -p 2>/dev/null" | getline > 0) if ($NF ~ libc || $NF ~ libcpp) if (!seen[ver = version("readlink " $NF)]++) printversion("Linux C" ($NF ~ libcpp? "++" : "") " Library", ver) - printversion("Dynamic linker (ldd)", version("ldd --version")) - printversion("Procps", version("ps --version")) + printversion("mcelog", version("mcelog --version")) + printversion("mkimage", version("mkimage --version")) + printversion("Module-init-tools", version("depmod -V")) + printversion("Mount", version("mount --version")) printversion("Net-tools", version("ifconfig --version")) - printversion("Kbd", version("loadkeys -V")) - printversion("Console-tools", version("loadkeys -V")) + printversion("nfs-utils", version("showmount --version")) + printversion("openssl", version("openssl version")) + printversion("pahole", version("pahole --version")) + printversion("Pcmcia-cs", version("cardmgr -V")) + printversion("pcmciautils", version("pccardctl -V")) + printversion("PPP", version("pppd --version")) + printversion("procps", version("ps --version")) + printversion("Python", version("python3 -V")) + printversion("quota-tools", version("quota -V")) + printversion("Rust", version("rustc --version")) printversion("Sh-utils", version("expr --v")) - printversion("Udev", version("udevadm --version")) + printversion("Sphinx", version("sphinx-build --version")) + printversion("squashfs-tools", version("mksquashfs -version")) + printversion("udev", version("udevadm --version")) + printversion("util-linux", version("mount --version")) printversion("Wireless-tools", version("iwconfig --version")) + printversion("xfsprogs", version("xfs_db -V")) while ("sort /proc/modules" | getline > 0) { mods = mods sep $1 |