diff options
Diffstat (limited to 'rust')
417 files changed, 56202 insertions, 3932 deletions
diff --git a/rust/Makefile b/rust/Makefile index 9801af2e1e02..da1a7409d984 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -3,18 +3,27 @@ # Where to place rustdoc generated documentation rustdoc_output := $(objtree)/Documentation/output/rust/rustdoc +# Clean generated host directory +clean-files := host/ + obj-$(CONFIG_RUST) += core.o compiler_builtins.o ffi.o always-$(CONFIG_RUST) += exports_core_generated.h +obj-$(CONFIG_RUST) += zerocopy.o + +ifdef CONFIG_RUST_INLINE_HELPERS +always-$(CONFIG_RUST) += helpers/helpers.bc helpers/helpers_module.bc +else +obj-$(CONFIG_RUST) += helpers/helpers.o +always-$(CONFIG_RUST) += exports_helpers_generated.h +endif # Missing prototypes are expected in the helpers since these are exported # for Rust only, thus there is no header nor prototypes. -obj-$(CONFIG_RUST) += helpers/helpers.o CFLAGS_REMOVE_helpers/helpers.o = -Wmissing-prototypes -Wmissing-declarations always-$(CONFIG_RUST) += bindings/bindings_generated.rs bindings/bindings_helpers_generated.rs obj-$(CONFIG_RUST) += bindings.o pin_init.o kernel.o -always-$(CONFIG_RUST) += exports_helpers_generated.h \ - exports_bindings_generated.h exports_kernel_generated.h +always-$(CONFIG_RUST) += exports_bindings_generated.h exports_kernel_generated.h always-$(CONFIG_RUST) += uapi/uapi_generated.rs obj-$(CONFIG_RUST) += uapi.o @@ -27,7 +36,7 @@ endif obj-$(CONFIG_RUST) += exports.o -always-$(CONFIG_RUST) += libproc_macro2.rlib libquote.rlib libsyn.rlib +always-$(CONFIG_RUST) += host/libproc_macro2.rlib host/libquote.rlib host/libsyn.rlib always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.rs always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.c @@ -43,13 +52,14 @@ endif # Avoids running `$(RUSTC)` when it may not be available. ifdef CONFIG_RUST -libmacros_name := $(shell MAKEFLAGS= $(RUSTC) --print file-names --crate-name macros --crate-type proc-macro - </dev/null) -libmacros_extension := $(patsubst libmacros.%,%,$(libmacros_name)) +procmacro-name = $(shell MAKEFLAGS= $(RUSTC) --print file-names --crate-name $(1) --crate-type proc-macro - </dev/null) +procmacro-extension := $(patsubst libname.%,%,$(call procmacro-name,name)) -libpin_init_internal_name := $(shell MAKEFLAGS= $(RUSTC) --print file-names --crate-name pin_init_internal --crate-type proc-macro - </dev/null) -libpin_init_internal_extension := $(patsubst libpin_init_internal.%,%,$(libpin_init_internal_name)) +libzerocopy_derive_name := $(call procmacro-name,zerocopy_derive) +libmacros_name := $(call procmacro-name,macros) +libpin_init_internal_name := $(call procmacro-name,pin_init_internal) -always-$(CONFIG_RUST) += $(libmacros_name) $(libpin_init_internal_name) +always-$(CONFIG_RUST) += $(libzerocopy_derive_name) $(libmacros_name) $(libpin_init_internal_name) # `$(rust_flags)` is passed in case the user added `--sysroot`. rustc_sysroot := $(shell MAKEFLAGS= $(RUSTC) $(rust_flags) --print sysroot) @@ -71,22 +81,29 @@ core-edition := $(if $(call rustc-min-version,108700),2024,2021) core-skip_flags := \ --edition=2021 \ - -Wunreachable_pub \ - -Wrustdoc::unescaped_backticks + -Wunreachable_pub core-flags := \ --edition=$(core-edition) \ $(call cfgs-to-flags,$(core-cfgs)) +zerocopy-cfgs := \ + no_fp_fmt_parse + +zerocopy-flags := \ + --cap-lints=allow \ + $(call cfgs-to-flags,$(zerocopy-cfgs)) + +zerocopy-envs := \ + CARGO_PKG_VERSION=0.8.54 + proc_macro2-cfgs := \ feature="proc-macro" \ wrap_proc_macro \ $(if $(call rustc-min-version,108800),proc_macro_span_file proc_macro_span_location) -# Stable since Rust 1.79.0: `feature(proc_macro_byte_character,proc_macro_c_str_literals)`. proc_macro2-flags := \ --cap-lints=allow \ - -Zcrate-attr='feature(proc_macro_byte_character,proc_macro_c_str_literals)' \ $(call cfgs-to-flags,$(proc_macro2-cfgs)) quote-cfgs := \ @@ -109,6 +126,7 @@ syn-cfgs := \ feature="parsing" \ feature="printing" \ feature="proc-macro" \ + feature="visit" \ feature="visit-mut" syn-flags := \ @@ -117,8 +135,18 @@ syn-flags := \ --extern quote \ $(call cfgs-to-flags,$(syn-cfgs)) +zerocopy_derive-cfgs := \ + zerocopy_unstable_linux + +zerocopy_derive-flags := \ + --cap-lints=allow \ + --extern proc_macro2 \ + --extern quote \ + --extern syn \ + $(call cfgs-to-flags,$(zerocopy_derive-cfgs)) + pin_init_internal-cfgs := \ - kernel + kernel USE_RUSTC_FEATURES pin_init_internal-flags := \ --extern proc_macro2 \ @@ -127,7 +155,7 @@ pin_init_internal-flags := \ $(call cfgs-to-flags,$(pin_init_internal-cfgs)) pin_init-cfgs := \ - kernel + kernel USE_RUSTC_FEATURES pin_init-flags := \ --extern pin_init_internal \ @@ -141,16 +169,12 @@ rustdoc_modifiers_workaround := $(if $(call rustc-min-version,108800),-Cunsafe-a # Similarly, for doctests (https://github.com/rust-lang/rust/issues/146465). doctests_modifiers_workaround := $(rustdoc_modifiers_workaround)$(if $(call rustc-min-version,109100),$(comma)sanitizer) -# `rustc` recognizes `--remap-path-prefix` since 1.26.0, but `rustdoc` only -# since Rust 1.81.0. Moreover, `rustdoc` ICEs on out-of-tree builds since Rust -# 1.82.0 (https://github.com/rust-lang/rust/issues/138520). Thus workaround both -# issues skipping the flag. The former also applies to `RUSTDOC TK`. quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< cmd_rustdoc = \ + $(rustc_target_envs) \ OBJTREE=$(abspath $(objtree)) \ - $(RUSTDOC) $(filter-out $(skip_flags) --remap-path-prefix=% --remap-path-scope=%, \ - $(if $(rustdoc_host),$(rust_common_flags),$(rust_flags))) \ - $(rustc_target_flags) -L$(objtree)/$(obj) \ + $(RUSTDOC) $(filter-out $(skip_flags) --remap-path-scope=%,$(if $(rustdoc_host),$(rust_common_flags),$(rust_flags))) \ + $(rustc_target_flags) -L$(objtree)/$(obj)$(if $(rustdoc_host),/host) \ -Zunstable-options --generate-link-to-definition \ --output $(rustdoc_output) \ --crate-name $(subst rustdoc-,,$@) \ @@ -170,7 +194,7 @@ quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< # command-like flags to solve the issue. Meanwhile, we use the non-custom case # and then retouch the generated files. rustdoc: rustdoc-core rustdoc-macros rustdoc-compiler_builtins \ - rustdoc-kernel rustdoc-pin_init + rustdoc-kernel rustdoc-pin_init rustdoc-zerocopy rustdoc-zerocopy_derive $(Q)grep -Ehro '<a href="srctree/([^"]+)"' $(rustdoc_output) | \ cut -d'"' -f2 | cut -d/ -f2- | while read f; do \ if [ ! -e "$(srctree)/$$f" ]; then \ @@ -203,6 +227,12 @@ rustdoc-syn: private rustc_target_flags = $(syn-flags) rustdoc-syn: $(src)/syn/lib.rs rustdoc-clean rustdoc-quote FORCE +$(call if_changed,rustdoc) +rustdoc-zerocopy_derive: private rustdoc_host = yes +rustdoc-zerocopy_derive: private rustc_target_flags = $(zerocopy_derive-flags) \ + --extern proc_macro --crate-type proc-macro +rustdoc-zerocopy_derive: $(src)/zerocopy-derive/lib.rs rustdoc-clean rustdoc-syn FORCE + +$(call if_changed,rustdoc) + rustdoc-macros: private rustdoc_host = yes rustdoc-macros: private rustc_target_flags = --crate-type proc-macro \ --extern proc_macro --extern proc_macro2 --extern quote --extern syn @@ -210,8 +240,6 @@ rustdoc-macros: $(src)/macros/lib.rs rustdoc-clean rustdoc-proc_macro2 \ rustdoc-quote rustdoc-syn FORCE +$(call if_changed,rustdoc) -# Starting with Rust 1.82.0, skipping `-Wrustdoc::unescaped_backticks` should -# not be needed -- see https://github.com/rust-lang/rust/pull/128307. rustdoc-core: private skip_flags = $(core-skip_flags) rustdoc-core: private rustc_target_flags = $(core-flags) rustdoc-core: $(RUST_LIB_SRC)/core/src/lib.rs rustdoc-clean FORCE @@ -224,6 +252,13 @@ rustdoc-compiler_builtins: private is-kernel-object := y rustdoc-compiler_builtins: $(src)/compiler_builtins.rs rustdoc-core FORCE +$(call if_changed,rustdoc) +rustdoc-zerocopy: private rustc_target_envs := $(zerocopy-envs) +rustdoc-zerocopy: private is-kernel-object := y +rustdoc-zerocopy: private rustc_target_flags = $(zerocopy-flags) \ + --extend-css $(src)/zerocopy/rustdoc/style.css +rustdoc-zerocopy: $(src)/zerocopy/src/lib.rs rustdoc-clean rustdoc-core FORCE + +$(call if_changed,rustdoc) + rustdoc-ffi: private is-kernel-object := y rustdoc-ffi: $(src)/ffi.rs rustdoc-core FORCE +$(call if_changed,rustdoc) @@ -237,6 +272,7 @@ rustdoc-pin_init_internal: $(src)/pin-init/internal/src/lib.rs \ rustdoc-pin_init: private rustdoc_host = yes rustdoc-pin_init: private rustc_target_flags = $(pin_init-flags) \ + --extern pin_init_internal=$(objtree)/$(obj)/$(libpin_init_internal_name) \ --extern alloc --cfg feature=\"alloc\" rustdoc-pin_init: $(src)/pin-init/src/lib.rs rustdoc-pin_init_internal \ rustdoc-macros FORCE @@ -245,7 +281,8 @@ rustdoc-pin_init: $(src)/pin-init/src/lib.rs rustdoc-pin_init_internal \ rustdoc-kernel: private is-kernel-object := y rustdoc-kernel: private rustc_target_flags = --extern ffi --extern pin_init \ --extern build_error --extern macros \ - --extern bindings --extern uapi + --extern bindings --extern uapi \ + --extern zerocopy --extern zerocopy_derive rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-ffi rustdoc-macros \ rustdoc-pin_init rustdoc-compiler_builtins $(obj)/$(libmacros_name) \ $(obj)/bindings.o FORCE @@ -256,6 +293,7 @@ rustdoc-clean: FORCE quiet_cmd_rustc_test_library = $(RUSTC_OR_CLIPPY_QUIET) TL $< cmd_rustc_test_library = \ + $(rustc_target_envs) \ OBJTREE=$(abspath $(objtree)) \ $(RUSTC_OR_CLIPPY) $(filter-out $(skip_flags),$(rust_common_flags) $(rustc_target_flags)) \ @$(objtree)/include/generated/rustc_cfg \ @@ -264,6 +302,11 @@ quiet_cmd_rustc_test_library = $(RUSTC_OR_CLIPPY_QUIET) TL $< -L$(objtree)/$(obj)/test \ --crate-name $(subst rusttest-,,$(subst rusttestlib-,,$@)) $< +rusttestlib-zerocopy: private rustc_target_envs := $(zerocopy-envs) +rusttestlib-zerocopy: private rustc_target_flags = $(zerocopy-flags) +rusttestlib-zerocopy: $(src)/zerocopy/src/lib.rs FORCE + +$(call if_changed,rustc_test_library) + rusttestlib-build_error: $(src)/build_error.rs FORCE +$(call if_changed,rustc_test_library) @@ -283,6 +326,12 @@ rusttestlib-syn: private rustc_target_flags = $(syn-flags) rusttestlib-syn: $(src)/syn/lib.rs rusttestlib-quote FORCE +$(call if_changed,rustc_test_library) +rusttestlib-zerocopy_derive: private rustc_target_flags = $(zerocopy_derive-flags) \ + --extern proc_macro +rusttestlib-zerocopy_derive: private rustc_test_library_proc = yes +rusttestlib-zerocopy_derive: $(src)/zerocopy-derive/lib.rs rusttestlib-syn FORCE + +$(call if_changed,rustc_test_library) + rusttestlib-macros: private rustc_target_flags = --extern proc_macro \ --extern proc_macro2 --extern quote --extern syn rusttestlib-macros: private rustc_test_library_proc = yes @@ -304,10 +353,12 @@ rusttestlib-pin_init: $(src)/pin-init/src/lib.rs rusttestlib-macros \ rusttestlib-kernel: private rustc_target_flags = --extern ffi \ --extern build_error --extern macros --extern pin_init \ - --extern bindings --extern uapi + --extern bindings --extern uapi \ + --extern zerocopy=$(objtree)/$(obj)/test/libzerocopy.rlib \ + --extern zerocopy_derive=$(objtree)/$(obj)/test/$(libzerocopy_derive_name) rusttestlib-kernel: $(src)/kernel/lib.rs rusttestlib-bindings rusttestlib-uapi \ rusttestlib-build_error rusttestlib-pin_init $(obj)/$(libmacros_name) \ - $(obj)/bindings.o FORCE + $(obj)/bindings.o rusttestlib-zerocopy rusttestlib-zerocopy_derive FORCE +$(call if_changed,rustc_test_library) rusttestlib-bindings: private rustc_target_flags = --extern ffi --extern pin_init @@ -320,6 +371,7 @@ rusttestlib-uapi: $(src)/uapi/lib.rs rusttestlib-ffi rusttestlib-pin_init FORCE quiet_cmd_rustdoc_test = RUSTDOC T $< cmd_rustdoc_test = \ + $(rustc_target_envs) \ RUST_MODFILE=test.rs \ OBJTREE=$(abspath $(objtree)) \ $(RUSTDOC) --test $(rust_common_flags) \ @@ -334,11 +386,13 @@ quiet_cmd_rustdoc_test_kernel = RUSTDOC TK $< cmd_rustdoc_test_kernel = \ rm -rf $(objtree)/$(obj)/test/doctests/kernel; \ mkdir -p $(objtree)/$(obj)/test/doctests/kernel; \ + $(rustc_target_envs) \ OBJTREE=$(abspath $(objtree)) \ - $(RUSTDOC) --test $(filter-out --remap-path-prefix=% --remap-path-scope=%,$(rust_flags)) \ + $(RUSTDOC) --test $(filter-out --remap-path-scope=%,$(rust_flags)) \ -L$(objtree)/$(obj) --extern ffi --extern pin_init \ --extern kernel --extern build_error --extern macros \ --extern bindings --extern uapi \ + --extern zerocopy --extern zerocopy_derive \ --no-run --crate-name kernel -Zunstable-options \ --sysroot=/dev/null \ $(doctests_modifiers_workaround) \ @@ -356,6 +410,7 @@ quiet_cmd_rustdoc_test_kernel = RUSTDOC TK $< # so for the moment we skip `-Cpanic=abort`. quiet_cmd_rustc_test = $(RUSTC_OR_CLIPPY_QUIET) T $< cmd_rustc_test = \ + $(rustc_target_envs) \ OBJTREE=$(abspath $(objtree)) \ $(RUSTC_OR_CLIPPY) --test $(rust_common_flags) \ @$(objtree)/include/generated/rustc_cfg \ @@ -402,13 +457,24 @@ bindgen_skip_c_flags := -mno-fp-ret-in-387 -mpreferred-stack-boundary=% \ -fstrict-flex-arrays=% -fmin-function-alignment=% \ -fzero-init-padding-bits=% -mno-fdpic \ -fdiagnostics-show-context -fdiagnostics-show-context=% \ - --param=% --param asan-% -fno-isolate-erroneous-paths-dereference + --param=% --param asan-% -fno-isolate-erroneous-paths-dereference \ + -ffixed-r2 -mmultiple -mno-readonly-in-sdata # Derived from `scripts/Makefile.clang`. BINDGEN_TARGET_x86 := x86_64-linux-gnu BINDGEN_TARGET_arm64 := aarch64-linux-gnu BINDGEN_TARGET_arm := arm-linux-gnueabi BINDGEN_TARGET_loongarch := loongarch64-linux-gnusf +BINDGEN_TARGET_s390 := s390x-linux-gnu +# This is only for i386 UM builds, which need the 32-bit target not -m32 +BINDGEN_TARGET_i386 := i386-linux-gnu + +ifdef CONFIG_PPC64 +BINDGEN_TARGET_powerpc := powerpc64le-linux-gnu +else +BINDGEN_TARGET_powerpc := powerpc-linux-gnu +endif + BINDGEN_TARGET_um := $(BINDGEN_TARGET_$(SUBARCH)) BINDGEN_TARGET := $(BINDGEN_TARGET_$(SRCARCH)) @@ -447,22 +513,10 @@ endif # architecture instead of generating `usize`. bindgen_c_flags_final = $(bindgen_c_flags_lto) -fno-builtin -D__BINDGEN__ -# Each `bindgen` release may upgrade the list of Rust target versions. By -# default, the highest stable release in their list is used. Thus we need to set -# a `--rust-target` to avoid future `bindgen` releases emitting code that -# `rustc` may not understand. On top of that, `bindgen` does not support passing -# an unknown Rust target version. -# -# Therefore, the Rust target for `bindgen` can be only as high as the minimum -# Rust version the kernel supports and only as high as the greatest stable Rust -# target supported by the minimum `bindgen` version the kernel supports (that -# is, if we do not test the actual `rustc`/`bindgen` versions running). -# -# Starting with `bindgen` 0.71.0, we will be able to set any future Rust version -# instead, i.e. we will be able to set here our minimum supported Rust version. +# `--rust-target` points to our minimum supported Rust version. quiet_cmd_bindgen = BINDGEN $@ cmd_bindgen = \ - $(BINDGEN) $< $(bindgen_target_flags) --rust-target 1.68 \ + $(BINDGEN) $< $(bindgen_target_flags) --rust-target 1.85 \ --use-core --with-derive-default --ctypes-prefix ffi --no-layout-tests \ --no-debug '.*' --enable-function-attribute-detection \ -o $@ -- $(bindgen_c_flags_final) -DMODULE \ @@ -496,6 +550,16 @@ $(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_extra = ; $(obj)/bindings/bindings_helpers_generated.rs: $(src)/helpers/helpers.c FORCE $(call if_changed_dep,bindgen) +quiet_cmd_rust_helper = HELPER $@ + cmd_rust_helper = \ + $(CC) $(filter-out $(CFLAGS_REMOVE_helpers/helpers.o), $(c_flags)) \ + -c -g0 $< -emit-llvm -o $@ + +$(obj)/helpers/helpers.bc: private part-of-builtin := y +$(obj)/helpers/helpers_module.bc: private part-of-module := y +$(obj)/helpers/helpers.bc $(obj)/helpers/helpers_module.bc: $(src)/helpers/helpers.c FORCE + +$(call if_changed_dep,rust_helper) + rust_exports = $(NM) -p --defined-only $(1) | awk '$$2~/(T|R|D|B)/ && $$3!~/__(pfx|cfi|odr_asan)/ { printf $(2),$$3 }' quiet_cmd_exports = EXPORTS $@ @@ -523,74 +587,93 @@ $(obj)/exports_bindings_generated.h: $(obj)/bindings.o FORCE $(obj)/exports_kernel_generated.h: $(obj)/kernel.o FORCE $(call if_changed,exports) -quiet_cmd_rustc_procmacrolibrary = $(RUSTC_OR_CLIPPY_QUIET) PL $@ +quiet_cmd_rustc_procmacrolibrary = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) PL $@ cmd_rustc_procmacrolibrary = \ + $(rustc_target_envs) \ $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ $(filter-out $(skip_flags),$(rust_common_flags) $(rustc_target_flags)) \ --emit=dep-info=$(depfile) --emit=link=$@ --crate-type rlib -O \ - --out-dir $(objtree)/$(obj) -L$(objtree)/$(obj) \ + --out-dir $(objtree)/$(obj)/host -L$(objtree)/$(obj)/host \ --crate-name $(patsubst lib%.rlib,%,$(notdir $@)) $< -$(obj)/libproc_macro2.rlib: private skip_clippy = 1 -$(obj)/libproc_macro2.rlib: private rustc_target_flags = $(proc_macro2-flags) -$(obj)/libproc_macro2.rlib: $(src)/proc-macro2/lib.rs FORCE +$(obj)/host/libproc_macro2.rlib: private skip_clippy = 1 +$(obj)/host/libproc_macro2.rlib: private rustc_target_flags = $(proc_macro2-flags) +$(obj)/host/libproc_macro2.rlib: $(src)/proc-macro2/lib.rs FORCE +$(call if_changed_dep,rustc_procmacrolibrary) -$(obj)/libquote.rlib: private skip_clippy = 1 -$(obj)/libquote.rlib: private skip_flags = $(quote-skip_flags) -$(obj)/libquote.rlib: private rustc_target_flags = $(quote-flags) -$(obj)/libquote.rlib: $(src)/quote/lib.rs $(obj)/libproc_macro2.rlib FORCE +$(obj)/host/libquote.rlib: private skip_clippy = 1 +$(obj)/host/libquote.rlib: private skip_flags = $(quote-skip_flags) +$(obj)/host/libquote.rlib: private rustc_target_flags = $(quote-flags) +$(obj)/host/libquote.rlib: $(src)/quote/lib.rs $(obj)/host/libproc_macro2.rlib FORCE +$(call if_changed_dep,rustc_procmacrolibrary) -$(obj)/libsyn.rlib: private skip_clippy = 1 -$(obj)/libsyn.rlib: private rustc_target_flags = $(syn-flags) -$(obj)/libsyn.rlib: $(src)/syn/lib.rs $(obj)/libquote.rlib FORCE +$(obj)/host/libsyn.rlib: private skip_clippy = 1 +$(obj)/host/libsyn.rlib: private rustc_target_flags = $(syn-flags) +$(obj)/host/libsyn.rlib: $(src)/syn/lib.rs $(obj)/host/libquote.rlib FORCE +$(call if_changed_dep,rustc_procmacrolibrary) -quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ +quiet_cmd_rustc_procmacro = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) P $@ cmd_rustc_procmacro = \ - $(RUSTC_OR_CLIPPY) $(rust_common_flags) $(rustc_target_flags) \ + $(rustc_target_envs) \ + $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) $(rust_common_flags) $(rustc_target_flags) \ -Clinker-flavor=gcc -Clinker=$(HOSTCC) \ -Clink-args='$(call escsq,$(KBUILD_PROCMACROLDFLAGS))' \ --emit=dep-info=$(depfile) --emit=link=$@ --extern proc_macro \ - --crate-type proc-macro -L$(objtree)/$(obj) \ - --crate-name $(patsubst lib%.$(libmacros_extension),%,$(notdir $@)) \ + --crate-type proc-macro -L$(objtree)/$(obj)/host \ + --crate-name $(patsubst lib%.$(procmacro-extension),%,$(notdir $@)) \ @$(objtree)/include/generated/rustc_cfg $< # Procedural macros can only be used with the `rustc` that compiled it. +$(obj)/$(libzerocopy_derive_name): private skip_clippy = 1 +$(obj)/$(libzerocopy_derive_name): private rustc_target_flags = $(zerocopy_derive-flags) +$(obj)/$(libzerocopy_derive_name): $(src)/zerocopy-derive/lib.rs $(obj)/host/libproc_macro2.rlib \ + $(obj)/host/libquote.rlib $(obj)/host/libsyn.rlib FORCE + +$(call if_changed_dep,rustc_procmacro) + $(obj)/$(libmacros_name): private rustc_target_flags = \ --extern proc_macro2 --extern quote --extern syn -$(obj)/$(libmacros_name): $(src)/macros/lib.rs $(obj)/libproc_macro2.rlib \ - $(obj)/libquote.rlib $(obj)/libsyn.rlib FORCE +$(obj)/$(libmacros_name): $(src)/macros/lib.rs $(obj)/host/libproc_macro2.rlib \ + $(obj)/host/libquote.rlib $(obj)/host/libsyn.rlib FORCE +$(call if_changed_dep,rustc_procmacro) $(obj)/$(libpin_init_internal_name): private rustc_target_flags = $(pin_init_internal-flags) $(obj)/$(libpin_init_internal_name): $(src)/pin-init/internal/src/lib.rs \ - $(obj)/libproc_macro2.rlib $(obj)/libquote.rlib $(obj)/libsyn.rlib FORCE + $(obj)/host/libproc_macro2.rlib $(obj)/host/libquote.rlib $(obj)/host/libsyn.rlib FORCE +$(call if_changed_dep,rustc_procmacro) # `rustc` requires `-Zunstable-options` to use custom target specifications # since Rust 1.95.0 (https://github.com/rust-lang/rust/pull/151534). quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@ cmd_rustc_library = \ + $(rustc_target_envs) \ OBJTREE=$(abspath $(objtree)) \ $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ $(filter-out $(skip_flags),$(rust_flags)) $(rustc_target_flags) \ - --emit=dep-info=$(depfile) --emit=obj=$@ \ + --emit=dep-info=$(depfile) --emit=$(if $(link_helper),llvm-bc=$(patsubst %.o,%.bc,$@),obj=$@) \ --emit=metadata=$(dir $@)$(patsubst %.o,lib%.rmeta,$(notdir $@)) \ --crate-type rlib -L$(objtree)/$(obj) \ --crate-name $(patsubst %.o,%,$(notdir $@)) $< \ --sysroot=/dev/null \ -Zunstable-options \ + $(if $(link_helper),;$(LLVM_LINK) --internalize --suppress-warnings $(patsubst %.o,%.bc,$@) \ + $(obj)/helpers/helpers$(if $(part-of-module),_module).bc -o $(patsubst %.o,%.m.bc,$@); \ + $(CC) $(CLANG_FLAGS) $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS)) \ + $(CC_FLAGS_RUST_INLINE_HELPERS) -Wno-override-module -c $(patsubst %.o,%.m.bc,$@) -o $@ \ + $(cmd_ld_single)) \ $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) \ $(cmd_objtool) rust-analyzer: $(Q)MAKEFLAGS= $(srctree)/scripts/generate_rust_analyzer.py \ --cfgs='core=$(core-cfgs)' $(core-edition) \ + --cfgs='zerocopy=$(zerocopy-cfgs)' \ --cfgs='proc_macro2=$(proc_macro2-cfgs)' \ --cfgs='quote=$(quote-cfgs)' \ --cfgs='syn=$(syn-cfgs)' \ + --cfgs='zerocopy_derive=$(zerocopy_derive-cfgs)' \ + --cfgs='pin_init_internal=$(pin_init_internal-cfgs)' \ + --cfgs='pin_init=$(pin_init-cfgs)' \ + --envs='zerocopy=$(zerocopy-envs)' \ $(realpath $(srctree)) $(realpath $(objtree)) \ $(rustc_sysroot) $(RUST_LIB_SRC) $(if $(KBUILD_EXTMOD),$(srcroot)) \ > rust-project.json @@ -614,6 +697,10 @@ ifneq ($(or $(CONFIG_ARM64),$(and $(CONFIG_RISCV),$(CONFIG_64BIT))),) __ashrti3 \ __ashlti3 __lshrti3 endif +ifdef CONFIG_PPC32 + redirect-intrinsics += \ + __udivdi3 __umoddi3 +endif ifdef CONFIG_MODVERSIONS cmd_gendwarfksyms = $(if $(skip_gendwarfksyms),, \ @@ -662,6 +749,13 @@ $(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*' $(obj)/compiler_builtins.o: $(src)/compiler_builtins.rs $(obj)/core.o FORCE +$(call if_changed_rule,rustc_library) +$(obj)/zerocopy.o: private skip_clippy = 1 +$(obj)/zerocopy.o: private skip_gendwarfksyms = 1 +$(obj)/zerocopy.o: private rustc_target_envs := $(zerocopy-envs) +$(obj)/zerocopy.o: private rustc_target_flags = $(zerocopy-flags) +$(obj)/zerocopy.o: $(src)/zerocopy/src/lib.rs $(obj)/compiler_builtins.o FORCE + +$(call if_changed_rule,rustc_library) + $(obj)/pin_init.o: private skip_gendwarfksyms = 1 $(obj)/pin_init.o: private rustc_target_flags = $(pin_init-flags) $(obj)/pin_init.o: $(src)/pin-init/src/lib.rs $(obj)/compiler_builtins.o \ @@ -697,9 +791,11 @@ $(obj)/uapi.o: $(src)/uapi/lib.rs \ +$(call if_changed_rule,rustc_library) $(obj)/kernel.o: private rustc_target_flags = --extern ffi --extern pin_init \ - --extern build_error --extern macros --extern bindings --extern uapi + --extern build_error --extern macros --extern bindings --extern uapi \ + --extern zerocopy --extern zerocopy_derive $(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/build_error.o $(obj)/pin_init.o \ - $(obj)/$(libmacros_name) $(obj)/bindings.o $(obj)/uapi.o FORCE + $(obj)/$(libmacros_name) $(obj)/bindings.o $(obj)/uapi.o \ + $(obj)/zerocopy.o $(obj)/$(libzerocopy_derive_name) FORCE +$(call if_changed_rule,rustc_library) ifdef CONFIG_JUMP_LABEL @@ -711,4 +807,9 @@ $(obj)/kernel.o: $(obj)/kernel/generated_arch_warn_asm.rs $(obj)/kernel/generate endif endif +ifdef CONFIG_RUST_INLINE_HELPERS +$(obj)/kernel.o: private link_helper = 1 +$(obj)/kernel.o: $(obj)/helpers/helpers.bc +endif + endif # CONFIG_RUST diff --git a/rust/bindgen_parameters b/rust/bindgen_parameters index fd2fd1c3cb9a..8402b0c93545 100644 --- a/rust/bindgen_parameters +++ b/rust/bindgen_parameters @@ -14,15 +14,22 @@ --opaque-type alt_instr --opaque-type x86_msi_data --opaque-type x86_msi_addr_lo - -# `try` is a reserved keyword since Rust 2018; solved in `bindgen` v0.59.2, -# commit 2aed6b021680 ("context: Escape the try keyword properly"). ---opaque-type kunit_try_catch +# s390-only: same packed/align issue as above (E0588). +--opaque-type lowcore +--opaque-type tod_clock +--opaque-type tpi_info +--opaque-type uv_cb.* +--opaque-type uv_secret.* +--opaque-type zpci_fib # If SMP is disabled, `arch_spinlock_t` is defined as a ZST which triggers a Rust # warning. We don't need to peek into it anyway. --opaque-type spinlock +# enums that appear in indirect function calls should specify a cfi type +--newtype-enum lru_status +--with-attribute-custom-enum=lru_status='#[cfi_encoding="10lru_status"]' + # `seccomp`'s comment gets understood as a doctest --no-doc-comments diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 083cc44aa952..4b31aa7f432f 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -29,10 +29,13 @@ #include <linux/hrtimer_types.h> #include <linux/acpi.h> +#include <linux/gpu_buddy.h> #include <drm/drm_device.h> #include <drm/drm_drv.h> #include <drm/drm_file.h> #include <drm/drm_gem.h> +#include <drm/drm_gem_shmem_helper.h> +#include <drm/drm_gpuvm.h> #include <drm/drm_ioctl.h> #include <kunit/test.h> #include <linux/auxiliary_bus.h> @@ -51,16 +54,19 @@ #include <linux/device/faux.h> #include <linux/dma-direction.h> #include <linux/dma-mapping.h> +#include <linux/dma-resv.h> #include <linux/errname.h> #include <linux/ethtool.h> #include <linux/fdtable.h> #include <linux/file.h> #include <linux/firmware.h> +#include <linux/fwctl.h> #include <linux/fs.h> #include <linux/i2c.h> #include <linux/interrupt.h> #include <linux/io-pgtable.h> #include <linux/ioport.h> +#include <linux/iosys-map.h> #include <linux/jiffies.h> #include <linux/jump_label.h> #include <linux/mdio.h> @@ -80,6 +86,7 @@ #include <linux/regulator/consumer.h> #include <linux/sched.h> #include <linux/security.h> +#include <linux/serdev.h> #include <linux/slab.h> #include <linux/sys_soc.h> #include <linux/task_work.h> @@ -88,6 +95,8 @@ #include <linux/wait.h> #include <linux/workqueue.h> #include <linux/xarray.h> +#include <net/genetlink.h> +#include <net/netlink.h> #include <trace/events/rust_sample.h> /* @@ -105,6 +114,7 @@ const size_t RUST_CONST_HELPER_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN; const size_t RUST_CONST_HELPER_ARCH_KMALLOC_MINALIGN = ARCH_KMALLOC_MINALIGN; const size_t RUST_CONST_HELPER_PAGE_SIZE = PAGE_SIZE; +const size_t RUST_CONST_HELPER_GENLMSG_DEFAULT_SIZE = GENLMSG_DEFAULT_SIZE; const gfp_t RUST_CONST_HELPER_GFP_ATOMIC = GFP_ATOMIC; const gfp_t RUST_CONST_HELPER_GFP_KERNEL = GFP_KERNEL; const gfp_t RUST_CONST_HELPER_GFP_KERNEL_ACCOUNT = GFP_KERNEL_ACCOUNT; @@ -146,8 +156,17 @@ const vm_flags_t RUST_CONST_HELPER_VM_MIXEDMAP = VM_MIXEDMAP; const vm_flags_t RUST_CONST_HELPER_VM_HUGEPAGE = VM_HUGEPAGE; const vm_flags_t RUST_CONST_HELPER_VM_NOHUGEPAGE = VM_NOHUGEPAGE; +#if IS_ENABLED(CONFIG_GPU_BUDDY) +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_RANGE_ALLOCATION = GPU_BUDDY_RANGE_ALLOCATION; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_TOPDOWN_ALLOCATION = GPU_BUDDY_TOPDOWN_ALLOCATION; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CONTIGUOUS_ALLOCATION = + GPU_BUDDY_CONTIGUOUS_ALLOCATION; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CLEAR_ALLOCATION = GPU_BUDDY_CLEAR_ALLOCATION; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CLEARED = GPU_BUDDY_CLEARED; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_TRIM_DISABLE = GPU_BUDDY_TRIM_DISABLE; +#endif + #if IS_ENABLED(CONFIG_ANDROID_BINDER_IPC_RUST) #include "../../drivers/android/binder/rust_binder.h" #include "../../drivers/android/binder/rust_binder_events.h" -#include "../../drivers/android/binder/page_range_helper.h" #endif diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index 19f57c5b2fa2..439ab88a5da1 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -9,10 +9,6 @@ //! using this crate. #![no_std] -// See <https://github.com/rust-lang/rust-bindgen/issues/1651>. -#![cfg_attr(test, allow(deref_nullptr))] -#![cfg_attr(test, allow(unaligned_references))] -#![cfg_attr(test, allow(unsafe_op_in_unsafe_fn))] #![allow( clippy::all, missing_docs, @@ -23,13 +19,20 @@ unreachable_pub, unsafe_op_in_unsafe_fn )] +#![feature(cfi_encoding)] #[allow(dead_code)] +#[allow(clippy::as_underscore)] #[allow(clippy::cast_lossless)] #[allow(clippy::ptr_as_ptr)] #[allow(clippy::ref_as_ptr)] #[allow(clippy::undocumented_unsafe_blocks)] -#[cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))] +#[cfg_attr(not(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES), allow(unknown_lints))] +#[allow(unnecessary_transmutes)] +#[cfg_attr( + CONFIG_RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS, + allow(suspicious_runtime_symbol_definitions) +)] mod bindings_raw { use pin_init::{MaybeZeroable, Zeroable}; diff --git a/rust/compiler_builtins.rs b/rust/compiler_builtins.rs index dd16c1dc899c..fc6b54636dd5 100644 --- a/rust/compiler_builtins.rs +++ b/rust/compiler_builtins.rs @@ -97,5 +97,11 @@ define_panicking_intrinsics!("`u64` division/modulo should not be used", { __aeabi_uldivmod, }); +#[cfg(target_arch = "powerpc")] +define_panicking_intrinsics!("`u64` division/modulo should not be used", { + __udivdi3, + __umoddi3, +}); + // NOTE: if you are adding a new intrinsic here, you should also add it to // `redirect-intrinsics` in `rust/Makefile`. diff --git a/rust/exports.c b/rust/exports.c index 587f0e776aba..1b52460b0f4e 100644 --- a/rust/exports.c +++ b/rust/exports.c @@ -16,10 +16,13 @@ #define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym) #include "exports_core_generated.h" -#include "exports_helpers_generated.h" #include "exports_bindings_generated.h" #include "exports_kernel_generated.h" +#ifndef CONFIG_RUST_INLINE_HELPERS +#include "exports_helpers_generated.h" +#endif + // For modules using `rust/build_error.rs`. #ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW EXPORT_SYMBOL_RUST_GPL(rust_build_error); diff --git a/rust/helpers/atomic_ext.c b/rust/helpers/atomic_ext.c index 7d0c2bd340da..c267d5190529 100644 --- a/rust/helpers/atomic_ext.c +++ b/rust/helpers/atomic_ext.c @@ -4,45 +4,39 @@ #include <asm/rwonce.h> #include <linux/atomic.h> -__rust_helper s8 rust_helper_atomic_i8_read(s8 *ptr) -{ - return READ_ONCE(*ptr); +#define GEN_READ_HELPER(tname, type) \ +__rust_helper type rust_helper_atomic_##tname##_read(type *ptr) \ +{ \ + return READ_ONCE(*ptr); \ } -__rust_helper s8 rust_helper_atomic_i8_read_acquire(s8 *ptr) -{ - return smp_load_acquire(ptr); +#define GEN_SET_HELPER(tname, type) \ +__rust_helper void rust_helper_atomic_##tname##_set(type *ptr, type val) \ +{ \ + WRITE_ONCE(*ptr, val); \ } -__rust_helper s16 rust_helper_atomic_i16_read(s16 *ptr) -{ - return READ_ONCE(*ptr); +#define GEN_READ_ACQUIRE_HELPER(tname, type) \ +__rust_helper type rust_helper_atomic_##tname##_read_acquire(type *ptr) \ +{ \ + return smp_load_acquire(ptr); \ } -__rust_helper s16 rust_helper_atomic_i16_read_acquire(s16 *ptr) -{ - return smp_load_acquire(ptr); +#define GEN_SET_RELEASE_HELPER(tname, type) \ +__rust_helper void rust_helper_atomic_##tname##_set_release(type *ptr, type val)\ +{ \ + smp_store_release(ptr, val); \ } -__rust_helper void rust_helper_atomic_i8_set(s8 *ptr, s8 val) -{ - WRITE_ONCE(*ptr, val); -} +#define GEN_READ_SET_HELPERS(tname, type) \ + GEN_READ_HELPER(tname, type) \ + GEN_SET_HELPER(tname, type) \ + GEN_READ_ACQUIRE_HELPER(tname, type) \ + GEN_SET_RELEASE_HELPER(tname, type) \ -__rust_helper void rust_helper_atomic_i8_set_release(s8 *ptr, s8 val) -{ - smp_store_release(ptr, val); -} - -__rust_helper void rust_helper_atomic_i16_set(s16 *ptr, s16 val) -{ - WRITE_ONCE(*ptr, val); -} - -__rust_helper void rust_helper_atomic_i16_set_release(s16 *ptr, s16 val) -{ - smp_store_release(ptr, val); -} +GEN_READ_SET_HELPERS(i8, s8) +GEN_READ_SET_HELPERS(i16, s16) +GEN_READ_SET_HELPERS(ptr, const void *) /* * xchg helpers depend on ARCH_SUPPORTS_ATOMIC_RMW and on the @@ -51,45 +45,22 @@ __rust_helper void rust_helper_atomic_i16_set_release(s16 *ptr, s16 val) * The architectures that currently support Rust (x86_64, armv7, * arm64, riscv, and loongarch) satisfy these requirements. */ -__rust_helper s8 rust_helper_atomic_i8_xchg(s8 *ptr, s8 new) -{ - return xchg(ptr, new); +#define GEN_XCHG_HELPER(tname, type, suffix) \ +__rust_helper type \ +rust_helper_atomic_##tname##_xchg##suffix(type *ptr, type new) \ +{ \ + return xchg##suffix(ptr, new); \ } -__rust_helper s16 rust_helper_atomic_i16_xchg(s16 *ptr, s16 new) -{ - return xchg(ptr, new); -} +#define GEN_XCHG_HELPERS(tname, type) \ + GEN_XCHG_HELPER(tname, type, ) \ + GEN_XCHG_HELPER(tname, type, _acquire) \ + GEN_XCHG_HELPER(tname, type, _release) \ + GEN_XCHG_HELPER(tname, type, _relaxed) \ -__rust_helper s8 rust_helper_atomic_i8_xchg_acquire(s8 *ptr, s8 new) -{ - return xchg_acquire(ptr, new); -} - -__rust_helper s16 rust_helper_atomic_i16_xchg_acquire(s16 *ptr, s16 new) -{ - return xchg_acquire(ptr, new); -} - -__rust_helper s8 rust_helper_atomic_i8_xchg_release(s8 *ptr, s8 new) -{ - return xchg_release(ptr, new); -} - -__rust_helper s16 rust_helper_atomic_i16_xchg_release(s16 *ptr, s16 new) -{ - return xchg_release(ptr, new); -} - -__rust_helper s8 rust_helper_atomic_i8_xchg_relaxed(s8 *ptr, s8 new) -{ - return xchg_relaxed(ptr, new); -} - -__rust_helper s16 rust_helper_atomic_i16_xchg_relaxed(s16 *ptr, s16 new) -{ - return xchg_relaxed(ptr, new); -} +GEN_XCHG_HELPERS(i8, s8) +GEN_XCHG_HELPERS(i16, s16) +GEN_XCHG_HELPERS(ptr, const void *) /* * try_cmpxchg helpers depend on ARCH_SUPPORTS_ATOMIC_RMW and on the @@ -98,42 +69,19 @@ __rust_helper s16 rust_helper_atomic_i16_xchg_relaxed(s16 *ptr, s16 new) * The architectures that currently support Rust (x86_64, armv7, * arm64, riscv, and loongarch) satisfy these requirements. */ -__rust_helper bool rust_helper_atomic_i8_try_cmpxchg(s8 *ptr, s8 *old, s8 new) -{ - return try_cmpxchg(ptr, old, new); -} - -__rust_helper bool rust_helper_atomic_i16_try_cmpxchg(s16 *ptr, s16 *old, s16 new) -{ - return try_cmpxchg(ptr, old, new); -} - -__rust_helper bool rust_helper_atomic_i8_try_cmpxchg_acquire(s8 *ptr, s8 *old, s8 new) -{ - return try_cmpxchg_acquire(ptr, old, new); -} - -__rust_helper bool rust_helper_atomic_i16_try_cmpxchg_acquire(s16 *ptr, s16 *old, s16 new) -{ - return try_cmpxchg_acquire(ptr, old, new); -} - -__rust_helper bool rust_helper_atomic_i8_try_cmpxchg_release(s8 *ptr, s8 *old, s8 new) -{ - return try_cmpxchg_release(ptr, old, new); -} - -__rust_helper bool rust_helper_atomic_i16_try_cmpxchg_release(s16 *ptr, s16 *old, s16 new) -{ - return try_cmpxchg_release(ptr, old, new); -} - -__rust_helper bool rust_helper_atomic_i8_try_cmpxchg_relaxed(s8 *ptr, s8 *old, s8 new) -{ - return try_cmpxchg_relaxed(ptr, old, new); -} - -__rust_helper bool rust_helper_atomic_i16_try_cmpxchg_relaxed(s16 *ptr, s16 *old, s16 new) -{ - return try_cmpxchg_relaxed(ptr, old, new); -} +#define GEN_TRY_CMPXCHG_HELPER(tname, type, suffix) \ +__rust_helper bool \ +rust_helper_atomic_##tname##_try_cmpxchg##suffix(type *ptr, type *old, type new)\ +{ \ + return try_cmpxchg##suffix(ptr, old, new); \ +} + +#define GEN_TRY_CMPXCHG_HELPERS(tname, type) \ + GEN_TRY_CMPXCHG_HELPER(tname, type, ) \ + GEN_TRY_CMPXCHG_HELPER(tname, type, _acquire) \ + GEN_TRY_CMPXCHG_HELPER(tname, type, _release) \ + GEN_TRY_CMPXCHG_HELPER(tname, type, _relaxed) \ + +GEN_TRY_CMPXCHG_HELPERS(i8, s8) +GEN_TRY_CMPXCHG_HELPERS(i16, s16) +GEN_TRY_CMPXCHG_HELPERS(ptr, const void *) diff --git a/rust/helpers/barrier.c b/rust/helpers/barrier.c index fed8853745c8..dbc7a3017c78 100644 --- a/rust/helpers/barrier.c +++ b/rust/helpers/barrier.c @@ -2,6 +2,36 @@ #include <asm/barrier.h> +__rust_helper void rust_helper_mb(void) +{ + mb(); +} + +__rust_helper void rust_helper_rmb(void) +{ + rmb(); +} + +__rust_helper void rust_helper_wmb(void) +{ + wmb(); +} + +__rust_helper void rust_helper_dma_mb(void) +{ + dma_mb(); +} + +__rust_helper void rust_helper_dma_rmb(void) +{ + dma_rmb(); +} + +__rust_helper void rust_helper_dma_wmb(void) +{ + dma_wmb(); +} + __rust_helper void rust_helper_smp_mb(void) { smp_mb(); diff --git a/rust/helpers/clk.c b/rust/helpers/clk.c index 6d04372c9f3b..15fd7e469cdd 100644 --- a/rust/helpers/clk.c +++ b/rust/helpers/clk.c @@ -7,60 +7,62 @@ * CONFIG_HAVE_CLK or CONFIG_HAVE_CLK_PREPARE aren't set. */ #ifndef CONFIG_HAVE_CLK -struct clk *rust_helper_clk_get(struct device *dev, const char *id) +__rust_helper struct clk *rust_helper_clk_get(struct device *dev, + const char *id) { return clk_get(dev, id); } -void rust_helper_clk_put(struct clk *clk) +__rust_helper void rust_helper_clk_put(struct clk *clk) { clk_put(clk); } -int rust_helper_clk_enable(struct clk *clk) +__rust_helper int rust_helper_clk_enable(struct clk *clk) { return clk_enable(clk); } -void rust_helper_clk_disable(struct clk *clk) +__rust_helper void rust_helper_clk_disable(struct clk *clk) { clk_disable(clk); } -unsigned long rust_helper_clk_get_rate(struct clk *clk) +__rust_helper unsigned long rust_helper_clk_get_rate(struct clk *clk) { return clk_get_rate(clk); } -int rust_helper_clk_set_rate(struct clk *clk, unsigned long rate) +__rust_helper int rust_helper_clk_set_rate(struct clk *clk, unsigned long rate) { return clk_set_rate(clk, rate); } #endif #ifndef CONFIG_HAVE_CLK_PREPARE -int rust_helper_clk_prepare(struct clk *clk) +__rust_helper int rust_helper_clk_prepare(struct clk *clk) { return clk_prepare(clk); } -void rust_helper_clk_unprepare(struct clk *clk) +__rust_helper void rust_helper_clk_unprepare(struct clk *clk) { clk_unprepare(clk); } #endif -struct clk *rust_helper_clk_get_optional(struct device *dev, const char *id) +__rust_helper struct clk *rust_helper_clk_get_optional(struct device *dev, + const char *id) { return clk_get_optional(dev, id); } -int rust_helper_clk_prepare_enable(struct clk *clk) +__rust_helper int rust_helper_clk_prepare_enable(struct clk *clk) { return clk_prepare_enable(clk); } -void rust_helper_clk_disable_unprepare(struct clk *clk) +__rust_helper void rust_helper_clk_disable_unprepare(struct clk *clk) { clk_disable_unprepare(clk); } diff --git a/rust/helpers/device.c b/rust/helpers/device.c index a8ab931a9bd1..3be4ee590784 100644 --- a/rust/helpers/device.c +++ b/rust/helpers/device.c @@ -25,3 +25,8 @@ __rust_helper void rust_helper_dev_set_drvdata(struct device *dev, void *data) { dev_set_drvdata(dev, data); } + +__rust_helper const char *rust_helper_dev_name(const struct device *dev) +{ + return dev_name(dev); +} diff --git a/rust/helpers/dma-resv.c b/rust/helpers/dma-resv.c new file mode 100644 index 000000000000..71914d8241e2 --- /dev/null +++ b/rust/helpers/dma-resv.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/dma-resv.h> + +__rust_helper +int rust_helper_dma_resv_lock(struct dma_resv *obj, struct ww_acquire_ctx *ctx) +{ + return dma_resv_lock(obj, ctx); +} + +__rust_helper void rust_helper_dma_resv_unlock(struct dma_resv *obj) +{ + dma_resv_unlock(obj); +} diff --git a/rust/helpers/drm.c b/rust/helpers/drm.c index fe226f7b53ef..65f3f22b0e1d 100644 --- a/rust/helpers/drm.c +++ b/rust/helpers/drm.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <drm/drm_gem.h> +#include <drm/drm_gem_shmem_helper.h> #include <drm/drm_vma_manager.h> #ifdef CONFIG_DRM @@ -21,4 +22,57 @@ rust_helper_drm_vma_node_offset_addr(struct drm_vma_offset_node *node) return drm_vma_node_offset_addr(node); } -#endif +#ifdef CONFIG_DRM_GEM_SHMEM_HELPER +__rust_helper void +rust_helper_drm_gem_shmem_object_free(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_free(obj); +} + +__rust_helper void +rust_helper_drm_gem_shmem_object_print_info(struct drm_printer *p, unsigned int indent, + const struct drm_gem_object *obj) +{ + drm_gem_shmem_object_print_info(p, indent, obj); +} + +__rust_helper int +rust_helper_drm_gem_shmem_object_pin(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_pin(obj); +} + +__rust_helper void +rust_helper_drm_gem_shmem_object_unpin(struct drm_gem_object *obj) +{ + drm_gem_shmem_object_unpin(obj); +} + +__rust_helper struct sg_table * +rust_helper_drm_gem_shmem_object_get_sg_table(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_get_sg_table(obj); +} + +__rust_helper int +rust_helper_drm_gem_shmem_object_vmap(struct drm_gem_object *obj, + struct iosys_map *map) +{ + return drm_gem_shmem_object_vmap(obj, map); +} + +__rust_helper void +rust_helper_drm_gem_shmem_object_vunmap(struct drm_gem_object *obj, + struct iosys_map *map) +{ + drm_gem_shmem_object_vunmap(obj, map); +} + +__rust_helper int +rust_helper_drm_gem_shmem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) +{ + return drm_gem_shmem_object_mmap(obj, vma); +} + +#endif /* CONFIG_DRM_GEM_SHMEM_HELPER */ +#endif /* CONFIG_DRM */ diff --git a/rust/helpers/drm_gpuvm.c b/rust/helpers/drm_gpuvm.c new file mode 100644 index 000000000000..4130b6325213 --- /dev/null +++ b/rust/helpers/drm_gpuvm.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +#ifdef CONFIG_RUST_DRM_GPUVM + +#include <drm/drm_gpuvm.h> + +__rust_helper +struct drm_gpuvm_bo *rust_helper_drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo) +{ + return drm_gpuvm_bo_get(vm_bo); +} + +__rust_helper +struct drm_gpuvm *rust_helper_drm_gpuvm_get(struct drm_gpuvm *obj) +{ + return drm_gpuvm_get(obj); +} + +__rust_helper +bool rust_helper_drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm, + struct drm_gem_object *obj) +{ + return drm_gpuvm_is_extobj(gpuvm, obj); +} + +#endif // CONFIG_RUST_DRM_GPUVM diff --git a/rust/helpers/fwctl.c b/rust/helpers/fwctl.c new file mode 100644 index 000000000000..c7eecd4336a7 --- /dev/null +++ b/rust/helpers/fwctl.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/fwctl.h> + +#if IS_ENABLED(CONFIG_RUST_FWCTL_ABSTRACTIONS) + +__rust_helper struct fwctl_device *rust_helper_fwctl_get(struct fwctl_device *fwctl) +{ + return fwctl_get(fwctl); +} + +__rust_helper void rust_helper_fwctl_put(struct fwctl_device *fwctl) +{ + fwctl_put(fwctl); +} + +#endif diff --git a/rust/helpers/gpu.c b/rust/helpers/gpu.c new file mode 100644 index 000000000000..a25448d54d72 --- /dev/null +++ b/rust/helpers/gpu.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/gpu_buddy.h> + +#ifdef CONFIG_GPU_BUDDY + +__rust_helper u64 rust_helper_gpu_buddy_block_offset(const struct gpu_buddy_block *block) +{ + return gpu_buddy_block_offset(block); +} + +__rust_helper unsigned int rust_helper_gpu_buddy_block_order(struct gpu_buddy_block *block) +{ + return gpu_buddy_block_order(block); +} + +#endif /* CONFIG_GPU_BUDDY */ diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index a3c42e51f00a..440fb7638e3c 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -7,7 +7,36 @@ * Sorted alphabetically. */ +#include <linux/compiler_types.h> + +#ifdef __BINDGEN__ +// Omit `inline` for bindgen as it ignores inline functions. #define __rust_helper +#else +// The helper functions are all inline functions. +// +// We use `__always_inline` here to bypass LLVM inlining checks, in case the +// helpers are inlined directly into Rust CGUs. +// +// The LLVM inlining checks are false positives: +// * LLVM doesn't want to inline functions compiled with +// `-fno-delete-null-pointer-checks` with code compiled without. +// The C CGUs all have this enabled and Rust CGUs don't. Inlining is okay +// since this is one of the hardening features that does not change the ABI, +// and we shouldn't have null pointer dereferences in these helpers. +// * LLVM doesn't want to inline functions with different list of builtins. C +// side has `-fno-builtin-wcslen`; `wcslen` is not a Rust builtin, so they +// should be compatible, but LLVM does not perform inlining due to attributes +// mismatch. +// * clang and Rust doesn't have the exact target string. Clang generates +// `+cmov,+cx8,+fxsr` but Rust doesn't enable them (in fact, Rust will +// complain if `-Ctarget-feature=+cmov,+cx8,+fxsr` is used). x86-64 always +// enable these features, so they are in fact the same target string, but +// LLVM doesn't understand this and so inlining is inhibited. This can be +// bypassed with `--ignore-tti-inline-compatible`, but this is a hidden +// option. +#define __rust_helper __always_inline +#endif #include "atomic.c" #include "atomic_ext.c" @@ -28,16 +57,25 @@ #include "cred.c" #include "device.c" #include "dma.c" +#ifdef CONFIG_DMA_SHARED_BUFFER +#include "dma-resv.c" +#endif #include "drm.c" +#include "drm_gpuvm.c" #include "err.c" -#include "irq.c" #include "fs.c" +#include "fwctl.c" +#include "gpu.c" +#include "interrupt.c" #include "io.c" +#include "irq.c" #include "jump_label.c" #include "kunit.c" +#include "list.c" #include "maple_tree.c" #include "mm.c" #include "mutex.c" +#include "net/genetlink.c" #include "of.c" #include "page.c" #include "pci.c" @@ -53,9 +91,12 @@ #include "regulator.c" #include "scatterlist.c" #include "security.c" +#include "serdev.c" #include "signal.c" #include "slab.c" #include "spinlock.c" +#include "string.c" +#include "srcu.c" #include "sync.c" #include "task.c" #include "time.c" diff --git a/rust/helpers/interrupt.c b/rust/helpers/interrupt.c new file mode 100644 index 000000000000..69595498620f --- /dev/null +++ b/rust/helpers/interrupt.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/spinlock.h> + +__rust_helper void rust_helper_local_interrupt_disable(void) +{ + local_interrupt_disable(); +} + +__rust_helper void rust_helper_local_interrupt_enable(void) +{ + local_interrupt_enable(); +} diff --git a/rust/helpers/io.c b/rust/helpers/io.c index 397810864a24..308950aae19c 100644 --- a/rust/helpers/io.c +++ b/rust/helpers/io.c @@ -3,6 +3,7 @@ #include <linux/io.h> #include <linux/ioport.h> +#ifdef CONFIG_HAS_IOMEM __rust_helper void __iomem *rust_helper_ioremap(phys_addr_t offset, size_t size) { return ioremap(offset, size); @@ -18,6 +19,20 @@ __rust_helper void rust_helper_iounmap(void __iomem *addr) { iounmap(addr); } +#endif /* CONFIG_HAS_IOMEM */ + +__rust_helper void rust_helper_memcpy_fromio(void *dst, + const volatile void __iomem *src, + size_t count) +{ + memcpy_fromio(dst, src, count); +} + +__rust_helper void rust_helper_memcpy_toio(volatile void __iomem *dst, + const void *src, size_t count) +{ + memcpy_toio(dst, src, count); +} __rust_helper u8 rust_helper_readb(const void __iomem *addr) { diff --git a/rust/helpers/jump_label.c b/rust/helpers/jump_label.c index fc1f1e0df08e..7ca384e73121 100644 --- a/rust/helpers/jump_label.c +++ b/rust/helpers/jump_label.c @@ -7,7 +7,7 @@ #include <linux/jump_label.h> #ifndef CONFIG_JUMP_LABEL -int rust_helper_static_key_count(struct static_key *key) +__rust_helper int rust_helper_static_key_count(struct static_key *key) { return static_key_count(key); } diff --git a/rust/helpers/list.c b/rust/helpers/list.c new file mode 100644 index 000000000000..18095a5593c5 --- /dev/null +++ b/rust/helpers/list.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Helpers for C circular doubly linked list implementation. + */ + +#include <linux/list.h> + +__rust_helper void rust_helper_INIT_LIST_HEAD(struct list_head *list) +{ + INIT_LIST_HEAD(list); +} + +__rust_helper void rust_helper_list_add_tail(struct list_head *new, struct list_head *head) +{ + list_add_tail(new, head); +} diff --git a/rust/helpers/net/genetlink.c b/rust/helpers/net/genetlink.c new file mode 100644 index 000000000000..3530b69f6cf7 --- /dev/null +++ b/rust/helpers/net/genetlink.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2026 Google LLC. + */ + +#include <net/genetlink.h> + +#ifdef CONFIG_NET + +__rust_helper struct sk_buff *rust_helper_genlmsg_new(size_t payload, gfp_t flags) +{ + return genlmsg_new(payload, flags); +} + +__rust_helper +int rust_helper_genlmsg_multicast(const struct genl_family *family, + struct sk_buff *skb, u32 portid, + unsigned int group, gfp_t flags) +{ + return genlmsg_multicast(family, skb, portid, group, flags); +} + +__rust_helper void rust_helper_genlmsg_cancel(struct sk_buff *skb, void *hdr) +{ + genlmsg_cancel(skb, hdr); +} + +__rust_helper void rust_helper_genlmsg_end(struct sk_buff *skb, void *hdr) +{ + genlmsg_end(skb, hdr); +} + +__rust_helper void rust_helper_nlmsg_free(struct sk_buff *skb) +{ + nlmsg_free(skb); +} + +__rust_helper +int rust_helper_genl_has_listeners(const struct genl_family *family, + struct net *net, unsigned int group) +{ + return genl_has_listeners(family, net, group); +} + +#endif diff --git a/rust/helpers/pci.c b/rust/helpers/pci.c index e44905317d75..a714cc2bfb7a 100644 --- a/rust/helpers/pci.c +++ b/rust/helpers/pci.c @@ -24,6 +24,19 @@ __rust_helper bool rust_helper_dev_is_pci(const struct device *dev) return dev_is_pci(dev); } +__rust_helper unsigned int rust_helper_pci_irq_type(struct pci_dev *pdev) +{ + return pci_irq_type(pdev); +} + +#ifndef CONFIG_PCI_IOV +__rust_helper unsigned int +rust_helper_pci_sriov_get_totalvfs(struct pci_dev *pdev) +{ + return pci_sriov_get_totalvfs(pdev); +} +#endif + #ifndef CONFIG_PCI_MSI __rust_helper int rust_helper_pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs, diff --git a/rust/helpers/serdev.c b/rust/helpers/serdev.c new file mode 100644 index 000000000000..c52b78ca3fc7 --- /dev/null +++ b/rust/helpers/serdev.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/serdev.h> + +__rust_helper +void rust_helper_serdev_device_driver_unregister(struct serdev_device_driver *sdrv) +{ + serdev_device_driver_unregister(sdrv); +} + +__rust_helper +void rust_helper_serdev_device_put(struct serdev_device *serdev) +{ + serdev_device_put(serdev); +} + +__rust_helper +void rust_helper_serdev_device_set_client_ops(struct serdev_device *serdev, + const struct serdev_device_ops *ops) +{ + serdev_device_set_client_ops(serdev, ops); +} diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c index 4d13062cf253..d53400c15022 100644 --- a/rust/helpers/spinlock.c +++ b/rust/helpers/spinlock.c @@ -36,3 +36,18 @@ __rust_helper void rust_helper_spin_assert_is_held(spinlock_t *lock) { lockdep_assert_held(lock); } + +__rust_helper void rust_helper_spin_lock_irq_disable(spinlock_t *lock) +{ + spin_lock_irq_disable(lock); +} + +__rust_helper void rust_helper_spin_unlock_irq_enable(spinlock_t *lock) +{ + spin_unlock_irq_enable(lock); +} + +__rust_helper int rust_helper_spin_trylock_irq_disable(spinlock_t *lock) +{ + return spin_trylock_irq_disable(lock); +} diff --git a/rust/helpers/srcu.c b/rust/helpers/srcu.c new file mode 100644 index 000000000000..1a2f563640e0 --- /dev/null +++ b/rust/helpers/srcu.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/srcu.h> + +__rust_helper int rust_helper_init_srcu_struct_with_key(struct srcu_struct *ssp, + const char *name, + struct lock_class_key *key) +{ + return __init_srcu_struct(ssp, name, key); +} + +__rust_helper bool rust_helper_srcu_readers_active(struct srcu_struct *ssp) +{ + return srcu_readers_active(ssp); +} + +__rust_helper int rust_helper_srcu_read_lock(struct srcu_struct *ssp) +{ + return srcu_read_lock(ssp); +} + +__rust_helper void rust_helper_srcu_read_unlock(struct srcu_struct *ssp, int idx) +{ + srcu_read_unlock(ssp, idx); +} + +__rust_helper void rust_helper_srcu_barrier(struct srcu_struct *ssp) +{ + srcu_barrier(ssp); +} + +__rust_helper void rust_helper_synchronize_srcu_expedited(struct srcu_struct *ssp) +{ + synchronize_srcu_expedited(ssp); +} diff --git a/rust/helpers/string.c b/rust/helpers/string.c new file mode 100644 index 000000000000..8ef30eb07a15 --- /dev/null +++ b/rust/helpers/string.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/string.h> + +__rust_helper void *rust_helper_memchr(const void *s, int c, size_t n) +{ + return memchr(s, c, n); +} diff --git a/rust/helpers/sync.c b/rust/helpers/sync.c index 82d6aff73b04..4f474fe847c4 100644 --- a/rust/helpers/sync.c +++ b/rust/helpers/sync.c @@ -11,3 +11,8 @@ __rust_helper void rust_helper_lockdep_unregister_key(struct lock_class_key *k) { lockdep_unregister_key(k); } + +__rust_helper void rust_helper_lockdep_assert_irqs_disabled(void) +{ + lockdep_assert_irqs_disabled(); +} diff --git a/rust/helpers/task.c b/rust/helpers/task.c index c0e1a06ede78..b46b1433a67e 100644 --- a/rust/helpers/task.c +++ b/rust/helpers/task.c @@ -28,11 +28,6 @@ __rust_helper kuid_t rust_helper_task_uid(struct task_struct *task) return task_uid(task); } -__rust_helper kuid_t rust_helper_task_euid(struct task_struct *task) -{ - return task_euid(task); -} - #ifndef CONFIG_USER_NS __rust_helper uid_t rust_helper_from_kuid(struct user_namespace *to, kuid_t uid) { diff --git a/rust/helpers/uaccess.c b/rust/helpers/uaccess.c index d9625b9ee046..6e59cc9c665c 100644 --- a/rust/helpers/uaccess.c +++ b/rust/helpers/uaccess.c @@ -14,7 +14,7 @@ rust_helper_copy_to_user(void __user *to, const void *from, unsigned long n) return copy_to_user(to, from, n); } -#ifdef INLINE_COPY_FROM_USER +#ifdef INLINE_COPY_USER __rust_helper unsigned long rust_helper__copy_from_user(void *to, const void __user *from, unsigned long n) { diff --git a/rust/helpers/vmalloc.c b/rust/helpers/vmalloc.c index 326b030487a2..6aed13292313 100644 --- a/rust/helpers/vmalloc.c +++ b/rust/helpers/vmalloc.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include <linux/mm.h> #include <linux/vmalloc.h> __rust_helper void *__must_check __realloc_size(2) @@ -8,3 +9,8 @@ rust_helper_vrealloc_node_align(const void *p, size_t size, unsigned long align, { return vrealloc_node_align(p, size, align, flags, node); } + +__rust_helper bool rust_helper_is_vmalloc_addr(const void *x) +{ + return is_vmalloc_addr(x); +} diff --git a/rust/kernel/Kconfig.test b/rust/kernel/Kconfig.test new file mode 100644 index 000000000000..e6a5c7a795f0 --- /dev/null +++ b/rust/kernel/Kconfig.test @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: GPL-2.0-only +menuconfig RUST_KUNIT_TESTS + bool "Rust KUnit tests" + depends on KUNIT && RUST + default KUNIT_ALL_TESTS + help + This menu collects all options for Rust KUnit tests. + See Documentation/rust/testing.rst for how to protect + unit tests with these options. + + Say Y here to enable Rust KUnit tests. + + If unsure, say N. + +if RUST_KUNIT_TESTS +config RUST_ALLOCATOR_KUNIT_TEST + bool "KUnit tests for Rust allocator API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust allocator API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_KVEC_KUNIT_TEST + bool "KUnit tests for Rust KVec API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust KVec API. + These are only for development and testing, not for + regular kernel use cases. + + If unsure, say N. + +config RUST_BITMAP_KUNIT_TEST + bool "KUnit tests for Rust bitmap API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust bitmap API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_KUNIT_SELFTEST + bool "KUnit selftests for Rust" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit selftests. These are only + for development and testing, not for regular kernel + use cases. + + If unsure, say N. + +config RUST_STR_KUNIT_TEST + bool "KUnit tests for Rust strings API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust strings API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_ATOMICS_KUNIT_TEST + bool "KUnit tests for Rust atomics API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust atomics API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_BITFIELD_KUNIT_TEST + bool "KUnit tests for the Rust `bitfield!` macro" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust `bitfield!` macro. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +endif diff --git a/rust/kernel/acpi.rs b/rust/kernel/acpi.rs index 9b8efa623130..ea2ce61ee393 100644 --- a/rust/kernel/acpi.rs +++ b/rust/kernel/acpi.rs @@ -25,10 +25,6 @@ unsafe impl RawDeviceId for DeviceId { // SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::acpi_device_id, driver_data); - - fn index(&self) -> usize { - self.0.driver_data - } } impl DeviceId { @@ -53,13 +49,7 @@ impl DeviceId { /// Create an ACPI `IdTable` with an "alias" for modpost. #[macro_export] macro_rules! acpi_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::acpi::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("acpi", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("acpi", $crate::acpi::DeviceId, $($tt)*); }; } diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index e38720349dcf..21067bde6860 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -22,8 +22,12 @@ pub use self::kvec::Vec; #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct AllocError; -use crate::error::{code::EINVAL, Result}; -use core::{alloc::Layout, ptr::NonNull}; +use crate::prelude::*; + +use core::{ + alloc::Layout, + ptr::NonNull, // +}; /// Flags to be used when allocating memory. /// diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 63bfb91b3671..cd4203f27aed 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -8,14 +8,25 @@ //! //! Reference: <https://docs.kernel.org/core-api/memory-allocation.html> -use super::Flags; -use core::alloc::Layout; -use core::ptr; -use core::ptr::NonNull; - -use crate::alloc::{AllocError, Allocator, NumaNode}; -use crate::bindings; -use crate::page; +use super::{ + AllocError, + Allocator, + Flags, + NumaNode, // +}; + +use crate::{ + bindings, + page, // +}; + +use core::{ + alloc::Layout, + ptr::{ + self, + NonNull, // + }, // +}; const ARCH_KMALLOC_MINALIGN: usize = bindings::ARCH_KMALLOC_MINALIGN; @@ -163,8 +174,11 @@ impl Vmalloc { /// # Examples /// /// ``` - /// # use core::ptr::{NonNull, from_mut}; - /// # use kernel::{page, prelude::*}; + /// # use core::ptr::{ + /// # from_mut, + /// # NonNull, // + /// # }; + /// # use kernel::page; /// use kernel::alloc::allocator::Vmalloc; /// /// let mut vbox = VBox::<[u8; page::PAGE_SIZE]>::new_uninit(GFP_KERNEL)?; @@ -251,6 +265,7 @@ unsafe impl Allocator for KVmalloc { } } +#[cfg(CONFIG_RUST_ALLOCATOR_KUNIT_TEST)] #[macros::kunit_tests(rust_allocator)] mod tests { use super::*; diff --git a/rust/kernel/alloc/allocator/iter.rs b/rust/kernel/alloc/allocator/iter.rs index 5759f86029b7..02fda3ea5cae 100644 --- a/rust/kernel/alloc/allocator/iter.rs +++ b/rust/kernel/alloc/allocator/iter.rs @@ -1,9 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 use super::Vmalloc; + use crate::page; -use core::marker::PhantomData; -use core::ptr::NonNull; + +use core::{ + marker::PhantomData, + ptr::NonNull, // +}; /// An [`Iterator`] of [`page::BorrowedPage`] items owned by a [`Vmalloc`] allocation. /// @@ -42,15 +46,9 @@ impl<'a> Iterator for VmallocPageIter<'a> { return None; } - // TODO: Use `NonNull::add()` instead, once the minimum supported compiler version is - // bumped to 1.80 or later. - // // SAFETY: `offset` is in the interval `[0, (self.page_count() - 1) * page::PAGE_SIZE]`, // hence the resulting pointer is guaranteed to be within the same allocation. - let ptr = unsafe { self.buf.as_ptr().add(offset) }; - - // SAFETY: `ptr` is guaranteed to be non-null given that it is derived from `self.buf`. - let ptr = unsafe { NonNull::new_unchecked(ptr) }; + let ptr = unsafe { self.buf.add(offset) }; // SAFETY: // - `ptr` is a valid pointer to a `Vmalloc` allocation. diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index 622b3529edfc..c63d6acdbb6f 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -3,24 +3,47 @@ //! Implementation of [`Box`]. #[allow(unused_imports)] // Used in doc comments. -use super::allocator::{KVmalloc, Kmalloc, Vmalloc, VmallocPageIter}; -use super::{AllocError, Allocator, Flags, NumaNode}; -use core::alloc::Layout; -use core::borrow::{Borrow, BorrowMut}; -use core::marker::PhantomData; -use core::mem::ManuallyDrop; -use core::mem::MaybeUninit; -use core::ops::{Deref, DerefMut}; -use core::pin::Pin; -use core::ptr::NonNull; -use core::result::Result; - -use crate::ffi::c_void; -use crate::fmt; -use crate::init::InPlaceInit; -use crate::page::AsPageIter; -use crate::types::ForeignOwnable; -use pin_init::{InPlaceWrite, Init, PinInit, ZeroableOption}; +use super::allocator::{ + KVmalloc, + Kmalloc, + Vmalloc, + VmallocPageIter, // +}; + +use super::{ + AllocError, + Allocator, + Flags, + NumaNode, // +}; + +use crate::{ + fmt, + page::AsPageIter, + prelude::*, + types::ForeignOwnable, // +}; + +use core::{ + alloc::Layout, + borrow::{ + Borrow, + BorrowMut, // + }, + marker::PhantomData, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, + ops::{ + Deref, + DerefMut, // + }, + ptr::NonNull, + result::Result, // +}; + +use pin_init::ZeroableOption; /// The kernel's [`Box`] type -- a heap allocation for a single value of type `T`. /// @@ -77,33 +100,8 @@ use pin_init::{InPlaceWrite, Init, PinInit, ZeroableOption}; /// `self.0` is always properly aligned and either points to memory allocated with `A` or, for /// zero-sized types, is a dangling, well aligned pointer. #[repr(transparent)] -#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))] -pub struct Box<#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, pointee)] T: ?Sized, A: Allocator>( - NonNull<T>, - PhantomData<A>, -); - -// This is to allow coercion from `Box<T, A>` to `Box<U, A>` if `T` can be converted to the -// dynamically-sized type (DST) `U`. -#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))] -impl<T, U, A> core::ops::CoerceUnsized<Box<U, A>> for Box<T, A> -where - T: ?Sized + core::marker::Unsize<U>, - U: ?Sized, - A: Allocator, -{ -} - -// This is to allow `Box<U, A>` to be dispatched on when `Box<T, A>` can be coerced into `Box<U, -// A>`. -#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))] -impl<T, U, A> core::ops::DispatchFromDyn<Box<U, A>> for Box<T, A> -where - T: ?Sized + core::marker::Unsize<U>, - U: ?Sized, - A: Allocator, -{ -} +#[derive(core::marker::CoercePointee)] +pub struct Box<#[pointee] T: ?Sized, A: Allocator>(NonNull<T>, PhantomData<A>); /// Type alias for [`Box`] with a [`Kmalloc`] allocator. /// @@ -281,6 +279,27 @@ where Ok(Box(ptr.cast(), PhantomData)) } + /// Creates a new zero-initialized `Box<T, A>`. + /// + /// New memory is allocated with `A` and the [`__GFP_ZERO`] flag. The allocation may fail, in + /// which case an error is returned. For ZSTs no memory is allocated. + /// + /// # Examples + /// + /// ``` + /// let b = KBox::<[u8; 128]>::zeroed(GFP_KERNEL)?; + /// assert_eq!(*b, [0; 128]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn zeroed(flags: Flags) -> Result<Self, AllocError> + where + T: Zeroable, + { + // SAFETY: `__GFP_ZERO` guarantees the memory is zeroed; `T: Zeroable` guarantees that + // all-zeroes is a valid bit pattern for `T`. + Ok(unsafe { Self::new_uninit(flags | __GFP_ZERO)?.assume_init() }) + } + /// Constructs a new `Pin<Box<T, A>>`. If `T` does not implement [`Unpin`], then `x` will be /// pinned in memory and can't be moved. #[inline] @@ -299,7 +318,10 @@ where /// # Examples /// /// ``` - /// use kernel::sync::{new_spinlock, SpinLock}; + /// use kernel::sync::{ + /// new_spinlock, + /// SpinLock, // + /// }; /// /// struct Inner { /// a: u32, @@ -350,13 +372,13 @@ where // - `ptr` is a valid pointer to uninitialized memory. // - `ptr` is not used if an error is returned. // - `ptr` won't be moved until it is dropped, i.e. it is pinned. - unsafe { init(i).__pinned_init(ptr)? }; + unsafe { pin_init::raw_try_init(ptr, init(i))? }; // SAFETY: // - `i + 1 <= len`, hence we don't exceed the capacity, due to the call to // `with_capacity()` above. // - The new value at index buffer.len() + 1 is the only element being added here, and - // it has been initialized above by `init(i).__pinned_init(ptr)`. + // it has been initialized above by `raw_try_init(ptr, i)`. unsafe { buffer.inc_len(1) }; } @@ -436,20 +458,22 @@ where { type Initialized = Box<T, A>; + #[inline] fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid. - unsafe { init.__init(slot)? }; + unsafe { pin_init::raw_try_init(slot, init)? }; // SAFETY: All fields have been initialized. Ok(unsafe { Box::assume_init(self) }) } + #[inline] fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; + unsafe { pin_init::raw_try_init(slot, init)? }; // SAFETY: All fields have been initialized. Ok(unsafe { Box::assume_init(self) }.into()) } @@ -480,7 +504,7 @@ where // SAFETY: The pointer returned by `into_foreign` comes from a well aligned // pointer to `T` allocated by `A`. -unsafe impl<T: 'static, A> ForeignOwnable for Box<T, A> +unsafe impl<T, A> ForeignOwnable for Box<T, A> where A: Allocator, { @@ -490,8 +514,14 @@ where core::mem::align_of::<T>() }; - type Borrowed<'a> = &'a T; - type BorrowedMut<'a> = &'a mut T; + type Borrowed<'a> + = &'a T + where + Self: 'a; + type BorrowedMut<'a> + = &'a mut T + where + Self: 'a; fn into_foreign(self) -> *mut c_void { Box::into_raw(self).cast() @@ -519,13 +549,19 @@ where // SAFETY: The pointer returned by `into_foreign` comes from a well aligned // pointer to `T` allocated by `A`. -unsafe impl<T: 'static, A> ForeignOwnable for Pin<Box<T, A>> +unsafe impl<T, A> ForeignOwnable for Pin<Box<T, A>> where A: Allocator, { const FOREIGN_ALIGN: usize = <Box<T, A> as ForeignOwnable>::FOREIGN_ALIGN; - type Borrowed<'a> = Pin<&'a T>; - type BorrowedMut<'a> = Pin<&'a mut T>; + type Borrowed<'a> + = Pin<&'a T> + where + Self: 'a; + type BorrowedMut<'a> + = Pin<&'a mut T> + where + Self: 'a; fn into_foreign(self) -> *mut c_void { // SAFETY: We are still treating the box as pinned. @@ -592,7 +628,6 @@ where /// /// ``` /// # use core::borrow::Borrow; -/// # use kernel::alloc::KBox; /// struct Foo<B: Borrow<u32>>(B); /// /// // Owned instance. @@ -620,7 +655,6 @@ where /// /// ``` /// # use core::borrow::BorrowMut; -/// # use kernel::alloc::KBox; /// struct Foo<B: BorrowMut<u32>>(B); /// /// // Owned instance. @@ -685,9 +719,13 @@ where /// # Examples /// /// ``` -/// # use kernel::prelude::*; -/// use kernel::alloc::allocator::VmallocPageIter; -/// use kernel::page::{AsPageIter, PAGE_SIZE}; +/// use kernel::{ +/// alloc::allocator::VmallocPageIter, +/// page::{ +/// AsPageIter, +/// PAGE_SIZE, // +/// }, // +/// }; /// /// let mut vbox = VBox::new((), GFP_KERNEL)?; /// diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index ac8d6f763ae8..c7546b9da4fa 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -3,28 +3,57 @@ //! Implementation of [`Vec`]. use super::{ - allocator::{KVmalloc, Kmalloc, Vmalloc, VmallocPageIter}, + allocator::{ + KVmalloc, + Kmalloc, + Vmalloc, + VmallocPageIter, // + }, + flags::__GFP_ZERO, layout::ArrayLayout, - AllocError, Allocator, Box, Flags, NumaNode, + AllocError, + Allocator, + Box, + Flags, + NumaNode, // }; + use crate::{ fmt, - page::AsPageIter, // + page::{ + AsPageIter, + PAGE_SIZE, // + }, // }; + use core::{ - borrow::{Borrow, BorrowMut}, + borrow::{ + Borrow, + BorrowMut, // + }, marker::PhantomData, - mem::{ManuallyDrop, MaybeUninit}, - ops::Deref, - ops::DerefMut, - ops::Index, - ops::IndexMut, - ptr, - ptr::NonNull, - slice, - slice::SliceIndex, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, + ops::{ + Deref, + DerefMut, + Index, + IndexMut, // + }, + ptr::{ + self, + NonNull, // + }, + slice::{ + self, + SliceIndex, // + }, // }; +use pin_init::Zeroable; + mod errors; pub use self::errors::{InsertError, PushError, RemoveError}; @@ -506,6 +535,30 @@ where Ok(v) } + /// Creates a new [`Vec`] with `n` zero-initialized elements. + /// + /// # Examples + /// + /// ``` + /// let v = KVec::<u32>::zeroed(20, GFP_KERNEL)?; + /// + /// assert!(v.iter().all(|&x| x == 0)); + /// # Ok::<(), Error>(()) + /// ``` + pub fn zeroed(n: usize, flags: Flags) -> Result<Self, AllocError> + where + T: Zeroable, + { + let mut v = Self::with_capacity(n, flags | __GFP_ZERO)?; + + // SAFETY: + // - `n <= capacity - len`: `with_capacity(n)` guarantees capacity >= n, len is 0. + // - All elements in `[0, n)` are initialized: `__GFP_ZERO` zeroes the allocation, + // and `T: Zeroable` guarantees all-zeroes is a valid bit pattern. + unsafe { v.inc_len(n) }; + Ok(v) + } + /// Creates a `Vec<T, A>` from a pointer, a length and a capacity using the allocator `A`. /// /// # Examples @@ -611,7 +664,7 @@ where /// /// v.reserve(10, GFP_KERNEL)?; /// let cap = v.capacity(); - /// assert!(cap >= 10); + /// assert!(cap >= v.len() + 10); /// /// v.reserve(10, GFP_KERNEL)?; /// let new_cap = v.capacity(); @@ -734,9 +787,136 @@ where self.truncate(num_kept); } } +// TODO: This is a temporary KVVec-specific implementation. It should be replaced with a generic +// `shrink_to()` for `impl<T, A: Allocator> Vec<T, A>` that uses `A::realloc()` once the +// underlying allocators properly support shrinking via realloc. +impl<T> Vec<T, KVmalloc> { + /// Shrinks the capacity of the vector with a lower bound. + /// + /// The capacity will remain at least as large as both the length and the supplied value. + /// If the current capacity is less than the lower limit, this is a no-op. + /// + /// For `kmalloc` allocations, this delegates to `realloc()`, which decides whether + /// shrinking is worthwhile. For `vmalloc` allocations, shrinking only occurs if the + /// operation would free at least one page of memory, and performs a deep copy since + /// `vrealloc` does not yet support in-place shrinking. + /// + /// # Examples + /// + /// ``` + /// // Allocate enough capacity to span multiple pages. + /// let elements_per_page = kernel::page::PAGE_SIZE / core::mem::size_of::<u32>(); + /// let mut v = KVVec::with_capacity(elements_per_page * 4, GFP_KERNEL)?; + /// v.push(1, GFP_KERNEL)?; + /// v.push(2, GFP_KERNEL)?; + /// + /// v.shrink_to(0, GFP_KERNEL)?; + /// # Ok::<(), Error>(()) + /// ``` + pub fn shrink_to(&mut self, min_capacity: usize, flags: Flags) -> Result<(), AllocError> { + let target_cap = core::cmp::max(self.len(), min_capacity); + + if self.capacity() <= target_cap { + return Ok(()); + } + + if Self::is_zst() { + return Ok(()); + } + + // For kmalloc allocations, delegate to realloc() and let the allocator decide + // whether shrinking is worthwhile. + // + // SAFETY: `self.ptr` points to a valid `KVmalloc` allocation. + if !unsafe { bindings::is_vmalloc_addr(self.ptr.as_ptr().cast()) } { + let new_layout = ArrayLayout::<T>::new(target_cap).map_err(|_| AllocError)?; + + // SAFETY: + // - `self.ptr` is valid and was previously allocated with `KVmalloc`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + let ptr = unsafe { + KVmalloc::realloc( + Some(self.ptr.cast()), + new_layout.into(), + self.layout.into(), + flags, + NumaNode::NO_NODE, + )? + }; + + self.ptr = ptr.cast(); + self.layout = new_layout; + return Ok(()); + } + + // Only shrink if we would free at least one page. + let current_size = self.capacity() * core::mem::size_of::<T>(); + let target_size = target_cap * core::mem::size_of::<T>(); + let current_pages = current_size.div_ceil(PAGE_SIZE); + let target_pages = target_size.div_ceil(PAGE_SIZE); + + if current_pages <= target_pages { + return Ok(()); + } + + if target_cap == 0 { + if !self.layout.is_empty() { + // SAFETY: + // - `self.ptr` was previously allocated with `KVmalloc`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + unsafe { KVmalloc::free(self.ptr.cast(), self.layout.into()) }; + } + self.ptr = NonNull::dangling(); + self.layout = ArrayLayout::empty(); + return Ok(()); + } + + // SAFETY: `target_cap <= self.capacity()` and original capacity was valid. + let new_layout = unsafe { ArrayLayout::<T>::new_unchecked(target_cap) }; + + let new_ptr = KVmalloc::alloc(new_layout.into(), flags, NumaNode::NO_NODE)?; + + // SAFETY: + // - `self.as_ptr()` is valid for reads of `self.len()` elements of `T`. + // - `new_ptr` is valid for writes of at least `target_cap >= self.len()` elements. + // - The two allocations do not overlap since `new_ptr` is freshly allocated. + // - Both pointers are properly aligned for `T`. + unsafe { + ptr::copy_nonoverlapping(self.as_ptr(), new_ptr.as_ptr().cast::<T>(), self.len()) + }; + + // SAFETY: + // - `self.ptr` was previously allocated with `KVmalloc`. + // - `self.layout` matches the `ArrayLayout` of the preceding allocation. + unsafe { KVmalloc::free(self.ptr.cast(), self.layout.into()) }; + + self.ptr = new_ptr.cast::<T>(); + self.layout = new_layout; + + Ok(()) + } +} impl<T: Clone, A: Allocator> Vec<T, A> { /// Extend the vector by `n` clones of `value`. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.extend_with(3, 5, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 5, 5, 5]); + /// + /// v.extend_with(2, 8, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 5, 5, 5, 8, 8]); + /// + /// v.extend_with(0, 3, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 5, 5, 5, 8, 8]); + /// + /// # Ok::<(), Error>(()) + /// ``` pub fn extend_with(&mut self, n: usize, value: T, flags: Flags) -> Result<(), AllocError> { if n == 0 { return Ok(()); @@ -754,7 +934,7 @@ impl<T: Clone, A: Allocator> Vec<T, A> { spare[n - 1].write(value); // SAFETY: - // - `self.len() + n < self.capacity()` due to the call to reserve above, + // - `self.len() + n <= self.capacity()` due to the call to reserve above, // - the loop and the line above initialized the next `n` elements. unsafe { self.inc_len(n) }; @@ -1034,9 +1214,13 @@ where /// # Examples /// /// ``` -/// # use kernel::prelude::*; -/// use kernel::alloc::allocator::VmallocPageIter; -/// use kernel::page::{AsPageIter, PAGE_SIZE}; +/// use kernel::{ +/// alloc::allocator::VmallocPageIter, +/// page::{ +/// AsPageIter, +/// PAGE_SIZE, // +/// }, // +/// }; /// /// let mut vec = VVec::<u8>::new(); /// @@ -1351,6 +1535,7 @@ impl<'vec, T> Drop for DrainAll<'vec, T> { } } +#[cfg(CONFIG_RUST_KVEC_KUNIT_TEST)] #[macros::kunit_tests(rust_kvec)] mod tests { use super::*; @@ -1398,4 +1583,106 @@ mod tests { func.push_within_capacity(false).unwrap(); } } + + #[test] + fn test_kvvec_shrink_to() { + use crate::page::PAGE_SIZE; + + // Create a vector with capacity spanning multiple pages. + let mut v = KVVec::<u8>::with_capacity(PAGE_SIZE * 4, GFP_KERNEL).unwrap(); + + // Add a few elements. + v.push(1, GFP_KERNEL).unwrap(); + v.push(2, GFP_KERNEL).unwrap(); + v.push(3, GFP_KERNEL).unwrap(); + + let initial_capacity = v.capacity(); + assert!(initial_capacity >= PAGE_SIZE * 4); + + // Shrink to a capacity that would free at least one page. + v.shrink_to(PAGE_SIZE, GFP_KERNEL).unwrap(); + + // Capacity should have been reduced. + assert!(v.capacity() < initial_capacity); + assert!(v.capacity() >= PAGE_SIZE); + + // Elements should be preserved. + assert_eq!(v.len(), 3); + assert_eq!(v[0], 1); + assert_eq!(v[1], 2); + assert_eq!(v[2], 3); + + // Shrink to zero (should shrink to len). + v.shrink_to(0, GFP_KERNEL).unwrap(); + + // Capacity should be at least the length. + assert!(v.capacity() >= v.len()); + + // Elements should still be preserved. + assert_eq!(v.len(), 3); + assert_eq!(v[0], 1); + assert_eq!(v[1], 2); + assert_eq!(v[2], 3); + } + + #[test] + fn test_kvvec_shrink_to_empty() { + use crate::page::PAGE_SIZE; + + // Create a vector with large capacity but no elements. + let mut v = KVVec::<u8>::with_capacity(PAGE_SIZE * 4, GFP_KERNEL).unwrap(); + + assert!(v.is_empty()); + + // Shrink empty vector to zero. + v.shrink_to(0, GFP_KERNEL).unwrap(); + + // Should have freed the allocation. + assert_eq!(v.capacity(), 0); + assert!(v.is_empty()); + } + + #[test] + fn test_kvvec_shrink_to_no_op() { + use crate::page::PAGE_SIZE; + + // Create a small vector. + let mut v = KVVec::<u8>::with_capacity(PAGE_SIZE, GFP_KERNEL).unwrap(); + v.push(1, GFP_KERNEL).unwrap(); + + let capacity_before = v.capacity(); + + // Try to shrink to a capacity larger than current - should be no-op. + v.shrink_to(capacity_before + 100, GFP_KERNEL).unwrap(); + + assert_eq!(v.capacity(), capacity_before); + assert_eq!(v.len(), 1); + assert_eq!(v[0], 1); + } + + #[test] + fn test_kvvec_shrink_to_respects_min_capacity() { + use crate::page::PAGE_SIZE; + + // Create a vector with large capacity. + let mut v = KVVec::<u8>::with_capacity(PAGE_SIZE * 4, GFP_KERNEL).unwrap(); + + // Add some elements. + for i in 0..10u8 { + v.push(i, GFP_KERNEL).unwrap(); + } + + // Shrink to a min_capacity larger than length. + let min_cap = PAGE_SIZE * 2; + v.shrink_to(min_cap, GFP_KERNEL).unwrap(); + + // Capacity should be at least min_capacity. + assert!(v.capacity() >= min_cap); + + // All elements preserved. + assert_eq!(v.len(), 10); + for i in 0..10u8 { + assert_eq!(v[i as usize], i); + } + } } diff --git a/rust/kernel/alloc/kvec/errors.rs b/rust/kernel/alloc/kvec/errors.rs index e7de5049ee47..aaca6446516a 100644 --- a/rust/kernel/alloc/kvec/errors.rs +++ b/rust/kernel/alloc/kvec/errors.rs @@ -2,8 +2,10 @@ //! Errors for the [`Vec`] type. -use kernel::fmt; -use kernel::prelude::*; +use crate::{ + fmt, + prelude::*, // +}; /// Error type for [`Vec::push_within_capacity`]. pub struct PushError<T>(pub T); @@ -15,6 +17,7 @@ impl<T> fmt::Debug for PushError<T> { } impl<T> From<PushError<T>> for Error { + #[inline] fn from(_: PushError<T>) -> Error { // Returning ENOMEM isn't appropriate because the system is not out of memory. The vector // is just full and we are refusing to resize it. @@ -32,6 +35,7 @@ impl fmt::Debug for RemoveError { } impl From<RemoveError> for Error { + #[inline] fn from(_: RemoveError) -> Error { EINVAL } @@ -55,6 +59,7 @@ impl<T> fmt::Debug for InsertError<T> { } impl<T> From<InsertError<T>> for Error { + #[inline] fn from(_: InsertError<T>) -> Error { EINVAL } diff --git a/rust/kernel/alloc/layout.rs b/rust/kernel/alloc/layout.rs index 9f8be72feb7a..62a459c66baf 100644 --- a/rust/kernel/alloc/layout.rs +++ b/rust/kernel/alloc/layout.rs @@ -4,7 +4,10 @@ //! //! Custom layout types extending or improving [`Layout`]. -use core::{alloc::Layout, marker::PhantomData}; +use core::{ + alloc::Layout, + marker::PhantomData, // +}; /// Error when constructing an [`ArrayLayout`]. pub struct LayoutError; @@ -47,7 +50,10 @@ impl<T> ArrayLayout<T> { /// # Examples /// /// ``` - /// # use kernel::alloc::layout::{ArrayLayout, LayoutError}; + /// # use kernel::alloc::layout::{ + /// # ArrayLayout, + /// # LayoutError, // + /// # }; /// let layout = ArrayLayout::<i32>::new(15)?; /// assert_eq!(layout.len(), 15); /// diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs index 93c0db1f6655..60dfbec8f330 100644 --- a/rust/kernel/auxiliary.rs +++ b/rust/kernel/auxiliary.rs @@ -12,19 +12,26 @@ use crate::{ RawDeviceId, RawDeviceIdIndex, // }, - devres::Devres, + driver, error::{ from_result, to_result, // }, prelude::*, - types::Opaque, + types::{ + CovariantForLt, + ForLt, + ForeignOwnable, + Opaque, // + }, ThisModule, // }; use core::{ + any::TypeId, marker::PhantomData, mem::offset_of, + pin::Pin, ptr::{ addr_of_mut, NonNull, // @@ -36,18 +43,18 @@ pub struct Adapter<T: Driver>(T); // SAFETY: // - `bindings::auxiliary_driver` is a C type declared as `repr(C)`. -// - `T` is the type of the driver's device private data. +// - `T::Data` is the type of the driver's device private data. // - `struct auxiliary_driver` embeds a `struct device_driver`. // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. -unsafe impl<T: Driver + 'static> driver::DriverLayout for Adapter<T> { +unsafe impl<T: Driver> driver::DriverLayout for Adapter<T> { type DriverType = bindings::auxiliary_driver; - type DriverData = T; + type DriverData<'bound> = T::Data<'bound>; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); } // SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if // a preceding call to `register` has been successful. -unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { +unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { unsafe fn register( adrv: &Opaque<Self::DriverType>, name: &'static CStr, @@ -63,7 +70,7 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { // SAFETY: `adrv` is guaranteed to be a valid `DriverType`. to_result(unsafe { - bindings::__auxiliary_driver_register(adrv.get(), module.0, name.as_char_ptr()) + bindings::__auxiliary_driver_register(adrv.get(), module.as_ptr(), name.as_char_ptr()) }) } @@ -73,7 +80,7 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { } } -impl<T: Driver + 'static> Adapter<T> { +impl<T: Driver> Adapter<T> { extern "C" fn probe_callback( adev: *mut bindings::auxiliary_device, id: *const bindings::auxiliary_device_id, @@ -82,12 +89,14 @@ impl<T: Driver + 'static> Adapter<T> { // `struct auxiliary_device`. // // INVARIANT: `adev` is valid for the duration of `probe_callback()`. - let adev = unsafe { &*adev.cast::<Device<device::CoreInternal>>() }; + let adev = unsafe { &*adev.cast::<Device<device::CoreInternal<'_>>>() }; // SAFETY: `DeviceId` is a `#[repr(transparent)`] wrapper of `struct auxiliary_device_id` // and does not add additional invariants, so it's safe to transmute. let id = unsafe { &*id.cast::<DeviceId>() }; - let info = T::ID_TABLE.info(id.index()); + + // SAFETY: `id` comes from `T::ID_TABLE` which is of type `IdArray<_, T::IdInfo>`. + let info = unsafe { id.info_unchecked::<T::IdInfo>() }; from_result(|| { let data = T::probe(adev, info); @@ -102,12 +111,12 @@ impl<T: Driver + 'static> Adapter<T> { // `struct auxiliary_device`. // // INVARIANT: `adev` is valid for the duration of `remove_callback()`. - let adev = unsafe { &*adev.cast::<Device<device::CoreInternal>>() }; + let adev = unsafe { &*adev.cast::<Device<device::CoreInternal<'_>>>() }; // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called - // and stored a `Pin<KBox<T>>`. - let data = unsafe { adev.as_ref().drvdata_borrow::<T>() }; + // and stored a `Pin<KBox<T::Data<'_>>>`. + let data = unsafe { adev.as_ref().drvdata_borrow::<T::Data<'_>>() }; T::unbind(adev, data); } @@ -163,10 +172,6 @@ unsafe impl RawDeviceId for DeviceId { unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::auxiliary_device_id, driver_data); - - fn index(&self) -> usize { - self.0.driver_data - } } /// IdTable type for auxiliary drivers. @@ -175,14 +180,8 @@ pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; /// Create a auxiliary `IdTable` with its alias for modpost. #[macro_export] macro_rules! auxiliary_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::auxiliary::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("auxiliary", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("auxiliary", $crate::auxiliary::DeviceId, $($tt)*); }; } @@ -197,13 +196,19 @@ pub trait Driver { /// type IdInfo: 'static = (); type IdInfo: 'static; + /// The type of the driver's bus device private data. + type Data<'bound>: Send + 'bound; + /// The table of device ids supported by the driver. const ID_TABLE: IdTable<Self::IdInfo>; /// Auxiliary driver probe. /// /// Called when an auxiliary device is matches a corresponding driver. - fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> impl PinInit<Self, Error>; + fn probe<'bound>( + dev: &'bound Device<device::Core<'_>>, + id_info: &'bound Self::IdInfo, + ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound; /// Auxiliary driver unbind. /// @@ -214,8 +219,8 @@ pub trait Driver { /// `&Device<Core>` or `&Device<Bound>` reference. For instance, drivers may try to perform I/O /// operations to gracefully tear down the device. /// - /// Otherwise, release operations for driver resources should be performed in `Self::drop`. - fn unbind(dev: &Device<device::Core>, this: Pin<&Self>) { + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(dev: &'bound Device<device::Core<'_>>, this: Pin<&Self::Data<'bound>>) { let _ = (dev, this); } } @@ -257,6 +262,88 @@ impl Device<device::Bound> { // SAFETY: A bound auxiliary device always has a bound parent device. unsafe { parent.as_bound() } } + + /// Returns the stored registration data as a pinned reference. + /// + /// Performs null and [`TypeId`] checks, then borrows the stored [`KBox`]. + /// + /// # Safety + /// + /// Callers must ensure that the lifetime shortening from the original `'static` storage to + /// `'_` is sound, e.g. via an HRTB closure or [`CovariantForLt`] guarantee. + unsafe fn registration_data_pinned<F: ForLt + 'static>(&self) -> Result<Pin<&F::Of<'_>>> { + // SAFETY: By the type invariant, `self.as_raw()` is a valid `struct auxiliary_device`. + let ptr = unsafe { (*self.as_raw()).registration_data_rust }; + if ptr.is_null() { + dev_warn!( + self.as_ref(), + "No registration data set; parent is not a Rust driver.\n" + ); + return Err(ENOENT); + } + + // SAFETY: `ptr` is non-null and was set via `into_foreign()` in `Registration::new()`; + // `RegistrationData` is `#[repr(C)]` with `type_id` at offset 0, so reading a `TypeId` + // at the start of the allocation is valid regardless of `F`. + let type_id = unsafe { ptr.cast::<TypeId>().read() }; + if type_id != TypeId::of::<F>() { + return Err(EINVAL); + } + + // SAFETY: The `TypeId` check above confirms that the stored type matches `F`'s + // encoding; lifetimes are erased at runtime, so borrowing as `F::Of<'_>` is + // layout-compatible with the stored `F::Of<'static>`. `ptr` remains valid until + // `Registration::drop()` calls `from_foreign()`. + let wrapper = unsafe { Pin::<KBox<RegistrationData<F::Of<'_>>>>::borrow(ptr) }; + + // SAFETY: `data` is a structurally pinned field of `RegistrationData`. + Ok(unsafe { wrapper.map_unchecked(|w| &w.data) }) + } + + /// Access the registration data set by the registering (parent) driver through a closure. + /// + /// `F` is the [`ForLt`](trait@ForLt) encoding of the data type. The closure receives a pinned + /// reference to the registration data. + /// + /// For covariant types that implement [`trait@CovariantForLt`], prefer + /// [`registration_data`](Self::registration_data) which returns a direct reference. + /// + /// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling + /// [`Registration::new()`]. + /// + /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was + /// registered by a C driver. + #[inline] + pub fn registration_data_with<F: ForLt + 'static, R>( + &self, + f: impl for<'a> FnOnce(Pin<&'a F::Of<'a>>) -> R, + ) -> Result<R> { + // SAFETY: The HRTB closure prevents the caller from smuggling in references with a + // concrete short lifetime, making the round-trip from `'static` sound regardless of + // variance. + let pinned = unsafe { self.registration_data_pinned::<F>()? }; + + Ok(f(pinned)) + } + + /// Returns a pinned reference to the registration data set by the registering (parent) driver. + /// + /// This method is only available when `F` implements [`trait@CovariantForLt`], which guarantees + /// that the lifetime shortening is sound. + /// + /// For non-covariant types, use the closure-based [`Self::registration_data_with`]. + /// + /// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling + /// [`Registration::new()`]. + /// + /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was + /// registered by a C driver. + #[inline] + pub fn registration_data<F: CovariantForLt + 'static>(&self) -> Result<Pin<&F::Of<'_>>> { + // SAFETY: `CovariantForLt` guarantees covariance, which makes the lifetime shortening + // from `'static` to `'_` performed by `registration_data_pinned` sound. + unsafe { self.registration_data_pinned::<F>() } + } } impl Device { @@ -326,87 +413,175 @@ unsafe impl Send for Device {} // (i.e. `Device<Normal>) are thread safe. unsafe impl Sync for Device {} +// SAFETY: Same as `Device<Normal>` -- the underlying `struct auxiliary_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device<device::Bound> {} + +/// Wrapper that stores a [`TypeId`] alongside the registration data for runtime type checking. +#[repr(C)] +#[pin_data] +struct RegistrationData<T> { + type_id: TypeId, + #[pin] + data: T, +} + /// The registration of an auxiliary device. /// /// This type represents the registration of a [`struct auxiliary_device`]. When its parent device /// is unbound, the corresponding auxiliary device will be unregistered from the system. /// +/// The type parameter `F` is a [`ForLt`](trait@ForLt) encoding of the registration +/// data type. For non-lifetime-parameterized types, use [`ForLt!(T)`](macro@ForLt). +/// +/// The data can be accessed by the auxiliary driver through [`Device::registration_data()`] and +/// [`Device::registration_data_with()`]. +/// /// # Invariants /// -/// `self.0` always holds a valid pointer to an initialized and registered -/// [`struct auxiliary_device`]. -pub struct Registration(NonNull<bindings::auxiliary_device>); +/// `self.adev` always holds a valid pointer to an initialized and registered +/// [`struct auxiliary_device`] whose `registration_data_rust` field points to a +/// valid `Pin<KBox<RegistrationData<F::Of<'static>>>>`. +pub struct Registration<'a, F: ForLt + 'static> { + adev: NonNull<bindings::auxiliary_device>, + _phantom: PhantomData<F::Of<'a>>, +} -impl Registration { - /// Create and register a new auxiliary device. - pub fn new<'a>( +impl<'a, F: ForLt> Registration<'a, F> +where + for<'b> F::Of<'b>: Send + Sync, +{ + /// Create and register a new auxiliary device with the given registration data. + /// + /// The `data` is owned by the registration and can be accessed through the auxiliary device + /// via [`Device::registration_data()`]. + /// + /// # Safety + /// + /// The caller must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running, since the registration data may contain borrowed + /// references that become invalid after `'a` ends. + /// + /// If the registration data is `'static`, use the safe [`Registration::new()`] instead. + pub unsafe fn new_with_lt<E>( parent: &'a device::Device<device::Bound>, - name: &'a CStr, + name: &CStr, id: u32, - modname: &'a CStr, - ) -> impl PinInit<Devres<Self>, Error> + 'a { - pin_init::pin_init_scope(move || { - let boxed = KBox::new(Opaque::<bindings::auxiliary_device>::zeroed(), GFP_KERNEL)?; - let adev = boxed.get(); - - // SAFETY: It's safe to set the fields of `struct auxiliary_device` on initialization. - unsafe { - (*adev).dev.parent = parent.as_raw(); - (*adev).dev.release = Some(Device::release); - (*adev).name = name.as_char_ptr(); - (*adev).id = id; - } - - // SAFETY: `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, - // which has not been initialized yet. - unsafe { bindings::auxiliary_device_init(adev) }; - - // Now that `adev` is initialized, leak the `Box`; the corresponding memory will be - // freed by `Device::release` when the last reference to the `struct auxiliary_device` - // is dropped. - let _ = KBox::into_raw(boxed); - - // SAFETY: - // - `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, which - // has been initialized, - // - `modname.as_char_ptr()` is a NULL terminated string. - let ret = unsafe { bindings::__auxiliary_device_add(adev, modname.as_char_ptr()) }; - if ret != 0 { - // SAFETY: `adev` is guaranteed to be a valid pointer to a - // `struct auxiliary_device`, which has been initialized. - unsafe { bindings::auxiliary_device_uninit(adev) }; - - return Err(Error::from_errno(ret)); - } - - // INVARIANT: The device will remain registered until `auxiliary_device_delete()` is - // called, which happens in `Self::drop()`. - Ok(Devres::new( - parent, - // SAFETY: `adev` is guaranteed to be non-null, since the `KBox` was allocated - // successfully. - Self(unsafe { NonNull::new_unchecked(adev) }), - )) + modname: &CStr, + data: impl PinInit<F::Of<'a>, E>, + ) -> Result<Self> + where + Error: From<E>, + { + let data = KBox::pin_init::<Error>( + try_pin_init!(RegistrationData { + type_id: TypeId::of::<F>(), + data <- data, + }), + GFP_KERNEL, + )?; + + // SAFETY: `'a` is invariant (via `Registration`'s `PhantomData`). Lifetimes do not + // affect layout, so RegistrationData<F::Of<'a>> and RegistrationData<F::Of<'static>> + // have identical representation. + let data: Pin<KBox<RegistrationData<F::Of<'static>>>> = + unsafe { core::mem::transmute(data) }; + + let boxed: KBox<Opaque<bindings::auxiliary_device>> = KBox::zeroed(GFP_KERNEL)?; + let adev = boxed.get(); + + // SAFETY: It's safe to set the fields of `struct auxiliary_device` on initialization. + unsafe { + (*adev).dev.parent = parent.as_raw(); + (*adev).dev.release = Some(Device::release); + (*adev).name = name.as_char_ptr(); + (*adev).id = id; + (*adev).registration_data_rust = data.into_foreign(); + } + + // SAFETY: `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, + // which has not been initialized yet. + unsafe { bindings::auxiliary_device_init(adev) }; + + // Now that `adev` is initialized, leak the `Box`; the corresponding memory will be + // freed by `Device::release` when the last reference to the `struct auxiliary_device` + // is dropped. + let _ = KBox::into_raw(boxed); + + // SAFETY: + // - `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, which + // has been initialized, + // - `modname.as_char_ptr()` is a NULL terminated string. + let ret = unsafe { bindings::__auxiliary_device_add(adev, modname.as_char_ptr()) }; + if ret != 0 { + // SAFETY: `registration_data` was set above via `into_foreign()`. + drop(unsafe { + Pin::<KBox<RegistrationData<F::Of<'static>>>>::from_foreign( + (*adev).registration_data_rust, + ) + }); + + // SAFETY: `adev` is guaranteed to be a valid pointer to a + // `struct auxiliary_device`, which has been initialized. + unsafe { bindings::auxiliary_device_uninit(adev) }; + + return Err(Error::from_errno(ret)); + } + + // INVARIANT: The device will remain registered until `auxiliary_device_delete()` is + // called, which happens in `Self::drop()`. + Ok(Self { + // SAFETY: `adev` is guaranteed to be non-null, since the `KBox` was allocated + // successfully. + adev: unsafe { NonNull::new_unchecked(adev) }, + _phantom: PhantomData, }) } + + /// Create and register a new auxiliary device with `'static` registration data. + /// + /// Safe variant of [`Registration::new_with_lt()`] for registration data that does not contain + /// borrowed references. + pub fn new<E>( + parent: &'a device::Device<device::Bound>, + name: &CStr, + id: u32, + modname: &CStr, + data: impl PinInit<F::Of<'a>, E>, + ) -> Result<Self> + where + F::Of<'a>: 'static, + Error: From<E>, + { + // SAFETY: `F::Of<'a>: 'static` guarantees the data contains no borrowed references, + // so forgetting the `Registration` cannot cause use-after-free. + unsafe { Self::new_with_lt(parent, name, id, modname, data) } + } } -impl Drop for Registration { +impl<F: ForLt> Drop for Registration<'_, F> { fn drop(&mut self) { - // SAFETY: By the type invariant of `Self`, `self.0.as_ptr()` is a valid registered + // SAFETY: By the type invariant of `Self`, `self.adev.as_ptr()` is a valid registered // `struct auxiliary_device`. - unsafe { bindings::auxiliary_device_delete(self.0.as_ptr()) }; + unsafe { bindings::auxiliary_device_delete(self.adev.as_ptr()) }; + + // SAFETY: `registration_data` was set in `new()` via `into_foreign()`. + drop(unsafe { + Pin::<KBox<RegistrationData<F::Of<'static>>>>::from_foreign( + (*self.adev.as_ptr()).registration_data_rust, + ) + }); // This drops the reference we acquired through `auxiliary_device_init()`. // - // SAFETY: By the type invariant of `Self`, `self.0.as_ptr()` is a valid registered + // SAFETY: By the type invariant of `Self`, `self.adev.as_ptr()` is a valid registered // `struct auxiliary_device`. - unsafe { bindings::auxiliary_device_uninit(self.0.as_ptr()) }; + unsafe { bindings::auxiliary_device_uninit(self.adev.as_ptr()) }; } } // SAFETY: A `Registration` of a `struct auxiliary_device` can be released from any thread. -unsafe impl Send for Registration {} +unsafe impl<F: ForLt> Send for Registration<'_, F> where for<'a> F::Of<'a>: Send {} // SAFETY: `Registration` does not expose any methods or fields that need synchronization. -unsafe impl Sync for Registration {} +unsafe impl<F: ForLt> Sync for Registration<'_, F> where for<'a> F::Of<'a>: Send {} diff --git a/rust/kernel/bitfield.rs b/rust/kernel/bitfield.rs new file mode 100644 index 000000000000..a0d089423f21 --- /dev/null +++ b/rust/kernel/bitfield.rs @@ -0,0 +1,865 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Support for defining bitfields as Rust structures. +//! +//! The [`bitfield!`](kernel::bitfield!) macro declares integer types that are split into distinct +//! bit fields of arbitrary length. Each field is typed using [`Bounded`](kernel::num::Bounded) to +//! ensure values are properly validated and to avoid implicit data loss. +//! +//! # Example +//! +//! ```rust +//! use kernel::bitfield; +//! use kernel::num::Bounded; +//! +//! bitfield! { +//! pub struct Rgb(u16) { +//! 15:11 blue; +//! 10:5 green; +//! 4:0 red; +//! } +//! } +//! +//! // Valid value for the `blue` field. +//! let blue = Bounded::<u16, 5>::new::<0x18>(); +//! +//! // Setters can be chained. Values ranges are checked at compile-time. +//! let color = Rgb::zeroed() +//! // Compile-time bounds check of constant value. +//! .with_const_red::<0x10>() +//! .with_const_green::<0x1f>() +//! // A `Bounded` can also be passed. +//! .with_blue(blue); +//! +//! assert_eq!(color.red(), 0x10); +//! assert_eq!(color.green(), 0x1f); +//! assert_eq!(color.blue(), 0x18); +//! assert_eq!( +//! color.into_raw(), +//! (0x18 << Rgb::BLUE_SHIFT) + (0x1f << Rgb::GREEN_SHIFT) + 0x10, +//! ); +//! +//! // Convert to/from the backing storage type. +//! let raw: u16 = color.into(); +//! assert_eq!(Rgb::from(raw), color); +//! ``` +//! +//! # Syntax +//! +//! ```text +//! bitfield! { +//! #[attributes] +//! // Documentation for `Name`. +//! pub struct Name(storage_type) { +//! // `field_1` documentation. +//! hi:lo field_1; +//! // `field_2` documentation. +//! hi:lo field_2 => ConvertedType; +//! // `field_3` documentation. +//! hi:lo field_3 ?=> ConvertedType; +//! ... +//! } +//! } +//! ``` +//! +//! - `storage_type`: The underlying unsigned integer type ([`u8`], [`u16`], [`u32`], [`u64`]). +//! Signed integer storage types are not supported. +//! - `hi:lo`: Bit range (inclusive), where `hi >= lo`. +//! - `=> Type`: Optional infallible conversion (see [below](#infallible-conversion-)). +//! - `?=> Type`: Optional fallible conversion (see [below](#fallible-conversion-)). +//! - Documentation strings and attributes are optional. +//! +//! # Generated code +//! +//! Each field is internally represented as a [`Bounded`] parameterized by its bit width. Field +//! values can either be set/retrieved directly, or converted from/to another type. +//! +//! The use of [`Bounded`] for each field enforces bounds-checking (at build time or runtime) of +//! every value assigned to a field. This ensures that data is never accidentally truncated. +//! +//! The macro generates the bitfield type, [`From`] and [`Into`] implementations for its storage +//! type, as well as [`Debug`] and [`Zeroable`](pin_init::Zeroable) implementations. +//! +//! For each field, it also generates: +//! +//! - `field()`: Getter method for the field value. +//! - `with_field(value)`: Infallible setter; the argument type must fit within the field's width. +//! - `with_const_field::<VALUE>()`: `const` setter; the value is validated at compile time. +//! Usually shorter to use than `with_field` for constant values as it doesn't require +//! constructing a [`Bounded`]. +//! - `try_with_field(value)`: Fallible setter. Returns an error if the value is out of range. +//! - `FIELD_MASK`, `FIELD_SHIFT`, `FIELD_RANGE`: Constants for manual bit manipulation. +//! +//! # Reserved names for field identifiers +//! +//! Field identifiers are used to generate methods and associated constants on the bitfield type. +//! For a field named `field`, the macro may generate methods named `field`, `with_field`, +//! `with_const_field`, `try_with_field`, `__field` and `__with_field`, as well as constants named +//! `FIELD_MASK`, `FIELD_SHIFT` and `FIELD_RANGE`. +//! +//! Therefore, field identifiers must not use names that would collide with generated items for +//! any field in the same bitfield. The following prefixes are thus reserved for field identifiers: +//! +//! - `with_` +//! - `const_` +//! - `try_with_` +//! - `__` +//! +//! The field identifiers `from_raw`, `into_raw`, and `into` are also reserved. +//! +//! In addition, field identifiers should follow Rust `snake_case` conventions, since the associated +//! constants are generated by uppercasing the field name. +//! +//! # Implicit conversions +//! +//! Types that fit entirely within a field's bit width can be used directly with setters. For +//! example, [`bool`] works with single-bit fields, and [`u8`] works with 8-bit fields: +//! +//! ```rust +//! use kernel::bitfield; +//! +//! bitfield! { +//! pub struct Flags(u32) { +//! 15:8 byte_field; +//! 0:0 flag; +//! } +//! } +//! +//! let flags = Flags::zeroed() +//! .with_byte_field(0x42_u8) +//! .with_flag(true); +//! +//! assert_eq!(flags.into_raw(), (0x42 << Flags::BYTE_FIELD_SHIFT) | 1); +//! ``` +//! +//! # Runtime bounds checking +//! +//! When a value is not known at compile time, use `try_with_field()` to check bounds at runtime: +//! +//! ```rust +//! use kernel::bitfield; +//! +//! bitfield! { +//! pub struct Config(u8) { +//! 3:0 nibble; +//! } +//! } +//! +//! fn set_nibble(config: Config, value: u8) -> Result<Config, Error> { +//! // Returns `EOVERFLOW` if `value > 0xf`. +//! config.try_with_nibble(value) +//! } +//! # Ok::<(), Error>(()) +//! ``` +//! +//! # Type conversion +//! +//! Fields can be automatically converted to/from a custom type using `=>` (infallible) or `?=>` +//! (fallible). The custom type must implement the appropriate [`From`] or [`TryFrom`] traits with +//! [`Bounded`]. +//! +//! ## Infallible conversion (`=>`) +//! +//! Use this when all possible bit patterns of a field map to valid values: +//! +//! ```rust +//! use kernel::bitfield; +//! use kernel::num::Bounded; +//! +//! #[derive(Debug, Clone, Copy, PartialEq)] +//! enum Power { +//! Off, +//! On, +//! } +//! +//! impl From<Bounded<u32, 1>> for Power { +//! fn from(v: Bounded<u32, 1>) -> Self { +//! match *v { +//! 0 => Power::Off, +//! _ => Power::On, +//! } +//! } +//! } +//! +//! impl From<Power> for Bounded<u32, 1> { +//! fn from(p: Power) -> Self { +//! (p as u32 != 0).into() +//! } +//! } +//! +//! bitfield! { +//! pub struct Control(u32) { +//! 0:0 power => Power; +//! } +//! } +//! +//! let ctrl = Control::zeroed().with_power(Power::On); +//! assert_eq!(ctrl.power(), Power::On); +//! ``` +//! +//! ## Fallible conversion (`?=>`) +//! +//! Use this when some bit patterns of a field are invalid. The getter returns a [`Result`]: +//! +//! ```rust +//! use kernel::bitfield; +//! use kernel::num::Bounded; +//! +//! #[derive(Debug, Clone, Copy, PartialEq)] +//! enum Mode { +//! Low = 0, +//! High = 1, +//! Auto = 2, +//! // 3 is invalid +//! } +//! +//! impl TryFrom<Bounded<u32, 2>> for Mode { +//! type Error = u32; +//! +//! fn try_from(v: Bounded<u32, 2>) -> Result<Self, u32> { +//! match *v { +//! 0 => Ok(Mode::Low), +//! 1 => Ok(Mode::High), +//! 2 => Ok(Mode::Auto), +//! n => Err(n), +//! } +//! } +//! } +//! +//! impl From<Mode> for Bounded<u32, 2> { +//! fn from(m: Mode) -> Self { +//! match m { +//! Mode::Low => Bounded::<u32, _>::new::<0>(), +//! Mode::High => Bounded::<u32, _>::new::<1>(), +//! Mode::Auto => Bounded::<u32, _>::new::<2>(), +//! } +//! } +//! } +//! +//! bitfield! { +//! pub struct Config(u32) { +//! 1:0 mode ?=> Mode; +//! } +//! } +//! +//! let cfg = Config::zeroed().with_mode(Mode::Auto); +//! assert_eq!(cfg.mode(), Ok(Mode::Auto)); +//! +//! // Invalid bit pattern returns an error. +//! assert_eq!(Config::from(0b11).mode(), Err(3)); +//! ``` +//! +//! # Bits outside of declared fields +//! +//! Bits of the storage type that are not part of any declared field are preserved by the setter +//! methods, and can only be modified through `from_raw` or the [`From`] implementation from the +//! storage type. +//! +//! ```rust +//! use kernel::bitfield; +//! +//! bitfield! { +//! pub struct Sparse(u8) { +//! 7:6 high; +//! // Bits 5:1 are not covered by any field. +//! 0:0 low; +//! } +//! } +//! +//! // Set the gap bits via `from_raw`, then mutate the declared fields. +//! let val = Sparse::from_raw(0b0010_1010) +//! .with_const_high::<0b11>() +//! .with_low(true); +//! +//! // Bits 5:1 are unchanged. +//! assert_eq!(val.into_raw(), 0b1110_1011); +//! ``` +//! +//! # Signed field values +//! +//! Bitfield storage types are unsigned. Since field getter methods return a [`Bounded`] of the +//! storage type, fields are also unsigned by default. +//! +//! If a field needs to encode a signed value, use a custom conversion type with `=>` or `?=>` to +//! perform the sign interpretation explicitly. +//! +//! [`Bounded`]: kernel::num::Bounded + +/// Defines a bitfield struct with bounds-checked accessors for individual bit ranges. +/// +/// See the [`mod@kernel::bitfield`] module for full documentation and examples. +#[macro_export] +macro_rules! bitfield { + // Entry point defining the bitfield struct, its implementations and its field accessors. + ( + $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* } + ) => { + $crate::bitfield!(@core + #[allow(non_camel_case_types)] + $(#[$attr])* $vis $name $storage + ); + $crate::bitfield!(@fields $vis $name $storage { $($fields)* }); + }; + + // All rules below are helpers. + + // Defines the wrapper `$name` type and its conversions from/to the storage type. + (@core $(#[$attr:meta])* $vis:vis $name:ident $storage:ty) => { + $(#[$attr])* + #[repr(transparent)] + #[derive(Clone, Copy, PartialEq, Eq)] + $vis struct $name { + inner: $storage, + } + + #[allow(dead_code)] + impl $name { + /// Creates a bitfield from a raw value. + #[inline(always)] + $vis const fn from_raw(value: $storage) -> Self { + Self{ inner: value } + } + + /// Turns this bitfield into its raw value. + /// + /// This is similar to the [`From`] implementation, but is shorter to invoke in + /// most cases. + #[inline(always)] + $vis const fn into_raw(self) -> $storage { + self.inner + } + } + + // SAFETY: `$storage` is `Zeroable` and `$name` is transparent. + unsafe impl ::pin_init::Zeroable for $name {} + + impl ::core::convert::From<$name> for $storage { + #[inline(always)] + fn from(val: $name) -> $storage { + val.into_raw() + } + } + + impl ::core::convert::From<$storage> for $name { + #[inline(always)] + fn from(val: $storage) -> $name { + Self::from_raw(val) + } + } + }; + + // Definitions requiring knowledge of individual fields: private and public field accessors, + // and `Debug` implementation. + (@fields $vis:vis $name:ident $storage:ty { + $($(#[doc = $doc:expr])* $hi:literal:$lo:literal $field:ident + $(?=> $try_into_type:ty)? + $(=> $into_type:ty)? + ; + )* + } + ) => { + #[allow(dead_code)] + impl $name { + $( + $crate::bitfield!(@private_field_accessors $vis $name $storage : $hi:$lo $field); + $crate::bitfield!( + @public_field_accessors $(#[doc = $doc])* $vis $name $storage : $hi:$lo $field + $(?=> $try_into_type)? + $(=> $into_type)? + ); + )* + } + + $crate::bitfield!(@debug $name { $($field;)* }); + }; + + // Private field accessors working with the exact `Bounded` type for the field. + ( + @private_field_accessors $vis:vis $name:ident $storage:ty : $hi:tt:$lo:tt $field:ident + ) => { + ::kernel::macros::paste!( + $vis const [<$field:upper _RANGE>]: ::core::ops::RangeInclusive<u8> = $lo..=$hi; + $vis const [<$field:upper _MASK>]: $storage = + ((((1 << $hi) - 1) << 1) + 1) - ((1 << $lo) - 1); + $vis const [<$field:upper _SHIFT>]: u32 = $lo; + ); + + ::kernel::macros::paste!( + #[inline(always)] + fn [<__ $field>](self) -> + ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> { + // Left shift to align the field's MSB with the storage MSB. + const ALIGN_TOP: u32 = $storage::BITS - ($hi + 1); + // Right shift to move the top-aligned field to bit 0 of the storage. + const ALIGN_BOTTOM: u32 = ALIGN_TOP + $lo; + + // Extract the field using two shifts. `Bounded::shr` produces the correctly-sized + // output type. + let val = ::kernel::num::Bounded::<$storage, { $storage::BITS }>::from( + self.inner << ALIGN_TOP + ); + val.shr::<ALIGN_BOTTOM, { $hi + 1 - $lo } >() + } + + #[inline(always)] + const fn [<__with_ $field>]( + mut self, + value: ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }>, + ) -> Self + { + const MASK: $storage = <$name>::[<$field:upper _MASK>]; + const SHIFT: u32 = <$name>::[<$field:upper _SHIFT>]; + + let value = value.get() << SHIFT; + self.inner = (self.inner & !MASK) | value; + + self + } + ); + }; + + // Public accessors for fields infallibly (`=>`) converted to a type. + ( + @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : + $hi:literal:$lo:literal $field:ident => $into_type:ty + ) => { + ::kernel::macros::paste!( + + $(#[doc = $doc])* + #[doc = "Returns the value of this field."] + #[inline(always)] + $vis fn $field(self) -> $into_type + { + self.[<__ $field>]().into() + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the given `value`."] + #[inline(always)] + $vis fn [<with_ $field>](self, value: $into_type) -> Self + { + self.[<__with_ $field>](value.into()) + } + + ); + }; + + // Public accessors for fields fallibly (`?=>`) converted to a type. + ( + @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : + $hi:tt:$lo:tt $field:ident ?=> $try_into_type:ty + ) => { + ::kernel::macros::paste!( + + $(#[doc = $doc])* + #[doc = "Returns the value of this field."] + #[inline(always)] + $vis fn $field(self) -> + ::core::result::Result< + $try_into_type, + <$try_into_type as ::core::convert::TryFrom< + ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> + >>::Error + > + { + self.[<__ $field>]().try_into() + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the given `value`."] + #[inline(always)] + $vis fn [<with_ $field>](self, value: $try_into_type) -> Self + { + self.[<__with_ $field>](value.into()) + } + + ); + }; + + // Public accessors for fields not converted to a type. + ( + @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : + $hi:tt:$lo:tt $field:ident + ) => { + ::kernel::macros::paste!( + + $(#[doc = $doc])* + #[doc = "Returns the value of this field."] + #[inline(always)] + $vis fn $field(self) -> + ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> + { + self.[<__ $field>]() + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the compile-time constant `VALUE`."] + #[inline(always)] + $vis const fn [<with_const_ $field>]<const VALUE: $storage>(self) -> Self { + self.[<__with_ $field>]( + ::kernel::num::Bounded::<$storage, { $hi + 1 - $lo }>::new::<VALUE>() + ) + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the given `value`."] + #[inline(always)] + $vis fn [<with_ $field>]<T>( + self, + value: T, + ) -> Self + where T: ::core::convert::Into<::kernel::num::Bounded<$storage, { $hi + 1 - $lo }>>, + { + self.[<__with_ $field>](value.into()) + } + + $(#[doc = $doc])* + #[doc = "Tries to set this field to `value`, returning an error if it is out of range."] + #[inline(always)] + $vis fn [<try_with_ $field>]<T>( + self, + value: T, + ) -> ::kernel::error::Result<Self> + where T: ::kernel::num::TryIntoBounded<$storage, { $hi + 1 - $lo }>, + { + Ok( + self.[<__with_ $field>]( + value.try_into_bounded().ok_or(::kernel::error::code::EOVERFLOW)? + ) + ) + } + + ); + }; + + // `Debug` implementation. + (@debug $name:ident { $($field:ident;)* }) => { + impl ::kernel::fmt::Debug for $name { + #[inline] + fn fmt(&self, f: &mut ::kernel::fmt::Formatter<'_>) -> ::kernel::fmt::Result { + f.debug_struct(stringify!($name)) + .field("<raw>", &::kernel::prelude::fmt!("{:#x}", self.inner)) + $( + .field(stringify!($field), &self.$field()) + )* + .finish() + } + } + }; +} + +#[cfg(CONFIG_RUST_BITFIELD_KUNIT_TEST)] +#[::kernel::macros::kunit_tests(rust_kernel_bitfield)] +mod tests { + use core::convert::TryFrom; + + use pin_init::Zeroable; + + use kernel::num::Bounded; + + // Enum types for testing `=>` and `?=>` conversions. + + #[derive(Debug, Clone, Copy, PartialEq)] + enum MemoryType { + Unmapped = 0, + Normal = 1, + Device = 2, + Reserved = 3, + } + + impl TryFrom<Bounded<u64, 4>> for MemoryType { + type Error = u64; + fn try_from(value: Bounded<u64, 4>) -> Result<Self, Self::Error> { + match value.get() { + 0 => Ok(MemoryType::Unmapped), + 1 => Ok(MemoryType::Normal), + 2 => Ok(MemoryType::Device), + 3 => Ok(MemoryType::Reserved), + _ => Err(value.get()), + } + } + } + + impl From<MemoryType> for Bounded<u64, 4> { + #[inline(always)] + fn from(mt: MemoryType) -> Bounded<u64, 4> { + Bounded::from_expr(mt as u64) + } + } + + #[derive(Debug, Clone, Copy, PartialEq)] + enum Priority { + Low = 0, + Medium = 1, + High = 2, + Critical = 3, + } + + impl From<Bounded<u16, 2>> for Priority { + fn from(value: Bounded<u16, 2>) -> Self { + match value & 0x3 { + 0 => Priority::Low, + 1 => Priority::Medium, + 2 => Priority::High, + _ => Priority::Critical, + } + } + } + + impl From<Priority> for Bounded<u16, 2> { + #[inline(always)] + fn from(p: Priority) -> Bounded<u16, 2> { + Bounded::from_expr(p as u16) + } + } + + bitfield! { + struct TestU64(u64) { + 63:63 field_63; + 61:52 field_61_52; + 51:16 field_51_16; + 15:12 field_15_12 ?=> MemoryType; + 11:9 field_11_9; + 1:1 field_1; + 0:0 field_0; + } + } + + bitfield! { + struct TestU16(u16) { + 15:8 field_15_8; + 7:4 field_7_4; // Partial overlap with `field_5_4`. + 5:4 field_5_4 => Priority; + 3:1 field_3_1; + 0:0 field_0; + } + } + + bitfield! { + struct TestU8(u8) { + 7:0 field_7_0; // Full byte overlap. + 7:4 field_7_4; + 3:2 field_3_2; + 1:1 field_1; + 0:0 field_0; + } + } + + // Single and multi-bit fields basic access. + #[test] + fn test_basic_access() { + // `TestU64`. + let mut val = TestU64::zeroed(); + assert_eq!(val.into_raw(), 0x0); + + val = val.with_field_0(true); + assert!(val.field_0().into_bool()); + assert_eq!(val.into_raw(), 0x1); + + val = val.with_field_1(true); + assert!(val.field_1().into_bool()); + val = val.with_field_1(false); + assert!(!val.field_1().into_bool()); + assert_eq!(val.into_raw(), 0x1); + + val = val.with_const_field_11_9::<0x5>(); + assert_eq!(val.field_11_9(), 0x5); + assert_eq!(val.into_raw(), 0xA01); + + val = val.with_const_field_51_16::<0x123456>(); + assert_eq!(val.field_51_16(), 0x123456); + assert_eq!(val.into_raw(), 0x0012_3456_0A01); + + const MAX_FIELD_51_16: u64 = ::kernel::bits::genmask_u64(0..=35); + val = val.with_const_field_51_16::<{ MAX_FIELD_51_16 }>(); + assert_eq!(val.field_51_16(), MAX_FIELD_51_16); + + val = val.with_const_field_61_52::<0x3FF>(); + assert_eq!(val.field_61_52(), 0x3FF); + + val = val.with_field_63(true); + assert!(val.field_63().into_bool()); + + // `TestU16`. + let mut val = TestU16::zeroed(); + assert_eq!(val.into_raw(), 0x0); + + val = val.with_field_0(true); + assert!(val.field_0().into_bool()); + assert_eq!(val.into_raw(), 0x1); + + val = val.with_const_field_3_1::<0x5>(); + assert_eq!(val.field_3_1(), 0x5); + assert_eq!(val.into_raw(), 0xB); + + val = val.with_const_field_7_4::<0xA>(); + assert_eq!(val.field_7_4(), 0xA); + assert_eq!(val.into_raw(), 0xAB); + + val = val.with_const_field_15_8::<0x42>(); + assert_eq!(val.field_15_8(), 0x42); + assert_eq!(val.into_raw(), 0x42AB); + + // `TestU8`. + let mut val = TestU8::zeroed(); + assert_eq!(val.into_raw(), 0x0); + + val = val.with_field_0(true); + assert!(val.field_0().into_bool()); + assert_eq!(val.into_raw(), 0x1); + + val = val.with_field_1(true); + assert!(val.field_1().into_bool()); + assert_eq!(val.into_raw(), 0x3); + + val = val.with_const_field_3_2::<0x3>(); + assert_eq!(val.field_3_2(), 0x3); + assert_eq!(val.into_raw(), 0xF); + + val = val.with_const_field_7_4::<0xA>(); + assert_eq!(val.field_7_4(), 0xA); + assert_eq!(val.into_raw(), 0xAF); + } + + // `=>` infallible conversion. + #[test] + fn test_infallible_conversion() { + let mut val = TestU16::zeroed(); + + val = val.with_field_5_4(Priority::Low); + assert_eq!(val.field_5_4(), Priority::Low); + assert_eq!(val.into_raw() & 0x30, 0x00); + + val = val.with_field_5_4(Priority::Medium); + assert_eq!(val.field_5_4(), Priority::Medium); + assert_eq!(val.into_raw() & 0x30, 0x10); + + val = val.with_field_5_4(Priority::High); + assert_eq!(val.field_5_4(), Priority::High); + assert_eq!(val.into_raw() & 0x30, 0x20); + + val = val.with_field_5_4(Priority::Critical); + assert_eq!(val.field_5_4(), Priority::Critical); + assert_eq!(val.into_raw() & 0x30, 0x30); + } + + // `?=>` fallible conversion. + #[test] + fn test_fallible_conversion() { + let mut val = TestU64::zeroed(); + + val = val.with_field_15_12(MemoryType::Unmapped); + assert_eq!(val.field_15_12(), Ok(MemoryType::Unmapped)); + val = val.with_field_15_12(MemoryType::Normal); + assert_eq!(val.field_15_12(), Ok(MemoryType::Normal)); + val = val.with_field_15_12(MemoryType::Device); + assert_eq!(val.field_15_12(), Ok(MemoryType::Device)); + val = val.with_field_15_12(MemoryType::Reserved); + assert_eq!(val.field_15_12(), Ok(MemoryType::Reserved)); + + // `field_15_12` is 4 bits wide (0-15); `MemoryType` only covers 0-3, so 4-15 return `Err`. + let raw = (val.into_raw() & !::kernel::bits::genmask_u64(12..=15)) | (0x7 << 12); + assert_eq!(TestU64::from_raw(raw).field_15_12(), Err(0x7)); + } + + // Test that setting an overlapping field affects the overlapped one as expected. + #[test] + fn test_overlapping_fields() { + let mut val = TestU16::zeroed(); + + val = val.with_field_5_4(Priority::High); // High == 2 == 0b10. + assert_eq!(val.field_5_4(), Priority::High); + assert_eq!(val.field_7_4(), 0x2); // Bits 7:6 == 0, bits 5:4 == 0b10. + + val = val.with_const_field_7_4::<0xF>(); + assert_eq!(val.field_7_4(), 0xF); + assert_eq!(val.field_5_4(), Priority::Critical); // Bits 5:4 == 0b11. + + // `field_7_0` should encompass all other fields. + let mut val = TestU8::zeroed() + .with_field_0(true) + .with_field_1(true) + .with_const_field_3_2::<0x3>() + .with_const_field_7_4::<0xA>(); + assert_eq!(val.into_raw(), 0xAF); + + val = val.with_field_7_0(0x55); + assert_eq!(val.field_7_0(), 0x55); + assert!(val.field_0().into_bool()); + assert!(!val.field_1().into_bool()); + assert_eq!(val.field_3_2(), 0x1); + assert_eq!(val.field_7_4(), 0x5); + } + + // Checks that bits not mapped to any field are left untouched. + #[test] + fn test_unallocated_bits() { + let gap_bits = (1u64 << 62) | 0x1FC; + + let set_all_fields = |val: TestU64| { + val.with_field_63(true) + .with_const_field_61_52::<0x155>() + .with_const_field_51_16::<0x123456>() + .with_field_15_12(MemoryType::Device) + .with_const_field_11_9::<0x5>() + .with_field_1(true) + .with_field_0(true) + }; + + // Gap bits to 0. + let val = set_all_fields(TestU64::from_raw(0)); + assert_eq!(val.into_raw() & gap_bits, 0); + + // Gap bits to 1. + let val = set_all_fields(TestU64::from_raw(gap_bits)); + assert_eq!(val.into_raw() & gap_bits, gap_bits); + } + + #[test] + fn test_try_with() { + let val = TestU64::zeroed().try_with_field_51_16(0x123456).unwrap(); + assert_eq!(val.field_51_16(), 0x123456); + + let err = TestU64::zeroed().try_with_field_51_16(u64::MAX); + assert_eq!(err, Err(::kernel::error::code::EOVERFLOW)); + + let val = TestU64::zeroed() + .try_with_field_51_16(0xABCDEF) + .and_then(|p| p.try_with_field_0(1)) + .unwrap(); + assert_eq!(val.field_51_16(), 0xABCDEF); + assert!(val.field_0().into_bool()); + } + + // `from_raw`/`into_raw` and `From`/`Into` round-trips. + #[test] + fn test_raw() { + let raw: u64 = 0xBFF0_0000_3123_3E03; + let val = TestU64::from_raw(raw); + assert_eq!(u64::from(val), raw); + assert!(val.field_0().into_bool()); + assert!(val.field_1().into_bool()); + assert_eq!(val.field_11_9(), 0x7); + assert_eq!(val.field_51_16(), 0x3123); + assert_eq!(val.field_15_12(), Ok(MemoryType::Reserved)); + assert_eq!(val.field_61_52(), 0x3FF); + assert!(val.field_63().into_bool()); + + let raw: u16 = 0x42AB; + let val = TestU16::from_raw(raw); + assert_eq!(u16::from(val), raw); + assert!(val.field_0().into_bool()); + assert_eq!(val.field_3_1(), 0x5); + assert_eq!(val.field_7_4(), 0xA); + assert_eq!(val.field_15_8(), 0x42); + + let raw: u8 = 0xAF; + let val = TestU8::from_raw(raw); + assert_eq!(u8::from(val), raw); + assert!(val.field_0().into_bool()); + assert!(val.field_1().into_bool()); + assert_eq!(val.field_3_2(), 0x3); + assert_eq!(val.field_7_4(), 0xA); + assert_eq!(val.field_7_0(), 0xAF); + } +} diff --git a/rust/kernel/bitmap.rs b/rust/kernel/bitmap.rs index 83d7dea99137..b27e0ec80d64 100644 --- a/rust/kernel/bitmap.rs +++ b/rust/kernel/bitmap.rs @@ -499,9 +499,8 @@ impl Bitmap { } } -use macros::kunit_tests; - -#[kunit_tests(rust_kernel_bitmap)] +#[cfg(CONFIG_RUST_BITMAP_KUNIT_TEST)] +#[macros::kunit_tests(rust_kernel_bitmap)] mod tests { use super::*; use kernel::alloc::flags::GFP_KERNEL; diff --git a/rust/kernel/block/mq/gen_disk.rs b/rust/kernel/block/mq/gen_disk.rs index c8b0ecb17082..fc97dd873974 100644 --- a/rust/kernel/block/mq/gen_disk.rs +++ b/rust/kernel/block/mq/gen_disk.rs @@ -140,9 +140,7 @@ impl GenDiskBuilder { devnode: None, alternative_gpt_sector: None, get_unique_id: None, - // TODO: Set to THIS_MODULE. Waiting for const_refs_to_static feature to - // be merged (unstable in rustc 1.78 which is staged for linux 6.10) - // <https://github.com/rust-lang/rust/issues/119618> + // TODO: Set to `THIS_MODULE`. owner: core::ptr::null_mut(), pr_ops: core::ptr::null_mut(), free_disk: None, @@ -152,6 +150,19 @@ impl GenDiskBuilder { // SAFETY: `gendisk` is a valid pointer as we initialized it above unsafe { (*gendisk).fops = &TABLE }; + let cleanup_failure = ScopeGuard::new_with_data((gendisk, data), |(gendisk, data)| { + // SAFETY: `gendisk` came from `__blk_mq_alloc_disk()` above and + // has not been added to the VFS on this cleanup path. + unsafe { bindings::put_disk(gendisk) }; + // SAFETY: `data` came from `into_foreign()` above and has not been + // converted back on this cleanup path. + drop(unsafe { T::QueueData::from_foreign(data) }); + }); + + // The failure guard now owns both pieces of cleanup; the early guard + // must not run on this path anymore. + recover_data.dismiss(); + let mut writer = NullTerminatedFormatter::new( // SAFETY: `gendisk` points to a valid and initialized instance. We // have exclusive access, since the disk is not added to the VFS @@ -174,7 +185,7 @@ impl GenDiskBuilder { }, )?; - recover_data.dismiss(); + cleanup_failure.dismiss(); // INVARIANT: `gendisk` was initialized above. // INVARIANT: `gendisk` was added to the VFS via `device_add_disk` above. @@ -217,6 +228,11 @@ impl<T: Operations> Drop for GenDisk<T> { // to the VFS. unsafe { bindings::del_gendisk(self.gendisk) }; + // SAFETY: By type invariant, `self.gendisk` was added to the VFS, so + // `put_disk()` must follow `del_gendisk()` to drop the final gendisk + // reference and trigger the remaining release path. + unsafe { bindings::put_disk(self.gendisk) }; + // SAFETY: `queue.queuedata` was created by `GenDiskBuilder::build` with // a call to `ForeignOwnable::into_foreign` to create `queuedata`. // `ForeignOwnable::from_foreign` is only called here. diff --git a/rust/kernel/block/mq/operations.rs b/rust/kernel/block/mq/operations.rs index 8ad46129a52c..861903e18fbf 100644 --- a/rust/kernel/block/mq/operations.rs +++ b/rust/kernel/block/mq/operations.rs @@ -218,7 +218,7 @@ impl<T: Operations> OperationsVTable<T> { _set: *mut bindings::blk_mq_tag_set, rq: *mut bindings::request, _hctx_idx: crate::ffi::c_uint, - _numa_node: crate::ffi::c_uint, + _numa_node: crate::ffi::c_int, ) -> crate::ffi::c_int { from_result(|| { // SAFETY: By the safety requirements of this function, `rq` points diff --git a/rust/kernel/bug.rs b/rust/kernel/bug.rs index ed943960f851..3566f0234ca4 100644 --- a/rust/kernel/bug.rs +++ b/rust/kernel/bug.rs @@ -8,6 +8,7 @@ #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))] #[cfg(CONFIG_DEBUG_BUGVERBOSE)] macro_rules! warn_flags { @@ -47,12 +48,17 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))] #[cfg(not(CONFIG_DEBUG_BUGVERBOSE))] macro_rules! warn_flags { ($file:expr, $flags:expr) => { const FLAGS: u32 = $crate::bindings::BUGFLAG_WARNING | $flags; + if false { + _ = $file; + } + // SAFETY: // - `flags` and `size` are all compile-time constants, preventing // any invalid memory access. @@ -73,14 +79,19 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, CONFIG_UML))] macro_rules! warn_flags { ($file:expr, $flags:expr) => { + if false { + _ = $file; + } + // SAFETY: It is always safe to call `warn_slowpath_fmt()` // with a valid null-terminated string. unsafe { $crate::bindings::warn_slowpath_fmt( - $crate::c_str!(::core::file!()).as_char_ptr(), + $crate::str::CStrExt::as_char_ptr($crate::c_str!(::core::file!())), line!() as $crate::ffi::c_int, $flags as $crate::ffi::c_uint, ::core::ptr::null(), @@ -91,9 +102,15 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, any(CONFIG_LOONGARCH, CONFIG_ARM)))] macro_rules! warn_flags { ($file:expr, $flags:expr) => { + if false { + _ = $file; + _ = $flags; + } + // SAFETY: It is always safe to call `WARN_ON()`. unsafe { $crate::bindings::WARN_ON(true) } }; @@ -101,9 +118,14 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] -#[cfg(not(CONFIG_BUG))] +#[cfg(any(testlib, not(CONFIG_BUG)))] macro_rules! warn_flags { - ($file:expr, $flags:expr) => {}; + ($file:expr, $flags:expr) => { + if false { + _ = $file; + _ = $flags; + } + }; } #[doc(hidden)] @@ -118,14 +140,14 @@ macro_rules! warn_on { let cond = $cond; #[cfg(CONFIG_DEBUG_BUGVERBOSE_DETAILED)] - const _COND_STR: &str = concat!("[", stringify!($cond), "] ", file!()); + const COND_STR: &str = concat!("[", stringify!($cond), "] ", file!()); #[cfg(not(CONFIG_DEBUG_BUGVERBOSE_DETAILED))] - const _COND_STR: &str = file!(); + const COND_STR: &str = file!(); if cond { const WARN_ON_FLAGS: u32 = $crate::bug::bugflag_taint($crate::bindings::TAINT_WARN); - $crate::warn_flags!(_COND_STR, WARN_ON_FLAGS); + $crate::warn_flags!(COND_STR, WARN_ON_FLAGS); } cond }}; diff --git a/rust/kernel/build_assert.rs b/rust/kernel/build_assert.rs index f8124dbc663f..c3acb9b68a65 100644 --- a/rust/kernel/build_assert.rs +++ b/rust/kernel/build_assert.rs @@ -1,9 +1,146 @@ // SPDX-License-Identifier: GPL-2.0 -//! Build-time assert. +//! Various assertions that happen during build-time. +//! +//! There are three types of build-time assertions that you can use: +//! - [`static_assert!`] +//! - [`const_assert!`] +//! - [`build_assert!`] +//! +//! The ones towards the bottom of the list are more expressive, while the ones towards the top of +//! the list are more robust and trigger earlier in the compilation pipeline. Therefore, you should +//! prefer the ones towards the top of the list wherever possible. +//! +//! # Choosing the correct assertion +//! +//! If you're asserting outside any bodies (e.g. initializers or function bodies), you should use +//! [`static_assert!`] as it is the only assertion that can be used in that context. +//! +//! Inside bodies, if your assertion condition does not depend on any variable or generics, you +//! should use [`static_assert!`]. If the condition depends on generics, but not variables +//! (including function arguments), you should use [`const_assert!`]. Otherwise, use +//! [`build_assert!`]. The same is true regardless if the function is `const fn`. +//! +//! ``` +//! // Outside any bodies. +//! static_assert!(core::mem::size_of::<u8>() == 1); +//! // `const_assert!` and `build_assert!` cannot be used here, they will fail to compile. +//! +//! #[inline(always)] +//! fn foo<const N: usize>(v: usize) { +//! static_assert!(core::mem::size_of::<u8>() == 1); // Preferred. +//! const_assert!(core::mem::size_of::<u8>() == 1); // Discouraged. +//! build_assert!(core::mem::size_of::<u8>() == 1); // Discouraged. +//! +//! // `static_assert!(N > 1);` is not allowed. +//! const_assert!(N > 1); // Preferred. +//! build_assert!(N > 1); // Discouraged. +//! +//! // `static_assert!(v > 1);` is not allowed. +//! // `const_assert!(v > 1);` is not allowed. +//! build_assert!(v > 1); // Works. +//! } +//! ``` +//! +//! # Detailed behavior +//! +//! `static_assert!()` is equivalent to `static_assert` in C. It requires `expr` to be a constant +//! expression. This expression cannot refer to any generics. A `static_assert!(expr)` in a program +//! is always evaluated, regardless if the function it appears in is used or not. This is also the +//! only usable assertion outside a body. +//! +//! `const_assert!()` has no direct C equivalence. It is a more powerful version of +//! `static_assert!()`, where it may refer to generics in a function. Note that due to the ability +//! to refer to generics, the assertion is tied to a specific instance of a function. So if it is +//! used in a generic function that is not instantiated, the assertion will not be checked. For this +//! reason, `static_assert!()` is preferred wherever possible. +//! +//! `build_assert!()` is equivalent to `BUILD_BUG_ON`. It is even more powerful than +//! `const_assert!()` because it can be used to check tautologies that depend on runtime value (this +//! is the same as `BUILD_BUG_ON`). However, the assertion failure mechanism can possibly be +//! undefined symbols and linker errors, it is not developer friendly to debug, so it is recommended +//! to avoid it and prefer other two assertions where possible. +#[doc(inline)] +pub use crate::{ + build_assert_macro as build_assert, + build_error, + const_assert, + static_assert, // +}; + +#[doc(hidden)] +pub use build_error::build_error as build_error_fn; + +/// Static assert (i.e. compile-time assert). +/// +/// Similar to C11 [`_Static_assert`] and C++11 [`static_assert`]. +/// +/// An optional panic message can be supplied after the expression. +/// Currently only a string literal without formatting is supported +/// due to constness limitations of the [`assert!`] macro. +/// +/// The feature may be added to Rust in the future: see [RFC 2790]. +/// +/// You cannot refer to generics or variables with [`static_assert!`]. If you need to refer to +/// generics, use [`const_assert!`]; if you need to refer to variables, use [`build_assert!`]. See +/// the [module documentation](self). +/// +/// [`_Static_assert`]: https://en.cppreference.com/w/c/language/_Static_assert +/// [`static_assert`]: https://en.cppreference.com/w/cpp/language/static_assert +/// [RFC 2790]: https://github.com/rust-lang/rfcs/issues/2790 +/// +/// # Examples +/// +/// ``` +/// static_assert!(42 > 24); +/// static_assert!(core::mem::size_of::<u8>() == 1); +/// +/// const X: &[u8] = b"bar"; +/// static_assert!(X[1] == b'a'); +/// +/// const fn f(x: i32) -> i32 { +/// x + 2 +/// } +/// static_assert!(f(40) == 42); +/// static_assert!(f(40) == 42, "f(x) must add 2 to the given input."); +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! static_assert { + ($condition:expr $(,$arg:literal)?) => { + const _: () = ::core::assert!($condition $(,$arg)?); + }; +} + +/// Assertion during constant evaluation. +/// +/// This is a more powerful version of [`static_assert!`] that can refer to generics inside +/// functions or implementation blocks. However, it also has a limitation where it can only appear +/// in places where statements can appear; for example, you cannot use it as an item in the module. +/// +/// [`static_assert!`] should be preferred if no generics are referred to in the condition. You +/// cannot refer to variables with [`const_assert!`] (even inside `const fn`); if you need the +/// capability, use [`build_assert!`]. See the [module documentation](self). +/// +/// # Examples +/// +/// ``` +/// fn foo<const N: usize>() { +/// const_assert!(N > 1); +/// } +/// +/// fn bar<T>() { +/// const_assert!(size_of::<T>() > 0, "T cannot be ZST"); +/// } +/// ``` +#[macro_export] #[doc(hidden)] -pub use build_error::build_error; +macro_rules! const_assert { + ($condition:expr $(,$arg:literal)?) => { + const { ::core::assert!($condition $(,$arg)?) }; + }; +} /// Fails the build if the code path calling `build_error!` can possibly be executed. /// @@ -23,12 +160,13 @@ pub use build_error::build_error; /// // foo(usize::MAX); // Fails to compile. /// ``` #[macro_export] +#[doc(hidden)] macro_rules! build_error { () => {{ - $crate::build_assert::build_error("") + $crate::build_assert::build_error_fn("") }}; ($msg:expr) => {{ - $crate::build_assert::build_error($msg) + $crate::build_assert::build_error_fn($msg) }}; } @@ -38,54 +176,44 @@ macro_rules! build_error { /// will panic. If the compiler or optimizer cannot guarantee the condition will /// be evaluated to `true`, a build error will be triggered. /// -/// [`static_assert!`] should be preferred to `build_assert!` whenever possible. +/// When a condition depends on a function argument, the function must be annotated with +/// `#[inline(always)]`. Without this attribute, the compiler may choose to not inline the +/// function, preventing it from optimizing out the error path. +/// +/// If the assertion condition does not depend on any variables or generics, you should use +/// [`static_assert!`]. If the assertion condition does not depend on variables, but does depend on +/// generics, you should use [`const_assert!`]. See the [module documentation](self). /// /// # Examples /// -/// These examples show that different types of [`assert!`] will trigger errors -/// at different stage of compilation. It is preferred to err as early as -/// possible, so [`static_assert!`] should be used whenever possible. -/// ```ignore -/// fn foo() { -/// static_assert!(1 > 1); // Compile-time error -/// build_assert!(1 > 1); // Build-time error -/// assert!(1 > 1); // Run-time error -/// } /// ``` +/// #[inline(always)] // Important. +/// fn bar(n: usize) { +/// build_assert!(n > 1); +/// } /// -/// When the condition refers to generic parameters or parameters of an inline function, -/// [`static_assert!`] cannot be used. Use `build_assert!` in this scenario. -/// ``` -/// fn foo<const N: usize>() { -/// // `static_assert!(N > 1);` is not allowed -/// build_assert!(N > 1); // Build-time check -/// assert!(N > 1); // Run-time check +/// fn foo() { +/// bar(2); /// } -/// ``` /// -/// When a condition depends on a function argument, the function must be annotated with -/// `#[inline(always)]`. Without this attribute, the compiler may choose to not inline the -/// function, preventing it from optimizing out the error path. -/// ``` -/// #[inline(always)] -/// fn bar(n: usize) { -/// // `static_assert!(n > 1);` is not allowed -/// build_assert!(n > 1); // Build-time check -/// assert!(n > 1); // Run-time check +/// #[inline(always)] // Important. +/// const fn const_bar(n: usize) { +/// build_assert!(n > 1); /// } -/// ``` /// -/// [`static_assert!`]: crate::static_assert! +/// const _: () = const_bar(2); +/// ``` #[macro_export] -macro_rules! build_assert { +#[doc(hidden)] +macro_rules! build_assert_macro { ($cond:expr $(,)?) => {{ if !$cond { - $crate::build_assert::build_error(concat!("assertion failed: ", stringify!($cond))); + $crate::build_assert::build_error_fn(concat!("assertion failed: ", stringify!($cond))); } }}; ($cond:expr, $msg:expr) => {{ if !$cond { - $crate::build_assert::build_error($msg); + $crate::build_assert::build_error_fn($msg); } }}; } diff --git a/rust/kernel/clk.rs b/rust/kernel/clk.rs index 4059aff34d09..7abbd0767d8c 100644 --- a/rust/kernel/clk.rs +++ b/rust/kernel/clk.rs @@ -128,6 +128,13 @@ mod common_clk { #[repr(transparent)] pub struct Clk(*mut bindings::clk); + // SAFETY: It is safe to call `clk_put` on another thread than where `clk_get` was called. + unsafe impl Send for Clk {} + + // SAFETY: It is safe to call any combination of the `&self` methods in parallel, as the + // methods are synchronized internally. + unsafe impl Sync for Clk {} + impl Clk { /// Gets [`Clk`] corresponding to a [`Device`] and a connection id. /// diff --git a/rust/kernel/configfs.rs b/rust/kernel/configfs.rs index 2339c6467325..cd082b83e9e7 100644 --- a/rust/kernel/configfs.rs +++ b/rust/kernel/configfs.rs @@ -875,13 +875,14 @@ impl<Container, Data> ItemType<Container, Data> { /// configfs::Subsystem<Configuration>, /// Configuration /// >::new_with_child_ctor::<N,Child>( -/// &THIS_MODULE, +/// ::kernel::module::this_module::<crate::LocalModule>(), /// &CONFIGURATION_ATTRS /// ); /// /// &CONFIGURATION_TPE /// } /// ``` +#[allow(clippy::crate_in_macro_def)] #[macro_export] macro_rules! configfs_attrs { ( @@ -1021,7 +1022,8 @@ macro_rules! configfs_attrs { static [< $data:upper _TPE >] : $crate::configfs::ItemType<$container, $data> = $crate::configfs::ItemType::<$container, $data>::new::<N>( - &THIS_MODULE, &[<$ data:upper _ATTRS >] + $crate::module::this_module::<crate::LocalModule>(), + &[<$ data:upper _ATTRS >] ); )? @@ -1030,7 +1032,8 @@ macro_rules! configfs_attrs { $crate::configfs::ItemType<$container, $data> = $crate::configfs::ItemType::<$container, $data>:: new_with_child_ctor::<N, $child>( - &THIS_MODULE, &[<$ data:upper _ATTRS >] + $crate::module::this_module::<crate::LocalModule>(), + &[<$ data:upper _ATTRS >] ); )? diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index f5adee48d40c..affa2b9490ef 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -361,23 +361,28 @@ impl TableBuilder { } } - /// Adds a new entry to the table. - pub fn add(&mut self, freq: Hertz, flags: u32, driver_data: u32) -> Result { + /// Adds a raw frequency-table entry. + fn push(&mut self, frequency: u32, flags: u32, driver_data: u32) -> Result { // Adds the new entry at the end of the vector. Ok(self.entries.push( bindings::cpufreq_frequency_table { flags, driver_data, - frequency: freq.as_khz() as u32, + frequency, }, GFP_KERNEL, )?) } + /// Adds a new entry to the table. + pub fn add(&mut self, freq: Hertz, flags: u32, driver_data: u32) -> Result { + self.push(freq.as_khz() as u32, flags, driver_data) + } + /// Consumes the [`TableBuilder`] and returns [`TableBox`]. pub fn to_table(mut self) -> Result<TableBox> { // Add last entry to the table. - self.add(Hertz(c_ulong::MAX), 0, 0)?; + self.push(bindings::CPUFREQ_TABLE_END as u32, 0, 0)?; TableBox::new(self.entries) } @@ -792,7 +797,13 @@ pub trait Driver { } /// Driver's `adjust_perf` callback. - fn adjust_perf(_policy: &mut Policy, _min_perf: usize, _target_perf: usize, _capacity: usize) { + fn adjust_perf( + _policy: &mut Policy, + _min_perf: usize, + _target_perf: usize, + _max_perf: usize, + _capacity: usize, + ) { build_error!(VTABLE_DEFAULT_ERROR) } @@ -817,7 +828,9 @@ pub trait Driver { } /// Driver's `bios_limit` callback. - fn bios_limit(_policy: &mut Policy, _limit: &mut u32) -> Result { + /// + /// Returns HW/BIOS max frequency limitations for the CPU. + fn bios_limit(_policy: &mut Policy) -> Result<u32> { build_error!(VTABLE_DEFAULT_ERROR) } @@ -888,12 +901,13 @@ pub trait Driver { /// /// impl platform::Driver for SampleDriver { /// type IdInfo = (); +/// type Data<'bound> = Self; /// const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = None; /// -/// fn probe( -/// pdev: &platform::Device<Core>, -/// _id_info: Option<&Self::IdInfo>, -/// ) -> impl PinInit<Self, Error> { +/// fn probe<'bound>( +/// pdev: &'bound platform::Device<Core<'_>>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit<Self, Error> + 'bound { /// cpufreq::Registration::<SampleDriver>::new_foreign_owned(pdev.as_ref())?; /// Ok(Self {}) /// } @@ -1257,18 +1271,18 @@ impl<T: Driver> Registration<T> { /// # Safety /// /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. unsafe extern "C" fn adjust_perf_callback( - cpu: c_uint, + ptr: *mut bindings::cpufreq_policy, min_perf: c_ulong, target_perf: c_ulong, + max_perf: c_ulong, capacity: c_ulong, ) { - // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. - let cpu_id = unsafe { CpuId::from_u32_unchecked(cpu) }; - - if let Ok(mut policy) = PolicyCpu::from_cpu(cpu_id) { - T::adjust_perf(&mut policy, min_perf, target_perf, capacity); - } + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::adjust_perf(policy, min_perf, target_perf, max_perf, capacity); } /// Driver's `get_intermediate` callback. @@ -1324,7 +1338,7 @@ impl<T: Driver> Registration<T> { // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. let cpu_id = unsafe { CpuId::from_u32_unchecked(cpu) }; - PolicyCpu::from_cpu(cpu_id).map_or(0, |mut policy| T::get(&mut policy).map_or(0, |f| f)) + PolicyCpu::from_cpu(cpu_id).map_or(0, |mut policy| T::get(&mut policy).unwrap_or(0)) } /// Driver's `update_limit` callback. @@ -1352,9 +1366,12 @@ impl<T: Driver> Registration<T> { from_result(|| { let mut policy = PolicyCpu::from_cpu(cpu_id)?; - + let val = T::bios_limit(&mut policy)?; // SAFETY: `limit` is guaranteed by the C code to be valid. - T::bios_limit(&mut policy, &mut (unsafe { *limit })).map(|()| 0) + unsafe { + *limit = val; + } + Ok(0) }) } diff --git a/rust/kernel/debugfs/traits.rs b/rust/kernel/debugfs/traits.rs index 8c39524b6a99..b295f8420abd 100644 --- a/rust/kernel/debugfs/traits.rs +++ b/rust/kernel/debugfs/traits.rs @@ -18,10 +18,6 @@ use crate::{ Arc, Mutex, // }, - transmute::{ - AsBytes, - FromBytes, // - }, uaccess::{ UserSliceReader, UserSliceWriter, // @@ -36,6 +32,8 @@ use core::{ str::FromStr, }; +use zerocopy::Immutable; + /// A trait for types that can be written into a string. /// /// This works very similarly to `Debug`, and is automatically implemented if `Debug` is @@ -76,8 +74,8 @@ pub trait BinaryWriter { ) -> Result<usize>; } -// Base implementation for any `T: AsBytes`. -impl<T: AsBytes> BinaryWriter for T { +// Base implementation for any `T: Immutable + IntoBytes`. +impl<T: Immutable + IntoBytes> BinaryWriter for T { fn write_to_slice( &self, writer: &mut UserSliceWriter, @@ -147,7 +145,7 @@ where // Delegate for `Vec<T, A>`. impl<T, A> BinaryWriter for Vec<T, A> where - T: AsBytes, + T: Immutable + IntoBytes, A: Allocator, { fn write_to_slice( @@ -155,14 +153,7 @@ where writer: &mut UserSliceWriter, offset: &mut file::Offset, ) -> Result<usize> { - let slice = self.as_slice(); - - // SAFETY: `T: AsBytes` allows us to treat `&[T]` as `&[u8]`. - let buffer = unsafe { - core::slice::from_raw_parts(slice.as_ptr().cast(), core::mem::size_of_val(slice)) - }; - - writer.write_slice_file(buffer, offset) + writer.write_slice_file(self.as_bytes(), offset) } } @@ -230,14 +221,14 @@ pub trait BinaryReaderMut { ) -> Result<usize>; } -// Base implementation for any `T: AsBytes + FromBytes`. -impl<T: AsBytes + FromBytes> BinaryReaderMut for T { +// Base implementation for any `T: FromBytes + IntoBytes`. +impl<T: FromBytes + IntoBytes> BinaryReaderMut for T { fn read_from_slice_mut( &mut self, reader: &mut UserSliceReader, offset: &mut file::Offset, ) -> Result<usize> { - reader.read_slice_file(self.as_bytes_mut(), offset) + reader.read_slice_file(self.as_mut_bytes(), offset) } } @@ -255,7 +246,7 @@ impl<T: ?Sized + BinaryReaderMut, A: Allocator> BinaryReaderMut for Box<T, A> { // Delegate for `Vec<T, A>`: Support a `Vec<T, A>` with an outer lock. impl<T, A> BinaryReaderMut for Vec<T, A> where - T: AsBytes + FromBytes, + T: FromBytes + IntoBytes, A: Allocator, { fn read_from_slice_mut( @@ -263,17 +254,7 @@ where reader: &mut UserSliceReader, offset: &mut file::Offset, ) -> Result<usize> { - let slice = self.as_mut_slice(); - - // SAFETY: `T: AsBytes + FromBytes` allows us to treat `&mut [T]` as `&mut [u8]`. - let buffer = unsafe { - core::slice::from_raw_parts_mut( - slice.as_mut_ptr().cast(), - core::mem::size_of_val(slice), - ) - }; - - reader.read_slice_file(buffer, offset) + reader.read_slice_file(self.as_mut_bytes(), offset) } } diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 94e0548e7687..2291d85b6849 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -15,16 +15,12 @@ use crate::{ }, // }; use core::{ - any::TypeId, marker::PhantomData, ptr, // }; pub mod property; -// Assert that we can `read()` / `write()` a `TypeId` instance from / into `struct driver_type`. -static_assert!(core::mem::size_of::<bindings::driver_type>() >= core::mem::size_of::<TypeId>()); - /// The core representation of a device in the kernel's driver model. /// /// This structure represents the Rust abstraction for a C `struct device`. A [`Device`] can either @@ -58,7 +54,8 @@ static_assert!(core::mem::size_of::<bindings::driver_type>() >= core::mem::size_ /// reference is valid in. For instance, the [`Bound`] context guarantees that the [`Device`] is /// bound to a driver for the entire duration of the existence of a [`Device<Bound>`] reference. /// -/// Other [`DeviceContext`] types besides [`Bound`] are [`Normal`], [`Core`] and [`CoreInternal`]. +/// Other [`DeviceContext`] types besides [`Bound`] are [`Normal`], [`Core`], [`CoreInternal`] and +/// [`BoundInternal`]. /// /// Unless selected otherwise [`Device`] defaults to the [`Normal`] [`DeviceContext`], which by /// itself has no additional requirements. @@ -205,30 +202,13 @@ impl Device { } } -impl Device<CoreInternal> { - fn set_type_id<T: 'static>(&self) { - // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. - let private = unsafe { (*self.as_raw()).p }; - - // SAFETY: For a bound device (implied by the `CoreInternal` device context), `private` is - // guaranteed to be a valid pointer to a `struct device_private`. - let driver_type = unsafe { &raw mut (*private).driver_type }; - - // SAFETY: `driver_type` is valid for (unaligned) writes of a `TypeId`. - unsafe { - driver_type - .cast::<TypeId>() - .write_unaligned(TypeId::of::<T>()) - }; - } - +impl<'a> Device<CoreInternal<'a>> { /// Store a pointer to the bound driver's private data. - pub fn set_drvdata<T: 'static>(&self, data: impl PinInit<T, Error>) -> Result { + pub fn set_drvdata<T>(&self, data: impl PinInit<T, Error>) -> Result { let data = KBox::pin_init(data, GFP_KERNEL)?; // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. unsafe { bindings::dev_set_drvdata(self.as_raw(), data.into_foreign().cast()) }; - self.set_type_id::<T>(); Ok(()) } @@ -239,7 +219,7 @@ impl Device<CoreInternal> { /// /// - The type `T` must match the type of the `ForeignOwnable` previously stored by /// [`Device::set_drvdata`]. - pub(crate) unsafe fn drvdata_obtain<T: 'static>(&self) -> Option<Pin<KBox<T>>> { + pub(crate) unsafe fn drvdata_obtain<T>(&self) -> Option<Pin<KBox<T>>> { // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; @@ -256,7 +236,9 @@ impl Device<CoreInternal> { // in `into_foreign()`. Some(unsafe { Pin::<KBox<T>>::from_foreign(ptr.cast()) }) } +} +impl<Ctx: InternalBoundContext> Device<Ctx> { /// Borrow the driver's private data bound to this [`Device`]. /// /// # Safety @@ -265,23 +247,7 @@ impl Device<CoreInternal> { /// device is fully unbound. /// - The type `T` must match the type of the `ForeignOwnable` previously stored by /// [`Device::set_drvdata`]. - pub unsafe fn drvdata_borrow<T: 'static>(&self) -> Pin<&T> { - // SAFETY: `drvdata_unchecked()` has the exact same safety requirements as the ones - // required by this method. - unsafe { self.drvdata_unchecked() } - } -} - -impl Device<Bound> { - /// Borrow the driver's private data bound to this [`Device`]. - /// - /// # Safety - /// - /// - Must only be called after a preceding call to [`Device::set_drvdata`] and before - /// the device is fully unbound. - /// - The type `T` must match the type of the `ForeignOwnable` previously stored by - /// [`Device::set_drvdata`]. - unsafe fn drvdata_unchecked<T: 'static>(&self) -> Pin<&T> { + pub unsafe fn drvdata_borrow<T>(&self) -> Pin<&T> { // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; @@ -292,45 +258,6 @@ impl Device<Bound> { // in `into_foreign()`. unsafe { Pin::<KBox<T>>::borrow(ptr.cast()) } } - - fn match_type_id<T: 'static>(&self) -> Result { - // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. - let private = unsafe { (*self.as_raw()).p }; - - // SAFETY: For a bound device, `private` is guaranteed to be a valid pointer to a - // `struct device_private`. - let driver_type = unsafe { &raw mut (*private).driver_type }; - - // SAFETY: - // - `driver_type` is valid for (unaligned) reads of a `TypeId`. - // - A bound device guarantees that `driver_type` contains a valid `TypeId` value. - let type_id = unsafe { driver_type.cast::<TypeId>().read_unaligned() }; - - if type_id != TypeId::of::<T>() { - return Err(EINVAL); - } - - Ok(()) - } - - /// Access a driver's private data. - /// - /// Returns a pinned reference to the driver's private data or [`EINVAL`] if it doesn't match - /// the asserted type `T`. - pub fn drvdata<T: 'static>(&self) -> Result<Pin<&T>> { - // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. - if unsafe { bindings::dev_get_drvdata(self.as_raw()) }.is_null() { - return Err(ENOENT); - } - - self.match_type_id::<T>()?; - - // SAFETY: - // - The above check of `dev_get_drvdata()` guarantees that we are called after - // `set_drvdata()`. - // - We've just checked that the type of the driver's private data is in fact `T`. - Ok(unsafe { self.drvdata_unchecked() }) - } } impl<Ctx: DeviceContext> Device<Ctx> { @@ -489,6 +416,17 @@ impl<Ctx: DeviceContext> Device<Ctx> { // defined as a `#[repr(transparent)]` wrapper around `fwnode_handle`. Some(unsafe { &*fwnode_handle.cast() }) } + + /// Returns the name of the device. + /// + /// This is the kobject name of the device, or its initial name if the kobject is not yet + /// available. + #[inline] + pub fn name(&self) -> &CStr { + // SAFETY: By its type invariant `self.as_raw()` is a valid pointer to a `struct device`. + // The returned string is valid for the lifetime of the device. + unsafe { CStr::from_char_ptr(bindings::dev_name(self.as_raw())) } + } } // SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic @@ -516,12 +454,17 @@ unsafe impl Send for Device {} // synchronization in `struct device`. unsafe impl Sync for Device {} +// SAFETY: Same as `Device<Normal>` -- the underlying `struct device` is the same; `Bound` is a +// zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device<Bound> {} + /// Marker trait for the context or scope of a bus specific device. /// /// [`DeviceContext`] is a marker trait for types representing the context of a bus specific /// [`Device`]. /// -/// The specific device context types are: [`CoreInternal`], [`Core`], [`Bound`] and [`Normal`]. +/// The specific device context types are: [`CoreInternal`], [`Core`], [`BoundInternal`], [`Bound`] +/// and [`Normal`]. /// /// [`DeviceContext`] types are hierarchical, which means that there is a strict hierarchy that /// defines which [`DeviceContext`] type can be derived from another. For instance, any @@ -530,6 +473,11 @@ unsafe impl Sync for Device {} /// The following enumeration illustrates the dereference hierarchy of [`DeviceContext`] types. /// /// - [`CoreInternal`] => [`Core`] => [`Bound`] => [`Normal`] +/// - [`BoundInternal`] => [`Bound`] => [`Normal`] +/// +/// Both [`CoreInternal`] and [`BoundInternal`] implement the [`InternalBoundContext`] trait, +/// which provides access to internal bus abstraction methods on [`Device`] that are not available +/// to drivers. /// /// Bus devices can automatically implement the dereference hierarchy by using /// [`impl_device_context_deref`]. @@ -556,7 +504,11 @@ pub struct Normal; /// callback it appears in. It is intended to be used for synchronization purposes. Bus device /// implementations can implement methods for [`Device<Core>`], such that they can only be called /// from bus callbacks. -pub struct Core; +/// +/// The lifetime `'a` is for "lifetime branding" purpose. Callbacks need to polymorphic over this +/// lifetime so the `&'bound Device<Core<'_>>` provided to them cannot outlive the scope of the +/// function. For this reason, it needs to be invariant. +pub struct Core<'a>(PhantomData<fn(&'a ()) -> &'a ()>); /// Semantically the same as [`Core`], but reserved for internal usage of the corresponding bus /// abstraction. @@ -567,7 +519,21 @@ pub struct Core; /// /// This context mainly exists to share generic [`Device`] infrastructure that should only be called /// from bus callbacks with bus abstractions, but without making them accessible for drivers. -pub struct CoreInternal; +/// +/// Lifetime `'a` is invariant for the same reason as [`Core`]. +pub struct CoreInternal<'a>(PhantomData<fn(&'a ()) -> &'a ()>); + +/// Semantically the same as [`Bound`], but reserved for internal usage of the corresponding bus +/// abstraction. +/// +/// The internal bound context is intended to be used in exactly the same way as the [`Bound`] +/// context, with the difference that this [`DeviceContext`] is internal to the corresponding bus +/// abstraction. +/// +/// This context exists for cases where the bus abstraction needs access to internal device +/// infrastructure (such as [`Device::drvdata_borrow`]), where [`CoreInternal`] would not be +/// justified. +pub struct BoundInternal; /// The [`Bound`] context is the [`DeviceContext`] of a bus specific device when it is guaranteed to /// be bound to a driver. @@ -575,7 +541,7 @@ pub struct CoreInternal; /// The bound context indicates that for the entire duration of the lifetime of a [`Device<Bound>`] /// reference, the [`Device`] is guaranteed to be bound to a driver. /// -/// Some APIs, such as [`dma::CoherentAllocation`] or [`Devres`] rely on the [`Device`] to be bound, +/// Some APIs, such as [`dma::Coherent`] or [`Devres`] rely on the [`Device`] to be bound, /// which can be proven with the [`Bound`] device context. /// /// Any abstraction that can guarantee a scope where the corresponding bus device is bound, should @@ -584,23 +550,35 @@ pub struct CoreInternal; /// /// [`Devres`]: kernel::devres::Devres /// [`Devres::access`]: kernel::devres::Devres::access -/// [`dma::CoherentAllocation`]: kernel::dma::CoherentAllocation +/// [`dma::Coherent`]: kernel::dma::Coherent pub struct Bound; mod private { pub trait Sealed {} impl Sealed for super::Bound {} - impl Sealed for super::Core {} - impl Sealed for super::CoreInternal {} + impl Sealed for super::BoundInternal {} + impl<'a> Sealed for super::Core<'a> {} + impl<'a> Sealed for super::CoreInternal<'a> {} impl Sealed for super::Normal {} } impl DeviceContext for Bound {} -impl DeviceContext for Core {} -impl DeviceContext for CoreInternal {} +impl DeviceContext for BoundInternal {} +impl<'a> DeviceContext for Core<'a> {} +impl<'a> DeviceContext for CoreInternal<'a> {} impl DeviceContext for Normal {} +/// Marker trait for [`DeviceContext`] types that have internal bound-level access. +/// +/// This trait is implemented by [`CoreInternal`] and [`BoundInternal`], allowing methods that +/// require internal bus abstraction access to a bound device to be generic over both contexts. +/// +/// Methods bounded by this trait are available to bus abstractions but not to drivers. +pub trait InternalBoundContext: DeviceContext {} +impl<'a> InternalBoundContext for CoreInternal<'a> {} +impl InternalBoundContext for BoundInternal {} + impl<Ctx: DeviceContext> AsRef<Device<Ctx>> for Device<Ctx> { #[inline] fn as_ref(&self) -> &Device<Ctx> { @@ -648,6 +626,22 @@ pub unsafe trait AsBusDevice<Ctx: DeviceContext>: AsRef<Device<Ctx>> { #[doc(hidden)] #[macro_export] macro_rules! __impl_device_context_deref { + (unsafe { $device:ident, <$lt:lifetime> $src:ty => $dst:ty }) => { + impl<$lt> ::core::ops::Deref for $device<$src> { + type Target = $device<$dst>; + + fn deref(&self) -> &Self::Target { + let ptr: *const Self = self; + + // CAST: `$device<$src>` and `$device<$dst>` transparently wrap the same type by the + // safety requirement of the macro. + let ptr = ptr.cast::<Self::Target>(); + + // SAFETY: `ptr` was derived from `&self`. + unsafe { &*ptr } + } + } + }; (unsafe { $device:ident, $src:ty => $dst:ty }) => { impl ::core::ops::Deref for $device<$src> { type Target = $device<$dst>; @@ -680,14 +674,21 @@ macro_rules! impl_device_context_deref { // `__impl_device_context_deref!`. ::kernel::__impl_device_context_deref!(unsafe { $device, - $crate::device::CoreInternal => $crate::device::Core + <'a> $crate::device::CoreInternal<'a> => $crate::device::Core<'a> }); // SAFETY: This macro has the exact same safety requirement as // `__impl_device_context_deref!`. ::kernel::__impl_device_context_deref!(unsafe { $device, - $crate::device::Core => $crate::device::Bound + <'a> $crate::device::Core<'a> => $crate::device::Bound + }); + + // SAFETY: This macro has the exact same safety requirement as + // `__impl_device_context_deref!`. + ::kernel::__impl_device_context_deref!(unsafe { + $device, + $crate::device::BoundInternal => $crate::device::Bound }); // SAFETY: This macro has the exact same safety requirement as @@ -702,6 +703,13 @@ macro_rules! impl_device_context_deref { #[doc(hidden)] #[macro_export] macro_rules! __impl_device_context_into_aref { + (<$lt:lifetime> $src:ty, $device:tt) => { + impl<$lt> ::core::convert::From<&$device<$src>> for $crate::sync::aref::ARef<$device> { + fn from(dev: &$device<$src>) -> Self { + (&**dev).into() + } + } + }; ($src:ty, $device:tt) => { impl ::core::convert::From<&$device<$src>> for $crate::sync::aref::ARef<$device> { fn from(dev: &$device<$src>) -> Self { @@ -716,8 +724,13 @@ macro_rules! __impl_device_context_into_aref { #[macro_export] macro_rules! impl_device_context_into_aref { ($device:tt) => { - ::kernel::__impl_device_context_into_aref!($crate::device::CoreInternal, $device); - ::kernel::__impl_device_context_into_aref!($crate::device::Core, $device); + ::kernel::__impl_device_context_into_aref!( + <'a> $crate::device::CoreInternal<'a>, $device + ); + ::kernel::__impl_device_context_into_aref!( + <'a> $crate::device::Core<'a>, $device + ); + ::kernel::__impl_device_context_into_aref!($crate::device::BoundInternal, $device); ::kernel::__impl_device_context_into_aref!($crate::device::Bound, $device); }; } @@ -726,9 +739,7 @@ macro_rules! impl_device_context_into_aref { #[macro_export] macro_rules! dev_printk { ($method:ident, $dev:expr, $($f:tt)*) => { - { - $crate::device::Device::$method($dev.as_ref(), $crate::prelude::fmt!($($f)*)) - } + $crate::device::Device::$method($dev.as_ref(), $crate::prelude::fmt!($($f)*)) } } @@ -755,7 +766,7 @@ macro_rules! dev_printk { /// ``` #[macro_export] macro_rules! dev_emerg { - ($($f:tt)*) => { $crate::dev_printk!(pr_emerg, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_emerg, $($f)*) } } /// Prints an alert-level message (level 1) prefixed with device information. @@ -781,7 +792,7 @@ macro_rules! dev_emerg { /// ``` #[macro_export] macro_rules! dev_alert { - ($($f:tt)*) => { $crate::dev_printk!(pr_alert, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_alert, $($f)*) } } /// Prints a critical-level message (level 2) prefixed with device information. @@ -807,7 +818,7 @@ macro_rules! dev_alert { /// ``` #[macro_export] macro_rules! dev_crit { - ($($f:tt)*) => { $crate::dev_printk!(pr_crit, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_crit, $($f)*) } } /// Prints an error-level message (level 3) prefixed with device information. @@ -833,7 +844,7 @@ macro_rules! dev_crit { /// ``` #[macro_export] macro_rules! dev_err { - ($($f:tt)*) => { $crate::dev_printk!(pr_err, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_err, $($f)*) } } /// Prints a warning-level message (level 4) prefixed with device information. @@ -859,7 +870,7 @@ macro_rules! dev_err { /// ``` #[macro_export] macro_rules! dev_warn { - ($($f:tt)*) => { $crate::dev_printk!(pr_warn, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_warn, $($f)*) } } /// Prints a notice-level message (level 5) prefixed with device information. @@ -885,7 +896,7 @@ macro_rules! dev_warn { /// ``` #[macro_export] macro_rules! dev_notice { - ($($f:tt)*) => { $crate::dev_printk!(pr_notice, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_notice, $($f)*) } } /// Prints an info-level message (level 6) prefixed with device information. @@ -911,7 +922,7 @@ macro_rules! dev_notice { /// ``` #[macro_export] macro_rules! dev_info { - ($($f:tt)*) => { $crate::dev_printk!(pr_info, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_info, $($f)*) } } /// Prints a debug-level message (level 7) prefixed with device information. @@ -937,5 +948,5 @@ macro_rules! dev_info { /// ``` #[macro_export] macro_rules! dev_dbg { - ($($f:tt)*) => { $crate::dev_printk!(pr_dbg, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_dbg, $($f)*) } } diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs index 8e9721446014..c81fca5b4986 100644 --- a/rust/kernel/device_id.rs +++ b/rust/kernel/device_id.rs @@ -5,7 +5,10 @@ //! Each bus / subsystem that matches device and driver through a bus / subsystem specific ID is //! expected to implement [`RawDeviceId`]. -use core::mem::MaybeUninit; +use core::{ + marker::PhantomData, + mem::MaybeUninit, // +}; /// Marker trait to indicate a Rust device ID type represents a corresponding C device ID type. /// @@ -47,112 +50,110 @@ pub unsafe trait RawDeviceIdIndex: RawDeviceId { /// The offset (in bytes) to the context/data field in the raw device ID. const DRIVER_DATA_OFFSET: usize; - /// The index stored at `DRIVER_DATA_OFFSET` of the implementor of the [`RawDeviceIdIndex`] - /// trait. - fn index(&self) -> usize; -} - -/// A zero-terminated device id array. -#[repr(C)] -pub struct RawIdArray<T: RawDeviceId, const N: usize> { - ids: [T::RawType; N], - sentinel: MaybeUninit<T::RawType>, -} + /// Obtain the data pointer stored inside the device ID. + /// + /// # Safety + /// + /// `&Self` must be stored inside a `IdArray<Self, U>`. + unsafe fn info_unchecked<U>(&self) -> &'static U { + // SAFETY: By safety requirement of the trait, this is `self.driver_data as *const U` and by + // the safety requirement of the function, this is stored in `IdArray<Self, U>` so is + // convertible to `&'static U`. + unsafe { + core::ptr::from_ref(self) + .byte_add(Self::DRIVER_DATA_OFFSET) + .cast::<&U>() + .read() + } + } -impl<T: RawDeviceId, const N: usize> RawIdArray<T, N> { - #[doc(hidden)] - pub const fn size(&self) -> usize { - core::mem::size_of::<Self>() + /// Obtain the data pointer stored inside the device ID. + /// + /// # Safety + /// + /// `&Self` must be stored inside a `IdArray<Self, U>`, or has NULL (or 0) as driver data. + unsafe fn info_unchecked_opt<U>(&self) -> Option<&'static U> { + // SAFETY: By safety requirement of the trait, this is `self.driver_data as *const U` and by + // the safety requirement of the function, if this is stored in `IdArray<Self, U>`, this is + // convertible to `Option<&'static U>`. Otherwise it is NULL which is `None` as + // `Option<&U>`. + unsafe { + core::ptr::from_ref(self) + .byte_add(Self::DRIVER_DATA_OFFSET) + .cast::<Option<&U>>() + .read() + } } } /// A zero-terminated device id array, followed by context data. #[repr(C)] -pub struct IdArray<T: RawDeviceId, U, const N: usize> { - raw_ids: RawIdArray<T, N>, - id_infos: [U; N], +pub struct IdArray<T: RawDeviceId, U: 'static, const N: usize> { + // This is `MaybeUninit<T::RawType>` so any bytes inside it can carry provenance in CTFE. + // If this were `T::RawType`, integer fields would not be able to contain pointers. + ids: [MaybeUninit<T::RawType>; N], + sentinel: MaybeUninit<T::RawType>, + phantom: PhantomData<&'static U>, } -impl<T: RawDeviceId, U, const N: usize> IdArray<T, U, N> { +// SAFETY: device ID is plain data plus a `&'static U` and can thus be sent between threads safely +// if `&U` can. +unsafe impl<T: RawDeviceId, U: Sync + 'static, const N: usize> Send for IdArray<T, U, N> {} + +// SAFETY: device ID is plain data plus a `&'static U` and can thus be shared between threads safely +// if `&U` can. +unsafe impl<T: RawDeviceId, U: Sync + 'static, const N: usize> Sync for IdArray<T, U, N> {} + +impl<T: RawDeviceId + RawDeviceIdIndex, U: 'static, const N: usize> IdArray<T, U, N> { /// Creates a new instance of the array. /// /// The contents are derived from the given identifiers and context information. - /// - /// # Safety - /// - /// `data_offset` as `None` is always safe. - /// If `data_offset` is `Some(data_offset)`, then: - /// - `data_offset` must be the correct offset (in bytes) to the context/data field - /// (e.g., the `driver_data` field) within the raw device ID structure. - /// - The field at `data_offset` must be correctly sized to hold a `usize`. - const unsafe fn build(ids: [(T, U); N], data_offset: Option<usize>) -> Self { + pub const fn new(ids: [(T, &'static U); N]) -> Self { let mut raw_ids = [const { MaybeUninit::<T::RawType>::uninit() }; N]; - let mut infos = [const { MaybeUninit::uninit() }; N]; let mut i = 0usize; while i < N { // SAFETY: by the safety requirement of `RawDeviceId`, we're guaranteed that `T` is // layout-wise compatible with `RawType`. raw_ids[i] = unsafe { core::mem::transmute_copy(&ids[i].0) }; - if let Some(data_offset) = data_offset { - // SAFETY: by the safety requirement of this function, this would be effectively - // `raw_ids[i].driver_data = i;`. - unsafe { - raw_ids[i] - .as_mut_ptr() - .byte_add(data_offset) - .cast::<usize>() - .write(i); - } + // SAFETY: by the safety requirement of `RawDeviceIdIndex`, this would be effectively + // `raw_ids[i].driver_data = ids[i].1;`. + unsafe { + raw_ids[i] + .as_mut_ptr() + .byte_add(T::DRIVER_DATA_OFFSET) + .cast::<&U>() + .write(ids[i].1); } - // SAFETY: this is effectively a move: `infos[i] = ids[i].1`. We make a copy here but - // later forget `ids`. - infos[i] = MaybeUninit::new(unsafe { core::ptr::read(&ids[i].1) }); i += 1; } core::mem::forget(ids); Self { - raw_ids: RawIdArray { - // SAFETY: this is effectively `array_assume_init`, which is unstable, so we use - // `transmute_copy` instead. We have initialized all elements of `raw_ids` so this - // `array_assume_init` is safe. - ids: unsafe { core::mem::transmute_copy(&raw_ids) }, - sentinel: MaybeUninit::zeroed(), - }, - // SAFETY: We have initialized all elements of `infos` so this `array_assume_init` is - // safe. - id_infos: unsafe { core::mem::transmute_copy(&infos) }, + ids: raw_ids, + sentinel: MaybeUninit::zeroed(), + phantom: PhantomData, } } +} +impl<T: RawDeviceId, const N: usize> IdArray<T, (), N> { /// Creates a new instance of the array without writing index values. /// /// The contents are derived from the given identifiers and context information. /// If the device implements [`RawDeviceIdIndex`], consider using [`IdArray::new`] instead. - pub const fn new_without_index(ids: [(T, U); N]) -> Self { - // SAFETY: Calling `Self::build` with `offset = None` is always safe, - // because no raw memory writes are performed in this case. - unsafe { Self::build(ids, None) } - } - - /// Reference to the contained [`RawIdArray`]. - pub const fn raw_ids(&self) -> &RawIdArray<T, N> { - &self.raw_ids - } -} + pub const fn new_without_index(ids: [T; N]) -> Self { + // SAFETY: `T` is layout-wise compatible with `T::RawType`, so is the array of them. + let raw_ids: [MaybeUninit<T::RawType>; N] = unsafe { core::mem::transmute_copy(&ids) }; + core::mem::forget(ids); -impl<T: RawDeviceId + RawDeviceIdIndex, U, const N: usize> IdArray<T, U, N> { - /// Creates a new instance of the array. - /// - /// The contents are derived from the given identifiers and context information. - pub const fn new(ids: [(T, U); N]) -> Self { - // SAFETY: by the safety requirement of `RawDeviceIdIndex`, - // `T::DRIVER_DATA_OFFSET` is guaranteed to be the correct offset (in bytes) to - // a field within `T::RawType`. - unsafe { Self::build(ids, Some(T::DRIVER_DATA_OFFSET)) } + Self { + ids: raw_ids, + sentinel: MaybeUninit::zeroed(), + phantom: PhantomData, + } } } @@ -165,12 +166,6 @@ impl<T: RawDeviceId + RawDeviceIdIndex, U, const N: usize> IdArray<T, U, N> { pub trait IdTable<T: RawDeviceId, U> { /// Obtain the pointer to the ID table. fn as_ptr(&self) -> *const T::RawType; - - /// Obtain the pointer to the bus specific device ID from an index. - fn id(&self, index: usize) -> &T::RawType; - - /// Obtain the pointer to the driver-specific information from an index. - fn info(&self, index: usize) -> &U; } impl<T: RawDeviceId, U, const N: usize> IdTable<T, U> for IdArray<T, U, N> { @@ -179,28 +174,45 @@ impl<T: RawDeviceId, U, const N: usize> IdTable<T, U> for IdArray<T, U, N> { // to access the sentinel. core::ptr::from_ref(self).cast() } - - fn id(&self, index: usize) -> &T::RawType { - &self.raw_ids.ids[index] - } - - fn info(&self, index: usize) -> &U { - &self.id_infos[index] - } } /// Create device table alias for modpost. #[macro_export] macro_rules! module_device_table { - ($table_type: literal, $module_table_name:ident, $table_name:ident) => { - #[rustfmt::skip] + ( + $table_type: literal, $device_id_ty: ty, + $table_name: ident, $id_info_type: ty, + [$(($id: expr, $info:expr $(,)?)),* $(,)?] + ) => { + #[export_name = + concat!("__mod_device_table__", ::core::line!(), + "__kmod_", module_path!(), + "__", $table_type, + "__", stringify!($table_name)) + ] + static $table_name: $crate::device_id::IdArray< + $device_id_ty, + $id_info_type, + { <[$device_id_ty]>::len(&[$($id,)*]) }, + > = $crate::device_id::IdArray::new([$(($id, &$info),)*]); + }; + + // Case for no ID info. + ( + $table_type: literal, $device_id_ty: ty, + $table_name: ident, @none, + [$($id: expr),* $(,)?] + ) => { #[export_name = - concat!("__mod_device_table__", line!(), + concat!("__mod_device_table__", ::core::line!(), "__kmod_", module_path!(), "__", $table_type, "__", stringify!($table_name)) ] - static $module_table_name: [::core::mem::MaybeUninit<u8>; $table_name.raw_ids().size()] = - unsafe { ::core::mem::transmute_copy($table_name.raw_ids()) }; + static $table_name: $crate::device_id::IdArray< + $device_id_ty, + (), + { <[$device_id_ty]>::len(&[$($id,)*]) }, + > = $crate::device_id::IdArray::new_without_index([$($id),*]); }; } diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index 6afe196be42c..d2924aaae008 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -21,11 +21,29 @@ use crate::{ sync::{ aref::ARef, rcu, - Arc, // + Arc, + Completion, // + }, + types::{ + CovariantForLt, + ForLt, + ForeignOwnable, + Opaque, // }, - types::ForeignOwnable, }; +/// Inner type that embeds a `struct devres_node` and the `Revocable<T>`. +#[repr(C)] +#[pin_data] +struct Inner<T> { + #[pin] + node: Opaque<bindings::devres_node>, + #[pin] + data: Revocable<T>, + #[pin] + revocation: Completion, +} + /// This abstraction is meant to be used by subsystems to containerize [`Device`] bound resources to /// manage their lifetime. /// @@ -40,12 +58,17 @@ use crate::{ /// After the [`Devres`] has been unbound it is not possible to access the encapsulated resource /// anymore. /// +/// When a [`Devres`] is dropped, it is guaranteed that `T` has been fully dropped by the time +/// [`Devres::drop`] returns, even if a concurrent revocation through the release callback is in +/// progress. +/// /// [`Devres`] users should make sure to simply free the corresponding backing resource in `T`'s /// [`Drop`] implementation. /// /// # Examples /// /// ```no_run +/// # #![cfg(CONFIG_HAS_IOMEM)] /// use kernel::{ /// bindings, /// device::{ @@ -55,17 +78,19 @@ use crate::{ /// devres::Devres, /// io::{ /// Io, -/// IoKnownSize, +/// IoBase, /// Mmio, /// MmioRaw, -/// PhysAddr, // +/// MmioBackend, +/// PhysAddr, +/// Region, // /// }, /// prelude::*, /// }; /// use core::ops::Deref; /// /// // See also [`pci::Bar`] for a real example. -/// struct IoMem<const SIZE: usize>(MmioRaw<SIZE>); +/// struct IoMem<const SIZE: usize>(MmioRaw<Region<SIZE>>); /// /// impl<const SIZE: usize> IoMem<SIZE> { /// /// # Safety @@ -80,7 +105,7 @@ use crate::{ /// return Err(ENOMEM); /// } /// -/// Ok(IoMem(MmioRaw::new(addr as usize, SIZE)?)) +/// Ok(IoMem(MmioRaw::new_region(addr as usize, SIZE)?)) /// } /// } /// @@ -91,12 +116,13 @@ use crate::{ /// } /// } /// -/// impl<const SIZE: usize> Deref for IoMem<SIZE> { -/// type Target = Mmio<SIZE>; +/// impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem<SIZE> { +/// type Backend = MmioBackend; +/// type Target = Region<SIZE>; /// -/// fn deref(&self) -> &Self::Target { +/// fn as_view(self) -> Mmio<'a, Region<SIZE>> { /// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`. -/// unsafe { Mmio::from_raw(&self.0) } +/// unsafe { Mmio::from_raw(self.0) } /// } /// } /// # fn no_run(dev: &Device<Bound>) -> Result<(), Error> { @@ -109,17 +135,69 @@ use crate::{ /// # Ok(()) /// # } /// ``` -pub struct Devres<T: Send> { +pub struct Devres<T: Send + 'static> { dev: ARef<Device>, - /// Pointer to [`Self::devres_callback`]. - /// - /// Has to be stored, since Rust does not guarantee to always return the same address for a - /// function. However, the C API uses the address as a key. - callback: unsafe extern "C" fn(*mut c_void), - data: Arc<Revocable<T>>, + inner: Arc<Inner<T>>, +} + +// Calling the FFI functions from the `base` module directly from the `Devres<T>` impl may result in +// them being called directly from driver modules. This happens since the Rust compiler will use +// monomorphisation, so it might happen that functions are instantiated within the calling driver +// module. For now, work around this with `#[inline(never)]` helpers. +// +// TODO: Remove once a more generic solution has been implemented. For instance, we may be able to +// leverage `bindgen` to take care of this depending on whether a symbol is (already) exported. +mod base { + use kernel::{ + bindings, + prelude::*, // + }; + + #[inline(never)] + #[allow(clippy::missing_safety_doc)] + pub(super) unsafe fn devres_node_init( + node: *mut bindings::devres_node, + release: bindings::dr_node_release_t, + free: bindings::dr_node_free_t, + ) { + // SAFETY: Safety requirements are the same as `bindings::devres_node_init`. + unsafe { bindings::devres_node_init(node, release, free) } + } + + #[inline(never)] + #[allow(clippy::missing_safety_doc)] + pub(super) unsafe fn devres_set_node_dbginfo( + node: *mut bindings::devres_node, + name: *const c_char, + size: usize, + ) { + // SAFETY: Safety requirements are the same as `bindings::devres_set_node_dbginfo`. + unsafe { bindings::devres_set_node_dbginfo(node, name, size) } + } + + #[inline(never)] + #[allow(clippy::missing_safety_doc)] + pub(super) unsafe fn devres_node_add( + dev: *mut bindings::device, + node: *mut bindings::devres_node, + ) { + // SAFETY: Safety requirements are the same as `bindings::devres_node_add`. + unsafe { bindings::devres_node_add(dev, node) } + } + + #[must_use] + #[inline(never)] + #[allow(clippy::missing_safety_doc)] + pub(super) unsafe fn devres_node_remove( + dev: *mut bindings::device, + node: *mut bindings::devres_node, + ) -> bool { + // SAFETY: Safety requirements are the same as `bindings::devres_node_remove`. + unsafe { bindings::devres_node_remove(dev, node) } + } } -impl<T: Send> Devres<T> { +impl<T: Send + 'static> Devres<T> { /// Creates a new [`Devres`] instance of the given `data`. /// /// The `data` encapsulated within the returned `Devres` instance' `data` will be @@ -128,58 +206,94 @@ impl<T: Send> Devres<T> { where Error: From<E>, { - let callback = Self::devres_callback; - let data = Arc::pin_init(Revocable::new(data), GFP_KERNEL)?; - let devres_data = data.clone(); + let inner = Arc::pin_init::<Error>( + try_pin_init!(Inner { + node <- Opaque::ffi_init(|node: *mut bindings::devres_node| { + // SAFETY: `node` is a valid pointer to an uninitialized `struct devres_node`. + unsafe { + base::devres_node_init( + node, + Some(Self::devres_node_release), + Some(Self::devres_node_free_node), + ) + }; + + // SAFETY: `node` is a valid pointer to an uninitialized `struct devres_node`. + unsafe { + base::devres_set_node_dbginfo( + node, + // TODO: Use `core::any::type_name::<T>()` once it is a `const fn`, + // such that we can convert the `&str` to a `&CStr` at compile-time. + c"Devres<T>".as_char_ptr(), + core::mem::size_of::<Revocable<T>>(), + ) + }; + }), + data <- Revocable::new(data), + revocation <- Completion::new(), + }), + GFP_KERNEL, + )?; // SAFETY: - // - `dev.as_raw()` is a pointer to a valid bound device. - // - `data` is guaranteed to be a valid for the duration of the lifetime of `Self`. - // - `devm_add_action()` is guaranteed not to call `callback` for the entire lifetime of - // `dev`. - to_result(unsafe { - bindings::devm_add_action( - dev.as_raw(), - Some(callback), - Arc::as_ptr(&data).cast_mut().cast(), - ) - })?; - - // `devm_add_action()` was successful and has consumed the reference count. - core::mem::forget(devres_data); + // - `dev` is a valid pointer to a bound `struct device`. + // - `node` is a valid pointer to a `struct devres_node`. + // - `devres_node_add()` is guaranteed not to call `devres_node_release()` for the entire + // lifetime of `dev`. + unsafe { base::devres_node_add(dev.as_raw(), inner.node.get()) }; + + // Take additional reference count for `devres_node_add()`. + core::mem::forget(inner.clone()); Ok(Self { dev: dev.into(), - callback, - data, + inner, }) } fn data(&self) -> &Revocable<T> { - &self.data + &self.inner.data + } + + #[allow(clippy::missing_safety_doc)] + unsafe extern "C" fn devres_node_release( + _dev: *mut bindings::device, + node: *mut bindings::devres_node, + ) { + let node = Opaque::cast_from(node); + + // SAFETY: `node` is in the same allocation as its container. + let inner = unsafe { kernel::container_of!(node, Inner<T>, node) }; + + // SAFETY: `inner` is a valid `Inner<T>` pointer. + let inner = unsafe { &*inner }; + + if inner.data.revoke() { + inner.revocation.complete_all(); + } else { + // Devres::drop() is concurrently revoking; wait for it to finish `drop_in_place()` + // before returning to `devres_release_all()`, ensuring `T` is fully torn down before + // the device finishes unbinding. + inner.revocation.wait_for_completion(); + } } #[allow(clippy::missing_safety_doc)] - unsafe extern "C" fn devres_callback(ptr: *mut kernel::ffi::c_void) { - // SAFETY: In `Self::new` we've passed a valid pointer of `Revocable<T>` to - // `devm_add_action()`, hence `ptr` must be a valid pointer to `Revocable<T>`. - let data = unsafe { Arc::from_raw(ptr.cast::<Revocable<T>>()) }; + unsafe extern "C" fn devres_node_free_node(node: *mut bindings::devres_node) { + let node = Opaque::cast_from(node); + + // SAFETY: `node` is in the same allocation as its container. + let inner = unsafe { kernel::container_of!(node, Inner<T>, node) }; - data.revoke(); + // SAFETY: `inner` points to the entire `Inner<T>` allocation. + drop(unsafe { Arc::from_raw(inner) }); } - fn remove_action(&self) -> bool { + fn remove_node(&self) -> bool { // SAFETY: - // - `self.dev` is a valid `Device`, - // - the `action` and `data` pointers are the exact same ones as given to - // `devm_add_action()` previously, - (unsafe { - bindings::devm_remove_action_nowarn( - self.dev.as_raw(), - Some(self.callback), - core::ptr::from_ref(self.data()).cast_mut().cast(), - ) - } == 0) + // - `self.device().as_raw()` is a valid pointer to a bound `struct device`. + // - `self.inner.node.get()` is a valid pointer to a `struct devres_node`. + unsafe { base::devres_node_remove(self.device().as_raw(), self.inner.node.get()) } } /// Return a reference of the [`Device`] this [`Devres`] instance has been created with. @@ -204,14 +318,11 @@ impl<T: Send> Devres<T> { /// use kernel::{ /// device::Core, /// devres::Devres, - /// io::{ - /// Io, - /// IoKnownSize, // - /// }, + /// io::Io, /// pci, // /// }; /// - /// fn from_core(dev: &pci::Device<Core>, devres: Devres<pci::Bar<0x4>>) -> Result { + /// fn from_core(dev: &pci::Device<Core<'_>>, devres: Devres<pci::Bar<'_, 0x4>>) -> Result { /// let bar = devres.access(dev.as_ref())?; /// /// let _ = bar.read32(0x0); @@ -256,22 +367,128 @@ unsafe impl<T: Send> Send for Devres<T> {} // SAFETY: `Devres` can be shared with any task, if `T: Sync`. unsafe impl<T: Send + Sync> Sync for Devres<T> {} -impl<T: Send> Drop for Devres<T> { +impl<T: Send + 'static> Drop for Devres<T> { fn drop(&mut self) { // SAFETY: When `drop` runs, it is guaranteed that nobody is accessing the revocable data // anymore, hence it is safe not to wait for the grace period to finish. if unsafe { self.data().revoke_nosync() } { - // We revoked `self.data` before the devres action did, hence try to remove it. - if self.remove_action() { + self.inner.revocation.complete_all(); + + // We revoked `self.data` before devres did, hence try to remove it. + if self.remove_node() { // SAFETY: In `Self::new` we have taken an additional reference count of `self.data` - // for `devm_add_action()`. Since `remove_action()` was successful, we have to drop + // for `devres_node_add()`. Since `remove_node()` was successful, we have to drop // this additional reference count. - drop(unsafe { Arc::from_raw(Arc::as_ptr(&self.data)) }); + drop(unsafe { Arc::from_raw(Arc::as_ptr(&self.inner)) }); } + } else { + // The release callback is concurrently revoking; wait for it to finish + // `drop_in_place()` of the wrapped object before returning. + self.inner.revocation.wait_for_completion(); } } } +/// Guard returned by [`DevresLt::try_access`]. +/// +/// Dereferences to `F::Of<'a>`, shortening the lifetime of the stored data to the guard's borrow +/// lifetime. +pub struct DevresGuard<'a, F: CovariantForLt>(RevocableGuard<'a, F::Of<'static>>); + +impl<'a, F: CovariantForLt> core::ops::Deref for DevresGuard<'a, F> { + type Target = F::Of<'a>; + + #[inline] + fn deref(&self) -> &Self::Target { + F::cast_ref(&*self.0) + } +} + +/// Device-managed resource with [`ForLt`](trait@ForLt)-aware access. +/// +/// `DevresLt` wraps [`Devres`] and shortens the stored `'static` lifetime to the caller's borrow +/// lifetime in all access methods. +/// +/// Types that implement [`trait@CovariantForLt`] get direct-reference accessors ([`Self::access`], +/// [`Self::try_access`]). Plain [`ForLt`](trait@ForLt) types use closure-based accessors +/// ([`Self::access_with`], [`Self::try_access_with`]). +pub struct DevresLt<F: ForLt>(Devres<F::Of<'static>>) +where + for<'a> F::Of<'a>: Send; + +impl<F: ForLt> DevresLt<F> +where + for<'a> F::Of<'a>: Send, +{ + /// Creates a new [`DevresLt`] instance of the given `data`. + /// + /// # Safety + /// + /// The data must remain valid for the device's full bound scope. [`DevresLt`] allows + /// access until the device is unbound, which may outlast `'a`. + pub unsafe fn new<'a, E>( + dev: &'a Device<Bound>, + data: impl PinInit<F::Of<'a>, E>, + ) -> Result<Self> + where + Error: From<E>, + { + // SAFETY: The caller guarantees the data is valid for the device's full bound scope. + // Lifetimes do not affect layout, so F::Of<'a> and F::Of<'static> have identical + // representation; casting the slot pointer is sound. + let data = unsafe { pin_init::cast_pin_init(data) }; + + Ok(Self(Devres::new(dev, data)?)) + } + + /// Return a reference of the [`Device`] this [`DevresLt`] instance has been created with. + #[inline] + pub fn device(&self) -> &Device { + self.0.device() + } + + /// Obtain `&F::Of<'_>`, bypassing the [`Revocable`], through a closure. + /// + /// This method works like [`DevresLt::access`](DevresLt::access) but accepts any + /// [`trait@ForLt`] type, not just [`trait@CovariantForLt`]. + #[inline] + pub fn access_with<R, G>(&self, dev: &Device<Bound>, f: G) -> Result<R> + where + G: for<'a> FnOnce(&F::Of<'a>) -> R, + { + self.0.access(dev).map(f) + } + + /// [`DevresLt`] accessor for [`Revocable::try_access_with`]. + #[inline] + pub fn try_access_with<R, G>(&self, f: G) -> Option<R> + where + G: for<'a> FnOnce(&F::Of<'a>) -> R, + { + self.0.data().try_access_with(f) + } +} + +impl<F: CovariantForLt> DevresLt<F> +where + for<'a> F::Of<'a>: Send, +{ + /// Obtain `&'a F::Of<'a>`, bypassing the [`Revocable`]. + /// + /// This method works like [`Devres::access`], but shortens the returned reference's lifetime + /// from `'static` to `'a` via [`CovariantForLt::cast_ref`]. + #[inline] + pub fn access<'a>(&'a self, dev: &'a Device<Bound>) -> Result<&'a F::Of<'a>> { + self.0.access(dev).map(F::cast_ref) + } + + /// [`DevresLt`] accessor for [`Revocable::try_access`]. + #[inline] + pub fn try_access(&self) -> Option<DevresGuard<'_, F>> { + self.0.data().try_access().map(DevresGuard) + } +} + /// Consume `data` and [`Drop::drop`] `data` once `dev` is unbound. fn register_foreign<P>(dev: &Device<Bound>, data: P) -> Result where diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index a396f8435739..2ce09f8e90c6 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -5,14 +5,39 @@ //! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h) use crate::{ - bindings, build_assert, device, - device::{Bound, Core}, - error::{to_result, Result}, + bindings, + debugfs, + device::{ + self, + Bound, + Core, // + }, + error::to_result, + fs::file, + io::{ + IoBackend, + IoBase, + IoCapable, + IoCopyable, + SysMem, + SysMemBackend, // + }, prelude::*, + ptr::KnownSize, sync::aref::ARef, - transmute::{AsBytes, FromBytes}, + transmute::{ + AsBytes, + FromBytes, // + }, + uaccess::UserSliceWriter, // +}; +use core::{ + ops::{ + Deref, + DerefMut, // + }, + ptr::NonNull, // }; -use core::ptr::NonNull; /// DMA address type. /// @@ -30,7 +55,7 @@ pub type DmaAddress = bindings::dma_addr_t; /// where the underlying bus is DMA capable, such as: #[cfg_attr(CONFIG_PCI, doc = "* [`pci::Device`](kernel::pci::Device)")] /// * [`platform::Device`](::kernel::platform::Device) -pub trait Device: AsRef<device::Device<Core>> { +pub trait Device<'a>: AsRef<device::Device<Core<'a>>> { /// Set up the device's DMA streaming addressing capabilities. /// /// This method is usually called once from `probe()` as soon as the device capabilities are @@ -39,7 +64,7 @@ pub trait Device: AsRef<device::Device<Core>> { /// # Safety /// /// This method must not be called concurrently with any DMA allocation or mapping primitives, - /// such as [`CoherentAllocation::alloc_attrs`]. + /// such as [`Coherent::zeroed`]. unsafe fn dma_set_mask(&self, mask: DmaMask) -> Result { // SAFETY: // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. @@ -56,7 +81,7 @@ pub trait Device: AsRef<device::Device<Core>> { /// # Safety /// /// This method must not be called concurrently with any DMA allocation or mapping primitives, - /// such as [`CoherentAllocation::alloc_attrs`]. + /// such as [`Coherent::zeroed`]. unsafe fn dma_set_coherent_mask(&self, mask: DmaMask) -> Result { // SAFETY: // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. @@ -75,7 +100,7 @@ pub trait Device: AsRef<device::Device<Core>> { /// # Safety /// /// This method must not be called concurrently with any DMA allocation or mapping primitives, - /// such as [`CoherentAllocation::alloc_attrs`]. + /// such as [`Coherent::zeroed`]. unsafe fn dma_set_mask_and_coherent(&self, mask: DmaMask) -> Result { // SAFETY: // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. @@ -94,7 +119,7 @@ pub trait Device: AsRef<device::Device<Core>> { /// # Safety /// /// This method must not be called concurrently with any DMA allocation or mapping primitives, - /// such as [`CoherentAllocation::alloc_attrs`]. + /// such as [`Coherent::zeroed`]. unsafe fn dma_set_max_seg_size(&self, size: u32) { // SAFETY: // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. @@ -194,12 +219,12 @@ impl DmaMask { /// /// ``` /// # use kernel::device::{Bound, Device}; -/// use kernel::dma::{attrs::*, CoherentAllocation}; +/// use kernel::dma::{attrs::*, Coherent}; /// /// # fn test(dev: &Device<Bound>) -> Result { /// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN; -/// let c: CoherentAllocation<u64> = -/// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, attribs)?; +/// let c: Coherent<[u64]> = +/// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, attribs)?; /// # Ok::<(), Error>(()) } /// ``` #[derive(Clone, Copy, PartialEq)] @@ -250,9 +275,6 @@ pub mod attrs { /// Specifies that writes to the mapping may be buffered to improve performance. pub const DMA_ATTR_WRITE_COMBINE: Attrs = Attrs(bindings::DMA_ATTR_WRITE_COMBINE); - /// Lets the platform to avoid creating a kernel virtual mapping for the allocated buffer. - pub const DMA_ATTR_NO_KERNEL_MAPPING: Attrs = Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING); - /// Allows platform code to skip synchronization of the CPU cache for the given buffer assuming /// that it has been already transferred to 'device' domain. pub const DMA_ATTR_SKIP_CPU_SYNC: Attrs = Attrs(bindings::DMA_ATTR_SKIP_CPU_SYNC); @@ -344,23 +366,228 @@ impl From<DataDirection> for bindings::dma_data_direction { } } +/// CPU-owned DMA allocation that can be converted into a device-shared [`Coherent`] object. +/// +/// Unlike [`Coherent`], a [`CoherentBox`] is guaranteed to be fully owned by the CPU -- its DMA +/// address is not exposed and it cannot be accessed by a device. This means it can safely be used +/// like a normal boxed allocation (e.g. direct reads, writes, and mutable slices are all safe). +/// +/// A typical use is to allocate a [`CoherentBox`], populate it with normal CPU access, and then +/// convert it into a [`Coherent`] object to share it with the device. +/// +/// # Examples +/// +/// `CoherentBox<T>`: +/// +/// ``` +/// # use kernel::device::{ +/// # Bound, +/// # Device, +/// # }; +/// use kernel::dma::{attrs::*, +/// Coherent, +/// CoherentBox, +/// }; +/// +/// # fn test(dev: &Device<Bound>) -> Result { +/// let mut dmem: CoherentBox<u64> = CoherentBox::zeroed(dev, GFP_KERNEL)?; +/// *dmem = 42; +/// let dmem: Coherent<u64> = dmem.into(); +/// # Ok::<(), Error>(()) } +/// ``` +/// +/// `CoherentBox<[T]>`: +/// +/// +/// ``` +/// # use kernel::device::{ +/// # Bound, +/// # Device, +/// # }; +/// use kernel::dma::{attrs::*, +/// Coherent, +/// CoherentBox, +/// }; +/// +/// # fn test(dev: &Device<Bound>) -> Result { +/// let mut dmem: CoherentBox<[u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?; +/// dmem.fill(42); +/// let dmem: Coherent<[u64]> = dmem.into(); +/// # Ok::<(), Error>(()) } +/// ``` +pub struct CoherentBox<T: KnownSize + ?Sized>(Coherent<T>); + +impl<T: AsBytes + FromBytes> CoherentBox<[T]> { + /// [`CoherentBox`] variant of [`Coherent::zeroed_slice_with_attrs`]. + #[inline] + pub fn zeroed_slice_with_attrs( + dev: &device::Device<Bound>, + count: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result<Self> { + Coherent::zeroed_slice_with_attrs(dev, count, gfp_flags, dma_attrs).map(Self) + } + + /// Same as [CoherentBox::zeroed_slice_with_attrs], but with `dma::Attrs(0)`. + #[inline] + pub fn zeroed_slice( + dev: &device::Device<Bound>, + count: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result<Self> { + Self::zeroed_slice_with_attrs(dev, count, gfp_flags, Attrs(0)) + } + + /// Initializes the element at `i` using the given initializer. + /// + /// Returns `EINVAL` if `i` is out of bounds. + pub fn init_at<E>(&mut self, i: usize, init: impl Init<T, E>) -> Result + where + Error: From<E>, + { + if i >= self.0.len() { + return Err(EINVAL); + } + + let ptr = &raw mut self[i]; + + // SAFETY: + // - `ptr` is valid, properly aligned, and within this allocation. + // - `T: AsBytes + FromBytes` guarantees all bit patterns are valid, so partial writes on + // error cannot leave the element in an invalid state. + // - The DMA address has not been exposed yet, so there is no concurrent device access. + unsafe { pin_init::raw_try_init(ptr, init)? }; + + Ok(()) + } + + /// Allocates a region of coherent memory of the same size as `data` and initializes it with a + /// copy of its contents. + /// + /// This is the [`CoherentBox`] variant of [`Coherent::from_slice_with_attrs`]. + /// + /// # Examples + /// + /// ``` + /// use core::ops::Deref; + /// + /// # use kernel::device::{Bound, Device}; + /// use kernel::dma::{ + /// attrs::*, + /// CoherentBox + /// }; + /// + /// # fn test(dev: &Device<Bound>) -> Result { + /// let data = [0u8, 1u8, 2u8, 3u8]; + /// let c: CoherentBox<[u8]> = + /// CoherentBox::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// + /// assert_eq!(c.deref(), &data); + /// # Ok::<(), Error>(()) } + /// ``` + pub fn from_slice_with_attrs( + dev: &device::Device<Bound>, + data: &[T], + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result<Self> + where + T: Copy, + { + let mut slice = Self(Coherent::<T>::alloc_slice_with_attrs( + dev, + data.len(), + gfp_flags, + dma_attrs, + )?); + + // PANIC: `slice` was created with length `data.len()`. + slice.copy_from_slice(data); + + Ok(slice) + } + + /// Performs the same functionality as [`CoherentBox::from_slice_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn from_slice( + dev: &device::Device<Bound>, + data: &[T], + gfp_flags: kernel::alloc::Flags, + ) -> Result<Self> + where + T: Copy, + { + Self::from_slice_with_attrs(dev, data, gfp_flags, Attrs(0)) + } +} + +impl<T: AsBytes + FromBytes> CoherentBox<T> { + /// Same as [`CoherentBox::zeroed_slice_with_attrs`], but for a single element. + #[inline] + pub fn zeroed_with_attrs( + dev: &device::Device<Bound>, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result<Self> { + Coherent::zeroed_with_attrs(dev, gfp_flags, dma_attrs).map(Self) + } + + /// Same as [`CoherentBox::zeroed_slice`], but for a single element. + #[inline] + pub fn zeroed(dev: &device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> { + Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) + } +} + +impl<T: KnownSize + ?Sized> Deref for CoherentBox<T> { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: + // - We have not exposed the DMA address yet, so there can't be any concurrent access by a + // device. + // - We have exclusive access to `self.0`. + unsafe { self.0.as_ref() } + } +} + +impl<T: AsBytes + FromBytes + KnownSize + ?Sized> DerefMut for CoherentBox<T> { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: + // - We have not exposed the DMA address yet, so there can't be any concurrent access by a + // device. + // - We have exclusive access to `self.0`. + unsafe { self.0.as_mut() } + } +} + +impl<T: AsBytes + FromBytes + KnownSize + ?Sized> From<CoherentBox<T>> for Coherent<T> { + #[inline] + fn from(value: CoherentBox<T>) -> Self { + value.0 + } +} + /// An abstraction of the `dma_alloc_coherent` API. /// /// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map /// large coherent DMA regions. /// -/// A [`CoherentAllocation`] instance contains a pointer to the allocated region (in the +/// A [`Coherent`] instance contains a pointer to the allocated region (in the /// processor's virtual address space) and the device address which can be given to the device -/// as the DMA address base of the region. The region is released once [`CoherentAllocation`] +/// as the DMA address base of the region. The region is released once [`Coherent`] /// is dropped. /// /// # Invariants /// -/// - For the lifetime of an instance of [`CoherentAllocation`], the `cpu_addr` is a valid pointer -/// to an allocated region of coherent memory and `dma_handle` is the DMA address base of the +/// - For the lifetime of an instance of [`Coherent`], the `cpu_addr` is a valid pointer +/// to an allocated region of coherent memory and `dma_addr` is the DMA address base of the /// region. -/// - The size in bytes of the allocation is equal to `size_of::<T> * count`. -/// - `size_of::<T> * count` fits into a `usize`. +/// - The size in bytes of the allocation is equal to size information via pointer. // TODO // // DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness @@ -371,380 +598,654 @@ impl From<DataDirection> for bindings::dma_data_direction { // allocation from surviving device unbind; it would require RCU read side critical sections to // access the memory, which may require subsequent unnecessary copies. // -// Hence, find a way to revoke the device resources of a `CoherentAllocation`, but not the -// entire `CoherentAllocation` including the allocated memory itself. -pub struct CoherentAllocation<T: AsBytes + FromBytes> { +// Hence, find a way to revoke the device resources of a `Coherent`, but not the +// entire `Coherent` including the allocated memory itself. +pub struct Coherent<T: KnownSize + ?Sized> { dev: ARef<device::Device>, - dma_handle: DmaAddress, - count: usize, + dma_addr: DmaAddress, cpu_addr: NonNull<T>, dma_attrs: Attrs, } -impl<T: AsBytes + FromBytes> CoherentAllocation<T> { - /// Allocates a region of `size_of::<T> * count` of coherent memory. +impl<T: KnownSize + ?Sized> Coherent<T> { + /// Returns the size in bytes of this allocation. + #[inline] + pub fn size(&self) -> usize { + T::size(self.cpu_addr.as_ptr()) + } + + /// Returns the raw pointer to the allocated region in the CPU's virtual address space. + #[inline] + pub fn as_ptr(&self) -> *const T { + self.cpu_addr.as_ptr() + } + + /// Returns the raw pointer to the allocated region in the CPU's virtual address space as + /// a mutable pointer. + #[inline] + pub fn as_mut_ptr(&self) -> *mut T { + self.cpu_addr.as_ptr() + } + + /// Returns a DMA address which may be given to the device as the base of the region. + #[inline] + pub fn dma_address(&self) -> DmaAddress { + self.dma_addr + } + + /// Returns a reference to the data in the region. /// - /// # Examples + /// # Safety /// - /// ``` - /// # use kernel::device::{Bound, Device}; - /// use kernel::dma::{attrs::*, CoherentAllocation}; + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// slice is live. + /// * Callers must ensure that this call does not race with a write to the same region while + /// the returned slice is live. + #[inline] + pub unsafe fn as_ref(&self) -> &T { + // SAFETY: per safety requirement. + unsafe { &*self.as_ptr() } + } + + /// Returns a mutable reference to the data in the region. /// - /// # fn test(dev: &Device<Bound>) -> Result { - /// let c: CoherentAllocation<u64> = - /// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; - /// # Ok::<(), Error>(()) } - /// ``` - pub fn alloc_attrs( + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// slice is live. + /// * Callers must ensure that this call does not race with a read or write to the same region + /// while the returned slice is live. + #[expect(clippy::mut_from_ref, reason = "unsafe to use API")] + #[inline] + pub unsafe fn as_mut(&self) -> &mut T { + // SAFETY: per safety requirement. + unsafe { &mut *self.as_mut_ptr() } + } +} + +impl<T: AsBytes + FromBytes> Coherent<T> { + /// Allocates a region of `T` of coherent memory. + fn alloc_with_attrs( dev: &device::Device<Bound>, - count: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, - ) -> Result<CoherentAllocation<T>> { - build_assert!( - core::mem::size_of::<T>() > 0, - "It doesn't make sense for the allocated type to be a ZST" - ); - - let size = count - .checked_mul(core::mem::size_of::<T>()) - .ok_or(EOVERFLOW)?; - let mut dma_handle = 0; + ) -> Result<Self> { + const { + assert!( + core::mem::size_of::<T>() > 0, + "It doesn't make sense for the allocated type to be a ZST" + ); + } + + let mut dma_addr = 0; // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. let addr = unsafe { bindings::dma_alloc_attrs( dev.as_raw(), - size, - &mut dma_handle, + core::mem::size_of::<T>(), + &mut dma_addr, gfp_flags.as_raw(), dma_attrs.as_raw(), ) }; - let addr = NonNull::new(addr).ok_or(ENOMEM)?; + let cpu_addr = NonNull::new(addr.cast()).ok_or(ENOMEM)?; // INVARIANT: - // - We just successfully allocated a coherent region which is accessible for - // `count` elements, hence the cpu address is valid. We also hold a refcounted reference - // to the device. - // - The allocated `size` is equal to `size_of::<T> * count`. - // - The allocated `size` fits into a `usize`. + // - We just successfully allocated a coherent region which is adequately sized for `T`, + // hence the cpu address is valid. + // - We also hold a refcounted reference to the device. Ok(Self { dev: dev.into(), - dma_handle, - count, - cpu_addr: addr.cast(), + dma_addr, + cpu_addr, dma_attrs, }) } - /// Performs the same functionality as [`CoherentAllocation::alloc_attrs`], except the - /// `dma_attrs` is 0 by default. - pub fn alloc_coherent( + /// Allocates a region of type `T` of coherent memory. + /// + /// # Examples + /// + /// ``` + /// # use kernel::device::{ + /// # Bound, + /// # Device, + /// # }; + /// use kernel::dma::{ + /// attrs::*, + /// Coherent, + /// }; + /// + /// # fn test(dev: &Device<Bound>) -> Result { + /// let c: Coherent<[u64; 4]> = + /// Coherent::zeroed_with_attrs(dev, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + pub fn zeroed_with_attrs( dev: &device::Device<Bound>, - count: usize, gfp_flags: kernel::alloc::Flags, - ) -> Result<CoherentAllocation<T>> { - CoherentAllocation::alloc_attrs(dev, count, gfp_flags, Attrs(0)) - } - - /// Returns the number of elements `T` in this allocation. - /// - /// Note that this is not the size of the allocation in bytes, which is provided by - /// [`Self::size`]. - pub fn count(&self) -> usize { - self.count - } - - /// Returns the size in bytes of this allocation. - pub fn size(&self) -> usize { - // INVARIANT: The type invariant of `Self` guarantees that `size_of::<T> * count` fits into - // a `usize`. - self.count * core::mem::size_of::<T>() + dma_attrs: Attrs, + ) -> Result<Self> { + Self::alloc_with_attrs(dev, gfp_flags | __GFP_ZERO, dma_attrs) } - /// Returns the raw pointer to the allocated region in the CPU's virtual address space. + /// Performs the same functionality as [`Coherent::zeroed_with_attrs`], except the + /// `dma_attrs` is 0 by default. #[inline] - pub fn as_ptr(&self) -> *const [T] { - core::ptr::slice_from_raw_parts(self.cpu_addr.as_ptr(), self.count) + pub fn zeroed(dev: &device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> { + Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) } - /// Returns the raw pointer to the allocated region in the CPU's virtual address space as - /// a mutable pointer. - #[inline] - pub fn as_mut_ptr(&self) -> *mut [T] { - core::ptr::slice_from_raw_parts_mut(self.cpu_addr.as_ptr(), self.count) - } + /// Same as [`Coherent::zeroed_with_attrs`], but instead of a zero-initialization the memory is + /// initialized with `init`. + pub fn init_with_attrs<E>( + dev: &device::Device<Bound>, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + init: impl Init<T, E>, + ) -> Result<Self> + where + Error: From<E>, + { + let dmem = Self::alloc_with_attrs(dev, gfp_flags, dma_attrs)?; + let ptr = dmem.as_mut_ptr(); - /// Returns the base address to the allocated region in the CPU's virtual address space. - pub fn start_ptr(&self) -> *const T { - self.cpu_addr.as_ptr() - } + // SAFETY: + // - `ptr` is valid, properly aligned, and points to exclusively owned memory. + // - If `raw_try_init` fails, `self` is dropped, which safely frees the underlying + // `Coherent`'s DMA memory. `T: AsBytes + FromBytes` ensures there are no complex `Drop` + // requirements we are bypassing. + unsafe { pin_init::raw_try_init(ptr, init)? }; - /// Returns the base address to the allocated region in the CPU's virtual address space as - /// a mutable pointer. - pub fn start_ptr_mut(&mut self) -> *mut T { - self.cpu_addr.as_ptr() + Ok(dmem) } - /// Returns a DMA handle which may be given to the device as the DMA address base of - /// the region. - pub fn dma_handle(&self) -> DmaAddress { - self.dma_handle + /// Same as [`Coherent::zeroed`], but instead of a zero-initialization the memory is initialized + /// with `init`. + #[inline] + pub fn init<E>( + dev: &device::Device<Bound>, + gfp_flags: kernel::alloc::Flags, + init: impl Init<T, E>, + ) -> Result<Self> + where + Error: From<E>, + { + Self::init_with_attrs(dev, gfp_flags, Attrs(0), init) } - /// Returns a DMA handle starting at `offset` (in units of `T`) which may be given to the - /// device as the DMA address base of the region. - /// - /// Returns `EINVAL` if `offset` is not within the bounds of the allocation. - pub fn dma_handle_with_offset(&self, offset: usize) -> Result<DmaAddress> { - if offset >= self.count { - Err(EINVAL) - } else { - // INVARIANT: The type invariant of `Self` guarantees that `size_of::<T> * count` fits - // into a `usize`, and `offset` is inferior to `count`. - Ok(self.dma_handle + (offset * core::mem::size_of::<T>()) as DmaAddress) + /// Allocates a region of `[T; len]` of coherent memory. + fn alloc_slice_with_attrs( + dev: &device::Device<Bound>, + len: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result<Coherent<[T]>> { + const { + assert!( + core::mem::size_of::<T>() > 0, + "It doesn't make sense for the allocated type to be a ZST" + ); } - } - /// Common helper to validate a range applied from the allocated region in the CPU's virtual - /// address space. - fn validate_range(&self, offset: usize, count: usize) -> Result { - if offset.checked_add(count).ok_or(EOVERFLOW)? > self.count { - return Err(EINVAL); + // `dma_alloc_attrs` cannot handle zero-length allocation, bail early. + if len == 0 { + Err(EINVAL)?; } - Ok(()) + + let size = core::mem::size_of::<T>().checked_mul(len).ok_or(ENOMEM)?; + let mut dma_addr = 0; + // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. + let addr = unsafe { + bindings::dma_alloc_attrs( + dev.as_raw(), + size, + &mut dma_addr, + gfp_flags.as_raw(), + dma_attrs.as_raw(), + ) + }; + let cpu_addr = NonNull::slice_from_raw_parts(NonNull::new(addr.cast()).ok_or(ENOMEM)?, len); + // INVARIANT: + // - We just successfully allocated a coherent region which is adequately sized for + // `[T; len]`, hence the cpu address is valid. + // - We also hold a refcounted reference to the device. + Ok(Coherent { + dev: dev.into(), + dma_addr, + cpu_addr, + dma_attrs, + }) } - /// Returns the data from the region starting from `offset` as a slice. - /// `offset` and `count` are in units of `T`, not the number of bytes. + /// Allocates a zeroed region of type `T` of coherent memory. /// - /// For ringbuffer type of r/w access or use-cases where the pointer to the live data is needed, - /// [`CoherentAllocation::start_ptr`] or [`CoherentAllocation::start_ptr_mut`] could be used - /// instead. + /// Unlike `Coherent::<[T; N]>::zeroed_with_attrs`, `Coherent::<T>::zeroed_slices` support + /// a runtime length. /// - /// # Safety + /// # Examples /// - /// * Callers must ensure that the device does not read/write to/from memory while the returned - /// slice is live. - /// * Callers must ensure that this call does not race with a write to the same region while - /// the returned slice is live. - pub unsafe fn as_slice(&self, offset: usize, count: usize) -> Result<&[T]> { - self.validate_range(offset, count)?; - // SAFETY: - // - The pointer is valid due to type invariant on `CoherentAllocation`, - // we've just checked that the range and index is within bounds. The immutability of the - // data is also guaranteed by the safety requirements of the function. - // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked - // that `self.count` won't overflow early in the constructor. - Ok(unsafe { core::slice::from_raw_parts(self.start_ptr().add(offset), count) }) + /// ``` + /// # use kernel::device::{ + /// # Bound, + /// # Device, + /// # }; + /// use kernel::dma::{ + /// attrs::*, + /// Coherent, + /// }; + /// + /// # fn test(dev: &Device<Bound>) -> Result { + /// let c: Coherent<[u64]> = + /// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + pub fn zeroed_slice_with_attrs( + dev: &device::Device<Bound>, + len: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result<Coherent<[T]>> { + Coherent::alloc_slice_with_attrs(dev, len, gfp_flags | __GFP_ZERO, dma_attrs) } - /// Performs the same functionality as [`CoherentAllocation::as_slice`], except that a mutable - /// slice is returned. - /// - /// # Safety - /// - /// * Callers must ensure that the device does not read/write to/from memory while the returned - /// slice is live. - /// * Callers must ensure that this call does not race with a read or write to the same region - /// while the returned slice is live. - pub unsafe fn as_slice_mut(&mut self, offset: usize, count: usize) -> Result<&mut [T]> { - self.validate_range(offset, count)?; - // SAFETY: - // - The pointer is valid due to type invariant on `CoherentAllocation`, - // we've just checked that the range and index is within bounds. The immutability of the - // data is also guaranteed by the safety requirements of the function. - // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked - // that `self.count` won't overflow early in the constructor. - Ok(unsafe { core::slice::from_raw_parts_mut(self.start_ptr_mut().add(offset), count) }) + /// Performs the same functionality as [`Coherent::zeroed_slice_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn zeroed_slice( + dev: &device::Device<Bound>, + len: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result<Coherent<[T]>> { + Self::zeroed_slice_with_attrs(dev, len, gfp_flags, Attrs(0)) } - /// Writes data to the region starting from `offset`. `offset` is in units of `T`, not the - /// number of bytes. - /// - /// # Safety - /// - /// * Callers must ensure that this call does not race with a read or write to the same region - /// that overlaps with this write. + /// Allocates a region of coherent memory of the same size as `data` and initializes it with a + /// copy of its contents. /// /// # Examples /// /// ``` - /// # fn test(alloc: &mut kernel::dma::CoherentAllocation<u8>) -> Result { - /// let somedata: [u8; 4] = [0xf; 4]; - /// let buf: &[u8] = &somedata; - /// // SAFETY: There is no concurrent HW operation on the device and no other R/W access to the - /// // region. - /// unsafe { alloc.write(buf, 0)?; } + /// # use kernel::device::{Bound, Device}; + /// use kernel::dma::{ + /// attrs::*, + /// Coherent + /// }; + /// + /// # fn test(dev: &Device<Bound>) -> Result { + /// let data = [0u8, 1u8, 2u8, 3u8]; + /// // `c` has the same content as `data`. + /// let c: Coherent<[u8]> = + /// Coherent::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// /// # Ok::<(), Error>(()) } /// ``` - pub unsafe fn write(&mut self, src: &[T], offset: usize) -> Result { - self.validate_range(offset, src.len())?; - // SAFETY: - // - The pointer is valid due to type invariant on `CoherentAllocation` - // and we've just checked that the range and index is within bounds. - // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked - // that `self.count` won't overflow early in the constructor. - unsafe { - core::ptr::copy_nonoverlapping( - src.as_ptr(), - self.start_ptr_mut().add(offset), - src.len(), - ) - }; - Ok(()) + #[inline] + pub fn from_slice_with_attrs( + dev: &device::Device<Bound>, + data: &[T], + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result<Coherent<[T]>> + where + T: Copy, + { + CoherentBox::from_slice_with_attrs(dev, data, gfp_flags, dma_attrs).map(Into::into) } - /// Reads the value of `field` and ensures that its type is [`FromBytes`]. - /// - /// # Safety - /// - /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is - /// validated beforehand. - /// - /// Public but hidden since it should only be used from [`dma_read`] macro. - #[doc(hidden)] - pub unsafe fn field_read<F: FromBytes>(&self, field: *const F) -> F { - // SAFETY: - // - By the safety requirements field is valid. - // - Using read_volatile() here is not sound as per the usual rules, the usage here is - // a special exception with the following notes in place. When dealing with a potential - // race from a hardware or code outside kernel (e.g. user-space program), we need that - // read on a valid memory is not UB. Currently read_volatile() is used for this, and the - // rationale behind is that it should generate the same code as READ_ONCE() which the - // kernel already relies on to avoid UB on data races. Note that the usage of - // read_volatile() is limited to this particular case, it cannot be used to prevent - // the UB caused by racing between two kernel functions nor do they provide atomicity. - unsafe { field.read_volatile() } + /// Performs the same functionality as [`Coherent::from_slice_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn from_slice( + dev: &device::Device<Bound>, + data: &[T], + gfp_flags: kernel::alloc::Flags, + ) -> Result<Coherent<[T]>> + where + T: Copy, + { + Self::from_slice_with_attrs(dev, data, gfp_flags, Attrs(0)) } +} - /// Writes a value to `field` and ensures that its type is [`AsBytes`]. - /// - /// # Safety - /// - /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is - /// validated beforehand. +impl<T> Coherent<[T]> { + /// Returns the number of elements `T` in this allocation. /// - /// Public but hidden since it should only be used from [`dma_write`] macro. - #[doc(hidden)] - pub unsafe fn field_write<F: AsBytes>(&self, field: *mut F, val: F) { - // SAFETY: - // - By the safety requirements field is valid. - // - Using write_volatile() here is not sound as per the usual rules, the usage here is - // a special exception with the following notes in place. When dealing with a potential - // race from a hardware or code outside kernel (e.g. user-space program), we need that - // write on a valid memory is not UB. Currently write_volatile() is used for this, and the - // rationale behind is that it should generate the same code as WRITE_ONCE() which the - // kernel already relies on to avoid UB on data races. Note that the usage of - // write_volatile() is limited to this particular case, it cannot be used to prevent - // the UB caused by racing between two kernel functions nor do they provide atomicity. - unsafe { field.write_volatile(val) } + /// Note that this is not the size of the allocation in bytes, which is provided by + /// [`Self::size`]. + #[inline] + #[expect(clippy::len_without_is_empty, reason = "Coherent slice is never empty")] + pub fn len(&self) -> usize { + self.cpu_addr.len() } } /// Note that the device configured to do DMA must be halted before this object is dropped. -impl<T: AsBytes + FromBytes> Drop for CoherentAllocation<T> { +impl<T: KnownSize + ?Sized> Drop for Coherent<T> { fn drop(&mut self) { - let size = self.count * core::mem::size_of::<T>(); + let size = T::size(self.cpu_addr.as_ptr()); // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. - // The cpu address, and the dma handle are valid due to the type invariants on - // `CoherentAllocation`. + // The cpu address, and the dma address are valid due to the type invariants on + // `Coherent`. unsafe { bindings::dma_free_attrs( self.dev.as_raw(), size, - self.start_ptr_mut().cast(), - self.dma_handle, + self.cpu_addr.as_ptr().cast(), + self.dma_addr, self.dma_attrs.as_raw(), ) } } } -// SAFETY: It is safe to send a `CoherentAllocation` to another thread if `T` +// SAFETY: It is safe to send a `Coherent` to another thread if `T` // can be sent to another thread. -unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {} +unsafe impl<T: KnownSize + Send + ?Sized> Send for Coherent<T> {} -/// Reads a field of an item from an allocated region of structs. -/// -/// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating -/// to a [`CoherentAllocation`] and `proj` is a [projection specification](kernel::ptr::project!). -/// -/// # Examples -/// -/// ``` -/// use kernel::device::Device; -/// use kernel::dma::{attrs::*, CoherentAllocation}; -/// -/// struct MyStruct { field: u32, } -/// -/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. -/// unsafe impl kernel::transmute::FromBytes for MyStruct{}; -/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. -/// unsafe impl kernel::transmute::AsBytes for MyStruct{}; -/// -/// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result { -/// let whole = kernel::dma_read!(alloc, [2]?); -/// let field = kernel::dma_read!(alloc, [1]?.field); -/// # Ok::<(), Error>(()) } -/// ``` -#[macro_export] -macro_rules! dma_read { - ($dma:expr, $($proj:tt)*) => {{ - let dma = &$dma; - let ptr = $crate::ptr::project!( - $crate::dma::CoherentAllocation::as_ptr(dma), $($proj)* - ); - // SAFETY: The pointer created by the projection is within the DMA region. - unsafe { $crate::dma::CoherentAllocation::field_read(dma, ptr) } - }}; +// SAFETY: Sharing `&Coherent` across threads is safe if `T` is `Sync`, because all +// methods that access the buffer contents (`field_read`, `field_write`, `as_slice`, +// `as_slice_mut`) are `unsafe`, and callers are responsible for ensuring no data races occur. +// The safe methods only return metadata or raw pointers whose use requires `unsafe`. +unsafe impl<T: KnownSize + ?Sized + AsBytes + FromBytes + Sync> Sync for Coherent<T> {} + +impl<T: KnownSize + AsBytes + ?Sized> debugfs::BinaryWriter for Coherent<T> { + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result<usize> { + if offset.is_negative() { + return Err(EINVAL); + } + + // If the offset is too large for a usize (e.g. on 32-bit platforms), + // then consider that as past EOF and just return 0 bytes. + let Ok(offset_val) = usize::try_from(*offset) else { + return Ok(0); + }; + + if offset_val >= self.size() { + return Ok(0); + } + + let count = (self.size() - offset_val).min(writer.len()); + + writer.write_dma(self, offset_val, count)?; + + *offset += count as i64; + Ok(count) + } } -/// Writes to a field of an item from an allocated region of structs. +/// An opaque DMA allocation without a kernel virtual mapping. /// -/// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression -/// evaluating to a [`CoherentAllocation`], `proj` is a -/// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the -/// projected location. +/// Unlike [`Coherent`], a `CoherentHandle` does not provide CPU access to the allocated memory. +/// The allocation is always performed with `DMA_ATTR_NO_KERNEL_MAPPING`, meaning no kernel +/// virtual mapping is created for the buffer. The value returned by the C API as the CPU +/// address is an opaque handle used only to free the allocation. /// -/// # Examples +/// This is useful for buffers that are only ever accessed by hardware. /// -/// ``` -/// use kernel::device::Device; -/// use kernel::dma::{attrs::*, CoherentAllocation}; -/// -/// struct MyStruct { member: u32, } +/// # Invariants /// -/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. -/// unsafe impl kernel::transmute::FromBytes for MyStruct{}; -/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. -/// unsafe impl kernel::transmute::AsBytes for MyStruct{}; +/// - `cpu_handle` holds the opaque handle returned by `dma_alloc_attrs` with +/// `DMA_ATTR_NO_KERNEL_MAPPING` set, and is only valid for passing back to `dma_free_attrs`. +/// - `dma_addr` is the corresponding bus address for device DMA. +/// - `size` is the allocation size in bytes as passed to `dma_alloc_attrs`. +/// - `dma_attrs` contains the attributes used for the allocation, always including +/// `DMA_ATTR_NO_KERNEL_MAPPING`. +pub struct CoherentHandle { + dev: ARef<device::Device>, + dma_addr: DmaAddress, + cpu_handle: NonNull<c_void>, + size: usize, + dma_attrs: Attrs, +} + +impl CoherentHandle { + /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping. + /// + /// Additional DMA attributes may be passed via `dma_attrs`; `DMA_ATTR_NO_KERNEL_MAPPING` is + /// always set implicitly. + /// + /// Returns `EINVAL` if `size` is zero, `ENOMEM` if the allocation fails. + pub fn alloc_with_attrs( + dev: &device::Device<Bound>, + size: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result<Self> { + if size == 0 { + return Err(EINVAL); + } + + let dma_attrs = dma_attrs | Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING); + let mut dma_addr = 0; + // SAFETY: `dev.as_raw()` is valid by the type invariant on `device::Device`. + let cpu_handle = unsafe { + bindings::dma_alloc_attrs( + dev.as_raw(), + size, + &mut dma_addr, + gfp_flags.as_raw(), + dma_attrs.as_raw(), + ) + }; + + let cpu_handle = NonNull::new(cpu_handle).ok_or(ENOMEM)?; + + // INVARIANT: `cpu_handle` is the opaque handle from a successful `dma_alloc_attrs` call + // with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_addr` is the corresponding DMA address, + // and we hold a refcounted reference to the device. + Ok(Self { + dev: dev.into(), + dma_addr, + cpu_handle, + size, + dma_attrs, + }) + } + + /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping. + #[inline] + pub fn alloc( + dev: &device::Device<Bound>, + size: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result<Self> { + Self::alloc_with_attrs(dev, size, gfp_flags, Attrs(0)) + } + + /// Returns the DMA address for this allocation. + /// + /// This address can be programmed into device hardware for DMA access. + #[inline] + pub fn dma_address(&self) -> DmaAddress { + self.dma_addr + } + + /// Returns the size in bytes of this allocation. + #[inline] + pub fn size(&self) -> usize { + self.size + } +} + +impl Drop for CoherentHandle { + fn drop(&mut self) { + // SAFETY: All values are valid by the type invariants on `CoherentHandle`. + // `cpu_handle` is the opaque handle from `dma_alloc_attrs` and is passed back unchanged. + unsafe { + bindings::dma_free_attrs( + self.dev.as_raw(), + self.size, + self.cpu_handle.as_ptr(), + self.dma_addr, + self.dma_attrs.as_raw(), + ) + } + } +} + +// SAFETY: `CoherentHandle` only holds a device reference, a DMA address, an opaque CPU handle, +// and a size. None of these are tied to a specific thread. +unsafe impl Send for CoherentHandle {} + +// SAFETY: `CoherentHandle` provides no CPU access to the underlying allocation. The only +// operations on `&CoherentHandle` are reading the DMA address and size, both of which are +// plain `Copy` values. +unsafe impl Sync for CoherentHandle {} + +/// View type for `Coherent`. /// -/// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result { -/// kernel::dma_write!(alloc, [2]?.member, 0xf); -/// kernel::dma_write!(alloc, [1]?, MyStruct { member: 0xf }); -/// # Ok::<(), Error>(()) } -/// ``` -#[macro_export] -macro_rules! dma_write { - (@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{ - let dma = &$dma; - let ptr = $crate::ptr::project!( - mut $crate::dma::CoherentAllocation::as_mut_ptr(dma), $($proj)* - ); - let val = $val; - // SAFETY: The pointer created by the projection is within the DMA region. - unsafe { $crate::dma::CoherentAllocation::field_write(dma, ptr, val) } - }}; - (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { - $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*]) - }; - (@parse [$dma:expr] [$($proj:tt)*] [[$index:expr]? $($rest:tt)*]) => { - $crate::dma_write!(@parse [$dma] [$($proj)* [$index]?] [$($rest)*]) - }; - (@parse [$dma:expr] [$($proj:tt)*] [[$index:expr] $($rest:tt)*]) => { - $crate::dma_write!(@parse [$dma] [$($proj)* [$index]] [$($rest)*]) - }; - ($dma:expr, $($rest:tt)*) => { - $crate::dma_write!(@parse [$dma] [] [$($rest)*]) - }; +/// This is same as [`SysMem`] but with additional information that allows handing out a DMA +/// address. +pub struct CoherentView<'a, T: ?Sized> { + cpu_addr: SysMem<'a, T>, + dma_addr: DmaAddress, +} + +impl<T: ?Sized> Copy for CoherentView<'_, T> {} +impl<T: ?Sized> Clone for CoherentView<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> CoherentView<'a, T> { + /// Erase the DMA address information and obtain a [`SysMem`] view of the same memory region. + #[inline] + pub fn as_sys_mem(self) -> SysMem<'a, T> { + self.cpu_addr + } + + /// Returns the DMA address which may be given to the device as base of the region. + #[inline] + pub fn dma_address(self) -> DmaAddress { + self.dma_addr + } + + /// Returns a reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// reference is live. + /// * Callers must ensure that this call does not race with a write (including call to `as_mut`) + /// to the same region while the returned reference is live. + #[inline] + pub unsafe fn as_ref(self) -> &'a T { + // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per + // safety requirement. + unsafe { &*self.cpu_addr.as_ptr() } + } + + /// Returns a mutable reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// reference is live. + /// * Callers must ensure that this call does not race with a read (including call to `as_ref`) + /// or write (including call to `as_mut`) to the same region while the returned reference is + /// live. + #[inline] + pub unsafe fn as_mut(self) -> &'a mut T { + // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per + // safety requirement. + unsafe { &mut *self.cpu_addr.as_ptr() } + } +} + +/// `IoBackend` implementation for `Coherent`. +pub struct CoherentIoBackend; + +impl IoBackend for CoherentIoBackend { + type View<'a, T: ?Sized + KnownSize> = CoherentView<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + SysMemBackend::as_ptr(view.cpu_addr) + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + let offset = ptr.addr() - view.cpu_addr.as_ptr().addr(); + // CAST: The offset DMA address can never overflow. + let dma_addr = view.dma_addr + offset as DmaAddress; + CoherentView { + dma_addr, + // SAFETY: Per safety requirement. + cpu_addr: unsafe { SysMemBackend::project_view(view.cpu_addr, ptr) }, + } + } +} + +impl<T> IoCapable<T> for CoherentIoBackend +where + SysMemBackend: IoCapable<T>, +{ + #[inline] + fn io_read<'a>(view: Self::View<'a, T>) -> T { + SysMemBackend::io_read(view.cpu_addr) + } + + #[inline] + fn io_write<'a>(view: Self::View<'a, T>, value: T) { + SysMemBackend::io_write(view.cpu_addr, value) + } +} + +impl IoCopyable for CoherentIoBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // SAFETY: Per safety requirement. + unsafe { SysMemBackend::copy_from_io(view.cpu_addr, buffer) } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // SAFETY: Per safety requirement. + unsafe { SysMemBackend::copy_to_io(view.cpu_addr, buffer) } + } + + #[inline] + fn copy_read<T: zerocopy::FromBytes>(view: Self::View<'_, T>) -> T { + SysMemBackend::copy_read(view.cpu_addr) + } + + #[inline] + fn copy_write<T: zerocopy::IntoBytes>(view: Self::View<'_, T>, value: T) { + SysMemBackend::copy_write(view.cpu_addr, value) + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> { + type Backend = CoherentIoBackend; + type Target = T; + + #[inline] + fn as_view(self) -> CoherentView<'a, Self::Target> { + self + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<T> { + type Backend = CoherentIoBackend; + type Target = T; + + #[inline] + fn as_view(self) -> CoherentView<'a, Self::Target> { + CoherentView { + // SAFETY: `cpu_addr` is valid and aligned kernel accessible memory. + cpu_addr: unsafe { SysMem::new(self.cpu_addr.as_ptr()) }, + dma_addr: self.dma_addr, + } + } } diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs index 36de8098754d..c9c74c4dde8f 100644 --- a/rust/kernel/driver.rs +++ b/rust/kernel/driver.rs @@ -13,10 +13,13 @@ //! The main driver interface is defined by a bus specific driver trait. For instance: //! //! ```ignore -//! pub trait Driver: Send { +//! pub trait Driver { //! /// The type holding information about each device ID supported by the driver. //! type IdInfo: 'static; //! +//! /// The type of the driver's bus device private data. +//! type Data<'bound>: Send + 'bound; +//! //! /// The table of OF device ids supported by the driver. //! const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = None; //! @@ -24,10 +27,16 @@ //! const ACPI_ID_TABLE: Option<acpi::IdTable<Self::IdInfo>> = None; //! //! /// Driver probe. -//! fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> impl PinInit<Self, Error>; +//! fn probe<'bound>( +//! dev: &'bound Device<device::Core<'_>>, +//! id_info: &'bound Self::IdInfo, +//! ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound; //! //! /// Driver unbind (optional). -//! fn unbind(dev: &Device<device::Core>, this: Pin<&Self>) { +//! fn unbind<'bound>( +//! dev: &'bound Device<device::Core<'_>>, +//! this: Pin<&Self::Data<'bound>>, +//! ) { //! let _ = (dev, this); //! } //! } @@ -42,8 +51,9 @@ )] #")] //! -//! The `probe()` callback should return a `impl PinInit<Self, Error>`, i.e. the driver's private -//! data. The bus abstraction should store the pointer in the corresponding bus device. The generic +//! The `probe()` callback should return a +//! `impl PinInit<Self::Data<'bound>, Error>`, i.e. the driver's private data. The bus +//! abstraction should store the pointer in the corresponding bus device. The generic //! [`Device`] infrastructure provides common helpers for this purpose on its //! [`Device<CoreInternal>`] implementation. //! @@ -118,8 +128,8 @@ pub unsafe trait DriverLayout { /// The specific driver type embedding a `struct device_driver`. type DriverType: Default; - /// The type of the driver's device private data. - type DriverData; + /// The type of the driver's bus device private data. + type DriverData<'bound>; /// Byte offset of the embedded `struct device_driver` within `DriverType`. /// @@ -181,20 +191,20 @@ unsafe impl<T: RegistrationOps> Sync for Registration<T> {} // any thread, so `Registration` is `Send`. unsafe impl<T: RegistrationOps> Send for Registration<T> {} -impl<T: RegistrationOps + 'static> Registration<T> { +impl<T: RegistrationOps> Registration<T> { extern "C" fn post_unbind_callback(dev: *mut bindings::device) { // SAFETY: The driver core only ever calls the post unbind callback with a valid pointer to // a `struct device`. // // INVARIANT: `dev` is valid for the duration of the `post_unbind_callback()`. - let dev = unsafe { &*dev.cast::<device::Device<device::CoreInternal>>() }; + let dev = unsafe { &*dev.cast::<device::Device<device::CoreInternal<'_>>>() }; - // `remove()` and all devres callbacks have been completed at this point, hence drop the - // driver's device private data. + // `remove()` has been completed at this point; devres resources are still valid and will + // be released after the driver's bus device private data is dropped. // // SAFETY: By the safety requirements of the `Driver` trait, `T::DriverData` is the - // driver's device private data type. - drop(unsafe { dev.drvdata_obtain::<T::DriverData>() }); + // driver's bus device private data type. + drop(unsafe { dev.drvdata_obtain::<T::DriverData<'_>>() }); } /// Attach generic `struct device_driver` callbacks. @@ -215,7 +225,10 @@ impl<T: RegistrationOps + 'static> Registration<T> { } /// Creates a new instance of the registration object. - pub fn new(name: &'static CStr, module: &'static ThisModule) -> impl PinInit<Self, Error> { + pub fn new(name: &'static CStr, module: &'static ThisModule) -> impl PinInit<Self, Error> + where + T: 'static, + { try_pin_init!(Self { reg <- Opaque::try_ffi_init(|ptr: *mut T::DriverType| { // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write. @@ -291,90 +304,23 @@ pub trait Adapter { /// The [`acpi::IdTable`] of the corresponding driver fn acpi_id_table() -> Option<acpi::IdTable<Self::IdInfo>>; - /// Returns the driver's private data from the matching entry in the [`acpi::IdTable`], if any. - /// - /// If this returns `None`, it means there is no match with an entry in the [`acpi::IdTable`]. - fn acpi_id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { - #[cfg(not(CONFIG_ACPI))] - { - let _ = dev; - None - } - - #[cfg(CONFIG_ACPI)] - { - let table = Self::acpi_id_table()?; - - // SAFETY: - // - `table` has static lifetime, hence it's valid for read, - // - `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. - let raw_id = unsafe { bindings::acpi_match_device(table.as_ptr(), dev.as_raw()) }; - - if raw_id.is_null() { - None - } else { - // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct acpi_device_id` - // and does not add additional invariants, so it's safe to transmute. - let id = unsafe { &*raw_id.cast::<acpi::DeviceId>() }; - - Some(table.info(<acpi::DeviceId as crate::device_id::RawDeviceIdIndex>::index(id))) - } - } - } - /// The [`of::IdTable`] of the corresponding driver. fn of_id_table() -> Option<of::IdTable<Self::IdInfo>>; - /// Returns the driver's private data from the matching entry in the [`of::IdTable`], if any. - /// - /// If this returns `None`, it means there is no match with an entry in the [`of::IdTable`]. - fn of_id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { - #[cfg(not(CONFIG_OF))] - { - let _ = dev; - None - } - - #[cfg(CONFIG_OF)] - { - let table = Self::of_id_table()?; - - // SAFETY: - // - `table` has static lifetime, hence it's valid for read, - // - `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. - let raw_id = unsafe { bindings::of_match_device(table.as_ptr(), dev.as_raw()) }; - - if raw_id.is_null() { - None - } else { - // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct of_device_id` - // and does not add additional invariants, so it's safe to transmute. - let id = unsafe { &*raw_id.cast::<of::DeviceId>() }; - - Some( - table.info(<of::DeviceId as crate::device_id::RawDeviceIdIndex>::index( - id, - )), - ) - } - } - } - /// Returns the driver's private data from the matching entry of any of the ID tables, if any. /// /// If this returns `None`, it means that there is no match in any of the ID tables directly /// associated with a [`device::Device`]. - fn id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { - let id = Self::acpi_id_info(dev); - if id.is_some() { - return id; - } - - let id = Self::of_id_info(dev); - if id.is_some() { - return id; - } - - None + /// + /// # Safety + /// + /// The caller must ensure that the `dev` matched data is of type `Self::IdInfo`. + #[inline] + unsafe fn id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { + // SAFETY: `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. + let data = unsafe { bindings::device_get_match_data(dev.as_raw()) }; + + // SAFETY: Per safety requirement, `data` is of type `Self::IdInfo`. + unsafe { data.cast::<Self::IdInfo>().as_ref() } } } diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 3ce8f62a0056..6b88ade28e24 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -6,15 +6,41 @@ use crate::{ alloc::allocator::Kmalloc, - bindings, device, drm, - drm::driver::AllocImpl, + bindings, + device, + drm::{ + self, + driver::AllocImpl, + private::Sealed, // + }, error::from_err_ptr, - error::Result, prelude::*, - sync::aref::{ARef, AlwaysRefCounted}, - types::Opaque, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::{ + NotThreadSafe, + Opaque, // + }, + workqueue::{ + HasDelayedWork, + HasWork, + Work, + WorkItem, // + }, // +}; +use core::{ + alloc::Layout, + cell::UnsafeCell, + marker::PhantomData, + mem, + ops::Deref, + ptr::{ + self, + NonNull, // + }, }; -use core::{alloc::Layout, mem, ops::Deref, ptr, ptr::NonNull}; #[cfg(CONFIG_DRM_LEGACY)] macro_rules! drm_legacy_fields { @@ -47,29 +73,105 @@ macro_rules! drm_legacy_fields { } } -/// A typed DRM device with a specific `drm::Driver` implementation. +/// A trait implemented by all possible contexts a [`Device`] can be used in. +/// +/// A [`Device`] can be in one of the following contexts: +/// +/// - [`Normal`]: The general-purpose, reference-counted context. A [`Device`] in this context may +/// or may not be registered with userspace. +/// - [`Ioctl`]: The device has been registered with userspace at some point; used in ioctl +/// dispatch context. +/// - [`Registered`]: The device is currently registered with userspace and the parent bus device +/// is bound. +/// +/// Both `Device<T, Ioctl>` and `Device<T, Registered>` dereference to `Device<T>` ([`Normal`]), +/// so any method available on a [`Normal`] device is also available in the other contexts. +pub trait DeviceContext: Sealed + Send + Sync + 'static {} + +/// The general-purpose, reference-counted [`DeviceContext`]. +/// +/// A [`Device`] in this context may or may not be registered with userspace. This context is used +/// for reference-counted device handles and during device setup via [`UnregisteredDevice`]. /// -/// The device is always reference-counted. +/// [`AlwaysRefCounted`] is only implemented for `Device<T, Normal>`, making this the required +/// context for [`ARef`]-based device handles. +pub struct Normal; + +impl Sealed for Normal {} +impl DeviceContext for Normal {} + +/// The [`DeviceContext`] of a [`Device`] that is currently registered with userspace. +/// +/// A [`Device`] in this context is guaranteed to be registered and its parent bus device is +/// guaranteed to be bound. This is enforced at runtime by [`RegistrationGuard`], which holds a +/// `drm_dev_enter()` / `drm_dev_exit()` SRCU critical section. /// /// # Invariants /// -/// `self.dev` is a valid instance of a `struct device`. -#[repr(C)] -pub struct Device<T: drm::Driver> { - dev: Opaque<bindings::drm_device>, - data: T::Data, +/// The parent bus device is bound for the duration of any reference to a `Device<T, Registered>`. +pub struct Registered; + +impl Sealed for Registered {} +impl DeviceContext for Registered {} + +/// The [`DeviceContext`] of a [`Device`] that has been registered with userspace previously. +/// +/// A [`Device`] in this context has been registered at some point, but may be concurrently +/// unregistering or already unregistered. `drm_dev_enter()` can guard against this, ensuring the +/// device remains registered for the duration of the critical section. +/// +/// # Invariants +/// +/// A [`Device`] in this context has been registered with userspace via `drm_dev_register()` at +/// some point. +pub struct Ioctl; + +impl Sealed for Ioctl {} +impl DeviceContext for Ioctl {} + +/// A [`Device`] which is known at compile-time to be unregistered with userspace. +/// +/// This type allows performing operations which are only safe to do before userspace registration, +/// and can be used to create a [`Registration`](drm::driver::Registration) once the driver is ready +/// to register the device with userspace. +/// +/// Since DRM device initialization must be single-threaded, this object is not thread-safe. +/// +/// # Invariants +/// +/// The device in `self.0` is guaranteed to be a newly created [`Device`] that has not yet been +/// registered with userspace until this type is dropped. +pub struct UnregisteredDevice<T: drm::Driver>(ARef<Device<T, Normal>>, NotThreadSafe); + +impl<T: drm::Driver> Deref for UnregisteredDevice<T> { + type Target = Device<T, Normal>; + + fn deref(&self) -> &Self::Target { + &self.0 + } } -impl<T: drm::Driver> Device<T> { +impl<T: drm::Driver> UnregisteredDevice<T> { + const fn compute_features() -> u32 { + let mut features = drm::driver::FEAT_GEM; + + if T::FEAT_RENDER { + features |= drm::driver::FEAT_RENDER; + } + + features + } + const VTABLE: bindings::drm_driver = drm_legacy_fields! { load: None, open: Some(drm::File::<T::File>::open_callback), postclose: Some(drm::File::<T::File>::postclose_callback), unload: None, - release: Some(Self::release), + release: Some(Device::<T>::release), master_set: None, master_drop: None, debugfs_init: None, + gem_create_object: T::Object::ALLOC_OPS.gem_create_object, prime_handle_to_fd: T::Object::ALLOC_OPS.prime_handle_to_fd, prime_fd_to_handle: T::Object::ALLOC_OPS.prime_fd_to_handle, @@ -77,6 +179,7 @@ impl<T: drm::Driver> Device<T> { gem_prime_import_sg_table: T::Object::ALLOC_OPS.gem_prime_import_sg_table, dumb_create: T::Object::ALLOC_OPS.dumb_create, dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset, + show_fdinfo: None, fbdev_probe: None, @@ -86,55 +189,97 @@ impl<T: drm::Driver> Device<T> { name: crate::str::as_char_ptr_in_const_context(T::INFO.name).cast_mut(), desc: crate::str::as_char_ptr_in_const_context(T::INFO.desc).cast_mut(), - driver_features: drm::driver::FEAT_GEM, + driver_features: Self::compute_features(), ioctls: T::IOCTLS.as_ptr(), num_ioctls: T::IOCTLS.len() as i32, fops: &Self::GEM_FOPS, }; - const GEM_FOPS: bindings::file_operations = drm::gem::create_fops(); + const GEM_FOPS: bindings::file_operations = + drm::gem::create_fops(crate::module::this_module::<T::OwnerModule>().as_ptr()); - /// Create a new `drm::Device` for a `drm::Driver`. - pub fn new(dev: &device::Device, data: impl PinInit<T::Data, Error>) -> Result<ARef<Self>> { + /// Create a new `UnregisteredDevice` for a `drm::Driver`. + /// + /// This can be used to create a [`Registration`](kernel::drm::Registration). + pub fn new( + dev: &T::ParentDevice<device::Bound>, + data: impl PinInit<T::Data, Error>, + ) -> Result<Self> { // `__drm_dev_alloc` uses `kmalloc()` to allocate memory, hence ensure a `kmalloc()` // compatible `Layout`. - let layout = Kmalloc::aligned_layout(Layout::new::<Self>()); + let layout = Kmalloc::aligned_layout(Layout::new::<Device<T, Normal>>()); + + // Use a temporary vtable without a `release` callback until `data` is initialized, so + // init failure can release the DRM device without dropping uninitialized fields. + let alloc_vtable = bindings::drm_driver { + release: None, + ..Self::VTABLE + }; // SAFETY: - // - `VTABLE`, as a `const` is pinned to the read-only section of the compilation, + // - `alloc_vtable` reference remains valid until no longer used, // - `dev` is valid by its type invarants, - let raw_drm: *mut Self = unsafe { + let raw_drm: *mut Device<T, Normal> = unsafe { bindings::__drm_dev_alloc( - dev.as_raw(), - &Self::VTABLE, + dev.as_ref().as_raw(), + &alloc_vtable, layout.size(), - mem::offset_of!(Self, dev), + mem::offset_of!(Device<T, Normal>, dev), ) } .cast(); let raw_drm = NonNull::new(from_err_ptr(raw_drm)?).ok_or(ENOMEM)?; + // SAFETY: `raw_drm` is a valid pointer to `Self`, given that `__drm_dev_alloc` was + // successful. + let drm_dev = unsafe { Device::into_drm_device(raw_drm) }; + // SAFETY: `raw_drm` is a valid pointer to `Self`. let raw_data = unsafe { ptr::addr_of_mut!((*raw_drm.as_ptr()).data) }; // SAFETY: // - `raw_data` is a valid pointer to uninitialized memory. // - `raw_data` will not move until it is dropped. - unsafe { data.__pinned_init(raw_data) }.inspect_err(|_| { - // SAFETY: `raw_drm` is a valid pointer to `Self`, given that `__drm_dev_alloc` was - // successful. - let drm_dev = unsafe { Self::into_drm_device(raw_drm) }; - + unsafe { pin_init::raw_try_init(raw_data, data) }.inspect_err(|_| { // SAFETY: `__drm_dev_alloc()` was successful, hence `drm_dev` must be valid and the // refcount must be non-zero. unsafe { bindings::drm_dev_put(drm_dev) }; })?; + // SAFETY: `drm_dev` is still private to this function. + unsafe { (*drm_dev).driver = const { &Self::VTABLE } }; + + // SAFETY: `raw_drm` is valid; no concurrent access before registration. + unsafe { (*raw_drm.as_ptr()).registration_data = UnsafeCell::new(NonNull::dangling()) }; + // SAFETY: The reference count is one, and now we take ownership of that reference as a // `drm::Device`. - Ok(unsafe { ARef::from_raw(raw_drm) }) + // INVARIANT: We just created the device above, but have yet to call `drm_dev_register`. + // `Self` cannot be copied or sent to another thread - ensuring that `drm_dev_register` + // won't be called during its lifetime and that the device is unregistered. + Ok(Self(unsafe { ARef::from_raw(raw_drm) }, NotThreadSafe)) } +} +/// A typed DRM device with a specific [`drm::Driver`] implementation and [`DeviceContext`]. +/// +/// A device in the [`Registered`] context is currently registered with userspace and its parent +/// bus device is bound. The [`Normal`] context is the general-purpose, reference-counted context. +/// +/// # Invariants +/// +/// * `self.dev` is a valid instance of a `struct device`. +/// * The data layout of `Self` remains the same across all implementations of `C`. +/// * Any invariants for `C` also apply. +#[repr(C)] +pub struct Device<T: drm::Driver, C: DeviceContext = Normal> { + dev: Opaque<bindings::drm_device>, + data: T::Data, + pub(super) registration_data: UnsafeCell<NonNull<T::RegistrationData<'static>>>, + _ctx: PhantomData<C>, +} + +impl<T: drm::Driver, C: DeviceContext> Device<T, C> { pub(crate) fn as_raw(&self) -> *mut bindings::drm_device { self.dev.get() } @@ -160,13 +305,13 @@ impl<T: drm::Driver> Device<T> { /// /// # Safety /// - /// Callers must ensure that `ptr` is valid, non-null, and has a non-zero reference count, - /// i.e. it must be ensured that the reference count of the C `struct drm_device` `ptr` points - /// to can't drop to zero, for the duration of this function call and the entire duration when - /// the returned reference exists. - /// - /// Additionally, callers must ensure that the `struct device`, `ptr` is pointing to, is - /// embedded in `Self`. + /// * Callers must ensure that `ptr` is valid, non-null, and has a non-zero reference count, + /// i.e. it must be ensured that the reference count of the C `struct drm_device` `ptr` points + /// to can't drop to zero, for the duration of this function call and the entire duration when + /// the returned reference exists. + /// * Additionally, callers must ensure that the `struct device`, `ptr` is pointing to, is + /// embedded in `Self`. + /// * Callers promise that any type invariants of `C` will be upheld. #[doc(hidden)] pub unsafe fn from_raw<'a>(ptr: *const bindings::drm_device) -> &'a Self { // SAFETY: By the safety requirements of this function `ptr` is a valid pointer to a @@ -186,6 +331,121 @@ impl<T: drm::Driver> Device<T> { // - `this` is valid for dropping. unsafe { core::ptr::drop_in_place(this) }; } + + /// Change the [`DeviceContext`] for a [`Device`]. + /// + /// # Safety + /// + /// The caller promises that `self` fulfills all of the guarantees provided by the given + /// [`DeviceContext`]. + pub(crate) unsafe fn assume_ctx<NewCtx: DeviceContext>(&self) -> &Device<T, NewCtx> { + // SAFETY: The data layout is identical via our type invariants. + unsafe { mem::transmute(self) } + } +} + +impl<T: drm::Driver> Device<T, Ioctl> { + /// Guard against the parent bus device being unbound. + /// + /// Returns a [`RegistrationGuard`] if the device has not been unplugged, [`None`] otherwise. + /// + /// While [`RegistrationGuard`] is held the parent device is guaranteed to be bound. + #[must_use] + pub fn registration_guard(&self) -> Option<RegistrationGuard<'_, T>> { + let mut idx: i32 = 0; + // SAFETY: `self.as_raw()` is a valid pointer to a `struct drm_device`. + if unsafe { bindings::drm_dev_enter(self.as_raw(), &mut idx) } { + // INVARIANT: + // - `idx` is the SRCU index from the successful `drm_dev_enter()` above. + // - The parent bus device is bound: `drm_dev_enter()` succeeded, meaning + // `drm_dev_unplug()` has not completed; since it is only called from + // `Registration::drop()` during parent unbind, the parent is still bound. + Some(RegistrationGuard { + // SAFETY: See INVARIANT above; the `Registered` context invariant holds. + dev: unsafe { self.assume_ctx() }, + idx, + _not_send: NotThreadSafe, + }) + } else { + None + } + } +} + +/// A guard proving the DRM device is registered and the parent bus device is bound. +/// +/// The guard dereferences to [`Device<T, Registered>`], providing access to the DRM device with +/// the guarantee that the parent bus device is bound for the entire duration of the critical +/// section. +/// +/// Internally this is backed by a `drm_dev_enter()` / `drm_dev_exit()` SRCU critical section. +/// +/// # Invariants +/// +/// - `idx` is the SRCU read lock index returned by a successful `drm_dev_enter()` call. +/// - The parent bus device of `dev` is bound for the lifetime of this guard. +#[must_use] +pub struct RegistrationGuard<'a, T: drm::Driver> { + dev: &'a Device<T, Registered>, + idx: i32, + _not_send: NotThreadSafe, +} + +impl<T: drm::Driver> Device<T, Registered> { + /// Returns a reference to the registration data with lifetime shortened from `'static`. + /// + /// # Safety + /// + /// The returned reference must not be exposed to code that can choose a concrete lifetime for + /// it, as that would be unsound for types that are invariant over their lifetime parameter + /// (e.g. it must be passed through an HRTB-bounded closure). + #[inline] + unsafe fn registration_data_unchecked(&self) -> &T::RegistrationData<'_> { + // SAFETY: + // - `Registered` guarantees the parent bus device is bound, hence the pointer is valid. + // - The pointer cast from `Of<'static>` to `Of<'_>` is layout-compatible since lifetimes + // are erased at runtime. + // - Caller guarantees the reference is only used behind an HRTB, making the lifetime + // shortening sound regardless of variance. + unsafe { (*self.registration_data.get()).cast::<_>().as_ref() } + } + + /// Access the registration data through a closure, with the lifetime tied to the closure + /// scope. + /// + /// The data is owned by [`Registration`](drm::Registration) and is guaranteed to remain valid + /// as long as the device is registered, since [`Registration`](drm::Registration)'s `drop` + /// calls `drm_dev_unplug()` which waits for all `drm_dev_enter()` critical sections to + /// complete. + #[inline] + pub fn registration_data_with<R, F>(&self, f: F) -> R + where + F: for<'a> FnOnce(&'a T::RegistrationData<'a>) -> R, + { + // SAFETY: `Registered` guarantees the device is registered and the parent bus device is + // bound. The closure's HRTB `for<'a>` prevents the caller from smuggling in references + // with a concrete short lifetime, satisfying the lifetime requirement of + // `registration_data_unchecked`. + f(unsafe { self.registration_data_unchecked() }) + } +} + +impl<T: drm::Driver> Deref for RegistrationGuard<'_, T> { + type Target = Device<T, Registered>; + + #[inline] + fn deref(&self) -> &Self::Target { + self.dev + } +} + +impl<T: drm::Driver> Drop for RegistrationGuard<'_, T> { + #[inline] + fn drop(&mut self) { + // SAFETY: `self.idx` was returned by a successful `drm_dev_enter()` call, as guaranteed + // by the type invariants of `RegistrationGuard`. + unsafe { bindings::drm_dev_exit(self.idx) }; + } } impl<T: drm::Driver> Deref for Device<T> { @@ -196,6 +456,28 @@ impl<T: drm::Driver> Deref for Device<T> { } } +impl<T: drm::Driver> Deref for Device<T, Registered> { + type Target = Device<T>; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: The caller holds a `Device<T, Registered>`, which guarantees all invariants + // of the weaker `Normal` context. + unsafe { self.assume_ctx() } + } +} + +impl<T: drm::Driver> Deref for Device<T, Ioctl> { + type Target = Device<T>; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: The caller holds a `Device<T, Ioctl>`, which guarantees all invariants + // of the weaker `Normal` context. + unsafe { self.assume_ctx() } + } +} + // SAFETY: DRM device objects are always reference counted and the get/put functions // satisfy the requirements. unsafe impl<T: drm::Driver> AlwaysRefCounted for Device<T> { @@ -213,17 +495,94 @@ unsafe impl<T: drm::Driver> AlwaysRefCounted for Device<T> { } } -impl<T: drm::Driver> AsRef<device::Device> for Device<T> { - fn as_ref(&self) -> &device::Device { +impl<T: drm::Driver> AsRef<T::ParentDevice<device::Normal>> for Device<T> { + fn as_ref(&self) -> &T::ParentDevice<device::Normal> { // SAFETY: `bindings::drm_device::dev` is valid as long as the DRM device itself is valid, // which is guaranteed by the type invariant. - unsafe { device::Device::from_raw((*self.as_raw()).dev) } + let dev = unsafe { device::Device::from_raw((*self.as_raw()).dev) }; + + // SAFETY: The DRM device was constructed in `UnregisteredDevice::new()` with a parent + // device of type `T::ParentDevice`, hence `dev` is contained in a `T::ParentDevice`. + unsafe { device::AsBusDevice::from_device(dev) } + } +} + +impl<T: drm::Driver> AsRef<T::ParentDevice<device::Bound>> for Device<T, Registered> { + #[inline] + fn as_ref(&self) -> &T::ParentDevice<device::Bound> { + let dev = (**self).as_ref().as_ref(); + + // SAFETY: A `Device<T, Registered>` guarantees that the parent device is bound. + let dev = unsafe { dev.as_bound() }; + + // SAFETY: The DRM device was constructed in `UnregisteredDevice::new()` with a parent + // device of type `T::ParentDevice`, hence `dev` is contained in a `T::ParentDevice`. + unsafe { device::AsBusDevice::from_device(dev) } } } // SAFETY: A `drm::Device` can be released from any thread. -unsafe impl<T: drm::Driver> Send for Device<T> {} +unsafe impl<T: drm::Driver, C: DeviceContext> Send for Device<T, C> {} // SAFETY: A `drm::Device` can be shared among threads because all immutable methods are protected // by the synchronization in `struct drm_device`. -unsafe impl<T: drm::Driver> Sync for Device<T> {} +unsafe impl<T: drm::Driver, C: DeviceContext> Sync for Device<T, C> {} + +impl<T: drm::Driver, const ID: u64> WorkItem<ID> for Device<T> +where + T::Data: WorkItem<ID, Pointer = ARef<Self>>, + T::Data: HasWork<Self, ID>, +{ + type Pointer = ARef<Self>; + + fn run(ptr: ARef<Self>) { + T::Data::run(ptr); + } +} + +// SAFETY: +// +// - `raw_get_work` and `work_container_of` return valid pointers by relying on +// `T::Data::raw_get_work` and `container_of`. In particular, `T::Data` is +// stored inline in `drm::Device`, so the `container_of` call is valid. +// +// - The two methods are true inverses of each other: given `ptr: *mut +// Device<T, C>`, `raw_get_work` will return a `*mut Work<Device<T, C>, ID>` through +// `T::Data::raw_get_work` and given a `ptr: *mut Work<Device<T, C>, ID>`, +// `work_container_of` will return a `*mut Device<T, C>` through `container_of`. +unsafe impl<T, C, const ID: u64> HasWork<Self, ID> for Device<T, C> +where + T: drm::Driver, + T::Data: HasWork<Self, ID>, + C: DeviceContext, +{ + unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work<Self, ID> { + // SAFETY: The caller promises that `ptr` points to a valid `Device<T, C>`. + let data_ptr = unsafe { &raw mut (*ptr).data }; + + // SAFETY: `data_ptr` is a valid pointer to `T::Data`. + unsafe { T::Data::raw_get_work(data_ptr) } + } + + unsafe fn work_container_of(ptr: *mut Work<Self, ID>) -> *mut Self { + // SAFETY: The caller promises that `ptr` points at a `Work` field in + // `T::Data`. + let data_ptr = unsafe { T::Data::work_container_of(ptr) }; + + // SAFETY: `T::Data` is stored as the `data` field in `Device<T, C>`. + unsafe { crate::container_of!(data_ptr, Self, data) } + } +} + +// SAFETY: Our `HasWork<T, ID>` implementation returns a `work_struct` that is +// stored in the `work` field of a `delayed_work` with the same access rules as +// the `work_struct` owing to the bound on `T::Data: HasDelayedWork<Device<T, C>, +// ID>`, which requires that `T::Data::raw_get_work` return a `work_struct` that +// is inside a `delayed_work`. +unsafe impl<T, C, const ID: u64> HasDelayedWork<Self, ID> for Device<T, C> +where + T: drm::Driver, + T::Data: HasDelayedWork<Self, ID>, + C: DeviceContext, +{ +} diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index e09f977b5b51..74f6ed690d8b 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -5,15 +5,19 @@ //! C header: [`include/drm/drm_drv.h`](srctree/include/drm/drm_drv.h) use crate::{ - bindings, device, devres, drm, - error::{to_result, Result}, + bindings, + device, + drm, + error::to_result, prelude::*, - sync::aref::ARef, + sync::aref::ARef, // }; -use macros::vtable; +use core::ptr::NonNull; /// Driver use the GEM memory manager. This should be set for all modern drivers. pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; +/// Driver supports render nodes, i.e.: /dev/dri/renderDXX devices. +pub(crate) const FEAT_RENDER: u32 = bindings::drm_driver_feature_DRIVER_RENDER; /// Information data for a DRM Driver. pub struct DriverInfo { @@ -102,69 +106,121 @@ pub trait Driver { /// Context data associated with the DRM driver type Data: Sync + Send; + /// Data owned by the [`Registration`] and accessible within a + /// [`RegistrationGuard`](drm::RegistrationGuard) critical section via + /// [`Device::registration_data_with()`](drm::Device::registration_data_with). + /// + /// The lifetime parameter is tied to the [`Registration`] scope, which is enclosed in the + /// parent bus device binding scope but may be shorter. + type RegistrationData<'a>: Send + Sync + 'a; + /// The type used to manage memory for this driver. type Object: AllocImpl; /// The type used to represent a DRM File (client) type File: drm::file::DriverFile; + /// The bus device type of the parent device that the DRM device is associated with. + type ParentDevice<Ctx: device::DeviceContext>: device::AsBusDevice<Ctx>; + /// Driver metadata const INFO: DriverInfo; /// IOCTL list. See `kernel::drm::ioctl::declare_drm_ioctls!{}`. const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor]; + + /// Sets the `DRIVER_RENDER` feature for this driver. + /// + /// When enabled, the driver exposes `/dev/dri/renderDXX` render nodes to + /// userspace. The render node is an alternate low-privilege way to access + /// the driver, which is enforced on a per-ioctl level. Userspace processes + /// that open the render node can only invoke ioctls explicitly listed as + /// usable from the render node (i.e. marked DRM_RENDER_ALLOW), whereas + /// userspace processes using the master node can invoke any ioctl. + const FEAT_RENDER: bool = false; } /// The registration type of a `drm::Device`. /// /// Once the `Registration` structure is dropped, the device is unregistered. -pub struct Registration<T: Driver>(ARef<drm::Device<T>>); - -impl<T: Driver> Registration<T> { - fn new(drm: &drm::Device<T>, flags: usize) -> Result<Self> { - // SAFETY: `drm.as_raw()` is valid by the invariants of `drm::Device`. - to_result(unsafe { bindings::drm_dev_register(drm.as_raw(), flags) })?; - - Ok(Self(drm.into())) - } +pub struct Registration<'a, T: Driver> { + drm: ARef<drm::Device<T>>, + _reg_data: Pin<KBox<T::RegistrationData<'a>>>, +} - /// Registers a new [`Device`](drm::Device) with userspace. +impl<'a, T: Driver> Registration<'a, T> { + /// Register a new [`UnregisteredDevice`](drm::UnregisteredDevice) with userspace. /// - /// Ownership of the [`Registration`] object is passed to [`devres::register`]. - pub fn new_foreign_owned( - drm: &drm::Device<T>, - dev: &device::Device<device::Bound>, + /// # Safety + /// + /// The caller must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running, since the registration data may contain borrowed + /// references that become invalid after `'a` ends. + pub unsafe fn new<E>( + dev: &'a device::Device<device::Bound>, + drm: drm::UnregisteredDevice<T>, + reg_data: impl PinInit<T::RegistrationData<'a>, E>, flags: usize, - ) -> Result + ) -> Result<Self> where - T: 'static, + Error: From<E>, { - if drm.as_ref().as_raw() != dev.as_raw() { + let parent = drm.as_ref(); + if parent.as_ref().as_raw() != dev.as_raw() { return Err(EINVAL); } - let reg = Registration::<T>::new(drm, flags)?; + let reg_data: Pin<KBox<T::RegistrationData<'a>>> = KBox::pin_init(reg_data, GFP_KERNEL)?; + + // Store the registration data pointer in the device before registration, so that it is + // visible once ioctls can be called. + let ptr: NonNull<T::RegistrationData<'static>> = + NonNull::from(Pin::get_ref(reg_data.as_ref())).cast(); + + // SAFETY: No concurrent access; the device is not yet registered. + unsafe { *drm.registration_data.get() = ptr }; + + // SAFETY: `drm` is a valid, initialized but not yet registered DRM device. + let ret = unsafe { bindings::drm_dev_register(drm.as_raw(), flags) }; + if let Err(e) = to_result(ret) { + // SAFETY: `drm_dev_register()` synchronizes SRCU on failure, so no concurrent + // access to `registration_data` is possible at this point. + unsafe { *drm.registration_data.get() = NonNull::dangling() }; + return Err(e); + } - devres::register(dev, reg, GFP_KERNEL) + Ok(Self { + drm: (&*drm).into(), + _reg_data: reg_data, + }) } /// Returns a reference to the `Device` instance for this registration. pub fn device(&self) -> &drm::Device<T> { - &self.0 + &self.drm } } // SAFETY: `Registration` doesn't offer any methods or access to fields when shared between // threads, hence it's safe to share it. -unsafe impl<T: Driver> Sync for Registration<T> {} +unsafe impl<T: Driver> Sync for Registration<'_, T> {} // SAFETY: Registration with and unregistration from the DRM subsystem can happen from any thread. -unsafe impl<T: Driver> Send for Registration<T> {} +unsafe impl<T: Driver> Send for Registration<'_, T> {} -impl<T: Driver> Drop for Registration<T> { +impl<T: Driver> Drop for Registration<'_, T> { fn drop(&mut self) { + // Use `drm_dev_unplug` rather than `drm_dev_unregister` to ensure that existing + // `drm_dev_enter()` critical sections complete before unregistration proceeds. This + // is required for the safety of `RegistrationGuard`, which relies on the SRCU barrier in + // `drm_dev_unplug()` to guarantee that the parent device is still bound within the + // critical section. + // // SAFETY: Safe by the invariant of `ARef<drm::Device<T>>`. The existence of this - // `Registration` also guarantees the this `drm::Device` is actually registered. - unsafe { bindings::drm_dev_unregister(self.0.as_raw()) }; + // `Registration` also guarantees that this `drm::Device` is actually registered. + unsafe { bindings::drm_dev_unplug(self.drm.as_raw()) }; + // After drm_dev_unplug(), the SRCU barrier guarantees that all RegistrationGuard critical + // sections have completed, so no one holds a reference to reg_data anymore. + // reg_data is dropped here automatically. } } diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 8c46f8d51951..10160601ce5a 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -4,9 +4,13 @@ //! //! C header: [`include/drm/drm_file.h`](srctree/include/drm/drm_file.h) -use crate::{bindings, drm, error::Result, prelude::*, types::Opaque}; +use crate::{ + bindings, + drm, + prelude::*, + types::Opaque, // +}; use core::marker::PhantomData; -use core::pin::Pin; /// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance). pub trait DriverFile { diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index d49a9ba02635..e1ebad77ebe2 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -5,15 +5,71 @@ //! C header: [`include/drm/drm_gem.h`](srctree/include/drm/drm_gem.h) use crate::{ - alloc::flags::*, - bindings, drm, - drm::driver::{AllocImpl, AllocOps}, - error::{to_result, Result}, + bindings, + drm::{ + self, + device::{ + DeviceContext, + Normal, // + }, + driver::{ + AllocImpl, + AllocOps, // + }, + }, + error::to_result, prelude::*, - sync::aref::{ARef, AlwaysRefCounted}, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, types::Opaque, }; -use core::{ops::Deref, ptr::NonNull}; +use core::{ + marker::PhantomData, + ops::Deref, + ptr::NonNull, // +}; + +#[cfg(CONFIG_RUST_DRM_GEM_SHMEM_HELPER)] +pub mod shmem; + +/// A macro for implementing [`AlwaysRefCounted`] for any GEM object type. +/// +/// Since all GEM objects use the same refcounting scheme. +#[macro_export] +macro_rules! impl_aref_for_gem_obj { + ( + impl $( <$( $tparam_id:ident ),+> )? for $type:ty + $( + where + $( $bind_param:path : $bind_trait:path ),+ + )? + ) => { + // SAFETY: All GEM objects are refcounted. + unsafe impl $( <$( $tparam_id ),+> )? $crate::sync::aref::AlwaysRefCounted for $type + where + Self: IntoGEMObject, + $( $( $bind_param : $bind_trait ),+ )? + { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is + // non-zero. + unsafe { bindings::drm_gem_object_get(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: core::ptr::NonNull<Self>) { + // SAFETY: `obj` is a valid pointer to an `Object<T>`. + let obj = unsafe { obj.as_ref() }.as_raw(); + + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::drm_gem_object_put(obj) }; + } + } + }; +} +#[cfg_attr(not(CONFIG_RUST_DRM_GEM_SHMEM_HELPER), allow(unused))] +pub(crate) use impl_aref_for_gem_obj; /// A type alias for retrieving a [`Driver`]s [`DriverFile`] implementation from its /// [`DriverObject`] implementation. @@ -22,25 +78,37 @@ use core::{ops::Deref, ptr::NonNull}; /// [`DriverFile`]: drm::file::DriverFile pub type DriverFile<T> = drm::File<<<T as DriverObject>::Driver as drm::Driver>::File>; +/// A type alias for retrieving the current [`AllocImpl`] for a given [`DriverObject`]. +/// +/// [`Driver`]: drm::Driver +pub type DriverAllocImpl<T> = <<T as DriverObject>::Driver as drm::Driver>::Object; + /// GEM object functions, which must be implemented by drivers. -pub trait DriverObject: Sync + Send + Sized { +pub trait DriverObject: Sync + Send + Sized + 'static { /// Parent `Driver` for this object. type Driver: drm::Driver; + /// The data type to use for passing arguments to [`DriverObject::new`]. + type Args; + /// Create a new driver data object for a GEM object of a given size. - fn new(dev: &drm::Device<Self::Driver>, size: usize) -> impl PinInit<Self, Error>; + fn new( + dev: &drm::Device<Self::Driver>, + size: usize, + args: Self::Args, + ) -> impl PinInit<Self, Error>; /// Open a new handle to an existing object, associated with a File. - fn open(_obj: &<Self::Driver as drm::Driver>::Object, _file: &DriverFile<Self>) -> Result { + fn open(_obj: &DriverAllocImpl<Self>, _file: &DriverFile<Self>) -> Result { Ok(()) } /// Close a handle to an existing object, associated with a File. - fn close(_obj: &<Self::Driver as drm::Driver>::Object, _file: &DriverFile<Self>) {} + fn close(_obj: &DriverAllocImpl<Self>, _file: &DriverFile<Self>) {} } /// Trait that represents a GEM object subtype -pub trait IntoGEMObject: Sized + super::private::Sealed + AlwaysRefCounted { +pub trait IntoGEMObject: Sized + super::private::Sealed { /// Returns a reference to the raw `drm_gem_object` structure, which must be valid as long as /// this owning object is valid. fn as_raw(&self) -> *mut bindings::drm_gem_object; @@ -49,7 +117,8 @@ pub trait IntoGEMObject: Sized + super::private::Sealed + AlwaysRefCounted { /// /// # Safety /// - /// - `self_ptr` must be a valid pointer to `Self`. + /// - `self_ptr` must be a valid pointer to the `struct drm_gem_object` embedded in a + /// valid instance of `Self`. /// - The caller promises that holding the immutable reference returned by this function does /// not violate rust's data aliasing rules and remains valid throughout the lifetime of `'a`. unsafe fn from_raw<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self; @@ -62,9 +131,12 @@ extern "C" fn open_callback<T: DriverObject>( // SAFETY: `open_callback` is only ever called with a valid pointer to a `struct drm_file`. let file = unsafe { DriverFile::<T>::from_raw(raw_file) }; - // SAFETY: `open_callback` is specified in the AllocOps structure for `DriverObject<T>`, - // ensuring that `raw_obj` is contained within a `DriverObject<T>` - let obj = unsafe { <<T::Driver as drm::Driver>::Object as IntoGEMObject>::from_raw(raw_obj) }; + // SAFETY: + // * `open_callback` is specified in the AllocOps structure for `DriverObject`, ensuring that + // `raw_obj` is contained within a `DriverAllocImpl<T>` + // * It is only possible for `open_callback` to be called after device registration, ensuring + // that the object's device is in the `Registered` state. + let obj: &DriverAllocImpl<T> = unsafe { IntoGEMObject::from_raw(raw_obj) }; match T::open(obj, file) { Err(e) => e.to_errno(), @@ -81,12 +153,12 @@ extern "C" fn close_callback<T: DriverObject>( // SAFETY: `close_callback` is specified in the AllocOps structure for `Object<T>`, ensuring // that `raw_obj` is indeed contained within a `Object<T>`. - let obj = unsafe { <<T::Driver as drm::Driver>::Object as IntoGEMObject>::from_raw(raw_obj) }; + let obj: &DriverAllocImpl<T> = unsafe { IntoGEMObject::from_raw(raw_obj) }; T::close(obj, file); } -impl<T: DriverObject> IntoGEMObject for Object<T> { +impl<T: DriverObject, Ctx: DeviceContext> IntoGEMObject for Object<T, Ctx> { fn as_raw(&self) -> *mut bindings::drm_gem_object { self.obj.get() } @@ -94,7 +166,7 @@ impl<T: DriverObject> IntoGEMObject for Object<T> { unsafe fn from_raw<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self { // SAFETY: `obj` is guaranteed to be in an `Object<T>` via the safety contract of this // function - unsafe { &*crate::container_of!(Opaque::cast_from(self_ptr), Object<T>, obj) } + unsafe { &*crate::container_of!(Opaque::cast_from(self_ptr), Object<T, Ctx>, obj) } } } @@ -125,7 +197,7 @@ pub trait BaseObject: IntoGEMObject { /// Looks up an object by its handle for a given `File`. fn lookup_handle<D, F>(file: &drm::File<F>, handle: u32) -> Result<ARef<Self>> where - Self: AllocImpl<Driver = D>, + Self: AllocImpl<Driver = D> + AlwaysRefCounted, D: drm::Driver<Object = Self, File = F>, F: drm::file::DriverFile<Driver = D>, { @@ -162,20 +234,34 @@ pub trait BaseObject: IntoGEMObject { impl<T: IntoGEMObject> BaseObject for T {} +/// Crate-private base operations shared by all GEM object classes. +#[cfg_attr(not(CONFIG_RUST_DRM_GEM_SHMEM_HELPER), expect(unused))] +pub(crate) trait BaseObjectPrivate: IntoGEMObject { + /// Return a pointer to this object's dma_resv. + fn raw_dma_resv(&self) -> *mut bindings::dma_resv { + // SAFETY: `self.as_raw()` always returns a valid pointer to the base DRM GEM object. + unsafe { (*self.as_raw()).resv } + } +} + +impl<T: IntoGEMObject> BaseObjectPrivate for T {} + /// A base GEM object. /// /// # Invariants /// -/// - `self.obj` is a valid instance of a `struct drm_gem_object`. +/// * `self.obj` is a valid instance of a `struct drm_gem_object`. +/// * Any type invariants of `Ctx` apply to the parent DRM device for this GEM object. #[repr(C)] #[pin_data] -pub struct Object<T: DriverObject + Send + Sync> { +pub struct Object<T: DriverObject + Send + Sync, Ctx: DeviceContext = Normal> { obj: Opaque<bindings::drm_gem_object>, #[pin] data: T, + _ctx: PhantomData<Ctx>, } -impl<T: DriverObject> Object<T> { +impl<T: DriverObject, Ctx: DeviceContext> Object<T, Ctx> { const OBJECT_FUNCS: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { free: Some(Self::free_callback), open: Some(open_callback::<T>), @@ -194,40 +280,16 @@ impl<T: DriverObject> Object<T> { rss: None, }; - /// Create a new GEM object. - pub fn new(dev: &drm::Device<T::Driver>, size: usize) -> Result<ARef<Self>> { - let obj: Pin<KBox<Self>> = KBox::pin_init( - try_pin_init!(Self { - obj: Opaque::new(bindings::drm_gem_object::default()), - data <- T::new(dev, size), - }), - GFP_KERNEL, - )?; - - // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. - unsafe { (*obj.as_raw()).funcs = &Self::OBJECT_FUNCS }; - - // SAFETY: The arguments are all valid per the type invariants. - to_result(unsafe { bindings::drm_gem_object_init(dev.as_raw(), obj.obj.get(), size) })?; - - // SAFETY: We will never move out of `Self` as `ARef<Self>` is always treated as pinned. - let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(obj) }); - - // SAFETY: `ptr` comes from `KBox::into_raw` and hence can't be NULL. - let ptr = unsafe { NonNull::new_unchecked(ptr) }; - - // SAFETY: We take over the initial reference count from `drm_gem_object_init()`. - Ok(unsafe { ARef::from_raw(ptr) }) - } - /// Returns the `Device` that owns this GEM object. - pub fn dev(&self) -> &drm::Device<T::Driver> { + pub fn dev(&self) -> &drm::Device<T::Driver, Ctx> { // SAFETY: // - `struct drm_gem_object.dev` is initialized and valid for as long as the GEM // object lives. // - The device we used for creating the gem object is passed as &drm::Device<T::Driver> to // Object::<T>::new(), so we know that `T::Driver` is the right generic parameter to use // here. + // - Any type invariants of `Ctx` are upheld by using the same `Ctx` for the `Device` we + // return. unsafe { drm::Device::from_raw((*self.as_raw()).dev) } } @@ -252,25 +314,55 @@ impl<T: DriverObject> Object<T> { } } -// SAFETY: Instances of `Object<T>` are always reference-counted. -unsafe impl<T: DriverObject> crate::sync::aref::AlwaysRefCounted for Object<T> { - fn inc_ref(&self) { - // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. - unsafe { bindings::drm_gem_object_get(self.as_raw()) }; - } +impl<T: DriverObject> Object<T> { + /// Create a new GEM object. + pub fn new(dev: &drm::Device<T::Driver>, size: usize, args: T::Args) -> Result<ARef<Self>> { + let obj: Pin<KBox<Self>> = KBox::pin_init( + try_pin_init!(Self { + obj: Opaque::new(bindings::drm_gem_object::default()), + data <- T::new(dev, size, args), + _ctx: PhantomData, + }), + GFP_KERNEL, + )?; + + // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. + unsafe { (*obj.as_raw()).funcs = &Self::OBJECT_FUNCS }; + + // INVARIANT: `dev` and the GEM object are in the same state at the moment, and upgrading + // the typestate in `dev` will not carry over to the GEM object. + if let Err(err) = + // SAFETY: The arguments are all valid per the type invariants. + to_result(unsafe { + bindings::drm_gem_object_init(dev.as_raw(), obj.obj.get(), size) + }) + { + // SAFETY: `drm_gem_object_init()` initializes the private GEM object state before + // failing, so `drm_gem_private_object_fini()` is the matching cleanup. + unsafe { bindings::drm_gem_private_object_fini(obj.obj.get()) }; + return Err(err); + } + + // SAFETY: We will never move out of `Self` as `ARef<Self>` is always treated as pinned. + let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(obj) }); - unsafe fn dec_ref(obj: NonNull<Self>) { - // SAFETY: `obj` is a valid pointer to an `Object<T>`. - let obj = unsafe { obj.as_ref() }; + // SAFETY: `ptr` comes from `KBox::into_raw` and hence can't be NULL. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; - // SAFETY: The safety requirements guarantee that the refcount is non-zero. - unsafe { bindings::drm_gem_object_put(obj.as_raw()) } + // SAFETY: We take over the initial reference count from `drm_gem_object_init()`. + Ok(unsafe { ARef::from_raw(ptr) }) } } -impl<T: DriverObject> super::private::Sealed for Object<T> {} +impl_aref_for_gem_obj! { + impl<T> for Object<T> + where + T: DriverObject +} + +impl<T: DriverObject, Ctx: DeviceContext> super::private::Sealed for Object<T, Ctx> {} -impl<T: DriverObject> Deref for Object<T> { +impl<T: DriverObject, Ctx: DeviceContext> Deref for Object<T, Ctx> { type Target = T; fn deref(&self) -> &Self::Target { @@ -278,7 +370,7 @@ impl<T: DriverObject> Deref for Object<T> { } } -impl<T: DriverObject> AllocImpl for Object<T> { +impl<T: DriverObject, Ctx: DeviceContext> AllocImpl for Object<T, Ctx> { type Driver = T::Driver; const ALLOC_OPS: AllocOps = AllocOps { @@ -292,10 +384,10 @@ impl<T: DriverObject> AllocImpl for Object<T> { }; } -pub(super) const fn create_fops() -> bindings::file_operations { +pub(super) const fn create_fops(owner: *mut bindings::module) -> bindings::file_operations { let mut fops: bindings::file_operations = pin_init::zeroed(); - fops.owner = core::ptr::null_mut(); + fops.owner = owner; fops.open = Some(bindings::drm_open); fops.release = Some(bindings::drm_release); fops.unlocked_ioctl = Some(bindings::drm_ioctl); diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs new file mode 100644 index 000000000000..a687d46d170d --- /dev/null +++ b/rust/kernel/drm/gem/shmem.rs @@ -0,0 +1,721 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! DRM GEM shmem helper objects +//! +//! C header: [`include/linux/drm/drm_gem_shmem_helper.h`](srctree/include/drm/drm_gem_shmem_helper.h) + +// TODO: +// - There are a number of spots here that manually acquire/release the DMA reservation lock using +// dma_resv_(un)lock(). In the future we should add support for ww mutex, expose a method to +// acquire a reference to the WwMutex, and then use that directly instead of the C functions here. + +use crate::{ + container_of, + device::{ + self, + Bound, // + }, + devres::*, + drm::{ + driver, + gem, + private::Sealed, + Device, // + }, + error::{ + from_err_ptr, + to_result, // + }, + io::{ + IoBase, + Region, + SysMem, + SysMemBackend, // + }, + prelude::*, + scatterlist, + sync::{ + aref::ARef, + new_mutex, + Mutex, + SetOnce, // + }, + types::{ + NotThreadSafe, + Opaque, // + }, +}; +use core::{ + ffi::c_void, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, + ops::{ + Deref, + DerefMut, // + }, + ptr::{ + self, + NonNull, // + }, +}; +use gem::{ + BaseObject, + BaseObjectPrivate, + DriverObject, + IntoGEMObject, // +}; + +/// A struct for controlling the creation of shmem-backed GEM objects. +/// +/// This is used with [`Object::new()`] to control various properties that can only be set when +/// initially creating a shmem-backed GEM object. +pub struct ObjectConfig<'a, T: DriverObject> { + /// Whether to set the write-combine map flag. + pub map_wc: bool, + + /// Reuse the DMA reservation from another GEM object. + /// + /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. + pub parent_resv_obj: Option<&'a Object<T>>, +} + +impl<'a, T: DriverObject> Default for ObjectConfig<'a, T> { + #[inline(always)] + fn default() -> Self { + Self { + map_wc: false, + parent_resv_obj: None, + } + } +} + +/// A shmem-backed GEM object. +/// +/// # Invariants +/// +/// - `obj` contains a valid initialized `struct drm_gem_shmem_object` for the lifetime of this +/// object. +#[repr(C)] +#[pin_data] +pub struct Object<T: DriverObject> { + #[pin] + obj: Opaque<bindings::drm_gem_shmem_object>, + /// Parent object that owns this object's DMA reservation object. + parent_resv_obj: Option<ARef<Object<T>>>, + /// Devres object for unmapping any SGTable on driver-unbind. + sgt_res: ManuallyDrop<SetOnce<Devres<SGTableMap<T>>>>, + #[pin] + /// Lock for protecting initialization of `sgt_res`. + sgt_lock: Mutex<()>, + #[pin] + inner: T, +} + +super::impl_aref_for_gem_obj! { + impl<T> for Object<T> + where + T: DriverObject +} + +// SAFETY: All GEM objects are thread-safe. +unsafe impl<T: DriverObject> Send for Object<T> {} + +// SAFETY: All GEM objects are thread-safe. +unsafe impl<T: DriverObject> Sync for Object<T> {} + +impl<T: DriverObject> Object<T> { + /// `drm_gem_object_funcs` vtable suitable for GEM shmem objects. + const VTABLE: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { + free: Some(Self::free_callback), + open: Some(super::open_callback::<T>), + close: Some(super::close_callback::<T>), + print_info: Some(bindings::drm_gem_shmem_object_print_info), + export: None, + pin: Some(bindings::drm_gem_shmem_object_pin), + unpin: Some(bindings::drm_gem_shmem_object_unpin), + get_sg_table: Some(bindings::drm_gem_shmem_object_get_sg_table), + vmap: Some(bindings::drm_gem_shmem_object_vmap), + vunmap: Some(bindings::drm_gem_shmem_object_vunmap), + mmap: Some(bindings::drm_gem_shmem_object_mmap), + status: None, + rss: None, + #[allow(unused_unsafe, reason = "Safe since Rust 1.82.0")] + // SAFETY: `drm_gem_shmem_vm_ops` is a valid, static const on the C side. + vm_ops: unsafe { &raw const bindings::drm_gem_shmem_vm_ops }, + evict: None, + }; + + /// Return a raw pointer to the embedded drm_gem_shmem_object. + fn as_raw_shmem(&self) -> *mut bindings::drm_gem_shmem_object { + self.obj.get() + } + + /// Returns the `Device` that owns this GEM object. + pub fn dev(&self) -> &Device<T::Driver> { + // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`. + unsafe { Device::from_raw((*self.as_raw()).dev) } + } + + extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { + // SAFETY: + // - DRM always passes a valid gem object here + // - We used drm_gem_shmem_create() in our create_gem_object callback, so we know that + // `obj` is contained within a drm_gem_shmem_object + let base = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) }; + + // SAFETY: + // - We verified above that `obj` is valid, which makes `this` valid + // - This function is set in AllocOps, so we know that `this` is contained within an + // `Object<T>` + let this = unsafe { container_of!(Opaque::cast_from(base), Self, obj) }.cast_mut(); + + // We need to drop `sgt_res` first, since doing so requires that the GEM object is still + // alive. + // SAFETY: + // - We verified above that `this` is valid. + // - We are in free_callback, guaranteeing we have exclusive access to `this` and that + // `sgt_res` will not be used after dropping it here. + unsafe { ManuallyDrop::drop(&mut (*this).sgt_res) }; + + // SAFETY: + // - We're in free_callback - so this function is safe to call. + // - We won't be using the gem resources on `this` after this call. + unsafe { bindings::drm_gem_shmem_release(base) }; + + // SAFETY: We're recovering the Kbox<> we created in gem_create_object() + let _ = unsafe { KBox::from_raw(this) }; + } + + /// Attempt to create a vmap from the gem object, and confirm the size of said vmap. + fn make_vmap<'a, R, const SIZE: usize>(&'a self) -> Result<VMap<T, R, SIZE>> + where + R: Deref<Target = Self> + From<&'a Self>, + { + // INVARIANT: We check here that the gem object is at least as large as `SIZE`. + if self.size() < SIZE { + return Err(ENOSPC); + } + + let mut map: MaybeUninit<bindings::iosys_map> = MaybeUninit::uninit(); + let guard = DmaResvGuard::new(self); + + // SAFETY: `drm_gem_shmem_vmap()` can be called with the DMA reservation lock held. + to_result(unsafe { + bindings::drm_gem_shmem_vmap_locked(self.as_raw_shmem(), map.as_mut_ptr()) + })?; + + // Drop the guard explicitly here, since we may need to call `raw_vunmap()` (which + // re-acquires the lock). + drop(guard); + + // SAFETY: The call to `drm_gem_shmem_vmap_locked()` succeeded above, so we are guaranteed + // that map is properly initialized. + let map = unsafe { map.assume_init() }; + + // XXX: We don't currently support iomem allocations + if map.is_iomem { + // SAFETY: The vmap operation above succeeded, guaranteeing that `map` points to a valid + // memory mapping. + unsafe { self.raw_vunmap(map) }; + + Err(ENOTSUPP) + } else { + Ok(VMap { + // INVARIANT: `addr` remains valid for as long as `owner` does, which extends to the + // lifetime of `VMap` itself. + // SAFETY: We checked that this is not an iomem allocation, making it safe to read + // vaddr. + addr: unsafe { map.__bindgen_anon_1.vaddr }, + owner: self.into(), + }) + } + } + + /// Unmap a vmap from the gem object. + /// + /// # Safety + /// + /// - The caller promises that `map` is a valid vmap on this gem object. + /// - The caller promises that the memory pointed to by map will no longer be accesed through + /// this instance. + unsafe fn raw_vunmap(&self, mut map: bindings::iosys_map) { + let _guard = DmaResvGuard::new(self); + + // SAFETY: + // - This function is safe to call with the DMA reservation lock held. + // - The caller promises that `map` is a valid vmap on this gem object. + unsafe { bindings::drm_gem_shmem_vunmap_locked(self.as_raw_shmem(), &mut map) }; + } + + /// Creates and returns a virtual kernel memory mapping for this object. + #[inline] + pub fn vmap<const SIZE: usize>(&self) -> Result<VMapRef<'_, T, SIZE>> { + self.make_vmap() + } + + /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA + /// pages for this object. + /// + /// This will pin the object in memory. It is expected that `dev` should be a pointer to the + /// same [`device::Device`] which `self` belongs to, otherwise this function will return + /// `Err(EINVAL)`. + pub fn sg_table<'a>( + &'a self, + dev: &'a device::Device<Bound>, + ) -> Result<&'a scatterlist::SGTable> { + let parent = self.dev().as_ref(); + if dev.as_raw() != parent.as_ref().as_raw() { + return Err(EINVAL); + } + + let sgt_res = 'out: { + // Fast path: sgt_res is already initialized + if let Some(sgt_res) = self.sgt_res.as_ref() { + break 'out sgt_res; + } + + // Slow path: Grab the lock and see if we need to initialize sgt_res. + let _guard = self.sgt_lock.lock(); + + // If someone initialized it while we were waiting, we can exit early. + if let Some(sgt_res) = self.sgt_res.as_ref() { + break 'out sgt_res; + } + + // If not, finish initializing and return. `populate()` cannot return false, as + // `sgt_res` must be unpopulated, and we must hold `sgt_lock` to reach this point. + self.sgt_res + .populate(Devres::new(dev, SGTableMap::new(self))?); + + // SAFETY: We just populated sgt_res above. + unsafe { self.sgt_res.as_ref().unwrap_unchecked() } + }; + + Ok(sgt_res.access(dev)?) + } + + /// Create a new shmem-backed DRM object of the given size. + /// + /// Additional config options can be specified using `config`. + pub fn new( + dev: &Device<T::Driver>, + size: usize, + config: ObjectConfig<'_, T>, + args: T::Args, + ) -> Result<ARef<Self>> { + let new: Pin<KBox<Self>> = KBox::try_pin_init( + try_pin_init!(Self { + obj <- Opaque::init_zeroed(), + parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), + sgt_res: ManuallyDrop::new(SetOnce::new()), + sgt_lock <- new_mutex!(()), + inner <- T::new(dev, size, args), + }), + GFP_KERNEL, + )?; + + // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. + unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; + + // SAFETY: The arguments are all valid via the type invariants. + to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_raw_shmem(), size) })?; + + // SAFETY: We never move out of `self`. + let new = KBox::into_raw(unsafe { Pin::into_inner_unchecked(new) }); + + // SAFETY: We're taking over the owned refcount from `drm_gem_shmem_init`. + let obj = unsafe { ARef::from_raw(NonNull::new_unchecked(new)) }; + + // Start filling out values from `config` + if let Some(parent_resv) = config.parent_resv_obj { + // SAFETY: We have yet to expose the new gem object outside of this function, so it is + // safe to modify this field. + unsafe { (*obj.obj.get()).base.resv = parent_resv.raw_dma_resv() }; + } + + // SAFETY: We have yet to expose this object outside of this function, so we're guaranteed + // to have exclusive access - thus making this safe to hold a mutable reference to. + let shmem = unsafe { &mut *obj.as_raw_shmem() }; + shmem.set_map_wc(config.map_wc); + + Ok(obj) + } + + /// Creates and returns an owned reference to a virtual kernel memory mapping for this object. + #[inline] + pub fn owned_vmap<const SIZE: usize>(&self) -> Result<VMapOwned<T, SIZE>> { + self.make_vmap() + } +} + +impl<T: DriverObject> Deref for Object<T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl<T: DriverObject> DerefMut for Object<T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl<T: DriverObject> Sealed for Object<T> {} + +impl<T: DriverObject> gem::IntoGEMObject for Object<T> { + fn as_raw(&self) -> *mut bindings::drm_gem_object { + // SAFETY: + // - Our immutable reference is proof that this is safe to dereference. + // - `obj` is always a valid drm_gem_shmem_object via our type invariants. + unsafe { &raw mut (*self.obj.get()).base } + } + + unsafe fn from_raw<'a>(obj: *mut bindings::drm_gem_object) -> &'a Self { + // SAFETY: The safety contract of from_gem_obj() guarantees that `obj` is contained within + // `Self` + unsafe { + let obj = Opaque::cast_from(container_of!(obj, bindings::drm_gem_shmem_object, base)); + + &*container_of!(obj, Self, obj) + } + } +} + +impl<T: DriverObject> driver::AllocImpl for Object<T> { + type Driver = T::Driver; + + const ALLOC_OPS: driver::AllocOps = driver::AllocOps { + gem_create_object: None, + prime_handle_to_fd: None, + prime_fd_to_handle: None, + gem_prime_import: None, + gem_prime_import_sg_table: Some(bindings::drm_gem_shmem_prime_import_sg_table), + dumb_create: Some(bindings::drm_gem_shmem_dumb_create), + dumb_map_offset: None, + }; +} + +/// Private helper-type for holding the `dma_resv` object for a GEM shmem object. +/// +/// When this is dropped, the `dma_resv` lock is dropped as well. +/// +// TODO: This should be replace with a WwMutex equivalent once we have such bindings in the kernel. +struct DmaResvGuard<'a, T: DriverObject>(&'a Object<T>, NotThreadSafe); + +impl<'a, T: DriverObject> DmaResvGuard<'a, T> { + #[inline] + fn new(obj: &'a Object<T>) -> Self { + // SAFETY: This lock is initialized throughout the lifetime of `object`. + unsafe { bindings::dma_resv_lock(obj.raw_dma_resv(), ptr::null_mut()) }; + + Self(obj, NotThreadSafe) + } +} + +impl<'a, T: DriverObject> Drop for DmaResvGuard<'a, T> { + #[inline] + fn drop(&mut self) { + // SAFETY: We are releasing the lock grabbed during the creation of this object. + unsafe { bindings::dma_resv_unlock(self.0.raw_dma_resv()) }; + } +} + +/// A reference to a virtual mapping for an shmem-based GEM object in kernel address space. +/// +/// # Invariants +/// +/// - The size of `owner` is >= SIZE. +/// - The memory pointed to by `addr` remains valid at least until this object is dropped. +pub struct VMap<D, R, const SIZE: usize = 0> +where + D: DriverObject, + R: Deref<Target = Object<D>>, +{ + addr: *mut c_void, + owner: R, +} + +/// An alias type for a reference to a shmem-based GEM object's VMap. +pub type VMapRef<'a, D, const SIZE: usize = 0> = VMap<D, &'a Object<D>, SIZE>; + +/// An alias type for an owned reference to a shmem-based GEM object's VMap. +pub type VMapOwned<D, const SIZE: usize = 0> = VMap<D, ARef<Object<D>>, SIZE>; + +impl<D, R, const SIZE: usize> VMap<D, R, SIZE> +where + D: DriverObject, + R: Deref<Target = Object<D>>, +{ + /// Borrows a reference to the object that owns this virtual mapping. + #[inline] + pub fn owner(&self) -> &Object<D> { + &self.owner + } +} + +impl<'a, D, R, const SIZE: usize> IoBase<'a> for &'a VMap<D, R, SIZE> +where + D: DriverObject, + R: Deref<Target = Object<D>>, +{ + type Backend = SysMemBackend; + type Target = Region<SIZE>; + + #[inline] + fn as_view(self) -> SysMem<'a, Region<SIZE>> { + let ptr = Region::ptr_from_raw_parts_mut(self.addr.cast(), self.owner.size()); + + // SAFETY: Per type invariants of `VMap`: + // - `addr .. addr + owner.size()` is a valid kernel accessible memory region. + // - `addr` is page-aligned, which satisfies `Region`'s 4-byte alignment requirement. + // - The memory remains valid until this `VMap` is dropped; since `self` is `&'a VMap`, + // the borrow prevents the `VMap` from being dropped for the lifetime `'a`. + unsafe { SysMem::new(ptr) } + } +} + +impl<D, R, const SIZE: usize> Drop for VMap<D, R, SIZE> +where + D: DriverObject, + R: Deref<Target = Object<D>>, +{ + #[inline] + fn drop(&mut self) { + // SAFETY: + // - Our existence is proof that this map was previously created using self.owner. + // - Since we are in Drop, we are guaranteed that no one will access the memory + // through this mapping after calling this. + unsafe { + self.owner.raw_vunmap(bindings::iosys_map { + is_iomem: false, + __bindgen_anon_1: bindings::iosys_map__bindgen_ty_1 { vaddr: self.addr }, + }) + }; + } +} + +// SAFETY: `addr` points to a valid memory address for as long as `owner` exists, meaning that so +// long as `owner` is `Send` so is `VMap`. +unsafe impl<D, R, const SIZE: usize> Send for VMap<D, R, SIZE> +where + D: DriverObject, + R: Deref<Target = Object<D>> + Send, +{ +} + +// SAFETY: `addr` points to a valid memory address for as long as `owner` exists, meaning that so +// long as `owner` is `Sync` so is `VMap`. +unsafe impl<D, R, const SIZE: usize> Sync for VMap<D, R, SIZE> +where + D: DriverObject, + R: Deref<Target = Object<D>> + Sync, +{ +} + +/// A reference to a GEM object that is known to have a mapped [`SGTable`]. +/// +/// This is used by the Rust bindings with [`Devres`] in order to ensure that mappings for SGTables +/// on GEM shmem objects are revoked on driver-unbind. +/// +/// # Invariants +/// +/// - `self.obj` always points to a valid GEM object. +/// - This object is proof that `self.obj.owner.sgt_res` has an initialized and valid pointer to an +/// [`SGTable`]. +/// +/// [`SGTable`]: scatterlist::SGTable +pub struct SGTableMap<T: DriverObject> { + obj: NonNull<Object<T>>, +} + +impl<T: DriverObject> Deref for SGTableMap<T> { + type Target = scatterlist::SGTable; + + fn deref(&self) -> &Self::Target { + // SAFETY: + // - The NonNull is guaranteed to be valid via our type invariants. + // - The sgt field is guaranteed to be initialized and valid via our type invariants. + unsafe { scatterlist::SGTable::from_raw((*self.obj.as_ref().as_raw_shmem()).sgt) } + } +} + +impl<T: DriverObject> Drop for SGTableMap<T> { + fn drop(&mut self) { + // SAFETY: `obj` is always valid via our type invariants + let obj = unsafe { self.obj.as_ref() }; + let _lock = DmaResvGuard::new(obj); + + // SAFETY: We acquired the lock needed for calling this function above + unsafe { bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()) }; + } +} + +impl<T: DriverObject> SGTableMap<T> { + fn new(obj: &Object<T>) -> impl Init<Self, Error> { + // INVARIANT: + // - We call drm_gem_shmem_get_pages_sgt below and check whether or not it succeeds, + // fulfilling the invariant of SGTableMap that the object's `sgt` field is initialized. + // SAFETY: + // - `obj` is fully initialized, making this function safe to call. + from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(obj.as_raw_shmem()) })?; + + Ok(Self { obj: obj.into() }) + } +} + +// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object +// it points to is guaranteed to be thread-safe. +unsafe impl<T: DriverObject> Send for SGTableMap<T> {} +// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object +// it points to is guaranteed to be thread-safe. +unsafe impl<T: DriverObject> Sync for SGTableMap<T> {} + +#[kunit_tests(rust_drm_gem_shmem)] +mod tests { + use super::*; + use crate::{ + drm::{ + self, + UnregisteredDevice, // + }, + faux, + io::Io, + page::PAGE_SIZE, // + }; + + // The bare minimum needed to create a fake drm driver for kunit + + #[pin_data] + struct KunitData {} + struct KunitDriver; + struct KunitFile; + #[pin_data] + struct KunitObject {} + + const INFO: drm::DriverInfo = drm::DriverInfo { + major: 0, + minor: 0, + patchlevel: 0, + name: c"kunit", + desc: c"Kunit", + }; + + impl drm::file::DriverFile for KunitFile { + type Driver = KunitDriver; + + fn open(_dev: &drm::Device<KunitDriver>) -> Result<Pin<KBox<Self>>> { + Ok(KBox::new(Self, GFP_KERNEL)?.into()) + } + } + + impl gem::DriverObject for KunitObject { + type Driver = KunitDriver; + type Args = (); + + fn new( + _dev: &drm::Device<KunitDriver>, + _size: usize, + _args: Self::Args, + ) -> impl PinInit<Self, Error> { + try_pin_init!(KunitObject {}) + } + } + + #[vtable] + impl drm::Driver for KunitDriver { + type Data = KunitData; + type RegistrationData<'a> = (); + type File = KunitFile; + type Object = Object<KunitObject>; + type ParentDevice<Ctx: device::DeviceContext> = faux::Device<Ctx>; + + const INFO: drm::DriverInfo = INFO; + const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor] = &[]; + } + + fn create_drm_dev() -> Result<(faux::Registration, UnregisteredDevice<KunitDriver>)> { + // Create a faux DRM device so we can test gem object creation. + let data = try_pin_init!(KunitData {}); + let reg = faux::Registration::new(c"Kunit", None)?; + let fdev = reg.as_ref(); + let drm = UnregisteredDevice::new(fdev, data)?; + + Ok((reg, drm)) + } + + #[test] + fn compile_time_vmap_sizes() -> Result { + let (_dev, drm) = create_drm_dev()?; + + let obj = Object::<KunitObject>::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + // Try creating a normal vmap + obj.vmap::<PAGE_SIZE>()?; + + // Try creating a vmap that's smaller then the size we specified + let vmap = obj.vmap::<{ PAGE_SIZE - 100 }>()?; + + // Verify the owner matches + assert!(ptr::eq(vmap.owner(), obj.deref())); + + // Verify the size matches the actual object size + assert_eq!(vmap.size(), PAGE_SIZE); + + // Make sure creating a vmap that's too large fails + assert!(obj.vmap::<{ PAGE_SIZE + 200 }>().is_err()); + + Ok(()) + } + + #[test] + fn vmap_io() -> Result { + let (_dev, drm) = create_drm_dev()?; + + let obj = Object::<KunitObject>::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + let vmap = obj.vmap::<PAGE_SIZE>()?; + + vmap.write8(0xDE, 0x0); + assert_eq!(vmap.read8(0x0), 0xDE); + vmap.write32(0xFEDCBA98, 0x20); + + assert_eq!(vmap.read32(0x20), 0xFEDCBA98); + + // Ensure the ordering in memory is correct + let expected = 0xFEDCBA98_u32.to_ne_bytes().into_iter(); + for (offset, expected) in (0x20..=0x23).zip(expected) { + assert_eq!(vmap.try_read8(offset).unwrap(), expected); + } + + Ok(()) + } + + // TODO: I would love to actually test the success paths of sg_table(), but that would require + // also implementing dummy dma_ops so that trying to create a mapping doesn't explode. So, leave + // that for someone else. + + // Ensures that passing the wrong device to sg_table() fails as we expect, and also ensure it + // skips initializing `sgt_res` since we could otherwise create `sgt_res` with the wrong device + // bound to it. + #[test] + fn fail_sg_table_on_wrong_dev() -> Result { + let (_dev, drm) = create_drm_dev()?; + let reg = faux::Registration::new(c"EvilKunit", None)?; + let wrong_dev = reg.as_ref(); + + let obj = Object::<KunitObject>::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); + + // If sgt_res was not initialized mistakenly with the wrong device, this should still fail. + assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); + + // TODO: Someday, we should test that creating an sg_table here still succeeds. + + Ok(()) + } +} diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs new file mode 100644 index 000000000000..d9d43d719761 --- /dev/null +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +#![cfg(CONFIG_RUST_DRM_GPUVM)] + +//! DRM GPUVM in immediate mode +//! +//! Rust abstractions for using GPUVM in immediate mode. This is when the GPUVM state is updated +//! during `run_job()`, i.e., in the DMA fence signalling critical path, to ensure that the GPUVM +//! and the GPU's virtual address space has the same state at all times. +//! +//! C header: [`include/drm/drm_gpuvm.h`](srctree/include/drm/drm_gpuvm.h) + +use kernel::{ + alloc::{ + AllocError, + Flags as AllocFlags, // + }, + bindings, + drm, + drm::gem::IntoGEMObject, + error::to_result, + prelude::*, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, // +}; + +use core::{ + cell::UnsafeCell, + marker::PhantomData, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, + ops::{ + Deref, + DerefMut, + Range, // + }, + ptr::{ + self, + NonNull, // + }, // +}; + +mod sm_ops; +pub use self::sm_ops::*; + +mod vm_bo; +pub use self::vm_bo::*; + +mod va; +pub use self::va::*; + +/// A DRM GPU VA manager. +/// +/// This object is refcounted, but the locations of mapped ranges may only be accessed or changed +/// via the special unique handle [`UniqueRefGpuVm`]. +/// +/// # Invariants +/// +/// * Stored in an allocation managed by the refcount in `self.vm`. +/// * Access to `data` and the gpuvm interval tree is controlled via the [`UniqueRefGpuVm`] type. +/// * Does not contain any sparse [`GpuVa<T>`] instances. +#[pin_data] +pub struct GpuVm<T: DriverGpuVm> { + #[pin] + vm: Opaque<bindings::drm_gpuvm>, + /// Accessed only through the [`UniqueRefGpuVm`] reference. + data: UnsafeCell<T>, +} + +// SAFETY: It is safe to send a `GpuVm<T>` to another thread: all data reachable through it +// (`T`, `T::VmBoData`, and the GEM `T::Object`) is `Send` by the `DriverGpuVm` bounds. +unsafe impl<T: DriverGpuVm> Send for GpuVm<T> {} +// SAFETY: It is safe to share a `&GpuVm<T>` between threads: `&self` methods only alias data +// that is `Sync` by the `DriverGpuVm` bounds, and any thread may drop that data, or upgrade the +// reference and ultimately drop `T`, which the same bounds make `Send`. +unsafe impl<T: DriverGpuVm> Sync for GpuVm<T> {} + +// SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. +unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVm<T> { + fn inc_ref(&self) { + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. + unsafe { bindings::drm_gpuvm_get(self.vm.get()) }; + } + + unsafe fn dec_ref(obj: NonNull<Self>) { + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. + unsafe { bindings::drm_gpuvm_put((*obj.as_ptr()).vm.get()) }; + } +} + +impl<T: DriverGpuVm> PartialEq for GpuVm<T> { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self.as_raw(), other.as_raw()) + } +} +impl<T: DriverGpuVm> Eq for GpuVm<T> {} + +impl<T: DriverGpuVm> GpuVm<T> { + const fn vtable() -> &'static bindings::drm_gpuvm_ops { + &bindings::drm_gpuvm_ops { + vm_free: Some(Self::vm_free), + op_alloc: None, + op_free: None, + vm_bo_alloc: GpuVmBo::<T>::ALLOC_FN, + vm_bo_free: GpuVmBo::<T>::FREE_FN, + vm_bo_validate: None, + sm_step_map: Some(Self::sm_step_map), + sm_step_unmap: Some(Self::sm_step_unmap), + sm_step_remap: Some(Self::sm_step_remap), + } + } + + /// Creates a GPUVM instance. + #[expect(clippy::new_ret_no_self)] + pub fn new<E, Ctx: drm::DeviceContext>( + name: &'static CStr, + dev: &drm::Device<T::Driver, Ctx>, + r_obj: &T::Object, + range: Range<u64>, + reserve_range: Range<u64>, + data: T, + ) -> Result<UniqueRefGpuVm<T>, E> + where + E: From<AllocError>, + E: From<core::convert::Infallible>, + { + let obj = KBox::try_pin_init::<E>( + try_pin_init!(Self { + data: UnsafeCell::new(data), + vm <- Opaque::ffi_init(|vm| { + // SAFETY: These arguments are valid. `vm` is valid until refcount drops to + // zero. The `vm` is zeroed before calling this method by `__GFP_ZERO` flag + // below. + unsafe { + bindings::drm_gpuvm_init( + vm, + name.as_char_ptr(), + bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE + | bindings::drm_gpuvm_flags_DRM_GPUVM_RESV_PROTECTED, + dev.as_raw(), + r_obj.as_raw(), + range.start, + range.end - range.start, + reserve_range.start, + reserve_range.end - reserve_range.start, + const { Self::vtable() }, + ) + } + }), + }? E), + GFP_KERNEL | __GFP_ZERO, + )?; + // SAFETY: This transfers the initial refcount to the ARef. + let aref = unsafe { + ARef::from_raw(NonNull::new_unchecked(KBox::into_raw( + Pin::into_inner_unchecked(obj), + ))) + }; + // INVARIANT: This reference is unique. + Ok(UniqueRefGpuVm(aref)) + } + + /// Access this [`GpuVm`] from a raw pointer. + /// + /// # Safety + /// + /// The pointer must reference the `struct drm_gpuvm` in a valid [`GpuVm<T>`] that remains + /// valid for at least `'a`. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm) -> &'a Self { + // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. Caller ensures the + // pointer is valid for 'a. + unsafe { &*kernel::container_of!(Opaque::cast_from(ptr), Self, vm) } + } + + /// Returns a raw pointer to the embedded `struct drm_gpuvm`. + #[inline] + pub fn as_raw(&self) -> *mut bindings::drm_gpuvm { + self.vm.get() + } + + /// The start of the VA space. + #[inline] + pub fn va_start(&self) -> u64 { + // SAFETY: The `mm_start` field is immutable. + unsafe { (*self.as_raw()).mm_start } + } + + /// The length of the GPU's virtual address space. + #[inline] + pub fn va_length(&self) -> u64 { + // SAFETY: The `mm_range` field is immutable. + unsafe { (*self.as_raw()).mm_range } + } + + /// Returns the range of the GPU virtual address space. + #[inline] + pub fn va_range(&self) -> Range<u64> { + let start = self.va_start(); + // OVERFLOW: This reconstructs the Range<u64> passed to the constructor, so it won't fail. + let end = start + self.va_length(); + Range { start, end } + } + + /// Get or create the [`GpuVmBo`] for this gem object. + #[inline] + pub fn obtain( + &self, + obj: &T::Object, + data: impl PinInit<T::VmBoData>, + ) -> Result<ARef<GpuVmBo<T>>, AllocError> { + Ok(GpuVmBoAlloc::new(self, obj, data)?.obtain()) + } + + /// Clean up buffer objects that are no longer used. + #[inline] + pub fn deferred_cleanup(&self) { + // SAFETY: This GPUVM uses immediate mode. + unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.as_raw()) } + } + + /// Check if this GEM object is an external object for this GPUVM. + #[inline] + pub fn is_extobj(&self, obj: &T::Object) -> bool { + // SAFETY: We may call this with any GPUVM and GEM object. + unsafe { bindings::drm_gpuvm_is_extobj(self.as_raw(), obj.as_raw()) } + } + + /// Free this GPUVM. + /// + /// # Safety + /// + /// Called when refcount hits zero. + unsafe extern "C" fn vm_free(me: *mut bindings::drm_gpuvm) { + // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. + let me = unsafe { kernel::container_of!(Opaque::cast_from(me), Self, vm).cast_mut() }; + // SAFETY: By type invariants we can free it when refcount hits zero. + drop(unsafe { KBox::from_raw(me) }) + } + + #[inline] + fn raw_resv(&self) -> *mut bindings::dma_resv { + // SAFETY: `r_obj` is immutable and valid for duration of GPUVM. + unsafe { (*(*self.as_raw()).r_obj).resv } + } +} + +/// The manager for a GPUVM. +pub trait DriverGpuVm: Sized + Send + Sync { + /// Parent `Driver` for this object. + type Driver: drm::Driver; + + /// The kind of GEM object stored in this GPUVM. + type Object: drm::driver::AllocImpl<Driver = Self::Driver> + Send + Sync; + + /// Data stored with each [`struct drm_gpuva`](struct@GpuVa). + /// + /// Only `Send` is required: the data has a single owner at all times, moving + /// between threads by value (handed back as a [`GpuVaRemoved`]) but never + /// accessed by two threads concurrently. + type VaData: Send; + + /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo). + type VmBoData: Send + Sync; + + /// The private data passed to callbacks. + type SmContext<'ctx> + where + Self: 'ctx; + + /// Indicates that a new mapping should be created. + fn sm_step_map<'op, 'ctx>( + &mut self, + op: OpMap<'op, Self>, + context: &mut Self::SmContext<'ctx>, + ) -> Result<OpMapped<'op, Self>, Error>; + + /// Indicates that an existing mapping should be removed. + fn sm_step_unmap<'op, 'ctx>( + &mut self, + op: OpUnmap<'op, Self>, + context: &mut Self::SmContext<'ctx>, + ) -> Result<OpUnmapped<'op, Self>, Error>; + + /// Indicates that an existing mapping should be split up. + fn sm_step_remap<'op, 'ctx>( + &mut self, + op: OpRemap<'op, Self>, + context: &mut Self::SmContext<'ctx>, + ) -> Result<OpRemapped<'op, Self>, Error>; +} + +/// The core of the DRM GPU VA manager. +/// +/// This object is a unique reference to the VM that can access the interval tree and the Rust +/// `data` field. +/// +/// # Invariants +/// +/// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference. +// `Send`/`Sync` derive from `ARef<GpuVm<T>>`; the trait bounds make them correct for the unique +// handle's `&mut T` access. +pub struct UniqueRefGpuVm<T: DriverGpuVm>(ARef<GpuVm<T>>); + +impl<T: DriverGpuVm> UniqueRefGpuVm<T> { + /// Access the data owned by this `UniqueRefGpuVm` immutably. + #[inline] + pub fn data_ref(&self) -> &T { + // SAFETY: By the type invariants we may access `data`. + unsafe { &*self.0.data.get() } + } + + /// Access the data owned by this `UniqueRefGpuVm` mutably. + #[inline] + pub fn data(&mut self) -> &mut T { + // SAFETY: By the type invariants we may access `data`. + unsafe { &mut *self.0.data.get() } + } +} + +impl<T: DriverGpuVm> Deref for UniqueRefGpuVm<T> { + type Target = GpuVm<T>; + + #[inline] + fn deref(&self) -> &GpuVm<T> { + &self.0 + } +} diff --git a/rust/kernel/drm/gpuvm/sm_ops.rs b/rust/kernel/drm/gpuvm/sm_ops.rs new file mode 100644 index 000000000000..742c151b2540 --- /dev/null +++ b/rust/kernel/drm/gpuvm/sm_ops.rs @@ -0,0 +1,429 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +use super::*; + +/// The actual data that gets threaded through the callbacks. +struct SmData<'a, 'ctx, T: DriverGpuVm + 'ctx> { + gpuvm: &'a mut UniqueRefGpuVm<T>, + user_context: &'a mut T::SmContext<'ctx>, +} + +/// Adds an extra field to `SmData` for `sm_map()` callbacks. +/// +/// # Invariants +/// +/// `self.vm_bo.gpuvm() == self.sm_data.gpuvm`. +#[repr(C)] +struct SmMapData<'a, 'ctx, T: DriverGpuVm> { + sm_data: SmData<'a, 'ctx, T>, + vm_bo: &'a GpuVmBo<T>, +} + +/// The argument for [`UniqueRefGpuVm::sm_map`]. +pub struct OpMapRequest<'a, 'ctx, T: DriverGpuVm + 'ctx> { + /// Address in GPU virtual address space. + pub addr: u64, + /// Length of mapping to create. + pub range: u64, + /// Offset in GEM object. + pub gem_offset: u64, + /// The GEM object to map. + pub vm_bo: &'a GpuVmBo<T>, + /// The user-provided context type. + pub context: &'a mut T::SmContext<'ctx>, +} + +impl<'a, 'ctx, T: DriverGpuVm> OpMapRequest<'a, 'ctx, T> { + fn raw_request(&self) -> bindings::drm_gpuvm_map_req { + bindings::drm_gpuvm_map_req { + map: bindings::drm_gpuva_op_map { + va: bindings::drm_gpuva_op_map__bindgen_ty_1 { + addr: self.addr, + range: self.range, + }, + gem: bindings::drm_gpuva_op_map__bindgen_ty_2 { + offset: self.gem_offset, + obj: self.vm_bo.obj().as_raw(), + }, + }, + } + } +} + +/// Represents an `sm_step_map` operation that has not yet been completed. +pub struct OpMap<'op, T: DriverGpuVm> { + op: &'op bindings::drm_gpuva_op_map, + // Since these abstractions are designed for immediate mode, the VM BO needs to be + // pre-allocated, so we always have it available when we reach this point. + vm_bo: &'op GpuVmBo<T>, + // This ensures that 'op is invariant, so that `OpMap<'long, T>` does not + // coerce to `OpMap<'short, T>`. This ensures that the user can't return + // the wrong `OpMapped` value. + _invariant: PhantomData<*mut &'op mut T>, +} + +impl<'op, T: DriverGpuVm> OpMap<'op, T> { + /// The base address of the new mapping. + pub fn addr(&self) -> u64 { + self.op.va.addr + } + + /// The length of the new mapping. + pub fn length(&self) -> u64 { + self.op.va.range + } + + /// The offset within the [`drm_gem_object`](DriverGpuVm::Object). + pub fn gem_offset(&self) -> u64 { + self.op.gem.offset + } + + /// The [`drm_gem_object`](DriverGpuVm::Object) to map. + pub fn obj(&self) -> &T::Object { + // SAFETY: The `obj` pointer is guaranteed to be valid. + unsafe { <T::Object as IntoGEMObject>::from_raw(self.op.gem.obj) } + } + + /// The [`GpuVmBo`] that the new VA will be associated with. + pub fn vm_bo(&self) -> &GpuVmBo<T> { + self.vm_bo + } + + /// Use the pre-allocated VA to carry out this map operation. + pub fn insert(self, va: GpuVaAlloc<T>, va_data: impl PinInit<T::VaData>) -> OpMapped<'op, T> { + let va = va.prepare(va_data); + // SAFETY: By the type invariants we may access the interval tree. + unsafe { bindings::drm_gpuva_map(self.vm_bo.gpuvm().as_raw(), va, self.op) }; + + let _gpuva_guard = self.vm_bo().lock_gpuva(); + // SAFETY: The va is prepared for insertion, and we hold the GEM lock. + unsafe { bindings::drm_gpuva_link(va, self.vm_bo.as_raw()) }; + + OpMapped { + _invariant: self._invariant, + } + } +} + +/// Represents a completed [`OpMap`] operation. +pub struct OpMapped<'op, T> { + _invariant: PhantomData<*mut &'op mut T>, +} + +/// Represents an `sm_step_unmap` operation that has not yet been completed. +pub struct OpUnmap<'op, T: DriverGpuVm> { + op: &'op bindings::drm_gpuva_op_unmap, + // This ensures that 'op is invariant, so that `OpUnmap<'long, T>` does not + // coerce to `OpUnmap<'short, T>`. This ensures that the user can't return the + // wrong`OpUnmapped` value. + _invariant: PhantomData<*mut &'op mut T>, +} + +impl<'op, T: DriverGpuVm> OpUnmap<'op, T> { + /// Indicates whether this [`GpuVa`] is physically contiguous with the + /// original mapping request. + /// + /// Optionally, if `keep` is set, drivers may keep the actual page table + /// mappings for this `drm_gpuva`, adding the missing page table entries + /// only and update the `drm_gpuvm` accordingly. + pub fn keep(&self) -> bool { + self.op.keep + } + + /// The range being unmapped. + pub fn va(&self) -> &GpuVa<T> { + // SAFETY: This is a valid va. It's not the `kernel_alloc_node` because you can't unmap it, + // and it's not sparse by the `GpuVm<T>` type invariants. + unsafe { GpuVa::<T>::from_raw(self.op.va) } + } + + /// Remove the VA. + pub fn remove(self) -> (OpUnmapped<'op, T>, GpuVaRemoved<T>) { + // SAFETY: The op references a valid drm_gpuva in the GPUVM. + unsafe { bindings::drm_gpuva_unmap(self.op) }; + // SAFETY: The va is no longer in the interval tree so we may unlink it. + unsafe { bindings::drm_gpuva_unlink_defer(self.op.va) }; + + // SAFETY: We just removed this va from the `GpuVm<T>`. + let va = unsafe { GpuVaRemoved::from_raw(self.op.va) }; + + ( + OpUnmapped { + _invariant: self._invariant, + }, + va, + ) + } +} + +/// Represents a completed [`OpUnmap`] operation. +pub struct OpUnmapped<'op, T> { + _invariant: PhantomData<*mut &'op mut T>, +} + +/// Represents an `sm_step_remap` operation that has not yet been completed. +pub struct OpRemap<'op, T: DriverGpuVm> { + op: &'op bindings::drm_gpuva_op_remap, + // This ensures that 'op is invariant, so that `OpRemap<'long, T>` does not + // coerce to `OpRemap<'short, T>`. This ensures that the user can't return the + // wrong`OpRemapped` value. + _invariant: PhantomData<*mut &'op mut T>, +} + +impl<'op, T: DriverGpuVm> OpRemap<'op, T> { + /// The preceding part of a split mapping. + #[inline] + pub fn prev(&self) -> Option<&OpRemapMapData> { + // SAFETY: We checked for null, so the pointer must be valid. + NonNull::new(self.op.prev).map(|ptr| unsafe { OpRemapMapData::from_raw(ptr) }) + } + + /// The subsequent part of a split mapping. + #[inline] + pub fn next(&self) -> Option<&OpRemapMapData> { + // SAFETY: We checked for null, so the pointer must be valid. + NonNull::new(self.op.next).map(|ptr| unsafe { OpRemapMapData::from_raw(ptr) }) + } + + /// Indicates whether the `drm_gpuva` being removed is physically contiguous with the original + /// mapping request. + /// + /// Optionally, if `keep` is set, drivers may keep the actual page table mappings for this + /// `drm_gpuva`, adding the missing page table entries only and update the `drm_gpuvm` + /// accordingly. + #[inline] + pub fn keep(&self) -> bool { + // SAFETY: The unmap pointer is always valid. + unsafe { (*self.op.unmap).keep } + } + + /// The range being unmapped. + #[inline] + pub fn va_to_unmap(&self) -> &GpuVa<T> { + // SAFETY: This is a valid va. It's not the `kernel_alloc_node` because you can't unmap it, + // and it's not sparse by the `GpuVm<T>` type invariants. + unsafe { GpuVa::<T>::from_raw((*self.op.unmap).va) } + } + + /// The [`drm_gem_object`](DriverGpuVm::Object) whose VA is being remapped. + #[inline] + pub fn obj(&self) -> &T::Object { + self.va_to_unmap().obj() + } + + /// The [`GpuVmBo`] that is being remapped. + #[inline] + pub fn vm_bo(&self) -> &GpuVmBo<T> { + self.va_to_unmap().vm_bo() + } + + /// Update the GPUVM to perform the remapping. + pub fn remap( + self, + va_alloc: [GpuVaAlloc<T>; 2], + prev_data: impl PinInit<T::VaData>, + next_data: impl PinInit<T::VaData>, + ) -> (OpRemapped<'op, T>, OpRemapRet<T>) { + let [va1, va2] = va_alloc; + + let mut unused_va = None; + let mut prev_ptr = ptr::null_mut(); + let mut next_ptr = ptr::null_mut(); + if self.prev().is_some() { + prev_ptr = va1.prepare(prev_data); + } else { + unused_va = Some(va1); + } + if self.next().is_some() { + next_ptr = va2.prepare(next_data); + } else { + unused_va = Some(va2); + } + + // SAFETY: the pointers are non-null when required + unsafe { bindings::drm_gpuva_remap(prev_ptr, next_ptr, self.op) }; + + let gpuva_guard = self.vm_bo().lock_gpuva(); + if !prev_ptr.is_null() { + // SAFETY: The prev_ptr is a valid drm_gpuva prepared for insertion. The vm_bo is still + // valid as the not-yet-unlinked gpuva holds a refcount on the vm_bo. + unsafe { bindings::drm_gpuva_link(prev_ptr, self.vm_bo().as_raw()) }; + } + if !next_ptr.is_null() { + // SAFETY: The next_ptr is a valid drm_gpuva prepared for insertion. The vm_bo is still + // valid as the not-yet-unlinked gpuva holds a refcount on the vm_bo. + unsafe { bindings::drm_gpuva_link(next_ptr, self.vm_bo().as_raw()) }; + } + drop(gpuva_guard); + + // SAFETY: The va is no longer in the interval tree so we may unlink it. + unsafe { bindings::drm_gpuva_unlink_defer((*self.op.unmap).va) }; + + ( + OpRemapped { + _invariant: self._invariant, + }, + OpRemapRet { + // SAFETY: We just removed this va from the `GpuVm<T>`. + unmapped_va: unsafe { GpuVaRemoved::from_raw((*self.op.unmap).va) }, + unused_va, + }, + ) + } +} + +/// Part of an [`OpRemap`] that represents a new mapping. +#[repr(transparent)] +pub struct OpRemapMapData(bindings::drm_gpuva_op_map); + +impl OpRemapMapData { + /// # Safety + /// Must reference a valid `drm_gpuva_op_map` for duration of `'a`. + unsafe fn from_raw<'a>(ptr: NonNull<bindings::drm_gpuva_op_map>) -> &'a Self { + // SAFETY: ok per safety requirements + unsafe { ptr.cast().as_ref() } + } + + /// The base address of the new mapping. + pub fn addr(&self) -> u64 { + self.0.va.addr + } + + /// The length of the new mapping. + pub fn length(&self) -> u64 { + self.0.va.range + } + + /// The offset within the [`drm_gem_object`](DriverGpuVm::Object). + pub fn gem_offset(&self) -> u64 { + self.0.gem.offset + } +} + +/// Struct containing objects removed or not used by [`OpRemap::remap`]. +pub struct OpRemapRet<T: DriverGpuVm> { + /// The `drm_gpuva` that was removed. + pub unmapped_va: GpuVaRemoved<T>, + /// If the remap did not split the region into two pieces, then the unused `drm_gpuva` is + /// returned here. + pub unused_va: Option<GpuVaAlloc<T>>, +} + +/// Represents a completed [`OpRemap`] operation. +pub struct OpRemapped<'op, T> { + _invariant: PhantomData<*mut &'op mut T>, +} + +impl<T: DriverGpuVm> UniqueRefGpuVm<T> { + /// Create a mapping, removing or remapping anything that overlaps. + /// + /// Internally calls the [`DriverGpuVm`] callbacks similar to [`Self::sm_unmap`], except that + /// the [`DriverGpuVm::sm_step_map`] is called once to create the requested mapping. + #[inline] + pub fn sm_map(&mut self, req: OpMapRequest<'_, '_, T>) -> Result { + if req.vm_bo.gpuvm() != &**self { + return Err(EINVAL); + } + + let gpuvm = self.as_raw(); + let raw_req = req.raw_request(); + // INVARIANT: Checked above that `vm_bo.gpuvm() == self`. + let mut p = SmMapData { + sm_data: SmData { + gpuvm: self, + user_context: req.context, + }, + vm_bo: req.vm_bo, + }; + // SAFETY: + // * raw_request() creates a valid request. + // * The private data is valid to be interpreted as both SmData and SmMapData since the + // first field of SmMapData is SmData. + to_result(unsafe { + bindings::drm_gpuvm_sm_map(gpuvm, (&raw mut p).cast(), &raw const raw_req) + }) + } + + /// Remove any mappings in the given region. + /// + /// Internally calls [`DriverGpuVm::sm_step_unmap`] for ranges entirely contained within the + /// given range, and [`DriverGpuVm::sm_step_remap`] for ranges that overlap with the range. + #[inline] + pub fn sm_unmap(&mut self, addr: u64, length: u64, context: &mut T::SmContext<'_>) -> Result { + let gpuvm = self.as_raw(); + let mut p = SmData { + gpuvm: self, + user_context: context, + }; + // SAFETY: + // * raw_request() creates a valid request. + // * The private data is a valid SmData. + to_result(unsafe { bindings::drm_gpuvm_sm_unmap(gpuvm, (&raw mut p).cast(), addr, length) }) + } +} + +impl<T: DriverGpuVm> GpuVm<T> { + /// # Safety + /// Must be called from `sm_map` with a pointer to `SmMapData`. + pub(super) unsafe extern "C" fn sm_step_map( + op: *mut bindings::drm_gpuva_op, + p: *mut c_void, + ) -> c_int { + // SAFETY: If we reach `sm_step_map` then we were called from `sm_map` which always passes + // an `SmMapData` as private data. + let p = unsafe { &mut *p.cast::<SmMapData<'_, '_, T>>() }; + let op = OpMap { + // SAFETY: sm_step_map is called with a map operation. + op: unsafe { &(*op).__bindgen_anon_1.map }, + vm_bo: p.vm_bo, + _invariant: PhantomData, + }; + match p + .sm_data + .gpuvm + .data() + .sm_step_map(op, p.sm_data.user_context) + { + Ok(OpMapped { .. }) => 0, + Err(err) => err.to_errno(), + } + } + + /// # Safety + /// Must be called from `sm_map` or `sm_unmap` with a pointer to `SmMapData` or `SmData`. + pub(super) unsafe extern "C" fn sm_step_unmap( + op: *mut bindings::drm_gpuva_op, + p: *mut c_void, + ) -> c_int { + // SAFETY: The caller provides a pointer that can be treated as `SmData`. + let p = unsafe { &mut *p.cast::<SmData<'_, '_, T>>() }; + let op = OpUnmap { + // SAFETY: sm_step_unmap is called with an unmap operation. + op: unsafe { &(*op).__bindgen_anon_1.unmap }, + _invariant: PhantomData, + }; + match p.gpuvm.data().sm_step_unmap(op, p.user_context) { + Ok(OpUnmapped { .. }) => 0, + Err(err) => err.to_errno(), + } + } + + /// # Safety + /// Must be called from `sm_map` or `sm_unmap` with a pointer to `SmMapData` or `SmData`. + pub(super) unsafe extern "C" fn sm_step_remap( + op: *mut bindings::drm_gpuva_op, + p: *mut c_void, + ) -> c_int { + // SAFETY: The caller provides a pointer that can be treated as `SmData`. + let p = unsafe { &mut *p.cast::<SmData<'_, '_, T>>() }; + let op = OpRemap { + // SAFETY: sm_step_remap is called with a remap operation. + op: unsafe { &(*op).__bindgen_anon_1.remap }, + _invariant: PhantomData, + }; + match p.gpuvm.data().sm_step_remap(op, p.user_context) { + Ok(OpRemapped { .. }) => 0, + Err(err) => err.to_errno(), + } + } +} diff --git a/rust/kernel/drm/gpuvm/va.rs b/rust/kernel/drm/gpuvm/va.rs new file mode 100644 index 000000000000..46493f0ec5a6 --- /dev/null +++ b/rust/kernel/drm/gpuvm/va.rs @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +use super::*; + +/// Represents that a range of a GEM object is mapped in this [`GpuVm`] instance. +/// +/// Does not assume that GEM lock is held. +/// +/// # Invariants +/// +/// * This is a valid `drm_gpuva` object that is resident in a [`GpuVm<T>`] instance. +/// * It is associated with a [`GpuVmBo<T>`]. Or in other words, it's not an +/// `gpuvm->kernel_alloc_node` and `DRM_GPUVA_SPARSE` is not set. +/// * The associated [`GpuVmBo<T>`] is part of the GEM list. +#[repr(C)] +#[pin_data] +pub struct GpuVa<T: DriverGpuVm> { + #[pin] + inner: Opaque<bindings::drm_gpuva>, + #[pin] + data: T::VaData, +} + +impl<T: DriverGpuVm> PartialEq for GpuVa<T> { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self.as_raw(), other.as_raw()) + } +} +impl<T: DriverGpuVm> Eq for GpuVa<T> {} + +impl<T: DriverGpuVm> GpuVa<T> { + /// Access this [`GpuVa`] from a raw pointer. + /// + /// # Safety + /// + /// * For the duration of `'a`, the pointer must reference a valid `drm_gpuva` associated with + /// a [`GpuVm<T>`]. + /// * It must be associated with a [`GpuVmBo<T>`]. + /// * The associated [`GpuVmBo<T>`] is part of the GEM list. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuva) -> &'a Self { + // CAST: `drm_gpuva` is first field and `repr(C)`. + // SAFETY: The safety requirements match the invariants of `GpuVa`. + unsafe { &*ptr.cast() } + } + + /// Returns a raw pointer to underlying C value. + #[inline] + pub fn as_raw(&self) -> *mut bindings::drm_gpuva { + self.inner.get() + } + + /// Returns the address of this mapping in the GPU virtual address space. + #[inline] + pub fn addr(&self) -> u64 { + // SAFETY: The `va.addr` field of `drm_gpuva` is immutable. + unsafe { (*self.as_raw()).va.addr } + } + + /// Returns the length of this mapping. + #[inline] + pub fn length(&self) -> u64 { + // SAFETY: The `va.range` field of `drm_gpuva` is immutable. + unsafe { (*self.as_raw()).va.range } + } + + /// Returns `addr..addr+length`. + #[inline] + pub fn range(&self) -> Range<u64> { + let addr = self.addr(); + addr..addr + self.length() + } + + /// Returns the offset within the GEM object. + #[inline] + pub fn gem_offset(&self) -> u64 { + // SAFETY: The `gem.offset` field of `drm_gpuva` is immutable. + unsafe { (*self.as_raw()).gem.offset } + } + + /// Returns the GEM object. + #[inline] + pub fn obj(&self) -> &T::Object { + // SAFETY: The `gem.obj` field of `drm_gpuva` is immutable. We know that it's not null + // because this VA is associated with a `GpuVmBo<T>`. + unsafe { <T::Object as IntoGEMObject>::from_raw((*self.as_raw()).gem.obj) } + } + + /// Returns the underlying [`GpuVmBo`] object that backs this [`GpuVa`]. + #[inline] + pub fn vm_bo(&self) -> &GpuVmBo<T> { + // SAFETY: The `vm_bo` field of `drm_gpuva` is immutable. We know that it's not null + // because this VA is associated with a `GpuVmBo<T>`. The BO is in the GEM list by the type + // invariants. + unsafe { GpuVmBo::from_raw((*self.as_raw()).vm_bo) } + } +} + +/// A pre-allocated [`GpuVa`] object. +/// +/// # Invariants +/// +/// The memory is zeroed. +pub struct GpuVaAlloc<T: DriverGpuVm>(KBox<MaybeUninit<GpuVa<T>>>); + +// SAFETY: A `GpuVaAlloc` is an owned, uninitialised allocation with no live `T::VaData` and no +// thread-bound state. +unsafe impl<T: DriverGpuVm> Send for GpuVaAlloc<T> {} + +// SAFETY: A `GpuVaAlloc` has no `&self` method that reaches its contents, so a shared +// `&GpuVaAlloc` cannot access the allocation. +unsafe impl<T: DriverGpuVm> Sync for GpuVaAlloc<T> {} + +impl<T: DriverGpuVm> GpuVaAlloc<T> { + /// Pre-allocate a [`GpuVa`] object. + pub fn new(flags: AllocFlags) -> Result<GpuVaAlloc<T>, AllocError> { + // INVARIANTS: Memory allocated with __GFP_ZERO. + Ok(GpuVaAlloc(KBox::new_uninit(flags | __GFP_ZERO)?)) + } + + /// Prepare this `drm_gpuva` for insertion into the GPUVM. + #[must_use] + pub(super) fn prepare(mut self, va_data: impl PinInit<T::VaData>) -> *mut bindings::drm_gpuva { + let va_ptr = MaybeUninit::as_mut_ptr(&mut self.0); + // SAFETY: The `data` field is pinned. + unsafe { pin_init::raw_init(&raw mut (*va_ptr).data, va_data) }; + KBox::into_raw(self.0).cast() + } +} + +/// A [`GpuVa`] object that has been removed. +/// +/// # Invariants +/// +/// The `drm_gpuva` is not resident in the [`GpuVm`]. +pub struct GpuVaRemoved<T: DriverGpuVm>(KBox<GpuVa<T>>); + +impl<T: DriverGpuVm> GpuVaRemoved<T> { + /// Convert a raw pointer into a [`GpuVaRemoved`]. + /// + /// # Safety + /// + /// * Must have been removed from a [`GpuVm<T>`]. + /// * It must not be a `gpuvm->kernel_alloc_node` va. + pub(super) unsafe fn from_raw(ptr: *mut bindings::drm_gpuva) -> Self { + // SAFETY: Since it used to be a VA in a `GpuVm<T>` and it's not a kernel_alloc_node, this + // pointer references a `GpuVa<T>` with a valid `T::VaData`. Since it has been removed, we + // can take ownership of the allocation. + GpuVaRemoved(unsafe { KBox::from_raw(ptr.cast()) }) + } + + /// Take ownership of the VA data. + pub fn into_inner(self) -> T::VaData + where + T::VaData: Unpin, + { + KBox::into_inner(self.0).data + } +} + +impl<T: DriverGpuVm> Deref for GpuVaRemoved<T> { + type Target = T::VaData; + fn deref(&self) -> &T::VaData { + &self.0.data + } +} + +impl<T: DriverGpuVm> DerefMut for GpuVaRemoved<T> +where + T::VaData: Unpin, +{ + fn deref_mut(&mut self) -> &mut T::VaData { + &mut self.0.data + } +} diff --git a/rust/kernel/drm/gpuvm/vm_bo.rs b/rust/kernel/drm/gpuvm/vm_bo.rs new file mode 100644 index 000000000000..5989972da829 --- /dev/null +++ b/rust/kernel/drm/gpuvm/vm_bo.rs @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +use super::*; + +/// Represents that a given GEM object has at least one mapping on this [`GpuVm`] instance. +/// +/// Does not assume that GEM lock is held. +/// +/// # Invariants +/// +/// * Allocated with `kmalloc` and refcounted via `inner`. +/// * Is present in the gem list. +#[repr(C)] +#[pin_data] +pub struct GpuVmBo<T: DriverGpuVm> { + #[pin] + inner: Opaque<bindings::drm_gpuvm_bo>, + #[pin] + data: T::VmBoData, +} + +// SAFETY: It is safe to send a `GpuVmBo<T>` to another thread: dropping it there drops +// `T::VmBoData` and the GEM `T::Object`, both `Send` by the `DriverGpuVm` bounds. +unsafe impl<T: DriverGpuVm> Send for GpuVmBo<T> {} + +// SAFETY: It is safe to share a `&GpuVmBo<T>` between threads: it effectively shares +// `&T::VmBoData` and the GEM `&T::Object` (both `Sync`), and any thread may upgrade to an +// `ARef` and ultimately drop them (both `Send`), per the `DriverGpuVm` bounds. +unsafe impl<T: DriverGpuVm> Sync for GpuVmBo<T> {} + +// SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`. +unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVmBo<T> { + fn inc_ref(&self) { + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`. + unsafe { bindings::drm_gpuvm_bo_get(self.inner.get()) }; + } + + unsafe fn dec_ref(obj: NonNull<Self>) { + // CAST: `drm_gpuvm_bo` is first field of repr(C) struct. + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`. + // This GPUVM instance uses immediate mode, so we may put the refcount using the deferred + // mechanism. + unsafe { bindings::drm_gpuvm_bo_put_deferred(obj.as_ptr().cast()) }; + } +} + +impl<T: DriverGpuVm> PartialEq for GpuVmBo<T> { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self.as_raw(), other.as_raw()) + } +} +impl<T: DriverGpuVm> Eq for GpuVmBo<T> {} + +impl<T: DriverGpuVm> GpuVmBo<T> { + /// The function pointer for allocating a GpuVmBo stored in the gpuvm vtable. + /// + /// Allocation is always implemented according to [`Self::vm_bo_alloc`], but it is set to + /// `None` if the default gpuvm behavior is the same as `vm_bo_alloc`. + /// + /// This may be `Some` even if `FREE_FN` is `None`, or vice-versa. + pub(super) const ALLOC_FN: Option<unsafe extern "C" fn() -> *mut bindings::drm_gpuvm_bo> = { + use core::alloc::Layout; + let base = Layout::new::<bindings::drm_gpuvm_bo>(); + let rust = Layout::new::<Self>(); + assert!(base.size() <= rust.size()); + if base.size() != rust.size() || base.align() != rust.align() { + Some(Self::vm_bo_alloc) + } else { + // This causes GPUVM to allocate a `GpuVmBo<T>` with `kzalloc(sizeof(drm_gpuvm_bo))`. + None + } + }; + + /// The function pointer for freeing a GpuVmBo stored in the gpuvm vtable. + /// + /// Freeing is always implemented according to [`Self::vm_bo_free`], but it is set to `None` if + /// the default gpuvm behavior is the same as `vm_bo_free`. + /// + /// This may be `Some` even if `ALLOC_FN` is `None`, or vice-versa. + pub(super) const FREE_FN: Option<unsafe extern "C" fn(*mut bindings::drm_gpuvm_bo)> = { + if core::mem::needs_drop::<Self>() { + Some(Self::vm_bo_free) + } else { + // This causes GPUVM to free a `GpuVmBo<T>` with `kfree`. + None + } + }; + + /// Custom function for allocating a `drm_gpuvm_bo`. + /// + /// # Safety + /// + /// Always safe to call. + unsafe extern "C" fn vm_bo_alloc() -> *mut bindings::drm_gpuvm_bo { + let raw_ptr = KBox::<Self>::new_uninit(GFP_KERNEL | __GFP_ZERO) + .map(KBox::into_raw) + .unwrap_or(ptr::null_mut()); + + // CAST: `drm_gpuvm_bo` is first field of `Self`. + raw_ptr.cast() + } + + /// Custom function for freeing a `drm_gpuvm_bo`. + /// + /// # Safety + /// + /// The pointer must have been allocated with [`GpuVmBo::ALLOC_FN`], and must not be used after + /// this call. + unsafe extern "C" fn vm_bo_free(ptr: *mut bindings::drm_gpuvm_bo) { + // CAST: `drm_gpuvm_bo` is first field of `Self`. + // SAFETY: + // * The ptr was allocated from kmalloc with the layout of `GpuVmBo<T>`. + // * `ptr->inner` has no destructor. + // * `ptr->data` contains a valid `T::VmBoData` that we can drop. + drop(unsafe { KBox::<Self>::from_raw(ptr.cast()) }); + } + + /// Access this [`GpuVmBo`] from a raw pointer. + /// + /// # Safety + /// + /// For the duration of `'a`, the pointer must reference a valid `drm_gpuvm_bo` associated with + /// a [`GpuVm<T>`]. The BO must also be present in the GEM list. + #[inline] + pub(crate) unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm_bo) -> &'a Self { + // SAFETY: `drm_gpuvm_bo` is first field and `repr(C)`. + unsafe { &*ptr.cast() } + } + + /// Returns a raw pointer to underlying C value. + #[inline] + pub fn as_raw(&self) -> *mut bindings::drm_gpuvm_bo { + self.inner.get() + } + + /// The [`GpuVm`] that this GEM object is mapped in. + #[inline] + pub fn gpuvm(&self) -> &GpuVm<T> { + // SAFETY: The `obj` pointer is guaranteed to be valid. + unsafe { GpuVm::<T>::from_raw((*self.inner.get()).vm) } + } + + /// The [`drm_gem_object`](DriverGpuVm::Object) for these mappings. + #[inline] + pub fn obj(&self) -> &T::Object { + // SAFETY: The `obj` pointer is guaranteed to be valid. + unsafe { <T::Object as IntoGEMObject>::from_raw((*self.inner.get()).obj) } + } + + /// The driver data with this buffer object. + #[inline] + pub fn data(&self) -> &T::VmBoData { + &self.data + } + + pub(super) fn lock_gpuva(&self) -> crate::sync::MutexGuard<'_, ()> { + // SAFETY: The GEM object is valid. + let ptr = unsafe { &raw mut (*self.obj().as_raw()).gpuva.lock }; + // SAFETY: The GEM object is valid, so the mutex is properly initialized. + let mutex = unsafe { crate::sync::Mutex::from_raw(ptr) }; + mutex.lock() + } +} + +/// A pre-allocated [`GpuVmBo`] object. +/// +/// # Invariants +/// +/// Points at a `drm_gpuvm_bo` that contains a valid `T::VmBoData`, has a refcount of one, and is +/// absent from any gem, extobj, or evict lists. +pub(super) struct GpuVmBoAlloc<T: DriverGpuVm>(NonNull<GpuVmBo<T>>); + +impl<T: DriverGpuVm> GpuVmBoAlloc<T> { + /// Create a new pre-allocated [`GpuVmBo`]. + /// + /// It's intentional that the initializer is infallible because `drm_gpuvm_bo_put` will call + /// drop on the data, so we don't have a way to free it when the data is missing. + #[inline] + pub(super) fn new( + gpuvm: &GpuVm<T>, + gem: &T::Object, + value: impl PinInit<T::VmBoData>, + ) -> Result<GpuVmBoAlloc<T>, AllocError> { + // CAST: `GpuVmBoAlloc::vm_bo_alloc` ensures that this memory was allocated with the layout + // of `GpuVmBo<T>`. The type is repr(C), so `container_of` is not required. + // SAFETY: The provided gpuvm and gem ptrs are valid for the duration of this call. + let raw_ptr = unsafe { + bindings::drm_gpuvm_bo_create(gpuvm.as_raw(), gem.as_raw()).cast::<GpuVmBo<T>>() + }; + let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; + // SAFETY: `ptr->data` is a valid pinned location. + unsafe { pin_init::raw_init(&raw mut (*raw_ptr).data, value) }; + // INVARIANTS: We just created the vm_bo so it's absent from lists, and the data is valid + // as we just initialized it. + Ok(GpuVmBoAlloc(ptr)) + } + + /// Returns a raw pointer to underlying C value. + #[inline] + pub(super) fn as_raw(&self) -> *mut bindings::drm_gpuvm_bo { + // SAFETY: The pointer references a valid `drm_gpuvm_bo`. + unsafe { (*self.0.as_ptr()).inner.get() } + } + + /// Look up whether there is an existing [`GpuVmBo`] for this gem object. + /// + /// The caller should not hold the GEM mutex or DMA resv lock. + #[inline] + pub(super) fn obtain(self) -> ARef<GpuVmBo<T>> { + let me = ManuallyDrop::new(self); + // SAFETY: Valid `drm_gpuvm_bo` not already in the lists. We do not access `me` after this + // call. + let ptr = unsafe { bindings::drm_gpuvm_bo_obtain_prealloc(me.as_raw()) }; + + // SAFETY: `drm_gpuvm_bo_obtain_prealloc` always returns a non-null ptr + let nonnull = unsafe { NonNull::new_unchecked(ptr.cast()) }; + + // INVARIANTS: `drm_gpuvm_bo_obtain_prealloc` ensures that the bo is in the GEM list. + // SAFETY: We received one refcount from `drm_gpuvm_bo_obtain_prealloc`. + let ret = unsafe { ARef::<GpuVmBo<T>>::from_raw(nonnull) }; + + // Ensure that external objects are in the extobj list. + // + // Note that we must call `extobj_add` even if `ptr != me` to avoid a race condition where + // we could end up using the extobj before the thread with `ptr == me` calls extobj_add. + if ret.gpuvm().is_extobj(ret.obj()) { + let resv_lock = ret.gpuvm().raw_resv(); + // TODO: Use a proper lock guard here once a dma_resv lock abstraction exists. + // SAFETY: The GPUVM is still alive, so its resv lock is too. + unsafe { bindings::dma_resv_lock(resv_lock, ptr::null_mut()) }; + // SAFETY: We hold the GPUVMs resv lock. + unsafe { bindings::drm_gpuvm_bo_extobj_add(ptr) }; + // SAFETY: We took the lock, so we can unlock it. + unsafe { bindings::dma_resv_unlock(resv_lock) }; + } + + ret + } +} + +impl<T: DriverGpuVm> Deref for GpuVmBoAlloc<T> { + type Target = GpuVmBo<T>; + #[inline] + fn deref(&self) -> &GpuVmBo<T> { + // SAFETY: By the type invariants we may deref while `Self` exists. + unsafe { self.0.as_ref() } + } +} + +impl<T: DriverGpuVm> Drop for GpuVmBoAlloc<T> { + #[inline] + fn drop(&mut self) { + // TODO: Call drm_gpuvm_bo_destroy_not_in_lists() directly. + // SAFETY: It's safe to perform a deferred put in any context. + unsafe { bindings::drm_gpuvm_bo_put_deferred(self.as_raw()) }; + } +} diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs index cf328101dde4..64af9eacc306 100644 --- a/rust/kernel/drm/ioctl.rs +++ b/rust/kernel/drm/ioctl.rs @@ -70,6 +70,18 @@ pub mod internal { pub use bindings::drm_device; pub use bindings::drm_file; pub use bindings::drm_ioctl_desc; + + /// Cast an [`Ioctl`] DRM device pointer to [`Registered`], preserving the driver type + /// parameter `T`. + /// + /// Used by [`declare_drm_ioctls!`] to anchor type inference. + #[doc(hidden)] + #[inline] + pub const fn __dev_ctx_cast<T: crate::drm::Driver>( + ptr: *const crate::drm::Device<T, crate::drm::Ioctl>, + ) -> *const crate::drm::Device<T, crate::drm::Registered> { + ptr.cast() + } } /// Declare the DRM ioctls for a driver. @@ -82,7 +94,8 @@ pub mod internal { /// `user_callback` should have the following prototype: /// /// ```ignore -/// fn foo(device: &kernel::drm::Device<Self>, +/// fn foo(device: &kernel::drm::Device<Self, kernel::drm::Registered>, +/// reg_data: &Self::RegistrationData<'_>, /// data: &mut uapi::argument_type, /// file: &kernel::drm::File<Self::File>, /// ) -> Result<u32> @@ -131,10 +144,45 @@ macro_rules! declare_drm_ioctls { // - The DRM device must have been registered when we're called through // an IOCTL. // + // INVARIANT: The `Ioctl` context requires that the device has been + // registered via `drm_dev_register()` at some point; the DRM core + // guarantees this for ioctl dispatch callbacks. + // // FIXME: Currently there is nothing enforcing that the types of the // dev/file match the current driver these ioctls are being declared // for, and it's not clear how to enforce this within the type system. - let dev = $crate::drm::device::Device::from_raw(raw_dev); + let dev: &$crate::drm::device::Device<_, $crate::drm::Ioctl> = + $crate::drm::device::Device::from_raw(raw_dev); + + // Type-inference anchor: the closure is never called but ties `dev`'s + // type to `$func`'s first parameter, which the compiler cannot infer + // through method resolution and associated-type projections alone. + #[allow(unreachable_code)] + let _ = || { + let __ptr = $crate::drm::ioctl::internal::__dev_ctx_cast( + ::core::ptr::from_ref(dev), + ); + + $func( + // SAFETY: This closure is never executed; the dereference + // exists purely to unify the type parameter with `$func`. + // The pointer is valid regardless. + unsafe { &*__ptr }, + unreachable!(), + unreachable!(), + unreachable!(), + ) + }; + + // Enforce that the handler accepts higher-ranked + // lifetimes, preventing it from requiring 'static + // references that could escape this scope. + let _: for<'a> fn(&'a _, &'a _, &'a mut _, &'a _) -> _ = $func; + + let Some(guard) = dev.registration_guard() else { + return $crate::error::code::ENODEV.to_errno(); + }; + // SAFETY: The ioctl argument has size `_IOC_SIZE(cmd)`, which we // asserted above matches the size of this type, and all bit patterns of // UAPI structs must be valid. @@ -147,7 +195,9 @@ macro_rules! declare_drm_ioctls { // SAFETY: This is just the DRM file structure let file = unsafe { $crate::drm::File::from_raw(raw_file) }; - match $func(dev, data, file) { + match guard.registration_data_with(|reg_data| { + $func(&*guard, reg_data, data, file) + }) { Err(e) => e.to_errno(), Ok(i) => i.try_into() .unwrap_or($crate::error::code::ERANGE.to_errno()), diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs index 1b82b6945edf..fd6ed35bc35a 100644 --- a/rust/kernel/drm/mod.rs +++ b/rust/kernel/drm/mod.rs @@ -6,9 +6,16 @@ pub mod device; pub mod driver; pub mod file; pub mod gem; +pub mod gpuvm; pub mod ioctl; pub use self::device::Device; +pub use self::device::DeviceContext; +pub use self::device::Ioctl; +pub use self::device::Normal; +pub use self::device::Registered; +pub use self::device::RegistrationGuard; +pub use self::device::UnregisteredDevice; pub use self::driver::Driver; pub use self::driver::DriverInfo; pub use self::driver::Registration; diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 258b12afdcba..e52793f77196 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -25,13 +25,12 @@ pub mod code { #[doc = $doc] )* pub const $err: super::Error = - match super::Error::try_from_errno(-(crate::bindings::$err as i32)) { - Some(err) => err, - None => panic!("Invalid errno in `declare_err!`"), - }; + super::Error::try_from_errno(-(crate::bindings::$err as i32)) + .expect("Invalid errno in `declare_err!`"); }; } + // From `include/uapi/asm-generic/errno-base.h`. declare_err!(EPERM, "Operation not permitted."); declare_err!(ENOENT, "No such file or directory."); declare_err!(ESRCH, "No such process."); @@ -66,8 +65,110 @@ pub mod code { declare_err!(EPIPE, "Broken pipe."); declare_err!(EDOM, "Math argument out of domain of func."); declare_err!(ERANGE, "Math result not representable."); + + // From `include/uapi/asm-generic/errno.h`. + declare_err!(EDEADLK, "Resource deadlock would occur."); + declare_err!(ENAMETOOLONG, "File name too long."); + declare_err!(ENOLCK, "No record locks available."); + declare_err!(ENOSYS, "Invalid system call number."); + declare_err!(ENOTEMPTY, "Directory not empty."); + declare_err!(ELOOP, "Too many symbolic links encountered."); + declare_err!(ENOMSG, "No message of desired type."); + declare_err!(EIDRM, "Identifier removed."); + declare_err!(ECHRNG, "Channel number out of range."); + declare_err!(EL2NSYNC, "Level 2 not synchronized."); + declare_err!(EL3HLT, "Level 3 halted."); + declare_err!(EL3RST, "Level 3 reset."); + declare_err!(ELNRNG, "Link number out of range."); + declare_err!(EUNATCH, "Protocol driver not attached."); + declare_err!(ENOCSI, "No CSI structure available."); + declare_err!(EL2HLT, "Level 2 halted."); + declare_err!(EBADE, "Invalid exchange."); + declare_err!(EBADR, "Invalid request descriptor."); + declare_err!(EXFULL, "Exchange full."); + declare_err!(ENOANO, "No anode."); + declare_err!(EBADRQC, "Invalid request code."); + declare_err!(EBADSLT, "Invalid slot."); + declare_err!(EBFONT, "Bad font file format."); + declare_err!(ENOSTR, "Device not a stream."); + declare_err!(ENODATA, "No data available."); + declare_err!(ETIME, "Timer expired."); + declare_err!(ENOSR, "Out of streams resources."); + declare_err!(ENONET, "Machine is not on the network."); + declare_err!(ENOPKG, "Package not installed."); + declare_err!(EREMOTE, "Object is remote."); + declare_err!(ENOLINK, "Link has been severed."); + declare_err!(EADV, "Advertise error."); + declare_err!(ESRMNT, "Srmount error."); + declare_err!(ECOMM, "Communication error on send."); + declare_err!(EPROTO, "Protocol error."); + declare_err!(EMULTIHOP, "Multihop attempted."); + declare_err!(EDOTDOT, "RFS specific error."); + declare_err!(EBADMSG, "Not a data message."); + declare_err!(EFSBADCRC, "Bad CRC detected."); declare_err!(EOVERFLOW, "Value too large for defined data type."); + declare_err!(ENOTUNIQ, "Name not unique on network."); + declare_err!(EBADFD, "File descriptor in bad state."); + declare_err!(EREMCHG, "Remote address changed."); + declare_err!(ELIBACC, "Can not access a needed shared library."); + declare_err!(ELIBBAD, "Accessing a corrupted shared library."); + declare_err!(ELIBSCN, ".lib section in a.out corrupted."); + declare_err!(ELIBMAX, "Attempting to link in too many shared libraries."); + declare_err!(ELIBEXEC, "Cannot exec a shared library directly."); + declare_err!(EILSEQ, "Illegal byte sequence."); + declare_err!(ERESTART, "Interrupted system call should be restarted."); + declare_err!(ESTRPIPE, "Streams pipe error."); + declare_err!(EUSERS, "Too many users."); + declare_err!(ENOTSOCK, "Socket operation on non-socket."); + declare_err!(EDESTADDRREQ, "Destination address required."); + declare_err!(EMSGSIZE, "Message too long."); + declare_err!(EPROTOTYPE, "Protocol wrong type for socket."); + declare_err!(ENOPROTOOPT, "Protocol not available."); + declare_err!(EPROTONOSUPPORT, "Protocol not supported."); + declare_err!(ESOCKTNOSUPPORT, "Socket type not supported."); + declare_err!(EOPNOTSUPP, "Operation not supported on transport endpoint."); + declare_err!(EPFNOSUPPORT, "Protocol family not supported."); + declare_err!(EAFNOSUPPORT, "Address family not supported by protocol."); + declare_err!(EADDRINUSE, "Address already in use."); + declare_err!(EADDRNOTAVAIL, "Cannot assign requested address."); + declare_err!(ENETDOWN, "Network is down."); + declare_err!(ENETUNREACH, "Network is unreachable."); + declare_err!(ENETRESET, "Network dropped connection because of reset."); + declare_err!(ECONNABORTED, "Software caused connection abort."); + declare_err!(ECONNRESET, "Connection reset by peer."); + declare_err!(ENOBUFS, "No buffer space available."); + declare_err!(EISCONN, "Transport endpoint is already connected."); + declare_err!(ENOTCONN, "Transport endpoint is not connected."); + declare_err!(ESHUTDOWN, "Cannot send after transport endpoint shutdown."); + declare_err!(ETOOMANYREFS, "Too many references: cannot splice."); declare_err!(ETIMEDOUT, "Connection timed out."); + declare_err!(ECONNREFUSED, "Connection refused."); + declare_err!(EHOSTDOWN, "Host is down."); + declare_err!(EHOSTUNREACH, "No route to host."); + declare_err!(EALREADY, "Operation already in progress."); + declare_err!(EINPROGRESS, "Operation now in progress."); + declare_err!(ESTALE, "Stale file handle."); + declare_err!(EUCLEAN, "Structure needs cleaning."); + declare_err!(EFSCORRUPTED, "Filesystem is corrupted."); + declare_err!(ENOTNAM, "Not a XENIX named type file."); + declare_err!(ENAVAIL, "No XENIX semaphores available."); + declare_err!(EISNAM, "Is a named type file."); + declare_err!(EREMOTEIO, "Remote I/O error."); + declare_err!(EDQUOT, "Quota exceeded."); + declare_err!(ENOMEDIUM, "No medium found."); + declare_err!(EMEDIUMTYPE, "Wrong medium type."); + declare_err!(ECANCELED, "Operation Canceled."); + declare_err!(ENOKEY, "Required key not available."); + declare_err!(EKEYEXPIRED, "Key has expired."); + declare_err!(EKEYREVOKED, "Key has been revoked."); + declare_err!(EKEYREJECTED, "Key was rejected by service."); + declare_err!(EOWNERDEAD, "Owner died."); + declare_err!(ENOTRECOVERABLE, "State not recoverable."); + declare_err!(ERFKILL, "Operation not possible due to RF-kill."); + declare_err!(EHWPOISON, "Memory page has hardware error."); + declare_err!(EFTYPE, "Wrong file type for the intended operation."); + + // From `include/linux/errno.h`. declare_err!(ERESTARTSYS, "Restart the system call."); declare_err!(ERESTARTNOINTR, "System call was interrupted by a signal and will be restarted."); declare_err!(ERESTARTNOHAND, "Restart if no handler."); @@ -216,36 +317,42 @@ impl fmt::Debug for Error { } impl From<AllocError> for Error { + #[inline] fn from(_: AllocError) -> Error { code::ENOMEM } } impl From<TryFromIntError> for Error { + #[inline] fn from(_: TryFromIntError) -> Error { code::EINVAL } } impl From<Utf8Error> for Error { + #[inline] fn from(_: Utf8Error) -> Error { code::EINVAL } } impl From<LayoutError> for Error { + #[inline] fn from(_: LayoutError) -> Error { code::ENOMEM } } impl From<fmt::Error> for Error { + #[inline] fn from(_: fmt::Error) -> Error { code::EINVAL } } impl From<core::convert::Infallible> for Error { + #[inline] fn from(e: core::convert::Infallible) -> Error { match e {} } @@ -446,6 +553,9 @@ pub fn to_result(err: crate::ffi::c_int) -> Result { /// for errors. This function performs the check and converts the "error pointer" /// to a normal pointer in an idiomatic fashion. /// +/// Note that a `NULL` pointer is not considered an error pointer, and is returned +/// as-is, wrapped in [`Ok`]. +/// /// # Examples /// /// ```ignore @@ -460,6 +570,34 @@ pub fn to_result(err: crate::ffi::c_int) -> Result { /// from_err_ptr(unsafe { bindings::devm_platform_ioremap_resource(pdev.to_ptr(), index) }) /// } /// ``` +/// +/// ``` +/// # use kernel::error::from_err_ptr; +/// # mod bindings { +/// # #![expect(clippy::missing_safety_doc)] +/// # use kernel::prelude::*; +/// # pub(super) unsafe fn einval_err_ptr() -> *mut kernel::ffi::c_void { +/// # EINVAL.to_ptr() +/// # } +/// # pub(super) unsafe fn null_ptr() -> *mut kernel::ffi::c_void { +/// # core::ptr::null_mut() +/// # } +/// # pub(super) unsafe fn non_null_ptr() -> *mut kernel::ffi::c_void { +/// # 0x1234 as *mut kernel::ffi::c_void +/// # } +/// # } +/// // SAFETY: ... +/// let einval_err = from_err_ptr(unsafe { bindings::einval_err_ptr() }); +/// assert_eq!(einval_err, Err(EINVAL)); +/// +/// // SAFETY: ... +/// let null_ok = from_err_ptr(unsafe { bindings::null_ptr() }); +/// assert_eq!(null_ok, Ok(core::ptr::null_mut())); +/// +/// // SAFETY: ... +/// let non_null = from_err_ptr(unsafe { bindings::non_null_ptr() }).unwrap(); +/// assert_ne!(non_null, core::ptr::null_mut()); +/// ``` pub fn from_err_ptr<T>(ptr: *mut T) -> Result<*mut T> { // CAST: Casting a pointer to `*const crate::ffi::c_void` is always valid. let const_ptr: *const crate::ffi::c_void = ptr.cast(); diff --git a/rust/kernel/faux.rs b/rust/kernel/faux.rs index 43b4974f48cd..cd4198fbb232 100644 --- a/rust/kernel/faux.rs +++ b/rust/kernel/faux.rs @@ -9,15 +9,63 @@ use crate::{ bindings, device, - prelude::*, // + prelude::*, + types::Opaque, // }; -use core::ptr::{ - addr_of_mut, - null, - null_mut, - NonNull, // +use core::{ + marker::PhantomData, + ptr::{ + null, + null_mut, + NonNull, // + }, }; +/// A faux device. +/// +/// A faux device is a virtual device backed by the faux bus, primarily used for scenarios where a +/// real hardware device is not available or for testing. +/// +/// # Invariants +/// +/// The underlying `struct faux_device` is valid. +#[repr(transparent)] +pub struct Device<Ctx: device::DeviceContext = device::Normal>( + Opaque<bindings::faux_device>, + PhantomData<Ctx>, +); + +impl<Ctx: device::DeviceContext> Device<Ctx> { + #[inline] + fn as_raw(&self) -> *mut bindings::faux_device { + self.0.get() + } + + /// # Safety + /// + /// `ptr` must be a valid pointer to a `struct faux_device`. + #[inline] + unsafe fn from_raw<'a>(ptr: *mut bindings::faux_device) -> &'a Self { + // SAFETY: `Device` is a transparent wrapper of `Opaque<bindings::faux_device>`. + unsafe { &*ptr.cast() } + } +} + +impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> { + #[inline] + fn as_ref(&self) -> &device::Device<Ctx> { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct faux_device`. `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(&raw mut (*self.as_raw()).dev) } + } +} + +// SAFETY: `faux::Device` is a transparent wrapper of `struct faux_device`. +// The offset is guaranteed to point to a valid device field inside `faux::Device`. +unsafe impl<Ctx: device::DeviceContext> device::AsBusDevice<Ctx> for Device<Ctx> { + const OFFSET: usize = core::mem::offset_of!(bindings::faux_device, dev); +} + /// The registration of a faux device. /// /// This type represents the registration of a [`struct faux_device`]. When an instance of this type @@ -25,7 +73,8 @@ use core::ptr::{ /// /// # Invariants /// -/// `self.0` always holds a valid pointer to an initialized and registered [`struct faux_device`]. +/// - `self.0` always holds a valid pointer to an initialized and registered [`struct faux_device`]. +/// - This object is proof that the object described by this `Registration` is bound to a device. /// /// [`struct faux_device`]: srctree/include/linux/device/faux.h pub struct Registration(NonNull<bindings::faux_device>); @@ -59,11 +108,19 @@ impl Registration { } } -impl AsRef<device::Device> for Registration { - fn as_ref(&self) -> &device::Device { - // SAFETY: The underlying `device` in `faux_device` is guaranteed by the C API to be - // a valid initialized `device`. - unsafe { device::Device::from_raw(addr_of_mut!((*self.as_raw()).dev)) } +impl AsRef<Device<device::Bound>> for Registration { + #[inline] + fn as_ref(&self) -> &Device<device::Bound> { + // SAFETY: + // - The underlying `struct faux_device` is guaranteed by the C API to be a valid + // initialized `device`. + // - `faux_match()` always returns 1, and probe runs synchronously + // (PROBE_FORCE_SYNCHRONOUS). + // - `suppress_bind_attrs = true` on faux_driver prevents userspace-triggered unbind via + // sysfs. + // - `mem::forget(Registration)` is not a problem; if the `Registration` is leaked, the faux + // device stays bound forever. + unsafe { Device::from_raw(self.as_raw()) } } } diff --git a/rust/kernel/firmware.rs b/rust/kernel/firmware.rs index 71168d8004e2..a18f8b84f3e3 100644 --- a/rust/kernel/firmware.rs +++ b/rust/kernel/firmware.rs @@ -7,9 +7,9 @@ use crate::{ bindings, device::Device, - error::Error, - error::Result, + error::to_result, ffi, + prelude::*, str::{CStr, CStrExt as _}, }; use core::ptr::NonNull; @@ -51,12 +51,8 @@ impl FwFunc { /// # Examples /// /// ```no_run -/// # use kernel::{device::Device, firmware::Firmware}; -/// -/// # fn no_run() -> Result<(), Error> { -/// # // SAFETY: *NOT* safe, just for the example to get an `ARef<Device>` instance -/// # let dev = unsafe { Device::get_device(core::ptr::null_mut()) }; -/// +/// # use kernel::{device::Device, firmware::Firmware, sync::aref::ARef}; +/// # fn no_run(dev: ARef<Device>) -> Result<(), Error> { /// let fw = Firmware::request(c"path/to/firmware.bin", &dev)?; /// let blob = fw.data(); /// @@ -120,6 +116,48 @@ impl Drop for Firmware { } } +/// Load firmware directly into the caller-provided `buf`. +/// +/// On success the firmware image has been copied into `buf`; the caller accesses the data +/// through `buf` itself. +/// +/// This is intentionally a stand-alone function rather than a `Firmware` constructor. For +/// the `into_buf` path, the firmware data lives in the caller's `buf`, not in a +/// kernel-owned buffer, so returning a `Firmware` would expose `Firmware::data()` as a +/// second handle aliasing `buf` (and `release_firmware()` does not free `buf` anyway). +pub fn request_into_buf(name: &CStr, dev: &Device, buf: &mut [u8]) -> Result { + // `as_mut_ptr()` on an empty slice returns a non-NULL pointer to + // memory which the loader does not own. Passing that pointer with `size == 0` + // makes the loader believe that it is buffer it allocated itself, so when + // `release_firmware()` is called, it will vfree the pointer and trigger a + // bug. Reject empty slices to avoid this situation. + if buf.is_empty() { + return Err(EINVAL); + } + + let mut fw: *const bindings::firmware = core::ptr::null(); + + // SAFETY: `&raw mut fw` is a valid pointer to a NULL initialized `bindings::firmware` pointer. + // `name` and `dev` are valid as by their type invariants. `buf` is a valid writable + // buffer of `buf.len()` bytes. + to_result(unsafe { + bindings::request_firmware_into_buf( + &raw mut fw, + name.as_char_ptr(), + dev.as_raw(), + buf.as_mut_ptr().cast(), + buf.len(), + ) + })?; + + // The firmware bytes are now in `buf`, which the caller owns, so we don't need + // the kernel to hang on to it any more. + // SAFETY: `fw` is a valid pointer returned by `request_firmware_into_buf`. + unsafe { bindings::release_firmware(fw) }; + + Ok(()) +} + // SAFETY: `Firmware` only holds a pointer to a C `struct firmware`, which is safe to be used from // any thread. unsafe impl Send for Firmware {} diff --git a/rust/kernel/fmt.rs b/rust/kernel/fmt.rs index 1e8725eb44ed..29582b053ab1 100644 --- a/rust/kernel/fmt.rs +++ b/rust/kernel/fmt.rs @@ -4,7 +4,16 @@ //! //! This module is intended to be used in place of `core::fmt` in kernel code. -pub use core::fmt::{Arguments, Debug, Error, Formatter, Result, Write}; +use kernel::prelude::*; + +pub use core::fmt::{ + Arguments, + Debug, + Error, + Formatter, + Result, + Write, // +}; /// Internal adapter used to route and allow implementations of formatting traits for foreign types. /// @@ -27,8 +36,120 @@ macro_rules! impl_fmt_adapter_forward { }; } -use core::fmt::{Binary, LowerExp, LowerHex, Octal, Pointer, UpperExp, UpperHex}; -impl_fmt_adapter_forward!(Debug, LowerHex, UpperHex, Octal, Binary, Pointer, LowerExp, UpperExp); +use core::fmt::{ + Binary, + LowerExp, + LowerHex, + Octal, + UpperExp, + UpperHex, // +}; +use core::ptr::NonNull; +impl_fmt_adapter_forward!(Debug, LowerHex, UpperHex, Octal, Binary, LowerExp, UpperExp); + +/// A copy of [`core::fmt::Pointer`] that allows implementing pointer formatting for foreign types. +/// +/// Together with the [`Adapter`] type and [`fmt!`] macro, it enables raw pointer formatting to be +/// intercepted and routed to [`HashedPtr`] (kernel's `%p` hashed format), preventing kernel address +/// leaks. +/// +/// [`fmt!`]: crate::prelude::fmt! +pub trait Pointer { + /// Same as [`core::fmt::Pointer::fmt`]. + fn fmt(&self, f: &mut Formatter<'_>) -> Result; +} + +/// A wrapper for pointers that formats them using kernel's `%p` format specifier. +/// +/// By default, `%p` prints a hashed representation of the pointer address to prevent kernel address +/// leaks. When the `no_hash_pointers` kernel command-line parameter is enabled, the real address is +/// printed instead (for debugging purposes). +pub struct HashedPtr<T: ?Sized>(pub *const T); + +impl<T: ?Sized> Pointer for HashedPtr<T> { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + use crate::str::CStrExt as _; + + let mut buf = [0u8; 32]; + + // Use `%#0*p` for the `0x` prefix and zero-padding; `+2` compensates for + // the prefix counting toward the field width. + let default_width = (2 * size_of::<usize>() + 2) as c_int; + let width = match (f.sign_aware_zero_pad(), f.width()) { + (true, Some(w)) if w > 0 => w.min(buf.len() - 1) as c_int, + _ => default_width, + }; + + // SAFETY: `buf` is a valid, writable 32-byte buffer, sufficient for + // all architectures (max 19 bytes for 64-bit under the default width). + // The format string is null-terminated; `width` (c_int) and pointer + // match the `%*` and `%p` specifiers. + let len = unsafe { + crate::bindings::scnprintf( + buf.as_mut_ptr().cast(), + buf.len(), + c"%#0*p".as_char_ptr(), + width, + self.0.cast::<c_void>(), + ) + }; + + // SAFETY: `%#0*p` produces only ASCII, which is valid UTF-8. + let s = unsafe { core::str::from_utf8_unchecked(&buf[..len as usize]) }; + + if f.sign_aware_zero_pad() { + // `scnprintf` already applied the width and zero-padding via `%#0*p`. + f.write_str(s) + } else { + f.pad(s) + } + } +} + +// Raw pointers are formatted via `HashedPtr` (kernel `%p`: hashed by default, plain with +// `no_hash_pointers`). +impl<T: ?Sized> Pointer for *const T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(*self), f) + } +} + +impl<T: ?Sized> Pointer for *mut T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(*self), f) + } +} + +impl<T: ?Sized> Pointer for &T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(*self), f) + } +} + +impl<T: ?Sized> Pointer for &mut T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(core::ptr::from_ref(*self)), f) + } +} + +impl<T: ?Sized> Pointer for NonNull<T> { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(self.as_ptr()), f) + } +} + +// `Adapter<&T>` bridges our `Pointer` trait to `core::fmt::Pointer` +impl<T: Pointer> core::fmt::Pointer for Adapter<&T> { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(self.0, f) + } +} /// A copy of [`core::fmt::Display`] that allows us to implement it for foreign types. /// @@ -90,3 +211,88 @@ impl_display_forward!( {<T: ?Sized>} crate::sync::Arc<T> {where crate::sync::Arc<T>: core::fmt::Display}, {<T: ?Sized>} crate::sync::UniqueArc<T> {where crate::sync::UniqueArc<T>: core::fmt::Display}, ); + +#[macros::kunit_tests(rust_kernel_fmt)] +mod tests { + use crate::{ + bindings, + prelude::fmt, + str::CString, // + }; + + #[cfg(CONFIG_64BIT)] + mod expected { + pub(super) const PTR_VALUE: usize = 0xffffffffdeadbeef; + pub(super) const PTR_VAL_NO_CRNG: &str = "(____ptrval____)"; + pub(super) const HASHED_PREFIX: &str = "0x00000000"; + pub(super) const RAW_POINTER: &str = "0xffffffffdeadbeef"; + pub(super) const PADDED_RIGHT: &str = " 0xffffffffdeadbeef"; + pub(super) const ZERO_PADDED: &str = "0x000000ffffffffdeadbeef"; + pub(super) const HASHED_PADDED_RIGHT_PREFIX: &str = " "; + pub(super) const HASHED_ZERO_PADDED_PREFIX: &str = "0x00000000000000"; + pub(super) const CLAMPED: &str = "0x0000000000000ffffffffdeadbeef"; + } + + #[cfg(not(CONFIG_64BIT))] + mod expected { + pub(super) const PTR_VALUE: usize = 0xdeadbeef; + pub(super) const PTR_VAL_NO_CRNG: &str = "(ptrval)"; + pub(super) const HASHED_PREFIX: &str = "0x"; + pub(super) const RAW_POINTER: &str = "0xdeadbeef"; + pub(super) const PADDED_RIGHT: &str = " 0xdeadbeef"; + pub(super) const ZERO_PADDED: &str = "0x00000000000000deadbeef"; + pub(super) const HASHED_PADDED_RIGHT_PREFIX: &str = " "; + pub(super) const HASHED_ZERO_PADDED_PREFIX: &str = "0x00000000000000"; + pub(super) const CLAMPED: &str = "0x0000000000000000000000deadbeef"; + } + + #[test] + fn test_ptr_formatting() -> core::result::Result<(), crate::error::Error> { + let ptr: *const u8 = core::ptr::without_provenance(expected::PTR_VALUE); + + // SAFETY: `no_hash_pointers` is a global variable that is never concurrently modified — + // KUnit tests may run at boot (before `mark_readonly()`) or manually afterwards (when the + // variable is read-only). Reading is always safe. + let no_hash = unsafe { bindings::no_hash_pointers }; + + if no_hash { + let cstr = CString::try_from_fmt(fmt!("{:p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::RAW_POINTER); + + let cstr = CString::try_from_fmt(fmt!("{:>24p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::PADDED_RIGHT); + + let cstr = CString::try_from_fmt(fmt!("{:024p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::ZERO_PADDED); + + let cstr = CString::try_from_fmt(fmt!("{:0100p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::CLAMPED); + } else { + let cstr = CString::try_from_fmt(fmt!("{:p}", ptr))?; + let formatted = cstr.to_str()?; + // If the RNG is not yet ready, `%p` falls back to a placeholder. + if formatted == expected::PTR_VAL_NO_CRNG { + return Ok(()); + } + assert!(formatted.starts_with(expected::HASHED_PREFIX)); + assert_ne!(formatted, expected::RAW_POINTER); + + let cstr = CString::try_from_fmt(fmt!("{:>24p}", ptr))?; + assert!(cstr + .to_str()? + .starts_with(expected::HASHED_PADDED_RIGHT_PREFIX)); + + let cstr = CString::try_from_fmt(fmt!("{:024p}", ptr))?; + assert!(cstr + .to_str()? + .starts_with(expected::HASHED_ZERO_PADDED_PREFIX)); + + let cstr = CString::try_from_fmt(fmt!("{:0100p}", ptr))?; + let output = cstr.to_str()?; + assert!(output.starts_with("0x")); + assert!(!output[2..].chars().all(|c| c == '0')); + } + + Ok(()) + } +} diff --git a/rust/kernel/fwctl.rs b/rust/kernel/fwctl.rs new file mode 100644 index 000000000000..f29244fb0d1d --- /dev/null +++ b/rust/kernel/fwctl.rs @@ -0,0 +1,593 @@ +// SPDX-License-Identifier: GPL-2.0-only + +//! Abstractions for the fwctl subsystem. +//! +//! C header: `include/linux/fwctl.h` + +use crate::{ + bindings, + container_of, + device, + prelude::*, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, // +}; +use core::{ + alloc::Layout, + cell::UnsafeCell, + marker::PhantomData, + ptr::NonNull, + slice, // +}; + +/// Returns a kmalloc-compatible allocation size for `T`. +const fn kmalloc_aligned_size<T>() -> usize { + Layout::new::<T>().pad_to_align().size() +} + +/// Represents a fwctl device type. +/// +/// Corresponds to the C `enum fwctl_device_type`. All non-error UAPI values are represented so +/// Rust drivers can select a device type without passing an untyped integer, while +/// `FWCTL_DEVICE_TYPE_ERROR` remains unrepresentable. +#[repr(u32)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum DeviceType { + /// Mellanox ConnectX (mlx5) device. + Mlx5 = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_MLX5, + /// CXL (Compute Express Link) device. + Cxl = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_CXL, + /// AMD/Pensando PDS device. + Pds = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_PDS, + /// Broadcom NetXtreme (bnxt) device. + Bnxt = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_BNXT, +} + +/// Scope of access for an RPC request. +/// +/// Corresponds to the C `enum fwctl_rpc_scope`. +#[repr(u32)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum RpcScope { + /// Read/write access to device configuration. + Configuration = bindings::fwctl_rpc_scope_FWCTL_RPC_CONFIGURATION, + /// Read-only access to debug information. + DebugReadOnly = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_READ_ONLY, + /// Write access to lockdown-compatible debug information. + DebugWrite = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_WRITE, + /// Full read/write access to all debug information (requires `CAP_SYS_RAWIO`). + DebugWriteFull = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_WRITE_FULL, +} + +impl TryFrom<u32> for RpcScope { + type Error = Error; + + #[inline] + fn try_from(value: u32) -> Result<Self, Error> { + match value { + v if v == Self::Configuration as u32 => Ok(Self::Configuration), + v if v == Self::DebugReadOnly as u32 => Ok(Self::DebugReadOnly), + v if v == Self::DebugWrite as u32 => Ok(Self::DebugWrite), + v if v == Self::DebugWriteFull as u32 => Ok(Self::DebugWriteFull), + _ => Err(EINVAL), + } + } +} + +/// Response from a [`Operations::fw_rpc`] call. +pub enum FwRpcResponse { + /// Reuse the input buffer as the output, with the given output length. + /// + /// The callback returns `EINVAL` if the output length exceeds the input buffer length. + InPlace(usize), + /// Return a newly allocated buffer as the output. + NewBuffer(KVVec<u8>), +} + +/// Trait implemented by each Rust driver that integrates with the fwctl subsystem. +/// +/// The implementing type **is** the per-FD user context: one instance is +/// created for each `open()` call and dropped when the FD is closed. +/// +/// Each implementation corresponds to a specific device type and provides the +/// vtable used by the core `fwctl` layer to manage per-FD user contexts and +/// handle RPC requests. +pub trait Operations: Sized + Send + Sync + 'static { + /// Data owned by the [`Registration`] and accessible during callbacks. + /// + /// The lifetime `'a` is tied to the [`Registration`] scope (which lives within the parent bus + /// device binding scope). Drivers use it to store references to resources bound to this scope, + /// such as PCI BARs or typed bus device references. + type RegistrationData<'a>: Send + Sync + 'a + where + Self: 'a; + + /// fwctl device type identifier. + const DEVICE_TYPE: DeviceType; + + /// Called when a new user context is opened. + /// + /// Returns a [`PinInit`] initializer for `Self`. The instance is dropped + /// automatically when the FD is closed (after [`close`](Self::close)). + fn open<'a>( + device: &Device<Self>, + reg_data: &Self::RegistrationData<'a>, + ) -> impl PinInit<Self, Error>; + + /// Called when the user context is closed. + /// + /// The driver may perform additional cleanup here that requires access + /// to the owning [`Device`]. `Self` is dropped automatically after this + /// returns. + fn close<'a>( + _this: Pin<&mut Self>, + _device: &Device<Self>, + _reg_data: &Self::RegistrationData<'a>, + ) { + } + + /// Return device information to userspace. + /// + /// The default implementation returns no device-specific data. + fn info<'a>( + _this: Pin<&Self>, + _device: &Device<Self>, + _reg_data: &Self::RegistrationData<'a>, + ) -> Result<KVec<u8>, Error> { + Ok(KVec::new()) + } + + /// Handle a userspace RPC request. + /// + /// `max_output_len` is the size of the userspace output buffer. A driver may return a larger + /// response to report the required size; the fwctl core copies only the bytes that fit and + /// reports the full response length to userspace. + fn fw_rpc<'a>( + this: Pin<&Self>, + device: &Device<Self>, + reg_data: &Self::RegistrationData<'a>, + scope: RpcScope, + rpc_buf: &mut [u8], + max_output_len: usize, + ) -> Result<FwRpcResponse, Error>; +} + +/// A fwctl device. +/// +/// `#[repr(C)]` with the `fwctl_device` at offset 0, matching the C `fwctl_alloc_device()` layout +/// convention. Contains a pointer to the [`Registration`]'s data, set at registration time and +/// cleared on unregistration. +/// +/// # Invariants +/// +/// - `dev` is embedded at offset 0 and is initialised by fwctl. +/// - The fwctl refcount owns the allocation lifetime. +/// - `registration_data` is either [`NonNull::dangling()`] (before registration / after +/// unregistration) or points to valid data owned by the [`Registration`]. +#[repr(C)] +pub struct Device<T: Operations> { + dev: Opaque<bindings::fwctl_device>, + registration_data: UnsafeCell<NonNull<T::RegistrationData<'static>>>, +} + +impl<T: Operations> Device<T> { + /// Allocate a new fwctl device. + /// + /// Returns an [`ARef`] that can be passed to [`Registration::new()`] + /// to make the device visible to userspace. + pub fn new(parent: &device::Device<device::Bound>) -> Result<ARef<Self>> { + const_assert!( + core::mem::offset_of!(Self, dev) == 0, + "struct fwctl_device must be at offset 0" + ); + + let size = kmalloc_aligned_size::<Self>(); + let ops = core::ptr::from_ref::<bindings::fwctl_ops>(&VTable::<T>::VTABLE).cast_mut(); + + // SAFETY: `ops` is static, `parent` is bound, and `size` is padded so the allocation made + // by `_fwctl_alloc_device` satisfies the size and alignment required by `Device<T>`. + let raw = unsafe { bindings::_fwctl_alloc_device(parent.as_raw(), ops, size) }; + let this = NonNull::new(raw.cast::<Self>()).ok_or(ENOMEM)?; + + // INVARIANT: Set `registration_data` to dangling (no registration yet). + // SAFETY: `this` points to the allocation just returned by fwctl. + unsafe { + (&raw mut (*this.as_ptr()).registration_data) + .write(UnsafeCell::new(NonNull::dangling())); + }; + + // SAFETY: `this` owns the initial reference. + Ok(unsafe { ARef::from_raw(this) }) + } + + /// Returns the underlying `fwctl_device` pointer. + #[inline] + fn as_raw(&self) -> *mut bindings::fwctl_device { + self.dev.get() + } + + /// Borrows a Rust fwctl device from its raw C pointer. + /// + /// # Safety + /// + /// `ptr` must point to a valid `fwctl_device` embedded in a [`Device<T>`]. + #[inline] + unsafe fn from_raw<'a>(ptr: *mut bindings::fwctl_device) -> &'a Self { + // SAFETY: The caller upholds the offset-0 `Device<T>` invariant. + unsafe { &*ptr.cast() } + } + + /// Invokes `f` with the registration data. + /// + /// The higher-ranked callback prevents the erased registration lifetime from escaping and + /// permits registration data that is invariant over its lifetime parameter. + /// + /// # Safety + /// + /// The caller must ensure that the device is registered and that this is called from a fwctl + /// callback protected by `registration_lock`. + #[inline] + unsafe fn with_registration_data<R>( + &self, + f: impl for<'a> FnOnce(&Device<T>, &'a T::RegistrationData<'a>) -> R, + ) -> R { + // SAFETY: Caller guarantees the device is registered, so the pointer is valid. + // Lifetimes do not affect layout. The higher-ranked callback prevents the shortened + // lifetime from escaping or being selected by the caller. + let reg_data = unsafe { + (*self.registration_data.get()) + .cast::<T::RegistrationData<'_>>() + .as_ref() + }; + + f(self, reg_data) + } +} + +impl<T: Operations> AsRef<device::Device> for Device<T> { + #[inline] + fn as_ref(&self) -> &device::Device { + // SAFETY: `self` contains a live fwctl_device. + let dev = unsafe { &raw mut (*self.as_raw()).dev }; + // SAFETY: The embedded device is initialised by fwctl. + unsafe { device::Device::from_raw(dev) } + } +} + +// SAFETY: `fwctl_get` increments the refcount of a valid fwctl_device. +// `fwctl_put` decrements it and frees the device when it reaches zero. +unsafe impl<T: Operations> AlwaysRefCounted for Device<T> { + #[inline] + fn inc_ref(&self) { + // SAFETY: `self` holds a live reference. + unsafe { bindings::fwctl_get(self.as_raw()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull<Self>) { + // SAFETY: The caller owns a live reference. + unsafe { bindings::fwctl_put(obj.cast().as_ptr()) }; + } +} + +// SAFETY: `Device<T>` is refcounted by the fwctl core and may be released from any thread. +unsafe impl<T: Operations> Send for Device<T> {} + +// SAFETY: Shared access to the embedded `fwctl_device` is protected by the fwctl core. The +// `registration_data` field is only mutated before registration and after unregistration (both +// single-threaded with respect to callbacks). +unsafe impl<T: Operations> Sync for Device<T> {} + +/// A registered fwctl device. +/// +/// Owns the [`RegistrationData`](Operations::RegistrationData) made available to driver callbacks. +/// The parent device lifetime ensures that [`fwctl_unregister`] runs before the parent driver +/// unbinds. +/// +/// On drop the device is unregistered (all user contexts are closed and `ops` is set to `NULL`) +/// and the registration data is dropped. +/// +/// [`fwctl_unregister`]: srctree/drivers/fwctl/main.c +pub struct Registration<'a, T: Operations> { + dev: ARef<Device<T>>, + _reg_data: Pin<KBox<T::RegistrationData<'a>>>, +} + +impl<'a, T: Operations> Registration<'a, T> { + /// Register a previously allocated fwctl device with the given registration data. + /// + /// The `reg_data` is owned by the registration and accessible during callbacks. + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running, since `fwctl_unregister` must be called before the + /// parent device is unbound. + /// + /// `dev` must be an unregistered [`Device`] that is not associated with any live + /// [`Registration`], and no other thread may attempt to register the same device concurrently. + pub unsafe fn new( + parent: &'a device::Device<device::Bound>, + dev: &Device<T>, + reg_data: impl PinInit<T::RegistrationData<'a>, Error>, + ) -> Result<Self> { + let actual_parent = dev.as_ref().parent().ok_or(EINVAL)?; + let parent_device: &device::Device = parent; + if !core::ptr::eq(actual_parent, parent_device) { + return Err(EINVAL); + } + + let reg_data: Pin<KBox<T::RegistrationData<'a>>> = KBox::pin_init(reg_data, GFP_KERNEL)?; + + // Store the registration data pointer in the device before registration, so that it is + // visible once callbacks can be invoked. The `'static` type is only an erased storage + // handle; callbacks access the pointer through a higher-ranked closure. + let ptr: NonNull<T::RegistrationData<'static>> = + NonNull::from(Pin::get_ref(reg_data.as_ref())).cast(); + + // SAFETY: No concurrent access; the device is not yet registered. + unsafe { *dev.registration_data.get() = ptr }; + + // SAFETY: `dev` is a valid fwctl_device backed by an ARef. + let ret = unsafe { bindings::fwctl_register(dev.as_raw()) }; + if ret != 0 { + // SAFETY: No concurrent readers; registration failed. + unsafe { *dev.registration_data.get() = NonNull::dangling() }; + return Err(Error::from_errno(ret)); + } + + Ok(Self { + dev: dev.into(), + _reg_data: reg_data, + }) + } +} + +impl<T: Operations> Drop for Registration<'_, T> { + fn drop(&mut self) { + // SAFETY: The Registration lifetime guarantees that the parent device is still bound. + // `fwctl_unregister` takes the write lock, closes all user contexts, and sets ops=NULL. + // After it returns, no callbacks can be running or will run. + unsafe { bindings::fwctl_unregister(self.dev.as_raw()) }; + + // SAFETY: `fwctl_unregister` guarantees no concurrent readers. + unsafe { *self.dev.registration_data.get() = NonNull::dangling() }; + + // `self._reg_data` is dropped here, after callbacks have stopped. + } +} + +/// Internal per-FD user context wrapping `struct fwctl_uctx` and `T`. +/// +/// Not exposed to drivers; they work with `&T` / `Pin<&mut T>` directly. +#[repr(C)] +#[pin_data] +struct UserCtx<T: Operations> { + #[pin] + fwctl_uctx: Opaque<bindings::fwctl_uctx>, + #[pin] + uctx: T, +} + +impl<T: Operations> UserCtx<T> { + /// Borrows a pinned Rust user context from its raw C pointer. + /// + /// # Safety + /// + /// `ptr` must point to a `fwctl_uctx` embedded in a live, pinned `UserCtx<T>` that remains + /// valid and does not move for the duration of `'a`. + #[inline] + unsafe fn from_raw<'a>(ptr: *mut bindings::fwctl_uctx) -> Pin<&'a Self> { + // SAFETY: The caller upholds the `UserCtx<T>` embedding, lifetime, and pinning invariants. + unsafe { Pin::new_unchecked(&*container_of!(Opaque::cast_from(ptr), Self, fwctl_uctx)) } + } + + /// Mutably borrows a pinned Rust user context from its raw C pointer. + /// + /// # Safety + /// + /// - `ptr` must point to a `fwctl_uctx` embedded in a live, pinned `UserCtx<T>` that remains + /// valid and does not move for the duration of `'a`. + /// - The caller must ensure exclusive access to the `UserCtx<T>` for the duration of `'a`. + #[inline] + unsafe fn from_raw_mut<'a>(ptr: *mut bindings::fwctl_uctx) -> Pin<&'a mut Self> { + // SAFETY: The caller upholds the embedding, lifetime, pinning, and exclusivity invariants. + unsafe { + Pin::new_unchecked( + &mut *container_of!(Opaque::cast_from(ptr), Self, fwctl_uctx).cast_mut(), + ) + } + } + + /// Returns a reference to the fwctl [`Device`] that owns this context. + #[inline] + fn device(self: Pin<&Self>) -> &Device<T> { + // SAFETY: fwctl initialises this pointer before any driver callback. + let raw_fwctl = unsafe { (*self.fwctl_uctx.get()).fwctl }; + // SAFETY: Rust fwctl devices use the offset-0 `Device<T>` layout. + unsafe { Device::from_raw(raw_fwctl) } + } + + /// Returns a pinned reference to the driver context. + #[inline] + fn uctx(self: Pin<&Self>) -> Pin<&T> { + ::pin_init::assert_pinned!(UserCtx<T>, uctx, T, inline); + + // SAFETY: `uctx` is structurally pinned. + unsafe { self.map_unchecked(|ctx| &ctx.uctx) } + } +} + +/// Static vtable mapping Rust trait methods to C callbacks. +struct VTable<T: Operations>(PhantomData<T>); + +impl<T: Operations> VTable<T> { + /// The fwctl operations vtable for this driver type. + const VTABLE: bindings::fwctl_ops = bindings::fwctl_ops { + // CAST: `DeviceType` has the same `u32` representation as the C enum field. + device_type: T::DEVICE_TYPE as u32, + uctx_size: kmalloc_aligned_size::<UserCtx<T>>(), + open_uctx: Some(Self::open_uctx_callback), + close_uctx: Some(Self::close_uctx_callback), + info: Some(Self::info_callback), + fw_rpc: Some(Self::fw_rpc_callback), + }; + + /// Initialises a newly opened Rust user context. + /// + /// # Safety + /// + /// `uctx` must be a valid `fwctl_uctx` embedded in a `UserCtx<T>` with + /// sufficient allocated space for the uctx field. + unsafe extern "C" fn open_uctx_callback(uctx: *mut bindings::fwctl_uctx) -> ffi::c_int { + const_assert!( + core::mem::offset_of!(UserCtx<T>, fwctl_uctx) == 0, + "struct fwctl_uctx must be at offset 0" + ); + + // SAFETY: fwctl sets this pointer before calling `open_uctx`. + let raw_fwctl = unsafe { (*uctx).fwctl }; + // SAFETY: Rust fwctl devices use the offset-0 `Device<T>` layout. + let device = unsafe { Device::<T>::from_raw(raw_fwctl) }; + + let uctx_offset = core::mem::offset_of!(UserCtx<T>, uctx); + // SAFETY: `uctx_size` reserves space for the full `UserCtx<T>`. + let uctx_ptr: *mut T = unsafe { uctx.byte_add(uctx_offset).cast() }; + + // SAFETY: `open_uctx` is called under `registration_lock` read, so the device is + // registered. `uctx_ptr` addresses the uninitialised pinned context reserved by + // `uctx_size`. + unsafe { + device.with_registration_data(|device, reg_data| { + match pin_init::raw_try_init(uctx_ptr, T::open(device, reg_data)) { + Ok(()) => 0, + Err(e) => e.to_errno(), + } + }) + } + } + + /// Closes and drops an opened Rust user context. + /// + /// # Safety + /// + /// `uctx` must point to a fully initialised `UserCtx<T>`. + unsafe extern "C" fn close_uctx_callback(uctx: *mut bindings::fwctl_uctx) { + // SAFETY: fwctl keeps the owning device live for this callback. + let device = unsafe { Device::<T>::from_raw((*uctx).fwctl) }; + + // SAFETY: close is called for an opened Rust user context. + let mut ctx = unsafe { UserCtx::<T>::from_raw_mut(uctx) }; + + // SAFETY: `close_uctx` is called under `registration_lock` write (from + // `fwctl_unregister`) or read (from `fwctl_fops_release`), so the device is registered. + unsafe { + device.with_registration_data(|device, reg_data| { + T::close(ctx.as_mut().project().uctx, device, reg_data); + }); + } + + // SAFETY: close is the last callback before fwctl frees the allocation. + unsafe { core::ptr::drop_in_place(ctx.project().uctx.get_unchecked_mut()) }; + } + + /// Returns device-specific information for an opened Rust user context. + /// + /// # Safety + /// + /// - `uctx` must point to a fully initialised `UserCtx<T>`. + /// - `length` must be a valid pointer. + unsafe extern "C" fn info_callback( + uctx: *mut bindings::fwctl_uctx, + length: *mut usize, + ) -> *mut ffi::c_void { + // SAFETY: info is called for an opened Rust user context. + let ctx = unsafe { UserCtx::<T>::from_raw(uctx) }; + let device = ctx.device(); + + // SAFETY: `info` is called under `registration_lock` read, so the device is registered. + let result = unsafe { + device.with_registration_data(|device, reg_data| T::info(ctx.uctx(), device, reg_data)) + }; + + match result { + Ok(kvec) if kvec.is_empty() => { + // SAFETY: `length` is a valid out-parameter. + unsafe { *length = 0 }; + // Return NULL for empty data; kfree(NULL) is safe. + core::ptr::null_mut() + } + Ok(kvec) => { + let (ptr, len, _cap) = kvec.into_raw_parts(); + // SAFETY: `length` is a valid out-parameter. + unsafe { *length = len }; + ptr.cast::<ffi::c_void>() + } + Err(e) => Error::to_ptr(e), + } + } + + /// Dispatches a firmware RPC for an opened Rust user context. + /// + /// # Safety + /// + /// - `uctx` must point to a fully initialised `UserCtx<T>`. + /// - `rpc_in` must be valid, initialised, and exclusively accessible for `in_len` bytes. + /// - `out_len` must be valid for reading and writing an initialised `usize`. + unsafe extern "C" fn fw_rpc_callback( + uctx: *mut bindings::fwctl_uctx, + scope: u32, + rpc_in: *mut ffi::c_void, + in_len: usize, + out_len: *mut usize, + ) -> *mut ffi::c_void { + let scope = match RpcScope::try_from(scope) { + Ok(s) => s, + Err(e) => return Error::to_ptr(e), + }; + + // SAFETY: `out_len` points to an initialised `usize` supplied by fwctl. + let max_output_len = unsafe { *out_len }; + + // SAFETY: RPC is called for an opened Rust user context. + let ctx = unsafe { UserCtx::<T>::from_raw(uctx) }; + let device = ctx.device(); + + // SAFETY: fwctl passes an exclusively owned buffer that is valid and initialised for + // `in_len` bytes. It remains live for the duration of this callback. + let rpc_buf = unsafe { slice::from_raw_parts_mut(rpc_in.cast::<u8>(), in_len) }; + + // SAFETY: `fw_rpc` is called under `registration_lock` read, so the device is registered. + let result = unsafe { + device.with_registration_data(|device, reg_data| { + T::fw_rpc(ctx.uctx(), device, reg_data, scope, rpc_buf, max_output_len) + }) + }; + + let (response, response_len) = match result { + Ok(FwRpcResponse::InPlace(len)) => { + if len > in_len { + return Error::to_ptr(EINVAL); + } + + (rpc_in, len) + } + Ok(FwRpcResponse::NewBuffer(kvec)) if kvec.is_empty() => { + // Return NULL for empty data; kvfree(NULL) is safe. + (core::ptr::null_mut(), 0) + } + Ok(FwRpcResponse::NewBuffer(kvec)) => { + let (ptr, len, _cap) = kvec.into_raw_parts(); + (ptr.cast::<ffi::c_void>(), len) + } + Err(e) => return Error::to_ptr(e), + }; + + // SAFETY: `out_len` is a valid out-parameter. + unsafe { *out_len = response_len }; + response + } +} diff --git a/rust/kernel/gpu.rs b/rust/kernel/gpu.rs new file mode 100644 index 000000000000..1dc5d0c8c09d --- /dev/null +++ b/rust/kernel/gpu.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! GPU subsystem abstractions. + +#[cfg(CONFIG_GPU_BUDDY = "y")] +pub mod buddy; diff --git a/rust/kernel/gpu/buddy.rs b/rust/kernel/gpu/buddy.rs new file mode 100644 index 000000000000..d502ada6ebbd --- /dev/null +++ b/rust/kernel/gpu/buddy.rs @@ -0,0 +1,614 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! GPU buddy allocator bindings. +//! +//! C header: [`include/linux/gpu_buddy.h`](srctree/include/linux/gpu_buddy.h) +//! +//! This module provides Rust abstractions over the Linux kernel's GPU buddy +//! allocator, which implements a binary buddy memory allocator. +//! +//! The buddy allocator manages a contiguous address space and allocates blocks +//! in power-of-two sizes, useful for GPU physical memory management. +//! +//! # Examples +//! +//! Create a buddy allocator and perform a basic range allocation: +//! +//! ``` +//! use kernel::{ +//! gpu::buddy::{ +//! GpuBuddy, +//! GpuBuddyAllocFlags, +//! GpuBuddyAllocMode, +//! GpuBuddyParams, // +//! }, +//! prelude::*, +//! ptr::Alignment, +//! sizes::*, // +//! }; +//! +//! // Create a 1GB buddy allocator with 4KB minimum chunk size. +//! let buddy = GpuBuddy::new(GpuBuddyParams { +//! base_offset: 0, +//! size: SZ_1G as u64, +//! chunk_size: Alignment::new::<SZ_4K>(), +//! })?; +//! +//! assert_eq!(buddy.size(), SZ_1G as u64); +//! assert_eq!(buddy.chunk_size(), Alignment::new::<SZ_4K>()); +//! let initial_free = buddy.avail(); +//! +//! // Allocate 16MB. Block lands at the top of the address range. +//! let allocated = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Simple, +//! SZ_16M as u64, +//! Alignment::new::<SZ_16M>(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_16M as u64); +//! +//! let block = allocated.iter().next().expect("expected one block"); +//! assert_eq!(block.offset(), (SZ_1G - SZ_16M) as u64); +//! assert_eq!(block.order(), 12); // 2^12 pages = 16MB +//! assert_eq!(block.size(), SZ_16M as u64); +//! assert_eq!(allocated.iter().count(), 1); +//! +//! // Dropping the allocation returns the range to the buddy allocator. +//! drop(allocated); +//! assert_eq!(buddy.avail(), initial_free); +//! # Ok::<(), Error>(()) +//! ``` +//! +//! Top-down allocation allocates from the highest addresses: +//! +//! ``` +//! # use kernel::{ +//! # gpu::buddy::{GpuBuddy, GpuBuddyAllocMode, GpuBuddyAllocFlags, GpuBuddyParams}, +//! # prelude::*, +//! # ptr::Alignment, +//! # sizes::*, // +//! # }; +//! # let buddy = GpuBuddy::new(GpuBuddyParams { +//! # base_offset: 0, +//! # size: SZ_1G as u64, +//! # chunk_size: Alignment::new::<SZ_4K>(), +//! # })?; +//! # let initial_free = buddy.avail(); +//! let topdown = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::TopDown, +//! SZ_16M as u64, +//! Alignment::new::<SZ_16M>(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_16M as u64); +//! +//! let block = topdown.iter().next().expect("expected one block"); +//! assert_eq!(block.offset(), (SZ_1G - SZ_16M) as u64); +//! assert_eq!(block.order(), 12); +//! assert_eq!(block.size(), SZ_16M as u64); +//! +//! // Dropping the allocation returns the range to the buddy allocator. +//! drop(topdown); +//! assert_eq!(buddy.avail(), initial_free); +//! # Ok::<(), Error>(()) +//! ``` +//! +//! Non-contiguous allocation can fill fragmented memory by returning multiple +//! blocks: +//! +//! ``` +//! # use kernel::{ +//! # gpu::buddy::{ +//! # GpuBuddy, GpuBuddyAllocFlags, GpuBuddyAllocMode, GpuBuddyParams, +//! # }, +//! # prelude::*, +//! # ptr::Alignment, +//! # sizes::*, // +//! # }; +//! # let buddy = GpuBuddy::new(GpuBuddyParams { +//! # base_offset: 0, +//! # size: SZ_1G as u64, +//! # chunk_size: Alignment::new::<SZ_4K>(), +//! # })?; +//! # let initial_free = buddy.avail(); +//! // Create fragmentation by allocating 4MB blocks at [0,4M) and [8M,12M). +//! let frag1 = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Range(0..SZ_4M as u64), +//! SZ_4M as u64, +//! Alignment::new::<SZ_4M>(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_4M as u64); +//! +//! let frag2 = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Range(SZ_8M as u64..(SZ_8M + SZ_4M) as u64), +//! SZ_4M as u64, +//! Alignment::new::<SZ_4M>(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_8M as u64); +//! +//! // Allocate 8MB, this returns 2 blocks from the holes. +//! let fragmented = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Range(0..SZ_16M as u64), +//! SZ_8M as u64, +//! Alignment::new::<SZ_4M>(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_16M as u64); +//! +//! let (mut count, mut total) = (0u32, 0u64); +//! for block in fragmented.iter() { +//! assert_eq!(block.size(), SZ_4M as u64); +//! total += block.size(); +//! count += 1; +//! } +//! assert_eq!(total, SZ_8M as u64); +//! assert_eq!(count, 2); +//! # Ok::<(), Error>(()) +//! ``` +//! +//! Contiguous allocation fails when only fragmented space is available: +//! +//! ``` +//! # use kernel::{ +//! # gpu::buddy::{ +//! # GpuBuddy, GpuBuddyAllocFlag, GpuBuddyAllocFlags, GpuBuddyAllocMode, GpuBuddyParams, +//! # }, +//! # prelude::*, +//! # ptr::Alignment, +//! # sizes::*, // +//! # }; +//! // Create a small 16MB buddy allocator with fragmented memory. +//! let small = GpuBuddy::new(GpuBuddyParams { +//! base_offset: 0, +//! size: SZ_16M as u64, +//! chunk_size: Alignment::new::<SZ_4K>(), +//! })?; +//! +//! let _hole1 = KBox::pin_init( +//! small.alloc_blocks( +//! GpuBuddyAllocMode::Range(0..SZ_4M as u64), +//! SZ_4M as u64, +//! Alignment::new::<SZ_4M>(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! +//! let _hole2 = KBox::pin_init( +//! small.alloc_blocks( +//! GpuBuddyAllocMode::Range(SZ_8M as u64..(SZ_8M + SZ_4M) as u64), +//! SZ_4M as u64, +//! Alignment::new::<SZ_4M>(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! +//! // 8MB contiguous should fail, only two non-contiguous 4MB holes exist. +//! let result = KBox::pin_init( +//! small.alloc_blocks( +//! GpuBuddyAllocMode::Simple, +//! SZ_8M as u64, +//! Alignment::new::<SZ_4M>(), +//! GpuBuddyAllocFlag::Contiguous, +//! ), +//! GFP_KERNEL, +//! ); +//! assert!(result.is_err()); +//! # Ok::<(), Error>(()) +//! ``` + +use core::ops::Range; + +use crate::{ + bindings, + clist_create, + error::to_result, + interop::list::CListHead, + new_mutex, + prelude::*, + ptr::Alignment, + sync::{ + lock::mutex::MutexGuard, + Arc, + Mutex, // + }, + types::Opaque, // +}; + +/// Allocation mode for the GPU buddy allocator. +/// +/// The mode determines the primary allocation strategy. Modes are mutually +/// exclusive: an allocation is either simple, range-constrained, or top-down. +/// +/// Orthogonal modifier flags (e.g., contiguous, clear) are specified separately +/// via [`GpuBuddyAllocFlags`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum GpuBuddyAllocMode { + /// Simple allocation without constraints. + Simple, + /// Range-based allocation within the given address range. + Range(Range<u64>), + /// Allocate from top of address space downward. + TopDown, +} + +impl GpuBuddyAllocMode { + /// Returns the C flags corresponding to the allocation mode. + fn as_flags(&self) -> usize { + match self { + Self::Simple => 0, + Self::Range(_) => bindings::GPU_BUDDY_RANGE_ALLOCATION, + Self::TopDown => bindings::GPU_BUDDY_TOPDOWN_ALLOCATION, + } + } + + /// Extracts the range start/end, defaulting to `(0, 0)` for non-range modes. + fn range(&self) -> (u64, u64) { + match self { + Self::Range(range) => (range.start, range.end), + _ => (0, 0), + } + } +} + +crate::impl_flags!( + /// Modifier flags for GPU buddy allocation. + /// + /// These flags can be combined with any [`GpuBuddyAllocMode`] to control + /// additional allocation behavior. + #[derive(Clone, Copy, Default, PartialEq, Eq)] + pub struct GpuBuddyAllocFlags(usize); + + /// Individual modifier flag for GPU buddy allocation. + #[derive(Clone, Copy, PartialEq, Eq)] + pub enum GpuBuddyAllocFlag { + /// Allocate physically contiguous blocks. + Contiguous = bindings::GPU_BUDDY_CONTIGUOUS_ALLOCATION, + + /// Request allocation from cleared (zeroed) memory. + Clear = bindings::GPU_BUDDY_CLEAR_ALLOCATION, + + /// Disable trimming of partially used blocks. + TrimDisable = bindings::GPU_BUDDY_TRIM_DISABLE, + } +); + +/// Parameters for creating a GPU buddy allocator. +pub struct GpuBuddyParams { + /// Base offset (in bytes) where the managed memory region starts. + /// Allocations will be offset by this value. + pub base_offset: u64, + /// Total size (in bytes) of the address space managed by the allocator. + pub size: u64, + /// Minimum allocation unit / chunk size; must be >= 4KB. + pub chunk_size: Alignment, +} + +/// Inner structure holding the actual buddy allocator. +/// +/// # Synchronization +/// +/// The C `gpu_buddy` API requires synchronization (see `include/linux/gpu_buddy.h`). +/// Internal locking ensures all allocator and free operations are properly +/// synchronized, preventing races between concurrent allocations and the +/// freeing that occurs when [`AllocatedBlocks`] is dropped. +/// +/// # Invariants +/// +/// The inner [`Opaque`] contains an initialized buddy allocator. +#[pin_data(PinnedDrop)] +struct GpuBuddyInner { + #[pin] + inner: Opaque<bindings::gpu_buddy>, + + // TODO: Replace `Mutex<()>` with `Mutex<Opaque<..>>` once `Mutex::new()` + // accepts `impl PinInit<T>`. + #[pin] + lock: Mutex<()>, + /// Cached creation parameters (do not change after init). + params: GpuBuddyParams, +} + +impl GpuBuddyInner { + /// Create a pin-initializer for the buddy allocator. + fn new(params: GpuBuddyParams) -> impl PinInit<Self, Error> { + let size = params.size; + let chunk_size = params.chunk_size; + + // INVARIANT: `gpu_buddy_init` returns 0 on success, at which point the + // `gpu_buddy` structure is initialized and ready for use with all + // `gpu_buddy_*` APIs. `try_pin_init!` only completes if all fields succeed, + // so the invariant holds when construction finishes. + try_pin_init!(Self { + inner <- Opaque::try_ffi_init(|ptr| { + // SAFETY: `ptr` points to valid uninitialized memory from the pin-init + // infrastructure. `gpu_buddy_init` will initialize the structure. + to_result(unsafe { + bindings::gpu_buddy_init(ptr, size, chunk_size.as_usize() as u64) + }) + }), + lock <- new_mutex!(()), + params, + }) + } + + /// Lock the mutex and return a guard for accessing the allocator. + fn lock(&self) -> GpuBuddyGuard<'_> { + GpuBuddyGuard { + inner: self, + _guard: self.lock.lock(), + } + } +} + +#[pinned_drop] +impl PinnedDrop for GpuBuddyInner { + fn drop(self: Pin<&mut Self>) { + let guard = self.lock(); + + // SAFETY: Per the type invariant, `inner` contains an initialized + // allocator. `guard` provides exclusive access. + unsafe { bindings::gpu_buddy_fini(guard.as_raw()) }; + } +} + +// SAFETY: `GpuBuddyInner` can be sent between threads. +unsafe impl Send for GpuBuddyInner {} + +// SAFETY: `GpuBuddyInner` is `Sync` because `GpuBuddyInner::lock` +// serializes all access to the C allocator, preventing data races. +unsafe impl Sync for GpuBuddyInner {} + +/// Guard that proves the lock is held, enabling access to the allocator. +/// +/// The `_guard` holds the lock for the duration of this guard's lifetime. +struct GpuBuddyGuard<'a> { + inner: &'a GpuBuddyInner, + _guard: MutexGuard<'a, ()>, +} + +impl GpuBuddyGuard<'_> { + /// Get a raw pointer to the underlying C `gpu_buddy` structure. + fn as_raw(&self) -> *mut bindings::gpu_buddy { + self.inner.inner.get() + } +} + +/// GPU buddy allocator instance. +/// +/// This structure wraps the C `gpu_buddy` allocator using reference counting. +/// The allocator is automatically cleaned up when all references are dropped. +/// +/// Refer to the module-level documentation for usage examples. +pub struct GpuBuddy(Arc<GpuBuddyInner>); + +impl GpuBuddy { + /// Create a new buddy allocator. + /// + /// The allocator manages a contiguous address space of the given size, with the + /// specified minimum allocation unit (chunk_size must be at least 4KB). + pub fn new(params: GpuBuddyParams) -> Result<Self> { + Arc::pin_init(GpuBuddyInner::new(params), GFP_KERNEL).map(Self) + } + + /// Get the base offset for allocations. + pub fn base_offset(&self) -> u64 { + self.0.params.base_offset + } + + /// Get the chunk size (minimum allocation unit). + pub fn chunk_size(&self) -> Alignment { + self.0.params.chunk_size + } + + /// Get the total managed size. + pub fn size(&self) -> u64 { + self.0.params.size + } + + /// Get the available (free) memory in bytes. + pub fn avail(&self) -> u64 { + let guard = self.0.lock(); + + // SAFETY: Per the type invariant, `inner` contains an initialized allocator. + // `guard` provides exclusive access. + unsafe { (*guard.as_raw()).avail } + } + + /// Allocate blocks from the buddy allocator. + /// + /// Returns a pin-initializer for [`AllocatedBlocks`]. + pub fn alloc_blocks( + &self, + mode: GpuBuddyAllocMode, + size: u64, + min_block_size: Alignment, + flags: impl Into<GpuBuddyAllocFlags>, + ) -> impl PinInit<AllocatedBlocks, Error> { + let buddy_arc = Arc::clone(&self.0); + let (start, end) = mode.range(); + let mode_flags = mode.as_flags(); + let modifier_flags = flags.into(); + + // Create pin-initializer that initializes list and allocates blocks. + try_pin_init!(AllocatedBlocks { + buddy: buddy_arc, + list <- CListHead::new(), + _: { + // Reject zero-sized or inverted ranges. + if let GpuBuddyAllocMode::Range(range) = &mode { + if range.is_empty() { + Err::<(), Error>(EINVAL)?; + } + } + + // Lock while allocating to serialize with concurrent frees. + let guard = buddy.lock(); + + // SAFETY: Per the type invariant, `inner` contains an initialized + // allocator. `guard` provides exclusive access. + to_result(unsafe { + bindings::gpu_buddy_alloc_blocks( + guard.as_raw(), + start, + end, + size, + min_block_size.as_usize() as u64, + list.as_raw(), + mode_flags | usize::from(modifier_flags), + ) + })? + } + }) + } +} + +/// Allocated blocks from the buddy allocator with automatic cleanup. +/// +/// This structure owns a list of allocated blocks and ensures they are +/// automatically freed when dropped. Use `iter()` to iterate over all +/// allocated blocks. +/// +/// # Invariants +/// +/// - `list` is an initialized, valid list head containing allocated blocks. +#[pin_data(PinnedDrop)] +pub struct AllocatedBlocks { + #[pin] + list: CListHead, + buddy: Arc<GpuBuddyInner>, +} + +impl AllocatedBlocks { + /// Check if the block list is empty. + pub fn is_empty(&self) -> bool { + // An empty list head points to itself. + !self.list.is_linked() + } + + /// Iterate over allocated blocks. + /// + /// Returns an iterator yielding [`AllocatedBlock`] values. Each [`AllocatedBlock`] + /// borrows `self` and is only valid for the duration of that borrow. + pub fn iter(&self) -> impl Iterator<Item = AllocatedBlock<'_>> + '_ { + let head = self.list.as_raw(); + // SAFETY: Per the type invariant, `list` is an initialized sentinel `list_head` + // and is not concurrently modified (we hold a `&self` borrow). The list contains + // `gpu_buddy_block` items linked via `__bindgen_anon_1.link`. `Block` is + // `#[repr(transparent)]` over `gpu_buddy_block`. + let clist = unsafe { + clist_create!( + head, + Block, + bindings::gpu_buddy_block, + __bindgen_anon_1.link + ) + }; + + clist + .iter() + .map(|this| AllocatedBlock { this, blocks: self }) + } +} + +#[pinned_drop] +impl PinnedDrop for AllocatedBlocks { + fn drop(self: Pin<&mut Self>) { + let guard = self.buddy.lock(); + + // SAFETY: + // - list is valid per the type's invariants. + // - guard provides exclusive access to the allocator. + unsafe { + bindings::gpu_buddy_free_list(guard.as_raw(), self.list.as_raw(), 0); + } + } +} + +/// A GPU buddy block. +/// +/// Transparent wrapper over C `gpu_buddy_block` structure. This type is returned +/// as references during iteration over [`AllocatedBlocks`]. +/// +/// # Invariants +/// +/// The inner [`Opaque`] contains a valid, allocated `gpu_buddy_block`. +#[repr(transparent)] +struct Block(Opaque<bindings::gpu_buddy_block>); + +impl Block { + /// Get a raw pointer to the underlying C block. + fn as_raw(&self) -> *mut bindings::gpu_buddy_block { + self.0.get() + } + + /// Get the block's raw offset in the buddy address space (without base offset). + fn offset(&self) -> u64 { + // SAFETY: `self.as_raw()` is valid per the type's invariants. + unsafe { bindings::gpu_buddy_block_offset(self.as_raw()) } + } + + /// Get the block order. + fn order(&self) -> u32 { + // SAFETY: `self.as_raw()` is valid per the type's invariants. + unsafe { bindings::gpu_buddy_block_order(self.as_raw()) } + } +} + +// SAFETY: `Block` is a wrapper around `gpu_buddy_block` which can be +// sent across threads safely. +unsafe impl Send for Block {} + +// SAFETY: `Block` is only accessed through shared references after +// allocation, and thus safe to access concurrently across threads. +unsafe impl Sync for Block {} + +/// A buddy block paired with its owning [`AllocatedBlocks`] context. +/// +/// Unlike a raw block, which only knows its offset within the buddy address +/// space, an [`AllocatedBlock`] also has access to the allocator's `base_offset` +/// and `chunk_size`, enabling it to compute absolute offsets and byte sizes. +/// +/// Returned by [`AllocatedBlocks::iter()`]. +pub struct AllocatedBlock<'a> { + this: &'a Block, + blocks: &'a AllocatedBlocks, +} + +impl AllocatedBlock<'_> { + /// Get the block's offset in the address space. + /// + /// Returns the absolute offset including the allocator's base offset. + /// This is the actual address to use for accessing the allocated memory. + pub fn offset(&self) -> u64 { + self.blocks.buddy.params.base_offset + self.this.offset() + } + + /// Get the block order (size = chunk_size << order). + pub fn order(&self) -> u32 { + self.this.order() + } + + /// Get the block's size in bytes. + pub fn size(&self) -> u64 { + (self.blocks.buddy.params.chunk_size.as_usize() as u64) << self.this.order() + } +} diff --git a/rust/kernel/i2c.rs b/rust/kernel/i2c.rs index bb5b830f48c3..0487bae811fb 100644 --- a/rust/kernel/i2c.rs +++ b/rust/kernel/i2c.rs @@ -16,10 +16,11 @@ use crate::{ error::*, of, prelude::*, - types::{ - AlwaysRefCounted, - Opaque, // - }, // + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, // }; use core::{ @@ -31,8 +32,6 @@ use core::{ }, // }; -use kernel::types::ARef; - /// An I2C device id table. #[repr(transparent)] #[derive(Clone, Copy)] @@ -66,10 +65,6 @@ unsafe impl RawDeviceId for DeviceId { // SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::i2c_device_id, driver_data); - - fn index(&self) -> usize { - self.0.driver_data - } } /// IdTable type for I2C @@ -78,14 +73,8 @@ pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; /// Create a I2C `IdTable` with its alias for modpost. #[macro_export] macro_rules! i2c_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::i2c::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("i2c", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("i2c", $crate::i2c::DeviceId, $($tt)*); }; } @@ -94,18 +83,18 @@ pub struct Adapter<T: Driver>(T); // SAFETY: // - `bindings::i2c_driver` is a C type declared as `repr(C)`. -// - `T` is the type of the driver's device private data. +// - `T::Data` is the type of the driver's device private data. // - `struct i2c_driver` embeds a `struct device_driver`. // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. -unsafe impl<T: Driver + 'static> driver::DriverLayout for Adapter<T> { +unsafe impl<T: Driver> driver::DriverLayout for Adapter<T> { type DriverType = bindings::i2c_driver; - type DriverData = T; + type DriverData<'bound> = T::Data<'bound>; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); } // SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if // a preceding call to `register` has been successful. -unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { +unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { unsafe fn register( idrv: &Opaque<Self::DriverType>, name: &'static CStr, @@ -143,7 +132,7 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { } // SAFETY: `idrv` is guaranteed to be a valid `DriverType`. - to_result(unsafe { bindings::i2c_register_driver(module.0, idrv.get()) }) + to_result(unsafe { bindings::i2c_register_driver(module.as_ptr(), idrv.get()) }) } unsafe fn unregister(idrv: &Opaque<Self::DriverType>) { @@ -152,16 +141,18 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { } } -impl<T: Driver + 'static> Adapter<T> { +impl<T: Driver> Adapter<T> { extern "C" fn probe_callback(idev: *mut bindings::i2c_client) -> kernel::ffi::c_int { // SAFETY: The I2C bus only ever calls the probe callback with a valid pointer to a // `struct i2c_client`. // // INVARIANT: `idev` is valid for the duration of `probe_callback()`. - let idev = unsafe { &*idev.cast::<I2cClient<device::CoreInternal>>() }; + let idev = unsafe { &*idev.cast::<I2cClient<device::CoreInternal<'_>>>() }; - let info = - Self::i2c_id_info(idev).or_else(|| <Self as driver::Adapter>::id_info(idev.as_ref())); + let info = Self::i2c_id_info(idev).or_else(|| { + // SAFETY: `idev` matched data is of type `Self::IdInfo`. + unsafe { <Self as driver::Adapter>::id_info(idev.as_ref()) } + }); from_result(|| { let data = T::probe(idev, info); @@ -173,24 +164,24 @@ impl<T: Driver + 'static> Adapter<T> { extern "C" fn remove_callback(idev: *mut bindings::i2c_client) { // SAFETY: `idev` is a valid pointer to a `struct i2c_client`. - let idev = unsafe { &*idev.cast::<I2cClient<device::CoreInternal>>() }; + let idev = unsafe { &*idev.cast::<I2cClient<device::CoreInternal<'_>>>() }; // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `I2cClient::set_drvdata()` has been called - // and stored a `Pin<KBox<T>>`. - let data = unsafe { idev.as_ref().drvdata_borrow::<T>() }; + // and stored a `Pin<KBox<T::Data<'_>>>`. + let data = unsafe { idev.as_ref().drvdata_borrow::<T::Data<'_>>() }; T::unbind(idev, data); } extern "C" fn shutdown_callback(idev: *mut bindings::i2c_client) { // SAFETY: `shutdown_callback` is only ever called for a valid `idev` - let idev = unsafe { &*idev.cast::<I2cClient<device::CoreInternal>>() }; + let idev = unsafe { &*idev.cast::<I2cClient<device::CoreInternal<'_>>>() }; // SAFETY: `shutdown_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called - // and stored a `Pin<KBox<T>>`. - let data = unsafe { idev.as_ref().drvdata_borrow::<T>() }; + // and stored a `Pin<KBox<T::Data<'_>>>`. + let data = unsafe { idev.as_ref().drvdata_borrow::<T::Data<'_>>() }; T::shutdown(idev, data); } @@ -219,11 +210,12 @@ impl<T: Driver + 'static> Adapter<T> { // does not add additional invariants, so it's safe to transmute. let id = unsafe { &*raw_id.cast::<DeviceId>() }; - Some(table.info(<DeviceId as RawDeviceIdIndex>::index(id))) + // SAFETY: `id` comes from `table` which is of type `IdArray<_, Self::IdInfo>`. + Some(unsafe { id.info_unchecked::<T::IdInfo>() }) } } -impl<T: Driver + 'static> driver::Adapter for Adapter<T> { +impl<T: Driver> driver::Adapter for Adapter<T> { type IdInfo = T::IdInfo; fn of_id_table() -> Option<of::IdTable<Self::IdInfo>> { @@ -268,7 +260,6 @@ macro_rules! module_i2c_driver { /// /// kernel::acpi_device_table!( /// ACPI_TABLE, -/// MODULE_ACPI_TABLE, /// <MyDriver as i2c::Driver>::IdInfo, /// [ /// (acpi::DeviceId::new(c"LNUXBEEF"), ()) @@ -277,7 +268,6 @@ macro_rules! module_i2c_driver { /// /// kernel::i2c_device_table!( /// I2C_TABLE, -/// MODULE_I2C_TABLE, /// <MyDriver as i2c::Driver>::IdInfo, /// [ /// (i2c::DeviceId::new(c"rust_driver_i2c"), ()) @@ -286,7 +276,6 @@ macro_rules! module_i2c_driver { /// /// kernel::of_device_table!( /// OF_TABLE, -/// MODULE_OF_TABLE, /// <MyDriver as i2c::Driver>::IdInfo, /// [ /// (of::DeviceId::new(c"test,device"), ()) @@ -295,22 +284,26 @@ macro_rules! module_i2c_driver { /// /// impl i2c::Driver for MyDriver { /// type IdInfo = (); +/// type Data<'bound> = Self; /// const I2C_ID_TABLE: Option<i2c::IdTable<Self::IdInfo>> = Some(&I2C_TABLE); /// const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = Some(&OF_TABLE); /// const ACPI_ID_TABLE: Option<acpi::IdTable<Self::IdInfo>> = Some(&ACPI_TABLE); /// -/// fn probe( -/// _idev: &i2c::I2cClient<Core>, -/// _id_info: Option<&Self::IdInfo>, -/// ) -> impl PinInit<Self, Error> { +/// fn probe<'bound>( +/// _idev: &'bound i2c::I2cClient<Core<'_>>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound { /// Err(ENODEV) /// } /// -/// fn shutdown(_idev: &i2c::I2cClient<Core>, this: Pin<&Self>) { +/// fn shutdown<'bound>( +/// _idev: &'bound i2c::I2cClient<Core<'_>>, +/// this: Pin<&Self::Data<'bound>>, +/// ) { /// } /// } ///``` -pub trait Driver: Send { +pub trait Driver { /// The type holding information about each device id supported by the driver. // TODO: Use `associated_type_defaults` once stabilized: // @@ -319,6 +312,9 @@ pub trait Driver: Send { // ``` type IdInfo: 'static; + /// The type of the driver's bus device private data. + type Data<'bound>: Send + 'bound; + /// The table of device ids supported by the driver. const I2C_ID_TABLE: Option<IdTable<Self::IdInfo>> = None; @@ -332,10 +328,10 @@ pub trait Driver: Send { /// /// Called when a new i2c client is added or discovered. /// Implementers should attempt to initialize the client here. - fn probe( - dev: &I2cClient<device::Core>, - id_info: Option<&Self::IdInfo>, - ) -> impl PinInit<Self, Error>; + fn probe<'bound>( + dev: &'bound I2cClient<device::Core<'_>>, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound; /// I2C driver shutdown. /// @@ -347,8 +343,8 @@ pub trait Driver: Send { /// /// This callback is distinct from final resource cleanup, as the driver instance remains valid /// after it returns. Any deallocation or teardown of driver-owned resources should instead be - /// handled in `Self::drop`. - fn shutdown(dev: &I2cClient<device::Core>, this: Pin<&Self>) { + /// handled in `Drop`. + fn shutdown<'bound>(dev: &'bound I2cClient<device::Core<'_>>, this: Pin<&Self::Data<'bound>>) { let _ = (dev, this); } @@ -361,8 +357,8 @@ pub trait Driver: Send { /// `&Device<Core>` or `&Device<Bound>` reference. For instance, drivers may try to perform I/O /// operations to gracefully tear down the device. /// - /// Otherwise, release operations for driver resources should be performed in `Self::drop`. - fn unbind(dev: &I2cClient<device::Core>, this: Pin<&Self>) { + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(dev: &'bound I2cClient<device::Core<'_>>, this: Pin<&Self::Data<'bound>>) { let _ = (dev, this); } } @@ -398,6 +394,7 @@ impl I2cAdapter { } /// Gets pointer to an `i2c_adapter` by index. + #[inline] pub fn get(index: i32) -> Result<ARef<Self>> { // SAFETY: `index` must refer to a valid I2C adapter; the kernel // guarantees that `i2c_get_adapter(index)` returns either a valid @@ -406,7 +403,9 @@ impl I2cAdapter { // SAFETY: `adapter` is non-null and points to a live `i2c_adapter`. // `I2cAdapter` is #[repr(transparent)], so this cast is valid. - Ok(unsafe { (&*adapter.as_ptr().cast::<I2cAdapter<device::Normal>>()).into() }) + // `i2c_get_adapter` returned the adapter with an incremented refcount, which we pass to + // the `ARef`. + Ok(unsafe { ARef::from_raw(adapter.cast::<I2cAdapter<device::Normal>>()) }) } } @@ -416,12 +415,14 @@ kernel::impl_device_context_deref!(unsafe { I2cAdapter }); kernel::impl_device_context_into_aref!(I2cAdapter); // SAFETY: Instances of `I2cAdapter` are always reference-counted. -unsafe impl crate::types::AlwaysRefCounted for I2cAdapter { +unsafe impl AlwaysRefCounted for I2cAdapter { + #[inline] fn inc_ref(&self) { // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. unsafe { bindings::i2c_get_adapter(self.index()) }; } + #[inline] unsafe fn dec_ref(obj: NonNull<Self>) { // SAFETY: The safety requirements guarantee that the refcount is non-zero. unsafe { bindings::i2c_put_adapter(obj.as_ref().as_raw()) } diff --git a/rust/kernel/impl_flags.rs b/rust/kernel/impl_flags.rs index e2bd7639da12..fdf44d5eea9c 100644 --- a/rust/kernel/impl_flags.rs +++ b/rust/kernel/impl_flags.rs @@ -19,7 +19,10 @@ /// # Examples /// /// ``` -/// use kernel::impl_flags; +/// use kernel::{ +/// bits::bit_u32, +/// impl_flags, // +/// }; /// /// impl_flags!( /// /// Represents multiple permissions. @@ -30,13 +33,13 @@ /// #[derive(Debug, Clone, Copy, PartialEq, Eq)] /// pub enum Permission { /// /// Read permission. -/// Read = 1 << 0, +/// Read = bit_u32(0), /// /// /// Write permission. -/// Write = 1 << 1, +/// Write = bit_u32(1), /// /// /// Execute permission. -/// Execute = 1 << 2, +/// Execute = bit_u32(2), /// } /// ); /// diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 7a0d4559d7b5..1fdc3963e3e3 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -151,13 +151,16 @@ pub trait InPlaceInit<T>: Sized { /// type. /// /// If `T: !Unpin` it will not be able to move afterwards. + #[inline] fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self::PinnedSelf> where Error: From<E>, { // SAFETY: We delegate to `init` and only change the error type. let init = unsafe { - pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) + pin_init_from_closure(|slot| { + pin_init::raw_try_init(slot, init).map_err(|e| Error::from(e)) + }) }; Self::try_pin_init(init, flags) } @@ -168,13 +171,14 @@ pub trait InPlaceInit<T>: Sized { E: From<AllocError>; /// Use the given initializer to in-place initialize a `T`. + #[inline] fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self> where Error: From<E>, { // SAFETY: We delegate to `init` and only change the error type. let init = unsafe { - init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) + init_from_closure(|slot| pin_init::raw_try_init(slot, init).map_err(|e| Error::from(e))) }; Self::try_init(init, flags) } diff --git a/rust/kernel/interop.rs b/rust/kernel/interop.rs new file mode 100644 index 000000000000..3b371d782a59 --- /dev/null +++ b/rust/kernel/interop.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Infrastructure for interfacing Rust code with C kernel subsystems. +//! +//! This module is intended for low-level, unsafe Rust infrastructure code +//! that interoperates between Rust and C. It is *not* for use directly in +//! Rust drivers. + +pub mod list; diff --git a/rust/kernel/interop/list.rs b/rust/kernel/interop/list.rs new file mode 100644 index 000000000000..54265ea036bb --- /dev/null +++ b/rust/kernel/interop/list.rs @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust interface for C doubly circular intrusive linked lists. +//! +//! This module provides Rust abstractions for iterating over C `list_head`-based +//! linked lists. It should only be used for cases where C and Rust code share +//! direct access to the same linked list through a C interop interface. +//! +//! Note: This *must not* be used by Rust components that just need a linked list +//! primitive. Use [`kernel::list::List`] instead. +//! +//! # Examples +//! +//! ``` +//! use kernel::{ +//! bindings, +//! interop::list::clist_create, +//! types::Opaque, +//! }; +//! # // Create test list with values (0, 10, 20) - normally done by C code but it is +//! # // emulated here for doctests using the C bindings. +//! # use core::mem::MaybeUninit; +//! # +//! # /// C struct with embedded `list_head` (typically will be allocated by C code). +//! # #[repr(C)] +//! # pub struct SampleItemC { +//! # pub value: i32, +//! # pub link: bindings::list_head, +//! # } +//! # +//! # let mut head = MaybeUninit::<bindings::list_head>::uninit(); +//! # +//! # let head = head.as_mut_ptr(); +//! # // SAFETY: `head` and all the items are test objects allocated in this scope. +//! # unsafe { bindings::INIT_LIST_HEAD(head) }; +//! # +//! # let mut items = [ +//! # MaybeUninit::<SampleItemC>::uninit(), +//! # MaybeUninit::<SampleItemC>::uninit(), +//! # MaybeUninit::<SampleItemC>::uninit(), +//! # ]; +//! # +//! # for (i, item) in items.iter_mut().enumerate() { +//! # let ptr = item.as_mut_ptr(); +//! # // SAFETY: `ptr` points to a valid `MaybeUninit<SampleItemC>`. +//! # unsafe { (*ptr).value = i as i32 * 10 }; +//! # // SAFETY: `&raw mut` creates a pointer valid for `INIT_LIST_HEAD`. +//! # unsafe { bindings::INIT_LIST_HEAD(&raw mut (*ptr).link) }; +//! # // SAFETY: `link` was just initialized and `head` is a valid list head. +//! # unsafe { bindings::list_add_tail(&mut (*ptr).link, head) }; +//! # } +//! +//! /// Rust wrapper for the C struct. +//! /// +//! /// The list item struct in this example is defined in C code as: +//! /// +//! /// ```c +//! /// struct SampleItemC { +//! /// int value; +//! /// struct list_head link; +//! /// }; +//! /// ``` +//! #[repr(transparent)] +//! pub struct Item(Opaque<SampleItemC>); +//! +//! impl Item { +//! pub fn value(&self) -> i32 { +//! // SAFETY: `Item` has the same layout as `SampleItemC`. +//! unsafe { (*self.0.get()).value } +//! } +//! } +//! +//! // Create typed [`CList`] from sentinel head. +//! // SAFETY: `head` is valid and initialized, items are `SampleItemC` with +//! // embedded `link` field, and `Item` is `#[repr(transparent)]` over `SampleItemC`. +//! let list = unsafe { clist_create!(head, Item, SampleItemC, link) }; +//! +//! // Iterate directly over typed items. +//! let mut found_0 = false; +//! let mut found_10 = false; +//! let mut found_20 = false; +//! +//! for item in list.iter() { +//! let val = item.value(); +//! if val == 0 { found_0 = true; } +//! if val == 10 { found_10 = true; } +//! if val == 20 { found_20 = true; } +//! } +//! +//! assert!(found_0 && found_10 && found_20); +//! ``` + +use core::{ + iter::FusedIterator, + marker::PhantomData, // +}; + +use crate::{ + bindings, + types::Opaque, // +}; + +use pin_init::{ + pin_data, + pin_init, + PinInit, // +}; + +/// FFI wrapper for a C `list_head` object used in intrusive linked lists. +/// +/// # Invariants +/// +/// - The underlying `list_head` is initialized with valid non-`NULL` `next`/`prev` pointers. +#[pin_data] +#[repr(transparent)] +pub struct CListHead { + #[pin] + inner: Opaque<bindings::list_head>, +} + +impl CListHead { + /// Create a `&CListHead` reference from a raw `list_head` pointer. + /// + /// # Safety + /// + /// - `ptr` must be a valid pointer to an initialized `list_head` (e.g. via + /// `INIT_LIST_HEAD()`), with valid non-`NULL` `next`/`prev` pointers. + /// - `ptr` must remain valid for the lifetime `'a`. + /// - The list and all linked `list_head` nodes must not be modified from + /// anywhere for the lifetime `'a`, unless done so via any [`CListHead`] APIs. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::list_head) -> &'a Self { + // SAFETY: + // - `CListHead` has the same layout as `list_head`. + // - `ptr` is valid and unmodified for `'a` per caller guarantees. + unsafe { &*ptr.cast() } + } + + /// Get the raw `list_head` pointer. + #[inline] + pub fn as_raw(&self) -> *mut bindings::list_head { + self.inner.get() + } + + /// Get the next [`CListHead`] in the list. + #[inline] + pub fn next(&self) -> &Self { + let raw = self.as_raw(); + // SAFETY: + // - `self.as_raw()` is valid and initialized per type invariants. + // - The `next` pointer is valid and non-`NULL` per type invariants + // (initialized via `INIT_LIST_HEAD()` or equivalent). + unsafe { Self::from_raw((*raw).next) } + } + + /// Check if this node is linked in a list (not isolated). + #[inline] + pub fn is_linked(&self) -> bool { + let raw = self.as_raw(); + // SAFETY: `self.as_raw()` is valid per type invariants. + unsafe { (*raw).next != raw && (*raw).prev != raw } + } + + /// Returns a pin-initializer for the list head. + pub fn new() -> impl PinInit<Self> { + pin_init!(Self { + // SAFETY: `INIT_LIST_HEAD` initializes `slot` to a valid empty list. + inner <- Opaque::ffi_init(|slot| unsafe { bindings::INIT_LIST_HEAD(slot) }), + }) + } +} + +// SAFETY: `list_head` contains no thread-bound state; it only holds +// `next`/`prev` pointers. +unsafe impl Send for CListHead {} + +// SAFETY: `CListHead` can be shared among threads as modifications are +// not allowed at the moment. +unsafe impl Sync for CListHead {} + +impl PartialEq for CListHead { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self, other) + } +} + +impl Eq for CListHead {} + +/// Low-level iterator over `list_head` nodes. +/// +/// An iterator used to iterate over a C intrusive linked list (`list_head`). The caller has to +/// perform conversion of returned [`CListHead`] to an item (using [`container_of`] or similar). +/// +/// # Invariants +/// +/// `current` and `sentinel` are valid references into an initialized linked list. +struct CListHeadIter<'a> { + /// Current position in the list. + current: &'a CListHead, + /// The sentinel head (used to detect end of iteration). + sentinel: &'a CListHead, +} + +impl<'a> Iterator for CListHeadIter<'a> { + type Item = &'a CListHead; + + #[inline] + fn next(&mut self) -> Option<Self::Item> { + // Check if we've reached the sentinel (end of list). + if self.current == self.sentinel { + return None; + } + + let item = self.current; + self.current = item.next(); + Some(item) + } +} + +impl<'a> FusedIterator for CListHeadIter<'a> {} + +/// A typed C linked list with a sentinel head intended for FFI use-cases where +/// a C subsystem manages a linked list that Rust code needs to read. Generally +/// required only for special cases. +/// +/// A sentinel head [`CListHead`] represents the entire linked list and can be used +/// for iteration over items of type `T`; it is not associated with a specific item. +/// +/// The const generic `OFFSET` specifies the byte offset of the `list_head` field within +/// the struct that `T` wraps. +/// +/// # Invariants +/// +/// - The sentinel [`CListHead`] has valid non-`NULL` `next`/`prev` pointers. +/// - `OFFSET` is the byte offset of the `list_head` field within the struct that `T` wraps. +/// - All the list's `list_head` nodes have valid non-`NULL` `next`/`prev` pointers. +#[repr(transparent)] +pub struct CList<T, const OFFSET: usize>(CListHead, PhantomData<T>); + +impl<T, const OFFSET: usize> CList<T, OFFSET> { + /// Create a typed [`CList`] reference from a raw sentinel `list_head` pointer. + /// + /// # Safety + /// + /// - `ptr` must be a valid pointer to an initialized sentinel `list_head` (e.g. via + /// `INIT_LIST_HEAD()`), with valid non-`NULL` `next`/`prev` pointers. + /// - `ptr` must remain valid for the lifetime `'a`. + /// - The list and all linked nodes must not be concurrently modified for the lifetime `'a`. + /// - The list must contain items where the `list_head` field is at byte offset `OFFSET`. + /// - `T` must be `#[repr(transparent)]` over the C struct. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::list_head) -> &'a Self { + // SAFETY: + // - `CList` has the same layout as `CListHead` due to `#[repr(transparent)]`. + // - Caller guarantees `ptr` is a valid, sentinel `list_head` object. + unsafe { &*ptr.cast() } + } + + /// Check if the list is empty. + #[inline] + pub fn is_empty(&self) -> bool { + !self.0.is_linked() + } + + /// Create an iterator over typed items. + #[inline] + pub fn iter(&self) -> CListIter<'_, T, OFFSET> { + let head = &self.0; + CListIter { + head_iter: CListHeadIter { + current: head.next(), + sentinel: head, + }, + _phantom: PhantomData, + } + } +} + +/// High-level iterator over typed list items. +pub struct CListIter<'a, T, const OFFSET: usize> { + head_iter: CListHeadIter<'a>, + _phantom: PhantomData<&'a T>, +} + +impl<'a, T, const OFFSET: usize> Iterator for CListIter<'a, T, OFFSET> { + type Item = &'a T; + + #[inline] + fn next(&mut self) -> Option<Self::Item> { + let head = self.head_iter.next()?; + + // Convert to item using `OFFSET`. + // + // SAFETY: The pointer calculation is valid because `OFFSET` is derived + // from `offset_of!` per type invariants. + Some(unsafe { &*head.as_raw().byte_sub(OFFSET).cast::<T>() }) + } +} + +impl<'a, T, const OFFSET: usize> FusedIterator for CListIter<'a, T, OFFSET> {} + +/// Create a C doubly-circular linked list interface [`CList`] from a raw `list_head` pointer. +/// +/// This macro creates a `CList<T, OFFSET>` that can iterate over items of type `$rust_type` +/// linked via the `$field` field in the underlying C struct `$c_type`. +/// +/// # Arguments +/// +/// - `$head`: Raw pointer to the sentinel `list_head` object (`*mut bindings::list_head`). +/// - `$rust_type`: Each item's Rust wrapper type. +/// - `$c_type`: Each item's C struct type that contains the embedded `list_head`. +/// - `$field`: The name of the `list_head` field within the C struct. +/// +/// # Safety +/// +/// The caller must ensure: +/// +/// - `$head` is a valid, initialized sentinel `list_head` (e.g. via `INIT_LIST_HEAD()`) +/// pointing to a list that is not concurrently modified for the lifetime of the [`CList`]. +/// - The list contains items of type `$c_type` linked via an embedded `$field`. +/// - `$rust_type` is `#[repr(transparent)]` over `$c_type` or has compatible layout. +/// +/// # Examples +/// +/// Refer to the examples in the [`crate::interop::list`] module documentation. +#[macro_export] +macro_rules! clist_create { + ($head:expr, $rust_type:ty, $c_type:ty, $($field:tt).+) => {{ + // Compile-time check that field path is a `list_head`. + let _: fn(*const $c_type) -> *const $crate::bindings::list_head = + |p| &raw const (*p).$($field).+; + + // Calculate offset and create `CList`. + const OFFSET: usize = ::core::mem::offset_of!($c_type, $($field).+); + $crate::interop::list::CList::<$rust_type, OFFSET>::from_raw($head) + }}; +} +pub use clist_create; diff --git a/rust/kernel/interrupt.rs b/rust/kernel/interrupt.rs new file mode 100644 index 000000000000..a880ec3b8538 --- /dev/null +++ b/rust/kernel/interrupt.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Interrupt controls +//! +//! This module allows Rust code to annotate areas of code where local processor interrupts should +//! be disabled, along with actually disabling local processor interrupts. +//! +//! # ⚠️ Warning! ⚠️ +//! +//! The usage of this module can be more complicated than meets the eye, especially surrounding +//! [preemptible kernels]. It's recommended to take care when using the functions and types defined +//! here and familiarize yourself with the various documentation we have before using them, along +//! with the various documents we link to here. +//! +//! # Reading material +//! +//! - [Software interrupts and realtime (LWN)](https://lwn.net/Articles/520076) +//! +//! [preemptible kernels]: https://www.kernel.org/doc/html/latest/locking/preempt-locking.html + +use crate::types::NotThreadSafe; + +/// A guard that represents local processor interrupt disablement on preemptible kernels. +/// +/// [`LocalInterruptDisabled`] is a guard type that represents that local processor interrupts have +/// been disabled on a preemptible kernel. +/// +/// Certain functions take an immutable reference of [`LocalInterruptDisabled`] in order to require +/// that they may only be run in local-interrupt-disabled contexts on preemptible kernels. +/// +/// This is a marker type; it has no size, and is simply used as a compile-time guarantee that local +/// processor interrupts are disabled on preemptible kernels. Note that no guarantees about the +/// state of interrupts are made by this type on non-preemptible kernels. +/// +/// # Invariants +/// +/// Local processor interrupts are disabled on preemptible kernels for as long as an object of this +/// type exists. +pub struct LocalInterruptDisabled(NotThreadSafe); + +/// Disable local processor interrupts on a preemptible kernel. +/// +/// This function disables local processor interrupts on a preemptible kernel, and returns a +/// [`LocalInterruptDisabled`] token as proof of this. On non-preemptible kernels, this function is +/// a no-op. +/// +/// **Usage of this function is discouraged** unless you are absolutely sure you know what you are +/// doing, as kernel interfaces for Rust that deal with interrupt state will typically handle local +/// processor interrupt state management on their own and managing this by hand is quite error +/// prone. +#[inline] +pub fn local_interrupt_disable() -> LocalInterruptDisabled { + // SAFETY: It's always safe to call `local_interrupt_disable()`. + unsafe { bindings::local_interrupt_disable() }; + + LocalInterruptDisabled(NotThreadSafe) +} + +impl Drop for LocalInterruptDisabled { + #[inline] + fn drop(&mut self) { + // SAFETY: Per type invariants, a `local_interrupt_disable()` must be called to create this + // object, hence calling the corresponding `local_interrupt_enable()` is safe. + unsafe { bindings::local_interrupt_enable() }; + } +} + +impl LocalInterruptDisabled { + /// Assume that local processor interrupts are disabled on preemptible kernels. + /// + /// This can be used for annotating code that is known to be run in contexts where local + /// processor interrupts are disabled on preemptible kernels. It makes no changes to the local + /// interrupt state on its own. + /// + /// # Safety + /// + /// For the whole life `'a`, local interrupts must be disabled on preemptible kernels. This + /// could be a context like, for example, an interrupt handler. + #[inline] + pub unsafe fn assume_disabled<'a>() -> &'a LocalInterruptDisabled { + const ASSUME_DISABLED: &LocalInterruptDisabled = &LocalInterruptDisabled(NotThreadSafe); + + // Confirm they're actually disabled if lockdep is available + // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()`. + unsafe { bindings::lockdep_assert_irqs_disabled() }; + + ASSUME_DISABLED + } +} diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index e5fba6bf6db0..5ce9fd129068 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -4,17 +4,31 @@ //! //! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h) +use core::{ + marker::PhantomData, + mem::MaybeUninit, // +}; + use crate::{ bindings, - prelude::*, // + prelude::*, + ptr::{ + Alignment, + KnownSize, // + }, // }; +#[cfg(CONFIG_HAS_IOMEM)] pub mod mem; pub mod poll; +pub mod register; pub mod resource; +pub use crate::register; pub use resource::Resource; +use register::LocatedRegister; + /// Physical address type. /// /// This is a type alias to either `u32` or `u64` depending on the config option @@ -27,629 +41,1714 @@ pub type PhysAddr = bindings::phys_addr_t; /// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. pub type ResourceSize = bindings::resource_size_t; +/// Untyped I/O region. +/// +/// This type can be used when an I/O region without known type information has a compile-time known +/// minimum size (and a runtime known actual size). +/// +/// # Invariants +/// +/// - Size of the region is at least as large as the `SIZE` generic parameter. +/// - Size of the region is multiple of 4. +#[repr(C, align(4))] +#[derive(FromBytes)] +pub struct Region<const SIZE: usize = 0> { + inner: [u8], +} + +impl<const SIZE: usize> Region<SIZE> { + /// Create a raw mutable pointer from given base address and size. + /// + /// `size` should be at least as large as the minimum size `SIZE`, and `base` and `size` should + /// be 4-byte aligned to uphold the type invariant. + /// + /// Just like other methods on raw pointers, it is not unsafe to create a raw pointer + /// that does not uphold the type invariants. However such pointers are not valid. + #[inline] + pub fn ptr_from_raw_parts_mut(base: *mut u8, size: usize) -> *mut Self { + core::ptr::slice_from_raw_parts_mut(base, size) as *mut Region<SIZE> + } + + /// Create a raw mutable pointer from given base address and size. + /// + /// The alignment of `base` is checked, and `size` is checked against the minimum size specified + /// via const generics. + #[inline] + pub fn ptr_try_from_raw_parts_mut(base: *mut u8, size: usize) -> Result<*mut Self> { + if size < SIZE || base.align_offset(4) != 0 || !size.is_multiple_of(4) { + return Err(EINVAL); + } + + Ok(Self::ptr_from_raw_parts_mut(base, size)) + } +} + +impl<const SIZE: usize> KnownSize for Region<SIZE> { + const MIN_SIZE: usize = SIZE; + // Alignment of 4 is the most common; different base types can be added once required. + const MIN_ALIGN: Alignment = Alignment::new::<4>(); + + #[inline(always)] + fn size(p: *const Self) -> usize { + (p as *const [u8]).len() + } +} + +// SAFETY: +// - Values read from I/O are always treated as initialized. +// - Per type invariant the size is multiple of 4 and the type is 4-byte aligned, so it is padding +// free. +// +// This cannot be derived as `derive(IntoBytes)` as the padding free property comes from type +// invariant which the macro does not know. +unsafe impl<const SIZE: usize> IntoBytes for Region<SIZE> { + #[inline] + #[allow(unused)] // Rust 1.87+ stops requiring this and will emit unused warnings. + fn only_derive_is_allowed_to_implement_this_trait() {} +} + /// Raw representation of an MMIO region. /// +/// `MmioRaw<T>` is equivalent to `T __iomem *` in C. +/// /// By itself, the existence of an instance of this structure does not provide any guarantees that /// the represented MMIO region does exist or is properly mapped. /// /// Instead, the bus specific MMIO implementation must convert this raw representation into an /// `Mmio` instance providing the actual memory accessors. Only by the conversion into an `Mmio` /// structure any guarantees are given. -pub struct MmioRaw<const SIZE: usize = 0> { - addr: usize, - maxsize: usize, +pub struct MmioRaw<T: ?Sized> { + /// Pointer is in I/O address space. + /// + /// The provenance does not matter, only the address and metadata do. + ptr: *mut T, } -impl<const SIZE: usize> MmioRaw<SIZE> { - /// Returns a new `MmioRaw` instance on success, an error otherwise. - pub fn new(addr: usize, maxsize: usize) -> Result<Self> { - if maxsize < SIZE { - return Err(EINVAL); +impl<T: ?Sized> Copy for MmioRaw<T> {} +impl<T: ?Sized> Clone for MmioRaw<T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +// SAFETY: `MmioRaw` is just an address, so is thread-safe. +unsafe impl<T: ?Sized> Send for MmioRaw<T> {} +// SAFETY: `MmioRaw` is just an address, so is thread-safe. +unsafe impl<T: ?Sized> Sync for MmioRaw<T> {} + +impl<T> MmioRaw<T> { + /// Create a `MmioRaw` from address. + #[inline] + pub fn new(addr: usize) -> Self { + Self { + ptr: core::ptr::without_provenance_mut(addr), } + } +} - Ok(Self { addr, maxsize }) +impl<const SIZE: usize> MmioRaw<Region<SIZE>> { + /// Create a `MmioRaw` representing a I/O region with given size. + /// + /// The size is checked against the minimum size specified via const generics. + #[inline] + pub fn new_region(addr: usize, size: usize) -> Result<Self> { + Ok(Self { + ptr: Region::ptr_try_from_raw_parts_mut(core::ptr::without_provenance_mut(addr), size)?, + }) } +} +impl<T: ?Sized + KnownSize> MmioRaw<T> { /// Returns the base address of the MMIO region. #[inline] pub fn addr(&self) -> usize { - self.addr + self.ptr.addr() } - /// Returns the maximum size of the MMIO region. + /// Returns the size of the MMIO region. #[inline] - pub fn maxsize(&self) -> usize { - self.maxsize + pub fn size(&self) -> usize { + KnownSize::size(self.ptr) } } -/// IO-mapped memory region. -/// -/// The creator (usually a subsystem / bus such as PCI) is responsible for creating the -/// mapping, performing an additional region request etc. -/// -/// # Invariant -/// -/// `addr` is the start and `maxsize` the length of valid I/O mapped memory region of size -/// `maxsize`. -/// -/// # Examples +/// Checks whether an access of type `U` at the given `base` and the given `offset` +/// is valid within this region. /// -/// ```no_run -/// use kernel::{ -/// bindings, -/// ffi::c_void, -/// io::{ -/// Io, -/// IoKnownSize, -/// Mmio, -/// MmioRaw, -/// PhysAddr, -/// }, -/// }; -/// use core::ops::Deref; -/// -/// // See also `pci::Bar` for a real example. -/// struct IoMem<const SIZE: usize>(MmioRaw<SIZE>); -/// -/// impl<const SIZE: usize> IoMem<SIZE> { -/// /// # Safety -/// /// -/// /// [`paddr`, `paddr` + `SIZE`) must be a valid MMIO region that is mappable into the CPUs -/// /// virtual address space. -/// unsafe fn new(paddr: usize) -> Result<Self>{ -/// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is -/// // valid for `ioremap`. -/// let addr = unsafe { bindings::ioremap(paddr as PhysAddr, SIZE) }; -/// if addr.is_null() { -/// return Err(ENOMEM); -/// } -/// -/// Ok(IoMem(MmioRaw::new(addr as usize, SIZE)?)) -/// } -/// } -/// -/// impl<const SIZE: usize> Drop for IoMem<SIZE> { -/// fn drop(&mut self) { -/// // SAFETY: `self.0.addr()` is guaranteed to be properly mapped by `Self::new`. -/// unsafe { bindings::iounmap(self.0.addr() as *mut c_void); }; -/// } -/// } -/// -/// impl<const SIZE: usize> Deref for IoMem<SIZE> { -/// type Target = Mmio<SIZE>; -/// -/// fn deref(&self) -> &Self::Target { -/// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`. -/// unsafe { Mmio::from_raw(&self.0) } -/// } -/// } -/// -///# fn no_run() -> Result<(), Error> { -/// // SAFETY: Invalid usage for example purposes. -/// let iomem = unsafe { IoMem::<{ core::mem::size_of::<u32>() }>::new(0xBAAAAAAD)? }; -/// iomem.write32(0x42, 0x0); -/// assert!(iomem.try_write32(0x42, 0x0).is_ok()); -/// assert!(iomem.try_write32(0x42, 0x4).is_err()); -/// # Ok(()) -/// # } -/// ``` -#[repr(transparent)] -pub struct Mmio<const SIZE: usize = 0>(MmioRaw<SIZE>); +/// The `base` is used for alignment checking only. This can be set to 0 to skip the check. +#[inline] +const fn offset_valid<U>(base: usize, offset: usize, size: usize) -> bool { + if let Some(end) = offset.checked_add(size_of::<U>()) { + end <= size && (base.wrapping_add(offset) % align_of::<U>() == 0) + } else { + false + } +} -/// Internal helper macros used to invoke C MMIO read functions. -/// -/// This macro is intended to be used by higher-level MMIO access macros (io_define_read) and -/// provides a unified expansion for infallible vs. fallible read semantics. It emits a direct call -/// into the corresponding C helper and performs the required cast to the Rust return type. +/// Returns a view for a given `offset`, performing compile-time bound checks. +// Always inline to optimize out error path of `build_assert`. +#[inline(always)] +fn io_view_assert<'a, IO: Io<'a>, U>( + this: IO, + offset: usize, +) -> <IO::Backend as IoBackend>::View<'a, U> { + // We cannot check alignment with `offset_valid` using `ptr.addr()`. So set 0 for it and + // ensure alignment by checking that the alignment of `U` is smaller or equal to the + // alignment of `IO::Target`. + const_assert!(Alignment::of::<U>().as_usize() <= IO::Target::MIN_ALIGN.as_usize()); + build_assert!(offset_valid::<U>(0, offset, IO::Target::MIN_SIZE)); + + let view = this.as_view(); + let ptr = IO::Backend::as_ptr(view); + let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset); + // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a + // valid projection. + unsafe { IO::Backend::project_view(view, projected_ptr) } +} + +/// Returns a view for a given `offset`, performing runtime bound checks. +#[inline] +fn io_view<'a, IO: Io<'a>, U>( + this: IO, + offset: usize, +) -> Result<<IO::Backend as IoBackend>::View<'a, U>> { + let view = this.as_view(); + let ptr = IO::Backend::as_ptr(view); + + if !offset_valid::<U>(ptr.addr(), offset, KnownSize::size(ptr)) { + return Err(EINVAL); + } + + let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset); + // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a + // valid projection. + Ok(unsafe { IO::Backend::project_view(view, projected_ptr) }) +} + +/// I/O backends. /// -/// # Parameters +/// This is an abstract representation to be implemented by arbitrary I/O +/// backends (e.g. MMIO, PCI config space, etc.). /// -/// * `$c_fn` – The C function performing the MMIO read. -/// * `$self` – The I/O backend object. -/// * `$ty` – The type of the value to be read. -/// * `$addr` – The MMIO address to read. +/// The base trait only defines the projection operations; which I/O methods are available depends +/// on which [`IoCapable<T>`] traits are implemented for the type. For example, for MMIO regions, +/// all widths (u8, u16, u32, and u64 on 64-bit systems) are typically supported. For PCI +/// configuration space, u8, u16, and u32 are supported but u64 is not. /// -/// This macro does not perform any validation; all invariants must be upheld by the higher-level -/// abstraction invoking it. -macro_rules! call_mmio_read { - (infallible, $c_fn:ident, $self:ident, $type:ty, $addr:expr) => { - // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. - unsafe { bindings::$c_fn($addr as *const c_void) as $type } - }; +/// This trait is separate from the `Io` trait as multiple different I/O types may share the same +/// operation. +pub trait IoBackend { + /// View type for this I/O backend. + type View<'a, T: ?Sized + KnownSize>: IoBase<'a, Backend = Self, Target = T>; - (fallible, $c_fn:ident, $self:ident, $type:ty, $addr:expr) => {{ - // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. - Ok(unsafe { bindings::$c_fn($addr as *const c_void) as $type }) - }}; + /// Convert a `view` to a raw pointer for projection. + /// + /// The returned pointer is private implementation detail of the backend; it is likely not + /// valid. It should not be dereferenced. + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T; + + /// Project `view` to its subregion indicated by `ptr`. + /// + /// If input `view` is valid, returned view must also be valid. + /// + /// # Safety + /// + /// `ptr` must be a projection of `Self::as_ptr(view)`. + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U>; } -/// Internal helper macros used to invoke C MMIO write functions. +/// Trait indicating that an I/O backend supports operations of a certain type and providing an +/// implementation for these operations. /// -/// This macro is intended to be used by higher-level MMIO access macros (io_define_write) and -/// provides a unified expansion for infallible vs. fallible write semantics. It emits a direct call -/// into the corresponding C helper and performs the required cast to the Rust return type. -/// -/// # Parameters -/// -/// * `$c_fn` – The C function performing the MMIO write. -/// * `$self` – The I/O backend object. -/// * `$ty` – The type of the written value. -/// * `$addr` – The MMIO address to write. -/// * `$value` – The value to write. +/// Different I/O backends can implement this trait to expose only the operations they support. /// -/// This macro does not perform any validation; all invariants must be upheld by the higher-level -/// abstraction invoking it. -macro_rules! call_mmio_write { - (infallible, $c_fn:ident, $self:ident, $ty:ty, $addr:expr, $value:expr) => { - // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. - unsafe { bindings::$c_fn($value, $addr as *mut c_void) } - }; +/// For example, a PCI configuration space may implement `IoCapable<u8>`, `IoCapable<u16>`, +/// and `IoCapable<u32>`, but not `IoCapable<u64>`, while an MMIO region on a 64-bit +/// system might implement all four. +pub trait IoCapable<T>: IoBackend { + /// Performs an I/O read of type `T` at `view` and returns the result. + fn io_read<'a>(view: Self::View<'a, T>) -> T; - (fallible, $c_fn:ident, $self:ident, $ty:ty, $addr:expr, $value:expr) => {{ - // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. - unsafe { bindings::$c_fn($value, $addr as *mut c_void) }; - Ok(()) - }}; + /// Performs an I/O write of `value` at `view`. + fn io_write<'a>(view: Self::View<'a, T>, value: T); } -/// Generates an accessor method for reading from an I/O backend. -/// -/// This macro reduces boilerplate by automatically generating either compile-time bounds-checked -/// (infallible) or runtime bounds-checked (fallible) read methods. It abstracts the address -/// calculation and bounds checking, and delegates the actual I/O read operation to a specified -/// helper macro, making it generic over different I/O backends. -/// -/// # Parameters -/// -/// * `infallible` / `fallible` - Determines the bounds-checking strategy. `infallible` relies on -/// `IoKnownSize` for compile-time checks and returns the value directly. `fallible` performs -/// runtime checks against `maxsize()` and returns a `Result<T>`. -/// * `$(#[$attr:meta])*` - Optional attributes to apply to the generated method (e.g., -/// `#[cfg(CONFIG_64BIT)]` or inline directives). -/// * `$vis:vis` - The visibility of the generated method (e.g., `pub`). -/// * `$name:ident` / `$try_name:ident` - The name of the generated method (e.g., `read32`, -/// `try_read8`). -/// * `$call_macro:ident` - The backend-specific helper macro used to emit the actual I/O call -/// (e.g., `call_mmio_read`). -/// * `$c_fn:ident` - The backend-specific C function or identifier to be passed into the -/// `$call_macro`. -/// * `$type_name:ty` - The Rust type of the value being read (e.g., `u8`, `u32`). -#[macro_export] -macro_rules! io_define_read { - (infallible, $(#[$attr:meta])* $vis:vis $name:ident, $call_macro:ident($c_fn:ident) -> - $type_name:ty) => { - /// Read IO data from a given offset known at compile time. - /// - /// Bound checks are performed on compile time, hence if the offset is not known at compile - /// time, the build will fail. - $(#[$attr])* - // Always inline to optimize out error path of `io_addr_assert`. - #[inline(always)] - $vis fn $name(&self, offset: usize) -> $type_name { - let addr = self.io_addr_assert::<$type_name>(offset); - - // SAFETY: By the type invariant `addr` is a valid address for IO operations. - $call_macro!(infallible, $c_fn, self, $type_name, addr) - } - }; +/// Trait indicating that an I/O backend supports memory copy operations. +pub trait IoCopyable: IoBackend { + /// Copy contents of `view` to `buffer`. + /// + /// # Safety + /// + /// - `buffer` is valid for volatile write for `view.size()` bytes. + /// - `buffer` should not overlap with `view`. + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8); - (fallible, $(#[$attr:meta])* $vis:vis $try_name:ident, $call_macro:ident($c_fn:ident) -> - $type_name:ty) => { - /// Read IO data from a given offset. - /// - /// Bound checks are performed on runtime, it fails if the offset (plus the type size) is - /// out of bounds. - $(#[$attr])* - $vis fn $try_name(&self, offset: usize) -> Result<$type_name> { - let addr = self.io_addr::<$type_name>(offset)?; - - // SAFETY: By the type invariant `addr` is a valid address for IO operations. - $call_macro!(fallible, $c_fn, self, $type_name, addr) - } - }; -} -pub use io_define_read; - -/// Generates an accessor method for writing to an I/O backend. -/// -/// This macro reduces boilerplate by automatically generating either compile-time bounds-checked -/// (infallible) or runtime bounds-checked (fallible) write methods. It abstracts the address -/// calculation and bounds checking, and delegates the actual I/O write operation to a specified -/// helper macro, making it generic over different I/O backends. -/// -/// # Parameters -/// -/// * `infallible` / `fallible` - Determines the bounds-checking strategy. `infallible` relies on -/// `IoKnownSize` for compile-time checks and returns `()`. `fallible` performs runtime checks -/// against `maxsize()` and returns a `Result`. -/// * `$(#[$attr:meta])*` - Optional attributes to apply to the generated method (e.g., -/// `#[cfg(CONFIG_64BIT)]` or inline directives). -/// * `$vis:vis` - The visibility of the generated method (e.g., `pub`). -/// * `$name:ident` / `$try_name:ident` - The name of the generated method (e.g., `write32`, -/// `try_write8`). -/// * `$call_macro:ident` - The backend-specific helper macro used to emit the actual I/O call -/// (e.g., `call_mmio_write`). -/// * `$c_fn:ident` - The backend-specific C function or identifier to be passed into the -/// `$call_macro`. -/// * `$type_name:ty` - The Rust type of the value being written (e.g., `u8`, `u32`). Note the use -/// of `<-` before the type to denote a write operation. -#[macro_export] -macro_rules! io_define_write { - (infallible, $(#[$attr:meta])* $vis:vis $name:ident, $call_macro:ident($c_fn:ident) <- - $type_name:ty) => { - /// Write IO data from a given offset known at compile time. - /// - /// Bound checks are performed on compile time, hence if the offset is not known at compile - /// time, the build will fail. - $(#[$attr])* - // Always inline to optimize out error path of `io_addr_assert`. - #[inline(always)] - $vis fn $name(&self, value: $type_name, offset: usize) { - let addr = self.io_addr_assert::<$type_name>(offset); - - $call_macro!(infallible, $c_fn, self, $type_name, addr, value); - } - }; + /// Copy contents from `buffer` to `view`. + /// + /// # Safety + /// + /// - `buffer` is valid for volatile read for `view.size()` bytes. + /// - `buffer` should not overlap with `view`. + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8); - (fallible, $(#[$attr:meta])* $vis:vis $try_name:ident, $call_macro:ident($c_fn:ident) <- - $type_name:ty) => { - /// Write IO data from a given offset. - /// - /// Bound checks are performed on runtime, it fails if the offset (plus the type size) is - /// out of bounds. - $(#[$attr])* - $vis fn $try_name(&self, value: $type_name, offset: usize) -> Result { - let addr = self.io_addr::<$type_name>(offset)?; - - $call_macro!(fallible, $c_fn, self, $type_name, addr, value) - } - }; -} -pub use io_define_write; + /// Copy from `view` and return the value. + #[inline] + fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T { + // Project `self` to `[u8]`. + let ptr = Self::as_ptr(view); + // SAFETY: This is a identity projection. + let slice_view = unsafe { + Self::project_view( + view, + core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()), + ) + }; -/// Checks whether an access of type `U` at the given `offset` -/// is valid within this region. -#[inline] -const fn offset_valid<U>(offset: usize, size: usize) -> bool { - let type_size = core::mem::size_of::<U>(); - if let Some(end) = offset.checked_add(type_size) { - end <= size && offset % type_size == 0 - } else { - false + let mut buf = MaybeUninit::<T>::uninit(); + // SAFETY: + // - `buf.as_mut_ptr()` is valid for write for `size_of::<T>()` bytes. + // - `buf` is local so `buf.as_mut_ptr()` cannot overlap with `slice_view`. + unsafe { Self::copy_from_io(slice_view, buf.as_mut_ptr().cast()) }; + // SAFETY: `T: FromBytes` guarantee that all bit patterns are valid. + unsafe { buf.assume_init() } + } + + /// Copy `value` to `view`. + /// + /// Destructor of `value` will not be executed, consistent with [`zerocopy::transmute`]. + #[inline] + fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) { + // Project `self` to `[u8]`. + let ptr = Self::as_ptr(view); + // SAFETY: This is a identity projection. + let slice_view = unsafe { + Self::project_view( + view, + core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()), + ) + }; + + // SAFETY: + // - `&raw const value` is valid for read for `size_of::<T>()` bytes. + // - `value` is local so `&raw const value` cannot overlap with `slice_view`. + unsafe { Self::copy_to_io(slice_view, (&raw const value).cast()) }; + core::mem::forget(value); } } -/// Marker trait indicating that an I/O backend supports operations of a certain type. +/// Describes a given I/O location: its offset, width, and type to convert the raw value from and +/// into. /// -/// Different I/O backends can implement this trait to expose only the operations they support. +/// This trait is the key abstraction allowing [`Io::read`], [`Io::write`], and [`Io::update`] (and +/// their fallible [`try_read`](Io::try_read), [`try_write`](Io::try_write) and +/// [`try_update`](Io::try_update) counterparts) to work uniformly with both raw [`usize`] offsets +/// (for primitive types like [`u32`]) and typed ones (like those generated by the [`register!`] +/// macro). /// -/// For example, a PCI configuration space may implement `IoCapable<u8>`, `IoCapable<u16>`, -/// and `IoCapable<u32>`, but not `IoCapable<u64>`, while an MMIO region on a 64-bit -/// system might implement all four. -pub trait IoCapable<T> {} +/// An `IoLoc<Base, T>` carries the following pieces of information: +/// +/// - The valid `Base` to operate on. For most registers, this should be [`Region`]. +/// - The offset to access (returned by [`IoLoc::offset`]), +/// - The width of the access (determined by [`IoLoc::IoType`]), +/// - The type `T` in which the raw data is returned or provided. +/// +/// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type +/// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`). +pub trait IoLoc<Base: ?Sized, T> { + /// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset). + type IoType: Into<T> + From<T>; + + /// Consumes `self` and returns the offset of this location. + fn offset(self) -> usize; +} + +/// Implements [`IoLoc<Region<SIZE>, $ty>`] for [`usize`], allowing [`usize`] to be used as a +/// parameter of [`Io::read`] and [`Io::write`]. +macro_rules! impl_usize_ioloc { + ($($ty:ty),*) => { + $( + impl<const SIZE: usize> IoLoc<Region<SIZE>, $ty> for usize { + type IoType = $ty; + + #[inline(always)] + fn offset(self) -> usize { + self + } + } + )* + } +} + +// Provide the ability to read any primitive type from a [`usize`]. +impl_usize_ioloc!(u8, u16, u32, u64); /// Types implementing this trait (e.g. MMIO BARs or PCI config regions) /// can perform I/O operations on regions of memory. /// -/// This is an abstract representation to be implemented by arbitrary I/O -/// backends (e.g. MMIO, PCI config space, etc.). +/// This trait defines which backend shall be used for I/O operations and provides a method to +/// convert into [`IoBackend::View`]. Users should use the [`Io`] trait which provides the actual +/// methods to perform I/O operations. +/// +/// This should be implemented on cheaply copyable handles, such as references or view types. +pub trait IoBase<'a>: Copy { + /// Type that defines all I/O operations. + type Backend: IoBackend; + + /// Type of this I/O region. For untyped regions, [`Region`] can be used. + type Target: ?Sized + KnownSize; + + /// Return a view that covers the full region. + fn as_view(self) -> <Self::Backend as IoBackend>::View<'a, Self::Target>; +} + +/// Extension trait to provide I/O operation methods to types that implement [`IoBase`]. /// -/// The [`Io`] trait provides: -/// - Base address and size information +/// This trait provides: /// - Helper methods for offset validation and address calculation /// - Fallible (runtime checked) accessors for different data widths /// -/// Which I/O methods are available depends on which [`IoCapable<T>`] traits -/// are implemented for the type. -/// -/// # Examples -/// -/// For MMIO regions, all widths (u8, u16, u32, and u64 on 64-bit systems) are typically -/// supported. For PCI configuration space, u8, u16, and u32 are supported but u64 is not. -pub trait Io { - /// Returns the base address of this mapping. - fn addr(&self) -> usize; +/// Which I/O methods are available depends on the associated [`IoBackend`] implementation. +pub trait Io<'a>: IoBase<'a> { + /// Returns the size of this I/O region. + #[inline] + fn size(self) -> usize { + KnownSize::size(Self::Backend::as_ptr(self.as_view())) + } - /// Returns the maximum size of this mapping. - fn maxsize(&self) -> usize; + /// Returns the length of the slice in number of elements. + #[inline] + fn len<T>(self) -> usize + where + Self: Io<'a, Target = [T]>, + { + Self::Backend::as_ptr(self.as_view()).len() + } + + /// Returns `true` if the slice has a length of 0. + #[inline] + fn is_empty<T>(self) -> bool + where + Self: Io<'a, Target = [T]>, + { + self.len() == 0 + } - /// Returns the absolute I/O address for a given `offset`, - /// performing runtime bound checks. + /// Try to convert into a different typed I/O view. + /// + /// A runtime check is performed to ensure that the target type is of same or smaller size to + /// current type, and the current view is properly aligned for the target type. Returns + /// `Err(EINVAL)` if the runtime check fails. + /// + /// # Examples + /// + /// ```no_run + /// use kernel::io::{ + /// io_project, + /// Mmio, + /// Io, + /// Region, + /// }; + /// #[derive(FromBytes, IntoBytes)] + /// #[repr(C)] + /// struct MyStruct { field: u32, } + /// + /// # fn test(mmio: &Mmio<'_, Region>) -> Result { + /// // let mmio: Mmio<'_, Region>; + /// let whole: Mmio<'_, MyStruct> = mmio.try_cast()?; + /// # Ok::<(), Error>(()) } + /// ``` #[inline] - fn io_addr<U>(&self, offset: usize) -> Result<usize> { - if !offset_valid::<U>(offset, self.maxsize()) { + fn try_cast<U>(self) -> Result<<Self::Backend as IoBackend>::View<'a, U>> + where + Self::Target: FromBytes + IntoBytes, + U: FromBytes + IntoBytes, + { + let view = self.as_view(); + let ptr = Self::Backend::as_ptr(view); + + if size_of::<U>() > KnownSize::size(ptr) { return Err(EINVAL); } - // Probably no need to check, since the safety requirements of `Self::new` guarantee that - // this can't overflow. - self.addr().checked_add(offset).ok_or(EINVAL) + if ptr.addr() % align_of::<U>() != 0 { + return Err(EINVAL); + } + + // SAFETY: We have checked bounds and alignment, so this is a valid projection. + Ok(unsafe { Self::Backend::project_view(view, ptr.cast()) }) + } + + /// Read a value from I/O. + /// + /// This only works for primitives supported by the I/O backend. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_read_val(mmio: Mmio<'_, u32>) { + /// // let mmio: Mmio<'_, u32>; + /// let val: u32 = mmio.read_val(); + /// # } + /// ``` + #[inline] + fn read_val(self) -> Self::Target + where + Self::Backend: IoCapable<Self::Target>, + Self::Target: Sized, + { + Self::Backend::io_read(self.as_view()) + } + + /// Write a value to I/O. + /// + /// This only works for primitives supported by the I/O backend. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_write_val(mmio: Mmio<'_, u32>) { + /// // let mmio: Mmio<'_, u32>; + /// mmio.write_val(1u32); + /// # } + /// ``` + #[inline] + fn write_val(self, value: Self::Target) + where + Self::Backend: IoCapable<Self::Target>, + Self::Target: Sized, + { + Self::Backend::io_write(self.as_view(), value) + } + + /// Copy-read from I/O memory. + /// + /// This is equivalent to reading from the I/O memory with byte-wise copy, although the actual + /// implementation might be more efficient. There is no atomicity guarantee. Note that for some + /// backends (e.g. `Mmio`), this can read different value compared to [`read_val`] as + /// byte-swapping is not performed. + /// + /// [`read_val`]: Io::read_val + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_read(mmio: Mmio<'_, [u8; 6]>) { + /// // let mmio: Mmio<'_, [u8; 6]>; + /// let val: [u8; 6] = mmio.copy_read(); + /// # } + /// ``` + #[inline] + fn copy_read(self) -> Self::Target + where + Self::Backend: IoCopyable, + Self::Target: Sized + FromBytes, + { + Self::Backend::copy_read(self.as_view()) + } + + /// Copy-write to I/O memory. + /// + /// This is equivalent to writing to the I/O memory with byte-wise copy, although the actual + /// implementation might be more efficient. There is no atomicity guarantee. Note that for some + /// backends (e.g. `Mmio`), this can write different value compared to [`write_val`] as + /// byte-swapping is not performed. + /// + /// [`write_val`]: Io::write_val + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8; 6]>) { + /// // let mmio: Mmio<'_, [u8; 6]>; + /// mmio.copy_write([0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]); + /// # } + /// ``` + #[inline] + fn copy_write(self, value: Self::Target) + where + Self::Backend: IoCopyable, + Self::Target: Sized + IntoBytes, + { + Self::Backend::copy_write(self.as_view(), value); + } + + /// Copy bytes from `data` to I/O memory. + /// + /// # Panics + /// + /// This function will panic if the length of `self` differs from the length of `data`, similar + /// to [`[u8]::copy_from_slice`]. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) { + /// // let mmio: Mmio<'_, [u8]>; + /// mmio.copy_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]); + /// # } + /// ``` + #[inline] + fn copy_from_slice(self, data: &[u8]) + where + Self::Backend: IoCopyable, + Self: Io<'a, Target = [u8]>, + { + assert_eq!(self.len(), data.len()); + + // SAFETY: `data.as_ptr()` is valid for read for `self.size()` bytes. + unsafe { + Self::Backend::copy_to_io(self.as_view(), data.as_ptr()); + } + } + + /// Copy bytes from I/O memory to `data`. + /// + /// # Panics + /// + /// This function will panic if the length of `self` differs from the length of `data`, similar + /// to [`[u8]::copy_from_slice`]. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) { + /// // let mmio: Mmio<'_, [u8]>; + /// let mut buf = [0; 6]; + /// mmio.copy_to_slice(&mut buf); + /// # } + /// ``` + #[inline] + fn copy_to_slice(self, data: &mut [u8]) + where + Self::Backend: IoCopyable, + Self: Io<'a, Target = [u8]>, + { + assert_eq!(self.len(), data.len()); + + // SAFETY: `data.as_mut_ptr()` is valid for write for `self.size()` bytes. + unsafe { + Self::Backend::copy_from_io(self.as_view(), data.as_mut_ptr()); + } } /// Fallible 8-bit read with runtime bounds check. #[inline(always)] - fn try_read8(&self, _offset: usize) -> Result<u8> + fn try_read8(self, offset: usize) -> Result<u8> where - Self: IoCapable<u8>, + usize: IoLoc<Self::Target, u8, IoType = u8>, + Self::Backend: IoCapable<u8>, { - build_error!("Backend does not support fallible 8-bit read") + self.try_read(offset) } /// Fallible 16-bit read with runtime bounds check. #[inline(always)] - fn try_read16(&self, _offset: usize) -> Result<u16> + fn try_read16(self, offset: usize) -> Result<u16> where - Self: IoCapable<u16>, + usize: IoLoc<Self::Target, u16, IoType = u16>, + Self::Backend: IoCapable<u16>, { - build_error!("Backend does not support fallible 16-bit read") + self.try_read(offset) } /// Fallible 32-bit read with runtime bounds check. #[inline(always)] - fn try_read32(&self, _offset: usize) -> Result<u32> + fn try_read32(self, offset: usize) -> Result<u32> where - Self: IoCapable<u32>, + usize: IoLoc<Self::Target, u32, IoType = u32>, + Self::Backend: IoCapable<u32>, { - build_error!("Backend does not support fallible 32-bit read") + self.try_read(offset) } /// Fallible 64-bit read with runtime bounds check. #[inline(always)] - fn try_read64(&self, _offset: usize) -> Result<u64> + fn try_read64(self, offset: usize) -> Result<u64> where - Self: IoCapable<u64>, + usize: IoLoc<Self::Target, u64, IoType = u64>, + Self::Backend: IoCapable<u64>, { - build_error!("Backend does not support fallible 64-bit read") + self.try_read(offset) } /// Fallible 8-bit write with runtime bounds check. #[inline(always)] - fn try_write8(&self, _value: u8, _offset: usize) -> Result + fn try_write8(self, value: u8, offset: usize) -> Result where - Self: IoCapable<u8>, + usize: IoLoc<Self::Target, u8, IoType = u8>, + Self::Backend: IoCapable<u8>, { - build_error!("Backend does not support fallible 8-bit write") + self.try_write(offset, value) } /// Fallible 16-bit write with runtime bounds check. #[inline(always)] - fn try_write16(&self, _value: u16, _offset: usize) -> Result + fn try_write16(self, value: u16, offset: usize) -> Result where - Self: IoCapable<u16>, + usize: IoLoc<Self::Target, u16, IoType = u16>, + Self::Backend: IoCapable<u16>, { - build_error!("Backend does not support fallible 16-bit write") + self.try_write(offset, value) } /// Fallible 32-bit write with runtime bounds check. #[inline(always)] - fn try_write32(&self, _value: u32, _offset: usize) -> Result + fn try_write32(self, value: u32, offset: usize) -> Result where - Self: IoCapable<u32>, + usize: IoLoc<Self::Target, u32, IoType = u32>, + Self::Backend: IoCapable<u32>, { - build_error!("Backend does not support fallible 32-bit write") + self.try_write(offset, value) } /// Fallible 64-bit write with runtime bounds check. #[inline(always)] - fn try_write64(&self, _value: u64, _offset: usize) -> Result + fn try_write64(self, value: u64, offset: usize) -> Result where - Self: IoCapable<u64>, + usize: IoLoc<Self::Target, u64, IoType = u64>, + Self::Backend: IoCapable<u64>, { - build_error!("Backend does not support fallible 64-bit write") + self.try_write(offset, value) } /// Infallible 8-bit read with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn read8(&self, _offset: usize) -> u8 + fn read8(self, offset: usize) -> u8 where - Self: IoKnownSize + IoCapable<u8>, + usize: IoLoc<Self::Target, u8, IoType = u8>, + Self::Backend: IoCapable<u8>, { - build_error!("Backend does not support infallible 8-bit read") + self.read(offset) } /// Infallible 16-bit read with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn read16(&self, _offset: usize) -> u16 + fn read16(self, offset: usize) -> u16 where - Self: IoKnownSize + IoCapable<u16>, + usize: IoLoc<Self::Target, u16, IoType = u16>, + Self::Backend: IoCapable<u16>, { - build_error!("Backend does not support infallible 16-bit read") + self.read(offset) } /// Infallible 32-bit read with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn read32(&self, _offset: usize) -> u32 + fn read32(self, offset: usize) -> u32 where - Self: IoKnownSize + IoCapable<u32>, + usize: IoLoc<Self::Target, u32, IoType = u32>, + Self::Backend: IoCapable<u32>, { - build_error!("Backend does not support infallible 32-bit read") + self.read(offset) } /// Infallible 64-bit read with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn read64(&self, _offset: usize) -> u64 + fn read64(self, offset: usize) -> u64 where - Self: IoKnownSize + IoCapable<u64>, + usize: IoLoc<Self::Target, u64, IoType = u64>, + Self::Backend: IoCapable<u64>, { - build_error!("Backend does not support infallible 64-bit read") + self.read(offset) } /// Infallible 8-bit write with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn write8(&self, _value: u8, _offset: usize) + fn write8(self, value: u8, offset: usize) where - Self: IoKnownSize + IoCapable<u8>, + usize: IoLoc<Self::Target, u8, IoType = u8>, + Self::Backend: IoCapable<u8>, { - build_error!("Backend does not support infallible 8-bit write") + self.write(offset, value) } /// Infallible 16-bit write with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn write16(&self, _value: u16, _offset: usize) + fn write16(self, value: u16, offset: usize) where - Self: IoKnownSize + IoCapable<u16>, + usize: IoLoc<Self::Target, u16, IoType = u16>, + Self::Backend: IoCapable<u16>, { - build_error!("Backend does not support infallible 16-bit write") + self.write(offset, value) } /// Infallible 32-bit write with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn write32(&self, _value: u32, _offset: usize) + fn write32(self, value: u32, offset: usize) where - Self: IoKnownSize + IoCapable<u32>, + usize: IoLoc<Self::Target, u32, IoType = u32>, + Self::Backend: IoCapable<u32>, { - build_error!("Backend does not support infallible 32-bit write") + self.write(offset, value) } /// Infallible 64-bit write with compile-time bounds check. + /// + /// `offset` should be constant. + #[inline(always)] + fn write64(self, value: u64, offset: usize) + where + usize: IoLoc<Self::Target, u64, IoType = u64>, + Self::Backend: IoCapable<u64>, + { + self.write(offset, value) + } + + /// Generic fallible read with runtime bounds check. + /// + /// # Examples + /// + /// Read a primitive type from an I/O address: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_reads(io: Mmio<'_, Region>) -> Result { + /// // 32-bit read from address `0x10`. + /// let v: u32 = io.try_read(0x10)?; + /// + /// // 8-bit read from address `0xfff`. + /// let v: u8 = io.try_read(0xfff)?; + /// + /// Ok(()) + /// } + /// ``` + #[inline(always)] + fn try_read<T, L>(self, location: L) -> Result<T> + where + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, + { + let view = io_view::<Self, L::IoType>(self, location.offset())?; + Ok(Self::Backend::io_read(view).into()) + } + + /// Generic fallible write with runtime bounds check. + /// + /// # Examples + /// + /// Write a primitive type to an I/O address: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_writes(io: Mmio<'_, Region>) -> Result { + /// // 32-bit write of value `1` at address `0x10`. + /// io.try_write(0x10, 1u32)?; + /// + /// // 8-bit write of value `0xff` at address `0xfff`. + /// io.try_write(0xfff, 0xffu8)?; + /// + /// Ok(()) + /// } + /// ``` + #[inline(always)] + fn try_write<T, L>(self, location: L, value: T) -> Result + where + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, + { + let view = io_view::<Self, L::IoType>(self, location.offset())?; + let io_value = value.into(); + Self::Backend::io_write(view, io_value); + Ok(()) + } + + /// Generic fallible write of a fully-located register value. + /// + /// # Examples + /// + /// Tuples carrying a location and a value can be used with this method: + /// + /// ```no_run + /// use kernel::io::{ + /// register, + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// register! { + /// VERSION(u32) @ 0x100 { + /// 15:8 major; + /// 7:0 minor; + /// } + /// } + /// + /// impl VERSION { + /// fn new(major: u8, minor: u8) -> Self { + /// VERSION::zeroed().with_major(major).with_minor(minor) + /// } + /// } + /// + /// fn do_write_reg(io: Mmio<'_, Region>) -> Result { + /// + /// io.try_write_reg(VERSION::new(1, 0)) + /// } + /// ``` + #[inline(always)] + fn try_write_reg<T, L, V>(self, value: V) -> Result + where + L: IoLoc<Self::Target, T>, + V: LocatedRegister<Self::Target, Location = L, Value = T>, + Self::Backend: IoCapable<L::IoType>, + { + let (location, value) = value.into_io_op(); + + self.try_write(location, value) + } + + /// Generic fallible update with runtime bounds check. + /// + /// Note: this does not perform any synchronization. The caller is responsible for ensuring + /// exclusive access if required. + /// + /// # Examples + /// + /// Read the u32 value at address `0x10`, increment it, and store the updated value back: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_update(io: Mmio<'_, Region<0x1000>>) -> Result { + /// io.try_update(0x10, |v: u32| { + /// v + 1 + /// }) + /// } + /// ``` + #[inline(always)] + fn try_update<T, L, F>(self, location: L, f: F) -> Result + where + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, + F: FnOnce(T) -> T, + { + let view = io_view::<Self, L::IoType>(self, location.offset())?; + + let value: T = Self::Backend::io_read(view).into(); + let io_value = f(value).into(); + Self::Backend::io_write(view, io_value); + + Ok(()) + } + + /// Generic infallible read with compile-time bounds check. + /// + /// # Examples + /// + /// Read a primitive type from an I/O address: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_reads(io: Mmio<'_, Region<0x1000>>) { + /// // 32-bit read from address `0x10`. + /// let v: u32 = io.read(0x10); + /// + /// // 8-bit read from the top of the I/O space. + /// let v: u8 = io.read(0xfff); + /// } + /// ``` + #[inline(always)] + fn read<T, L>(self, location: L) -> T + where + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, + { + let view = io_view_assert::<Self, L::IoType>(self, location.offset()); + Self::Backend::io_read(view).into() + } + + /// Generic infallible write with compile-time bounds check. + /// + /// # Examples + /// + /// Write a primitive type to an I/O address: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_writes(io: Mmio<'_, Region<0x1000>>) { + /// // 32-bit write of value `1` at address `0x10`. + /// io.write(0x10, 1u32); + /// + /// // 8-bit write of value `0xff` at the top of the I/O space. + /// io.write(0xfff, 0xffu8); + /// } + /// ``` #[inline(always)] - fn write64(&self, _value: u64, _offset: usize) + fn write<T, L>(self, location: L, value: T) where - Self: IoKnownSize + IoCapable<u64>, + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, { - build_error!("Backend does not support infallible 64-bit write") + let view = io_view_assert::<Self, L::IoType>(self, location.offset()); + let io_value = value.into(); + Self::Backend::io_write(view, io_value); + } + + /// Generic infallible write of a fully-located register value. + /// + /// # Examples + /// + /// Tuples carrying a location and a value can be used with this method: + /// + /// ```no_run + /// use kernel::io::{ + /// register, + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// register! { + /// VERSION(u32) @ 0x100 { + /// 15:8 major; + /// 7:0 minor; + /// } + /// } + /// + /// impl VERSION { + /// fn new(major: u8, minor: u8) -> Self { + /// VERSION::zeroed().with_major(major).with_minor(minor) + /// } + /// } + /// + /// fn do_write_reg(io: Mmio<'_, Region<0x1000>>) { + /// io.write_reg(VERSION::new(1, 0)); + /// } + /// ``` + #[inline(always)] + fn write_reg<T, L, V>(self, value: V) + where + L: IoLoc<Self::Target, T>, + V: LocatedRegister<Self::Target, Location = L, Value = T>, + Self::Backend: IoCapable<L::IoType>, + { + let (location, value) = value.into_io_op(); + + self.write(location, value) + } + + /// Generic infallible update with compile-time bounds check. + /// + /// Note: this does not perform any synchronization. The caller is responsible for ensuring + /// exclusive access if required. + /// + /// # Examples + /// + /// Read the u32 value at address `0x10`, increment it, and store the updated value back: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// }; + /// + /// fn do_update(io: Mmio<'_, Region<0x1000>>) { + /// io.update(0x10, |v: u32| { + /// v + 1 + /// }) + /// } + /// ``` + #[inline(always)] + fn update<T, L, F>(self, location: L, f: F) + where + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, + F: FnOnce(T) -> T, + { + let view = io_view_assert::<Self, L::IoType>(self, location.offset()); + let value: T = Self::Backend::io_read(view).into(); + let io_value = f(value).into(); + Self::Backend::io_write(view, io_value); } } -/// Trait for types with a known size at compile time. +// Blanket implementation ensures that provided methods cannot be arbitrarily overridden by +// implementers, which is relied upon for correctness and soundness. +impl<'a, T: IoBase<'a>> Io<'a> for T {} + +/// A view of memory-mapped I/O region. /// -/// This trait is implemented by I/O backends that have a compile-time known size, -/// enabling the use of infallible I/O accessors with compile-time bounds checking. +/// # Invariant /// -/// Types implementing this trait can use the infallible methods in [`Io`] trait -/// (e.g., `read8`, `write32`), which require `Self: IoKnownSize` bound. -pub trait IoKnownSize: Io { - /// Minimum usable size of this region. - const MIN_SIZE: usize; +/// `ptr` points to a valid and aligned memory-mapped I/O region for the duration lifetime `'a`. +pub struct Mmio<'a, T: ?Sized> { + ptr: *mut T, + phantom: PhantomData<&'a ()>, +} - /// Returns the absolute I/O address for a given `offset`, - /// performing compile-time bound checks. - // Always inline to optimize out error path of `build_assert`. - #[inline(always)] - fn io_addr_assert<U>(&self, offset: usize) -> usize { - build_assert!(offset_valid::<U>(offset, Self::MIN_SIZE)); +impl<T: ?Sized> Copy for Mmio<'_, T> {} +impl<T: ?Sized> Clone for Mmio<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> Mmio<'a, T> { + /// Create a `Mmio`, providing the accessors to the MMIO mapping. + /// + /// # Safety + /// + /// `raw` represents a valid and aligned memory-mapped I/O region while `'a` is alive. + #[inline] + pub unsafe fn from_raw(raw: MmioRaw<T>) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr: raw.ptr, + phantom: PhantomData, + } + } +} + +// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl<T: ?Sized + Sync> Send for Mmio<'_, T> {} + +// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl<T: ?Sized + Sync> Sync for Mmio<'_, T> {} - self.addr() + offset +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for Mmio<'a, T> { + type Backend = MmioBackend; + type Target = T; + + #[inline] + fn as_view(self) -> Mmio<'a, T> { + self + } +} + +/// I/O Backend for memory-mapped I/O. +pub struct MmioBackend; + +impl IoBackend for MmioBackend { + type View<'a, T: ?Sized + KnownSize> = Mmio<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + _view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid + // memory-mapped I/O region. + Mmio { + ptr, + phantom: PhantomData, + } } } +/// Implements [`IoCapable`] on `$backend` for `$ty` using `$read_fn` and `$write_fn`. +macro_rules! impl_mmio_io_capable { + ($backend: ident, $ty:ty, $read_fn:ident, $write_fn:ident) => { + impl IoCapable<$ty> for $backend { + #[inline] + fn io_read(view: <$backend as IoBackend>::View<'_, $ty>) -> $ty { + // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both + // `MmioBackend` and `RelaxedMmioBackend`. + unsafe { bindings::$read_fn($backend::as_ptr(view).cast_const().cast()) } + } + + #[inline] + fn io_write(view: <$backend as IoBackend>::View<'_, $ty>, value: $ty) { + // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both + // `MmioBackend` and `RelaxedMmioBackend`. + unsafe { bindings::$write_fn(value, $backend::as_ptr(view).cast()) } + } + } + }; +} + // MMIO regions support 8, 16, and 32-bit accesses. -impl<const SIZE: usize> IoCapable<u8> for Mmio<SIZE> {} -impl<const SIZE: usize> IoCapable<u16> for Mmio<SIZE> {} -impl<const SIZE: usize> IoCapable<u32> for Mmio<SIZE> {} +impl_mmio_io_capable!(MmioBackend, u8, readb, writeb); +impl_mmio_io_capable!(MmioBackend, u16, readw, writew); +impl_mmio_io_capable!(MmioBackend, u32, readl, writel); +// MMIO regions on 64-bit systems also support 64-bit accesses. +#[cfg(CONFIG_64BIT)] +impl_mmio_io_capable!(MmioBackend, u64, readq, writeq); + +impl IoCopyable for MmioBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // SAFETY: + // - `view.ptr` is valid MMIO memory for `view.size()` bytes. + // - `buffer` is valid for write for `view.size()` bytes. + unsafe { + bindings::memcpy_fromio(buffer.cast(), view.ptr.cast(), view.size()); + } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // SAFETY: + // - `view.ptr` is valid MMIO memory for `view.size()` bytes. + // - `buffer` is valid for read for `view.size()` bytes. + unsafe { + bindings::memcpy_toio(view.ptr.cast(), buffer.cast(), view.size()); + } + } +} + +/// [`Mmio`] but using relaxed accessors. +/// +/// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO operands instead of +/// the regular ones. +/// +/// See [`Mmio::relaxed`] for a usage example. +pub struct RelaxedMmio<'a, T: ?Sized>(Mmio<'a, T>); + +impl<T: ?Sized> Copy for RelaxedMmio<'_, T> {} +impl<T: ?Sized> Clone for RelaxedMmio<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +/// I/O Backend for memory-mapped I/O, with relaxed access semantics. +pub struct RelaxedMmioBackend; + +impl IoBackend for RelaxedMmioBackend { + type View<'a, T: ?Sized + KnownSize> = RelaxedMmio<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + MmioBackend::as_ptr(view.0) + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // SAFETY: Per safety requirement. + RelaxedMmio(unsafe { MmioBackend::project_view(view.0, ptr) }) + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for RelaxedMmio<'a, T> { + type Backend = RelaxedMmioBackend; + type Target = T; + + #[inline] + fn as_view(self) -> RelaxedMmio<'a, T> { + self + } +} +impl<'a, T: ?Sized> Mmio<'a, T> { + /// Returns a [`RelaxedMmio`] that performs relaxed I/O operations. + /// + /// Relaxed accessors do not provide ordering guarantees with respect to DMA or memory accesses + /// and can be used when such ordering is not required. + /// + /// # Examples + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// Region, + /// RelaxedMmio, + /// }; + /// + /// fn do_io(io: Mmio<'_, Region<0x100>>) { + /// // The access is performed using `readl_relaxed` instead of `readl`. + /// let v = io.relaxed().read32(0x10); + /// } + /// + /// ``` + #[inline] + pub fn relaxed(self) -> RelaxedMmio<'a, T> { + RelaxedMmio(self) + } +} + +// MMIO regions support 8, 16, and 32-bit accesses. +impl_mmio_io_capable!(RelaxedMmioBackend, u8, readb_relaxed, writeb_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u16, readw_relaxed, writew_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u32, readl_relaxed, writel_relaxed); // MMIO regions on 64-bit systems also support 64-bit accesses. #[cfg(CONFIG_64BIT)] -impl<const SIZE: usize> IoCapable<u64> for Mmio<SIZE> {} - -impl<const SIZE: usize> Io for Mmio<SIZE> { - /// Returns the base address of this mapping. - #[inline] - fn addr(&self) -> usize { - self.0.addr() - } - - /// Returns the maximum size of this mapping. - #[inline] - fn maxsize(&self) -> usize { - self.0.maxsize() - } - - io_define_read!(fallible, try_read8, call_mmio_read(readb) -> u8); - io_define_read!(fallible, try_read16, call_mmio_read(readw) -> u16); - io_define_read!(fallible, try_read32, call_mmio_read(readl) -> u32); - io_define_read!( - fallible, - #[cfg(CONFIG_64BIT)] - try_read64, - call_mmio_read(readq) -> u64 - ); - - io_define_write!(fallible, try_write8, call_mmio_write(writeb) <- u8); - io_define_write!(fallible, try_write16, call_mmio_write(writew) <- u16); - io_define_write!(fallible, try_write32, call_mmio_write(writel) <- u32); - io_define_write!( - fallible, - #[cfg(CONFIG_64BIT)] - try_write64, - call_mmio_write(writeq) <- u64 - ); - - io_define_read!(infallible, read8, call_mmio_read(readb) -> u8); - io_define_read!(infallible, read16, call_mmio_read(readw) -> u16); - io_define_read!(infallible, read32, call_mmio_read(readl) -> u32); - io_define_read!( - infallible, - #[cfg(CONFIG_64BIT)] - read64, - call_mmio_read(readq) -> u64 - ); - - io_define_write!(infallible, write8, call_mmio_write(writeb) <- u8); - io_define_write!(infallible, write16, call_mmio_write(writew) <- u16); - io_define_write!(infallible, write32, call_mmio_write(writel) <- u32); - io_define_write!( - infallible, - #[cfg(CONFIG_64BIT)] - write64, - call_mmio_write(writeq) <- u64 - ); -} - -impl<const SIZE: usize> IoKnownSize for Mmio<SIZE> { - const MIN_SIZE: usize = SIZE; +impl_mmio_io_capable!(RelaxedMmioBackend, u64, readq_relaxed, writeq_relaxed); + +/// I/O Backend for system memory. +pub struct SysMemBackend; + +impl IoBackend for SysMemBackend { + type View<'a, T: ?Sized + KnownSize> = SysMem<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + _view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid + // kernel accessible memory region. + SysMem { + ptr, + phantom: PhantomData, + } + } +} + +/// Implements [`IoCapable`] on `SysMemBackend` for `$ty` using `read_volatile` and +/// `write_volatile`. +macro_rules! impl_sysmem_io_capable { + ($ty:ty) => { + impl IoCapable<$ty> for SysMemBackend { + #[inline] + fn io_read(view: SysMem<'_, $ty>) -> $ty { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using read_volatile() here so that race with hardware is well-defined. + // - Using read_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + // - The macro is only used on primitives so all bit patterns are valid. + unsafe { view.ptr.read_volatile() } + } + + #[inline] + fn io_write(view: SysMem<'_, $ty>, value: $ty) { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using write_volatile() here so that race with hardware is well-defined. + // - Using write_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + unsafe { view.ptr.write_volatile(value) } + } + } + }; +} + +impl_sysmem_io_capable!(u8); +impl_sysmem_io_capable!(u16); +impl_sysmem_io_capable!(u32); +#[cfg(CONFIG_64BIT)] +impl_sysmem_io_capable!(u64); + +impl IoCopyable for SysMemBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile. + // SAFETY: + // - `view.ptr` is in CPU address space and valid for read. + // - `buffer` is valid for write for `view.size()` bytes which is equal to `view.ptr.len()`. + unsafe { bindings::memcpy(buffer.cast(), view.ptr.cast(), view.ptr.len()) }; + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile. + // SAFETY: + // - `view.ptr` is in CPU address space and valid for write. + // - `buffer` is valid for read for `view.size()` bytes which is equal to `view.ptr.len()`. + unsafe { bindings::memcpy(view.ptr.cast(), buffer.cast(), view.ptr.len()) }; + } + + #[inline] + fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using read_volatile() here so that race with hardware is well-defined. + // - Using read_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + // - `T: FromBytes` so all bit patterns are valid. + unsafe { view.ptr.read_volatile() } + } + + #[inline] + fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using write_volatile() here so that race with hardware is well-defined. + // - Using write_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + unsafe { view.ptr.write_volatile(value) } + } +} + +/// A view of a system memory region. +/// +/// Provides `Io` trait implementation for kernel virtual address ranges, +/// using volatile read/write to safely access shared memory that may be +/// concurrently accessed by external hardware. +/// +/// # Invariants +/// +/// `self.ptr.addr() .. self.ptr.addr() + KnownSize::size(self.ptr)` is valid and aligned kernel +/// accessible memory region for the lifetime `'a`. +pub struct SysMem<'a, T: ?Sized> { + ptr: *mut T, + phantom: PhantomData<&'a ()>, +} + +impl<T: ?Sized> Copy for SysMem<'_, T> {} +impl<T: ?Sized> Clone for SysMem<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } } -impl<const SIZE: usize> Mmio<SIZE> { - /// Converts an `MmioRaw` into an `Mmio` instance, providing the accessors to the MMIO mapping. +// SAFETY: `SysMem<'_, T>` is conceptually `&T`. +unsafe impl<T: ?Sized + Sync> Send for SysMem<'_, T> {} + +// SAFETY: `SysMem<'_, T>` is conceptually `&T`. +unsafe impl<T: ?Sized + Sync> Sync for SysMem<'_, T> {} + +impl<'a, T: ?Sized> SysMem<'a, T> { + /// Create a `SysMem` from a raw pointer. /// /// # Safety /// - /// Callers must ensure that `addr` is the start of a valid I/O mapped memory region of size - /// `maxsize`. - pub unsafe fn from_raw(raw: &MmioRaw<SIZE>) -> &Self { - // SAFETY: `Mmio` is a transparent wrapper around `MmioRaw`. - unsafe { &*core::ptr::from_ref(raw).cast() } - } - - io_define_read!(infallible, pub read8_relaxed, call_mmio_read(readb_relaxed) -> u8); - io_define_read!(infallible, pub read16_relaxed, call_mmio_read(readw_relaxed) -> u16); - io_define_read!(infallible, pub read32_relaxed, call_mmio_read(readl_relaxed) -> u32); - io_define_read!( - infallible, - #[cfg(CONFIG_64BIT)] - pub read64_relaxed, - call_mmio_read(readq_relaxed) -> u64 - ); - - io_define_read!(fallible, pub try_read8_relaxed, call_mmio_read(readb_relaxed) -> u8); - io_define_read!(fallible, pub try_read16_relaxed, call_mmio_read(readw_relaxed) -> u16); - io_define_read!(fallible, pub try_read32_relaxed, call_mmio_read(readl_relaxed) -> u32); - io_define_read!( - fallible, - #[cfg(CONFIG_64BIT)] - pub try_read64_relaxed, - call_mmio_read(readq_relaxed) -> u64 - ); - - io_define_write!(infallible, pub write8_relaxed, call_mmio_write(writeb_relaxed) <- u8); - io_define_write!(infallible, pub write16_relaxed, call_mmio_write(writew_relaxed) <- u16); - io_define_write!(infallible, pub write32_relaxed, call_mmio_write(writel_relaxed) <- u32); - io_define_write!( - infallible, - #[cfg(CONFIG_64BIT)] - pub write64_relaxed, - call_mmio_write(writeq_relaxed) <- u64 - ); - - io_define_write!(fallible, pub try_write8_relaxed, call_mmio_write(writeb_relaxed) <- u8); - io_define_write!(fallible, pub try_write16_relaxed, call_mmio_write(writew_relaxed) <- u16); - io_define_write!(fallible, pub try_write32_relaxed, call_mmio_write(writel_relaxed) <- u32); - io_define_write!( - fallible, - #[cfg(CONFIG_64BIT)] - pub try_write64_relaxed, - call_mmio_write(writeq_relaxed) <- u64 - ); + /// `ptr.addr() .. ptr.addr() + KnownSize::size(ptr)` must be valid and aligned kernel + /// accessible memory region for the lifetime `'a`. + #[inline] + pub unsafe fn new(ptr: *mut T) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr, + phantom: PhantomData, + } + } + + /// Obtain the raw pointer to the memory. + #[inline] + pub fn as_ptr(self) -> *mut T { + self.ptr + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for SysMem<'a, T> { + type Backend = SysMemBackend; + type Target = T; + + #[inline] + fn as_view(self) -> <Self::Backend as IoBackend>::View<'a, Self::Target> { + self + } +} + +/// I/O Backend for [`IoSysMap`]. +pub struct IoSysMapBackend; + +/// Either [`Mmio`] or [`SysMem`]. +/// +/// This can be used when a piece of logic may wish to handle both MMIO or system memory but does +/// not want or cannot be generic over I/O backends. This serves a similar purpose to +/// [`include/linux/iosys-map.h`] in C. +/// +/// This type can be used like any other types that implements [`Io`]; this also include +/// [`io_project!`], [`io_read!`], [`io_write!`]. +/// +/// [`include/linux/iosys-map.h`]: srctree/include/linux/iosys-map.h +pub enum IoSysMap<'a, T: ?Sized> { + /// The view is I/O memory. + Io(Mmio<'a, T>), + /// The view is system memory. + Sys(SysMem<'a, T>), +} + +impl<T: ?Sized> Copy for IoSysMap<'_, T> {} +impl<T: ?Sized> Clone for IoSysMap<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> From<Mmio<'a, T>> for IoSysMap<'a, T> { + #[inline] + fn from(value: Mmio<'a, T>) -> Self { + IoSysMap::Io(value) + } +} + +impl<'a, T: ?Sized> From<SysMem<'a, T>> for IoSysMap<'a, T> { + #[inline] + fn from(value: SysMem<'a, T>) -> Self { + IoSysMap::Sys(value) + } +} + +impl IoBackend for IoSysMapBackend { + type View<'a, T: ?Sized + KnownSize> = IoSysMap<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + match view { + IoSysMap::Io(l) => MmioBackend::as_ptr(l), + IoSysMap::Sys(r) => SysMemBackend::as_ptr(r), + } + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => IoSysMap::Io(unsafe { MmioBackend::project_view(l, ptr) }), + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => IoSysMap::Sys(unsafe { SysMemBackend::project_view(r, ptr) }), + } + } +} + +impl<T> IoCapable<T> for IoSysMapBackend +where + MmioBackend: IoCapable<T>, + SysMemBackend: IoCapable<T>, +{ + #[inline] + fn io_read(view: Self::View<'_, T>) -> T { + match view { + IoSysMap::Io(l) => MmioBackend::io_read(l), + IoSysMap::Sys(r) => SysMemBackend::io_read(r), + } + } + + #[inline] + fn io_write<'a>(view: Self::View<'a, T>, value: T) { + match view { + IoSysMap::Io(l) => MmioBackend::io_write(l, value), + IoSysMap::Sys(r) => SysMemBackend::io_write(r, value), + } + } +} + +impl IoCopyable for IoSysMapBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => unsafe { MmioBackend::copy_from_io(l, buffer) }, + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_from_io(r, buffer) }, + } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => unsafe { MmioBackend::copy_to_io(l, buffer) }, + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_to_io(r, buffer) }, + } + } + + #[inline] + fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T { + match view { + IoSysMap::Io(l) => MmioBackend::copy_read(l), + IoSysMap::Sys(r) => SysMemBackend::copy_read(r), + } + } + + #[inline] + fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) { + match view { + IoSysMap::Io(l) => MmioBackend::copy_write(l, value), + IoSysMap::Sys(r) => SysMemBackend::copy_write(r, value), + } + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for IoSysMap<'a, T> { + type Backend = IoSysMapBackend; + type Target = T; + + #[inline] + fn as_view(self) -> IoSysMap<'a, T> { + self + } +} + +// This helper turns associated functions to methods so it can be invoked in macro. +// Used by `io_project!()` only. +#[doc(hidden)] +#[derive(Clone, Copy)] +pub struct ProjectHelper<T>(pub T); + +impl<'a, T> ProjectHelper<T> +where + T: Io<'a, Backend: IoBackend<View<'a, T::Target> = T>>, +{ + // These helper methods must not have symbols present in the binary to avoid confusion. + #[inline(always)] + pub fn as_ptr(self) -> *mut T::Target { + T::Backend::as_ptr(self.0) + } + + /// # Safety + /// + /// Same as `IoBackend::project_view` + #[inline(always)] + pub unsafe fn project_view<U: ?Sized + KnownSize>( + self, + ptr: *mut U, + ) -> <T::Backend as IoBackend>::View<'a, U> { + // SAFETY: Per safety requirement. + unsafe { T::Backend::project_view::<T::Target, _>(self.0, ptr) } + } +} + +/// Project an I/O type to a subview of it. +/// +/// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). +/// +/// # Examples +/// +/// ``` +/// use kernel::io::{ +/// io_project, +/// Mmio, +/// }; +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<[MyStruct]>; +/// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field); +/// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]); +/// let nested: Mmio<'_, u32> = io_project!(whole, .field); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_project { + ($io:expr, $($proj:tt)*) => {{ + #[allow(unused)] + use $crate::io::IoBase as _; + let view = $crate::io::ProjectHelper($io.as_view()); + let ptr = $crate::ptr::project!( + mut view.as_ptr(), $($proj)* + ); + #[allow(unused_unsafe)] + // SAFETY: `ptr` is a projection. + unsafe { view.project_view(ptr) } + }}; +} +#[doc(inline)] +pub use crate::io_project; + +/// Read from I/O memory. +/// +/// The syntax is of form `io_read!(io, proj)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). +/// +/// # Examples +/// +/// ``` +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<'_, [MyStruct]>; +/// let field: u32 = kernel::io::io_read!(mmio, [try: 2].field); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_read { + ($io:expr, $($proj:tt)*) => { + $crate::io::Io::read_val($crate::io_project!($io, $($proj)*)) + }; +} +#[doc(inline)] +pub use crate::io_read; + +/// Writes to I/O memory. +/// +/// The syntax is of form `io_write!(io, proj, val)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!), +/// and `val` is the value to be written to the projected location. +/// +/// # Examples +/// +/// ``` +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<'_, [MyStruct]>; +/// kernel::io::io_write!(mmio, [try: 2].field, 10); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_write { + (@parse [$io:expr] [$($proj:tt)*] [, $val:expr]) => { + $crate::io::Io::write_val($crate::io_project!($io, $($proj)*), $val) + }; + (@parse [$io:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [$($proj)* .$field] [$($rest)*]) + }; + (@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*]) + }; + ($io:expr, $($rest:tt)*) => { + $crate::io_write!(@parse [$io] [] [$($rest)*]) + }; } +#[doc(inline)] +pub use crate::io_write; diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index 620022cff401..32a919099dcd 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -2,24 +2,28 @@ //! Generic memory-mapped IO. -use core::ops::Deref; - use crate::{ device::{ Bound, Device, // }, - devres::Devres, + devres::DevresLt, io::{ self, resource::{ Region, Resource, // }, + IoBase, Mmio, + MmioBackend, MmioRaw, // }, prelude::*, + types::{ + CovariantForLt, + ForLt, // + }, }; /// An IO request for a specific device and resource. @@ -54,6 +58,7 @@ impl<'a> IoRequest<'a> { /// use kernel::{ /// bindings, /// device::Core, + /// io::Io, /// of, /// platform, /// }; @@ -61,33 +66,31 @@ impl<'a> IoRequest<'a> { /// /// impl platform::Driver for SampleDriver { /// # type IdInfo = (); + /// # type Data<'bound> = Self; /// - /// fn probe( - /// pdev: &platform::Device<Core>, - /// info: Option<&Self::IdInfo>, - /// ) -> impl PinInit<Self, Error> { + /// fn probe<'bound>( + /// pdev: &'bound platform::Device<Core<'_>>, + /// info: Option<&'bound Self::IdInfo>, + /// ) -> impl PinInit<Self, Error> + 'bound { /// let offset = 0; // Some offset. /// /// // If the size is known at compile time, use [`Self::iomap_sized`]. /// // /// // No runtime checks will apply when reading and writing. /// let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; - /// let iomem = request.iomap_sized::<42>(); - /// let iomem = KBox::pin_init(iomem, GFP_KERNEL)?; - /// - /// let io = iomem.access(pdev.as_ref())?; + /// let iomem = request.iomap_sized::<42>()?; /// /// // Read and write a 32-bit value at `offset`. - /// let data = io.read32_relaxed(offset); + /// let data = iomem.read32(offset); /// - /// io.write32_relaxed(data, offset); + /// iomem.write32(data, offset); /// /// # Ok(SampleDriver) /// } /// } /// ``` - pub fn iomap_sized<const SIZE: usize>(self) -> impl PinInit<Devres<IoMem<SIZE>>, Error> + 'a { - IoMem::new(self) + pub fn iomap_sized<const SIZE: usize>(self) -> Result<IoMem<'a, SIZE>> { + IoMem::ioremap(self.device, self.resource) } /// Same as [`Self::iomap_sized`] but with exclusive access to the @@ -96,10 +99,8 @@ impl<'a> IoRequest<'a> { /// This uses the [`ioremap()`] C API. /// /// [`ioremap()`]: https://docs.kernel.org/driver-api/device-io.html#getting-access-to-the-device - pub fn iomap_exclusive_sized<const SIZE: usize>( - self, - ) -> impl PinInit<Devres<ExclusiveIoMem<SIZE>>, Error> + 'a { - ExclusiveIoMem::new(self) + pub fn iomap_exclusive_sized<const SIZE: usize>(self) -> Result<ExclusiveIoMem<'a, SIZE>> { + ExclusiveIoMem::ioremap(self.device, self.resource) } /// Maps an [`IoRequest`] where the size is not known at compile time, @@ -117,6 +118,7 @@ impl<'a> IoRequest<'a> { /// use kernel::{ /// bindings, /// device::Core, + /// io::Io, /// of, /// platform, /// }; @@ -124,11 +126,12 @@ impl<'a> IoRequest<'a> { /// /// impl platform::Driver for SampleDriver { /// # type IdInfo = (); + /// # type Data<'bound> = Self; /// - /// fn probe( - /// pdev: &platform::Device<Core>, - /// info: Option<&Self::IdInfo>, - /// ) -> impl PinInit<Self, Error> { + /// fn probe<'bound>( + /// pdev: &'bound platform::Device<Core<'_>>, + /// info: Option<&'bound Self::IdInfo>, + /// ) -> impl PinInit<Self, Error> + 'bound { /// let offset = 0; // Some offset. /// /// // Unlike [`Self::iomap_sized`], here the size of the memory region @@ -136,27 +139,24 @@ impl<'a> IoRequest<'a> { /// // family of functions should be used, leading to runtime checks on every /// // access. /// let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; - /// let iomem = request.iomap(); - /// let iomem = KBox::pin_init(iomem, GFP_KERNEL)?; - /// - /// let io = iomem.access(pdev.as_ref())?; + /// let iomem = request.iomap()?; /// - /// let data = io.try_read32_relaxed(offset)?; + /// let data = iomem.try_read32(offset)?; /// - /// io.try_write32_relaxed(data, offset)?; + /// iomem.try_write32(data, offset)?; /// /// # Ok(SampleDriver) /// } /// } /// ``` - pub fn iomap(self) -> impl PinInit<Devres<IoMem<0>>, Error> + 'a { - Self::iomap_sized::<0>(self) + pub fn iomap(self) -> Result<IoMem<'a>> { + self.iomap_sized::<0>() } /// Same as [`Self::iomap`] but with exclusive access to the underlying /// region. - pub fn iomap_exclusive(self) -> impl PinInit<Devres<ExclusiveIoMem<0>>, Error> + 'a { - Self::iomap_exclusive_sized::<0>(self) + pub fn iomap_exclusive(self) -> Result<ExclusiveIoMem<'a, 0>> { + self.iomap_exclusive_sized::<0>() } } @@ -165,9 +165,9 @@ impl<'a> IoRequest<'a> { /// # Invariants /// /// - [`ExclusiveIoMem`] has exclusive access to the underlying [`IoMem`]. -pub struct ExclusiveIoMem<const SIZE: usize> { +pub struct ExclusiveIoMem<'a, const SIZE: usize> { /// The underlying `IoMem` instance. - iomem: IoMem<SIZE>, + iomem: IoMem<'a, SIZE>, /// The region abstraction. This represents exclusive access to the /// range represented by the underlying `iomem`. @@ -176,9 +176,22 @@ pub struct ExclusiveIoMem<const SIZE: usize> { _region: Region, } -impl<const SIZE: usize> ExclusiveIoMem<SIZE> { +impl<const SIZE: usize> ForLt for ExclusiveIoMem<'static, SIZE> { + type Of<'a> = ExclusiveIoMem<'a, SIZE>; +} + +// SAFETY: `ExclusiveIoMem<'a, SIZE>` is covariant over `'a`; it holds an `IoMem<'a, SIZE>`, +// which holds `&'a Device<Bound>`, which is covariant. +unsafe impl<const SIZE: usize> CovariantForLt for ExclusiveIoMem<'static, SIZE> {} + +/// A device-managed exclusive I/O memory region. +/// +/// See [`ExclusiveIoMem::into_devres`]. +pub type DevresExclusiveIoMem<const SIZE: usize> = DevresLt<ExclusiveIoMem<'static, SIZE>>; + +impl<'a, const SIZE: usize> ExclusiveIoMem<'a, SIZE> { /// Creates a new `ExclusiveIoMem` instance. - fn ioremap(resource: &Resource) -> Result<Self> { + fn ioremap(dev: &'a Device<Bound>, resource: &Resource) -> Result<Self> { let start = resource.start(); let size = resource.size(); let name = resource.name().unwrap_or_default(); @@ -192,30 +205,33 @@ impl<const SIZE: usize> ExclusiveIoMem<SIZE> { ) .ok_or(EBUSY)?; - let iomem = IoMem::ioremap(resource)?; + let iomem = IoMem::ioremap(dev, resource)?; - let iomem = ExclusiveIoMem { + Ok(ExclusiveIoMem { iomem, _region: region, - }; - - Ok(iomem) + }) } - /// Creates a new `ExclusiveIoMem` instance from a previously acquired [`IoRequest`]. - pub fn new<'a>(io_request: IoRequest<'a>) -> impl PinInit<Devres<Self>, Error> + 'a { - let dev = io_request.device; - let res = io_request.resource; - - Devres::new(dev, Self::ioremap(res)) + /// Consume the `ExclusiveIoMem` and register it as a device-managed resource. + /// + /// The returned [`DevresExclusiveIoMem`] can outlive the original borrow and be stored in + /// driver data. Access to the I/O memory is revoked automatically when the device is unbound. + pub fn into_devres(self) -> Result<DevresExclusiveIoMem<SIZE>> { + let dev = self.iomem.dev; + // SAFETY: `ExclusiveIoMem` only holds a device reference and an I/O mapping, both of + // which remain valid for the device's full bound scope, not just for `'a`. + unsafe { DevresLt::new(dev, self) } } } -impl<const SIZE: usize> Deref for ExclusiveIoMem<SIZE> { - type Target = Mmio<SIZE>; +impl<'a, const SIZE: usize> IoBase<'a> for &'a ExclusiveIoMem<'_, SIZE> { + type Backend = MmioBackend; + type Target = super::Region<SIZE>; - fn deref(&self) -> &Self::Target { - &self.iomem + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { + self.iomem.as_view() } } @@ -228,12 +244,26 @@ impl<const SIZE: usize> Deref for ExclusiveIoMem<SIZE> { /// /// [`IoMem`] always holds an [`MmioRaw`] instance that holds a valid pointer to the /// start of the I/O memory mapped region. -pub struct IoMem<const SIZE: usize = 0> { - io: MmioRaw<SIZE>, +pub struct IoMem<'a, const SIZE: usize = 0> { + dev: &'a Device<Bound>, + io: MmioRaw<super::Region<SIZE>>, } -impl<const SIZE: usize> IoMem<SIZE> { - fn ioremap(resource: &Resource) -> Result<Self> { +impl<const SIZE: usize> ForLt for IoMem<'static, SIZE> { + type Of<'a> = IoMem<'a, SIZE>; +} + +// SAFETY: `IoMem<'a, SIZE>` is covariant over `'a`; it holds `&'a Device<Bound>`, +// which is covariant. +unsafe impl<const SIZE: usize> CovariantForLt for IoMem<'static, SIZE> {} + +/// A device-managed I/O memory region. +/// +/// See [`IoMem::into_devres`]. +pub type DevresIoMem<const SIZE: usize = 0> = DevresLt<IoMem<'static, SIZE>>; + +impl<'a, const SIZE: usize> IoMem<'a, SIZE> { + fn ioremap(dev: &'a Device<Bound>, resource: &Resource) -> Result<Self> { // Note: Some ioremap() implementations use types that depend on the CPU // word width rather than the bus address width. // @@ -264,33 +294,36 @@ impl<const SIZE: usize> IoMem<SIZE> { return Err(ENOMEM); } - let io = MmioRaw::new(addr as usize, size)?; - let io = IoMem { io }; - - Ok(io) + let io = MmioRaw::new_region(addr as usize, size)?; + Ok(IoMem { dev, io }) } - /// Creates a new `IoMem` instance from a previously acquired [`IoRequest`]. - pub fn new<'a>(io_request: IoRequest<'a>) -> impl PinInit<Devres<Self>, Error> + 'a { - let dev = io_request.device; - let res = io_request.resource; - - Devres::new(dev, Self::ioremap(res)) + /// Consume the `IoMem` and register it as a device-managed resource. + /// + /// The returned [`DevresIoMem`] can outlive the original borrow and be stored in driver data. + /// Access to the I/O memory is revoked automatically when the device is unbound. + pub fn into_devres(self) -> Result<DevresIoMem<SIZE>> { + let dev = self.dev; + // SAFETY: `IoMem` only holds a device reference and an I/O mapping, both of which + // remain valid for the device's full bound scope, not just for `'a`. + unsafe { DevresLt::new(dev, self) } } } -impl<const SIZE: usize> Drop for IoMem<SIZE> { +impl<const SIZE: usize> Drop for IoMem<'_, SIZE> { fn drop(&mut self) { // SAFETY: Safe as by the invariant of `Io`. unsafe { bindings::iounmap(self.io.addr() as *mut c_void) } } } -impl<const SIZE: usize> Deref for IoMem<SIZE> { - type Target = Mmio<SIZE>; +impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem<'_, SIZE> { + type Backend = MmioBackend; + type Target = super::Region<SIZE>; - fn deref(&self) -> &Self::Target { + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { // SAFETY: Safe as by the invariant of `IoMem`. - unsafe { Mmio::from_raw(&self.io) } + unsafe { Mmio::from_raw(self.io) } } } diff --git a/rust/kernel/io/poll.rs b/rust/kernel/io/poll.rs index 75d1b3e8596c..d75f2fcf46f2 100644 --- a/rust/kernel/io/poll.rs +++ b/rust/kernel/io/poll.rs @@ -48,13 +48,14 @@ use crate::{ /// use kernel::io::{ /// Io, /// Mmio, +/// Region, /// poll::read_poll_timeout, // /// }; /// use kernel::time::Delta; /// /// const HW_READY: u16 = 0x01; /// -/// fn wait_for_hardware<const SIZE: usize>(io: &Mmio<SIZE>) -> Result { +/// fn wait_for_hardware<const SIZE: usize>(io: Mmio<'_, Region<SIZE>>) -> Result { /// read_poll_timeout( /// // The `op` closure reads the value of a specific status register. /// || io.try_read16(0x1000), @@ -135,13 +136,14 @@ where /// use kernel::io::{ /// Io, /// Mmio, +/// Region, /// poll::read_poll_timeout_atomic, // /// }; /// use kernel::time::Delta; /// /// const HW_READY: u16 = 0x01; /// -/// fn wait_for_hardware<const SIZE: usize>(io: &Mmio<SIZE>) -> Result { +/// fn wait_for_hardware<const SIZE: usize>(io: Mmio<'_, Region<SIZE>>) -> Result { /// read_poll_timeout_atomic( /// // The `op` closure reads the value of a specific status register. /// || io.try_read16(0x1000), diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs new file mode 100644 index 000000000000..03dfd2ff48c7 --- /dev/null +++ b/rust/kernel/io/register.rs @@ -0,0 +1,1027 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Macro to define register layout and accessors. +//! +//! The [`register!`](kernel::io::register!) macro provides an intuitive and readable syntax for +//! defining a dedicated type for each register and accessing it using [`Io`](super::Io). Each such +//! type comes with its own field accessors that can return an error if a field's value is invalid. +//! +//! Note: most of the items in this module are public so they can be referenced by the macro, but +//! most are not to be used directly by users. Outside of the `register!` macro itself, the only +//! items you might want to import from this module are [`WithBase`] and [`Array`]. +//! +//! # Simple example +//! +//! ```no_run +//! use kernel::io::register; +//! +//! register! { +//! /// Basic information about the chip. +//! pub BOOT_0(u32) @ 0x00000100 { +//! /// Vendor ID. +//! 15:8 vendor_id; +//! /// Major revision of the chip. +//! 7:4 major_revision; +//! /// Minor revision of the chip. +//! 3:0 minor_revision; +//! } +//! } +//! ``` +//! +//! This defines a 32-bit `BOOT_0` type which can be read from or written to offset `0x100` of an +//! `Io` region, with the described bitfields. For instance, `minor_revision` consists of the 4 +//! least significant bits of the type. +//! +//! Fields are instances of [`Bounded`](kernel::num::Bounded) and can be read by calling their +//! getter method, which is named after them. They also have setter methods prefixed with `with_` +//! for runtime values and `with_const_` for constant values. All setters return the updated +//! register value. +//! +//! Fields can also be transparently converted from/to an arbitrary type by using the `=>` and +//! `?=>` syntaxes. +//! +//! If present, doc comments above register or fields definitions are added to the relevant item +//! they document (the register type itself, or the field's setter and getter methods). +//! +//! Note that multiple registers can be defined in a single `register!` invocation. This can be +//! useful to group related registers together. +//! +//! Here is how the register defined above can be used in code: +//! +//! +//! ```no_run +//! use kernel::{ +//! io::{ +//! register, +//! Io, +//! IoLoc, +//! }, +//! num::Bounded, +//! }; +//! # use kernel::io::{Mmio, Region}; +//! # register! { +//! # pub BOOT_0(u32) @ 0x00000100 { +//! # 15:8 vendor_id; +//! # 7:4 major_revision; +//! # 3:0 minor_revision; +//! # } +//! # } +//! # fn test(io: Mmio<'_, Region<0x1000>>) { +//! # fn obtain_vendor_id() -> u8 { 0xff } +//! +//! // Read from the register's defined offset (0x100). +//! let boot0 = io.read(BOOT_0); +//! pr_info!("chip revision: {}.{}", boot0.major_revision().get(), boot0.minor_revision().get()); +//! +//! // Update some fields and write the new value back. +//! let new_boot0 = boot0 +//! // Constant values. +//! .with_const_major_revision::<3>() +//! .with_const_minor_revision::<10>() +//! // Runtime value. +//! .with_vendor_id(obtain_vendor_id()); +//! io.write_reg(new_boot0); +//! +//! // Or, build a new value from zero and write it: +//! io.write_reg(BOOT_0::zeroed() +//! .with_const_major_revision::<3>() +//! .with_const_minor_revision::<10>() +//! .with_vendor_id(obtain_vendor_id()) +//! ); +//! +//! // Or, read and update the register in a single step. +//! io.update(BOOT_0, |r| r +//! .with_const_major_revision::<3>() +//! .with_const_minor_revision::<10>() +//! .with_vendor_id(obtain_vendor_id()) +//! ); +//! +//! // Constant values can also be built using the const setters. +//! const V: BOOT_0 = pin_init::zeroed::<BOOT_0>() +//! .with_const_major_revision::<3>() +//! .with_const_minor_revision::<10>(); +//! # } +//! ``` +//! +//! For more extensive documentation about how to define registers, see the +//! [`register!`](kernel::io::register!) macro. + +use core::marker::PhantomData; + +use crate::{ + build_assert::build_assert, + io::IoLoc, // +}; + +use super::Region; + +/// Trait implemented by all registers. +pub trait Register: Sized { + /// Backing primitive type of the register. + type Storage: Into<Self> + From<Self>; + + /// Start offset of the register. + /// + /// The interpretation of this offset depends on the type of the register. + const OFFSET: usize; +} + +/// Trait implemented by registers with a fixed offset. +pub trait FixedRegister: Register {} + +/// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when +/// passing a [`FixedRegister`] value. +impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for () +where + T: FixedRegister, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + T::OFFSET + } +} + +/// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used +/// as an [`IoLoc`]. +impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for T +where + T: FixedRegister, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + T::OFFSET + } +} + +/// Location of a fixed register. +pub struct FixedRegisterLoc<T: FixedRegister>(PhantomData<T>); + +impl<T: FixedRegister> FixedRegisterLoc<T> { + /// Returns the location of `T`. + #[inline(always)] + // We do not implement `Default` so we can be const. + #[expect(clippy::new_without_default)] + pub const fn new() -> Self { + Self(PhantomData) + } +} + +impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for FixedRegisterLoc<T> +where + T: FixedRegister, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + T::OFFSET + } +} + +/// Trait providing a base address to be added to the offset of a relative register to obtain +/// its actual offset. +/// +/// The `T` generic argument is used to distinguish which base to use, in case a type provides +/// several bases. It is given to the `register!` macro to restrict the use of the register to +/// implementors of this particular variant. +pub trait RegisterBase<T> { + /// Base address to which register offsets are added. + const BASE: usize; +} + +/// Trait implemented by all registers that are relative to a base. +pub trait WithBase { + /// Family of bases applicable to this register. + type BaseFamily; + + /// Returns the absolute location of this type when using `B` as its base. + #[inline(always)] + fn of<B: RegisterBase<Self::BaseFamily>>() -> RelativeRegisterLoc<Self, B> + where + Self: Register, + { + RelativeRegisterLoc::new() + } +} + +/// Trait implemented by relative registers. +pub trait RelativeRegister: Register + WithBase {} + +/// Location of a relative register. +/// +/// This can either be an immediately accessible regular [`RelativeRegister`], or a +/// [`RelativeRegisterArray`] that needs one additional resolution through +/// [`RelativeRegisterLoc::at`]. +pub struct RelativeRegisterLoc<T: WithBase, B: ?Sized>(PhantomData<T>, PhantomData<B>); + +impl<T, B> RelativeRegisterLoc<T, B> +where + T: Register + WithBase, + B: RegisterBase<T::BaseFamily> + ?Sized, +{ + /// Returns the location of a relative register or register array. + #[inline(always)] + // We do not implement `Default` so we can be const. + #[expect(clippy::new_without_default)] + pub const fn new() -> Self { + Self(PhantomData, PhantomData) + } + + // Returns the absolute offset of the relative register using base `B`. + // + // This is implemented as a private const method so it can be reused by the [`IoLoc`] + // implementations of both [`RelativeRegisterLoc`] and [`RelativeRegisterArrayLoc`]. + #[inline] + const fn offset(self) -> usize { + B::BASE + T::OFFSET + } +} + +impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterLoc<T, B> +where + T: RelativeRegister, + B: RegisterBase<T::BaseFamily> + ?Sized, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + RelativeRegisterLoc::offset(self) + } +} + +/// Trait implemented by arrays of registers. +pub trait RegisterArray: Register { + /// Number of elements in the registers array. + const SIZE: usize; + /// Number of bytes between the start of elements in the registers array. + const STRIDE: usize; +} + +/// Location of an array register. +pub struct RegisterArrayLoc<T: RegisterArray>(usize, PhantomData<T>); + +impl<T: RegisterArray> RegisterArrayLoc<T> { + /// Returns the location of register `T` at position `idx`, with build-time validation. + #[inline(always)] + pub fn new(idx: usize) -> Self { + build_assert!(idx < T::SIZE); + + Self(idx, PhantomData) + } + + /// Attempts to return the location of register `T` at position `idx`, with runtime validation. + #[inline(always)] + pub fn try_new(idx: usize) -> Option<Self> { + if idx < T::SIZE { + Some(Self(idx, PhantomData)) + } else { + None + } + } +} + +impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for RegisterArrayLoc<T> +where + T: RegisterArray, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + T::OFFSET + self.0 * T::STRIDE + } +} + +/// Trait providing location builders for [`RegisterArray`]s. +pub trait Array { + /// Returns the location of the register at position `idx`, with build-time validation. + #[inline(always)] + fn at(idx: usize) -> RegisterArrayLoc<Self> + where + Self: RegisterArray, + { + RegisterArrayLoc::new(idx) + } + + /// Returns the location of the register at position `idx`, with runtime validation. + #[inline(always)] + fn try_at(idx: usize) -> Option<RegisterArrayLoc<Self>> + where + Self: RegisterArray, + { + RegisterArrayLoc::try_new(idx) + } +} + +/// Trait implemented by arrays of relative registers. +pub trait RelativeRegisterArray: RegisterArray + WithBase {} + +/// Location of a relative array register. +pub struct RelativeRegisterArrayLoc< + T: RelativeRegisterArray, + B: RegisterBase<T::BaseFamily> + ?Sized, +>(RelativeRegisterLoc<T, B>, usize); + +impl<T, B> RelativeRegisterArrayLoc<T, B> +where + T: RelativeRegisterArray, + B: RegisterBase<T::BaseFamily> + ?Sized, +{ + /// Returns the location of register `T` from the base `B` at index `idx`, with build-time + /// validation. + #[inline(always)] + pub fn new(idx: usize) -> Self { + build_assert!(idx < T::SIZE); + + Self(RelativeRegisterLoc::new(), idx) + } + + /// Attempts to return the location of register `T` from the base `B` at index `idx`, with + /// runtime validation. + #[inline(always)] + pub fn try_new(idx: usize) -> Option<Self> { + if idx < T::SIZE { + Some(Self(RelativeRegisterLoc::new(), idx)) + } else { + None + } + } +} + +/// Methods exclusive to [`RelativeRegisterLoc`]s created with a [`RelativeRegisterArray`]. +impl<T, B> RelativeRegisterLoc<T, B> +where + T: RelativeRegisterArray, + B: RegisterBase<T::BaseFamily> + ?Sized, +{ + /// Returns the location of the register at position `idx`, with build-time validation. + #[inline(always)] + pub fn at(self, idx: usize) -> RelativeRegisterArrayLoc<T, B> { + RelativeRegisterArrayLoc::new(idx) + } + + /// Returns the location of the register at position `idx`, with runtime validation. + #[inline(always)] + pub fn try_at(self, idx: usize) -> Option<RelativeRegisterArrayLoc<T, B>> { + RelativeRegisterArrayLoc::try_new(idx) + } +} + +impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterArrayLoc<T, B> +where + T: RelativeRegisterArray, + B: RegisterBase<T::BaseFamily> + ?Sized, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + self.0.offset() + self.1 * T::STRIDE + } +} + +/// Trait implemented by items that contain both a register value and the absolute I/O location at +/// which to write it. +/// +/// Implementors can be used with [`Io::write_reg`](super::Io::write_reg). +pub trait LocatedRegister<Base: ?Sized> { + /// Register value to write. + type Value: Register; + /// Full location information at which to write the value. + type Location: IoLoc<Base, Self::Value>; + + /// Consumes `self` and returns a `(location, value)` tuple describing a valid I/O write + /// operation. + fn into_io_op(self) -> (Self::Location, Self::Value); +} + +impl<const SIZE: usize, T> LocatedRegister<Region<SIZE>> for T +where + T: FixedRegister, +{ + type Location = FixedRegisterLoc<Self::Value>; + type Value = T; + + #[inline(always)] + fn into_io_op(self) -> (FixedRegisterLoc<T>, T) { + (FixedRegisterLoc::new(), self) + } +} + +/// Defines a dedicated type for a register, including getter and setter methods for its fields and +/// methods to read and write it from an [`Io`](kernel::io::Io) region. +/// +/// This documentation focuses on how to declare registers. See the [module-level +/// documentation](mod@kernel::io::register) for examples of how to access them. +/// +/// There are 4 possible kinds of registers: fixed offset registers, relative registers, arrays of +/// registers, and relative arrays of registers. +/// +/// ## Fixed offset registers +/// +/// These are the simplest kind of registers. Their location is simply an offset inside the I/O +/// region. For instance: +/// +/// ```ignore +/// register! { +/// pub FIXED_REG(u16) @ 0x80 { +/// ... +/// } +/// } +/// ``` +/// +/// This creates a 16-bit register named `FIXED_REG` located at offset `0x80` of an I/O region. +/// +/// These registers' location can be built simply by referencing their name: +/// +/// ```no_run +/// use kernel::{ +/// io::{ +/// register, +/// Io, +/// }, +/// }; +/// # use kernel::io::{Mmio, Region}; +/// +/// register! { +/// FIXED_REG(u32) @ 0x100 { +/// 15:8 high_byte; +/// 7:0 low_byte; +/// } +/// } +/// +/// # fn test(io: Mmio<'_, Region<0x1000>>) { +/// let val = io.read(FIXED_REG); +/// +/// // Write from an already-existing value. +/// io.write(FIXED_REG, val.with_low_byte(0xff)); +/// +/// // Create a register value from scratch. +/// let val2 = FIXED_REG::zeroed().with_high_byte(0x80); +/// +/// // The location of fixed offset registers is already contained in their type. Thus, the +/// // `location` argument of `Io::write` is technically redundant and can be replaced by `()`. +/// io.write((), val2); +/// +/// // Or, the single-argument `Io::write_reg` can be used. +/// io.write_reg(val2); +/// # } +/// +/// ``` +/// +/// It is possible to create an alias of an existing register with new field definitions by using +/// the `=> ALIAS` syntax. This is useful for cases where a register's interpretation depends on +/// the context: +/// +/// ```no_run +/// use kernel::io::register; +/// +/// register! { +/// /// Scratch register. +/// pub SCRATCH(u32) @ 0x00000200 { +/// 31:0 value; +/// } +/// +/// /// Boot status of the firmware. +/// pub SCRATCH_BOOT_STATUS(u32) => SCRATCH { +/// 0:0 completed; +/// } +/// } +/// ``` +/// +/// In this example, `SCRATCH_BOOT_STATUS` uses the same I/O address as `SCRATCH`, while providing +/// its own `completed` field. +/// +/// ## Relative registers +/// +/// Relative registers can be instantiated several times at a relative offset of a group of bases. +/// For instance, imagine the following I/O space: +/// +/// ```text +/// +-----------------------------+ +/// | ... | +/// | | +/// 0x100--->+------------CPU0-------------+ +/// | | +/// 0x110--->+-----------------------------+ +/// | CPU_CTL | +/// +-----------------------------+ +/// | ... | +/// | | +/// | | +/// 0x200--->+------------CPU1-------------+ +/// | | +/// 0x210--->+-----------------------------+ +/// | CPU_CTL | +/// +-----------------------------+ +/// | ... | +/// +-----------------------------+ +/// ``` +/// +/// `CPU0` and `CPU1` both have a `CPU_CTL` register that starts at offset `0x10` of their I/O +/// space segment. Since both instances of `CPU_CTL` share the same layout, we don't want to define +/// them twice and would prefer a way to select which one to use from a single definition. +/// +/// This can be done using the `Base + Offset` syntax when specifying the register's address: +/// +/// ```ignore +/// register! { +/// pub RELATIVE_REG(u32) @ Base + 0x80 { +/// ... +/// } +/// } +/// ``` +/// +/// This creates a register with an offset of `0x80` from a given base. +/// +/// `Base` is an arbitrary type (typically a ZST) to be used as a generic parameter of the +/// [`RegisterBase`] trait to provide the base as a constant, i.e. each type providing a base for +/// this register needs to implement `RegisterBase<Base>`. +/// +/// The location of relative registers can be built using the [`WithBase::of`] method to specify +/// its base. All relative registers implement [`WithBase`]. +/// +/// Here is the above layout translated into code: +/// +/// ```no_run +/// use kernel::{ +/// io::{ +/// register, +/// register::{ +/// RegisterBase, +/// WithBase, +/// }, +/// Io, +/// }, +/// }; +/// # use kernel::io::{Mmio, Region}; +/// +/// // Type used to identify the base. +/// pub struct CpuCtlBase; +/// +/// // ZST describing `CPU0`. +/// struct Cpu0; +/// impl RegisterBase<CpuCtlBase> for Cpu0 { +/// const BASE: usize = 0x100; +/// } +/// +/// // ZST describing `CPU1`. +/// struct Cpu1; +/// impl RegisterBase<CpuCtlBase> for Cpu1 { +/// const BASE: usize = 0x200; +/// } +/// +/// // This makes `CPU_CTL` accessible from all implementors of `RegisterBase<CpuCtlBase>`. +/// register! { +/// /// CPU core control. +/// pub CPU_CTL(u32) @ CpuCtlBase + 0x10 { +/// 0:0 start; +/// } +/// } +/// +/// # fn test(io: Mmio<'_, Region<0x1000>>) { +/// // Read the status of `Cpu0`. +/// let cpu0_started = io.read(CPU_CTL::of::<Cpu0>()); +/// +/// // Stop `Cpu0`. +/// io.write(WithBase::of::<Cpu0>(), CPU_CTL::zeroed()); +/// # } +/// +/// // Aliases can also be defined for relative register. +/// register! { +/// /// Alias to CPU core control. +/// pub CPU_CTL_ALIAS(u32) => CpuCtlBase + CPU_CTL { +/// /// Start the aliased CPU core. +/// 1:1 alias_start; +/// } +/// } +/// +/// # fn test2(io: Mmio<'_, Region<0x1000>>) { +/// // Start the aliased `CPU0`, leaving its other fields untouched. +/// io.update(CPU_CTL_ALIAS::of::<Cpu0>(), |r| r.with_alias_start(true)); +/// # } +/// ``` +/// +/// ## Arrays of registers +/// +/// Some I/O areas contain consecutive registers that share the same field layout. These areas can +/// be defined as an array of identical registers, allowing them to be accessed by index with +/// compile-time or runtime bound checking: +/// +/// ```ignore +/// register! { +/// pub REGISTER_ARRAY(u8)[10, stride = 4] @ 0x100 { +/// ... +/// } +/// } +/// ``` +/// +/// This defines `REGISTER_ARRAY`, an array of 10 byte registers starting at offset `0x100`. Each +/// register is separated from its neighbor by 4 bytes. +/// +/// The `stride` parameter is optional; if unspecified, the registers are placed consecutively from +/// each other. +/// +/// A location for a register in a register array is built using the [`Array::at`] trait method. +/// All arrays of registers implement [`Array`]. +/// +/// ```no_run +/// use kernel::{ +/// io::{ +/// register, +/// register::Array, +/// Io, +/// }, +/// }; +/// # use kernel::io::{Mmio, Region}; +/// # fn get_scratch_idx() -> usize { +/// # 0x15 +/// # } +/// +/// // Array of 64 consecutive registers with the same layout starting at offset `0x80`. +/// register! { +/// /// Scratch registers. +/// pub SCRATCH(u32)[64] @ 0x00000080 { +/// 31:0 value; +/// } +/// } +/// +/// # fn test(io: Mmio<'_, Region<0x1000>>) +/// # -> Result<(), Error>{ +/// // Read scratch register 0, i.e. I/O address `0x80`. +/// let scratch_0 = io.read(SCRATCH::at(0)).value(); +/// +/// // Write scratch register 15, i.e. I/O address `0x80 + (15 * 4)`. +/// io.write(Array::at(15), SCRATCH::from(0xffeeaabb)); +/// +/// // This is out of bounds and won't build. +/// // let scratch_128 = io.read(SCRATCH::at(128)).value(); +/// +/// // Runtime-obtained array index. +/// let idx = get_scratch_idx(); +/// // Access on a runtime index returns an error if it is out-of-bounds. +/// let some_scratch = io.read(SCRATCH::try_at(idx).ok_or(EINVAL)?).value(); +/// +/// // Alias to a specific register in an array. +/// // Here `SCRATCH[8]` is used to convey the firmware exit code. +/// register! { +/// /// Firmware exit status code. +/// pub FIRMWARE_STATUS(u32) => SCRATCH[8] { +/// 7:0 status; +/// } +/// } +/// +/// let status = io.read(FIRMWARE_STATUS).status(); +/// +/// // Non-contiguous register arrays can be defined by adding a stride parameter. +/// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the +/// // registers of the two declarations below are interleaved. +/// register! { +/// /// Scratch registers bank 0. +/// pub SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ 0x000000c0 { +/// 31:0 value; +/// } +/// +/// /// Scratch registers bank 1. +/// pub SCRATCH_INTERLEAVED_1(u32)[16, stride = 8] @ 0x000000c4 { +/// 31:0 value; +/// } +/// } +/// # Ok(()) +/// # } +/// ``` +/// +/// ## Relative arrays of registers +/// +/// Combining the two features described in the sections above, arrays of registers accessible from +/// a base can also be defined: +/// +/// ```ignore +/// register! { +/// pub RELATIVE_REGISTER_ARRAY(u8)[10, stride = 4] @ Base + 0x100 { +/// ... +/// } +/// } +/// ``` +/// +/// Like relative registers, they implement the [`WithBase`] trait. However the return value of +/// [`WithBase::of`] cannot be used directly as a location and must be further specified using the +/// [`at`](RelativeRegisterLoc::at) method. +/// +/// ```no_run +/// use kernel::{ +/// io::{ +/// register, +/// register::{ +/// RegisterBase, +/// WithBase, +/// }, +/// Io, +/// }, +/// }; +/// # use kernel::io::{Mmio, Region}; +/// # fn get_scratch_idx() -> usize { +/// # 0x15 +/// # } +/// +/// // Type used as parameter of `RegisterBase` to specify the base. +/// pub struct CpuCtlBase; +/// +/// // ZST describing `CPU0`. +/// struct Cpu0; +/// impl RegisterBase<CpuCtlBase> for Cpu0 { +/// const BASE: usize = 0x100; +/// } +/// +/// // ZST describing `CPU1`. +/// struct Cpu1; +/// impl RegisterBase<CpuCtlBase> for Cpu1 { +/// const BASE: usize = 0x200; +/// } +/// +/// // 64 per-cpu scratch registers, arranged as a contiguous array. +/// register! { +/// /// Per-CPU scratch registers. +/// pub CPU_SCRATCH(u32)[64] @ CpuCtlBase + 0x00000080 { +/// 31:0 value; +/// } +/// } +/// +/// # fn test(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { +/// // Read scratch register 0 of CPU0. +/// let scratch = io.read(CPU_SCRATCH::of::<Cpu0>().at(0)); +/// +/// // Write the retrieved value into scratch register 15 of CPU1. +/// io.write(WithBase::of::<Cpu1>().at(15), scratch); +/// +/// // This won't build. +/// // let cpu0_scratch_128 = io.read(CPU_SCRATCH::of::<Cpu0>().at(128)).value(); +/// +/// // Runtime-obtained array index. +/// let scratch_idx = get_scratch_idx(); +/// // Access on a runtime index returns an error if it is out-of-bounds. +/// let cpu0_scratch = io.read( +/// CPU_SCRATCH::of::<Cpu0>().try_at(scratch_idx).ok_or(EINVAL)? +/// ).value(); +/// # Ok(()) +/// # } +/// +/// // Alias to `SCRATCH[8]` used to convey the firmware exit code. +/// register! { +/// /// Per-CPU firmware exit status code. +/// pub CPU_FIRMWARE_STATUS(u32) => CpuCtlBase + CPU_SCRATCH[8] { +/// 7:0 status; +/// } +/// } +/// +/// // Non-contiguous relative register arrays can be defined by adding a stride parameter. +/// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the +/// // registers of the two declarations below are interleaved. +/// register! { +/// /// Scratch registers bank 0. +/// pub CPU_SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d00 { +/// 31:0 value; +/// } +/// +/// /// Scratch registers bank 1. +/// pub CPU_SCRATCH_INTERLEAVED_1(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d04 { +/// 31:0 value; +/// } +/// } +/// +/// # fn test2(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { +/// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::<Cpu0>()).status(); +/// # Ok(()) +/// # } +/// ``` +#[macro_export] +macro_rules! register { + // Entry point for the macro, allowing multiple registers to be defined in one call. + // It matches all possible register declaration patterns to dispatch them to corresponding + // `@reg` rule that defines a single register. + // + // TODO: change `alias:ident` to `alias:path` once relative registers are replaced by I/O + // projections. + ( + $( + $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) + $([ $size:expr $(, stride = $stride:expr)? ])? + $(@ $($base:ident +)? $offset:literal)? + $(=> $alias:ident $(+ $alias_offset:ident)? $([$alias_idx:expr])? )? + { $($fields:tt)* } + )* + ) => { + $( + $crate::register!( + @reg $(#[$attr])* $vis $name ($storage) $([$size $(, stride = $stride)?])? + $(@ $($base +)? $offset)? + $(=> $alias $(+ $alias_offset)? $([$alias_idx])? )? + { $($fields)* } + ); + )* + }; + + // All the rules below are private helpers. + + // Creates a register at a fixed offset of the MMIO space. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $offset:literal + { $($fields:tt)* } + ) => { + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!(@io_base $name($storage) @ $offset); + $crate::register!(@io_fixed $(#[$attr])* $vis $name); + }; + + // Creates an alias register of fixed offset register `alias` with its own fields. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path + { $($fields:tt)* } + ) => { + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!( + @io_base $name($storage) @ + <$alias as $crate::io::register::Register>::OFFSET + ); + $crate::register!(@io_fixed $(#[$attr])* $vis $name); + }; + + // Creates a register at a relative offset from a base address provider. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $base:ident + $offset:literal + { $($fields:tt)* } + ) => { + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!(@io_base $name($storage) @ $offset); + $crate::register!(@io_relative $name @ $base); + }; + + // Creates an alias register of relative offset register `alias` with its own fields. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $base:ident + $alias:ident + { $($fields:tt)* } + ) => { + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!( + @io_base $name($storage) @ <$alias as $crate::io::register::Register>::OFFSET + ); + $crate::register!(@io_relative $name @ $base); + }; + + // Creates an array of registers at a fixed offset of the MMIO space. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) + [ $size:expr, stride = $stride:expr ] @ $offset:literal { $($fields:tt)* } + ) => { + $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride); + + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!(@io_base $name($storage) @ $offset); + $crate::register!(@io_array $name [ $size, stride = $stride ]); + }; + + // Shortcut for contiguous array of registers (stride == size of element). + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] @ $offset:literal + { $($fields:tt)* } + ) => { + $crate::register!( + @reg $(#[$attr])* $vis $name($storage) + [ $size, stride = ::core::mem::size_of::<$storage>() ] + @ $offset { $($fields)* } + ); + }; + + // Creates an alias of register `idx` of array of registers `alias` with its own fields. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path [ $idx:expr ] + { $($fields:tt)* } + ) => { + $crate::build_assert::static_assert!( + $idx < <$alias as $crate::io::register::RegisterArray>::SIZE + ); + + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!( + @io_base $name($storage) @ + <$alias as $crate::io::register::Register>::OFFSET + + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE + ); + $crate::register!(@io_fixed $(#[$attr])* $vis $name); + }; + + // Creates an array of registers at a relative offset from a base address provider. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) + [ $size:expr, stride = $stride:expr ] + @ $base:ident + $offset:literal { $($fields:tt)* } + ) => { + $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride); + + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!(@io_base $name($storage) @ $offset); + $crate::register!(@io_relative_array $name [ $size, stride = $stride ] @ $base); + }; + + // Shortcut for contiguous array of relative registers (stride == size of element). + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] + @ $base:ident + $offset:literal { $($fields:tt)* } + ) => { + $crate::register!( + @reg $(#[$attr])* $vis $name($storage) + [ $size, stride = ::core::mem::size_of::<$storage>() ] + @ $base + $offset { $($fields)* } + ); + }; + + // Creates an alias of register `idx` of relative array of registers `alias` with its own + // fields. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) + => $base:ident + $alias:ident [ $idx:expr ] { $($fields:tt)* } + ) => { + $crate::build_assert::static_assert!( + $idx < <$alias as $crate::io::register::RegisterArray>::SIZE + ); + + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!( + @io_base $name($storage) @ + <$alias as $crate::io::register::Register>::OFFSET + + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE + ); + $crate::register!(@io_relative $name @ $base); + }; + + // Generates the bitfield for the register. + // + // `#[allow(non_camel_case_types)]` is added since register names typically use + // `SCREAMING_CASE`. + ( + @bitfield $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* } + ) => { + $crate::bitfield!( + #[allow(non_camel_case_types)] + $(#[$attr])* $vis struct $name($storage) { $($fields)* } + ); + }; + + // Implementations shared by all registers types. + (@io_base $name:ident($storage:ty) @ $offset:expr) => { + impl $crate::io::register::Register for $name { + type Storage = $storage; + + const OFFSET: usize = $offset; + } + }; + + // Implementations of fixed registers. + (@io_fixed $(#[$attr:meta])* $vis:vis $name:ident) => { + impl $crate::io::register::FixedRegister for $name {} + + $(#[$attr])* + $vis const $name: $crate::io::register::FixedRegisterLoc<$name> = + $crate::io::register::FixedRegisterLoc::<$name>::new(); + }; + + // Implementations of relative registers. + (@io_relative $name:ident @ $base:ident) => { + impl $crate::io::register::WithBase for $name { + type BaseFamily = $base; + } + + impl $crate::io::register::RelativeRegister for $name {} + }; + + // Implementations of register arrays. + (@io_array $name:ident [ $size:expr, stride = $stride:expr ]) => { + impl $crate::io::register::Array for $name {} + + impl $crate::io::register::RegisterArray for $name { + const SIZE: usize = $size; + const STRIDE: usize = $stride; + } + }; + + // Implementations of relative array registers. + ( + @io_relative_array $name:ident [ $size:expr, stride = $stride:expr ] @ $base:ident + ) => { + impl $crate::io::register::WithBase for $name { + type BaseFamily = $base; + } + + impl $crate::io::register::RegisterArray for $name { + const SIZE: usize = $size; + const STRIDE: usize = $stride; + } + + impl $crate::io::register::RelativeRegisterArray for $name {} + }; +} diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs index b7ac9faf141d..17b0c174cfc5 100644 --- a/rust/kernel/io/resource.rs +++ b/rust/kernel/io/resource.rs @@ -229,7 +229,7 @@ impl Flags { // Always inline to optimize out error path of `build_assert`. #[inline(always)] const fn new(value: u32) -> Self { - crate::build_assert!(value as u64 <= c_ulong::MAX as u64); + build_assert!(value as u64 <= c_ulong::MAX as u64); Flags(value as c_ulong) } } diff --git a/rust/kernel/ioctl.rs b/rust/kernel/ioctl.rs index 2fc7662339e5..5bb5b48cf949 100644 --- a/rust/kernel/ioctl.rs +++ b/rust/kernel/ioctl.rs @@ -6,7 +6,7 @@ #![expect(non_snake_case)] -use crate::build_assert; +use crate::build_assert::build_assert; /// Build an ioctl number, analogous to the C macro of the same name. #[inline(always)] diff --git a/rust/kernel/iommu/pgtable.rs b/rust/kernel/iommu/pgtable.rs index c88e38fd938a..5f9b42ca92c2 100644 --- a/rust/kernel/iommu/pgtable.rs +++ b/rust/kernel/iommu/pgtable.rs @@ -16,7 +16,6 @@ use crate::{ Bound, Device, // }, - devres::Devres, error::to_result, io::PhysAddr, prelude::*, // @@ -59,15 +58,16 @@ pub struct Config { /// # Invariants /// /// The pointer references a valid io page table. -pub struct IoPageTable<F: IoPageTableFmt> { +pub struct IoPageTable<'a, F: IoPageTableFmt> { ptr: NonNull<bindings::io_pgtable_ops>, + _dev: PhantomData<&'a Device<Bound>>, _marker: PhantomData<F>, } // SAFETY: `struct io_pgtable_ops` is not restricted to a single thread. -unsafe impl<F: IoPageTableFmt> Send for IoPageTable<F> {} +unsafe impl<F: IoPageTableFmt> Send for IoPageTable<'_, F> {} // SAFETY: `struct io_pgtable_ops` may be accessed concurrently. -unsafe impl<F: IoPageTableFmt> Sync for IoPageTable<F> {} +unsafe impl<F: IoPageTableFmt> Sync for IoPageTable<'_, F> {} /// The format used by this page table. pub trait IoPageTableFmt: 'static { @@ -75,25 +75,10 @@ pub trait IoPageTableFmt: 'static { const FORMAT: io_pgtable_fmt; } -impl<F: IoPageTableFmt> IoPageTable<F> { - /// Create a new `IoPageTable` as a device resource. - #[inline] - pub fn new( - dev: &Device<Bound>, - config: Config, - ) -> impl PinInit<Devres<IoPageTable<F>>, Error> + '_ { - // SAFETY: Devres ensures that the value is dropped during device unbind. - Devres::new(dev, unsafe { Self::new_raw(dev, config) }) - } - +impl<'a, F: IoPageTableFmt> IoPageTable<'a, F> { /// Create a new `IoPageTable`. - /// - /// # Safety - /// - /// If successful, then the returned `IoPageTable` must be dropped before the device is - /// unbound. #[inline] - pub unsafe fn new_raw(dev: &Device<Bound>, config: Config) -> Result<IoPageTable<F>> { + pub fn new(dev: &'a Device<Bound>, config: Config) -> Result<IoPageTable<'a, F>> { let mut raw_cfg = bindings::io_pgtable_cfg { quirks: config.quirks, pgsize_bitmap: config.pgsize_bitmap, @@ -102,8 +87,7 @@ impl<F: IoPageTableFmt> IoPageTable<F> { coherent_walk: config.coherent_walk, tlb: &raw const NOOP_FLUSH_OPS, iommu_dev: dev.as_raw(), - // SAFETY: All zeroes is a valid value for `struct io_pgtable_cfg`. - ..unsafe { core::mem::zeroed() } + ..Zeroable::zeroed() }; // SAFETY: @@ -118,6 +102,7 @@ impl<F: IoPageTableFmt> IoPageTable<F> { // INVARIANT: We successfully created a valid page table. Ok(IoPageTable { ptr: NonNull::new(ops).ok_or(ENOMEM)?, + _dev: PhantomData, _marker: PhantomData, }) } @@ -240,7 +225,7 @@ extern "C" fn rust_tlb_flush_walk_noop( ) { } -impl<F: IoPageTableFmt> Drop for IoPageTable<F> { +impl<F: IoPageTableFmt> Drop for IoPageTable<'_, F> { fn drop(&mut self) { // SAFETY: The caller of `Self::ttbr()` promised that the page table is not live when this // destructor runs. @@ -255,7 +240,7 @@ impl IoPageTableFmt for ARM64LPAES1 { const FORMAT: io_pgtable_fmt = bindings::io_pgtable_fmt_ARM_64_LPAE_S1 as io_pgtable_fmt; } -impl IoPageTable<ARM64LPAES1> { +impl IoPageTable<'_, ARM64LPAES1> { /// Access the `ttbr` field of the configuration. /// /// This is the physical address of the page table, which may be passed to the device that diff --git a/rust/kernel/irq.rs b/rust/kernel/irq.rs index 20abd4056655..09ef1e7f853c 100644 --- a/rust/kernel/irq.rs +++ b/rust/kernel/irq.rs @@ -8,7 +8,7 @@ //! The current abstractions handle IRQ requests and handlers, i.e.: it allows //! drivers to register a handler for a given IRQ line. //! -//! C header: [`include/linux/device.h`](srctree/include/linux/interrupt.h) +//! C header: [`include/linux/interrupt.h`](srctree/include/linux/interrupt.h) /// Flags to be used when registering IRQ handlers. mod flags; diff --git a/rust/kernel/irq/request.rs b/rust/kernel/irq/request.rs index 7a36f790593e..c1c6525a676a 100644 --- a/rust/kernel/irq/request.rs +++ b/rust/kernel/irq/request.rs @@ -5,16 +5,21 @@ //! [`ThreadedRegistration`], which allow users to register handlers for a given //! IRQ line. -use core::marker::PhantomPinned; - -use crate::alloc::Allocator; -use crate::device::{Bound, Device}; -use crate::devres::Devres; -use crate::error::to_result; -use crate::irq::flags::Flags; -use crate::prelude::*; -use crate::str::CStr; -use crate::sync::Arc; +use core::marker::{ + PhantomData, + PhantomPinned, // +}; + +use crate::{ + device::{ + Bound, + Device, // + }, + error::to_result, + irq::flags::Flags, + prelude::*, + str::CStr, +}; /// The value that can be returned from a [`Handler`] or a [`ThreadedHandler`]. #[repr(u32)] @@ -36,73 +41,20 @@ pub trait Handler: Sync { /// All work that does not necessarily need to be executed from /// interrupt context, should be deferred to a threaded handler. /// See also [`ThreadedRegistration`]. - fn handle(&self, device: &Device<Bound>) -> IrqReturn; -} - -impl<T: ?Sized + Handler + Send> Handler for Arc<T> { - fn handle(&self, device: &Device<Bound>) -> IrqReturn { - T::handle(self, device) - } -} - -impl<T: ?Sized + Handler, A: Allocator> Handler for Box<T, A> { - fn handle(&self, device: &Device<Bound>) -> IrqReturn { - T::handle(self, device) - } -} - -/// # Invariants -/// -/// - `self.irq` is the same as the one passed to `request_{threaded}_irq`. -/// - `cookie` was passed to `request_{threaded}_irq` as the cookie. It is guaranteed to be unique -/// by the type system, since each call to `new` will return a different instance of -/// `Registration`. -#[pin_data(PinnedDrop)] -struct RegistrationInner { - irq: u32, - cookie: *mut c_void, -} - -impl RegistrationInner { - fn synchronize(&self) { - // SAFETY: safe as per the invariants of `RegistrationInner` - unsafe { bindings::synchronize_irq(self.irq) }; - } -} - -#[pinned_drop] -impl PinnedDrop for RegistrationInner { - fn drop(self: Pin<&mut Self>) { - // SAFETY: - // - // Safe as per the invariants of `RegistrationInner` and: - // - // - The containing struct is `!Unpin` and was initialized using - // pin-init, so it occupied the same memory location for the entirety of - // its lifetime. - // - // Notice that this will block until all handlers finish executing, - // i.e.: at no point will &self be invalid while the handler is running. - unsafe { bindings::free_irq(self.irq, self.cookie) }; - } + fn handle(&self) -> IrqReturn; } -// SAFETY: We only use `inner` on drop, which called at most once with no -// concurrent access. -unsafe impl Sync for RegistrationInner {} - -// SAFETY: It is safe to send `RegistrationInner` across threads. -unsafe impl Send for RegistrationInner {} - /// A request for an IRQ line for a given device. /// /// # Invariants /// /// - `ìrq` is the number of an interrupt source of `dev`. -/// - `irq` has not been registered yet. +/// - `irq` has not been registered yet; this is consumed by [`Registration::new()`]. pub struct IrqRequest<'a> { - dev: &'a Device<Bound>, irq: u32, + /// Proves the device is bound at registration time and ties `'a` to the device's bound + /// lifetime, ensuring the [`Registration`] cannot outlive it. + _dev: PhantomData<&'a Device<Bound>>, } impl<'a> IrqRequest<'a> { @@ -111,12 +63,16 @@ impl<'a> IrqRequest<'a> { /// # Safety /// /// - `irq` should be a valid IRQ number for `dev`. - pub(crate) unsafe fn new(dev: &'a Device<Bound>, irq: u32) -> Self { + pub(crate) unsafe fn new(_dev: &'a Device<Bound>, irq: u32) -> Self { // INVARIANT: `irq` is a valid IRQ number for `dev`. - IrqRequest { dev, irq } + IrqRequest { + irq, + _dev: PhantomData, + } } /// Returns the IRQ number of an [`IrqRequest`]. + #[inline] pub fn irq(&self) -> u32 { self.irq } @@ -139,10 +95,18 @@ impl<'a> IrqRequest<'a> { /// [`Completion::wait_for_completion()`]: kernel::sync::Completion::wait_for_completion /// /// ``` -/// use kernel::device::{Bound, Device}; -/// use kernel::irq::{self, Flags, IrqRequest, IrqReturn, Registration}; -/// use kernel::prelude::*; -/// use kernel::sync::{Arc, Completion}; +/// use core::pin::Pin; +/// use kernel::{ +/// irq::{ +/// self, +/// Flags, +/// IrqRequest, +/// IrqReturn, +/// Registration, +/// }, +/// prelude::*, +/// sync::Completion, +/// }; /// /// // Data shared between process and IRQ context. /// #[pin_data] @@ -153,7 +117,7 @@ impl<'a> IrqRequest<'a> { /// /// impl irq::Handler for Data { /// // Executed in IRQ context. -/// fn handle(&self, _dev: &Device<Bound>) -> IrqReturn { +/// fn handle(&self) -> IrqReturn { /// self.completion.complete_all(); /// IrqReturn::Handled /// } @@ -163,12 +127,21 @@ impl<'a> IrqRequest<'a> { /// // /// // This runs in process context and assumes `request` was previously acquired from a device. /// fn register_irq( -/// handler: impl PinInit<Data, Error>, /// request: IrqRequest<'_>, -/// ) -> Result<Arc<Registration<Data>>> { -/// let registration = Registration::new(request, Flags::SHARED, c"my_device", handler); +/// ) -> Result<Pin<KBox<Registration<'_, Data>>>> { +/// // SAFETY: The returned Registration is not leaked. +/// let registration = unsafe { +/// Registration::new( +/// request, +/// Flags::SHARED, +/// c"my_device", +/// try_pin_init!(Data { +/// completion <- Completion::new(), +/// }? Error), +/// ) +/// }; /// -/// let registration = Arc::pin_init(registration, GFP_KERNEL)?; +/// let registration = KBox::pin_init(registration, GFP_KERNEL)?; /// /// registration.handler().completion.wait_for_completion(); /// @@ -179,11 +152,10 @@ impl<'a> IrqRequest<'a> { /// /// # Invariants /// -/// * We own an irq handler whose cookie is a pointer to `Self`. -#[pin_data] -pub struct Registration<T: Handler + 'static> { - #[pin] - inner: Devres<RegistrationInner>, +/// * We own an irq handler registered via `request_irq` whose cookie is a pointer to `Self`. +#[pin_data(PinnedDrop)] +pub struct Registration<'a, T: Handler> { + request: IrqRequest<'a>, #[pin] handler: T, @@ -194,44 +166,46 @@ pub struct Registration<T: Handler + 'static> { _pin: PhantomPinned, } -impl<T: Handler + 'static> Registration<T> { +impl<'a, T: Handler> Registration<'a, T> { /// Registers the IRQ handler with the system for the given IRQ number. - pub fn new<'a>( + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running. + pub unsafe fn new( request: IrqRequest<'a>, flags: Flags, name: &'static CStr, handler: impl PinInit<T, Error> + 'a, - ) -> impl PinInit<Self, Error> + 'a { + ) -> impl PinInit<Self, Error> + 'a + where + T: 'a, + { + // INVARIANT: If initialization completes successfully, we own an IRQ handler registered + // via `request_irq` whose cookie is a pointer to `Self`. try_pin_init!(&this in Self { handler <- handler, - inner <- Devres::new( - request.dev, - try_pin_init!(RegistrationInner { - // INVARIANT: `this` is a valid pointer to the `Registration` instance - cookie: this.as_ptr().cast::<c_void>(), - irq: { - // SAFETY: - // - The callbacks are valid for use with request_irq. - // - If this succeeds, the slot is guaranteed to be valid until the - // destructor of Self runs, which will deregister the callbacks - // before the memory location becomes invalid. - // - When request_irq is called, everything that handle_irq_callback will - // touch has already been initialized, so it's safe for the callback to - // be called immediately. - to_result(unsafe { - bindings::request_irq( - request.irq, - Some(handle_irq_callback::<T>), - flags.into_inner(), - name.as_char_ptr(), - this.as_ptr().cast::<c_void>(), - ) - })?; - request.irq - } - }) - ), + request, _pin: PhantomPinned, + _: { + // SAFETY: + // - The callbacks are valid for use with request_irq. + // - If this succeeds, the slot is guaranteed to be valid until the destructor of + // Self runs, which will deregister the callbacks before the memory location + // becomes invalid. + // - All fields are already initialized, so it's safe for the callback to be + // called immediately. + to_result(unsafe { + bindings::request_irq( + request.irq, + Some(handle_irq_callback::<T>), + flags.into_inner(), + name.as_char_ptr(), + this.as_ptr().cast::<c_void>(), + ) + })?; + }, }) } @@ -241,36 +215,37 @@ impl<T: Handler + 'static> Registration<T> { } /// Wait for pending IRQ handlers on other CPUs. - /// - /// This will attempt to access the inner [`Devres`] container. - pub fn try_synchronize(&self) -> Result { - let inner = self.inner.try_access().ok_or(ENODEV)?; - inner.synchronize(); - Ok(()) + #[inline] + pub fn synchronize(&self) { + // SAFETY: `self.request.irq` is a valid registered IRQ number (type invariant). + unsafe { bindings::synchronize_irq(self.request.irq) }; } +} - /// Wait for pending IRQ handlers on other CPUs. - pub fn synchronize(&self, dev: &Device<Bound>) -> Result { - let inner = self.inner.access(dev)?; - inner.synchronize(); - Ok(()) +#[pinned_drop] +impl<T: Handler> PinnedDrop for Registration<'_, T> { + fn drop(self: Pin<&mut Self>) { + // SAFETY: The cookie was set to a pointer to `Self` in `Registration::new()`. This blocks + // until all in-flight handlers complete, so no references to `self` remain after this + // returns. + unsafe { + bindings::free_irq( + self.request.irq, + core::ptr::from_mut::<Self>(self.get_unchecked_mut()).cast::<c_void>(), + ) + }; } } /// # Safety /// /// This function should be only used as the callback in `request_irq`. -unsafe extern "C" fn handle_irq_callback<T: Handler + 'static>( - _irq: i32, - ptr: *mut c_void, -) -> c_uint { - // SAFETY: `ptr` is a pointer to `Registration<T>` set in `Registration::new` - let registration = unsafe { &*(ptr as *const Registration<T>) }; - // SAFETY: The irq callback is removed before the device is unbound, so the fact that the irq - // callback is running implies that the device has not yet been unbound. - let device = unsafe { registration.inner.device().as_bound() }; +unsafe extern "C" fn handle_irq_callback<T: Handler>(_irq: i32, ptr: *mut c_void) -> c_uint { + let ptr = ptr.cast_const().cast::<Registration<'_, T>>(); + // SAFETY: `ptr` is a pointer to `Registration<'_, T>` set in `Registration::new()`. + let registration = unsafe { &*ptr }; - T::handle(®istration.handler, device) as c_uint + T::handle(®istration.handler) as c_uint } /// The value that can be returned from [`ThreadedHandler::handle`]. @@ -296,8 +271,7 @@ pub trait ThreadedHandler: Sync { /// handler, i.e. [`ThreadedHandler::handle_threaded`]. /// /// The default implementation returns [`ThreadedIrqReturn::WakeThread`]. - #[expect(unused_variables)] - fn handle(&self, device: &Device<Bound>) -> ThreadedIrqReturn { + fn handle(&self) -> ThreadedIrqReturn { ThreadedIrqReturn::WakeThread } @@ -305,27 +279,7 @@ pub trait ThreadedHandler: Sync { /// /// This is executed in process context. The kernel creates a dedicated /// `kthread` for this purpose. - fn handle_threaded(&self, device: &Device<Bound>) -> IrqReturn; -} - -impl<T: ?Sized + ThreadedHandler + Send> ThreadedHandler for Arc<T> { - fn handle(&self, device: &Device<Bound>) -> ThreadedIrqReturn { - T::handle(self, device) - } - - fn handle_threaded(&self, device: &Device<Bound>) -> IrqReturn { - T::handle_threaded(self, device) - } -} - -impl<T: ?Sized + ThreadedHandler, A: Allocator> ThreadedHandler for Box<T, A> { - fn handle(&self, device: &Device<Bound>) -> ThreadedIrqReturn { - T::handle(self, device) - } - - fn handle_threaded(&self, device: &Device<Bound>) -> IrqReturn { - T::handle_threaded(self, device) - } + fn handle_threaded(&self) -> IrqReturn; } /// A registration of a threaded IRQ handler for a given IRQ line. @@ -342,13 +296,20 @@ impl<T: ?Sized + ThreadedHandler, A: Allocator> ThreadedHandler for Box<T, A> { /// [`Mutex`](kernel::sync::Mutex) to provide interior mutability. /// /// ``` -/// use kernel::device::{Bound, Device}; -/// use kernel::irq::{ -/// self, Flags, IrqRequest, IrqReturn, ThreadedHandler, ThreadedIrqReturn, -/// ThreadedRegistration, +/// use core::pin::Pin; +/// use kernel::{ +/// irq::{ +/// self, +/// Flags, +/// IrqRequest, +/// IrqReturn, +/// ThreadedHandler, +/// ThreadedIrqReturn, +/// ThreadedRegistration, +/// }, +/// prelude::*, +/// sync::Mutex, /// }; -/// use kernel::prelude::*; -/// use kernel::sync::{Arc, Mutex}; /// /// // Declare a struct that will be passed in when the interrupt fires. The u32 /// // merely serves as an example of some internal data. @@ -366,7 +327,7 @@ impl<T: ?Sized + ThreadedHandler, A: Allocator> ThreadedHandler for Box<T, A> { /// // This will run (in a separate kthread) if and only if /// // [`ThreadedHandler::handle`] returns [`WakeThread`], which it does by /// // default. -/// fn handle_threaded(&self, _dev: &Device<Bound>) -> IrqReturn { +/// fn handle_threaded(&self) -> IrqReturn { /// let mut data = self.value.lock(); /// *data += 1; /// IrqReturn::Handled @@ -378,13 +339,21 @@ impl<T: ?Sized + ThreadedHandler, A: Allocator> ThreadedHandler for Box<T, A> { /// // This is executing in process context and assumes that `request` was /// // previously acquired from a device. /// fn register_threaded_irq( -/// handler: impl PinInit<Data, Error>, /// request: IrqRequest<'_>, -/// ) -> Result<Arc<ThreadedRegistration<Data>>> { -/// let registration = -/// ThreadedRegistration::new(request, Flags::SHARED, c"my_device", handler); +/// ) -> Result<Pin<KBox<ThreadedRegistration<'_, Data>>>> { +/// // SAFETY: The returned Registration is not leaked. +/// let registration = unsafe { +/// ThreadedRegistration::new( +/// request, +/// Flags::SHARED, +/// c"my_device", +/// try_pin_init!(Data { +/// value <- kernel::new_mutex!(0), +/// }? Error), +/// ) +/// }; /// -/// let registration = Arc::pin_init(registration, GFP_KERNEL)?; +/// let registration = KBox::pin_init(registration, GFP_KERNEL)?; /// /// { /// // The data can be accessed from process context too. @@ -399,11 +368,11 @@ impl<T: ?Sized + ThreadedHandler, A: Allocator> ThreadedHandler for Box<T, A> { /// /// # Invariants /// -/// * We own an irq handler whose cookie is a pointer to `Self`. -#[pin_data] -pub struct ThreadedRegistration<T: ThreadedHandler + 'static> { - #[pin] - inner: Devres<RegistrationInner>, +/// * We own an irq handler registered via `request_threaded_irq` whose cookie is a pointer to +/// `Self`. +#[pin_data(PinnedDrop)] +pub struct ThreadedRegistration<'a, T: ThreadedHandler> { + request: IrqRequest<'a>, #[pin] handler: T, @@ -414,45 +383,47 @@ pub struct ThreadedRegistration<T: ThreadedHandler + 'static> { _pin: PhantomPinned, } -impl<T: ThreadedHandler + 'static> ThreadedRegistration<T> { +impl<'a, T: ThreadedHandler> ThreadedRegistration<'a, T> { /// Registers the IRQ handler with the system for the given IRQ number. - pub fn new<'a>( + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the returned [`ThreadedRegistration`] or otherwise prevent + /// its [`Drop`] implementation from running. + pub unsafe fn new( request: IrqRequest<'a>, flags: Flags, name: &'static CStr, handler: impl PinInit<T, Error> + 'a, - ) -> impl PinInit<Self, Error> + 'a { + ) -> impl PinInit<Self, Error> + 'a + where + T: 'a, + { + // INVARIANT: If initialization completes successfully, we own an IRQ handler registered + // via `request_threaded_irq` whose cookie is a pointer to `Self`. try_pin_init!(&this in Self { handler <- handler, - inner <- Devres::new( - request.dev, - try_pin_init!(RegistrationInner { - // INVARIANT: `this` is a valid pointer to the `ThreadedRegistration` instance. - cookie: this.as_ptr().cast::<c_void>(), - irq: { - // SAFETY: - // - The callbacks are valid for use with request_threaded_irq. - // - If this succeeds, the slot is guaranteed to be valid until the - // destructor of Self runs, which will deregister the callbacks - // before the memory location becomes invalid. - // - When request_threaded_irq is called, everything that the two callbacks - // will touch has already been initialized, so it's safe for the - // callbacks to be called immediately. - to_result(unsafe { - bindings::request_threaded_irq( - request.irq, - Some(handle_threaded_irq_callback::<T>), - Some(thread_fn_callback::<T>), - flags.into_inner(), - name.as_char_ptr(), - this.as_ptr().cast::<c_void>(), - ) - })?; - request.irq - } - }) - ), + request, _pin: PhantomPinned, + _: { + // SAFETY: + // - The callbacks are valid for use with request_threaded_irq. + // - If this succeeds, the slot is guaranteed to be valid until the destructor of + // Self runs, which will deregister the callbacks before the memory location + // becomes invalid. + // - All fields are already initialized, so it's safe for the callbacks to be + // called immediately. + to_result(unsafe { + bindings::request_threaded_irq( + request.irq, + Some(handle_threaded_irq_callback::<T>), + Some(thread_fn_callback::<T>), + flags.into_inner(), + name.as_char_ptr(), + this.as_ptr().cast::<c_void>(), + ) + })?; + }, }) } @@ -462,50 +433,51 @@ impl<T: ThreadedHandler + 'static> ThreadedRegistration<T> { } /// Wait for pending IRQ handlers on other CPUs. - /// - /// This will attempt to access the inner [`Devres`] container. - pub fn try_synchronize(&self) -> Result { - let inner = self.inner.try_access().ok_or(ENODEV)?; - inner.synchronize(); - Ok(()) + #[inline] + pub fn synchronize(&self) { + // SAFETY: `self.request.irq` is a valid registered IRQ number (type invariant). + unsafe { bindings::synchronize_irq(self.request.irq) }; } +} - /// Wait for pending IRQ handlers on other CPUs. - pub fn synchronize(&self, dev: &Device<Bound>) -> Result { - let inner = self.inner.access(dev)?; - inner.synchronize(); - Ok(()) +#[pinned_drop] +impl<T: ThreadedHandler> PinnedDrop for ThreadedRegistration<'_, T> { + fn drop(self: Pin<&mut Self>) { + // SAFETY: The cookie was set to a pointer to `Self` in `ThreadedRegistration::new()`. This + // blocks until all in-flight handlers complete, so no references to `self` remain after + // this returns. + unsafe { + bindings::free_irq( + self.request.irq, + core::ptr::from_mut::<Self>(self.get_unchecked_mut()).cast::<c_void>(), + ) + }; } } /// # Safety /// /// This function should be only used as the callback in `request_threaded_irq`. -unsafe extern "C" fn handle_threaded_irq_callback<T: ThreadedHandler + 'static>( +unsafe extern "C" fn handle_threaded_irq_callback<T: ThreadedHandler>( _irq: i32, ptr: *mut c_void, ) -> c_uint { - // SAFETY: `ptr` is a pointer to `ThreadedRegistration<T>` set in `ThreadedRegistration::new` - let registration = unsafe { &*(ptr as *const ThreadedRegistration<T>) }; - // SAFETY: The irq callback is removed before the device is unbound, so the fact that the irq - // callback is running implies that the device has not yet been unbound. - let device = unsafe { registration.inner.device().as_bound() }; + let ptr = ptr.cast_const().cast::<ThreadedRegistration<'_, T>>(); + // SAFETY: `ptr` is a pointer to `ThreadedRegistration<'_, T>` set in + // `ThreadedRegistration::new()`. + let registration = unsafe { &*ptr }; - T::handle(®istration.handler, device) as c_uint + T::handle(®istration.handler) as c_uint } /// # Safety /// /// This function should be only used as the callback in `request_threaded_irq`. -unsafe extern "C" fn thread_fn_callback<T: ThreadedHandler + 'static>( - _irq: i32, - ptr: *mut c_void, -) -> c_uint { - // SAFETY: `ptr` is a pointer to `ThreadedRegistration<T>` set in `ThreadedRegistration::new` - let registration = unsafe { &*(ptr as *const ThreadedRegistration<T>) }; - // SAFETY: The irq callback is removed before the device is unbound, so the fact that the irq - // callback is running implies that the device has not yet been unbound. - let device = unsafe { registration.inner.device().as_bound() }; +unsafe extern "C" fn thread_fn_callback<T: ThreadedHandler>(_irq: i32, ptr: *mut c_void) -> c_uint { + let ptr = ptr.cast_const().cast::<ThreadedRegistration<'_, T>>(); + // SAFETY: `ptr` is a pointer to `ThreadedRegistration<'_, T>` set in + // `ThreadedRegistration::new()`. + let registration = unsafe { &*ptr }; - T::handle_threaded(®istration.handler, device) as c_uint + T::handle_threaded(®istration.handler) as c_uint } diff --git a/rust/kernel/jump_label.rs b/rust/kernel/jump_label.rs index 4e974c768dbd..f54cedcb6fd5 100644 --- a/rust/kernel/jump_label.rs +++ b/rust/kernel/jump_label.rs @@ -44,6 +44,7 @@ const _: &str = include!(concat!( #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(CONFIG_JUMP_LABEL)] macro_rules! arch_static_branch { ($key:path, $keytyp:ty, $field:ident, $branch:expr) => {'my_label: { @@ -61,6 +62,17 @@ macro_rules! arch_static_branch { }}; } +#[macro_export] +#[doc(hidden)] +#[cfg(testlib)] +#[cfg(CONFIG_JUMP_LABEL)] +macro_rules! arch_static_branch { + ($key:path, $keytyp:ty, $field:ident, $branch:expr) => { + // The asm falls through until patched, which never happens on the host. + false + }; +} + #[cfg(CONFIG_JUMP_LABEL)] pub use arch_static_branch; diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs index a1edf7491579..91eaff8c186a 100644 --- a/rust/kernel/kunit.rs +++ b/rust/kernel/kunit.rs @@ -288,6 +288,7 @@ macro_rules! kunit_unsafe_test_suite { log: ::core::ptr::null_mut(), suite_init_err: 0, is_init: false, + status: kernel::bindings::kunit_status_KUNIT_SUCCESS, }; #[used(compiler)] @@ -329,6 +330,7 @@ pub fn in_kunit_test() -> bool { !unsafe { bindings::kunit_get_current_test() }.is_null() } +#[cfg(CONFIG_RUST_KUNIT_SELFTEST)] #[kunit_tests(rust_kernel_kunit)] mod tests { use super::*; diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index d93292d47420..4d5c96ddc49c 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -16,45 +16,20 @@ // Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on // the unstable features in use. // -// Stable since Rust 1.79.0. -#![feature(generic_nonzero)] -#![feature(inline_const)] -#![feature(pointer_is_aligned)] -#![feature(slice_ptr_len)] +// Stable since Rust 1.87.0. +#![feature(unsigned_is_multiple_of)] // -// Stable since Rust 1.80.0. -#![feature(slice_flatten)] -// -// Stable since Rust 1.81.0. -#![feature(lint_reasons)] -// -// Stable since Rust 1.82.0. -#![feature(raw_ref_op)] -// -// Stable since Rust 1.83.0. -#![feature(const_maybe_uninit_as_mut_ptr)] -#![feature(const_mut_refs)] -#![feature(const_option)] -#![feature(const_ptr_write)] -#![feature(const_refs_to_cell)] -// -// Stable since Rust 1.84.0. -#![feature(strict_provenance)] +// Stable since Rust 1.89.0. +#![feature(generic_arg_infer)] // // Expected to become stable. #![feature(arbitrary_self_types)] +#![feature(derive_coerce_pointee)] // // To be determined. #![feature(used_with_arg)] // -// `feature(derive_coerce_pointee)` is expected to become stable. Before Rust -// 1.84.0, it did not exist, so enable the predecessor features. -#![cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, feature(derive_coerce_pointee))] -#![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(coerce_unsized))] -#![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(dispatch_from_dyn))] -#![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(unsize))] -// -// `feature(file_with_nul)` is expected to become stable. Before Rust 1.89.0, it did not exist, so +// `feature(file_with_nul)` is stable since Rust 1.92.0. Before Rust 1.89.0, it did not exist, so // enable it conditionally. #![cfg_attr(CONFIG_RUSTC_HAS_FILE_WITH_NUL, feature(file_with_nul))] @@ -72,12 +47,12 @@ pub mod acpi; pub mod alloc; #[cfg(CONFIG_AUXILIARY_BUS)] pub mod auxiliary; +pub mod bitfield; pub mod bitmap; pub mod bits; #[cfg(CONFIG_BLOCK)] pub mod block; pub mod bug; -#[doc(hidden)] pub mod build_assert; pub mod clk; #[cfg(CONFIG_CONFIGFS_FS)] @@ -101,12 +76,18 @@ pub mod faux; pub mod firmware; pub mod fmt; pub mod fs; +#[cfg(CONFIG_RUST_FWCTL_ABSTRACTIONS)] +pub mod fwctl; +#[cfg(CONFIG_GPU_BUDDY = "y")] +pub mod gpu; #[cfg(CONFIG_I2C = "y")] pub mod i2c; pub mod id_pool; #[doc(hidden)] pub mod impl_flags; pub mod init; +pub mod interop; +pub mod interrupt; pub mod io; pub mod ioctl; pub mod iommu; @@ -119,6 +100,7 @@ pub mod list; pub mod maple_tree; pub mod miscdevice; pub mod mm; +pub mod module; pub mod module_param; #[cfg(CONFIG_NET)] pub mod net; @@ -144,11 +126,11 @@ pub mod safety; pub mod scatterlist; pub mod security; pub mod seq_file; +#[cfg(CONFIG_RUST_SERIAL_DEV_BUS_ABSTRACTIONS)] +pub mod serdev; pub mod sizes; -pub mod slice; #[cfg(CONFIG_SOC_BUS)] pub mod soc; -mod static_assert; #[doc(hidden)] pub mod std_vendor; pub mod str; @@ -167,77 +149,29 @@ pub mod xarray; #[doc(hidden)] pub use bindings; pub use macros; +pub use module::{ + InPlaceModule, + Module, + ModuleMetadata, + ThisModule, // +}; pub use uapi; /// Prefix to appear before log messages printed from within the `kernel` crate. const __LOG_PREFIX: &[u8] = b"rust_kernel\0"; -/// The top level entrypoint to implementing a kernel module. -/// -/// For any teardown or cleanup operations, your type may implement [`Drop`]. -pub trait Module: Sized + Sync + Send { - /// Called at module initialization time. - /// - /// Use this method to perform whatever setup or registration your module - /// should do. - /// - /// Equivalent to the `module_init` macro in the C API. - fn init(module: &'static ThisModule) -> error::Result<Self>; -} - -/// A module that is pinned and initialised in-place. -pub trait InPlaceModule: Sync + Send { - /// Creates an initialiser for the module. - /// - /// It is called when the module is loaded. - fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error>; -} - -impl<T: Module> InPlaceModule for T { - fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error> { - let initer = move |slot: *mut Self| { - let m = <Self as Module>::init(module)?; +/// Dummy module type for `#[vtable]` `impl` blocks within the `kernel` crate (e.g. KUnit tests). +// The `allow` is needed since it may be unused (e.g. KUnit tests may be disabled). +#[allow(dead_code)] +struct LocalModule; - // SAFETY: `slot` is valid for write per the contract with `pin_init_from_closure`. - unsafe { slot.write(m) }; - Ok(()) - }; +impl ModuleMetadata for LocalModule { + const NAME: &'static str::CStr = c"rust_kernel"; - // SAFETY: On success, `initer` always fully initialises an instance of `Self`. - unsafe { pin_init::pin_init_from_closure(initer) } - } -} - -/// Metadata attached to a [`Module`] or [`InPlaceModule`]. -pub trait ModuleMetadata { - /// The name of the module as specified in the `module!` macro. - const NAME: &'static crate::str::CStr; -} - -/// Equivalent to `THIS_MODULE` in the C API. -/// -/// C header: [`include/linux/init.h`](srctree/include/linux/init.h) -pub struct ThisModule(*mut bindings::module); - -// SAFETY: `THIS_MODULE` may be used from all threads within a module. -unsafe impl Sync for ThisModule {} - -impl ThisModule { - /// Creates a [`ThisModule`] given the `THIS_MODULE` pointer. - /// - /// # Safety - /// - /// The pointer must be equal to the right `THIS_MODULE`. - pub const unsafe fn from_ptr(ptr: *mut bindings::module) -> ThisModule { - ThisModule(ptr) - } - - /// Access the raw pointer for this module. - /// - /// It is up to the user to use it correctly. - pub const fn as_ptr(&self) -> *mut bindings::module { - self.0 - } + const THIS_MODULE: ThisModule = { + // SAFETY: `try_module_get`/`module_put` handle null module pointers gracefully. + unsafe { ThisModule::from_ptr(core::ptr::null_mut()) } + }; } #[cfg(not(testlib))] diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs index 8349ff32fc37..0f367264ee2e 100644 --- a/rust/kernel/list.rs +++ b/rust/kernel/list.rs @@ -12,15 +12,31 @@ use core::ptr; use pin_init::PinInit; mod impl_list_item_mod; +#[doc(inline)] pub use self::impl_list_item_mod::{ - impl_has_list_links, impl_has_list_links_self_ptr, impl_list_item, HasListLinks, HasSelfPtr, + impl_has_list_links, + impl_has_list_links_self_ptr, + impl_list_item, + HasListLinks, + HasSelfPtr, // }; mod arc; -pub use self::arc::{impl_list_arc_safe, AtomicTracker, ListArc, ListArcSafe, TryNewListArc}; +#[doc(inline)] +pub use self::arc::{ + impl_list_arc_safe, + AtomicTracker, + ListArc, + ListArcSafe, + TryNewListArc, // +}; mod arc_field; -pub use self::arc_field::{define_list_arc_field_getter, ListArcField}; +#[doc(inline)] +pub use self::arc_field::{ + define_list_arc_field_getter, + ListArcField, // +}; /// A linked list. /// @@ -233,7 +249,7 @@ pub use self::arc_field::{define_list_arc_field_getter, ListArcField}; /// assert_eq!(list.iter().count(), 3); /// } /// -/// // Pop the items from the list using `pop_front()` and verify the content. +/// // Pop the items from the list using `pop_back()` and verify the content. /// { /// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 15)); /// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 32)); diff --git a/rust/kernel/list/arc.rs b/rust/kernel/list/arc.rs index 2282f33913ee..209b1173c826 100644 --- a/rust/kernel/list/arc.rs +++ b/rust/kernel/list/arc.rs @@ -6,7 +6,7 @@ use crate::alloc::{AllocError, Flags}; use crate::prelude::*; -use crate::sync::atomic::{ordering, Atomic}; +use crate::sync::atomic::{ordering, AtomicFlag}; use crate::sync::{Arc, ArcBorrow, UniqueArc}; use core::marker::PhantomPinned; use core::ops::Deref; @@ -82,6 +82,7 @@ pub unsafe trait TryNewListArc<const ID: u64 = 0>: ListArcSafe<ID> { /// [`AtomicTracker`]. However, it is also possible to defer the tracking to another struct /// using also using this macro. #[macro_export] +#[doc(hidden)] macro_rules! impl_list_arc_safe { (impl$({$($generics:tt)*})? ListArcSafe<$num:tt> for $t:ty { untracked; } $($rest:tt)*) => { impl$(<$($generics)*>)? $crate::list::ListArcSafe<$num> for $t { @@ -159,7 +160,7 @@ pub use impl_list_arc_safe; /// /// [`List`]: crate::list::List #[repr(transparent)] -#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))] +#[derive(core::marker::CoercePointee)] pub struct ListArc<T, const ID: u64 = 0> where T: ListArcSafe<ID> + ?Sized, @@ -442,26 +443,6 @@ where } } -// This is to allow coercion from `ListArc<T>` to `ListArc<U>` if `T` can be converted to the -// dynamically-sized type (DST) `U`. -#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))] -impl<T, U, const ID: u64> core::ops::CoerceUnsized<ListArc<U, ID>> for ListArc<T, ID> -where - T: ListArcSafe<ID> + core::marker::Unsize<U> + ?Sized, - U: ListArcSafe<ID> + ?Sized, -{ -} - -// This is to allow `ListArc<U>` to be dispatched on when `ListArc<T>` can be coerced into -// `ListArc<U>`. -#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))] -impl<T, U, const ID: u64> core::ops::DispatchFromDyn<ListArc<U, ID>> for ListArc<T, ID> -where - T: ListArcSafe<ID> + core::marker::Unsize<U> + ?Sized, - U: ListArcSafe<ID> + ?Sized, -{ -} - /// A utility for tracking whether a [`ListArc`] exists using an atomic. /// /// # Invariants @@ -469,7 +450,7 @@ where /// If the boolean is `false`, then there is no [`ListArc`] for this value. #[repr(transparent)] pub struct AtomicTracker<const ID: u64 = 0> { - inner: Atomic<bool>, + inner: AtomicFlag, // This value needs to be pinned to justify the INVARIANT: comment in `AtomicTracker::new`. _pin: PhantomPinned, } @@ -480,12 +461,12 @@ impl<const ID: u64> AtomicTracker<ID> { // INVARIANT: Pin-init initializers can't be used on an existing `Arc`, so this value will // not be constructed in an `Arc` that already has a `ListArc`. Self { - inner: Atomic::new(false), + inner: AtomicFlag::new(false), _pin: PhantomPinned, } } - fn project_inner(self: Pin<&mut Self>) -> &mut Atomic<bool> { + fn project_inner(self: Pin<&mut Self>) -> &mut AtomicFlag { // SAFETY: The `inner` field is not structurally pinned, so we may obtain a mutable // reference to it even if we only have a pinned reference to `self`. unsafe { &mut Pin::into_inner_unchecked(self).inner } diff --git a/rust/kernel/list/arc_field.rs b/rust/kernel/list/arc_field.rs index c4b9dd503982..2ad8aea55993 100644 --- a/rust/kernel/list/arc_field.rs +++ b/rust/kernel/list/arc_field.rs @@ -66,6 +66,7 @@ impl<T, const ID: u64> ListArcField<T, ID> { /// Defines getters for a [`ListArcField`]. #[macro_export] +#[doc(hidden)] macro_rules! define_list_arc_field_getter { ($pub:vis fn $name:ident(&self $(<$id:tt>)?) -> &$typ:ty { $field:ident } $($rest:tt)* diff --git a/rust/kernel/list/impl_list_item_mod.rs b/rust/kernel/list/impl_list_item_mod.rs index ee53d0387e63..5a3eac9f3cf0 100644 --- a/rust/kernel/list/impl_list_item_mod.rs +++ b/rust/kernel/list/impl_list_item_mod.rs @@ -29,6 +29,7 @@ pub unsafe trait HasListLinks<const ID: u64 = 0> { /// Implements the [`HasListLinks`] trait for the given type. #[macro_export] +#[doc(hidden)] macro_rules! impl_has_list_links { ($(impl$({$($generics:tt)*})? HasListLinks$(<$id:tt>)? @@ -74,6 +75,7 @@ where /// Implements the [`HasListLinks`] and [`HasSelfPtr`] traits for the given type. #[macro_export] +#[doc(hidden)] macro_rules! impl_has_list_links_self_ptr { ($(impl$({$($generics:tt)*})? HasSelfPtr<$item_type:ty $(, $id:tt)?> @@ -181,6 +183,7 @@ pub use impl_has_list_links_self_ptr; /// } /// ``` #[macro_export] +#[doc(hidden)] macro_rules! impl_list_item { ( $(impl$({$($generics:tt)*})? ListItem<$num:tt> for $self:ty { diff --git a/rust/kernel/miscdevice.rs b/rust/kernel/miscdevice.rs index c3c2052c9206..8d4b719bd83f 100644 --- a/rust/kernel/miscdevice.rs +++ b/rust/kernel/miscdevice.rs @@ -11,16 +11,28 @@ use crate::{ bindings, device::Device, - error::{to_result, Error, Result, VTABLE_DEFAULT_ERROR}, - ffi::{c_int, c_long, c_uint, c_ulong}, - fs::{File, Kiocb}, - iov::{IovIterDest, IovIterSource}, + error::{ + to_result, + VTABLE_DEFAULT_ERROR, // + }, + fs::{ + File, + Kiocb, // + }, + iov::{ + IovIterDest, + IovIterSource, // + }, mm::virt::VmaNew, + module::this_module, prelude::*, seq_file::SeqFile, - types::{ForeignOwnable, Opaque}, + types::{ + ForeignOwnable, + Opaque, // + }, // }; -use core::{marker::PhantomData, pin::Pin}; +use core::marker::PhantomData; /// Options for creating a misc device. #[derive(Copy, Clone)] @@ -278,7 +290,7 @@ impl<T: MiscDevice> MiscdeviceVTable<T> { /// # Safety /// /// `kiocb` must be correspond to a valid file that is associated with a - /// `MiscDeviceRegistration<T>`. `iter` must be a valid `struct iov_iter` for writing. + /// `MiscDeviceRegistration<T>`. `iter` must be a valid `struct iov_iter` for reading. unsafe extern "C" fn write_iter( kiocb: *mut bindings::kiocb, iter: *mut bindings::iov_iter, @@ -419,6 +431,7 @@ impl<T: MiscDevice> MiscdeviceVTable<T> { } else { None }, + owner: this_module::<T::OwnerModule>().as_ptr(), ..pin_init::zeroed() }; diff --git a/rust/kernel/mm/virt.rs b/rust/kernel/mm/virt.rs index da21d65ccd20..63eb730b0b05 100644 --- a/rust/kernel/mm/virt.rs +++ b/rust/kernel/mm/virt.rs @@ -113,7 +113,7 @@ impl VmaRef { /// kernel goes further in freeing unused page tables, but for the purposes of this operation /// we must only assume that the leaf level is cleared. #[inline] - pub fn zap_page_range_single(&self, address: usize, size: usize) { + pub fn zap_vma_range(&self, address: usize, size: usize) { let (end, did_overflow) = address.overflowing_add(size); if did_overflow || address < self.start() || self.end() < end { // TODO: call WARN_ONCE once Rust version of it is added @@ -123,9 +123,7 @@ impl VmaRef { // SAFETY: By the type invariants, the caller has read access to this VMA, which is // sufficient for this method call. This method has no requirements on the vma flags. The // address range is checked to be within the vma. - unsafe { - bindings::zap_page_range_single(self.as_ptr(), address, size, core::ptr::null_mut()) - }; + unsafe { bindings::zap_vma_range(self.as_ptr(), address, size) }; } /// If the [`VM_MIXEDMAP`] flag is set, returns a [`VmaMixedMap`] to this VMA, otherwise diff --git a/rust/kernel/module.rs b/rust/kernel/module.rs new file mode 100644 index 000000000000..d71370598447 --- /dev/null +++ b/rust/kernel/module.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Module-related types and helpers. + +/// The entrypoint to implementing a kernel module. +/// +/// For any teardown or cleanup operations, your type may implement [`Drop`]. +pub trait Module: Sized + Sync + Send { + /// Called at module initialization time. + /// + /// Use this method to perform whatever setup or registration your module + /// should do. + /// + /// Equivalent to the `module_init` macro in the C API. + fn init(module: &'static ThisModule) -> crate::error::Result<Self>; +} + +/// A module that is pinned and initialised in-place. +pub trait InPlaceModule: Sync + Send { + /// Creates an initialiser for the module. + /// + /// It is called when the module is loaded. + fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, crate::error::Error>; +} + +impl<T: Module> InPlaceModule for T { + fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, crate::error::Error> { + let initer = move |slot: *mut Self| { + let m = <Self as Module>::init(module)?; + + // SAFETY: `slot` is valid for write per the contract with `pin_init_from_closure`. + unsafe { slot.write(m) }; + Ok(()) + }; + + // SAFETY: On success, `initer` always fully initialises an instance of `Self`. + unsafe { pin_init::pin_init_from_closure(initer) } + } +} + +/// Metadata attached to a [`Module`] or [`InPlaceModule`]. +pub trait ModuleMetadata { + /// The name of the module as specified in the `module!` macro. + const NAME: &'static crate::str::CStr; + + /// The module's `THIS_MODULE` pointer. + const THIS_MODULE: ThisModule; +} + +/// Returns a reference to the `THIS_MODULE` of the given module type. +#[inline] +pub const fn this_module<M: ModuleMetadata>() -> &'static ThisModule { + &M::THIS_MODULE +} + +/// Equivalent to `THIS_MODULE` in the C API. +/// +/// C header: [`include/linux/init.h`](srctree/include/linux/init.h) +pub struct ThisModule(*mut crate::bindings::module); + +// SAFETY: `THIS_MODULE` may be used from all threads within a module. +unsafe impl Sync for ThisModule {} + +impl ThisModule { + /// Creates a [`ThisModule`] given the `THIS_MODULE` pointer. + /// + /// # Safety + /// + /// The pointer must be equal to the right `THIS_MODULE`. + pub const unsafe fn from_ptr(ptr: *mut crate::bindings::module) -> ThisModule { + ThisModule(ptr) + } + + /// Access the raw pointer for this module. + /// + /// It is up to the user to use it correctly. + pub const fn as_ptr(&self) -> *mut crate::bindings::module { + self.0 + } +} diff --git a/rust/kernel/module_param.rs b/rust/kernel/module_param.rs index 6a8a7a875643..f9a14765a926 100644 --- a/rust/kernel/module_param.rs +++ b/rust/kernel/module_param.rs @@ -5,7 +5,7 @@ //! C header: [`include/linux/moduleparam.h`](srctree/include/linux/moduleparam.h) use crate::prelude::*; -use crate::str::BStr; +use crate::str::{kstrtobool_bytes, BStr}; use bindings; use kernel::sync::SetOnce; @@ -62,8 +62,7 @@ where // NOTE: If we start supporting arguments without values, val _is_ allowed // to be null here. if val.is_null() { - // TODO: Use pr_warn_once available. - crate::pr_warn!("Null pointer passed to `module_param::set_param`"); + crate::pr_warn_once!("Null pointer passed to `module_param::set_param`\n"); return EINVAL.to_errno(); } @@ -106,6 +105,12 @@ impl_int_module_param!(u64); impl_int_module_param!(isize); impl_int_module_param!(usize); +impl ModuleParam for bool { + fn try_from_param_arg(arg: &BStr) -> Result<Self> { + kstrtobool_bytes(arg) + } +} + /// A wrapper for kernel parameters. /// /// This type is instantiated by the [`module!`] macro when module parameters are @@ -131,10 +136,26 @@ impl<T> ModuleParamAccess<T> { } } + /// Get a copy of the parameter value. + /// + /// Returns the value supplied at module load time, or the default value + /// if the parameter has not been set. + #[inline] + pub fn value(&self) -> T + where + T: Copy, + { + self.value.copy().unwrap_or(self.default) + } + /// Get a shared reference to the parameter value. + /// + /// Returns a reference to the value supplied at module load time, or a + /// reference to the default value if the parameter has not been set. // Note: When sysfs access to parameters are enabled, we have to pass in a // held lock guard here. - pub fn value(&self) -> &T { + #[inline] + pub fn value_ref(&self) -> &T { self.value.as_ref().unwrap_or(&self.default) } @@ -180,3 +201,4 @@ make_param_ops!(PARAM_OPS_I64, i64); make_param_ops!(PARAM_OPS_U64, u64); make_param_ops!(PARAM_OPS_ISIZE, isize); make_param_ops!(PARAM_OPS_USIZE, usize); +make_param_ops!(PARAM_OPS_BOOL, bool); diff --git a/rust/kernel/net.rs b/rust/kernel/net/mod.rs index fe415cb369d3..8ecae7577ed2 100644 --- a/rust/kernel/net.rs +++ b/rust/kernel/net/mod.rs @@ -4,3 +4,5 @@ #[cfg(CONFIG_RUST_PHYLIB_ABSTRACTIONS)] pub mod phy; + +pub mod netlink; diff --git a/rust/kernel/net/netlink.rs b/rust/kernel/net/netlink.rs new file mode 100644 index 000000000000..22ef3dde36fa --- /dev/null +++ b/rust/kernel/net/netlink.rs @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2026 Google LLC. + +//! Rust support for generic netlink. +//! +//! Currently only supports exposing multicast groups. +//! +//! C header: [`include/net/genetlink.h`](srctree/include/net/genetlink.h) + +use kernel::{ + alloc::{self, AllocError}, + error::to_result, + prelude::*, + transmute::AsBytes, + types::Opaque, + ThisModule, +}; + +use core::{ + mem::ManuallyDrop, + ptr::NonNull, // +}; + +/// The default netlink message size. +pub const GENLMSG_DEFAULT_SIZE: usize = bindings::GENLMSG_DEFAULT_SIZE; + +/// A wrapper around `struct sk_buff` for generic netlink messages. +/// +/// This type is intended to be specific for buffers used with netlink only, and other usecases for +/// `struct sk_buff` are out-of-scope for this abstraction. +/// +/// # Invariants +/// +/// The pointer has ownership over a valid `sk_buff`. +pub struct NetlinkSkBuff { + skb: NonNull<kernel::bindings::sk_buff>, +} + +impl NetlinkSkBuff { + /// Creates a new `NetlinkSkBuff` with the given size. + pub fn new(size: usize, flags: alloc::Flags) -> Result<NetlinkSkBuff, AllocError> { + // SAFETY: `genlmsg_new` only requires its arguments to be valid integers. + let skb = unsafe { bindings::genlmsg_new(size, flags.as_raw()) }; + let skb = NonNull::new(skb).ok_or(AllocError)?; + Ok(NetlinkSkBuff { skb }) + } + + /// Puts a generic netlink header into the `NetlinkSkBuff`. + pub fn genlmsg_put( + self, + portid: u32, + seq: u32, + family: &'static Family, + cmd: u8, + ) -> Result<GenlMsg, AllocError> { + let skb = self.skb.as_ptr(); + // SAFETY: The skb and family pointers are valid. + let hdr = unsafe { bindings::genlmsg_put(skb, portid, seq, family.as_raw(), 0, cmd) }; + let hdr = NonNull::new(hdr).ok_or(AllocError)?; + Ok(GenlMsg { skb: self, hdr }) + } +} + +impl Drop for NetlinkSkBuff { + fn drop(&mut self) { + // SAFETY: We have ownership over the `sk_buff`, so we may free it. + unsafe { bindings::nlmsg_free(self.skb.as_ptr()) } + } +} + +/// A generic netlink message being constructed. +/// +/// # Invariants +/// +/// `hdr` references the header in this netlink message. +pub struct GenlMsg { + skb: NetlinkSkBuff, + hdr: NonNull<c_void>, +} + +impl GenlMsg { + /// Puts an attribute into the message. + #[inline] + fn put<T>(&mut self, attrtype: c_int, value: &T) -> Result + where + T: ?Sized + AsBytes, + { + let skb = self.skb.skb.as_ptr(); + let len = size_of_val(value); + let ptr = core::ptr::from_ref(value).cast::<c_void>(); + // SAFETY: `skb` is valid by `NetlinkSkBuff` type invariants, and the provided value is + // readable and initialized for its `size_of` bytes. + to_result(unsafe { bindings::nla_put(skb, attrtype, len as c_int, ptr) }) + } + + /// Puts a `u32` attribute into the message. + #[inline] + pub fn put_u32(&mut self, attrtype: c_int, value: u32) -> Result { + self.put(attrtype, &value) + } + + /// Puts a string attribute into the message. + #[inline] + pub fn put_string(&mut self, attrtype: c_int, value: &CStr) -> Result { + self.put(attrtype, value.to_bytes_with_nul()) + } + + /// Puts a flag attribute into the message. + #[inline] + pub fn put_flag(&mut self, attrtype: c_int) -> Result { + let skb = self.skb.skb.as_ptr(); + // SAFETY: `skb` is valid by `NetlinkSkBuff` type invariants, and a null pointer is valid + // when the length is zero. + to_result(unsafe { bindings::nla_put(skb, attrtype, 0, core::ptr::null()) }) + } + + /// Sends the generic netlink message as a multicast message. + #[inline] + pub fn multicast( + self, + family: &'static Family, + portid: u32, + group: u32, + flags: alloc::Flags, + ) -> Result { + let me = ManuallyDrop::new(self); + // SAFETY: The `skb` and `family` pointers are valid. We pass ownership of the `skb` to + // `genlmsg_multicast` by not dropping `self`. + unsafe { + bindings::genlmsg_end(me.skb.skb.as_ptr(), me.hdr.as_ptr()); + to_result(bindings::genlmsg_multicast( + family.as_raw(), + me.skb.skb.as_ptr(), + portid, + group, + flags.as_raw(), + )) + } + } +} +impl Drop for GenlMsg { + fn drop(&mut self) { + // SAFETY: The `hdr` pointer references the header of this generic netlink message. + unsafe { bindings::genlmsg_cancel(self.skb.skb.as_ptr(), self.hdr.as_ptr()) }; + } +} + +/// Flags for a generic netlink family. +struct FamilyFlags { + /// Whether the family supports network namespaces. + netnsok: bool, + /// Whether the family supports parallel operations. + parallel_ops: bool, +} + +impl FamilyFlags { + /// Converts the flags to the bitfield representation used by `genl_family`. + const fn into_bitfield(self) -> bindings::__BindgenBitfieldUnit<[u8; 1]> { + // The below shifts are verified correct by test_family_flags_bitfield() below. + // + // Although bindgen generates helpers to change bitfields based on the C headers, these + // helpers unfortunately can't be used in const context. Since `Family` needs to be filled + // out at build-time, we use this helper instead. + let mut bits = 0; + if self.netnsok { + bits |= 1 << 0; + } + if self.parallel_ops { + bits |= 1 << 1; + } + // Convert from little endian to the target's endianness. + bits = u8::from_le(bits); + // SAFETY: This bitfield is represented as an u8. + unsafe { core::mem::transmute::<u8, bindings::__BindgenBitfieldUnit<[u8; 1]>>(bits) } + } +} + +/// A generic netlink family. +#[repr(transparent)] +pub struct Family { + inner: Opaque<bindings::genl_family>, +} + +// SAFETY: The `Family` type is thread safe. +unsafe impl Sync for Family {} + +impl Family { + /// Creates a new `Family` instance. + /// + /// Intended to be used from const context only. Will panic if provided with invalid arguments. + /// + /// The name must be a nul-terminated string, but it is taken as `&[u8]` so that it can be used + /// more conveniently with the strings generated by bindgen. + pub const fn const_new( + module: &ThisModule, + name: &[u8], + version: u32, + mcgrps: &'static [MulticastGroup], + ) -> Family { + let n_mcgrps = mcgrps.len() as u8; + if n_mcgrps as usize != mcgrps.len() { + panic!("too many mcgrps"); + } + let mut genl_family = bindings::genl_family { + version, + _bitfield_1: FamilyFlags { + netnsok: true, + parallel_ops: true, + } + .into_bitfield(), + module: module.as_ptr(), + mcgrps: mcgrps.as_ptr().cast(), + n_mcgrps, + ..pin_init::zeroed() + }; + if CStr::from_bytes_with_nul(name).is_err() { + panic!("genl_family name not nul-terminated"); + } + if genl_family.name.len() < name.len() { + panic!("genl_family name too long"); + } + let mut i = 0; + while i < name.len() { + genl_family.name[i] = name[i]; + i += 1; + } + Family { + inner: Opaque::new(genl_family), + } + } + + /// Checks if there are any listeners for the given multicast group. + pub fn has_listeners(&self, group: u32) -> bool { + // SAFETY: The family and init_net pointers are valid. + unsafe { + bindings::genl_has_listeners(self.as_raw(), &raw mut bindings::init_net, group) != 0 + } + } + + /// Returns a raw pointer to the underlying `genl_family` structure. + pub fn as_raw(&self) -> *mut bindings::genl_family { + self.inner.get() + } +} + +/// A generic netlink multicast group. +#[repr(transparent)] +pub struct MulticastGroup { + // No Opaque because fully immutable + group: bindings::genl_multicast_group, +} + +// SAFETY: Pure data so thread safe. +unsafe impl Sync for MulticastGroup {} + +impl MulticastGroup { + /// Creates a new `MulticastGroup` instance. + /// + /// Intended to be used from const context only. Will panic if provided with invalid arguments. + pub const fn const_new(name: &CStr) -> MulticastGroup { + let mut group: bindings::genl_multicast_group = pin_init::zeroed(); + + let name = name.to_bytes_with_nul(); + if group.name.len() < name.len() { + panic!("genl_multicast_group name too long"); + } + let mut i = 0; + while i < name.len() { + group.name[i] = name[i]; + i += 1; + } + + MulticastGroup { group } + } +} + +/// A registration of a generic netlink family. +/// +/// This type represents the registration of a [`Family`]. When an instance of this type is +/// dropped, its respective generic netlink family will be unregistered from the system. +/// +/// # Invariants +/// +/// `self.family` always holds a valid reference to an initialized and registered [`Family`]. +pub struct Registration { + family: &'static Family, +} + +impl Family { + /// Registers the generic netlink family with the kernel. + pub fn register(&'static self) -> Result<Registration> { + // SAFETY: `self.as_raw()` is a valid pointer to a `genl_family` struct. + // The `genl_family` struct is static, so it will outlive the registration. + to_result(unsafe { bindings::genl_register_family(self.as_raw()) })?; + Ok(Registration { family: self }) + } +} + +impl Drop for Registration { + fn drop(&mut self) { + // SAFETY: `self.family.as_raw()` is a valid pointer to a registered `genl_family` struct. + // The `Registration` struct ensures that `genl_unregister_family` is called exactly once + // for this family when it goes out of scope. + unsafe { bindings::genl_unregister_family(self.family.as_raw()) }; + } +} + +#[macros::kunit_tests(rust_netlink)] +mod tests { + use super::*; + + #[test] + fn test_family_flags_bitfield() { + for netnsok in [false, true] { + for parallel_ops in [false, true] { + let mut b_fam = bindings::genl_family { + ..Default::default() + }; + b_fam.set_netnsok(if netnsok { 1 } else { 0 }); + b_fam.set_parallel_ops(if parallel_ops { 1 } else { 0 }); + + let c_bitfield = FamilyFlags { + netnsok, + parallel_ops, + } + .into_bitfield(); + + // SAFETY: The bit field is stored as u8. + let b_val: u8 = unsafe { core::mem::transmute(b_fam._bitfield_1) }; + // SAFETY: The bit field is stored as u8. + let c_val: u8 = unsafe { core::mem::transmute(c_bitfield) }; + assert_eq!(b_val, c_val); + } + } + } +} diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs index 3ca99db5cccf..c4e7b1d6c6f4 100644 --- a/rust/kernel/net/phy.rs +++ b/rust/kernel/net/phy.rs @@ -123,39 +123,37 @@ impl Device { /// Gets the current link state. /// /// It returns true if the link is up. + #[inline] pub fn is_link_up(&self) -> bool { - const LINK_IS_UP: u64 = 1; - // TODO: the code to access to the bit field will be replaced with automatically - // generated code by bindgen when it becomes possible. - // SAFETY: The struct invariant ensures that we may access - // this field without additional synchronization. - let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; - bit_field.get(14, 1) == LINK_IS_UP + let phydev = self.0.get().cast_const(); + // SAFETY: By the type invariant of `Device`, `phydev` points to a valid + // `struct phy_device`, and there is no concurrent write to this field. + let link = unsafe { bindings::phy_device::link_raw(phydev) }; + link == 1 } /// Gets the current auto-negotiation configuration. /// /// It returns true if auto-negotiation is enabled. + #[inline] pub fn is_autoneg_enabled(&self) -> bool { - // TODO: the code to access to the bit field will be replaced with automatically - // generated code by bindgen when it becomes possible. - // SAFETY: The struct invariant ensures that we may access - // this field without additional synchronization. - let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; - bit_field.get(13, 1) == u64::from(bindings::AUTONEG_ENABLE) + let phydev = self.0.get().cast_const(); + // SAFETY: By the type invariant of `Device`, `phydev` points to a valid + // `struct phy_device`, and there is no concurrent write to this field. + let autoneg = unsafe { bindings::phy_device::autoneg_raw(phydev) }; + autoneg == bindings::AUTONEG_ENABLE } /// Gets the current auto-negotiation state. /// /// It returns true if auto-negotiation is completed. + #[inline] pub fn is_autoneg_completed(&self) -> bool { - const AUTONEG_COMPLETED: u64 = 1; - // TODO: the code to access to the bit field will be replaced with automatically - // generated code by bindgen when it becomes possible. - // SAFETY: The struct invariant ensures that we may access - // this field without additional synchronization. - let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; - bit_field.get(15, 1) == AUTONEG_COMPLETED + let phydev = self.0.get().cast_const(); + // SAFETY: By the type invariant of `Device`, `phydev` points to a valid + // `struct phy_device`, and there is no concurrent write to this field. + let completed = unsafe { bindings::phy_device::autoneg_complete_raw(phydev) }; + completed == 1 } /// Sets the speed of the PHY. @@ -659,7 +657,11 @@ impl Registration { // the `drivers` slice are initialized properly. `drivers` will not be moved. // So it's just an FFI call. to_result(unsafe { - bindings::phy_drivers_register(drivers[0].0.get(), drivers.len().try_into()?, module.0) + bindings::phy_drivers_register( + drivers[0].0.get(), + drivers.len().try_into()?, + module.as_ptr(), + ) })?; // INVARIANT: The `drivers` slice is successfully registered to the kernel via `phy_drivers_register`. Ok(Registration { drivers }) @@ -800,62 +802,6 @@ impl DeviceMask { /// } /// # } /// ``` -/// -/// This expands to the following code: -/// -/// ```ignore -/// use kernel::net::phy::{self, DeviceId}; -/// use kernel::prelude::*; -/// -/// struct Module { -/// _reg: ::kernel::net::phy::Registration, -/// } -/// -/// module! { -/// type: Module, -/// name: "rust_sample_phy", -/// authors: ["Rust for Linux Contributors"], -/// description: "Rust sample PHYs driver", -/// license: "GPL", -/// } -/// -/// struct PhySample; -/// -/// #[vtable] -/// impl phy::Driver for PhySample { -/// const NAME: &'static CStr = c"PhySample"; -/// const PHY_DEVICE_ID: phy::DeviceId = phy::DeviceId::new_with_exact_mask(0x00000001); -/// } -/// -/// const _: () = { -/// static mut DRIVERS: [::kernel::net::phy::DriverVTable; 1] = -/// [::kernel::net::phy::create_phy_driver::<PhySample>()]; -/// -/// impl ::kernel::Module for Module { -/// fn init(module: &'static ::kernel::ThisModule) -> Result<Self> { -/// let drivers = unsafe { &mut DRIVERS }; -/// let mut reg = ::kernel::net::phy::Registration::register( -/// module, -/// ::core::pin::Pin::static_mut(drivers), -/// )?; -/// Ok(Module { _reg: reg }) -/// } -/// } -/// }; -/// -/// const N: usize = 1; -/// -/// const TABLE: ::kernel::device_id::IdArray<::kernel::net::phy::DeviceId, (), N> = -/// ::kernel::device_id::IdArray::new_without_index([ -/// ::kernel::net::phy::DeviceId( -/// ::kernel::bindings::mdio_device_id { -/// phy_id: 0x00000001, -/// phy_id_mask: 0xffffffff, -/// }), -/// ]); -/// -/// ::kernel::module_device_table!("mdio", phydev, TABLE); -/// ``` #[macro_export] macro_rules! module_phy_driver { (@replace_expr $_t:tt $sub:expr) => {$sub}; @@ -865,12 +811,10 @@ macro_rules! module_phy_driver { }; (@device_table [$($dev:expr),+]) => { - const N: usize = $crate::module_phy_driver!(@count_devices $($dev),+); - - const TABLE: $crate::device_id::IdArray<$crate::net::phy::DeviceId, (), N> = - $crate::device_id::IdArray::new_without_index([ $(($dev,())),+, ]); - - $crate::module_device_table!("mdio", phydev, TABLE); + $crate::module_device_table!( + "mdio", $crate::net::phy::DeviceId, + TABLE, @none, [$($dev),+] + ); }; (drivers: [$($driver:ident),+ $(,)?], device_table: [$($dev:expr),+ $(,)?], $($f:tt)*) => { diff --git a/rust/kernel/net/phy/reg.rs b/rust/kernel/net/phy/reg.rs index a7db0064cb7d..80e22c264ea8 100644 --- a/rust/kernel/net/phy/reg.rs +++ b/rust/kernel/net/phy/reg.rs @@ -9,9 +9,11 @@ //! defined in IEEE 802.3. use super::Device; -use crate::build_assert; -use crate::error::*; -use crate::uapi; +use crate::{ + build_assert::build_assert, + error::*, + uapi, // +}; mod private { /// Marker that a trait cannot be implemented outside of this crate diff --git a/rust/kernel/num.rs b/rust/kernel/num.rs index 8532b511384c..de589792a77a 100644 --- a/rust/kernel/num.rs +++ b/rust/kernel/num.rs @@ -5,6 +5,8 @@ use core::ops; pub mod bounded; +pub mod casts; + pub use bounded::*; /// Designates unsigned primitive types. @@ -13,9 +15,14 @@ pub enum Unsigned {} /// Designates signed primitive types. pub enum Signed {} +mod private { + pub trait Sealed {} +} + /// Describes core properties of integer types. pub trait Integer: - Sized + private::Sealed + + Sized + Copy + Clone + PartialEq @@ -54,6 +61,8 @@ pub trait Integer: macro_rules! impl_integer { ($($type:ty: $signedness:ty), *) => { $( + impl private::Sealed for $type {} + impl Integer for $type { type Signedness = $signedness; diff --git a/rust/kernel/num/bounded.rs b/rust/kernel/num/bounded.rs index fa81acbdc8c2..2a2b0a4bca5e 100644 --- a/rust/kernel/num/bounded.rs +++ b/rust/kernel/num/bounded.rs @@ -13,7 +13,10 @@ use core::{ }; use kernel::{ - num::Integer, + num::{ + Integer, + Unsigned, // + }, prelude::*, // }; @@ -174,13 +177,16 @@ fn fits_within<T: Integer>(value: T, num_bits: u32) -> bool { /// // `u8` (regardless of the passed value). /// // let _ = Bounded::<u32, 6>::from(10u8); /// -/// // Booleans can be converted into single-bit `Bounded`s. +/// // Booleans can be converted into unsigned `Bounded`s. /// /// let v = Bounded::<u64, 1>::from(false); /// assert_eq!(v.get(), 0); /// /// let v = Bounded::<u64, 1>::from(true); /// assert_eq!(v.get(), 1); +/// +/// // This does not build because `i8` is signed. +/// // let _ = Bounded::<i8, 2>::from(true); /// ``` /// /// Infallible conversions from a [`Bounded`] to a primitive integer are also supported, and @@ -203,12 +209,16 @@ fn fits_within<T: Integer>(value: T, num_bits: u32) -> bool { /// let _v = Bounded::<u32, 10>::new::<10>(); /// // assert_eq!(u8::from(_v), 10); /// -/// // Single-bit `Bounded`s can be converted into a boolean. +/// // Unsigned single-bit `Bounded`s can be converted into a boolean. /// let v = Bounded::<u8, 1>::new::<1>(); /// assert_eq!(bool::from(v), true); /// /// let v = Bounded::<u8, 1>::new::<0>(); /// assert_eq!(bool::from(v), false); +/// +/// // This does not build because `i8` is signed. +/// // let v = Bounded::<i8, 1>::new::<-1>(); +/// // let _ = bool::from(v); /// ``` /// /// Fallible conversions from any primitive integer to any [`Bounded`] are also supported using the @@ -255,9 +265,7 @@ macro_rules! impl_const_new { /// ``` pub const fn new<const VALUE: $type>() -> Self { // Statically assert that `VALUE` fits within the set number of bits. - const { - assert!(fits_within!(VALUE, $type, N)); - } + const_assert!(fits_within!(VALUE, $type, N)); // SAFETY: `fits_within` confirmed that `VALUE` can be represented within // `N` bits. @@ -287,12 +295,10 @@ where /// The caller must ensure that `value` can be represented within `N` bits. const unsafe fn __new(value: T) -> Self { // Enforce the type invariants. - const { - // `N` cannot be zero. - assert!(N != 0); - // The backing type is at least as large as `N` bits. - assert!(N <= T::BITS); - } + // `N` cannot be zero. + const_assert!(N != 0); + // The backing type is at least as large as `N` bits. + const_assert!(N <= T::BITS); // INVARIANT: The caller ensures `value` fits within `N` bits. Self(value) @@ -368,7 +374,7 @@ where // Always inline to optimize out error path of `build_assert`. #[inline(always)] pub fn from_expr(expr: T) -> Self { - crate::build_assert!( + crate::build_assert::build_assert!( fits_within(expr, N), "Requested value larger than maximal representable value." ); @@ -379,6 +385,9 @@ where /// Returns the wrapped value as the backing type. /// + /// This is similar to the [`Deref`] implementation, but doesn't enforce the size invariant of + /// the [`Bounded`], which might produce slightly less optimal code. + /// /// # Examples /// /// ``` @@ -387,8 +396,8 @@ where /// let v = Bounded::<u32, 4>::new::<7>(); /// assert_eq!(v.get(), 7u32); /// ``` - pub fn get(self) -> T { - *self.deref() + pub const fn get(self) -> T { + self.0 } /// Increases the number of bits usable for `self`. @@ -406,12 +415,10 @@ where /// assert_eq!(larger_v, v); /// ``` pub const fn extend<const M: u32>(self) -> Bounded<T, M> { - const { - assert!( - M >= N, - "Requested number of bits is less than the current representation." - ); - } + const_assert!( + M >= N, + "Requested number of bits is less than the current representation." + ); // SAFETY: The value did fit within `N` bits, so it will all the more fit within // the larger `M` bits. @@ -473,6 +480,80 @@ where // `N` bits, and with the same signedness. unsafe { Bounded::__new(value) } } + + /// Right-shifts `self` by `SHIFT` and returns the result as a `Bounded<_, RES>`, where `RES >= + /// N - SHIFT`. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::<u32, 16>::new::<0xff00>(); + /// let v_shifted: Bounded::<u32, 8> = v.shr::<8, _>(); + /// + /// assert_eq!(v_shifted.get(), 0xff); + /// ``` + pub fn shr<const SHIFT: u32, const RES: u32>(self) -> Bounded<T, RES> { + const_assert!(SHIFT < T::BITS); + const_assert!(RES + SHIFT >= N); + + // SAFETY: We shift the value right by `SHIFT`, reducing the number of bits needed to + // represent the shifted value by as much, and just asserted that `RES >= N - SHIFT`. + unsafe { Bounded::__new(self.0 >> SHIFT) } + } + + /// Right-shifts `self` by `SHIFT` if that loses no set bits, and returns the result as a + /// `Bounded<_, RES>`, where `RES >= N - SHIFT`. + /// + /// Returns [`None`] if any of the `SHIFT` least significant bits of `self` is set. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::<u32, 16>::new::<0xff00>(); + /// let v_shifted: Option<Bounded<u32, 8>> = v.shr_exact::<8, _>(); + /// + /// assert_eq!(v_shifted.map(|v| v.get()), Some(0xff)); + /// + /// // A set bit would be shifted out. + /// let v = Bounded::<u32, 16>::new::<0xff01>(); + /// let v_shifted: Option<Bounded<u32, 8>> = v.shr_exact::<8, _>(); + /// + /// assert!(v_shifted.is_none()); + /// ``` + #[inline] + pub fn shr_exact<const SHIFT: u32, const RES: u32>(self) -> Option<Bounded<T, RES>> { + let shifted = self.shr::<SHIFT, RES>(); + if shifted.get() << SHIFT == self.0 { + Some(shifted) + } else { + None + } + } + + /// Left-shifts `self` by `SHIFT` and returns the result as a `Bounded<_, RES>`, where `RES >= + /// N + SHIFT`. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::<u32, 8>::new::<0xff>(); + /// let v_shifted: Bounded::<u32, 16> = v.shl::<8, _>(); + /// + /// assert_eq!(v_shifted.get(), 0xff00); + /// ``` + pub fn shl<const SHIFT: u32, const RES: u32>(self) -> Bounded<T, RES> { + const_assert!(RES >= N + SHIFT); + + // SAFETY: We shift the value left by `SHIFT`, augmenting the number of bits needed to + // represent the shifted value by as much, and just asserted that `RES >= N + SHIFT`. + unsafe { Bounded::__new(self.0 << SHIFT) } + } } impl<T, const N: u32> Deref for Bounded<T, N> @@ -1038,24 +1119,47 @@ impl_into_primitive!( i8 i16 i32 i64 isize ); -// Single-bit `Bounded`s can be converted from/to a boolean. +// Unsigned single-bit `Bounded`s can be converted to a boolean. impl<T> From<Bounded<T, 1>> for bool where - T: Integer + Zeroable, + T: Integer<Signedness = Unsigned> + Zeroable, { fn from(value: Bounded<T, 1>) -> Self { value.get() != Zeroable::zeroed() } } +// Booleans can be converted to unsigned `Bounded`s. + impl<T, const N: u32> From<bool> for Bounded<T, N> where - T: Integer + From<bool>, + T: Integer<Signedness = Unsigned> + From<bool>, { fn from(value: bool) -> Self { - // SAFETY: A boolean can be represented using a single bit, and thus fits within any - // integer type for any `N` > 0. + // SAFETY: A boolean is represented by `0` or `1`, so it fits within any valid unsigned + // `Bounded` width. unsafe { Self::__new(T::from(value)) } } } + +impl<T> Bounded<T, 1> +where + T: Integer<Signedness = Unsigned> + Zeroable, +{ + /// Converts this [`Bounded`] into a [`bool`]. + /// + /// This is a shorter way of writing `bool::from(self)`. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// assert_eq!(Bounded::<u8, 1>::new::<0>().into_bool(), false); + /// assert_eq!(Bounded::<u8, 1>::new::<1>().into_bool(), true); + /// ``` + pub fn into_bool(self) -> bool { + self.into() + } +} diff --git a/rust/kernel/num/casts.rs b/rust/kernel/num/casts.rs new file mode 100644 index 000000000000..7e6c7dec747d --- /dev/null +++ b/rust/kernel/num/casts.rs @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Helpers for performing lossless integer casts. +//! +//! The `as` keyword can be used to perform casts between integer types, but it unfortunately makes +//! no distinction between casts that are lossless, and casts from a larger type into a smaller one +//! that might silently strip data away. Thus, its use in the kernel is discouraged in favor of +//! [`From`] implementations. +//! +//! Conversely, there are casts that are lossless depending on the build architecture (such as +//! casting [`usize`] to [`u64`] on 32 or 64 bit archs), but not supported by [`From`] +//! implementations in the standard library because they are not portable. It does however make +//! sense for the kernel to support these, if only for code that is architecture-specific. +//! +//! This module provides ways to perform such conversions safely: +//! +//! - A series of const functions (e.g. [`usize_as_u64`]) supporting safe conversions in const +//! context. Conversions supported by [`From`] implementations in the standard library are also +//! covered as the [`From`] trait cannot be used in const context. +//! - Two extension traits, [`FromSafeCast`] and [`IntoSafeCast`], providing conversion methods +//! similar to [`From`] and [`Into`] for conversions that are safe to perform in the kernel, but +//! not supported by the standard library. +//! - Another series of const functions (e.g. [`u64_into_u8`]) supporting the conversion of a const +//! value from a larger type into a smaller one, provided the value fits into the destination +//! type. This is useful if a constant is defined as a larger type, but needs to be used as a +//! smaller one. +//! - An [`arch`] sub-module, defining more conversion functions that are only guaranteed to be +//! lossless for a given pointer size. These can only be used in code that is specific to a +//! given pointer size. +//! +//! # Examples +//! +//! ``` +//! use kernel::num::casts::{self, FromSafeCast, IntoSafeCast}; +//! +//! // Conversion from const context. +//! const USIZED_CONST: usize = casts::u8_as_usize(255u8); +//! +//! // Non-const conversions. +//! let a = u64::from_safe_cast(4096usize); +//! let b: u64 = 4096usize.into_safe_cast(); +//! ``` + +use crate::prelude::*; + +/// Implements safe `as` conversion functions from a given type into a series of target types. +/// +/// These functions can be used in place of `as`, with the guarantee that they will be lossless. +macro_rules! impl_safe_as { + ($from:ty as { $($into:ty),* }) => { + $( + $crate::macros::paste! { + #[doc = ::core::concat!( + "Losslessly converts a [`", + ::core::stringify!($from), + "`] into a [`", + ::core::stringify!($into), + "`].")] + /// + /// This conversion is allowed as it is always lossless. Prefer this over the `as` + /// keyword to ensure no lossy casts are performed. + /// + /// This is for use from a `const` context. For non `const` use, prefer the + /// [`FromSafeCast`] and [`IntoSafeCast`] traits. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::casts; + /// + #[doc = ::core::concat!( + "assert_eq!(casts::", + ::core::stringify!($from), + "_as_", + ::core::stringify!($into), + "(1", + ::core::stringify!($from), + "), 1", + ::core::stringify!($into), + ");")] + /// ``` + #[inline] + pub const fn [<$from _as_ $into>](value: $from) -> $into { + $crate::static_assert!(size_of::<$into>() >= size_of::<$from>()); + + value as $into + } + } + )* + }; +} + +// Valid `Into` transformations. +impl_safe_as!(u8 as { u16, u32, u64, usize }); +impl_safe_as!(u16 as { u32, u64, usize }); +impl_safe_as!(u32 as { u64 }); +// A `usize` fits into a `u64` on all supported platforms. +impl_safe_as!(usize as { u64 }); +// A `u32` fits into a `usize` on all supported platforms. +impl_safe_as!(u32 as { usize }); + +/// Extension trait providing guaranteed lossless cast to [`Self`] from `T`. +/// +/// The standard library's [`From`] implementations do not cover conversions that are not portable +/// or future-proof. For instance, even though it is safe today, [`From<usize>`] is not implemented +/// for [`u64`] because of the possibility of needing to support larger-than-64bit architectures in +/// the future. +/// +/// The workaround is to either deal with the error handling of [`TryFrom`] for an operation that +/// technically cannot fail, or to use the `as` keyword, which can silently strip data if the +/// destination type is smaller than the source. +/// +/// Both options are hardly acceptable for the kernel. It is also a much more architecture +/// dependent environment, supporting only 32 and 64 bit architectures, with some modules +/// explicitly depending on a specific bus width that could greatly benefit from infallible +/// conversion operations. +/// +/// Thus this extension trait that provides, for all architectures supported by the kernel, +/// conversion methods between types for which such a cast is lossless. +/// +/// In other words, this trait is implemented if, for all supported targets and with `t: T`, the +/// `t as Self` operation is completely lossless. +/// +/// Prefer this over the `as` keyword to guarantee that no lossy casts are performed. +/// +/// If you need to perform a conversion in `const` context, use [`u32_as_usize`], [`usize_as_u64`], +/// etc. +/// +/// # Examples +/// +/// ``` +/// use kernel::num::casts::FromSafeCast; +/// +/// assert_eq!(usize::from_safe_cast(0xf00u32), 0xf00usize); +/// ``` +pub trait FromSafeCast<T> { + /// Create a [`Self`] from `value`. This operation is guaranteed to be lossless. + fn from_safe_cast(value: T) -> Self; +} + +// A `usize` fits into a `u64` on all supported platforms. +impl FromSafeCast<usize> for u64 { + #[inline] + fn from_safe_cast(value: usize) -> Self { + usize_as_u64(value) + } +} + +// A `u32` fits into a `usize` on all supported platforms. +impl FromSafeCast<u32> for usize { + #[inline] + fn from_safe_cast(value: u32) -> Self { + u32_as_usize(value) + } +} + +/// Counterpart to the [`FromSafeCast`] trait, i.e. this trait is to [`FromSafeCast`] what [`Into`] +/// is to [`From`]. +/// +/// See the documentation of [`FromSafeCast`] for the motivation. +/// +/// # Examples +/// +/// ``` +/// use kernel::num::casts::IntoSafeCast; +/// +/// assert_eq!(0xf00usize, 0xf00u32.into_safe_cast()); +/// ``` +pub trait IntoSafeCast<T> { + /// Convert `self` into a `T`. This operation is guaranteed to be lossless. + fn into_safe_cast(self) -> T; +} + +/// Reverse operation for types implementing [`FromSafeCast`]. +impl<S, T> IntoSafeCast<T> for S +where + T: FromSafeCast<S>, +{ + #[inline] + fn into_safe_cast(self) -> T { + T::from_safe_cast(self) + } +} + +/// Implements lossless conversion of a constant from a larger type into a smaller one. +macro_rules! impl_const_into { + ($from:ty => { $($into:ty),* }) => { + $( + $crate::macros::paste! { + #[doc = ::core::concat!( + "Performs a build-time safe conversion of a [`", + ::core::stringify!($from), + "`] constant value into a [`", + ::core::stringify!($into), + "`].")] + /// + /// This checks at compile-time that the conversion is lossless, and triggers a build + /// error if it isn't. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::casts; + /// + /// // Succeeds because the value of the source fits into the destination's type. + #[doc = ::core::concat!( + "assert_eq!(casts::", + ::core::stringify!($from), + "_into_", + ::core::stringify!($into), + "::<1", + ::core::stringify!($from), + ">(), 1", + ::core::stringify!($into), + ");")] + /// ``` + #[inline] + pub const fn [<$from _into_ $into>]<const N: $from>() -> $into { + // Make sure that the target type is smaller than the source one. + $crate::static_assert!($from::BITS >= $into::BITS); + // CAST: we statically enforced above that `$from` is larger than `$into`, so the + // `as` conversion will be lossless. + $crate::const_assert!(N >= $into::MIN as $from && N <= $into::MAX as $from); + + N as $into + } + } + )* + }; +} + +impl_const_into!(usize => { u8, u16, u32 }); +impl_const_into!(u64 => { u8, u16, u32 }); +impl_const_into!(u32 => { u8, u16 }); +impl_const_into!(u16 => { u8 }); + +/// Conversions that are only lossless for the current architecture. +/// +/// # Portability +/// +/// Callers of this module become dependent on the setting of `CONFIG_64BIT`. Use with caution, and +/// never in code that is portable across pointer sizes. +pub mod arch { + /// Trait identical to [`FromSafeCast`](super::FromSafeCast), but for conversions that are not + /// available on all architectures. + pub trait FromSafeCastArch<T> { + /// Create a [`Self`] from `value`. This operation is guaranteed to be lossless. + fn from_safe_cast_arch(value: T) -> Self; + } + + /// Trait identical to [`IntoSafeCast`](super::IntoSafeCast), but for conversions that are not + /// available on all architectures. + pub trait IntoSafeCastArch<T> { + /// Convert `self` into a `T`. This operation is guaranteed to be lossless. + fn into_safe_cast_arch(self) -> T; + } + + /// Reverse operation for types implementing [`FromSafeCastArch`]. + impl<S, T> IntoSafeCastArch<T> for S + where + T: FromSafeCastArch<S>, + { + #[inline] + fn into_safe_cast_arch(self) -> T { + T::from_safe_cast_arch(self) + } + } + + /// A [`u64`] fits into a [`usize`] on 64-bit platforms. + #[cfg(CONFIG_64BIT)] + #[inline] + pub const fn u64_as_usize(value: u64) -> usize { + value as usize + } + + #[cfg(CONFIG_64BIT)] + impl FromSafeCastArch<u64> for usize { + #[inline] + fn from_safe_cast_arch(value: u64) -> Self { + u64_as_usize(value) + } + } + + /// A [`usize`] fits into a [`u32`] on 32-bit platforms. + #[cfg(not(CONFIG_64BIT))] + #[inline] + pub const fn usize_as_u32(value: usize) -> u32 { + value as u32 + } + + #[cfg(not(CONFIG_64BIT))] + impl FromSafeCastArch<usize> for u32 { + #[inline] + fn from_safe_cast_arch(value: usize) -> Self { + usize_as_u32(value) + } + } +} diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs index 58b20c367f99..d0318f62afd7 100644 --- a/rust/kernel/of.rs +++ b/rust/kernel/of.rs @@ -25,10 +25,6 @@ unsafe impl RawDeviceId for DeviceId { // SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `data` field. unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::of_device_id, data); - - fn index(&self) -> usize { - self.0.data as usize - } } impl DeviceId { @@ -53,13 +49,7 @@ impl DeviceId { /// Create an OF `IdTable` with an "alias" for modpost. #[macro_export] macro_rules! of_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::of::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("of", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("of", $crate::of::DeviceId, $($tt)*); }; } diff --git a/rust/kernel/opp.rs b/rust/kernel/opp.rs index a760fac28765..62e44676125d 100644 --- a/rust/kernel/opp.rs +++ b/rust/kernel/opp.rs @@ -1042,11 +1042,13 @@ unsafe impl Sync for OPP {} /// SAFETY: The type invariants guarantee that [`OPP`] is always refcounted. unsafe impl AlwaysRefCounted for OPP { + #[inline] fn inc_ref(&self) { // SAFETY: The existence of a shared reference means that the refcount is nonzero. unsafe { bindings::dev_pm_opp_get(self.0.get()) }; } + #[inline] unsafe fn dec_ref(obj: ptr::NonNull<Self>) { // SAFETY: The safety requirements guarantee that the refcount is nonzero. unsafe { bindings::dev_pm_opp_put(obj.cast().as_ptr()) } @@ -1095,6 +1097,7 @@ impl OPP { } /// Returns the frequency of an [`OPP`]. + #[inline] pub fn freq(&self, index: Option<u32>) -> Hertz { let index = index.unwrap_or(0); diff --git a/rust/kernel/page.rs b/rust/kernel/page.rs index adecb200c654..1c0796ea229f 100644 --- a/rust/kernel/page.rs +++ b/rust/kernel/page.rs @@ -3,17 +3,25 @@ //! Kernel page allocation and management. use crate::{ - alloc::{AllocError, Flags}, + alloc::{ + AllocError, + Flags, // + }, bindings, - error::code::*, - error::Result, - uaccess::UserSliceReader, + error::{ + code::*, + Result, // + }, + uaccess::UserSliceReader, // }; use core::{ marker::PhantomData, mem::ManuallyDrop, ops::Deref, - ptr::{self, NonNull}, + ptr::{ + self, + NonNull, // + }, // }; /// A bitwise shift for the page size. @@ -193,6 +201,7 @@ impl Page { } /// Get the node id containing this page. + #[inline] pub fn nid(&self) -> i32 { // SAFETY: Always safe to call with a valid page. unsafe { bindings::page_to_nid(self.as_ptr()) } diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index af74ddff6114..3ec897709e89 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -25,6 +25,7 @@ use crate::{ use core::{ marker::PhantomData, mem::offset_of, + num::NonZero, ptr::{ addr_of_mut, NonNull, // @@ -43,15 +44,16 @@ pub use self::id::{ pub use self::io::{ Bar, ConfigSpace, - ConfigSpaceKind, ConfigSpaceSize, + DevresBar, Extended, Normal, // }; pub use self::irq::{ IrqType, IrqTypes, - IrqVector, // + IrqVector, + IrqVectorRegistration, // }; /// An adapter for the registration of PCI drivers. @@ -59,18 +61,18 @@ pub struct Adapter<T: Driver>(T); // SAFETY: // - `bindings::pci_driver` is a C type declared as `repr(C)`. -// - `T` is the type of the driver's device private data. +// - `T::Data` is the type of the driver's device private data. // - `struct pci_driver` embeds a `struct device_driver`. // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. -unsafe impl<T: Driver + 'static> driver::DriverLayout for Adapter<T> { +unsafe impl<T: Driver> driver::DriverLayout for Adapter<T> { type DriverType = bindings::pci_driver; - type DriverData = T; + type DriverData<'bound> = T::Data<'bound>; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); } // SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if // a preceding call to `register` has been successful. -unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { +unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { unsafe fn register( pdrv: &Opaque<Self::DriverType>, name: &'static CStr, @@ -86,7 +88,7 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. to_result(unsafe { - bindings::__pci_register_driver(pdrv.get(), module.0, name.as_char_ptr()) + bindings::__pci_register_driver(pdrv.get(), module.as_ptr(), name.as_char_ptr()) }) } @@ -96,7 +98,7 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { } } -impl<T: Driver + 'static> Adapter<T> { +impl<T: Driver> Adapter<T> { extern "C" fn probe_callback( pdev: *mut bindings::pci_dev, id: *const bindings::pci_device_id, @@ -105,12 +107,16 @@ impl<T: Driver + 'static> Adapter<T> { // `struct pci_dev`. // // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. - let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal>>() }; + let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal<'_>>>() }; // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct pci_device_id` and // does not add additional invariants, so it's safe to transmute. let id = unsafe { &*id.cast::<DeviceId>() }; - let info = T::ID_TABLE.info(id.index()); + + // SAFETY: `id` comes from `T::ID_TABLE` which is of type `IdArray<_, T::IdInfo>` or + // `pci_device_id_any` which has 0 as driver_data. It can also come from dynamic IDs, which + // will ensure that `driver_data` exists in `T::ID_TABLE`. + let info = unsafe { id.info_unchecked_opt::<T::IdInfo>() }; from_result(|| { let data = T::probe(pdev, info); @@ -125,12 +131,12 @@ impl<T: Driver + 'static> Adapter<T> { // `struct pci_dev`. // // INVARIANT: `pdev` is valid for the duration of `remove_callback()`. - let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal>>() }; + let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal<'_>>>() }; // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called - // and stored a `Pin<KBox<T>>`. - let data = unsafe { pdev.as_ref().drvdata_borrow::<T>() }; + // and stored a `Pin<KBox<T::Data<'_>>>`. + let data = unsafe { pdev.as_ref().drvdata_borrow::<T::Data<'_>>() }; T::unbind(pdev, data); } @@ -233,10 +239,6 @@ unsafe impl RawDeviceId for DeviceId { // SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::pci_device_id, driver_data); - - fn index(&self) -> usize { - self.0.driver_data - } } /// `IdTable` type for PCI. @@ -245,14 +247,8 @@ pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; /// Create a PCI `IdTable` with its alias for modpost. #[macro_export] macro_rules! pci_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::pci::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("pci", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("pci", $crate::pci::DeviceId, $($tt)*); }; } @@ -267,7 +263,6 @@ macro_rules! pci_device_table { /// /// kernel::pci_device_table!( /// PCI_TABLE, -/// MODULE_PCI_TABLE, /// <MyDriver as pci::Driver>::IdInfo, /// [ /// ( @@ -279,19 +274,20 @@ macro_rules! pci_device_table { /// /// impl pci::Driver for MyDriver { /// type IdInfo = (); +/// type Data<'bound> = Self; /// const ID_TABLE: pci::IdTable<Self::IdInfo> = &PCI_TABLE; /// -/// fn probe( -/// _pdev: &pci::Device<Core>, -/// _id_info: &Self::IdInfo, -/// ) -> impl PinInit<Self, Error> { +/// fn probe<'bound>( +/// _pdev: &'bound pci::Device<Core<'_>>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound { /// Err(ENODEV) /// } /// } ///``` /// Drivers must implement this trait in order to get a PCI driver registered. Please refer to the /// `Adapter` documentation for an example. -pub trait Driver: Send { +pub trait Driver { /// The type holding information about each device id supported by the driver. // TODO: Use `associated_type_defaults` once stabilized: // @@ -300,6 +296,9 @@ pub trait Driver: Send { // ``` type IdInfo: 'static; + /// The type of the driver's bus device private data. + type Data<'bound>: Send + 'bound; + /// The table of device ids supported by the driver. const ID_TABLE: IdTable<Self::IdInfo>; @@ -307,7 +306,10 @@ pub trait Driver: Send { /// /// Called when a new pci device is added or discovered. Implementers should /// attempt to initialize the device here. - fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> impl PinInit<Self, Error>; + fn probe<'bound>( + dev: &'bound Device<device::Core<'_>>, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound; /// PCI driver unbind. /// @@ -318,8 +320,8 @@ pub trait Driver: Send { /// `&Device<Core>` or `&Device<Bound>` reference. For instance, drivers may try to perform I/O /// operations to gracefully tear down the device. /// - /// Otherwise, release operations for driver resources should be performed in `Self::drop`. - fn unbind(dev: &Device<device::Core>, this: Pin<&Self>) { + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(dev: &'bound Device<device::Core<'_>>, this: Pin<&Self::Data<'bound>>) { let _ = (dev, this); } } @@ -354,7 +356,7 @@ impl Device { /// /// ``` /// # use kernel::{device::Core, pci::{self, Vendor}, prelude::*}; - /// fn log_device_info(pdev: &pci::Device<Core>) -> Result { + /// fn log_device_info(pdev: &pci::Device<Core<'_>>) -> Result { /// // Get an instance of `Vendor`. /// let vendor = pdev.vendor_id(); /// dev_info!( @@ -445,7 +447,19 @@ impl Device { } } -impl Device<device::Core> { +impl<'a> Device<device::Core<'a>> { + /// Returns the total number of VFs, or [`None`] if SR-IOV is not available. + #[inline] + pub fn sriov_get_totalvfs(&self) -> Option<NonZero<u16>> { + // SAFETY: `self.as_raw()` is a valid pointer to a `struct pci_dev`. + let total_vfs = unsafe { bindings::pci_sriov_get_totalvfs(self.as_raw()) }; + + // CAST: The C function returns `unsigned int`, but the value originates + // from TotalVFs/driver_max_VFs (which are defined as `u16`), so this cast + // cannot truncate. + NonZero::new(total_vfs as u16) + } + /// Enable memory resources for this device. pub fn enable_device_mem(&self) -> Result { // SAFETY: `self.as_raw` is guaranteed to be a pointer to a valid `struct pci_dev`. @@ -471,15 +485,17 @@ unsafe impl<Ctx: device::DeviceContext> device::AsBusDevice<Ctx> for Device<Ctx> kernel::impl_device_context_deref!(unsafe { Device }); kernel::impl_device_context_into_aref!(Device); -impl crate::dma::Device for Device<device::Core> {} +impl<'a> crate::dma::Device<'a> for Device<device::Core<'a>> {} // SAFETY: Instances of `Device` are always reference-counted. unsafe impl crate::sync::aref::AlwaysRefCounted for Device { + #[inline] fn inc_ref(&self) { // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. unsafe { bindings::pci_dev_get(self.as_raw()) }; } + #[inline] unsafe fn dec_ref(obj: NonNull<Self>) { // SAFETY: The safety requirements guarantee that the refcount is non-zero. unsafe { bindings::pci_dev_put(obj.cast().as_ptr()) } @@ -523,3 +539,7 @@ unsafe impl Send for Device {} // SAFETY: `Device` can be shared among threads because all methods of `Device` // (i.e. `Device<Normal>) are thread safe. unsafe impl Sync for Device {} + +// SAFETY: Same as `Device<Normal>` -- the underlying `struct pci_dev` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device<device::Bound> {} diff --git a/rust/kernel/pci/id.rs b/rust/kernel/pci/id.rs index 50005d176561..dbaf301666e7 100644 --- a/rust/kernel/pci/id.rs +++ b/rust/kernel/pci/id.rs @@ -19,7 +19,7 @@ use crate::{ /// /// ``` /// # use kernel::{device::Core, pci::{self, Class}, prelude::*}; -/// fn probe_device(pdev: &pci::Device<Core>) -> Result { +/// fn probe_device(pdev: &pci::Device<Core<'_>>) -> Result { /// let pci_class = pdev.pci_class(); /// dev_info!( /// pdev, diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index fb6edab2aea7..953e16735c6e 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -6,22 +6,22 @@ use super::Device; use crate::{ bindings, device, - devres::Devres, + devres::DevresLt, io::{ - io_define_read, - io_define_write, - Io, + IoBackend, + IoBase, IoCapable, - IoKnownSize, Mmio, - MmioRaw, // + MmioBackend, + MmioRaw, + Region, // }, prelude::*, - sync::aref::ARef, // -}; -use core::{ - marker::PhantomData, - ops::Deref, // + ptr::KnownSize, + types::{ + CovariantForLt, + ForLt, // + }, // }; /// Represents the size of a PCI configuration space. @@ -49,131 +49,113 @@ impl ConfigSpaceSize { } } -/// Marker type for normal (256-byte) PCI configuration space. -pub struct Normal; +/// Alias for normal (256-byte) PCI configuration space. +pub type Normal = Region<256>; -/// Marker type for extended (4096-byte) PCIe configuration space. -pub struct Extended; +/// Alias for extended (4096-byte) PCIe configuration space. +pub type Extended = Region<4096>; -/// Trait for PCI configuration space size markers. -/// -/// This trait is implemented by [`Normal`] and [`Extended`] to provide -/// compile-time knowledge of the configuration space size. -pub trait ConfigSpaceKind { - /// The size of this configuration space in bytes. - const SIZE: usize; -} - -impl ConfigSpaceKind for Normal { - const SIZE: usize = 256; -} - -impl ConfigSpaceKind for Extended { - const SIZE: usize = 4096; -} - -/// The PCI configuration space of a device. +/// A view of PCI configuration space of a device. /// /// Provides typed read and write accessors for configuration registers /// using the standard `pci_read_config_*` and `pci_write_config_*` helpers. /// -/// The generic parameter `S` indicates the maximum size of the configuration space. -/// Use [`Normal`] for 256-byte legacy configuration space or [`Extended`] for -/// 4096-byte PCIe extended configuration space (default). -pub struct ConfigSpace<'a, S: ConfigSpaceKind = Extended> { - pub(crate) pdev: &'a Device<device::Bound>, - _marker: PhantomData<S>, -} - -/// Internal helper macros used to invoke C PCI configuration space read functions. -/// -/// This macro is intended to be used by higher-level PCI configuration space access macros -/// (io_define_read) and provides a unified expansion for infallible vs. fallible read semantics. It -/// emits a direct call into the corresponding C helper and performs the required cast to the Rust -/// return type. -/// -/// # Parameters +/// The generic parameter `T` is the type of the view. The full configuration space is also a +/// special type of view; in such cases, `T` can be [`Normal`] for 256-byte legacy configuration +/// space or [`Extended`] for 4096-byte PCIe extended configuration space (default). /// -/// * `$c_fn` – The C function performing the PCI configuration space write. -/// * `$self` – The I/O backend object. -/// * `$ty` – The type of the value to read. -/// * `$addr` – The PCI configuration space offset to read. +/// # Invariants /// -/// This macro does not perform any validation; all invariants must be upheld by the higher-level -/// abstraction invoking it. -macro_rules! call_config_read { - (infallible, $c_fn:ident, $self:ident, $ty:ty, $addr:expr) => {{ - let mut val: $ty = 0; - // SAFETY: By the type invariant `$self.pdev` is a valid address. - // CAST: The offset is cast to `i32` because the C functions expect a 32-bit signed offset - // parameter. PCI configuration space size is at most 4096 bytes, so the value always fits - // within `i32` without truncation or sign change. - // Return value from C function is ignored in infallible accessors. - let _ret = unsafe { bindings::$c_fn($self.pdev.as_raw(), $addr as i32, &mut val) }; - val - }}; +/// `ptr` is aligned and range `ptr..ptr + KnownSize::size(ptr)` is within +/// `0..pdev.cfg_size().into_raw()`. +pub struct ConfigSpace<'a, T: ?Sized = Extended> { + pub(crate) pdev: &'a Device<device::Bound>, + ptr: *mut T, } -/// Internal helper macros used to invoke C PCI configuration space write functions. -/// -/// This macro is intended to be used by higher-level PCI configuration space access macros -/// (io_define_write) and provides a unified expansion for infallible vs. fallible read semantics. -/// It emits a direct call into the corresponding C helper and performs the required cast to the -/// Rust return type. -/// -/// # Parameters -/// -/// * `$c_fn` – The C function performing the PCI configuration space write. -/// * `$self` – The I/O backend object. -/// * `$ty` – The type of the written value. -/// * `$addr` – The configuration space offset to write. -/// * `$value` – The value to write. -/// -/// This macro does not perform any validation; all invariants must be upheld by the higher-level -/// abstraction invoking it. -macro_rules! call_config_write { - (infallible, $c_fn:ident, $self:ident, $ty:ty, $addr:expr, $value:expr) => { - // SAFETY: By the type invariant `$self.pdev` is a valid address. - // CAST: The offset is cast to `i32` because the C functions expect a 32-bit signed offset - // parameter. PCI configuration space size is at most 4096 bytes, so the value always fits - // within `i32` without truncation or sign change. - // Return value from C function is ignored in infallible accessors. - let _ret = unsafe { bindings::$c_fn($self.pdev.as_raw(), $addr as i32, $value) }; - }; +impl<T: ?Sized> Copy for ConfigSpace<'_, T> {} +impl<T: ?Sized> Clone for ConfigSpace<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } } -// PCI configuration space supports 8, 16, and 32-bit accesses. -impl<'a, S: ConfigSpaceKind> IoCapable<u8> for ConfigSpace<'a, S> {} -impl<'a, S: ConfigSpaceKind> IoCapable<u16> for ConfigSpace<'a, S> {} -impl<'a, S: ConfigSpaceKind> IoCapable<u32> for ConfigSpace<'a, S> {} +// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl<T: ?Sized + Sync> Send for ConfigSpace<'_, T> {} + +// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl<T: ?Sized + Sync> Sync for ConfigSpace<'_, T> {} + +/// I/O Backend for PCI configuration space. +pub struct ConfigSpaceBackend; + +impl IoBackend for ConfigSpaceBackend { + type View<'a, T: ?Sized + KnownSize> = ConfigSpace<'a, T>; -impl<'a, S: ConfigSpaceKind> Io for ConfigSpace<'a, S> { - /// Returns the base address of the I/O region. It is always 0 for configuration space. #[inline] - fn addr(&self) -> usize { - 0 + fn as_ptr<'a, T: ?Sized + KnownSize>(view: ConfigSpace<'a, T>) -> *mut T { + view.ptr } - /// Returns the maximum size of the configuration space. #[inline] - fn maxsize(&self) -> usize { - self.pdev.cfg_size().into_raw() + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement. + ConfigSpace { + pdev: view.pdev, + ptr, + } } +} - // PCI configuration space does not support fallible operations. - // The default implementations from the Io trait are not used. +/// Implements [`IoCapable`] on [`ConfigSpace`] for `$ty` using `$read_fn` and `$write_fn`. +macro_rules! impl_config_space_io_capable { + ($ty:ty, $read_fn:ident, $write_fn:ident) => { + impl IoCapable<$ty> for ConfigSpaceBackend { + fn io_read(view: ConfigSpace<'_, $ty>) -> $ty { + // CAST: The offset is cast to `i32` because the C functions expect a 32-bit + // signed offset parameter. PCI configuration space size is at most 4096 bytes, + // so the value always fits within `i32` without truncation or sign change. + let addr = view.ptr.addr() as i32; + + let mut val: $ty = 0; + + // Return value from C function is ignored in infallible accessors. + // SAFETY: By the type invariant `pdev` is a valid address. + let _ = unsafe { bindings::$read_fn(view.pdev.as_raw(), addr, &mut val) }; + val + } - io_define_read!(infallible, read8, call_config_read(pci_read_config_byte) -> u8); - io_define_read!(infallible, read16, call_config_read(pci_read_config_word) -> u16); - io_define_read!(infallible, read32, call_config_read(pci_read_config_dword) -> u32); + fn io_write(view: ConfigSpace<'_, $ty>, value: $ty) { + // CAST: The offset is cast to `i32` because the C functions expect a 32-bit + // signed offset parameter. PCI configuration space size is at most 4096 bytes, + // so the value always fits within `i32` without truncation or sign change. + let addr = view.ptr.addr() as i32; - io_define_write!(infallible, write8, call_config_write(pci_write_config_byte) <- u8); - io_define_write!(infallible, write16, call_config_write(pci_write_config_word) <- u16); - io_define_write!(infallible, write32, call_config_write(pci_write_config_dword) <- u32); + // Return value from C function is ignored in infallible accessors. + // SAFETY: By the type invariant `pdev` is a valid address. + let _ = unsafe { bindings::$write_fn(view.pdev.as_raw(), addr, value) }; + } + } + }; } -impl<'a, S: ConfigSpaceKind> IoKnownSize for ConfigSpace<'a, S> { - const MIN_SIZE: usize = S::SIZE; +// PCI configuration space supports 8, 16, and 32-bit accesses. +impl_config_space_io_capable!(u8, pci_read_config_byte, pci_write_config_byte); +impl_config_space_io_capable!(u16, pci_read_config_word, pci_write_config_word); +impl_config_space_io_capable!(u32, pci_read_config_dword, pci_write_config_dword); + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for ConfigSpace<'a, T> { + type Backend = ConfigSpaceBackend; + type Target = T; + + #[inline] + fn as_view(self) -> ConfigSpace<'a, T> { + self + } } /// A PCI BAR to perform I/O-Operations on. @@ -185,14 +167,31 @@ impl<'a, S: ConfigSpaceKind> IoKnownSize for ConfigSpace<'a, S> { /// /// `Bar` always holds an `IoRaw` instance that holds a valid pointer to the start of the I/O /// memory mapped PCI BAR and its size. -pub struct Bar<const SIZE: usize = 0> { - pdev: ARef<Device>, - io: MmioRaw<SIZE>, +pub struct Bar<'a, const SIZE: usize = 0> { + pdev: &'a Device<device::Bound>, + io: MmioRaw<crate::io::Region<SIZE>>, num: i32, } -impl<const SIZE: usize> Bar<SIZE> { - pub(super) fn new(pdev: &Device, num: u32, name: &CStr) -> Result<Self> { +impl<const SIZE: usize> ForLt for Bar<'static, SIZE> { + type Of<'a> = Bar<'a, SIZE>; +} + +// SAFETY: `Bar<'a, SIZE>` is covariant over `'a`; it holds `&'a Device<Bound>`, +// which is covariant. +unsafe impl<const SIZE: usize> CovariantForLt for Bar<'static, SIZE> {} + +/// A device-managed PCI BAR mapping. +/// +/// See [`Bar::into_devres`]. +pub type DevresBar<const SIZE: usize = 0> = DevresLt<Bar<'static, SIZE>>; + +impl<'a, const SIZE: usize> Bar<'a, SIZE> { + pub(super) fn new( + pdev: &'a Device<device::Bound>, + num: u32, + name: &'static CStr, + ) -> Result<Self> { let len = pdev.resource_len(num)?; if len == 0 { return Err(ENOMEM); @@ -223,7 +222,7 @@ impl<const SIZE: usize> Bar<SIZE> { return Err(ENOMEM); } - let io = match MmioRaw::new(ioptr, len as usize) { + let io = match MmioRaw::new_region(ioptr, len as usize) { Ok(io) => io, Err(err) => { // SAFETY: @@ -235,11 +234,7 @@ impl<const SIZE: usize> Bar<SIZE> { } }; - Ok(Bar { - pdev: pdev.into(), - io, - num, - }) + Ok(Bar { pdev, io, num }) } /// # Safety @@ -258,11 +253,22 @@ impl<const SIZE: usize> Bar<SIZE> { fn release(&self) { // SAFETY: The safety requirements are guaranteed by the type invariant of `self.pdev`. - unsafe { Self::do_release(&self.pdev, self.io.addr(), self.num) }; + unsafe { Self::do_release(self.pdev, self.io.addr(), self.num) }; + } + + /// Consume the `Bar` and register it as a device-managed resource. + /// + /// The returned [`DevresBar`] can outlive the original borrow and be stored in driver data. + /// Access to the BAR is revoked automatically when the device is unbound. + pub fn into_devres(self) -> Result<DevresBar<SIZE>> { + let pdev = self.pdev; + // SAFETY: `Bar` only holds a reference to the device and an I/O mapping, both of which + // remain valid for the device's full bound scope, not just for `'a`. + unsafe { DevresLt::new(pdev.as_ref(), self) } } } -impl Bar { +impl Bar<'_> { #[inline] pub(super) fn index_is_valid(index: u32) -> bool { // A `struct pci_dev` owns an array of resources with at most `PCI_NUM_RESOURCES` entries. @@ -270,18 +276,20 @@ impl Bar { } } -impl<const SIZE: usize> Drop for Bar<SIZE> { +impl<const SIZE: usize> Drop for Bar<'_, SIZE> { fn drop(&mut self) { self.release(); } } -impl<const SIZE: usize> Deref for Bar<SIZE> { - type Target = Mmio<SIZE>; +impl<'a, const SIZE: usize> IoBase<'a> for &'a Bar<'_, SIZE> { + type Backend = MmioBackend; + type Target = crate::io::Region<SIZE>; - fn deref(&self) -> &Self::Target { + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { // SAFETY: By the type invariant of `Self`, the MMIO range in `self.io` is properly mapped. - unsafe { Mmio::from_raw(&self.io) } + unsafe { Mmio::from_raw(self.io) } } } @@ -291,17 +299,13 @@ impl Device<device::Bound> { pub fn iomap_region_sized<'a, const SIZE: usize>( &'a self, bar: u32, - name: &'a CStr, - ) -> impl PinInit<Devres<Bar<SIZE>>, Error> + 'a { - Devres::new(self.as_ref(), Bar::<SIZE>::new(self, bar, name)) + name: &'static CStr, + ) -> Result<Bar<'a, SIZE>> { + Bar::new(self, bar, name) } /// Maps an entire PCI BAR after performing a region-request on it. - pub fn iomap_region<'a>( - &'a self, - bar: u32, - name: &'a CStr, - ) -> impl PinInit<Devres<Bar>, Error> + 'a { + pub fn iomap_region<'a>(&'a self, bar: u32, name: &'static CStr) -> Result<Bar<'a>> { self.iomap_region_sized::<0>(bar, name) } @@ -320,23 +324,25 @@ impl Device<device::Bound> { } } - /// Return an initialized normal (256-byte) config space object. + /// Return a view of the normal (256-byte) config space. pub fn config_space<'a>(&'a self) -> ConfigSpace<'a, Normal> { + // INVARIANT: null is aligned and the range is within config space. ConfigSpace { pdev: self, - _marker: PhantomData, + ptr: Normal::ptr_from_raw_parts_mut(core::ptr::null_mut(), self.cfg_size().into_raw()), } } - /// Return an initialized extended (4096-byte) config space object. + /// Return a view of the extended (4096-byte) config space. pub fn config_space_extended<'a>(&'a self) -> Result<ConfigSpace<'a, Extended>> { if self.cfg_size() != ConfigSpaceSize::Extended { return Err(EINVAL); } + // INVARIANT: null is aligned and we just checked the `cfg_size`. Ok(ConfigSpace { pdev: self, - _marker: PhantomData, + ptr: Extended::ptr_from_raw_parts_mut(core::ptr::null_mut(), 4096), }) } } diff --git a/rust/kernel/pci/irq.rs b/rust/kernel/pci/irq.rs index d9230e105541..22e2cdf82a21 100644 --- a/rust/kernel/pci/irq.rs +++ b/rust/kernel/pci/irq.rs @@ -7,17 +7,11 @@ use crate::{ bindings, device, device::Bound, - devres, error::to_result, - irq::{ - self, - IrqRequest, // - }, - prelude::*, - str::CStr, - sync::aref::ARef, // + irq::IrqRequest, + prelude::*, // }; -use core::ops::RangeInclusive; +use core::num::NonZero; /// IRQ type flags for PCI interrupt allocation. #[derive(Debug, Clone, Copy)] @@ -39,6 +33,16 @@ impl IrqType { IrqType::MsiX => bindings::PCI_IRQ_MSIX, } } + + /// Construct from raw value. + #[inline] + const fn from_raw(raw: u32) -> Self { + match raw { + bindings::PCI_IRQ_MSIX => IrqType::MsiX, + bindings::PCI_IRQ_MSI => IrqType::Msi, + _ => IrqType::Intx, + } + } } /// Set of IRQ types that can be used for PCI interrupt allocation. @@ -71,148 +75,115 @@ impl IrqTypes { } } -/// Represents an allocated IRQ vector for a specific PCI device. +/// A resolved IRQ vector from a PCI interrupt vector allocation. /// -/// This type ties an IRQ vector to the device it was allocated for, -/// ensuring the vector is only used with the correct device. -#[derive(Clone, Copy)] +/// Created by [`IrqVectorRegistration::index`]. Convert to [`IrqRequest`] via [`From`] to register +/// a handler with [`irq::Registration::new`](crate::irq::Registration::new). pub struct IrqVector<'a> { - dev: &'a Device<Bound>, - index: u32, + request: IrqRequest<'a>, + reg: &'a IrqVectorRegistration<'a>, } impl<'a> IrqVector<'a> { - /// Creates a new [`IrqVector`] for the given device and index. + /// Creates a new [`IrqVector`] with an already resolved [`IrqRequest`]. /// /// # Safety /// - /// - `index` must be a valid IRQ vector index for `dev`. - /// - `dev` must point to a [`Device`] that has successfully allocated IRQ vectors. - unsafe fn new(dev: &'a Device<Bound>, index: u32) -> Self { - Self { dev, index } + /// `request` must have been resolved from `reg`. + #[inline] + unsafe fn new(request: IrqRequest<'a>, reg: &'a IrqVectorRegistration<'a>) -> Self { + Self { request, reg } } - /// Returns the raw vector index. - fn index(&self) -> u32 { - self.index + /// Returns the [`IrqVectorRegistration`] this vector was derived from. + #[inline] + pub fn vectors(&self) -> &'a IrqVectorRegistration<'a> { + self.reg } -} -impl<'a> TryInto<IrqRequest<'a>> for IrqVector<'a> { - type Error = Error; + /// Returns the interrupt type the PCI core selected for this vector's allocation. + #[inline] + pub fn irq_type(&self) -> IrqType { + self.reg.irq_type() + } +} - fn try_into(self) -> Result<IrqRequest<'a>> { - // SAFETY: `self.as_raw` returns a valid pointer to a `struct pci_dev`. - let irq = unsafe { bindings::pci_irq_vector(self.dev.as_raw(), self.index()) }; - if irq < 0 { - return Err(crate::error::Error::from_errno(irq)); - } - // SAFETY: `irq` is guaranteed to be a valid IRQ number for `&self`. - Ok(unsafe { IrqRequest::new(self.dev.as_ref(), irq as u32) }) +impl<'a> From<IrqVector<'a>> for IrqRequest<'a> { + #[inline] + fn from(vector: IrqVector<'a>) -> Self { + vector.request } } -/// Represents an IRQ vector allocation for a PCI device. +/// An allocation of PCI interrupt vectors for a device. /// -/// This type ensures that IRQ vectors are properly allocated and freed by -/// tying the allocation to the lifetime of this registration object. +/// This type owns the vector allocation; dropping it frees the vectors. IRQ handlers borrow from +/// this registration and must be dropped before it is. /// /// # Invariants /// -/// The [`Device`] has successfully allocated IRQ vectors. -struct IrqVectorRegistration { - dev: ARef<Device>, +/// `dev` has an allocation of `len` interrupt vectors. +pub struct IrqVectorRegistration<'a> { + dev: &'a Device<Bound>, + len: NonZero<usize>, } -impl IrqVectorRegistration { - /// Allocate and register IRQ vectors for the given PCI device. +impl<'a> IrqVectorRegistration<'a> { + /// Returns the number of allocated vectors. /// - /// Allocates IRQ vectors and registers them with devres for automatic cleanup. - /// Returns a range of valid IRQ vectors. - fn register<'a>( - dev: &'a Device<Bound>, - min_vecs: u32, - max_vecs: u32, - irq_types: IrqTypes, - ) -> Result<RangeInclusive<IrqVector<'a>>> { - // SAFETY: - // - `dev.as_raw()` is guaranteed to be a valid pointer to a `struct pci_dev` - // by the type invariant of `Device`. - // - `pci_alloc_irq_vectors` internally validates all other parameters - // and returns error codes. - let ret = unsafe { - bindings::pci_alloc_irq_vectors(dev.as_raw(), min_vecs, max_vecs, irq_types.as_raw()) - }; - - to_result(ret)?; - let count = ret as u32; + /// This is at least the `min_vecs` that [`Device::alloc_irq_vectors`] was asked for. + #[inline] + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.len.get() + } - // SAFETY: - // - `pci_alloc_irq_vectors` returns the number of allocated vectors on success. - // - Vectors are 0-based, so valid indices are [0, count-1]. - // - `pci_alloc_irq_vectors` guarantees `count >= min_vecs > 0`, so both `0` and - // `count - 1` are valid IRQ vector indices for `dev`. - let range = unsafe { IrqVector::new(dev, 0)..=IrqVector::new(dev, count - 1) }; + /// Returns the interrupt type the PCI core selected for this allocation. + #[inline] + pub fn irq_type(&self) -> IrqType { + // SAFETY: `self.dev.as_raw()` is a valid pointer to a `struct pci_dev`. + IrqType::from_raw(unsafe { bindings::pci_irq_type(self.dev.as_raw()) }) + } - // INVARIANT: The IRQ vector allocation for `dev` above was successful. - let irq_vecs = Self { dev: dev.into() }; - devres::register(dev.as_ref(), irq_vecs, GFP_KERNEL)?; + /// Returns the [`IrqVector`] at `index`. + /// + /// Returns [`EINVAL`] if the `index` is out of bounds for the length reported by + /// [`Self::len()`]. + #[inline] + pub fn index(&self, index: usize) -> Result<IrqVector<'_>> { + let index = u32::try_from(index)?; + + // SAFETY: `self.dev.as_raw()` is a valid pointer to a `struct pci_dev`. + let irq = unsafe { bindings::pci_irq_vector(self.dev.as_raw(), index) }; + if irq < 0 { + return Err(Error::from_errno(irq)); + } - Ok(range) + // SAFETY: `irq` is a valid IRQ number for `self.dev`, resolved from this registration. + Ok(unsafe { IrqVector::new(IrqRequest::new(self.dev.as_ref(), irq as u32), self) }) } } -impl Drop for IrqVectorRegistration { +impl Drop for IrqVectorRegistration<'_> { + #[inline] fn drop(&mut self) { - // SAFETY: - // - By the type invariant, `self.dev.as_raw()` is a valid pointer to a `struct pci_dev`. - // - `self.dev` has successfully allocated IRQ vectors. + // SAFETY: By the type invariant, `self.dev.as_raw()` is a valid pointer to a + // `struct pci_dev` that has successfully allocated IRQ vectors. unsafe { bindings::pci_free_irq_vectors(self.dev.as_raw()) }; } } impl Device<device::Bound> { - /// Returns a [`kernel::irq::Registration`] for the given IRQ vector. - pub fn request_irq<'a, T: crate::irq::Handler + 'static>( - &'a self, - vector: IrqVector<'a>, - flags: irq::Flags, - name: &'static CStr, - handler: impl PinInit<T, Error> + 'a, - ) -> impl PinInit<irq::Registration<T>, Error> + 'a { - pin_init::pin_init_scope(move || { - let request = vector.try_into()?; - - Ok(irq::Registration::<T>::new(request, flags, name, handler)) - }) - } - - /// Returns a [`kernel::irq::ThreadedRegistration`] for the given IRQ vector. - pub fn request_threaded_irq<'a, T: crate::irq::ThreadedHandler + 'static>( - &'a self, - vector: IrqVector<'a>, - flags: irq::Flags, - name: &'static CStr, - handler: impl PinInit<T, Error> + 'a, - ) -> impl PinInit<irq::ThreadedRegistration<T>, Error> + 'a { - pin_init::pin_init_scope(move || { - let request = vector.try_into()?; - - Ok(irq::ThreadedRegistration::<T>::new( - request, flags, name, handler, - )) - }) - } - - /// Allocate IRQ vectors for this PCI device with automatic cleanup. + /// Allocate IRQ vectors for this PCI device. /// /// Allocates between `min_vecs` and `max_vecs` interrupt vectors for the device. /// The allocation will use MSI-X, MSI, or INTx interrupts based on the `irq_types` /// parameter and hardware capabilities. When multiple types are specified, the kernel /// will try them in order of preference: MSI-X first, then MSI, then INTx interrupts. /// - /// The allocated vectors are automatically freed when the device is unbound, using the - /// devres (device resource management) system. + /// The allocated vectors are freed when the returned [`IrqVectorRegistration`] is dropped. + /// Use [`IrqVectorRegistration::index`] to obtain an [`IrqVector`] for a given vector + /// index. /// /// # Arguments /// @@ -222,8 +193,8 @@ impl Device<device::Bound> { /// /// # Returns /// - /// Returns a range of IRQ vectors that were successfully allocated, or an error if the - /// allocation fails or cannot meet the minimum requirement. + /// Returns the IRQ vector registration, or an error if `min_vecs` vectors cannot be + /// allocated. /// /// # Examples /// @@ -246,7 +217,20 @@ impl Device<device::Bound> { min_vecs: u32, max_vecs: u32, irq_types: IrqTypes, - ) -> Result<RangeInclusive<IrqVector<'_>>> { - IrqVectorRegistration::register(self, min_vecs, max_vecs, irq_types) + ) -> Result<IrqVectorRegistration<'_>> { + // SAFETY: + // - `self.as_raw()` is guaranteed to be a valid pointer to a `struct pci_dev` + // by the type invariant of `Device`. + // - `pci_alloc_irq_vectors` internally validates all other parameters + // and returns error codes. + let ret = unsafe { + bindings::pci_alloc_irq_vectors(self.as_raw(), min_vecs, max_vecs, irq_types.as_raw()) + }; + to_result(ret)?; + + let len = NonZero::new(ret as usize).ok_or(EINVAL)?; + + // INVARIANT: `pci_alloc_irq_vectors()` allocated `len` vectors for `self`. + Ok(IrqVectorRegistration { dev: self, len }) } } diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 8917d4ee499f..ac0a012ae1bb 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -17,10 +17,7 @@ use crate::{ from_result, to_result, // }, - io::{ - mem::IoRequest, - Resource, // - }, + io::Resource, irq::{ self, IrqRequest, // @@ -31,6 +28,9 @@ use crate::{ ThisModule, // }; +#[cfg(CONFIG_HAS_IOMEM)] +use crate::io::mem::IoRequest; + use core::{ marker::PhantomData, mem::offset_of, @@ -45,18 +45,18 @@ pub struct Adapter<T: Driver>(T); // SAFETY: // - `bindings::platform_driver` is a C type declared as `repr(C)`. -// - `T` is the type of the driver's device private data. +// - `T::Data` is the type of the driver's device private data. // - `struct platform_driver` embeds a `struct device_driver`. // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. -unsafe impl<T: Driver + 'static> driver::DriverLayout for Adapter<T> { +unsafe impl<T: Driver> driver::DriverLayout for Adapter<T> { type DriverType = bindings::platform_driver; - type DriverData = T; + type DriverData<'bound> = T::Data<'bound>; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); } // SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if // a preceding call to `register` has been successful. -unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { +unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { unsafe fn register( pdrv: &Opaque<Self::DriverType>, name: &'static CStr, @@ -82,7 +82,9 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { } // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. - to_result(unsafe { bindings::__platform_driver_register(pdrv.get(), module.0) }) + to_result(unsafe { + bindings::__platform_driver_register(pdrv.get(), module.as_ptr(), name.as_char_ptr()) + }) } unsafe fn unregister(pdrv: &Opaque<Self::DriverType>) { @@ -91,14 +93,15 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { } } -impl<T: Driver + 'static> Adapter<T> { +impl<T: Driver> Adapter<T> { extern "C" fn probe_callback(pdev: *mut bindings::platform_device) -> kernel::ffi::c_int { // SAFETY: The platform bus only ever calls the probe callback with a valid pointer to a // `struct platform_device`. // // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. - let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal>>() }; - let info = <Self as driver::Adapter>::id_info(pdev.as_ref()); + let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal<'_>>>() }; + // SAFETY: `pdev` matched data is of type `Self::IdInfo`. + let info = unsafe { <Self as driver::Adapter>::id_info(pdev.as_ref()) }; from_result(|| { let data = T::probe(pdev, info); @@ -113,18 +116,18 @@ impl<T: Driver + 'static> Adapter<T> { // `struct platform_device`. // // INVARIANT: `pdev` is valid for the duration of `remove_callback()`. - let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal>>() }; + let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal<'_>>>() }; // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called - // and stored a `Pin<KBox<T>>`. - let data = unsafe { pdev.as_ref().drvdata_borrow::<T>() }; + // and stored a `Pin<KBox<T::Data<'_>>>`. + let data = unsafe { pdev.as_ref().drvdata_borrow::<T::Data<'_>>() }; T::unbind(pdev, data); } } -impl<T: Driver + 'static> driver::Adapter for Adapter<T> { +impl<T: Driver> driver::Adapter for Adapter<T> { type IdInfo = T::IdInfo; fn of_id_table() -> Option<of::IdTable<Self::IdInfo>> { @@ -174,7 +177,6 @@ macro_rules! module_platform_driver { /// /// kernel::of_device_table!( /// OF_TABLE, -/// MODULE_OF_TABLE, /// <MyDriver as platform::Driver>::IdInfo, /// [ /// (of::DeviceId::new(c"test,device"), ()) @@ -183,7 +185,6 @@ macro_rules! module_platform_driver { /// /// kernel::acpi_device_table!( /// ACPI_TABLE, -/// MODULE_ACPI_TABLE, /// <MyDriver as platform::Driver>::IdInfo, /// [ /// (acpi::DeviceId::new(c"LNUXBEEF"), ()) @@ -192,18 +193,19 @@ macro_rules! module_platform_driver { /// /// impl platform::Driver for MyDriver { /// type IdInfo = (); +/// type Data<'bound> = Self; /// const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = Some(&OF_TABLE); /// const ACPI_ID_TABLE: Option<acpi::IdTable<Self::IdInfo>> = Some(&ACPI_TABLE); /// -/// fn probe( -/// _pdev: &platform::Device<Core>, -/// _id_info: Option<&Self::IdInfo>, -/// ) -> impl PinInit<Self, Error> { +/// fn probe<'bound>( +/// _pdev: &'bound platform::Device<Core<'_>>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound { /// Err(ENODEV) /// } /// } ///``` -pub trait Driver: Send { +pub trait Driver { /// The type holding driver private data about each device id supported by the driver. // TODO: Use associated_type_defaults once stabilized: // @@ -212,6 +214,9 @@ pub trait Driver: Send { // ``` type IdInfo: 'static; + /// The type of the driver's bus device private data. + type Data<'bound>: Send + 'bound; + /// The table of OF device ids supported by the driver. const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = None; @@ -222,10 +227,10 @@ pub trait Driver: Send { /// /// Called when a new platform device is added or discovered. /// Implementers should attempt to initialize the device here. - fn probe( - dev: &Device<device::Core>, - id_info: Option<&Self::IdInfo>, - ) -> impl PinInit<Self, Error>; + fn probe<'bound>( + dev: &'bound Device<device::Core<'_>>, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound; /// Platform driver unbind. /// @@ -236,8 +241,8 @@ pub trait Driver: Send { /// `&Device<Core>` or `&Device<Bound>` reference. For instance, drivers may try to perform I/O /// operations to gracefully tear down the device. /// - /// Otherwise, release operations for driver resources should be performed in `Self::drop`. - fn unbind(dev: &Device<device::Core>, this: Pin<&Self>) { + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(dev: &'bound Device<device::Core<'_>>, this: Pin<&Self::Data<'bound>>) { let _ = (dev, this); } } @@ -301,6 +306,7 @@ impl<Ctx: device::DeviceContext> Device<Ctx> { } } +#[cfg(CONFIG_HAS_IOMEM)] impl Device<Bound> { /// Returns an `IoRequest` for the resource at `index`, if any. pub fn io_request_by_index(&self, index: u32) -> Option<IoRequest<'_>> { @@ -333,22 +339,30 @@ macro_rules! define_irq_accessor_by_index { $handler_trait:ident ) => { $(#[$meta])* - pub fn $fn_name<'a, T: irq::$handler_trait + 'static>( + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the resulting registration or otherwise prevent its + /// [`Drop`] implementation from running. + pub unsafe fn $fn_name<'a, T: irq::$handler_trait + 'a>( &'a self, flags: irq::Flags, index: u32, name: &'static CStr, handler: impl PinInit<T, Error> + 'a, - ) -> impl PinInit<irq::$reg_type<T>, Error> + 'a { + ) -> impl PinInit<irq::$reg_type<'a, T>, Error> + 'a { pin_init::pin_init_scope(move || { let request = self.$request_fn(index)?; - Ok(irq::$reg_type::<T>::new( - request, - flags, - name, - handler, - )) + // SAFETY: Caller guarantees the Registration will not be leaked. + Ok(unsafe { + irq::$reg_type::<T>::new( + request, + flags, + name, + handler, + ) + }) }) } }; @@ -362,22 +376,30 @@ macro_rules! define_irq_accessor_by_name { $handler_trait:ident ) => { $(#[$meta])* - pub fn $fn_name<'a, T: irq::$handler_trait + 'static>( + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the resulting registration or otherwise prevent its + /// [`Drop`] implementation from running. + pub unsafe fn $fn_name<'a, T: irq::$handler_trait + 'a>( &'a self, flags: irq::Flags, irq_name: &'a CStr, name: &'static CStr, handler: impl PinInit<T, Error> + 'a, - ) -> impl PinInit<irq::$reg_type<T>, Error> + 'a { + ) -> impl PinInit<irq::$reg_type<'a, T>, Error> + 'a { pin_init::pin_init_scope(move || { let request = self.$request_fn(irq_name)?; - Ok(irq::$reg_type::<T>::new( - request, - flags, - name, - handler, - )) + // SAFETY: Caller guarantees the Registration will not be leaked. + Ok(unsafe { + irq::$reg_type::<T>::new( + request, + flags, + name, + handler, + ) + }) }) } }; @@ -509,7 +531,7 @@ impl Device<Bound> { kernel::impl_device_context_deref!(unsafe { Device }); kernel::impl_device_context_into_aref!(Device); -impl crate::dma::Device for Device<device::Core> {} +impl<'a> crate::dma::Device<'a> for Device<device::Core<'a>> {} // SAFETY: Instances of `Device` are always reference-counted. unsafe impl crate::sync::aref::AlwaysRefCounted for Device { @@ -561,3 +583,7 @@ unsafe impl Send for Device {} // SAFETY: `Device` can be shared among threads because all methods of `Device` // (i.e. `Device<Normal>) are thread safe. unsafe impl Sync for Device {} + +// SAFETY: Same as `Device<Normal>` -- the underlying `struct platform_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device<device::Bound> {} diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index 2877e3f7b6d3..ca396f1f78a6 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -13,43 +13,114 @@ #[doc(no_inline)] pub use core::{ - mem::{align_of, align_of_val, size_of, size_of_val}, - pin::Pin, + mem::{ + align_of, + align_of_val, + size_of, + size_of_val, // + }, + pin::Pin, // }; +#[doc(no_inline)] pub use ::ffi::{ - c_char, c_int, c_long, c_longlong, c_schar, c_short, c_uchar, c_uint, c_ulong, c_ulonglong, - c_ushort, c_void, CStr, + c_char, + c_int, + c_long, + c_longlong, + c_schar, + c_short, + c_uchar, + c_uint, + c_ulong, + c_ulonglong, + c_ushort, + c_void, + CStr, // }; -pub use crate::alloc::{flags::*, Box, KBox, KVBox, KVVec, KVec, VBox, VVec, Vec}; +#[doc(no_inline)] +pub use macros::{ + export, + fmt, + kunit_tests, + module, + vtable, // +}; #[doc(no_inline)] -pub use macros::{export, fmt, kunit_tests, module, vtable}; +pub use pin_init::{ + init, + pin_data, + pin_init, + pinned_drop, + InPlaceWrite, + Init, + PinInit, + Zeroable, // +}; -pub use pin_init::{init, pin_data, pin_init, pinned_drop, InPlaceWrite, Init, PinInit, Zeroable}; +#[doc(no_inline)] +pub use zerocopy::{ + FromBytes, + IntoBytes, // +}; -pub use super::{build_assert, build_error}; +#[doc(no_inline)] +pub use zerocopy_derive::{ + FromBytes, + IntoBytes, // +}; + +#[doc(no_inline)] +pub use super::{ + alloc::{ + flags::*, + Box, + KBox, + KVBox, + KVVec, + KVec, + VBox, + VVec, + Vec, // + }, + build_assert::{ + build_assert, + build_error, + const_assert, + static_assert, // + }, + current, + dev_alert, + dev_crit, + dev_dbg, + dev_emerg, + dev_err, + dev_info, + dev_notice, + dev_warn, + error::{ + code::*, + Error, + Result, // + }, + init::InPlaceInit, + pr_alert, + pr_crit, + pr_debug, + pr_emerg, + pr_err, + pr_info, + pr_notice, + pr_warn, + str::CStrExt as _, + try_init, + try_pin_init, + uaccess::UserPtr, + ThisModule, // +}; // `super::std_vendor` is hidden, which makes the macro inline for some reason. #[doc(no_inline)] pub use super::dbg; -pub use super::{dev_alert, dev_crit, dev_dbg, dev_emerg, dev_err, dev_info, dev_notice, dev_warn}; -pub use super::{pr_alert, pr_crit, pr_debug, pr_emerg, pr_err, pr_info, pr_notice, pr_warn}; - -pub use super::{try_init, try_pin_init}; - -pub use super::static_assert; - -pub use super::error::{code::*, Error, Result}; - -pub use super::{str::CStrExt as _, ThisModule}; - -pub use super::init::InPlaceInit; - -pub use super::current; - -pub use super::uaccess::UserPtr; - -#[cfg(not(CONFIG_RUSTC_HAS_SLICE_AS_FLATTENED))] -pub use super::slice::AsFlattened; diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs index 6fd84389a858..0d62beeedca5 100644 --- a/rust/kernel/print.rs +++ b/rust/kernel/print.rs @@ -99,7 +99,7 @@ pub mod format_strings { /// The format string must be one of the ones in [`format_strings`], and /// the module name must be null-terminated. /// -/// [`_printk`]: srctree/include/linux/_printk.h +/// [`_printk`]: srctree/include/linux/printk.h #[doc(hidden)] #[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] pub unsafe fn call_printk( diff --git a/rust/kernel/ptr.rs b/rust/kernel/ptr.rs index bdc2d79ff669..82acb531b17b 100644 --- a/rust/kernel/ptr.rs +++ b/rust/kernel/ptr.rs @@ -11,6 +11,8 @@ use core::mem::{ }; use core::num::NonZero; +use crate::const_assert; + /// Type representing an alignment, which is always a power of two. /// /// It is used to validate that a given value is a valid alignment, and to perform masking and @@ -44,12 +46,10 @@ impl Alignment { /// ``` #[inline(always)] pub const fn new<const ALIGN: usize>() -> Self { - const { - assert!( - ALIGN.is_power_of_two(), - "Provided alignment is not a power of two." - ); - } + const_assert!( + ALIGN.is_power_of_two(), + "Provided alignment is not a power of two." + ); // INVARIANT: `align` is a power of two. // SAFETY: `align` is a power of two, and thus non-zero. @@ -87,7 +87,6 @@ impl Alignment { /// This is equivalent to [`align_of`], but with the return value provided as an [`Alignment`]. #[inline(always)] pub const fn of<T>() -> Self { - #![allow(clippy::incompatible_msrv)] // This cannot panic since alignments are always powers of two. // // We unfortunately cannot use `new` as it would require the `generic_const_exprs` feature. @@ -236,11 +235,20 @@ impl_alignable_uint!(u8, u16, u32, u64, usize); /// /// This is a generalization of [`size_of`] that works for dynamically sized types. pub trait KnownSize { + /// Minimum size of this type known at compile-time. + const MIN_SIZE: usize; + + /// Minimum alignment of this type known at compile-time. + const MIN_ALIGN: Alignment; + /// Get the size of an object of this type in bytes, with the metadata of the given pointer. fn size(p: *const Self) -> usize; } impl<T> KnownSize for T { + const MIN_SIZE: usize = size_of::<T>(); + const MIN_ALIGN: Alignment = Alignment::of::<T>(); + #[inline(always)] fn size(_: *const Self) -> usize { size_of::<T>() @@ -248,8 +256,40 @@ impl<T> KnownSize for T { } impl<T> KnownSize for [T] { + const MIN_SIZE: usize = 0; + const MIN_ALIGN: Alignment = Alignment::of::<T>(); + #[inline(always)] fn size(p: *const Self) -> usize { p.len() * size_of::<T>() } } + +/// Aligns `value` up to `align`. +/// +/// This is the const-compatible equivalent of [`Alignable::align_up`]. +/// +/// Returns [`None`] on overflow. +/// +/// # Examples +/// +/// ``` +/// use kernel::{ +/// ptr::{ +/// const_align_up, +/// Alignment, // +/// }, +/// sizes::SZ_4K, // +/// }; +/// +/// assert_eq!(const_align_up(0x4f, Alignment::new::<16>()), Some(0x50)); +/// assert_eq!(const_align_up(0x40, Alignment::new::<16>()), Some(0x40)); +/// assert_eq!(const_align_up(1, Alignment::new::<SZ_4K>()), Some(SZ_4K)); +/// ``` +#[inline(always)] +pub const fn const_align_up(value: usize, align: Alignment) -> Option<usize> { + match value.checked_add(align.as_usize() - 1) { + Some(v) => Some(v & align.mask()), + None => None, + } +} diff --git a/rust/kernel/ptr/projection.rs b/rust/kernel/ptr/projection.rs index 140ea8e21617..af72d3b0e2a3 100644 --- a/rust/kernel/ptr/projection.rs +++ b/rust/kernel/ptr/projection.rs @@ -26,14 +26,14 @@ impl From<OutOfBound> for Error { /// /// # Safety /// -/// The implementation of `index` and `get` (if [`Some`] is returned) must ensure that, if provided -/// input pointer `slice` and returned pointer `output`, then: +/// For a given input pointer `slice` and return value `output`, the implementation of `index`, +/// `build_index` and `get` (if [`Some`] is returned) must ensure that: /// - `output` has the same provenance as `slice`; /// - `output.byte_offset_from(slice)` is between 0 to /// `KnownSize::size(slice) - KnownSize::size(output)`. /// -/// This means that if the input pointer is valid, then pointer returned by `get` or `index` is -/// also valid. +/// This means that if the input pointer is valid, then the pointer returned by `get`, `index` +/// or `build_index` is also valid. #[diagnostic::on_unimplemented(message = "`{Self}` cannot be used to index `{T}`")] #[doc(hidden)] pub unsafe trait ProjectIndex<T: ?Sized>: Sized { @@ -42,10 +42,16 @@ pub unsafe trait ProjectIndex<T: ?Sized>: Sized { /// Returns an index-projected pointer, if in bounds. fn get(self, slice: *mut T) -> Option<*mut Self::Output>; + /// Returns an index-projected pointer; panic if out of bounds. + fn index(self, slice: *mut T) -> *mut Self::Output; + /// Returns an index-projected pointer; fail the build if it cannot be proved to be in bounds. #[inline(always)] - fn index(self, slice: *mut T) -> *mut Self::Output { - Self::get(self, slice).unwrap_or_else(|| build_error!()) + fn build_index(self, slice: *mut T) -> *mut Self::Output { + match Self::get(self, slice) { + Some(v) => v, + None => build_error!(), + } } } @@ -67,6 +73,11 @@ where fn index(self, slice: *mut [T; N]) -> *mut Self::Output { <I as ProjectIndex<[T]>>::index(self, slice) } + + #[inline(always)] + fn build_index(self, slice: *mut [T; N]) -> *mut Self::Output { + <I as ProjectIndex<[T]>>::build_index(self, slice) + } } // SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to @@ -82,6 +93,16 @@ unsafe impl<T> ProjectIndex<[T]> for usize { Some(slice.cast::<T>().wrapping_add(self)) } } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut T { + // Leverage Rust built-in operators for bounds checking. + // SAFETY: All non-null and aligned pointers are valid for ZST read. + let zst_slice = + unsafe { core::slice::from_raw_parts::<()>(core::ptr::dangling(), slice.len()) }; + let () = zst_slice[self]; + slice.cast::<T>().wrapping_add(self) + } } // SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to @@ -100,6 +121,18 @@ unsafe impl<T> ProjectIndex<[T]> for core::ops::Range<usize> { new_len, )) } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut [T] { + // Leverage Rust built-in operators for bounds checking. + // SAFETY: All non-null and aligned pointers are valid for ZST read. + let zst_slice = + unsafe { core::slice::from_raw_parts::<()>(core::ptr::dangling(), slice.len()) }; + _ = zst_slice[self.clone()]; + + // SAFETY: Bounds checked. + unsafe { self.get(slice).unwrap_unchecked() } + } } // SAFETY: Safety requirement guaranteed by the forwarded impl. @@ -110,6 +143,11 @@ unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeTo<usize> { fn get(self, slice: *mut [T]) -> Option<*mut [T]> { (0..self.end).get(slice) } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut [T] { + (0..self.end).index(slice) + } } // SAFETY: Safety requirement guaranteed by the forwarded impl. @@ -120,6 +158,11 @@ unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeFrom<usize> { fn get(self, slice: *mut [T]) -> Option<*mut [T]> { (self.start..slice.len()).get(slice) } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut [T] { + (self.start..slice.len()).index(slice) + } } // SAFETY: `get` returned the pointer as is, so it always has the same provenance and offset of 0. @@ -130,6 +173,11 @@ unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeFull { fn get(self, slice: *mut [T]) -> Option<*mut [T]> { Some(slice) } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut [T] { + slice + } } /// A helper trait to perform field projection. @@ -207,10 +255,13 @@ unsafe impl<T: Deref> ProjectField<true> for T { /// If a mutable pointer is needed, the macro input can be prefixed with the `mut` keyword, i.e. /// `kernel::ptr::project!(mut ptr, projection)`. By default, a const pointer is created. /// -/// `ptr::project!` macro can perform both fallible indexing and build-time checked indexing. -/// `[index]` form performs build-time bounds checking; if compiler fails to prove `[index]` is in -/// bounds, compilation will fail. `[index]?` can be used to perform runtime bounds checking; -/// `OutOfBound` error is raised via `?` if the index is out of bounds. +/// The `ptr::project!` macro can perform both fallible indexing and build-time checked indexing. +/// The syntax is of the form `[<flavor>: index]` where `flavor` indicates the way of handling +/// index out-of-bounds errors. +/// - `try` will raise an [`OutOfBound`] error (which is convertible to [`ERANGE`]). +/// - `build` will use the [`build_assert!`] mechanism to have the compiler validate the index is +/// in bounds. +/// - `panic` will cause a Rust [`panic!`] if the index goes out of bounds. /// /// # Examples /// @@ -228,17 +279,21 @@ unsafe impl<T: Deref> ProjectField<true> for T { /// } /// ``` /// -/// Index projections are performed with `[index]`: +/// Index projections are performed with `[<flavor>: index]`, where `flavor` is `try`, `build` or +/// `panic`: /// /// ``` /// fn proj(ptr: *const [u8; 32]) -> Result { -/// let field_ptr: *const u8 = kernel::ptr::project!(ptr, [1]); +/// let field_ptr: *const u8 = kernel::ptr::project!(ptr, [build: 1]); /// // The following invocation, if uncommented, would fail the build. /// // -/// // kernel::ptr::project!(ptr, [128]); +/// // kernel::ptr::project!(ptr, [build: 128]); /// /// // This will raise an `OutOfBound` error (which is convertible to `ERANGE`). -/// kernel::ptr::project!(ptr, [128]?); +/// kernel::ptr::project!(ptr, [try: 128]); +/// +/// // This will panic at runtime if executed. +/// kernel::ptr::project!(ptr, [panic: 128]); /// Ok(()) /// } /// ``` @@ -248,7 +303,7 @@ unsafe impl<T: Deref> ProjectField<true> for T { /// ``` /// let ptr: *const [u8; 32] = core::ptr::dangling(); /// let field_ptr: Result<*const u8> = (|| -> Result<_> { -/// Ok(kernel::ptr::project!(ptr, [128]?)) +/// Ok(kernel::ptr::project!(ptr, [try: 128])) /// })(); /// assert!(field_ptr.is_err()); /// ``` @@ -257,7 +312,7 @@ unsafe impl<T: Deref> ProjectField<true> for T { /// /// ``` /// let ptr: *mut [(u8, u16); 32] = core::ptr::dangling_mut(); -/// let field_ptr: *mut u16 = kernel::ptr::project!(mut ptr, [1].1); +/// let field_ptr: *mut u16 = kernel::ptr::project!(mut ptr, [build: 1].1); /// ``` #[macro_export] macro_rules! project_pointer { @@ -280,16 +335,22 @@ macro_rules! project_pointer { $crate::ptr::project!(@gen $ptr, $($rest)*) }; // Fallible index projection. - (@gen $ptr:ident, [$index:expr]? $($rest:tt)*) => { + (@gen $ptr:ident, [try: $index:expr] $($rest:tt)*) => { let $ptr = $crate::ptr::projection::ProjectIndex::get($index, $ptr) .ok_or($crate::ptr::projection::OutOfBound)?; $crate::ptr::project!(@gen $ptr, $($rest)*) }; - // Build-time checked index projection. - (@gen $ptr:ident, [$index:expr] $($rest:tt)*) => { + // Panicking index projection. + (@gen $ptr:ident, [panic: $index:expr] $($rest:tt)*) => { let $ptr = $crate::ptr::projection::ProjectIndex::index($index, $ptr); $crate::ptr::project!(@gen $ptr, $($rest)*) }; + // Build-time checked index projection. + (@gen $ptr:ident, [build: $index:expr] $($rest:tt)*) => { + let $ptr = $crate::ptr::projection::ProjectIndex::build_index($index, $ptr); + $crate::ptr::project!(@gen $ptr, $($rest)*) + }; + (mut $ptr:expr, $($proj:tt)*) => {{ let ptr: *mut _ = $ptr; $crate::ptr::project!(@gen ptr, $($proj)*); diff --git a/rust/kernel/pwm.rs b/rust/kernel/pwm.rs index 6c9d667009ef..8aa47304bec3 100644 --- a/rust/kernel/pwm.rs +++ b/rust/kernel/pwm.rs @@ -494,9 +494,7 @@ impl PwmOpsVTable { /// This is used to bridge Rust trait implementations to the C `struct pwm_ops` /// expected by the kernel. pub const fn create_pwm_ops<T: PwmOps>() -> PwmOpsVTable { - // SAFETY: `core::mem::zeroed()` is unsafe. For `pwm_ops`, all fields are - // `Option<extern "C" fn(...)>` or data, so a zeroed pattern (None/0) is valid initially. - let mut ops: bindings::pwm_ops = unsafe { core::mem::zeroed() }; + let mut ops: bindings::pwm_ops = pin_init::zeroed(); ops.request = Some(Adapter::<T>::request_callback); ops.capture = Some(Adapter::<T>::capture_callback); @@ -600,7 +598,7 @@ impl<T: PwmOps> Chip<T> { let drvdata_ptr = unsafe { bindings::pwmchip_get_drvdata(c_chip_ptr) }; // SAFETY: We construct the `T` object in-place in the allocated private memory. - unsafe { data.__pinned_init(drvdata_ptr.cast()) }.inspect_err(|_| { + unsafe { pin_init::raw_try_init(drvdata_ptr.cast(), data) }.inspect_err(|_| { // SAFETY: It is safe to call `pwmchip_put()` with a valid pointer obtained // from `pwmchip_alloc()`. We will not use pointer after this. unsafe { bindings::pwmchip_put(c_chip_ptr) } diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs index 0f4ae673256d..0e55e2a0fb37 100644 --- a/rust/kernel/revocable.rs +++ b/rust/kernel/revocable.rs @@ -7,12 +7,21 @@ use pin_init::Wrapper; -use crate::{bindings, prelude::*, sync::rcu, types::Opaque}; +use crate::{ + prelude::*, + sync::{ + atomic::{ + AtomicFlag, + Relaxed, // + }, + rcu, // + }, + types::Opaque, // +}; use core::{ marker::PhantomData, ops::Deref, - ptr::drop_in_place, - sync::atomic::{AtomicBool, Ordering}, + ptr::drop_in_place, // }; /// An object that can become inaccessible at runtime. @@ -65,7 +74,7 @@ use core::{ /// ``` #[pin_data(PinnedDrop)] pub struct Revocable<T> { - is_available: AtomicBool, + is_available: AtomicFlag, #[pin] data: Opaque<T>, } @@ -84,7 +93,7 @@ impl<T> Revocable<T> { /// Creates a new revocable instance of the given data. pub fn new<E>(data: impl PinInit<T, E>) -> impl PinInit<Self, E> { try_pin_init!(Self { - is_available: AtomicBool::new(true), + is_available: AtomicFlag::new(true), data <- Opaque::pin_init(data), }? E) } @@ -98,7 +107,7 @@ impl<T> Revocable<T> { /// because another CPU may be waiting to complete the revocation of this object. pub fn try_access(&self) -> Option<RevocableGuard<'_, T>> { let guard = rcu::read_lock(); - if self.is_available.load(Ordering::Relaxed) { + if self.is_available.load(Relaxed) { // Since `self.is_available` is true, data is initialised and has to remain valid // because the RCU read side lock prevents it from being dropped. Some(RevocableGuard::new(self.data.get(), guard)) @@ -116,7 +125,7 @@ impl<T> Revocable<T> { /// allowed to sleep because another CPU may be waiting to complete the revocation of this /// object. pub fn try_access_with_guard<'a>(&'a self, _guard: &'a rcu::Guard) -> Option<&'a T> { - if self.is_available.load(Ordering::Relaxed) { + if self.is_available.load(Relaxed) { // SAFETY: Since `self.is_available` is true, data is initialised and has to remain // valid because the RCU read side lock prevents it from being dropped. Some(unsafe { &*self.data.get() }) @@ -157,12 +166,11 @@ impl<T> Revocable<T> { /// /// Callers must ensure that there are no more concurrent users of the revocable object. unsafe fn revoke_internal<const SYNC: bool>(&self) -> bool { - let revoke = self.is_available.swap(false, Ordering::Relaxed); + let revoke = self.is_available.xchg(false, Relaxed); if revoke { if SYNC { - // SAFETY: Just an FFI call, there are no further requirements. - unsafe { bindings::synchronize_rcu() }; + rcu::synchronize_rcu(); } // SAFETY: We know `self.data` is valid because only one CPU can succeed the diff --git a/rust/kernel/serdev.rs b/rust/kernel/serdev.rs new file mode 100644 index 000000000000..17ca504b7f8d --- /dev/null +++ b/rust/kernel/serdev.rs @@ -0,0 +1,604 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for the serial device bus. +//! +//! C header: [`include/linux/serdev.h`](srctree/include/linux/serdev.h) + +use crate::{ + acpi, + device, + driver, + error::{ + from_result, + to_result, + VTABLE_DEFAULT_ERROR, // + }, + new_mutex, + of, + prelude::*, + sync::{ + aref::AlwaysRefCounted, + Mutex, // + }, + time::Jiffies, + types::{ + Opaque, + ScopeGuard, // + }, // +}; + +use core::{ + cell::UnsafeCell, + marker::PhantomData, + mem::{offset_of, MaybeUninit}, + ptr::NonNull, // +}; + +/// Parity bit to use with a serial device. +#[repr(u32)] +pub enum Parity { + /// No parity bit. + None = bindings::serdev_parity_SERDEV_PARITY_NONE, + /// Even partiy. + Even = bindings::serdev_parity_SERDEV_PARITY_EVEN, + /// Odd parity. + Odd = bindings::serdev_parity_SERDEV_PARITY_ODD, +} + +/// An adapter for the registration of serial device bus device drivers. +pub struct Adapter<T: Driver>(T); + +// SAFETY: +// - `bindings::serdev_device_driver` is a C type declared as `repr(C)`. +// - `PrivateData<'bound, T>` is the type of the driver's device private data. +// - `struct serdev_device_driver` embeds a `struct device_driver`. +// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. +unsafe impl<T: Driver> driver::DriverLayout for Adapter<T> { + type DriverType = bindings::serdev_device_driver; + type DriverData<'bound> = PrivateData<'bound, T>; + const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); +} + +// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { + unsafe fn register( + sdrv: &Opaque<Self::DriverType>, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + let of_table = match T::OF_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + let acpi_table = match T::ACPI_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + // SAFETY: It's safe to set the fields of `struct serdev_device_driver` on initialization. + unsafe { + (*sdrv.get()).driver.name = name.as_char_ptr(); + (*sdrv.get()).probe = Some(Self::probe_callback); + (*sdrv.get()).remove = Some(Self::remove_callback); + (*sdrv.get()).driver.of_match_table = of_table; + (*sdrv.get()).driver.acpi_match_table = acpi_table; + } + + // SAFETY: `sdrv` is guaranteed to be a valid `DriverType`. + to_result(unsafe { bindings::__serdev_device_driver_register(sdrv.get(), module.as_ptr()) }) + } + + unsafe fn unregister(sdrv: &Opaque<Self::DriverType>) { + // SAFETY: `sdrv` is guaranteed to be a valid `DriverType`. + unsafe { bindings::serdev_device_driver_unregister(sdrv.get()) }; + } +} + +#[doc(hidden)] +#[pin_data(PinnedDrop)] +pub struct PrivateData<'bound, T: Driver> { + sdev: &'bound Device<device::Bound>, + #[pin] + driver: UnsafeCell<MaybeUninit<T::Data<'bound>>>, + open: UnsafeCell<bool>, + /// Whether `receive_buf_callback` is allowed to call `Driver::receive`. + /// + /// If locked, the receive_buf_callback will be blocked on data reception. + /// This is the case while the driver is being probed or while [`PrivateData`] is being dropped. + /// This is necessary, because we need to open the serdev device before the driver has been + /// probed in order to allow it to be configured, which allows `receive_buf_callback` to be + /// called. Thus we need to block data until probe completes and the driver data becomes + /// initialized. + /// + /// If unlocked and true, the receive_buf_callback will forward the data to + /// `Driver::receive`. This is the normal state of operation. + /// + /// If unlocked and false, the receive_buf_callback will throw away the data. + /// This is only the case, if the serdev device is open and + /// - the driver returned an error in probe + /// or + /// - the driver data already has been dropped, because it was unbound. + #[pin] + active: Mutex<bool>, +} + +#[pinned_drop] +impl<T: Driver> PinnedDrop for PrivateData<'_, T> { + fn drop(self: Pin<&mut Self>) { + let mut active = self.active.lock(); + if *active { + // SAFETY: + // - We have exclusive access to `self.driver`. + // - `self.driver` is guaranteed to be initialized. + unsafe { (*self.driver.get()).assume_init_drop() }; + *active = false; + } + drop(active); + + // SAFETY: We have exclusive access to `self.open`. + if unsafe { *self.open.get() } { + // SAFETY: `self.sdev.as_raw()` is guaranteed to be a pointer to a valid + // `struct serdev_device`. + unsafe { bindings::serdev_device_close(self.sdev.as_raw()) }; + } + } +} + +impl<T: Driver> Adapter<T> { + const OPS: &'static bindings::serdev_device_ops = &bindings::serdev_device_ops { + receive_buf: if T::HAS_RECEIVE { + Some(Self::receive_buf_callback) + } else { + None + }, + write_wakeup: Some(bindings::serdev_device_write_wakeup), + }; + + extern "C" fn probe_callback(sdev: *mut bindings::serdev_device) -> kernel::ffi::c_int { + // SAFETY: The serial device bus only ever calls the probe callback with a valid pointer to + // a `struct serdev_device`. + // + // INVARIANT: `sdev` is valid for the duration of `probe_callback()`. + let sdev = unsafe { &*sdev.cast::<Device<device::CoreInternal<'_>>>() }; + // SAFETY: `sdev` matched data is of type `Self::IdInfo`. + let info = unsafe { <Self as driver::Adapter>::id_info(sdev.as_ref()) }; + + from_result(|| { + sdev.as_ref().set_drvdata(try_pin_init!(PrivateData::<T> { + sdev: &**sdev, + driver: MaybeUninit::<T::Data<'_>>::zeroed().into(), + open: false.into(), + active <- new_mutex!(false), + }))?; + // SAFETY: We just set drvdata to `PrivateData<'_, T>`. + let private_data = unsafe { sdev.as_ref().drvdata_borrow::<PrivateData<'_, T>>() }; + let private_data = ScopeGuard::new_with_data(private_data, |_| { + // SAFETY: We just set drvdata to `PrivateData<'_, T>`. + drop(unsafe { sdev.as_ref().drvdata_obtain::<PrivateData<'_, T>>() }); + }); + let mut active = private_data.active.lock(); + + // SAFETY: `sdev.as_raw()` is guaranteed to be a valid pointer to `serdev_device`. + unsafe { bindings::serdev_device_set_client_ops(sdev.as_raw(), Self::OPS) }; + + // SAFETY: The serial device bus only ever calls the probe callback with a valid pointer + // to a `serdev_device`. + to_result(unsafe { bindings::serdev_device_open(sdev.as_raw()) })?; + + // SAFETY: We have exclusive access to `private_data.open`. + unsafe { *private_data.open.get() = true }; + + let data = T::probe(sdev, info); + + // SAFETY: We have exclusive access to `private_data.driver`. + let driver = unsafe { &mut *private_data.driver.get() }; + // SAFETY: + // - `driver.as_mut_ptr()` is a valid pointer to uninitialized data. + // - `private_data.driver` is pinned. + let result = unsafe { pin_init::raw_try_init(driver.as_mut_ptr(), data) }; + + *active = result.is_ok(); + + drop(active); + + result.map(|()| { + private_data.dismiss(); + 0 + }) + }) + } + + extern "C" fn remove_callback(sdev: *mut bindings::serdev_device) { + // SAFETY: The serial device bus only ever calls the remove callback with a valid pointer + // to a `struct serdev_device`. + // + // INVARIANT: `sdev` is valid for the duration of `remove_callback()`. + let sdev = unsafe { &*sdev.cast::<Device<device::CoreInternal<'_>>>() }; + + // SAFETY: `remove_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin<KBox<PrivateData<'_, T>>>`. + let private_data = unsafe { sdev.as_ref().drvdata_borrow::<PrivateData<'_, T>>() }; + + // SAFETY: No one has exclusive access to `private_data.driver`. + let data = unsafe { &*private_data.driver.get() }; + // SAFETY: + // - `private_data.driver` is pinned. + // - `remove_callback` is only ever called after a successful call to `probe_callback`, + // hence it's guaranteed that `private_data.driver` was initialized. + let data_pinned = unsafe { Pin::new_unchecked(data.assume_init_ref()) }; + + T::unbind(sdev, data_pinned); + } + + extern "C" fn receive_buf_callback( + sdev: *mut bindings::serdev_device, + buf: *const u8, + length: usize, + ) -> usize { + // SAFETY: The serial device bus only ever calls the receive buf callback with a valid + // pointer to a `struct serdev_device`. + // + // INVARIANT: `sdev` is valid for the duration of `receive_buf_callback()`. + let sdev = unsafe { &*sdev.cast::<Device<device::BoundInternal>>() }; + + // SAFETY: `receive_buf_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin<KBox<PrivateData<'_, T>>>`. + let private_data = unsafe { sdev.as_ref().drvdata_borrow::<PrivateData<'_, T>>() }; + let active = private_data.active.lock(); + + if !*active { + return length; + } + + // SAFETY: No one has exclusive access to `private_data.driver`. + let data = unsafe { &*private_data.driver.get() }; + // SAFETY: + // - `private_data.driver` is pinned. + // - `receive_buf_callback` is only ever called after a successful call to `probe_callback`, + // hence it's guaranteed that `private_data.driver` was initialized. + let data_pinned = unsafe { Pin::new_unchecked(data.assume_init_ref()) }; + + // SAFETY: `buf` is guaranteed to be non-null and has the size of `length`. + let buf = unsafe { core::slice::from_raw_parts(buf, length) }; + + T::receive(sdev, data_pinned, buf) + } +} + +impl<T: Driver> driver::Adapter for Adapter<T> { + type IdInfo = T::IdInfo; + + fn of_id_table() -> Option<of::IdTable<Self::IdInfo>> { + T::OF_ID_TABLE + } + + fn acpi_id_table() -> Option<acpi::IdTable<Self::IdInfo>> { + T::ACPI_ID_TABLE + } +} + +/// Declares a kernel module that exposes a single serial device bus device driver. +/// +/// # Examples +/// +/// ```ignore +/// kernel::module_serdev_device_driver! { +/// type: MyDriver, +/// name: "Module name", +/// authors: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +/// ``` +#[macro_export] +macro_rules! module_serdev_device_driver { + ($($f:tt)*) => { + $crate::module_driver!(<T>, $crate::serdev::Adapter<T>, { $($f)* }); + }; +} + +/// The serial device bus device driver trait. +/// +/// Drivers must implement this trait in order to get a serial device bus device driver registered. +/// +/// # Examples +/// +///``` +/// # use kernel::{ +/// acpi, +/// bindings, +/// device::{ +/// Bound, +/// Core, // +/// }, +/// of, +/// serdev, // +/// }; +/// +/// struct MyDriver; +/// +/// kernel::of_device_table!( +/// OF_TABLE, +/// <MyDriver as serdev::Driver>::IdInfo, +/// [ +/// (of::DeviceId::new(c"test,device"), ()) +/// ] +/// ); +/// +/// kernel::acpi_device_table!( +/// ACPI_TABLE, +/// <MyDriver as serdev::Driver>::IdInfo, +/// [ +/// (acpi::DeviceId::new(c"LNUXBEEF"), ()) +/// ] +/// ); +/// +/// #[vtable] +/// impl serdev::Driver for MyDriver { +/// type IdInfo = (); +/// type Data<'bound> = Self; +/// const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = Some(&OF_TABLE); +/// const ACPI_ID_TABLE: Option<acpi::IdTable<Self::IdInfo>> = Some(&ACPI_TABLE); +/// +/// fn probe<'bound>( +/// sdev: &'bound serdev::Device<Core<'_>>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound { +/// sdev.set_baudrate(115200); +/// sdev.write_all(b"Hello\n", 0)?; +/// Ok(MyDriver) +/// } +/// } +///``` +#[vtable] +pub trait Driver { + /// The type holding driver private data about each device id supported by the driver. + // TODO: Use associated_type_defaults once stabilized: + // + // ``` + // type IdInfo: 'static = (); + // ``` + type IdInfo: 'static; + + /// The type of the driver's bus device private data. + type Data<'bound>: Send + Sync + 'bound; + + /// The table of OF device ids supported by the driver. + const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = None; + + /// The table of ACPI device ids supported by the driver. + const ACPI_ID_TABLE: Option<acpi::IdTable<Self::IdInfo>> = None; + + /// Serial device bus device driver probe. + /// + /// Called when a new serial device bus device is added or discovered. + /// Implementers should attempt to initialize the device here. + fn probe<'bound>( + sdev: &'bound Device<device::Core<'_>>, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound; + + /// Serial device bus device driver unbind. + /// + /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback + /// is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device<Core>` or `&Device<Bound>` reference. For instance. + /// + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(sdev: &'bound Device<device::Core<'_>>, this: Pin<&Self::Data<'bound>>) { + let _ = (sdev, this); + } + + /// Serial device bus device data receive callback. + /// + /// Called when data got received from device. + /// + /// Returns the number of bytes accepted. + fn receive<'bound>( + sdev: &'bound Device<device::Bound>, + this: Pin<&Self::Data<'bound>>, + data: &[u8], + ) -> usize { + let _ = (sdev, this, data); + build_error!(VTABLE_DEFAULT_ERROR) + } +} + +/// The serial device bus device representation. +/// +/// This structure represents the Rust abstraction for a C `struct serdev_device`. The +/// implementation abstracts the usage of an already existing C `struct serdev_device` within Rust +/// code that we get passed from the C side. +/// +/// # Invariants +/// +/// A [`Device`] instance represents a valid `struct serdev_device` created by the C portion of +/// the kernel. +#[repr(transparent)] +pub struct Device<Ctx: device::DeviceContext = device::Normal>( + Opaque<bindings::serdev_device>, + PhantomData<Ctx>, +); + +impl<Ctx: device::DeviceContext> Device<Ctx> { + #[inline] + fn as_raw(&self) -> *mut bindings::serdev_device { + self.0.get() + } +} + +impl Device<device::Bound> { + /// Set the baudrate in bits per second. + /// + /// Common baudrates are 115200, 9600, 19200, 57600, 4800. + /// + /// Use [`Device::write_flush`] before calling this if you have written data prior to this call. + #[inline] + pub fn set_baudrate(&self, speed: u32) -> Result<(), u32> { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + let ret = unsafe { bindings::serdev_device_set_baudrate(self.as_raw(), speed) }; + if ret == speed { + Ok(()) + } else { + Err(ret) + } + } + + /// Set if flow control should be enabled. + /// + /// Use [`Device::write_flush`] before calling this if you have written data prior to this call. + #[inline] + pub fn set_flow_control(&self, enable: bool) { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + unsafe { bindings::serdev_device_set_flow_control(self.as_raw(), enable) }; + } + + /// Set parity to use. + /// + /// Use [`Device::write_flush`] before calling this if you have written data prior to this call. + #[inline] + pub fn set_parity(&self, parity: Parity) -> Result { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + to_result(unsafe { bindings::serdev_device_set_parity(self.as_raw(), parity as u32) }) + } + + /// Write data to the serial device until the controller has accepted all the data or has + /// been interrupted by a timeout or signal. + /// + /// Note that any accepted data has only been buffered by the controller. Use + /// [`Device::wait_until_sent`] to make sure the controller write buffer has actually been + /// emptied. + /// + /// Use a timeout of 0 to wait indefinitely. + /// + /// Returns the number of bytes written (less than `data.len()` if interrupted). + /// [`kernel::error::code::ETIMEDOUT`] or [`kernel::error::code::ERESTARTSYS`] if interrupted + /// before any bytes were written. [`kernel::error::code::EINVAL`] if `data.len() > i32::MAX`. + #[inline] + pub fn write_all(&self, data: &[u8], timeout: Jiffies) -> Result<usize> { + if data.len() > i32::MAX as usize { + return Err(EINVAL); + } + + // SAFETY: + // - `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + // - `data.as_ptr()` is guaranteed to be a valid array pointer with the size of + // `data.len()`. + let ret = unsafe { + bindings::serdev_device_write( + self.as_raw(), + data.as_ptr(), + data.len(), + isize::try_from(timeout).unwrap_or_default(), + ) + }; + // CAST: negative return values are guaranteed to be between `-MAX_ERRNO` and `-1`, + // which always fit into a `i32`. + to_result(ret as i32).map(|()| ret.unsigned_abs()) + } + + /// Write data to the serial device. + /// + /// If you want to write until the controller has accepted all the data, use + /// [`Device::write_all`]. + /// + /// Note that any accepted data has only been buffered by the controller. Use + /// [`Device::wait_until_sent`] to make sure the controller write buffer has actually been + /// emptied. + /// + /// Returns the number of bytes written (less than `data.len()` if not enough room in the + /// write buffer). + #[inline] + pub fn write(&self, data: &[u8]) -> Result<u32> { + if data.len() > i32::MAX as usize { + return Err(EINVAL); + } + + // SAFETY: + // - `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + // - `data.as_ptr()` is guaranteed to be a valid array pointer with the size of + // `data.len()`. + let ret = + unsafe { bindings::serdev_device_write_buf(self.as_raw(), data.as_ptr(), data.len()) }; + + to_result(ret as i32).map(|()| ret.unsigned_abs()) + } + + /// Send data to the serial device immediately. + /// + /// Note that this doesn't guarantee that the data has been transmitted. + /// Use [`Device::wait_until_sent`] for this purpose. + #[inline] + pub fn write_flush(&self) { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + unsafe { bindings::serdev_device_write_flush(self.as_raw()) }; + } + + /// Wait for the data to be sent. + /// + /// After this function, the write buffer of the controller should be empty or the timeout + /// elapsed. + /// + /// Use a timeout of 0 to wait indefinitely. + #[inline] + pub fn wait_until_sent(&self, timeout: Jiffies) { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + unsafe { + bindings::serdev_device_wait_until_sent( + self.as_raw(), + isize::try_from(timeout).unwrap_or_default(), + ) + }; + } +} + +// SAFETY: `serdev::Device` is a transparent wrapper of `struct serdev_device`. +// The offset is guaranteed to point to a valid device field inside `serdev::Device`. +unsafe impl<Ctx: device::DeviceContext> device::AsBusDevice<Ctx> for Device<Ctx> { + const OFFSET: usize = offset_of!(bindings::serdev_device, dev); +} + +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); +kernel::impl_device_context_into_aref!(Device); + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl AlwaysRefCounted for Device { + fn inc_ref(&self) { + self.as_ref().inc_ref(); + } + + unsafe fn dec_ref(obj: NonNull<Self>) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::serdev_device_put(obj.cast().as_ptr()) } + } +} + +impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> { + fn as_ref(&self) -> &device::Device<Ctx> { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct serdev_device`. + let dev = unsafe { &raw mut (*self.as_raw()).dev }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +// SAFETY: A `Device` is always reference-counted and can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` can be shared among threads because all methods of `Device` +// (i.e. `Device<Normal>) are thread safe. +unsafe impl Sync for Device {} + +// SAFETY: Same as `Device<Normal>` -- the underlying `struct serdev_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device<device::Bound> {} diff --git a/rust/kernel/sizes.rs b/rust/kernel/sizes.rs index 661e680d9330..521b2b38bfe7 100644 --- a/rust/kernel/sizes.rs +++ b/rust/kernel/sizes.rs @@ -3,48 +3,132 @@ //! Commonly used sizes. //! //! C headers: [`include/linux/sizes.h`](srctree/include/linux/sizes.h). +//! +//! The top-level `SZ_*` constants are [`usize`]-typed, for use in kernel page +//! arithmetic and similar CPU-side work. +//! +//! The [`SizeConstants`] trait provides the same constants as associated constants +//! on [`u32`], [`u64`], and [`usize`], for use in device address spaces where +//! the address width depends on the hardware. Device drivers frequently need +//! these constants as [`u64`] (or [`u32`]) rather than [`usize`], because +//! device address spaces are sized independently of the CPU pointer width. +//! +//! # Examples +//! +//! ``` +//! use kernel::{ +//! page::PAGE_SIZE, +//! sizes::{ +//! SizeConstants, +//! SZ_1M, // +//! }, // +//! }; +//! +//! // Module-level constants continue to work without a type qualifier. +//! let num_pages_in_1m = SZ_1M / PAGE_SIZE; +//! +//! // Trait associated constants require a type qualifier. +//! let heap_size = 14 * u64::SZ_1M; +//! let small = u32::SZ_4K; +//! ``` + +macro_rules! define_sizes { + ($($type:ty),* $(,)?) => { + define_sizes!(@internal [$($type),*] + /// `0x0000_0400`. + SZ_1K, + /// `0x0000_0800`. + SZ_2K, + /// `0x0000_1000`. + SZ_4K, + /// `0x0000_2000`. + SZ_8K, + /// `0x0000_4000`. + SZ_16K, + /// `0x0000_8000`. + SZ_32K, + /// `0x0001_0000`. + SZ_64K, + /// `0x0002_0000`. + SZ_128K, + /// `0x0004_0000`. + SZ_256K, + /// `0x0008_0000`. + SZ_512K, + /// `0x0010_0000`. + SZ_1M, + /// `0x0020_0000`. + SZ_2M, + /// `0x0040_0000`. + SZ_4M, + /// `0x0080_0000`. + SZ_8M, + /// `0x0100_0000`. + SZ_16M, + /// `0x0200_0000`. + SZ_32M, + /// `0x0400_0000`. + SZ_64M, + /// `0x0800_0000`. + SZ_128M, + /// `0x1000_0000`. + SZ_256M, + /// `0x2000_0000`. + SZ_512M, + /// `0x4000_0000`. + SZ_1G, + /// `0x8000_0000`. + SZ_2G, + ); + }; + + (@internal [$($type:ty),*] $($names_and_metas:tt)*) => { + define_sizes!(@consts_and_trait $($names_and_metas)*); + define_sizes!(@impls [$($type),*] $($names_and_metas)*); + }; + + (@consts_and_trait $($(#[$meta:meta])* $name:ident,)*) => { + $( + $(#[$meta])* + pub const $name: usize = bindings::$name as usize; + )* + + /// Size constants for device address spaces. + /// + /// Implemented for [`u32`], [`u64`], and [`usize`] so drivers can + /// choose the width that matches their hardware. All `SZ_*` values fit + /// in a [`u32`], so all implementations are lossless. + /// + /// # Examples + /// + /// ``` + /// use kernel::sizes::SizeConstants; + /// + /// let gpu_heap = 14 * u64::SZ_1M; + /// let mmio_window = u32::SZ_16M; + /// ``` + pub trait SizeConstants { + $( + $(#[$meta])* + const $name: Self; + )* + } + }; + + (@impls [] $($(#[$meta:meta])* $name:ident,)*) => {}; + + (@impls [$first:ty $(, $rest:ty)*] $($(#[$meta:meta])* $name:ident,)*) => { + impl SizeConstants for $first { + $( + const $name: Self = { + assert!((self::$name as u128) <= (<$first>::MAX as u128)); + self::$name as $first + }; + )* + } + + define_sizes!(@impls [$($rest),*] $($(#[$meta])* $name,)*); + }; +} -/// 0x00000400 -pub const SZ_1K: usize = bindings::SZ_1K as usize; -/// 0x00000800 -pub const SZ_2K: usize = bindings::SZ_2K as usize; -/// 0x00001000 -pub const SZ_4K: usize = bindings::SZ_4K as usize; -/// 0x00002000 -pub const SZ_8K: usize = bindings::SZ_8K as usize; -/// 0x00004000 -pub const SZ_16K: usize = bindings::SZ_16K as usize; -/// 0x00008000 -pub const SZ_32K: usize = bindings::SZ_32K as usize; -/// 0x00010000 -pub const SZ_64K: usize = bindings::SZ_64K as usize; -/// 0x00020000 -pub const SZ_128K: usize = bindings::SZ_128K as usize; -/// 0x00040000 -pub const SZ_256K: usize = bindings::SZ_256K as usize; -/// 0x00080000 -pub const SZ_512K: usize = bindings::SZ_512K as usize; -/// 0x00100000 -pub const SZ_1M: usize = bindings::SZ_1M as usize; -/// 0x00200000 -pub const SZ_2M: usize = bindings::SZ_2M as usize; -/// 0x00400000 -pub const SZ_4M: usize = bindings::SZ_4M as usize; -/// 0x00800000 -pub const SZ_8M: usize = bindings::SZ_8M as usize; -/// 0x01000000 -pub const SZ_16M: usize = bindings::SZ_16M as usize; -/// 0x02000000 -pub const SZ_32M: usize = bindings::SZ_32M as usize; -/// 0x04000000 -pub const SZ_64M: usize = bindings::SZ_64M as usize; -/// 0x08000000 -pub const SZ_128M: usize = bindings::SZ_128M as usize; -/// 0x10000000 -pub const SZ_256M: usize = bindings::SZ_256M as usize; -/// 0x20000000 -pub const SZ_512M: usize = bindings::SZ_512M as usize; -/// 0x40000000 -pub const SZ_1G: usize = bindings::SZ_1G as usize; -/// 0x80000000 -pub const SZ_2G: usize = bindings::SZ_2G as usize; +define_sizes!(u32, u64, usize); diff --git a/rust/kernel/slice.rs b/rust/kernel/slice.rs deleted file mode 100644 index ca2cde135061..000000000000 --- a/rust/kernel/slice.rs +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! Additional (and temporary) slice helpers. - -/// Extension trait providing a portable version of [`as_flattened`] and -/// [`as_flattened_mut`]. -/// -/// In Rust 1.80, the previously unstable `slice::flatten` family of methods -/// have been stabilized and renamed from `flatten` to `as_flattened`. -/// -/// This creates an issue for as long as the MSRV is < 1.80, as the same functionality is provided -/// by different methods depending on the compiler version. -/// -/// This extension trait solves this by abstracting `as_flatten` and calling the correct method -/// depending on the Rust version. -/// -/// This trait can be removed once the MSRV passes 1.80. -/// -/// [`as_flattened`]: https://doc.rust-lang.org/std/primitive.slice.html#method.as_flattened -/// [`as_flattened_mut`]: https://doc.rust-lang.org/std/primitive.slice.html#method.as_flattened_mut -#[cfg(not(CONFIG_RUSTC_HAS_SLICE_AS_FLATTENED))] -pub trait AsFlattened<T> { - /// Takes a `&[[T; N]]` and flattens it to a `&[T]`. - /// - /// This is an portable layer on top of [`as_flattened`]; see its documentation for details. - /// - /// [`as_flattened`]: https://doc.rust-lang.org/std/primitive.slice.html#method.as_flattened - fn as_flattened(&self) -> &[T]; - - /// Takes a `&mut [[T; N]]` and flattens it to a `&mut [T]`. - /// - /// This is an portable layer on top of [`as_flattened_mut`]; see its documentation for details. - /// - /// [`as_flattened_mut`]: https://doc.rust-lang.org/std/primitive.slice.html#method.as_flattened_mut - fn as_flattened_mut(&mut self) -> &mut [T]; -} - -#[cfg(not(CONFIG_RUSTC_HAS_SLICE_AS_FLATTENED))] -impl<T, const N: usize> AsFlattened<T> for [[T; N]] { - #[allow(clippy::incompatible_msrv)] - fn as_flattened(&self) -> &[T] { - self.flatten() - } - - #[allow(clippy::incompatible_msrv)] - fn as_flattened_mut(&mut self) -> &mut [T] { - self.flatten_mut() - } -} diff --git a/rust/kernel/static_assert.rs b/rust/kernel/static_assert.rs deleted file mode 100644 index a57ba14315a0..000000000000 --- a/rust/kernel/static_assert.rs +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! Static assert. - -/// Static assert (i.e. compile-time assert). -/// -/// Similar to C11 [`_Static_assert`] and C++11 [`static_assert`]. -/// -/// An optional panic message can be supplied after the expression. -/// Currently only a string literal without formatting is supported -/// due to constness limitations of the [`assert!`] macro. -/// -/// The feature may be added to Rust in the future: see [RFC 2790]. -/// -/// [`_Static_assert`]: https://en.cppreference.com/w/c/language/_Static_assert -/// [`static_assert`]: https://en.cppreference.com/w/cpp/language/static_assert -/// [RFC 2790]: https://github.com/rust-lang/rfcs/issues/2790 -/// -/// # Examples -/// -/// ``` -/// static_assert!(42 > 24); -/// static_assert!(core::mem::size_of::<u8>() == 1); -/// -/// const X: &[u8] = b"bar"; -/// static_assert!(X[1] == b'a'); -/// -/// const fn f(x: i32) -> i32 { -/// x + 2 -/// } -/// static_assert!(f(40) == 42); -/// static_assert!(f(40) == 42, "f(x) must add 2 to the given input."); -/// ``` -#[macro_export] -macro_rules! static_assert { - ($condition:expr $(,$arg:literal)?) => { - const _: () = ::core::assert!($condition $(,$arg)?); - }; -} diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index 3f8918764640..b3caa9a1c898 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -3,14 +3,27 @@ //! String representations. use crate::{ - alloc::{flags::*, AllocError, KVec}, - error::{to_result, Result}, - fmt::{self, Write}, - prelude::*, + alloc::{ + AllocError, + KVec, // + }, + error::{ + to_result, + Result, // + }, + fmt::{ + self, + Write, // + }, + prelude::*, // }; use core::{ marker::PhantomData, - ops::{Deref, DerefMut, Index}, + ops::{ + Deref, + DerefMut, + Index, // + }, // }; pub use crate::prelude::CStr; @@ -189,6 +202,7 @@ macro_rules! b_str { // // - error[E0379]: functions in trait impls cannot be declared const #[inline] +#[expect(clippy::disallowed_methods, reason = "internal implementation")] pub const fn as_char_ptr_in_const_context(c_str: &CStr) -> *const c_char { c_str.as_ptr().cast() } @@ -319,6 +333,7 @@ unsafe fn to_bytes_mut(s: &mut CStr) -> &mut [u8] { impl CStrExt for CStr { #[inline] + #[expect(clippy::disallowed_methods, reason = "internal implementation")] unsafe fn from_char_ptr<'a>(ptr: *const c_char) -> &'a Self { // SAFETY: The safety preconditions are the same as for `CStr::from_ptr`. unsafe { CStr::from_ptr(ptr.cast()) } @@ -334,6 +349,7 @@ impl CStrExt for CStr { } #[inline] + #[expect(clippy::disallowed_methods, reason = "internal implementation")] fn as_char_ptr(&self) -> *const c_char { self.as_ptr().cast() } @@ -376,19 +392,32 @@ impl AsRef<BStr> for CStr { } } -/// Creates a new [`CStr`] from a string literal. +/// Creates a new [`CStr`] at compile time. /// -/// The string literal should not contain any `NUL` bytes. +/// Rust supports C string literals since Rust 1.77, and they should be used instead of this macro +/// where possible. This macro exists to allow static *non-literal* C strings to be created at +/// compile time. This is most often used in other macros. +/// +/// # Panics +/// +/// This macro panics if the operand contains an interior `NUL` byte. /// /// # Examples /// /// ``` /// # use kernel::c_str; /// # use kernel::str::CStr; -/// const MY_CSTR: &CStr = c_str!("My awesome CStr!"); +/// // This is allowed, but `c"literal"` should be preferred for literals. +/// const BAD: &CStr = c_str!("literal"); +/// +/// // `c_str!` is still needed for static non-literal C strings. +/// const GOOD: &CStr = c_str!(concat!(file!(), ":", line!(), ": My CStr!")); /// ``` #[macro_export] macro_rules! c_str { + // NB: We could write `($str:lit) => compile_error!("use a C string literal instead");` here but + // that would trigger when the literal is at the top of several macro expansions. That would be + // too limiting to macro authors. ($str:expr) => {{ const S: &str = concat!($str, "\0"); const C: &$crate::str::CStr = match $crate::str::CStr::from_bytes_with_nul(S.as_bytes()) { @@ -399,6 +428,7 @@ macro_rules! c_str { }}; } +#[cfg(CONFIG_RUST_STR_KUNIT_TEST)] #[kunit_tests(rust_kernel_str)] mod tests { use super::*; @@ -828,7 +858,10 @@ impl CString { f.write_str("\0")?; // SAFETY: The number of bytes that can be written to `f` is bounded by `size`, which is - // `buf`'s capacity. The contents of the buffer have been initialised by writes to `f`. + // `buf`'s capacity. The `Formatter` is created with `size` as its limit, and the `?` + // operators on `write_fmt` and `write_str` above ensure that if writing exceeds this + // limit, an error is returned early. The contents of the buffer have been initialised + // by writes to `f`. unsafe { buf.inc_len(f.bytes_written()) }; // Check that there are no `NUL` bytes before the end. diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 993dbf2caa0e..3e55de3b1636 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -21,16 +21,25 @@ pub mod poll; pub mod rcu; mod refcount; mod set_once; +pub mod srcu; pub use arc::{Arc, ArcBorrow, UniqueArc}; pub use completion::Completion; pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult}; pub use lock::global::{global_lock, GlobalGuard, GlobalLock, GlobalLockBackend, GlobalLockedBy}; pub use lock::mutex::{new_mutex, Mutex, MutexGuard}; -pub use lock::spinlock::{new_spinlock, SpinLock, SpinLockGuard}; +pub use lock::spinlock::{ + new_spinlock, + new_spinlock_irq, + SpinLock, + SpinLockGuard, + SpinLockIrq, + SpinLockIrqGuard, // +}; pub use locked_by::LockedBy; pub use refcount::Refcount; pub use set_once::SetOnce; +pub use srcu::Srcu; /// Represents a lockdep class. /// diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 921e19333b89..8ae0fe6f19ec 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -128,7 +128,7 @@ mod std_vendor; /// # Ok::<(), Error>(()) /// ``` #[repr(transparent)] -#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))] +#[derive(core::marker::CoercePointee)] pub struct Arc<T: ?Sized> { ptr: NonNull<ArcInner<T>>, // NB: this informs dropck that objects of type `ArcInner<T>` may be used in `<Arc<T> as @@ -154,7 +154,7 @@ impl<T: ?Sized> ArcInner<T> { /// /// # Safety /// - /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must + /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the [`Arc`] must /// not yet have been destroyed. unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> { let refcount_layout = Layout::new::<Refcount>(); @@ -182,15 +182,6 @@ impl<T: ?Sized> ArcInner<T> { } } -// This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the -// dynamically-sized type (DST) `U`. -#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))] -impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {} - -// This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`. -#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))] -impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {} - // SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because // it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs // `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` using a @@ -262,7 +253,7 @@ impl<T: ?Sized> Arc<T> { /// Convert the [`Arc`] into a raw pointer. /// - /// The raw pointer has ownership of the refcount that this Arc object owned. + /// The raw pointer has ownership of the refcount that this [`Arc`] object owned. pub fn into_raw(self) -> *const T { let ptr = self.ptr.as_ptr(); core::mem::forget(self); @@ -270,7 +261,7 @@ impl<T: ?Sized> Arc<T> { unsafe { core::ptr::addr_of!((*ptr).data) } } - /// Return a raw pointer to the data in this arc. + /// Return a raw pointer to the data in this [`Arc`]. pub fn as_ptr(this: &Self) -> *const T { let ptr = this.ptr.as_ptr(); @@ -314,7 +305,7 @@ impl<T: ?Sized> Arc<T> { /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique. /// - /// When this destroys the `Arc`, it does so while properly avoiding races. This means that + /// When this destroys the [`Arc`], it does so while properly avoiding races. This means that /// this method will never call the destructor of the value. /// /// # Examples @@ -354,11 +345,11 @@ impl<T: ?Sized> Arc<T> { // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will // return without further touching the `Arc`. If the refcount reaches zero, then there are - // no other arcs, and we can create a `UniqueArc`. + // no other `Arc`s, and we can create a `UniqueArc`. if refcount.dec_and_test() { refcount.set(1); - // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We + // INVARIANT: We own the only refcount to this `Arc`, so we may create a `UniqueArc`. We // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin // their values. Some(Pin::from(UniqueArc { @@ -547,20 +538,12 @@ impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> { /// # Ok::<(), Error>(()) /// ``` #[repr(transparent)] -#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))] +#[derive(core::marker::CoercePointee)] pub struct ArcBorrow<'a, T: ?Sized + 'a> { inner: NonNull<ArcInner<T>>, _p: PhantomData<&'a ()>, } -// This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into -// `ArcBorrow<U>`. -#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))] -impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>> - for ArcBorrow<'_, T> -{ -} - impl<T: ?Sized> Clone for ArcBorrow<'_, T> { fn clone(&self) -> Self { *self @@ -729,20 +712,22 @@ impl<T> InPlaceInit<T> for UniqueArc<T> { impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> { type Initialized = UniqueArc<T>; + #[inline] fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid. - unsafe { init.__init(slot)? }; + unsafe { pin_init::raw_try_init(slot, init)? }; // SAFETY: All fields have been initialized. Ok(unsafe { self.assume_init() }) } + #[inline] fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; + unsafe { pin_init::raw_try_init(slot, init)? }; // SAFETY: All fields have been initialized. Ok(unsafe { self.assume_init() }.into()) } @@ -775,6 +760,14 @@ impl<T> UniqueArc<T> { } } +impl<T: ?Sized> UniqueArc<T> { + /// Return a raw pointer to the data in this [`UniqueArc`]. + #[inline] + pub fn as_ptr(this: &Self) -> *const T { + Arc::as_ptr(&this.inner) + } +} + impl<T> UniqueArc<MaybeUninit<T>> { /// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it. pub fn write(mut self, value: T) -> UniqueArc<T> { @@ -799,9 +792,10 @@ impl<T> UniqueArc<MaybeUninit<T>> { } /// Initialize `self` using the given initializer. + #[inline] pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> { // SAFETY: The supplied pointer is valid for initialization. - match unsafe { init.__init(self.as_mut_ptr()) } { + match unsafe { pin_init::raw_try_init(self.as_mut_ptr(), init) } { // SAFETY: Initialization completed successfully. Ok(()) => Ok(unsafe { self.assume_init() }), Err(err) => Err(err), @@ -809,13 +803,14 @@ impl<T> UniqueArc<MaybeUninit<T>> { } /// Pin-initialize `self` using the given pin-initializer. + #[inline] pub fn pin_init_with<E>( mut self, init: impl PinInit<T, E>, ) -> core::result::Result<Pin<UniqueArc<T>>, E> { // SAFETY: The supplied pointer is valid for initialization and we will later pin the value // to ensure it does not move. - match unsafe { init.__pinned_init(self.as_mut_ptr()) } { + match unsafe { pin_init::raw_try_init(self.as_mut_ptr(), init) } { // SAFETY: Initialization completed successfully. Ok(()) => Ok(unsafe { self.assume_init() }.into()), Err(err) => Err(err), diff --git a/rust/kernel/sync/aref.rs b/rust/kernel/sync/aref.rs index 0616c0353c2b..9983ee085248 100644 --- a/rust/kernel/sync/aref.rs +++ b/rust/kernel/sync/aref.rs @@ -17,7 +17,17 @@ //! [`Arc`]: crate::sync::Arc //! [`Arc<T>`]: crate::sync::Arc -use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, + ptr::NonNull, // +}; + +use crate::{ + prelude::*, + types::ForeignOwnable, // +}; /// Types that are _always_ reference counted. /// @@ -170,3 +180,70 @@ impl<T: AlwaysRefCounted> Drop for ARef<T> { unsafe { T::dec_ref(self.ptr) }; } } + +impl<T, U> PartialEq<ARef<U>> for ARef<T> +where + T: AlwaysRefCounted + PartialEq<U>, + U: AlwaysRefCounted, +{ + #[inline] + fn eq(&self, other: &ARef<U>) -> bool { + T::eq(&**self, &**other) + } +} +impl<T: AlwaysRefCounted + Eq> Eq for ARef<T> {} + +// SAFETY: `into_foreign` returns a pointer from `NonNull::as_ptr`, so it's non-null. The +// `ARef` invariant guarantees that `ptr` points to a valid `T`, so it's aligned to `T`. +unsafe impl<T: AlwaysRefCounted> ForeignOwnable for ARef<T> { + const FOREIGN_ALIGN: usize = core::mem::align_of::<T>(); + + type Borrowed<'a> + = &'a T + where + Self: 'a; + type BorrowedMut<'a> + = &'a T + where + Self: 'a; + + #[inline] + fn into_foreign(self) -> *mut c_void { + ARef::into_raw(self).as_ptr().cast() + } + + #[inline] + unsafe fn from_foreign(ptr: *mut c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + let ptr = unsafe { NonNull::new_unchecked(ptr.cast()) }; + + // SAFETY: `ptr` came from `into_foreign`, which consumed an `ARef` without decrementing + // the refcount, so we can transfer the ownership to the new `ARef`. + unsafe { ARef::from_raw(ptr) } + } + + #[inline] + unsafe fn borrow<'a>(ptr: *mut c_void) -> &'a T { + // SAFETY: The safety requirements of this method ensure that the object remains alive and + // immutable for the duration of 'a. + unsafe { &*ptr.cast() } + } + + #[inline] + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> &'a T { + // SAFETY: The safety requirements for `borrow_mut` are a superset of the safety + // requirements for `borrow`. + unsafe { <Self as ForeignOwnable>::borrow(ptr) } + } +} + +impl<T, U> PartialEq<&'_ U> for ARef<T> +where + T: AlwaysRefCounted + PartialEq<U>, +{ + #[inline] + fn eq(&self, other: &&U) -> bool { + T::eq(&**self, other) + } +} diff --git a/rust/kernel/sync/atomic.rs b/rust/kernel/sync/atomic.rs index 4aebeacb961a..9cd009d57e35 100644 --- a/rust/kernel/sync/atomic.rs +++ b/rust/kernel/sync/atomic.rs @@ -51,6 +51,10 @@ use ordering::OrderingType; #[repr(transparent)] pub struct Atomic<T: AtomicType>(AtomicRepr<T::Repr>); +// SAFETY: `Atomic<T>` is safe to transfer between execution contexts because of the safety +// requirement of `AtomicType`. +unsafe impl<T: AtomicType> Send for Atomic<T> {} + // SAFETY: `Atomic<T>` is safe to share among execution contexts because all accesses are atomic. unsafe impl<T: AtomicType> Sync for Atomic<T> {} @@ -68,6 +72,11 @@ unsafe impl<T: AtomicType> Sync for Atomic<T> {} /// /// - [`Self`] must have the same size and alignment as [`Self::Repr`]. /// - [`Self`] must be [round-trip transmutable] to [`Self::Repr`]. +/// - [`Self`] must be safe to transfer between execution contexts, if it's [`Send`], this is +/// automatically satisfied. The exception is pointer types that are even though marked as +/// `!Send` (e.g. raw pointers and [`NonNull<T>`]) but requiring `unsafe` to do anything +/// meaningful on them. This is because transferring pointer values between execution contexts is +/// safe as long as the actual `unsafe` dereferencing is justified. /// /// Note that this is more relaxed than requiring the bi-directional transmutability (i.e. /// [`transmute()`] is always sound between `U` and `T`) because of the support for atomic @@ -108,7 +117,8 @@ unsafe impl<T: AtomicType> Sync for Atomic<T> {} /// [`transmute()`]: core::mem::transmute /// [round-trip transmutable]: AtomicType#round-trip-transmutability /// [Examples]: AtomicType#examples -pub unsafe trait AtomicType: Sized + Send + Copy { +/// [`NonNull<T>`]: core::ptr::NonNull +pub unsafe trait AtomicType: Sized + Copy { /// The backing atomic implementation type. type Repr: AtomicImpl; } @@ -204,10 +214,7 @@ impl<T: AtomicType> Atomic<T> { /// // no data race. /// unsafe { Atomic::from_ptr(foo_a_ptr) }.store(2, Release); /// ``` - pub unsafe fn from_ptr<'a>(ptr: *mut T) -> &'a Self - where - T: Sync, - { + pub unsafe fn from_ptr<'a>(ptr: *mut T) -> &'a Self { // CAST: `T` and `Atomic<T>` have the same size, alignment and bit validity. // SAFETY: Per function safety requirement, `ptr` is a valid pointer and the object will // live long enough. It's safe to return a `&Atomic<T>` because function safety requirement @@ -235,6 +242,17 @@ impl<T: AtomicType> Atomic<T> { /// Returns a mutable reference to the underlying atomic `T`. /// /// This is safe because the mutable reference of the atomic `T` guarantees exclusive access. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Relaxed}; + /// + /// let mut atomic_val = Atomic::new(0u32); + /// let val_mut = atomic_val.get_mut(); + /// *val_mut = 101; + /// assert_eq!(101, atomic_val.load(Relaxed)); + /// ``` pub fn get_mut(&mut self) -> &mut T { // CAST: `T` and `T::Repr` has the same size and alignment per the safety requirement of // `AtomicType`, and per the type invariants `self.0` is a valid `T`, therefore the casting @@ -527,16 +545,14 @@ where /// use kernel::sync::atomic::{Atomic, Acquire, Full, Relaxed}; /// /// let x = Atomic::new(42); - /// /// assert_eq!(42, x.load(Relaxed)); - /// - /// assert_eq!(54, { x.fetch_add(12, Acquire); x.load(Relaxed) }); + /// assert_eq!(42, x.fetch_add(12, Acquire)); + /// assert_eq!(54, x.load(Relaxed)); /// /// let x = Atomic::new(42); - /// /// assert_eq!(42, x.load(Relaxed)); - /// - /// assert_eq!(54, { x.fetch_add(12, Full); x.load(Relaxed) } ); + /// assert_eq!(42, x.fetch_add(12, Full)); + /// assert_eq!(54, x.load(Relaxed)); /// ``` #[inline(always)] pub fn fetch_add<Rhs, Ordering: ordering::Ordering>(&self, v: Rhs, _: Ordering) -> T @@ -559,4 +575,276 @@ where // SAFETY: `ret` comes from reading `self.0`, which is a valid `T` per type invariants. unsafe { from_repr(ret) } } + + /// Atomic fetch and subtract. + /// + /// Atomically updates `*self` to `(*self).wrapping_sub(v)`, and returns the value of `*self` + /// before the update. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{Atomic, Acquire, Full, Relaxed}; + /// + /// let x = Atomic::new(42); + /// assert_eq!(42, x.load(Relaxed)); + /// assert_eq!(42, x.fetch_sub(12, Acquire)); + /// assert_eq!(30, x.load(Relaxed)); + /// + /// let x = Atomic::new(42); + /// assert_eq!(42, x.load(Relaxed)); + /// assert_eq!(42, x.fetch_sub(12, Full)); + /// assert_eq!(30, x.load(Relaxed)); + /// ``` + #[inline(always)] + pub fn fetch_sub<Rhs, Ordering: ordering::Ordering>(&self, v: Rhs, _: Ordering) -> T + where + // Types that support addition also support subtraction. + T: AtomicAdd<Rhs>, + { + let v = T::rhs_into_delta(v); + + // INVARIANT: `self.0` is a valid `T` after `atomic_fetch_sub*()` due to safety requirement + // of `AtomicAdd`. + let ret = { + match Ordering::TYPE { + OrderingType::Full => T::Repr::atomic_fetch_sub(&self.0, v), + OrderingType::Acquire => T::Repr::atomic_fetch_sub_acquire(&self.0, v), + OrderingType::Release => T::Repr::atomic_fetch_sub_release(&self.0, v), + OrderingType::Relaxed => T::Repr::atomic_fetch_sub_relaxed(&self.0, v), + } + }; + + // SAFETY: `ret` comes from reading `self.0`, which is a valid `T` per type invariants. + unsafe { from_repr(ret) } + } +} + +#[cfg(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64))] +#[repr(C)] +#[derive(Clone, Copy)] +struct Flag { + bool_field: bool, +} + +/// # Invariants +/// +/// `padding` must be all zeroes. +#[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] +#[repr(C, align(4))] +#[derive(Clone, Copy)] +struct Flag { + #[cfg(target_endian = "big")] + padding: [u8; 3], + bool_field: bool, + #[cfg(target_endian = "little")] + padding: [u8; 3], +} + +impl Flag { + #[inline(always)] + const fn new(b: bool) -> Self { + // INVARIANT: `padding` is all zeroes. + Self { + bool_field: b, + #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] + padding: [0; 3], + } + } +} + +// SAFETY: `Flag` and `Repr` have the same size and alignment, and `Flag` is round-trip +// transmutable to the selected representation (`i8` or `i32`). +unsafe impl AtomicType for Flag { + #[cfg(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64))] + type Repr = i8; + #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] + type Repr = i32; +} + +/// An atomic flag type intended to be backed by performance-optimal integer type. +/// +/// The backing integer type is an implementation detail; it may vary by architecture and change +/// in the future. +/// +/// [`AtomicFlag`] is generally preferable to [`Atomic<bool>`] when you need read-modify-write +/// (RMW) operations (e.g. [`Atomic::xchg()`]/[`Atomic::cmpxchg()`]) or when [`Atomic<bool>`] does +/// not save memory due to padding. On some architectures that do not support byte-sized atomic +/// RMW operations, RMW operations on [`Atomic<bool>`] are slower. +/// +/// If you only use [`Atomic::load()`]/[`Atomic::store()`], [`Atomic<bool>`] is fine. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::atomic::{AtomicFlag, Relaxed}; +/// +/// let flag = AtomicFlag::new(false); +/// assert_eq!(false, flag.load(Relaxed)); +/// flag.store(true, Relaxed); +/// assert_eq!(true, flag.load(Relaxed)); +/// ``` +pub struct AtomicFlag(Atomic<Flag>); + +impl AtomicFlag { + /// Creates a new atomic flag. + #[inline(always)] + pub const fn new(b: bool) -> Self { + Self(Atomic::new(Flag::new(b))) + } + + /// Returns a mutable reference to the underlying flag as a [`bool`]. + /// + /// This is safe because the mutable reference of the atomic flag guarantees exclusive access. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::atomic::{AtomicFlag, Relaxed}; + /// + /// let mut atomic_flag = AtomicFlag::new(false); + /// assert_eq!(false, atomic_flag.load(Relaxed)); + /// *atomic_flag.get_mut() = true; + /// assert_eq!(true, atomic_flag.load(Relaxed)); + /// ``` + #[inline(always)] + pub fn get_mut(&mut self) -> &mut bool { + &mut self.0.get_mut().bool_field + } + + /// Loads the value from the atomic flag. + #[inline(always)] + pub fn load<Ordering: ordering::AcquireOrRelaxed>(&self, o: Ordering) -> bool { + self.0.load(o).bool_field + } + + /// Stores a value to the atomic flag. + #[inline(always)] + pub fn store<Ordering: ordering::ReleaseOrRelaxed>(&self, v: bool, o: Ordering) { + self.0.store(Flag::new(v), o); + } + + /// Stores a value to the atomic flag and returns the previous value. + #[inline(always)] + pub fn xchg<Ordering: ordering::Ordering>(&self, new: bool, o: Ordering) -> bool { + self.0.xchg(Flag::new(new), o).bool_field + } + + /// Store a value to the atomic flag if the current value is equal to `old`. + #[inline(always)] + pub fn cmpxchg<Ordering: ordering::Ordering>( + &self, + old: bool, + new: bool, + o: Ordering, + ) -> Result<bool, bool> { + match self.0.cmpxchg(Flag::new(old), Flag::new(new), o) { + Ok(_) => Ok(old), + Err(f) => Err(f.bool_field), + } + } +} + +/// Atomic load over raw pointers. +/// +/// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work +/// with C side on synchronizations: +/// +/// - `atomic_load(.., Relaxed)` maps to `READ_ONCE()` when used for inter-thread communication. +/// - `atomic_load(.., Acquire)` maps to `smp_load_acquire()`. +/// +/// # Safety +/// +/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`. +/// - If there is a concurrent store from kernel (C or Rust), it has to be atomic. +#[doc(alias("READ_ONCE", "smp_load_acquire"))] +#[inline(always)] +pub unsafe fn atomic_load<T: AtomicType, Ordering: ordering::AcquireOrRelaxed>( + ptr: *mut T, + o: Ordering, +) -> T +where + T::Repr: AtomicBasicOps, +{ + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to + // `align_of::<T>()`, and all concurrent stores from kernel are atomic, hence no data race per + // LKMM. + unsafe { Atomic::from_ptr(ptr) }.load(o) +} + +/// Atomic store over raw pointers. +/// +/// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work +/// with C side on synchronizations: +/// +/// - `atomic_store(.., Relaxed)` maps to `WRITE_ONCE()` when used for inter-thread communication. +/// - `atomic_load(.., Release)` maps to `smp_store_release()`. +/// +/// # Safety +/// +/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`. +/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic. +#[doc(alias("WRITE_ONCE", "smp_store_release"))] +#[inline(always)] +pub unsafe fn atomic_store<T: AtomicType, Ordering: ordering::ReleaseOrRelaxed>( + ptr: *mut T, + v: T, + o: Ordering, +) where + T::Repr: AtomicBasicOps, +{ + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to + // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race + // per LKMM. + unsafe { Atomic::from_ptr(ptr) }.store(v, o); +} + +/// Atomic exchange over raw pointers. +/// +/// This function provides a short-cut of `Atomic::from_ptr().xchg(..)`, and can be used to work +/// with C side on synchronizations. +/// +/// # Safety +/// +/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`. +/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic. +#[inline(always)] +pub unsafe fn xchg<T: AtomicType, Ordering: ordering::Ordering>( + ptr: *mut T, + new: T, + o: Ordering, +) -> T +where + T::Repr: AtomicExchangeOps, +{ + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to + // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race + // per LKMM. + unsafe { Atomic::from_ptr(ptr) }.xchg(new, o) +} + +/// Atomic compare and exchange over raw pointers. +/// +/// This function provides a short-cut of `Atomic::from_ptr().cmpxchg(..)`, and can be used to work +/// with C side on synchronizations. +/// +/// # Safety +/// +/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`. +/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic. +#[doc(alias("try_cmpxchg"))] +#[inline(always)] +pub unsafe fn cmpxchg<T: AtomicType, Ordering: ordering::Ordering>( + ptr: *mut T, + old: T, + new: T, + o: Ordering, +) -> Result<T, T> +where + T::Repr: AtomicExchangeOps, +{ + // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to + // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race + // per LKMM. + unsafe { Atomic::from_ptr(ptr) }.cmpxchg(old, new, o) } diff --git a/rust/kernel/sync/atomic/internal.rs b/rust/kernel/sync/atomic/internal.rs index 0dac58bca2b3..9c8a7a203abd 100644 --- a/rust/kernel/sync/atomic/internal.rs +++ b/rust/kernel/sync/atomic/internal.rs @@ -4,9 +4,13 @@ //! //! Provides 1:1 mapping to the C atomic operations. -use crate::bindings; -use crate::macros::paste; +use crate::{ + bindings, + build_assert::static_assert, + macros::paste, // +}; use core::cell::UnsafeCell; +use ffi::c_void; mod private { /// Sealed trait marker to disable customized impls on atomic implementation traits. @@ -14,10 +18,11 @@ mod private { } // The C side supports atomic primitives only for `i32` and `i64` (`atomic_t` and `atomic64_t`), -// while the Rust side also layers provides atomic support for `i8` and `i16` -// on top of lower-level C primitives. +// while the Rust side also provides atomic support for `i8`, `i16` and `*const c_void` on top of +// lower-level C primitives. impl private::Sealed for i8 {} impl private::Sealed for i16 {} +impl private::Sealed for *const c_void {} impl private::Sealed for i32 {} impl private::Sealed for i64 {} @@ -26,10 +31,10 @@ impl private::Sealed for i64 {} /// This trait is sealed, and only types that map directly to the C side atomics /// or can be implemented with lower-level C primitives are allowed to implement this: /// -/// - `i8` and `i16` are implemented with lower-level C primitives. +/// - `i8`, `i16` and `*const c_void` are implemented with lower-level C primitives. /// - `i32` map to `atomic_t` /// - `i64` map to `atomic64_t` -pub trait AtomicImpl: Sized + Send + Copy + private::Sealed { +pub trait AtomicImpl: Sized + Copy + private::Sealed { /// The type of the delta in arithmetic or logical operations. /// /// For example, in `atomic_add(ptr, v)`, it's the type of `v`. Usually it's the same type of @@ -37,20 +42,31 @@ pub trait AtomicImpl: Sized + Send + Copy + private::Sealed { type Delta; } -// The current helpers of load/store uses `{WRITE,READ}_ONCE()` hence the atomicity is only -// guaranteed against read-modify-write operations if the architecture supports native atomic RmW. -#[cfg(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW)] +// The current helpers of load/store of atomic `i8`, `i16` and pointers use `{WRITE,READ}_ONCE()` +// hence the atomicity is only guaranteed against read-modify-write operations if the architecture +// supports native atomic RmW. +// +// In the future when a CONFIG_ARCH_SUPPORTS_ATOMIC_RMW=n architecture plans to support Rust, the +// load/store helpers that guarantee atomicity against RmW operations (usually via a lock) need to +// be added. +static_assert!( + cfg!(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW), + "The current implementation of atomic i8/i16/ptr relies on the architecure being \ + ARCH_SUPPORTS_ATOMIC_RMW" +); + impl AtomicImpl for i8 { type Delta = Self; } -// The current helpers of load/store uses `{WRITE,READ}_ONCE()` hence the atomicity is only -// guaranteed against read-modify-write operations if the architecture supports native atomic RmW. -#[cfg(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW)] impl AtomicImpl for i16 { type Delta = Self; } +impl AtomicImpl for *const c_void { + type Delta = isize; +} + // `atomic_t` implements atomic operations on `i32`. impl AtomicImpl for i32 { type Delta = Self; @@ -262,7 +278,7 @@ macro_rules! declare_and_impl_atomic_methods { } declare_and_impl_atomic_methods!( - [ i8 => atomic_i8, i16 => atomic_i16, i32 => atomic, i64 => atomic64 ] + [ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ] /// Basic atomic operations pub trait AtomicBasicOps { /// Atomic read (load). @@ -280,7 +296,7 @@ declare_and_impl_atomic_methods!( ); declare_and_impl_atomic_methods!( - [ i8 => atomic_i8, i16 => atomic_i16, i32 => atomic, i64 => atomic64 ] + [ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ] /// Exchange and compare-and-exchange atomic operations pub trait AtomicExchangeOps { /// Atomic exchange. @@ -324,7 +340,12 @@ declare_and_impl_atomic_methods!( /// Atomically updates `*a` to `(*a).wrapping_add(v)`, and returns the value of `*a` /// before the update. fn fetch_add[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self::Delta) -> Self { - // SAFETY: `a.as_ptr()` is valid and properly aligned. + // SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned. + unsafe { bindings::#call(v, a.as_ptr().cast()) } + } + + fn fetch_sub[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self::Delta) -> Self { + // SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned. unsafe { bindings::#call(v, a.as_ptr().cast()) } } } diff --git a/rust/kernel/sync/atomic/ordering.rs b/rust/kernel/sync/atomic/ordering.rs index 3f103aa8db99..c4e732e7212f 100644 --- a/rust/kernel/sync/atomic/ordering.rs +++ b/rust/kernel/sync/atomic/ordering.rs @@ -15,7 +15,7 @@ //! - It provides ordering between the annotated operation and all the following memory accesses. //! - It provides ordering between all the preceding memory accesses and all the following memory //! accesses. -//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb()`). +//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb(Full)`). //! - [`Relaxed`] provides no ordering except the dependency orderings. Dependency orderings are //! described in "DEPENDENCY RELATIONS" in [`LKMM`]'s [`explanation`]. //! diff --git a/rust/kernel/sync/atomic/predefine.rs b/rust/kernel/sync/atomic/predefine.rs index 67a0406d3ea4..3d63f40791fa 100644 --- a/rust/kernel/sync/atomic/predefine.rs +++ b/rust/kernel/sync/atomic/predefine.rs @@ -2,8 +2,7 @@ //! Pre-defined atomic types -use crate::static_assert; -use core::mem::{align_of, size_of}; +use crate::prelude::*; // Ensure size and alignment requirements are checked. static_assert!(size_of::<bool>() == size_of::<i8>()); @@ -28,6 +27,26 @@ unsafe impl super::AtomicType for i16 { type Repr = i16; } +// SAFETY: +// +// - `*mut T` has the same size and alignment with `*const c_void`, and is round-trip +// transmutable to `*const c_void`. +// - `*mut T` is safe to transfer between execution contexts. See the safety requirement of +// [`AtomicType`]. +unsafe impl<T: Sized> super::AtomicType for *mut T { + type Repr = *const c_void; +} + +// SAFETY: +// +// - `*const T` has the same size and alignment with `*const c_void`, and is round-trip +// transmutable to `*const c_void`. +// - `*const T` is safe to transfer between execution contexts. See the safety requirement of +// [`AtomicType`]. +unsafe impl<T: Sized> super::AtomicType for *const T { + type Repr = *const c_void; +} + // SAFETY: `i32` has the same size and alignment with itself, and is round-trip transmutable to // itself. unsafe impl super::AtomicType for i32 { @@ -133,9 +152,8 @@ unsafe impl super::AtomicAdd<usize> for usize { } } -use crate::macros::kunit_tests; - -#[kunit_tests(rust_atomics)] +#[cfg(CONFIG_RUST_ATOMICS_KUNIT_TEST)] +#[macros::kunit_tests(rust_atomics)] mod tests { use super::super::*; @@ -157,6 +175,14 @@ mod tests { assert_eq!(v, x.load(Relaxed)); }); + + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + let ptr = x.as_ptr(); + + // SAFETY: `ptr` is a valid pointer and no concurrent access. + assert_eq!(v, unsafe { atomic_load(ptr, Relaxed) }); + }); } #[test] @@ -167,6 +193,17 @@ mod tests { x.store(v, Release); assert_eq!(v, x.load(Acquire)); }); + + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(0); + let ptr = x.as_ptr(); + + // SAFETY: `ptr` is a valid pointer and no concurrent access. + unsafe { atomic_store(ptr, v, Release) }; + + // SAFETY: `ptr` is a valid pointer and no concurrent access. + assert_eq!(v, unsafe { atomic_load(ptr, Acquire) }); + }); } #[test] @@ -180,6 +217,18 @@ mod tests { assert_eq!(old, x.xchg(new, Full)); assert_eq!(new, x.load(Relaxed)); }); + + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + let ptr = x.as_ptr(); + + let old = v; + let new = v + 1; + + // SAFETY: `ptr` is a valid pointer and no concurrent access. + assert_eq!(old, unsafe { xchg(ptr, new, Full) }); + assert_eq!(new, x.load(Relaxed)); + }); } #[test] @@ -195,6 +244,21 @@ mod tests { assert_eq!(Ok(old), x.cmpxchg(old, new, Relaxed)); assert_eq!(new, x.load(Relaxed)); }); + + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(v); + let ptr = x.as_ptr(); + + let old = v; + let new = v + 1; + + // SAFETY: `ptr` is a valid pointer and no concurrent access. + assert_eq!(Err(old), unsafe { cmpxchg(ptr, new, new, Full) }); + assert_eq!(old, x.load(Relaxed)); + // SAFETY: `ptr` is a valid pointer and no concurrent access. + assert_eq!(Ok(old), unsafe { cmpxchg(ptr, old, new, Relaxed) }); + assert_eq!(new, x.load(Relaxed)); + }); } #[test] @@ -226,4 +290,46 @@ mod tests { assert_eq!(false, x.load(Relaxed)); assert_eq!(Ok(false), x.cmpxchg(false, true, Full)); } + + #[test] + fn atomic_ptr_tests() { + let mut v = 42; + let mut u = 43; + let x = Atomic::new(&raw mut v); + + assert_eq!(x.load(Acquire), &raw mut v); + assert_eq!(x.cmpxchg(&raw mut u, &raw mut u, Relaxed), Err(&raw mut v)); + assert_eq!(x.cmpxchg(&raw mut v, &raw mut u, Relaxed), Ok(&raw mut v)); + assert_eq!(x.load(Relaxed), &raw mut u); + + let x = Atomic::new(&raw const v); + + assert_eq!(x.load(Acquire), &raw const v); + assert_eq!( + x.cmpxchg(&raw const u, &raw const u, Relaxed), + Err(&raw const v) + ); + assert_eq!( + x.cmpxchg(&raw const v, &raw const u, Relaxed), + Ok(&raw const v) + ); + assert_eq!(x.load(Relaxed), &raw const u); + } + + #[test] + fn atomic_flag_tests() { + let mut flag = AtomicFlag::new(false); + + assert_eq!(false, flag.load(Relaxed)); + + *flag.get_mut() = true; + assert_eq!(true, flag.load(Relaxed)); + + assert_eq!(true, flag.xchg(false, Relaxed)); + assert_eq!(false, flag.load(Relaxed)); + + *flag.get_mut() = true; + assert_eq!(Ok(true), flag.cmpxchg(true, false, Full)); + assert_eq!(false, flag.load(Relaxed)); + } } diff --git a/rust/kernel/sync/barrier.rs b/rust/kernel/sync/barrier.rs index 8f2d435fcd94..1180695d533a 100644 --- a/rust/kernel/sync/barrier.rs +++ b/rust/kernel/sync/barrier.rs @@ -7,6 +7,38 @@ //! //! [`LKMM`]: srctree/tools/memory-model/ +#![expect(private_bounds, reason = "sealed implementation")] + +/// Memory barrier orderings. +/// +/// The semantics of these orderings follows the [`LKMM`] definitions and rules. +/// +/// - [`Read`] provides ordering between preceding load operations and succeeding load operations. +/// - [`Write`] provides ordering between preceding store operations and succeeding store +/// operations. +/// - [`Full`] provides ordering between all the preceding memory accesses and succeeding memory +/// accesses. +/// +/// [`LKMM`]: srctree/tools/memory-model/ +pub mod ordering { + pub use crate::sync::atomic::ordering::Full; + + /// The annotation type for read-read barrier ordering. + pub struct Read; + + /// The annotation type for write-write barrier ordering. + pub struct Write; +} + +pub use ordering::{ + Full, + Read, + Write, // +}; + +struct Smp; +struct Dma; + /// A compiler barrier. /// /// A barrier that prevents compiler from reordering memory accesses across the barrier. @@ -19,43 +51,82 @@ pub(crate) fn barrier() { unsafe { core::arch::asm!("") }; } -/// A full memory barrier. +trait MemoryBarrier<Flavour = ()> { + fn run(); +} + +macro_rules! define_barrier { + ($([$flavour:ident])? $ordering:ident, $binding:ident) => { + impl MemoryBarrier$(<$flavour>)? for $ordering { + #[inline] + fn run() { + // SAFETY: barrier methods are safe to call. + unsafe { bindings::$binding() }; + } + } + }; +} + +define_barrier!(Full, mb); +define_barrier!(Read, rmb); +define_barrier!(Write, wmb); +define_barrier!([Dma] Full, dma_mb); +define_barrier!([Dma] Read, dma_rmb); +define_barrier!([Dma] Write, dma_wmb); +define_barrier!([Smp] Full, smp_mb); +define_barrier!([Smp] Read, smp_rmb); +define_barrier!([Smp] Write, smp_wmb); + +/// Memory barrier. /// /// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. -#[inline(always)] -pub fn smp_mb() { - if cfg!(CONFIG_SMP) { - // SAFETY: `smp_mb()` is safe to call. - unsafe { bindings::smp_mb() }; - } else { - barrier(); - } +/// +/// The specific forms of reordering can be specified using the parameter. +/// - `mb(Read)` provides a read-read barrier. +/// - `mb(Write)` provides a write-write barrier. +/// - `mb(Full)` provides a full barrier. +/// +/// # Examples +/// +/// ``` +/// # use kernel::sync::barrier::*; +/// mb(Read); +/// mb(Write); +/// mb(Full); +/// ``` +#[inline] +#[doc(alias = "rmb")] +#[doc(alias = "wmb")] +pub fn mb<T: MemoryBarrier>(_: T) { + T::run() } -/// A write-write memory barrier. +/// Memory barrier between CPUs. /// -/// A barrier that prevents compiler and CPU from reordering memory write accesses across the -/// barrier. -#[inline(always)] -pub fn smp_wmb() { +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// Does not prevent re-ordering with respect to other bus-mastering devices. +/// +/// See [`mb`] for usage. +#[inline] +#[doc(alias = "smp_rmb")] +#[doc(alias = "smp_wmb")] +pub fn smp_mb<T: MemoryBarrier<Smp>>(_: T) { if cfg!(CONFIG_SMP) { - // SAFETY: `smp_wmb()` is safe to call. - unsafe { bindings::smp_wmb() }; + T::run() } else { - barrier(); + barrier() } } -/// A read-read memory barrier. +/// Memory barrier between local CPU and bus-mastering devices. /// -/// A barrier that prevents compiler and CPU from reordering memory read accesses across the -/// barrier. -#[inline(always)] -pub fn smp_rmb() { - if cfg!(CONFIG_SMP) { - // SAFETY: `smp_rmb()` is safe to call. - unsafe { bindings::smp_rmb() }; - } else { - barrier(); - } +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// Does not prevent re-ordering with respect to other CPUs. +/// +/// See [`mb`] for usage. +#[inline] +#[doc(alias = "dma_rmb")] +#[doc(alias = "dma_wmb")] +pub fn dma_mb<T: MemoryBarrier<Dma>>(_: T) { + T::run() } diff --git a/rust/kernel/sync/completion.rs b/rust/kernel/sync/completion.rs index c50012a940a3..35ff049ff078 100644 --- a/rust/kernel/sync/completion.rs +++ b/rust/kernel/sync/completion.rs @@ -94,6 +94,7 @@ impl Completion { /// /// This method wakes up all tasks waiting on this completion; after this operation the /// completion is permanently done, i.e. signals all current and future waiters. + #[inline] pub fn complete_all(&self) { // SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`. unsafe { bindings::complete_all(self.as_raw()) }; @@ -105,6 +106,7 @@ impl Completion { /// timeout. /// /// See also [`Completion::complete_all`]. + #[inline] pub fn wait_for_completion(&self) { // SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`. unsafe { bindings::wait_for_completion(self.as_raw()) }; diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global.rs index aecbdc34738f..ebb10521d8bd 100644 --- a/rust/kernel/sync/lock/global.rs +++ b/rust/kernel/sync/lock/global.rs @@ -85,6 +85,7 @@ impl<B: GlobalLockBackend> GlobalLock<B> { } /// Try to lock this global lock. + #[must_use = "if unused, the lock will be immediately unlocked"] #[inline] pub fn try_lock(&'static self) -> Option<GlobalGuard<B>> { Some(GlobalGuard { @@ -96,6 +97,7 @@ impl<B: GlobalLockBackend> GlobalLock<B> { /// A guard for a [`GlobalLock`]. /// /// See [`global_lock!`] for examples. +#[must_use = "the lock unlocks immediately when the guard is unused"] pub struct GlobalGuard<B: GlobalLockBackend> { inner: Guard<'static, B::Item, B::Backend>, } @@ -304,4 +306,7 @@ macro_rules! global_lock_inner { (backend SpinLock) => { $crate::sync::lock::spinlock::SpinLockBackend }; + (backend SpinLockIrq) => { + $crate::sync::lock::spinlock::SpinLockIrqBackend + }; } diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index ef76fa07ca3a..aafc80125f59 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -3,6 +3,11 @@ //! A kernel spinlock. //! //! This module allows Rust code to use the kernel's `spinlock_t`. +use super::*; +use crate::{ + interrupt::LocalInterruptDisabled, + prelude::*, // +}; /// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class. /// @@ -82,7 +87,7 @@ pub use new_spinlock; /// ``` /// /// [`spinlock_t`]: srctree/include/linux/spinlock.h -pub type SpinLock<T> = super::Lock<T, SpinLockBackend>; +pub type SpinLock<T> = Lock<T, SpinLockBackend>; /// A kernel `spinlock_t` lock backend. pub struct SpinLockBackend; @@ -91,13 +96,11 @@ pub struct SpinLockBackend; /// /// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLock`]. It will unlock /// the [`SpinLock`] upon being dropped. -/// -/// [`Guard`]: super::Guard -pub type SpinLockGuard<'a, T> = super::Guard<'a, T, SpinLockBackend>; +pub type SpinLockGuard<'a, T> = Guard<'a, T, SpinLockBackend>; // SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the // default implementation that always calls the same locking method. -unsafe impl super::Backend for SpinLockBackend { +unsafe impl Backend for SpinLockBackend { type State = bindings::spinlock_t; type GuardState = (); @@ -144,3 +147,319 @@ unsafe impl super::Backend for SpinLockBackend { unsafe { bindings::spin_assert_is_held(ptr) } } } + +/// Creates a [`SpinLockIrq`] initialiser with the given name and a newly-created lock class. +/// +/// It uses the name if one is given, otherwise it generates one based on the file name and line +/// number. +#[macro_export] +macro_rules! new_spinlock_irq { + ($inner:expr $(, $name:literal)? $(,)?) => { + $crate::sync::SpinLockIrq::new( + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_spinlock_irq; + +/// A variant of `SpinLock` that ensures interrupts are disabled in the critical section. +/// +/// This lock can be acquired in two ways: +/// +/// - Using [`lock()`] like any other type of lock, in which case the bindings will modify the +/// interrupt state to ensure that local processor interrupts remain disabled for at least as +/// long as the [`SpinLockIrqGuard`] exists. +/// - Using [`lock_with()`] in contexts where a [`LocalInterruptDisabled`] token is present and +/// local processor interrupts are already known to be disabled, in which case the local +/// interrupt state will not be touched. This method should be preferred if a +/// [`LocalInterruptDisabled`] token is present in the scope. +/// +/// For more info on spinlocks, see [`SpinLock`]. For more information on interrupts, +/// [see the interrupt module](kernel::interrupt). +/// +/// # Examples +/// +/// The following example shows how to declare, allocate initialise and access a struct (`Example`) +/// that contains an inner struct (`Inner`) that is protected by a spinlock that requires local +/// processor interrupts to be disabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// +/// struct Inner { +/// a: u32, +/// b: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// c: SpinLockIrq<Inner>, +/// #[pin] +/// d: SpinLockIrq<Inner>, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit<Self> { +/// pin_init!(Self { +/// c <- new_spinlock_irq!(Inner { a: 0, b: 10 }), +/// d <- new_spinlock_irq!(Inner { a: 20, b: 30 }), +/// }) +/// } +/// } +/// +/// // Allocate a boxed `Example` +/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// +/// // Accessing an `Example` from a context where interrupts may not be disabled already. +/// let c_guard = e.c.lock(); // interrupts are disabled now, +1 interrupt disable refcount +/// let d_guard = e.d.lock(); // no interrupt state change, +1 interrupt disable refcount +/// +/// assert_eq!(c_guard.a, 0); +/// assert_eq!(c_guard.b, 10); +/// assert_eq!(d_guard.a, 20); +/// assert_eq!(d_guard.b, 30); +/// +/// drop(c_guard); // Dropping c_guard will not re-enable interrupts just yet, since d_guard is +/// // still in scope. +/// drop(d_guard); // Last interrupt disable reference dropped here, so interrupts are re-enabled +/// // now +/// # Ok::<(), Error>(()) +/// ``` +/// +/// The next example demonstrates locking a [`SpinLockIrq`] using [`lock_with()`] in a function +/// which can only be called when local processor interrupts are already disabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// use kernel::interrupt::*; +/// +/// struct Inner { +/// a: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// inner: SpinLockIrq<Inner>, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit<Self> { +/// pin_init!(Self { +/// inner <- new_spinlock_irq!(Inner { a: 20 }), +/// }) +/// } +/// } +/// +/// // Accessing an `Example` from a function that can only be called in no-interrupt contexts. +/// fn noirq_work(e: &Example, interrupt_disabled: &LocalInterruptDisabled) { +/// // Because we know interrupts are disabled from interrupt_disable, we can skip toggling +/// // interrupt state using lock_with() and the provided token +/// assert_eq!(e.inner.lock_with(interrupt_disabled).a, 20); +/// } +/// +/// # let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// # let interrupt_guard = local_interrupt_disable(); +/// # noirq_work(&e, &interrupt_guard); +/// # +/// # Ok::<(), Error>(()) +/// ``` +/// +/// [`lock()`]: SpinLockIrq::lock +/// [`lock_with()`]: SpinLockIrq::lock_with +pub type SpinLockIrq<T> = super::Lock<T, SpinLockIrqBackend>; + +/// A kernel `spinlock_t` lock backend that can only be acquired in interrupt disabled contexts. +pub struct SpinLockIrqBackend; + +/// A [`Guard`] acquired from locking a [`SpinLockIrq`] using [`lock()`]. +/// +/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLockIrq`] using +/// [`lock()`]. It will unlock the [`SpinLockIrq`] and decrement the local processor's interrupt +/// disablement refcount upon being dropped. +/// +/// [`lock()`]: SpinLockIrq::lock +pub type SpinLockIrqGuard<'a, T> = Guard<'a, T, SpinLockIrqBackend>; + +// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the +// default implementation that always calls the same locking method. +unsafe impl Backend for SpinLockIrqBackend { + type State = bindings::spinlock_t; + type GuardState = (); + + #[inline] + unsafe fn init( + ptr: *mut Self::State, + name: *const crate::ffi::c_char, + key: *mut bindings::lock_class_key, + ) { + // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and + // `key` are valid for read indefinitely. + unsafe { bindings::__spin_lock_init(ptr, name, key) } + } + + #[inline] + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { + // SAFETY: The safety requirements of this function ensure that `ptr` points to valid + // memory, and that it has been initialised before. + unsafe { bindings::spin_lock_irq_disable(ptr) } + } + + #[inline] + unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { + // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the + // caller is the owner of the spinlock. + unsafe { bindings::spin_unlock_irq_enable(ptr) } + } + + #[inline] + unsafe fn try_lock(ptr: *mut Self::State) -> Option<Self::GuardState> { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + let result = unsafe { bindings::spin_trylock_irq_disable(ptr) }; + + if result != 0 { + Some(()) + } else { + None + } + } + + #[inline] + unsafe fn assert_is_held(ptr: *mut Self::State) { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + unsafe { bindings::spin_assert_is_held(ptr) } + } +} + +impl<T: ?Sized> Lock<T, SpinLockIrqBackend> { + /// Casts the lock as a `Lock<T, SpinLockBackend>`. + #[inline] + fn as_lock_in_interrupt<'a>(&'a self, _context: &'a LocalInterruptDisabled) -> &'a SpinLock<T> { + // SAFETY: + // - `Lock<T, SpinLockBackend>` and `Lock<T, SpinLockIrqBackend>` both have identical data + // layouts. + // - As long as local interrupts are disabled (which is proven to be true by _context), it + // is safe to treat a lock with SpinLockIrqBackend as a SpinLockBackend lock. + unsafe { core::mem::transmute(self) } + } + + /// Acquires the lock without modifying local interrupt state. + /// + /// This function should be used in place of the more expensive [`Lock::lock()`] function when + /// possible for [`SpinLockIrq`] locks. + #[inline] + pub fn lock_with<'a>(&'a self, context: &'a LocalInterruptDisabled) -> SpinLockGuard<'a, T> { + self.as_lock_in_interrupt(context).lock() + } + + /// Tries to acquire the lock without modifying local interrupt state. + /// + /// This function should be used in place of the more expensive [`Lock::try_lock()`] function + /// when possible for [`SpinLockIrq`] locks. + /// + /// Returns a guard that can be used to access the data protected by the lock if successful. + #[must_use = "if unused, the lock will be immediately unlocked"] + #[inline] + pub fn try_lock_with<'a>( + &'a self, + context: &'a LocalInterruptDisabled, + ) -> Option<SpinLockGuard<'a, T>> { + self.as_lock_in_interrupt(context).try_lock() + } +} + +#[kunit_tests(rust_spinlock_irq_condvar)] +mod tests { + use super::*; + use crate::{ + sync::*, + workqueue::{ + self, + impl_has_work, + new_work, + Work, + WorkItem, // + }, + }; + + struct TestState { + value: u32, + waiter_ready: bool, + } + + #[pin_data] + struct Test { + #[pin] + state: SpinLockIrq<TestState>, + + #[pin] + state_changed: CondVar, + + #[pin] + waiter_state_changed: CondVar, + + #[pin] + wait_work: Work<Self>, + } + + impl_has_work! { + impl HasWork<Self> for Test { self.wait_work } + } + + impl Test { + pub(crate) fn new() -> Result<Arc<Self>> { + Arc::try_pin_init( + try_pin_init!( + Self { + state <- new_spinlock_irq!(TestState { + value: 1, + waiter_ready: false + }), + state_changed <- new_condvar!(), + waiter_state_changed <- new_condvar!(), + wait_work <- new_work!("IrqCondvarTest::wait_work") + } + ), + GFP_KERNEL, + ) + } + } + + impl WorkItem for Test { + type Pointer = Arc<Self>; + + fn run(this: Arc<Self>) { + // Wait for the test to be ready to wait for us + let mut state = this.state.lock(); + + // Make sure the interrupts actually turned off + // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()` + unsafe { bindings::lockdep_assert_irqs_disabled() }; + + while !state.waiter_ready { + this.waiter_state_changed.wait(&mut state); + } + + // Deliver the exciting value update our test has been waiting for + state.value += 1; + this.state_changed.notify_sync(); + } + } + + #[test] + fn spinlock_irq_condvar() -> Result { + let testdata = Test::new()?; + + let _ = workqueue::system().enqueue(testdata.clone()); + + // Let the updater know when we're ready to wait + let mut state = testdata.state.lock(); + state.waiter_ready = true; + testdata.waiter_state_changed.notify_sync(); + + // Wait for the exciting value update + testdata.state_changed.wait(&mut state); + assert_eq!(state.value, 2); + Ok(()) + } +} diff --git a/rust/kernel/sync/locked_by.rs b/rust/kernel/sync/locked_by.rs index 61f100a45b35..fb4a1430b3b4 100644 --- a/rust/kernel/sync/locked_by.rs +++ b/rust/kernel/sync/locked_by.rs @@ -3,7 +3,7 @@ //! A wrapper for data protected by a lock that does not wrap it. use super::{lock::Backend, lock::Lock}; -use crate::build_assert; +use crate::build_assert::build_assert; use core::{cell::UnsafeCell, mem::size_of, ptr}; /// Allows access to some data to be serialised by a lock that does not wrap it. diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs index 0ec985d560c8..dc40bfaa57e6 100644 --- a/rust/kernel/sync/poll.rs +++ b/rust/kernel/sync/poll.rs @@ -5,12 +5,22 @@ //! Utilities for working with `struct poll_table`. use crate::{ + alloc::AllocError, bindings, fs::File, prelude::*, - sync::{CondVar, LockClassKey}, + sync::{ + rcu::synchronize_rcu, + CondVar, + LockClassKey, // + }, // + types::Opaque, // +}; +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, // }; -use core::{marker::PhantomData, ops::Deref}; /// Creates a [`PollCondVar`] initialiser with the given name and a newly-created lock class. #[macro_export] @@ -66,6 +76,7 @@ impl<'a> PollTable<'a> { /// /// [`CondVar`]: crate::sync::CondVar #[pin_data(PinnedDrop)] +#[repr(transparent)] pub struct PollCondVar { #[pin] inner: CondVar, @@ -99,8 +110,70 @@ impl PinnedDrop for PollCondVar { unsafe { bindings::__wake_up_pollfree(self.inner.wait_queue_head.get()) }; // Wait for epoll items to be properly removed. - // - // SAFETY: Just an FFI call. - unsafe { bindings::synchronize_rcu() }; + synchronize_rcu(); + } +} + +/// A [`KBox<PollCondVar>`] that uses `kfree_rcu`. +/// +/// [`KBox<PollCondVar>`]: PollCondVar +pub struct PollCondVarBox { + inner: ManuallyDrop<Pin<KBox<PollCondVarBoxInner>>>, +} + +#[pin_data] +#[repr(C)] +struct PollCondVarBoxInner { + #[pin] + inner: PollCondVar, + rcu: Opaque<bindings::kvfree_rcu_head>, +} + +// SAFETY: PollCondVar is Send +unsafe impl Send for PollCondVarBoxInner {} +// SAFETY: PollCondVar is Sync +unsafe impl Sync for PollCondVarBoxInner {} + +impl PollCondVarBox { + /// Constructs a new boxed [`PollCondVar`]. + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> Result<Self, AllocError> { + let b = KBox::pin_init( + pin_init!(PollCondVarBoxInner { + inner <- PollCondVar::new(name, key), + rcu: Opaque::uninit(), + }), + GFP_KERNEL, + ) + .map_err(|_| AllocError)?; + + Ok(PollCondVarBox { + inner: ManuallyDrop::new(b), + }) + } +} + +impl Deref for PollCondVarBox { + type Target = PollCondVar; + fn deref(&self) -> &PollCondVar { + &self.inner.inner + } +} + +impl Drop for PollCondVarBox { + #[inline] + fn drop(&mut self) { + // SAFETY: ManuallyDrop::take ok because not already taken. + let boxed = unsafe { ManuallyDrop::take(&mut self.inner) }; + + // SAFETY: The code below frees the box without calling the actual destructor of the type, + // but it's okay because it re-implements the destructor using `kfree_rcu()` in place of + // `synchronize_rcu()`. + let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(boxed) }); + + // SAFETY: The pointer points at a valid `wait_queue_head`. + unsafe { bindings::__wake_up_pollfree((*ptr).inner.inner.wait_queue_head.get()) }; + + // SAFETY: This was allocated using `KBox::pin_init`, so it can be freed with `kvfree`. + unsafe { bindings::kvfree_call_rcu((*ptr).rcu.get(), ptr.cast::<ffi::c_void>()) }; } } diff --git a/rust/kernel/sync/rcu.rs b/rust/kernel/sync/rcu.rs index a32bef6e490b..0daa1ac87d81 100644 --- a/rust/kernel/sync/rcu.rs +++ b/rust/kernel/sync/rcu.rs @@ -50,3 +50,39 @@ impl Drop for Guard { pub fn read_lock() -> Guard { Guard::new() } + +/// Wait until all in-flight `call_rcu()` callbacks complete. +/// +/// Note that this primitive does not necessarily wait for an RCU grace period +/// to complete. For example, if there are no RCU callbacks queued anywhere +/// in the system, then [`rcu_barrier()`] is within its rights to return +/// immediately, without waiting for anything, much less an RCU grace period. +/// In fact, [`rcu_barrier()`] will normally not result in any RCU grace periods +/// beyond those that were already destined to be executed. +/// +/// In kernels built with `CONFIG_RCU_LAZY=y`, this function also hurries all +/// pending lazy RCU callbacks. +/// +/// Note that this is one of the RCU primitives which must not be called in +/// atomic context. +#[inline] +pub fn rcu_barrier() { + // SAFETY: `rcu_barrier()` is always safe to be called. It just might wait for a grace period. + unsafe { bindings::rcu_barrier() }; +} + +/// Wait for one RCU grace period. +/// +/// Waits for all RCU read-side critical sections (such as those established by +/// a [`Guard`]) at the moment of the function call to finish. +/// +/// Does not prevent new read-side critical sections from starting, which may +/// begin and run while this call is blocking. +/// +/// Note that this is one of the RCU primitives which must not be called in +/// atomic context. +#[inline] +pub fn synchronize_rcu() { + // SAFETY: `synchronize_rcu()` is always safe to be called from process context. + unsafe { bindings::synchronize_rcu() }; +} diff --git a/rust/kernel/sync/refcount.rs b/rust/kernel/sync/refcount.rs index 6c7ae8b05a0b..23a5d201f343 100644 --- a/rust/kernel/sync/refcount.rs +++ b/rust/kernel/sync/refcount.rs @@ -4,9 +4,11 @@ //! //! C header: [`include/linux/refcount.h`](srctree/include/linux/refcount.h) -use crate::build_assert; -use crate::sync::atomic::Atomic; -use crate::types::Opaque; +use crate::{ + build_assert::build_assert, + sync::atomic::Atomic, + types::Opaque, // +}; /// Atomic reference counter. /// diff --git a/rust/kernel/sync/srcu.rs b/rust/kernel/sync/srcu.rs new file mode 100644 index 000000000000..723e5e277fd6 --- /dev/null +++ b/rust/kernel/sync/srcu.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Sleepable read-copy update (SRCU) support. +//! +//! C header: [`include/linux/srcu.h`](srctree/include/linux/srcu.h) + +use crate::{ + bindings, + error::to_result, + prelude::*, + sync::LockClassKey, + types::{ + NotThreadSafe, + Opaque, // + }, +}; + +use pin_init::pin_data; + +/// Creates an [`Srcu`] initialiser with the given name and a newly-created lock class. +#[doc(hidden)] +#[macro_export] +macro_rules! new_srcu { + ($($name:literal)?) => { + $crate::sync::Srcu::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_srcu; + +/// Sleepable read-copy update primitive. +/// +/// SRCU readers may sleep while holding the read-side guard. +/// +/// The destructor waits for active readers and callbacks, so it may sleep. +/// If a read-side guard has been leaked, dropping an [`Srcu`] may never return. +/// +/// # Invariants +/// +/// This represents a valid `struct srcu_struct` initialized by the C SRCU API +/// and it remains pinned and valid until the pinned destructor runs. +#[repr(transparent)] +#[pin_data(PinnedDrop)] +pub struct Srcu { + #[pin] + inner: Opaque<bindings::srcu_struct>, +} + +impl Srcu { + /// Creates a new SRCU instance. + #[inline] + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self, Error> { + try_pin_init!(Self { + // INVARIANT: On success, the C initializer creates a valid `srcu_struct` and + // it remains pinned until `PinnedDrop` runs. + inner <- Opaque::try_ffi_init(|ptr: *mut bindings::srcu_struct| { + // SAFETY: `ptr` points to valid uninitialised memory for a `srcu_struct`. + to_result(unsafe { + bindings::init_srcu_struct_with_key(ptr, name.as_char_ptr(), key.as_ptr()) + }) + }), + }) + } + + /// Enters an SRCU read-side critical section. + /// + /// Leaking the returned [`Guard`] leaves the SRCU read-side critical + /// section active and makes `drop` sleep forever. + #[inline] + pub fn read_lock(&self) -> Guard<'_> { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + let idx = unsafe { bindings::srcu_read_lock(self.inner.get()) }; + + // INVARIANT: `idx` was returned by `srcu_read_lock()` for this `Srcu`. + Guard { + srcu: self, + idx, + _not_send: NotThreadSafe, + } + } + + /// Waits until all pre-existing SRCU readers have completed. + #[inline] + pub fn synchronize(&self) { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + unsafe { bindings::synchronize_srcu(self.inner.get()) }; + } + + /// Waits until all pre-existing SRCU readers have completed, expedited. + /// + /// This requests a lower-latency grace period than [`Srcu::synchronize`] typically + /// at the cost of higher system-wide overhead. Prefer [`Srcu::synchronize`] by default + /// and use this variant only when reducing reset or teardown latency is more important + /// than the extra cost. + #[inline] + pub fn synchronize_expedited(&self) { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + unsafe { bindings::synchronize_srcu_expedited(self.inner.get()) }; + } +} + +#[pinned_drop] +impl PinnedDrop for Srcu { + fn drop(self: Pin<&mut Self>) { + let ptr = self.inner.get(); + + if crate::warn_on!( + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct` + // and `srcu_readers_active()` only checks the active reader count. + unsafe { bindings::srcu_readers_active(ptr) } + ) { + // `cleanup_srcu_struct()` may return early if there are still active readers. + // This should only happen if a guard was leaked with `mem::forget`, which is + // "WRONG" code and may cause a UAF because Rust will free the `srcu_struct` + // while it is still referenced from the C side (e.g. by `call_srcu()` callbacks). + // + // Another consequence of leaking guards is that `call_srcu()` callbacks will + // never run because the grace period can never complete due to permanently + // active readers (i.e. leaked guards). + // + // If this ever happens, that means the guard was leaked by mistake and the + // caller must fix the bug. Sleeping here is intentional and less harmful + // than risking a UAF. + // + // SAFETY: By the type invariants, `self` contains a valid and pinned + // `struct srcu_struct`. + unsafe { bindings::synchronize_srcu(ptr) }; + } + + // Ensure all SRCU callbacks have been finished before freeing. + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. + unsafe { bindings::srcu_barrier(ptr) }; + + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. + unsafe { bindings::cleanup_srcu_struct(ptr) }; + } +} + +// SAFETY: `srcu_struct` may be shared and used across threads. +unsafe impl Send for Srcu {} +// SAFETY: `srcu_struct` may be shared and used concurrently. +unsafe impl Sync for Srcu {} + +/// Guard for an active SRCU read-side critical section on a particular [`Srcu`]. +/// +/// Leaking this guard with [`core::mem::forget`] leaves the SRCU read-side +/// critical section active and makes dropping the associated [`Srcu`] sleep forever. +/// +/// # Invariants +/// +/// `idx` is the index returned by `srcu_read_lock()` for `srcu`. +#[must_use = "if unused, the lock will be immediately unlocked"] +pub struct Guard<'a> { + srcu: &'a Srcu, + idx: i32, + _not_send: NotThreadSafe, +} + +impl Guard<'_> { + /// Explicitly releases the SRCU read-side critical section. + #[inline] + pub fn unlock(self) {} +} + +impl Drop for Guard<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: `Guard` is only constructible through `Srcu::read_lock()`, + // which returns a valid index for the SRCU instance. + unsafe { bindings::srcu_read_unlock(self.srcu.inner.get(), self.idx) }; + } +} diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs index cc907fb531bc..3336df493dec 100644 --- a/rust/kernel/task.rs +++ b/rust/kernel/task.rs @@ -6,16 +6,15 @@ use crate::{ bindings, - ffi::{c_int, c_long, c_uint}, mm::MmWithUser, pid_namespace::PidNamespace, + prelude::*, sync::aref::ARef, types::{NotThreadSafe, Opaque}, }; use core::{ - cmp::{Eq, PartialEq}, ops::Deref, - ptr, + ptr, // }; /// A sentinel value used for infinite timeouts. @@ -211,20 +210,20 @@ impl Task { unsafe { *ptr::addr_of!((*self.as_ptr()).pid) } } - /// Returns the UID of the given task. + /// Returns the TGID (Thread Group ID / Process ID) of the given task. + pub fn tgid(&self) -> Pid { + // SAFETY: The tgid of a task never changes after initialization, so reading this field is + // not a data race. + unsafe { *ptr::addr_of!((*self.as_ptr()).tgid) } + } + + /// Returns the objective real UID of the given task. #[inline] pub fn uid(&self) -> Kuid { // SAFETY: It's always safe to call `task_uid` on a valid task. Kuid::from_raw(unsafe { bindings::task_uid(self.as_ptr()) }) } - /// Returns the effective UID of the given task. - #[inline] - pub fn euid(&self) -> Kuid { - // SAFETY: It's always safe to call `task_euid` on a valid task. - Kuid::from_raw(unsafe { bindings::task_euid(self.as_ptr()) }) - } - /// Determines whether the given task has pending signals. #[inline] pub fn signal_pending(&self) -> bool { @@ -362,8 +361,17 @@ unsafe impl crate::sync::aref::AlwaysRefCounted for Task { } } +impl PartialEq for Task { + #[inline] + fn eq(&self, other: &Self) -> bool { + ptr::eq(self.as_ptr(), other.as_ptr()) + } +} + +impl Eq for Task {} + impl Kuid { - /// Get the current euid. + /// Get the current subjective effective UID. #[inline] pub fn current_euid() -> Kuid { // SAFETY: Just an FFI call. @@ -419,7 +427,7 @@ pub fn might_sleep() { let file = kernel::file_from_location(loc); // SAFETY: `file.as_ptr()` is valid for reading and guaranteed to be nul-terminated. - unsafe { crate::bindings::__might_sleep(file.as_ptr().cast(), loc.line() as i32) } + unsafe { crate::bindings::__might_sleep(file.as_char_ptr(), loc.line() as i32) } } // SAFETY: Always safe to call. diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs index 6ea98dfcd027..6c0a5e8090d0 100644 --- a/rust/kernel/time.rs +++ b/rust/kernel/time.rs @@ -60,7 +60,13 @@ pub fn msecs_to_jiffies(msecs: Msecs) -> Jiffies { /// cases the user of the clock has to decide which clock is best suited for the /// purpose. In most scenarios clock [`Monotonic`] is the best choice as it /// provides a accurate monotonic notion of time (leap second smearing ignored). -pub trait ClockSource { +/// +/// # Safety +/// +/// Implementers must ensure that `ktime_get()` returns a value in the inclusive range +/// `0..=KTIME_MAX` (i.e., greater than or equal to 0 and less than or equal to +/// `KTIME_MAX`, where `KTIME_MAX` equals `i64::MAX`). +pub unsafe trait ClockSource { /// The kernel clock ID associated with this clock source. /// /// This constant corresponds to the C side `clockid_t` value. @@ -68,7 +74,7 @@ pub trait ClockSource { /// Get the current time from the clock source. /// - /// The function must return a value in the range from 0 to `KTIME_MAX`. + /// The function must return a value in the range `0..=KTIME_MAX`. fn ktime_get() -> bindings::ktime_t; } @@ -85,7 +91,9 @@ pub trait ClockSource { /// count time that the system is suspended. pub struct Monotonic; -impl ClockSource for Monotonic { +// SAFETY: The kernel's `ktime_get()` is guaranteed to return a value +// in `0..=KTIME_MAX`. +unsafe impl ClockSource for Monotonic { const ID: bindings::clockid_t = bindings::CLOCK_MONOTONIC as bindings::clockid_t; fn ktime_get() -> bindings::ktime_t { @@ -110,7 +118,9 @@ impl ClockSource for Monotonic { /// the clock will experience discontinuity around leap second adjustment. pub struct RealTime; -impl ClockSource for RealTime { +// SAFETY: The kernel's `ktime_get_real()` is guaranteed to return a value +// in `0..=KTIME_MAX`. +unsafe impl ClockSource for RealTime { const ID: bindings::clockid_t = bindings::CLOCK_REALTIME as bindings::clockid_t; fn ktime_get() -> bindings::ktime_t { @@ -128,7 +138,9 @@ impl ClockSource for RealTime { /// discontinuities if the time is changed using settimeofday(2) or similar. pub struct BootTime; -impl ClockSource for BootTime { +// SAFETY: The kernel's `ktime_get_boottime()` is guaranteed to return a value +// in `0..=KTIME_MAX`. +unsafe impl ClockSource for BootTime { const ID: bindings::clockid_t = bindings::CLOCK_BOOTTIME as bindings::clockid_t; fn ktime_get() -> bindings::ktime_t { @@ -150,7 +162,9 @@ impl ClockSource for BootTime { /// The acronym TAI refers to International Atomic Time. pub struct Tai; -impl ClockSource for Tai { +// SAFETY: The kernel's `ktime_get_clocktai()` is guaranteed to return a value +// in `0..=KTIME_MAX`. +unsafe impl ClockSource for Tai { const ID: bindings::clockid_t = bindings::CLOCK_TAI as bindings::clockid_t; fn ktime_get() -> bindings::ktime_t { @@ -232,7 +246,7 @@ impl<C: ClockSource> ops::Sub for Instant<C> { #[inline] fn sub(self, other: Instant<C>) -> Delta { Delta { - nanos: self.inner - other.inner, + value: self.inner - other.inner, } } } @@ -244,7 +258,7 @@ impl<T: ClockSource> ops::Add<Delta> for Instant<T> { fn add(self, rhs: Delta) -> Self::Output { // INVARIANT: With arithmetic over/underflow checks enabled, this will panic if we overflow // (e.g. go above `KTIME_MAX`) - let res = self.inner + rhs.nanos; + let res = self.inner + rhs.value; // INVARIANT: With overflow checks enabled, we verify here that the value is >= 0 #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)] @@ -264,7 +278,7 @@ impl<T: ClockSource> ops::Sub<Delta> for Instant<T> { fn sub(self, rhs: Delta) -> Self::Output { // INVARIANT: With arithmetic over/underflow checks enabled, this will panic if we overflow // (e.g. go above `KTIME_MAX`) - let res = self.inner - rhs.nanos; + let res = self.inner - rhs.value; // INVARIANT: With overflow checks enabled, we verify here that the value is >= 0 #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)] @@ -277,14 +291,64 @@ impl<T: ClockSource> ops::Sub<Delta> for Instant<T> { } } +mod private { + pub trait Sealed {} + + impl Sealed for super::Nsec {} + impl Sealed for super::Jiffy {} +} + +/// A trait for time units. +pub trait TimeUnit: private::Sealed { + /// The underlying representation of the time unit. + type Repr: Copy + Clone + PartialEq + PartialOrd + Eq + Ord + core::fmt::Debug; +} + +/// A time unit of nanoseconds. +/// +/// A [`Delta<Nsec>`] stores its value as [`i64`] nanoseconds and can represent +/// any [`i64`] value, including negative, zero, and positive numbers. +#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] +pub enum Nsec {} + +impl TimeUnit for Nsec { + type Repr = i64; +} + +/// A time unit of jiffies. +/// +/// A [`Delta<Jiffy>`] stores its value as [`isize`] jiffies and can represent +/// any [`isize`] value, including negative, zero, and positive numbers. +#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] +pub enum Jiffy {} + +impl TimeUnit for Jiffy { + type Repr = isize; +} + /// A span of time. /// -/// This struct represents a span of time, with its value stored as nanoseconds. -/// The value can represent any valid i64 value, including negative, zero, and -/// positive numbers. +/// The span is stored in the unit given by the type parameter `U` (see +/// [`TimeUnit`]); its value has type `U::Repr`. `U` defaults to [`Nsec`], so a +/// plain [`Delta`] is a span in nanoseconds. The value can be negative, zero, or +/// positive. #[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] -pub struct Delta { - nanos: i64, +pub struct Delta<U: TimeUnit = Nsec> { + value: U::Repr, +} + +impl Delta<Jiffy> { + /// Create a new [`Delta`] from a number of jiffies. + #[inline] + pub const fn from_jiffies(jiffies: isize) -> Self { + Self { value: jiffies } + } + + /// Return the number of jiffies in the [`Delta`]. + #[inline] + pub const fn as_jiffies(self) -> isize { + self.value + } } impl ops::Add for Delta { @@ -293,7 +357,7 @@ impl ops::Add for Delta { #[inline] fn add(self, rhs: Self) -> Self { Self { - nanos: self.nanos + rhs.nanos, + value: self.value + rhs.value, } } } @@ -301,7 +365,7 @@ impl ops::Add for Delta { impl ops::AddAssign for Delta { #[inline] fn add_assign(&mut self, rhs: Self) { - self.nanos += rhs.nanos; + self.value += rhs.value; } } @@ -311,7 +375,7 @@ impl ops::Sub for Delta { #[inline] fn sub(self, rhs: Self) -> Self::Output { Self { - nanos: self.nanos - rhs.nanos, + value: self.value - rhs.value, } } } @@ -319,7 +383,7 @@ impl ops::Sub for Delta { impl ops::SubAssign for Delta { #[inline] fn sub_assign(&mut self, rhs: Self) { - self.nanos -= rhs.nanos; + self.value -= rhs.value; } } @@ -329,7 +393,7 @@ impl ops::Mul<i64> for Delta { #[inline] fn mul(self, rhs: i64) -> Self::Output { Self { - nanos: self.nanos * rhs, + value: self.value * rhs, } } } @@ -337,7 +401,7 @@ impl ops::Mul<i64> for Delta { impl ops::MulAssign<i64> for Delta { #[inline] fn mul_assign(&mut self, rhs: i64) { - self.nanos *= rhs; + self.value *= rhs; } } @@ -348,20 +412,26 @@ impl ops::Div for Delta { fn div(self, rhs: Self) -> Self::Output { #[cfg(CONFIG_64BIT)] { - self.nanos / rhs.nanos + self.value / rhs.value } #[cfg(not(CONFIG_64BIT))] { // SAFETY: This function is always safe to call regardless of the input values - unsafe { bindings::div64_s64(self.nanos, rhs.nanos) } + unsafe { bindings::div64_s64(self.value, rhs.value) } } } } impl Delta { /// A span of time equal to zero. - pub const ZERO: Self = Self { nanos: 0 }; + pub const ZERO: Self = Self { value: 0 }; + + /// Create a new [`Delta`] from a number of nanoseconds. + #[inline] + pub const fn from_nanos(nanos: i64) -> Self { + Self { value: nanos } + } /// Create a new [`Delta`] from a number of microseconds. /// @@ -371,7 +441,7 @@ impl Delta { #[inline] pub const fn from_micros(micros: i64) -> Self { Self { - nanos: micros.saturating_mul(NSEC_PER_USEC), + value: micros.saturating_mul(NSEC_PER_USEC), } } @@ -383,7 +453,7 @@ impl Delta { #[inline] pub const fn from_millis(millis: i64) -> Self { Self { - nanos: millis.saturating_mul(NSEC_PER_MSEC), + value: millis.saturating_mul(NSEC_PER_MSEC), } } @@ -395,7 +465,7 @@ impl Delta { #[inline] pub const fn from_secs(secs: i64) -> Self { Self { - nanos: secs.saturating_mul(NSEC_PER_SEC), + value: secs.saturating_mul(NSEC_PER_SEC), } } @@ -414,22 +484,32 @@ impl Delta { /// Return the number of nanoseconds in the [`Delta`]. #[inline] pub const fn as_nanos(self) -> i64 { - self.nanos + self.value } /// Return the smallest number of microseconds greater than or equal /// to the value in the [`Delta`]. #[inline] pub fn as_micros_ceil(self) -> i64 { + // Only positive values need to be rounded up: truncating division already + // rounds towards zero, i.e. up, for negative values. + // + // The usual `(nanos + d - 1) / d` is not used because the addition overflows + // once `nanos` exceeds `i64::MAX - (d - 1)`; saturating the addition instead + // would drop the rounding bias and return a result one unit too small. + let n = self.as_nanos(); + + let (n, add) = if n > 0 { (n - 1, 1) } else { (n, 0) }; + #[cfg(CONFIG_64BIT)] { - self.as_nanos().saturating_add(NSEC_PER_USEC - 1) / NSEC_PER_USEC + n / NSEC_PER_USEC + add } #[cfg(not(CONFIG_64BIT))] // SAFETY: It is always safe to call `ktime_to_us()` with any value. unsafe { - bindings::ktime_to_us(self.as_nanos().saturating_add(NSEC_PER_USEC - 1)) + bindings::ktime_to_us(n) + add } } @@ -448,6 +528,32 @@ impl Delta { } } + /// Return the smallest number of milliseconds greater than or equal + /// to the value in the [`Delta`]. + #[inline] + pub fn as_millis_ceil(self) -> i64 { + // Only positive values need to be rounded up: truncating division already + // rounds towards zero, i.e. up, for negative values. + // + // The usual `(nanos + d - 1) / d` is not used because the addition overflows + // once `nanos` exceeds `i64::MAX - (d - 1)`; saturating the addition instead + // would drop the rounding bias and return a result one unit too small. + let n = self.as_nanos(); + + let (n, add) = if n > 0 { (n - 1, 1) } else { (n, 0) }; + + #[cfg(CONFIG_64BIT)] + { + n / NSEC_PER_MSEC + add + } + + #[cfg(not(CONFIG_64BIT))] + // SAFETY: It is always safe to call `ktime_to_ms()` with any value. + unsafe { + bindings::ktime_to_ms(n) + add + } + } + /// Return `self % dividend` where `dividend` is in nanoseconds. /// /// The kernel doesn't have any emulation for `s64 % s64` on 32 bit platforms, so this is @@ -457,7 +563,7 @@ impl Delta { #[cfg(CONFIG_64BIT)] { Self { - nanos: self.as_nanos() % i64::from(dividend), + value: self.as_nanos() % i64::from(dividend), } } @@ -469,7 +575,7 @@ impl Delta { unsafe { bindings::div_s64_rem(self.as_nanos(), dividend, &mut rem) }; Self { - nanos: i64::from(rem), + value: i64::from(rem), } } } diff --git a/rust/kernel/time/hrtimer.rs b/rust/kernel/time/hrtimer.rs index 856d2d929a00..2d7f1131a813 100644 --- a/rust/kernel/time/hrtimer.rs +++ b/rust/kernel/time/hrtimer.rs @@ -66,6 +66,342 @@ //! //! A `restart` operation on a timer in the **stopped** state is equivalent to a //! `start` operation. +//! +//! When a type implements both `HrTimerPointer` and `Clone`, it is possible to +//! issue the `start` operation while the timer is in the **started** state. In +//! this case the `start` operation is equivalent to the `restart` operation. +//! +//! # Examples +//! +//! ## Using an intrusive timer living in a [`Box`] +//! +//! ``` +//! # use kernel::{ +//! # alloc::flags, +//! # impl_has_hr_timer, +//! # prelude::*, +//! # sync::{ +//! # atomic::{ordering, Atomic}, +//! # completion::Completion, +//! # Arc, +//! # }, +//! # time::{ +//! # hrtimer::{ +//! # RelativeMode, HrTimer, HrTimerCallback, HrTimerPointer, +//! # HrTimerRestart, HrTimerCallbackContext +//! # }, +//! # Delta, Monotonic, +//! # }, +//! # }; +//! +//! #[pin_data] +//! struct Shared { +//! #[pin] +//! flag: Atomic<u64>, +//! #[pin] +//! cond: Completion, +//! } +//! +//! impl Shared { +//! fn new() -> impl PinInit<Self> { +//! pin_init!(Self { +//! flag <- Atomic::new(0), +//! cond <- Completion::new(), +//! }) +//! } +//! } +//! +//! #[pin_data] +//! struct BoxIntrusiveHrTimer { +//! #[pin] +//! timer: HrTimer<Self>, +//! shared: Arc<Shared>, +//! } +//! +//! impl BoxIntrusiveHrTimer { +//! fn new() -> impl PinInit<Self, kernel::error::Error> { +//! try_pin_init!(Self { +//! timer <- HrTimer::new(), +//! shared: Arc::pin_init(Shared::new(), flags::GFP_KERNEL)?, +//! }) +//! } +//! } +//! +//! impl HrTimerCallback for BoxIntrusiveHrTimer { +//! type Pointer<'a> = Pin<KBox<Self>>; +//! +//! fn run(this: Pin<&mut Self>, _ctx: HrTimerCallbackContext<'_, Self>) -> HrTimerRestart { +//! pr_info!("Timer called\n"); +//! +//! let flag = this.shared.flag.fetch_add(1, ordering::Full); +//! this.shared.cond.complete_all(); +//! +//! if flag == 4 { +//! HrTimerRestart::NoRestart +//! } else { +//! HrTimerRestart::Restart +//! } +//! } +//! } +//! +//! impl_has_hr_timer! { +//! impl HasHrTimer<Self> for BoxIntrusiveHrTimer { +//! mode: RelativeMode<Monotonic>, field: self.timer +//! } +//! } +//! +//! let has_timer = Box::pin_init(BoxIntrusiveHrTimer::new(), GFP_KERNEL)?; +//! let shared = has_timer.shared.clone(); +//! let _handle = has_timer.start(Delta::from_micros(200)); +//! +//! while shared.flag.load(ordering::Relaxed) != 5 { +//! shared.cond.wait_for_completion(); +//! } +//! +//! pr_info!("Counted to 5\n"); +//! # Ok::<(), kernel::error::Error>(()) +//! ``` +//! +//! ## Using an intrusive timer in an [`Arc`] +//! +//! ``` +//! # use kernel::{ +//! # alloc::flags, +//! # impl_has_hr_timer, +//! # prelude::*, +//! # sync::{ +//! # atomic::{ordering, Atomic}, +//! # completion::Completion, +//! # Arc, ArcBorrow, +//! # }, +//! # time::{ +//! # hrtimer::{ +//! # RelativeMode, HrTimer, HrTimerCallback, HrTimerPointer, HrTimerRestart, +//! # HasHrTimer, HrTimerCallbackContext +//! # }, +//! # Delta, Monotonic, +//! # }, +//! # }; +//! +//! #[pin_data] +//! struct ArcIntrusiveHrTimer { +//! #[pin] +//! timer: HrTimer<Self>, +//! #[pin] +//! flag: Atomic<u64>, +//! #[pin] +//! cond: Completion, +//! } +//! +//! impl ArcIntrusiveHrTimer { +//! fn new() -> impl PinInit<Self> { +//! pin_init!(Self { +//! timer <- HrTimer::new(), +//! flag <- Atomic::new(0), +//! cond <- Completion::new(), +//! }) +//! } +//! } +//! +//! impl HrTimerCallback for ArcIntrusiveHrTimer { +//! type Pointer<'a> = Arc<Self>; +//! +//! fn run( +//! this: ArcBorrow<'_, Self>, +//! _ctx: HrTimerCallbackContext<'_, Self>, +//! ) -> HrTimerRestart { +//! pr_info!("Timer called\n"); +//! +//! let flag = this.flag.fetch_add(1, ordering::Full); +//! this.cond.complete_all(); +//! +//! if flag == 4 { +//! HrTimerRestart::NoRestart +//! } else { +//! HrTimerRestart::Restart +//! } +//! } +//! } +//! +//! impl_has_hr_timer! { +//! impl HasHrTimer<Self> for ArcIntrusiveHrTimer { +//! mode: RelativeMode<Monotonic>, field: self.timer +//! } +//! } +//! +//! let has_timer = Arc::pin_init(ArcIntrusiveHrTimer::new(), GFP_KERNEL)?; +//! let _handle = has_timer.clone().start(Delta::from_micros(200)); +//! +//! while has_timer.flag.load(ordering::Relaxed) != 5 { +//! has_timer.cond.wait_for_completion(); +//! } +//! +//! pr_info!("Counted to 5\n"); +//! # Ok::<(), kernel::error::Error>(()) +//! ``` +//! +//! ## Using a stack-based timer +//! +//! ``` +//! # use kernel::{ +//! # impl_has_hr_timer, +//! # prelude::*, +//! # sync::{ +//! # atomic::{ordering, Atomic}, +//! # completion::Completion, +//! # }, +//! # time::{ +//! # hrtimer::{ +//! # ScopedHrTimerPointer, HrTimer, HrTimerCallback, HrTimerPointer, HrTimerRestart, +//! # HasHrTimer, RelativeMode, HrTimerCallbackContext +//! # }, +//! # Delta, Monotonic, +//! # }, +//! # }; +//! # use pin_init::stack_pin_init; +//! +//! #[pin_data] +//! struct IntrusiveHrTimer { +//! #[pin] +//! timer: HrTimer<Self>, +//! #[pin] +//! flag: Atomic<u64>, +//! #[pin] +//! cond: Completion, +//! } +//! +//! impl IntrusiveHrTimer { +//! fn new() -> impl PinInit<Self> { +//! pin_init!(Self { +//! timer <- HrTimer::new(), +//! flag <- Atomic::new(0), +//! cond <- Completion::new(), +//! }) +//! } +//! } +//! +//! impl HrTimerCallback for IntrusiveHrTimer { +//! type Pointer<'a> = Pin<&'a Self>; +//! +//! fn run(this: Pin<&Self>, _ctx: HrTimerCallbackContext<'_, Self>) -> HrTimerRestart { +//! pr_info!("Timer called\n"); +//! +//! this.flag.store(1, ordering::Release); +//! this.cond.complete_all(); +//! +//! HrTimerRestart::NoRestart +//! } +//! } +//! +//! impl_has_hr_timer! { +//! impl HasHrTimer<Self> for IntrusiveHrTimer { +//! mode: RelativeMode<Monotonic>, field: self.timer +//! } +//! } +//! +//! stack_pin_init!( let has_timer = IntrusiveHrTimer::new() ); +//! has_timer.as_ref().start_scoped(Delta::from_micros(200), || { +//! while has_timer.flag.load(ordering::Relaxed) != 1 { +//! has_timer.cond.wait_for_completion(); +//! } +//! }); +//! +//! pr_info!("Flag raised\n"); +//! # Ok::<(), kernel::error::Error>(()) +//! ``` +//! +//! ## Using a mutable stack-based timer +//! +//! ``` +//! # use kernel::{ +//! # alloc::flags, +//! # impl_has_hr_timer, +//! # prelude::*, +//! # sync::{ +//! # atomic::{ordering, Atomic}, +//! # completion::Completion, +//! # Arc, +//! # }, +//! # time::{ +//! # hrtimer::{ +//! # ScopedHrTimerPointer, HrTimer, HrTimerCallback, HrTimerPointer, HrTimerRestart, +//! # HasHrTimer, RelativeMode, HrTimerCallbackContext +//! # }, +//! # Delta, Monotonic, +//! # }, +//! # }; +//! # use pin_init::stack_try_pin_init; +//! +//! #[pin_data] +//! struct Shared { +//! #[pin] +//! flag: Atomic<u64>, +//! #[pin] +//! cond: Completion, +//! } +//! +//! impl Shared { +//! fn new() -> impl PinInit<Self> { +//! pin_init!(Self { +//! flag <- Atomic::new(0), +//! cond <- Completion::new(), +//! }) +//! } +//! } +//! +//! #[pin_data] +//! struct IntrusiveHrTimer { +//! #[pin] +//! timer: HrTimer<Self>, +//! shared: Arc<Shared>, +//! } +//! +//! impl IntrusiveHrTimer { +//! fn new() -> impl PinInit<Self, kernel::error::Error> { +//! try_pin_init!(Self { +//! timer <- HrTimer::new(), +//! shared: Arc::pin_init(Shared::new(), flags::GFP_KERNEL)?, +//! }) +//! } +//! } +//! +//! impl HrTimerCallback for IntrusiveHrTimer { +//! type Pointer<'a> = Pin<&'a mut Self>; +//! +//! fn run(this: Pin<&mut Self>, _ctx: HrTimerCallbackContext<'_, Self>) -> HrTimerRestart { +//! pr_info!("Timer called\n"); +//! +//! let flag = this.shared.flag.fetch_add(1, ordering::Full); +//! this.shared.cond.complete_all(); +//! +//! if flag == 4 { +//! HrTimerRestart::NoRestart +//! } else { +//! HrTimerRestart::Restart +//! } +//! } +//! } +//! +//! impl_has_hr_timer! { +//! impl HasHrTimer<Self> for IntrusiveHrTimer { +//! mode: RelativeMode<Monotonic>, field: self.timer +//! } +//! } +//! +//! stack_try_pin_init!( let has_timer =? IntrusiveHrTimer::new() ); +//! let shared = has_timer.shared.clone(); +//! +//! has_timer.as_mut().start_scoped(Delta::from_micros(200), || { +//! while shared.flag.load(ordering::Relaxed) != 5 { +//! shared.cond.wait_for_completion(); +//! } +//! }); +//! +//! pr_info!("Counted to 5\n"); +//! # Ok::<(), kernel::error::Error>(()) +//! ``` +//! +//! [`Arc`]: kernel::sync::Arc use super::{ClockSource, Delta, Instant}; use crate::{prelude::*, types::Opaque}; diff --git a/rust/kernel/transmute.rs b/rust/kernel/transmute.rs index 5711580c9f9b..654b5ede2fe2 100644 --- a/rust/kernel/transmute.rs +++ b/rust/kernel/transmute.rs @@ -49,7 +49,6 @@ pub unsafe trait FromBytes { let slice_ptr = bytes.as_ptr().cast::<Self>(); let size = size_of::<Self>(); - #[allow(clippy::incompatible_msrv)] if bytes.len() == size && slice_ptr.is_aligned() { // SAFETY: Size and alignment were just checked. unsafe { Some(&*slice_ptr) } @@ -67,16 +66,9 @@ pub unsafe trait FromBytes { where Self: Sized, { - if bytes.len() < size_of::<Self>() { - None - } else { - // PANIC: We checked that `bytes.len() >= size_of::<Self>`, thus `split_at` cannot - // panic. - // TODO: replace with `split_at_checked` once the MSRV is >= 1.80. - let (prefix, remainder) = bytes.split_at(size_of::<Self>()); + let (prefix, remainder) = bytes.split_at_checked(size_of::<Self>())?; - Self::from_bytes(prefix).map(|s| (s, remainder)) - } + Self::from_bytes(prefix).map(|s| (s, remainder)) } /// Converts a mutable slice of bytes to a reference to `Self`. @@ -92,7 +84,6 @@ pub unsafe trait FromBytes { let slice_ptr = bytes.as_mut_ptr().cast::<Self>(); let size = size_of::<Self>(); - #[allow(clippy::incompatible_msrv)] if bytes.len() == size && slice_ptr.is_aligned() { // SAFETY: Size and alignment were just checked. unsafe { Some(&mut *slice_ptr) } @@ -110,16 +101,9 @@ pub unsafe trait FromBytes { where Self: AsBytes + Sized, { - if bytes.len() < size_of::<Self>() { - None - } else { - // PANIC: We checked that `bytes.len() >= size_of::<Self>`, thus `split_at_mut` cannot - // panic. - // TODO: replace with `split_at_mut_checked` once the MSRV is >= 1.80. - let (prefix, remainder) = bytes.split_at_mut(size_of::<Self>()); + let (prefix, remainder) = bytes.split_at_mut_checked(size_of::<Self>())?; - Self::from_bytes_mut(prefix).map(|s| (s, remainder)) - } + Self::from_bytes_mut(prefix).map(|s| (s, remainder)) } /// Creates an owned instance of `Self` by copying `bytes`. @@ -149,16 +133,9 @@ pub unsafe trait FromBytes { where Self: Sized, { - if bytes.len() < size_of::<Self>() { - None - } else { - // PANIC: We checked that `bytes.len() >= size_of::<Self>`, thus `split_at` cannot - // panic. - // TODO: replace with `split_at_checked` once the MSRV is >= 1.80. - let (prefix, remainder) = bytes.split_at(size_of::<Self>()); + let (prefix, remainder) = bytes.split_at_checked(size_of::<Self>())?; - Self::from_bytes_copy(prefix).map(|s| (s, remainder)) - } + Self::from_bytes_copy(prefix).map(|s| (s, remainder)) } } diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 9c5e7dbf1632..132dd428c1f6 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -11,7 +11,12 @@ use core::{ }; use pin_init::{PinInit, Wrapper, Zeroable}; -pub use crate::sync::aref::{ARef, AlwaysRefCounted}; +#[doc(hidden)] +pub mod for_lt; +pub use for_lt::{ + CovariantForLt, + ForLt, // +}; /// Used to transfer ownership to and from foreign (non-Rust) languages. /// @@ -29,10 +34,14 @@ pub unsafe trait ForeignOwnable: Sized { const FOREIGN_ALIGN: usize; /// Type used to immutably borrow a value that is currently foreign-owned. - type Borrowed<'a>; + type Borrowed<'a> + where + Self: 'a; /// Type used to mutably borrow a value that is currently foreign-owned. - type BorrowedMut<'a>; + type BorrowedMut<'a> + where + Self: 'a; /// Converts a Rust-owned object to a foreign-owned one. /// @@ -411,13 +420,13 @@ impl<T> Opaque<T> { impl<T> Wrapper<T> for Opaque<T> { /// Create an opaque pin-initializer from the given pin-initializer. - fn pin_init<E>(slot: impl PinInit<T, E>) -> impl PinInit<Self, E> { - Self::try_ffi_init(|ptr: *mut T| { + fn pin_init<E>(init: impl PinInit<T, E>) -> impl PinInit<Self, E> { + Self::try_ffi_init(|slot: *mut T| { // SAFETY: - // - `ptr` is a valid pointer to uninitialized memory, + // - `slot` is a valid pointer to uninitialized memory, // - `slot` is not accessed on error, // - `slot` is pinned in memory. - unsafe { PinInit::<T, E>::__pinned_init(slot, ptr) } + unsafe { pin_init::raw_try_init(slot, init) } }) } } diff --git a/rust/kernel/types/for_lt.rs b/rust/kernel/types/for_lt.rs new file mode 100644 index 000000000000..b8f422c802dc --- /dev/null +++ b/rust/kernel/types/for_lt.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Provide implementation and test of the [`trait@ForLt`] and [`trait@CovariantForLt`] traits and +//! macros. +//! +//! This module is hidden and users should just use [`ForLt!`](macro@ForLt) / +//! [`CovariantForLt!`](macro@CovariantForLt) directly. + +use core::marker::PhantomData; + +/// Representation of types generic over a lifetime. +/// +/// # Macro +/// +/// It is not recommended to implement this trait directly. [`ForLt!`](macro@ForLt) macro is +/// provided to obtain a type that implements this trait. +/// +/// The full syntax is +/// +/// ``` +/// # use kernel::types::ForLt; +/// # fn expect_lt<F: ForLt>() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// ForLt!(for<'a> TypeThatUse<'a>) +/// # >(); +/// ``` +/// +/// which gives a type so that `<ForLt!(for<'a> TypeThatUse<'a>) as ForLt>::Of<'b>` +/// is `TypeThatUse<'b>`. +/// +/// You may also use a short-hand syntax which works similar to lifetime elision. +/// The macro also accepts types that do not involve a lifetime at all. +/// +/// ``` +/// # use kernel::types::ForLt; +/// # fn expect_lt<F: ForLt>() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// ForLt!(TypeThatUse<'_>) // Equivalent to `ForLt!(for<'a> TypeThatUse<'a>)`. +/// # >(); +/// # expect_lt::< +/// ForLt!(&u32) // Equivalent to `ForLt!(for<'a> &'a u32)`. +/// # >(); +/// # expect_lt::< +/// ForLt!(u32) // Equivalent to `ForLt!(for<'a> u32)`. +/// # >(); +/// ``` +pub trait ForLt { + /// The type parameterized by the lifetime. + type Of<'a>: 'a; +} +pub use macros::ForLt; + +/// [`trait@ForLt`] subtrait for types that are covariant over their lifetime parameter. +/// +/// Provides a safe [`cast_ref`](CovariantForLt::cast_ref) method for types that are proven to be +/// covariant. The `CovariantForLt!` macro syntax is the same as `ForLt!`. +/// +/// # Macro +/// +/// It is not recommended to implement this trait directly. +/// [`CovariantForLt!`](macro@CovariantForLt) macro is provided to obtain a type that implements +/// this trait. +/// +/// The full syntax is +/// +/// ``` +/// # use kernel::types::CovariantForLt; +/// # fn expect_lt<F: CovariantForLt>() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// CovariantForLt!(for<'a> TypeThatUse<'a>) +/// # >(); +/// ``` +/// +/// which gives a type so that +/// `<CovariantForLt!(for<'a> TypeThatUse<'a>) as CovariantForLt>::Of<'b>` +/// is `TypeThatUse<'b>`. +/// +/// You may also use a short-hand syntax which works similar to lifetime elision. +/// The macro also accepts types that do not involve a lifetime at all. +/// +/// ``` +/// # use kernel::types::CovariantForLt; +/// # fn expect_lt<F: CovariantForLt>() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// CovariantForLt!(TypeThatUse<'_>) // Equivalent to `CovariantForLt!(for<'a> TypeThatUse<'a>)`. +/// # >(); +/// # expect_lt::< +/// CovariantForLt!(&u32) // Equivalent to `CovariantForLt!(for<'a> &'a u32)`. +/// # >(); +/// # expect_lt::< +/// CovariantForLt!(u32) // Equivalent to `CovariantForLt!(for<'a> u32)`. +/// # >(); +/// ``` +/// +/// The macro will attempt to prove that the type is indeed covariant over the lifetime supplied. +/// When it cannot be syntactically proven, it will emit checks to ask the Rust compiler to prove +/// it. +/// +/// ```ignore,compile_fail +/// # use kernel::types::CovariantForLt; +/// # fn expect_lt<F: CovariantForLt>() {} +/// # expect_lt::< +/// CovariantForLt!(fn(&u32)) // Contravariant, will fail compilation. +/// # >(); +/// ``` +/// +/// There is a limitation if the type refers to generic parameters; if the macro cannot prove the +/// covariance syntactically, the emitted checks will fail the compilation as it needs to refer to +/// the generic parameter but is in a separate item. +/// +/// ``` +/// # use kernel::types::CovariantForLt; +/// fn expect_lt<F: CovariantForLt>() {} +/// # #[allow(clippy::unnecessary_safety_comment, reason = "false positive")] +/// fn generic_fn<T: 'static>() { +/// // Syntactically proven by the macro +/// expect_lt::<CovariantForLt!(&T)>(); +/// // Syntactically proven by the macro +/// expect_lt::<CovariantForLt!(&KBox<T>)>(); +/// // Cannot be syntactically proven, need to check covariance of `KBox` +/// // expect_lt::<CovariantForLt!(&KBox<&T>)>(); +/// } +/// ``` +/// +/// # Safety +/// +/// `Self::Of<'a>` must be covariant over the lifetime `'a`. +pub unsafe trait CovariantForLt: ForLt { + /// Cast a reference to a shorter lifetime. + #[inline(always)] + fn cast_ref<'r, 'short: 'r, 'long: 'short>(long: &'r Self::Of<'long>) -> &'r Self::Of<'short> { + // SAFETY: This is sound as this trait guarantees covariance. + unsafe { core::mem::transmute(long) } + } +} +pub use macros::CovariantForLt; + +/// This is intended to be an "unsafe-to-refer-to" type. +/// +/// Must only be used by the [`ForLt!`](macro@ForLt) / [`CovariantForLt!`](macro@CovariantForLt) +/// macros. +/// +/// `T` is the magic `dyn for<'a> WithLt<'a, TypeThatUse<'a>>` generated by macro. +/// +/// `WF` is a type that the macro can use to assert some specific type is well-formed. +/// +/// `N` is to provide the macro a place to emit arbitrary items, in case it needs to prove +/// additional properties. [`ForLt!`](macro@ForLt) emits `N = 0`; +/// [`CovariantForLt!`](macro@CovariantForLt) emits `N = 1` after a covariance proof. +#[doc(hidden)] +pub struct UnsafeForLtImpl<T: ?Sized, WF, const N: usize>(PhantomData<(WF, T)>); + +// This is a helper trait for implementation of `ForLt` / `CovariantForLt` to be able to use HRTB. +#[doc(hidden)] +pub trait WithLt<'a> { + type Of: 'a; +} + +impl<T: ?Sized + for<'a> WithLt<'a>, WF, const N: usize> ForLt for UnsafeForLtImpl<T, WF, N> { + type Of<'a> = <T as WithLt<'a>>::Of; +} + +// SAFETY: In `CovariantForLt!` macro, a covariance proof is generated in the `N` const generic +// and it will fail to evaluate if the type is not covariant. Only `N = 1` gets this impl. +unsafe impl<T: ?Sized + for<'a> WithLt<'a>, WF> CovariantForLt for UnsafeForLtImpl<T, WF, 1> {} diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs index f989539a31b4..5f6c4d7a1a51 100644 --- a/rust/kernel/uaccess.rs +++ b/rust/kernel/uaccess.rs @@ -7,10 +7,12 @@ use crate::{ alloc::{Allocator, Flags}, bindings, + dma::Coherent, error::Result, ffi::{c_char, c_void}, fs::file, prelude::*, + ptr::KnownSize, transmute::{AsBytes, FromBytes}, }; use core::mem::{size_of, MaybeUninit}; @@ -19,7 +21,7 @@ use core::mem::{size_of, MaybeUninit}; /// /// This is the Rust equivalent to C pointers tagged with `__user`. #[repr(transparent)] -#[derive(Copy, Clone)] +#[derive(Copy, Clone, Zeroable)] pub struct UserPtr(*mut c_void); impl UserPtr { @@ -459,20 +461,19 @@ impl UserSliceWriter { self.length == 0 } - /// Writes raw data to this user pointer from a kernel buffer. + /// Low-level write from a raw pointer. /// - /// Fails with [`EFAULT`] if the write happens on a bad address, or if the write goes out of - /// bounds of this [`UserSliceWriter`]. This call may modify the associated userspace slice even - /// if it returns an error. - pub fn write_slice(&mut self, data: &[u8]) -> Result { - let len = data.len(); - let data_ptr = data.as_ptr().cast::<c_void>(); + /// # Safety + /// + /// The caller must ensure that `from` is valid for reads of `len` bytes. + unsafe fn write_raw(&mut self, from: *const u8, len: usize) -> Result { if len > self.length { return Err(EFAULT); } - // SAFETY: `data_ptr` points into an immutable slice of length `len`, so we may read - // that many bytes from it. - let res = unsafe { bindings::copy_to_user(self.ptr.as_mut_ptr(), data_ptr, len) }; + + // SAFETY: Caller guarantees `from` is valid for `len` bytes (see this function's + // safety contract). + let res = unsafe { bindings::copy_to_user(self.ptr.as_mut_ptr(), from.cast(), len) }; if res != 0 { return Err(EFAULT); } @@ -481,6 +482,76 @@ impl UserSliceWriter { Ok(()) } + /// Writes raw data to this user pointer from a kernel buffer. + /// + /// Fails with [`EFAULT`] if the write happens on a bad address, or if the write goes out of + /// bounds of this [`UserSliceWriter`]. This call may modify the associated userspace slice even + /// if it returns an error. + pub fn write_slice(&mut self, data: &[u8]) -> Result { + // SAFETY: `data` is a valid slice, so `data.as_ptr()` is valid for + // reading `data.len()` bytes. + unsafe { self.write_raw(data.as_ptr(), data.len()) } + } + + /// Writes raw data to this user pointer from a DMA coherent allocation. + /// + /// Copies `count` bytes from `alloc` starting from `offset` into this userspace slice. + /// + /// # Errors + /// + /// - [`EOVERFLOW`]: `offset + count` overflows. + /// - [`ERANGE`]: `offset + count` exceeds the size of `alloc`, or `count` exceeds the + /// size of the user-space buffer. + /// - [`EFAULT`]: the write hits a bad address or goes out of bounds of this + /// [`UserSliceWriter`]. + /// + /// This call may modify the associated userspace slice even if it returns an error. + /// + /// Note: The memory may be concurrently modified by hardware (e.g., DMA). In such cases, + /// the copied data may be inconsistent, but this does not cause undefined behavior. + /// + /// # Example + /// + /// Copy the first 256 bytes of a DMA coherent allocation into a userspace buffer: + /// + /// ```no_run + /// use kernel::uaccess::UserSliceWriter; + /// use kernel::dma::Coherent; + /// + /// fn copy_dma_to_user( + /// mut writer: UserSliceWriter, + /// alloc: &Coherent<[u8]>, + /// ) -> Result { + /// writer.write_dma(alloc, 0, 256) + /// } + /// ``` + pub fn write_dma<T: KnownSize + AsBytes + ?Sized>( + &mut self, + alloc: &Coherent<T>, + offset: usize, + count: usize, + ) -> Result { + let len = alloc.size(); + if offset.checked_add(count).ok_or(EOVERFLOW)? > len { + return Err(ERANGE); + } + + if count > self.len() { + return Err(ERANGE); + } + + // SAFETY: `as_ptr()` returns a valid pointer to a memory region of `count()` bytes, as + // guaranteed by the `Coherent` invariants. The check above ensures `offset + count <= len`. + let src_ptr = unsafe { alloc.as_ptr().cast::<u8>().add(offset) }; + + // Note: Use `write_raw` instead of `write_slice` because the allocation is coherent + // memory that hardware may modify (e.g., DMA); we cannot form a `&[u8]` slice over + // such volatile memory. + // + // SAFETY: `src_ptr` points into the allocation and is valid for `count` bytes (see above). + unsafe { self.write_raw(src_ptr, count) } + } + /// Writes raw data to this user pointer from a kernel buffer partially. /// /// This is the same as [`Self::write_slice`] but considers the given `offset` into `data` and diff --git a/rust/kernel/usb.rs b/rust/kernel/usb.rs index 0e1b9a88f4f1..993760ff332b 100644 --- a/rust/kernel/usb.rs +++ b/rust/kernel/usb.rs @@ -18,19 +18,14 @@ use crate::{ to_result, // }, prelude::*, - types::{ - AlwaysRefCounted, - Opaque, // - }, + sync::aref::AlwaysRefCounted, + types::Opaque, ThisModule, // }; use core::{ marker::PhantomData, - mem::{ - offset_of, - MaybeUninit, // - }, - ptr::NonNull, + mem::offset_of, + ptr::NonNull, // }; /// An adapter for the registration of USB drivers. @@ -38,18 +33,18 @@ pub struct Adapter<T: Driver>(T); // SAFETY: // - `bindings::usb_driver` is a C type declared as `repr(C)`. -// - `T` is the type of the driver's device private data. +// - `T::Data` is the type of the driver's device private data. // - `struct usb_driver` embeds a `struct device_driver`. // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. -unsafe impl<T: Driver + 'static> driver::DriverLayout for Adapter<T> { +unsafe impl<T: Driver> driver::DriverLayout for Adapter<T> { type DriverType = bindings::usb_driver; - type DriverData = T; + type DriverData<'bound> = T::Data<'bound>; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); } // SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if // a preceding call to `register` has been successful. -unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { +unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { unsafe fn register( udrv: &Opaque<Self::DriverType>, name: &'static CStr, @@ -65,7 +60,7 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { // SAFETY: `udrv` is guaranteed to be a valid `DriverType`. to_result(unsafe { - bindings::usb_register_driver(udrv.get(), module.0, name.as_char_ptr()) + bindings::usb_register_driver(udrv.get(), module.as_ptr(), name.as_char_ptr()) }) } @@ -75,7 +70,7 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { } } -impl<T: Driver + 'static> Adapter<T> { +impl<T: Driver> Adapter<T> { extern "C" fn probe_callback( intf: *mut bindings::usb_interface, id: *const bindings::usb_device_id, @@ -84,17 +79,20 @@ impl<T: Driver + 'static> Adapter<T> { // `struct usb_interface` and `struct usb_device_id`. // // INVARIANT: `intf` is valid for the duration of `probe_callback()`. - let intf = unsafe { &*intf.cast::<Interface<device::CoreInternal>>() }; + let intf = unsafe { &*intf.cast::<Interface<device::CoreInternal<'_>>>() }; from_result(|| { // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct usb_device_id` and // does not add additional invariants, so it's safe to transmute. let id = unsafe { &*id.cast::<DeviceId>() }; - let info = T::ID_TABLE.info(id.index()); + // SAFETY: `id` comes from `T::ID_TABLE` which is of type `IdArray<_, T::IdInfo>`. It + // can also come from dynamic IDs, which will ensure that `driver_data` exists in + // `T::ID_TABLE` or is 0. + let info = unsafe { id.info_unchecked_opt::<T::IdInfo>() }; let data = T::probe(intf, id, info); - let dev: &device::Device<device::CoreInternal> = intf.as_ref(); + let dev: &device::Device<device::CoreInternal<'_>> = intf.as_ref(); dev.set_drvdata(data)?; Ok(0) }) @@ -105,14 +103,14 @@ impl<T: Driver + 'static> Adapter<T> { // `struct usb_interface`. // // INVARIANT: `intf` is valid for the duration of `disconnect_callback()`. - let intf = unsafe { &*intf.cast::<Interface<device::CoreInternal>>() }; + let intf = unsafe { &*intf.cast::<Interface<device::CoreInternal<'_>>>() }; - let dev: &device::Device<device::CoreInternal> = intf.as_ref(); + let dev: &device::Device<device::CoreInternal<'_>> = intf.as_ref(); // SAFETY: `disconnect_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called - // and stored a `Pin<KBox<T>>`. - let data = unsafe { dev.drvdata_borrow::<T>() }; + // and stored a `Pin<KBox<T::Data<'_>>>`. + let data = unsafe { dev.drvdata_borrow::<T::Data<'_>>() }; T::disconnect(intf, data); } @@ -132,8 +130,7 @@ impl DeviceId { match_flags: bindings::USB_DEVICE_ID_MATCH_DEVICE as u16, idVendor: vendor, idProduct: product, - // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }) } @@ -145,8 +142,7 @@ impl DeviceId { idProduct: product, bcdDevice_lo: bcd_lo, bcdDevice_hi: bcd_hi, - // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }) } @@ -157,8 +153,7 @@ impl DeviceId { bDeviceClass: class, bDeviceSubClass: subclass, bDeviceProtocol: protocol, - // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }) } @@ -169,8 +164,7 @@ impl DeviceId { bInterfaceClass: class, bInterfaceSubClass: subclass, bInterfaceProtocol: protocol, - // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }) } @@ -182,8 +176,7 @@ impl DeviceId { idVendor: vendor, idProduct: product, bInterfaceClass: class, - // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }) } @@ -195,8 +188,7 @@ impl DeviceId { idVendor: vendor, idProduct: product, bInterfaceProtocol: protocol, - // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }) } @@ -208,8 +200,7 @@ impl DeviceId { idVendor: vendor, idProduct: product, bInterfaceNumber: number, - // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }) } @@ -229,8 +220,7 @@ impl DeviceId { bInterfaceClass: class, bInterfaceSubClass: subclass, bInterfaceProtocol: protocol, - // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }) } } @@ -244,10 +234,6 @@ unsafe impl RawDeviceId for DeviceId { // SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_info` field. unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::usb_device_id, driver_info); - - fn index(&self) -> usize { - self.0.driver_info - } } /// [`IdTable`](kernel::device_id::IdTable) type for USB. @@ -256,14 +242,8 @@ pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; /// Create a USB `IdTable` with its alias for modpost. #[macro_export] macro_rules! usb_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::usb::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("usb", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("usb", $crate::usb::DeviceId, $($tt)*); }; } @@ -279,7 +259,6 @@ macro_rules! usb_device_table { /// /// kernel::usb_device_table!( /// USB_TABLE, -/// MODULE_USB_TABLE, /// <MyDriver as usb::Driver>::IdInfo, /// [ /// (usb::DeviceId::from_id(0x1234, 0x5678), ()), @@ -289,23 +268,31 @@ macro_rules! usb_device_table { /// /// impl usb::Driver for MyDriver { /// type IdInfo = (); +/// type Data<'bound> = Self; /// const ID_TABLE: usb::IdTable<Self::IdInfo> = &USB_TABLE; /// -/// fn probe( -/// _interface: &usb::Interface<Core>, +/// fn probe<'bound>( +/// _interface: &'bound usb::Interface<Core<'_>>, /// _id: &usb::DeviceId, -/// _info: &Self::IdInfo, -/// ) -> impl PinInit<Self, Error> { +/// _info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound { /// Err(ENODEV) /// } /// -/// fn disconnect(_interface: &usb::Interface<Core>, _data: Pin<&Self>) {} +/// fn disconnect<'bound>( +/// _interface: &'bound usb::Interface<Core<'_>>, +/// _data: Pin<&Self::Data<'bound>>, +/// ) { +/// } /// } ///``` pub trait Driver { /// The type holding information about each one of the device ids supported by the driver. type IdInfo: 'static; + /// The type of the driver's bus device private data. + type Data<'bound>: Send + 'bound; + /// The table of device ids supported by the driver. const ID_TABLE: IdTable<Self::IdInfo>; @@ -313,16 +300,19 @@ pub trait Driver { /// /// Called when a new USB interface is bound to this driver. /// Implementers should attempt to initialize the interface here. - fn probe( - interface: &Interface<device::Core>, + fn probe<'bound>( + interface: &'bound Interface<device::Core<'_>>, id: &DeviceId, - id_info: &Self::IdInfo, - ) -> impl PinInit<Self, Error>; + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound; /// USB driver disconnect. /// /// Called when the USB interface is about to be unbound from this driver. - fn disconnect(interface: &Interface<device::Core>, data: Pin<&Self>); + fn disconnect<'bound>( + interface: &'bound Interface<device::Core<'_>>, + data: Pin<&Self::Data<'bound>>, + ); } /// A USB interface. @@ -384,6 +374,7 @@ impl<Ctx: device::DeviceContext> AsRef<Device> for Interface<Ctx> { // SAFETY: Instances of `Interface` are always reference-counted. unsafe impl AlwaysRefCounted for Interface { + #[inline] fn inc_ref(&self) { // SAFETY: The invariants of `Interface` guarantee that `self.as_raw()` // returns a valid `struct usb_interface` pointer, for which we will @@ -391,6 +382,7 @@ unsafe impl AlwaysRefCounted for Interface { unsafe { bindings::usb_get_intf(self.as_raw()) }; } + #[inline] unsafe fn dec_ref(obj: NonNull<Self>) { // SAFETY: The safety requirements guarantee that the refcount is non-zero. unsafe { bindings::usb_put_intf(obj.cast().as_ptr()) } @@ -435,6 +427,7 @@ kernel::impl_device_context_into_aref!(Device); // SAFETY: Instances of `Device` are always reference-counted. unsafe impl AlwaysRefCounted for Device { + #[inline] fn inc_ref(&self) { // SAFETY: The invariants of `Device` guarantee that `self.as_raw()` // returns a valid `struct usb_device` pointer, for which we will @@ -442,6 +435,7 @@ unsafe impl AlwaysRefCounted for Device { unsafe { bindings::usb_get_dev(self.as_raw()) }; } + #[inline] unsafe fn dec_ref(obj: NonNull<Self>) { // SAFETY: The safety requirements guarantee that the refcount is non-zero. unsafe { bindings::usb_put_dev(obj.cast().as_ptr()) } @@ -466,6 +460,10 @@ unsafe impl Send for Device {} // allow any mutation through a shared reference. unsafe impl Sync for Device {} +// SAFETY: Same as `Device<Normal>` -- the underlying `struct usb_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device<device::Bound> {} + /// Declares a kernel module that exposes a single USB driver. /// /// # Examples diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 706e833e9702..7e253b6f299c 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -189,12 +189,18 @@ use crate::{ alloc::{AllocError, Flags}, container_of, prelude::*, - sync::Arc, - sync::LockClassKey, + sync::{ + aref::{ + ARef, + AlwaysRefCounted, // + }, + Arc, + LockClassKey, // + }, time::Jiffies, types::Opaque, }; -use core::marker::PhantomData; +use core::{marker::PhantomData, ptr::NonNull}; /// Creates a [`Work`] initialiser with the given name and a newly-created lock class. #[macro_export] @@ -425,10 +431,11 @@ pub unsafe trait RawDelayedWorkItem<const ID: u64>: RawWorkItem<ID> {} /// Defines the method that should be called directly when a work item is executed. /// -/// This trait is implemented by `Pin<KBox<T>>` and [`Arc<T>`], and is mainly intended to be -/// implemented for smart pointer types. For your own structs, you would implement [`WorkItem`] -/// instead. The [`run`] method on this trait will usually just perform the appropriate -/// `container_of` translation and then call into the [`run`][WorkItem::run] method from the +/// This trait is implemented by `Pin<KBox<T>>`, [`Arc<T>`] and [`ARef<T>`], and +/// is mainly intended to be implemented for smart pointer types. For your own +/// structs, you would implement [`WorkItem`] instead. The [`run`] method on +/// this trait will usually just perform the appropriate `container_of` +/// translation and then call into the [`run`][WorkItem::run] method from the /// [`WorkItem`] trait. /// /// This trait is used when the `work_struct` field is defined using the [`Work`] helper. @@ -934,6 +941,89 @@ where { } +// SAFETY: Like the `Arc<T>` implementation, the `__enqueue` implementation for +// `ARef<T>` obtains a `work_struct` from the `Work` field using +// `T::raw_get_work`, so the same safety reasoning applies: +// +// - `__enqueue` gets the `work_struct` from the `Work` field, using `T::raw_get_work`. +// - The only safe way to create a `Work` object is through `Work::new`. +// - `Work::new` makes sure that `T::Pointer::run` is passed to `init_work_with_key`. +// - Finally `Work` and `RawWorkItem` guarantee that the correct `Work` field +// will be used because of the ID const generic bound. This makes sure that `T::raw_get_work` +// uses the correct offset for the `Work` field, and `Work::new` picks the correct +// implementation of `WorkItemPointer` for `ARef<T>`. +unsafe impl<T, const ID: u64> WorkItemPointer<ID> for ARef<T> +where + T: AlwaysRefCounted, + T: WorkItem<ID, Pointer = Self>, + T: HasWork<T, ID>, +{ + unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { + // The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. + let ptr = ptr.cast::<Work<T, ID>>(); + + // SAFETY: This computes the pointer that `__enqueue` got from + // `ARef::into_raw`. + let ptr = unsafe { T::work_container_of(ptr) }; + + // SAFETY: The safety contract of `work_container_of` ensures that it + // returns a valid non-null pointer. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + + // SAFETY: This pointer comes from `ARef::into_raw` and we've been given + // back ownership. + let aref = unsafe { ARef::from_raw(ptr) }; + + T::run(aref) + } +} + +// SAFETY: The `work_struct` raw pointer is guaranteed to be valid for the duration of the call to +// the closure because we get it from an `ARef`, which means that the ref count will be at least 1, +// and we don't drop the `ARef` ourselves. If `queue_work_on` returns true, it is further guaranteed +// to be valid until a call to the function pointer in `work_struct` because we leak the memory it +// points to, and only reclaim it if the closure returns false, or in `WorkItemPointer::run`, which +// is what the function pointer in the `work_struct` must be pointing to, according to the safety +// requirements of `WorkItemPointer`. +unsafe impl<T, const ID: u64> RawWorkItem<ID> for ARef<T> +where + T: AlwaysRefCounted, + T: WorkItem<ID, Pointer = Self>, + T: HasWork<T, ID>, +{ + type EnqueueOutput = Result<(), Self>; + + unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool, + { + let ptr = ARef::into_raw(self); + + // SAFETY: Pointers from ARef::into_raw are valid and non-null. + let work_ptr = unsafe { T::raw_get_work(ptr.as_ptr()) }; + // SAFETY: `raw_get_work` returns a pointer to a valid value. + let work_ptr = unsafe { Work::raw_get(work_ptr) }; + + if queue_work_on(work_ptr) { + Ok(()) + } else { + // SAFETY: The work queue has not taken ownership of the pointer. + Err(unsafe { ARef::from_raw(ptr) }) + } + } +} + +// SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in +// `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of +// the `delayed_work` has the same access rules as its `work` field. +unsafe impl<T, const ID: u64> RawDelayedWorkItem<ID> for ARef<T> +where + T: WorkItem<ID, Pointer = Self>, + T: HasDelayedWork<T, ID>, + T: AlwaysRefCounted, +{ +} + /// Returns the system work queue (`system_wq`). /// /// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are diff --git a/rust/kernel/xarray.rs b/rust/kernel/xarray.rs index a49d6db28845..987c9c0c2198 100644 --- a/rust/kernel/xarray.rs +++ b/rust/kernel/xarray.rs @@ -5,10 +5,16 @@ //! C header: [`include/linux/xarray.h`](srctree/include/linux/xarray.h) use crate::{ - alloc, bindings, build_assert, + alloc, + bindings, + build_assert::build_assert, error::{Error, Result}, ffi::c_void, - types::{ForeignOwnable, NotThreadSafe, Opaque}, + types::{ + ForeignOwnable, + NotThreadSafe, + Opaque, // + }, // }; use core::{iter, marker::PhantomData, pin::Pin, ptr::NonNull}; use pin_init::{pin_data, pin_init, pinned_drop, PinInit}; @@ -172,6 +178,7 @@ pub struct StoreError<T> { } impl<T> From<StoreError<T>> for Error { + #[inline] fn from(value: StoreError<T>) -> Self { value.error } diff --git a/rust/macros/for_lt.rs b/rust/macros/for_lt.rs new file mode 100644 index 000000000000..4372cbad3ec4 --- /dev/null +++ b/rust/macros/for_lt.rs @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use proc_macro2::{ + Span, + TokenStream, // +}; +use quote::{ + format_ident, + quote, // +}; +use syn::{ + parse::{ + Parse, + ParseStream, // + }, + visit::Visit, + visit_mut::VisitMut, + Lifetime, + Result, + Token, + Type, // +}; + +pub(crate) enum HigherRankedType { + Explicit { + _for_token: Token![for], + _lt_token: Token![<], + lifetime: Lifetime, + _gt_token: Token![>], + ty: Type, + }, + Implicit { + ty: Type, + }, +} + +impl Parse for HigherRankedType { + fn parse(input: ParseStream<'_>) -> Result<Self> { + if input.peek(Token![for]) { + Ok(Self::Explicit { + _for_token: input.parse()?, + _lt_token: input.parse()?, + lifetime: input.parse()?, + _gt_token: input.parse()?, + ty: input.parse()?, + }) + } else { + Ok(Self::Implicit { ty: input.parse()? }) + } + } +} + +trait TypeExt { + fn expand_elided_lifetime(&self, explicit_lt: &Lifetime) -> Type; + fn replace_lifetime(&self, src: &Lifetime, dst: &Lifetime) -> Type; + fn has_lifetime(&self, lt: &Lifetime) -> bool; +} + +impl TypeExt for Type { + fn expand_elided_lifetime(&self, explicit_lt: &Lifetime) -> Type { + struct ElidedLifetimeExpander<'a>(&'a Lifetime); + + impl VisitMut for ElidedLifetimeExpander<'_> { + fn visit_lifetime_mut(&mut self, lifetime: &mut Lifetime) { + // Expand explicit `'_` + if lifetime.ident == "_" { + *lifetime = self.0.clone(); + } + } + + fn visit_type_reference_mut(&mut self, reference: &mut syn::TypeReference) { + syn::visit_mut::visit_type_reference_mut(self, reference); + + if reference.lifetime.is_none() { + reference.lifetime = Some(self.0.clone()); + } + } + } + + let mut ret = self.clone(); + ElidedLifetimeExpander(explicit_lt).visit_type_mut(&mut ret); + ret + } + + fn replace_lifetime(&self, src: &Lifetime, dst: &Lifetime) -> Type { + struct LifetimeReplacer<'a>(&'a Lifetime, &'a Lifetime); + + impl VisitMut for LifetimeReplacer<'_> { + fn visit_lifetime_mut(&mut self, lifetime: &mut Lifetime) { + if lifetime.ident == self.0.ident { + *lifetime = self.1.clone(); + } + } + } + + let mut ret = self.clone(); + LifetimeReplacer(src, dst).visit_type_mut(&mut ret); + ret + } + + fn has_lifetime(&self, lt: &Lifetime) -> bool { + struct HasLifetime<'a>(&'a Lifetime, bool); + + impl Visit<'_> for HasLifetime<'_> { + fn visit_lifetime(&mut self, lifetime: &Lifetime) { + if lifetime.ident == self.0.ident { + self.1 = true; + } + } + + // Macro invocations are opaque; conservatively assume they may + // reference the lifetime. + fn visit_macro(&mut self, _: &syn::Macro) { + self.1 = true; + } + } + + let mut visitor = HasLifetime(lt, false); + visitor.visit_type(self); + visitor.1 + } +} + +struct Prover<'a>(&'a Lifetime, Vec<&'a Type>); + +impl<'a> Prover<'a> { + /// Prove that `ty` is covariant over `'lt`. + /// + /// This also needs to prove that it'll be wellformed for any instance of `'lt`. + /// It can be assumed that `ty` will be wellformed if `'lt` is substituted to `'static`. + fn prove(&mut self, ty: &'a Type) { + match ty { + Type::Paren(ty) => self.prove(&ty.elem), + Type::Group(ty) => self.prove(&ty.elem), + + // No lifetime involved + Type::Never(_) => {} + + // `[T; N]` and `[T]` is covariant over `T`. + Type::Array(ty) => self.prove(&ty.elem), + Type::Slice(ty) => self.prove(&ty.elem), + + Type::Tuple(ty) => { + for elem in &ty.elems { + self.prove(elem); + } + } + + // `*const T` is covariant over `T` + Type::Ptr(ty) if ty.const_token.is_some() => self.prove(&ty.elem), + + // `&T` is covariant over `T` and lifetime. + // + // Note that if we encounter `&'other_lt T`, then we still need to make sure the type + // is wellformed if `T` involves `&'lt`, so we defer to the compiler. + // + // This is to block cases like `CovariantForLt!(for<'a> &'static &'a u32)`, as the + // presence of the type implies `'a: 'static` but this is unsound. + Type::Reference(ty) + if ty.mutability.is_none() && ty.lifetime.as_ref() == Some(self.0) => + { + self.prove(&ty.elem) + } + + // `&[mut] T` is covariant over lifetime. + // In case we have `&[mut] NoLifetime`, we don't need to do additional checks. + Type::Reference(ty) if !ty.elem.has_lifetime(self.0) => (), + + // No mention of lifetime at all, no need to perform compiler check. + ty if !ty.has_lifetime(self.0) => (), + + // Otherwise, we need to emit checks so that compiler can determine if the types are + // actually covariant. + ty => self.1.push(ty), + } + } +} + +/// Shared implementation for both `ForLt!` and `CovariantForLt!`. +/// +/// Both macros run the prover and emit `ProveWf` structs to check well-formedness for all lifetime +/// instances (workaround for <https://github.com/rust-lang/rust/issues/152489>). `CovariantForLt!` +/// additionally emits covariance proof functions and sets `N = 1`. +fn for_lt_inner(input: HigherRankedType, prove_covariance: bool) -> TokenStream { + let (ty, lifetime) = match input { + HigherRankedType::Explicit { lifetime, ty, .. } => (ty, lifetime), + HigherRankedType::Implicit { ty } => { + // If there's no explicit `for<'a>` binder, inject a synthetic `'__elided` lifetime + // and expand elided sites. + let lifetime = Lifetime { + apostrophe: Span::mixed_site(), + ident: format_ident!("__elided", span = Span::mixed_site()), + }; + (ty.expand_elided_lifetime(&lifetime), lifetime) + } + }; + + let mut prover = Prover(&lifetime, Vec::new()); + prover.prove(&ty); + + let mut proof = Vec::new(); + + // Emit proofs for every type that requires additional compiler help in proving covariance. + for (idx, required_proof) in prover.1.into_iter().enumerate() { + // Insert a proof that the type is well-formed. + // + // This is intended to workaround a Rust compiler soundness bug related to HRTB. + // https://github.com/rust-lang/rust/issues/152489 + // + // This needs to be a struct instead of fn to avoid the implied WF bounds. + let wf_proof_name = format_ident!("ProveWf{idx}"); + proof.push(quote!( + struct #wf_proof_name<#lifetime>( + ::core::marker::PhantomData<&#lifetime ()>, #required_proof + ); + )); + + // Insert a proof that the type is covariant. + if prove_covariance { + let cov_proof_name = format_ident!("prove_covariant_{idx}"); + proof.push(quote!( + fn #cov_proof_name<'__short, '__long: '__short>( + long: #wf_proof_name<'__long> + ) -> #wf_proof_name<'__short> { + long + } + )); + } + } + + // Make sure that the type is wellformed when substituting lifetime with `'static`. + // + // Currently the Rust compiler doesn't check this, see the above `ProveWf` documentation. + // + // We prefer to use this way of proving WF-ness as it can work when generics are involved. + let ty_static = ty.replace_lifetime( + &lifetime, + &Lifetime { + apostrophe: Span::mixed_site(), + ident: format_ident!("static"), + }, + ); + + let n: usize = prove_covariance.into(); + + quote!( + ::kernel::types::for_lt::UnsafeForLtImpl::< + dyn for<#lifetime> ::kernel::types::for_lt::WithLt<#lifetime, Of = #ty>, + #ty_static, + { + #(#proof)* + + #n + } + > + ) +} + +pub(crate) fn for_lt(input: HigherRankedType) -> TokenStream { + for_lt_inner(input, false) +} + +pub(crate) fn covariant_for_lt(input: HigherRankedType) -> TokenStream { + for_lt_inner(input, true) +} diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs index 37ef6a6f2c85..d18fbf4daa0a 100644 --- a/rust/macros/helpers.rs +++ b/rust/macros/helpers.rs @@ -49,7 +49,6 @@ pub(crate) fn file() -> String { } #[cfg(CONFIG_RUSTC_HAS_SPAN_FILE)] - #[allow(clippy::incompatible_msrv)] { proc_macro::Span::call_site().file() } diff --git a/rust/macros/kunit.rs b/rust/macros/kunit.rs index 6be880d634e2..ae20ed6768f1 100644 --- a/rust/macros/kunit.rs +++ b/rust/macros/kunit.rs @@ -87,10 +87,11 @@ pub(crate) fn kunit_tests(test_suite: Ident, mut module: ItemMod) -> Result<Toke continue; }; - // TODO: Replace below with `extract_if` when MSRV is bumped above 1.85. - let before_len = f.attrs.len(); - f.attrs.retain(|attr| !attr.path().is_ident("test")); - if f.attrs.len() == before_len { + if f.attrs + .extract_if(.., |attr| attr.path().is_ident("test")) + .count() + == 0 + { processed_items.push(Item::Fn(f)); continue; } diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 0c36194d9971..24f96feaeb34 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -6,6 +6,9 @@ // and thus add a dependency on `include/config/RUSTC_VERSION_TEXT`, which is // touched by Kconfig when the version string from the compiler changes. +// Stable since Rust 1.87.0. +#![feature(extract_if)] +// // Stable since Rust 1.88.0 under a different name, `proc_macro_span_file`, // which was added in Rust 1.88.0. This is why `cfg_attr` is used here, i.e. // to avoid depending on the full `proc_macro_span` on Rust >= 1.88.0. @@ -14,6 +17,7 @@ mod concat_idents; mod export; mod fmt; +mod for_lt; mod helpers; mod kunit; mod module; @@ -52,6 +56,7 @@ use syn::parse_macro_input; /// - [`u64`] /// - [`isize`] /// - [`usize`] +/// - [`bool`] /// /// C header: [`include/linux/moduleparam.h`](srctree/include/linux/moduleparam.h) /// @@ -173,12 +178,29 @@ pub fn module(input: TokenStream) -> TokenStream { /// /// This macro should not be used when all functions are required. /// +/// Additionally, this macro automatically handles the `OwnerModule` +/// associated type: on the trait side, `type OwnerModule: ModuleMetadata;` +/// is added as a required associated type if not already defined; on the +/// impl side, `type OwnerModule = LocalModule;` is automatically inserted +/// if not explicitly defined. +/// /// # Examples /// /// ``` /// use kernel::error::VTABLE_DEFAULT_ERROR; /// use kernel::prelude::*; /// +/// # struct LocalModule; +/// # impl kernel::ModuleMetadata for LocalModule { +/// # const NAME: &'static kernel::str::CStr = c"vtable_doctest"; +/// # +/// # // SAFETY: This doctest runs on the host: there is no `THIS_MODULE`. +/// # const THIS_MODULE: kernel::ThisModule = unsafe { +/// # kernel::ThisModule::from_ptr(core::ptr::null_mut()) +/// # }; +/// # } +/// # +/// # fn main() { /// // Declares a `#[vtable]` trait /// #[vtable] /// pub trait Operations: Send + Sync + Sized { @@ -204,6 +226,7 @@ pub fn module(input: TokenStream) -> TokenStream { /// /// assert_eq!(<Foo as Operations>::HAS_FOO, true); /// assert_eq!(<Foo as Operations>::HAS_BAR, false); +/// # } /// ``` /// /// [`kernel::error::VTABLE_DEFAULT_ERROR`]: ../kernel/error/constant.VTABLE_DEFAULT_ERROR.html @@ -486,3 +509,31 @@ pub fn kunit_tests(attr: TokenStream, input: TokenStream) -> TokenStream { .unwrap_or_else(|e| e.into_compile_error()) .into() } + +/// Obtain a type that implements [`ForLt`] for the given higher-ranked type. +/// +/// Please refer to the documentation of the [`ForLt`] trait. +/// +/// [`ForLt`]: trait.ForLt.html +#[proc_macro] +#[allow(non_snake_case)] +pub fn ForLt(input: TokenStream) -> TokenStream { + for_lt::for_lt(parse_macro_input!(input)).into() +} + +/// Obtain a type that implements [`CovariantForLt`] (and [`ForLt`]) for the given higher-ranked +/// type. +/// +/// Unlike [`ForLt!`], this macro additionally proves that the type is covariant over the lifetime, +/// providing a safe [`CovariantForLt::cast_ref`] method. +/// +/// Please refer to the documentation of the [`CovariantForLt`] trait. +/// +/// [`CovariantForLt`]: trait.CovariantForLt.html +/// [`CovariantForLt::cast_ref`]: trait.CovariantForLt.html#method.cast_ref +/// [`ForLt`]: trait.ForLt.html +#[proc_macro] +#[allow(non_snake_case)] +pub fn CovariantForLt(input: TokenStream) -> TokenStream { + for_lt::covariant_for_lt(parse_macro_input!(input)).into() +} diff --git a/rust/macros/module.rs b/rust/macros/module.rs index e16298e520c7..bc7027f8dbb2 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -52,12 +52,7 @@ impl<'a> ModInfoBuilder<'a> { fn emit_base(&mut self, field: &str, content: &str, builtin: bool, param: bool) { let string = if builtin { // Built-in modules prefix their modinfo strings by `module.`. - format!( - "{module}.{field}={content}\0", - module = self.module, - field = field, - content = content - ) + format!("{module}.{field}={content}\0", module = self.module) } else { // Loadable modules' modinfo strings go as-is. format!("{field}={content}\0") @@ -109,7 +104,7 @@ impl<'a> ModInfoBuilder<'a> { } fn emit_param(&mut self, field: &str, param: &str, content: &str) { - let content = format!("{param}:{content}", param = param, content = content); + let content = format!("{param}:{content}"); self.emit_internal(field, &content, true); } @@ -197,6 +192,7 @@ fn param_ops_path(param_type: &str) -> Path { "u64" => parse_quote!(::kernel::module_param::PARAM_OPS_U64), "isize" => parse_quote!(::kernel::module_param::PARAM_OPS_ISIZE), "usize" => parse_quote!(::kernel::module_param::PARAM_OPS_USIZE), + "bool" => parse_quote!(::kernel::module_param::PARAM_OPS_BOOL), t => panic!("Unsupported parameter type {}", t), } } @@ -502,28 +498,28 @@ pub(crate) fn module(info: ModuleInfo) -> Result<TokenStream> { /// Used by the printing macros, e.g. [`info!`]. const __LOG_PREFIX: &[u8] = #name_cstr.to_bytes_with_nul(); - // SAFETY: `__this_module` is constructed by the kernel at load time and will not be - // freed until the module is unloaded. - #[cfg(MODULE)] - static THIS_MODULE: ::kernel::ThisModule = unsafe { - extern "C" { - static __this_module: ::kernel::types::Opaque<::kernel::bindings::module>; - }; - - ::kernel::ThisModule::from_ptr(__this_module.get()) - }; - - #[cfg(not(MODULE))] - static THIS_MODULE: ::kernel::ThisModule = unsafe { - ::kernel::ThisModule::from_ptr(::core::ptr::null_mut()) - }; - /// The `LocalModule` type is the type of the module created by `module!`, /// `module_pci_driver!`, `module_platform_driver!`, etc. type LocalModule = #type_; impl ::kernel::ModuleMetadata for #type_ { const NAME: &'static ::kernel::str::CStr = #name_cstr; + + #[cfg(MODULE)] + const THIS_MODULE: ::kernel::ThisModule = { + extern "C" { + static __this_module: ::kernel::types::Opaque<::kernel::bindings::module>; + } + + // SAFETY: `__this_module` is constructed by the kernel at load time + // and lives until the module is unloaded. + unsafe { ::kernel::ThisModule::from_ptr(__this_module.get()) } + }; + + #[cfg(not(MODULE))] + const THIS_MODULE: ::kernel::ThisModule = unsafe { + ::kernel::ThisModule::from_ptr(::core::ptr::null_mut()) + }; } // Double nested modules, since then nobody can access the public items inside. @@ -621,12 +617,12 @@ pub(crate) fn module(info: ModuleInfo) -> Result<TokenStream> { /// This function must only be called once. unsafe fn __init() -> ::kernel::ffi::c_int { let initer = <super::super::LocalModule as ::kernel::InPlaceModule>::init( - &super::super::THIS_MODULE + ::kernel::module::this_module::<super::super::LocalModule>() ); // SAFETY: No data race, since `__MOD` can only be accessed by this module // and there only `__init` and `__exit` access it. These functions are only // called once and `__exit` cannot be called before or during `__init`. - match unsafe { initer.__pinned_init(__MOD.as_mut_ptr()) } { + match unsafe { ::pin_init::raw_try_init(__MOD.as_mut_ptr(), initer) } { Ok(m) => 0, Err(e) => e.to_errno(), } diff --git a/rust/macros/vtable.rs b/rust/macros/vtable.rs index c6510b0c4ea1..be9a5ed8abe5 100644 --- a/rust/macros/vtable.rs +++ b/rust/macros/vtable.rs @@ -30,6 +30,22 @@ fn handle_trait(mut item: ItemTrait) -> Result<ItemTrait> { const USE_VTABLE_ATTR: (); }); + // Add `type OwnerModule: ModuleMetadata` as a required associated type if + // the trait does not already define it. + if !item + .items + .iter() + .any(|i| matches!(i, TraitItem::Type(t) if t.ident == "OwnerModule")) + { + gen_items.push(parse_quote! { + /// The module implementing this vtable trait. + /// + /// Automatically set to `crate::LocalModule` by the `#[vtable]` + /// impl macro. + type OwnerModule: ::kernel::ModuleMetadata; + }); + } + for item in &item.items { if let TraitItem::Fn(fn_item) = item { let name = &fn_item.sig.ident; @@ -57,12 +73,18 @@ fn handle_trait(mut item: ItemTrait) -> Result<ItemTrait> { fn handle_impl(mut item: ItemImpl) -> Result<ItemImpl> { let mut gen_items = Vec::new(); - let mut defined_consts = HashSet::new(); + let mut defined_items = HashSet::new(); - // Iterate over all user-defined constants to gather any possible explicit overrides. + // Iterate over all user-defined items to gather any possible explicit overrides. for item in &item.items { - if let ImplItem::Const(const_item) = item { - defined_consts.insert(const_item.ident.clone()); + match item { + ImplItem::Const(const_item) => { + defined_items.insert(const_item.ident.clone()); + } + ImplItem::Type(type_item) => { + defined_items.insert(type_item.ident.clone()); + } + _ => {} } } @@ -70,6 +92,15 @@ fn handle_impl(mut item: ItemImpl) -> Result<ItemImpl> { const USE_VTABLE_ATTR: () = (); }); + // Auto-insert `type OwnerModule = crate::LocalModule` if not explicitly defined. + // `crate::LocalModule` resolves to the real module type (via `module!`) or a + // dummy fallback in non-module contexts (e.g., doctests). + if !defined_items.contains(&parse_quote!(OwnerModule)) { + gen_items.push(parse_quote! { + type OwnerModule = crate::LocalModule; + }); + } + for item in &item.items { if let ImplItem::Fn(fn_item) = item { let name = &fn_item.sig.ident; @@ -78,7 +109,7 @@ fn handle_impl(mut item: ItemImpl) -> Result<ItemImpl> { name.span(), ); // Skip if it's declared already -- this allows user override. - if defined_consts.contains(&gen_const_name) { + if defined_items.contains(&gen_const_name) { continue; } let cfg_attrs = crate::helpers::gather_cfg_attrs(&fn_item.attrs); diff --git a/rust/pin-init/README.md b/rust/pin-init/README.md index 6cee6ab1eb57..2312c9e75f8c 100644 --- a/rust/pin-init/README.md +++ b/rust/pin-init/README.md @@ -3,7 +3,7 @@ [](https://deps.rs/repo/github/Rust-for-Linux/pin-init)  [](#nightly-only) - + # `pin-init` > [!NOTE] @@ -160,7 +160,6 @@ actually does the initialization in the correct way. Here are the things to look ```rust use pin_init::{pin_data, pinned_drop, PinInit, PinnedDrop, pin_init_from_closure}; use core::{ - ptr::addr_of_mut, marker::PhantomPinned, cell::UnsafeCell, pin::Pin, @@ -199,7 +198,7 @@ impl RawFoo { unsafe { pin_init_from_closure(move |slot: *mut Self| { // `slot` contains uninit memory, avoid creating a reference. - let foo = addr_of_mut!((*slot).foo); + let foo = &raw mut (*slot).foo; let foo = UnsafeCell::raw_get(foo).cast::<bindings::foo>(); // Initialize the `foo` diff --git a/rust/pin-init/examples/error.rs b/rust/pin-init/examples/error.rs index 8f4e135eb8ba..96f095398e8d 100644 --- a/rust/pin-init/examples/error.rs +++ b/rust/pin-init/examples/error.rs @@ -11,6 +11,7 @@ use std::alloc::AllocError; pub struct Error; impl From<Infallible> for Error { + #[inline] fn from(e: Infallible) -> Self { match e {} } @@ -18,6 +19,7 @@ impl From<Infallible> for Error { #[cfg(feature = "alloc")] impl From<AllocError> for Error { + #[inline] fn from(_: AllocError) -> Self { Self } diff --git a/rust/pin-init/examples/linked_list.rs b/rust/pin-init/examples/linked_list.rs index 8445a5890cb7..424585fe226d 100644 --- a/rust/pin-init/examples/linked_list.rs +++ b/rust/pin-init/examples/linked_list.rs @@ -2,7 +2,6 @@ #![allow(clippy::undocumented_unsafe_blocks)] #![cfg_attr(feature = "alloc", feature(allocator_api))] -#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] use core::{ cell::Cell, diff --git a/rust/pin-init/examples/mutex.rs b/rust/pin-init/examples/mutex.rs index 9f295226cd64..e8d4dbb664fe 100644 --- a/rust/pin-init/examples/mutex.rs +++ b/rust/pin-init/examples/mutex.rs @@ -2,7 +2,6 @@ #![allow(clippy::undocumented_unsafe_blocks)] #![cfg_attr(feature = "alloc", feature(allocator_api))] -#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] #![allow(clippy::missing_safety_doc)] use core::{ @@ -80,11 +79,7 @@ impl<T> CMutex<T> { wait_list <- ListHead::new(), spin_lock: SpinLock::new(), locked: Cell::new(false), - data <- unsafe { - pin_init_from_closure(|slot: *mut UnsafeCell<T>| { - val.__pinned_init(slot.cast::<T>()) - }) - }, + data <- UnsafeCell::pin_init(val), }) } @@ -92,7 +87,7 @@ impl<T> CMutex<T> { pub fn lock(&self) -> Pin<CMutexGuard<'_, T>> { let mut sguard = self.spin_lock.acquire(); if self.locked.get() { - stack_pin_init!(let wait_entry = WaitEntry::insert_new(&self.wait_list)); + stack_pin_init!(let _wait_entry = WaitEntry::insert_new(&self.wait_list)); // println!("wait list length: {}", self.wait_list.size()); while self.locked.get() { drop(sguard); @@ -100,9 +95,6 @@ impl<T> CMutex<T> { thread::park(); sguard = self.spin_lock.acquire(); } - // This does have an effect, as the ListHead inside wait_entry implements Drop! - #[expect(clippy::drop_non_drop)] - drop(wait_entry); } self.locked.set(true); unsafe { @@ -219,7 +211,7 @@ fn main() { for h in handles { h.join().expect("thread panicked"); } - println!("{:?}", &*mtx.lock()); + println!("{:?}", *mtx.lock()); assert_eq!(*mtx.lock(), workload * thread_count * 2); } } diff --git a/rust/pin-init/examples/pthread_mutex.rs b/rust/pin-init/examples/pthread_mutex.rs index 4e082ec7d5de..00f457e68827 100644 --- a/rust/pin-init/examples/pthread_mutex.rs +++ b/rust/pin-init/examples/pthread_mutex.rs @@ -3,7 +3,6 @@ // inspired by <https://github.com/nbdd0121/pin-init/blob/trunk/examples/pthread_mutex.rs> #![allow(clippy::undocumented_unsafe_blocks)] #![cfg_attr(feature = "alloc", feature(allocator_api))] -#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] #[cfg(not(windows))] mod pthread_mtx { @@ -178,7 +177,7 @@ fn main() { for h in handles { h.join().expect("thread panicked"); } - println!("{:?}", &*mtx.lock()); + println!("{:?}", *mtx.lock()); assert_eq!(*mtx.lock(), workload * thread_count * 2); } } diff --git a/rust/pin-init/examples/static_init.rs b/rust/pin-init/examples/static_init.rs index 0e165daa9798..8dd52313c1b8 100644 --- a/rust/pin-init/examples/static_init.rs +++ b/rust/pin-init/examples/static_init.rs @@ -2,7 +2,6 @@ #![allow(clippy::undocumented_unsafe_blocks)] #![cfg_attr(feature = "alloc", feature(allocator_api))] -#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] #![allow(unused_imports)] use core::{ @@ -60,7 +59,7 @@ impl<T, I: PinInit<T>> ops::Deref for StaticInit<T, I> { println!("doing init"); let ptr = self.cell.get().cast::<T>(); match self.init.take() { - Some(f) => unsafe { f.__pinned_init(ptr).unwrap() }, + Some(f) => unsafe { pin_init::raw_init(ptr, f) }, None => unsafe { core::hint::unreachable_unchecked() }, } self.present.set(true); @@ -72,13 +71,11 @@ impl<T, I: PinInit<T>> ops::Deref for StaticInit<T, I> { pub struct CountInit; unsafe impl PinInit<CMutex<usize>> for CountInit { - unsafe fn __pinned_init( - self, - slot: *mut CMutex<usize>, - ) -> Result<(), core::convert::Infallible> { + unsafe fn __init(self, slot: *mut CMutex<usize>) -> Result<(), core::convert::Infallible> { let init = CMutex::new(0); std::thread::sleep(std::time::Duration::from_millis(1000)); - unsafe { init.__pinned_init(slot) } + unsafe { pin_init::raw_init(slot, init) }; + Ok(()) } } @@ -118,7 +115,7 @@ fn main() { for h in handles { h.join().expect("thread panicked"); } - println!("{:?}, {:?}", &*mtx.lock(), &*COUNT.lock()); + println!("{:?}, {:?}", *mtx.lock(), *COUNT.lock()); assert_eq!(*mtx.lock(), workload * thread_count * 2); } } diff --git a/rust/pin-init/internal/src/diagnostics.rs b/rust/pin-init/internal/src/diagnostics.rs index 3bdb477c2f2b..c7d9b3e624fc 100644 --- a/rust/pin-init/internal/src/diagnostics.rs +++ b/rust/pin-init/internal/src/diagnostics.rs @@ -3,6 +3,7 @@ use std::fmt::Display; use proc_macro2::TokenStream; +use quote::quote_spanned; use syn::{spanned::Spanned, Error}; pub(crate) struct DiagCtxt(TokenStream); @@ -15,6 +16,19 @@ impl DiagCtxt { ErrorGuaranteed(()) } + pub(crate) fn warn(&mut self, span: impl Spanned, msg: impl Display) { + // Have the message start on a new line for visual clarity. + let msg = format!("\n{}", msg); + self.0.extend(quote_spanned!(span.span() => + // Approximate using deprecated warning while `proc_macro_diagnostic` is unstable. + const _: () = { + #[deprecated = #msg] + const fn warn() {} + warn(); + }; + )); + } + pub(crate) fn with( fun: impl FnOnce(&mut DiagCtxt) -> Result<TokenStream, ErrorGuaranteed>, ) -> TokenStream { diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index 2fe918f4d82a..fd0b5ea4a0a3 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use proc_macro2::{Span, TokenStream}; -use quote::{format_ident, quote, quote_spanned}; +use quote::{format_ident, quote}; use syn::{ braced, parse::{End, Parse}, @@ -103,17 +103,15 @@ pub(crate) fn expand( |(_, err)| Box::new(err), ); let slot = format_ident!("slot"); - let (has_data_trait, data_trait, get_data, init_from_closure) = if pinned { + let (has_data_trait, get_data, init_from_closure) = if pinned { ( format_ident!("HasPinData"), - format_ident!("PinData"), format_ident!("__pin_data"), format_ident!("pin_init_from_closure"), ) } else { ( format_ident!("HasInitData"), - format_ident!("InitData"), format_ident!("__init_data"), format_ident!("init_from_closure"), ) @@ -157,8 +155,7 @@ pub(crate) fn expand( #path::#get_data() }; // Ensure that `#data` really is of type `#data` and help with type inference: - let init = ::pin_init::__internal::#data_trait::make_closure::<_, #error>( - #data, + let init = #data.__make_closure::<_, #error>( move |slot| { #zeroable_check #this @@ -172,8 +169,7 @@ pub(crate) fn expand( init(slot).map(|__InitOk| ()) }; // SAFETY: TODO - let init = unsafe { ::pin_init::#init_from_closure::<_, #error>(init) }; - init + unsafe { ::pin_init::#init_from_closure::<_, #error>(init) } }}) } @@ -232,123 +228,84 @@ fn init_fields( cfgs.retain(|attr| attr.path().is_ident("cfg")); cfgs }; + + let ident = match kind { + InitializerKind::Value { ident, .. } => ident, + InitializerKind::Init { ident, .. } => ident, + InitializerKind::Code { block, .. } => { + let stmt = &block.stmts; + res.extend(quote! { + #(#attrs)* + { + #(#stmt)* + } + }); + continue; + } + }; + + let slot = if pinned { + quote! { + // SAFETY: + // - `slot` is valid and properly aligned. + // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned. + // - `make_field_check` prevents `#ident` from being used twice, therefore + // `(*slot).#ident` is exclusively accessed and has not been initialized. + (unsafe { #data.#ident(#slot) }) + } + } else { + quote! { + // For `init!()` macro, everything is unpinned. + // SAFETY: + // - `&raw mut (*slot).#ident` is valid. + // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned. + // - `make_field_check` prevents `#ident` from being used twice, therefore + // `(*slot).#ident` is exclusively accessed and has not been initialized. + (unsafe { + ::pin_init::__internal::Slot::<::pin_init::__internal::Unpinned, _>::new( + &raw mut (*#slot).#ident + ) + }) + } + }; + + // `mixed_site` ensures that the guard is not accessible to the user-controlled code. + let guard = format_ident!("__{ident}_guard", span = Span::mixed_site()); + let init = match kind { InitializerKind::Value { ident, value } => { - let mut value_ident = ident.clone(); - let value_prep = value.as_ref().map(|value| &value.1).map(|value| { - // Setting the span of `value_ident` to `value`'s span improves error messages - // when the type of `value` is wrong. - value_ident.set_span(value.span()); - quote!(let #value_ident = #value;) - }); - // Again span for better diagnostics - let write = quote_spanned!(ident.span()=> ::core::ptr::write); - // NOTE: the field accessor ensures that the initialized field is properly aligned. - // Unaligned fields will cause the compiler to emit E0793. We do not support - // unaligned fields since `Init::__init` requires an aligned pointer; the call to - // `ptr::write` below has the same requirement. - let accessor = if pinned { - let project_ident = format_ident!("__project_{ident}"); - quote! { - // SAFETY: TODO - unsafe { #data.#project_ident(&mut (*#slot).#ident) } - } - } else { - quote! { - // SAFETY: TODO - unsafe { &mut (*#slot).#ident } - } - }; + let value = value + .as_ref() + .map(|(_, value)| quote!(#value)) + .unwrap_or_else(|| quote!(#ident)); + quote! { #(#attrs)* - { - #value_prep - // SAFETY: TODO - unsafe { #write(::core::ptr::addr_of_mut!((*#slot).#ident), #value_ident) }; - } - #(#cfgs)* - #[allow(unused_variables)] - let #ident = #accessor; + let mut #guard = #slot.write(#value); + } } - InitializerKind::Init { ident, value, .. } => { - // Again span for better diagnostics - let init = format_ident!("init", span = value.span()); - // NOTE: the field accessor ensures that the initialized field is properly aligned. - // Unaligned fields will cause the compiler to emit E0793. We do not support - // unaligned fields since `Init::__init` requires an aligned pointer; the call to - // `ptr::write` below has the same requirement. - let (value_init, accessor) = if pinned { - let project_ident = format_ident!("__project_{ident}"); - ( - quote! { - // SAFETY: - // - `slot` is valid, because we are inside of an initializer closure, we - // return when an error/panic occurs. - // - We also use `#data` to require the correct trait (`Init` or `PinInit`) - // for `#ident`. - unsafe { #data.#ident(::core::ptr::addr_of_mut!((*#slot).#ident), #init)? }; - }, - quote! { - // SAFETY: TODO - unsafe { #data.#project_ident(&mut (*#slot).#ident) } - }, - ) - } else { - ( - quote! { - // SAFETY: `slot` is valid, because we are inside of an initializer - // closure, we return when an error/panic occurs. - unsafe { - ::pin_init::Init::__init( - #init, - ::core::ptr::addr_of_mut!((*#slot).#ident), - )? - }; - }, - quote! { - // SAFETY: TODO - unsafe { &mut (*#slot).#ident } - }, - ) - }; + InitializerKind::Init { value, .. } => { quote! { #(#attrs)* - { - let #init = #value; - #value_init - } - #(#cfgs)* - #[allow(unused_variables)] - let #ident = #accessor; + let mut #guard = #slot.init(#value)?; } } - InitializerKind::Code { block: value, .. } => quote! { - #(#attrs)* - #[allow(unused_braces)] - #value - }, + InitializerKind::Code { .. } => unreachable!(), }; - res.extend(init); - if let Some(ident) = kind.ident() { - // `mixed_site` ensures that the guard is not accessible to the user-controlled code. - let guard = format_ident!("__{ident}_guard", span = Span::mixed_site()); - res.extend(quote! { - #(#cfgs)* - // Create the drop guard: - // - // We rely on macro hygiene to make it impossible for users to access this local - // variable. - // SAFETY: We forget the guard later when initialization has succeeded. - let #guard = unsafe { - ::pin_init::__internal::DropGuard::new( - ::core::ptr::addr_of_mut!((*slot).#ident) - ) - }; - }); - guards.push(guard); - guard_attrs.push(cfgs); - } + + res.extend(quote! { + #init + + #(#cfgs)* + // Allow `non_snake_case` since the same warning is going to be reported for the struct + // field. + #[allow(unused_variables, non_snake_case)] + let #ident = #guard.let_binding(); + }); + + guards.push(guard); + guard_attrs.push(cfgs); } quote! { #res @@ -361,49 +318,49 @@ fn init_fields( } } -/// Generate the check for ensuring that every field has been initialized. +/// Generate the check for ensuring that every field has been initialized and aligned. fn make_field_check( fields: &Punctuated<InitializerField, Token![,]>, init_kind: InitKind, path: &Path, ) -> TokenStream { - let field_attrs = fields + let field_attrs: Vec<_> = fields .iter() - .filter_map(|f| f.kind.ident().map(|_| &f.attrs)); - let field_name = fields.iter().filter_map(|f| f.kind.ident()); - match init_kind { - InitKind::Normal => quote! { - // We use unreachable code to ensure that all fields have been mentioned exactly once, - // this struct initializer will still be type-checked and complain with a very natural - // error message if a field is forgotten/mentioned more than once. - #[allow(unreachable_code, clippy::diverging_sub_expression)] - // SAFETY: this code is never executed. - let _ = || unsafe { - ::core::ptr::write(slot, #path { - #( - #(#field_attrs)* - #field_name: ::core::panic!(), - )* - }) - }; - }, - InitKind::Zeroing => quote! { - // We use unreachable code to ensure that all fields have been mentioned at most once. - // Since the user specified `..Zeroable::zeroed()` at the end, all missing fields will - // be zeroed. This struct initializer will still be type-checked and complain with a - // very natural error message if a field is mentioned more than once, or doesn't exist. - #[allow(unreachable_code, clippy::diverging_sub_expression, unused_assignments)] - // SAFETY: this code is never executed. - let _ = || unsafe { - ::core::ptr::write(slot, #path { - #( - #(#field_attrs)* - #field_name: ::core::panic!(), - )* - ..::core::mem::zeroed() - }) - }; - }, + .filter_map(|f| f.kind.ident().map(|_| &f.attrs)) + .collect(); + let field_name: Vec<_> = fields.iter().filter_map(|f| f.kind.ident()).collect(); + let zeroing_trailer = match init_kind { + InitKind::Normal => None, + InitKind::Zeroing => Some(quote! { + ..::core::mem::zeroed() + }), + }; + quote! { + #[allow(unreachable_code)] + // We use unreachable code to perform field checks. They're still checked by the compiler. + // SAFETY: this code is never executed. + let _ = || unsafe { + // Create references to ensure that the initialized field is properly aligned. + // Unaligned fields will cause the compiler to emit E0793. We do not support + // unaligned fields since `Init::__init` requires an aligned pointer; the call to + // `ptr::write` for value-initialization case has the same requirement. + #( + #(#field_attrs)* + let _ = &(*slot).#field_name; + )* + + // If the zeroing trailer is not present, this checks that all fields have been + // mentioned exactly once. If the zeroing trailer is present, all missing fields will be + // zeroed, so this checks that all fields have been mentioned at most once. The use of + // struct initializer will still generate very natural error messages for any misuse. + ::core::ptr::write(slot, #path { + #( + #(#field_attrs)* + #field_name: loop {}, + )* + #zeroing_trailer + }) + }; } } diff --git a/rust/pin-init/internal/src/lib.rs b/rust/pin-init/internal/src/lib.rs index 08372c8f65f0..60d5093f3128 100644 --- a/rust/pin-init/internal/src/lib.rs +++ b/rust/pin-init/internal/src/lib.rs @@ -6,7 +6,6 @@ //! `pin-init` proc macros. -#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] // Documentation is done in the pin-init crate instead. #![allow(missing_docs)] diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 7d871236b49c..ff194d27565e 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -1,13 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use proc_macro2::TokenStream; -use quote::{format_ident, quote}; +use quote::{format_ident, quote, ToTokens}; use syn::{ parse::{End, Nothing, Parse}, parse_quote, parse_quote_spanned, spanned::Spanned, visit_mut::VisitMut, - Field, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause, + Field, Fields, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause, }; use crate::diagnostics::{DiagCtxt, ErrorGuaranteed}; @@ -35,6 +35,20 @@ impl Parse for Args { } } +impl ToTokens for Args { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Self::Nothing(_) => (), + Self::PinnedDrop(kw) => kw.to_tokens(tokens), + } + } +} + +struct FieldInfo<'a> { + field: &'a Field, + pinned: bool, +} + pub(crate) fn pin_data( args: Args, input: Item, @@ -62,6 +76,55 @@ pub(crate) fn pin_data( } }; + // Handling cfg can gets very complicated, especially for tuple structs. Therefore, resolve all + // field cfgs first before continuing. + // + // We need to perform this after parsing so we can reliably detect field cfgs. + for (field_idx, field) in struct_.fields.iter_mut().enumerate() { + let cfg: Vec<_> = field + .attrs + .iter() + .filter(|a| a.path().is_ident("cfg")) + .map(|a| { + a.parse_args::<TokenStream>() + .expect("parse as token stream cannot fail") + }) + .collect(); + + if cfg.is_empty() { + continue; + } + + field.attrs.retain(|a| !a.path().is_ident("cfg")); + let cfg_true_struct = quote!(#struct_); + + let punctuated = match &mut struct_.fields { + Fields::Named(fields) => &mut fields.named, + Fields::Unnamed(fields) => &mut fields.unnamed, + Fields::Unit => unreachable!(), + }; + *punctuated = std::mem::take(punctuated) + .into_pairs() + .enumerate() + .filter(|&(i, _)| i != field_idx) + .map(|(_, p)| p) + .collect(); + let cfg_false_struct = quote!(#struct_); + + // Resolve one field at a time until we've got no more field cfgs. + // + // This is linear time because macro invocations with false cfg will not be expanded. + return Ok(quote!( + #[cfg(all(#(#cfg,)*))] + #[::pin_init::pin_data(#args)] + #cfg_true_struct + + #[cfg(not(all(#(#cfg,)*)))] + #[::pin_init::pin_data(#args)] + #cfg_false_struct + )); + } + // The generics might contain the `Self` type. Since this macro will define a new type with the // same generics and bounds, this poses a problem: `Self` will refer to the new type as opposed // to this struct definition. Therefore we have to replace `Self` with the concrete name. @@ -73,24 +136,38 @@ pub(crate) fn pin_data( replacer.visit_generics_mut(&mut struct_.generics); replacer.visit_fields_mut(&mut struct_.fields); - let fields: Vec<(bool, &Field)> = struct_ + let fields: Vec<FieldInfo<'_>> = struct_ .fields .iter_mut() .map(|field| { let len = field.attrs.len(); field.attrs.retain(|a| !a.path().is_ident("pin")); - (len != field.attrs.len(), &*field) + let pinned_count = len - field.attrs.len(); + if pinned_count > 1 { + dcx.error(&field, "#[pin] attribute specified more than once"); + } + + assert!( + !field.attrs.iter().any(|a| a.path().is_ident("cfg")), + "cfgs should be all resolved at this point" + ); + + FieldInfo { + field: &*field, + pinned: pinned_count != 0, + } }) .collect(); - for (pinned, field) in &fields { - if !pinned && is_phantom_pinned(&field.ty) { - dcx.error( - field, + for field in &fields { + let ident = field.field.ident.as_ref().unwrap(); + + if !field.pinned && is_phantom_pinned(&field.field.ty) { + dcx.warn( + field.field, format!( - "The field `{}` of type `PhantomPinned` only has an effect \ + "The field `{ident}` of type `PhantomPinned` only has an effect \ if it has the `#[pin]` attribute", - field.ident.as_ref().unwrap(), ), ); } @@ -143,7 +220,7 @@ fn is_phantom_pinned(ty: &Type) -> bool { fn generate_unpin_impl( ident: &Ident, generics: &Generics, - fields: &[(bool, &Field)], + fields: &[FieldInfo<'_>], ) -> TokenStream { let (_, ty_generics, _) = generics.split_for_impl(); let mut generics_with_pin_lt = generics.clone(); @@ -160,19 +237,26 @@ fn generate_unpin_impl( else { unreachable!() }; - let pinned_fields = fields.iter().filter_map(|(b, f)| b.then_some(f)); + let pinned_fields = fields.iter().filter(|f| f.pinned).map(|f| { + let ident = f.field.ident.as_ref().unwrap(); + let ty = &f.field.ty; + quote!( + #ident: #ty + ) + }); quote! { // This struct will be used for the unpin analysis. It is needed, because only structurally // pinned fields are relevant whether the struct should implement `Unpin`. - #[allow(dead_code)] // The fields below are never used. + #[allow( + dead_code, // The fields below are never used. + non_snake_case // The warning will be emitted on the struct definition. + )] struct __Unpin #generics_with_pin_lt #where_token #predicates { - __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, - __phantom: ::core::marker::PhantomData< - fn(#ident #ty_generics) -> #ident #ty_generics - >, + __phantom_pin: ::pin_init::__internal::PhantomInvariantLifetime<'__pin>, + __phantom: ::pin_init::__internal::PhantomInvariant<#ident #ty_generics>, #(#pinned_fields),* } @@ -214,7 +298,6 @@ fn generate_drop_impl(ident: &Ident, generics: &Generics, args: Args) -> TokenSt // `Drop`. Additionally we will implement this trait for the struct leading to a conflict, // if it also implements `Drop` trait MustNotImplDrop {} - #[expect(drop_bounds)] impl<T: ::core::ops::Drop + ?::core::marker::Sized> MustNotImplDrop for T {} impl #impl_generics MustNotImplDrop for #ident #ty_generics #whr @@ -222,7 +305,6 @@ fn generate_drop_impl(ident: &Ident, generics: &Generics, args: Args) -> TokenSt // We also take care to prevent users from writing a useless `PinnedDrop` implementation. // They might implement `PinnedDrop` correctly for the struct, but forget to give // `PinnedDrop` as the parameter to `#[pin_data]`. - #[expect(non_camel_case_types)] trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} impl<T: ::pin_init::PinnedDrop + ?::core::marker::Sized> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} @@ -238,7 +320,7 @@ fn generate_projections( vis: &Visibility, ident: &Ident, generics: &Generics, - fields: &[(bool, &Field)], + fields: &[FieldInfo<'_>], ) -> TokenStream { let (impl_generics, ty_generics, _) = generics.split_for_impl(); let mut generics_with_pin_lt = generics.clone(); @@ -247,32 +329,20 @@ fn generate_projections( let projection = format_ident!("{ident}Projection"); let this = format_ident!("this"); - let (fields_decl, fields_proj) = collect_tuple(fields.iter().map( - |( - pinned, - Field { - vis, - ident, - ty, - attrs, - .. - }, - )| { - let mut attrs = attrs.clone(); - attrs.retain(|a| !a.path().is_ident("pin")); - let mut no_doc_attrs = attrs.clone(); - no_doc_attrs.retain(|a| !a.path().is_ident("doc")); + let (fields_decl, fields_proj): (Vec<_>, Vec<_>) = fields + .iter() + .map(|field| { + let Field { vis, ident, ty, .. } = &field.field; + let ident = ident .as_ref() .expect("only structs with named fields are supported"); - if *pinned { + if field.pinned { ( quote!( - #(#attrs)* #vis #ident: ::core::pin::Pin<&'__pin mut #ty>, ), quote!( - #(#no_doc_attrs)* // SAFETY: this field is structurally pinned. #ident: unsafe { ::core::pin::Pin::new_unchecked(&mut #this.#ident) }, ), @@ -280,31 +350,33 @@ fn generate_projections( } else { ( quote!( - #(#attrs)* #vis #ident: &'__pin mut #ty, ), quote!( - #(#no_doc_attrs)* #ident: &mut #this.#ident, ), ) } - }, - )); + }) + .collect(); let structurally_pinned_fields_docs = fields .iter() - .filter_map(|(pinned, field)| pinned.then_some(field)) - .map(|Field { ident, .. }| format!(" - `{}`", ident.as_ref().unwrap())); + .filter(|f| f.pinned) + .map(|f| format!(" - `{}`", f.field.ident.as_ref().unwrap())); let not_structurally_pinned_fields_docs = fields .iter() - .filter_map(|(pinned, field)| (!pinned).then_some(field)) - .map(|Field { ident, .. }| format!(" - `{}`", ident.as_ref().unwrap())); + .filter(|f| !f.pinned) + .map(|f| format!(" - `{}`", f.field.ident.as_ref().unwrap())); let docs = format!(" Pin-projections of [`{ident}`]"); quote! { #[doc = #docs] - #[allow(dead_code)] + // Allow `non_snake_case` since the same warning will be emitted on + // the struct definition. + #[allow(dead_code, non_snake_case)] #[doc(hidden)] - #vis struct #projection #generics_with_pin_lt { + #vis struct #projection #generics_with_pin_lt + #whr + { #(#fields_decl)* ___pin_phantom_data: ::core::marker::PhantomData<&'__pin mut ()>, } @@ -336,91 +408,52 @@ fn generate_projections( fn generate_the_pin_data( vis: &Visibility, - ident: &Ident, + struct_name: &Ident, generics: &Generics, - fields: &[(bool, &Field)], + fields: &[FieldInfo<'_>], ) -> TokenStream { let (impl_generics, ty_generics, whr) = generics.split_for_impl(); // For every field, we create an initializing projection function according to its projection - // type. If a field is structurally pinned, then it must be initialized via `PinInit`, if it is - // not structurally pinned, then it can be initialized via `Init`. - // - // The functions are `unsafe` to prevent accidentally calling them. - fn handle_field( - Field { - vis, - ident, - ty, - attrs, - .. - }: &Field, - struct_ident: &Ident, - pinned: bool, - ) -> TokenStream { - let mut attrs = attrs.clone(); - attrs.retain(|a| !a.path().is_ident("pin")); - let ident = ident - .as_ref() - .expect("only structs with named fields are supported"); - let project_ident = format_ident!("__project_{ident}"); - let (init_ty, init_fn, project_ty, project_body, pin_safety) = if pinned { - ( - quote!(PinInit), - quote!(__pinned_init), - quote!(::core::pin::Pin<&'__slot mut #ty>), - // SAFETY: this field is structurally pinned. - quote!(unsafe { ::core::pin::Pin::new_unchecked(slot) }), - quote!( - /// - `slot` will not move until it is dropped, i.e. it will be pinned. - ), - ) - } else { - ( - quote!(Init), - quote!(__init), - quote!(&'__slot mut #ty), - quote!(slot), - quote!(), - ) - }; - let slot_safety = format!( - " `slot` points at the field `{ident}` inside of `{struct_ident}`, which is pinned.", - ); - quote! { - /// # Safety - /// - /// - `slot` is a valid pointer to uninitialized memory. - /// - the caller does not touch `slot` when `Err` is returned, they are only permitted - /// to deallocate. - #pin_safety - #(#attrs)* - #vis unsafe fn #ident<E>( - self, - slot: *mut #ty, - init: impl ::pin_init::#init_ty<#ty, E>, - ) -> ::core::result::Result<(), E> { - // SAFETY: this function has the same safety requirements as the __init function - // called below. - unsafe { ::pin_init::#init_ty::#init_fn(init, slot) } - } - - /// # Safety - /// - #[doc = #slot_safety] - #(#attrs)* - #vis unsafe fn #project_ident<'__slot>( - self, - slot: &'__slot mut #ty, - ) -> #project_ty { - #project_body - } - } - } - + // type. If a field is structurally pinned, we create a `Slot` with `Pinned` which must be + // initialized via `PinInit`; if it is not structurally pinned, then we create a `Slot` with + // `Unpinned` which allows initialization via `Init`. let field_accessors = fields .iter() - .map(|(pinned, field)| handle_field(field, ident, *pinned)) + .map(|f| { + let Field { vis, ident, ty, .. } = f.field; + + let field_name = ident + .as_ref() + .expect("only structs with named fields are supported"); + let pin_marker = if f.pinned { + quote!(Pinned) + } else { + quote!(Unpinned) + }; + quote! { + /// # Safety + /// + /// - `slot` is valid and properly aligned. + /// - `(*slot).#field_name` is properly aligned. + /// - `(*slot).#field_name` points to uninitialized and exclusively accessed + /// memory. + // Allow `non_snake_case` since the same warning will be emitted on + // the struct definition. + #[allow(non_snake_case)] + #[inline(always)] + #vis unsafe fn #field_name( + self, + slot: *mut #struct_name #ty_generics, + ) -> ::pin_init::__internal::Slot<::pin_init::__internal::#pin_marker, #ty> { + // SAFETY: + // - If `#pin_marker` is `Pinned`, the corresponding field is structurally + // pinned. + // - Other safety requirements follows the safety requirement. + unsafe { ::pin_init::__internal::Slot::new(&raw mut (*slot).#field_name) } + } + } + }) .collect::<TokenStream>(); quote! { // We declare this struct which will host all of the projection function for our type. It @@ -429,14 +462,13 @@ fn generate_the_pin_data( #vis struct __ThePinData #generics #whr { - __phantom: ::core::marker::PhantomData< - fn(#ident #ty_generics) -> #ident #ty_generics - >, + __phantom: ::pin_init::__internal::PhantomInvariant<#struct_name #ty_generics>, } impl #impl_generics ::core::clone::Clone for __ThePinData #ty_generics #whr { + #[inline] fn clone(&self) -> Self { *self } } @@ -445,31 +477,34 @@ fn generate_the_pin_data( {} #[allow(dead_code)] // Some functions might never be used and private. - #[expect(clippy::missing_safety_doc)] impl #impl_generics __ThePinData #ty_generics #whr { + /// Type inference helper function. + #[inline(always)] + #vis fn __make_closure<__F, __E>(self, f: __F) -> __F + where + __F: FnOnce(*mut #struct_name #ty_generics) -> + ::core::result::Result<::pin_init::__internal::InitOk, __E>, + { + f + } + #field_accessors } // SAFETY: We have added the correct projection functions above to `__ThePinData` and // we also use the least restrictive generics possible. - unsafe impl #impl_generics ::pin_init::__internal::HasPinData for #ident #ty_generics + unsafe impl #impl_generics ::pin_init::__internal::HasPinData for #struct_name #ty_generics #whr { type PinData = __ThePinData #ty_generics; + #[inline] unsafe fn __pin_data() -> Self::PinData { - __ThePinData { __phantom: ::core::marker::PhantomData } + __ThePinData { __phantom: ::pin_init::__internal::PhantomInvariant::new() } } } - - // SAFETY: TODO - unsafe impl #impl_generics ::pin_init::__internal::PinData for __ThePinData #ty_generics - #whr - { - type Datee = #ident #ty_generics; - } } } @@ -500,14 +535,3 @@ impl VisitMut for SelfReplacer { // Do not descend into items, since items reset/change what `Self` refers to. } } - -// replace with `.collect()` once MSRV is above 1.79 -fn collect_tuple<A, B>(iter: impl Iterator<Item = (A, B)>) -> (Vec<A>, Vec<B>) { - let mut res_a = vec![]; - let mut res_b = vec![]; - for (a, b) in iter { - res_a.push(a); - res_b.push(b); - } - (res_a, res_b) -} diff --git a/rust/pin-init/internal/src/zeroable.rs b/rust/pin-init/internal/src/zeroable.rs index 05683319b0f7..b11feaeb1ca6 100644 --- a/rust/pin-init/internal/src/zeroable.rs +++ b/rust/pin-init/internal/src/zeroable.rs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: Apache-2.0 OR MIT use proc_macro2::TokenStream; use quote::quote; diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index 90adbdc1893b..8e9fd18b993f 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -7,42 +7,54 @@ use super::*; -/// See the [nomicon] for what subtyping is. See also [this table]. +/// Zero-sized type used to mark a type as invariant. +/// +/// This is a polyfill for the [unstable type] in the standard library of the same name. /// -/// The reason for not using `PhantomData<*mut T>` is that that type never implements [`Send`] and -/// [`Sync`]. Hence `fn(*mut T) -> *mut T` is used, as that type always implements them. +/// See the [nomicon] for what subtyping is. See also [this table]. /// +/// [unstable type]: https://doc.rust-lang.org/nightly/std/marker/struct.PhantomInvariant.html /// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html /// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns -pub(crate) type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>; +#[repr(transparent)] +pub struct PhantomInvariant<T: ?Sized>(PhantomData<fn(T) -> T>); -/// Module-internal type implementing `PinInit` and `Init`. -/// -/// It is unsafe to create this type, since the closure needs to fulfill the same safety -/// requirement as the `__pinned_init`/`__init` functions. -pub(crate) struct InitClosure<F, T: ?Sized, E>(pub(crate) F, pub(crate) Invariant<(E, T)>); - -// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the -// `__init` invariants. -unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T, E> -where - F: FnOnce(*mut T) -> Result<(), E>, -{ - #[inline] - unsafe fn __init(self, slot: *mut T) -> Result<(), E> { - (self.0)(slot) +impl<T: ?Sized> Clone for PhantomInvariant<T> { + #[inline(always)] + fn clone(&self) -> Self { + *self } } -// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the -// `__pinned_init` invariants. -unsafe impl<T: ?Sized, F, E> PinInit<T, E> for InitClosure<F, T, E> -where - F: FnOnce(*mut T) -> Result<(), E>, -{ - #[inline] - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { - (self.0)(slot) +impl<T: ?Sized> Copy for PhantomInvariant<T> {} + +impl<T: ?Sized> Default for PhantomInvariant<T> { + #[inline(always)] + fn default() -> Self { + Self::new() + } +} + +impl<T: ?Sized> PhantomInvariant<T> { + #[inline(always)] + pub const fn new() -> Self { + Self(PhantomData) + } +} + +/// Zero-sized type used to mark a lifetime as invariant. +/// +/// This is a polyfill for the [unstable type] in the standard library of the same name. +/// +/// [unstable type]: https://doc.rust-lang.org/nightly/std/marker/struct.PhantomInvariantLifetime.html +#[repr(transparent)] +#[derive(Clone, Copy, Default)] +pub struct PhantomInvariantLifetime<'a>(PhantomInvariant<&'a ()>); + +impl PhantomInvariantLifetime<'_> { + #[inline(always)] + pub const fn new() -> Self { + Self(PhantomInvariant::new()) } } @@ -71,30 +83,12 @@ impl InitOk { /// /// Only the `init` module is allowed to use this trait. pub unsafe trait HasPinData { - type PinData: PinData; + type PinData; #[expect(clippy::missing_safety_doc)] unsafe fn __pin_data() -> Self::PinData; } -/// Marker trait for pinning data of structs. -/// -/// # Safety -/// -/// Only the `init` module is allowed to use this trait. -pub unsafe trait PinData: Copy { - type Datee: ?Sized + HasPinData; - - /// Type inference helper function. - #[inline(always)] - fn make_closure<F, E>(self, f: F) -> F - where - F: FnOnce(*mut Self::Datee) -> Result<InitOk, E>, - { - f - } -} - /// This trait is automatically implemented for every type. It aims to provide the same type /// inference help as `HasPinData`. /// @@ -102,33 +96,16 @@ pub unsafe trait PinData: Copy { /// /// Only the `init` module is allowed to use this trait. pub unsafe trait HasInitData { - type InitData: InitData; + type InitData; #[expect(clippy::missing_safety_doc)] unsafe fn __init_data() -> Self::InitData; } -/// Same function as `PinData`, but for arbitrary data. -/// -/// # Safety -/// -/// Only the `init` module is allowed to use this trait. -pub unsafe trait InitData: Copy { - type Datee: ?Sized + HasInitData; - - /// Type inference helper function. - #[inline(always)] - fn make_closure<F, E>(self, f: F) -> F - where - F: FnOnce(*mut Self::Datee) -> Result<InitOk, E>, - { - f - } -} - -pub struct AllData<T: ?Sized>(Invariant<T>); +pub struct AllData<T: ?Sized>(PhantomInvariant<T>); impl<T: ?Sized> Clone for AllData<T> { + #[inline] fn clone(&self) -> Self { *self } @@ -136,17 +113,24 @@ impl<T: ?Sized> Clone for AllData<T> { impl<T: ?Sized> Copy for AllData<T> {} -// SAFETY: TODO. -unsafe impl<T: ?Sized> InitData for AllData<T> { - type Datee = T; +impl<T: ?Sized> AllData<T> { + /// Type inference helper function. + #[inline(always)] + pub fn __make_closure<F, E>(self, f: F) -> F + where + F: FnOnce(*mut T) -> Result<InitOk, E>, + { + f + } } // SAFETY: TODO. unsafe impl<T: ?Sized> HasInitData for T { type InitData = AllData<T>; + #[inline] unsafe fn __init_data() -> Self::InitData { - AllData(PhantomData) + AllData(PhantomInvariant::new()) } } @@ -199,7 +183,7 @@ impl<T> StackInit<T> { unsafe { this.value.assume_init_drop() }; } // SAFETY: The memory slot is valid and this type ensures that it will stay pinned. - unsafe { init.__pinned_init(this.value.as_mut_ptr())? }; + unsafe { init.__init(this.value.as_mut_ptr())? }; // INVARIANT: `this.value` is initialized above. this.is_init = true; // SAFETY: The slot is now pinned, since we will never give access to `&mut T`. @@ -235,35 +219,144 @@ fn stack_init_reuse() { println!("{value:?}"); } +// Marker types that determines type of `DropGuard`'s let bindings. +pub struct Pinned; +pub struct Unpinned; + +/// Represent an uninitialized field. +/// +/// # Invariants +/// +/// - `ptr` is valid, properly aligned and points to uninitialized and exclusively accessed memory. +/// - If `P` is `Pinned`, then `ptr` is structurally pinned. +pub struct Slot<P, T: ?Sized> { + ptr: *mut T, + _phantom: PhantomData<P>, +} + +impl<P, T: ?Sized> Slot<P, T> { + /// # Safety + /// + /// - `ptr` is valid, properly aligned and points to uninitialized and exclusively accessed + /// memory. + /// - If `P` is `Pinned`, then `ptr` is structurally pinned. + #[inline(always)] + pub unsafe fn new(ptr: *mut T) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr, + _phantom: PhantomData, + } + } + + /// Initialize the field by value. + #[inline(always)] + pub fn write(self, value: T) -> DropGuard<P, T> + where + T: Sized, + { + // SAFETY: `self.ptr` is a valid and aligned pointer for write. + unsafe { self.ptr.write(value) } + // SAFETY: + // - `self.ptr` is valid and properly aligned per type invariant. + // - `*self.ptr` is initialized above and the ownership is transferred to the guard. + // - If `P` is `Pinned`, `self.ptr` is pinned. + unsafe { DropGuard::new(self.ptr) } + } +} + +impl<T: ?Sized> Slot<Unpinned, T> { + /// Initialize the field. + #[inline(always)] + pub fn init<E>(self, init: impl Init<T, E>) -> Result<DropGuard<Unpinned, T>, E> { + // SAFETY: + // - `self.ptr` is valid and properly aligned. + // - when `Err` is returned, we also propagate the error without touching `slot`; + // also `self` is consumed so it cannot be touched further. + unsafe { init.__init(self.ptr)? }; + + // SAFETY: + // - `self.ptr` is valid and properly aligned per type invariant. + // - `*self.ptr` is initialized above and the ownership is transferred to the guard. + Ok(unsafe { DropGuard::new(self.ptr) }) + } +} + +impl<T: ?Sized> Slot<Pinned, T> { + /// Initialize the field. + #[inline(always)] + pub fn init<E>(self, init: impl PinInit<T, E>) -> Result<DropGuard<Pinned, T>, E> { + // SAFETY: + // - `self.ptr` is valid and properly aligned. + // - when `Err` is returned, we also propagate the error without touching `ptr`; + // also `self` is consumed so it cannot be touched further. + // - the drop guard will not hand out `&mut` (only `Pin<&mut T>`). + unsafe { init.__init(self.ptr)? }; + + // SAFETY: + // - `self.ptr` is valid, properly aligned and pinned per type invariant. + // - `*self.ptr` is initialized above and the ownership is transferred to the guard. + Ok(unsafe { DropGuard::new(self.ptr) }) + } +} + /// When a value of this type is dropped, it drops a `T`. /// /// Can be forgotten to prevent the drop. -pub struct DropGuard<T: ?Sized> { +/// +/// # Invariants +/// +/// - `ptr` is valid and properly aligned. +/// - `*ptr` is initialized and owned by this guard. +/// - if `P` is `Pinned`, `ptr` is pinned. +pub struct DropGuard<P, T: ?Sized> { ptr: *mut T, + phantom: PhantomData<P>, } -impl<T: ?Sized> DropGuard<T> { - /// Creates a new [`DropGuard<T>`]. It will [`ptr::drop_in_place`] `ptr` when it gets dropped. +impl<P, T: ?Sized> DropGuard<P, T> { + /// Creates a drop guard and transfer the ownership of the pointer content. /// - /// # Safety + /// The ownership is only relinguished if the guard is forgotten via [`core::mem::forget`]. /// - /// `ptr` must be a valid pointer. + /// # Safety /// - /// It is the callers responsibility that `self` will only get dropped if the pointee of `ptr`: - /// - has not been dropped, - /// - is not accessible by any other means, - /// - will not be dropped by any other means. + /// - `ptr` is valid and properly aligned. + /// - `*ptr` is initialized, and the ownership is transferred to this guard. + /// - if `P` is `Pinned`, `ptr` is pinned. #[inline] pub unsafe fn new(ptr: *mut T) -> Self { - Self { ptr } + // INVARIANT: By safety requirement. + Self { + ptr, + phantom: PhantomData, + } } } -impl<T: ?Sized> Drop for DropGuard<T> { +impl<T: ?Sized> DropGuard<Unpinned, T> { + /// Create a let binding for accessor use. + #[inline] + pub fn let_binding(&mut self) -> &mut T { + // SAFETY: Per type invariant. + unsafe { &mut *self.ptr } + } +} + +impl<T: ?Sized> DropGuard<Pinned, T> { + /// Create a let binding for accessor use. + #[inline] + pub fn let_binding(&mut self) -> Pin<&mut T> { + // SAFETY: `self.ptr` is valid, properly aligned, initialized, exclusively accessible and + // pinned per type invariant. + unsafe { Pin::new_unchecked(&mut *self.ptr) } + } +} + +impl<P, T: ?Sized> Drop for DropGuard<P, T> { #[inline] fn drop(&mut self) { - // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function - // ensuring that this operation is safe. + // SAFETY: `self.ptr` is valid, properly aligned and `*self.ptr` is owned by this guard. unsafe { ptr::drop_in_place(self.ptr) } } } @@ -294,20 +387,23 @@ pub struct AlwaysFail<T: ?Sized> { impl<T: ?Sized> AlwaysFail<T> { /// Creates a new initializer that always fails. + #[inline] pub fn new() -> Self { Self { _t: PhantomData } } } impl<T: ?Sized> Default for AlwaysFail<T> { + #[inline] fn default() -> Self { Self::new() } } -// SAFETY: `__pinned_init` always fails, which is always okay. +// SAFETY: `__init` always fails, which is always okay. unsafe impl<T: ?Sized> PinInit<T, ()> for AlwaysFail<T> { - unsafe fn __pinned_init(self, _slot: *mut T) -> Result<(), ()> { + #[inline] + unsafe fn __init(self, _slot: *mut T) -> Result<(), ()> { Err(()) } } diff --git a/rust/pin-init/src/alloc.rs b/rust/pin-init/src/alloc.rs index 5017f57442d8..471652e8663a 100644 --- a/rust/pin-init/src/alloc.rs +++ b/rust/pin-init/src/alloc.rs @@ -35,10 +35,11 @@ pub trait InPlaceInit<T>: Sized { /// type. /// /// If `T: !Unpin` it will not be able to move afterwards. + #[inline] fn pin_init(init: impl PinInit<T>) -> Result<Pin<Self>, AllocError> { // SAFETY: We delegate to `init` and only change the error type. let init = unsafe { - pin_init_from_closure(|slot| match init.__pinned_init(slot) { + pin_init_from_closure(|slot| match init.__init(slot) { Ok(()) => Ok(()), Err(i) => match i {}, }) @@ -52,6 +53,7 @@ pub trait InPlaceInit<T>: Sized { E: From<AllocError>; /// Use the given initializer to in-place initialize a `T`. + #[inline] fn init(init: impl Init<T>) -> Result<Self, AllocError> { // SAFETY: We delegate to `init` and only change the error type. let init = unsafe { @@ -109,7 +111,7 @@ impl<T> InPlaceInit<T> for Arc<T> { let slot = slot.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; + unsafe { init.__init(slot)? }; // SAFETY: All fields have been initialized and this is the only `Arc` to that data. Ok(unsafe { Pin::new_unchecked(this.assume_init()) }) } @@ -136,6 +138,7 @@ impl<T> InPlaceInit<T> for Arc<T> { impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> { type Initialized = Box<T>; + #[inline] fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, @@ -145,11 +148,12 @@ impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> { Ok(unsafe { self.assume_init() }) } + #[inline] fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; + unsafe { init.__init(slot)? }; // SAFETY: All fields have been initialized. Ok(unsafe { self.assume_init() }.into()) } diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index fe4c85ae3f02..f1463be9479d 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -70,7 +70,6 @@ //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. //! //! ```rust -//! # #![expect(clippy::disallowed_names)] //! # #![feature(allocator_api)] //! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; //! # use core::pin::Pin; @@ -94,7 +93,6 @@ //! (or just the stack) to actually initialize a `Foo`: //! //! ```rust -//! # #![expect(clippy::disallowed_names)] //! # #![feature(allocator_api)] //! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; //! # use core::{alloc::AllocError, pin::Pin}; @@ -172,7 +170,6 @@ //! # #![feature(extern_types)] //! use pin_init::{pin_data, pinned_drop, PinInit, PinnedDrop, pin_init_from_closure}; //! use core::{ -//! ptr::addr_of_mut, //! marker::PhantomPinned, //! cell::UnsafeCell, //! pin::Pin, @@ -211,7 +208,7 @@ //! unsafe { //! pin_init_from_closure(move |slot: *mut Self| { //! // `slot` contains uninit memory, avoid creating a reference. -//! let foo = addr_of_mut!((*slot).foo); +//! let foo = &raw mut (*slot).foo; //! let foo = UnsafeCell::raw_get(foo).cast::<bindings::foo>(); //! //! // Initialize the `foo` @@ -264,14 +261,6 @@ //! [`impl Init<T, E>`]: crate::Init //! [Rust-for-Linux]: https://rust-for-linux.com/ -#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] -#![cfg_attr( - all( - any(feature = "alloc", feature = "std"), - not(RUSTC_NEW_UNINIT_IS_STABLE) - ), - feature(new_uninit) -)] #![forbid(missing_docs, unsafe_op_in_unsafe_fn)] #![cfg_attr(not(feature = "std"), no_std)] #![cfg_attr(feature = "alloc", feature(allocator_api))] @@ -279,6 +268,8 @@ all(feature = "unsafe-pinned", CONFIG_RUSTC_HAS_UNSAFE_PINNED), feature(unsafe_pinned) )] +#![cfg_attr(all(USE_RUSTC_FEATURES, doc), allow(internal_features))] +#![cfg_attr(all(USE_RUSTC_FEATURES, doc), feature(rustdoc_internals))] use core::{ cell::UnsafeCell, @@ -438,7 +429,7 @@ pub use ::pin_init_internal::Zeroable; /// ``` /// use pin_init::MaybeZeroable; /// -/// // implmements `Zeroable` +/// // implements `Zeroable` /// #[derive(MaybeZeroable)] /// pub struct DriverData { /// pub(crate) id: i64, @@ -446,7 +437,7 @@ pub use ::pin_init_internal::Zeroable; /// len: usize, /// } /// -/// // does not implmement `Zeroable` +/// // does not implement `Zeroable` /// #[derive(MaybeZeroable)] /// pub struct DriverData2 { /// pub(crate) id: i64, @@ -463,7 +454,6 @@ pub use ::pin_init_internal::MaybeZeroable; /// # Examples /// /// ```rust -/// # #![expect(clippy::disallowed_names)] /// # #![feature(allocator_api)] /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; /// # use pin_init::*; @@ -500,13 +490,7 @@ macro_rules! stack_pin_init { (let $var:ident $(: $t:ty)? = $val:expr) => { let val = $val; let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit()); - let mut $var = match $crate::__internal::StackInit::init($var, val) { - Ok(res) => res, - Err(x) => { - let x: ::core::convert::Infallible = x; - match x {} - } - }; + let Ok(mut $var) = $crate::__internal::StackInit::init($var, val); }; } @@ -515,7 +499,6 @@ macro_rules! stack_pin_init { /// # Examples /// /// ```rust -/// # #![expect(clippy::disallowed_names)] /// # #![feature(allocator_api)] /// # #[path = "../examples/error.rs"] mod error; use error::Error; /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; @@ -542,7 +525,6 @@ macro_rules! stack_pin_init { /// ``` /// /// ```rust -/// # #![expect(clippy::disallowed_names)] /// # #![feature(allocator_api)] /// # #[path = "../examples/error.rs"] mod error; use error::Error; /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; @@ -665,7 +647,6 @@ macro_rules! stack_try_pin_init { /// Users of `Foo` can now create it like this: /// /// ```rust -/// # #![expect(clippy::disallowed_names)] /// # use pin_init::*; /// # use core::pin::Pin; /// # #[pin_data] @@ -755,7 +736,7 @@ macro_rules! stack_try_pin_init { /// /// ```rust /// # use pin_init::*; -/// # use core::{ptr::addr_of_mut, marker::PhantomPinned}; +/// # use core::marker::PhantomPinned; /// #[pin_data] /// #[derive(Zeroable)] /// struct Buf { @@ -769,7 +750,7 @@ macro_rules! stack_try_pin_init { /// let init = pin_init!(&this in Buf { /// buf: [0; 64], /// // SAFETY: TODO. -/// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() }, +/// ptr: unsafe { (&raw mut (*this.as_ptr()).buf).cast() }, /// pin: PhantomPinned, /// }); /// let init = pin_init!(Buf { @@ -874,12 +855,12 @@ pub use pin_init_internal::init; #[macro_export] macro_rules! assert_pinned { ($ty:ty, $field:ident, $field_ty:ty, inline) => { - let _ = move |ptr: *mut $field_ty| { - // SAFETY: This code is unreachable. - let data = unsafe { <$ty as $crate::__internal::HasPinData>::__pin_data() }; - let init = $crate::__internal::AlwaysFail::<$field_ty>::new(); - // SAFETY: This code is unreachable. - unsafe { data.$field(ptr, init) }.ok(); + // SAFETY: This code is unreachable. + let _ = move |ptr: *mut $ty| unsafe { + let data = <$ty as $crate::__internal::HasPinData>::__pin_data(); + _ = data + .$field(ptr) + .init($crate::__internal::AlwaysFail::<$field_ty>::new()); }; }; @@ -902,7 +883,7 @@ macro_rules! assert_pinned { /// When implementing this trait you will need to take great care. Also there are probably very few /// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. /// -/// The [`PinInit::__pinned_init`] function: +/// The [`PinInit::__init`] function: /// - returns `Ok(())` if it initialized every field of `slot`, /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: /// - `slot` can be deallocated without UB occurring, @@ -922,15 +903,33 @@ macro_rules! assert_pinned { #[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")] #[must_use = "An initializer must be used in order to create its value."] pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { + /// Alias of [`PinInit::__init`]. + /// + /// New code should use `__init` instead. + /// + /// # Safety + /// + /// Same as `__init`. + #[inline(always)] + #[cfg(not(kernel))] + #[deprecated = "use `raw_try_init` instead"] + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: Per safety requirement. + unsafe { self.__init(slot) } + } + /// Initializes `slot`. /// + /// It is not recommended to call this directly. Use [`raw_init`] or [`raw_try_init`]. + /// /// # Safety /// /// - `slot` is a valid pointer to uninitialized memory. /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to /// deallocate. /// - `slot` will not move until it is dropped, i.e. it will be pinned. - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>; + /// If `Self: Init<T, E>`, this requirement is cancelled and it may be moved. + unsafe fn __init(self, slot: *mut T) -> Result<(), E>; /// First initializes the value using `self` then calls the function `f` with the initialized /// value. @@ -950,18 +949,47 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { /// Ok(()) /// }); /// ``` + #[inline] fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E> where F: FnOnce(Pin<&mut T>) -> Result<(), E>, { - ChainPinInit(self, f, PhantomData) + ChainPinInit(self, f, __internal::PhantomInvariant::new()) } } +/// Initializes `slot` with an initializer. +/// +/// # Safety +/// +/// - `slot` is a valid pointer to uninitialized memory. +/// - `slot` will not move until it is dropped, i.e. it will be pinned. +/// If `init` implements `Init<T, E>`, this requirement is cancelled and it may be moved. +#[inline(always)] +pub unsafe fn raw_init<T>(slot: *mut T, init: impl PinInit<T>) { + // SAFETY: Per safety requirement. + unsafe { init.__init(slot).unwrap_or_else(|e| match e {}) } +} + +/// Fallibly initializes `slot` with an initializer. +/// +/// # Safety +/// +/// - `slot` is a valid pointer to uninitialized memory. +/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to +/// deallocate. +/// - `slot` will not move until it is dropped, i.e. it will be pinned. +/// If `init` implements `Init<T, E>`, this requirement is cancelled and it may be moved. +#[inline(always)] +pub unsafe fn raw_try_init<T, E>(slot: *mut T, init: impl PinInit<T, E>) -> Result<(), E> { + // SAFETY: Per safety requirement. + unsafe { init.__init(slot) } +} + /// An initializer returned by [`PinInit::pin_chain`]. -pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, T)>); +pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::PhantomInvariant<(E, T)>); -// SAFETY: The `__pinned_init` function is implemented such that it +// SAFETY: The `__init` function is implemented such that it // - returns `Ok(())` on successful initialization, // - returns `Err(err)` on error and in this case `slot` will be dropped. // - considers `slot` pinned. @@ -970,15 +998,14 @@ where I: PinInit<T, E>, F: FnOnce(Pin<&mut T>) -> Result<(), E>, { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: All requirements fulfilled since this function is `__pinned_init`. - unsafe { self.0.__pinned_init(slot)? }; - // SAFETY: The above call initialized `slot` and we still have unique access. - let val = unsafe { &mut *slot }; - // SAFETY: `slot` is considered pinned. - let val = unsafe { Pin::new_unchecked(val) }; - // SAFETY: `slot` was initialized above. - (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) }) + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__init`. + let slot = unsafe { __internal::Slot::<__internal::Pinned, _>::new(slot) }; + let mut guard = slot.init(self.0)?; + (self.1)(guard.let_binding())?; + core::mem::forget(guard); + Ok(()) } } @@ -995,19 +1022,8 @@ where /// When implementing this trait you will need to take great care. Also there are probably very few /// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. /// -/// The [`Init::__init`] function: -/// - returns `Ok(())` if it initialized every field of `slot`, -/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: -/// - `slot` can be deallocated without UB occurring, -/// - `slot` does not need to be dropped, -/// - `slot` is not partially initialized. -/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. -/// -/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same -/// code as `__init`. -/// -/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to -/// move the pointee after initialization. +/// The [`PinInit::__init`] function must work without the pinning requirement; the caller is +/// allowed to move the pointee after initialization. /// #[cfg_attr( kernel, @@ -1021,15 +1037,6 @@ where #[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")] #[must_use = "An initializer must be used in order to create its value."] pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { - /// Initializes `slot`. - /// - /// # Safety - /// - /// - `slot` is a valid pointer to uninitialized memory. - /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to - /// deallocate. - unsafe fn __init(self, slot: *mut T) -> Result<(), E>; - /// First initializes the value using `self` then calls the function `f` with the initialized /// value. /// @@ -1038,7 +1045,6 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { /// # Examples /// /// ```rust - /// # #![expect(clippy::disallowed_names)] /// use pin_init::{init, init_zeroed, Init}; /// /// struct Foo { @@ -1058,44 +1064,68 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { /// Ok(()) /// }); /// ``` + #[inline] fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E> where F: FnOnce(&mut T) -> Result<(), E>, { - ChainInit(self, f, PhantomData) + ChainInit(self, f, __internal::PhantomInvariant::new()) } } /// An initializer returned by [`Init::chain`]. -pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, T)>); +pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::PhantomInvariant<(E, T)>); + +// SAFETY: The `__init` function does not rely on the pinning requirement. +unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E> +where + I: Init<T, E>, + F: FnOnce(&mut T) -> Result<(), E>, +{ +} // SAFETY: The `__init` function is implemented such that it // - returns `Ok(())` on successful initialization, // - returns `Err(err)` on error and in this case `slot` will be dropped. -unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E> +unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E> where I: Init<T, E>, F: FnOnce(&mut T) -> Result<(), E>, { + #[inline] unsafe fn __init(self, slot: *mut T) -> Result<(), E> { // SAFETY: All requirements fulfilled since this function is `__init`. - unsafe { self.0.__pinned_init(slot)? }; - // SAFETY: The above call initialized `slot` and we still have unique access. - (self.1)(unsafe { &mut *slot }).inspect_err(|_| - // SAFETY: `slot` was initialized above. - unsafe { core::ptr::drop_in_place(slot) }) + let slot = unsafe { __internal::Slot::<__internal::Unpinned, _>::new(slot) }; + let mut guard = slot.init(self.0)?; + (self.1)(guard.let_binding())?; + core::mem::forget(guard); + Ok(()) } } -// SAFETY: `__pinned_init` behaves exactly the same as `__init`. -unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E> +/// Implement `PinInit` and `Init` for closures. +/// +/// It is unsafe to create this type, since the closure needs to fulfill the same safety +/// requirement as the `__init` functions. +struct InitClosure<F, T: ?Sized>(F, __internal::PhantomInvariant<T>); + +// SAFETY: When constructing via `init_from_closure`, the `__init` function does not rely on the +// pinning requirement. When constructing via `pin_init_from_closure`, the opaque type prevents this +// implementation from being visible. +unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T> where + F: FnOnce(*mut T) -> Result<(), E> +{ +} + +// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the +// `__init` invariants. +unsafe impl<T: ?Sized, F, E> PinInit<T, E> for InitClosure<F, T> where - I: Init<T, E>, - F: FnOnce(&mut T) -> Result<(), E>, + F: FnOnce(*mut T) -> Result<(), E>, { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: `__init` has less strict requirements compared to `__pinned_init`. - unsafe { self.__init(slot) } + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + (self.0)(slot) } } @@ -1115,7 +1145,7 @@ where pub const unsafe fn pin_init_from_closure<T: ?Sized, E>( f: impl FnOnce(*mut T) -> Result<(), E>, ) -> impl PinInit<T, E> { - __internal::InitClosure(f, PhantomData) + InitClosure(f, __internal::PhantomInvariant::new()) } /// Creates a new [`Init<T, E>`] from the given closure. @@ -1134,7 +1164,7 @@ pub const unsafe fn pin_init_from_closure<T: ?Sized, E>( pub const unsafe fn init_from_closure<T: ?Sized, E>( f: impl FnOnce(*mut T) -> Result<(), E>, ) -> impl Init<T, E> { - __internal::InitClosure(f, PhantomData) + InitClosure(f, __internal::PhantomInvariant::new()) } /// Changes the to be initialized type. @@ -1143,14 +1173,11 @@ pub const unsafe fn init_from_closure<T: ?Sized, E>( /// /// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a /// pointer must result in a valid `U`. +#[inline] pub const unsafe fn cast_pin_init<T, U, E>(init: impl PinInit<T, E>) -> impl PinInit<U, E> { // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety // requirements. - let res = unsafe { pin_init_from_closure(|ptr: *mut U| init.__pinned_init(ptr.cast::<T>())) }; - // FIXME: remove the let statement once the nightly-MSRV allows it (1.78 otherwise encounters a - // cycle when computing the type returned by this function) - #[allow(clippy::let_and_return)] - res + unsafe { pin_init_from_closure(|ptr: *mut U| init.__init(ptr.cast::<T>())) } } /// Changes the to be initialized type. @@ -1159,14 +1186,11 @@ pub const unsafe fn cast_pin_init<T, U, E>(init: impl PinInit<T, E>) -> impl Pin /// /// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a /// pointer must result in a valid `U`. +#[inline] pub const unsafe fn cast_init<T, U, E>(init: impl Init<T, E>) -> impl Init<U, E> { // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety // requirements. - let res = unsafe { init_from_closure(|ptr: *mut U| init.__init(ptr.cast::<T>())) }; - // FIXME: remove the let statement once the nightly-MSRV allows it (1.78 otherwise encounters a - // cycle when computing the type returned by this function) - #[allow(clippy::let_and_return)] - res + unsafe { init_from_closure(|ptr: *mut U| init.__init(ptr.cast::<T>())) } } /// An initializer that leaves the memory uninitialized. @@ -1178,6 +1202,77 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { unsafe { init_from_closure(|_| Ok(())) } } +/// Array initializer from element initializer. +struct ArrayInit<T: ?Sized, F>(F, __internal::PhantomInvariant<T>); + +// SAFETY: On success, all `N` elements of the array have been initialized. On error or panic, the +// elements that have been initialized so far are dropped, thus leaving the array uninitialized and +// ready to deallocate. +unsafe impl<T, F, I, E, const N: usize> PinInit<[T; N], E> for ArrayInit<T, F> +where + F: FnMut(usize) -> I, + I: PinInit<T, E>, +{ + unsafe fn __init(mut self, slot: *mut [T; N]) -> Result<(), E> { + /// # Invariants + /// + /// - `ptr[..num_init]` contains initialized elements of type `T` + /// - `ptr[num_init..N]` (where N is the size of the array) contains uninitialized memory + struct ArrayInitGuard<T> { + /// A pointer to the first element of the array. + ptr: *mut T, + /// The number of initialized elements in the array. + num_init: usize, + } + + impl<T> Drop for ArrayInitGuard<T> { + #[inline] + fn drop(&mut self) { + // SAFETY: Per type invariant, `self.ptr[..self.num_init]` are initialized. + unsafe { + core::ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut( + self.ptr, + self.num_init, + )) + }; + } + } + + // INVARIANT: nothing is initialized yet. + let mut guard = ArrayInitGuard { + ptr: slot.cast::<T>(), + num_init: 0, + }; + + for i in 0..N { + // INVARIANT: Elements `self.ptr[..self.num_init]` have been initialized + // thus far. This holds true for every `self.num_init = i`. + guard.num_init = i; + + let init = (self.0)(i); + // SAFETY: + // - The subslot is derived from `slot` with a valid offset. + // - If `Err` is touched, the subslot is not touched further, the guard will drop + // previously initialized elements only. + // - `slot` is pinned so is the subslot. + unsafe { init.__init(&raw mut (*slot)[i]) }?; + } + + // Dismiss the drop guard now that all elements are initialized. + core::mem::forget(guard); + Ok(()) + } +} + +// SAFETY: `I: Init` cancels out the pinning requirement on subslots, which is the only place in the +// `__init` function that relies on `slot` being pinned. +unsafe impl<T, F, I, E, const N: usize> Init<[T; N], E> for ArrayInit<T, F> +where + F: FnMut(usize) -> I, + I: Init<T, E>, +{ +} + /// Initializes an array by initializing each element via the provided initializer. /// /// # Examples @@ -1188,32 +1283,14 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { /// let array: Box<[usize; 1_000]> = Box::init(init_array_from_fn(|i| i)).unwrap(); /// assert_eq!(array.len(), 1_000); /// ``` +#[inline] pub fn init_array_from_fn<I, const N: usize, T, E>( - mut make_init: impl FnMut(usize) -> I, + make_init: impl FnMut(usize) -> I, ) -> impl Init<[T; N], E> where I: Init<T, E>, { - let init = move |slot: *mut [T; N]| { - let slot = slot.cast::<T>(); - for i in 0..N { - let init = make_init(i); - // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. - let ptr = unsafe { slot.add(i) }; - // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` - // requirements. - if let Err(e) = unsafe { init.__init(ptr) } { - // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return - // `Err` below, `slot` will be considered uninitialized memory. - unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; - return Err(e); - } - } - Ok(()) - }; - // SAFETY: The initializer above initializes every element of the array. On failure it drops - // any initialized elements and returns `Err`. - unsafe { init_from_closure(init) } + ArrayInit(make_init, __internal::PhantomInvariant::new()) } /// Initializes an array by initializing each element via the provided initializer. @@ -1231,32 +1308,14 @@ where /// Arc::pin_init(pin_init_array_from_fn(|i| CMutex::new(i))).unwrap(); /// assert_eq!(array.len(), 1_000); /// ``` +#[inline] pub fn pin_init_array_from_fn<I, const N: usize, T, E>( - mut make_init: impl FnMut(usize) -> I, + make_init: impl FnMut(usize) -> I, ) -> impl PinInit<[T; N], E> where I: PinInit<T, E>, { - let init = move |slot: *mut [T; N]| { - let slot = slot.cast::<T>(); - for i in 0..N { - let init = make_init(i); - // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. - let ptr = unsafe { slot.add(i) }; - // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` - // requirements. - if let Err(e) = unsafe { init.__pinned_init(ptr) } { - // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return - // `Err` below, `slot` will be considered uninitialized memory. - unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; - return Err(e); - } - } - Ok(()) - }; - // SAFETY: The initializer above initializes every element of the array. On failure it drops - // any initialized elements and returns `Err`. - unsafe { pin_init_from_closure(init) } + ArrayInit(make_init, __internal::PhantomInvariant::new()) } /// Construct an initializer in a closure and run it. @@ -1285,6 +1344,7 @@ where /// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the /// initializer itself will fail with that error. If it returned `Ok`, then it will run the /// initializer returned by the [`pin_init!`] invocation. +#[inline] pub fn pin_init_scope<T, E, F, I>(make_init: F) -> impl PinInit<T, E> where F: FnOnce() -> Result<I, E>, @@ -1292,13 +1352,13 @@ where { // SAFETY: // - If `make_init` returns `Err`, `Err` is returned and `slot` is completely uninitialized, - // - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__pinned_init`. - // - The safety requirements of `init.__pinned_init` are fulfilled, since it's being called - // from an initializer. + // - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__init`. + // - The safety requirements of `init.__init` are fulfilled, since it's being called from an + // initializer. unsafe { pin_init_from_closure(move |slot: *mut T| -> Result<(), E> { let init = make_init()?; - init.__pinned_init(slot) + init.__init(slot) }) } } @@ -1328,6 +1388,7 @@ where /// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the /// initializer itself will fail with that error. If it returned `Ok`, then it will run the /// initializer returned by the [`init!`] invocation. +#[inline] pub fn init_scope<T, E, F, I>(make_init: F) -> impl Init<T, E> where F: FnOnce() -> Result<I, E>, @@ -1346,41 +1407,29 @@ where } } -// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of `slot`. -unsafe impl<T> Init<T> for T { - unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> { - // SAFETY: `slot` is valid for writes by the safety requirements of this function. - unsafe { slot.write(self) }; - Ok(()) - } -} +// SAFETY: The `__init` function does not rely on slot being pinned after it returns. +unsafe impl<T> Init<T> for T {} -// SAFETY: the `__pinned_init` function always returns `Ok(())` and initializes every field of +// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of // `slot`. Additionally, all pinning invariants of `T` are upheld. unsafe impl<T> PinInit<T> for T { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), Infallible> { + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> { // SAFETY: `slot` is valid for writes by the safety requirements of this function. unsafe { slot.write(self) }; Ok(()) } } -// SAFETY: when the `__init` function returns with -// - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld. -// - `Err(err)`, slot was not written to. -unsafe impl<T, E> Init<T, E> for Result<T, E> { - unsafe fn __init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: `slot` is valid for writes by the safety requirements of this function. - unsafe { slot.write(self?) }; - Ok(()) - } -} +// SAFETY: The `__init` function does not rely on slot being pinned after it returns. +unsafe impl<T, E> Init<T, E> for Result<T, E> {} -// SAFETY: when the `__pinned_init` function returns with +// SAFETY: when the `__init` function returns with // - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld. // - `Err(err)`, slot was not written to. unsafe impl<T, E> PinInit<T, E> for Result<T, E> { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { // SAFETY: `slot` is valid for writes by the safety requirements of this function. unsafe { slot.write(self?) }; Ok(()) @@ -1406,6 +1455,7 @@ pub trait InPlaceWrite<T> { impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> { type Initialized = &'static mut T; + #[inline] fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { let slot = self.as_mut_ptr(); @@ -1416,6 +1466,7 @@ impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> { unsafe { Ok(self.assume_init_mut()) } } + #[inline] fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { let slot = self.as_mut_ptr(); @@ -1423,7 +1474,7 @@ impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> { // // The `'static` borrow guarantees the data will not be // moved/invalidated until it gets dropped (which is never). - unsafe { init.__pinned_init(slot)? }; + unsafe { init.__init(slot)? }; // SAFETY: The above call initialized the memory. Ok(Pin::static_mut(unsafe { self.assume_init_mut() })) @@ -1495,10 +1546,13 @@ pub unsafe trait Zeroable { /// Whenever a type implements [`Zeroable`], this function should be preferred over /// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`. /// + /// As const traits are not yet stable, [`pin_init::zeroed()`] can be used instead + /// when initialization is required in a `const` context. + /// /// # Examples /// /// ``` - /// use pin_init::{Zeroable, zeroed}; + /// use pin_init::Zeroable; /// /// #[derive(Zeroable)] /// struct Point { @@ -1506,10 +1560,11 @@ pub unsafe trait Zeroable { /// y: u32, /// } /// - /// let point: Point = zeroed(); + /// let point: Point = Zeroable::zeroed(); /// assert_eq!(point.x, 0); /// assert_eq!(point.y, 0); /// ``` + #[inline] fn zeroed() -> Self where Self: Sized, @@ -1518,27 +1573,6 @@ pub unsafe trait Zeroable { } } -/// Marker trait for types that allow `Option<Self>` to be set to all zeroes in order to write -/// `None` to that location. -/// -/// # Safety -/// -/// The implementer needs to ensure that `unsafe impl Zeroable for Option<Self> {}` is sound. -pub unsafe trait ZeroableOption {} - -// SAFETY: by the safety requirement of `ZeroableOption`, this is valid. -unsafe impl<T: ZeroableOption> Zeroable for Option<T> {} - -// SAFETY: `Option<&T>` is part of the option layout optimization guarantee: -// <https://doc.rust-lang.org/stable/std/option/index.html#representation>. -unsafe impl<T> ZeroableOption for &T {} -// SAFETY: `Option<&mut T>` is part of the option layout optimization guarantee: -// <https://doc.rust-lang.org/stable/std/option/index.html#representation>. -unsafe impl<T> ZeroableOption for &mut T {} -// SAFETY: `Option<NonNull<T>>` is part of the option layout optimization guarantee: -// <https://doc.rust-lang.org/stable/std/option/index.html#representation>. -unsafe impl<T> ZeroableOption for NonNull<T> {} - /// Create an initializer for a zeroed `T`. /// /// The returned initializer will write `0x00` to every byte of the given `slot`. @@ -1559,6 +1593,9 @@ pub fn init_zeroed<T: Zeroable>() -> impl Init<T> { /// Whenever a type implements [`Zeroable`], this function should be preferred over /// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`. /// +/// While const traits remain unstable, this function serves as the `const` version of +/// [`Zeroable::zeroed()`]. +/// /// # Examples /// /// ``` @@ -1574,6 +1611,7 @@ pub fn init_zeroed<T: Zeroable>() -> impl Init<T> { /// assert_eq!(point.x, 0); /// assert_eq!(point.y, 0); /// ``` +#[inline] pub const fn zeroed<T: Zeroable>() -> T { // SAFETY:By the type invariants of `Zeroable`, all zeroes is a valid bit pattern for `T`. unsafe { core::mem::zeroed() } @@ -1610,13 +1648,6 @@ impl_zeroable! { // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`. {<T: ?Sized + Zeroable>} UnsafeCell<T>, - // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee: - // <https://doc.rust-lang.org/stable/std/option/index.html#representation>). - Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>, - Option<NonZeroU128>, Option<NonZeroUsize>, - Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>, - Option<NonZeroI128>, Option<NonZeroIsize>, - // SAFETY: `null` pointer is valid. // // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be @@ -1635,8 +1666,14 @@ impl_zeroable! { } macro_rules! impl_tuple_zeroable { - ($(,)?) => {}; + ($first:ident, $(,)?) => { + #[cfg_attr(all(USE_RUSTC_FEATURES, doc), doc(fake_variadic))] + /// Implemented for tuples up to 10 items long. + // SAFETY: All elements are zeroable and padding can be zero. + unsafe impl<$first: Zeroable> Zeroable for ($first,) {} + }; ($first:ident, $($t:ident),* $(,)?) => { + #[cfg_attr(doc, doc(hidden))] // SAFETY: All elements are zeroable and padding can be zero. unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {} impl_tuple_zeroable!($($t),* ,); @@ -1645,13 +1682,33 @@ macro_rules! impl_tuple_zeroable { impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J); +/// Marker trait for types that allow `Option<Self>` to be set to all zeroes in order to write +/// `None` to that location. +/// +/// # Safety +/// +/// The implementer needs to ensure that `unsafe impl Zeroable for Option<Self> {}` is sound. +pub unsafe trait ZeroableOption {} + +// SAFETY: by the safety requirement of `ZeroableOption`, this is valid. +unsafe impl<T: ZeroableOption> Zeroable for Option<T> {} + macro_rules! impl_fn_zeroable_option { ([$($abi:literal),* $(,)?] $args:tt) => { $(impl_fn_zeroable_option!({extern $abi} $args);)* $(impl_fn_zeroable_option!({unsafe extern $abi} $args);)* }; ({$($prefix:tt)*} {$(,)?}) => {}; + ({$($prefix:tt)*} {$ret:ident, $arg:ident $(,)?}) => { + #[cfg_attr(all(USE_RUSTC_FEATURES, doc), doc(fake_variadic))] + /// Implemented for function pointers with up to 20 arity. + // SAFETY: function pointers are part of the option layout optimization: + // <https://doc.rust-lang.org/stable/std/option/index.html#representation>. + unsafe impl<$ret, $arg> ZeroableOption for $($prefix)* fn($arg) -> $ret {} + impl_fn_zeroable_option!({$($prefix)*} {$arg,}); + }; ({$($prefix:tt)*} {$ret:ident, $($rest:ident),* $(,)?}) => { + #[cfg_attr(doc, doc(hidden))] // SAFETY: function pointers are part of the option layout optimization: // <https://doc.rust-lang.org/stable/std/option/index.html#representation>. unsafe impl<$ret, $($rest),*> ZeroableOption for $($prefix)* fn($($rest),*) -> $ret {} @@ -1661,6 +1718,29 @@ macro_rules! impl_fn_zeroable_option { impl_fn_zeroable_option!(["Rust", "C"] { A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U }); +macro_rules! impl_zeroable_option { + ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. + $(unsafe impl$($($generics)*)? ZeroableOption for $t {})* + }; +} + +impl_zeroable_option! { + // SAFETY: `Option<&T>` is part of the option layout optimization guarantee: + // <https://doc.rust-lang.org/stable/std/option/index.html#representation>. + {<T: ?Sized>} &T, + // SAFETY: `Option<&mut T>` is part of the option layout optimization guarantee: + // <https://doc.rust-lang.org/stable/std/option/index.html#representation>. + {<T: ?Sized>} &mut T, + // SAFETY: `Option<NonNull<T>>` is part of the option layout optimization guarantee: + // <https://doc.rust-lang.org/stable/std/option/index.html#representation>. + {<T: ?Sized>} NonNull<T>, + // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee: + // <https://doc.rust-lang.org/stable/std/option/index.html#representation>). + NonZero<u8>, NonZero<u16>, NonZero<u32>, NonZero<u64>, NonZero<u128>, NonZero<usize>, + NonZero<i8>, NonZero<i16>, NonZero<i32>, NonZero<i64>, NonZero<i128>, NonZero<isize>, +} + /// This trait allows creating an instance of `Self` which contains exactly one /// [structurally pinned value](https://doc.rust-lang.org/std/pin/index.html#projections-and-structural-pinning). /// @@ -1692,6 +1772,7 @@ pub trait Wrapper<T> { } impl<T> Wrapper<T> for UnsafeCell<T> { + #[inline] fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> { // SAFETY: `UnsafeCell<T>` has a compatible layout to `T`. unsafe { cast_pin_init(value_init) } @@ -1699,6 +1780,7 @@ impl<T> Wrapper<T> for UnsafeCell<T> { } impl<T> Wrapper<T> for MaybeUninit<T> { + #[inline] fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> { // SAFETY: `MaybeUninit<T>` has a compatible layout to `T`. unsafe { cast_pin_init(value_init) } @@ -1707,6 +1789,7 @@ impl<T> Wrapper<T> for MaybeUninit<T> { #[cfg(all(feature = "unsafe-pinned", CONFIG_RUSTC_HAS_UNSAFE_PINNED))] impl<T> Wrapper<T> for core::pin::UnsafePinned<T> { + #[inline] fn pin_init<E>(init: impl PinInit<T, E>) -> impl PinInit<Self, E> { // SAFETY: `UnsafePinned<T>` has a compatible layout to `T`. unsafe { cast_pin_init(init) } diff --git a/rust/uapi/lib.rs b/rust/uapi/lib.rs index 1d5fd9efb93e..003e6d4f7c4b 100644 --- a/rust/uapi/lib.rs +++ b/rust/uapi/lib.rs @@ -8,12 +8,9 @@ //! userspace APIs. #![no_std] -// See <https://github.com/rust-lang/rust-bindgen/issues/1651>. -#![cfg_attr(test, allow(deref_nullptr))] -#![cfg_attr(test, allow(unaligned_references))] -#![cfg_attr(test, allow(unsafe_op_in_unsafe_fn))] #![allow( clippy::all, + clippy::as_underscore, clippy::cast_lossless, clippy::ptr_as_ptr, clippy::ref_as_ptr, @@ -27,7 +24,13 @@ unreachable_pub, unsafe_op_in_unsafe_fn )] -#![cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))] +#![cfg_attr(not(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES), allow(unknown_lints))] +#![allow(unnecessary_transmutes)] +#![cfg_attr( + CONFIG_RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS, + allow(suspicious_runtime_symbol_definitions) +)] +#![feature(cfi_encoding)] // Manual definition of blocklisted types. type __kernel_size_t = usize; diff --git a/rust/uapi/uapi_helper.h b/rust/uapi/uapi_helper.h index 06d7d1a2e8da..489748ef642c 100644 --- a/rust/uapi/uapi_helper.h +++ b/rust/uapi/uapi_helper.h @@ -6,11 +6,12 @@ * Sorted alphabetically. */ -#include <uapi/asm-generic/ioctl.h> #include <uapi/drm/drm.h> #include <uapi/drm/nova_drm.h> #include <uapi/drm/panthor_drm.h> #include <uapi/linux/android/binder.h> +#include <uapi/linux/android/binder_netlink.h> +#include <uapi/linux/ioctl.h> #include <uapi/linux/mdio.h> #include <uapi/linux/mii.h> #include <uapi/linux/ethtool.h> diff --git a/rust/zerocopy-derive/README.md b/rust/zerocopy-derive/README.md new file mode 100644 index 000000000000..d62c79804342 --- /dev/null +++ b/rust/zerocopy-derive/README.md @@ -0,0 +1,14 @@ +# `zerocopy-derive` + +These source files come from the Rust `zerocopy-derive` crate, version v0.8.54 +(released 2026-07-08), hosted in the <https://github.com/google/zerocopy> +repository, licensed under "BSD-2-Clause OR Apache-2.0 OR MIT" and only +modified to tweak the SPDX license identifiers and to remove the generation of +non-ASCII identifiers. + +For copyright details, please see: + + https://github.com/google/zerocopy/blob/v0.8.54/README.md?plain=1 + https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-BSD + https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-APACHE + https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-MIT diff --git a/rust/zerocopy-derive/derive/from_bytes.rs b/rust/zerocopy-derive/derive/from_bytes.rs new file mode 100644 index 000000000000..66d820f6ad4c --- /dev/null +++ b/rust/zerocopy-derive/derive/from_bytes.rs @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +use proc_macro2::{Span, TokenStream}; +use syn::{ + parse_quote, Data, DataEnum, DataStruct, DataUnion, Error, Expr, ExprLit, ExprUnary, Lit, UnOp, + WherePredicate, +}; + +use crate::{ + derive::try_from_bytes::derive_try_from_bytes, + repr::{CompoundRepr, EnumRepr, Repr, Spanned}, + util::{enum_size_from_repr, Ctx, FieldBounds, ImplBlockBuilder, Trait, TraitBound}, +}; +/// Returns `Ok(index)` if variant `index` of the enum has a discriminant of +/// zero. If `Err(bool)` is returned, the boolean is true if the enum has +/// unknown discriminants (e.g. discriminants set to const expressions which we +/// can't evaluate in a proc macro). If the enum has unknown discriminants, then +/// it might have a zero variant that we just can't detect. +pub(crate) fn find_zero_variant(enm: &DataEnum) -> Result<usize, bool> { + // Discriminants can be anywhere in the range [i128::MIN, u128::MAX] because + // the discriminant type may be signed or unsigned. Since we only care about + // tracking the discriminant when it's less than or equal to zero, we can + // avoid u128 -> i128 conversions and bounds checking by making the "next + // discriminant" value implicitly negative. + // Technically 64 bits is enough, but 128 is better for future compatibility + // with https://github.com/rust-lang/rust/issues/56071 + let mut next_negative_discriminant = Some(0); + + // Sometimes we encounter explicit discriminants that we can't know the + // value of (e.g. a constant expression that requires evaluation). These + // could evaluate to zero or a negative number, but we can't assume that + // they do (no false positives allowed!). So we treat them like strictly- + // positive values that can't result in any zero variants, and track whether + // we've encountered any unknown discriminants. + let mut has_unknown_discriminants = false; + + for (i, v) in enm.variants.iter().enumerate() { + match v.discriminant.as_ref() { + // Implicit discriminant + None => { + match next_negative_discriminant.as_mut() { + Some(0) => return Ok(i), + // n is nonzero so subtraction is always safe + Some(n) => *n -= 1, + None => (), + } + } + // Explicit positive discriminant + Some((_, Expr::Lit(ExprLit { lit: Lit::Int(int), .. }))) => { + match int.base10_parse::<u128>().ok() { + Some(0) => return Ok(i), + Some(_) => next_negative_discriminant = None, + None => { + // Numbers should never fail to parse, but just in case: + has_unknown_discriminants = true; + next_negative_discriminant = None; + } + } + } + // Explicit negative discriminant + Some((_, Expr::Unary(ExprUnary { op: UnOp::Neg(_), expr, .. }))) => match &**expr { + Expr::Lit(ExprLit { lit: Lit::Int(int), .. }) => { + match int.base10_parse::<u128>().ok() { + Some(0) => return Ok(i), + // x is nonzero so subtraction is always safe + Some(x) => next_negative_discriminant = Some(x - 1), + None => { + // Numbers should never fail to parse, but just in + // case: + has_unknown_discriminants = true; + next_negative_discriminant = None; + } + } + } + // Unknown negative discriminant (e.g. const repr) + _ => { + has_unknown_discriminants = true; + next_negative_discriminant = None; + } + }, + // Unknown discriminant (e.g. const expr) + _ => { + has_unknown_discriminants = true; + next_negative_discriminant = None; + } + } + } + + Err(has_unknown_discriminants) +} +pub(crate) fn derive_from_zeros(ctx: &Ctx, top_level: Trait) -> Result<TokenStream, Error> { + let try_from_bytes = derive_try_from_bytes(ctx, top_level)?; + let from_zeros = match &ctx.ast.data { + Data::Struct(strct) => derive_from_zeros_struct(ctx, strct), + Data::Enum(enm) => derive_from_zeros_enum(ctx, enm)?, + Data::Union(unn) => derive_from_zeros_union(ctx, unn), + }; + Ok(IntoIterator::into_iter([try_from_bytes, from_zeros]).collect()) +} +pub(crate) fn derive_from_bytes(ctx: &Ctx, top_level: Trait) -> Result<TokenStream, Error> { + let from_zeros = derive_from_zeros(ctx, top_level)?; + let from_bytes = match &ctx.ast.data { + Data::Struct(strct) => derive_from_bytes_struct(ctx, strct), + Data::Enum(enm) => derive_from_bytes_enum(ctx, enm)?, + Data::Union(unn) => derive_from_bytes_union(ctx, unn), + }; + + Ok(IntoIterator::into_iter([from_zeros, from_bytes]).collect()) +} +fn derive_from_zeros_struct(ctx: &Ctx, strct: &DataStruct) -> TokenStream { + ImplBlockBuilder::new(ctx, strct, Trait::FromZeros, FieldBounds::ALL_SELF).build() +} +fn derive_from_zeros_enum(ctx: &Ctx, enm: &DataEnum) -> Result<TokenStream, Error> { + let repr = EnumRepr::from_attrs(&ctx.ast.attrs)?; + + // We don't actually care what the repr is; we just care that it's one of + // the allowed ones. + match repr { + Repr::Compound(Spanned { t: CompoundRepr::C | CompoundRepr::Primitive(_), span: _ }, _) => { + } + Repr::Transparent(_) | Repr::Compound(Spanned { t: CompoundRepr::Rust, span: _ }, _) => { + return ctx.error_or_skip( + Error::new( + Span::call_site(), + "must have #[repr(C)] or #[repr(Int)] attribute in order to guarantee this type's memory layout", + ), + ); + } + } + + let zero_variant = match find_zero_variant(enm) { + Ok(index) => enm.variants.iter().nth(index).unwrap(), + // Has unknown variants + Err(true) => { + return ctx.error_or_skip(Error::new_spanned( + &ctx.ast, + "FromZeros only supported on enums with a variant that has a discriminant of `0`\n\ + help: This enum has discriminants which are not literal integers. One of those may \ + define or imply which variant has a discriminant of zero. Use a literal integer to \ + define or imply the variant with a discriminant of zero.", + )); + } + // Does not have unknown variants + Err(false) => { + return ctx.error_or_skip(Error::new_spanned( + &ctx.ast, + "FromZeros only supported on enums with a variant that has a discriminant of `0`", + )); + } + }; + + let zerocopy_crate = &ctx.zerocopy_crate; + let explicit_bounds = zero_variant + .fields + .iter() + .map(|field| { + let ty = &field.ty; + parse_quote! { #ty: #zerocopy_crate::FromZeros } + }) + .collect::<Vec<WherePredicate>>(); + + Ok(ImplBlockBuilder::new(ctx, enm, Trait::FromZeros, FieldBounds::Explicit(explicit_bounds)) + .build()) +} +fn derive_from_zeros_union(ctx: &Ctx, unn: &DataUnion) -> TokenStream { + let field_type_trait_bounds = FieldBounds::All(&[TraitBound::Slf]); + ImplBlockBuilder::new(ctx, unn, Trait::FromZeros, field_type_trait_bounds).build() +} +fn derive_from_bytes_struct(ctx: &Ctx, strct: &DataStruct) -> TokenStream { + ImplBlockBuilder::new(ctx, strct, Trait::FromBytes, FieldBounds::ALL_SELF).build() +} +fn derive_from_bytes_enum(ctx: &Ctx, enm: &DataEnum) -> Result<TokenStream, Error> { + let repr = EnumRepr::from_attrs(&ctx.ast.attrs)?; + + let variants_required = 1usize << enum_size_from_repr(&repr)?; + if enm.variants.len() != variants_required { + return ctx.error_or_skip(Error::new_spanned( + &ctx.ast, + format!( + "FromBytes only supported on {} enum with {} variants", + repr.repr_type_name(), + variants_required + ), + )); + } + + Ok(ImplBlockBuilder::new(ctx, enm, Trait::FromBytes, FieldBounds::ALL_SELF).build()) +} +fn derive_from_bytes_union(ctx: &Ctx, unn: &DataUnion) -> TokenStream { + let field_type_trait_bounds = FieldBounds::All(&[TraitBound::Slf]); + ImplBlockBuilder::new(ctx, unn, Trait::FromBytes, field_type_trait_bounds).build() +} diff --git a/rust/zerocopy-derive/derive/into_bytes.rs b/rust/zerocopy-derive/derive/into_bytes.rs new file mode 100644 index 000000000000..0103a78d087f --- /dev/null +++ b/rust/zerocopy-derive/derive/into_bytes.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +use proc_macro2::{Span, TokenStream}; +use quote::quote; +use syn::{Data, DataEnum, DataStruct, DataUnion, Error, Type}; + +use crate::{ + repr::{EnumRepr, StructUnionRepr}, + util::{ + generate_tag_enum, Ctx, DataExt, FieldBounds, ImplBlockBuilder, PaddingCheck, Trait, + TraitBound, + }, +}; +pub(crate) fn derive_into_bytes(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> { + match &ctx.ast.data { + Data::Struct(strct) => derive_into_bytes_struct(ctx, strct), + Data::Enum(enm) => derive_into_bytes_enum(ctx, enm), + Data::Union(unn) => derive_into_bytes_union(ctx, unn), + } +} +fn derive_into_bytes_struct(ctx: &Ctx, strct: &DataStruct) -> Result<TokenStream, Error> { + let repr = StructUnionRepr::from_attrs(&ctx.ast.attrs)?; + + let is_transparent = repr.is_transparent(); + let is_c = repr.is_c(); + let is_packed_1 = repr.is_packed_1(); + let num_fields = strct.fields().len(); + + let (padding_check, require_unaligned_fields) = if is_transparent || is_packed_1 { + // No padding check needed. + // - repr(transparent): The layout and ABI of the whole struct is the + // same as its only non-ZST field (meaning there's no padding outside + // of that field) and we require that field to be `IntoBytes` (meaning + // there's no padding in that field). + // - repr(packed): Any inter-field padding bytes are removed, meaning + // that any padding bytes would need to come from the fields, all of + // which we require to be `IntoBytes` (meaning they don't have any + // padding). Note that this holds regardless of other `repr` + // attributes, including `repr(Rust)`. [1] + // + // [1] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#the-alignment-modifiers: + // + // An important consequence of these rules is that a type with + // `#[repr(packed(1))]`` (or `#[repr(packed)]``) will have no + // inter-field padding. + (None, false) + } else if is_c && !repr.is_align_gt_1() && num_fields <= 1 { + // No padding check needed. A repr(C) struct with zero or one field has + // no padding unless #[repr(align)] explicitly adds padding, which we + // check for in this branch's condition. + (None, false) + } else if ctx.ast.generics.params.is_empty() { + // Is the last field a syntactic slice, i.e., `[SomeType]`. + let is_syntactic_dst = + strct.fields().last().map(|(_, _, ty)| matches!(ty, Type::Slice(_))).unwrap_or(false); + // Since there are no generics, we can emit a padding check. All reprs + // guarantee that fields won't overlap [1], so the padding check is + // sound. This is more permissive than the next case, which requires + // that all field types implement `Unaligned`. + // + // [1] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#the-rust-representation: + // + // The only data layout guarantees made by [`repr(Rust)`] are those + // required for soundness. They are: + // ... + // 2. The fields do not overlap. + // ... + if is_c && is_syntactic_dst { + (Some(PaddingCheck::ReprCStruct), false) + } else { + (Some(PaddingCheck::Struct), false) + } + } else if is_c && !repr.is_align_gt_1() { + // We can't use a padding check since there are generic type arguments. + // Instead, we require all field types to implement `Unaligned`. This + // ensures that the `repr(C)` layout algorithm will not insert any + // padding unless #[repr(align)] explicitly adds padding, which we check + // for in this branch's condition. + // + // FIXME(#10): Support type parameters for non-transparent, non-packed + // structs without requiring `Unaligned`. + (None, true) + } else { + return ctx.error_or_skip(Error::new( + Span::call_site(), + "must have a non-align #[repr(...)] attribute in order to guarantee this type's memory layout", + )); + }; + + let field_bounds = if require_unaligned_fields { + FieldBounds::All(&[TraitBound::Slf, TraitBound::Other(Trait::Unaligned)]) + } else { + FieldBounds::ALL_SELF + }; + + Ok(ImplBlockBuilder::new(ctx, strct, Trait::IntoBytes, field_bounds) + .padding_check(padding_check) + .build()) +} + +fn derive_into_bytes_enum(ctx: &Ctx, enm: &DataEnum) -> Result<TokenStream, Error> { + let repr = EnumRepr::from_attrs(&ctx.ast.attrs)?; + if !repr.is_c() && !repr.is_primitive() { + return ctx.error_or_skip(Error::new( + Span::call_site(), + "must have #[repr(C)] or #[repr(Int)] attribute in order to guarantee this type's memory layout", + )); + } + + let tag_type_definition = generate_tag_enum(ctx, &repr, enm); + Ok(ImplBlockBuilder::new(ctx, enm, Trait::IntoBytes, FieldBounds::ALL_SELF) + .padding_check(PaddingCheck::Enum { tag_type_definition }) + .build()) +} + +fn derive_into_bytes_union(ctx: &Ctx, unn: &DataUnion) -> Result<TokenStream, Error> { + // See #1792 for more context. + // + // By checking for `zerocopy_derive_union_into_bytes` both here and in the + // generated code, we ensure that `--cfg zerocopy_derive_union_into_bytes` + // need only be passed *either* when compiling this crate *or* when + // compiling the user's crate. The former is preferable, but in some + // situations (such as when cross-compiling using `cargo build --target`), + // it doesn't get propagated to this crate's build by default. + let cfg_compile_error = if cfg!(zerocopy_derive_union_into_bytes) { + quote!() + } else { + let core = ctx.core_path(); + let error_message = "requires --cfg zerocopy_derive_union_into_bytes; +please let us know you use this feature: https://github.com/google/zerocopy/discussions/1802"; + quote!( + #[allow(unused_attributes, unexpected_cfgs)] + const _: () = { + #[cfg(not(zerocopy_derive_union_into_bytes))] + #core::compile_error!(#error_message); + }; + ) + }; + + // FIXME(#10): Support type parameters. + if !ctx.ast.generics.params.is_empty() { + return ctx.error_or_skip(Error::new( + Span::call_site(), + "unsupported on types with type parameters", + )); + } + + // Because we don't support generics, we don't need to worry about + // special-casing different reprs. So long as there is *some* repr which + // guarantees the layout, our `PaddingCheck::Union` guarantees that there is + // no padding. + let repr = StructUnionRepr::from_attrs(&ctx.ast.attrs)?; + if !repr.is_c() && !repr.is_transparent() && !repr.is_packed_1() { + return ctx.error_or_skip(Error::new( + Span::call_site(), + "must be #[repr(C)], #[repr(packed)], or #[repr(transparent)]", + )); + } + + let impl_block = ImplBlockBuilder::new(ctx, unn, Trait::IntoBytes, FieldBounds::ALL_SELF) + .padding_check(PaddingCheck::Union) + .build(); + Ok(quote!(#cfg_compile_error #impl_block)) +} diff --git a/rust/zerocopy-derive/derive/known_layout.rs b/rust/zerocopy-derive/derive/known_layout.rs new file mode 100644 index 000000000000..d0c4cecfff15 --- /dev/null +++ b/rust/zerocopy-derive/derive/known_layout.rs @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +use proc_macro2::TokenStream; +use quote::quote; +use syn::{parse_quote, Data, Error, Type}; + +use crate::{ + repr::StructUnionRepr, + util::{Ctx, DataExt, FieldBounds, ImplBlockBuilder, SelfBounds, Trait}, +}; + +fn derive_known_layout_for_repr_c_struct<'a>( + ctx: &'a Ctx, + repr: &StructUnionRepr, + fields: &[(&'a syn::Visibility, TokenStream, &'a Type)], +) -> Option<(SelfBounds<'a>, TokenStream, Option<TokenStream>)> { + let (trailing_field, leading_fields) = fields.split_last()?; + + let (_vis, trailing_field_name, trailing_field_ty) = trailing_field; + let leading_fields_tys = leading_fields.iter().map(|(_vis, _name, ty)| ty); + + let core = ctx.core_path(); + let repr_align = repr + .get_align() + .map(|align| { + let align = align.t.get(); + quote!(#core::num::NonZeroUsize::new(#align as usize)) + }) + .unwrap_or_else(|| quote!(#core::option::Option::None)); + let repr_packed = repr + .get_packed() + .map(|packed| { + let packed = packed.get(); + quote!(#core::num::NonZeroUsize::new(#packed as usize)) + }) + .unwrap_or_else(|| quote!(#core::option::Option::None)); + + let zerocopy_crate = &ctx.zerocopy_crate; + let make_methods = |trailing_field_ty| { + quote! { + // SAFETY: + // - The returned pointer has the same address and provenance as + // `bytes`: + // - The recursive call to `raw_from_ptr_len` preserves both + // address and provenance. + // - The `as` cast preserves both address and provenance. + // - `NonNull::new_unchecked` preserves both address and + // provenance. + // - If `Self` is a slice DST, the returned pointer encodes + // `elems` elements in the trailing slice: + // - This is true of the recursive call to `raw_from_ptr_len`. + // - `trailing.as_ptr() as *mut Self` preserves trailing slice + // element count [1]. + // - `NonNull::new_unchecked` preserves trailing slice element + // count. + // + // [1] Per https://doc.rust-lang.org/reference/expressions/operator-expr.html#pointer-to-pointer-cast: + // + // `*const T`` / `*mut T` can be cast to `*const U` / `*mut U` + // with the following behavior: + // ... + // - If `T` and `U` are both unsized, the pointer is also + // returned unchanged. In particular, the metadata is + // preserved exactly. + // + // For instance, a cast from `*const [T]` to `*const [U]` + // preserves the number of elements. ... The same holds + // for str and any compound type whose unsized tail is a + // slice type, such as struct `Foo(i32, [u8])` or + // `(u64, Foo)`. + #[inline(always)] + fn raw_from_ptr_len( + bytes: #core::ptr::NonNull<u8>, + meta: <Self as #zerocopy_crate::KnownLayout>::PointerMetadata, + ) -> #core::ptr::NonNull<Self> { + let trailing = <#trailing_field_ty as #zerocopy_crate::KnownLayout>::raw_from_ptr_len(bytes, meta); + let slf = trailing.as_ptr() as *mut Self; + // SAFETY: Constructed from `trailing`, which is non-null. + unsafe { #core::ptr::NonNull::new_unchecked(slf) } + } + + #[inline(always)] + fn pointer_to_metadata(ptr: *mut Self) -> <Self as #zerocopy_crate::KnownLayout>::PointerMetadata { + <#trailing_field_ty>::pointer_to_metadata(ptr as *mut _) + } + } + }; + + let inner_extras = { + let methods = make_methods(*trailing_field_ty); + let (_, ty_generics, _) = ctx.ast.generics.split_for_impl(); + + quote!( + type PointerMetadata = <#trailing_field_ty as #zerocopy_crate::KnownLayout>::PointerMetadata; + + type MaybeUninit = __ZerocopyKnownLayoutMaybeUninit #ty_generics; + + // SAFETY: `LAYOUT` accurately describes the layout of `Self`. + // The documentation of `DstLayout::for_repr_c_struct` vows that + // invocations in this manner will accurately describe a type, + // so long as: + // + // - that type is `repr(C)`, + // - its fields are enumerated in the order they appear, + // - the presence of `repr_align` and `repr_packed` are + // correctly accounted for. + // + // We respect all three of these preconditions here. This + // expansion is only used if `is_repr_c_struct`, we enumerate + // the fields in order, and we extract the values of `align(N)` + // and `packed(N)`. + const LAYOUT: #zerocopy_crate::DstLayout = #zerocopy_crate::DstLayout::for_repr_c_struct( + #repr_align, + #repr_packed, + &[ + #(#zerocopy_crate::DstLayout::for_type::<#leading_fields_tys>(),)* + <#trailing_field_ty as #zerocopy_crate::KnownLayout>::LAYOUT + ], + ); + + #methods + ) + }; + + let outer_extras = { + let ident = &ctx.ast.ident; + let vis = &ctx.ast.vis; + let params = &ctx.ast.generics.params; + let (impl_generics, ty_generics, where_clause) = ctx.ast.generics.split_for_impl(); + + let predicates = if let Some(where_clause) = where_clause { + where_clause.predicates.clone() + } else { + Default::default() + }; + + // Generate a valid ident for a type-level handle to a field of a + // given `name`. + let field_index = |name: &TokenStream| ident!(("__Zerocopy_Field_{}", name), ident.span()); + + let field_indices: Vec<_> = + fields.iter().map(|(_vis, name, _ty)| field_index(name)).collect(); + + // Define the collection of type-level field handles. + let field_defs = field_indices.iter().zip(fields).map(|(idx, (vis, _, _))| { + quote! { + #vis struct #idx; + } + }); + + let field_impls = field_indices.iter().zip(fields).map(|(idx, (_, _, ty))| quote! { + // SAFETY: `#ty` is the type of `#ident`'s field at `#idx`. + // + // We implement `Field` for each field of the struct to create a + // projection from the field index to its type. This allows us + // to refer to the field's type in a way that respects `Self` + // hygiene. If we just copy-pasted the tokens of `#ty`, we + // would not respect `Self` hygiene, as `Self` would refer to + // the helper struct we are generating, not the derive target + // type. + unsafe impl #impl_generics #zerocopy_crate::util::macro_util::Field<#idx> for #ident #ty_generics + where + #predicates + { + type Type = #ty; + } + }); + + let trailing_field_index = field_index(trailing_field_name); + let leading_field_indices = + leading_fields.iter().map(|(_vis, name, _ty)| field_index(name)); + + // We use `Field` to project the type of the trailing field. This is + // required to ensure that if the field type uses `Self`, it + // resolves to the derive target type, not the helper struct we are + // generating. + let trailing_field_ty = quote! { + <#ident #ty_generics as + #zerocopy_crate::util::macro_util::Field<#trailing_field_index> + >::Type + }; + + let methods = make_methods(&parse_quote! { + <#trailing_field_ty as #zerocopy_crate::KnownLayout>::MaybeUninit + }); + + let core = ctx.core_path(); + + quote! { + #(#field_defs)* + + #(#field_impls)* + + // SAFETY: This has the same layout as the derive target type, + // except that it admits uninit bytes. This is ensured by using + // the same repr as the target type, and by using field types + // which have the same layout as the target type's fields, + // except that they admit uninit bytes. We indirect through + // `Field` to ensure that occurrences of `Self` resolve to + // `#ty`, not `__ZerocopyKnownLayoutMaybeUninit` (see #2116). + #repr + #[doc(hidden)] + #vis struct __ZerocopyKnownLayoutMaybeUninit<#params> ( + #(#core::mem::MaybeUninit< + <#ident #ty_generics as + #zerocopy_crate::util::macro_util::Field<#leading_field_indices> + >::Type + >,)* + // NOTE(#2302): We wrap in `ManuallyDrop` here in case the + // type we're operating on is both generic and + // `repr(packed)`. In that case, Rust needs to know that the + // type is *either* `Sized` or has a trivial `Drop`. + // `ManuallyDrop` has a trivial `Drop`, and so satisfies + // this requirement. + #core::mem::ManuallyDrop< + <#trailing_field_ty as #zerocopy_crate::KnownLayout>::MaybeUninit + > + ) + where + #trailing_field_ty: #zerocopy_crate::KnownLayout, + #predicates; + + // SAFETY: We largely defer to the `KnownLayout` implementation + // on the derive target type (both by using the same tokens, and + // by deferring to impl via type-level indirection). This is + // sound, since `__ZerocopyKnownLayoutMaybeUninit` is guaranteed + // to have the same layout as the derive target type, except + // that `__ZerocopyKnownLayoutMaybeUninit` admits uninit bytes. + unsafe impl #impl_generics #zerocopy_crate::KnownLayout for __ZerocopyKnownLayoutMaybeUninit #ty_generics + where + #trailing_field_ty: #zerocopy_crate::KnownLayout, + #predicates + { + fn only_derive_is_allowed_to_implement_this_trait() {} + + type PointerMetadata = <#ident #ty_generics as #zerocopy_crate::KnownLayout>::PointerMetadata; + + type MaybeUninit = Self; + + const LAYOUT: #zerocopy_crate::DstLayout = <#ident #ty_generics as #zerocopy_crate::KnownLayout>::LAYOUT; + + #methods + } + } + }; + + Some((SelfBounds::None, inner_extras, Some(outer_extras))) +} + +pub(crate) fn derive(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> { + // If this is a `repr(C)` struct, then `c_struct_repr` contains the entire + // `repr` attribute. + let c_struct_repr = match &ctx.ast.data { + Data::Struct(..) => { + let repr = StructUnionRepr::from_attrs(&ctx.ast.attrs)?; + if repr.is_c() { + Some(repr) + } else { + None + } + } + Data::Enum(..) | Data::Union(..) => None, + }; + + let fields = ctx.ast.data.fields(); + + let (self_bounds, inner_extras, outer_extras) = c_struct_repr + .as_ref() + .and_then(|repr| { + derive_known_layout_for_repr_c_struct(ctx, repr, &fields) + }) + .unwrap_or_else(|| { + let zerocopy_crate = &ctx.zerocopy_crate; + let core = ctx.core_path(); + + // For enums, unions, and non-`repr(C)` structs, we require that + // `Self` is sized, and as a result don't need to reason about the + // internals of the type. + ( + SelfBounds::SIZED, + quote!( + type PointerMetadata = (); + type MaybeUninit = + #core::mem::MaybeUninit<Self>; + + // SAFETY: `LAYOUT` is guaranteed to accurately describe the + // layout of `Self`, because that is the documented safety + // contract of `DstLayout::for_type`. + const LAYOUT: #zerocopy_crate::DstLayout = #zerocopy_crate::DstLayout::for_type::<Self>(); + + // SAFETY: `.cast` preserves address and provenance. + // + // FIXME(#429): Add documentation to `.cast` that promises that + // it preserves provenance. + #[inline(always)] + fn raw_from_ptr_len(bytes: #core::ptr::NonNull<u8>, _meta: ()) -> #core::ptr::NonNull<Self> { + bytes.cast::<Self>() + } + + #[inline(always)] + fn pointer_to_metadata(_ptr: *mut Self) -> () {} + ), + None, + ) + }); + Ok(match &ctx.ast.data { + Data::Struct(strct) => { + let require_trait_bound_on_field_types = + if matches!(self_bounds, SelfBounds::All(&[Trait::Sized])) { + FieldBounds::None + } else { + FieldBounds::TRAILING_SELF + }; + + // A bound on the trailing field is required, since structs are + // unsized if their trailing field is unsized. Reflecting the layout + // of an usized trailing field requires that the field is + // `KnownLayout`. + ImplBlockBuilder::new( + ctx, + strct, + Trait::KnownLayout, + require_trait_bound_on_field_types, + ) + .self_type_trait_bounds(self_bounds) + .inner_extras(inner_extras) + .outer_extras(outer_extras) + .build() + } + Data::Enum(enm) => { + // A bound on the trailing field is not required, since enums cannot + // currently be unsized. + ImplBlockBuilder::new(ctx, enm, Trait::KnownLayout, FieldBounds::None) + .self_type_trait_bounds(SelfBounds::SIZED) + .inner_extras(inner_extras) + .outer_extras(outer_extras) + .build() + } + Data::Union(unn) => { + // A bound on the trailing field is not required, since unions + // cannot currently be unsized. + ImplBlockBuilder::new(ctx, unn, Trait::KnownLayout, FieldBounds::None) + .self_type_trait_bounds(SelfBounds::SIZED) + .inner_extras(inner_extras) + .outer_extras(outer_extras) + .build() + } + }) +} diff --git a/rust/zerocopy-derive/derive/mod.rs b/rust/zerocopy-derive/derive/mod.rs new file mode 100644 index 000000000000..b3839fcf73c9 --- /dev/null +++ b/rust/zerocopy-derive/derive/mod.rs @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +pub mod from_bytes; +pub mod into_bytes; +pub mod known_layout; +pub mod try_from_bytes; +pub mod unaligned; + +use proc_macro2::{Span, TokenStream}; +use quote::quote; +use syn::{Data, Error}; + +use crate::{ + repr::StructUnionRepr, + util::{Ctx, DataExt, FieldBounds, ImplBlockBuilder, Trait}, +}; + +pub(crate) fn derive_immutable(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> { + Ok(match &ctx.ast.data { + Data::Struct(strct) => { + ImplBlockBuilder::new(ctx, strct, Trait::Immutable, FieldBounds::ALL_SELF).build() + } + Data::Enum(enm) => { + ImplBlockBuilder::new(ctx, enm, Trait::Immutable, FieldBounds::ALL_SELF).build() + } + Data::Union(unn) => { + ImplBlockBuilder::new(ctx, unn, Trait::Immutable, FieldBounds::ALL_SELF).build() + } + }) +} + +pub(crate) fn derive_hash(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> { + // This doesn't delegate to `impl_block` because `impl_block` assumes it is + // deriving a `zerocopy`-defined trait, and these trait impls share a common + // shape that `Hash` does not. In particular, `zerocopy` traits contain a + // method that only `zerocopy_derive` macros are supposed to implement, and + // `impl_block` generating this trait method is incompatible with `Hash`. + let type_ident = &ctx.ast.ident; + let (impl_generics, ty_generics, where_clause) = ctx.ast.generics.split_for_impl(); + let where_predicates = where_clause.map(|clause| &clause.predicates); + let zerocopy_crate = &ctx.zerocopy_crate; + let core = ctx.core_path(); + Ok(quote! { + impl #impl_generics #core::hash::Hash for #type_ident #ty_generics + where + Self: #zerocopy_crate::IntoBytes + #zerocopy_crate::Immutable, + #where_predicates + { + fn hash<H: #core::hash::Hasher>(&self, state: &mut H) { + #core::hash::Hasher::write(state, #zerocopy_crate::IntoBytes::as_bytes(self)) + } + + fn hash_slice<H: #core::hash::Hasher>(data: &[Self], state: &mut H) { + #core::hash::Hasher::write(state, #zerocopy_crate::IntoBytes::as_bytes(data)) + } + } + }) +} + +pub(crate) fn derive_eq(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> { + // This doesn't delegate to `impl_block` because `impl_block` assumes it is + // deriving a `zerocopy`-defined trait, and these trait impls share a common + // shape that `Eq` does not. In particular, `zerocopy` traits contain a + // method that only `zerocopy_derive` macros are supposed to implement, and + // `impl_block` generating this trait method is incompatible with `Eq`. + let type_ident = &ctx.ast.ident; + let (impl_generics, ty_generics, where_clause) = ctx.ast.generics.split_for_impl(); + let where_predicates = where_clause.map(|clause| &clause.predicates); + let zerocopy_crate = &ctx.zerocopy_crate; + let core = ctx.core_path(); + Ok(quote! { + impl #impl_generics #core::cmp::PartialEq for #type_ident #ty_generics + where + Self: #zerocopy_crate::IntoBytes + #zerocopy_crate::Immutable, + #where_predicates + { + fn eq(&self, other: &Self) -> bool { + #core::cmp::PartialEq::eq( + #zerocopy_crate::IntoBytes::as_bytes(self), + #zerocopy_crate::IntoBytes::as_bytes(other), + ) + } + } + + impl #impl_generics #core::cmp::Eq for #type_ident #ty_generics + where + Self: #zerocopy_crate::IntoBytes + #zerocopy_crate::Immutable, + #where_predicates + { + } + }) +} + +pub(crate) fn derive_split_at(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> { + let repr = StructUnionRepr::from_attrs(&ctx.ast.attrs)?; + + match &ctx.ast.data { + Data::Struct(_) => {} + Data::Enum(_) | Data::Union(_) => { + return ctx + .error_or_skip(Error::new(Span::call_site(), "can only be applied to structs")); + } + }; + + if repr.get_packed().is_some() { + return ctx.error_or_skip(Error::new( + Span::call_site(), + "must not have #[repr(packed)] attribute", + )); + } + + if !(repr.is_c() || repr.is_transparent()) { + return ctx.error_or_skip(Error::new( + Span::call_site(), + "must have #[repr(C)] or #[repr(transparent)] in order to guarantee this type's layout is splitable", + )); + } + + let fields = ctx.ast.data.fields(); + let trailing_field = if let Some(((_, _, trailing_field), _)) = fields.split_last() { + trailing_field + } else { + return ctx.error_or_skip(Error::new(Span::call_site(), "must at least one field")); + }; + + let zerocopy_crate = &ctx.zerocopy_crate; + // SAFETY: `#ty`, per the above checks, is `repr(C)` or `repr(transparent)` + // and is not packed; its trailing field is guaranteed to be well-aligned + // for its type. By invariant on `FieldBounds::TRAILING_SELF`, the trailing + // slice of the trailing field is also well-aligned for its type. + Ok(ImplBlockBuilder::new(ctx, &ctx.ast.data, Trait::SplitAt, FieldBounds::TRAILING_SELF) + .inner_extras(quote! { + type Elem = <#trailing_field as #zerocopy_crate::SplitAt>::Elem; + }) + .build()) +} diff --git a/rust/zerocopy-derive/derive/try_from_bytes.rs b/rust/zerocopy-derive/derive/try_from_bytes.rs new file mode 100644 index 000000000000..44f083328786 --- /dev/null +++ b/rust/zerocopy-derive/derive/try_from_bytes.rs @@ -0,0 +1,765 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +use proc_macro2::TokenStream; +use quote::quote; +use syn::{ + parse_quote, spanned::Spanned as _, Data, DataEnum, DataStruct, DataUnion, DeriveInput, Error, + Expr, Fields, Ident, Index, Type, +}; + +use crate::{ + repr::{EnumRepr, StructUnionRepr}, + util::{ + const_block, enum_size_from_repr, generate_tag_enum, Ctx, DataExt, FieldBounds, + ImplBlockBuilder, Trait, TraitBound, + }, +}; +fn tag_ident(variant_ident: &Ident) -> Ident { + ident!(("___ZEROCOPY_TAG_{}", variant_ident), variant_ident.span()) +} + +/// Generates a constant for the tag associated with each variant of the enum. +/// When we match on the enum's tag, each arm matches one of these constants. We +/// have to use constants here because: +/// +/// - The type that we're matching on is not the type of the tag, it's an +/// integer of the same size as the tag type and with the same bit patterns. +/// - We can't read the enum tag as an enum because the bytes may not represent +/// a valid variant. +/// - Patterns do not currently support const expressions, so we have to assign +/// these constants to names rather than use them inline in the `match` +/// statement. +fn generate_tag_consts(data: &DataEnum) -> TokenStream { + let tags = data.variants.iter().map(|v| { + let variant_ident = &v.ident; + let tag_ident = tag_ident(variant_ident); + + quote! { + // This casts the enum variant to its discriminant, and then + // converts the discriminant to the target integral type via a + // numeric cast [1]. + // + // Because these are the same size, this is defined to be a no-op + // and therefore is a lossless conversion [2]. + // + // [1] Per https://doc.rust-lang.org/1.81.0/reference/expressions/operator-expr.html#enum-cast: + // + // Casts an enum to its discriminant. + // + // [2] Per https://doc.rust-lang.org/1.81.0/reference/expressions/operator-expr.html#numeric-cast: + // + // Casting between two integers of the same size (e.g. i32 -> u32) + // is a no-op. + const #tag_ident: ___ZerocopyTagPrimitive = + ___ZerocopyTag::#variant_ident as ___ZerocopyTagPrimitive; + } + }); + + quote! { + #(#tags)* + } +} + +fn variant_struct_ident(variant_ident: &Ident) -> Ident { + ident!(("___ZerocopyVariantStruct_{}", variant_ident), variant_ident.span()) +} + +/// Generates variant structs for the given enum variant. +/// +/// These are structs associated with each variant of an enum. They are +/// `repr(C)` tuple structs with the same fields as the variant after a +/// `MaybeUninit<___ZerocopyInnerTag>`. +/// +/// In order to unify the generated types for `repr(C)` and `repr(int)` enums, +/// we use a "fused" representation with fields for both an inner tag and an +/// outer tag. Depending on the repr, we will set one of these tags to the tag +/// type and the other to `()`. This lets us generate the same code but put the +/// tags in different locations. +fn generate_variant_structs(ctx: &Ctx, data: &DataEnum) -> TokenStream { + let (impl_generics, ty_generics, where_clause) = ctx.ast.generics.split_for_impl(); + + let enum_name = &ctx.ast.ident; + + // All variant structs have a `PhantomData<MyEnum<...>>` field because we + // don't know which generic parameters each variant will use, and unused + // generic parameters are a compile error. + let core = ctx.core_path(); + let phantom_ty = quote! { + #core::marker::PhantomData<#enum_name #ty_generics> + }; + + let variant_structs = data.variants.iter().filter_map(|variant| { + // We don't generate variant structs for unit variants because we only + // need to check the tag. This helps cut down our generated code a bit. + if matches!(variant.fields, Fields::Unit) { + return None; + } + + let variant_struct_ident = variant_struct_ident(&variant.ident); + let field_types = variant.fields.iter().map(|f| &f.ty); + + let variant_struct = parse_quote! { + #[repr(C)] + struct #variant_struct_ident #impl_generics ( + #core::mem::MaybeUninit<___ZerocopyInnerTag>, + #(#field_types,)* + #phantom_ty, + ) #where_clause; + }; + + // We do this rather than emitting `#[derive(::zerocopy::TryFromBytes)]` + // because that is not hygienic, and this is also more performant. + let try_from_bytes_impl = + derive_try_from_bytes(&ctx.with_input(&variant_struct), Trait::TryFromBytes) + .expect("derive_try_from_bytes should not fail on synthesized type"); + + Some(quote! { + #variant_struct + #try_from_bytes_impl + }) + }); + + quote! { + #(#variant_structs)* + } +} + +fn variants_union_field_ident(ident: &Ident) -> Ident { + // Field names are prefixed with `__field_` to prevent name collision + // with the `__nonempty` field. + ident!(("__field_{}", ident), ident.span()) +} + +fn generate_variants_union(ctx: &Ctx, data: &DataEnum) -> TokenStream { + let generics = &ctx.ast.generics; + let (_, ty_generics, _) = generics.split_for_impl(); + + let fields = data.variants.iter().filter_map(|variant| { + // We don't generate variant structs for unit variants because we only + // need to check the tag. This helps cut down our generated code a bit. + if matches!(variant.fields, Fields::Unit) { + return None; + } + + let field_name = variants_union_field_ident(&variant.ident); + let variant_struct_ident = variant_struct_ident(&variant.ident); + + let core = ctx.core_path(); + Some(quote! { + #field_name: #core::mem::ManuallyDrop<#variant_struct_ident #ty_generics>, + }) + }); + + let variants_union = parse_quote! { + #[repr(C)] + union ___ZerocopyVariants #generics { + #(#fields)* + // Enums can have variants with no fields, but unions must + // have at least one field. So we just add a trailing unit + // to ensure that this union always has at least one field. + // Because this union is `repr(C)`, this unit type does not + // affect the layout. + __nonempty: (), + } + }; + + let has_field = + derive_has_field_struct_union(&ctx.with_input(&variants_union), &variants_union.data); + + quote! { + #variants_union + #has_field + } +} + +/// Generates an implementation of `is_bit_valid` for an arbitrary enum. +/// +/// The general process is: +/// +/// 1. Generate a tag enum. This is an enum with the same repr, variants, and +/// corresponding discriminants as the original enum, but without any fields +/// on the variants. This gives us access to an enum where the variants have +/// the same discriminants as the one we're writing `is_bit_valid` for. +/// 2. Make constants from the variants of the tag enum. We need these because +/// we can't put const exprs in match arms. +/// 3. Generate variant structs. These are structs which have the same fields as +/// each variant of the enum, and are `#[repr(C)]` with an optional "inner +/// tag". +/// 4. Generate a variants union, with one field for each variant struct type. +/// 5. And finally, our raw enum is a `#[repr(C)]` struct of an "outer tag" and +/// the variants union. +/// +/// See these reference links for fully-worked example decompositions. +/// +/// - `repr(C)`: <https://doc.rust-lang.org/reference/type-layout.html#reprc-enums-with-fields> +/// - `repr(int)`: <https://doc.rust-lang.org/reference/type-layout.html#primitive-representation-of-enums-with-fields> +/// - `repr(C, int)`: <https://doc.rust-lang.org/reference/type-layout.html#combining-primitive-representations-of-enums-with-fields-and-reprc> +pub(crate) fn derive_is_bit_valid( + ctx: &Ctx, + data: &DataEnum, + repr: &EnumRepr, +) -> Result<TokenStream, Error> { + let trait_path = Trait::TryFromBytes.crate_path(ctx); + let tag_enum = generate_tag_enum(ctx, repr, data); + let tag_consts = generate_tag_consts(data); + + let (outer_tag_type, inner_tag_type) = if repr.is_c() { + (quote! { ___ZerocopyTag }, quote! { () }) + } else if repr.is_primitive() { + (quote! { () }, quote! { ___ZerocopyTag }) + } else { + return Err(Error::new( + ctx.ast.span(), + "must have #[repr(C)] or #[repr(Int)] attribute in order to guarantee this type's memory layout", + )); + }; + + let variant_structs = generate_variant_structs(ctx, data); + let variants_union = generate_variants_union(ctx, data); + + let (impl_generics, ty_generics, where_clause) = ctx.ast.generics.split_for_impl(); + + let zerocopy_crate = &ctx.zerocopy_crate; + let has_tag = ImplBlockBuilder::new(ctx, data, Trait::HasTag, FieldBounds::None) + .inner_extras(quote! { + type Tag = ___ZerocopyTag; + type ProjectToTag = #zerocopy_crate::pointer::cast::CastSized; + }) + .build(); + let has_fields = data.variants().into_iter().flat_map(|(variant, fields)| { + let variant_ident = &variant.unwrap().ident; + let variants_union_field_ident = variants_union_field_ident(variant_ident); + let field: Box<syn::Type> = parse_quote!(()); + fields.into_iter().enumerate().map(move |(idx, (vis, ident, ty))| { + // Rust does not presently support explicit visibility modifiers on + // enum fields, but we guard against the possibility to ensure this + // derive remains sound. + assert!(matches!(vis, syn::Visibility::Inherited)); + let variant_struct_field_index = Index::from(idx + 1); + let (_, ty_generics, _) = ctx.ast.generics.split_for_impl(); + let has_field_trait = Trait::HasField { + variant_id: parse_quote!({ #zerocopy_crate::ident_id!(#variant_ident) }), + // Since Rust does not presently support explicit visibility + // modifiers on enum fields, any public type is suitable here; + // we use `()`. + field: field.clone(), + field_id: parse_quote!({ #zerocopy_crate::ident_id!(#ident) }), + }; + let has_field_path = has_field_trait.crate_path(ctx); + let has_field = ImplBlockBuilder::new( + ctx, + data, + has_field_trait, + FieldBounds::None, + ) + .inner_extras(quote! { + type Type = #ty; + + #[inline(always)] + fn project(slf: #zerocopy_crate::pointer::PtrInner<'_, Self>) -> *mut <Self as #has_field_path>::Type { + use #zerocopy_crate::pointer::cast::{CastSized, Projection}; + + slf.project::<___ZerocopyRawEnum #ty_generics, CastSized>() + .project::<_, Projection<_, { #zerocopy_crate::STRUCT_VARIANT_ID }, { #zerocopy_crate::ident_id!(variants) }>>() + .project::<_, Projection<_, { #zerocopy_crate::REPR_C_UNION_VARIANT_ID }, { #zerocopy_crate::ident_id!(#variants_union_field_ident) }>>() + .project::<_, Projection<_, { #zerocopy_crate::STRUCT_VARIANT_ID }, { #zerocopy_crate::ident_id!(value) }>>() + .project::<_, Projection<_, { #zerocopy_crate::STRUCT_VARIANT_ID }, { #zerocopy_crate::ident_id!(#variant_struct_field_index) }>>() + .as_ptr() + } + }) + .build(); + + let project = ImplBlockBuilder::new( + ctx, + data, + Trait::ProjectField { + variant_id: parse_quote!({ #zerocopy_crate::ident_id!(#variant_ident) }), + // Since Rust does not presently support explicit visibility + // modifiers on enum fields, any public type is suitable + // here; we use `()`. + field: field.clone(), + field_id: parse_quote!({ #zerocopy_crate::ident_id!(#ident) }), + invariants: parse_quote!((Aliasing, Alignment, #zerocopy_crate::invariant::Initialized)), + }, + FieldBounds::None, + ) + .param_extras(vec![ + parse_quote!(Aliasing: #zerocopy_crate::invariant::Aliasing), + parse_quote!(Alignment: #zerocopy_crate::invariant::Alignment), + ]) + .inner_extras(quote! { + type Error = #zerocopy_crate::util::macro_util::core_reexport::convert::Infallible; + type Invariants = (Aliasing, Alignment, #zerocopy_crate::invariant::Initialized); + }) + .build(); + + quote! { + #has_field + #project + } + }) + }); + + let core = ctx.core_path(); + let match_arms = data.variants.iter().map(|variant| { + let tag_ident = tag_ident(&variant.ident); + let variant_struct_ident = variant_struct_ident(&variant.ident); + let variants_union_field_ident = variants_union_field_ident(&variant.ident); + + if matches!(variant.fields, Fields::Unit) { + // Unit variants don't need any further validation beyond checking + // the tag. + quote! { + #tag_ident => true + } + } else { + quote! { + #tag_ident => { + // SAFETY: Since we know that the tag is `#tag_ident`, we + // know that no other `&`s exist which refer to this enum + // as any other variant. + let variant_md = variants.cast::< + _, + #zerocopy_crate::pointer::cast::Projection< + // #zerocopy_crate::ReadOnly<_>, + _, + { #zerocopy_crate::REPR_C_UNION_VARIANT_ID }, + { #zerocopy_crate::ident_id!(#variants_union_field_ident) } + >, + _ + >(); + let variant = variant_md.cast::< + #zerocopy_crate::ReadOnly<#variant_struct_ident #ty_generics>, + #zerocopy_crate::pointer::cast::CastSized, + (#zerocopy_crate::pointer::BecauseRead, _) + >(); + < + #variant_struct_ident #ty_generics as #trait_path + >::is_bit_valid(variant) + } + } + } + }); + + let generics = &ctx.ast.generics; + let raw_enum: DeriveInput = parse_quote! { + #[repr(C)] + struct ___ZerocopyRawEnum #generics { + tag: ___ZerocopyOuterTag, + variants: ___ZerocopyVariants #ty_generics, + } + }; + + let self_ident = &ctx.ast.ident; + let invariants_eq_impl = quote! { + // SAFETY: `___ZerocopyRawEnum` is designed to have the same layout, + // validity, and invariants as `Self`. + unsafe impl #impl_generics #zerocopy_crate::pointer::InvariantsEq<___ZerocopyRawEnum #ty_generics> for #self_ident #ty_generics #where_clause {} + }; + + let raw_enum_projections = + derive_has_field_struct_union(&ctx.with_input(&raw_enum), &raw_enum.data); + + let raw_enum = quote! { + #raw_enum + #invariants_eq_impl + #raw_enum_projections + }; + + Ok(quote! { + // SAFETY: We use `is_bit_valid` to validate that the bit pattern of the + // enum's tag corresponds to one of the enum's discriminants. Then, we + // check the bit validity of each field of the corresponding variant. + // Thus, this is a sound implementation of `is_bit_valid`. + #[inline] + fn is_bit_valid<___ZcAlignment>( + mut candidate: #zerocopy_crate::Maybe<'_, Self, ___ZcAlignment>, + ) -> #core::primitive::bool + where + ___ZcAlignment: #zerocopy_crate::invariant::Alignment, + { + #tag_enum + + type ___ZerocopyTagPrimitive = #zerocopy_crate::util::macro_util::SizeToTag< + { #core::mem::size_of::<___ZerocopyTag>() }, + >; + + #tag_consts + + type ___ZerocopyOuterTag = #outer_tag_type; + type ___ZerocopyInnerTag = #inner_tag_type; + + #variant_structs + + #variants_union + + #raw_enum + + #has_tag + + #(#has_fields)* + + let tag = { + // SAFETY: + // - The provided cast addresses a subset of the bytes addressed + // by `candidate` because it addresses the starting tag of the + // enum. + // - Because the pointer is cast from `candidate`, it has the + // same provenance as it. + // - There are no `UnsafeCell`s in the tag because it is a + // primitive integer. + // - `tag_ptr` is casted from `candidate`, whose referent is + // `Initialized`. Since we have not written uninitialized + // bytes into the referent, `tag_ptr` is also `Initialized`. + // + // FIXME(#2874): Revise this to a `cast` once `candidate` + // references a `ReadOnly<Self>`. + let tag_ptr = unsafe { + candidate.reborrow().project_transmute_unchecked::< + _, + #zerocopy_crate::invariant::Initialized, + #zerocopy_crate::pointer::cast::CastSized + >() + }; + tag_ptr.recall_validity::<_, (_, (_, _))>().read::<#zerocopy_crate::BecauseImmutable>() + }; + + let mut raw_enum = candidate.cast::< + #zerocopy_crate::ReadOnly<___ZerocopyRawEnum #ty_generics>, + #zerocopy_crate::pointer::cast::CastSized, + (#zerocopy_crate::pointer::BecauseRead, _) + >(); + + let variants = #zerocopy_crate::into_inner!(raw_enum.project::< + _, + { #zerocopy_crate::STRUCT_VARIANT_ID }, + { #zerocopy_crate::ident_id!(variants) } + >()); + + match tag { + #(#match_arms,)* + _ => false, + } + } + }) +} +pub(crate) fn derive_try_from_bytes(ctx: &Ctx, top_level: Trait) -> Result<TokenStream, Error> { + match &ctx.ast.data { + Data::Struct(strct) => derive_try_from_bytes_struct(ctx, strct, top_level), + Data::Enum(enm) => derive_try_from_bytes_enum(ctx, enm, top_level), + Data::Union(unn) => Ok(derive_try_from_bytes_union(ctx, unn, top_level)), + } +} +fn derive_has_field_struct_union(ctx: &Ctx, data: &dyn DataExt) -> TokenStream { + let fields = ctx.ast.data.fields(); + if fields.is_empty() { + return quote! {}; + } + + let field_tokens = fields.iter().map(|(vis, ident, _)| { + let ident = ident!(("__z{}", ident), ident.span()); + quote!( + #vis enum #ident {} + ) + }); + + let zerocopy_crate = &ctx.zerocopy_crate; + let variant_id: Box<Expr> = match &ctx.ast.data { + Data::Struct(_) => parse_quote!({ #zerocopy_crate::STRUCT_VARIANT_ID }), + Data::Union(_) => { + let is_repr_c = StructUnionRepr::from_attrs(&ctx.ast.attrs) + .map(|repr| repr.is_c()) + .unwrap_or(false); + if is_repr_c { + parse_quote!({ #zerocopy_crate::REPR_C_UNION_VARIANT_ID }) + } else { + parse_quote!({ #zerocopy_crate::UNION_VARIANT_ID }) + } + } + _ => unreachable!(), + }; + + let core = ctx.core_path(); + let has_tag = ImplBlockBuilder::new(ctx, data, Trait::HasTag, FieldBounds::None) + .inner_extras(quote! { + type Tag = (); + type ProjectToTag = #zerocopy_crate::pointer::cast::CastToUnit; + }) + .build(); + let has_fields = fields.iter().map(move |(_, ident, ty)| { + let field_token = ident!(("__z{}", ident), ident.span()); + let field: Box<Type> = parse_quote!(#field_token); + let field_id: Box<Expr> = parse_quote!({ #zerocopy_crate::ident_id!(#ident) }); + let has_field_trait = Trait::HasField { + variant_id: variant_id.clone(), + field: field.clone(), + field_id: field_id.clone(), + }; + let has_field_path = has_field_trait.crate_path(ctx); + ImplBlockBuilder::new( + ctx, + data, + has_field_trait, + FieldBounds::None, + ) + .inner_extras(quote! { + type Type = #ty; + + #[inline(always)] + fn project(slf: #zerocopy_crate::pointer::PtrInner<'_, Self>) -> *mut <Self as #has_field_path>::Type { + let slf = slf.as_ptr(); + // SAFETY: By invariant on `PtrInner`, `slf` is a non-null + // pointer whose referent is zero-sized or lives in a valid + // allocation. Since `#ident` is a struct or union field of + // `Self`, this projection preserves or shrinks the referent + // size, and so the resulting referent also fits in the same + // allocation. + unsafe { #core::ptr::addr_of_mut!((*slf).#ident) } + } + }).outer_extras(if matches!(&ctx.ast.data, Data::Struct(..)) { + let fields_preserve_alignment = StructUnionRepr::from_attrs(&ctx.ast.attrs) + .map(|repr| repr.get_packed().is_none()) + .unwrap(); + let alignment = if fields_preserve_alignment { + quote! { Alignment } + } else { + quote! { #zerocopy_crate::invariant::Unaligned } + }; + // SAFETY: See comments on items. + ImplBlockBuilder::new( + ctx, + data, + Trait::ProjectField { + variant_id: variant_id.clone(), + field, + field_id, + invariants: parse_quote!((Aliasing, Alignment, #zerocopy_crate::invariant::Initialized)), + }, + FieldBounds::None, + ) + .param_extras(vec![ + parse_quote!(Aliasing: #zerocopy_crate::invariant::Aliasing), + parse_quote!(Alignment: #zerocopy_crate::invariant::Alignment), + ]) + .inner_extras(quote! { + // SAFETY: Projection into structs is always infallible. + type Error = #zerocopy_crate::util::macro_util::core_reexport::convert::Infallible; + // SAFETY: The alignment of the projected `Ptr` is `Unaligned` + // if the structure is packed; otherwise inherited from the + // outer `Ptr`. If the validity of the outer pointer is + // `Initialized`, so too is the validity of its fields. + type Invariants = (Aliasing, #alignment, #zerocopy_crate::invariant::Initialized); + }) + .build() + } else { + quote! {} + }) + .build() + }); + + const_block(field_tokens.into_iter().chain(Some(has_tag)).chain(has_fields).map(Some)) +} +fn derive_try_from_bytes_struct( + ctx: &Ctx, + strct: &DataStruct, + top_level: Trait, +) -> Result<TokenStream, Error> { + let extras = try_gen_trivial_is_bit_valid(ctx, top_level).unwrap_or_else(|| { + let zerocopy_crate = &ctx.zerocopy_crate; + let fields = strct.fields(); + let field_names = fields.iter().map(|(_vis, name, _ty)| name); + let field_tys = fields.iter().map(|(_vis, _name, ty)| ty); + let core = ctx.core_path(); + quote!( + // SAFETY: We use `is_bit_valid` to validate that each field is + // bit-valid, and only return `true` if all of them are. The bit + // validity of a struct is just the composition of the bit + // validities of its fields, so this is a sound implementation + // of `is_bit_valid`. + #[inline] + fn is_bit_valid<___ZcAlignment>( + mut candidate: #zerocopy_crate::Maybe<'_, Self, ___ZcAlignment>, + ) -> #core::primitive::bool + where + ___ZcAlignment: #zerocopy_crate::invariant::Alignment, + { + true #(&& { + let field_candidate = #zerocopy_crate::into_inner!(candidate.reborrow().project::< + _, + { #zerocopy_crate::STRUCT_VARIANT_ID }, + { #zerocopy_crate::ident_id!(#field_names) } + >()); + <#field_tys as #zerocopy_crate::TryFromBytes>::is_bit_valid(field_candidate) + })* + } + ) + }); + Ok(ImplBlockBuilder::new(ctx, strct, Trait::TryFromBytes, FieldBounds::ALL_SELF) + .inner_extras(extras) + .outer_extras(derive_has_field_struct_union(ctx, strct)) + .build()) +} +fn derive_try_from_bytes_union(ctx: &Ctx, unn: &DataUnion, top_level: Trait) -> TokenStream { + let field_type_trait_bounds = FieldBounds::All(&[TraitBound::Slf]); + + let zerocopy_crate = &ctx.zerocopy_crate; + let variant_id: Box<Expr> = { + let is_repr_c = + StructUnionRepr::from_attrs(&ctx.ast.attrs).map(|repr| repr.is_c()).unwrap_or(false); + if is_repr_c { + parse_quote!({ #zerocopy_crate::REPR_C_UNION_VARIANT_ID }) + } else { + parse_quote!({ #zerocopy_crate::UNION_VARIANT_ID }) + } + }; + + let extras = try_gen_trivial_is_bit_valid(ctx, top_level).unwrap_or_else(|| { + let fields = unn.fields(); + let field_names = fields.iter().map(|(_vis, name, _ty)| name); + let field_tys = fields.iter().map(|(_vis, _name, ty)| ty); + let core = ctx.core_path(); + quote!( + // SAFETY: We use `is_bit_valid` to validate that any field is + // bit-valid; we only return `true` if at least one of them is. + // The bit validity of a union is not yet well defined in Rust, + // but it is guaranteed to be no more strict than this + // definition. See #696 for a more in-depth discussion. + #[inline] + fn is_bit_valid<___ZcAlignment>( + mut candidate: #zerocopy_crate::Maybe<'_, Self, ___ZcAlignment>, + ) -> #core::primitive::bool + where + ___ZcAlignment: #zerocopy_crate::invariant::Alignment, + { + false #(|| { + // SAFETY: + // - Since `ReadOnly<Self>: Immutable` unconditionally, + // neither `*slf` nor the returned pointer's referent + // permit interior mutation. + // - Both source and destination validity are + // `Initialized`, which is always a sound + // transmutation. + let field_candidate = unsafe { + candidate.reborrow().project_transmute_unchecked::< + _, + _, + #zerocopy_crate::pointer::cast::Projection< + _, + #variant_id, + { #zerocopy_crate::ident_id!(#field_names) } + > + >() + }; + + <#field_tys as #zerocopy_crate::TryFromBytes>::is_bit_valid(field_candidate) + })* + } + ) + }); + ImplBlockBuilder::new(ctx, unn, Trait::TryFromBytes, field_type_trait_bounds) + .inner_extras(extras) + .outer_extras(derive_has_field_struct_union(ctx, unn)) + .build() +} +fn derive_try_from_bytes_enum( + ctx: &Ctx, + enm: &DataEnum, + top_level: Trait, +) -> Result<TokenStream, Error> { + let repr = EnumRepr::from_attrs(&ctx.ast.attrs)?; + + // If an enum has no fields, it has a well-defined integer representation, + // and every possible bit pattern corresponds to a valid discriminant tag, + // then it *could* be `FromBytes` (even if the user hasn't derived + // `FromBytes`). This holds if, for `repr(uN)` or `repr(iN)`, there are 2^N + // variants. + let could_be_from_bytes = enum_size_from_repr(&repr) + .map(|size| enm.fields().is_empty() && enm.variants.len() == 1usize << size) + .unwrap_or(false); + + let trivial_is_bit_valid = try_gen_trivial_is_bit_valid(ctx, top_level); + let extra = match (trivial_is_bit_valid, could_be_from_bytes) { + (Some(is_bit_valid), _) => is_bit_valid, + // SAFETY: It would be sound for the enum to implement `FromBytes`, as + // required by `gen_trivial_is_bit_valid_unchecked`. + (None, true) => unsafe { gen_trivial_is_bit_valid_unchecked(ctx) }, + (None, false) => match derive_is_bit_valid(ctx, enm, &repr) { + Ok(extra) => extra, + Err(_) if ctx.skip_on_error => return Ok(TokenStream::new()), + Err(e) => return Err(e), + }, + }; + + Ok(ImplBlockBuilder::new(ctx, enm, Trait::TryFromBytes, FieldBounds::ALL_SELF) + .inner_extras(extra) + .build()) +} +fn try_gen_trivial_is_bit_valid(ctx: &Ctx, top_level: Trait) -> Option<proc_macro2::TokenStream> { + // If the top-level trait is `FromBytes` and `Self` has no type parameters, + // then the `FromBytes` derive will fail compilation if `Self` is not + // actually soundly `FromBytes`, and so we can rely on that for our + // `is_bit_valid` impl. It's plausible that we could make changes - or Rust + // could make changes (such as the "trivial bounds" language feature) - that + // make this no longer true. To hedge against these, we include an explicit + // `Self: FromBytes` check in the generated `is_bit_valid`, which is + // bulletproof. + // + // If `ctx.skip_on_error` is true, we can't rely on the `FromBytes` derive + // to fail compilation if `Self` is not actually soundly `FromBytes`. + if matches!(top_level, Trait::FromBytes) + && ctx.ast.generics.params.is_empty() + && !ctx.skip_on_error + { + let zerocopy_crate = &ctx.zerocopy_crate; + let core = ctx.core_path(); + Some(quote!( + // SAFETY: See inline. + #[inline(always)] + fn is_bit_valid<___ZcAlignment>( + _candidate: #zerocopy_crate::Maybe<'_, Self, ___ZcAlignment>, + ) -> #core::primitive::bool + where + ___ZcAlignment: #zerocopy_crate::invariant::Alignment, + { + if false { + fn assert_is_from_bytes<T>() + where + T: #zerocopy_crate::FromBytes, + T: ?#core::marker::Sized, + { + } + + assert_is_from_bytes::<Self>(); + } + + // SAFETY: The preceding code only compiles if `Self: + // FromBytes`. Thus, this code only compiles if all initialized + // byte sequences represent valid instances of `Self`. + true + } + )) + } else { + None + } +} + +/// # Safety +/// +/// All initialized bit patterns must be valid for `Self`. +unsafe fn gen_trivial_is_bit_valid_unchecked(ctx: &Ctx) -> proc_macro2::TokenStream { + let zerocopy_crate = &ctx.zerocopy_crate; + let core = ctx.core_path(); + quote!( + // SAFETY: The caller of `gen_trivial_is_bit_valid_unchecked` has + // promised that all initialized bit patterns are valid for `Self`. + #[inline(always)] + fn is_bit_valid<___ZcAlignment>( + _candidate: #zerocopy_crate::Maybe<'_, Self, ___ZcAlignment>, + ) -> #core::primitive::bool + where + ___ZcAlignment: #zerocopy_crate::invariant::Alignment, + { + true + } + ) +} diff --git a/rust/zerocopy-derive/derive/unaligned.rs b/rust/zerocopy-derive/derive/unaligned.rs new file mode 100644 index 000000000000..7c97d62e2dcb --- /dev/null +++ b/rust/zerocopy-derive/derive/unaligned.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +use proc_macro2::{Span, TokenStream}; +use syn::{Data, DataEnum, DataStruct, DataUnion, Error}; + +use crate::{ + repr::{EnumRepr, StructUnionRepr}, + util::{Ctx, FieldBounds, ImplBlockBuilder, Trait}, +}; + +pub(crate) fn derive_unaligned(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> { + match &ctx.ast.data { + Data::Struct(strct) => derive_unaligned_struct(ctx, strct), + Data::Enum(enm) => derive_unaligned_enum(ctx, enm), + Data::Union(unn) => derive_unaligned_union(ctx, unn), + } +} + +/// A struct is `Unaligned` if: +/// - `repr(align)` is no more than 1 and either +/// - `repr(C)` or `repr(transparent)` and +/// - all fields `Unaligned` +/// - `repr(packed)` +fn derive_unaligned_struct(ctx: &Ctx, strct: &DataStruct) -> Result<TokenStream, Error> { + let repr = StructUnionRepr::from_attrs(&ctx.ast.attrs)?; + repr.unaligned_validate_no_align_gt_1()?; + + let field_bounds = if repr.is_packed_1() { + FieldBounds::None + } else if repr.is_c() || repr.is_transparent() { + FieldBounds::ALL_SELF + } else { + return ctx.error_or_skip(Error::new( + Span::call_site(), + "must have #[repr(C)], #[repr(transparent)], or #[repr(packed)] attribute in order to guarantee this type's alignment", + )); + }; + + Ok(ImplBlockBuilder::new(ctx, strct, Trait::Unaligned, field_bounds).build()) +} + +/// An enum is `Unaligned` if: +/// - No `repr(align(N > 1))` +/// - `repr(u8)` or `repr(i8)` +fn derive_unaligned_enum(ctx: &Ctx, enm: &DataEnum) -> Result<TokenStream, Error> { + let repr = EnumRepr::from_attrs(&ctx.ast.attrs)?; + repr.unaligned_validate_no_align_gt_1()?; + + if !repr.is_u8() && !repr.is_i8() { + return ctx.error_or_skip(Error::new( + Span::call_site(), + "must have #[repr(u8)] or #[repr(i8)] attribute in order to guarantee this type's alignment", + )); + } + + Ok(ImplBlockBuilder::new(ctx, enm, Trait::Unaligned, FieldBounds::ALL_SELF).build()) +} + +/// Like structs, a union is `Unaligned` if: +/// - `repr(align)` is no more than 1 and either +/// - `repr(C)` or `repr(transparent)` and +/// - all fields `Unaligned` +/// - `repr(packed)` +fn derive_unaligned_union(ctx: &Ctx, unn: &DataUnion) -> Result<TokenStream, Error> { + let repr = StructUnionRepr::from_attrs(&ctx.ast.attrs)?; + repr.unaligned_validate_no_align_gt_1()?; + + let field_type_trait_bounds = if repr.is_packed_1() { + FieldBounds::None + } else if repr.is_c() || repr.is_transparent() { + FieldBounds::ALL_SELF + } else { + return ctx.error_or_skip(Error::new( + Span::call_site(), + "must have #[repr(C)], #[repr(transparent)], or #[repr(packed)] attribute in order to guarantee this type's alignment", + )); + }; + + Ok(ImplBlockBuilder::new(ctx, unn, Trait::Unaligned, field_type_trait_bounds).build()) +} diff --git a/rust/zerocopy-derive/lib.rs b/rust/zerocopy-derive/lib.rs new file mode 100644 index 000000000000..d387de368367 --- /dev/null +++ b/rust/zerocopy-derive/lib.rs @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2019 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Derive macros for [zerocopy]'s traits. +//! +//! [zerocopy]: https://docs.rs/zerocopy + +// Sometimes we want to use lints which were added after our MSRV. +// `unknown_lints` is `warn` by default and we deny warnings in CI, so without +// this attribute, any unknown lint would cause a CI failure when testing with +// our MSRV. +#![allow(unknown_lints)] +#![deny(renamed_and_removed_lints)] +#![deny( + clippy::all, + clippy::missing_safety_doc, + clippy::multiple_unsafe_ops_per_block, + clippy::undocumented_unsafe_blocks +)] +// We defer to own discretion on type complexity. +#![allow(clippy::type_complexity)] +// Inlining format args isn't supported on our MSRV. +#![allow(clippy::uninlined_format_args)] +#![deny( + rustdoc::bare_urls, + rustdoc::broken_intra_doc_links, + rustdoc::invalid_codeblock_attributes, + rustdoc::invalid_html_tags, + rustdoc::invalid_rust_codeblocks, + rustdoc::missing_crate_level_docs, + rustdoc::private_intra_doc_links +)] +#![recursion_limit = "128"] + +macro_rules! ident { + (($fmt:literal $(, $arg:expr)*), $span:expr) => { + syn::Ident::new(&format!($fmt $(, crate::util::to_ident_str($arg))*), $span) + }; +} + +mod derive; +#[cfg(test)] +mod output_tests; +mod repr; +mod util; + +use syn::{DeriveInput, Error}; + +use crate::util::*; + +// FIXME(https://github.com/rust-lang/rust/issues/54140): Some errors could be +// made better if we could add multiple lines of error output like this: +// +// error: unsupported representation +// --> enum.rs:28:8 +// | +// 28 | #[repr(transparent)] +// | +// help: required by the derive of FromBytes +// +// Instead, we have more verbose error messages like "unsupported representation +// for deriving FromZeros, FromBytes, IntoBytes, or Unaligned on an enum" +// +// This will probably require Span::error +// (https://doc.rust-lang.org/nightly/proc_macro/struct.Span.html#method.error), +// which is currently unstable. Revisit this once it's stable. + +/// Defines a derive function named `$outer` which parses its input +/// `TokenStream` as a `DeriveInput` and then invokes the `$inner` function. +/// +/// Note that the separate `$outer` parameter is required - proc macro functions +/// are currently required to live at the crate root, and so the caller must +/// specify the name in order to avoid name collisions. +macro_rules! derive { + ($trait:ident => $outer:ident => $inner:path) => { + #[proc_macro_derive($trait, attributes(zerocopy))] + pub fn $outer(ts: proc_macro::TokenStream) -> proc_macro::TokenStream { + let ast = syn::parse_macro_input!(ts as DeriveInput); + let ctx = match Ctx::try_from_derive_input(ast) { + Ok(ctx) => ctx, + Err(e) => return e.into_compile_error().into(), + }; + let ts = $inner(&ctx, Trait::$trait).into_ts(); + // We wrap in `const_block` as a backstop in case any derive fails + // to wrap its output in `const_block` (and thus fails to annotate) + // with the full set of `#[allow(...)]` attributes). + let ts = const_block([Some(ts)]); + #[cfg(test)] + crate::util::testutil::check_hygiene(ts.clone()); + ts.into() + } + }; +} + +trait IntoTokenStream { + fn into_ts(self) -> proc_macro2::TokenStream; +} + +impl IntoTokenStream for proc_macro2::TokenStream { + fn into_ts(self) -> proc_macro2::TokenStream { + self + } +} + +impl IntoTokenStream for Result<proc_macro2::TokenStream, Error> { + fn into_ts(self) -> proc_macro2::TokenStream { + match self { + Ok(ts) => ts, + Err(err) => err.to_compile_error(), + } + } +} + +derive!(KnownLayout => derive_known_layout => crate::derive::known_layout::derive); +derive!(Immutable => derive_immutable => crate::derive::derive_immutable); +derive!(TryFromBytes => derive_try_from_bytes => crate::derive::try_from_bytes::derive_try_from_bytes); +derive!(FromZeros => derive_from_zeros => crate::derive::from_bytes::derive_from_zeros); +derive!(FromBytes => derive_from_bytes => crate::derive::from_bytes::derive_from_bytes); +derive!(IntoBytes => derive_into_bytes => crate::derive::into_bytes::derive_into_bytes); +derive!(Unaligned => derive_unaligned => crate::derive::unaligned::derive_unaligned); +derive!(ByteHash => derive_hash => crate::derive::derive_hash); +derive!(ByteEq => derive_eq => crate::derive::derive_eq); +derive!(SplitAt => derive_split_at => crate::derive::derive_split_at); + +#[cfg_attr(not(zerocopy_unstable_linux), doc(hidden))] +#[proc_macro_derive(most_traits, attributes(zerocopy))] +pub fn most_traits(ts: proc_macro::TokenStream) -> proc_macro::TokenStream { + let ast = syn::parse_macro_input!(ts as DeriveInput); + let ctx = match Ctx::try_from_derive_input(ast) { + Ok(ctx) => ctx, + Err(e) => return e.into_compile_error().into(), + } + .skip_on_error(); + + // top-level traits for which to attempt a derive + let derives: [(fn(&Ctx, Trait) -> _, _); 6] = [ + (crate::derive::known_layout::derive, Trait::KnownLayout), + (crate::derive::derive_immutable, Trait::Immutable), + (crate::derive::from_bytes::derive_from_bytes, Trait::FromBytes), + (crate::derive::into_bytes::derive_into_bytes, Trait::IntoBytes), + (crate::derive::derive_split_at, Trait::SplitAt), + (crate::derive::unaligned::derive_unaligned, Trait::Unaligned), + ]; + + let mut tokens = proc_macro2::TokenStream::new(); + for (derive, t) in derives { + tokens.extend(derive(&ctx, t)) + } + + // We wrap in `const_block` as a backstop in case any derive fails + // to wrap its output in `const_block` (and thus fails to annotate) + // with the full set of `#[allow(...)]` attributes). + let ts = const_block([Some(tokens)]); + #[cfg(test)] + crate::util::testutil::check_hygiene(ts.clone()); + ts.into() +} + +/// Deprecated: prefer [`FromZeros`] instead. +#[deprecated(since = "0.8.0", note = "`FromZeroes` was renamed to `FromZeros`")] +#[doc(hidden)] +#[proc_macro_derive(FromZeroes)] +pub fn derive_from_zeroes(ts: proc_macro::TokenStream) -> proc_macro::TokenStream { + derive_from_zeros(ts) +} + +/// Deprecated: prefer [`IntoBytes`] instead. +#[deprecated(since = "0.8.0", note = "`AsBytes` was renamed to `IntoBytes`")] +#[doc(hidden)] +#[proc_macro_derive(AsBytes)] +pub fn derive_as_bytes(ts: proc_macro::TokenStream) -> proc_macro::TokenStream { + derive_into_bytes(ts) +} diff --git a/rust/zerocopy-derive/repr.rs b/rust/zerocopy-derive/repr.rs new file mode 100644 index 000000000000..1525e94302d1 --- /dev/null +++ b/rust/zerocopy-derive/repr.rs @@ -0,0 +1,851 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2019 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use core::{ + convert::{Infallible, TryFrom}, + num::NonZeroU32, +}; + +use proc_macro2::{Span, TokenStream}; +use quote::{quote_spanned, ToTokens, TokenStreamExt as _}; +use syn::{ + punctuated::Punctuated, spanned::Spanned as _, token::Comma, Attribute, Error, LitInt, Meta, + MetaList, +}; + +/// The computed representation of a type. +/// +/// This is the result of processing all `#[repr(...)]` attributes on a type, if +/// any. A `Repr` is only capable of representing legal combinations of +/// `#[repr(...)]` attributes. +#[cfg_attr(test, derive(Copy, Clone, Debug))] +pub(crate) enum Repr<Prim, Packed> { + /// `#[repr(transparent)]` + Transparent(Span), + /// A compound representation: `repr(C)`, `repr(Rust)`, or `repr(Int)` + /// optionally combined with `repr(packed(...))` or `repr(align(...))` + Compound(Spanned<CompoundRepr<Prim>>, Option<Spanned<AlignRepr<Packed>>>), +} + +/// A compound representation: `repr(C)`, `repr(Rust)`, or `repr(Int)`. +#[cfg_attr(test, derive(Copy, Clone, Debug, Eq, PartialEq))] +pub(crate) enum CompoundRepr<Prim> { + C, + Rust, + Primitive(Prim), +} + +/// `repr(Int)` +#[derive(Copy, Clone)] +#[cfg_attr(test, derive(Debug, Eq, PartialEq))] +pub(crate) enum PrimitiveRepr { + U8, + U16, + U32, + U64, + U128, + Usize, + I8, + I16, + I32, + I64, + I128, + Isize, +} + +/// `repr(packed(...))` or `repr(align(...))` +#[cfg_attr(test, derive(Copy, Clone, Debug, Eq, PartialEq))] +pub(crate) enum AlignRepr<Packed> { + Packed(Packed), + Align(NonZeroU32), +} + +/// The representations which can legally appear on a struct or union type. +pub(crate) type StructUnionRepr = Repr<Infallible, NonZeroU32>; + +/// The representations which can legally appear on an enum type. +pub(crate) type EnumRepr = Repr<PrimitiveRepr, Infallible>; + +impl<Prim, Packed> Repr<Prim, Packed> { + /// Gets the name of this "repr type" - the non-align `repr(X)` that is used + /// in prose to refer to this type. + /// + /// For example, we would refer to `#[repr(C, align(4))] struct Foo { ... }` + /// as a "`repr(C)` struct". + pub(crate) fn repr_type_name(&self) -> &str + where + Prim: Copy + With<PrimitiveRepr>, + { + use CompoundRepr::*; + use PrimitiveRepr::*; + use Repr::*; + match self { + Transparent(_span) => "repr(transparent)", + Compound(Spanned { t: repr, span: _ }, _align) => match repr { + C => "repr(C)", + Rust => "repr(Rust)", + Primitive(prim) => prim.with(|prim| match prim { + U8 => "repr(u8)", + U16 => "repr(u16)", + U32 => "repr(u32)", + U64 => "repr(u64)", + U128 => "repr(u128)", + Usize => "repr(usize)", + I8 => "repr(i8)", + I16 => "repr(i16)", + I32 => "repr(i32)", + I64 => "repr(i64)", + I128 => "repr(i128)", + Isize => "repr(isize)", + }), + }, + } + } + + pub(crate) fn is_transparent(&self) -> bool { + matches!(self, Repr::Transparent(_)) + } + + pub(crate) fn is_c(&self) -> bool { + use CompoundRepr::*; + matches!(self, Repr::Compound(Spanned { t: C, span: _ }, _align)) + } + + pub(crate) fn is_primitive(&self) -> bool { + use CompoundRepr::*; + matches!(self, Repr::Compound(Spanned { t: Primitive(_), span: _ }, _align)) + } + + pub(crate) fn get_packed(&self) -> Option<&Packed> { + use AlignRepr::*; + use Repr::*; + if let Compound(_, Some(Spanned { t: Packed(p), span: _ })) = self { + Some(p) + } else { + None + } + } + + pub(crate) fn get_align(&self) -> Option<Spanned<NonZeroU32>> { + use AlignRepr::*; + use Repr::*; + if let Compound(_, Some(Spanned { t: Align(n), span })) = self { + Some(Spanned::new(*n, *span)) + } else { + None + } + } + + pub(crate) fn is_align_gt_1(&self) -> bool { + self.get_align().map(|n| n.t.get() > 1).unwrap_or(false) + } + + /// When deriving `Unaligned`, validate that the decorated type has no + /// `#[repr(align(N))]` attribute where `N > 1`. If no such attribute exists + /// (including if `N == 1`), this returns `Ok(())`, and otherwise it returns + /// a descriptive error. + pub(crate) fn unaligned_validate_no_align_gt_1(&self) -> Result<(), Error> { + if let Some(n) = self.get_align().filter(|n| n.t.get() > 1) { + Err(Error::new( + n.span, + "cannot derive `Unaligned` on type with alignment greater than 1", + )) + } else { + Ok(()) + } + } +} + +impl<Prim> Repr<Prim, NonZeroU32> { + /// Does `self` describe a `#[repr(packed)]` or `#[repr(packed(1))]` type? + pub(crate) fn is_packed_1(&self) -> bool { + self.get_packed().map(|n| n.get() == 1).unwrap_or(false) + } +} + +impl<Packed> Repr<PrimitiveRepr, Packed> { + fn get_primitive(&self) -> Option<&PrimitiveRepr> { + use CompoundRepr::*; + use Repr::*; + if let Compound(Spanned { t: Primitive(p), span: _ }, _align) = self { + Some(p) + } else { + None + } + } + + /// Does `self` describe a `#[repr(u8)]` type? + pub(crate) fn is_u8(&self) -> bool { + matches!(self.get_primitive(), Some(PrimitiveRepr::U8)) + } + + /// Does `self` describe a `#[repr(i8)]` type? + pub(crate) fn is_i8(&self) -> bool { + matches!(self.get_primitive(), Some(PrimitiveRepr::I8)) + } +} + +impl<Prim, Packed> ToTokens for Repr<Prim, Packed> +where + Prim: With<PrimitiveRepr> + Copy, + Packed: With<NonZeroU32> + Copy, +{ + fn to_tokens(&self, ts: &mut TokenStream) { + use Repr::*; + match self { + Transparent(span) => ts.append_all(quote_spanned! { *span=> #[repr(transparent)] }), + Compound(repr, align) => { + repr.to_tokens(ts); + if let Some(align) = align { + align.to_tokens(ts); + } + } + } + } +} + +impl<Prim: With<PrimitiveRepr> + Copy> ToTokens for Spanned<CompoundRepr<Prim>> { + fn to_tokens(&self, ts: &mut TokenStream) { + use CompoundRepr::*; + match &self.t { + C => ts.append_all(quote_spanned! { self.span=> #[repr(C)] }), + Rust => ts.append_all(quote_spanned! { self.span=> #[repr(Rust)] }), + Primitive(prim) => prim.with(|prim| Spanned::new(prim, self.span).to_tokens(ts)), + } + } +} + +impl ToTokens for Spanned<PrimitiveRepr> { + fn to_tokens(&self, ts: &mut TokenStream) { + use PrimitiveRepr::*; + match self.t { + U8 => ts.append_all(quote_spanned! { self.span => #[repr(u8)] }), + U16 => ts.append_all(quote_spanned! { self.span => #[repr(u16)] }), + U32 => ts.append_all(quote_spanned! { self.span => #[repr(u32)] }), + U64 => ts.append_all(quote_spanned! { self.span => #[repr(u64)] }), + U128 => ts.append_all(quote_spanned! { self.span => #[repr(u128)] }), + Usize => ts.append_all(quote_spanned! { self.span => #[repr(usize)] }), + I8 => ts.append_all(quote_spanned! { self.span => #[repr(i8)] }), + I16 => ts.append_all(quote_spanned! { self.span => #[repr(i16)] }), + I32 => ts.append_all(quote_spanned! { self.span => #[repr(i32)] }), + I64 => ts.append_all(quote_spanned! { self.span => #[repr(i64)] }), + I128 => ts.append_all(quote_spanned! { self.span => #[repr(i128)] }), + Isize => ts.append_all(quote_spanned! { self.span => #[repr(isize)] }), + } + } +} + +impl<Packed: With<NonZeroU32> + Copy> ToTokens for Spanned<AlignRepr<Packed>> { + fn to_tokens(&self, ts: &mut TokenStream) { + use AlignRepr::*; + // We use `syn::Index` instead of `u32` because `quote_spanned!` + // serializes `u32` literals as `123u32`, not just `123`. Rust doesn't + // recognize that as a valid argument to `#[repr(align(...))]` or + // `#[repr(packed(...))]`. + let to_index = |n: NonZeroU32| syn::Index { index: n.get(), span: self.span }; + match self.t { + Packed(n) => n.with(|n| { + let n = to_index(n); + ts.append_all(quote_spanned! { self.span => #[repr(packed(#n))] }) + }), + Align(n) => { + let n = to_index(n); + ts.append_all(quote_spanned! { self.span => #[repr(align(#n))] }) + } + } + } +} + +/// The result of parsing a single `#[repr(...)]` attribute or a single +/// directive inside a compound `#[repr(..., ...)]` attribute. +#[derive(Copy, Clone, PartialEq, Eq)] +#[cfg_attr(test, derive(Debug))] +pub(crate) enum RawRepr { + Transparent, + C, + Rust, + U8, + U16, + U32, + U64, + U128, + Usize, + I8, + I16, + I32, + I64, + I128, + Isize, + Align(NonZeroU32), + PackedN(NonZeroU32), + Packed, +} + +/// The error from converting from a `RawRepr`. +#[cfg_attr(test, derive(Debug, Eq, PartialEq))] +pub(crate) enum FromRawReprError<E> { + /// The `RawRepr` doesn't affect the high-level repr we're parsing (e.g. + /// it's `align(...)` and we're parsing a `CompoundRepr`). + None, + /// The `RawRepr` is invalid for the high-level repr we're parsing (e.g. + /// it's `packed` repr and we're parsing an `AlignRepr` for an enum type). + Err(E), +} + +/// The representation hint is not supported for the decorated type. +#[cfg_attr(test, derive(Copy, Clone, Debug, Eq, PartialEq))] +pub(crate) struct UnsupportedReprError; + +impl<Prim: With<PrimitiveRepr>> TryFrom<RawRepr> for CompoundRepr<Prim> { + type Error = FromRawReprError<UnsupportedReprError>; + fn try_from( + raw: RawRepr, + ) -> Result<CompoundRepr<Prim>, FromRawReprError<UnsupportedReprError>> { + use RawRepr::*; + match raw { + C => Ok(CompoundRepr::C), + Rust => Ok(CompoundRepr::Rust), + raw @ (U8 | U16 | U32 | U64 | U128 | Usize | I8 | I16 | I32 | I64 | I128 | Isize) => { + Prim::try_with_or( + || match raw { + U8 => Ok(PrimitiveRepr::U8), + U16 => Ok(PrimitiveRepr::U16), + U32 => Ok(PrimitiveRepr::U32), + U64 => Ok(PrimitiveRepr::U64), + U128 => Ok(PrimitiveRepr::U128), + Usize => Ok(PrimitiveRepr::Usize), + I8 => Ok(PrimitiveRepr::I8), + I16 => Ok(PrimitiveRepr::I16), + I32 => Ok(PrimitiveRepr::I32), + I64 => Ok(PrimitiveRepr::I64), + I128 => Ok(PrimitiveRepr::I128), + Isize => Ok(PrimitiveRepr::Isize), + Transparent | C | Rust | Align(_) | PackedN(_) | Packed => { + Err(UnsupportedReprError) + } + }, + UnsupportedReprError, + ) + .map(CompoundRepr::Primitive) + .map_err(FromRawReprError::Err) + } + Transparent | Align(_) | PackedN(_) | Packed => Err(FromRawReprError::None), + } + } +} + +impl<Pcked: With<NonZeroU32>> TryFrom<RawRepr> for AlignRepr<Pcked> { + type Error = FromRawReprError<UnsupportedReprError>; + fn try_from(raw: RawRepr) -> Result<AlignRepr<Pcked>, FromRawReprError<UnsupportedReprError>> { + use RawRepr::*; + match raw { + Packed | PackedN(_) => Pcked::try_with_or( + || match raw { + Packed => Ok(NonZeroU32::new(1).unwrap()), + PackedN(n) => Ok(n), + U8 | U16 | U32 | U64 | U128 | Usize | I8 | I16 | I32 | I64 | I128 | Isize + | Transparent | C | Rust | Align(_) => Err(UnsupportedReprError), + }, + UnsupportedReprError, + ) + .map(AlignRepr::Packed) + .map_err(FromRawReprError::Err), + Align(n) => Ok(AlignRepr::Align(n)), + U8 | U16 | U32 | U64 | U128 | Usize | I8 | I16 | I32 | I64 | I128 | Isize + | Transparent | C | Rust => Err(FromRawReprError::None), + } + } +} + +/// The error from extracting a high-level repr type from a list of `RawRepr`s. +#[cfg_attr(test, derive(Copy, Clone, Debug, Eq, PartialEq))] +enum FromRawReprsError<E> { + /// One of the `RawRepr`s is invalid for the high-level repr we're parsing + /// (e.g. there's a `packed` repr and we're parsing an `AlignRepr` for an + /// enum type). + Single(E), + /// Two `RawRepr`s appear which both affect the high-level repr we're + /// parsing (e.g., the list is `#[repr(align(2), packed)]`). Note that we + /// conservatively treat redundant reprs as conflicting (e.g. + /// `#[repr(packed, packed)]`). + Conflict, +} + +/// Tries to extract a high-level repr from a list of `RawRepr`s. +fn try_from_raw_reprs<'a, E, R: TryFrom<RawRepr, Error = FromRawReprError<E>>>( + r: impl IntoIterator<Item = &'a Spanned<RawRepr>>, +) -> Result<Option<Spanned<R>>, Spanned<FromRawReprsError<E>>> { + // Walk the list of `RawRepr`s and attempt to convert each to an `R`. Bail + // if we find any errors. If we find more than one which converts to an `R`, + // bail with a `Conflict` error. + r.into_iter().try_fold(None, |found: Option<Spanned<R>>, raw| { + let new = match Spanned::<R>::try_from(*raw) { + Ok(r) => r, + // This `RawRepr` doesn't convert to an `R`, so keep the current + // found `R`, if any. + Err(FromRawReprError::None) => return Ok(found), + // This repr is unsupported for the decorated type (e.g. + // `repr(packed)` on an enum). + Err(FromRawReprError::Err(Spanned { t: err, span })) => { + return Err(Spanned::new(FromRawReprsError::Single(err), span)) + } + }; + + if let Some(found) = found { + // We already found an `R`, but this `RawRepr` also converts to an + // `R`, so that's a conflict. + // + // `Span::join` returns `None` if the two spans are from different + // files or if we're not on the nightly compiler. In that case, just + // use `new`'s span. + let span = found.span.join(new.span).unwrap_or(new.span); + Err(Spanned::new(FromRawReprsError::Conflict, span)) + } else { + Ok(Some(new)) + } + }) +} + +/// The error returned from [`Repr::from_attrs`]. +#[cfg_attr(test, derive(Copy, Clone, Debug, Eq, PartialEq))] +enum FromAttrsError { + FromRawReprs(FromRawReprsError<UnsupportedReprError>), + Unrecognized, +} + +impl From<FromRawReprsError<UnsupportedReprError>> for FromAttrsError { + fn from(err: FromRawReprsError<UnsupportedReprError>) -> FromAttrsError { + FromAttrsError::FromRawReprs(err) + } +} + +impl From<UnrecognizedReprError> for FromAttrsError { + fn from(_err: UnrecognizedReprError) -> FromAttrsError { + FromAttrsError::Unrecognized + } +} + +impl From<Spanned<FromAttrsError>> for Error { + fn from(err: Spanned<FromAttrsError>) -> Error { + let Spanned { t: err, span } = err; + match err { + FromAttrsError::FromRawReprs(FromRawReprsError::Single( + _err @ UnsupportedReprError, + )) => Error::new(span, "unsupported representation hint for the decorated type"), + FromAttrsError::FromRawReprs(FromRawReprsError::Conflict) => { + // NOTE: This says "another" rather than "a preceding" because + // when one of the reprs involved is `transparent`, we detect + // that condition in `Repr::from_attrs`, and at that point we + // can't tell which repr came first, so we might report this on + // the first involved repr rather than the second, third, etc. + Error::new(span, "this conflicts with another representation hint") + } + FromAttrsError::Unrecognized => Error::new(span, "unrecognized representation hint"), + } + } +} + +impl<Prim, Packed> Repr<Prim, Packed> { + fn from_attrs_inner(attrs: &[Attribute]) -> Result<Repr<Prim, Packed>, Spanned<FromAttrsError>> + where + Prim: With<PrimitiveRepr>, + Packed: With<NonZeroU32>, + { + let raw_reprs = RawRepr::from_attrs(attrs).map_err(Spanned::from)?; + + let transparent = { + let mut transparents = raw_reprs.iter().filter_map(|Spanned { t, span }| match t { + RawRepr::Transparent => Some(span), + _ => None, + }); + let first = transparents.next(); + let second = transparents.next(); + match (first, second) { + (None, None) => None, + (Some(span), None) => Some(*span), + (Some(_), Some(second)) => { + return Err(Spanned::new( + FromAttrsError::FromRawReprs(FromRawReprsError::Conflict), + *second, + )) + } + // An iterator can't produce a value only on the second call to + // `.next()`. + (None, Some(_)) => unreachable!(), + } + }; + + let compound: Option<Spanned<CompoundRepr<Prim>>> = + try_from_raw_reprs(raw_reprs.iter()).map_err(Spanned::from)?; + let align: Option<Spanned<AlignRepr<Packed>>> = + try_from_raw_reprs(raw_reprs.iter()).map_err(Spanned::from)?; + + if let Some(span) = transparent { + if compound.is_some() || align.is_some() { + // Arbitrarily report the problem on the `transparent` span. Any + // span will do. + return Err(Spanned::new(FromRawReprsError::Conflict.into(), span)); + } + + Ok(Repr::Transparent(span)) + } else { + Ok(Repr::Compound( + compound.unwrap_or(Spanned::new(CompoundRepr::Rust, Span::call_site())), + align, + )) + } + } +} + +impl<Prim, Packed> Repr<Prim, Packed> { + pub(crate) fn from_attrs(attrs: &[Attribute]) -> Result<Repr<Prim, Packed>, Error> + where + Prim: With<PrimitiveRepr>, + Packed: With<NonZeroU32>, + { + Repr::from_attrs_inner(attrs).map_err(Into::into) + } +} + +/// The representation hint could not be parsed or was unrecognized. +struct UnrecognizedReprError; + +impl RawRepr { + fn from_attrs( + attrs: &[Attribute], + ) -> Result<Vec<Spanned<RawRepr>>, Spanned<UnrecognizedReprError>> { + let mut reprs = Vec::new(); + for attr in attrs { + // Ignore documentation attributes. + if attr.path().is_ident("doc") { + continue; + } + if let Meta::List(ref meta_list) = attr.meta { + if meta_list.path.is_ident("repr") { + let parsed: Punctuated<Meta, Comma> = + match meta_list.parse_args_with(Punctuated::parse_terminated) { + Ok(parsed) => parsed, + Err(_) => { + return Err(Spanned::new( + UnrecognizedReprError, + meta_list.tokens.span(), + )) + } + }; + for meta in parsed { + let s = meta.span(); + reprs.push( + RawRepr::from_meta(&meta) + .map(|r| Spanned::new(r, s)) + .map_err(|e| Spanned::new(e, s))?, + ); + } + } + } + } + + Ok(reprs) + } + + fn from_meta(meta: &Meta) -> Result<RawRepr, UnrecognizedReprError> { + let (path, list) = match meta { + Meta::Path(path) => (path, None), + Meta::List(list) => (&list.path, Some(list)), + _ => return Err(UnrecognizedReprError), + }; + + let ident = path.get_ident().ok_or(UnrecognizedReprError)?; + + // Only returns `Ok` for non-zero power-of-two values. + let parse_nzu64 = |list: &MetaList| { + list.parse_args::<LitInt>() + .and_then(|int| int.base10_parse::<NonZeroU32>()) + .map_err(|_| UnrecognizedReprError) + .and_then(|nz| { + if nz.get().is_power_of_two() { + Ok(nz) + } else { + Err(UnrecognizedReprError) + } + }) + }; + + use RawRepr::*; + Ok(match (ident.to_string().as_str(), list) { + ("u8", None) => U8, + ("u16", None) => U16, + ("u32", None) => U32, + ("u64", None) => U64, + ("u128", None) => U128, + ("usize", None) => Usize, + ("i8", None) => I8, + ("i16", None) => I16, + ("i32", None) => I32, + ("i64", None) => I64, + ("i128", None) => I128, + ("isize", None) => Isize, + ("C", None) => C, + ("transparent", None) => Transparent, + ("Rust", None) => Rust, + ("packed", None) => Packed, + ("packed", Some(list)) => PackedN(parse_nzu64(list)?), + ("align", Some(list)) => Align(parse_nzu64(list)?), + _ => return Err(UnrecognizedReprError), + }) + } +} + +pub(crate) use util::*; +mod util { + use super::*; + /// A value with an associated span. + #[derive(Copy, Clone)] + #[cfg_attr(test, derive(Debug))] + pub(crate) struct Spanned<T> { + pub(crate) t: T, + pub(crate) span: Span, + } + + impl<T> Spanned<T> { + pub(super) fn new(t: T, span: Span) -> Spanned<T> { + Spanned { t, span } + } + + pub(super) fn from<U>(s: Spanned<U>) -> Spanned<T> + where + T: From<U>, + { + let Spanned { t: u, span } = s; + Spanned::new(u.into(), span) + } + + /// Delegates to `T: TryFrom`, preserving span information in both the + /// success and error cases. + pub(super) fn try_from<E, U>( + u: Spanned<U>, + ) -> Result<Spanned<T>, FromRawReprError<Spanned<E>>> + where + T: TryFrom<U, Error = FromRawReprError<E>>, + { + let Spanned { t: u, span } = u; + T::try_from(u).map(|t| Spanned { t, span }).map_err(|err| match err { + FromRawReprError::None => FromRawReprError::None, + FromRawReprError::Err(e) => FromRawReprError::Err(Spanned::new(e, span)), + }) + } + } + + // Used to permit implementing `With<T> for T: Inhabited` and for + // `Infallible` without a blanket impl conflict. + pub(crate) trait Inhabited {} + impl Inhabited for PrimitiveRepr {} + impl Inhabited for NonZeroU32 {} + + pub(crate) trait With<T> { + fn with<O, F: FnOnce(T) -> O>(self, f: F) -> O; + fn try_with_or<E, F: FnOnce() -> Result<T, E>>(f: F, err: E) -> Result<Self, E> + where + Self: Sized; + } + + impl<T: Inhabited> With<T> for T { + fn with<O, F: FnOnce(T) -> O>(self, f: F) -> O { + f(self) + } + + fn try_with_or<E, F: FnOnce() -> Result<T, E>>(f: F, _err: E) -> Result<Self, E> { + f() + } + } + + impl<T> With<T> for Infallible { + fn with<O, F: FnOnce(T) -> O>(self, _f: F) -> O { + match self {} + } + + fn try_with_or<E, F: FnOnce() -> Result<T, E>>(_f: F, err: E) -> Result<Self, E> { + Err(err) + } + } +} + +#[cfg(test)] +mod tests { + use syn::parse_quote; + + use super::*; + + impl<T> From<T> for Spanned<T> { + fn from(t: T) -> Spanned<T> { + Spanned::new(t, Span::call_site()) + } + } + + // We ignore spans for equality in testing since real spans are hard to + // synthesize and don't implement `PartialEq`. + impl<T: PartialEq> PartialEq for Spanned<T> { + fn eq(&self, other: &Spanned<T>) -> bool { + self.t.eq(&other.t) + } + } + + impl<T: Eq> Eq for Spanned<T> {} + + impl<Prim: PartialEq, Packed: PartialEq> PartialEq for Repr<Prim, Packed> { + fn eq(&self, other: &Repr<Prim, Packed>) -> bool { + match (self, other) { + (Repr::Transparent(_), Repr::Transparent(_)) => true, + (Repr::Compound(sc, sa), Repr::Compound(oc, oa)) => (sc, sa) == (oc, oa), + _ => false, + } + } + } + + fn s() -> Span { + Span::call_site() + } + + #[test] + fn test() { + // Test that a given `#[repr(...)]` attribute parses and returns the + // given `Repr` or error. + macro_rules! test { + ($(#[$attr:meta])* => $repr:expr) => { + test!(@inner $(#[$attr])* => Repr => Ok($repr)); + }; + // In the error case, the caller must explicitly provide the name of + // the `Repr` type to assist in type inference. + (@error $(#[$attr:meta])* => $typ:ident => $repr:expr) => { + test!(@inner $(#[$attr])* => $typ => Err($repr)); + }; + (@inner $(#[$attr:meta])* => $typ:ident => $repr:expr) => { + let attr: Attribute = parse_quote!($(#[$attr])*); + let mut got = $typ::from_attrs_inner(&[attr]); + let expect: Result<Repr<_, _>, _> = $repr; + if false { + // Force Rust to infer `got` as having the same type as + // `expect`. + got = expect; + } + assert_eq!(got, expect, stringify!($(#[$attr])*)); + }; + } + + use AlignRepr::*; + use CompoundRepr::*; + use PrimitiveRepr::*; + let nz = |n: u32| NonZeroU32::new(n).unwrap(); + + test!(#[repr(transparent)] => StructUnionRepr::Transparent(s())); + test!(#[repr()] => StructUnionRepr::Compound(Rust.into(), None)); + test!(#[repr(packed)] => StructUnionRepr::Compound(Rust.into(), Some(Packed(nz(1)).into()))); + test!(#[repr(packed(2))] => StructUnionRepr::Compound(Rust.into(), Some(Packed(nz(2)).into()))); + test!(#[repr(align(1))] => StructUnionRepr::Compound(Rust.into(), Some(Align(nz(1)).into()))); + test!(#[repr(align(2))] => StructUnionRepr::Compound(Rust.into(), Some(Align(nz(2)).into()))); + test!(#[repr(C)] => StructUnionRepr::Compound(C.into(), None)); + test!(#[repr(C, packed)] => StructUnionRepr::Compound(C.into(), Some(Packed(nz(1)).into()))); + test!(#[repr(C, packed(2))] => StructUnionRepr::Compound(C.into(), Some(Packed(nz(2)).into()))); + test!(#[repr(C, align(1))] => StructUnionRepr::Compound(C.into(), Some(Align(nz(1)).into()))); + test!(#[repr(C, align(2))] => StructUnionRepr::Compound(C.into(), Some(Align(nz(2)).into()))); + + test!(#[repr(transparent)] => EnumRepr::Transparent(s())); + test!(#[repr()] => EnumRepr::Compound(Rust.into(), None)); + test!(#[repr(align(1))] => EnumRepr::Compound(Rust.into(), Some(Align(nz(1)).into()))); + test!(#[repr(align(2))] => EnumRepr::Compound(Rust.into(), Some(Align(nz(2)).into()))); + + macro_rules! for_each_compound_repr { + ($($r:tt => $var:expr),*) => { + $( + test!(#[repr($r)] => EnumRepr::Compound($var.into(), None)); + test!(#[repr($r, align(1))] => EnumRepr::Compound($var.into(), Some(Align(nz(1)).into()))); + test!(#[repr($r, align(2))] => EnumRepr::Compound($var.into(), Some(Align(nz(2)).into()))); + )* + } + } + + for_each_compound_repr!( + C => C, + u8 => Primitive(U8), + u16 => Primitive(U16), + u32 => Primitive(U32), + u64 => Primitive(U64), + usize => Primitive(Usize), + i8 => Primitive(I8), + i16 => Primitive(I16), + i32 => Primitive(I32), + i64 => Primitive(I64), + isize => Primitive(Isize) + ); + + use FromAttrsError::*; + use FromRawReprsError::*; + + // Run failure tests which are valid for both `StructUnionRepr` and + // `EnumRepr`. + macro_rules! for_each_repr_type { + ($($repr:ident),*) => { + $( + // Invalid packed or align attributes + test!(@error #[repr(packed(0))] => $repr => Unrecognized.into()); + test!(@error #[repr(packed(3))] => $repr => Unrecognized.into()); + test!(@error #[repr(align(0))] => $repr => Unrecognized.into()); + test!(@error #[repr(align(3))] => $repr => Unrecognized.into()); + + // Conflicts + test!(@error #[repr(transparent, transparent)] => $repr => FromRawReprs(Conflict).into()); + test!(@error #[repr(transparent, C)] => $repr => FromRawReprs(Conflict).into()); + test!(@error #[repr(transparent, Rust)] => $repr => FromRawReprs(Conflict).into()); + + test!(@error #[repr(C, transparent)] => $repr => FromRawReprs(Conflict).into()); + test!(@error #[repr(C, C)] => $repr => FromRawReprs(Conflict).into()); + test!(@error #[repr(C, Rust)] => $repr => FromRawReprs(Conflict).into()); + + test!(@error #[repr(Rust, transparent)] => $repr => FromRawReprs(Conflict).into()); + test!(@error #[repr(Rust, C)] => $repr => FromRawReprs(Conflict).into()); + test!(@error #[repr(Rust, Rust)] => $repr => FromRawReprs(Conflict).into()); + )* + } + } + + for_each_repr_type!(StructUnionRepr, EnumRepr); + + // Enum-specific conflicts. + // + // We don't bother to test every combination since that would be a huge + // number (enums can have primitive reprs u8, u16, u32, u64, usize, i8, + // i16, i32, i64, and isize). Instead, since the conflict logic doesn't + // care what specific value of `PrimitiveRepr` is present, we assume + // that testing against u8 alone is fine. + test!(@error #[repr(transparent, u8)] => EnumRepr => FromRawReprs(Conflict).into()); + test!(@error #[repr(u8, transparent)] => EnumRepr => FromRawReprs(Conflict).into()); + test!(@error #[repr(C, u8)] => EnumRepr => FromRawReprs(Conflict).into()); + test!(@error #[repr(u8, C)] => EnumRepr => FromRawReprs(Conflict).into()); + test!(@error #[repr(Rust, u8)] => EnumRepr => FromRawReprs(Conflict).into()); + test!(@error #[repr(u8, Rust)] => EnumRepr => FromRawReprs(Conflict).into()); + test!(@error #[repr(u8, u8)] => EnumRepr => FromRawReprs(Conflict).into()); + + // Illegal struct/union reprs + test!(@error #[repr(u8)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(u16)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(u32)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(u64)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(usize)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(i8)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(i16)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(i32)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(i64)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(isize)] => StructUnionRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + + // Illegal enum reprs + test!(@error #[repr(packed)] => EnumRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(packed(1))] => EnumRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + test!(@error #[repr(packed(2))] => EnumRepr => FromRawReprs(Single(UnsupportedReprError)).into()); + } +} diff --git a/rust/zerocopy-derive/util.rs b/rust/zerocopy-derive/util.rs new file mode 100644 index 000000000000..5c5e9d3bdcb8 --- /dev/null +++ b/rust/zerocopy-derive/util.rs @@ -0,0 +1,863 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2019 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use std::num::NonZeroU32; + +use proc_macro2::{Span, TokenStream}; +use quote::{quote, quote_spanned, ToTokens}; +use syn::{ + parse_quote, spanned::Spanned as _, Data, DataEnum, DataStruct, DataUnion, DeriveInput, Error, + Expr, ExprLit, Field, GenericParam, Ident, Index, Lit, LitStr, Meta, Path, Type, Variant, + Visibility, WherePredicate, +}; + +use crate::repr::{CompoundRepr, EnumRepr, PrimitiveRepr, Repr, Spanned}; + +pub(crate) struct Ctx { + pub(crate) ast: DeriveInput, + pub(crate) zerocopy_crate: Path, + + // The value of the last `#[zerocopy(on_error = ...)]` attribute, or `false` + // if none is provided. + pub(crate) skip_on_error: bool, + + // The span of the last `#[zerocopy(on_error = ...)]` attribute, if any. + pub(crate) on_error_span: Option<proc_macro2::Span>, +} + +impl Ctx { + /// Attempt to extract a crate path from the provided attributes. Defaults to + /// `::zerocopy` if not found. + pub(crate) fn try_from_derive_input(ast: DeriveInput) -> Result<Self, Error> { + let mut path = parse_quote!(::zerocopy); + let mut skip_on_error = false; + let mut on_error_span = None; + + for attr in &ast.attrs { + if let Meta::List(ref meta_list) = attr.meta { + if meta_list.path.is_ident("zerocopy") { + attr.parse_nested_meta(|meta| { + if meta.path.is_ident("crate") { + let expr = meta.value().and_then(|value| value.parse()); + if let Ok(Expr::Lit(ExprLit { lit: Lit::Str(lit), .. })) = expr { + if let Ok(path_lit) = lit.parse::<Ident>() { + path = parse_quote!(::#path_lit); + return Ok(()); + } + } + + return Err(Error::new( + Span::call_site(), + "`crate` attribute requires a path as the value", + )); + } + + if meta.path.is_ident("on_error") { + on_error_span = Some(meta.path.span()); + let value = meta.value()?; + let s: LitStr = value.parse()?; + match s.value().as_str() { + "skip" => skip_on_error = true, + "fail" => skip_on_error = false, + _ => return Err(Error::new( + s.span(), + "unrecognized value for `on_error` attribute from `zerocopy`; expected `skip` or `fail`", + )), + } + return Ok(()); + } + + Err(Error::new( + Span::call_site(), + format!( + "unknown attribute encountered: {}", + meta.path.into_token_stream() + ), + )) + })?; + } + } + } + + Ok(Self { ast, zerocopy_crate: path, skip_on_error, on_error_span }) + } + + pub(crate) fn with_input(&self, input: &DeriveInput) -> Self { + Self { + ast: input.clone(), + zerocopy_crate: self.zerocopy_crate.clone(), + skip_on_error: self.skip_on_error, + on_error_span: self.on_error_span, + } + } + + pub(crate) fn skip_on_error(mut self) -> Self { + self.skip_on_error = true; + self + } + + pub(crate) fn core_path(&self) -> TokenStream { + let zerocopy_crate = &self.zerocopy_crate; + quote!(#zerocopy_crate::util::macro_util::core_reexport) + } + + pub(crate) fn cfg_compile_error(&self) -> TokenStream { + // By checking both during the compilation of the proc macro *and* in + // the generated code, we ensure that `--cfg + // zerocopy_unstable_linux` need only be passed *either* when + // compiling this crate *or* when compiling the user's crate. The former + // is preferable, but in some situations (such as when cross-compiling + // using `cargo build --target`), it doesn't get propagated to this + // crate's build by default. + if cfg!(zerocopy_unstable_linux) { + quote!() + } else if let Some(span) = self.on_error_span { + let core = self.core_path(); + let error_message = + "`on_error` is experimental; pass '--cfg zerocopy_unstable_linux' to enable"; + quote::quote_spanned! {span=> + #[allow(unused_attributes, unexpected_cfgs)] + const _: () = { + #[cfg(not(zerocopy_unstable_linux))] + #core::compile_error!(#error_message); + }; + } + } else { + quote!() + } + } + + pub(crate) fn error_or_skip<E>(&self, error: E) -> Result<TokenStream, E> { + if self.skip_on_error { + Ok(self.cfg_compile_error()) + } else { + Err(error) + } + } +} + +pub(crate) trait DataExt { + /// Extracts the names and types of all fields. For enums, extracts the + /// names and types of fields from each variant. For tuple structs, the + /// names are the indices used to index into the struct (ie, `0`, `1`, etc). + /// + /// FIXME: Extracting field names for enums doesn't really make sense. Types + /// makes sense because we don't care about where they live - we just care + /// about transitive ownership. But for field names, we'd only use them when + /// generating is_bit_valid, which cares about where they live. + fn fields(&self) -> Vec<(&Visibility, TokenStream, &Type)>; + + fn variants(&self) -> Vec<(Option<&Variant>, Vec<(&Visibility, TokenStream, &Type)>)>; + + fn tag(&self) -> Option<Ident>; +} + +impl DataExt for Data { + fn fields(&self) -> Vec<(&Visibility, TokenStream, &Type)> { + match self { + Data::Struct(strc) => strc.fields(), + Data::Enum(enm) => enm.fields(), + Data::Union(un) => un.fields(), + } + } + + fn variants(&self) -> Vec<(Option<&Variant>, Vec<(&Visibility, TokenStream, &Type)>)> { + match self { + Data::Struct(strc) => strc.variants(), + Data::Enum(enm) => enm.variants(), + Data::Union(un) => un.variants(), + } + } + + fn tag(&self) -> Option<Ident> { + match self { + Data::Struct(strc) => strc.tag(), + Data::Enum(enm) => enm.tag(), + Data::Union(un) => un.tag(), + } + } +} + +impl DataExt for DataStruct { + fn fields(&self) -> Vec<(&Visibility, TokenStream, &Type)> { + map_fields(&self.fields) + } + + fn variants(&self) -> Vec<(Option<&Variant>, Vec<(&Visibility, TokenStream, &Type)>)> { + vec![(None, self.fields())] + } + + fn tag(&self) -> Option<Ident> { + None + } +} + +impl DataExt for DataEnum { + fn fields(&self) -> Vec<(&Visibility, TokenStream, &Type)> { + map_fields(self.variants.iter().flat_map(|var| &var.fields)) + } + + fn variants(&self) -> Vec<(Option<&Variant>, Vec<(&Visibility, TokenStream, &Type)>)> { + self.variants.iter().map(|var| (Some(var), map_fields(&var.fields))).collect() + } + + fn tag(&self) -> Option<Ident> { + Some(Ident::new("___ZerocopyTag", Span::call_site())) + } +} + +impl DataExt for DataUnion { + fn fields(&self) -> Vec<(&Visibility, TokenStream, &Type)> { + map_fields(&self.fields.named) + } + + fn variants(&self) -> Vec<(Option<&Variant>, Vec<(&Visibility, TokenStream, &Type)>)> { + vec![(None, self.fields())] + } + + fn tag(&self) -> Option<Ident> { + None + } +} + +fn map_fields<'a>( + fields: impl 'a + IntoIterator<Item = &'a Field>, +) -> Vec<(&'a Visibility, TokenStream, &'a Type)> { + fields + .into_iter() + .enumerate() + .map(|(idx, f)| { + ( + &f.vis, + f.ident + .as_ref() + .map(ToTokens::to_token_stream) + .unwrap_or_else(|| Index::from(idx).to_token_stream()), + &f.ty, + ) + }) + .collect() +} + +pub(crate) fn to_ident_str(t: &impl ToString) -> String { + let s = t.to_string(); + if let Some(stripped) = s.strip_prefix("r#") { + stripped.to_string() + } else { + s + } +} + +/// This enum describes what kind of padding check needs to be generated for the +/// associated impl. +pub(crate) enum PaddingCheck { + /// Check that the sum of the fields' sizes exactly equals the struct's + /// size. + Struct, + /// Check that a `repr(C)` struct has no padding. + ReprCStruct, + /// Check that the size of each field exactly equals the union's size. + Union, + /// Check that every variant of the enum contains no padding. + /// + /// Because doing so requires a tag enum, this padding check requires an + /// additional `TokenStream` which defines the tag enum as `___ZerocopyTag`. + Enum { tag_type_definition: TokenStream }, +} + +impl PaddingCheck { + /// Returns the idents of the trait to use and the macro to call in order to + /// validate that a type passes the relevant padding check. + pub(crate) fn validator_trait_and_macro_idents(&self) -> (Ident, Ident) { + let (trt, mcro) = match self { + PaddingCheck::Struct => ("PaddingFree", "struct_padding"), + PaddingCheck::ReprCStruct => ("DynamicPaddingFree", "repr_c_struct_has_padding"), + PaddingCheck::Union => ("PaddingFree", "union_padding"), + PaddingCheck::Enum { .. } => ("PaddingFree", "enum_padding"), + }; + + let trt = Ident::new(trt, Span::call_site()); + let mcro = Ident::new(mcro, Span::call_site()); + (trt, mcro) + } + + /// Sometimes performing the padding check requires some additional + /// "context" code. For enums, this is the definition of the tag enum. + pub(crate) fn validator_macro_context(&self) -> Option<&TokenStream> { + match self { + PaddingCheck::Struct | PaddingCheck::ReprCStruct | PaddingCheck::Union => None, + PaddingCheck::Enum { tag_type_definition } => Some(tag_type_definition), + } + } +} + +#[derive(Clone)] +pub(crate) enum Trait { + KnownLayout, + HasTag, + HasField { + variant_id: Box<Expr>, + field: Box<Type>, + field_id: Box<Expr>, + }, + ProjectField { + variant_id: Box<Expr>, + field: Box<Type>, + field_id: Box<Expr>, + invariants: Box<Type>, + }, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned, + Sized, + ByteHash, + ByteEq, + SplitAt, +} + +impl ToTokens for Trait { + fn to_tokens(&self, tokens: &mut TokenStream) { + // According to [1], the format of the derived `Debug`` output is not + // stable and therefore not guaranteed to represent the variant names. + // Indeed with the (unstable) `fmt-debug` compiler flag [2], it can + // return only a minimalized output or empty string. To make sure this + // code will work in the future and independent of the compiler flag, we + // translate the variants to their names manually here. + // + // [1] https://doc.rust-lang.org/1.81.0/std/fmt/trait.Debug.html#stability + // [2] https://doc.rust-lang.org/beta/unstable-book/compiler-flags/fmt-debug.html + let s = match self { + Trait::HasField { .. } => "HasField", + Trait::ProjectField { .. } => "ProjectField", + Trait::KnownLayout => "KnownLayout", + Trait::HasTag => "HasTag", + Trait::Immutable => "Immutable", + Trait::TryFromBytes => "TryFromBytes", + Trait::FromZeros => "FromZeros", + Trait::FromBytes => "FromBytes", + Trait::IntoBytes => "IntoBytes", + Trait::Unaligned => "Unaligned", + Trait::Sized => "Sized", + Trait::ByteHash => "ByteHash", + Trait::ByteEq => "ByteEq", + Trait::SplitAt => "SplitAt", + }; + let ident = Ident::new(s, Span::call_site()); + let arguments: Option<syn::AngleBracketedGenericArguments> = match self { + Trait::HasField { variant_id, field, field_id } => { + Some(parse_quote!(<#field, #variant_id, #field_id>)) + } + Trait::ProjectField { variant_id, field, field_id, invariants } => { + Some(parse_quote!(<#field, #invariants, #variant_id, #field_id>)) + } + Trait::KnownLayout + | Trait::HasTag + | Trait::Immutable + | Trait::TryFromBytes + | Trait::FromZeros + | Trait::FromBytes + | Trait::IntoBytes + | Trait::Unaligned + | Trait::Sized + | Trait::ByteHash + | Trait::ByteEq + | Trait::SplitAt => None, + }; + tokens.extend(quote!(#ident #arguments)); + } +} + +impl Trait { + pub(crate) fn crate_path(&self, ctx: &Ctx) -> Path { + let zerocopy_crate = &ctx.zerocopy_crate; + let core = ctx.core_path(); + match self { + Self::Sized => parse_quote!(#core::marker::#self), + _ => parse_quote!(#zerocopy_crate::#self), + } + } +} + +pub(crate) enum TraitBound { + Slf, + Other(Trait), +} + +pub(crate) enum FieldBounds<'a> { + None, + All(&'a [TraitBound]), + Trailing(&'a [TraitBound]), + Explicit(Vec<WherePredicate>), +} + +impl<'a> FieldBounds<'a> { + pub(crate) const ALL_SELF: FieldBounds<'a> = FieldBounds::All(&[TraitBound::Slf]); + pub(crate) const TRAILING_SELF: FieldBounds<'a> = FieldBounds::Trailing(&[TraitBound::Slf]); +} + +pub(crate) enum SelfBounds<'a> { + None, + All(&'a [Trait]), +} + +// FIXME(https://github.com/rust-lang/rust-clippy/issues/12908): This is a false +// positive. Explicit lifetimes are actually necessary here. +#[allow(clippy::needless_lifetimes)] +impl<'a> SelfBounds<'a> { + pub(crate) const SIZED: Self = Self::All(&[Trait::Sized]); +} + +/// Normalizes a slice of bounds by replacing [`TraitBound::Slf`] with `slf`. +pub(crate) fn normalize_bounds<'a>( + slf: &'a Trait, + bounds: &'a [TraitBound], +) -> impl 'a + Iterator<Item = Trait> { + bounds.iter().map(move |bound| match bound { + TraitBound::Slf => slf.clone(), + TraitBound::Other(trt) => trt.clone(), + }) +} + +pub(crate) struct ImplBlockBuilder<'a> { + ctx: &'a Ctx, + data: &'a dyn DataExt, + trt: Trait, + field_type_trait_bounds: FieldBounds<'a>, + self_type_trait_bounds: SelfBounds<'a>, + padding_check: Option<PaddingCheck>, + param_extras: Vec<GenericParam>, + inner_extras: Option<TokenStream>, + outer_extras: Option<TokenStream>, +} + +impl<'a> ImplBlockBuilder<'a> { + pub(crate) fn new( + ctx: &'a Ctx, + data: &'a dyn DataExt, + trt: Trait, + field_type_trait_bounds: FieldBounds<'a>, + ) -> Self { + Self { + ctx, + data, + trt, + field_type_trait_bounds, + self_type_trait_bounds: SelfBounds::None, + padding_check: None, + param_extras: Vec::new(), + inner_extras: None, + outer_extras: None, + } + } + + pub(crate) fn self_type_trait_bounds(mut self, self_type_trait_bounds: SelfBounds<'a>) -> Self { + self.self_type_trait_bounds = self_type_trait_bounds; + self + } + + pub(crate) fn padding_check<P: Into<Option<PaddingCheck>>>(mut self, padding_check: P) -> Self { + self.padding_check = padding_check.into(); + self + } + + pub(crate) fn param_extras(mut self, param_extras: Vec<GenericParam>) -> Self { + self.param_extras.extend(param_extras); + self + } + + pub(crate) fn inner_extras(mut self, inner_extras: TokenStream) -> Self { + self.inner_extras = Some(inner_extras); + self + } + + pub(crate) fn outer_extras<T: Into<Option<TokenStream>>>(mut self, outer_extras: T) -> Self { + self.outer_extras = outer_extras.into(); + self + } + + pub(crate) fn build(self) -> TokenStream { + // In this documentation, we will refer to this hypothetical struct: + // + // #[derive(FromBytes)] + // struct Foo<T, I: Iterator> + // where + // T: Copy, + // I: Clone, + // I::Item: Clone, + // { + // a: u8, + // b: T, + // c: I::Item, + // } + // + // We extract the field types, which in this case are `u8`, `T`, and + // `I::Item`. We re-use the existing parameters and where clauses. If + // `require_trait_bound == true` (as it is for `FromBytes), we add where + // bounds for each field's type: + // + // impl<T, I: Iterator> FromBytes for Foo<T, I> + // where + // T: Copy, + // I: Clone, + // I::Item: Clone, + // T: FromBytes, + // I::Item: FromBytes, + // { + // } + // + // NOTE: It is standard practice to only emit bounds for the type + // parameters themselves, not for field types based on those parameters + // (e.g., `T` vs `T::Foo`). For a discussion of why this is standard + // practice, see https://github.com/rust-lang/rust/issues/26925. + // + // The reason we diverge from this standard is that doing it that way + // for us would be unsound. E.g., consider a type, `T` where `T: + // FromBytes` but `T::Foo: !FromBytes`. It would not be sound for us to + // accept a type with a `T::Foo` field as `FromBytes` simply because `T: + // FromBytes`. + // + // While there's no getting around this requirement for us, it does have + // the pretty serious downside that, when lifetimes are involved, the + // trait solver ties itself in knots: + // + // #[derive(Unaligned)] + // #[repr(C)] + // struct Dup<'a, 'b> { + // a: PhantomData<&'a u8>, + // b: PhantomData<&'b u8>, + // } + // + // error[E0283]: type annotations required: cannot resolve `core::marker::PhantomData<&'a u8>: zerocopy::Unaligned` + // --> src/main.rs:6:10 + // | + // 6 | #[derive(Unaligned)] + // | ^^^^^^^^^ + // | + // = note: required by `zerocopy::Unaligned` + + let type_ident = &self.ctx.ast.ident; + let trait_path = self.trt.crate_path(self.ctx); + let fields = self.data.fields(); + let variants = self.data.variants(); + let tag = self.data.tag(); + let zerocopy_crate = &self.ctx.zerocopy_crate; + + fn bound_tt(ty: &Type, traits: impl Iterator<Item = Trait>, ctx: &Ctx) -> WherePredicate { + let traits = traits.map(|t| t.crate_path(ctx)); + parse_quote!(#ty: #(#traits)+*) + } + let field_type_bounds: Vec<_> = match (self.field_type_trait_bounds, &fields[..]) { + (FieldBounds::All(traits), _) => fields + .iter() + .map(|(_vis, _name, ty)| { + bound_tt(ty, normalize_bounds(&self.trt, traits), self.ctx) + }) + .collect(), + (FieldBounds::None, _) | (FieldBounds::Trailing(..), []) => vec![], + (FieldBounds::Trailing(traits), [.., last]) => { + vec![bound_tt(last.2, normalize_bounds(&self.trt, traits), self.ctx)] + } + (FieldBounds::Explicit(bounds), _) => bounds, + }; + + let padding_check_bound = self + .padding_check + .map(|check| { + // Parse the repr for `align` and `packed` modifiers. Note that + // `Repr::<PrimitiveRepr, NonZeroU32>` is more permissive than + // what Rust supports for structs, enums, or unions, and thus + // reliably extracts these modifiers for any kind of type. + let repr = + Repr::<PrimitiveRepr, NonZeroU32>::from_attrs(&self.ctx.ast.attrs).unwrap(); + let core = self.ctx.core_path(); + let option = quote! { #core::option::Option }; + let nonzero = quote! { #core::num::NonZeroUsize }; + let none = quote! { #option::None::<#nonzero> }; + let repr_align = + repr.get_align().map(|spanned| { + let n = spanned.t.get(); + quote_spanned! { spanned.span => (#nonzero::new(#n as usize)) } + }).unwrap_or(quote! { (#none) }); + let repr_packed = + repr.get_packed().map(|packed| { + let n = packed.get(); + quote! { (#nonzero::new(#n as usize)) } + }).unwrap_or(quote! { (#none) }); + let variant_types = variants.iter().map(|(_, fields)| { + let types = fields.iter().map(|(_vis, _name, ty)| ty); + quote!([#((#types)),*]) + }); + let validator_context = check.validator_macro_context(); + let (trt, validator_macro) = check.validator_trait_and_macro_idents(); + let t = tag.iter(); + parse_quote! { + (): #zerocopy_crate::util::macro_util::#trt< + Self, + { + #validator_context + #zerocopy_crate::#validator_macro!(Self, #repr_align, #repr_packed, #(#t,)* #(#variant_types),*) + } + > + } + }); + + let self_bounds: Option<WherePredicate> = match self.self_type_trait_bounds { + SelfBounds::None => None, + SelfBounds::All(traits) => { + Some(bound_tt(&parse_quote!(Self), traits.iter().cloned(), self.ctx)) + } + }; + + let zerocopy_bounds = + field_type_bounds + .into_iter() + .chain(padding_check_bound) + .chain(self_bounds) + .map(|bound| { + if self.ctx.skip_on_error { + parse_quote!(for<'zc> #bound) + } else { + bound.clone() + } + }) + .collect::<Vec<_>>(); + + let bounds = self + .ctx + .ast + .generics + .where_clause + .as_ref() + .map(|where_clause| where_clause.predicates.iter()) + .into_iter() + .flatten() + .chain(zerocopy_bounds.iter()); + + // The parameters with trait bounds, but without type defaults. + let mut params: Vec<_> = self + .ctx + .ast + .generics + .params + .clone() + .into_iter() + .map(|mut param| { + match &mut param { + GenericParam::Type(ty) => ty.default = None, + GenericParam::Const(cnst) => cnst.default = None, + GenericParam::Lifetime(_) => {} + } + parse_quote!(#param) + }) + .chain(self.param_extras) + .collect(); + + // For MSRV purposes, ensure that lifetimes precede types precede const + // generics. + params.sort_by_cached_key(|param| match param { + GenericParam::Lifetime(_) => 0, + GenericParam::Type(_) => 1, + GenericParam::Const(_) => 2, + }); + + // The identifiers of the parameters without trait bounds or type + // defaults. + let param_idents = self.ctx.ast.generics.params.iter().map(|param| match param { + GenericParam::Type(ty) => { + let ident = &ty.ident; + quote!(#ident) + } + GenericParam::Lifetime(l) => { + let ident = &l.lifetime; + quote!(#ident) + } + GenericParam::Const(cnst) => { + let ident = &cnst.ident; + quote!({#ident}) + } + }); + + let inner_extras = self.inner_extras; + let allow_trivial_bounds = + if self.ctx.skip_on_error { quote!(#[allow(trivial_bounds)]) } else { quote!() }; + let impl_tokens = quote! { + #allow_trivial_bounds + unsafe impl < #(#params),* > #trait_path for #type_ident < #(#param_idents),* > + where + #(#bounds,)* + { + fn only_derive_is_allowed_to_implement_this_trait() {} + + #inner_extras + } + }; + + let outer_extras = self.outer_extras.filter(|e| !e.is_empty()); + let cfg_compile_error = self.ctx.cfg_compile_error(); + const_block([Some(cfg_compile_error), Some(impl_tokens), outer_extras]) + } +} + +// A polyfill for `Option::then_some`, which was added after our MSRV. +// +// The `#[allow(unused)]` is necessary because, on sufficiently recent toolchain +// versions, `b.then_some(...)` resolves to the inherent method rather than to +// this trait, and so this trait is considered unused. +// +// FIXME(#67): Remove this once our MSRV is >= 1.62. +#[allow(unused)] +trait BoolExt { + fn then_some<T>(self, t: T) -> Option<T>; +} + +impl BoolExt for bool { + fn then_some<T>(self, t: T) -> Option<T> { + if self { + Some(t) + } else { + None + } + } +} + +pub(crate) fn const_block(items: impl IntoIterator<Item = Option<TokenStream>>) -> TokenStream { + let items = items.into_iter().flatten(); + quote! { + #[allow( + // FIXME(#553): Add a test that generates a warning when + // `#[allow(deprecated)]` isn't present. + deprecated, + // Required on some rustc versions due to a lint that is only + // triggered when `derive(KnownLayout)` is applied to `repr(C)` + // structs that are generated by macros. See #2177 for details. + private_bounds, + non_local_definitions, + non_camel_case_types, + non_upper_case_globals, + non_snake_case, + non_ascii_idents, + clippy::missing_inline_in_public_items, + )] + #[deny(ambiguous_associated_items)] + // While there are not currently any warnings that this suppresses + // (that we're aware of), it's good future-proofing hygiene. + #[automatically_derived] + const _: () = { + #(#items)* + }; + } +} +pub(crate) fn generate_tag_enum(ctx: &Ctx, repr: &EnumRepr, data: &DataEnum) -> TokenStream { + let zerocopy_crate = &ctx.zerocopy_crate; + let variants = data.variants.iter().map(|v| { + let ident = &v.ident; + if let Some((eq, discriminant)) = &v.discriminant { + quote! { #ident #eq #discriminant } + } else { + quote! { #ident } + } + }); + + // Don't include any `repr(align)` when generating the tag enum, as that + // could add padding after the tag but before any variants, which is not the + // correct behavior. + let repr = match repr { + EnumRepr::Transparent(span) => quote::quote_spanned! { *span => #[repr(transparent)] }, + EnumRepr::Compound(c, _) => quote! { #c }, + }; + + quote! { + #repr + #[allow(dead_code)] + pub enum ___ZerocopyTag { + #(#variants,)* + } + + // SAFETY: `___ZerocopyTag` has no fields, and so it does not permit + // interior mutation. + unsafe impl #zerocopy_crate::Immutable for ___ZerocopyTag { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + } +} +pub(crate) fn enum_size_from_repr(repr: &EnumRepr) -> Result<usize, Error> { + use CompoundRepr::*; + use PrimitiveRepr::*; + use Repr::*; + match repr { + Transparent(span) + | Compound( + Spanned { + t: C | Rust | Primitive(U32 | I32 | U64 | I64 | U128 | I128 | Usize | Isize), + span, + }, + _, + ) => Err(Error::new( + *span, + "`FromBytes` only supported on enums with `#[repr(...)]` attributes `u8`, `i8`, `u16`, or `i16`", + )), + Compound(Spanned { t: Primitive(U8 | I8), span: _ }, _align) => Ok(8), + Compound(Spanned { t: Primitive(U16 | I16), span: _ }, _align) => Ok(16), + } +} + +#[cfg(test)] +pub(crate) mod testutil { + use proc_macro2::TokenStream; + use syn::visit::{self, Visit}; + + /// Checks for hygiene violations in the generated code. + /// + /// # Panics + /// + /// Panics if a hygiene violation is found. + pub(crate) fn check_hygiene(ts: TokenStream) { + struct AmbiguousItemVisitor; + + impl<'ast> Visit<'ast> for AmbiguousItemVisitor { + fn visit_path(&mut self, i: &'ast syn::Path) { + if i.segments.len() > 1 && i.segments.first().unwrap().ident == "Self" { + panic!( + "Found ambiguous path `{}` in generated output. \ + All associated item access must be fully qualified (e.g., `<Self as Trait>::Item`) \ + to prevent hygiene issues.", + quote::quote!(#i) + ); + } + visit::visit_path(self, i); + } + } + + let file = syn::parse2::<syn::File>(ts).expect("failed to parse generated output as File"); + AmbiguousItemVisitor.visit_file(&file); + } + + #[test] + fn test_check_hygiene_success() { + check_hygiene(quote::quote! { + fn foo() { + let _ = <Self as Trait>::Item; + } + }); + } + + #[test] + #[should_panic(expected = "Found ambiguous path `Self :: Ambiguous`")] + fn test_check_hygiene_failure() { + check_hygiene(quote::quote! { + fn foo() { + let _ = Self::Ambiguous; + } + }); + } +} diff --git a/rust/zerocopy/README.md b/rust/zerocopy/README.md new file mode 100644 index 000000000000..3d11a6502cf0 --- /dev/null +++ b/rust/zerocopy/README.md @@ -0,0 +1,13 @@ +# `zerocopy` + +These source files come from the Rust `zerocopy` crate, version v0.8.54 +(released 2026-07-08), hosted in the <https://github.com/google/zerocopy> +repository, licensed under "BSD-2-Clause OR Apache-2.0 OR MIT" and only +modified to tweak the SPDX license identifiers. + +For copyright details, please see: + + https://github.com/google/zerocopy/blob/v0.8.54/README.md?plain=1 + https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-BSD + https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-APACHE + https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-MIT diff --git a/rust/zerocopy/benches/as_bytes_dynamic_size.rs b/rust/zerocopy/benches/as_bytes_dynamic_size.rs new file mode 100644 index 000000000000..68cd1d6f4111 --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_dynamic_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_as_bytes_dynamic_size(source: &format::CocoPacket) -> &[u8] { + source.as_bytes() +} diff --git a/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 b/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 new file mode 100644 index 000000000000..f68bad612695 --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64 @@ -0,0 +1,5 @@ +bench_as_bytes_dynamic_size: + mov rax, rdi + lea rdx, [2*rsi + 5] + and rdx, -2 + ret diff --git a/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64.mca b/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..c3b92a9a95fe --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_dynamic_size.x86-64.mca @@ -0,0 +1,47 @@ +Iterations: 100 +Instructions: 400 +Total Cycles: 137 +Total uOps: 400 + +Dispatch Width: 4 +uOps Per Cycle: 2.92 +IPC: 2.92 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.50 lea rdx, [2*rsi + 5] + 1 1 0.33 and rdx, -2 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.33 1.33 - 1.34 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.66 - 0.34 - - mov rax, rdi + - - 0.33 0.67 - - - - lea rdx, [2*rsi + 5] + - - 1.00 - - - - - and rdx, -2 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/as_bytes_static_size.rs b/rust/zerocopy/benches/as_bytes_static_size.rs new file mode 100644 index 000000000000..2ad738e95480 --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_static_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_as_bytes_static_size(source: &format::CocoPacket) -> &[u8] { + source.as_bytes() +} diff --git a/rust/zerocopy/benches/as_bytes_static_size.x86-64 b/rust/zerocopy/benches/as_bytes_static_size.x86-64 new file mode 100644 index 000000000000..213e74ab54ff --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_static_size.x86-64 @@ -0,0 +1,4 @@ +bench_as_bytes_static_size: + mov rax, rdi + mov edx, 6 + ret diff --git a/rust/zerocopy/benches/as_bytes_static_size.x86-64.mca b/rust/zerocopy/benches/as_bytes_static_size.x86-64.mca new file mode 100644 index 000000000000..ae04a6ba9061 --- /dev/null +++ b/rust/zerocopy/benches/as_bytes_static_size.x86-64.mca @@ -0,0 +1,45 @@ +Iterations: 100 +Instructions: 300 +Total Cycles: 104 +Total uOps: 300 + +Dispatch Width: 4 +uOps Per Cycle: 2.88 +IPC: 2.88 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov edx, 6 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 - 1.01 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.99 - - 0.01 - - mov rax, rdi + - - - 1.00 - - - - mov edx, 6 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/extend_vec_zeroed.rs b/rust/zerocopy/benches/extend_vec_zeroed.rs new file mode 100644 index 000000000000..1fbf772d1ea7 --- /dev/null +++ b/rust/zerocopy/benches/extend_vec_zeroed.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_extend_vec_zeroed(v: &mut Vec<format::LocoPacket>, additional: usize) -> Option<()> { + FromZeros::extend_vec_zeroed(v, additional).ok() +} diff --git a/rust/zerocopy/benches/extend_vec_zeroed.x86-64 b/rust/zerocopy/benches/extend_vec_zeroed.x86-64 new file mode 100644 index 000000000000..831b2a075fec --- /dev/null +++ b/rust/zerocopy/benches/extend_vec_zeroed.x86-64 @@ -0,0 +1,60 @@ +bench_extend_vec_zeroed: + push r15 + push r14 + push r13 + push r12 + push rbx + sub rsp, 32 + mov rbx, rdi + mov rax, qword ptr [rdi] + mov r12, qword ptr [rdi + 16] + mov rcx, rax + sub rcx, r12 + cmp rsi, rcx + jbe .LBB6_3 + mov r15, r12 + add r15, rsi + jae .LBB6_6 +.LBB6_2: + xor eax, eax + jmp .LBB6_5 +.LBB6_3: + mov rax, qword ptr [rbx + 8] + lea r15, [r12 + rsi] +.LBB6_4: + lea rcx, [r12 + 2*r12] + lea rdi, [rax + 2*rcx] + add rsi, rsi + lea rdx, [rsi + 2*rsi] + xor esi, esi + call qword ptr [rip + memset@GOTPCREL] + mov qword ptr [rbx + 16], r15 + mov al, 1 +.LBB6_5: + add rsp, 32 + pop rbx + pop r12 + pop r13 + pop r14 + pop r15 + ret +.LBB6_6: + mov r13, rsi + lea rcx, [rax + rax] + cmp r15, rcx + cmova rcx, r15 + cmp rcx, 5 + mov r14d, 4 + cmovae r14, rcx + mov rdx, qword ptr [rbx + 8] + lea rdi, [rsp + 8] + mov rsi, rax + mov rcx, r14 + call <alloc::raw_vec::RawVecInner>::finish_grow + cmp dword ptr [rsp + 8], 1 + je .LBB6_2 + mov rax, qword ptr [rsp + 16] + mov qword ptr [rbx + 8], rax + mov qword ptr [rbx], r14 + mov rsi, r13 + jmp .LBB6_4 diff --git a/rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca b/rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca new file mode 100644 index 000000000000..cfab1eea8f56 --- /dev/null +++ b/rust/zerocopy/benches/extend_vec_zeroed.x86-64.mca @@ -0,0 +1,147 @@ +Iterations: 100 +Instructions: 5400 +Total Cycles: 6595 +Total uOps: 6800 + +Dispatch Width: 4 +uOps Per Cycle: 1.03 +IPC: 0.82 +Block RThroughput: 17.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r15 + 2 5 1.00 * push r14 + 2 5 1.00 * push r13 + 2 5 1.00 * push r12 + 2 5 1.00 * push rbx + 1 1 0.33 sub rsp, 32 + 1 1 0.33 mov rbx, rdi + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 5 0.50 * mov r12, qword ptr [rdi + 16] + 1 1 0.33 mov rcx, rax + 1 1 0.33 sub rcx, r12 + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 jbe .LBB6_3 + 1 1 0.33 mov r15, r12 + 1 1 0.33 add r15, rsi + 1 1 1.00 jae .LBB6_6 + 1 0 0.25 xor eax, eax + 1 1 1.00 jmp .LBB6_5 + 1 5 0.50 * mov rax, qword ptr [rbx + 8] + 1 1 0.50 lea r15, [r12 + rsi] + 1 1 0.50 lea rcx, [r12 + 2*r12] + 1 1 0.50 lea rdi, [rax + 2*rcx] + 1 1 0.33 add rsi, rsi + 1 1 0.50 lea rdx, [rsi + 2*rsi] + 1 0 0.25 xor esi, esi + 4 7 1.00 * call qword ptr [rip + memset@GOTPCREL] + 1 1 1.00 * mov qword ptr [rbx + 16], r15 + 1 1 0.33 mov al, 1 + 1 1 0.33 add rsp, 32 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r12 + 1 6 0.50 * pop r13 + 1 6 0.50 * pop r14 + 1 6 0.50 * pop r15 + 1 1 1.00 U ret + 1 1 0.33 mov r13, rsi + 1 1 0.50 lea rcx, [rax + rax] + 1 1 0.33 cmp r15, rcx + 3 3 1.00 cmova rcx, r15 + 1 1 0.33 cmp rcx, 5 + 1 1 0.33 mov r14d, 4 + 2 2 0.67 cmovae r14, rcx + 1 5 0.50 * mov rdx, qword ptr [rbx + 8] + 1 1 0.50 lea rdi, [rsp + 8] + 1 1 0.33 mov rsi, rax + 1 1 0.33 mov rcx, r14 + 3 5 1.00 call <alloc::raw_vec::RawVecInner>::finish_grow + 2 6 0.50 * cmp dword ptr [rsp + 8], 1 + 1 1 1.00 je .LBB6_2 + 1 5 0.50 * mov rax, qword ptr [rsp + 16] + 1 1 1.00 * mov qword ptr [rbx + 8], rax + 1 1 1.00 * mov qword ptr [rbx], r14 + 1 1 0.33 mov rsi, r13 + 1 1 1.00 jmp .LBB6_4 + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 12.00 12.00 10.00 13.00 11.00 11.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - 0.49 0.51 push r15 + - - - - 1.00 - 0.51 0.49 push r14 + - - - - 1.00 - 0.50 0.50 push r13 + - - - - 1.00 - 0.50 0.50 push r12 + - - - - 1.00 - 0.50 0.50 push rbx + - - 0.01 0.99 - - - - sub rsp, 32 + - - - - - 1.00 - - mov rbx, rdi + - - - - - - 0.50 0.50 mov rax, qword ptr [rdi] + - - - - - - 0.50 0.50 mov r12, qword ptr [rdi + 16] + - - - 1.00 - - - - mov rcx, rax + - - - 0.99 - 0.01 - - sub rcx, r12 + - - - - - 1.00 - - cmp rsi, rcx + - - - - - 1.00 - - jbe .LBB6_3 + - - 0.01 0.98 - 0.01 - - mov r15, r12 + - - 0.99 0.01 - - - - add r15, rsi + - - - - - 1.00 - - jae .LBB6_6 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - jmp .LBB6_5 + - - - - - - 0.50 0.50 mov rax, qword ptr [rbx + 8] + - - 1.00 - - - - - lea r15, [r12 + rsi] + - - 0.98 0.02 - - - - lea rcx, [r12 + 2*r12] + - - 0.99 0.01 - - - - lea rdi, [rax + 2*rcx] + - - - 1.00 - - - - add rsi, rsi + - - 0.99 0.01 - - - - lea rdx, [rsi + 2*rsi] + - - - - - - - - xor esi, esi + - - - - 1.00 1.00 1.00 1.00 call qword ptr [rip + memset@GOTPCREL] + - - - - 1.00 - 0.50 0.50 mov qword ptr [rbx + 16], r15 + - - 0.01 0.99 - - - - mov al, 1 + - - 1.00 - - - - - add rsp, 32 + - - - - - - 0.50 0.50 pop rbx + - - - - - - 0.50 0.50 pop r12 + - - - - - - 0.50 0.50 pop r13 + - - - - - - 0.50 0.50 pop r14 + - - - - - - 0.50 0.50 pop r15 + - - - - - 1.00 - - ret + - - 1.00 - - - - - mov r13, rsi + - - 0.01 0.99 - - - - lea rcx, [rax + rax] + - - 0.99 0.01 - - - - cmp r15, rcx + - - 2.00 0.01 - 0.99 - - cmova rcx, r15 + - - 0.01 0.99 - - - - cmp rcx, 5 + - - 0.01 0.99 - - - - mov r14d, 4 + - - 1.00 0.01 - 0.99 - - cmovae r14, rcx + - - - - - - 0.50 0.50 mov rdx, qword ptr [rbx + 8] + - - 0.01 0.99 - - - - lea rdi, [rsp + 8] + - - - 1.00 - - - - mov rsi, rax + - - - 0.01 - 0.99 - - mov rcx, r14 + - - - - 1.00 1.00 0.50 0.50 call <alloc::raw_vec::RawVecInner>::finish_grow + - - - 0.99 - 0.01 0.50 0.50 cmp dword ptr [rsp + 8], 1 + - - - - - 1.00 - - je .LBB6_2 + - - - - - - 0.50 0.50 mov rax, qword ptr [rsp + 16] + - - - - 1.00 - 0.49 0.51 mov qword ptr [rbx + 8], rax + - - - - 1.00 - 0.51 0.49 mov qword ptr [rbx], r14 + - - 0.99 0.01 - - - - mov rsi, r13 + - - - - - 1.00 - - jmp .LBB6_4 diff --git a/rust/zerocopy/benches/formats/coco_dynamic_padding.rs b/rust/zerocopy/benches/formats/coco_dynamic_padding.rs new file mode 100644 index 000000000000..e494bce67312 --- /dev/null +++ b/rust/zerocopy/benches/formats/coco_dynamic_padding.rs @@ -0,0 +1,24 @@ +use zerocopy_derive::*; + +// The only valid value of this type are the bytes `0xC0C0`. +#[derive(TryFromBytes, KnownLayout, Immutable)] +#[repr(u16)] +pub enum C0C0 { + _XC0C0 = 0xC0C0, +} + +#[derive(FromBytes, KnownLayout, Immutable, SplitAt)] +#[repr(C, align(4))] +pub struct Packet<Magic> { + magic_number: Magic, + milk: u8, + mug_size: u8, + temperature: [u8; 5], + marshmallows: [[u8; 3]], +} + +/// A packet begining with the magic number `0xC0C0`. +pub type CocoPacket = Packet<C0C0>; + +/// A packet beginning with any two initialized bytes. +pub type LocoPacket = Packet<[u8; 2]>; diff --git a/rust/zerocopy/benches/formats/coco_dynamic_size.rs b/rust/zerocopy/benches/formats/coco_dynamic_size.rs new file mode 100644 index 000000000000..59364638e66c --- /dev/null +++ b/rust/zerocopy/benches/formats/coco_dynamic_size.rs @@ -0,0 +1,27 @@ +use zerocopy_derive::*; + +// The only valid value of this type are the bytes `0xC0C0`. +#[derive(TryFromBytes, KnownLayout, Immutable, IntoBytes)] +#[repr(u16)] +pub enum C0C0 { + _XC0C0 = 0xC0C0, +} + +macro_rules! define_packet { + ($name: ident, $trait: ident, $leading_field: ty) => { + #[derive($trait, KnownLayout, Immutable, IntoBytes, SplitAt)] + #[repr(C, align(2))] + pub struct $name { + magic_number: $leading_field, + mug_size: u8, + temperature: u8, + marshmallows: [[u8; 2]], + } + }; +} + +/// Packet begins with bytes 0xC0C0. +define_packet!(CocoPacket, TryFromBytes, C0C0); + +/// Packet begins with any two bytes. +define_packet!(LocoPacket, FromBytes, [u8; 2]); diff --git a/rust/zerocopy/benches/formats/coco_static_size.rs b/rust/zerocopy/benches/formats/coco_static_size.rs new file mode 100644 index 000000000000..0839497e1748 --- /dev/null +++ b/rust/zerocopy/benches/formats/coco_static_size.rs @@ -0,0 +1,27 @@ +use zerocopy_derive::*; + +// The only valid value of this type are the bytes `0xC0C0`. +#[derive(TryFromBytes, KnownLayout, Immutable, IntoBytes)] +#[repr(u16)] +pub enum C0C0 { + _XC0C0 = 0xC0C0, +} + +macro_rules! define_packet { + ($name: ident, $trait: ident, $leading_field: ty) => { + #[derive($trait, KnownLayout, Immutable, IntoBytes)] + #[repr(C, align(2))] + pub struct $name { + magic_number: $leading_field, + mug_size: u8, + temperature: u8, + marshmallows: [u8; 2], + } + }; +} + +/// Packet begins with bytes 0xC0C0. +define_packet!(CocoPacket, TryFromBytes, C0C0); + +/// Packet begins with any two bytes. +define_packet!(LocoPacket, FromBytes, [u8; 2]); diff --git a/rust/zerocopy/benches/insert_vec_zeroed.rs b/rust/zerocopy/benches/insert_vec_zeroed.rs new file mode 100644 index 000000000000..a5d685c2b027 --- /dev/null +++ b/rust/zerocopy/benches/insert_vec_zeroed.rs @@ -0,0 +1,13 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_insert_vec_zeroed( + v: &mut Vec<format::LocoPacket>, + position: usize, + additional: usize, +) -> Option<()> { + FromZeros::insert_vec_zeroed(v, position, additional).ok() +} diff --git a/rust/zerocopy/benches/insert_vec_zeroed.x86-64 b/rust/zerocopy/benches/insert_vec_zeroed.x86-64 new file mode 100644 index 000000000000..9db87403cb55 --- /dev/null +++ b/rust/zerocopy/benches/insert_vec_zeroed.x86-64 @@ -0,0 +1,79 @@ +bench_insert_vec_zeroed: + push rbp + push r15 + push r14 + push r13 + push r12 + push rbx + sub rsp, 24 + mov r12, qword ptr [rdi + 16] + mov r13, r12 + sub r13, rsi + jb .LBB6_10 + mov rbx, rdi + mov rax, qword ptr [rdi] + mov rcx, rax + sub rcx, r12 + cmp rdx, rcx + jbe .LBB6_4 + add r12, rdx + jae .LBB6_7 +.LBB6_3: + xor eax, eax + jmp .LBB6_6 +.LBB6_4: + mov rax, qword ptr [rbx + 8] + add r12, rdx +.LBB6_5: + lea rcx, [rsi + 2*rsi] + lea r14, [rax + 2*rcx] + add rdx, rdx + lea r15, [rdx + 2*rdx] + lea rdi, [r14 + r15] + add r13, r13 + lea rdx, [2*r13] + add rdx, r13 + mov rsi, r14 + call qword ptr [rip + memmove@GOTPCREL] + mov rdi, r14 + xor esi, esi + mov rdx, r15 + call qword ptr [rip + memset@GOTPCREL] + mov qword ptr [rbx + 16], r12 + mov al, 1 +.LBB6_6: + add rsp, 24 + pop rbx + pop r12 + pop r13 + pop r14 + pop r15 + pop rbp + ret +.LBB6_7: + mov r15, rsi + mov rbp, rdx + lea rcx, [rax + rax] + cmp r12, rcx + cmova rcx, r12 + cmp rcx, 5 + mov r14d, 4 + cmovae r14, rcx + mov rdx, qword ptr [rbx + 8] + mov rdi, rsp + mov rsi, rax + mov rcx, r14 + call <alloc::raw_vec::RawVecInner>::finish_grow + cmp dword ptr [rsp], 1 + je .LBB6_3 + mov rax, qword ptr [rsp + 8] + mov qword ptr [rbx + 8], rax + mov qword ptr [rbx], r14 + mov rdx, rbp + mov rsi, r15 + jmp .LBB6_5 +.LBB6_10: + lea rdi, [rip + .Lanon.HASH.1] + lea rdx, [rip + .Lanon.HASH.3] + mov esi, 37 + call qword ptr [rip + core::panicking::panic@GOTPCREL] diff --git a/rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca b/rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca new file mode 100644 index 000000000000..665240667844 --- /dev/null +++ b/rust/zerocopy/benches/insert_vec_zeroed.x86-64.mca @@ -0,0 +1,183 @@ +Iterations: 100 +Instructions: 7200 +Total Cycles: 7648 +Total uOps: 9300 + +Dispatch Width: 4 +uOps Per Cycle: 1.22 +IPC: 0.94 +Block RThroughput: 23.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push rbp + 2 5 1.00 * push r15 + 2 5 1.00 * push r14 + 2 5 1.00 * push r13 + 2 5 1.00 * push r12 + 2 5 1.00 * push rbx + 1 1 0.33 sub rsp, 24 + 1 5 0.50 * mov r12, qword ptr [rdi + 16] + 1 1 0.33 mov r13, r12 + 1 1 0.33 sub r13, rsi + 1 1 1.00 jb .LBB6_10 + 1 1 0.33 mov rbx, rdi + 1 5 0.50 * mov rax, qword ptr [rdi] + 1 1 0.33 mov rcx, rax + 1 1 0.33 sub rcx, r12 + 1 1 0.33 cmp rdx, rcx + 1 1 1.00 jbe .LBB6_4 + 1 1 0.33 add r12, rdx + 1 1 1.00 jae .LBB6_7 + 1 0 0.25 xor eax, eax + 1 1 1.00 jmp .LBB6_6 + 1 5 0.50 * mov rax, qword ptr [rbx + 8] + 1 1 0.33 add r12, rdx + 1 1 0.50 lea rcx, [rsi + 2*rsi] + 1 1 0.50 lea r14, [rax + 2*rcx] + 1 1 0.33 add rdx, rdx + 1 1 0.50 lea r15, [rdx + 2*rdx] + 1 1 0.50 lea rdi, [r14 + r15] + 1 1 0.33 add r13, r13 + 1 1 0.50 lea rdx, [2*r13] + 1 1 0.33 add rdx, r13 + 1 1 0.33 mov rsi, r14 + 4 7 1.00 * call qword ptr [rip + memmove@GOTPCREL] + 1 1 0.33 mov rdi, r14 + 1 0 0.25 xor esi, esi + 1 1 0.33 mov rdx, r15 + 4 7 1.00 * call qword ptr [rip + memset@GOTPCREL] + 1 1 1.00 * mov qword ptr [rbx + 16], r12 + 1 1 0.33 mov al, 1 + 1 1 0.33 add rsp, 24 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r12 + 1 6 0.50 * pop r13 + 1 6 0.50 * pop r14 + 1 6 0.50 * pop r15 + 1 6 0.50 * pop rbp + 1 1 1.00 U ret + 1 1 0.33 mov r15, rsi + 1 1 0.33 mov rbp, rdx + 1 1 0.50 lea rcx, [rax + rax] + 1 1 0.33 cmp r12, rcx + 3 3 1.00 cmova rcx, r12 + 1 1 0.33 cmp rcx, 5 + 1 1 0.33 mov r14d, 4 + 2 2 0.67 cmovae r14, rcx + 1 5 0.50 * mov rdx, qword ptr [rbx + 8] + 1 1 0.33 mov rdi, rsp + 1 1 0.33 mov rsi, rax + 1 1 0.33 mov rcx, r14 + 3 5 1.00 call <alloc::raw_vec::RawVecInner>::finish_grow + 2 6 0.50 * cmp dword ptr [rsp], 1 + 1 1 1.00 je .LBB6_3 + 1 5 0.50 * mov rax, qword ptr [rsp + 8] + 1 1 1.00 * mov qword ptr [rbx + 8], rax + 1 1 1.00 * mov qword ptr [rbx], r14 + 1 1 0.33 mov rdx, rbp + 1 1 0.33 mov rsi, r15 + 1 1 1.00 jmp .LBB6_5 + 1 1 0.50 lea rdi, [rip + .Lanon.HASH.1] + 1 1 0.50 lea rdx, [rip + .Lanon.HASH.3] + 1 1 0.33 mov esi, 37 + 4 7 1.00 * call qword ptr [rip + core::panicking::panic@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 17.02 16.50 13.00 19.48 14.00 14.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - 0.98 0.02 push rbp + - - - - 1.00 - 0.02 0.98 push r15 + - - - - 1.00 - 0.99 0.01 push r14 + - - - - 1.00 - 0.01 0.99 push r13 + - - - - 1.00 - 0.99 0.01 push r12 + - - - - 1.00 - 0.01 0.99 push rbx + - - 0.49 0.51 - - - - sub rsp, 24 + - - - - - - 0.04 0.96 mov r12, qword ptr [rdi + 16] + - - 0.49 0.50 - 0.01 - - mov r13, r12 + - - 0.48 0.51 - 0.01 - - sub r13, rsi + - - - - - 1.00 - - jb .LBB6_10 + - - 0.49 0.49 - 0.02 - - mov rbx, rdi + - - - - - - 0.97 0.03 mov rax, qword ptr [rdi] + - - 0.51 0.49 - - - - mov rcx, rax + - - 0.49 0.02 - 0.49 - - sub rcx, r12 + - - 0.49 0.50 - 0.01 - - cmp rdx, rcx + - - - - - 1.00 - - jbe .LBB6_4 + - - 0.02 0.49 - 0.49 - - add r12, rdx + - - - - - 1.00 - - jae .LBB6_7 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - jmp .LBB6_6 + - - - - - - 0.97 0.03 mov rax, qword ptr [rbx + 8] + - - 0.51 0.49 - - - - add r12, rdx + - - 0.49 0.51 - - - - lea rcx, [rsi + 2*rsi] + - - 0.50 0.50 - - - - lea r14, [rax + 2*rcx] + - - 0.51 0.49 - - - - add rdx, rdx + - - 0.50 0.50 - - - - lea r15, [rdx + 2*rdx] + - - 0.49 0.51 - - - - lea rdi, [r14 + r15] + - - 0.50 0.49 - 0.01 - - add r13, r13 + - - 0.51 0.49 - - - - lea rdx, [2*r13] + - - 0.01 0.01 - 0.98 - - add rdx, r13 + - - 0.01 - - 0.99 - - mov rsi, r14 + - - - - 1.00 1.00 1.98 0.02 call qword ptr [rip + memmove@GOTPCREL] + - - 0.49 0.50 - 0.01 - - mov rdi, r14 + - - - - - - - - xor esi, esi + - - 0.50 0.50 - - - - mov rdx, r15 + - - - - 1.00 1.00 1.96 0.04 call qword ptr [rip + memset@GOTPCREL] + - - - - 1.00 - 0.01 0.99 mov qword ptr [rbx + 16], r12 + - - 0.50 - - 0.50 - - mov al, 1 + - - 0.51 0.49 - - - - add rsp, 24 + - - - - - - 0.02 0.98 pop rbx + - - - - - - 0.03 0.97 pop r12 + - - - - - - 0.03 0.97 pop r13 + - - - - - - 0.97 0.03 pop r14 + - - - - - - 0.03 0.97 pop r15 + - - - - - - 0.01 0.99 pop rbp + - - - - - 1.00 - - ret + - - 0.49 0.51 - - - - mov r15, rsi + - - 0.51 0.49 - - - - mov rbp, rdx + - - 0.49 0.51 - - - - lea rcx, [rax + rax] + - - 0.49 0.50 - 0.01 - - cmp r12, rcx + - - 1.04 0.50 - 1.46 - - cmova rcx, r12 + - - 0.49 0.49 - 0.02 - - cmp rcx, 5 + - - 0.50 - - 0.50 - - mov r14d, 4 + - - 0.50 0.51 - 0.99 - - cmovae r14, rcx + - - - - - - 0.97 0.03 mov rdx, qword ptr [rbx + 8] + - - - 0.51 - 0.49 - - mov rdi, rsp + - - 0.01 0.50 - 0.49 - - mov rsi, rax + - - 0.49 0.50 - 0.01 - - mov rcx, r14 + - - - - 1.00 1.00 0.99 0.01 call <alloc::raw_vec::RawVecInner>::finish_grow + - - 0.51 0.49 - - 0.50 0.50 cmp dword ptr [rsp], 1 + - - - - - 1.00 - - je .LBB6_3 + - - - - - - 0.50 0.50 mov rax, qword ptr [rsp + 8] + - - - - 1.00 - 0.99 0.01 mov qword ptr [rbx + 8], rax + - - - - 1.00 - 0.01 0.99 mov qword ptr [rbx], r14 + - - 0.49 0.50 - 0.01 - - mov rdx, rbp + - - 0.50 0.01 - 0.49 - - mov rsi, r15 + - - - - - 1.00 - - jmp .LBB6_5 + - - 0.01 0.99 - - - - lea rdi, [rip + .Lanon.HASH.1] + - - 0.99 0.01 - - - - lea rdx, [rip + .Lanon.HASH.3] + - - 0.02 0.49 - 0.49 - - mov esi, 37 + - - - - 1.00 1.00 0.02 1.98 call qword ptr [rip + core::panicking::panic@GOTPCREL] diff --git a/rust/zerocopy/benches/new_box_zeroed.rs b/rust/zerocopy/benches/new_box_zeroed.rs new file mode 100644 index 000000000000..aa9a66cce353 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_new_box_zeroed() -> Option<Box<format::LocoPacket>> { + FromZeros::new_box_zeroed().ok() +} diff --git a/rust/zerocopy/benches/new_box_zeroed.x86-64 b/rust/zerocopy/benches/new_box_zeroed.x86-64 new file mode 100644 index 000000000000..ef74ea5388ac --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed.x86-64 @@ -0,0 +1,7 @@ +bench_new_box_zeroed: + push rax + call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + mov edi, 6 + mov esi, 2 + pop rax + jmp qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] diff --git a/rust/zerocopy/benches/new_box_zeroed.x86-64.mca b/rust/zerocopy/benches/new_box_zeroed.x86-64.mca new file mode 100644 index 000000000000..05afa7feb0b8 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed.x86-64.mca @@ -0,0 +1,51 @@ +Iterations: 100 +Instructions: 600 +Total Cycles: 1197 +Total uOps: 1100 + +Dispatch Width: 4 +uOps Per Cycle: 0.92 +IPC: 0.50 +Block RThroughput: 2.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push rax + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + 1 1 0.33 mov edi, 6 + 1 1 0.33 mov esi, 2 + 1 6 0.50 * pop rax + 2 6 1.00 * jmp qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 2.00 2.01 2.07 2.93 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - 0.93 0.07 push rax + - - - - 1.00 1.00 0.12 1.88 call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + - - 0.99 - - 0.01 - - mov edi, 6 + - - - 1.00 - - - - mov esi, 2 + - - - - - - 0.94 0.06 pop rax + - - - - - 1.00 0.08 0.92 jmp qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..0afde999bff8 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_new_box_zeroed_with_elems_dynamic_padding( + count: usize, +) -> Option<Box<format::LocoPacket>> { + FromZeros::new_box_zeroed_with_elems(count).ok() +} diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..22a8d048ce0f --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,24 @@ +bench_new_box_zeroed_with_elems_dynamic_padding: + push r14 + push rbx + push rax + mov rbx, rdi + movabs rax, 3074457345618258598 + cmp rdi, rax + ja .LBB5_1 + lea r14, [rbx + 2*rbx] + or r14, 3 + add r14, 9 + call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + mov esi, 4 + mov rdi, r14 + call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + jmp .LBB5_3 +.LBB5_1: + xor eax, eax +.LBB5_3: + mov rdx, rbx + add rsp, 8 + pop rbx + pop r14 + ret diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..e6efaeded476 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,81 @@ +Iterations: 100 +Instructions: 2100 +Total Cycles: 2990 +Total uOps: 3000 + +Dispatch Width: 4 +uOps Per Cycle: 1.00 +IPC: 0.70 +Block RThroughput: 7.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 2 5 1.00 * push rax + 1 1 0.33 mov rbx, rdi + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdi, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r14, [rbx + 2*rbx] + 1 1 0.33 or r14, 3 + 1 1 0.33 add r14, 9 + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + 1 1 0.33 mov esi, 4 + 1 1 0.33 mov rdi, r14 + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + 1 1 1.00 jmp .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rbx + 1 1 0.33 add rsp, 8 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.49 4.50 5.00 6.01 4.50 4.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - 0.50 0.50 push r14 + - - - - 1.00 - 0.50 0.50 push rbx + - - - - 1.00 - 0.50 0.50 push rax + - - 0.49 0.50 - 0.01 - - mov rbx, rdi + - - 0.50 0.50 - - - - movabs rax, 3074457345618258598 + - - 0.50 0.50 - - - - cmp rdi, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.50 0.50 - - - - lea r14, [rbx + 2*rbx] + - - 0.50 0.50 - - - - or r14, 3 + - - 0.50 - - 0.50 - - add r14, 9 + - - - - 1.00 1.00 1.00 1.00 call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + - - - 0.50 - 0.50 - - mov esi, 4 + - - 0.50 0.50 - - - - mov rdi, r14 + - - - - 1.00 1.00 1.00 1.00 call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + - - - - - 1.00 - - jmp .LBB5_3 + - - - - - - - - xor eax, eax + - - 0.51 0.49 - - - - mov rdx, rbx + - - 0.49 0.51 - - - - add rsp, 8 + - - - - - - 0.50 0.50 pop rbx + - - - - - - 0.50 0.50 pop r14 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.rs b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..1b12ca220692 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_new_box_zeroed_with_elems_dynamic_size(count: usize) -> Option<Box<format::LocoPacket>> { + FromZeros::new_box_zeroed_with_elems(count).ok() +} diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..bff15e55ad9f --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64 @@ -0,0 +1,22 @@ +bench_new_box_zeroed_with_elems_dynamic_size: + push r14 + push rbx + push rax + mov rbx, rdi + movabs rax, 4611686018427387901 + cmp rdi, rax + ja .LBB5_1 + lea r14, [2*rbx + 4] + call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + mov esi, 2 + mov rdi, r14 + call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + jmp .LBB5_3 +.LBB5_1: + xor eax, eax +.LBB5_3: + mov rdx, rbx + add rsp, 8 + pop rbx + pop r14 + ret diff --git a/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..153d36c01ce0 --- /dev/null +++ b/rust/zerocopy/benches/new_box_zeroed_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 2990 +Total uOps: 2800 + +Dispatch Width: 4 +uOps Per Cycle: 0.94 +IPC: 0.64 +Block RThroughput: 7.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 2 5 1.00 * push rax + 1 1 0.33 mov rbx, rdi + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdi, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r14, [2*rbx + 4] + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + 1 1 0.33 mov esi, 2 + 1 1 0.33 mov rdi, r14 + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + 1 1 1.00 jmp .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rbx + 1 1 0.33 add rsp, 8 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.97 3.98 5.00 5.05 4.50 4.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - 0.50 0.50 push r14 + - - - - 1.00 - 0.50 0.50 push rbx + - - - - 1.00 - 0.50 0.50 push rax + - - 0.05 0.94 - 0.01 - - mov rbx, rdi + - - 0.94 0.06 - - - - movabs rax, 4611686018427387901 + - - 0.06 0.94 - - - - cmp rdi, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.94 0.06 - - - - lea r14, [2*rbx + 4] + - - - - 1.00 1.00 1.00 1.00 call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + - - 0.98 0.02 - - - - mov esi, 2 + - - 0.02 0.94 - 0.04 - - mov rdi, r14 + - - - - 1.00 1.00 1.00 1.00 call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + - - - - - 1.00 - - jmp .LBB5_3 + - - - - - - - - xor eax, eax + - - 0.94 0.06 - - - - mov rdx, rbx + - - 0.04 0.96 - - - - add rsp, 8 + - - - - - - 0.50 0.50 pop rbx + - - - - - - 0.50 0.50 pop r14 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/new_vec_zeroed.rs b/rust/zerocopy/benches/new_vec_zeroed.rs new file mode 100644 index 000000000000..3d95b2b24d8d --- /dev/null +++ b/rust/zerocopy/benches/new_vec_zeroed.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_new_vec_zeroed(len: usize) -> Option<Vec<format::LocoPacket>> { + FromZeros::new_vec_zeroed(len).ok() +} diff --git a/rust/zerocopy/benches/new_vec_zeroed.x86-64 b/rust/zerocopy/benches/new_vec_zeroed.x86-64 new file mode 100644 index 000000000000..b5c083aa0d36 --- /dev/null +++ b/rust/zerocopy/benches/new_vec_zeroed.x86-64 @@ -0,0 +1,40 @@ +bench_new_vec_zeroed: + mov rax, rdi + movabs rcx, 1537228672809129301 + cmp rsi, rcx + ja .LBB5_5 + test rsi, rsi + je .LBB5_2 + push r15 + push r14 + push rbx + lea rcx, [rsi + rsi] + lea rbx, [rcx + 2*rcx] + mov r14, rax + mov r15, rsi + call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + mov esi, 2 + mov rdi, rbx + call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + mov rsi, r15 + mov rcx, rax + mov rax, r14 + test rcx, rcx + pop rbx + pop r14 + pop r15 + je .LBB5_5 + mov qword ptr [rax], rsi + mov qword ptr [rax + 8], rcx + mov qword ptr [rax + 16], rsi + ret +.LBB5_5: + movabs rcx, -9223372036854775808 + mov qword ptr [rax], rcx + ret +.LBB5_2: + mov ecx, 2 + mov qword ptr [rax], rsi + mov qword ptr [rax + 8], rcx + mov qword ptr [rax + 16], rsi + ret diff --git a/rust/zerocopy/benches/new_vec_zeroed.x86-64.mca b/rust/zerocopy/benches/new_vec_zeroed.x86-64.mca new file mode 100644 index 000000000000..b4fb4544ec39 --- /dev/null +++ b/rust/zerocopy/benches/new_vec_zeroed.x86-64.mca @@ -0,0 +1,113 @@ +Iterations: 100 +Instructions: 3700 +Total Cycles: 3486 +Total uOps: 4600 + +Dispatch Width: 4 +uOps Per Cycle: 1.32 +IPC: 1.06 +Block RThroughput: 12.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 movabs rcx, 1537228672809129301 + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 ja .LBB5_5 + 1 1 0.33 test rsi, rsi + 1 1 1.00 je .LBB5_2 + 2 5 1.00 * push r15 + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 1 1 0.50 lea rcx, [rsi + rsi] + 1 1 0.50 lea rbx, [rcx + 2*rcx] + 1 1 0.33 mov r14, rax + 1 1 0.33 mov r15, rsi + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + 1 1 0.33 mov esi, 2 + 1 1 0.33 mov rdi, rbx + 4 7 1.00 * call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + 1 1 0.33 mov rsi, r15 + 1 1 0.33 mov rcx, rax + 1 1 0.33 mov rax, r14 + 1 1 0.33 test rcx, rcx + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 6 0.50 * pop r15 + 1 1 1.00 je .LBB5_5 + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 * mov qword ptr [rax + 8], rcx + 1 1 1.00 * mov qword ptr [rax + 16], rsi + 1 1 1.00 U ret + 1 1 0.33 movabs rcx, -9223372036854775808 + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 U ret + 1 1 0.33 mov ecx, 2 + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 * mov qword ptr [rax + 8], rcx + 1 1 1.00 * mov qword ptr [rax + 16], rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.99 6.99 12.00 10.02 8.00 9.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.01 0.98 - 0.01 - - mov rax, rdi + - - 0.98 0.02 - - - - movabs rcx, 1537228672809129301 + - - 0.02 0.98 - - - - cmp rsi, rcx + - - - - - 1.00 - - ja .LBB5_5 + - - 0.98 - - 0.02 - - test rsi, rsi + - - - - - 1.00 - - je .LBB5_2 + - - - - 1.00 - - 1.00 push r15 + - - - - 1.00 - 1.00 - push r14 + - - - - 1.00 - - 1.00 push rbx + - - - 1.00 - - - - lea rcx, [rsi + rsi] + - - - 1.00 - - - - lea rbx, [rcx + 2*rcx] + - - 1.00 - - - - - mov r14, rax + - - 1.00 - - - - - mov r15, rsi + - - - - 1.00 1.00 2.00 - call qword ptr [rip + __rustc::__rust_no_alloc_shim_is_unstable_v2@GOTPCREL] + - - - 0.01 - 0.99 - - mov esi, 2 + - - 0.01 0.99 - - - - mov rdi, rbx + - - - - 1.00 1.00 - 2.00 call qword ptr [rip + __rustc::__rust_alloc_zeroed@GOTPCREL] + - - 0.01 - - 0.99 - - mov rsi, r15 + - - 0.99 0.01 - - - - mov rcx, rax + - - - 0.99 - 0.01 - - mov rax, r14 + - - 0.99 0.01 - - - - test rcx, rcx + - - - - - - - 1.00 pop rbx + - - - - - - 1.00 - pop r14 + - - - - - - - 1.00 pop r15 + - - - - - 1.00 - - je .LBB5_5 + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - 1.00 - - 1.00 mov qword ptr [rax + 8], rcx + - - - - 1.00 - 1.00 - mov qword ptr [rax + 16], rsi + - - - - - 1.00 - - ret + - - 0.01 0.99 - - - - movabs rcx, -9223372036854775808 + - - - - 1.00 - - 1.00 mov qword ptr [rax], rcx + - - - - - 1.00 - - ret + - - 0.99 0.01 - - - - mov ecx, 2 + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - 1.00 - - 1.00 mov qword ptr [rax + 8], rcx + - - - - 1.00 - 1.00 - mov qword ptr [rax + 16], rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/new_zeroed.rs b/rust/zerocopy/benches/new_zeroed.rs new file mode 100644 index 000000000000..b49f62edb146 --- /dev/null +++ b/rust/zerocopy/benches/new_zeroed.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_new_zeroed() -> format::LocoPacket { + FromZeros::new_zeroed() +} diff --git a/rust/zerocopy/benches/new_zeroed.x86-64 b/rust/zerocopy/benches/new_zeroed.x86-64 new file mode 100644 index 000000000000..b4d305e41fff --- /dev/null +++ b/rust/zerocopy/benches/new_zeroed.x86-64 @@ -0,0 +1,3 @@ +bench_new_zeroed: + xor eax, eax + ret diff --git a/rust/zerocopy/benches/new_zeroed.x86-64.mca b/rust/zerocopy/benches/new_zeroed.x86-64.mca new file mode 100644 index 000000000000..44583ca3089f --- /dev/null +++ b/rust/zerocopy/benches/new_zeroed.x86-64.mca @@ -0,0 +1,43 @@ +Iterations: 100 +Instructions: 200 +Total Cycles: 103 +Total uOps: 200 + +Dispatch Width: 4 +uOps Per Cycle: 1.94 +IPC: 1.94 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - - - - 1.00 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/read_from_bytes.rs b/rust/zerocopy/benches/read_from_bytes.rs new file mode 100644 index 000000000000..8a3baddad9cb --- /dev/null +++ b/rust/zerocopy/benches/read_from_bytes.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_read_from_bytes_static_size(source: &[u8]) -> Option<format::LocoPacket> { + zerocopy::FromBytes::read_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/read_from_bytes.x86-64 b/rust/zerocopy/benches/read_from_bytes.x86-64 new file mode 100644 index 000000000000..9082d79f1fd5 --- /dev/null +++ b/rust/zerocopy/benches/read_from_bytes.x86-64 @@ -0,0 +1,15 @@ +bench_read_from_bytes_static_size: + mov rcx, rsi + cmp rsi, 6 + jne .LBB5_2 + mov eax, dword ptr [rdi] + movzx ecx, word ptr [rdi + 4] + shl rcx, 32 + or rcx, rax +.LBB5_2: + shl rcx, 16 + inc rcx + xor eax, eax + cmp rsi, 6 + cmove rax, rcx + ret diff --git a/rust/zerocopy/benches/read_from_bytes.x86-64.mca b/rust/zerocopy/benches/read_from_bytes.x86-64.mca new file mode 100644 index 000000000000..77e787c19032 --- /dev/null +++ b/rust/zerocopy/benches/read_from_bytes.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 377 +Total uOps: 1400 + +Dispatch Width: 4 +uOps Per Cycle: 3.71 +IPC: 3.45 +Block RThroughput: 3.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rcx, rsi + 1 1 0.33 cmp rsi, 6 + 1 1 1.00 jne .LBB5_2 + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 5 0.50 * movzx ecx, word ptr [rdi + 4] + 1 1 0.50 shl rcx, 32 + 1 1 0.33 or rcx, rax + 1 1 0.50 shl rcx, 16 + 1 1 0.33 inc rcx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp rsi, 6 + 2 2 0.67 cmove rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.66 3.67 - 3.67 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.63 0.36 - 0.01 - - mov rcx, rsi + - - 0.05 0.05 - 0.90 - - cmp rsi, 6 + - - - - - 1.00 - - jne .LBB5_2 + - - - - - - - 1.00 mov eax, dword ptr [rdi] + - - - - - - 1.00 - movzx ecx, word ptr [rdi + 4] + - - 0.97 - - 0.03 - - shl rcx, 32 + - - 0.02 0.35 - 0.63 - - or rcx, rax + - - 0.98 - - 0.02 - - shl rcx, 16 + - - - 0.98 - 0.02 - - inc rcx + - - - - - - - - xor eax, eax + - - 0.03 0.93 - 0.04 - - cmp rsi, 6 + - - 0.98 1.00 - 0.02 - - cmove rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/read_from_prefix.rs b/rust/zerocopy/benches/read_from_prefix.rs new file mode 100644 index 000000000000..d49bf80ab785 --- /dev/null +++ b/rust/zerocopy/benches/read_from_prefix.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_read_from_prefix_static_size(source: &[u8]) -> Option<format::LocoPacket> { + match zerocopy::FromBytes::read_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/read_from_prefix.x86-64 b/rust/zerocopy/benches/read_from_prefix.x86-64 new file mode 100644 index 000000000000..c75b06c0c22a --- /dev/null +++ b/rust/zerocopy/benches/read_from_prefix.x86-64 @@ -0,0 +1,14 @@ +bench_read_from_prefix_static_size: + cmp rsi, 5 + jbe .LBB5_2 + mov eax, dword ptr [rdi] + movzx edi, word ptr [rdi + 4] + shl rdi, 32 + or rdi, rax +.LBB5_2: + shl rdi, 16 + inc rdi + xor eax, eax + cmp rsi, 6 + cmovae rax, rdi + ret diff --git a/rust/zerocopy/benches/read_from_prefix.x86-64.mca b/rust/zerocopy/benches/read_from_prefix.x86-64.mca new file mode 100644 index 000000000000..04e76cdd07ee --- /dev/null +++ b/rust/zerocopy/benches/read_from_prefix.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 905 +Total uOps: 1300 + +Dispatch Width: 4 +uOps Per Cycle: 1.44 +IPC: 1.33 +Block RThroughput: 3.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 cmp rsi, 5 + 1 1 1.00 jbe .LBB5_2 + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 5 0.50 * movzx edi, word ptr [rdi + 4] + 1 1 0.50 shl rdi, 32 + 1 1 0.33 or rdi, rax + 1 1 0.50 shl rdi, 16 + 1 1 0.33 inc rdi + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp rsi, 6 + 2 2 0.67 cmovae rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.32 3.32 - 3.36 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.05 0.94 - 0.01 - - cmp rsi, 5 + - - - - - 1.00 - - jbe .LBB5_2 + - - - - - - - 1.00 mov eax, dword ptr [rdi] + - - - - - - 1.00 - movzx edi, word ptr [rdi + 4] + - - 0.71 - - 0.29 - - shl rdi, 32 + - - - 0.64 - 0.36 - - or rdi, rax + - - 1.00 - - - - - shl rdi, 16 + - - 0.31 0.40 - 0.29 - - inc rdi + - - - - - - - - xor eax, eax + - - 0.34 0.35 - 0.31 - - cmp rsi, 6 + - - 0.91 0.99 - 0.10 - - cmovae rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/read_from_suffix.rs b/rust/zerocopy/benches/read_from_suffix.rs new file mode 100644 index 000000000000..4eaadb0d4dff --- /dev/null +++ b/rust/zerocopy/benches/read_from_suffix.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_read_from_suffix_static_size(source: &[u8]) -> Option<format::LocoPacket> { + match zerocopy::FromBytes::read_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/read_from_suffix.x86-64 b/rust/zerocopy/benches/read_from_suffix.x86-64 new file mode 100644 index 000000000000..5cff2a0e2f36 --- /dev/null +++ b/rust/zerocopy/benches/read_from_suffix.x86-64 @@ -0,0 +1,15 @@ +bench_read_from_suffix_static_size: + mov rcx, rsi + cmp rsi, 6 + jb .LBB5_2 + mov eax, dword ptr [rdi + rsi - 6] + movzx ecx, word ptr [rdi + rsi - 2] + shl rcx, 32 + or rcx, rax +.LBB5_2: + shl rcx, 16 + inc rcx + xor eax, eax + cmp rsi, 6 + cmovae rax, rcx + ret diff --git a/rust/zerocopy/benches/read_from_suffix.x86-64.mca b/rust/zerocopy/benches/read_from_suffix.x86-64.mca new file mode 100644 index 000000000000..0107de89562a --- /dev/null +++ b/rust/zerocopy/benches/read_from_suffix.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 377 +Total uOps: 1400 + +Dispatch Width: 4 +uOps Per Cycle: 3.71 +IPC: 3.45 +Block RThroughput: 3.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rcx, rsi + 1 1 0.33 cmp rsi, 6 + 1 1 1.00 jb .LBB5_2 + 1 5 0.50 * mov eax, dword ptr [rdi + rsi - 6] + 1 5 0.50 * movzx ecx, word ptr [rdi + rsi - 2] + 1 1 0.50 shl rcx, 32 + 1 1 0.33 or rcx, rax + 1 1 0.50 shl rcx, 16 + 1 1 0.33 inc rcx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp rsi, 6 + 2 2 0.67 cmovae rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.66 3.67 - 3.67 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.63 0.36 - 0.01 - - mov rcx, rsi + - - 0.05 0.05 - 0.90 - - cmp rsi, 6 + - - - - - 1.00 - - jb .LBB5_2 + - - - - - - - 1.00 mov eax, dword ptr [rdi + rsi - 6] + - - - - - - 1.00 - movzx ecx, word ptr [rdi + rsi - 2] + - - 0.97 - - 0.03 - - shl rcx, 32 + - - 0.02 0.35 - 0.63 - - or rcx, rax + - - 0.98 - - 0.02 - - shl rcx, 16 + - - - 0.98 - 0.02 - - inc rcx + - - - - - - - - xor eax, eax + - - 0.03 0.93 - 0.04 - - cmp rsi, 6 + - - 0.98 1.00 - 0.02 - - cmovae rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.rs new file mode 100644 index 000000000000..29708df55b45 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_bytes_dynamic_padding(source: &[u8]) -> Option<&format::LocoPacket> { + zerocopy::FromBytes::ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 new file mode 100644 index 000000000000..e844a4608fac --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 @@ -0,0 +1,22 @@ +bench_ref_from_bytes_dynamic_padding: + test dil, 3 + jne .LBB5_3 + movabs rax, 9223372036854775804 + and rax, rsi + cmp rax, 9 + jb .LBB5_3 + add rax, -9 + movabs rcx, -6148914691236517205 + mul rcx + shr rdx + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + cmp rsi, rax + je .LBB5_4 +.LBB5_3: + xor edi, edi + mov rdx, rsi +.LBB5_4: + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..423ed38ba28d --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 645 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 3.10 +IPC: 2.95 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 test dil, 3 + 1 1 1.00 jne .LBB5_3 + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rax, rsi + 1 1 0.33 cmp rax, 9 + 1 1 1.00 jb .LBB5_3 + 1 1 0.33 add rax, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 1 0.33 cmp rsi, rax + 1 1 1.00 je .LBB5_4 + 1 0 0.25 xor edi, edi + 1 1 0.33 mov rdx, rsi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.32 6.33 - 6.35 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.64 0.35 - 0.01 - - test dil, 3 + - - - - - 1.00 - - jne .LBB5_3 + - - 0.34 0.65 - 0.01 - - movabs rax, 9223372036854775804 + - - 0.35 0.65 - - - - and rax, rsi + - - 0.33 0.34 - 0.33 - - cmp rax, 9 + - - - - - 1.00 - - jb .LBB5_3 + - - 0.35 - - 0.65 - - add rax, -9 + - - 0.97 0.01 - 0.02 - - movabs rcx, -6148914691236517205 + - - 1.00 1.00 - - - - mul rcx + - - 0.99 - - 0.01 - - shr rdx + - - 0.33 0.67 - - - - lea rax, [rdx + 2*rdx] + - - 0.34 0.66 - - - - or rax, 3 + - - 0.33 0.66 - 0.01 - - add rax, 9 + - - 0.01 0.99 - - - - cmp rsi, rax + - - - - - 1.00 - - je .LBB5_4 + - - - - - - - - xor edi, edi + - - 0.32 0.01 - 0.67 - - mov rdx, rsi + - - 0.02 0.34 - 0.64 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_size.rs b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.rs new file mode 100644 index 000000000000..4eb4f970e365 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_bytes_dynamic_size(source: &[u8]) -> Option<&format::LocoPacket> { + zerocopy::FromBytes::ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 new file mode 100644 index 000000000000..cc905b76c06f --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 @@ -0,0 +1,20 @@ +bench_ref_from_bytes_dynamic_size: + mov rdx, rsi + cmp rsi, 4 + setb al + or al, dil + test al, 1 + je .LBB5_2 + xor eax, eax + ret +.LBB5_2: + lea rcx, [rdx - 4] + mov rsi, rcx + and rsi, -2 + add rsi, 4 + shr rcx + xor eax, eax + cmp rdx, rsi + cmove rdx, rcx + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..68aea583e401 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca @@ -0,0 +1,75 @@ +Iterations: 100 +Instructions: 1800 +Total Cycles: 704 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 2.84 +IPC: 2.56 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rdx, rsi + 1 1 0.33 cmp rsi, 4 + 1 1 0.50 setb al + 1 1 0.33 or al, dil + 1 1 0.33 test al, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.50 lea rcx, [rdx - 4] + 1 1 0.33 mov rsi, rcx + 1 1 0.33 and rsi, -2 + 1 1 0.33 add rsi, 4 + 1 1 0.50 shr rcx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp rdx, rsi + 2 2 0.67 cmove rdx, rcx + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.97 5.98 - 6.05 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.97 0.01 - 0.02 - - mov rdx, rsi + - - 0.01 0.02 - 0.97 - - cmp rsi, 4 + - - 0.03 - - 0.97 - - setb al + - - 0.01 0.02 - 0.97 - - or al, dil + - - - 0.98 - 0.02 - - test al, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.98 0.02 - - - - lea rcx, [rdx - 4] + - - 0.01 0.99 - - - - mov rsi, rcx + - - - 0.98 - 0.02 - - and rsi, -2 + - - 0.98 0.01 - 0.01 - - add rsi, 4 + - - 0.99 - - 0.01 - - shr rcx + - - - - - - - - xor eax, eax + - - 0.02 0.97 - 0.01 - - cmp rdx, rsi + - - 0.99 0.99 - 0.02 - - cmove rdx, rcx + - - 0.98 0.99 - 0.03 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_static_size.rs b/rust/zerocopy/benches/ref_from_bytes_static_size.rs new file mode 100644 index 000000000000..3742bba0780f --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_static_size.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_bytes_static_size(source: &[u8]) -> Option<&format::LocoPacket> { + zerocopy::FromBytes::ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64 b/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64 new file mode 100644 index 000000000000..2c8da68c8b5b --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64 @@ -0,0 +1,8 @@ +bench_ref_from_bytes_static_size: + mov ecx, edi + and ecx, 1 + xor rsi, 6 + xor eax, eax + or rsi, rcx + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64.mca new file mode 100644 index 000000000000..832697801ee2 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_static_size.x86-64.mca @@ -0,0 +1,53 @@ +Iterations: 100 +Instructions: 700 +Total Cycles: 240 +Total uOps: 800 + +Dispatch Width: 4 +uOps Per Cycle: 3.33 +IPC: 2.92 +Block RThroughput: 2.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov ecx, edi + 1 1 0.33 and ecx, 1 + 1 1 0.33 xor rsi, 6 + 1 0 0.25 xor eax, eax + 1 1 0.33 or rsi, rcx + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.33 2.33 - 2.34 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.01 0.98 - 0.01 - - mov ecx, edi + - - 0.02 0.66 - 0.32 - - and ecx, 1 + - - 0.33 0.66 - 0.01 - - xor rsi, 6 + - - - - - - - - xor eax, eax + - - 0.98 0.02 - - - - or rsi, rcx + - - 0.99 0.01 - 1.00 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..b4fea4aee51d --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_bytes_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + zerocopy::FromBytes::ref_from_bytes_with_elems(source, count).ok() +} diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..d579b3faefe7 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,19 @@ +bench_ref_from_bytes_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + seta cl + mov rax, rdi + test al, 3 + setne dil + or dil, cl + jne .LBB5_2 + lea rcx, [rdx + 2*rdx] + or rcx, 3 + add rcx, 9 + cmp rsi, rcx + je .LBB5_3 +.LBB5_2: + xor eax, eax + mov rdx, rsi +.LBB5_3: + ret diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..ea2d83dbd17a --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,71 @@ +Iterations: 100 +Instructions: 1600 +Total Cycles: 539 +Total uOps: 1700 + +Dispatch Width: 4 +uOps Per Cycle: 3.15 +IPC: 2.97 +Block RThroughput: 4.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 2 2 1.00 seta cl + 1 1 0.33 mov rax, rdi + 1 1 0.33 test al, 3 + 1 1 0.50 setne dil + 1 1 0.33 or dil, cl + 1 1 1.00 jne .LBB5_2 + 1 1 0.50 lea rcx, [rdx + 2*rdx] + 1 1 0.33 or rcx, 3 + 1 1 0.33 add rcx, 9 + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 je .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.33 5.32 - 5.35 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.01 0.98 - 0.01 - - movabs rax, 3074457345618258598 + - - - 1.00 - - - - cmp rdx, rax + - - 1.98 - - 0.02 - - seta cl + - - 0.02 0.98 - - - - mov rax, rdi + - - - 0.67 - 0.33 - - test al, 3 + - - 0.67 - - 0.33 - - setne dil + - - 0.99 - - 0.01 - - or dil, cl + - - - - - 1.00 - - jne .LBB5_2 + - - 0.01 0.99 - - - - lea rcx, [rdx + 2*rdx] + - - - 0.01 - 0.99 - - or rcx, 3 + - - 0.65 0.02 - 0.33 - - add rcx, 9 + - - 0.99 0.01 - - - - cmp rsi, rcx + - - - - - 1.00 - - je .LBB5_3 + - - - - - - - - xor eax, eax + - - 0.01 0.66 - 0.33 - - mov rdx, rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.rs b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..9d33a7c31bc3 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_bytes_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + zerocopy::FromBytes::ref_from_bytes_with_elems(source, count).ok() +} diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..3d8d15b7f6c1 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64 @@ -0,0 +1,16 @@ +bench_ref_from_bytes_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + seta cl + mov rax, rdi + or dil, cl + test dil, 1 + jne .LBB5_2 + lea rcx, [2*rdx + 4] + cmp rsi, rcx + je .LBB5_3 +.LBB5_2: + xor eax, eax + mov rdx, rsi +.LBB5_3: + ret diff --git a/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..602179f3c903 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_bytes_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 439 +Total uOps: 1400 + +Dispatch Width: 4 +uOps Per Cycle: 3.19 +IPC: 2.96 +Block RThroughput: 3.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 2 2 1.00 seta cl + 1 1 0.33 mov rax, rdi + 1 1 0.33 or dil, cl + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_2 + 1 1 0.50 lea rcx, [2*rdx + 4] + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 je .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.32 4.33 - 4.35 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - movabs rax, 4611686018427387901 + - - 0.33 0.67 - - - - cmp rdx, rax + - - 1.98 - - 0.02 - - seta cl + - - 0.01 0.99 - - - - mov rax, rdi + - - 1.00 - - - - - or dil, cl + - - 0.99 0.01 - - - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_2 + - - - 1.00 - - - - lea rcx, [2*rdx + 4] + - - 0.01 - - 0.99 - - cmp rsi, rcx + - - - - - 1.00 - - je .LBB5_3 + - - - - - - - - xor eax, eax + - - - 0.67 - 0.33 - - mov rdx, rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.rs new file mode 100644 index 000000000000..53c707b88256 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_prefix_dynamic_padding(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64 new file mode 100644 index 000000000000..a58592a24503 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64 @@ -0,0 +1,22 @@ +bench_ref_from_prefix_dynamic_padding: + xor edx, edx + mov eax, 0 + test dil, 3 + je .LBB5_1 + ret +.LBB5_1: + movabs rax, 9223372036854775804 + and rsi, rax + cmp rsi, 9 + jae .LBB5_3 + mov edx, 1 + xor eax, eax + ret +.LBB5_3: + add rsi, -9 + movabs rcx, -6148914691236517205 + mov rax, rsi + mul rcx + shr rdx + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..62ea4babaf28 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_padding.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 608 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 3.29 +IPC: 3.13 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 3 + 1 1 1.00 je .LBB5_1 + 1 1 1.00 U ret + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rsi, rax + 1 1 0.33 cmp rsi, 9 + 1 1 1.00 jae .LBB5_3 + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rsi, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 1 1 0.33 mov rax, rsi + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.00 6.00 - 6.00 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor edx, edx + - - 0.01 0.98 - 0.01 - - mov eax, 0 + - - 0.98 0.01 - 0.01 - - test dil, 3 + - - - - - 1.00 - - je .LBB5_1 + - - - - - 1.00 - - ret + - - 0.01 0.99 - - - - movabs rax, 9223372036854775804 + - - - 1.00 - - - - and rsi, rax + - - - 1.00 - - - - cmp rsi, 9 + - - - - - 1.00 - - jae .LBB5_3 + - - 1.00 - - - - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.02 0.02 - 0.96 - - add rsi, -9 + - - 0.99 0.01 - - - - movabs rcx, -6148914691236517205 + - - 0.01 0.99 - - - - mov rax, rsi + - - 1.00 1.00 - - - - mul rcx + - - 1.00 - - - - - shr rdx + - - 0.98 - - 0.02 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_size.rs b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.rs new file mode 100644 index 000000000000..a3f26f6b4e6c --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_prefix_dynamic_size(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64 new file mode 100644 index 000000000000..fe6332c9100c --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64 @@ -0,0 +1,17 @@ +bench_ref_from_prefix_dynamic_size: + xor edx, edx + mov eax, 0 + test dil, 1 + jne .LBB5_4 + cmp rsi, 4 + jae .LBB5_3 + mov edx, 1 + xor eax, eax + ret +.LBB5_3: + add rsi, -4 + shr rsi + mov rdx, rsi + mov rax, rdi +.LBB5_4: + ret diff --git a/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..3900a5946138 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_dynamic_size.x86-64.mca @@ -0,0 +1,67 @@ +Iterations: 100 +Instructions: 1400 +Total Cycles: 405 +Total uOps: 1400 + +Dispatch Width: 4 +uOps Per Cycle: 3.46 +IPC: 3.46 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_4 + 1 1 0.33 cmp rsi, 4 + 1 1 1.00 jae .LBB5_3 + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rsi, -4 + 1 1 0.50 shr rsi + 1 1 0.33 mov rdx, rsi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.99 3.99 - 4.02 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor edx, edx + - - 0.01 0.98 - 0.01 - - mov eax, 0 + - - 0.98 0.02 - - - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.02 0.98 - - - - cmp rsi, 4 + - - - - - 1.00 - - jae .LBB5_3 + - - 0.98 0.01 - 0.01 - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.01 0.99 - - - - add rsi, -4 + - - 1.00 - - - - - shr rsi + - - - 1.00 - - - - mov rdx, rsi + - - 0.99 0.01 - - - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_static_size.rs b/rust/zerocopy/benches/ref_from_prefix_static_size.rs new file mode 100644 index 000000000000..834fa3928611 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_static_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_prefix_static_size(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64 b/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64 new file mode 100644 index 000000000000..7c1bf45bb6c2 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64 @@ -0,0 +1,8 @@ +bench_ref_from_prefix_static_size: + xor eax, eax + cmp rsi, 6 + mov rcx, rdi + cmovb rcx, rax + test dil, 1 + cmove rax, rcx + ret diff --git a/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64.mca new file mode 100644 index 000000000000..9691b88fe03a --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_static_size.x86-64.mca @@ -0,0 +1,53 @@ +Iterations: 100 +Instructions: 700 +Total Cycles: 274 +Total uOps: 900 + +Dispatch Width: 4 +uOps Per Cycle: 3.28 +IPC: 2.55 +Block RThroughput: 2.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp rsi, 6 + 1 1 0.33 mov rcx, rdi + 2 2 0.67 cmovb rcx, rax + 1 1 0.33 test dil, 1 + 2 2 0.67 cmove rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.66 2.67 - 2.67 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor eax, eax + - - - 0.01 - 0.99 - - cmp rsi, 6 + - - 0.01 0.67 - 0.32 - - mov rcx, rdi + - - 1.00 0.99 - 0.01 - - cmovb rcx, rax + - - 0.66 0.01 - 0.33 - - test dil, 1 + - - 0.99 0.99 - 0.02 - - cmove rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..55d495e00c59 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_prefix_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_prefix_with_elems(source, count) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..5b31277bdebe --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,26 @@ +bench_ref_from_prefix_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + ja .LBB5_1 + xor ecx, ecx + mov eax, 0 + test dil, 3 + je .LBB5_3 + mov rdx, rcx + ret +.LBB5_1: + mov edx, 1 + xor eax, eax + ret +.LBB5_3: + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + xor r8d, r8d + cmp rax, rsi + mov ecx, 1 + cmovbe rcx, rdx + cmova rdi, r8 + mov rax, rdi + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..2f212ec6d03b --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,85 @@ +Iterations: 100 +Instructions: 2300 +Total Cycles: 807 +Total uOps: 2700 + +Dispatch Width: 4 +uOps Per Cycle: 3.35 +IPC: 2.85 +Block RThroughput: 6.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 3 + 1 1 1.00 je .LBB5_3 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 0 0.25 xor r8d, r8d + 1 1 0.33 cmp rax, rsi + 1 1 0.33 mov ecx, 1 + 3 3 1.00 cmovbe rcx, rdx + 3 3 1.00 cmova rdi, r8 + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 7.99 7.99 - 8.02 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.47 0.52 - 0.01 - - movabs rax, 3074457345618258598 + - - 0.94 0.01 - 0.05 - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - - - - - - - xor ecx, ecx + - - 0.03 0.97 - - - - mov eax, 0 + - - 0.01 0.52 - 0.47 - - test dil, 3 + - - - - - 1.00 - - je .LBB5_3 + - - 0.03 0.51 - 0.46 - - mov rdx, rcx + - - - - - 1.00 - - ret + - - 0.04 0.96 - - - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.01 0.99 - - - - lea rax, [rdx + 2*rdx] + - - 0.52 0.48 - - - - or rax, 3 + - - 0.51 0.49 - - - - add rax, 9 + - - - - - - - - xor r8d, r8d + - - 0.97 0.03 - - - - cmp rax, rsi + - - 0.01 0.99 - - - - mov ecx, 1 + - - 1.04 0.97 - 0.99 - - cmovbe rcx, rdx + - - 1.44 0.54 - 1.02 - - cmova rdi, r8 + - - 0.97 0.01 - 0.02 - - mov rax, rdi + - - 1.00 - - - - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.rs b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..e9663c721ece --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_prefix_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_prefix_with_elems(source, count) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..069fd4859c74 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64 @@ -0,0 +1,22 @@ +bench_ref_from_prefix_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + ja .LBB5_1 + mov rcx, rdx + xor edx, edx + mov eax, 0 + test dil, 1 + jne .LBB5_4 + lea rax, [2*rcx + 4] + xor r8d, r8d + cmp rax, rsi + mov edx, 1 + cmovbe rdx, rcx + cmova rdi, r8 + mov rax, rdi +.LBB5_4: + ret +.LBB5_1: + mov edx, 1 + xor eax, eax + ret diff --git a/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..6f227264066d --- /dev/null +++ b/rust/zerocopy/benches/ref_from_prefix_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 672 +Total uOps: 2300 + +Dispatch Width: 4 +uOps Per Cycle: 3.42 +IPC: 2.83 +Block RThroughput: 5.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.33 mov rcx, rdx + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_4 + 1 1 0.50 lea rax, [2*rcx + 4] + 1 0 0.25 xor r8d, r8d + 1 1 0.33 cmp rax, rsi + 1 1 0.33 mov edx, 1 + 3 3 1.00 cmovbe rdx, rcx + 3 3 1.00 cmova rdi, r8 + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.66 6.66 - 6.68 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - movabs rax, 4611686018427387901 + - - 0.37 0.63 - - - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.63 0.37 - - - - mov rcx, rdx + - - - - - - - - xor edx, edx + - - 0.01 0.98 - 0.01 - - mov eax, 0 + - - 0.98 0.02 - - - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.01 0.99 - - - - lea rax, [2*rcx + 4] + - - - - - - - - xor r8d, r8d + - - 1.00 - - - - - cmp rax, rsi + - - - 0.67 - 0.33 - - mov edx, 1 + - - 0.73 0.98 - 1.29 - - cmovbe rdx, rcx + - - 1.60 0.36 - 1.04 - - cmova rdi, r8 + - - 0.99 0.01 - - - - mov rax, rdi + - - - - - 1.00 - - ret + - - 0.34 0.66 - - - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.rs new file mode 100644 index 000000000000..5a6ea3a33dde --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_suffix_dynamic_padding(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64 new file mode 100644 index 000000000000..3e05f6023f38 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64 @@ -0,0 +1,23 @@ +bench_ref_from_suffix_dynamic_padding: + lea eax, [rsi + rdi] + test al, 3 + jne .LBB5_1 + movabs rax, 9223372036854775804 + and rax, rsi + cmp rax, 9 + jae .LBB5_3 +.LBB5_1: + xor eax, eax + ret +.LBB5_3: + add rax, -9 + movabs rcx, -6148914691236517205 + mul rcx + shr rdx + lea rax, [rdx + 2*rdx] + sub rsi, rax + or rax, -4 + add rsi, rdi + add rax, rsi + add rax, -8 + ret diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..73599d5b6aab --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_padding.x86-64.mca @@ -0,0 +1,79 @@ +Iterations: 100 +Instructions: 2000 +Total Cycles: 682 +Total uOps: 2100 + +Dispatch Width: 4 +uOps Per Cycle: 3.08 +IPC: 2.93 +Block RThroughput: 5.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea eax, [rsi + rdi] + 1 1 0.33 test al, 3 + 1 1 1.00 jne .LBB5_1 + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rax, rsi + 1 1 0.33 cmp rax, 9 + 1 1 1.00 jae .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rax, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 sub rsi, rax + 1 1 0.33 or rax, -4 + 1 1 0.33 add rsi, rdi + 1 1 0.33 add rax, rsi + 1 1 0.33 add rax, -8 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.65 6.67 - 6.68 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.90 0.10 - - - - lea eax, [rsi + rdi] + - - 0.93 - - 0.07 - - test al, 3 + - - - - - 1.00 - - jne .LBB5_1 + - - 0.51 0.47 - 0.02 - - movabs rax, 9223372036854775804 + - - - - - 1.00 - - and rax, rsi + - - - 0.09 - 0.91 - - cmp rax, 9 + - - - - - 1.00 - - jae .LBB5_3 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.43 0.47 - 0.10 - - add rax, -9 + - - 0.42 0.39 - 0.19 - - movabs rcx, -6148914691236517205 + - - 1.00 1.00 - - - - mul rcx + - - 0.69 - - 0.31 - - shr rdx + - - 0.54 0.46 - - - - lea rax, [rdx + 2*rdx] + - - 0.07 0.91 - 0.02 - - sub rsi, rax + - - 0.91 0.05 - 0.04 - - or rax, -4 + - - 0.08 0.90 - 0.02 - - add rsi, rdi + - - 0.09 0.91 - - - - add rax, rsi + - - 0.08 0.92 - - - - add rax, -8 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_size.rs b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.rs new file mode 100644 index 000000000000..3437b14f404a --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_suffix_dynamic_size(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64 new file mode 100644 index 000000000000..bd4ace89836a --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64 @@ -0,0 +1,13 @@ +bench_ref_from_suffix_dynamic_size: + mov rdx, rsi + lea ecx, [rsi + rdi] + mov eax, edx + and eax, 1 + add rax, rdi + xor esi, esi + sub rdx, 4 + cmovb rax, rsi + shr rdx + test cl, 1 + cmovne rax, rsi + ret diff --git a/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..1398bcfe27ae --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_dynamic_size.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 439 +Total uOps: 1400 + +Dispatch Width: 4 +uOps Per Cycle: 3.19 +IPC: 2.73 +Block RThroughput: 3.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rdx, rsi + 1 1 0.50 lea ecx, [rsi + rdi] + 1 1 0.33 mov eax, edx + 1 1 0.33 and eax, 1 + 1 1 0.33 add rax, rdi + 1 0 0.25 xor esi, esi + 1 1 0.33 sub rdx, 4 + 2 2 0.67 cmovb rax, rsi + 1 1 0.50 shr rdx + 1 1 0.33 test cl, 1 + 2 2 0.67 cmovne rax, rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.33 4.33 - 4.34 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.02 0.32 - 0.66 - - mov rdx, rsi + - - 0.32 0.68 - - - - lea ecx, [rsi + rdi] + - - 0.66 - - 0.34 - - mov eax, edx + - - 0.02 0.33 - 0.65 - - and eax, 1 + - - - 0.99 - 0.01 - - add rax, rdi + - - - - - - - - xor esi, esi + - - 0.65 - - 0.35 - - sub rdx, 4 + - - 1.00 1.00 - - - - cmovb rax, rsi + - - 0.66 - - 0.34 - - shr rdx + - - - 0.01 - 0.99 - - test cl, 1 + - - 1.00 1.00 - - - - cmovne rax, rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_static_size.rs b/rust/zerocopy/benches/ref_from_suffix_static_size.rs new file mode 100644 index 000000000000..c8435d10d38c --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_static_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_suffix_static_size(source: &[u8]) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64 b/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64 new file mode 100644 index 000000000000..9e90b9e2543f --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64 @@ -0,0 +1,13 @@ +bench_ref_from_suffix_static_size: + lea eax, [rsi + rdi] + cmp rsi, 6 + setb cl + or cl, al + test cl, 1 + je .LBB5_2 + xor eax, eax + ret +.LBB5_2: + lea rax, [rdi + rsi] + add rax, -6 + ret diff --git a/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64.mca new file mode 100644 index 000000000000..ef5892647b81 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_static_size.x86-64.mca @@ -0,0 +1,61 @@ +Iterations: 100 +Instructions: 1100 +Total Cycles: 338 +Total uOps: 1100 + +Dispatch Width: 4 +uOps Per Cycle: 3.25 +IPC: 3.25 +Block RThroughput: 3.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea eax, [rsi + rdi] + 1 1 0.33 cmp rsi, 6 + 1 1 0.50 setb cl + 1 1 0.33 or cl, al + 1 1 0.33 test cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.50 lea rax, [rdi + rsi] + 1 1 0.33 add rax, -6 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.32 3.33 - 3.35 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.97 0.03 - - - - lea eax, [rsi + rdi] + - - 0.33 0.32 - 0.35 - - cmp rsi, 6 + - - 1.00 - - - - - setb cl + - - - 1.00 - - - - or cl, al + - - - 1.00 - - - - test cl, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.34 0.66 - - - - lea rax, [rdi + rsi] + - - 0.68 0.32 - - - - add rax, -6 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..73d91cee5992 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_suffix_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_suffix_with_elems(source, count) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..c3d10b5fc685 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,27 @@ +bench_ref_from_suffix_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + ja .LBB5_1 + lea r8d, [rsi + rdi] + xor ecx, ecx + mov eax, 0 + test r8b, 3 + je .LBB5_3 + mov rdx, rcx + ret +.LBB5_3: + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + sub rsi, rax + jae .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + add rdi, rsi + mov rcx, rdx + mov rax, rdi + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..92e6280bb4cc --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,85 @@ +Iterations: 100 +Instructions: 2300 +Total Cycles: 706 +Total uOps: 2300 + +Dispatch Width: 4 +uOps Per Cycle: 3.26 +IPC: 3.26 +Block RThroughput: 6.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r8d, [rsi + rdi] + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test r8b, 3 + 1 1 1.00 je .LBB5_3 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 1 0.33 sub rsi, rax + 1 1 1.00 jae .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 1 0.33 add rdi, rsi + 1 1 0.33 mov rcx, rdx + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.99 7.00 - 7.01 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - movabs rax, 3074457345618258598 + - - 0.01 0.50 - 0.49 - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - - 1.00 - - - - lea r8d, [rsi + rdi] + - - - - - - - - xor ecx, ecx + - - 0.50 0.49 - 0.01 - - mov eax, 0 + - - 0.49 0.51 - - - - test r8b, 3 + - - - - - 1.00 - - je .LBB5_3 + - - 0.51 0.49 - - - - mov rdx, rcx + - - - - - 1.00 - - ret + - - 0.50 0.50 - - - - lea rax, [rdx + 2*rdx] + - - 1.00 - - - - - or rax, 3 + - - 1.00 - - - - - add rax, 9 + - - 0.99 0.01 - - - - sub rsi, rax + - - - - - 1.00 - - jae .LBB5_4 + - - - - - - - - xor eax, eax + - - - 1.00 - - - - mov edx, 1 + - - - - - 1.00 - - ret + - - 1.00 - - - - - add rdi, rsi + - - - 1.00 - - - - mov rcx, rdx + - - 0.99 0.01 - - - - mov rax, rdi + - - - 0.50 - 0.50 - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.rs b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..68a28baf55e6 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_ref_from_suffix_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::LocoPacket> { + match zerocopy::FromBytes::ref_from_suffix_with_elems(source, count) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..bdca57192455 --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64 @@ -0,0 +1,23 @@ +bench_ref_from_suffix_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + ja .LBB5_1 + lea r8d, [rsi + rdi] + xor ecx, ecx + mov eax, 0 + test r8b, 1 + jne .LBB5_5 + lea rax, [2*rdx + 4] + sub rsi, rax + jae .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + add rdi, rsi + mov rcx, rdx + mov rax, rdi +.LBB5_5: + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..6d9de0b3eb5c --- /dev/null +++ b/rust/zerocopy/benches/ref_from_suffix_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 571 +Total uOps: 1900 + +Dispatch Width: 4 +uOps Per Cycle: 3.33 +IPC: 3.33 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r8d, [rsi + rdi] + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test r8b, 1 + 1 1 1.00 jne .LBB5_5 + 1 1 0.50 lea rax, [2*rdx + 4] + 1 1 0.33 sub rsi, rax + 1 1 1.00 jae .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 1 0.33 add rdi, rsi + 1 1 0.33 mov rcx, rdx + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.66 5.66 - 5.68 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.66 0.33 - 0.01 - - movabs rax, 4611686018427387901 + - - 0.01 0.99 - - - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.99 0.01 - - - - lea r8d, [rsi + rdi] + - - - - - - - - xor ecx, ecx + - - 0.33 0.33 - 0.34 - - mov eax, 0 + - - 0.33 0.34 - 0.33 - - test r8b, 1 + - - - - - 1.00 - - jne .LBB5_5 + - - 0.34 0.66 - - - - lea rax, [2*rdx + 4] + - - - 1.00 - - - - sub rsi, rax + - - - - - 1.00 - - jae .LBB5_4 + - - - - - - - - xor eax, eax + - - 1.00 - - - - - mov edx, 1 + - - - - - 1.00 - - ret + - - - 1.00 - - - - add rdi, rsi + - - 1.00 - - - - - mov rcx, rdx + - - 0.32 0.68 - - - - mov rax, rdi + - - 0.68 0.32 - - - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_at_dynamic_padding.rs b/rust/zerocopy/benches/split_at_dynamic_padding.rs new file mode 100644 index 000000000000..bed90f60165e --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_padding.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_at_dynamic_padding( + source: &format::CocoPacket, + len: usize, +) -> Option<Split<&format::CocoPacket>> { + source.split_at(len) +} diff --git a/rust/zerocopy/benches/split_at_dynamic_padding.x86-64 b/rust/zerocopy/benches/split_at_dynamic_padding.x86-64 new file mode 100644 index 000000000000..6eaf5a004612 --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_padding.x86-64 @@ -0,0 +1,12 @@ +bench_split_at_dynamic_padding: + mov rax, rdi + cmp rcx, rdx + jbe .LBB5_2 + xor esi, esi + mov qword ptr [rax], rsi + ret +.LBB5_2: + mov qword ptr [rax + 8], rdx + mov qword ptr [rax + 16], rcx + mov qword ptr [rax], rsi + ret diff --git a/rust/zerocopy/benches/split_at_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/split_at_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..19ab3414d77a --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_padding.x86-64.mca @@ -0,0 +1,59 @@ +Iterations: 100 +Instructions: 1000 +Total Cycles: 404 +Total uOps: 1000 + +Dispatch Width: 4 +uOps Per Cycle: 2.48 +IPC: 2.48 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 cmp rcx, rdx + 1 1 1.00 jbe .LBB5_2 + 1 0 0.25 xor esi, esi + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + 1 1 1.00 * mov qword ptr [rax + 8], rdx + 1 1 1.00 * mov qword ptr [rax + 16], rcx + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 4.00 3.01 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.99 - - 0.01 - - mov rax, rdi + - - - 1.00 - - - - cmp rcx, rdx + - - - - - 1.00 - - jbe .LBB5_2 + - - - - - - - - xor esi, esi + - - - - 1.00 - - 1.00 mov qword ptr [rax], rsi + - - - - - 1.00 - - ret + - - - - 1.00 - 1.00 - mov qword ptr [rax + 8], rdx + - - - - 1.00 - - 1.00 mov qword ptr [rax + 16], rcx + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_at_dynamic_size.rs b/rust/zerocopy/benches/split_at_dynamic_size.rs new file mode 100644 index 000000000000..07a22ba2e5b5 --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_at_dynamic_size( + source: &format::CocoPacket, + len: usize, +) -> Option<Split<&format::CocoPacket>> { + source.split_at(len) +} diff --git a/rust/zerocopy/benches/split_at_dynamic_size.x86-64 b/rust/zerocopy/benches/split_at_dynamic_size.x86-64 new file mode 100644 index 000000000000..8d81b98bdab1 --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_size.x86-64 @@ -0,0 +1,12 @@ +bench_split_at_dynamic_size: + mov rax, rdi + cmp rcx, rdx + jbe .LBB5_2 + xor esi, esi + mov qword ptr [rax], rsi + ret +.LBB5_2: + mov qword ptr [rax + 8], rdx + mov qword ptr [rax + 16], rcx + mov qword ptr [rax], rsi + ret diff --git a/rust/zerocopy/benches/split_at_dynamic_size.x86-64.mca b/rust/zerocopy/benches/split_at_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..19ab3414d77a --- /dev/null +++ b/rust/zerocopy/benches/split_at_dynamic_size.x86-64.mca @@ -0,0 +1,59 @@ +Iterations: 100 +Instructions: 1000 +Total Cycles: 404 +Total uOps: 1000 + +Dispatch Width: 4 +uOps Per Cycle: 2.48 +IPC: 2.48 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 cmp rcx, rdx + 1 1 1.00 jbe .LBB5_2 + 1 0 0.25 xor esi, esi + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + 1 1 1.00 * mov qword ptr [rax + 8], rdx + 1 1 1.00 * mov qword ptr [rax + 16], rcx + 1 1 1.00 * mov qword ptr [rax], rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 4.00 3.01 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.99 - - 0.01 - - mov rax, rdi + - - - 1.00 - - - - cmp rcx, rdx + - - - - - 1.00 - - jbe .LBB5_2 + - - - - - - - - xor esi, esi + - - - - 1.00 - - 1.00 mov qword ptr [rax], rsi + - - - - - 1.00 - - ret + - - - - 1.00 - 1.00 - mov qword ptr [rax + 8], rdx + - - - - 1.00 - - 1.00 mov qword ptr [rax + 16], rcx + - - - - 1.00 - 1.00 - mov qword ptr [rax], rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.rs b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.rs new file mode 100644 index 000000000000..3c147d3bdd39 --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +unsafe fn bench_split_at_unchecked_dynamic_padding( + source: &format::CocoPacket, + len: usize, +) -> Split<&format::CocoPacket> { + unsafe { source.split_at_unchecked(len) } +} diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64 b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64 new file mode 100644 index 000000000000..74c3b52f63fa --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64 @@ -0,0 +1,6 @@ +bench_split_at_unchecked_dynamic_padding: + mov rax, rdi + mov qword ptr [rdi], rsi + mov qword ptr [rdi + 8], rdx + mov qword ptr [rdi + 16], rcx + ret diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..e8c61591c086 --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_padding.x86-64.mca @@ -0,0 +1,49 @@ +Iterations: 100 +Instructions: 500 +Total Cycles: 303 +Total uOps: 500 + +Dispatch Width: 4 +uOps Per Cycle: 1.65 +IPC: 1.65 +Block RThroughput: 3.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 1.00 * mov qword ptr [rdi], rsi + 1 1 1.00 * mov qword ptr [rdi + 8], rdx + 1 1 1.00 * mov qword ptr [rdi + 16], rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.49 0.50 3.00 1.01 1.50 1.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.49 0.50 - 0.01 - - mov rax, rdi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi], rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi + 8], rdx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi + 16], rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_size.rs b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.rs new file mode 100644 index 000000000000..b1aa1dfb35be --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +unsafe fn bench_split_at_unchecked_dynamic_size( + source: &format::CocoPacket, + len: usize, +) -> Split<&format::CocoPacket> { + unsafe { source.split_at_unchecked(len) } +} diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64 b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64 new file mode 100644 index 000000000000..56671d1ee8dc --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64 @@ -0,0 +1,6 @@ +bench_split_at_unchecked_dynamic_size: + mov rax, rdi + mov qword ptr [rdi], rsi + mov qword ptr [rdi + 8], rdx + mov qword ptr [rdi + 16], rcx + ret diff --git a/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64.mca b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..e8c61591c086 --- /dev/null +++ b/rust/zerocopy/benches/split_at_unchecked_dynamic_size.x86-64.mca @@ -0,0 +1,49 @@ +Iterations: 100 +Instructions: 500 +Total Cycles: 303 +Total uOps: 500 + +Dispatch Width: 4 +uOps Per Cycle: 1.65 +IPC: 1.65 +Block RThroughput: 3.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 1.00 * mov qword ptr [rdi], rsi + 1 1 1.00 * mov qword ptr [rdi + 8], rdx + 1 1 1.00 * mov qword ptr [rdi + 16], rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.49 0.50 3.00 1.01 1.50 1.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.49 0.50 - 0.01 - - mov rax, rdi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi], rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi + 8], rdx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rdi + 16], rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_padding.rs b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.rs new file mode 100644 index 000000000000..b86ad3614bf7 --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_via_immutable_dynamic_padding( + split: Split<&format::CocoPacket>, +) -> (&format::CocoPacket, &[[u8; 3]]) { + split.via_immutable() +} diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64 b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64 new file mode 100644 index 000000000000..dac183428b31 --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64 @@ -0,0 +1,14 @@ +bench_split_via_immutable_dynamic_padding: + mov rax, rdi + mov rcx, qword ptr [rsi] + mov rdx, qword ptr [rsi + 8] + mov rsi, qword ptr [rsi + 16] + lea rdi, [rsi + 2*rsi] + add rdi, rcx + add rdi, 9 + sub rdx, rsi + mov qword ptr [rax], rcx + mov qword ptr [rax + 8], rsi + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rdx + ret diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..6ab4e838767e --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_padding.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 510 +Total uOps: 1300 + +Dispatch Width: 4 +uOps Per Cycle: 2.55 +IPC: 2.55 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rdx, qword ptr [rsi + 8] + 1 5 0.50 * mov rsi, qword ptr [rsi + 16] + 1 1 0.50 lea rdi, [rsi + 2*rsi] + 1 1 0.33 add rdi, rcx + 1 1 0.33 add rdi, 9 + 1 1 0.33 sub rdx, rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 * mov qword ptr [rax + 8], rsi + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rdx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.00 2.00 4.00 2.00 3.50 3.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.03 0.93 - 0.04 - - mov rax, rdi + - - - - - - 0.49 0.51 mov rcx, qword ptr [rsi] + - - - - - - 1.00 - mov rdx, qword ptr [rsi + 8] + - - - - - - 0.01 0.99 mov rsi, qword ptr [rsi + 16] + - - 0.93 0.07 - - - - lea rdi, [rsi + 2*rsi] + - - 0.05 0.02 - 0.93 - - add rdi, rcx + - - 0.49 0.49 - 0.02 - - add rdi, 9 + - - 0.50 0.49 - 0.01 - - sub rdx, rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax], rcx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax + 8], rsi + - - - - 1.00 - 0.49 0.51 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.51 0.49 mov qword ptr [rax + 24], rdx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_size.rs b/rust/zerocopy/benches/split_via_immutable_dynamic_size.rs new file mode 100644 index 000000000000..7d115caa3c00 --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_size.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_via_immutable_dynamic_size( + split: Split<&format::CocoPacket>, +) -> (&format::CocoPacket, &[[u8; 2]]) { + split.via_immutable() +} diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64 b/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64 new file mode 100644 index 000000000000..58f6b09fc9d2 --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64 @@ -0,0 +1,13 @@ +bench_split_via_immutable_dynamic_size: + mov rax, rdi + mov rcx, qword ptr [rsi] + mov rdx, qword ptr [rsi + 8] + mov rsi, qword ptr [rsi + 16] + lea rdi, [rcx + 2*rsi] + add rdi, 4 + sub rdx, rsi + mov qword ptr [rax], rcx + mov qword ptr [rax + 8], rsi + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rdx + ret diff --git a/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64.mca b/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..4549f20de53c --- /dev/null +++ b/rust/zerocopy/benches/split_via_immutable_dynamic_size.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 509 +Total uOps: 1200 + +Dispatch Width: 4 +uOps Per Cycle: 2.36 +IPC: 2.36 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rdx, qword ptr [rsi + 8] + 1 5 0.50 * mov rsi, qword ptr [rsi + 16] + 1 1 0.50 lea rdi, [rcx + 2*rsi] + 1 1 0.33 add rdi, 4 + 1 1 0.33 sub rdx, rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 * mov qword ptr [rax + 8], rsi + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rdx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.66 1.66 4.00 1.68 3.50 3.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.34 0.33 - 0.33 - - mov rax, rdi + - - - - - - 0.49 0.51 mov rcx, qword ptr [rsi] + - - - - - - 0.51 0.49 mov rdx, qword ptr [rsi + 8] + - - - - - - 0.01 0.99 mov rsi, qword ptr [rsi + 16] + - - 0.33 0.67 - - - - lea rdi, [rcx + 2*rsi] + - - 0.63 0.34 - 0.03 - - add rdi, 4 + - - 0.36 0.32 - 0.32 - - sub rdx, rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax], rcx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax + 8], rsi + - - - - 1.00 - 0.98 0.02 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.51 0.49 mov qword ptr [rax + 24], rdx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.rs b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.rs new file mode 100644 index 000000000000..edba7bf34ba8 --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_via_runtime_check_dynamic_padding( + split: Split<&format::CocoPacket>, +) -> Option<(&format::CocoPacket, &[[u8; 3]])> { + split.via_runtime_check().ok() +} diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64 b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64 new file mode 100644 index 000000000000..03684cb09f8f --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64 @@ -0,0 +1,22 @@ +bench_split_via_runtime_check_dynamic_padding: + mov rax, rdi + mov rdx, qword ptr [rsi + 16] + mov ecx, edx + and ecx, 3 + cmp ecx, 1 + jne .LBB5_1 + mov rcx, qword ptr [rsi] + mov rsi, qword ptr [rsi + 8] + lea rdi, [rdx + 2*rdx] + add rdi, rcx + add rdi, 9 + sub rsi, rdx + mov qword ptr [rax + 8], rdx + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rsi + mov qword ptr [rax], rcx + ret +.LBB5_1: + xor ecx, ecx + mov qword ptr [rax], rcx + ret diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..5034ab0583bd --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_padding.x86-64.mca @@ -0,0 +1,79 @@ +Iterations: 100 +Instructions: 2000 +Total Cycles: 708 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 2.82 +IPC: 2.82 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rdx, qword ptr [rsi + 16] + 1 1 0.33 mov ecx, edx + 1 1 0.33 and ecx, 3 + 1 1 0.33 cmp ecx, 1 + 1 1 1.00 jne .LBB5_1 + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rsi, qword ptr [rsi + 8] + 1 1 0.50 lea rdi, [rdx + 2*rdx] + 1 1 0.33 add rdi, rcx + 1 1 0.33 add rdi, 9 + 1 1 0.33 sub rsi, rdx + 1 1 1.00 * mov qword ptr [rax + 8], rdx + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 U ret + 1 0 0.25 xor ecx, ecx + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.00 3.02 5.00 4.98 4.00 4.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - mov rax, rdi + - - - - - - - 1.00 mov rdx, qword ptr [rsi + 16] + - - 0.99 0.01 - - - - mov ecx, edx + - - 0.99 0.01 - - - - and ecx, 3 + - - 0.97 0.03 - - - - cmp ecx, 1 + - - - - - 1.00 - - jne .LBB5_1 + - - - - - - 1.00 - mov rcx, qword ptr [rsi] + - - - - - - 0.99 0.01 mov rsi, qword ptr [rsi + 8] + - - 0.01 0.99 - - - - lea rdi, [rdx + 2*rdx] + - - - 0.96 - 0.04 - - add rdi, rcx + - - 0.03 - - 0.97 - - add rdi, 9 + - - 0.01 0.03 - 0.96 - - sub rsi, rdx + - - - - 1.00 - 1.00 - mov qword ptr [rax + 8], rdx + - - - - 1.00 - - 1.00 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.01 0.99 mov qword ptr [rax + 24], rsi + - - - - 1.00 - 0.99 0.01 mov qword ptr [rax], rcx + - - - - - 1.00 - - ret + - - - - - - - - xor ecx, ecx + - - - - 1.00 - 0.01 0.99 mov qword ptr [rax], rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.rs b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.rs new file mode 100644 index 000000000000..ce22de909aca --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_split_via_runtime_check_dynamic_size( + split: Split<&format::CocoPacket>, +) -> Option<(&format::CocoPacket, &[[u8; 2]])> { + split.via_runtime_check().ok() +} diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64 b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64 new file mode 100644 index 000000000000..e54276bd85e8 --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64 @@ -0,0 +1,13 @@ +bench_split_via_runtime_check_dynamic_size: + mov rax, rdi + mov rcx, qword ptr [rsi] + mov rdx, qword ptr [rsi + 8] + mov rsi, qword ptr [rsi + 16] + lea rdi, [rcx + 2*rsi] + add rdi, 4 + sub rdx, rsi + mov qword ptr [rax], rcx + mov qword ptr [rax + 8], rsi + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rdx + ret diff --git a/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64.mca b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..4549f20de53c --- /dev/null +++ b/rust/zerocopy/benches/split_via_runtime_check_dynamic_size.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 509 +Total uOps: 1200 + +Dispatch Width: 4 +uOps Per Cycle: 2.36 +IPC: 2.36 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rdx, qword ptr [rsi + 8] + 1 5 0.50 * mov rsi, qword ptr [rsi + 16] + 1 1 0.50 lea rdi, [rcx + 2*rsi] + 1 1 0.33 add rdi, 4 + 1 1 0.33 sub rdx, rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 * mov qword ptr [rax + 8], rsi + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rdx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.66 1.66 4.00 1.68 3.50 3.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.34 0.33 - 0.33 - - mov rax, rdi + - - - - - - 0.49 0.51 mov rcx, qword ptr [rsi] + - - - - - - 0.51 0.49 mov rdx, qword ptr [rsi + 8] + - - - - - - 0.01 0.99 mov rsi, qword ptr [rsi + 16] + - - 0.33 0.67 - - - - lea rdi, [rcx + 2*rsi] + - - 0.63 0.34 - 0.03 - - add rdi, 4 + - - 0.36 0.32 - 0.32 - - sub rdx, rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax], rcx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax + 8], rsi + - - - - 1.00 - 0.98 0.02 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.51 0.49 mov qword ptr [rax + 24], rdx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.rs b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.rs new file mode 100644 index 000000000000..21d74dba176f --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +unsafe fn bench_split_via_unchecked_dynamic_padding( + split: Split<&format::CocoPacket>, +) -> (&format::CocoPacket, &[[u8; 3]]) { + unsafe { split.via_unchecked() } +} diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64 b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64 new file mode 100644 index 000000000000..3c2c4ec9f6ed --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64 @@ -0,0 +1,14 @@ +bench_split_via_unchecked_dynamic_padding: + mov rax, rdi + mov rcx, qword ptr [rsi] + mov rdx, qword ptr [rsi + 8] + mov rsi, qword ptr [rsi + 16] + lea rdi, [rsi + 2*rsi] + add rdi, rcx + add rdi, 9 + sub rdx, rsi + mov qword ptr [rax], rcx + mov qword ptr [rax + 8], rsi + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rdx + ret diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..6ab4e838767e --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_padding.x86-64.mca @@ -0,0 +1,65 @@ +Iterations: 100 +Instructions: 1300 +Total Cycles: 510 +Total uOps: 1300 + +Dispatch Width: 4 +uOps Per Cycle: 2.55 +IPC: 2.55 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rdx, qword ptr [rsi + 8] + 1 5 0.50 * mov rsi, qword ptr [rsi + 16] + 1 1 0.50 lea rdi, [rsi + 2*rsi] + 1 1 0.33 add rdi, rcx + 1 1 0.33 add rdi, 9 + 1 1 0.33 sub rdx, rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 * mov qword ptr [rax + 8], rsi + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rdx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.00 2.00 4.00 2.00 3.50 3.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.03 0.93 - 0.04 - - mov rax, rdi + - - - - - - 0.49 0.51 mov rcx, qword ptr [rsi] + - - - - - - 1.00 - mov rdx, qword ptr [rsi + 8] + - - - - - - 0.01 0.99 mov rsi, qword ptr [rsi + 16] + - - 0.93 0.07 - - - - lea rdi, [rsi + 2*rsi] + - - 0.05 0.02 - 0.93 - - add rdi, rcx + - - 0.49 0.49 - 0.02 - - add rdi, 9 + - - 0.50 0.49 - 0.01 - - sub rdx, rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax], rcx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax + 8], rsi + - - - - 1.00 - 0.49 0.51 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.51 0.49 mov qword ptr [rax + 24], rdx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_size.rs b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.rs new file mode 100644 index 000000000000..824e22d67a7b --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.rs @@ -0,0 +1,11 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +unsafe fn bench_split_via_unchecked_dynamic_size( + split: Split<&format::CocoPacket>, +) -> (&format::CocoPacket, &[[u8; 2]]) { + unsafe { split.via_unchecked() } +} diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64 b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64 new file mode 100644 index 000000000000..1e31268edc7f --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64 @@ -0,0 +1,13 @@ +bench_split_via_unchecked_dynamic_size: + mov rax, rdi + mov rcx, qword ptr [rsi] + mov rdx, qword ptr [rsi + 8] + mov rsi, qword ptr [rsi + 16] + lea rdi, [rcx + 2*rsi] + add rdi, 4 + sub rdx, rsi + mov qword ptr [rax], rcx + mov qword ptr [rax + 8], rsi + mov qword ptr [rax + 16], rdi + mov qword ptr [rax + 24], rdx + ret diff --git a/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64.mca b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..4549f20de53c --- /dev/null +++ b/rust/zerocopy/benches/split_via_unchecked_dynamic_size.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 509 +Total uOps: 1200 + +Dispatch Width: 4 +uOps Per Cycle: 2.36 +IPC: 2.36 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 5 0.50 * mov rcx, qword ptr [rsi] + 1 5 0.50 * mov rdx, qword ptr [rsi + 8] + 1 5 0.50 * mov rsi, qword ptr [rsi + 16] + 1 1 0.50 lea rdi, [rcx + 2*rsi] + 1 1 0.33 add rdi, 4 + 1 1 0.33 sub rdx, rsi + 1 1 1.00 * mov qword ptr [rax], rcx + 1 1 1.00 * mov qword ptr [rax + 8], rsi + 1 1 1.00 * mov qword ptr [rax + 16], rdi + 1 1 1.00 * mov qword ptr [rax + 24], rdx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.66 1.66 4.00 1.68 3.50 3.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.34 0.33 - 0.33 - - mov rax, rdi + - - - - - - 0.49 0.51 mov rcx, qword ptr [rsi] + - - - - - - 0.51 0.49 mov rdx, qword ptr [rsi + 8] + - - - - - - 0.01 0.99 mov rsi, qword ptr [rsi + 16] + - - 0.33 0.67 - - - - lea rdi, [rcx + 2*rsi] + - - 0.63 0.34 - 0.03 - - add rdi, 4 + - - 0.36 0.32 - 0.32 - - sub rdx, rsi + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax], rcx + - - - - 1.00 - 0.50 0.50 mov qword ptr [rax + 8], rsi + - - - - 1.00 - 0.98 0.02 mov qword ptr [rax + 16], rdi + - - - - 1.00 - 0.51 0.49 mov qword ptr [rax + 24], rdx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/transmute.rs b/rust/zerocopy/benches/transmute.rs new file mode 100644 index 000000000000..e60bfb252f5c --- /dev/null +++ b/rust/zerocopy/benches/transmute.rs @@ -0,0 +1,16 @@ +use zerocopy::Unalign; +use zerocopy_derive::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C)] +struct MinimalViableSource { + bytes: [u8; 6], +} + +#[unsafe(no_mangle)] +fn bench_transmute(source: MinimalViableSource) -> Unalign<format::LocoPacket> { + zerocopy::transmute!(source) +} diff --git a/rust/zerocopy/benches/transmute.x86-64 b/rust/zerocopy/benches/transmute.x86-64 new file mode 100644 index 000000000000..a4c6299d1112 --- /dev/null +++ b/rust/zerocopy/benches/transmute.x86-64 @@ -0,0 +1,3 @@ +bench_transmute: + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/transmute.x86-64.mca b/rust/zerocopy/benches/transmute.x86-64.mca new file mode 100644 index 000000000000..f297729c6523 --- /dev/null +++ b/rust/zerocopy/benches/transmute.x86-64.mca @@ -0,0 +1,43 @@ +Iterations: 100 +Instructions: 200 +Total Cycles: 104 +Total uOps: 200 + +Dispatch Width: 4 +uOps Per Cycle: 1.92 +IPC: 1.92 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.49 0.50 - 1.01 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.49 0.50 - 0.01 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/transmute_ref_dynamic_size.rs b/rust/zerocopy/benches/transmute_ref_dynamic_size.rs new file mode 100644 index 000000000000..825f0f2bed24 --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_dynamic_size.rs @@ -0,0 +1,16 @@ +use zerocopy_derive::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C, align(2))] +struct MinimalViableSource { + header: [u8; 6], + trailer: [[u8; 2]], +} + +#[unsafe(no_mangle)] +fn bench_transmute_ref_dynamic_size(source: &MinimalViableSource) -> &format::LocoPacket { + zerocopy::transmute_ref!(source) +} diff --git a/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64 b/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64 new file mode 100644 index 000000000000..80a0f5906610 --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64 @@ -0,0 +1,4 @@ +bench_transmute_ref_dynamic_size: + mov rax, rdi + lea rdx, [rsi + 1] + ret diff --git a/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64.mca b/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..ef1bcdca5ea3 --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_dynamic_size.x86-64.mca @@ -0,0 +1,45 @@ +Iterations: 100 +Instructions: 300 +Total Cycles: 104 +Total uOps: 300 + +Dispatch Width: 4 +uOps Per Cycle: 2.88 +IPC: 2.88 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.50 lea rdx, [rsi + 1] + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 - 1.01 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.99 - - 0.01 - - mov rax, rdi + - - - 1.00 - - - - lea rdx, [rsi + 1] + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/transmute_ref_static_size.rs b/rust/zerocopy/benches/transmute_ref_static_size.rs new file mode 100644 index 000000000000..a6db611fde0d --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_static_size.rs @@ -0,0 +1,15 @@ +use zerocopy_derive::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C, align(2))] +struct MinimalViableSource { + bytes: [u8; 6], +} + +#[unsafe(no_mangle)] +fn bench_transmute_ref_static_size(source: &MinimalViableSource) -> &format::LocoPacket { + zerocopy::transmute_ref!(source) +} diff --git a/rust/zerocopy/benches/transmute_ref_static_size.x86-64 b/rust/zerocopy/benches/transmute_ref_static_size.x86-64 new file mode 100644 index 000000000000..7a9229c129b0 --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_static_size.x86-64 @@ -0,0 +1,3 @@ +bench_transmute_ref_static_size: + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/transmute_ref_static_size.x86-64.mca b/rust/zerocopy/benches/transmute_ref_static_size.x86-64.mca new file mode 100644 index 000000000000..f297729c6523 --- /dev/null +++ b/rust/zerocopy/benches/transmute_ref_static_size.x86-64.mca @@ -0,0 +1,43 @@ +Iterations: 100 +Instructions: 200 +Total Cycles: 104 +Total uOps: 200 + +Dispatch Width: 4 +uOps Per Cycle: 1.92 +IPC: 1.92 +Block RThroughput: 1.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.49 0.50 - 1.01 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.49 0.50 - 0.01 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_read_from_bytes.rs b/rust/zerocopy/benches/try_read_from_bytes.rs new file mode 100644 index 000000000000..f8384b32f518 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_bytes.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_read_from_bytes_static_size(source: &[u8]) -> Option<format::CocoPacket> { + zerocopy::TryFromBytes::try_read_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/try_read_from_bytes.x86-64 b/rust/zerocopy/benches/try_read_from_bytes.x86-64 new file mode 100644 index 000000000000..08088a08fd85 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_bytes.x86-64 @@ -0,0 +1,23 @@ +bench_try_read_from_bytes_static_size: + mov ax, -16191 + cmp rsi, 6 + jne .LBB5_1 + mov ecx, dword ptr [rdi] + movzx edx, cx + cmp edx, 49344 + jne .LBB5_4 + movzx eax, word ptr [rdi + 4] + shl rax, 32 + or rcx, rax + shr rcx, 16 + mov ax, -16192 +.LBB5_4: + shl rcx, 16 + movzx eax, ax + or rax, rcx + ret +.LBB5_1: + shl rcx, 16 + movzx eax, ax + or rax, rcx + ret diff --git a/rust/zerocopy/benches/try_read_from_bytes.x86-64.mca b/rust/zerocopy/benches/try_read_from_bytes.x86-64.mca new file mode 100644 index 000000000000..385e6a480253 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_bytes.x86-64.mca @@ -0,0 +1,79 @@ +Iterations: 100 +Instructions: 2000 +Total Cycles: 608 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 3.29 +IPC: 3.29 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov ax, -16191 + 1 1 0.33 cmp rsi, 6 + 1 1 1.00 jne .LBB5_1 + 1 5 0.50 * mov ecx, dword ptr [rdi] + 1 1 0.33 movzx edx, cx + 1 1 0.33 cmp edx, 49344 + 1 1 1.00 jne .LBB5_4 + 1 5 0.50 * movzx eax, word ptr [rdi + 4] + 1 1 0.50 shl rax, 32 + 1 1 0.33 or rcx, rax + 1 1 0.50 shr rcx, 16 + 1 1 0.33 mov ax, -16192 + 1 1 0.50 shl rcx, 16 + 1 1 0.33 movzx eax, ax + 1 1 0.33 or rax, rcx + 1 1 1.00 U ret + 1 1 0.50 shl rcx, 16 + 1 1 0.33 movzx eax, ax + 1 1 0.33 or rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.99 5.99 - 6.02 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - mov ax, -16191 + - - - 0.01 - 0.99 - - cmp rsi, 6 + - - - - - 1.00 - - jne .LBB5_1 + - - - - - - - 1.00 mov ecx, dword ptr [rdi] + - - 0.98 - - 0.02 - - movzx edx, cx + - - 0.99 0.01 - - - - cmp edx, 49344 + - - - - - 1.00 - - jne .LBB5_4 + - - - - - - 1.00 - movzx eax, word ptr [rdi + 4] + - - 0.01 - - 0.99 - - shl rax, 32 + - - 0.02 0.98 - - - - or rcx, rax + - - 1.00 - - - - - shr rcx, 16 + - - 0.99 0.01 - - - - mov ax, -16192 + - - 1.00 - - - - - shl rcx, 16 + - - - 1.00 - - - - movzx eax, ax + - - - 1.00 - - - - or rax, rcx + - - - - - 1.00 - - ret + - - 1.00 - - - - - shl rcx, 16 + - - - 1.00 - - - - movzx eax, ax + - - - 0.99 - 0.01 - - or rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_read_from_prefix.rs b/rust/zerocopy/benches/try_read_from_prefix.rs new file mode 100644 index 000000000000..fabbac3bd7d9 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_prefix.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_read_from_prefix_static_size(source: &[u8]) -> Option<format::CocoPacket> { + match zerocopy::TryFromBytes::try_read_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_read_from_prefix.x86-64 b/rust/zerocopy/benches/try_read_from_prefix.x86-64 new file mode 100644 index 000000000000..d3e1edc3ea73 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_prefix.x86-64 @@ -0,0 +1,16 @@ +bench_try_read_from_prefix_static_size: + mov eax, 49345 + cmp rsi, 6 + jb .LBB5_2 + mov eax, dword ptr [rdi] + movzx ecx, word ptr [rdi + 4] + shl rcx, 32 + or rcx, rax + movzx eax, cx + and rcx, -65536 + or rcx, 49344 + cmp eax, 49344 + mov eax, 49345 + cmove rax, rcx +.LBB5_2: + ret diff --git a/rust/zerocopy/benches/try_read_from_prefix.x86-64.mca b/rust/zerocopy/benches/try_read_from_prefix.x86-64.mca new file mode 100644 index 000000000000..40401d89e83d --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_prefix.x86-64.mca @@ -0,0 +1,67 @@ +Iterations: 100 +Instructions: 1400 +Total Cycles: 442 +Total uOps: 1500 + +Dispatch Width: 4 +uOps Per Cycle: 3.39 +IPC: 3.17 +Block RThroughput: 3.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov eax, 49345 + 1 1 0.33 cmp rsi, 6 + 1 1 1.00 jb .LBB5_2 + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 5 0.50 * movzx ecx, word ptr [rdi + 4] + 1 1 0.50 shl rcx, 32 + 1 1 0.33 or rcx, rax + 1 1 0.33 movzx eax, cx + 1 1 0.33 and rcx, -65536 + 1 1 0.33 or rcx, 49344 + 1 1 0.33 cmp eax, 49344 + 1 1 0.33 mov eax, 49345 + 2 2 0.67 cmove rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.33 4.33 - 4.34 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.65 0.01 - 0.34 - - mov eax, 49345 + - - 0.01 0.33 - 0.66 - - cmp rsi, 6 + - - - - - 1.00 - - jb .LBB5_2 + - - - - - - - 1.00 mov eax, dword ptr [rdi] + - - - - - - 1.00 - movzx ecx, word ptr [rdi + 4] + - - 0.65 - - 0.35 - - shl rcx, 32 + - - - 0.67 - 0.33 - - or rcx, rax + - - 0.01 0.99 - - - - movzx eax, cx + - - 0.99 0.01 - - - - and rcx, -65536 + - - 0.01 0.99 - - - - or rcx, 49344 + - - 0.99 0.01 - - - - cmp eax, 49344 + - - 0.02 0.33 - 0.65 - - mov eax, 49345 + - - 1.00 0.99 - 0.01 - - cmove rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_read_from_suffix.rs b/rust/zerocopy/benches/try_read_from_suffix.rs new file mode 100644 index 000000000000..1e961647b3f3 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_suffix.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_read_from_suffix_static_size(source: &[u8]) -> Option<format::CocoPacket> { + match zerocopy::TryFromBytes::try_read_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_read_from_suffix.x86-64 b/rust/zerocopy/benches/try_read_from_suffix.x86-64 new file mode 100644 index 000000000000..095e326f0467 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_suffix.x86-64 @@ -0,0 +1,18 @@ +bench_try_read_from_suffix_static_size: + mov eax, 49345 + cmp rsi, 6 + jb .LBB5_2 + mov eax, dword ptr [rdi + rsi - 6] + movzx ecx, word ptr [rdi + rsi - 2] + shl rcx, 32 + or rcx, rax + movzx edx, cx + xor eax, eax + cmp edx, 49344 + cmovne rcx, rsi + sete al + and rcx, -65536 + xor rax, 49345 + or rax, rcx +.LBB5_2: + ret diff --git a/rust/zerocopy/benches/try_read_from_suffix.x86-64.mca b/rust/zerocopy/benches/try_read_from_suffix.x86-64.mca new file mode 100644 index 000000000000..d3eaadbb8a81 --- /dev/null +++ b/rust/zerocopy/benches/try_read_from_suffix.x86-64.mca @@ -0,0 +1,71 @@ +Iterations: 100 +Instructions: 1600 +Total Cycles: 478 +Total uOps: 1700 + +Dispatch Width: 4 +uOps Per Cycle: 3.56 +IPC: 3.35 +Block RThroughput: 4.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov eax, 49345 + 1 1 0.33 cmp rsi, 6 + 1 1 1.00 jb .LBB5_2 + 1 5 0.50 * mov eax, dword ptr [rdi + rsi - 6] + 1 5 0.50 * movzx ecx, word ptr [rdi + rsi - 2] + 1 1 0.50 shl rcx, 32 + 1 1 0.33 or rcx, rax + 1 1 0.33 movzx edx, cx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp edx, 49344 + 2 2 0.67 cmovne rcx, rsi + 1 1 0.50 sete al + 1 1 0.33 and rcx, -65536 + 1 1 0.33 xor rax, 49345 + 1 1 0.33 or rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.66 4.66 - 4.68 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.32 0.01 - 0.67 - - mov eax, 49345 + - - 0.62 0.02 - 0.36 - - cmp rsi, 6 + - - - - - 1.00 - - jb .LBB5_2 + - - - - - - - 1.00 mov eax, dword ptr [rdi + rsi - 6] + - - - - - - 1.00 - movzx ecx, word ptr [rdi + rsi - 2] + - - 0.37 - - 0.63 - - shl rcx, 32 + - - 0.99 0.01 - - - - or rcx, rax + - - 1.00 - - - - - movzx edx, cx + - - - - - - - - xor eax, eax + - - 0.35 0.64 - 0.01 - - cmp edx, 49344 + - - 1.00 1.00 - - - - cmovne rcx, rsi + - - - - - 1.00 - - sete al + - - 0.01 0.99 - - - - and rcx, -65536 + - - - 1.00 - - - - xor rax, 49345 + - - - 0.99 - 0.01 - - or rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.rs new file mode 100644 index 000000000000..126009cd71d5 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_bytes_dynamic_padding(source: &[u8]) -> Option<&format::CocoPacket> { + zerocopy::TryFromBytes::try_ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 new file mode 100644 index 000000000000..217c5fc61796 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 @@ -0,0 +1,24 @@ +bench_try_ref_from_bytes_dynamic_padding: + test dil, 3 + jne .LBB5_4 + movabs rax, 9223372036854775804 + and rax, rsi + cmp rax, 9 + jb .LBB5_4 + add rax, -9 + movabs rcx, -6148914691236517205 + mul rcx + shr rdx + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + cmp rsi, rax + jne .LBB5_4 + cmp word ptr [rdi], -16192 + je .LBB5_5 +.LBB5_4: + xor edi, edi + mov rdx, rsi +.LBB5_5: + mov rax, rdi + ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..95b993c7e0a8 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca @@ -0,0 +1,81 @@ +Iterations: 100 +Instructions: 2100 +Total Cycles: 709 +Total uOps: 2300 + +Dispatch Width: 4 +uOps Per Cycle: 3.24 +IPC: 2.96 +Block RThroughput: 5.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 test dil, 3 + 1 1 1.00 jne .LBB5_4 + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rax, rsi + 1 1 0.33 cmp rax, 9 + 1 1 1.00 jb .LBB5_4 + 1 1 0.33 add rax, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 1 0.33 cmp rsi, rax + 1 1 1.00 jne .LBB5_4 + 2 6 0.50 * cmp word ptr [rdi], -16192 + 1 1 1.00 je .LBB5_5 + 1 0 0.25 xor edi, edi + 1 1 0.33 mov rdx, rsi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.98 6.99 - 7.03 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.48 0.51 - 0.01 - - test dil, 3 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.51 0.49 - - - - movabs rax, 9223372036854775804 + - - 0.01 0.99 - - - - and rax, rsi + - - 0.51 0.49 - - - - cmp rax, 9 + - - - - - 1.00 - - jb .LBB5_4 + - - 0.98 - - 0.02 - - add rax, -9 + - - 0.98 0.02 - - - - movabs rcx, -6148914691236517205 + - - 1.00 1.00 - - - - mul rcx + - - 0.99 - - 0.01 - - shr rdx + - - - 1.00 - - - - lea rax, [rdx + 2*rdx] + - - - 0.51 - 0.49 - - or rax, 3 + - - 0.01 0.49 - 0.50 - - add rax, 9 + - - - 0.02 - 0.98 - - cmp rsi, rax + - - - - - 1.00 - - jne .LBB5_4 + - - 0.51 0.49 - - 0.50 0.50 cmp word ptr [rdi], -16192 + - - - - - 1.00 - - je .LBB5_5 + - - - - - - - - xor edi, edi + - - 0.50 0.50 - - - - mov rdx, rsi + - - 0.50 0.48 - 0.02 - - mov rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.rs new file mode 100644 index 000000000000..fc3cfbae27a2 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_bytes_dynamic_size(source: &[u8]) -> Option<&format::CocoPacket> { + zerocopy::TryFromBytes::try_ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 new file mode 100644 index 000000000000..cf67afd31ce0 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 @@ -0,0 +1,22 @@ +bench_try_ref_from_bytes_dynamic_size: + mov rdx, rsi + mov rax, rdi + cmp rsi, 4 + setb cl + or cl, al + test cl, 1 + jne .LBB5_4 + lea rcx, [rdx - 4] + mov rsi, rcx + and rsi, -2 + add rsi, 4 + cmp rdx, rsi + jne .LBB5_4 + cmp word ptr [rax], -16192 + jne .LBB5_4 + shr rcx + mov rdx, rcx + ret +.LBB5_4: + xor eax, eax + ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..ecd7a18f6d6d --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca @@ -0,0 +1,79 @@ +Iterations: 100 +Instructions: 2000 +Total Cycles: 639 +Total uOps: 2100 + +Dispatch Width: 4 +uOps Per Cycle: 3.29 +IPC: 3.13 +Block RThroughput: 5.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rdx, rsi + 1 1 0.33 mov rax, rdi + 1 1 0.33 cmp rsi, 4 + 1 1 0.50 setb cl + 1 1 0.33 or cl, al + 1 1 0.33 test cl, 1 + 1 1 1.00 jne .LBB5_4 + 1 1 0.50 lea rcx, [rdx - 4] + 1 1 0.33 mov rsi, rcx + 1 1 0.33 and rsi, -2 + 1 1 0.33 add rsi, 4 + 1 1 0.33 cmp rdx, rsi + 1 1 1.00 jne .LBB5_4 + 2 6 0.50 * cmp word ptr [rax], -16192 + 1 1 1.00 jne .LBB5_4 + 1 1 0.50 shr rcx + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.32 6.32 - 6.36 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.33 0.66 - 0.01 - - mov rdx, rsi + - - 0.66 0.34 - - - - mov rax, rdi + - - 0.34 0.66 - - - - cmp rsi, 4 + - - 0.99 - - 0.01 - - setb cl + - - 0.01 0.99 - - - - or cl, al + - - - 1.00 - - - - test cl, 1 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.66 0.34 - - - - lea rcx, [rdx - 4] + - - 0.33 0.66 - 0.01 - - mov rsi, rcx + - - 1.00 - - - - - and rsi, -2 + - - 0.66 0.34 - - - - add rsi, 4 + - - - 1.00 - - - - cmp rdx, rsi + - - - - - 1.00 - - jne .LBB5_4 + - - - - - 1.00 0.50 0.50 cmp word ptr [rax], -16192 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.67 - - 0.33 - - shr rcx + - - 0.67 0.33 - - - - mov rdx, rcx + - - - - - 1.00 - - ret + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_static_size.rs b/rust/zerocopy/benches/try_ref_from_bytes_static_size.rs new file mode 100644 index 000000000000..521557146324 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_static_size.rs @@ -0,0 +1,7 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_bytes_static_size(source: &[u8]) -> Option<&format::CocoPacket> { + zerocopy::TryFromBytes::try_ref_from_bytes(source).ok() +} diff --git a/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64 new file mode 100644 index 000000000000..a11f27189e90 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64 @@ -0,0 +1,13 @@ +bench_try_ref_from_bytes_static_size: + mov rax, rdi + cmp rsi, 6 + setne cl + or cl, al + test cl, 1 + jne .LBB5_2 + cmp word ptr [rax], -16192 + je .LBB5_3 +.LBB5_2: + xor eax, eax +.LBB5_3: + ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64.mca new file mode 100644 index 000000000000..e6bd20533a83 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_static_size.x86-64.mca @@ -0,0 +1,59 @@ +Iterations: 100 +Instructions: 1000 +Total Cycles: 308 +Total uOps: 1100 + +Dispatch Width: 4 +uOps Per Cycle: 3.57 +IPC: 3.25 +Block RThroughput: 3.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 cmp rsi, 6 + 1 1 0.50 setne cl + 1 1 0.33 or cl, al + 1 1 0.33 test cl, 1 + 1 1 1.00 jne .LBB5_2 + 2 6 0.50 * cmp word ptr [rax], -16192 + 1 1 1.00 je .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.98 2.98 - 3.04 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.02 0.97 - 0.01 - - mov rax, rdi + - - 0.02 0.98 - - - - cmp rsi, 6 + - - 1.00 - - - - - setne cl + - - 0.97 0.02 - 0.01 - - or cl, al + - - 0.96 0.03 - 0.01 - - test cl, 1 + - - - - - 1.00 - - jne .LBB5_2 + - - 0.01 0.98 - 0.01 0.50 0.50 cmp word ptr [rax], -16192 + - - - - - 1.00 - - je .LBB5_3 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..8b9e7355e367 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_bytes_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + zerocopy::TryFromBytes::try_ref_from_bytes_with_elems(source, count).ok() +} diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..3ef8d1448a50 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,21 @@ +bench_try_ref_from_bytes_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + seta cl + mov rax, rdi + test al, 3 + setne dil + or dil, cl + jne .LBB5_3 + lea rcx, [rdx + 2*rdx] + or rcx, 3 + add rcx, 9 + cmp rsi, rcx + jne .LBB5_3 + cmp word ptr [rax], -16192 + je .LBB5_4 +.LBB5_3: + xor eax, eax + mov rdx, rsi +.LBB5_4: + ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..8131f3bd549e --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,75 @@ +Iterations: 100 +Instructions: 1800 +Total Cycles: 607 +Total uOps: 2000 + +Dispatch Width: 4 +uOps Per Cycle: 3.29 +IPC: 2.97 +Block RThroughput: 5.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 2 2 1.00 seta cl + 1 1 0.33 mov rax, rdi + 1 1 0.33 test al, 3 + 1 1 0.50 setne dil + 1 1 0.33 or dil, cl + 1 1 1.00 jne .LBB5_3 + 1 1 0.50 lea rcx, [rdx + 2*rdx] + 1 1 0.33 or rcx, 3 + 1 1 0.33 add rcx, 9 + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 jne .LBB5_3 + 2 6 0.50 * cmp word ptr [rax], -16192 + 1 1 1.00 je .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.99 5.99 - 6.02 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - movabs rax, 3074457345618258598 + - - - 1.00 - - - - cmp rdx, rax + - - - - - 2.00 - - seta cl + - - 1.00 - - - - - mov rax, rdi + - - 0.99 0.01 - - - - test al, 3 + - - 1.00 - - - - - setne dil + - - - 0.99 - 0.01 - - or dil, cl + - - - - - 1.00 - - jne .LBB5_3 + - - 0.01 0.99 - - - - lea rcx, [rdx + 2*rdx] + - - - 1.00 - - - - or rcx, 3 + - - 0.99 0.01 - - - - add rcx, 9 + - - - 1.00 - - - - cmp rsi, rcx + - - - - - 1.00 - - jne .LBB5_3 + - - 1.00 - - - 0.50 0.50 cmp word ptr [rax], -16192 + - - - - - 1.00 - - je .LBB5_4 + - - - - - - - - xor eax, eax + - - 1.00 - - - - - mov rdx, rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..9ccd6fef558a --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_bytes_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + zerocopy::TryFromBytes::try_ref_from_bytes_with_elems(source, count).ok() +} diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..ba34b1855bf1 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64 @@ -0,0 +1,18 @@ +bench_try_ref_from_bytes_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + seta cl + mov rax, rdi + or dil, cl + test dil, 1 + jne .LBB5_3 + lea rcx, [2*rdx + 4] + cmp rsi, rcx + jne .LBB5_3 + cmp word ptr [rax], -16192 + je .LBB5_4 +.LBB5_3: + xor eax, eax + mov rdx, rsi +.LBB5_4: + ret diff --git a/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..ae049c03dfde --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_bytes_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,69 @@ +Iterations: 100 +Instructions: 1500 +Total Cycles: 507 +Total uOps: 1700 + +Dispatch Width: 4 +uOps Per Cycle: 3.35 +IPC: 2.96 +Block RThroughput: 4.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 2 2 1.00 seta cl + 1 1 0.33 mov rax, rdi + 1 1 0.33 or dil, cl + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_3 + 1 1 0.50 lea rcx, [2*rdx + 4] + 1 1 0.33 cmp rsi, rcx + 1 1 1.00 jne .LBB5_3 + 2 6 0.50 * cmp word ptr [rax], -16192 + 1 1 1.00 je .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov rdx, rsi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.98 4.99 - 5.03 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - 0.99 - 0.01 - - movabs rax, 4611686018427387901 + - - 0.50 0.50 - - - - cmp rdx, rax + - - 1.96 - - 0.04 - - seta cl + - - 0.01 0.99 - - - - mov rax, rdi + - - 1.00 - - - - - or dil, cl + - - 0.99 0.01 - - - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_3 + - - 0.01 0.99 - - - - lea rcx, [2*rdx + 4] + - - 0.02 0.49 - 0.49 - - cmp rsi, rcx + - - - - - 1.00 - - jne .LBB5_3 + - - - 0.51 - 0.49 0.50 0.50 cmp word ptr [rax], -16192 + - - - - - 1.00 - - je .LBB5_4 + - - - - - - - - xor eax, eax + - - 0.49 0.51 - - - - mov rdx, rsi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.rs new file mode 100644 index 000000000000..23b346f9c98d --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_prefix_dynamic_padding(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64 new file mode 100644 index 000000000000..d832cb7ecf7f --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64 @@ -0,0 +1,29 @@ +bench_try_ref_from_prefix_dynamic_padding: + xor edx, edx + mov eax, 0 + test dil, 3 + je .LBB5_1 + ret +.LBB5_1: + movabs rax, 9223372036854775804 + and rsi, rax + cmp rsi, 9 + jae .LBB5_3 + mov edx, 1 + xor eax, eax + ret +.LBB5_3: + add rsi, -9 + movabs rcx, -6148914691236517205 + mov rax, rsi + mul rcx + mov rax, rdx + shr rax + movzx ecx, word ptr [rdi] + cmp cx, -16192 + mov edx, 2 + cmove rdx, rax + xor eax, eax + cmp ecx, 49344 + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..482112a39b33 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_padding.x86-64.mca @@ -0,0 +1,91 @@ +Iterations: 100 +Instructions: 2600 +Total Cycles: 843 +Total uOps: 2900 + +Dispatch Width: 4 +uOps Per Cycle: 3.44 +IPC: 3.08 +Block RThroughput: 7.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 3 + 1 1 1.00 je .LBB5_1 + 1 1 1.00 U ret + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rsi, rax + 1 1 0.33 cmp rsi, 9 + 1 1 1.00 jae .LBB5_3 + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rsi, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 1 1 0.33 mov rax, rsi + 2 4 1.00 mul rcx + 1 1 0.33 mov rax, rdx + 1 1 0.50 shr rax + 1 5 0.50 * movzx ecx, word ptr [rdi] + 1 1 0.33 cmp cx, -16192 + 1 1 0.33 mov edx, 2 + 2 2 0.67 cmove rdx, rax + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp ecx, 49344 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 8.33 8.33 - 8.34 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor edx, edx + - - 0.32 0.34 - 0.34 - - mov eax, 0 + - - 0.34 0.33 - 0.33 - - test dil, 3 + - - - - - 1.00 - - je .LBB5_1 + - - - - - 1.00 - - ret + - - 0.35 0.65 - - - - movabs rax, 9223372036854775804 + - - 0.96 0.03 - 0.01 - - and rsi, rax + - - 0.01 0.97 - 0.02 - - cmp rsi, 9 + - - - - - 1.00 - - jae .LBB5_3 + - - 0.67 0.01 - 0.32 - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.02 0.34 - 0.64 - - add rsi, -9 + - - 0.33 0.66 - 0.01 - - movabs rcx, -6148914691236517205 + - - 0.66 0.34 - - - - mov rax, rsi + - - 1.00 1.00 - - - - mul rcx + - - 0.01 0.99 - - - - mov rax, rdx + - - 0.99 - - 0.01 - - shr rax + - - - - - - 0.50 0.50 movzx ecx, word ptr [rdi] + - - 0.33 0.03 - 0.64 - - cmp cx, -16192 + - - 0.01 0.31 - 0.68 - - mov edx, 2 + - - 1.00 1.00 - - - - cmove rdx, rax + - - - - - - - - xor eax, eax + - - 0.33 0.33 - 0.34 - - cmp ecx, 49344 + - - 1.00 1.00 - - - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.rs new file mode 100644 index 000000000000..41a466ec8e6a --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_prefix_dynamic_size(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64 new file mode 100644 index 000000000000..be7f34b9f8f5 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64 @@ -0,0 +1,22 @@ +bench_try_ref_from_prefix_dynamic_size: + xor edx, edx + mov eax, 0 + test dil, 1 + jne .LBB5_4 + cmp rsi, 4 + jae .LBB5_3 + mov edx, 1 + xor eax, eax + ret +.LBB5_3: + add rsi, -4 + shr rsi + movzx ecx, word ptr [rdi] + cmp ecx, 49344 + mov edx, 2 + cmove rdx, rsi + xor eax, eax + cmp cx, -16192 + cmove rax, rdi +.LBB5_4: + ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..11706defe11e --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 573 +Total uOps: 2100 + +Dispatch Width: 4 +uOps Per Cycle: 3.66 +IPC: 3.32 +Block RThroughput: 5.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_4 + 1 1 0.33 cmp rsi, 4 + 1 1 1.00 jae .LBB5_3 + 1 1 0.33 mov edx, 1 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rsi, -4 + 1 1 0.50 shr rsi + 1 5 0.50 * movzx ecx, word ptr [rdi] + 1 1 0.33 cmp ecx, 49344 + 1 1 0.33 mov edx, 2 + 2 2 0.67 cmove rdx, rsi + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp cx, -16192 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.66 5.67 - 5.67 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor edx, edx + - - 0.30 0.37 - 0.33 - - mov eax, 0 + - - 0.35 0.32 - 0.33 - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_4 + - - 0.32 0.33 - 0.35 - - cmp rsi, 4 + - - - - - 1.00 - - jae .LBB5_3 + - - 0.33 0.35 - 0.32 - - mov edx, 1 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.34 0.64 - 0.02 - - add rsi, -4 + - - 1.00 - - - - - shr rsi + - - - - - - 0.50 0.50 movzx ecx, word ptr [rdi] + - - 0.60 0.40 - - - - cmp ecx, 49344 + - - 0.05 0.95 - - - - mov edx, 2 + - - 1.00 1.00 - - - - cmove rdx, rsi + - - - - - - - - xor eax, eax + - - 0.37 0.31 - 0.32 - - cmp cx, -16192 + - - 1.00 1.00 - - - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_static_size.rs b/rust/zerocopy/benches/try_ref_from_prefix_static_size.rs new file mode 100644 index 000000000000..5f13d482b505 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_static_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_prefix_static_size(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_prefix(source) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64 new file mode 100644 index 000000000000..83212f776ea6 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64 @@ -0,0 +1,15 @@ +bench_try_ref_from_prefix_static_size: + cmp rsi, 6 + setb al + or al, dil + test al, 1 + jne .LBB5_2 + movzx eax, word ptr [rdi] + cmp eax, 49344 + mov eax, 2 + cmove rax, rdi + je .LBB5_3 +.LBB5_2: + xor eax, eax +.LBB5_3: + ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64.mca new file mode 100644 index 000000000000..5d02b863a741 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_static_size.x86-64.mca @@ -0,0 +1,63 @@ +Iterations: 100 +Instructions: 1200 +Total Cycles: 374 +Total uOps: 1300 + +Dispatch Width: 4 +uOps Per Cycle: 3.48 +IPC: 3.21 +Block RThroughput: 3.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 cmp rsi, 6 + 1 1 0.50 setb al + 1 1 0.33 or al, dil + 1 1 0.33 test al, 1 + 1 1 1.00 jne .LBB5_2 + 1 5 0.50 * movzx eax, word ptr [rdi] + 1 1 0.33 cmp eax, 49344 + 1 1 0.33 mov eax, 2 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 je .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 3.66 3.65 - 3.69 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.35 0.64 - 0.01 - - cmp rsi, 6 + - - 1.00 - - - - - setb al + - - 0.02 0.66 - 0.32 - - or al, dil + - - 0.03 0.65 - 0.32 - - test al, 1 + - - - - - 1.00 - - jne .LBB5_2 + - - - - - - 0.50 0.50 movzx eax, word ptr [rdi] + - - 0.92 0.07 - 0.01 - - cmp eax, 49344 + - - 0.37 0.63 - - - - mov eax, 2 + - - 0.97 1.00 - 0.03 - - cmove rax, rdi + - - - - - 1.00 - - je .LBB5_3 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..1744a40759b1 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_prefix_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_prefix_with_elems(source, count) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..80e66ba1601c --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,30 @@ +bench_try_ref_from_prefix_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + ja .LBB5_1 + xor ecx, ecx + mov eax, 0 + test dil, 3 + je .LBB5_3 + mov rdx, rcx + ret +.LBB5_3: + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + cmp rax, rsi + jbe .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + movzx esi, word ptr [rdi] + cmp si, -16192 + mov ecx, 2 + cmove rcx, rdx + xor eax, eax + cmp esi, 49344 + cmove rax, rdi + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..512e8ce64393 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,91 @@ +Iterations: 100 +Instructions: 2600 +Total Cycles: 806 +Total uOps: 2800 + +Dispatch Width: 4 +uOps Per Cycle: 3.47 +IPC: 3.23 +Block RThroughput: 7.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 3 + 1 1 1.00 je .LBB5_3 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 1 0.33 cmp rax, rsi + 1 1 1.00 jbe .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 5 0.50 * movzx esi, word ptr [rdi] + 1 1 0.33 cmp si, -16192 + 1 1 0.33 mov ecx, 2 + 2 2 0.67 cmove rcx, rdx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp esi, 49344 + 2 2 0.67 cmove rax, rdi + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 7.98 7.99 - 8.03 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.98 - - 0.02 - - movabs rax, 3074457345618258598 + - - - 1.00 - - - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - - - - - - - xor ecx, ecx + - - 0.99 0.01 - - - - mov eax, 0 + - - 0.01 0.96 - 0.03 - - test dil, 3 + - - - - - 1.00 - - je .LBB5_3 + - - 0.97 0.01 - 0.02 - - mov rdx, rcx + - - - - - 1.00 - - ret + - - 0.03 0.97 - - - - lea rax, [rdx + 2*rdx] + - - 0.03 0.97 - - - - or rax, 3 + - - 0.01 0.99 - - - - add rax, 9 + - - - 1.00 - - - - cmp rax, rsi + - - - - - 1.00 - - jbe .LBB5_4 + - - - - - - - - xor eax, eax + - - 0.98 0.01 - 0.01 - - mov edx, 1 + - - - - - 1.00 - - ret + - - - - - - 0.50 0.50 movzx esi, word ptr [rdi] + - - 0.97 0.03 - - - - cmp si, -16192 + - - 0.98 0.01 - 0.01 - - mov ecx, 2 + - - 1.00 0.03 - 0.97 - - cmove rcx, rdx + - - - - - - - - xor eax, eax + - - 0.03 0.97 - - - - cmp esi, 49344 + - - 1.00 1.00 - - - - cmove rax, rdi + - - - 0.03 - 0.97 - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..ed0f50941194 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_prefix_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_prefix_with_elems(source, count) { + Ok((packet, _rest)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..c12e87c137c5 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64 @@ -0,0 +1,26 @@ +bench_try_ref_from_prefix_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + ja .LBB5_1 + mov rcx, rdx + xor edx, edx + mov eax, 0 + test dil, 1 + jne .LBB5_5 + lea rax, [2*rcx + 4] + cmp rax, rsi + jbe .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + movzx esi, word ptr [rdi] + cmp si, -16192 + mov edx, 2 + cmove rdx, rcx + xor eax, eax + cmp esi, 49344 + cmove rax, rdi +.LBB5_5: + ret diff --git a/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..6c3f1a1ec97a --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_prefix_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,83 @@ +Iterations: 100 +Instructions: 2200 +Total Cycles: 674 +Total uOps: 2400 + +Dispatch Width: 4 +uOps Per Cycle: 3.56 +IPC: 3.26 +Block RThroughput: 6.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.33 mov rcx, rdx + 1 0 0.25 xor edx, edx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test dil, 1 + 1 1 1.00 jne .LBB5_5 + 1 1 0.50 lea rax, [2*rcx + 4] + 1 1 0.33 cmp rax, rsi + 1 1 1.00 jbe .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 5 0.50 * movzx esi, word ptr [rdi] + 1 1 0.33 cmp si, -16192 + 1 1 0.33 mov edx, 2 + 2 2 0.67 cmove rdx, rcx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp esi, 49344 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.65 6.66 - 6.69 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.66 0.33 - 0.01 - - movabs rax, 4611686018427387901 + - - 0.02 0.66 - 0.32 - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.66 0.33 - 0.01 - - mov rcx, rdx + - - - - - - - - xor edx, edx + - - 0.33 0.01 - 0.66 - - mov eax, 0 + - - 0.34 0.65 - 0.01 - - test dil, 1 + - - - - - 1.00 - - jne .LBB5_5 + - - 0.65 0.35 - - - - lea rax, [2*rcx + 4] + - - - 1.00 - - - - cmp rax, rsi + - - - - - 1.00 - - jbe .LBB5_4 + - - - - - - - - xor eax, eax + - - 0.34 0.01 - 0.65 - - mov edx, 1 + - - - - - 1.00 - - ret + - - - - - - 0.50 0.50 movzx esi, word ptr [rdi] + - - 0.65 0.34 - 0.01 - - cmp si, -16192 + - - 0.66 0.34 - - - - mov edx, 2 + - - 1.00 0.99 - 0.01 - - cmove rdx, rcx + - - - - - - - - xor eax, eax + - - 0.34 0.66 - - - - cmp esi, 49344 + - - 1.00 0.99 - 0.01 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.rs new file mode 100644 index 000000000000..981feca3ca24 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_suffix_dynamic_padding(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64 new file mode 100644 index 000000000000..b3e924442865 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64 @@ -0,0 +1,26 @@ +bench_try_ref_from_suffix_dynamic_padding: + lea eax, [rsi + rdi] + test al, 3 + jne .LBB5_1 + movabs rax, 9223372036854775804 + and rax, rsi + cmp rax, 9 + jae .LBB5_3 +.LBB5_1: + xor eax, eax + ret +.LBB5_3: + add rax, -9 + movabs rcx, -6148914691236517205 + mul rcx + shr rdx + lea rcx, [rdx + 2*rdx] + sub rsi, rcx + or rcx, -4 + add rsi, rdi + lea rdi, [rcx + rsi] + add rdi, -8 + xor eax, eax + cmp word ptr [rcx + rsi - 8], -16192 + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..d56ae56d854a --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_padding.x86-64.mca @@ -0,0 +1,85 @@ +Iterations: 100 +Instructions: 2300 +Total Cycles: 791 +Total uOps: 2600 + +Dispatch Width: 4 +uOps Per Cycle: 3.29 +IPC: 2.91 +Block RThroughput: 6.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea eax, [rsi + rdi] + 1 1 0.33 test al, 3 + 1 1 1.00 jne .LBB5_1 + 1 1 0.33 movabs rax, 9223372036854775804 + 1 1 0.33 and rax, rsi + 1 1 0.33 cmp rax, 9 + 1 1 1.00 jae .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.33 add rax, -9 + 1 1 0.33 movabs rcx, -6148914691236517205 + 2 4 1.00 mul rcx + 1 1 0.50 shr rdx + 1 1 0.50 lea rcx, [rdx + 2*rdx] + 1 1 0.33 sub rsi, rcx + 1 1 0.33 or rcx, -4 + 1 1 0.33 add rsi, rdi + 1 1 0.50 lea rdi, [rcx + rsi] + 1 1 0.33 add rdi, -8 + 1 0 0.25 xor eax, eax + 2 6 0.50 * cmp word ptr [rcx + rsi - 8], -16192 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 7.70 7.58 - 7.72 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.26 0.74 - - - - lea eax, [rsi + rdi] + - - 0.19 0.28 - 0.53 - - test al, 3 + - - - - - 1.00 - - jne .LBB5_1 + - - 0.93 0.06 - 0.01 - - movabs rax, 9223372036854775804 + - - 0.81 0.14 - 0.05 - - and rax, rsi + - - 0.55 0.43 - 0.02 - - cmp rax, 9 + - - - - - 1.00 - - jae .LBB5_3 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.42 0.56 - 0.02 - - add rax, -9 + - - 0.67 0.30 - 0.03 - - movabs rcx, -6148914691236517205 + - - 1.00 1.00 - - - - mul rcx + - - 0.71 - - 0.29 - - shr rdx + - - 0.32 0.68 - - - - lea rcx, [rdx + 2*rdx] + - - 0.57 0.04 - 0.39 - - sub rsi, rcx + - - 0.28 0.67 - 0.05 - - or rcx, -4 + - - 0.29 0.29 - 0.42 - - add rsi, rdi + - - 0.02 0.98 - - - - lea rdi, [rcx + rsi] + - - 0.02 0.41 - 0.57 - - add rdi, -8 + - - - - - - - - xor eax, eax + - - 0.57 0.01 - 0.42 0.50 0.50 cmp word ptr [rcx + rsi - 8], -16192 + - - 0.09 0.99 - 0.92 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.rs new file mode 100644 index 000000000000..c3d75ac9b3e1 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_suffix_dynamic_size(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64 new file mode 100644 index 000000000000..d51f7817e599 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64 @@ -0,0 +1,18 @@ +bench_try_ref_from_suffix_dynamic_size: + lea eax, [rsi + rdi] + cmp rsi, 4 + setb cl + or cl, al + test cl, 1 + je .LBB5_2 + xor eax, eax + ret +.LBB5_2: + lea rdx, [rsi - 4] + shr rdx + and esi, 1 + lea rcx, [rdi + rsi] + xor eax, eax + cmp word ptr [rdi + rsi], -16192 + cmove rax, rcx + ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..6cf7f8e493f5 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_dynamic_size.x86-64.mca @@ -0,0 +1,71 @@ +Iterations: 100 +Instructions: 1600 +Total Cycles: 510 +Total uOps: 1800 + +Dispatch Width: 4 +uOps Per Cycle: 3.53 +IPC: 3.14 +Block RThroughput: 4.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea eax, [rsi + rdi] + 1 1 0.33 cmp rsi, 4 + 1 1 0.50 setb cl + 1 1 0.33 or cl, al + 1 1 0.33 test cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.50 lea rdx, [rsi - 4] + 1 1 0.50 shr rdx + 1 1 0.33 and esi, 1 + 1 1 0.50 lea rcx, [rdi + rsi] + 1 0 0.25 xor eax, eax + 2 6 0.50 * cmp word ptr [rdi + rsi], -16192 + 2 2 0.67 cmove rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.99 5.00 - 5.01 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.98 0.02 - - - - lea eax, [rsi + rdi] + - - - 0.98 - 0.02 - - cmp rsi, 4 + - - 1.00 - - - - - setb cl + - - 0.01 0.99 - - - - or cl, al + - - 0.01 0.07 - 0.92 - - test cl, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.93 0.07 - - - - lea rdx, [rsi - 4] + - - 0.93 - - 0.07 - - shr rdx + - - 0.06 0.93 - 0.01 - - and esi, 1 + - - 0.07 0.93 - - - - lea rcx, [rdi + rsi] + - - - - - - - - xor eax, eax + - - - 0.01 - 0.99 0.50 0.50 cmp word ptr [rdi + rsi], -16192 + - - 1.00 1.00 - - - - cmove rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_static_size.rs b/rust/zerocopy/benches/try_ref_from_suffix_static_size.rs new file mode 100644 index 000000000000..d4b92f639a32 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_static_size.rs @@ -0,0 +1,10 @@ +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_suffix_static_size(source: &[u8]) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_suffix(source) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64 new file mode 100644 index 000000000000..cd39f70931bc --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64 @@ -0,0 +1,16 @@ +bench_try_ref_from_suffix_static_size: + lea eax, [rsi + rdi] + cmp rsi, 6 + setb cl + or cl, al + test cl, 1 + je .LBB5_2 + xor eax, eax + ret +.LBB5_2: + lea rcx, [rdi + rsi] + add rcx, -6 + xor eax, eax + cmp word ptr [rdi + rsi - 6], -16192 + cmove rax, rcx + ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64.mca new file mode 100644 index 000000000000..087d1e7ed971 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_static_size.x86-64.mca @@ -0,0 +1,67 @@ +Iterations: 100 +Instructions: 1400 +Total Cycles: 443 +Total uOps: 1600 + +Dispatch Width: 4 +uOps Per Cycle: 3.61 +IPC: 3.16 +Block RThroughput: 4.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea eax, [rsi + rdi] + 1 1 0.33 cmp rsi, 6 + 1 1 0.50 setb cl + 1 1 0.33 or cl, al + 1 1 0.33 test cl, 1 + 1 1 1.00 je .LBB5_2 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret + 1 1 0.50 lea rcx, [rdi + rsi] + 1 1 0.33 add rcx, -6 + 1 0 0.25 xor eax, eax + 2 6 0.50 * cmp word ptr [rdi + rsi - 6], -16192 + 2 2 0.67 cmove rax, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.33 4.33 - 4.34 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.32 0.68 - - - - lea eax, [rsi + rdi] + - - 0.05 0.94 - 0.01 - - cmp rsi, 6 + - - 1.00 - - - - - setb cl + - - 0.95 0.05 - - - - or cl, al + - - 0.95 0.02 - 0.03 - - test cl, 1 + - - - - - 1.00 - - je .LBB5_2 + - - - - - - - - xor eax, eax + - - - - - 1.00 - - ret + - - 0.04 0.96 - - - - lea rcx, [rdi + rsi] + - - 0.02 0.97 - 0.01 - - add rcx, -6 + - - - - - - - - xor eax, eax + - - 0.03 0.66 - 0.31 0.50 0.50 cmp word ptr [rdi + rsi - 6], -16192 + - - 0.97 0.05 - 0.98 - - cmove rax, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.rs b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.rs new file mode 100644 index 000000000000..1da455c9a238 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_suffix_with_elems_dynamic_padding( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_suffix_with_elems(source, count) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64 new file mode 100644 index 000000000000..c7530d8b6815 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64 @@ -0,0 +1,32 @@ +bench_try_ref_from_suffix_with_elems_dynamic_padding: + movabs rax, 3074457345618258598 + cmp rdx, rax + ja .LBB5_1 + lea r8d, [rsi + rdi] + xor ecx, ecx + mov eax, 0 + test r8b, 3 + je .LBB5_3 + mov rdx, rcx + ret +.LBB5_3: + lea rax, [rdx + 2*rdx] + or rax, 3 + add rax, 9 + sub rsi, rax + jae .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + lea r8, [rdi + rsi] + movzx esi, word ptr [rdi + rsi] + cmp si, -16192 + mov ecx, 2 + cmove rcx, rdx + xor eax, eax + cmp esi, 49344 + cmove rax, r8 + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..be736c00c250 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_padding.x86-64.mca @@ -0,0 +1,95 @@ +Iterations: 100 +Instructions: 2800 +Total Cycles: 878 +Total uOps: 3000 + +Dispatch Width: 4 +uOps Per Cycle: 3.42 +IPC: 3.19 +Block RThroughput: 7.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 3074457345618258598 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r8d, [rsi + rdi] + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test r8b, 3 + 1 1 1.00 je .LBB5_3 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + 1 1 0.50 lea rax, [rdx + 2*rdx] + 1 1 0.33 or rax, 3 + 1 1 0.33 add rax, 9 + 1 1 0.33 sub rsi, rax + 1 1 1.00 jae .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 1 0.50 lea r8, [rdi + rsi] + 1 5 0.50 * movzx esi, word ptr [rdi + rsi] + 1 1 0.33 cmp si, -16192 + 1 1 0.33 mov ecx, 2 + 2 2 0.67 cmove rcx, rdx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp esi, 49344 + 2 2 0.67 cmove rax, r8 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 8.65 8.65 - 8.70 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.67 0.30 - 0.03 - - movabs rax, 3074457345618258598 + - - 0.01 0.99 - - - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.99 0.01 - - - - lea r8d, [rsi + rdi] + - - - - - - - - xor ecx, ecx + - - 0.35 0.62 - 0.03 - - mov eax, 0 + - - 0.99 0.01 - - - - test r8b, 3 + - - - - - 1.00 - - je .LBB5_3 + - - 0.68 0.30 - 0.02 - - mov rdx, rcx + - - - - - 1.00 - - ret + - - 0.07 0.93 - - - - lea rax, [rdx + 2*rdx] + - - 0.06 0.35 - 0.59 - - or rax, 3 + - - 0.02 0.07 - 0.91 - - add rax, 9 + - - 0.01 0.04 - 0.95 - - sub rsi, rax + - - - - - 1.00 - - jae .LBB5_4 + - - - - - - - - xor eax, eax + - - 0.92 0.01 - 0.07 - - mov edx, 1 + - - - - - 1.00 - - ret + - - - 1.00 - - - - lea r8, [rdi + rsi] + - - - - - - 0.50 0.50 movzx esi, word ptr [rdi + rsi] + - - 0.01 0.99 - - - - cmp si, -16192 + - - 0.88 0.04 - 0.08 - - mov ecx, 2 + - - 1.00 0.99 - 0.01 - - cmove rcx, rdx + - - - - - - - - xor eax, eax + - - 0.99 0.01 - - - - cmp esi, 49344 + - - 1.00 1.00 - - - - cmove rax, r8 + - - - 0.99 - 0.01 - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.rs b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.rs new file mode 100644 index 000000000000..8c2b80f8762f --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.rs @@ -0,0 +1,13 @@ +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_try_ref_from_suffix_with_elems_dynamic_size( + source: &[u8], + count: usize, +) -> Option<&format::CocoPacket> { + match zerocopy::TryFromBytes::try_ref_from_suffix_with_elems(source, count) { + Ok((_rest, packet)) => Some(packet), + _ => None, + } +} diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64 b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64 new file mode 100644 index 000000000000..952eb12de8d6 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64 @@ -0,0 +1,28 @@ +bench_try_ref_from_suffix_with_elems_dynamic_size: + movabs rax, 4611686018427387901 + cmp rdx, rax + ja .LBB5_1 + lea r8d, [rsi + rdi] + xor ecx, ecx + mov eax, 0 + test r8b, 1 + jne .LBB5_5 + lea rax, [2*rdx + 4] + sub rsi, rax + jae .LBB5_4 +.LBB5_1: + xor eax, eax + mov edx, 1 + ret +.LBB5_4: + lea r8, [rdi + rsi] + movzx esi, word ptr [rdi + rsi] + cmp si, -16192 + mov ecx, 2 + cmove rcx, rdx + xor eax, eax + cmp esi, 49344 + cmove rax, r8 +.LBB5_5: + mov rdx, rcx + ret diff --git a/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..d4f78f67a252 --- /dev/null +++ b/rust/zerocopy/benches/try_ref_from_suffix_with_elems_dynamic_size.x86-64.mca @@ -0,0 +1,87 @@ +Iterations: 100 +Instructions: 2400 +Total Cycles: 1107 +Total uOps: 2600 + +Dispatch Width: 4 +uOps Per Cycle: 2.35 +IPC: 2.17 +Block RThroughput: 6.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movabs rax, 4611686018427387901 + 1 1 0.33 cmp rdx, rax + 1 1 1.00 ja .LBB5_1 + 1 1 0.50 lea r8d, [rsi + rdi] + 1 0 0.25 xor ecx, ecx + 1 1 0.33 mov eax, 0 + 1 1 0.33 test r8b, 1 + 1 1 1.00 jne .LBB5_5 + 1 1 0.50 lea rax, [2*rdx + 4] + 1 1 0.33 sub rsi, rax + 1 1 1.00 jae .LBB5_4 + 1 0 0.25 xor eax, eax + 1 1 0.33 mov edx, 1 + 1 1 1.00 U ret + 1 1 0.50 lea r8, [rdi + rsi] + 1 5 0.50 * movzx esi, word ptr [rdi + rsi] + 1 1 0.33 cmp si, -16192 + 1 1 0.33 mov ecx, 2 + 2 2 0.67 cmove rcx, rdx + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp esi, 49344 + 2 2 0.67 cmove rax, r8 + 1 1 0.33 mov rdx, rcx + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 6.99 7.00 - 8.01 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.02 0.95 - 0.03 - - movabs rax, 4611686018427387901 + - - 0.93 0.04 - 0.03 - - cmp rdx, rax + - - - - - 1.00 - - ja .LBB5_1 + - - 0.96 0.04 - - - - lea r8d, [rsi + rdi] + - - - - - - - - xor ecx, ecx + - - 0.95 0.02 - 0.03 - - mov eax, 0 + - - 0.95 0.05 - - - - test r8b, 1 + - - - - - 1.00 - - jne .LBB5_5 + - - 0.06 0.94 - - - - lea rax, [2*rdx + 4] + - - 0.93 0.07 - - - - sub rsi, rax + - - - - - 1.00 - - jae .LBB5_4 + - - - - - - - - xor eax, eax + - - 0.03 0.95 - 0.02 - - mov edx, 1 + - - - - - 1.00 - - ret + - - 0.97 0.03 - - - - lea r8, [rdi + rsi] + - - - - - - 0.50 0.50 movzx esi, word ptr [rdi + rsi] + - - 0.03 0.97 - - - - cmp si, -16192 + - - 0.05 0.94 - 0.01 - - mov ecx, 2 + - - 0.06 0.98 - 0.96 - - cmove rcx, rdx + - - - - - - - - xor eax, eax + - - 0.97 0.03 - - - - cmp esi, 49344 + - - 0.06 0.96 - 0.98 - - cmove rax, r8 + - - 0.02 0.03 - 0.95 - - mov rdx, rcx + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_transmute.rs b/rust/zerocopy/benches/try_transmute.rs new file mode 100644 index 000000000000..c0de07a8d094 --- /dev/null +++ b/rust/zerocopy/benches/try_transmute.rs @@ -0,0 +1,16 @@ +use zerocopy::Unalign; +use zerocopy_derive::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C)] +struct MinimalViableSource { + bytes: [u8; 6], +} + +#[unsafe(no_mangle)] +fn bench_try_transmute(source: MinimalViableSource) -> Option<Unalign<format::CocoPacket>> { + zerocopy::try_transmute!(source).ok() +} diff --git a/rust/zerocopy/benches/try_transmute.x86-64 b/rust/zerocopy/benches/try_transmute.x86-64 new file mode 100644 index 000000000000..9e16a663257c --- /dev/null +++ b/rust/zerocopy/benches/try_transmute.x86-64 @@ -0,0 +1,9 @@ +bench_try_transmute: + movzx ecx, di + xor eax, eax + cmp ecx, 49344 + sete al + and rdi, -65536 + xor rax, 49345 + or rax, rdi + ret diff --git a/rust/zerocopy/benches/try_transmute.x86-64.mca b/rust/zerocopy/benches/try_transmute.x86-64.mca new file mode 100644 index 000000000000..33abc3bf341e --- /dev/null +++ b/rust/zerocopy/benches/try_transmute.x86-64.mca @@ -0,0 +1,55 @@ +Iterations: 100 +Instructions: 800 +Total Cycles: 238 +Total uOps: 800 + +Dispatch Width: 4 +uOps Per Cycle: 3.36 +IPC: 3.36 +Block RThroughput: 2.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 movzx ecx, di + 1 0 0.25 xor eax, eax + 1 1 0.33 cmp ecx, 49344 + 1 1 0.50 sete al + 1 1 0.33 and rdi, -65536 + 1 1 0.33 xor rax, 49345 + 1 1 0.33 or rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 2.33 2.33 - 2.34 - - + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.32 0.67 - 0.01 - - movzx ecx, di + - - - - - - - - xor eax, eax + - - 0.33 0.67 - - - - cmp ecx, 49344 + - - 1.00 - - - - - sete al + - - 0.67 0.33 - - - - and rdi, -65536 + - - - 0.66 - 0.34 - - xor rax, 49345 + - - 0.01 - - 0.99 - - or rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_transmute_ref_dynamic_size.rs b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.rs new file mode 100644 index 000000000000..c9236e13d23c --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.rs @@ -0,0 +1,18 @@ +use zerocopy_derive::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C, align(2))] +struct MinimalViableSource { + header: [u8; 6], + trailer: [[u8; 2]], +} + +#[unsafe(no_mangle)] +fn bench_try_transmute_ref_dynamic_size( + source: &MinimalViableSource, +) -> Option<&format::CocoPacket> { + zerocopy::try_transmute_ref!(source).ok() +} diff --git a/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64 b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64 new file mode 100644 index 000000000000..d34d2b3eb3df --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64 @@ -0,0 +1,6 @@ +bench_try_transmute_ref_dynamic_size: + lea rdx, [rsi + 1] + xor eax, eax + cmp word ptr [rdi], -16192 + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64.mca b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..bc771b504713 --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_dynamic_size.x86-64.mca @@ -0,0 +1,49 @@ +Iterations: 100 +Instructions: 500 +Total Cycles: 209 +Total uOps: 700 + +Dispatch Width: 4 +uOps Per Cycle: 3.35 +IPC: 2.39 +Block RThroughput: 1.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea rdx, [rsi + 1] + 1 0 0.25 xor eax, eax + 2 6 0.50 * cmp word ptr [rdi], -16192 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.50 1.51 - 1.99 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.51 0.49 - - - - lea rdx, [rsi + 1] + - - - - - - - - xor eax, eax + - - - 0.02 - 0.98 0.50 0.50 cmp word ptr [rdi], -16192 + - - 0.99 1.00 - 0.01 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/try_transmute_ref_static_size.rs b/rust/zerocopy/benches/try_transmute_ref_static_size.rs new file mode 100644 index 000000000000..631cce2b0bb9 --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_static_size.rs @@ -0,0 +1,17 @@ +use zerocopy_derive::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[derive(IntoBytes, KnownLayout, Immutable)] +#[repr(C, align(2))] +struct MinimalViableSource { + bytes: [u8; 6], +} + +#[unsafe(no_mangle)] +fn bench_try_transmute_ref_static_size( + source: &MinimalViableSource, +) -> Option<&format::CocoPacket> { + zerocopy::try_transmute_ref!(source).ok() +} diff --git a/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64 b/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64 new file mode 100644 index 000000000000..8c16face9875 --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64 @@ -0,0 +1,5 @@ +bench_try_transmute_ref_static_size: + xor eax, eax + cmp word ptr [rdi], -16192 + cmove rax, rdi + ret diff --git a/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64.mca b/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64.mca new file mode 100644 index 000000000000..cf7384989536 --- /dev/null +++ b/rust/zerocopy/benches/try_transmute_ref_static_size.x86-64.mca @@ -0,0 +1,47 @@ +Iterations: 100 +Instructions: 400 +Total Cycles: 160 +Total uOps: 600 + +Dispatch Width: 4 +uOps Per Cycle: 3.75 +IPC: 2.50 +Block RThroughput: 1.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 0 0.25 xor eax, eax + 2 6 0.50 * cmp word ptr [rdi], -16192 + 2 2 0.67 cmove rax, rdi + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.02 1.48 - 1.50 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - - - - - xor eax, eax + - - 0.02 0.49 - 0.49 0.50 0.50 cmp word ptr [rdi], -16192 + - - 1.00 0.99 - 0.01 - - cmove rax, rdi + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_dynamic_size.rs b/rust/zerocopy/benches/write_to_dynamic_size.rs new file mode 100644 index 000000000000..c126a1468c9b --- /dev/null +++ b/rust/zerocopy/benches/write_to_dynamic_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_dynamic_size(source: &format::CocoPacket, destination: &mut [u8]) -> Option<()> { + source.write_to(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_dynamic_size.x86-64 b/rust/zerocopy/benches/write_to_dynamic_size.x86-64 new file mode 100644 index 000000000000..c5abb17f7eba --- /dev/null +++ b/rust/zerocopy/benches/write_to_dynamic_size.x86-64 @@ -0,0 +1,21 @@ +bench_write_to_dynamic_size: + push r14 + push rbx + push rax + mov rbx, rcx + lea r14, [2*rsi + 5] + and r14, -2 + cmp rcx, r14 + jne .LBB5_2 + mov rax, rdi + mov rdi, rdx + mov rsi, rax + mov rdx, rbx + call qword ptr [rip + memcpy@GOTPCREL] +.LBB5_2: + cmp rbx, r14 + sete al + add rsp, 8 + pop rbx + pop r14 + ret diff --git a/rust/zerocopy/benches/write_to_dynamic_size.x86-64.mca b/rust/zerocopy/benches/write_to_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..5b2c08a31a29 --- /dev/null +++ b/rust/zerocopy/benches/write_to_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 2890 +Total uOps: 2500 + +Dispatch Width: 4 +uOps Per Cycle: 0.87 +IPC: 0.66 +Block RThroughput: 6.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 2 5 1.00 * push rax + 1 1 0.33 mov rbx, rcx + 1 1 0.50 lea r14, [2*rsi + 5] + 1 1 0.33 and r14, -2 + 1 1 0.33 cmp rcx, r14 + 1 1 1.00 jne .LBB5_2 + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov rdi, rdx + 1 1 0.33 mov rsi, rax + 1 1 0.33 mov rdx, rbx + 4 7 1.00 * call qword ptr [rip + memcpy@GOTPCREL] + 1 1 0.33 cmp rbx, r14 + 1 1 0.50 sete al + 1 1 0.33 add rsp, 8 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.66 4.64 4.00 4.70 4.00 3.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - - 1.00 push r14 + - - - - 1.00 - 1.00 - push rbx + - - - - 1.00 - - 1.00 push rax + - - 0.02 0.97 - 0.01 - - mov rbx, rcx + - - 0.97 0.03 - - - - lea r14, [2*rsi + 5] + - - 0.63 0.35 - 0.02 - - and r14, -2 + - - 0.31 0.34 - 0.35 - - cmp rcx, r14 + - - - - - 1.00 - - jne .LBB5_2 + - - 0.33 0.33 - 0.34 - - mov rax, rdi + - - 0.36 0.31 - 0.33 - - mov rdi, rdx + - - 0.33 0.35 - 0.32 - - mov rsi, rax + - - 0.35 0.63 - 0.02 - - mov rdx, rbx + - - - - 1.00 1.00 2.00 - call qword ptr [rip + memcpy@GOTPCREL] + - - 0.65 0.35 - - - - cmp rbx, r14 + - - 0.69 - - 0.31 - - sete al + - - 0.02 0.98 - - - - add rsp, 8 + - - - - - - - 1.00 pop rbx + - - - - - - 1.00 - pop r14 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_prefix_dynamic_size.rs b/rust/zerocopy/benches/write_to_prefix_dynamic_size.rs new file mode 100644 index 000000000000..a54d32773113 --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_dynamic_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_prefix_dynamic_size( + source: &format::CocoPacket, + destination: &mut [u8], +) -> Option<()> { + source.write_to_prefix(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64 b/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64 new file mode 100644 index 000000000000..d7779c6c9178 --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64 @@ -0,0 +1,21 @@ +bench_write_to_prefix_dynamic_size: + push r14 + push rbx + push rax + mov rbx, rcx + lea r14, [2*rsi + 5] + and r14, -2 + cmp r14, rcx + ja .LBB5_2 + mov rax, rdi + mov rdi, rdx + mov rsi, rax + mov rdx, r14 + call qword ptr [rip + memcpy@GOTPCREL] +.LBB5_2: + cmp r14, rbx + setbe al + add rsp, 8 + pop rbx + pop r14 + ret diff --git a/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..4cebe24d4f3b --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_dynamic_size.x86-64.mca @@ -0,0 +1,77 @@ +Iterations: 100 +Instructions: 1900 +Total Cycles: 2890 +Total uOps: 2600 + +Dispatch Width: 4 +uOps Per Cycle: 0.90 +IPC: 0.66 +Block RThroughput: 6.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 2 5 1.00 * push rax + 1 1 0.33 mov rbx, rcx + 1 1 0.50 lea r14, [2*rsi + 5] + 1 1 0.33 and r14, -2 + 1 1 0.33 cmp r14, rcx + 1 1 1.00 ja .LBB5_2 + 1 1 0.33 mov rax, rdi + 1 1 0.33 mov rdi, rdx + 1 1 0.33 mov rsi, rax + 1 1 0.33 mov rdx, r14 + 4 7 1.00 * call qword ptr [rip + memcpy@GOTPCREL] + 1 1 0.33 cmp r14, rbx + 2 2 1.00 setbe al + 1 1 0.33 add rsp, 8 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 5.47 4.49 4.00 5.04 4.00 3.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - - 1.00 push r14 + - - - - 1.00 - 1.00 - push rbx + - - - - 1.00 - - 1.00 push rax + - - 0.48 0.51 - 0.01 - - mov rbx, rcx + - - 0.51 0.49 - - - - lea r14, [2*rsi + 5] + - - 0.48 0.05 - 0.47 - - and r14, -2 + - - 0.48 0.49 - 0.03 - - cmp r14, rcx + - - - - - 1.00 - - ja .LBB5_2 + - - 0.04 0.47 - 0.49 - - mov rax, rdi + - - 0.49 0.03 - 0.48 - - mov rdi, rdx + - - 0.03 0.48 - 0.49 - - mov rsi, rax + - - 0.48 0.51 - 0.01 - - mov rdx, r14 + - - - - 1.00 1.00 2.00 - call qword ptr [rip + memcpy@GOTPCREL] + - - 0.51 0.49 - - - - cmp r14, rbx + - - 1.94 - - 0.06 - - setbe al + - - 0.03 0.97 - - - - add rsp, 8 + - - - - - - - 1.00 pop rbx + - - - - - - 1.00 - pop r14 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_prefix_static_size.rs b/rust/zerocopy/benches/write_to_prefix_static_size.rs new file mode 100644 index 000000000000..826222c129fe --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_static_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_prefix_static_size( + source: &format::CocoPacket, + destination: &mut [u8], +) -> Option<()> { + source.write_to_prefix(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_prefix_static_size.x86-64 b/rust/zerocopy/benches/write_to_prefix_static_size.x86-64 new file mode 100644 index 000000000000..9cf066295304 --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_static_size.x86-64 @@ -0,0 +1,11 @@ +bench_write_to_prefix_static_size: + cmp rdx, 6 + jb .LBB5_2 + movzx eax, word ptr [rdi + 4] + mov word ptr [rsi + 4], ax + mov eax, dword ptr [rdi] + mov dword ptr [rsi], eax +.LBB5_2: + cmp rdx, 6 + setae al + ret diff --git a/rust/zerocopy/benches/write_to_prefix_static_size.x86-64.mca b/rust/zerocopy/benches/write_to_prefix_static_size.x86-64.mca new file mode 100644 index 000000000000..5d17200abd2d --- /dev/null +++ b/rust/zerocopy/benches/write_to_prefix_static_size.x86-64.mca @@ -0,0 +1,57 @@ +Iterations: 100 +Instructions: 900 +Total Cycles: 233 +Total uOps: 900 + +Dispatch Width: 4 +uOps Per Cycle: 3.86 +IPC: 3.86 +Block RThroughput: 2.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 cmp rdx, 6 + 1 1 1.00 jb .LBB5_2 + 1 5 0.50 * movzx eax, word ptr [rdi + 4] + 1 1 1.00 * mov word ptr [rsi + 4], ax + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 1 1.00 * mov dword ptr [rsi], eax + 1 1 0.33 cmp rdx, 6 + 1 1 0.50 setae al + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.50 1.49 2.00 2.01 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.25 0.74 - 0.01 - - cmp rdx, 6 + - - - - - 1.00 - - jb .LBB5_2 + - - - - - - 0.50 0.50 movzx eax, word ptr [rdi + 4] + - - - - 1.00 - 0.48 0.52 mov word ptr [rsi + 4], ax + - - - - - - 0.52 0.48 mov eax, dword ptr [rdi] + - - - - 1.00 - 0.50 0.50 mov dword ptr [rsi], eax + - - 0.25 0.75 - - - - cmp rdx, 6 + - - 1.00 - - - - - setae al + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_static_size.rs b/rust/zerocopy/benches/write_to_static_size.rs new file mode 100644 index 000000000000..3bb9435c5ade --- /dev/null +++ b/rust/zerocopy/benches/write_to_static_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_static_size(source: &format::CocoPacket, destination: &mut [u8]) -> Option<()> { + source.write_to(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_static_size.x86-64 b/rust/zerocopy/benches/write_to_static_size.x86-64 new file mode 100644 index 000000000000..d6413e0fd614 --- /dev/null +++ b/rust/zerocopy/benches/write_to_static_size.x86-64 @@ -0,0 +1,11 @@ +bench_write_to_static_size: + cmp rdx, 6 + jne .LBB5_2 + movzx eax, word ptr [rdi + 4] + mov word ptr [rsi + 4], ax + mov eax, dword ptr [rdi] + mov dword ptr [rsi], eax +.LBB5_2: + cmp rdx, 6 + sete al + ret diff --git a/rust/zerocopy/benches/write_to_static_size.x86-64.mca b/rust/zerocopy/benches/write_to_static_size.x86-64.mca new file mode 100644 index 000000000000..cc5bb1d26fc0 --- /dev/null +++ b/rust/zerocopy/benches/write_to_static_size.x86-64.mca @@ -0,0 +1,57 @@ +Iterations: 100 +Instructions: 900 +Total Cycles: 233 +Total uOps: 900 + +Dispatch Width: 4 +uOps Per Cycle: 3.86 +IPC: 3.86 +Block RThroughput: 2.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 cmp rdx, 6 + 1 1 1.00 jne .LBB5_2 + 1 5 0.50 * movzx eax, word ptr [rdi + 4] + 1 1 1.00 * mov word ptr [rsi + 4], ax + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 1 1.00 * mov dword ptr [rsi], eax + 1 1 0.33 cmp rdx, 6 + 1 1 0.50 sete al + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.50 1.49 2.00 2.01 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.25 0.74 - 0.01 - - cmp rdx, 6 + - - - - - 1.00 - - jne .LBB5_2 + - - - - - - 0.50 0.50 movzx eax, word ptr [rdi + 4] + - - - - 1.00 - 0.48 0.52 mov word ptr [rsi + 4], ax + - - - - - - 0.52 0.48 mov eax, dword ptr [rdi] + - - - - 1.00 - 0.50 0.50 mov dword ptr [rsi], eax + - - 0.25 0.75 - - - - cmp rdx, 6 + - - 1.00 - - - - - sete al + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_suffix_dynamic_size.rs b/rust/zerocopy/benches/write_to_suffix_dynamic_size.rs new file mode 100644 index 000000000000..9fa6b91cda41 --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_dynamic_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_suffix_dynamic_size( + source: &format::CocoPacket, + destination: &mut [u8], +) -> Option<()> { + source.write_to_suffix(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64 b/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64 new file mode 100644 index 000000000000..75f349562db6 --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64 @@ -0,0 +1,22 @@ +bench_write_to_suffix_dynamic_size: + push r14 + push rbx + push rax + mov rbx, rcx + lea r14, [2*rsi + 5] + and r14, -2 + sub rcx, r14 + jb .LBB5_2 + mov rax, rdi + add rdx, rcx + mov rdi, rdx + mov rsi, rax + mov rdx, r14 + call qword ptr [rip + memcpy@GOTPCREL] +.LBB5_2: + cmp rbx, r14 + setae al + add rsp, 8 + pop rbx + pop r14 + ret diff --git a/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64.mca b/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..95cb9dfe2eb7 --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_dynamic_size.x86-64.mca @@ -0,0 +1,79 @@ +Iterations: 100 +Instructions: 2000 +Total Cycles: 2890 +Total uOps: 2600 + +Dispatch Width: 4 +uOps Per Cycle: 0.90 +IPC: 0.69 +Block RThroughput: 6.5 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 2 5 1.00 * push r14 + 2 5 1.00 * push rbx + 2 5 1.00 * push rax + 1 1 0.33 mov rbx, rcx + 1 1 0.50 lea r14, [2*rsi + 5] + 1 1 0.33 and r14, -2 + 1 1 0.33 sub rcx, r14 + 1 1 1.00 jb .LBB5_2 + 1 1 0.33 mov rax, rdi + 1 1 0.33 add rdx, rcx + 1 1 0.33 mov rdi, rdx + 1 1 0.33 mov rsi, rax + 1 1 0.33 mov rdx, r14 + 4 7 1.00 * call qword ptr [rip + memcpy@GOTPCREL] + 1 1 0.33 cmp rbx, r14 + 1 1 0.50 setae al + 1 1 0.33 add rsp, 8 + 1 6 0.50 * pop rbx + 1 6 0.50 * pop r14 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 4.98 4.98 4.00 5.04 4.00 3.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - - 1.00 push r14 + - - - - 1.00 - 1.00 - push rbx + - - - - 1.00 - - 1.00 push rax + - - 0.94 0.05 - 0.01 - - mov rbx, rcx + - - 0.06 0.94 - - - - lea r14, [2*rsi + 5] + - - 0.93 0.02 - 0.05 - - and r14, -2 + - - 0.05 0.94 - 0.01 - - sub rcx, r14 + - - - - - 1.00 - - jb .LBB5_2 + - - 0.02 0.04 - 0.94 - - mov rax, rdi + - - 0.03 0.97 - - - - add rdx, rcx + - - 0.95 0.05 - - - - mov rdi, rdx + - - 0.94 0.03 - 0.03 - - mov rsi, rax + - - 0.01 0.03 - 0.96 - - mov rdx, r14 + - - - - 1.00 1.00 2.00 - call qword ptr [rip + memcpy@GOTPCREL] + - - 0.05 0.94 - 0.01 - - cmp rbx, r14 + - - 0.97 - - 0.03 - - setae al + - - 0.03 0.97 - - - - add rsp, 8 + - - - - - - - 1.00 pop rbx + - - - - - - 1.00 - pop r14 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/write_to_suffix_static_size.rs b/rust/zerocopy/benches/write_to_suffix_static_size.rs new file mode 100644 index 000000000000..1c95aba4b16c --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_static_size.rs @@ -0,0 +1,12 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_write_to_suffix_static_size( + source: &format::CocoPacket, + destination: &mut [u8], +) -> Option<()> { + source.write_to_suffix(destination).ok() +} diff --git a/rust/zerocopy/benches/write_to_suffix_static_size.x86-64 b/rust/zerocopy/benches/write_to_suffix_static_size.x86-64 new file mode 100644 index 000000000000..934aa370d4d6 --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_static_size.x86-64 @@ -0,0 +1,11 @@ +bench_write_to_suffix_static_size: + cmp rdx, 6 + jb .LBB5_2 + movzx eax, word ptr [rdi + 4] + mov word ptr [rsi + rdx - 2], ax + mov eax, dword ptr [rdi] + mov dword ptr [rsi + rdx - 6], eax +.LBB5_2: + cmp rdx, 6 + setae al + ret diff --git a/rust/zerocopy/benches/write_to_suffix_static_size.x86-64.mca b/rust/zerocopy/benches/write_to_suffix_static_size.x86-64.mca new file mode 100644 index 000000000000..6b18e4a44585 --- /dev/null +++ b/rust/zerocopy/benches/write_to_suffix_static_size.x86-64.mca @@ -0,0 +1,57 @@ +Iterations: 100 +Instructions: 900 +Total Cycles: 233 +Total uOps: 900 + +Dispatch Width: 4 +uOps Per Cycle: 3.86 +IPC: 3.86 +Block RThroughput: 2.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 cmp rdx, 6 + 1 1 1.00 jb .LBB5_2 + 1 5 0.50 * movzx eax, word ptr [rdi + 4] + 1 1 1.00 * mov word ptr [rsi + rdx - 2], ax + 1 5 0.50 * mov eax, dword ptr [rdi] + 1 1 1.00 * mov dword ptr [rsi + rdx - 6], eax + 1 1 0.33 cmp rdx, 6 + 1 1 0.50 setae al + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.50 1.49 2.00 2.01 2.00 2.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.25 0.74 - 0.01 - - cmp rdx, 6 + - - - - - 1.00 - - jb .LBB5_2 + - - - - - - 0.50 0.50 movzx eax, word ptr [rdi + 4] + - - - - 1.00 - 0.48 0.52 mov word ptr [rsi + rdx - 2], ax + - - - - - - 0.52 0.48 mov eax, dword ptr [rdi] + - - - - 1.00 - 0.50 0.50 mov dword ptr [rsi + rdx - 6], eax + - - 0.25 0.75 - - - - cmp rdx, 6 + - - 1.00 - - - - - setae al + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/benches/zero_dynamic_padding.rs b/rust/zerocopy/benches/zero_dynamic_padding.rs new file mode 100644 index 000000000000..8eda0953d1e6 --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_padding.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_padding.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_zero_dynamic_padding(source: &mut format::LocoPacket) { + source.zero() +} diff --git a/rust/zerocopy/benches/zero_dynamic_padding.x86-64 b/rust/zerocopy/benches/zero_dynamic_padding.x86-64 new file mode 100644 index 000000000000..7dccf1745f81 --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_padding.x86-64 @@ -0,0 +1,7 @@ +bench_zero_dynamic_padding: + lea rax, [rsi + 2*rsi] + movabs rdx, 9223372036854775804 + and rdx, rax + add rdx, 12 + xor esi, esi + jmp qword ptr [rip + memset@GOTPCREL] diff --git a/rust/zerocopy/benches/zero_dynamic_padding.x86-64.mca b/rust/zerocopy/benches/zero_dynamic_padding.x86-64.mca new file mode 100644 index 000000000000..098fc107875f --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_padding.x86-64.mca @@ -0,0 +1,51 @@ +Iterations: 100 +Instructions: 600 +Total Cycles: 209 +Total uOps: 700 + +Dispatch Width: 4 +uOps Per Cycle: 3.35 +IPC: 2.87 +Block RThroughput: 1.8 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea rax, [rsi + 2*rsi] + 1 1 0.33 movabs rdx, 9223372036854775804 + 1 1 0.33 and rdx, rax + 1 1 0.33 add rdx, 12 + 1 0 0.25 xor esi, esi + 2 6 1.00 * jmp qword ptr [rip + memset@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 1.66 1.66 - 1.68 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.33 0.67 - - - - lea rax, [rsi + 2*rsi] + - - 0.98 - - 0.02 - - movabs rdx, 9223372036854775804 + - - 0.01 0.66 - 0.33 - - and rdx, rax + - - 0.34 0.33 - 0.33 - - add rdx, 12 + - - - - - - - - xor esi, esi + - - - - - 1.00 0.50 0.50 jmp qword ptr [rip + memset@GOTPCREL] diff --git a/rust/zerocopy/benches/zero_dynamic_size.rs b/rust/zerocopy/benches/zero_dynamic_size.rs new file mode 100644 index 000000000000..536d800ebc76 --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_dynamic_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_zero_dynamic_size(source: &mut format::LocoPacket) { + source.zero() +} diff --git a/rust/zerocopy/benches/zero_dynamic_size.x86-64 b/rust/zerocopy/benches/zero_dynamic_size.x86-64 new file mode 100644 index 000000000000..2b31ed644eaf --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_size.x86-64 @@ -0,0 +1,5 @@ +bench_zero_dynamic_size: + lea rdx, [2*rsi + 5] + and rdx, -2 + xor esi, esi + jmp qword ptr [rip + memset@GOTPCREL] diff --git a/rust/zerocopy/benches/zero_dynamic_size.x86-64.mca b/rust/zerocopy/benches/zero_dynamic_size.x86-64.mca new file mode 100644 index 000000000000..0b086a29b0ed --- /dev/null +++ b/rust/zerocopy/benches/zero_dynamic_size.x86-64.mca @@ -0,0 +1,47 @@ +Iterations: 100 +Instructions: 400 +Total Cycles: 142 +Total uOps: 500 + +Dispatch Width: 4 +uOps Per Cycle: 3.52 +IPC: 2.82 +Block RThroughput: 1.3 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.50 lea rdx, [2*rsi + 5] + 1 1 0.33 and rdx, -2 + 1 0 0.25 xor esi, esi + 2 6 1.00 * jmp qword ptr [rip + memset@GOTPCREL] + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - 0.99 1.00 - 1.01 0.50 0.50 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - 0.99 0.01 - - - - lea rdx, [2*rsi + 5] + - - - 0.99 - 0.01 - - and rdx, -2 + - - - - - - - - xor esi, esi + - - - - - 1.00 0.50 0.50 jmp qword ptr [rip + memset@GOTPCREL] diff --git a/rust/zerocopy/benches/zero_static_size.rs b/rust/zerocopy/benches/zero_static_size.rs new file mode 100644 index 000000000000..fa7fa0839c15 --- /dev/null +++ b/rust/zerocopy/benches/zero_static_size.rs @@ -0,0 +1,9 @@ +use zerocopy::*; + +#[path = "formats/coco_static_size.rs"] +mod format; + +#[unsafe(no_mangle)] +fn bench_zero_static_size(source: &mut format::LocoPacket) { + source.zero() +} diff --git a/rust/zerocopy/benches/zero_static_size.x86-64 b/rust/zerocopy/benches/zero_static_size.x86-64 new file mode 100644 index 000000000000..ced8e18949b2 --- /dev/null +++ b/rust/zerocopy/benches/zero_static_size.x86-64 @@ -0,0 +1,4 @@ +bench_zero_static_size: + mov word ptr [rdi + 4], 0 + mov dword ptr [rdi], 0 + ret diff --git a/rust/zerocopy/benches/zero_static_size.x86-64.mca b/rust/zerocopy/benches/zero_static_size.x86-64.mca new file mode 100644 index 000000000000..042897ef2dfe --- /dev/null +++ b/rust/zerocopy/benches/zero_static_size.x86-64.mca @@ -0,0 +1,45 @@ +Iterations: 100 +Instructions: 300 +Total Cycles: 203 +Total uOps: 300 + +Dispatch Width: 4 +uOps Per Cycle: 1.48 +IPC: 1.48 +Block RThroughput: 2.0 + + +Instruction Info: +[1]: #uOps +[2]: Latency +[3]: RThroughput +[4]: MayLoad +[5]: MayStore +[6]: HasSideEffects (U) + +[1] [2] [3] [4] [5] [6] Instructions: + 1 1 1.00 * mov word ptr [rdi + 4], 0 + 1 1 1.00 * mov dword ptr [rdi], 0 + 1 1 1.00 U ret + + +Resources: +[0] - SBDivider +[1] - SBFPDivider +[2] - SBPort0 +[3] - SBPort1 +[4] - SBPort4 +[5] - SBPort5 +[6.0] - SBPort23 +[6.1] - SBPort23 + + +Resource pressure per iteration: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] + - - - - 2.00 1.00 1.00 1.00 + +Resource pressure by instruction: +[0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: + - - - - 1.00 - - 1.00 mov word ptr [rdi + 4], 0 + - - - - 1.00 - 1.00 - mov dword ptr [rdi], 0 + - - - - - 1.00 - - ret diff --git a/rust/zerocopy/rustdoc/style.css b/rust/zerocopy/rustdoc/style.css new file mode 100644 index 000000000000..d2d55ad2e689 --- /dev/null +++ b/rust/zerocopy/rustdoc/style.css @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT */ +/* +Copyright 2026 The Fuchsia Authors + +Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +<LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +This file may not be copied, modified, or distributed except according to +those terms. +*/ + +.codegen-tabs { + display: grid; + grid-template-columns: repeat(var(--arity), minmax(200px, 1fr)); + grid-template-rows: auto 1fr; + column-gap: 1rem; +} + +.codegen-tabs:not(:has(> details[open]))::after { + grid-column: 1/-1; + content: 'Click one of the above headers to expand its contents.'; + font-style: italic; + font-size: small; + text-align: center; +} + +.codegen-tabs details { + display: grid; + grid-column: 1 / -1; + grid-row: 1 / span 2; + grid-template-columns: subgrid; + grid-template-rows: subgrid; +} + +.codegen-tabs summary { + display: grid; + grid-column: var(--n) / span 1; + grid-row: 1; + z-index: 1; + border-bottom: 2px solid var(--headings-border-bottom-color); + cursor: pointer; +} + +.codegen-tabs details[open] > summary { + background-color: var(--code-block-background-color); + border-bottom-color: var(--target-border-color); +} + +.codegen-tabs details::details-content { + grid-column: 1 / -1; + grid-row: 2; +} + +.codegen-tabs details:not([open])::details-content { + display: none; +} diff --git a/rust/zerocopy/src/byte_slice.rs b/rust/zerocopy/src/byte_slice.rs new file mode 100644 index 000000000000..b7f85098dbc4 --- /dev/null +++ b/rust/zerocopy/src/byte_slice.rs @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2024 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Traits for types that encapsulate a `[u8]`. +//! +//! These traits are used to bound the `B` parameter of [`Ref`]. + +use core::{ + cell, + ops::{Deref, DerefMut}, +}; + +// For each trait polyfill, as soon as the corresponding feature is stable, the +// polyfill import will be unused because method/function resolution will prefer +// the inherent method/function over a trait method/function. Thus, we suppress +// the `unused_imports` warning. +// +// See the documentation on `util::polyfills` for more information. +#[allow(unused_imports)] +use crate::util::polyfills::{self, NonNullExt as _, NumExt as _}; +#[cfg(doc)] +use crate::Ref; + +/// A mutable or immutable reference to a byte slice. +/// +/// `ByteSlice` abstracts over the mutability of a byte slice reference, and is +/// implemented for various special reference types such as +/// [`Ref<[u8]>`](core::cell::Ref) and [`RefMut<[u8]>`](core::cell::RefMut). +/// +/// # Safety +/// +/// Implementations of `ByteSlice` must promise that their implementations of +/// [`Deref`] and [`DerefMut`] are "stable". In particular, given `B: ByteSlice` +/// and `b: B`, two calls, each to either `b.deref()` or `b.deref_mut()`, must +/// return a byte slice with the same address and length. This must hold even if +/// the two calls are separated by an arbitrary sequence of calls to methods on +/// `ByteSlice`, [`ByteSliceMut`], [`IntoByteSlice`], or [`IntoByteSliceMut`], +/// or on their super-traits. This does *not* need to hold if the two calls are +/// separated by any method calls, field accesses, or field modifications *other +/// than* those from these traits. +/// +/// Note that this also implies that, given `b: B`, the address and length +/// cannot be modified via objects other than `b`, either on the same thread or +/// on another thread. +pub unsafe trait ByteSlice: Deref<Target = [u8]> + Sized {} + +/// A mutable reference to a byte slice. +/// +/// `ByteSliceMut` abstracts over various ways of storing a mutable reference to +/// a byte slice, and is implemented for various special reference types such as +/// `RefMut<[u8]>`. +/// +/// `ByteSliceMut` is a shorthand for [`ByteSlice`] and [`DerefMut`]. +pub trait ByteSliceMut: ByteSlice + DerefMut {} +impl<B: ByteSlice + DerefMut> ByteSliceMut for B {} + +/// A [`ByteSlice`] which can be copied without violating dereference stability. +/// +/// # Safety +/// +/// If `B: CopyableByteSlice`, then the dereference stability properties +/// required by [`ByteSlice`] (see that trait's safety documentation) do not +/// only hold regarding two calls to `b.deref()` or `b.deref_mut()`, but also +/// hold regarding `c.deref()` or `c.deref_mut()`, where `c` is produced by +/// copying `b`. +pub unsafe trait CopyableByteSlice: ByteSlice + Copy + CloneableByteSlice {} + +/// A [`ByteSlice`] which can be cloned without violating dereference stability. +/// +/// # Safety +/// +/// If `B: CloneableByteSlice`, then the dereference stability properties +/// required by [`ByteSlice`] (see that trait's safety documentation) do not +/// only hold regarding two calls to `b.deref()` or `b.deref_mut()`, but also +/// hold regarding `c.deref()` or `c.deref_mut()`, where `c` is produced by +/// `b.clone()`, `b.clone().clone()`, etc. +pub unsafe trait CloneableByteSlice: ByteSlice + Clone {} + +/// A [`ByteSlice`] that can be split in two. +/// +/// # Safety +/// +/// Unsafe code may depend for its soundness on the assumption that `split_at` +/// and `split_at_unchecked` are implemented correctly. In particular, given `B: +/// SplitByteSlice` and `b: B`, if `b.deref()` returns a byte slice with address +/// `addr` and length `len`, then if `split <= len`, both of these +/// invocations: +/// - `b.split_at(split)` +/// - `b.split_at_unchecked(split)` +/// +/// ...will return `(first, second)` such that: +/// - `first`'s address is `addr` and its length is `split` +/// - `second`'s address is `addr + split` and its length is `len - split` +pub unsafe trait SplitByteSlice: ByteSlice { + /// Attempts to split `self` at the midpoint. + /// + /// `s.split_at(mid)` returns `Ok((s[..mid], s[mid..]))` if `mid <= + /// s.deref().len()` and otherwise returns `Err(s)`. + /// + /// # Safety + /// + /// Unsafe code may rely on this function correctly implementing the above + /// functionality. + #[inline] + fn split_at(self, mid: usize) -> Result<(Self, Self), Self> { + if mid <= self.deref().len() { + // SAFETY: Above, we ensure that `mid <= self.deref().len()`. By + // invariant on `ByteSlice`, a supertrait of `SplitByteSlice`, + // `.deref()` is guaranteed to be "stable"; i.e., it will always + // dereference to a byte slice of the same address and length. Thus, + // we can be sure that the above precondition remains satisfied + // through the call to `split_at_unchecked`. + unsafe { Ok(self.split_at_unchecked(mid)) } + } else { + Err(self) + } + } + + /// Splits the slice at the midpoint, possibly omitting bounds checks. + /// + /// `s.split_at_unchecked(mid)` returns `s[..mid]` and `s[mid..]`. + /// + /// # Safety + /// + /// `mid` must not be greater than `self.deref().len()`. + /// + /// # Panics + /// + /// Implementations of this method may choose to perform a bounds check and + /// panic if `mid > self.deref().len()`. They may also panic for any other + /// reason. Since it is optional, callers must not rely on this behavior for + /// soundness. + #[must_use] + unsafe fn split_at_unchecked(self, mid: usize) -> (Self, Self); +} + +/// A shorthand for [`SplitByteSlice`] and [`ByteSliceMut`]. +pub trait SplitByteSliceMut: SplitByteSlice + ByteSliceMut {} +impl<B: SplitByteSlice + ByteSliceMut> SplitByteSliceMut for B {} + +#[allow(clippy::missing_safety_doc)] // There's a `Safety` section on `into_byte_slice`. +/// A [`ByteSlice`] that conveys no ownership, and so can be converted into a +/// byte slice. +/// +/// Some `ByteSlice` types (notably, the standard library's [`Ref`] type) convey +/// ownership, and so they cannot soundly be moved by-value into a byte slice +/// type (`&[u8]`). Some methods in this crate's API (such as [`Ref::into_ref`]) +/// are only compatible with `ByteSlice` types without these ownership +/// semantics. +/// +/// [`Ref`]: core::cell::Ref +pub unsafe trait IntoByteSlice<'a>: ByteSlice { + /// Coverts `self` into a `&[u8]`. + /// + /// # Safety + /// + /// The returned reference has the same address and length as `self.deref()` + /// and `self.deref_mut()`. + /// + /// Note that, combined with the safety invariant on [`ByteSlice`], this + /// safety invariant implies that the returned reference is "stable" in the + /// sense described in the `ByteSlice` docs. + fn into_byte_slice(self) -> &'a [u8]; +} + +#[allow(clippy::missing_safety_doc)] // There's a `Safety` section on `into_byte_slice_mut`. +/// A [`ByteSliceMut`] that conveys no ownership, and so can be converted into a +/// mutable byte slice. +/// +/// Some `ByteSliceMut` types (notably, the standard library's [`RefMut`] type) +/// convey ownership, and so they cannot soundly be moved by-value into a byte +/// slice type (`&mut [u8]`). Some methods in this crate's API (such as +/// [`Ref::into_mut`]) are only compatible with `ByteSliceMut` types without +/// these ownership semantics. +/// +/// [`RefMut`]: core::cell::RefMut +pub unsafe trait IntoByteSliceMut<'a>: IntoByteSlice<'a> + ByteSliceMut { + /// Coverts `self` into a `&mut [u8]`. + /// + /// # Safety + /// + /// The returned reference has the same address and length as `self.deref()` + /// and `self.deref_mut()`. + /// + /// Note that, combined with the safety invariant on [`ByteSlice`], this + /// safety invariant implies that the returned reference is "stable" in the + /// sense described in the `ByteSlice` docs. + fn into_byte_slice_mut(self) -> &'a mut [u8]; +} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl ByteSlice for &[u8] {} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl CopyableByteSlice for &[u8] {} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl CloneableByteSlice for &[u8] {} + +// SAFETY: This delegates to `polyfills:split_at_unchecked`, which is documented +// to correctly split `self` into two slices at the given `mid` point. +unsafe impl SplitByteSlice for &[u8] { + #[inline] + unsafe fn split_at_unchecked(self, mid: usize) -> (Self, Self) { + // SAFETY: By contract on caller, `mid` is not greater than + // `self.len()`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + unsafe { + (<[u8]>::get_unchecked(self, ..mid), <[u8]>::get_unchecked(self, mid..)) + } + } +} + +// SAFETY: See inline. +unsafe impl<'a> IntoByteSlice<'a> for &'a [u8] { + #[inline(always)] + fn into_byte_slice(self) -> &'a [u8] { + // SAFETY: It would be patently insane to implement `<Deref for + // &[u8]>::deref` as anything other than `fn deref(&self) -> &[u8] { + // *self }`. Assuming this holds, then `self` is stable as required by + // `into_byte_slice`. + self + } +} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl ByteSlice for &mut [u8] {} + +// SAFETY: This delegates to `polyfills:split_at_mut_unchecked`, which is +// documented to correctly split `self` into two slices at the given `mid` +// point. +unsafe impl SplitByteSlice for &mut [u8] { + #[inline] + unsafe fn split_at_unchecked(self, mid: usize) -> (Self, Self) { + use core::slice::from_raw_parts_mut; + + // `l_ptr` is non-null, because `self` is non-null, by invariant on + // `&mut [u8]`. + let l_ptr = self.as_mut_ptr(); + + // SAFETY: By contract on caller, `mid` is not greater than + // `self.len()`. + let r_ptr = unsafe { l_ptr.add(mid) }; + + let l_len = mid; + + // SAFETY: By contract on caller, `mid` is not greater than + // `self.len()`. + // + // FIXME(#67): Remove this allow. See NumExt for more details. + #[allow(unstable_name_collisions)] + let r_len = unsafe { self.len().unchecked_sub(mid) }; + + // SAFETY: These invocations of `from_raw_parts_mut` satisfy its + // documented safety preconditions [1]: + // - The data `l_ptr` and `r_ptr` are valid for both reads and writes of + // `l_len` and `r_len` bytes, respectively, and they are trivially + // aligned. In particular: + // - The entire memory range of each slice is contained within a + // single allocated object, since `l_ptr` and `r_ptr` are both + // derived from within the address range of `self`. + // - Both `l_ptr` and `r_ptr` are non-null and trivially aligned. + // `self` is non-null by invariant on `&mut [u8]`, and the + // operations that derive `l_ptr` and `r_ptr` from `self` do not + // nullify either pointer. + // - The data `l_ptr` and `r_ptr` point to `l_len` and `r_len`, + // respectively, consecutive properly initialized values of type `u8`. + // This is true for `self` by invariant on `&mut [u8]`, and remains + // true for these two sub-slices of `self`. + // - The memory referenced by the returned slice cannot be accessed + // through any other pointer (not derived from the return value) for + // the duration of lifetime `'a``, because: + // - `split_at_unchecked` consumes `self` (which is not `Copy`), + // - `split_at_unchecked` does not exfiltrate any references to this + // memory, besides those references returned below, + // - the returned slices are non-overlapping. + // - The individual sizes of the sub-slices of `self` are no larger than + // `isize::MAX`, because their combined sizes are no larger than + // `isize::MAX`, by invariant on `self`. + // + // [1] https://doc.rust-lang.org/std/slice/fn.from_raw_parts_mut.html#safety + #[allow(clippy::multiple_unsafe_ops_per_block)] + unsafe { + (from_raw_parts_mut(l_ptr, l_len), from_raw_parts_mut(r_ptr, r_len)) + } + } +} + +// SAFETY: See inline. +unsafe impl<'a> IntoByteSlice<'a> for &'a mut [u8] { + #[inline(always)] + fn into_byte_slice(self) -> &'a [u8] { + // SAFETY: It would be patently insane to implement `<Deref for &mut + // [u8]>::deref` as anything other than `fn deref(&self) -> &[u8] { + // *self }`. Assuming this holds, then `self` is stable as required by + // `into_byte_slice`. + self + } +} + +// SAFETY: See inline. +unsafe impl<'a> IntoByteSliceMut<'a> for &'a mut [u8] { + #[inline(always)] + fn into_byte_slice_mut(self) -> &'a mut [u8] { + // SAFETY: It would be patently insane to implement `<DerefMut for &mut + // [u8]>::deref` as anything other than `fn deref_mut(&mut self) -> &mut + // [u8] { *self }`. Assuming this holds, then `self` is stable as + // required by `into_byte_slice_mut`. + self + } +} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl ByteSlice for cell::Ref<'_, [u8]> {} + +// SAFETY: This delegates to stdlib implementation of `Ref::map_split`, which is +// assumed to be correct, and `SplitByteSlice::split_at_unchecked`, which is +// documented to correctly split `self` into two slices at the given `mid` +// point. +unsafe impl SplitByteSlice for cell::Ref<'_, [u8]> { + #[inline] + unsafe fn split_at_unchecked(self, mid: usize) -> (Self, Self) { + cell::Ref::map_split(self, |slice| + // SAFETY: By precondition on caller, `mid` is not greater than + // `slice.len()`. + unsafe { + SplitByteSlice::split_at_unchecked(slice, mid) + }) + } +} + +// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. +#[allow(clippy::undocumented_unsafe_blocks)] +unsafe impl ByteSlice for cell::RefMut<'_, [u8]> {} + +// SAFETY: This delegates to stdlib implementation of `RefMut::map_split`, which +// is assumed to be correct, and `SplitByteSlice::split_at_unchecked`, which is +// documented to correctly split `self` into two slices at the given `mid` +// point. +unsafe impl SplitByteSlice for cell::RefMut<'_, [u8]> { + #[inline] + unsafe fn split_at_unchecked(self, mid: usize) -> (Self, Self) { + cell::RefMut::map_split(self, |slice| + // SAFETY: By precondition on caller, `mid` is not greater than + // `slice.len()` + unsafe { + SplitByteSlice::split_at_unchecked(slice, mid) + }) + } +} + +#[cfg(kani)] +mod proofs { + use super::*; + + fn any_vec() -> Vec<u8> { + let len = kani::any(); + kani::assume(len <= crate::DstLayout::MAX_SIZE); + vec![0u8; len] + } + + #[kani::proof] + fn prove_split_at_unchecked() { + let v = any_vec(); + let slc = v.as_slice(); + let mid = kani::any(); + kani::assume(mid <= slc.len()); + let (l, r) = unsafe { slc.split_at_unchecked(mid) }; + assert_eq!(l.len() + r.len(), slc.len()); + + let slc: *const _ = slc; + let l: *const _ = l; + let r: *const _ = r; + + assert_eq!(slc.cast::<u8>(), l.cast::<u8>()); + assert_eq!(unsafe { slc.cast::<u8>().add(mid) }, r.cast::<u8>()); + + let mut v = any_vec(); + let slc = v.as_mut_slice(); + let len = slc.len(); + let mid = kani::any(); + kani::assume(mid <= slc.len()); + let (l, r) = unsafe { slc.split_at_unchecked(mid) }; + assert_eq!(l.len() + r.len(), len); + + let l: *mut _ = l; + let r: *mut _ = r; + let slc: *mut _ = slc; + + assert_eq!(slc.cast::<u8>(), l.cast::<u8>()); + assert_eq!(unsafe { slc.cast::<u8>().add(mid) }, r.cast::<u8>()); + } +} + +#[cfg(test)] +mod tests { + use core::cell::RefCell; + + use super::*; + + #[test] + fn test_ref_split_at_unchecked() { + let cell = RefCell::new([1, 2, 3, 4]); + let borrow = cell.borrow(); + let slice_ref: cell::Ref<'_, [u8]> = cell::Ref::map(borrow, |a| &a[..]); + // SAFETY: 2 is within bounds of [1, 2, 3, 4] + let (l, r) = unsafe { slice_ref.split_at_unchecked(2) }; + assert_eq!(*l, [1, 2]); + assert_eq!(*r, [3, 4]); + } + + #[test] + fn test_ref_mut_split_at_unchecked() { + let cell = RefCell::new([1, 2, 3, 4]); + let borrow_mut = cell.borrow_mut(); + let slice_ref_mut: cell::RefMut<'_, [u8]> = cell::RefMut::map(borrow_mut, |a| &mut a[..]); + // SAFETY: 2 is within bounds of [1, 2, 3, 4] + let (l, r) = unsafe { slice_ref_mut.split_at_unchecked(2) }; + assert_eq!(*l, [1, 2]); + assert_eq!(*r, [3, 4]); + } +} diff --git a/rust/zerocopy/src/byteorder.rs b/rust/zerocopy/src/byteorder.rs new file mode 100644 index 000000000000..c761d5728320 --- /dev/null +++ b/rust/zerocopy/src/byteorder.rs @@ -0,0 +1,1596 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2019 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Byte order-aware numeric primitives. +//! +//! This module contains equivalents of the native multi-byte integer types with +//! no alignment requirement and supporting byte order conversions. +//! +//! For each native multi-byte integer type - `u16`, `i16`, `u32`, etc - and +//! floating point type - `f32` and `f64` - an equivalent type is defined by +//! this module - [`U16`], [`I16`], [`U32`], [`F32`], [`F64`], etc. Unlike their +//! native counterparts, these types have alignment 1, and take a type parameter +//! specifying the byte order in which the bytes are stored in memory. Each type +//! implements this crate's relevant conversion and marker traits. +//! +//! These two properties, taken together, make these types useful for defining +//! data structures whose memory layout matches a wire format such as that of a +//! network protocol or a file format. Such formats often have multi-byte values +//! at offsets that do not respect the alignment requirements of the equivalent +//! native types, and stored in a byte order not necessarily the same as that of +//! the target platform. +//! +//! Type aliases are provided for common byte orders in the [`big_endian`], +//! [`little_endian`], [`network_endian`], and [`native_endian`] submodules. +//! Note that network-endian is a synonym for big-endian. +//! +//! # Example +//! +//! One use of these types is for representing network packet formats, such as +//! UDP: +//! +//! ```rust +//! use zerocopy::{*, byteorder::network_endian::U16}; +//! # use zerocopy_derive::*; +//! +//! #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] +//! #[repr(C)] +//! struct UdpHeader { +//! src_port: U16, +//! dst_port: U16, +//! length: U16, +//! checksum: U16, +//! } +//! +//! #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] +//! #[repr(C, packed)] +//! struct UdpPacket { +//! header: UdpHeader, +//! body: [u8], +//! } +//! +//! impl UdpPacket { +//! fn parse(bytes: &[u8]) -> Option<&UdpPacket> { +//! UdpPacket::ref_from_bytes(bytes).ok() +//! } +//! } +//! ``` + +use core::{ + convert::{TryFrom, TryInto}, + fmt::{Binary, Debug, LowerHex, Octal, UpperHex}, + hash::Hash, + num::TryFromIntError, +}; + +use super::*; + +/// A type-level representation of byte order. +/// +/// This type is implemented by [`BigEndian`] and [`LittleEndian`], which +/// represent big-endian and little-endian byte order respectively. This module +/// also provides a number of useful aliases for those types: [`NativeEndian`], +/// [`NetworkEndian`], [`BE`], and [`LE`]. +/// +/// `ByteOrder` types can be used to specify the byte order of the types in this +/// module - for example, [`U32<BigEndian>`] is a 32-bit integer stored in +/// big-endian byte order. +/// +/// [`U32<BigEndian>`]: U32 +pub trait ByteOrder: + Copy + Clone + Debug + Display + Eq + PartialEq + Ord + PartialOrd + Hash + private::Sealed +{ + #[doc(hidden)] + const ORDER: Order; +} + +mod private { + pub trait Sealed {} + + impl Sealed for super::BigEndian {} + impl Sealed for super::LittleEndian {} +} + +#[allow(missing_copy_implementations, missing_debug_implementations)] +#[doc(hidden)] +#[derive(PartialEq)] +pub enum Order { + BigEndian, + LittleEndian, +} + +/// Big-endian byte order. +/// +/// See [`ByteOrder`] for more details. +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum BigEndian {} + +impl ByteOrder for BigEndian { + const ORDER: Order = Order::BigEndian; +} + +impl Display for BigEndian { + #[inline] + fn fmt(&self, _: &mut Formatter<'_>) -> fmt::Result { + match *self {} + } +} + +/// Little-endian byte order. +/// +/// See [`ByteOrder`] for more details. +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum LittleEndian {} + +impl ByteOrder for LittleEndian { + const ORDER: Order = Order::LittleEndian; +} + +impl Display for LittleEndian { + #[inline] + fn fmt(&self, _: &mut Formatter<'_>) -> fmt::Result { + match *self {} + } +} + +/// The endianness used by this platform. +/// +/// This is a type alias for [`BigEndian`] or [`LittleEndian`] depending on the +/// endianness of the target platform. +#[cfg(target_endian = "big")] +pub type NativeEndian = BigEndian; + +/// The endianness used by this platform. +/// +/// This is a type alias for [`BigEndian`] or [`LittleEndian`] depending on the +/// endianness of the target platform. +#[cfg(target_endian = "little")] +pub type NativeEndian = LittleEndian; + +/// The endianness used in many network protocols. +/// +/// This is a type alias for [`BigEndian`]. +pub type NetworkEndian = BigEndian; + +/// A type alias for [`BigEndian`]. +pub type BE = BigEndian; + +/// A type alias for [`LittleEndian`]. +pub type LE = LittleEndian; + +macro_rules! impl_dbg_trait { + ($name:ident, $native:ident) => { + impl<O: ByteOrder> Debug for $name<O> { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + // This results in a format like "U16(42)". + f.debug_tuple(stringify!($name)).field(&self.get()).finish() + } + } + }; +} + +macro_rules! impl_dbg_traits { + ($name:ident, $native:ident, "floating point number") => { + #[cfg(not(no_fp_fmt_parse))] + impl_dbg_trait!($name, $native); + + #[cfg(no_fp_fmt_parse)] + impl<O: ByteOrder> Debug for $name<O> { + #[inline] + fn fmt(&self, _f: &mut Formatter<'_>) -> fmt::Result { + panic!("floating point support is turned off"); + } + } + }; + ($name:ident, $native:ident, "unsigned integer") => { + impl_dbg_traits!($name, $native, @all_types); + }; + ($name:ident, $native:ident, "signed integer") => { + impl_dbg_traits!($name, $native, @all_types); + }; + ($name:ident, $native:ident, @all_types) => { + impl_dbg_trait!($name, $native); + }; +} + +macro_rules! impl_fmt_trait { + ($name:ident, $native:ident, $trait:ident) => { + impl<O: ByteOrder> $trait for $name<O> { + #[inline(always)] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + $trait::fmt(&self.get(), f) + } + } + }; +} + +macro_rules! impl_fmt_traits { + ($name:ident, $native:ident, "floating point number") => { + #[cfg(not(no_fp_fmt_parse))] + impl_fmt_trait!($name, $native, Display); + }; + ($name:ident, $native:ident, "unsigned integer") => { + impl_fmt_traits!($name, $native, @all_types); + }; + ($name:ident, $native:ident, "signed integer") => { + impl_fmt_traits!($name, $native, @all_types); + }; + ($name:ident, $native:ident, @all_types) => { + impl_fmt_trait!($name, $native, Display); + impl_fmt_trait!($name, $native, Octal); + impl_fmt_trait!($name, $native, LowerHex); + impl_fmt_trait!($name, $native, UpperHex); + impl_fmt_trait!($name, $native, Binary); + }; +} + +macro_rules! impl_ops_traits { + ($name:ident, $native:ident, "floating point number") => { + impl_ops_traits!($name, $native, @all_types); + impl_ops_traits!($name, $native, @signed_integer_floating_point); + + impl<O: ByteOrder> PartialOrd for $name<O> { + #[inline(always)] + fn partial_cmp(&self, other: &Self) -> Option<Ordering> { + self.get().partial_cmp(&other.get()) + } + } + }; + ($name:ident, $native:ident, "unsigned integer") => { + impl_ops_traits!($name, $native, @signed_unsigned_integer); + impl_ops_traits!($name, $native, @all_types); + }; + ($name:ident, $native:ident, "signed integer") => { + impl_ops_traits!($name, $native, @signed_unsigned_integer); + impl_ops_traits!($name, $native, @signed_integer_floating_point); + impl_ops_traits!($name, $native, @all_types); + }; + ($name:ident, $native:ident, @signed_unsigned_integer) => { + impl_ops_traits!(@without_byteorder_swap $name, $native, BitAnd, bitand, BitAndAssign, bitand_assign); + impl_ops_traits!(@without_byteorder_swap $name, $native, BitOr, bitor, BitOrAssign, bitor_assign); + impl_ops_traits!(@without_byteorder_swap $name, $native, BitXor, bitxor, BitXorAssign, bitxor_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Shl, shl, ShlAssign, shl_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Shr, shr, ShrAssign, shr_assign); + + impl<O> core::ops::Not for $name<O> { + type Output = $name<O>; + + #[inline(always)] + fn not(self) -> $name<O> { + let self_native = $native::from_ne_bytes(self.0); + $name((!self_native).to_ne_bytes(), PhantomData) + } + } + + impl<O: ByteOrder> PartialOrd for $name<O> { + #[inline(always)] + fn partial_cmp(&self, other: &Self) -> Option<Ordering> { + Some(self.cmp(other)) + } + } + + impl<O: ByteOrder> Ord for $name<O> { + #[inline(always)] + fn cmp(&self, other: &Self) -> Ordering { + self.get().cmp(&other.get()) + } + } + + impl<O: ByteOrder> PartialOrd<$native> for $name<O> { + #[inline(always)] + fn partial_cmp(&self, other: &$native) -> Option<Ordering> { + self.get().partial_cmp(other) + } + } + }; + ($name:ident, $native:ident, @signed_integer_floating_point) => { + impl<O: ByteOrder> core::ops::Neg for $name<O> { + type Output = $name<O>; + + #[inline(always)] + fn neg(self) -> $name<O> { + let self_native: $native = self.get(); + #[allow(clippy::arithmetic_side_effects)] + $name::<O>::new(-self_native) + } + } + }; + ($name:ident, $native:ident, @all_types) => { + impl_ops_traits!(@with_byteorder_swap $name, $native, Add, add, AddAssign, add_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Div, div, DivAssign, div_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Mul, mul, MulAssign, mul_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Rem, rem, RemAssign, rem_assign); + impl_ops_traits!(@with_byteorder_swap $name, $native, Sub, sub, SubAssign, sub_assign); + }; + (@with_byteorder_swap $name:ident, $native:ident, $trait:ident, $method:ident, $trait_assign:ident, $method_assign:ident) => { + impl<O: ByteOrder> core::ops::$trait<$name<O>> for $name<O> { + type Output = $name<O>; + + #[inline(always)] + fn $method(self, rhs: $name<O>) -> $name<O> { + let self_native: $native = self.get(); + let rhs_native: $native = rhs.get(); + let result_native = core::ops::$trait::$method(self_native, rhs_native); + $name::<O>::new(result_native) + } + } + + impl<O: ByteOrder> core::ops::$trait<$name<O>> for $native { + type Output = $name<O>; + + #[inline(always)] + fn $method(self, rhs: $name<O>) -> $name<O> { + let rhs_native: $native = rhs.get(); + let result_native = core::ops::$trait::$method(self, rhs_native); + $name::<O>::new(result_native) + } + } + + impl<O: ByteOrder> core::ops::$trait<$native> for $name<O> { + type Output = $name<O>; + + #[inline(always)] + fn $method(self, rhs: $native) -> $name<O> { + let self_native: $native = self.get(); + let result_native = core::ops::$trait::$method(self_native, rhs); + $name::<O>::new(result_native) + } + } + + impl<O: ByteOrder> core::ops::$trait_assign<$name<O>> for $name<O> { + #[inline(always)] + fn $method_assign(&mut self, rhs: $name<O>) { + *self = core::ops::$trait::$method(*self, rhs); + } + } + + impl<O: ByteOrder> core::ops::$trait_assign<$name<O>> for $native { + #[inline(always)] + fn $method_assign(&mut self, rhs: $name<O>) { + let rhs_native: $native = rhs.get(); + *self = core::ops::$trait::$method(*self, rhs_native); + } + } + + impl<O: ByteOrder> core::ops::$trait_assign<$native> for $name<O> { + #[inline(always)] + fn $method_assign(&mut self, rhs: $native) { + *self = core::ops::$trait::$method(*self, rhs); + } + } + }; + // Implement traits in terms of the same trait on the native type, but + // without performing a byte order swap when both operands are byteorder + // types. This only works for bitwise operations like `&`, `|`, etc. + // + // When only one operand is a byteorder type, we still need to perform a + // byteorder swap. + (@without_byteorder_swap $name:ident, $native:ident, $trait:ident, $method:ident, $trait_assign:ident, $method_assign:ident) => { + impl<O: ByteOrder> core::ops::$trait<$name<O>> for $name<O> { + type Output = $name<O>; + + #[inline(always)] + fn $method(self, rhs: $name<O>) -> $name<O> { + let self_native = $native::from_ne_bytes(self.0); + let rhs_native = $native::from_ne_bytes(rhs.0); + let result_native = core::ops::$trait::$method(self_native, rhs_native); + $name(result_native.to_ne_bytes(), PhantomData) + } + } + + impl<O: ByteOrder> core::ops::$trait<$name<O>> for $native { + type Output = $name<O>; + + #[inline(always)] + fn $method(self, rhs: $name<O>) -> $name<O> { + // No runtime cost - just byte packing + let rhs_native = $native::from_ne_bytes(rhs.0); + // (Maybe) runtime cost - byte order swap + let slf_byteorder = $name::<O>::new(self); + // No runtime cost - just byte packing + let slf_native = $native::from_ne_bytes(slf_byteorder.0); + // Runtime cost - perform the operation + let result_native = core::ops::$trait::$method(slf_native, rhs_native); + // No runtime cost - just byte unpacking + $name(result_native.to_ne_bytes(), PhantomData) + } + } + + impl<O: ByteOrder> core::ops::$trait<$native> for $name<O> { + type Output = $name<O>; + + #[inline(always)] + fn $method(self, rhs: $native) -> $name<O> { + // (Maybe) runtime cost - byte order swap + let rhs_byteorder = $name::<O>::new(rhs); + // No runtime cost - just byte packing + let rhs_native = $native::from_ne_bytes(rhs_byteorder.0); + // No runtime cost - just byte packing + let slf_native = $native::from_ne_bytes(self.0); + // Runtime cost - perform the operation + let result_native = core::ops::$trait::$method(slf_native, rhs_native); + // No runtime cost - just byte unpacking + $name(result_native.to_ne_bytes(), PhantomData) + } + } + + impl<O: ByteOrder> core::ops::$trait_assign<$name<O>> for $name<O> { + #[inline(always)] + fn $method_assign(&mut self, rhs: $name<O>) { + *self = core::ops::$trait::$method(*self, rhs); + } + } + + impl<O: ByteOrder> core::ops::$trait_assign<$name<O>> for $native { + #[inline(always)] + fn $method_assign(&mut self, rhs: $name<O>) { + // (Maybe) runtime cost - byte order swap + let rhs_native = rhs.get(); + // Runtime cost - perform the operation + *self = core::ops::$trait::$method(*self, rhs_native); + } + } + + impl<O: ByteOrder> core::ops::$trait_assign<$native> for $name<O> { + #[inline(always)] + fn $method_assign(&mut self, rhs: $native) { + *self = core::ops::$trait::$method(*self, rhs); + } + } + }; +} + +macro_rules! doc_comment { + ($x:expr, $($tt:tt)*) => { + #[doc = $x] + $($tt)* + }; +} + +macro_rules! define_max_value_constant { + ($name:ident, $bytes:expr, "unsigned integer") => { + /// The maximum value. + /// + /// This constant should be preferred to constructing a new value using + /// `new`, as `new` may perform an endianness swap depending on the + /// endianness `O` and the endianness of the platform. + pub const MAX_VALUE: $name<O> = $name([0xFFu8; $bytes], PhantomData); + }; + // We don't provide maximum and minimum value constants for signed values + // and floats because there's no way to do it generically - it would require + // a different value depending on the value of the `ByteOrder` type + // parameter. Currently, one workaround would be to provide implementations + // for concrete implementations of that trait. In the long term, if we are + // ever able to make the `new` constructor a const fn, we could use that + // instead. + ($name:ident, $bytes:expr, "signed integer") => {}; + ($name:ident, $bytes:expr, "floating point number") => {}; +} + +macro_rules! define_type { + ( + $article:ident, + $description:expr, + $name:ident, + $native:ident, + $bits:expr, + $bytes:expr, + $from_be_fn:path, + $to_be_fn:path, + $from_le_fn:path, + $to_le_fn:path, + $number_kind:tt, + [$($larger_native:ty),*], + [$($larger_native_try:ty),*], + [$($larger_byteorder:ident),*], + [$($larger_byteorder_try:ident),*] + ) => { + doc_comment! { + concat!($description, " stored in a given byte order. + +`", stringify!($name), "` is like the native `", stringify!($native), "` type with +two major differences: First, it has no alignment requirement (its alignment is 1). +Second, the endianness of its memory layout is given by the type parameter `O`, +which can be any type which implements [`ByteOrder`]. In particular, this refers +to [`BigEndian`], [`LittleEndian`], [`NativeEndian`], and [`NetworkEndian`]. + +", stringify!($article), " `", stringify!($name), "` can be constructed using +the [`new`] method, and its contained value can be obtained as a native +`",stringify!($native), "` using the [`get`] method, or updated in place with +the [`set`] method. In all cases, if the endianness `O` is not the same as the +endianness of the current platform, an endianness swap will be performed in +order to uphold the invariants that a) the layout of `", stringify!($name), "` +has endianness `O` and that, b) the layout of `", stringify!($native), "` has +the platform's native endianness. + +`", stringify!($name), "` implements [`FromBytes`], [`IntoBytes`], and [`Unaligned`], +making it useful for parsing and serialization. See the module documentation for an +example of how it can be used for parsing UDP packets. + +[`new`]: crate::byteorder::", stringify!($name), "::new +[`get`]: crate::byteorder::", stringify!($name), "::get +[`set`]: crate::byteorder::", stringify!($name), "::set +[`FromBytes`]: crate::FromBytes +[`IntoBytes`]: crate::IntoBytes +[`Unaligned`]: crate::Unaligned"), + #[derive(Copy, Clone, Eq, PartialEq, Hash)] + #[cfg_attr(any(feature = "derive", test), derive(KnownLayout, Immutable, FromBytes, IntoBytes, Unaligned))] + #[repr(transparent)] + pub struct $name<O>([u8; $bytes], PhantomData<O>); + } + + #[cfg(not(any(feature = "derive", test)))] + impl_known_layout!(O => $name<O>); + + #[allow(unused_unsafe)] // Unused when `feature = "derive"`. + // SAFETY: `$name<O>` is `repr(transparent)`, and so it has the same + // layout as its only non-zero field, which is a `u8` array. `u8` arrays + // are `Immutable`, `TryFromBytes`, `FromZeros`, `FromBytes`, + // `IntoBytes`, and `Unaligned`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + impl_or_verify!(O => Immutable for $name<O>); + impl_or_verify!(O => TryFromBytes for $name<O>); + impl_or_verify!(O => FromZeros for $name<O>); + impl_or_verify!(O => FromBytes for $name<O>); + impl_or_verify!(O => IntoBytes for $name<O>); + impl_or_verify!(O => Unaligned for $name<O>); + }; + + impl<O> Default for $name<O> { + #[inline(always)] + fn default() -> $name<O> { + $name::ZERO + } + } + + impl<O> $name<O> { + /// The value zero. + /// + /// This constant should be preferred to constructing a new value + /// using `new`, as `new` may perform an endianness swap depending + /// on the endianness and platform. + pub const ZERO: $name<O> = $name([0u8; $bytes], PhantomData); + + define_max_value_constant!($name, $bytes, $number_kind); + + /// Constructs a new value from bytes which are already in `O` byte + /// order. + #[must_use = "has no side effects"] + #[inline(always)] + pub const fn from_bytes(bytes: [u8; $bytes]) -> $name<O> { + $name(bytes, PhantomData) + } + + /// Extracts the bytes of `self` without swapping the byte order. + /// + /// The returned bytes will be in `O` byte order. + #[must_use = "has no side effects"] + #[inline(always)] + pub const fn to_bytes(self) -> [u8; $bytes] { + self.0 + } + } + + impl<O: ByteOrder> $name<O> { + maybe_const_trait_bounded_fn! { + /// Constructs a new value, possibly performing an endianness + /// swap to guarantee that the returned value has endianness + /// `O`. + #[must_use = "has no side effects"] + #[inline(always)] + pub const fn new(n: $native) -> $name<O> { + let bytes = match O::ORDER { + Order::BigEndian => $to_be_fn(n), + Order::LittleEndian => $to_le_fn(n), + }; + + $name(bytes, PhantomData) + } + } + + maybe_const_trait_bounded_fn! { + /// Returns the value as a primitive type, possibly performing + /// an endianness swap to guarantee that the return value has + /// the endianness of the native platform. + #[must_use = "has no side effects"] + #[inline(always)] + pub const fn get(self) -> $native { + match O::ORDER { + Order::BigEndian => $from_be_fn(self.0), + Order::LittleEndian => $from_le_fn(self.0), + } + } + } + + /// Updates the value in place as a primitive type, possibly + /// performing an endianness swap to guarantee that the stored value + /// has the endianness `O`. + #[inline(always)] + pub fn set(&mut self, n: $native) { + *self = Self::new(n); + } + } + + // The reasoning behind which traits to implement here is to only + // implement traits which won't cause inference issues. Notably, + // comparison traits like PartialEq and PartialOrd tend to cause + // inference issues. + + impl<O: ByteOrder> From<$name<O>> for [u8; $bytes] { + #[inline(always)] + fn from(x: $name<O>) -> [u8; $bytes] { + x.0 + } + } + + impl<O: ByteOrder> From<[u8; $bytes]> for $name<O> { + #[inline(always)] + fn from(bytes: [u8; $bytes]) -> $name<O> { + $name(bytes, PhantomData) + } + } + + impl<O: ByteOrder> From<$name<O>> for $native { + #[inline(always)] + fn from(x: $name<O>) -> $native { + x.get() + } + } + + impl<O: ByteOrder> From<$native> for $name<O> { + #[inline(always)] + fn from(x: $native) -> $name<O> { + $name::new(x) + } + } + + $( + impl<O: ByteOrder> From<$name<O>> for $larger_native { + #[inline(always)] + fn from(x: $name<O>) -> $larger_native { + x.get().into() + } + } + )* + + $( + impl<O: ByteOrder> TryFrom<$larger_native_try> for $name<O> { + type Error = TryFromIntError; + #[inline(always)] + fn try_from(x: $larger_native_try) -> Result<$name<O>, TryFromIntError> { + $native::try_from(x).map($name::new) + } + } + )* + + $( + impl<O: ByteOrder, P: ByteOrder> From<$name<O>> for $larger_byteorder<P> { + #[inline(always)] + fn from(x: $name<O>) -> $larger_byteorder<P> { + $larger_byteorder::new(x.get().into()) + } + } + )* + + $( + impl<O: ByteOrder, P: ByteOrder> TryFrom<$larger_byteorder_try<P>> for $name<O> { + type Error = TryFromIntError; + #[inline(always)] + fn try_from(x: $larger_byteorder_try<P>) -> Result<$name<O>, TryFromIntError> { + x.get().try_into().map($name::new) + } + } + )* + + impl<O> AsRef<[u8; $bytes]> for $name<O> { + #[inline(always)] + fn as_ref(&self) -> &[u8; $bytes] { + &self.0 + } + } + + impl<O> AsMut<[u8; $bytes]> for $name<O> { + #[inline(always)] + fn as_mut(&mut self) -> &mut [u8; $bytes] { + &mut self.0 + } + } + + impl<O> PartialEq<$name<O>> for [u8; $bytes] { + #[inline(always)] + fn eq(&self, other: &$name<O>) -> bool { + self.eq(&other.0) + } + } + + impl<O> PartialEq<[u8; $bytes]> for $name<O> { + #[inline(always)] + fn eq(&self, other: &[u8; $bytes]) -> bool { + self.0.eq(other) + } + } + + impl<O: ByteOrder> PartialEq<$native> for $name<O> { + #[inline(always)] + fn eq(&self, other: &$native) -> bool { + self.get().eq(other) + } + } + + impl_dbg_traits!($name, $native, $number_kind); + impl_fmt_traits!($name, $native, $number_kind); + impl_ops_traits!($name, $native, $number_kind); + }; +} + +define_type!( + A, + "A 16-bit unsigned integer", + U16, + u16, + 16, + 2, + u16::from_be_bytes, + u16::to_be_bytes, + u16::from_le_bytes, + u16::to_le_bytes, + "unsigned integer", + [u32, u64, u128, usize], + [u32, u64, u128, usize], + [U32, U64, U128, Usize], + [U32, U64, U128, Usize] +); +define_type!( + A, + "A 32-bit unsigned integer", + U32, + u32, + 32, + 4, + u32::from_be_bytes, + u32::to_be_bytes, + u32::from_le_bytes, + u32::to_le_bytes, + "unsigned integer", + [u64, u128], + [u64, u128], + [U64, U128], + [U64, U128] +); +define_type!( + A, + "A 64-bit unsigned integer", + U64, + u64, + 64, + 8, + u64::from_be_bytes, + u64::to_be_bytes, + u64::from_le_bytes, + u64::to_le_bytes, + "unsigned integer", + [u128], + [u128], + [U128], + [U128] +); +define_type!( + A, + "A 128-bit unsigned integer", + U128, + u128, + 128, + 16, + u128::from_be_bytes, + u128::to_be_bytes, + u128::from_le_bytes, + u128::to_le_bytes, + "unsigned integer", + [], + [], + [], + [] +); +define_type!( + A, + "A word-sized unsigned integer", + Usize, + usize, + mem::size_of::<usize>() * 8, + mem::size_of::<usize>(), + usize::from_be_bytes, + usize::to_be_bytes, + usize::from_le_bytes, + usize::to_le_bytes, + "unsigned integer", + [], + [], + [], + [] +); +define_type!( + An, + "A 16-bit signed integer", + I16, + i16, + 16, + 2, + i16::from_be_bytes, + i16::to_be_bytes, + i16::from_le_bytes, + i16::to_le_bytes, + "signed integer", + [i32, i64, i128, isize], + [i32, i64, i128, isize], + [I32, I64, I128, Isize], + [I32, I64, I128, Isize] +); +define_type!( + An, + "A 32-bit signed integer", + I32, + i32, + 32, + 4, + i32::from_be_bytes, + i32::to_be_bytes, + i32::from_le_bytes, + i32::to_le_bytes, + "signed integer", + [i64, i128], + [i64, i128], + [I64, I128], + [I64, I128] +); +define_type!( + An, + "A 64-bit signed integer", + I64, + i64, + 64, + 8, + i64::from_be_bytes, + i64::to_be_bytes, + i64::from_le_bytes, + i64::to_le_bytes, + "signed integer", + [i128], + [i128], + [I128], + [I128] +); +define_type!( + An, + "A 128-bit signed integer", + I128, + i128, + 128, + 16, + i128::from_be_bytes, + i128::to_be_bytes, + i128::from_le_bytes, + i128::to_le_bytes, + "signed integer", + [], + [], + [], + [] +); +define_type!( + An, + "A word-sized signed integer", + Isize, + isize, + mem::size_of::<isize>() * 8, + mem::size_of::<isize>(), + isize::from_be_bytes, + isize::to_be_bytes, + isize::from_le_bytes, + isize::to_le_bytes, + "signed integer", + [], + [], + [], + [] +); + +// FIXME(https://github.com/rust-lang/rust/issues/72447): Use the endianness +// conversion methods directly once those are const-stable. +macro_rules! define_float_conversion { + ($ty:ty, $bits:ident, $bytes:expr, $mod:ident) => { + mod $mod { + use super::*; + + define_float_conversion!($ty, $bits, $bytes, from_be_bytes, to_be_bytes); + define_float_conversion!($ty, $bits, $bytes, from_le_bytes, to_le_bytes); + } + }; + ($ty:ty, $bits:ident, $bytes:expr, $from:ident, $to:ident) => { + // Clippy: The suggestion of using `from_bits()` instead doesn't work + // because `from_bits` is not const-stable on our MSRV. + #[allow(clippy::unnecessary_transmutes)] + pub(crate) const fn $from(bytes: [u8; $bytes]) -> $ty { + transmute!($bits::$from(bytes)) + } + + pub(crate) const fn $to(f: $ty) -> [u8; $bytes] { + // Clippy: The suggestion of using `f.to_bits()` instead doesn't + // work because `to_bits` is not const-stable on our MSRV. + #[allow(clippy::unnecessary_transmutes)] + let bits: $bits = transmute!(f); + bits.$to() + } + }; +} + +define_float_conversion!(f32, u32, 4, f32_ext); +define_float_conversion!(f64, u64, 8, f64_ext); + +define_type!( + An, + "A 32-bit floating point number", + F32, + f32, + 32, + 4, + f32_ext::from_be_bytes, + f32_ext::to_be_bytes, + f32_ext::from_le_bytes, + f32_ext::to_le_bytes, + "floating point number", + [f64], + [], + [F64], + [] +); +define_type!( + An, + "A 64-bit floating point number", + F64, + f64, + 64, + 8, + f64_ext::from_be_bytes, + f64_ext::to_be_bytes, + f64_ext::from_le_bytes, + f64_ext::to_le_bytes, + "floating point number", + [], + [], + [], + [] +); + +macro_rules! module { + ($name:ident, $trait:ident, $endianness_str:expr) => { + /// Numeric primitives stored in + #[doc = $endianness_str] + /// byte order. + pub mod $name { + use super::$trait; + + module!(@ty U16, $trait, "16-bit unsigned integer", $endianness_str); + module!(@ty U32, $trait, "32-bit unsigned integer", $endianness_str); + module!(@ty U64, $trait, "64-bit unsigned integer", $endianness_str); + module!(@ty U128, $trait, "128-bit unsigned integer", $endianness_str); + module!(@ty I16, $trait, "16-bit signed integer", $endianness_str); + module!(@ty I32, $trait, "32-bit signed integer", $endianness_str); + module!(@ty I64, $trait, "64-bit signed integer", $endianness_str); + module!(@ty I128, $trait, "128-bit signed integer", $endianness_str); + module!(@ty F32, $trait, "32-bit floating point number", $endianness_str); + module!(@ty F64, $trait, "64-bit floating point number", $endianness_str); + } + }; + (@ty $ty:ident, $trait:ident, $desc_str:expr, $endianness_str:expr) => { + /// A + #[doc = $desc_str] + /// stored in + #[doc = $endianness_str] + /// byte order. + pub type $ty = crate::byteorder::$ty<$trait>; + }; +} + +module!(big_endian, BigEndian, "big-endian"); +module!(little_endian, LittleEndian, "little-endian"); +module!(network_endian, NetworkEndian, "network-endian"); +module!(native_endian, NativeEndian, "native-endian"); + +#[cfg(any(test, kani))] +mod tests { + use super::*; + + #[cfg(not(kani))] + mod compatibility { + pub(super) use rand::{ + distributions::{Distribution, Standard}, + rngs::SmallRng, + Rng, SeedableRng, + }; + + pub(crate) trait Arbitrary {} + + impl<T> Arbitrary for T {} + } + + #[cfg(kani)] + mod compatibility { + pub(crate) use kani::Arbitrary; + + pub(crate) struct SmallRng; + + impl SmallRng { + pub(crate) fn seed_from_u64(_state: u64) -> Self { + Self + } + } + + pub(crate) trait Rng { + fn sample<T, D: Distribution<T>>(&mut self, _distr: D) -> T + where + T: Arbitrary, + { + kani::any() + } + } + + impl Rng for SmallRng {} + + pub(crate) trait Distribution<T> {} + impl<T, U> Distribution<T> for U {} + + pub(crate) struct Standard; + } + + use compatibility::*; + + // A native integer type (u16, i32, etc). + trait Native: Arbitrary + FromBytes + IntoBytes + Immutable + Copy + PartialEq + Debug { + const ZERO: Self; + const MAX_VALUE: Self; + + type Distribution: Distribution<Self>; + const DIST: Self::Distribution; + + fn rand<R: Rng>(rng: &mut R) -> Self { + rng.sample(Self::DIST) + } + + #[cfg_attr(kani, allow(unused))] + fn checked_add(self, rhs: Self) -> Option<Self>; + + #[cfg_attr(kani, allow(unused))] + fn checked_div(self, rhs: Self) -> Option<Self>; + + #[cfg_attr(kani, allow(unused))] + fn checked_mul(self, rhs: Self) -> Option<Self>; + + #[cfg_attr(kani, allow(unused))] + fn checked_rem(self, rhs: Self) -> Option<Self>; + + #[cfg_attr(kani, allow(unused))] + fn checked_sub(self, rhs: Self) -> Option<Self>; + + #[cfg_attr(kani, allow(unused))] + fn checked_shl(self, rhs: Self) -> Option<Self>; + + #[cfg_attr(kani, allow(unused))] + fn checked_shr(self, rhs: Self) -> Option<Self>; + + fn is_nan(self) -> bool; + + /// For `f32` and `f64`, NaN values are not considered equal to + /// themselves. This method is like `assert_eq!`, but it treats NaN + /// values as equal. + fn assert_eq_or_nan(self, other: Self) { + let slf = (!self.is_nan()).then(|| self); + let other = (!other.is_nan()).then(|| other); + assert_eq!(slf, other); + } + } + + trait ByteArray: + FromBytes + IntoBytes + Immutable + Copy + AsRef<[u8]> + AsMut<[u8]> + Debug + Default + Eq + { + /// Invert the order of the bytes in the array. + fn invert(self) -> Self; + } + + trait ByteOrderType: + FromBytes + IntoBytes + Unaligned + Copy + Eq + Debug + Hash + From<Self::Native> + { + type Native: Native; + type ByteArray: ByteArray; + + const ZERO: Self; + + fn new(native: Self::Native) -> Self; + fn get(self) -> Self::Native; + fn set(&mut self, native: Self::Native); + fn from_bytes(bytes: Self::ByteArray) -> Self; + fn into_bytes(self) -> Self::ByteArray; + + /// For `f32` and `f64`, NaN values are not considered equal to + /// themselves. This method is like `assert_eq!`, but it treats NaN + /// values as equal. + fn assert_eq_or_nan(self, other: Self) { + let slf = (!self.get().is_nan()).then(|| self); + let other = (!other.get().is_nan()).then(|| other); + assert_eq!(slf, other); + } + } + + trait ByteOrderTypeUnsigned: ByteOrderType { + const MAX_VALUE: Self; + } + + macro_rules! impl_byte_array { + ($bytes:expr) => { + impl ByteArray for [u8; $bytes] { + fn invert(mut self) -> [u8; $bytes] { + self.reverse(); + self + } + } + }; + } + + impl_byte_array!(2); + impl_byte_array!(4); + impl_byte_array!(8); + impl_byte_array!(16); + + macro_rules! impl_byte_order_type_unsigned { + ($name:ident, unsigned) => { + impl<O: ByteOrder> ByteOrderTypeUnsigned for $name<O> { + const MAX_VALUE: $name<O> = $name::MAX_VALUE; + } + }; + ($name:ident, signed) => {}; + } + + macro_rules! impl_traits { + ($name:ident, $native:ident, $sign:ident $(, @$float:ident)?) => { + impl Native for $native { + // For some types, `0 as $native` is required (for example, when + // `$native` is a floating-point type; `0` is an integer), but + // for other types, it's a trivial cast. In all cases, Clippy + // thinks it's dangerous. + #[allow(trivial_numeric_casts, clippy::as_conversions)] + const ZERO: $native = 0 as $native; + const MAX_VALUE: $native = $native::MAX; + + type Distribution = Standard; + const DIST: Standard = Standard; + + impl_traits!(@float_dependent_methods $(@$float)?); + } + + impl<O: ByteOrder> ByteOrderType for $name<O> { + type Native = $native; + type ByteArray = [u8; mem::size_of::<$native>()]; + + const ZERO: $name<O> = $name::ZERO; + + fn new(native: $native) -> $name<O> { + $name::new(native) + } + + fn get(self) -> $native { + $name::get(self) + } + + fn set(&mut self, native: $native) { + $name::set(self, native) + } + + fn from_bytes(bytes: [u8; mem::size_of::<$native>()]) -> $name<O> { + $name::from(bytes) + } + + fn into_bytes(self) -> [u8; mem::size_of::<$native>()] { + <[u8; mem::size_of::<$native>()]>::from(self) + } + } + + impl_byte_order_type_unsigned!($name, $sign); + }; + (@float_dependent_methods) => { + fn checked_add(self, rhs: Self) -> Option<Self> { self.checked_add(rhs) } + fn checked_div(self, rhs: Self) -> Option<Self> { self.checked_div(rhs) } + fn checked_mul(self, rhs: Self) -> Option<Self> { self.checked_mul(rhs) } + fn checked_rem(self, rhs: Self) -> Option<Self> { self.checked_rem(rhs) } + fn checked_sub(self, rhs: Self) -> Option<Self> { self.checked_sub(rhs) } + fn checked_shl(self, rhs: Self) -> Option<Self> { self.checked_shl(rhs.try_into().unwrap_or(u32::MAX)) } + fn checked_shr(self, rhs: Self) -> Option<Self> { self.checked_shr(rhs.try_into().unwrap_or(u32::MAX)) } + fn is_nan(self) -> bool { false } + }; + (@float_dependent_methods @float) => { + fn checked_add(self, rhs: Self) -> Option<Self> { Some(self + rhs) } + fn checked_div(self, rhs: Self) -> Option<Self> { Some(self / rhs) } + fn checked_mul(self, rhs: Self) -> Option<Self> { Some(self * rhs) } + fn checked_rem(self, rhs: Self) -> Option<Self> { Some(self % rhs) } + fn checked_sub(self, rhs: Self) -> Option<Self> { Some(self - rhs) } + fn checked_shl(self, _rhs: Self) -> Option<Self> { unimplemented!() } + fn checked_shr(self, _rhs: Self) -> Option<Self> { unimplemented!() } + fn is_nan(self) -> bool { self.is_nan() } + }; + } + + impl_traits!(U16, u16, unsigned); + impl_traits!(U32, u32, unsigned); + impl_traits!(U64, u64, unsigned); + impl_traits!(U128, u128, unsigned); + impl_traits!(Usize, usize, unsigned); + impl_traits!(I16, i16, signed); + impl_traits!(I32, i32, signed); + impl_traits!(I64, i64, signed); + impl_traits!(I128, i128, signed); + impl_traits!(Isize, isize, unsigned); + impl_traits!(F32, f32, signed, @float); + impl_traits!(F64, f64, signed, @float); + + macro_rules! call_for_unsigned_types { + ($fn:ident, $byteorder:ident) => { + $fn::<U16<$byteorder>>(); + $fn::<U32<$byteorder>>(); + $fn::<U64<$byteorder>>(); + $fn::<U128<$byteorder>>(); + $fn::<Usize<$byteorder>>(); + }; + } + + macro_rules! call_for_signed_types { + ($fn:ident, $byteorder:ident) => { + $fn::<I16<$byteorder>>(); + $fn::<I32<$byteorder>>(); + $fn::<I64<$byteorder>>(); + $fn::<I128<$byteorder>>(); + $fn::<Isize<$byteorder>>(); + }; + } + + macro_rules! call_for_float_types { + ($fn:ident, $byteorder:ident) => { + $fn::<F32<$byteorder>>(); + $fn::<F64<$byteorder>>(); + }; + } + + macro_rules! call_for_all_types { + ($fn:ident, $byteorder:ident) => { + call_for_unsigned_types!($fn, $byteorder); + call_for_signed_types!($fn, $byteorder); + call_for_float_types!($fn, $byteorder); + }; + } + + #[cfg(target_endian = "big")] + type NonNativeEndian = LittleEndian; + #[cfg(target_endian = "little")] + type NonNativeEndian = BigEndian; + + // We use a `u64` seed so that we can use `SeedableRng::seed_from_u64`. + // `SmallRng`'s `SeedableRng::Seed` differs by platform, so if we wanted to + // call `SeedableRng::from_seed`, which takes a `Seed`, we would need + // conditional compilation by `target_pointer_width`. + const RNG_SEED: u64 = 0x7A03CAE2F32B5B8F; + + const RAND_ITERS: usize = if cfg!(any(miri, kani)) { + // The tests below which use this constant used to take a very long time + // on Miri, which slows down local development and CI jobs. We're not + // using Miri to check for the correctness of our code, but rather its + // soundness, and at least in the context of these particular tests, a + // single loop iteration is just as good for surfacing UB as multiple + // iterations are. + // + // As of the writing of this comment, here's one set of measurements: + // + // $ # RAND_ITERS == 1 + // $ cargo miri test -- -Z unstable-options --report-time endian + // test byteorder::tests::test_native_endian ... ok <0.049s> + // test byteorder::tests::test_non_native_endian ... ok <0.061s> + // + // $ # RAND_ITERS == 1024 + // $ cargo miri test -- -Z unstable-options --report-time endian + // test byteorder::tests::test_native_endian ... ok <25.716s> + // test byteorder::tests::test_non_native_endian ... ok <38.127s> + 1 + } else { + 1024 + }; + + #[test] + fn test_const_methods() { + use big_endian::*; + + #[rustversion::since(1.61.0)] + const _U: U16 = U16::new(0); + #[rustversion::since(1.61.0)] + const _NATIVE: u16 = _U.get(); + const _FROM_BYTES: U16 = U16::from_bytes([0, 1]); + const _BYTES: [u8; 2] = _FROM_BYTES.to_bytes(); + } + + #[cfg_attr(test, test)] + #[cfg_attr(kani, kani::proof)] + fn test_zero() { + fn test_zero<T: ByteOrderType>() { + assert_eq!(T::ZERO.get(), T::Native::ZERO); + } + + call_for_all_types!(test_zero, NativeEndian); + call_for_all_types!(test_zero, NonNativeEndian); + } + + #[cfg_attr(test, test)] + #[cfg_attr(kani, kani::proof)] + fn test_max_value() { + fn test_max_value<T: ByteOrderTypeUnsigned>() { + assert_eq!(T::MAX_VALUE.get(), T::Native::MAX_VALUE); + } + + call_for_unsigned_types!(test_max_value, NativeEndian); + call_for_unsigned_types!(test_max_value, NonNativeEndian); + } + + #[cfg_attr(test, test)] + #[cfg_attr(kani, kani::proof)] + fn test_endian() { + fn test<T: ByteOrderType>(invert: bool) { + let mut r = SmallRng::seed_from_u64(RNG_SEED); + for _ in 0..RAND_ITERS { + let native = T::Native::rand(&mut r); + let mut bytes = T::ByteArray::default(); + bytes.as_mut_bytes().copy_from_slice(native.as_bytes()); + if invert { + bytes = bytes.invert(); + } + let mut from_native = T::new(native); + let from_bytes = T::from_bytes(bytes); + + from_native.assert_eq_or_nan(from_bytes); + from_native.get().assert_eq_or_nan(native); + from_bytes.get().assert_eq_or_nan(native); + + assert_eq!(from_native.into_bytes(), bytes); + assert_eq!(from_bytes.into_bytes(), bytes); + + let updated = T::Native::rand(&mut r); + from_native.set(updated); + from_native.get().assert_eq_or_nan(updated); + } + } + + fn test_native<T: ByteOrderType>() { + test::<T>(false); + } + + fn test_non_native<T: ByteOrderType>() { + test::<T>(true); + } + + call_for_all_types!(test_native, NativeEndian); + call_for_all_types!(test_non_native, NonNativeEndian); + } + + #[test] + fn test_ops_impls() { + // Test implementations of traits in `core::ops`. Some of these are + // fairly banal, but some are optimized to perform the operation without + // swapping byte order (namely, bit-wise operations which are identical + // regardless of byte order). These are important to test, and while + // we're testing those anyway, it's trivial to test all of the impls. + + fn test<T, FTT, FTN, FNT, FNN, FNNChecked, FATT, FATN, FANT>( + op_t_t: FTT, + op_t_n: FTN, + op_n_t: FNT, + op_n_n: FNN, + op_n_n_checked: Option<FNNChecked>, + op_assign: Option<(FATT, FATN, FANT)>, + ) where + T: ByteOrderType, + FTT: Fn(T, T) -> T, + FTN: Fn(T, T::Native) -> T, + FNT: Fn(T::Native, T) -> T, + FNN: Fn(T::Native, T::Native) -> T::Native, + FNNChecked: Fn(T::Native, T::Native) -> Option<T::Native>, + FATT: Fn(&mut T, T), + FATN: Fn(&mut T, T::Native), + FANT: Fn(&mut T::Native, T), + { + let mut r = SmallRng::seed_from_u64(RNG_SEED); + for _ in 0..RAND_ITERS { + let n0 = T::Native::rand(&mut r); + let n1 = T::Native::rand(&mut r); + let t0 = T::new(n0); + let t1 = T::new(n1); + + // If this operation would overflow/underflow, skip it rather + // than attempt to catch and recover from panics. + if matches!(&op_n_n_checked, Some(checked) if checked(n0, n1).is_none()) { + continue; + } + + let t_t_res = op_t_t(t0, t1); + let t_n_res = op_t_n(t0, n1); + let n_t_res = op_n_t(n0, t1); + let n_n_res = op_n_n(n0, n1); + + // For `f32` and `f64`, NaN values are not considered equal to + // themselves. We store `Option<f32>`/`Option<f64>` and store + // NaN as `None` so they can still be compared. + let val_or_none = |t: T| (!T::Native::is_nan(t.get())).then(|| t.get()); + let t_t_res = val_or_none(t_t_res); + let t_n_res = val_or_none(t_n_res); + let n_t_res = val_or_none(n_t_res); + let n_n_res = (!T::Native::is_nan(n_n_res)).then(|| n_n_res); + assert_eq!(t_t_res, n_n_res); + assert_eq!(t_n_res, n_n_res); + assert_eq!(n_t_res, n_n_res); + + if let Some((op_assign_t_t, op_assign_t_n, op_assign_n_t)) = &op_assign { + let mut t_t_res = t0; + op_assign_t_t(&mut t_t_res, t1); + let mut t_n_res = t0; + op_assign_t_n(&mut t_n_res, n1); + let mut n_t_res = n0; + op_assign_n_t(&mut n_t_res, t1); + + // For `f32` and `f64`, NaN values are not considered equal to + // themselves. We store `Option<f32>`/`Option<f64>` and store + // NaN as `None` so they can still be compared. + let t_t_res = val_or_none(t_t_res); + let t_n_res = val_or_none(t_n_res); + let n_t_res = (!T::Native::is_nan(n_t_res)).then(|| n_t_res); + assert_eq!(t_t_res, n_n_res); + assert_eq!(t_n_res, n_n_res); + assert_eq!(n_t_res, n_n_res); + } + } + } + + macro_rules! test { + ( + @binary + $trait:ident, + $method:ident $([$checked_method:ident])?, + $trait_assign:ident, + $method_assign:ident, + $($call_for_macros:ident),* + ) => {{ + fn t<T>() + where + T: ByteOrderType, + T: core::ops::$trait<T, Output = T>, + T: core::ops::$trait<T::Native, Output = T>, + T::Native: core::ops::$trait<T, Output = T>, + T::Native: core::ops::$trait<T::Native, Output = T::Native>, + + T: core::ops::$trait_assign<T>, + T: core::ops::$trait_assign<T::Native>, + T::Native: core::ops::$trait_assign<T>, + T::Native: core::ops::$trait_assign<T::Native>, + { + test::<T, _, _, _, _, _, _, _, _>( + core::ops::$trait::$method, + core::ops::$trait::$method, + core::ops::$trait::$method, + core::ops::$trait::$method, + { + #[allow(unused_mut, unused_assignments)] + let mut op_native_checked = None::<fn(T::Native, T::Native) -> Option<T::Native>>; + $( + op_native_checked = Some(T::Native::$checked_method); + )? + op_native_checked + }, + Some(( + <T as core::ops::$trait_assign<T>>::$method_assign, + <T as core::ops::$trait_assign::<T::Native>>::$method_assign, + <T::Native as core::ops::$trait_assign::<T>>::$method_assign + )), + ); + } + + $( + $call_for_macros!(t, NativeEndian); + $call_for_macros!(t, NonNativeEndian); + )* + }}; + ( + @unary + $trait:ident, + $method:ident, + $($call_for_macros:ident),* + ) => {{ + fn t<T>() + where + T: ByteOrderType, + T: core::ops::$trait<Output = T>, + T::Native: core::ops::$trait<Output = T::Native>, + { + test::<T, _, _, _, _, _, _, _, _>( + |slf, _rhs| core::ops::$trait::$method(slf), + |slf, _rhs| core::ops::$trait::$method(slf), + |slf, _rhs| core::ops::$trait::$method(slf).into(), + |slf, _rhs| core::ops::$trait::$method(slf), + None::<fn(T::Native, T::Native) -> Option<T::Native>>, + None::<(fn(&mut T, T), fn(&mut T, T::Native), fn(&mut T::Native, T))>, + ); + } + + $( + $call_for_macros!(t, NativeEndian); + $call_for_macros!(t, NonNativeEndian); + )* + }}; + } + + test!(@binary Add, add[checked_add], AddAssign, add_assign, call_for_all_types); + test!(@binary Div, div[checked_div], DivAssign, div_assign, call_for_all_types); + test!(@binary Mul, mul[checked_mul], MulAssign, mul_assign, call_for_all_types); + test!(@binary Rem, rem[checked_rem], RemAssign, rem_assign, call_for_all_types); + test!(@binary Sub, sub[checked_sub], SubAssign, sub_assign, call_for_all_types); + + test!(@binary BitAnd, bitand, BitAndAssign, bitand_assign, call_for_unsigned_types, call_for_signed_types); + test!(@binary BitOr, bitor, BitOrAssign, bitor_assign, call_for_unsigned_types, call_for_signed_types); + test!(@binary BitXor, bitxor, BitXorAssign, bitxor_assign, call_for_unsigned_types, call_for_signed_types); + test!(@binary Shl, shl[checked_shl], ShlAssign, shl_assign, call_for_unsigned_types, call_for_signed_types); + test!(@binary Shr, shr[checked_shr], ShrAssign, shr_assign, call_for_unsigned_types, call_for_signed_types); + + test!(@unary Not, not, call_for_signed_types, call_for_unsigned_types); + test!(@unary Neg, neg, call_for_signed_types, call_for_float_types); + } + + #[test] + fn test_debug_impl() { + // Ensure that Debug applies format options to the inner value. + let val = U16::<LE>::new(10); + assert_eq!(format!("{:?}", val), "U16(10)"); + assert_eq!(format!("{:03?}", val), "U16(010)"); + assert_eq!(format!("{:x?}", val), "U16(a)"); + } + + #[test] + fn test_byteorder_traits_coverage() { + let val_be = U16::<BigEndian>::from_bytes([0, 1]); + let val_le = U16::<LittleEndian>::from_bytes([1, 0]); + + assert_eq!(val_be.get(), 1); + assert_eq!(val_le.get(), 1); + + // Debug + assert_eq!(format!("{:?}", val_be), "U16(1)"); + assert_eq!(format!("{:?}", val_le), "U16(1)"); + + // PartialOrd, Ord with same type + assert!(val_be >= val_be); + assert!(val_be <= val_be); + assert_eq!(val_be.cmp(&val_be), core::cmp::Ordering::Equal); + + // PartialOrd with native + assert!(val_be == 1u16); + assert!(val_be >= 1u16); + + // Default + let default_be: U16<BigEndian> = Default::default(); + assert_eq!(default_be.get(), 0); + + // I16 + let val_be_i16 = I16::<BigEndian>::from_bytes([0, 1]); + assert_eq!(val_be_i16.get(), 1); + assert_eq!(format!("{:?}", val_be_i16), "I16(1)"); + assert_eq!(val_be_i16.cmp(&val_be_i16), core::cmp::Ordering::Equal); + } +} diff --git a/rust/zerocopy/src/deprecated.rs b/rust/zerocopy/src/deprecated.rs new file mode 100644 index 000000000000..59ddd35c77c6 --- /dev/null +++ b/rust/zerocopy/src/deprecated.rs @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License <LICENSE-BSD or +// https://opensource.org/license/bsd-2-clause>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Deprecated items. These are kept separate so that they don't clutter up +//! other modules. + +use super::*; + +impl<B, T> Ref<B, T> +where + B: ByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + #[deprecated(since = "0.8.0", note = "renamed to `Ref::from_bytes`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new(bytes: B) -> Option<Ref<B, T>> { + Self::from_bytes(bytes).ok() + } +} + +impl<B, T> Ref<B, T> +where + B: SplitByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + #[deprecated(since = "0.8.0", note = "renamed to `Ref::from_prefix`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_from_prefix(bytes: B) -> Option<(Ref<B, T>, B)> { + Self::from_prefix(bytes).ok() + } +} + +impl<B, T> Ref<B, T> +where + B: SplitByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + #[deprecated(since = "0.8.0", note = "renamed to `Ref::from_suffix`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_from_suffix(bytes: B) -> Option<(B, Ref<B, T>)> { + Self::from_suffix(bytes).ok() + } +} + +impl<B, T> Ref<B, T> +where + B: ByteSlice, + T: Unaligned + KnownLayout + Immutable + ?Sized, +{ + #[deprecated( + since = "0.8.0", + note = "use `Ref::from_bytes`; for `T: Unaligned`, the returned `CastError` implements `Into<SizeError>`" + )] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_unaligned(bytes: B) -> Option<Ref<B, T>> { + Self::from_bytes(bytes).ok() + } +} + +impl<B, T> Ref<B, T> +where + B: SplitByteSlice, + T: Unaligned + KnownLayout + Immutable + ?Sized, +{ + #[deprecated( + since = "0.8.0", + note = "use `Ref::from_prefix`; for `T: Unaligned`, the returned `CastError` implements `Into<SizeError>`" + )] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_unaligned_from_prefix(bytes: B) -> Option<(Ref<B, T>, B)> { + Self::from_prefix(bytes).ok() + } +} + +impl<B, T> Ref<B, T> +where + B: SplitByteSlice, + T: Unaligned + KnownLayout + Immutable + ?Sized, +{ + #[deprecated( + since = "0.8.0", + note = "use `Ref::from_suffix`; for `T: Unaligned`, the returned `CastError` implements `Into<SizeError>`" + )] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_unaligned_from_suffix(bytes: B) -> Option<(B, Ref<B, T>)> { + Self::from_suffix(bytes).ok() + } +} + +impl<B, T> Ref<B, [T]> +where + B: ByteSlice, + T: Immutable, +{ + #[deprecated(since = "0.8.0", note = "`Ref::from_bytes` now supports slices")] + #[doc(hidden)] + #[inline(always)] + pub fn new_slice(bytes: B) -> Option<Ref<B, [T]>> { + Self::from_bytes(bytes).ok() + } +} + +impl<B, T> Ref<B, [T]> +where + B: ByteSlice, + T: Unaligned + Immutable, +{ + #[deprecated( + since = "0.8.0", + note = "`Ref::from_bytes` now supports slices; for `T: Unaligned`, the returned `CastError` implements `Into<SizeError>`" + )] + #[doc(hidden)] + #[inline(always)] + pub fn new_slice_unaligned(bytes: B) -> Option<Ref<B, [T]>> { + Ref::from_bytes(bytes).ok() + } +} + +impl<'a, B, T> Ref<B, [T]> +where + B: 'a + IntoByteSlice<'a>, + T: FromBytes + Immutable, +{ + #[deprecated(since = "0.8.0", note = "`Ref::into_ref` now supports slices")] + #[doc(hidden)] + #[inline(always)] + pub fn into_slice(self) -> &'a [T] { + Ref::into_ref(self) + } +} + +impl<'a, B, T> Ref<B, [T]> +where + B: 'a + IntoByteSliceMut<'a>, + T: FromBytes + IntoBytes + Immutable, +{ + #[deprecated(since = "0.8.0", note = "`Ref::into_mut` now supports slices")] + #[doc(hidden)] + #[inline(always)] + pub fn into_mut_slice(self) -> &'a mut [T] { + Ref::into_mut(self) + } +} + +impl<B, T> Ref<B, [T]> +where + B: SplitByteSlice, + T: Immutable, +{ + #[deprecated(since = "0.8.0", note = "replaced by `Ref::from_prefix_with_elems`")] + #[must_use = "has no side effects"] + #[doc(hidden)] + #[inline(always)] + pub fn new_slice_from_prefix(bytes: B, count: usize) -> Option<(Ref<B, [T]>, B)> { + Ref::from_prefix_with_elems(bytes, count).ok() + } + + #[deprecated(since = "0.8.0", note = "replaced by `Ref::from_suffix_with_elems`")] + #[must_use = "has no side effects"] + #[doc(hidden)] + #[inline(always)] + pub fn new_slice_from_suffix(bytes: B, count: usize) -> Option<(B, Ref<B, [T]>)> { + Ref::from_suffix_with_elems(bytes, count).ok() + } +} + +impl<B, T> Ref<B, [T]> +where + B: SplitByteSlice, + T: Unaligned + Immutable, +{ + #[deprecated( + since = "0.8.0", + note = "use `Ref::from_prefix_with_elems`; for `T: Unaligned`, the returned `CastError` implements `Into<SizeError>`" + )] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_slice_unaligned_from_prefix(bytes: B, count: usize) -> Option<(Ref<B, [T]>, B)> { + Ref::from_prefix_with_elems(bytes, count).ok() + } + + #[deprecated( + since = "0.8.0", + note = "use `Ref::from_suffix_with_elems`; for `T: Unaligned`, the returned `CastError` implements `Into<SizeError>`" + )] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn new_slice_unaligned_from_suffix(bytes: B, count: usize) -> Option<(B, Ref<B, [T]>)> { + Ref::from_suffix_with_elems(bytes, count).ok() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[allow(deprecated)] + fn test_deprecated_ref_methods() { + let bytes = &[0u8; 1][..]; + let bytes_slice = &[0u8; 4][..]; + + let r: Option<Ref<&[u8], u8>> = Ref::new(bytes); + assert!(r.is_some()); + + let r: Option<(Ref<&[u8], u8>, &[u8])> = Ref::new_from_prefix(bytes); + assert!(r.is_some()); + + let r: Option<(&[u8], Ref<&[u8], u8>)> = Ref::new_from_suffix(bytes); + assert!(r.is_some()); + + let r: Option<Ref<&[u8], u8>> = Ref::new_unaligned(bytes); + assert!(r.is_some()); + + let r: Option<(Ref<&[u8], u8>, &[u8])> = Ref::new_unaligned_from_prefix(bytes); + assert!(r.is_some()); + + let r: Option<(&[u8], Ref<&[u8], u8>)> = Ref::new_unaligned_from_suffix(bytes); + assert!(r.is_some()); + + let r: Option<Ref<&[u8], [u8]>> = Ref::new_slice(bytes_slice); + assert!(r.is_some()); + + let r: Option<Ref<&[u8], [u8]>> = Ref::new_slice_unaligned(bytes_slice); + assert!(r.is_some()); + + let r: Option<(Ref<&[u8], [u8]>, &[u8])> = Ref::new_slice_from_prefix(bytes_slice, 1); + assert!(r.is_some()); + + let r: Option<(&[u8], Ref<&[u8], [u8]>)> = Ref::new_slice_from_suffix(bytes_slice, 1); + assert!(r.is_some()); + + let r: Option<(Ref<&[u8], [u8]>, &[u8])> = + Ref::new_slice_unaligned_from_prefix(bytes_slice, 1); + assert!(r.is_some()); + + let r: Option<(&[u8], Ref<&[u8], [u8]>)> = + Ref::new_slice_unaligned_from_suffix(bytes_slice, 1); + assert!(r.is_some()); + } + + #[test] + #[allow(deprecated)] + fn test_deprecated_into_slice() { + let bytes = &[0u8; 4][..]; + let r: Ref<&[u8], [u8]> = Ref::from_bytes(bytes).unwrap(); + let slice: &[u8] = r.into_slice(); + assert_eq!(slice.len(), 4); + } + + #[test] + #[allow(deprecated)] + fn test_deprecated_into_mut_slice() { + let mut bytes = [0u8; 4]; + let r: Ref<&mut [u8], [u8]> = Ref::from_bytes(&mut bytes[..]).unwrap(); + let slice: &mut [u8] = r.into_mut_slice(); + assert_eq!(slice.len(), 4); + } +} diff --git a/rust/zerocopy/src/error.rs b/rust/zerocopy/src/error.rs new file mode 100644 index 000000000000..5eb30de934f3 --- /dev/null +++ b/rust/zerocopy/src/error.rs @@ -0,0 +1,1350 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License <LICENSE-BSD or +// https://opensource.org/license/bsd-2-clause>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Types related to error reporting. +//! +//! ## Single failure mode errors +//! +//! Generally speaking, zerocopy's conversions may fail for one of up to three +//! reasons: +//! - [`AlignmentError`]: the conversion source was improperly aligned +//! - [`SizeError`]: the conversion source was of incorrect size +//! - [`ValidityError`]: the conversion source contained invalid data +//! +//! Methods that only have one failure mode, like +//! [`FromBytes::read_from_bytes`], return that mode's corresponding error type +//! directly. +//! +//! ## Compound errors +//! +//! Conversion methods that have either two or three possible failure modes +//! return one of these error types: +//! - [`CastError`]: the error type of reference conversions +//! - [`TryCastError`]: the error type of fallible reference conversions +//! - [`TryReadError`]: the error type of fallible read conversions +//! +//! ## [`Unaligned`] destination types +//! +//! For [`Unaligned`] destination types, alignment errors are impossible. All +//! compound error types support infallibly discarding the alignment error via +//! [`From`] so long as `Dst: Unaligned`. For example, see [`<SizeError as +//! From<ConvertError>>::from`][size-error-from]. +//! +//! [size-error-from]: struct.SizeError.html#method.from-1 +//! +//! ## Accessing the conversion source +//! +//! All error types provide an `into_src` method that converts the error into +//! the source value underlying the failed conversion. +//! +//! ## Display formatting +//! +//! All error types provide a `Display` implementation that produces a +//! human-readable error message. When `debug_assertions` are enabled, these +//! error messages are verbose and may include potentially sensitive +//! information, including: +//! +//! - the names of the involved types +//! - the sizes of the involved types +//! - the addresses of the involved types +//! - the contents of the involved types +//! +//! When `debug_assertions` are disabled (as is default for `release` builds), +//! such potentially sensitive information is excluded. +//! +//! In the future, we may support manually configuring this behavior. If you are +//! interested in this feature, [let us know on GitHub][issue-1457] so we know +//! to prioritize it. +//! +//! [issue-1457]: https://github.com/google/zerocopy/issues/1457 +//! +//! ## Validation order +//! +//! Our conversion methods typically check alignment, then size, then bit +//! validity. However, we do not guarantee that this is always the case, and +//! this behavior may change between releases. +//! +//! ## `Send`, `Sync`, and `'static` +//! +//! Our error types are `Send`, `Sync`, and `'static` when their `Src` parameter +//! is `Send`, `Sync`, or `'static`, respectively. This can cause issues when an +//! error is sent or synchronized across threads; e.g.: +//! +//! ```compile_fail,E0515 +//! use zerocopy::*; +//! +//! let result: SizeError<&[u8], u32> = std::thread::spawn(|| { +//! let source = &mut [0u8, 1, 2][..]; +//! // Try (and fail) to read a `u32` from `source`. +//! u32::read_from_bytes(source).unwrap_err() +//! }).join().unwrap(); +//! ``` +//! +//! To work around this, use [`map_src`][CastError::map_src] to convert the +//! source parameter to an unproblematic type; e.g.: +//! +//! ``` +//! use zerocopy::*; +//! +//! let result: SizeError<(), u32> = std::thread::spawn(|| { +//! let source = &mut [0u8, 1, 2][..]; +//! // Try (and fail) to read a `u32` from `source`. +//! u32::read_from_bytes(source).unwrap_err() +//! // Erase the error source. +//! .map_src(drop) +//! }).join().unwrap(); +//! ``` +//! +//! Alternatively, use `.to_string()` to eagerly convert the error into a +//! human-readable message; e.g.: +//! +//! ``` +//! use zerocopy::*; +//! +//! let result: Result<u32, String> = std::thread::spawn(|| { +//! let source = &mut [0u8, 1, 2][..]; +//! // Try (and fail) to read a `u32` from `source`. +//! u32::read_from_bytes(source) +//! // Eagerly render the error message. +//! .map_err(|err| err.to_string()) +//! }).join().unwrap(); +//! ``` +#[cfg(not(no_zerocopy_core_error_1_81_0))] +use core::error::Error; +use core::{ + convert::Infallible, + fmt::{self, Debug, Write}, + ops::Deref, +}; +#[cfg(all(no_zerocopy_core_error_1_81_0, any(feature = "std", test)))] +use std::error::Error; + +use crate::{util::SendSyncPhantomData, KnownLayout, TryFromBytes, Unaligned}; +#[cfg(doc)] +use crate::{FromBytes, Ref}; + +/// Zerocopy's generic error type. +/// +/// Generally speaking, zerocopy's conversions may fail for one of up to three +/// reasons: +/// - [`AlignmentError`]: the conversion source was improperly aligned +/// - [`SizeError`]: the conversion source was of incorrect size +/// - [`ValidityError`]: the conversion source contained invalid data +/// +/// However, not all conversions produce all errors. For instance, +/// [`FromBytes::ref_from_bytes`] may fail due to alignment or size issues, but +/// not validity issues. This generic error type captures these +/// (im)possibilities via parameterization: `A` is parameterized with +/// [`AlignmentError`], `S` is parameterized with [`SizeError`], and `V` is +/// parameterized with [`Infallible`]. +/// +/// Zerocopy never uses this type directly in its API. Rather, we provide three +/// pre-parameterized aliases: +/// - [`CastError`]: the error type of reference conversions +/// - [`TryCastError`]: the error type of fallible reference conversions +/// - [`TryReadError`]: the error type of fallible read conversions +#[derive(PartialEq, Eq, Clone)] +pub enum ConvertError<A, S, V> { + /// The conversion source was improperly aligned. + Alignment(A), + /// The conversion source was of incorrect size. + Size(S), + /// The conversion source contained invalid data. + Validity(V), +} + +impl<Src, Dst: ?Sized + Unaligned, S, V> From<ConvertError<AlignmentError<Src, Dst>, S, V>> + for ConvertError<Infallible, S, V> +{ + /// Infallibly discards the alignment error from this `ConvertError` since + /// `Dst` is unaligned. + /// + /// Since [`Dst: Unaligned`], it is impossible to encounter an alignment + /// error. This method permits discarding that alignment error infallibly + /// and replacing it with [`Infallible`]. + /// + /// [`Dst: Unaligned`]: crate::Unaligned + /// + /// # Examples + /// + /// ``` + /// use core::convert::Infallible; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, KnownLayout, Unaligned, Immutable)] + /// #[repr(C, packed)] + /// struct Bools { + /// one: bool, + /// two: bool, + /// many: [bool], + /// } + /// + /// impl Bools { + /// fn parse(bytes: &[u8]) -> Result<&Bools, AlignedTryCastError<&[u8], Bools>> { + /// // Since `Bools: Unaligned`, we can infallibly discard + /// // the alignment error. + /// Bools::try_ref_from_bytes(bytes).map_err(Into::into) + /// } + /// } + /// ``` + #[inline] + fn from(err: ConvertError<AlignmentError<Src, Dst>, S, V>) -> ConvertError<Infallible, S, V> { + match err { + ConvertError::Alignment(e) => { + #[allow(unreachable_code)] + return ConvertError::Alignment(Infallible::from(e)); + } + ConvertError::Size(e) => ConvertError::Size(e), + ConvertError::Validity(e) => ConvertError::Validity(e), + } + } +} + +impl<A: fmt::Debug, S: fmt::Debug, V: fmt::Debug> fmt::Debug for ConvertError<A, S, V> { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Alignment(e) => f.debug_tuple("Alignment").field(e).finish(), + Self::Size(e) => f.debug_tuple("Size").field(e).finish(), + Self::Validity(e) => f.debug_tuple("Validity").field(e).finish(), + } + } +} + +/// Produces a human-readable error message. +/// +/// The message differs between debug and release builds. When +/// `debug_assertions` are enabled, this message is verbose and includes +/// potentially sensitive information. +impl<A: fmt::Display, S: fmt::Display, V: fmt::Display> fmt::Display for ConvertError<A, S, V> { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Alignment(e) => e.fmt(f), + Self::Size(e) => e.fmt(f), + Self::Validity(e) => e.fmt(f), + } + } +} + +#[cfg(any(not(no_zerocopy_core_error_1_81_0), feature = "std", test))] +#[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.81.0", feature = "std"))))] +impl<A, S, V> Error for ConvertError<A, S, V> +where + A: fmt::Display + fmt::Debug, + S: fmt::Display + fmt::Debug, + V: fmt::Display + fmt::Debug, +{ +} + +/// The error emitted if the conversion source is improperly aligned. +pub struct AlignmentError<Src, Dst: ?Sized> { + /// The source value involved in the conversion. + src: Src, + /// The inner destination type involved in the conversion. + /// + /// INVARIANT: An `AlignmentError` may only be constructed if `Dst`'s + /// alignment requirement is greater than one. + _dst: SendSyncPhantomData<Dst>, +} + +impl<Src, Dst: ?Sized> AlignmentError<Src, Dst> { + /// # Safety + /// + /// The caller must ensure that `Dst`'s alignment requirement is greater + /// than one. + pub(crate) unsafe fn new_unchecked(src: Src) -> Self { + // INVARIANT: The caller guarantees that `Dst`'s alignment requirement + // is greater than one. + Self { src, _dst: SendSyncPhantomData::default() } + } + + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + self.src + } + + pub(crate) fn with_src<NewSrc>(self, new_src: NewSrc) -> AlignmentError<NewSrc, Dst> { + // INVARIANT: `with_src` doesn't change the type of `Dst`, so the + // invariant that `Dst`'s alignment requirement is greater than one is + // preserved. + AlignmentError { src: new_src, _dst: SendSyncPhantomData::default() } + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::*; + /// + /// let unaligned = Unalign::new(0u16); + /// + /// // Attempt to deref `unaligned`. This might fail with an alignment error. + /// let maybe_n: Result<&u16, AlignmentError<&Unalign<u16>, u16>> = unaligned.try_deref(); + /// + /// // Map the error's source to its address as a usize. + /// let maybe_n: Result<&u16, AlignmentError<usize, u16>> = maybe_n.map_err(|err| { + /// err.map_src(|src| src as *const _ as usize) + /// }); + /// ``` + #[inline] + pub fn map_src<NewSrc>(self, f: impl FnOnce(Src) -> NewSrc) -> AlignmentError<NewSrc, Dst> { + AlignmentError { src: f(self.src), _dst: SendSyncPhantomData::default() } + } + + pub(crate) fn into<S, V>(self) -> ConvertError<Self, S, V> { + ConvertError::Alignment(self) + } + + /// Format extra details for a verbose, human-readable error message. + /// + /// This formatting may include potentially sensitive information. + fn display_verbose_extras(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + where + Src: Deref, + Dst: KnownLayout, + { + #[allow(clippy::as_conversions)] + let addr = self.src.deref() as *const _ as *const (); + let addr_align = 2usize.pow((crate::util::AsAddress::addr(addr)).trailing_zeros()); + + f.write_str("\n\nSource type: ")?; + f.write_str(core::any::type_name::<Src>())?; + + f.write_str("\nSource address: ")?; + addr.fmt(f)?; + f.write_str(" (a multiple of ")?; + addr_align.fmt(f)?; + f.write_str(")")?; + + f.write_str("\nDestination type: ")?; + f.write_str(core::any::type_name::<Dst>())?; + + f.write_str("\nDestination alignment: ")?; + <Dst as KnownLayout>::LAYOUT.align.get().fmt(f)?; + + Ok(()) + } +} + +impl<Src: Clone, Dst: ?Sized> Clone for AlignmentError<Src, Dst> { + #[inline] + fn clone(&self) -> Self { + Self { src: self.src.clone(), _dst: SendSyncPhantomData::default() } + } +} + +impl<Src: PartialEq, Dst: ?Sized> PartialEq for AlignmentError<Src, Dst> { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.src == other.src + } +} + +impl<Src: Eq, Dst: ?Sized> Eq for AlignmentError<Src, Dst> {} + +impl<Src, Dst: ?Sized + Unaligned> From<AlignmentError<Src, Dst>> for Infallible { + #[inline(always)] + fn from(_: AlignmentError<Src, Dst>) -> Infallible { + // SAFETY: `AlignmentError`s can only be constructed when `Dst`'s + // alignment requirement is greater than one. In this block, `Dst: + // Unaligned`, which means that its alignment requirement is equal to + // one. Thus, it's not possible to reach here at runtime. + unsafe { core::hint::unreachable_unchecked() } + } +} + +#[cfg(test)] +impl<Src, Dst> AlignmentError<Src, Dst> { + // A convenience constructor so that test code doesn't need to write + // `unsafe`. + fn new_checked(src: Src) -> AlignmentError<Src, Dst> { + assert_ne!(core::mem::align_of::<Dst>(), 1); + // SAFETY: The preceding assertion guarantees that `Dst`'s alignment + // requirement is greater than one. + unsafe { AlignmentError::new_unchecked(src) } + } +} + +impl<Src, Dst: ?Sized> fmt::Debug for AlignmentError<Src, Dst> { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AlignmentError").finish() + } +} + +/// Produces a human-readable error message. +/// +/// The message differs between debug and release builds. When +/// `debug_assertions` are enabled, this message is verbose and includes +/// potentially sensitive information. +impl<Src, Dst: ?Sized> fmt::Display for AlignmentError<Src, Dst> +where + Src: Deref, + Dst: KnownLayout, +{ + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("The conversion failed because the address of the source is not a multiple of the alignment of the destination type.")?; + + if cfg!(debug_assertions) { + self.display_verbose_extras(f) + } else { + Ok(()) + } + } +} + +#[cfg(any(not(no_zerocopy_core_error_1_81_0), feature = "std", test))] +#[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.81.0", feature = "std"))))] +impl<Src, Dst: ?Sized> Error for AlignmentError<Src, Dst> +where + Src: Deref, + Dst: KnownLayout, +{ +} + +impl<Src, Dst: ?Sized, S, V> From<AlignmentError<Src, Dst>> + for ConvertError<AlignmentError<Src, Dst>, S, V> +{ + #[inline(always)] + fn from(err: AlignmentError<Src, Dst>) -> Self { + Self::Alignment(err) + } +} + +/// The error emitted if the conversion source is of incorrect size. +pub struct SizeError<Src, Dst: ?Sized> { + /// The source value involved in the conversion. + src: Src, + /// The inner destination type involved in the conversion. + _dst: SendSyncPhantomData<Dst>, +} + +impl<Src, Dst: ?Sized> SizeError<Src, Dst> { + pub(crate) fn new(src: Src) -> Self { + Self { src, _dst: SendSyncPhantomData::default() } + } + + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + self.src + } + + /// Sets the source value associated with the conversion error. + pub(crate) fn with_src<NewSrc>(self, new_src: NewSrc) -> SizeError<NewSrc, Dst> { + SizeError { src: new_src, _dst: SendSyncPhantomData::default() } + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::*; + /// + /// let source: [u8; 3] = [0, 1, 2]; + /// + /// // Try to read a `u32` from `source`. This will fail because there are insufficient + /// // bytes in `source`. + /// let maybe_u32: Result<u32, SizeError<&[u8], u32>> = u32::read_from_bytes(&source[..]); + /// + /// // Map the error's source to its size. + /// let maybe_u32: Result<u32, SizeError<usize, u32>> = maybe_u32.map_err(|err| { + /// err.map_src(|src| src.len()) + /// }); + /// ``` + #[inline] + pub fn map_src<NewSrc>(self, f: impl FnOnce(Src) -> NewSrc) -> SizeError<NewSrc, Dst> { + SizeError { src: f(self.src), _dst: SendSyncPhantomData::default() } + } + + /// Sets the destination type associated with the conversion error. + pub(crate) fn with_dst<NewDst: ?Sized>(self) -> SizeError<Src, NewDst> { + SizeError { src: self.src, _dst: SendSyncPhantomData::default() } + } + + /// Converts the error into a general [`ConvertError`]. + pub(crate) fn into<A, V>(self) -> ConvertError<A, Self, V> { + ConvertError::Size(self) + } + + /// Format extra details for a verbose, human-readable error message. + /// + /// This formatting may include potentially sensitive information. + fn display_verbose_extras(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + where + Src: Deref, + Dst: KnownLayout, + { + // include the source type + f.write_str("\nSource type: ")?; + f.write_str(core::any::type_name::<Src>())?; + + // include the source.deref() size + let src_size = core::mem::size_of_val(&*self.src); + f.write_str("\nSource size: ")?; + src_size.fmt(f)?; + f.write_str(" byte")?; + if src_size != 1 { + f.write_char('s')?; + } + + // if `Dst` is `Sized`, include the `Dst` size + if let crate::SizeInfo::Sized { size } = Dst::LAYOUT.size_info { + f.write_str("\nDestination size: ")?; + size.fmt(f)?; + f.write_str(" byte")?; + if size != 1 { + f.write_char('s')?; + } + } + + // include the destination type + f.write_str("\nDestination type: ")?; + f.write_str(core::any::type_name::<Dst>())?; + + Ok(()) + } +} + +impl<Src: Clone, Dst: ?Sized> Clone for SizeError<Src, Dst> { + #[inline] + fn clone(&self) -> Self { + Self { src: self.src.clone(), _dst: SendSyncPhantomData::default() } + } +} + +impl<Src: PartialEq, Dst: ?Sized> PartialEq for SizeError<Src, Dst> { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.src == other.src + } +} + +impl<Src: Eq, Dst: ?Sized> Eq for SizeError<Src, Dst> {} + +impl<Src, Dst: ?Sized> fmt::Debug for SizeError<Src, Dst> { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SizeError").finish() + } +} + +/// Produces a human-readable error message. +/// +/// The message differs between debug and release builds. When +/// `debug_assertions` are enabled, this message is verbose and includes +/// potentially sensitive information. +impl<Src, Dst: ?Sized> fmt::Display for SizeError<Src, Dst> +where + Src: Deref, + Dst: KnownLayout, +{ + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("The conversion failed because the source was incorrectly sized to complete the conversion into the destination type.")?; + if cfg!(debug_assertions) { + f.write_str("\n")?; + self.display_verbose_extras(f)?; + } + Ok(()) + } +} + +#[cfg(any(not(no_zerocopy_core_error_1_81_0), feature = "std", test))] +#[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.81.0", feature = "std"))))] +impl<Src, Dst: ?Sized> Error for SizeError<Src, Dst> +where + Src: Deref, + Dst: KnownLayout, +{ +} + +impl<Src, Dst: ?Sized, A, V> From<SizeError<Src, Dst>> for ConvertError<A, SizeError<Src, Dst>, V> { + #[inline(always)] + fn from(err: SizeError<Src, Dst>) -> Self { + Self::Size(err) + } +} + +/// The error emitted if the conversion source contains invalid data. +pub struct ValidityError<Src, Dst: ?Sized + TryFromBytes> { + /// The source value involved in the conversion. + pub(crate) src: Src, + /// The inner destination type involved in the conversion. + _dst: SendSyncPhantomData<Dst>, +} + +impl<Src, Dst: ?Sized + TryFromBytes> ValidityError<Src, Dst> { + pub(crate) fn new(src: Src) -> Self { + Self { src, _dst: SendSyncPhantomData::default() } + } + + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + self.src + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::*; + /// + /// let source: u8 = 42; + /// + /// // Try to transmute the `source` to a `bool`. This will fail. + /// let maybe_bool: Result<bool, ValidityError<u8, bool>> = try_transmute!(source); + /// + /// // Drop the error's source. + /// let maybe_bool: Result<bool, ValidityError<(), bool>> = maybe_bool.map_err(|err| { + /// err.map_src(drop) + /// }); + /// ``` + #[inline] + pub fn map_src<NewSrc>(self, f: impl FnOnce(Src) -> NewSrc) -> ValidityError<NewSrc, Dst> { + ValidityError { src: f(self.src), _dst: SendSyncPhantomData::default() } + } + + /// Converts the error into a general [`ConvertError`]. + pub(crate) fn into<A, S>(self) -> ConvertError<A, S, Self> { + ConvertError::Validity(self) + } + + /// Format extra details for a verbose, human-readable error message. + /// + /// This formatting may include potentially sensitive information. + fn display_verbose_extras(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + where + Dst: KnownLayout, + { + f.write_str("Destination type: ")?; + f.write_str(core::any::type_name::<Dst>())?; + Ok(()) + } +} + +impl<Src: Clone, Dst: ?Sized + TryFromBytes> Clone for ValidityError<Src, Dst> { + #[inline] + fn clone(&self) -> Self { + Self { src: self.src.clone(), _dst: SendSyncPhantomData::default() } + } +} + +// SAFETY: `ValidityError` contains a single `Self::Inner = Src`, and no other +// non-ZST fields. `map` passes ownership of `self`'s sole `Self::Inner` to `f`. +unsafe impl<Src, NewSrc, Dst> crate::pointer::TryWithError<NewSrc> + for crate::ValidityError<Src, Dst> +where + Dst: TryFromBytes + ?Sized, +{ + type Inner = Src; + type Mapped = crate::ValidityError<NewSrc, Dst>; + #[inline] + fn map<F: FnOnce(Src) -> NewSrc>(self, f: F) -> Self::Mapped { + self.map_src(f) + } +} + +impl<Src: PartialEq, Dst: ?Sized + TryFromBytes> PartialEq for ValidityError<Src, Dst> { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.src == other.src + } +} + +impl<Src: Eq, Dst: ?Sized + TryFromBytes> Eq for ValidityError<Src, Dst> {} + +impl<Src, Dst: ?Sized + TryFromBytes> fmt::Debug for ValidityError<Src, Dst> { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ValidityError").finish() + } +} + +/// Produces a human-readable error message. +/// +/// The message differs between debug and release builds. When +/// `debug_assertions` are enabled, this message is verbose and includes +/// potentially sensitive information. +impl<Src, Dst: ?Sized> fmt::Display for ValidityError<Src, Dst> +where + Dst: KnownLayout + TryFromBytes, +{ + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("The conversion failed because the source bytes are not a valid value of the destination type.")?; + if cfg!(debug_assertions) { + f.write_str("\n\n")?; + self.display_verbose_extras(f)?; + } + Ok(()) + } +} + +#[cfg(any(not(no_zerocopy_core_error_1_81_0), feature = "std", test))] +#[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.81.0", feature = "std"))))] +impl<Src, Dst: ?Sized> Error for ValidityError<Src, Dst> where Dst: KnownLayout + TryFromBytes {} + +impl<Src, Dst: ?Sized + TryFromBytes, A, S> From<ValidityError<Src, Dst>> + for ConvertError<A, S, ValidityError<Src, Dst>> +{ + #[inline(always)] + fn from(err: ValidityError<Src, Dst>) -> Self { + Self::Validity(err) + } +} + +/// The error type of reference conversions. +/// +/// Reference conversions, like [`FromBytes::ref_from_bytes`] may emit +/// [alignment](AlignmentError) and [size](SizeError) errors. +// Bounds on generic parameters are not enforced in type aliases, but they do +// appear in rustdoc. +#[allow(type_alias_bounds)] +pub type CastError<Src, Dst: ?Sized> = + ConvertError<AlignmentError<Src, Dst>, SizeError<Src, Dst>, Infallible>; + +impl<Src, Dst: ?Sized> CastError<Src, Dst> { + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + match self { + Self::Alignment(e) => e.src, + Self::Size(e) => e.src, + Self::Validity(i) => match i {}, + } + } + + /// Sets the source value associated with the conversion error. + pub(crate) fn with_src<NewSrc>(self, new_src: NewSrc) -> CastError<NewSrc, Dst> { + match self { + Self::Alignment(e) => CastError::Alignment(e.with_src(new_src)), + Self::Size(e) => CastError::Size(e.with_src(new_src)), + Self::Validity(i) => match i {}, + } + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::*; + /// + /// let source: [u8; 3] = [0, 1, 2]; + /// + /// // Try to read a `u32` from `source`. This will fail because there are insufficient + /// // bytes in `source`. + /// let maybe_u32: Result<&u32, CastError<&[u8], u32>> = u32::ref_from_bytes(&source[..]); + /// + /// // Map the error's source to its size and address. + /// let maybe_u32: Result<&u32, CastError<(usize, usize), u32>> = maybe_u32.map_err(|err| { + /// err.map_src(|src| (src.len(), src.as_ptr() as usize)) + /// }); + /// ``` + #[inline] + pub fn map_src<NewSrc>(self, f: impl FnOnce(Src) -> NewSrc) -> CastError<NewSrc, Dst> { + match self { + Self::Alignment(e) => CastError::Alignment(e.map_src(f)), + Self::Size(e) => CastError::Size(e.map_src(f)), + Self::Validity(i) => match i {}, + } + } + + /// Converts the error into a general [`ConvertError`]. + pub(crate) fn into(self) -> TryCastError<Src, Dst> + where + Dst: TryFromBytes, + { + match self { + Self::Alignment(e) => TryCastError::Alignment(e), + Self::Size(e) => TryCastError::Size(e), + Self::Validity(i) => match i {}, + } + } +} + +// SAFETY: `CastError` is either a single `AlignmentError` or a single +// `SizeError`. In either case, it contains a single `Self::Inner = Src`, and no +// other non-ZST fields. `map` passes ownership of `self`'s sole `Self::Inner` +// to `f`. +unsafe impl<Src, NewSrc, Dst> crate::pointer::TryWithError<NewSrc> for crate::CastError<Src, Dst> +where + Dst: ?Sized, +{ + type Inner = Src; + type Mapped = crate::CastError<NewSrc, Dst>; + + #[inline] + fn map<F: FnOnce(Src) -> NewSrc>(self, f: F) -> Self::Mapped { + self.map_src(f) + } +} + +impl<Src, Dst: ?Sized + Unaligned> From<CastError<Src, Dst>> for SizeError<Src, Dst> { + /// Infallibly extracts the [`SizeError`] from this `CastError` since `Dst` + /// is unaligned. + /// + /// Since [`Dst: Unaligned`], it is impossible to encounter an alignment + /// error, and so the only error that can be encountered at runtime is a + /// [`SizeError`]. This method permits extracting that `SizeError` + /// infallibly. + /// + /// [`Dst: Unaligned`]: crate::Unaligned + /// + /// # Examples + /// + /// ```rust + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] + /// #[repr(C)] + /// struct UdpHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] + /// #[repr(C, packed)] + /// struct UdpPacket { + /// header: UdpHeader, + /// body: [u8], + /// } + /// + /// impl UdpPacket { + /// pub fn parse(bytes: &[u8]) -> Result<&UdpPacket, SizeError<&[u8], UdpPacket>> { + /// // Since `UdpPacket: Unaligned`, we can map the `CastError` to a `SizeError`. + /// UdpPacket::ref_from_bytes(bytes).map_err(Into::into) + /// } + /// } + /// ``` + #[inline(always)] + fn from(err: CastError<Src, Dst>) -> SizeError<Src, Dst> { + match err { + #[allow(unreachable_code)] + CastError::Alignment(e) => match Infallible::from(e) {}, + CastError::Size(e) => e, + CastError::Validity(i) => match i {}, + } + } +} + +/// The error type of fallible reference conversions. +/// +/// Fallible reference conversions, like [`TryFromBytes::try_ref_from_bytes`] +/// may emit [alignment](AlignmentError), [size](SizeError), and +/// [validity](ValidityError) errors. +// Bounds on generic parameters are not enforced in type aliases, but they do +// appear in rustdoc. +#[allow(type_alias_bounds)] +pub type TryCastError<Src, Dst: ?Sized + TryFromBytes> = + ConvertError<AlignmentError<Src, Dst>, SizeError<Src, Dst>, ValidityError<Src, Dst>>; + +// FIXME(#1139): Remove the `TryFromBytes` here and in other downstream +// locations (all the way to `ValidityError`) if we determine it's not necessary +// for rich validity errors. +impl<Src, Dst: ?Sized + TryFromBytes> TryCastError<Src, Dst> { + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + match self { + Self::Alignment(e) => e.src, + Self::Size(e) => e.src, + Self::Validity(e) => e.src, + } + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use core::num::NonZeroU32; + /// use zerocopy::*; + /// + /// let source: [u8; 3] = [0, 0, 0]; + /// + /// // Try to read a `NonZeroU32` from `source`. + /// let maybe_u32: Result<&NonZeroU32, TryCastError<&[u8], NonZeroU32>> + /// = NonZeroU32::try_ref_from_bytes(&source[..]); + /// + /// // Map the error's source to its size and address. + /// let maybe_u32: Result<&NonZeroU32, TryCastError<(usize, usize), NonZeroU32>> = + /// maybe_u32.map_err(|err| { + /// err.map_src(|src| (src.len(), src.as_ptr() as usize)) + /// }); + /// ``` + #[inline] + pub fn map_src<NewSrc>(self, f: impl FnOnce(Src) -> NewSrc) -> TryCastError<NewSrc, Dst> { + match self { + Self::Alignment(e) => TryCastError::Alignment(e.map_src(f)), + Self::Size(e) => TryCastError::Size(e.map_src(f)), + Self::Validity(e) => TryCastError::Validity(e.map_src(f)), + } + } +} + +impl<Src, Dst: ?Sized + TryFromBytes> From<CastError<Src, Dst>> for TryCastError<Src, Dst> { + #[inline] + fn from(value: CastError<Src, Dst>) -> Self { + match value { + CastError::Alignment(e) => Self::Alignment(e), + CastError::Size(e) => Self::Size(e), + CastError::Validity(i) => match i {}, + } + } +} + +/// The error type of fallible read-conversions. +/// +/// Fallible read-conversions, like [`TryFromBytes::try_read_from_bytes`] may +/// emit [size](SizeError) and [validity](ValidityError) errors, but not +/// alignment errors. +// Bounds on generic parameters are not enforced in type aliases, but they do +// appear in rustdoc. +#[allow(type_alias_bounds)] +pub type TryReadError<Src, Dst: ?Sized + TryFromBytes> = + ConvertError<Infallible, SizeError<Src, Dst>, ValidityError<Src, Dst>>; + +impl<Src, Dst: ?Sized + TryFromBytes> TryReadError<Src, Dst> { + /// Produces the source underlying the failed conversion. + #[inline] + pub fn into_src(self) -> Src { + match self { + Self::Alignment(i) => match i {}, + Self::Size(e) => e.src, + Self::Validity(e) => e.src, + } + } + + /// Maps the source value associated with the conversion error. + /// + /// This can help mitigate [issues with `Send`, `Sync` and `'static` + /// bounds][self#send-sync-and-static]. + /// + /// # Examples + /// + /// ``` + /// use core::num::NonZeroU32; + /// use zerocopy::*; + /// + /// let source: [u8; 3] = [0, 0, 0]; + /// + /// // Try to read a `NonZeroU32` from `source`. + /// let maybe_u32: Result<NonZeroU32, TryReadError<&[u8], NonZeroU32>> + /// = NonZeroU32::try_read_from_bytes(&source[..]); + /// + /// // Map the error's source to its size. + /// let maybe_u32: Result<NonZeroU32, TryReadError<usize, NonZeroU32>> = + /// maybe_u32.map_err(|err| { + /// err.map_src(|src| src.len()) + /// }); + /// ``` + #[inline] + pub fn map_src<NewSrc>(self, f: impl FnOnce(Src) -> NewSrc) -> TryReadError<NewSrc, Dst> { + match self { + Self::Alignment(i) => match i {}, + Self::Size(e) => TryReadError::Size(e.map_src(f)), + Self::Validity(e) => TryReadError::Validity(e.map_src(f)), + } + } +} + +/// The error type of well-aligned, fallible casts. +/// +/// This is like [`TryCastError`], but for casts that are always well-aligned. +/// It is identical to `TryCastError`, except that its alignment error is +/// [`Infallible`]. +/// +/// As of this writing, none of zerocopy's API produces this error directly. +/// However, it is useful since it permits users to infallibly discard alignment +/// errors when they can prove statically that alignment errors are impossible. +/// +/// # Examples +/// +/// ``` +/// use core::convert::Infallible; +/// use zerocopy::*; +/// # use zerocopy_derive::*; +/// +/// #[derive(TryFromBytes, KnownLayout, Unaligned, Immutable)] +/// #[repr(C, packed)] +/// struct Bools { +/// one: bool, +/// two: bool, +/// many: [bool], +/// } +/// +/// impl Bools { +/// fn parse(bytes: &[u8]) -> Result<&Bools, AlignedTryCastError<&[u8], Bools>> { +/// // Since `Bools: Unaligned`, we can infallibly discard +/// // the alignment error. +/// Bools::try_ref_from_bytes(bytes).map_err(Into::into) +/// } +/// } +/// ``` +#[allow(type_alias_bounds)] +pub type AlignedTryCastError<Src, Dst: ?Sized + TryFromBytes> = + ConvertError<Infallible, SizeError<Src, Dst>, ValidityError<Src, Dst>>; + +/// The error type of a failed allocation. +/// +/// This type is intended to be deprecated in favor of the standard library's +/// [`AllocError`] type once it is stabilized. When that happens, this type will +/// be replaced by a type alias to the standard library type. We do not intend +/// to treat this as a breaking change; users who wish to avoid breakage should +/// avoid writing code which assumes that this is *not* such an alias. For +/// example, implementing the same trait for both types will result in an impl +/// conflict once this type is an alias. +/// +/// [`AllocError`]: https://doc.rust-lang.org/alloc/alloc/struct.AllocError.html +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct AllocError; + +#[cfg(test)] +mod tests { + use core::convert::Infallible; + + use super::*; + + #[test] + fn test_send_sync() { + // Test that all error types are `Send + Sync` even if `Dst: !Send + + // !Sync`. + + #[allow(dead_code)] + fn is_send_sync<T: Send + Sync>(_t: T) {} + + #[allow(dead_code)] + fn alignment_err_is_send_sync<Src: Send + Sync, Dst>(err: AlignmentError<Src, Dst>) { + is_send_sync(err) + } + + #[allow(dead_code)] + fn size_err_is_send_sync<Src: Send + Sync, Dst>(err: SizeError<Src, Dst>) { + is_send_sync(err) + } + + #[allow(dead_code)] + fn validity_err_is_send_sync<Src: Send + Sync, Dst: TryFromBytes>( + err: ValidityError<Src, Dst>, + ) { + is_send_sync(err) + } + + #[allow(dead_code)] + fn convert_error_is_send_sync<Src: Send + Sync, Dst: TryFromBytes>( + err: ConvertError< + AlignmentError<Src, Dst>, + SizeError<Src, Dst>, + ValidityError<Src, Dst>, + >, + ) { + is_send_sync(err) + } + } + + #[test] + fn test_eq_partial_eq_clone() { + // Test that all error types implement `Eq`, `PartialEq` + // and `Clone` if src does + // even if `Dst: !Eq`, `!PartialEq`, `!Clone`. + + #[allow(dead_code)] + fn is_eq_partial_eq_clone<T: Eq + PartialEq + Clone>(_t: T) {} + + #[allow(dead_code)] + fn alignment_err_is_eq_partial_eq_clone<Src: Eq + PartialEq + Clone, Dst>( + err: AlignmentError<Src, Dst>, + ) { + is_eq_partial_eq_clone(err) + } + + #[allow(dead_code)] + fn size_err_is_eq_partial_eq_clone<Src: Eq + PartialEq + Clone, Dst>( + err: SizeError<Src, Dst>, + ) { + is_eq_partial_eq_clone(err) + } + + #[allow(dead_code)] + fn validity_err_is_eq_partial_eq_clone<Src: Eq + PartialEq + Clone, Dst: TryFromBytes>( + err: ValidityError<Src, Dst>, + ) { + is_eq_partial_eq_clone(err) + } + + #[allow(dead_code)] + fn convert_error_is_eq_partial_eq_clone<Src: Eq + PartialEq + Clone, Dst: TryFromBytes>( + err: ConvertError< + AlignmentError<Src, Dst>, + SizeError<Src, Dst>, + ValidityError<Src, Dst>, + >, + ) { + is_eq_partial_eq_clone(err) + } + } + + #[test] + fn alignment_display() { + #[repr(C, align(128))] + struct Aligned { + bytes: [u8; 128], + } + + impl_known_layout!(elain::Align::<8>); + + let aligned = Aligned { bytes: [0; 128] }; + + let bytes = &aligned.bytes[1..]; + let addr = crate::util::AsAddress::addr(bytes); + assert_eq!( + AlignmentError::<_, elain::Align::<8>>::new_checked(bytes).to_string(), + format!("The conversion failed because the address of the source is not a multiple of the alignment of the destination type.\n\ + \nSource type: &[u8]\ + \nSource address: 0x{:x} (a multiple of 1)\ + \nDestination type: elain::Align<8>\ + \nDestination alignment: 8", addr) + ); + + let bytes = &aligned.bytes[2..]; + let addr = crate::util::AsAddress::addr(bytes); + assert_eq!( + AlignmentError::<_, elain::Align::<8>>::new_checked(bytes).to_string(), + format!("The conversion failed because the address of the source is not a multiple of the alignment of the destination type.\n\ + \nSource type: &[u8]\ + \nSource address: 0x{:x} (a multiple of 2)\ + \nDestination type: elain::Align<8>\ + \nDestination alignment: 8", addr) + ); + + let bytes = &aligned.bytes[3..]; + let addr = crate::util::AsAddress::addr(bytes); + assert_eq!( + AlignmentError::<_, elain::Align::<8>>::new_checked(bytes).to_string(), + format!("The conversion failed because the address of the source is not a multiple of the alignment of the destination type.\n\ + \nSource type: &[u8]\ + \nSource address: 0x{:x} (a multiple of 1)\ + \nDestination type: elain::Align<8>\ + \nDestination alignment: 8", addr) + ); + + let bytes = &aligned.bytes[4..]; + let addr = crate::util::AsAddress::addr(bytes); + assert_eq!( + AlignmentError::<_, elain::Align::<8>>::new_checked(bytes).to_string(), + format!("The conversion failed because the address of the source is not a multiple of the alignment of the destination type.\n\ + \nSource type: &[u8]\ + \nSource address: 0x{:x} (a multiple of 4)\ + \nDestination type: elain::Align<8>\ + \nDestination alignment: 8", addr) + ); + } + + #[test] + fn size_display() { + assert_eq!( + SizeError::<_, [u8]>::new(&[0u8; 2][..]).to_string(), + "The conversion failed because the source was incorrectly sized to complete the conversion into the destination type.\n\ + \nSource type: &[u8]\ + \nSource size: 2 bytes\ + \nDestination type: [u8]" + ); + + assert_eq!( + SizeError::<_, [u8; 2]>::new(&[0u8; 1][..]).to_string(), + "The conversion failed because the source was incorrectly sized to complete the conversion into the destination type.\n\ + \nSource type: &[u8]\ + \nSource size: 1 byte\ + \nDestination size: 2 bytes\ + \nDestination type: [u8; 2]" + ); + } + + #[test] + fn validity_display() { + assert_eq!( + ValidityError::<_, bool>::new(&[2u8; 1][..]).to_string(), + "The conversion failed because the source bytes are not a valid value of the destination type.\n\ + \n\ + Destination type: bool" + ); + } + + #[test] + fn test_convert_error_debug() { + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Alignment(AlignmentError::new_checked(&[0u8])); + assert_eq!(format!("{:?}", err), "Alignment(AlignmentError)"); + + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Size(SizeError::new(&[0u8])); + assert_eq!(format!("{:?}", err), "Size(SizeError)"); + + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Validity(ValidityError::new(&[0u8])); + assert_eq!(format!("{:?}", err), "Validity(ValidityError)"); + } + + #[test] + fn test_convert_error_from_unaligned() { + // u8 is Unaligned + let err: ConvertError< + AlignmentError<&[u8], u8>, + SizeError<&[u8], u8>, + ValidityError<&[u8], bool>, + > = ConvertError::Size(SizeError::new(&[0u8])); + let converted: ConvertError<Infallible, SizeError<&[u8], u8>, ValidityError<&[u8], bool>> = + ConvertError::from(err); + match converted { + ConvertError::Size(_) => {} + _ => panic!("Expected Size error"), + } + } + + #[test] + fn test_alignment_error_display_debug() { + let err: AlignmentError<&[u8], u16> = AlignmentError::new_checked(&[0u8]); + assert!(format!("{:?}", err).contains("AlignmentError")); + assert!(format!("{}", err).contains("address of the source is not a multiple")); + } + + #[test] + fn test_size_error_display_debug() { + let err: SizeError<&[u8], u16> = SizeError::new(&[0u8]); + assert!(format!("{:?}", err).contains("SizeError")); + assert!(format!("{}", err).contains("source was incorrectly sized")); + } + + #[test] + fn test_validity_error_display_debug() { + let err: ValidityError<&[u8], bool> = ValidityError::new(&[0u8]); + assert!(format!("{:?}", err).contains("ValidityError")); + assert!(format!("{}", err).contains("source bytes are not a valid value")); + } + + #[test] + fn test_convert_error_display_debug_more() { + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Alignment(AlignmentError::new_checked(&[0u8])); + assert!(format!("{}", err).contains("address of the source is not a multiple")); + + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Size(SizeError::new(&[0u8])); + assert!(format!("{}", err).contains("source was incorrectly sized")); + + let err: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = ConvertError::Validity(ValidityError::new(&[0u8])); + assert!(format!("{}", err).contains("source bytes are not a valid value")); + } + + #[test] + fn test_alignment_error_methods() { + let err: AlignmentError<&[u8], u16> = AlignmentError::new_checked(&[0u8]); + + // into_src + let src = err.clone().into_src(); + assert_eq!(src, &[0u8]); + + // into + let converted: ConvertError< + AlignmentError<&[u8], u16>, + SizeError<&[u8], u16>, + ValidityError<&[u8], bool>, + > = err.clone().into(); + match converted { + ConvertError::Alignment(_) => {} + _ => panic!("Expected Alignment error"), + } + + // clone + let cloned = err.clone(); + assert_eq!(err, cloned); + + // eq + assert_eq!(err, cloned); + let err2: AlignmentError<&[u8], u16> = AlignmentError::new_checked(&[1u8]); + assert_ne!(err, err2); + } + + #[test] + fn test_convert_error_from_unaligned_variants() { + // u8 is Unaligned + let err: ConvertError< + AlignmentError<&[u8], u8>, + SizeError<&[u8], u8>, + ValidityError<&[u8], bool>, + > = ConvertError::Validity(ValidityError::new(&[0u8])); + let converted: ConvertError<Infallible, SizeError<&[u8], u8>, ValidityError<&[u8], bool>> = + ConvertError::from(err); + match converted { + ConvertError::Validity(_) => {} + _ => panic!("Expected Validity error"), + } + + let err: ConvertError< + AlignmentError<&[u8], u8>, + SizeError<&[u8], u8>, + ValidityError<&[u8], bool>, + > = ConvertError::Size(SizeError::new(&[0u8])); + let converted: ConvertError<Infallible, SizeError<&[u8], u8>, ValidityError<&[u8], bool>> = + ConvertError::from(err); + match converted { + ConvertError::Size(_) => {} + _ => panic!("Expected Size error"), + } + } +} diff --git a/rust/zerocopy/src/impls.rs b/rust/zerocopy/src/impls.rs new file mode 100644 index 000000000000..62e234c5202b --- /dev/null +++ b/rust/zerocopy/src/impls.rs @@ -0,0 +1,2389 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License <LICENSE-BSD or +// https://opensource.org/license/bsd-2-clause>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use core::{ + cell::{Cell, UnsafeCell}, + mem::MaybeUninit as CoreMaybeUninit, + ptr::NonNull, +}; + +use super::*; +use crate::pointer::cast::{CastSizedExact, CastUnsized}; + +// SAFETY: Per the reference [1], "the unit tuple (`()`) ... is guaranteed as a +// zero-sized type to have a size of 0 and an alignment of 1." +// - `Immutable`: `()` self-evidently does not contain any `UnsafeCell`s. +// - `TryFromBytes` (with no validator), `FromZeros`, `FromBytes`: There is only +// one possible sequence of 0 bytes, and `()` is inhabited. +// - `IntoBytes`: Since `()` has size 0, it contains no padding bytes. +// - `Unaligned`: `()` has alignment 1. +// +// [1] https://doc.rust-lang.org/1.81.0/reference/type-layout.html#tuple-layout +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!((): Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_unaligned!(()); +}; + +// SAFETY: +// - `Immutable`: These types self-evidently do not contain any `UnsafeCell`s. +// - `TryFromBytes` (with no validator), `FromZeros`, `FromBytes`: all bit +// patterns are valid for numeric types [1] +// - `IntoBytes`: numeric types have no padding bytes [1] +// - `Unaligned` (`u8` and `i8` only): The reference [2] specifies the size of +// `u8` and `i8` as 1 byte. We also know that: +// - Alignment is >= 1 [3] +// - Size is an integer multiple of alignment [4] +// - The only value >= 1 for which 1 is an integer multiple is 1 Therefore, +// the only possible alignment for `u8` and `i8` is 1. +// +// [1] Per https://doc.rust-lang.org/1.81.0/reference/types/numeric.html#bit-validity: +// +// For every numeric type, `T`, the bit validity of `T` is equivalent to +// the bit validity of `[u8; size_of::<T>()]`. An uninitialized byte is +// not a valid `u8`. +// +// [2] https://doc.rust-lang.org/1.81.0/reference/type-layout.html#primitive-data-layout +// +// [3] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#size-and-alignment: +// +// Alignment is measured in bytes, and must be at least 1. +// +// [4] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#size-and-alignment: +// +// The size of a value is always a multiple of its alignment. +// +// FIXME(#278): Once we've updated the trait docs to refer to `u8`s rather than +// bits or bytes, update this comment, especially the reference to [1]. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(u8: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + unsafe_impl!(i8: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_unaligned!(u8, i8); + unsafe_impl!(u16: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(i16: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(u32: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(i32: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(u64: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(i64: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(u128: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(i128: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(usize: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(isize: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(f32: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(f64: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + #[cfg(feature = "float-nightly")] + unsafe_impl!(#[cfg_attr(doc_cfg, doc(cfg(feature = "float-nightly")))] f16: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); + #[cfg(feature = "float-nightly")] + unsafe_impl!(#[cfg_attr(doc_cfg, doc(cfg(feature = "float-nightly")))] f128: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); +}; + +// SAFETY: +// - `Immutable`: `bool` self-evidently does not contain any `UnsafeCell`s. +// - `FromZeros`: Valid since "[t]he value false has the bit pattern 0x00" [1]. +// - `IntoBytes`: Since "the boolean type has a size and alignment of 1 each" +// and "The value false has the bit pattern 0x00 and the value true has the +// bit pattern 0x01" [1]. Thus, the only byte of the bool is always +// initialized. +// - `Unaligned`: Per the reference [1], "[a]n object with the boolean type has +// a size and alignment of 1 each." +// +// [1] https://doc.rust-lang.org/1.81.0/reference/types/boolean.html +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl!(bool: Immutable, FromZeros, IntoBytes, Unaligned) }; +assert_unaligned!(bool); + +// SAFETY: The impl must only return `true` for its argument if the original +// `Maybe<bool>` refers to a valid `bool`. We only return true if the `u8` value +// is 0 or 1, and both of these are valid values for `bool` [1]. +// +// [1] Per https://doc.rust-lang.org/1.81.0/reference/types/boolean.html: +// +// The value false has the bit pattern 0x00 and the value true has the bit +// pattern 0x01. +const _: () = unsafe { + unsafe_impl!(=> TryFromBytes for bool; |byte| { + let byte = byte.transmute_with::<u8, invariant::Valid, CastSizedExact, BecauseImmutable>(); + *byte.unaligned_as_ref() < 2 + }) +}; + +// SAFETY: +// - `Immutable`: `char` self-evidently does not contain any `UnsafeCell`s. +// - `FromZeros`: Per reference [1], "[a] value of type char is a Unicode scalar +// value (i.e. a code point that is not a surrogate), represented as a 32-bit +// unsigned word in the 0x0000 to 0xD7FF or 0xE000 to 0x10FFFF range" which +// contains 0x0000. +// - `IntoBytes`: `char` is per reference [1] "represented as a 32-bit unsigned +// word" (`u32`) which is `IntoBytes`. Note that unlike `u32`, not all bit +// patterns are valid for `char`. +// +// [1] https://doc.rust-lang.org/1.81.0/reference/types/textual.html +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl!(char: Immutable, FromZeros, IntoBytes) }; + +// SAFETY: The impl must only return `true` for its argument if the original +// `Maybe<char>` refers to a valid `char`. `char::from_u32` guarantees that it +// returns `None` if its input is not a valid `char` [1]. +// +// [1] Per https://doc.rust-lang.org/core/primitive.char.html#method.from_u32: +// +// `from_u32()` will return `None` if the input is not a valid value for a +// `char`. +const _: () = unsafe { + unsafe_impl!(=> TryFromBytes for char; |c| { + let c = c.transmute_with::<Unalign<u32>, invariant::Valid, CastSizedExact, BecauseImmutable>(); + let c = c.read().into_inner(); + char::from_u32(c).is_some() + }); +}; + +// SAFETY: Per the Reference [1], `str` has the same layout as `[u8]`. +// - `Immutable`: `[u8]` does not contain any `UnsafeCell`s. +// - `FromZeros`, `IntoBytes`, `Unaligned`: `[u8]` is `FromZeros`, `IntoBytes`, +// and `Unaligned`. +// +// Note that we don't `assert_unaligned!(str)` because `assert_unaligned!` uses +// `align_of`, which only works for `Sized` types. +// +// FIXME(#429): Improve safety proof for `FromZeros` and `IntoBytes`; having the same +// layout as `[u8]` isn't sufficient. +// +// [1] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#str-layout: +// +// String slices are a UTF-8 representation of characters that have the same +// layout as slices of type `[u8]`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl!(str: Immutable, FromZeros, IntoBytes, Unaligned) }; + +// SAFETY: The impl must only return `true` for its argument if the original +// `Maybe<str>` refers to a valid `str`. `str::from_utf8` guarantees that it +// returns `Err` if its input is not a valid `str` [1]. +// +// [1] Per https://doc.rust-lang.org/core/str/fn.from_utf8.html#errors: +// +// Returns `Err` if the slice is not UTF-8. +const _: () = unsafe { + unsafe_impl!(=> TryFromBytes for str; |c| { + let c = c.transmute_with::<[u8], invariant::Valid, CastUnsized, BecauseImmutable>(); + let c = c.unaligned_as_ref(); + core::str::from_utf8(c).is_ok() + }) +}; + +macro_rules! unsafe_impl_try_from_bytes_for_nonzero { + ($($nonzero:ident[$prim:ty]),*) => { + $( + unsafe_impl!(=> TryFromBytes for $nonzero; |n| { + let n = n.transmute_with::<Unalign<$prim>, invariant::Valid, CastSizedExact, BecauseImmutable>(); + $nonzero::new(n.read().into_inner()).is_some() + }); + )* + } +} + +// `NonZeroXxx` is `IntoBytes`, but not `FromZeros` or `FromBytes`. +// +// SAFETY: +// - `IntoBytes`: `NonZeroXxx` has the same layout as its associated primitive. +// Since it is the same size, this guarantees it has no padding - integers +// have no padding, and there's no room for padding if it can represent all +// of the same values except 0. +// - `Unaligned`: `NonZeroU8` and `NonZeroI8` document that `Option<NonZeroU8>` +// and `Option<NonZeroI8>` both have size 1. [1] [2] This is worded in a way +// that makes it unclear whether it's meant as a guarantee, but given the +// purpose of those types, it's virtually unthinkable that that would ever +// change. `Option` cannot be smaller than its contained type, which implies +// that, and `NonZeroX8` are of size 1 or 0. `NonZeroX8` can represent +// multiple states, so they cannot be 0 bytes, which means that they must be 1 +// byte. The only valid alignment for a 1-byte type is 1. +// +// FIXME(#429): +// - Add quotes from documentation. +// - Add safety comment for `Immutable`. How can we prove that `NonZeroXxx` +// doesn't contain any `UnsafeCell`s? It's obviously true, but it's not clear +// how we'd prove it short of adding text to the stdlib docs that says so +// explicitly, which likely wouldn't be accepted. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/num/type.NonZeroU8.html: +// +// `NonZeroU8` is guaranteed to have the same layout and bit validity as `u8` with +// the exception that 0 is not a valid instance. +// +// [2] Per https://doc.rust-lang.org/1.81.0/std/num/type.NonZeroI8.html: +// +// `NonZeroI8` is guaranteed to have the same layout and bit validity as `i8` with +// the exception that 0 is not a valid instance. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(NonZeroU8: Immutable, IntoBytes, Unaligned); + unsafe_impl!(NonZeroI8: Immutable, IntoBytes, Unaligned); + assert_unaligned!(NonZeroU8, NonZeroI8); + unsafe_impl!(NonZeroU16: Immutable, IntoBytes); + unsafe_impl!(NonZeroI16: Immutable, IntoBytes); + unsafe_impl!(NonZeroU32: Immutable, IntoBytes); + unsafe_impl!(NonZeroI32: Immutable, IntoBytes); + unsafe_impl!(NonZeroU64: Immutable, IntoBytes); + unsafe_impl!(NonZeroI64: Immutable, IntoBytes); + unsafe_impl!(NonZeroU128: Immutable, IntoBytes); + unsafe_impl!(NonZeroI128: Immutable, IntoBytes); + unsafe_impl!(NonZeroUsize: Immutable, IntoBytes); + unsafe_impl!(NonZeroIsize: Immutable, IntoBytes); + unsafe_impl_try_from_bytes_for_nonzero!( + NonZeroU8[u8], + NonZeroI8[i8], + NonZeroU16[u16], + NonZeroI16[i16], + NonZeroU32[u32], + NonZeroI32[i32], + NonZeroU64[u64], + NonZeroI64[i64], + NonZeroU128[u128], + NonZeroI128[i128], + NonZeroUsize[usize], + NonZeroIsize[isize] + ); +}; + +// SAFETY: +// - `TryFromBytes` (with no validator), `FromZeros`, `FromBytes`, `IntoBytes`: +// The Rust compiler reuses `0` value to represent `None`, so +// `size_of::<Option<NonZeroXxx>>() == size_of::<xxx>()`; see `NonZeroXxx` +// documentation. +// - `Unaligned`: `NonZeroU8` and `NonZeroI8` document that `Option<NonZeroU8>` +// and `Option<NonZeroI8>` both have size 1. [1] [2] This is worded in a way +// that makes it unclear whether it's meant as a guarantee, but given the +// purpose of those types, it's virtually unthinkable that that would ever +// change. The only valid alignment for a 1-byte type is 1. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/num/type.NonZeroU8.html: +// +// `Option<NonZeroU8>` is guaranteed to be compatible with `u8`, including in FFI. +// +// Thanks to the null pointer optimization, `NonZeroU8` and `Option<NonZeroU8>` +// are guaranteed to have the same size and alignment: +// +// [2] Per https://doc.rust-lang.org/1.81.0/std/num/type.NonZeroI8.html: +// +// `Option<NonZeroI8>` is guaranteed to be compatible with `i8`, including in FFI. +// +// Thanks to the null pointer optimization, `NonZeroI8` and `Option<NonZeroI8>` +// are guaranteed to have the same size and alignment: +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(Option<NonZeroU8>: TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + unsafe_impl!(Option<NonZeroI8>: TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_unaligned!(Option<NonZeroU8>, Option<NonZeroI8>); + unsafe_impl!(Option<NonZeroU16>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroI16>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroU32>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroI32>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroU64>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroI64>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroU128>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroI128>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroUsize>: TryFromBytes, FromZeros, FromBytes, IntoBytes); + unsafe_impl!(Option<NonZeroIsize>: TryFromBytes, FromZeros, FromBytes, IntoBytes); +}; + +// SAFETY: While it's not fully documented, the consensus is that `Box<T>` does +// not contain any `UnsafeCell`s for `T: Sized` [1]. This is not a complete +// proof, but we are accepting this as a known risk per #1358. +// +// [1] https://github.com/rust-lang/unsafe-code-guidelines/issues/492 +#[cfg(feature = "alloc")] +const _: () = unsafe { + unsafe_impl!( + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + T: Sized => Immutable for Box<T> + ) +}; + +// SAFETY: The following types can be transmuted from `[0u8; size_of::<T>()]`. [1] +// +// [1] Per https://doc.rust-lang.org/1.89.0/core/option/index.html#representation: +// +// Rust guarantees to optimize the following types `T` such that [`Option<T>`] +// has the same size and alignment as `T`. In some of these cases, Rust +// further guarantees that `transmute::<_, Option<T>>([0u8; size_of::<T>()])` +// is sound and produces `Option::<T>::None`. These cases are identified by +// the second column: +// +// | `T` | `transmute::<_, Option<T>>([0u8; size_of::<T>()])` sound? | +// |-----------------------------------|-----------------------------------------------------------| +// | [`Box<U>`] | when `U: Sized` | +// | `&U` | when `U: Sized` | +// | `&mut U` | when `U: Sized` | +// | [`ptr::NonNull<U>`] | when `U: Sized` | +// | `fn`, `extern "C" fn`[^extern_fn] | always | +// +// [^extern_fn]: this remains true for `unsafe` variants, any argument/return +// types, and any other ABI: `[unsafe] extern "abi" fn` (_e.g._, `extern +// "system" fn`) +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + #[cfg(feature = "alloc")] + unsafe_impl!( + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + T => TryFromBytes for Option<Box<T>>; |c| pointer::is_zeroed(c) + ); + #[cfg(feature = "alloc")] + unsafe_impl!( + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + T => FromZeros for Option<Box<T>> + ); + unsafe_impl!( + T => TryFromBytes for Option<&'_ T>; |c| pointer::is_zeroed(c) + ); + unsafe_impl!(T => FromZeros for Option<&'_ T>); + unsafe_impl!( + T => TryFromBytes for Option<&'_ mut T>; |c| pointer::is_zeroed(c) + ); + unsafe_impl!(T => FromZeros for Option<&'_ mut T>); + unsafe_impl!( + T => TryFromBytes for Option<NonNull<T>>; |c| pointer::is_zeroed(c) + ); + unsafe_impl!(T => FromZeros for Option<NonNull<T>>); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => FromZeros for opt_fn!(...)); + unsafe_impl_for_power_set!( + A, B, C, D, E, F, G, H, I, J, K, L -> M => TryFromBytes for opt_fn!(...); + |c| pointer::is_zeroed(c) + ); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => FromZeros for opt_unsafe_fn!(...)); + unsafe_impl_for_power_set!( + A, B, C, D, E, F, G, H, I, J, K, L -> M => TryFromBytes for opt_unsafe_fn!(...); + |c| pointer::is_zeroed(c) + ); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => FromZeros for opt_extern_c_fn!(...)); + unsafe_impl_for_power_set!( + A, B, C, D, E, F, G, H, I, J, K, L -> M => TryFromBytes for opt_extern_c_fn!(...); + |c| pointer::is_zeroed(c) + ); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => FromZeros for opt_unsafe_extern_c_fn!(...)); + unsafe_impl_for_power_set!( + A, B, C, D, E, F, G, H, I, J, K, L -> M => TryFromBytes for opt_unsafe_extern_c_fn!(...); + |c| pointer::is_zeroed(c) + ); +}; + +// SAFETY: `[unsafe] [extern "C"] fn()` self-evidently do not contain +// `UnsafeCell`s. This is not a proof, but we are accepting this as a known risk +// per #1358. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => Immutable for opt_fn!(...)); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => Immutable for opt_unsafe_fn!(...)); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => Immutable for opt_extern_c_fn!(...)); + unsafe_impl_for_power_set!(A, B, C, D, E, F, G, H, I, J, K, L -> M => Immutable for opt_unsafe_extern_c_fn!(...)); +}; + +#[cfg(all( + not(no_zerocopy_target_has_atomics_1_60_0), + any( + target_has_atomic = "8", + target_has_atomic = "16", + target_has_atomic = "32", + target_has_atomic = "64", + target_has_atomic = "ptr" + ) +))] +#[cfg_attr(doc_cfg, doc(cfg(rust = "1.60.0")))] +mod atomics { + use super::*; + + macro_rules! impl_traits_for_atomics { + ($($atomics:tt [$primitives:ty]),* $(,)?) => { + $( + impl_known_layout!($atomics); + impl_for_transmute_from!(=> FromZeros for $atomics [$primitives]); + impl_for_transmute_from!(=> FromBytes for $atomics [$primitives]); + impl_for_transmute_from!(=> TryFromBytes for $atomics [$primitives]); + impl_for_transmute_from!(=> IntoBytes for $atomics [$primitives]); + )* + }; + } + + /// Implements `TransmuteFrom` for `$atomic`, `$prim`, and + /// `UnsafeCell<$prim>`. + /// + /// # Safety + /// + /// `$atomic` must have the same size and bit validity as `$prim`. + macro_rules! unsafe_impl_transmute_from_for_atomic { + ($($($tyvar:ident)? => $atomic:ty [$prim:ty]),*) => {{ + crate::util::macros::__unsafe(); + + use crate::pointer::{SizeEq, TransmuteFrom, invariant::Valid}; + + $( + // SAFETY: The caller promised that `$atomic` and `$prim` have + // the same size and bit validity. + unsafe impl<$($tyvar)?> TransmuteFrom<$atomic, Valid, Valid> for $prim {} + // SAFETY: The caller promised that `$atomic` and `$prim` have + // the same size and bit validity. + unsafe impl<$($tyvar)?> TransmuteFrom<$prim, Valid, Valid> for $atomic {} + + impl<$($tyvar)?> SizeEq<ReadOnly<$atomic>> for ReadOnly<$prim> { + type CastFrom = $crate::pointer::cast::CastSizedExact; + } + + // SAFETY: The caller promised that `$atomic` and `$prim` have + // the same bit validity. `UnsafeCell<T>` has the same bit + // validity as `T` [1]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/cell/struct.UnsafeCell.html#memory-layout: + // + // `UnsafeCell<T>` has the same in-memory representation as + // its inner type `T`. A consequence of this guarantee is that + // it is possible to convert between `T` and `UnsafeCell<T>`. + unsafe impl<$($tyvar)?> TransmuteFrom<$atomic, Valid, Valid> for core::cell::UnsafeCell<$prim> {} + // SAFETY: See previous safety comment. + unsafe impl<$($tyvar)?> TransmuteFrom<core::cell::UnsafeCell<$prim>, Valid, Valid> for $atomic {} + )* + }}; + } + + #[cfg(target_has_atomic = "8")] + #[cfg_attr(doc_cfg, doc(cfg(target_has_atomic = "8")))] + mod atomic_8 { + use core::sync::atomic::{AtomicBool, AtomicI8, AtomicU8}; + + use super::*; + + impl_traits_for_atomics!(AtomicU8[u8], AtomicI8[i8]); + + impl_known_layout!(AtomicBool); + impl_for_transmute_from!(=> FromZeros for AtomicBool [bool]); + impl_for_transmute_from!(=> TryFromBytes for AtomicBool [bool]); + impl_for_transmute_from!(=> IntoBytes for AtomicBool [bool]); + + // SAFETY: Per [1], `AtomicBool`, `AtomicU8`, and `AtomicI8` have the + // same size as `bool`, `u8`, and `i8` respectively. Since a type's + // alignment cannot be smaller than 1 [2], and since its alignment + // cannot be greater than its size [3], the only possible value for the + // alignment is 1. Thus, it is sound to implement `Unaligned`. + // + // [1] Per (for example) https://doc.rust-lang.org/1.81.0/std/sync/atomic/struct.AtomicU8.html: + // + // This type has the same size, alignment, and bit validity as the + // underlying integer type + // + // [2] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#size-and-alignment: + // + // Alignment is measured in bytes, and must be at least 1. + // + // [3] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#size-and-alignment: + // + // The size of a value is always a multiple of its alignment. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl!(AtomicBool: Unaligned); + unsafe_impl!(AtomicU8: Unaligned); + unsafe_impl!(AtomicI8: Unaligned); + assert_unaligned!(AtomicBool, AtomicU8, AtomicI8); + }; + + // SAFETY: `AtomicU8`, `AtomicI8`, and `AtomicBool` have the same size + // and bit validity as `u8`, `i8`, and `bool` respectively [1][2][3]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicU8.html: + // + // This type has the same size, alignment, and bit validity as the + // underlying integer type, `u8`. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicI8.html: + // + // This type has the same size, alignment, and bit validity as the + // underlying integer type, `i8`. + // + // [3] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicBool.html: + // + // This type has the same size, alignment, and bit validity a `bool`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl_transmute_from_for_atomic!( + => AtomicU8 [u8], + => AtomicI8 [i8], + => AtomicBool [bool] + ) + }; + } + + #[cfg(target_has_atomic = "16")] + #[cfg_attr(doc_cfg, doc(cfg(target_has_atomic = "16")))] + mod atomic_16 { + use core::sync::atomic::{AtomicI16, AtomicU16}; + + use super::*; + + impl_traits_for_atomics!(AtomicU16[u16], AtomicI16[i16]); + + // SAFETY: `AtomicU16` and `AtomicI16` have the same size and bit + // validity as `u16` and `i16` respectively [1][2]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicU16.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `u16`. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicI16.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `i16`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl_transmute_from_for_atomic!(=> AtomicU16 [u16], => AtomicI16 [i16]) + }; + } + + #[cfg(target_has_atomic = "32")] + #[cfg_attr(doc_cfg, doc(cfg(target_has_atomic = "32")))] + mod atomic_32 { + use core::sync::atomic::{AtomicI32, AtomicU32}; + + use super::*; + + impl_traits_for_atomics!(AtomicU32[u32], AtomicI32[i32]); + + // SAFETY: `AtomicU32` and `AtomicI32` have the same size and bit + // validity as `u32` and `i32` respectively [1][2]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicU32.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `u32`. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicI32.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `i32`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl_transmute_from_for_atomic!(=> AtomicU32 [u32], => AtomicI32 [i32]) + }; + } + + #[cfg(target_has_atomic = "64")] + #[cfg_attr(doc_cfg, doc(cfg(target_has_atomic = "64")))] + mod atomic_64 { + use core::sync::atomic::{AtomicI64, AtomicU64}; + + use super::*; + + impl_traits_for_atomics!(AtomicU64[u64], AtomicI64[i64]); + + // SAFETY: `AtomicU64` and `AtomicI64` have the same size and bit + // validity as `u64` and `i64` respectively [1][2]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicU64.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `u64`. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicI64.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `i64`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl_transmute_from_for_atomic!(=> AtomicU64 [u64], => AtomicI64 [i64]) + }; + } + + #[cfg(target_has_atomic = "ptr")] + #[cfg_attr(doc_cfg, doc(cfg(target_has_atomic = "ptr")))] + mod atomic_ptr { + use core::sync::atomic::{AtomicIsize, AtomicPtr, AtomicUsize}; + + use super::*; + + impl_traits_for_atomics!(AtomicUsize[usize], AtomicIsize[isize]); + + // FIXME(#170): Implement `FromBytes` and `IntoBytes` once we implement + // those traits for `*mut T`. + impl_known_layout!(T => AtomicPtr<T>); + impl_for_transmute_from!(T => TryFromBytes for AtomicPtr<T> [*mut T]); + impl_for_transmute_from!(T => FromZeros for AtomicPtr<T> [*mut T]); + + // SAFETY: `AtomicUsize` and `AtomicIsize` have the same size and bit + // validity as `usize` and `isize` respectively [1][2]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicUsize.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `usize`. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicIsize.html: + // + // This type has the same size and bit validity as the underlying + // integer type, `isize`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + unsafe_impl_transmute_from_for_atomic!(=> AtomicUsize [usize], => AtomicIsize [isize]) + }; + + // SAFETY: Per + // https://doc.rust-lang.org/1.85.0/std/sync/atomic/struct.AtomicPtr.html: + // + // This type has the same size and bit validity as a `*mut T`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { unsafe_impl_transmute_from_for_atomic!(T => AtomicPtr<T> [*mut T]) }; + } +} + +// SAFETY: Per reference [1]: "For all T, the following are guaranteed: +// size_of::<PhantomData<T>>() == 0 align_of::<PhantomData<T>>() == 1". This +// gives: +// - `Immutable`: `PhantomData` has no fields. +// - `TryFromBytes` (with no validator), `FromZeros`, `FromBytes`: There is only +// one possible sequence of 0 bytes, and `PhantomData` is inhabited. +// - `IntoBytes`: Since `PhantomData` has size 0, it contains no padding bytes. +// - `Unaligned`: Per the preceding reference, `PhantomData` has alignment 1. +// +// [1] https://doc.rust-lang.org/1.81.0/std/marker/struct.PhantomData.html#layout-1 +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(T: ?Sized => Immutable for PhantomData<T>); + unsafe_impl!(T: ?Sized => TryFromBytes for PhantomData<T>); + unsafe_impl!(T: ?Sized => FromZeros for PhantomData<T>); + unsafe_impl!(T: ?Sized => FromBytes for PhantomData<T>); + unsafe_impl!(T: ?Sized => IntoBytes for PhantomData<T>); + unsafe_impl!(T: ?Sized => Unaligned for PhantomData<T>); + assert_unaligned!(PhantomData<()>, PhantomData<u8>, PhantomData<u64>); +}; + +impl_for_transmute_from!(T: TryFromBytes => TryFromBytes for Wrapping<T>[T]); +impl_for_transmute_from!(T: FromZeros => FromZeros for Wrapping<T>[T]); +impl_for_transmute_from!(T: FromBytes => FromBytes for Wrapping<T>[T]); +impl_for_transmute_from!(T: IntoBytes => IntoBytes for Wrapping<T>[T]); +assert_unaligned!(Wrapping<()>, Wrapping<u8>); + +// SAFETY: Per [1], `Wrapping<T>` has the same layout as `T`. Since its single +// field (of type `T`) is public, it would be a breaking change to add or remove +// fields. Thus, we know that `Wrapping<T>` contains a `T` (as opposed to just +// having the same size and alignment as `T`) with no pre- or post-padding. +// Thus, `Wrapping<T>` must have `UnsafeCell`s covering the same byte ranges as +// `Inner = T`. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/num/struct.Wrapping.html#layout-1: +// +// `Wrapping<T>` is guaranteed to have the same layout and ABI as `T` +const _: () = unsafe { unsafe_impl!(T: Immutable => Immutable for Wrapping<T>) }; + +// SAFETY: Per [1] in the preceding safety comment, `Wrapping<T>` has the same +// alignment as `T`. +const _: () = unsafe { unsafe_impl!(T: Unaligned => Unaligned for Wrapping<T>) }; + +// SAFETY: `TryFromBytes` (with no validator), `FromZeros`, `FromBytes`: +// `MaybeUninit<T>` has no restrictions on its contents. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(T => TryFromBytes for CoreMaybeUninit<T>); + unsafe_impl!(T => FromZeros for CoreMaybeUninit<T>); + unsafe_impl!(T => FromBytes for CoreMaybeUninit<T>); +}; + +// SAFETY: `MaybeUninit<T>` has `UnsafeCell`s covering the same byte ranges as +// `Inner = T`. This is not explicitly documented, but it can be inferred. Per +// [1], `MaybeUninit<T>` has the same size as `T`. Further, note the signature +// of `MaybeUninit::assume_init_ref` [2]: +// +// pub unsafe fn assume_init_ref(&self) -> &T +// +// If the argument `&MaybeUninit<T>` and the returned `&T` had `UnsafeCell`s at +// different offsets, this would be unsound. Its existence is proof that this is +// not the case. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/mem/union.MaybeUninit.html#layout-1: +// +// `MaybeUninit<T>` is guaranteed to have the same size, alignment, and ABI as +// `T`. +// +// [2] https://doc.rust-lang.org/1.81.0/std/mem/union.MaybeUninit.html#method.assume_init_ref +const _: () = unsafe { unsafe_impl!(T: Immutable => Immutable for CoreMaybeUninit<T>) }; + +// SAFETY: Per [1] in the preceding safety comment, `MaybeUninit<T>` has the +// same alignment as `T`. +const _: () = unsafe { unsafe_impl!(T: Unaligned => Unaligned for CoreMaybeUninit<T>) }; +assert_unaligned!(CoreMaybeUninit<()>, CoreMaybeUninit<u8>); + +// SAFETY: `ManuallyDrop<T>` has the same layout as `T` [1]. This strongly +// implies, but does not guarantee, that it contains `UnsafeCell`s covering the +// same byte ranges as in `T`. However, it also implements `Defer<Target = T>` +// [2], which provides the ability to convert `&ManuallyDrop<T> -> &T`. This, +// combined with having the same size as `T`, implies that `ManuallyDrop<T>` +// exactly contains a `T` with the same fields and `UnsafeCell`s covering the +// same byte ranges, or else the `Deref` impl would permit safe code to obtain +// different shared references to the same region of memory with different +// `UnsafeCell` coverage, which would in turn permit interior mutation that +// would violate the invariants of a shared reference. +// +// [1] Per https://doc.rust-lang.org/1.85.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop<T>` is guaranteed to have the same layout and bit validity as +// `T` +// +// [2] https://doc.rust-lang.org/1.85.0/std/mem/struct.ManuallyDrop.html#impl-Deref-for-ManuallyDrop%3CT%3E +const _: () = unsafe { unsafe_impl!(T: ?Sized + Immutable => Immutable for ManuallyDrop<T>) }; + +impl_for_transmute_from!(T: ?Sized + TryFromBytes => TryFromBytes for ManuallyDrop<T>[T]); +impl_for_transmute_from!(T: ?Sized + FromZeros => FromZeros for ManuallyDrop<T>[T]); +impl_for_transmute_from!(T: ?Sized + FromBytes => FromBytes for ManuallyDrop<T>[T]); +impl_for_transmute_from!(T: ?Sized + IntoBytes => IntoBytes for ManuallyDrop<T>[T]); +// SAFETY: `ManuallyDrop<T>` has the same layout as `T` [1], and thus has the +// same alignment as `T`. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop<T>` is guaranteed to have the same layout and bit validity as +// `T` +const _: () = unsafe { unsafe_impl!(T: ?Sized + Unaligned => Unaligned for ManuallyDrop<T>) }; +assert_unaligned!(ManuallyDrop<()>, ManuallyDrop<u8>); + +const _: () = { + #[allow( + non_camel_case_types, + missing_copy_implementations, + missing_debug_implementations, + missing_docs + )] + pub enum value {} + + // SAFETY: See safety comment on `ProjectToTag`. + unsafe impl<T: ?Sized> HasTag for ManuallyDrop<T> { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type Tag = (); + + // SAFETY: It is trivially sound to project any pointer to a pointer to + // a type of size zero and alignment 1 (which `()` is [1]). Such a + // pointer will trivially satisfy its aliasing and validity requirements + // (since it has a zero-sized referent), and its alignment requirement + // (since it is aligned to 1). + // + // [1] Per https://doc.rust-lang.org/1.92.0/reference/type-layout.html#r-layout.tuple.unit: + // + // [T]he unit tuple (`()`)... is guaranteed as a zero-sized type to + // have a size of 0 and an alignment of 1. + type ProjectToTag = crate::pointer::cast::CastToUnit; + } + + // SAFETY: `ManuallyDrop<T>` has a field of type `T` at offset `0` without + // any safety invariants beyond those of `T`. Its existence is not + // explicitly documented, but it can be inferred; per [1] `ManuallyDrop<T>` + // has the same size and bit validity as `T`. This field is not literally + // public, but is effectively so; the field can be transparently: + // + // - initialized via `ManuallyDrop::new` + // - moved via `ManuallyDrop::into_inner` + // - referenced via `ManuallyDrop::deref` + // - exclusively referenced via `ManuallyDrop::deref_mut` + // + // We call this field `value`, both because that is both the name of this + // private field, and because it is the name it is referred to in the public + // documentation of `ManuallyDrop::new`, `ManuallyDrop::into_inner`, + // `ManuallyDrop::take` and `ManuallyDrop::drop`. + unsafe impl<T: ?Sized> + HasField<value, { crate::STRUCT_VARIANT_ID }, { crate::ident_id!(value) }> + for ManuallyDrop<T> + { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type Type = T; + + #[inline(always)] + fn project(slf: PtrInner<'_, Self>) -> *mut T { + // SAFETY: `ManuallyDrop<T>` has the same layout and bit validity as + // `T` [1]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/mem/struct.ManuallyDrop.html: + // + // `ManuallyDrop<T>` is guaranteed to have the same layout and bit + // validity as `T` + #[allow(clippy::as_conversions)] + return slf.as_ptr() as *mut T; + } + } +}; + +impl_for_transmute_from!(T: ?Sized + TryFromBytes => TryFromBytes for Cell<T>[T]); +impl_for_transmute_from!(T: ?Sized + FromZeros => FromZeros for Cell<T>[T]); +impl_for_transmute_from!(T: ?Sized + FromBytes => FromBytes for Cell<T>[T]); +impl_for_transmute_from!(T: ?Sized + IntoBytes => IntoBytes for Cell<T>[T]); +// SAFETY: `Cell<T>` has the same in-memory representation as `T` [1], and thus +// has the same alignment as `T`. +// +// [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.Cell.html#memory-layout: +// +// `Cell<T>` has the same in-memory representation as its inner type `T`. +const _: () = unsafe { unsafe_impl!(T: ?Sized + Unaligned => Unaligned for Cell<T>) }; + +impl_for_transmute_from!(T: ?Sized + FromZeros => FromZeros for UnsafeCell<T>[T]); +impl_for_transmute_from!(T: ?Sized + FromBytes => FromBytes for UnsafeCell<T>[T]); +impl_for_transmute_from!(T: ?Sized + IntoBytes => IntoBytes for UnsafeCell<T>[T]); +// SAFETY: `UnsafeCell<T>` has the same in-memory representation as `T` [1], and +// thus has the same alignment as `T`. +// +// [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: +// +// `UnsafeCell<T>` has the same in-memory representation as its inner type +// `T`. +const _: () = unsafe { unsafe_impl!(T: ?Sized + Unaligned => Unaligned for UnsafeCell<T>) }; +assert_unaligned!(UnsafeCell<()>, UnsafeCell<u8>); + +// SAFETY: See safety comment in `is_bit_valid` impl. +unsafe impl<T: TryFromBytes + ?Sized> TryFromBytes for UnsafeCell<T> { + #[allow(clippy::missing_inline_in_public_items)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + #[inline(always)] + fn is_bit_valid<A>(candidate: Maybe<'_, Self, A>) -> bool + where + A: invariant::Alignment, + { + T::is_bit_valid(candidate.transmute::<_, _, BecauseImmutable>()) + } +} + +// SAFETY: Per the reference [1]: +// +// An array of `[T; N]` has a size of `size_of::<T>() * N` and the same +// alignment of `T`. Arrays are laid out so that the zero-based `nth` element +// of the array is offset from the start of the array by `n * size_of::<T>()` +// bytes. +// +// ... +// +// Slices have the same layout as the section of the array they slice. +// +// In other words, the layout of a `[T]` or `[T; N]` is a sequence of `T`s laid +// out back-to-back with no bytes in between. Therefore, `[T]` or `[T; N]` are +// `Immutable`, `TryFromBytes`, `FromZeros`, `FromBytes`, and `IntoBytes` if `T` +// is (respectively). Furthermore, since an array/slice has "the same alignment +// of `T`", `[T]` and `[T; N]` are `Unaligned` if `T` is. +// +// Note that we don't `assert_unaligned!` for slice types because +// `assert_unaligned!` uses `align_of`, which only works for `Sized` types. +// +// [1] https://doc.rust-lang.org/1.81.0/reference/type-layout.html#array-layout +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(const N: usize, T: Immutable => Immutable for [T; N]); + unsafe_impl!(const N: usize, T: TryFromBytes => TryFromBytes for [T; N]; |c| { + let c: Ptr<'_, [ReadOnly<T>; N], _> = c.cast::<_, crate::pointer::cast::CastSized, _>(); + let c: Ptr<'_, [ReadOnly<T>], _> = c.as_slice(); + let c: Ptr<'_, ReadOnly<[T]>, _> = c.cast::<_, crate::pointer::cast::CastUnsized, _>(); + + // Note that this call may panic, but it would still be sound even if it + // did. `is_bit_valid` does not promise that it will not panic (in fact, + // it explicitly warns that it's a possibility), and we have not + // violated any safety invariants that we must fix before returning. + <[T] as TryFromBytes>::is_bit_valid(c) + }); + unsafe_impl!(const N: usize, T: FromZeros => FromZeros for [T; N]); + unsafe_impl!(const N: usize, T: FromBytes => FromBytes for [T; N]); + unsafe_impl!(const N: usize, T: IntoBytes => IntoBytes for [T; N]); + unsafe_impl!(const N: usize, T: Unaligned => Unaligned for [T; N]); + assert_unaligned!([(); 0], [(); 1], [u8; 0], [u8; 1]); + unsafe_impl!(T: Immutable => Immutable for [T]); + unsafe_impl!(T: TryFromBytes => TryFromBytes for [T]; |c| { + let c: Ptr<'_, [ReadOnly<T>], _> = c.cast::<_, crate::pointer::cast::CastUnsized, _>(); + + // SAFETY: Per the reference [1]: + // + // An array of `[T; N]` has a size of `size_of::<T>() * N` and the + // same alignment of `T`. Arrays are laid out so that the zero-based + // `nth` element of the array is offset from the start of the array by + // `n * size_of::<T>()` bytes. + // + // ... + // + // Slices have the same layout as the section of the array they slice. + // + // In other words, the layout of a `[T] is a sequence of `T`s laid out + // back-to-back with no bytes in between. If all elements in `candidate` + // are `is_bit_valid`, so too is `candidate`. + // + // Note that any of the below calls may panic, but it would still be + // sound even if it did. `is_bit_valid` does not promise that it will + // not panic (in fact, it explicitly warns that it's a possibility), and + // we have not violated any safety invariants that we must fix before + // returning. + c.iter().all(<T as TryFromBytes>::is_bit_valid) + }); + unsafe_impl!(T: FromZeros => FromZeros for [T]); + unsafe_impl!(T: FromBytes => FromBytes for [T]); + unsafe_impl!(T: IntoBytes => IntoBytes for [T]); + unsafe_impl!(T: Unaligned => Unaligned for [T]); +}; + +// SAFETY: +// - `Immutable`: Raw pointers do not contain any `UnsafeCell`s. +// - `FromZeros`: For thin pointers (note that `T: Sized`), the zero pointer is +// considered "null". [1] No operations which require provenance are legal on +// null pointers, so this is not a footgun. +// - `TryFromBytes`: By the same reasoning as for `FromZeroes`, we can implement +// `TryFromBytes` for thin pointers provided that +// [`TryFromByte::is_bit_valid`] only produces `true` for zeroed bytes. +// +// NOTE(#170): Implementing `FromBytes` and `IntoBytes` for raw pointers would +// be sound, but carries provenance footguns. We want to support `FromBytes` and +// `IntoBytes` for raw pointers eventually, but we are holding off until we can +// figure out how to address those footguns. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/ptr/fn.null.html: +// +// Creates a null raw pointer. +// +// This function is equivalent to zero-initializing the pointer: +// `MaybeUninit::<*const T>::zeroed().assume_init()`. +// +// The resulting pointer has the address 0. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(T: ?Sized => Immutable for *const T); + unsafe_impl!(T: ?Sized => Immutable for *mut T); + unsafe_impl!(T => TryFromBytes for *const T; |c| pointer::is_zeroed(c)); + unsafe_impl!(T => FromZeros for *const T); + unsafe_impl!(T => TryFromBytes for *mut T; |c| pointer::is_zeroed(c)); + unsafe_impl!(T => FromZeros for *mut T); +}; + +// SAFETY: `NonNull<T>` self-evidently does not contain `UnsafeCell`s. This is +// not a proof, but we are accepting this as a known risk per #1358. +const _: () = unsafe { unsafe_impl!(T: ?Sized => Immutable for NonNull<T>) }; + +// SAFETY: Reference types do not contain any `UnsafeCell`s. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl!(T: ?Sized => Immutable for &'_ T); + unsafe_impl!(T: ?Sized => Immutable for &'_ mut T); +}; + +// SAFETY: `Option` is not `#[non_exhaustive]` [1], which means that the types +// in its variants cannot change, and no new variants can be added. `Option<T>` +// does not contain any `UnsafeCell`s outside of `T`. [1] +// +// [1] https://doc.rust-lang.org/core/option/enum.Option.html +const _: () = unsafe { unsafe_impl!(T: Immutable => Immutable for Option<T>) }; + +mod tuples { + use super::*; + + /// Generates various trait implementations for tuples. + /// + /// # Safety + /// + /// `impl_tuple!` should be provided name-number pairs, where each number is + /// the ordinal of the preceding type name. + macro_rules! impl_tuple { + // Entry point. + ($($T:ident $I:tt),+ $(,)?) => { + crate::util::macros::__unsafe(); + impl_tuple!(@all [] [$($T $I)+]); + }; + + // Build up the set of tuple types (i.e., `(A,)`, `(A, B)`, `(A, B, C)`, + // etc.) Trait implementations that do not depend on field index may be + // added to this branch. + (@all [$($head_T:ident $head_I:tt)*] [$next_T:ident $next_I:tt $($tail:tt)*]) => { + // SAFETY: If all fields of the tuple `Self` are `Immutable`, so too is `Self`. + unsafe_impl!($($head_T: Immutable,)* $next_T: Immutable => Immutable for ($($head_T,)* $next_T,)); + + // SAFETY: If all fields in `c` are `is_bit_valid`, so too is `c`. + unsafe_impl!($($head_T: TryFromBytes,)* $next_T: TryFromBytes => TryFromBytes for ($($head_T,)* $next_T,); |c| { + let mut c = c; + $(TryFromBytes::is_bit_valid(into_inner!(c.reborrow().project::<_, { crate::STRUCT_VARIANT_ID }, { crate::ident_id!($head_I) }>())) &&)* + TryFromBytes::is_bit_valid(into_inner!(c.reborrow().project::<_, { crate::STRUCT_VARIANT_ID }, { crate::ident_id!($next_I) }>())) + }); + + // SAFETY: If all fields in `Self` are `FromZeros`, so too is `Self`. + unsafe_impl!($($head_T: FromZeros,)* $next_T: FromZeros => FromZeros for ($($head_T,)* $next_T,)); + + // SAFETY: If all fields in `Self` are `FromBytes`, so too is `Self`. + unsafe_impl!($($head_T: FromBytes,)* $next_T: FromBytes => FromBytes for ($($head_T,)* $next_T,)); + + // SAFETY: See safety comment on `ProjectToTag`. + unsafe impl<$($head_T,)* $next_T> crate::HasTag for ($($head_T,)* $next_T,) { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized + {} + + type Tag = (); + + // SAFETY: It is trivially sound to project any pointer to a + // pointer to a type of size zero and alignment 1 (which `()` is + // [1]). Such a pointer will trivially satisfy its aliasing and + // validity requirements (since it has a zero-sized referent), + // and its alignment requirement (since it is aligned to 1). + // + // [1] Per https://doc.rust-lang.org/1.92.0/reference/type-layout.html#r-layout.tuple.unit: + // + // [T]he unit tuple (`()`)... is guaranteed as a zero-sized + // type to have a size of 0 and an alignment of 1. + type ProjectToTag = crate::pointer::cast::CastToUnit; + } + + // Generate impls that depend on tuple index. + impl_tuple!(@variants + [$($head_T $head_I)* $next_T $next_I] + [] + [$($head_T $head_I)* $next_T $next_I] + ); + + // Recurse to next tuple size + impl_tuple!(@all [$($head_T $head_I)* $next_T $next_I] [$($tail)*]); + }; + (@all [$($head_T:ident $head_I:tt)*] []) => {}; + + // Emit trait implementations that depend on field index. + (@variants + // The full tuple definition in type–index pairs. + [$($AllT:ident $AllI:tt)+] + // Types before the current index. + [$($BeforeT:ident)*] + // The types and indices at and after the current index. + [$CurrT:ident $CurrI:tt $($AfterT:ident $AfterI:tt)*] + ) => { + // SAFETY: + // - `Self` is a struct (albeit anonymous), so `VARIANT_ID` is + // `STRUCT_VARIANT_ID`. + // - `$CurrI` is the field at index `$CurrI`, so `FIELD_ID` is + // `zerocopy::ident_id!($CurrI)` + // - `()` has the same visibility as the `.$CurrI` field (ie, `.0`, + // `.1`, etc) + // - `Type` has the same type as `$CurrI`; i.e., `$CurrT`. + unsafe impl<$($AllT),+> crate::HasField< + (), + { crate::STRUCT_VARIANT_ID }, + { crate::ident_id!($CurrI)} + > for ($($AllT,)+) { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized + {} + + type Type = $CurrT; + + #[inline(always)] + fn project(slf: crate::PtrInner<'_, Self>) -> *mut Self::Type { + let slf = slf.as_non_null().as_ptr(); + // SAFETY: `PtrInner` promises it references either a zero-sized + // byte range, or else will reference a byte range that is + // entirely contained within an allocated object. In either + // case, this guarantees that `(*slf).$CurrI` is in-bounds of + // `slf`. + unsafe { core::ptr::addr_of_mut!((*slf).$CurrI) } + } + } + + // SAFETY: See comments on items. + unsafe impl<Aliasing, Alignment, $($AllT),+> crate::ProjectField< + (), + (Aliasing, Alignment, crate::invariant::Uninit), + { crate::STRUCT_VARIANT_ID }, + { crate::ident_id!($CurrI)} + > for ($($AllT,)+) + where + Aliasing: crate::invariant::Aliasing, + Alignment: crate::invariant::Alignment, + { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized + {} + + // SAFETY: Tuples are product types whose fields are + // well-aligned, so projection preserves both the alignment and + // validity invariants of the outer pointer. + type Invariants = (Aliasing, Alignment, crate::invariant::Uninit); + + // SAFETY: Tuples are product types and so projection is infallible; + type Error = core::convert::Infallible; + } + + // SAFETY: See comments on items. + unsafe impl<Aliasing, Alignment, $($AllT),+> crate::ProjectField< + (), + (Aliasing, Alignment, crate::invariant::Initialized), + { crate::STRUCT_VARIANT_ID }, + { crate::ident_id!($CurrI)} + > for ($($AllT,)+) + where + Aliasing: crate::invariant::Aliasing, + Alignment: crate::invariant::Alignment, + { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized + {} + + // SAFETY: Tuples are product types whose fields are + // well-aligned, so projection preserves both the alignment and + // validity invariants of the outer pointer. + type Invariants = (Aliasing, Alignment, crate::invariant::Initialized); + + // SAFETY: Tuples are product types and so projection is infallible; + type Error = core::convert::Infallible; + } + + // SAFETY: See comments on items. + unsafe impl<Aliasing, Alignment, $($AllT),+> crate::ProjectField< + (), + (Aliasing, Alignment, crate::invariant::Valid), + { crate::STRUCT_VARIANT_ID }, + { crate::ident_id!($CurrI)} + > for ($($AllT,)+) + where + Aliasing: crate::invariant::Aliasing, + Alignment: crate::invariant::Alignment, + { + #[inline] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized + {} + + // SAFETY: Tuples are product types whose fields are + // well-aligned, so projection preserves both the alignment and + // validity invariants of the outer pointer. + type Invariants = (Aliasing, Alignment, crate::invariant::Valid); + + // SAFETY: Tuples are product types and so projection is infallible; + type Error = core::convert::Infallible; + } + + // Recurse to the next index. + impl_tuple!(@variants [$($AllT $AllI)+] [$($BeforeT)* $CurrT] [$($AfterT $AfterI)*]); + }; + (@variants [$($AllT:ident $AllI:tt)+] [$($BeforeT:ident)*] []) => {}; + } + + // SAFETY: `impl_tuple` is provided name-number pairs, where number is the + // ordinal of the name. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + impl_tuple! { + A 0, + B 1, + C 2, + D 3, + E 4, + F 5, + G 6, + H 7, + I 8, + J 9, + K 10, + L 11, + M 12, + N 13, + O 14, + P 15, + Q 16, + R 17, + S 18, + T 19, + U 20, + V 21, + W 22, + X 23, + Y 24, + Z 25, + }; + }; +} + +// SIMD support +// +// Per the Unsafe Code Guidelines Reference [1]: +// +// Packed SIMD vector types are `repr(simd)` homogeneous tuple-structs +// containing `N` elements of type `T` where `N` is a power-of-two and the +// size and alignment requirements of `T` are equal: +// +// ```rust +// #[repr(simd)] +// struct Vector<T, N>(T_0, ..., T_(N - 1)); +// ``` +// +// ... +// +// The size of `Vector` is `N * size_of::<T>()` and its alignment is an +// implementation-defined function of `T` and `N` greater than or equal to +// `align_of::<T>()`. +// +// ... +// +// Vector elements are laid out in source field order, enabling random access +// to vector elements by reinterpreting the vector as an array: +// +// ```rust +// union U { +// vec: Vector<T, N>, +// arr: [T; N] +// } +// +// assert_eq!(size_of::<Vector<T, N>>(), size_of::<[T; N]>()); +// assert!(align_of::<Vector<T, N>>() >= align_of::<[T; N]>()); +// +// unsafe { +// let u = U { vec: Vector<T, N>(t_0, ..., t_(N - 1)) }; +// +// assert_eq!(u.vec.0, u.arr[0]); +// // ... +// assert_eq!(u.vec.(N - 1), u.arr[N - 1]); +// } +// ``` +// +// Given this background, we can observe that: +// - The size and bit pattern requirements of a SIMD type are equivalent to the +// equivalent array type. Thus, for any SIMD type whose primitive `T` is +// `Immutable`, `TryFromBytes`, `FromZeros`, `FromBytes`, or `IntoBytes`, that +// SIMD type is also `Immutable`, `TryFromBytes`, `FromZeros`, `FromBytes`, or +// `IntoBytes` respectively. +// - Since no upper bound is placed on the alignment, no SIMD type can be +// guaranteed to be `Unaligned`. +// +// Also per [1]: +// +// This chapter represents the consensus from issue #38. The statements in +// here are not (yet) "guaranteed" not to change until an RFC ratifies them. +// +// See issue #38 [2]. While this behavior is not technically guaranteed, the +// likelihood that the behavior will change such that SIMD types are no longer +// `TryFromBytes`, `FromZeros`, `FromBytes`, or `IntoBytes` is next to zero, as +// that would defeat the entire purpose of SIMD types. Nonetheless, we put this +// behavior behind the `simd` Cargo feature, which requires consumers to opt +// into this stability hazard. +// +// [1] https://rust-lang.github.io/unsafe-code-guidelines/layout/packed-simd-vectors.html +// [2] https://github.com/rust-lang/unsafe-code-guidelines/issues/38 +#[cfg(feature = "simd")] +#[cfg_attr(doc_cfg, doc(cfg(feature = "simd")))] +mod simd { + /// Defines a module which implements `TryFromBytes`, `FromZeros`, + /// `FromBytes`, and `IntoBytes` for a set of types from a module in + /// `core::arch`. + /// + /// `$arch` is both the name of the defined module and the name of the + /// module in `core::arch`, and `$typ` is the list of items from that module + /// to implement `FromZeros`, `FromBytes`, and `IntoBytes` for. + #[allow(unused_macros)] // `allow(unused_macros)` is needed because some + // target/feature combinations don't emit any impls + // and thus don't use this macro. + macro_rules! simd_arch_mod { + ($(#[cfg $cfg:tt])* $(#[cfg_attr $cfg_attr:tt])? $arch:ident, $mod:ident, $($typ:ident),*) => { + $(#[cfg $cfg])* + #[cfg_attr(doc_cfg, doc(cfg $($cfg)*))] + $(#[cfg_attr $cfg_attr])? + mod $mod { + use core::arch::$arch::{$($typ),*}; + + use crate::*; + impl_known_layout!($($typ),*); + // SAFETY: See comment on module definition for justification. + #[allow(clippy::multiple_unsafe_ops_per_block)] + const _: () = unsafe { + $( unsafe_impl!($typ: Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes); )* + }; + } + }; + } + + #[rustfmt::skip] + const _: () = { + simd_arch_mod!( + #[cfg(target_arch = "x86")] + x86, x86, __m128, __m128d, __m128i, __m256, __m256d, __m256i + ); + #[cfg(not(no_zerocopy_simd_x86_avx12_1_89_0))] + simd_arch_mod!( + #[cfg(target_arch = "x86")] + #[cfg_attr(doc_cfg, doc(cfg(rust = "1.89.0")))] + x86, x86_nightly, __m512bh, __m512, __m512d, __m512i + ); + simd_arch_mod!( + #[cfg(target_arch = "x86_64")] + x86_64, x86_64, __m128, __m128d, __m128i, __m256, __m256d, __m256i + ); + #[cfg(not(no_zerocopy_simd_x86_avx12_1_89_0))] + simd_arch_mod!( + #[cfg(target_arch = "x86_64")] + #[cfg_attr(doc_cfg, doc(cfg(rust = "1.89.0")))] + x86_64, x86_64_nightly, __m512bh, __m512, __m512d, __m512i + ); + simd_arch_mod!( + #[cfg(target_arch = "wasm32")] + wasm32, wasm32, v128 + ); + simd_arch_mod!( + #[cfg(all(feature = "simd-nightly", target_arch = "powerpc"))] + powerpc, powerpc, vector_bool_long, vector_double, vector_signed_long, vector_unsigned_long + ); + simd_arch_mod!( + #[cfg(all(feature = "simd-nightly", target_arch = "powerpc64"))] + powerpc64, powerpc64, vector_bool_long, vector_double, vector_signed_long, vector_unsigned_long + ); + // NOTE: NEON intrinsics were broken on big-endian platforms from their stabilization up to + // Rust 1.87. (Context in https://github.com/rust-lang/stdarch/issues/1484). Support is + // split in two different version ranges on top of the base configuration, requiring either + // little endian or the more recent version to be detected as well. + #[cfg(not(no_zerocopy_aarch64_simd_1_59_0))] + simd_arch_mod!( + #[cfg(all( + target_arch = "aarch64", + any( + target_endian = "little", + not(no_zerocopy_aarch64_simd_be_1_87_0) + ) + ))] + #[cfg_attr( + doc_cfg, + doc(cfg(all(target_arch = "aarch64", any( + all(rust = "1.59.0", target_endian = "little"), + rust = "1.87.0", + )))) + )] + aarch64, aarch64, float32x2_t, float32x4_t, float64x1_t, float64x2_t, int8x8_t, int8x8x2_t, + int8x8x3_t, int8x8x4_t, int8x16_t, int8x16x2_t, int8x16x3_t, int8x16x4_t, int16x4_t, + int16x8_t, int32x2_t, int32x4_t, int64x1_t, int64x2_t, poly8x8_t, poly8x8x2_t, poly8x8x3_t, + poly8x8x4_t, poly8x16_t, poly8x16x2_t, poly8x16x3_t, poly8x16x4_t, poly16x4_t, poly16x8_t, + poly64x1_t, poly64x2_t, uint8x8_t, uint8x8x2_t, uint8x8x3_t, uint8x8x4_t, uint8x16_t, + uint8x16x2_t, uint8x16x3_t, uint8x16x4_t, uint16x4_t, uint16x4x2_t, uint16x4x3_t, + uint16x4x4_t, uint16x8_t, uint32x2_t, uint32x4_t, uint64x1_t, uint64x2_t + ); + }; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_impls() { + // A type that can supply test cases for testing + // `TryFromBytes::is_bit_valid`. All types passed to `assert_impls!` + // must implement this trait; that macro uses it to generate runtime + // tests for `TryFromBytes` impls. + // + // All `T: FromBytes` types are provided with a blanket impl. Other + // types must implement `TryFromBytesTestable` directly (ie using + // `impl_try_from_bytes_testable!`). + trait TryFromBytesTestable { + fn with_passing_test_cases<F: Fn(Box<ReadOnly<Self>>)>(f: F); + fn with_failing_test_cases<F: Fn(&mut [u8])>(f: F); + } + + impl<T: FromBytes> TryFromBytesTestable for T { + fn with_passing_test_cases<F: Fn(Box<ReadOnly<Self>>)>(f: F) { + // Test with a zeroed value. + f(ReadOnly::<Self>::new_box_zeroed().unwrap()); + + let ffs = { + let mut t = ReadOnly::new(Self::new_zeroed()); + let ptr: *mut T = ReadOnly::as_mut(&mut t); + // SAFETY: `T: FromBytes` + unsafe { ptr::write_bytes(ptr.cast::<u8>(), 0xFF, mem::size_of::<T>()) }; + t + }; + + // Test with a value initialized with 0xFF. + f(Box::new(ffs)); + } + + fn with_failing_test_cases<F: Fn(&mut [u8])>(_f: F) {} + } + + macro_rules! impl_try_from_bytes_testable_for_null_pointer_optimization { + ($($tys:ty),*) => { + $( + impl TryFromBytesTestable for Option<$tys> { + fn with_passing_test_cases<F: Fn(Box<ReadOnly<Self>>)>(f: F) { + // Test with a zeroed value. + f(Box::new(ReadOnly::new(None))); + } + + fn with_failing_test_cases<F: Fn(&mut [u8])>(f: F) { + for pos in 0..mem::size_of::<Self>() { + let mut bytes = [0u8; mem::size_of::<Self>()]; + bytes[pos] = 0x01; + f(&mut bytes[..]); + } + } + } + )* + }; + } + + // Implements `TryFromBytesTestable`. + macro_rules! impl_try_from_bytes_testable { + // Base case for recursion (when the list of types has run out). + (=> @success $($success_case:expr),* $(, @failure $($failure_case:expr),*)?) => {}; + // Implements for type(s) with no type parameters. + ($ty:ty $(,$tys:ty)* => @success $($success_case:expr),* $(, @failure $($failure_case:expr),*)?) => { + impl TryFromBytesTestable for $ty { + impl_try_from_bytes_testable!( + @methods @success $($success_case),* + $(, @failure $($failure_case),*)? + ); + } + impl_try_from_bytes_testable!($($tys),* => @success $($success_case),* $(, @failure $($failure_case),*)?); + }; + // Implements for multiple types with no type parameters. + ($($($ty:ty),* => @success $($success_case:expr), * $(, @failure $($failure_case:expr),*)?;)*) => { + $( + impl_try_from_bytes_testable!($($ty),* => @success $($success_case),* $(, @failure $($failure_case),*)*); + )* + }; + // Implements only the methods; caller must invoke this from inside + // an impl block. + (@methods @success $($success_case:expr),* $(, @failure $($failure_case:expr),*)?) => { + fn with_passing_test_cases<F: Fn(Box<ReadOnly<Self>>)>(_f: F) { + $( + let bx = Box::<Self>::from($success_case); + let ro: Box<ReadOnly<_>> = { + let raw = Box::into_raw(bx); + // SAFETY: `ReadOnly<T>` has the same layout and bit + // validity as `T`. + #[allow(clippy::as_conversions)] + unsafe { Box::from_raw(raw as *mut _) } + }; + _f(ro); + )* + } + + fn with_failing_test_cases<F: Fn(&mut [u8])>(_f: F) { + $($( + let mut case = $failure_case; + _f(case.as_mut_bytes()); + )*)? + } + }; + } + + impl_try_from_bytes_testable_for_null_pointer_optimization!( + Box<UnsafeCell<NotZerocopy>>, + &'static UnsafeCell<NotZerocopy>, + &'static mut UnsafeCell<NotZerocopy>, + NonNull<UnsafeCell<NotZerocopy>>, + fn(), + FnManyArgs, + extern "C" fn(), + ECFnManyArgs + ); + + macro_rules! bx { + ($e:expr) => { + Box::new($e) + }; + } + + // Note that these impls are only for types which are not `FromBytes`. + // `FromBytes` types are covered by a preceding blanket impl. + impl_try_from_bytes_testable!( + bool => @success true, false, + @failure 2u8, 3u8, 0xFFu8; + char => @success '\u{0}', '\u{D7FF}', '\u{E000}', '\u{10FFFF}', + @failure 0xD800u32, 0xDFFFu32, 0x110000u32; + str => @success "", "hello", "❤️🧡💛💚💙💜", + @failure [0, 159, 146, 150]; + [u8] => @success vec![].into_boxed_slice(), vec![0, 1, 2].into_boxed_slice(); + NonZeroU8, NonZeroI8, NonZeroU16, NonZeroI16, NonZeroU32, + NonZeroI32, NonZeroU64, NonZeroI64, NonZeroU128, NonZeroI128, + NonZeroUsize, NonZeroIsize + => @success Self::new(1).unwrap(), + // Doing this instead of `0` ensures that we always satisfy + // the size and alignment requirements of `Self` (whereas `0` + // may be any integer type with a different size or alignment + // than some `NonZeroXxx` types). + @failure Option::<Self>::None; + [bool; 0] => @success []; + [bool; 1] + => @success [true], [false], + @failure [2u8], [3u8], [0xFFu8]; + [bool] + => @success vec![true, false].into_boxed_slice(), vec![false, true].into_boxed_slice(), + @failure [2u8], [3u8], [0xFFu8], [0u8, 1u8, 2u8]; + Unalign<bool> + => @success Unalign::new(false), Unalign::new(true), + @failure 2u8, 0xFFu8; + ManuallyDrop<bool> + => @success ManuallyDrop::new(false), ManuallyDrop::new(true), + @failure 2u8, 0xFFu8; + ManuallyDrop<[u8]> + => @success bx!(ManuallyDrop::new([])), bx!(ManuallyDrop::new([0u8])), bx!(ManuallyDrop::new([0u8, 1u8])); + ManuallyDrop<[bool]> + => @success bx!(ManuallyDrop::new([])), bx!(ManuallyDrop::new([false])), bx!(ManuallyDrop::new([false, true])), + @failure [2u8], [3u8], [0xFFu8], [0u8, 1u8, 2u8]; + ManuallyDrop<[UnsafeCell<u8>]> + => @success bx!(ManuallyDrop::new([UnsafeCell::new(0)])), bx!(ManuallyDrop::new([UnsafeCell::new(0), UnsafeCell::new(1)])); + ManuallyDrop<[UnsafeCell<bool>]> + => @success bx!(ManuallyDrop::new([UnsafeCell::new(false)])), bx!(ManuallyDrop::new([UnsafeCell::new(false), UnsafeCell::new(true)])), + @failure [2u8], [3u8], [0xFFu8], [0u8, 1u8, 2u8]; + Wrapping<bool> + => @success Wrapping(false), Wrapping(true), + @failure 2u8, 0xFFu8; + *const NotZerocopy + => @success ptr::null::<NotZerocopy>(), + @failure [0x01; mem::size_of::<*const NotZerocopy>()]; + *mut NotZerocopy + => @success ptr::null_mut::<NotZerocopy>(), + @failure [0x01; mem::size_of::<*mut NotZerocopy>()]; + ); + + // Use the trick described in [1] to allow us to call methods + // conditional on certain trait bounds. + // + // In all of these cases, methods return `Option<R>`, where `R` is the + // return type of the method we're conditionally calling. The "real" + // implementations (the ones defined in traits using `&self`) return + // `Some`, and the default implementations (the ones defined as inherent + // methods using `&mut self`) return `None`. + // + // [1] https://github.com/dtolnay/case-studies/blob/master/autoref-specialization/README.md + mod autoref_trick { + use super::*; + + pub(super) struct AutorefWrapper<T: ?Sized>(pub(super) PhantomData<T>); + + pub(super) trait TestIsBitValidShared<T: ?Sized> { + #[allow(clippy::needless_lifetimes)] + fn test_is_bit_valid_shared<'ptr>(&self, candidate: Maybe<'ptr, T>) + -> Option<bool>; + } + + impl<T: TryFromBytes + Immutable + ?Sized> TestIsBitValidShared<T> for AutorefWrapper<T> { + #[allow(clippy::needless_lifetimes)] + fn test_is_bit_valid_shared<'ptr>( + &self, + candidate: Maybe<'ptr, T>, + ) -> Option<bool> { + Some(T::is_bit_valid(candidate)) + } + } + + pub(super) trait TestTryFromRef<T: ?Sized> { + #[allow(clippy::needless_lifetimes)] + fn test_try_from_ref<'bytes>( + &self, + bytes: &'bytes [u8], + ) -> Option<Option<&'bytes T>>; + } + + impl<T: TryFromBytes + Immutable + KnownLayout + ?Sized> TestTryFromRef<T> for AutorefWrapper<T> { + #[allow(clippy::needless_lifetimes)] + fn test_try_from_ref<'bytes>( + &self, + bytes: &'bytes [u8], + ) -> Option<Option<&'bytes T>> { + Some(T::try_ref_from_bytes(bytes).ok()) + } + } + + pub(super) trait TestTryFromMut<T: ?Sized> { + #[allow(clippy::needless_lifetimes)] + fn test_try_from_mut<'bytes>( + &self, + bytes: &'bytes mut [u8], + ) -> Option<Option<&'bytes mut T>>; + } + + impl<T: TryFromBytes + IntoBytes + KnownLayout + ?Sized> TestTryFromMut<T> for AutorefWrapper<T> { + #[allow(clippy::needless_lifetimes)] + fn test_try_from_mut<'bytes>( + &self, + bytes: &'bytes mut [u8], + ) -> Option<Option<&'bytes mut T>> { + Some(T::try_mut_from_bytes(bytes).ok()) + } + } + + pub(super) trait TestTryReadFrom<T> { + fn test_try_read_from(&self, bytes: &[u8]) -> Option<Option<T>>; + } + + impl<T: TryFromBytes> TestTryReadFrom<T> for AutorefWrapper<T> { + fn test_try_read_from(&self, bytes: &[u8]) -> Option<Option<T>> { + Some(T::try_read_from_bytes(bytes).ok()) + } + } + + pub(super) trait TestAsBytes<T: ?Sized> { + #[allow(clippy::needless_lifetimes)] + fn test_as_bytes<'slf, 't>(&'slf self, t: &'t ReadOnly<T>) -> Option<&'t [u8]>; + } + + impl<T: IntoBytes + Immutable + ?Sized> TestAsBytes<T> for AutorefWrapper<T> { + #[allow(clippy::needless_lifetimes)] + fn test_as_bytes<'slf, 't>(&'slf self, t: &'t ReadOnly<T>) -> Option<&'t [u8]> { + Some(t.as_bytes()) + } + } + } + + use autoref_trick::*; + + // Asserts that `$ty` is one of a list of types which are allowed to not + // provide a "real" implementation for `$fn_name`. Since the + // `autoref_trick` machinery fails silently, this allows us to ensure + // that the "default" impls are only being used for types which we + // expect. + // + // Note that, since this is a runtime test, it is possible to have an + // allowlist which is too restrictive if the function in question is + // never called for a particular type. For example, if `as_bytes` is not + // supported for a particular type, and so `test_as_bytes` returns + // `None`, methods such as `test_try_from_ref` may never be called for + // that type. As a result, it's possible that, for example, adding + // `as_bytes` support for a type would cause other allowlist assertions + // to fail. This means that allowlist assertion failures should not + // automatically be taken as a sign of a bug. + macro_rules! assert_on_allowlist { + ($fn_name:ident($ty:ty) $(: $($tys:ty),*)?) => {{ + use core::any::TypeId; + + let allowlist: &[TypeId] = &[ $($(TypeId::of::<$tys>()),*)? ]; + let allowlist_names: &[&str] = &[ $($(stringify!($tys)),*)? ]; + + let id = TypeId::of::<$ty>(); + assert!(allowlist.contains(&id), "{} is not on allowlist for {}: {:?}", stringify!($ty), stringify!($fn_name), allowlist_names); + }}; + } + + // Asserts that `$ty` implements any `$trait` and doesn't implement any + // `!$trait`. Note that all `$trait`s must come before any `!$trait`s. + // + // For `T: TryFromBytes`, uses `TryFromBytesTestable` to test success + // and failure cases. + macro_rules! assert_impls { + ($ty:ty: TryFromBytes) => { + // "Default" implementations that match the "real" + // implementations defined in the `autoref_trick` module above. + #[allow(unused, non_local_definitions)] + impl AutorefWrapper<$ty> { + #[allow(clippy::needless_lifetimes)] + fn test_is_bit_valid_shared<'ptr>( + &mut self, + candidate: Maybe<'ptr, $ty>, + ) -> Option<bool> { + assert_on_allowlist!( + test_is_bit_valid_shared($ty): + ManuallyDrop<UnsafeCell<()>>, + ManuallyDrop<[UnsafeCell<u8>]>, + ManuallyDrop<[UnsafeCell<bool>]>, + CoreMaybeUninit<NotZerocopy>, + CoreMaybeUninit<UnsafeCell<()>>, + Wrapping<UnsafeCell<()>> + ); + + None + } + + #[allow(clippy::needless_lifetimes)] + fn test_try_from_ref<'bytes>(&mut self, _bytes: &'bytes [u8]) -> Option<Option<&'bytes $ty>> { + assert_on_allowlist!( + test_try_from_ref($ty): + ManuallyDrop<[UnsafeCell<bool>]> + ); + + None + } + + #[allow(clippy::needless_lifetimes)] + fn test_try_from_mut<'bytes>(&mut self, _bytes: &'bytes mut [u8]) -> Option<Option<&'bytes mut $ty>> { + assert_on_allowlist!( + test_try_from_mut($ty): + Option<Box<UnsafeCell<NotZerocopy>>>, + Option<&'static UnsafeCell<NotZerocopy>>, + Option<&'static mut UnsafeCell<NotZerocopy>>, + Option<NonNull<UnsafeCell<NotZerocopy>>>, + Option<fn()>, + Option<FnManyArgs>, + Option<extern "C" fn()>, + Option<ECFnManyArgs>, + *const NotZerocopy, + *mut NotZerocopy + ); + + None + } + + fn test_try_read_from(&mut self, _bytes: &[u8]) -> Option<Option<&$ty>> { + assert_on_allowlist!( + test_try_read_from($ty): + str, + ManuallyDrop<[u8]>, + ManuallyDrop<[bool]>, + ManuallyDrop<[UnsafeCell<bool>]>, + [u8], + [bool] + ); + + None + } + + fn test_as_bytes(&mut self, _t: &ReadOnly<$ty>) -> Option<&[u8]> { + assert_on_allowlist!( + test_as_bytes($ty): + Option<&'static UnsafeCell<NotZerocopy>>, + Option<&'static mut UnsafeCell<NotZerocopy>>, + Option<NonNull<UnsafeCell<NotZerocopy>>>, + Option<Box<UnsafeCell<NotZerocopy>>>, + Option<fn()>, + Option<FnManyArgs>, + Option<extern "C" fn()>, + Option<ECFnManyArgs>, + CoreMaybeUninit<u8>, + CoreMaybeUninit<NotZerocopy>, + CoreMaybeUninit<UnsafeCell<()>>, + ManuallyDrop<UnsafeCell<()>>, + ManuallyDrop<[UnsafeCell<u8>]>, + ManuallyDrop<[UnsafeCell<bool>]>, + Wrapping<UnsafeCell<()>>, + *const NotZerocopy, + *mut NotZerocopy + ); + + None + } + } + + <$ty as TryFromBytesTestable>::with_passing_test_cases(|mut val| { + // FIXME(#494): These tests only get exercised for types + // which are `IntoBytes`. Once we implement #494, we should + // be able to support non-`IntoBytes` types by zeroing + // padding. + + // We define `w` and `ww` since, in the case of the inherent + // methods, Rust thinks they're both borrowed mutably at the + // same time (given how we use them below). If we just + // defined a single `w` and used it for multiple operations, + // this would conflict. + // + // We `#[allow(unused_mut]` for the cases where the "real" + // impls are used, which take `&self`. + #[allow(unused_mut)] + let (mut w, mut ww) = (AutorefWrapper::<$ty>(PhantomData), AutorefWrapper::<$ty>(PhantomData)); + + let c = Ptr::from_ref(&*val); + let c = c.forget_aligned(); + // SAFETY: FIXME(#899): This is unsound. `$ty` is not + // necessarily `IntoBytes`, but that's the corner we've + // backed ourselves into by using `Ptr::from_ref`. + let c = unsafe { c.assume_initialized() }; + let res = w.test_is_bit_valid_shared(c); + if let Some(res) = res { + assert!(res, "{}::is_bit_valid (shared `Ptr`): got false, expected true", stringify!($ty)); + } + + let c = Ptr::from_mut(&mut *val); + let c = c.forget_aligned(); + // SAFETY: FIXME(#899): This is unsound. `$ty` is not + // necessarily `IntoBytes`, but that's the corner we've + // backed ourselves into by using `Ptr::from_ref`. + let mut c = unsafe { c.assume_initialized() }; + let res = <$ty as TryFromBytes>::is_bit_valid(c.reborrow_shared()); + assert!(res, "{}::is_bit_valid (exclusive `Ptr`): got false, expected true", stringify!($ty)); + + // `bytes` is `Some(val.as_bytes())` if `$ty: IntoBytes + + // Immutable` and `None` otherwise. + let bytes = w.test_as_bytes(&*val); + + // The inner closure returns + // `Some($ty::try_ref_from_bytes(bytes))` if `$ty: + // Immutable` and `None` otherwise. + let res = bytes.and_then(|bytes| ww.test_try_from_ref(bytes)); + if let Some(res) = res { + assert!(res.is_some(), "{}::try_ref_from_bytes: got `None`, expected `Some`", stringify!($ty)); + } + + if let Some(bytes) = bytes { + // We need to get a mutable byte slice, and so we clone + // into a `Vec`. However, we also need these bytes to + // satisfy `$ty`'s alignment requirement, which isn't + // guaranteed for `Vec<u8>`. In order to get around + // this, we create a `Vec` which is twice as long as we + // need. There is guaranteed to be an aligned byte range + // of size `size_of_val(val)` within that range. + let val = &*val; + let size = mem::size_of_val(val); + let align = mem::align_of_val(val); + + let mut vec = bytes.to_vec(); + vec.extend(bytes); + let slc = vec.as_slice(); + let offset = slc.as_ptr().align_offset(align); + let bytes_mut = &mut vec.as_mut_slice()[offset..offset+size]; + bytes_mut.copy_from_slice(bytes); + + let res = ww.test_try_from_mut(bytes_mut); + if let Some(res) = res { + assert!(res.is_some(), "{}::try_mut_from_bytes: got `None`, expected `Some`", stringify!($ty)); + } + } + + let res = bytes.and_then(|bytes| ww.test_try_read_from(bytes)); + if let Some(res) = res { + assert!(res.is_some(), "{}::try_read_from_bytes: got `None`, expected `Some`", stringify!($ty)); + } + }); + #[allow(clippy::as_conversions)] + <$ty as TryFromBytesTestable>::with_failing_test_cases(|c| { + #[allow(unused_mut)] // For cases where the "real" impls are used, which take `&self`. + let mut w = AutorefWrapper::<$ty>(PhantomData); + + // This is `Some($ty::try_ref_from_bytes(c))` if `$ty: + // Immutable` and `None` otherwise. + let res = w.test_try_from_ref(c); + if let Some(res) = res { + assert!(res.is_none(), "{}::try_ref_from_bytes({:?}): got Some, expected None", stringify!($ty), c); + } + + let res = w.test_try_from_mut(c); + if let Some(res) = res { + assert!(res.is_none(), "{}::try_mut_from_bytes({:?}): got Some, expected None", stringify!($ty), c); + } + + + let res = w.test_try_read_from(c); + if let Some(res) = res { + assert!(res.is_none(), "{}::try_read_from_bytes({:?}): got Some, expected None", stringify!($ty), c); + } + }); + + #[allow(dead_code)] + const _: () = { static_assertions::assert_impl_all!($ty: TryFromBytes); }; + }; + ($ty:ty: $trait:ident) => { + #[allow(dead_code)] + const _: () = { static_assertions::assert_impl_all!($ty: $trait); }; + }; + ($ty:ty: !$trait:ident) => { + #[allow(dead_code)] + const _: () = { static_assertions::assert_not_impl_any!($ty: $trait); }; + }; + ($ty:ty: $($trait:ident),* $(,)? $(!$negative_trait:ident),*) => { + $( + assert_impls!($ty: $trait); + )* + + $( + assert_impls!($ty: !$negative_trait); + )* + }; + } + + // NOTE: The negative impl assertions here are not necessarily + // prescriptive. They merely serve as change detectors to make sure + // we're aware of what trait impls are getting added with a given + // change. Of course, some impls would be invalid (e.g., `bool: + // FromBytes`), and so this change detection is very important. + + assert_impls!( + (): KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned + ); + assert_impls!( + u8: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned + ); + assert_impls!( + i8: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned + ); + assert_impls!( + u16: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + i16: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + u32: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + i32: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + u64: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + i64: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + u128: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + i128: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + usize: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + isize: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + #[cfg(feature = "float-nightly")] + assert_impls!( + f16: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + f32: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + f64: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + #[cfg(feature = "float-nightly")] + assert_impls!( + f128: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + !Unaligned + ); + assert_impls!( + bool: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + IntoBytes, + Unaligned, + !FromBytes + ); + assert_impls!( + char: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + str: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + IntoBytes, + Unaligned, + !FromBytes + ); + + assert_impls!( + NonZeroU8: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + Unaligned, + !FromZeros, + !FromBytes + ); + assert_impls!( + NonZeroI8: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + Unaligned, + !FromZeros, + !FromBytes + ); + assert_impls!( + NonZeroU16: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroI16: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroU32: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroI32: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroU64: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroI64: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroU128: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroI128: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroUsize: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + assert_impls!( + NonZeroIsize: KnownLayout, + Immutable, + TryFromBytes, + IntoBytes, + !FromBytes, + !Unaligned + ); + + assert_impls!(Option<NonZeroU8>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_impls!(Option<NonZeroI8>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_impls!(Option<NonZeroU16>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroI16>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroU32>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroI32>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroU64>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroI64>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroU128>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroI128>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroUsize>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + assert_impls!(Option<NonZeroIsize>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); + + // Implements none of the ZC traits. + struct NotZerocopy; + + #[rustfmt::skip] + type FnManyArgs = fn( + NotZerocopy, u8, u8, u8, u8, u8, u8, u8, u8, u8, u8, u8, + ) -> (NotZerocopy, NotZerocopy); + + // Allowed, because we're not actually using this type for FFI. + #[allow(improper_ctypes_definitions)] + #[rustfmt::skip] + type ECFnManyArgs = extern "C" fn( + NotZerocopy, u8, u8, u8, u8, u8, u8, u8, u8, u8, u8, u8, + ) -> (NotZerocopy, NotZerocopy); + + #[cfg(feature = "alloc")] + assert_impls!(Option<Box<UnsafeCell<NotZerocopy>>>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<Box<[UnsafeCell<NotZerocopy>]>>: KnownLayout, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<&'static UnsafeCell<NotZerocopy>>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<&'static [UnsafeCell<NotZerocopy>]>: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<&'static mut UnsafeCell<NotZerocopy>>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<&'static mut [UnsafeCell<NotZerocopy>]>: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<NonNull<UnsafeCell<NotZerocopy>>>: KnownLayout, TryFromBytes, FromZeros, Immutable, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<NonNull<[UnsafeCell<NotZerocopy>]>>: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<fn()>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<FnManyArgs>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<extern "C" fn()>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Option<ECFnManyArgs>: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + + assert_impls!(PhantomData<NotZerocopy>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_impls!(PhantomData<UnsafeCell<()>>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + assert_impls!(PhantomData<[u8]>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + + assert_impls!(ManuallyDrop<u8>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + // This test is important because it allows us to test our hand-rolled + // implementation of `<ManuallyDrop<T> as TryFromBytes>::is_bit_valid`. + assert_impls!(ManuallyDrop<bool>: KnownLayout, Immutable, TryFromBytes, FromZeros, IntoBytes, Unaligned, !FromBytes); + assert_impls!(ManuallyDrop<[u8]>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + // This test is important because it allows us to test our hand-rolled + // implementation of `<ManuallyDrop<T> as TryFromBytes>::is_bit_valid`. + assert_impls!(ManuallyDrop<[bool]>: KnownLayout, Immutable, TryFromBytes, FromZeros, IntoBytes, Unaligned, !FromBytes); + assert_impls!(ManuallyDrop<NotZerocopy>: !Immutable, !TryFromBytes, !KnownLayout, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(ManuallyDrop<[NotZerocopy]>: KnownLayout, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(ManuallyDrop<UnsafeCell<()>>: KnownLayout, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned, !Immutable); + assert_impls!(ManuallyDrop<[UnsafeCell<u8>]>: KnownLayout, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned, !Immutable); + assert_impls!(ManuallyDrop<[UnsafeCell<bool>]>: KnownLayout, TryFromBytes, FromZeros, IntoBytes, Unaligned, !Immutable, !FromBytes); + + assert_impls!(CoreMaybeUninit<u8>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, Unaligned, !IntoBytes); + assert_impls!(CoreMaybeUninit<NotZerocopy>: KnownLayout, TryFromBytes, FromZeros, FromBytes, !Immutable, !IntoBytes, !Unaligned); + assert_impls!(CoreMaybeUninit<UnsafeCell<()>>: KnownLayout, TryFromBytes, FromZeros, FromBytes, Unaligned, !Immutable, !IntoBytes); + + assert_impls!(Wrapping<u8>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + // This test is important because it allows us to test our hand-rolled + // implementation of `<Wrapping<T> as TryFromBytes>::is_bit_valid`. + assert_impls!(Wrapping<bool>: KnownLayout, Immutable, TryFromBytes, FromZeros, IntoBytes, Unaligned, !FromBytes); + assert_impls!(Wrapping<NotZerocopy>: KnownLayout, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(Wrapping<UnsafeCell<()>>: KnownLayout, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned, !Immutable); + + assert_impls!(Unalign<u8>: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, Unaligned); + // This test is important because it allows us to test our hand-rolled + // implementation of `<Unalign<T> as TryFromBytes>::is_bit_valid`. + assert_impls!(Unalign<bool>: KnownLayout, Immutable, TryFromBytes, FromZeros, IntoBytes, Unaligned, !FromBytes); + assert_impls!(Unalign<NotZerocopy>: KnownLayout, Unaligned, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes); + + assert_impls!( + [u8]: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned + ); + assert_impls!( + [bool]: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + IntoBytes, + Unaligned, + !FromBytes + ); + assert_impls!([NotZerocopy]: KnownLayout, !Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!( + [u8; 0]: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned, + ); + assert_impls!( + [NotZerocopy; 0]: KnownLayout, + !Immutable, + !TryFromBytes, + !FromZeros, + !FromBytes, + !IntoBytes, + !Unaligned + ); + assert_impls!( + [u8; 1]: KnownLayout, + Immutable, + TryFromBytes, + FromZeros, + FromBytes, + IntoBytes, + Unaligned, + ); + assert_impls!( + [NotZerocopy; 1]: KnownLayout, + !Immutable, + !TryFromBytes, + !FromZeros, + !FromBytes, + !IntoBytes, + !Unaligned + ); + + assert_impls!(*const NotZerocopy: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(*mut NotZerocopy: KnownLayout, Immutable, TryFromBytes, FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(*const [NotZerocopy]: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(*mut [NotZerocopy]: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(*const dyn Debug: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + assert_impls!(*mut dyn Debug: KnownLayout, Immutable, !TryFromBytes, !FromZeros, !FromBytes, !IntoBytes, !Unaligned); + + #[cfg(feature = "simd")] + { + #[allow(unused_macros)] + macro_rules! test_simd_arch_mod { + ($arch:ident, $($typ:ident),*) => { + { + use core::arch::$arch::{$($typ),*}; + use crate::*; + $( assert_impls!($typ: KnownLayout, Immutable, TryFromBytes, FromZeros, FromBytes, IntoBytes, !Unaligned); )* + } + }; + } + #[cfg(target_arch = "x86")] + test_simd_arch_mod!(x86, __m128, __m128d, __m128i, __m256, __m256d, __m256i); + + #[cfg(all(not(no_zerocopy_simd_x86_avx12_1_89_0), target_arch = "x86"))] + test_simd_arch_mod!(x86, __m512bh, __m512, __m512d, __m512i); + + #[cfg(target_arch = "x86_64")] + test_simd_arch_mod!(x86_64, __m128, __m128d, __m128i, __m256, __m256d, __m256i); + + #[cfg(all(not(no_zerocopy_simd_x86_avx12_1_89_0), target_arch = "x86_64"))] + test_simd_arch_mod!(x86_64, __m512bh, __m512, __m512d, __m512i); + + #[cfg(target_arch = "wasm32")] + test_simd_arch_mod!(wasm32, v128); + + #[cfg(all(feature = "simd-nightly", target_arch = "powerpc"))] + test_simd_arch_mod!( + powerpc, + vector_bool_long, + vector_double, + vector_signed_long, + vector_unsigned_long + ); + + #[cfg(all(feature = "simd-nightly", target_arch = "powerpc64"))] + test_simd_arch_mod!( + powerpc64, + vector_bool_long, + vector_double, + vector_signed_long, + vector_unsigned_long + ); + #[cfg(all(target_arch = "aarch64", not(no_zerocopy_aarch64_simd_1_59_0)))] + #[rustfmt::skip] + test_simd_arch_mod!( + aarch64, float32x2_t, float32x4_t, float64x1_t, float64x2_t, int8x8_t, int8x8x2_t, + int8x8x3_t, int8x8x4_t, int8x16_t, int8x16x2_t, int8x16x3_t, int8x16x4_t, int16x4_t, + int16x8_t, int32x2_t, int32x4_t, int64x1_t, int64x2_t, poly8x8_t, poly8x8x2_t, poly8x8x3_t, + poly8x8x4_t, poly8x16_t, poly8x16x2_t, poly8x16x3_t, poly8x16x4_t, poly16x4_t, poly16x8_t, + poly64x1_t, poly64x2_t, uint8x8_t, uint8x8x2_t, uint8x8x3_t, uint8x8x4_t, uint8x16_t, + uint8x16x2_t, uint8x16x3_t, uint8x16x4_t, uint16x4_t, uint16x4x2_t, uint16x4x3_t, + uint16x4x4_t, uint16x8_t, uint32x2_t, uint32x4_t, uint64x1_t, uint64x2_t + ); + } + } +} diff --git a/rust/zerocopy/src/layout.rs b/rust/zerocopy/src/layout.rs new file mode 100644 index 000000000000..b1fa0cd436db --- /dev/null +++ b/rust/zerocopy/src/layout.rs @@ -0,0 +1,2227 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License <LICENSE-BSD or +// https://opensource.org/license/bsd-2-clause>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use core::{mem, num::NonZeroUsize}; + +use crate::util; + +/// The target pointer width, counted in bits. +const POINTER_WIDTH_BITS: usize = mem::size_of::<usize>() * 8; + +/// The layout of a type which might be dynamically-sized. +/// +/// `DstLayout` describes the layout of sized types, slice types, and "slice +/// DSTs" - ie, those that are known by the type system to have a trailing slice +/// (as distinguished from `dyn Trait` types - such types *might* have a +/// trailing slice type, but the type system isn't aware of it). +/// +/// Note that `DstLayout` does not have any internal invariants, so no guarantee +/// is made that a `DstLayout` conforms to any of Rust's requirements regarding +/// the layout of real Rust types or instances of types. +#[doc(hidden)] +#[allow(missing_debug_implementations, missing_copy_implementations)] +#[cfg_attr(any(kani, test), derive(Debug, PartialEq, Eq))] +#[derive(Copy, Clone)] +pub struct DstLayout { + pub(crate) align: NonZeroUsize, + pub(crate) size_info: SizeInfo, + // Is it guaranteed statically (without knowing a value's runtime metadata) + // that the top-level type contains no padding? This does *not* apply + // recursively - for example, `[(u8, u16)]` has `statically_shallow_unpadded + // = true` even though this type likely has padding inside each `(u8, u16)`. + pub(crate) statically_shallow_unpadded: bool, +} + +#[cfg_attr(any(kani, test), derive(Debug, PartialEq, Eq))] +#[derive(Copy, Clone)] +pub(crate) enum SizeInfo<E = usize> { + Sized { size: usize }, + SliceDst(TrailingSliceLayout<E>), +} + +#[cfg_attr(any(kani, test), derive(Debug, PartialEq, Eq))] +#[derive(Copy, Clone)] +pub(crate) struct TrailingSliceLayout<E = usize> { + // The offset of the first byte of the trailing slice field. Note that this + // is NOT the same as the minimum size of the type. For example, consider + // the following type: + // + // struct Foo { + // a: u16, + // b: u8, + // c: [u8], + // } + // + // In `Foo`, `c` is at byte offset 3. When `c.len() == 0`, `c` is followed + // by a padding byte. + pub(crate) offset: usize, + // The size of the element type of the trailing slice field. + pub(crate) elem_size: E, +} + +impl SizeInfo { + /// Attempts to create a `SizeInfo` from `Self` in which `elem_size` is a + /// `NonZeroUsize`. If `elem_size` is 0, returns `None`. + #[allow(unused)] + #[cfg_attr(not(zerocopy_inline_always), inline)] + #[cfg_attr(zerocopy_inline_always, inline(always))] + const fn try_to_nonzero_elem_size(&self) -> Option<SizeInfo<NonZeroUsize>> { + Some(match *self { + SizeInfo::Sized { size } => SizeInfo::Sized { size }, + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + if let Some(elem_size) = NonZeroUsize::new(elem_size) { + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) + } else { + return None; + } + } + }) + } +} + +#[doc(hidden)] +#[derive(Copy, Clone)] +#[cfg_attr(test, derive(Debug))] +#[allow(missing_debug_implementations)] +pub enum CastType { + Prefix, + Suffix, +} + +#[cfg_attr(test, derive(Debug))] +pub(crate) enum MetadataCastError { + Alignment, + Size, +} + +impl DstLayout { + /// The minimum possible alignment of a type. + const MIN_ALIGN: NonZeroUsize = match NonZeroUsize::new(1) { + Some(min_align) => min_align, + None => const_unreachable!(), + }; + + /// The maximum theoretic possible alignment of a type. + /// + /// For compatibility with future Rust versions, this is defined as the + /// maximum power-of-two that fits into a `usize`. See also + /// [`DstLayout::CURRENT_MAX_ALIGN`]. + pub(crate) const THEORETICAL_MAX_ALIGN: NonZeroUsize = + match NonZeroUsize::new(1 << (POINTER_WIDTH_BITS - 1)) { + Some(max_align) => max_align, + None => const_unreachable!(), + }; + + /// The current, documented max alignment of a type \[1\]. + /// + /// \[1\] Per <https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers>: + /// + /// The alignment value must be a power of two from 1 up to + /// 2<sup>29</sup>. + #[cfg(not(kani))] + #[cfg(not(target_pointer_width = "16"))] + pub(crate) const CURRENT_MAX_ALIGN: NonZeroUsize = match NonZeroUsize::new(1 << 28) { + Some(max_align) => max_align, + None => const_unreachable!(), + }; + + #[cfg(not(kani))] + #[cfg(target_pointer_width = "16")] + pub(crate) const CURRENT_MAX_ALIGN: NonZeroUsize = match NonZeroUsize::new(1 << 15) { + Some(max_align) => max_align, + None => const_unreachable!(), + }; + + /// The maximum size of an allocation \[1\]. + /// + /// \[1\] Per <https://doc.rust-lang.org/1.91.1/std/ptr/index.html#allocation>: + /// + /// For any allocation with base `address`, `size`, and a set of `addresses`, + /// the following are guaranteed: [..] + /// + /// - `size <= isize::MAX` + /// + #[allow(clippy::as_conversions)] + pub(crate) const MAX_SIZE: usize = isize::MAX as usize; + + /// Assumes that this layout lacks static shallow padding. + /// + /// # Panics + /// + /// This method does not panic. + /// + /// # Safety + /// + /// If `self` describes the size and alignment of type that lacks static + /// shallow padding, unsafe code may assume that the result of this method + /// accurately reflects the size, alignment, and lack of static shallow + /// padding of that type. + const fn assume_shallow_unpadded(self) -> Self { + Self { statically_shallow_unpadded: true, ..self } + } + + /// Constructs a `DstLayout` for a zero-sized type with `repr_align` + /// alignment (or 1). If `repr_align` is provided, then it must be a power + /// of two. + /// + /// # Panics + /// + /// This function panics if the supplied `repr_align` is not a power of two. + /// + /// # Safety + /// + /// Unsafe code may assume that the contract of this function is satisfied. + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn new_zst(repr_align: Option<NonZeroUsize>) -> DstLayout { + let align = match repr_align { + Some(align) => align, + None => Self::MIN_ALIGN, + }; + + const_assert!(align.get().is_power_of_two()); + + DstLayout { + align, + size_info: SizeInfo::Sized { size: 0 }, + statically_shallow_unpadded: true, + } + } + + /// Constructs a `DstLayout` which describes `T` and assumes `T` may contain + /// padding. + /// + /// # Safety + /// + /// Unsafe code may assume that `DstLayout` is the correct layout for `T`. + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn for_type<T>() -> DstLayout { + // SAFETY: `align` is correct by construction. `T: Sized`, and so it is + // sound to initialize `size_info` to `SizeInfo::Sized { size }`; the + // `size` field is also correct by construction. `unpadded` can safely + // default to `false`. + DstLayout { + align: match NonZeroUsize::new(mem::align_of::<T>()) { + Some(align) => align, + None => const_unreachable!(), + }, + size_info: SizeInfo::Sized { size: mem::size_of::<T>() }, + statically_shallow_unpadded: false, + } + } + + /// Constructs a `DstLayout` which describes a `T` that does not contain + /// padding. + /// + /// # Safety + /// + /// Unsafe code may assume that `DstLayout` is the correct layout for `T`. + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn for_unpadded_type<T>() -> DstLayout { + Self::for_type::<T>().assume_shallow_unpadded() + } + + /// Constructs a `DstLayout` which describes `[T]`. + /// + /// # Safety + /// + /// Unsafe code may assume that `DstLayout` is the correct layout for `[T]`. + pub(crate) const fn for_slice<T>() -> DstLayout { + // SAFETY: The alignment of a slice is equal to the alignment of its + // element type, and so `align` is initialized correctly. + // + // Since this is just a slice type, there is no offset between the + // beginning of the type and the beginning of the slice, so it is + // correct to set `offset: 0`. The `elem_size` is correct by + // construction. Since `[T]` is a (degenerate case of a) slice DST, it + // is correct to initialize `size_info` to `SizeInfo::SliceDst`. + DstLayout { + align: match NonZeroUsize::new(mem::align_of::<T>()) { + Some(align) => align, + None => const_unreachable!(), + }, + size_info: SizeInfo::SliceDst(TrailingSliceLayout { + offset: 0, + elem_size: mem::size_of::<T>(), + }), + statically_shallow_unpadded: true, + } + } + + /// Constructs a complete `DstLayout` reflecting a `repr(C)` struct with the + /// given alignment modifiers and fields. + /// + /// This method cannot be used to match the layout of a record with the + /// default representation, as that representation is mostly unspecified. + /// + /// # Safety + /// + /// For any definition of a `repr(C)` struct, if this method is invoked with + /// alignment modifiers and fields corresponding to that definition, the + /// resulting `DstLayout` will correctly encode the layout of that struct. + /// + /// We make no guarantees to the behavior of this method when it is invoked + /// with arguments that cannot correspond to a valid `repr(C)` struct. + #[must_use] + #[inline] + pub const fn for_repr_c_struct( + repr_align: Option<NonZeroUsize>, + repr_packed: Option<NonZeroUsize>, + fields: &[DstLayout], + ) -> DstLayout { + let mut layout = DstLayout::new_zst(repr_align); + + let mut i = 0; + #[allow(clippy::arithmetic_side_effects)] + while i < fields.len() { + #[allow(clippy::indexing_slicing)] + let field = fields[i]; + layout = layout.extend(field, repr_packed); + i += 1; + } + + layout = layout.pad_to_align(); + + // SAFETY: `layout` accurately describes the layout of a `repr(C)` + // struct with `repr_align` or `repr_packed` alignment modifications and + // the given `fields`. The `layout` is constructed using a sequence of + // invocations of `DstLayout::{new_zst,extend,pad_to_align}`. The + // documentation of these items vows that invocations in this manner + // will accurately describe a type, so long as: + // + // - that type is `repr(C)`, + // - its fields are enumerated in the order they appear, + // - the presence of `repr_align` and `repr_packed` are correctly accounted for. + // + // We respect all three of these preconditions above. + layout + } + + /// Like `Layout::extend`, this creates a layout that describes a record + /// whose layout consists of `self` followed by `next` that includes the + /// necessary inter-field padding, but not any trailing padding. + /// + /// In order to match the layout of a `#[repr(C)]` struct, this method + /// should be invoked for each field in declaration order. To add trailing + /// padding, call `DstLayout::pad_to_align` after extending the layout for + /// all fields. If `self` corresponds to a type marked with + /// `repr(packed(N))`, then `repr_packed` should be set to `Some(N)`, + /// otherwise `None`. + /// + /// This method cannot be used to match the layout of a record with the + /// default representation, as that representation is mostly unspecified. + /// + /// # Safety + /// + /// If a (potentially hypothetical) valid `repr(C)` Rust type begins with + /// fields whose layout are `self`, and those fields are immediately + /// followed by a field whose layout is `field`, then unsafe code may rely + /// on `self.extend(field, repr_packed)` producing a layout that correctly + /// encompasses those two components. + /// + /// We make no guarantees to the behavior of this method if these fragments + /// cannot appear in a valid Rust type (e.g., the concatenation of the + /// layouts would lead to a size larger than `isize::MAX`). + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn extend(self, field: DstLayout, repr_packed: Option<NonZeroUsize>) -> Self { + use util::{max, min, padding_needed_for}; + + // If `repr_packed` is `None`, there are no alignment constraints, and + // the value can be defaulted to `THEORETICAL_MAX_ALIGN`. + let max_align = match repr_packed { + Some(max_align) => max_align, + None => Self::THEORETICAL_MAX_ALIGN, + }; + + const_assert!(max_align.get().is_power_of_two()); + + // We use Kani to prove that this method is robust to future increases + // in Rust's maximum allowed alignment. However, if such a change ever + // actually occurs, we'd like to be notified via assertion failures. + #[cfg(not(kani))] + { + const_debug_assert!(self.align.get() <= DstLayout::CURRENT_MAX_ALIGN.get()); + const_debug_assert!(field.align.get() <= DstLayout::CURRENT_MAX_ALIGN.get()); + if let Some(repr_packed) = repr_packed { + const_debug_assert!(repr_packed.get() <= DstLayout::CURRENT_MAX_ALIGN.get()); + } + } + + // The field's alignment is clamped by `repr_packed` (i.e., the + // `repr(packed(N))` attribute, if any) [1]. + // + // [1] Per https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers: + // + // The alignments of each field, for the purpose of positioning + // fields, is the smaller of the specified alignment and the alignment + // of the field's type. + let field_align = min(field.align, max_align); + + // The struct's alignment is the maximum of its previous alignment and + // `field_align`. + let align = max(self.align, field_align); + + let (interfield_padding, size_info) = match self.size_info { + // If the layout is already a DST, we panic; DSTs cannot be extended + // with additional fields. + SizeInfo::SliceDst(..) => const_panic!("Cannot extend a DST with additional fields."), + + SizeInfo::Sized { size: preceding_size } => { + // Compute the minimum amount of inter-field padding needed to + // satisfy the field's alignment, and offset of the trailing + // field. [1] + // + // [1] Per https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers: + // + // Inter-field padding is guaranteed to be the minimum + // required in order to satisfy each field's (possibly + // altered) alignment. + let padding = padding_needed_for(preceding_size, field_align); + + // This will not panic (and is proven to not panic, with Kani) + // if the layout components can correspond to a leading layout + // fragment of a valid Rust type, but may panic otherwise (e.g., + // combining or aligning the components would create a size + // exceeding `isize::MAX`). + let offset = match preceding_size.checked_add(padding) { + Some(offset) => offset, + None => const_panic!("Adding padding to `self`'s size overflows `usize`."), + }; + + ( + padding, + match field.size_info { + SizeInfo::Sized { size: field_size } => { + // If the trailing field is sized, the resulting layout + // will be sized. Its size will be the sum of the + // preceding layout, the size of the new field, and the + // size of inter-field padding between the two. + // + // This will not panic (and is proven with Kani to not + // panic) if the layout components can correspond to a + // leading layout fragment of a valid Rust type, but may + // panic otherwise (e.g., combining or aligning the + // components would create a size exceeding + // `usize::MAX`). + let size = match offset.checked_add(field_size) { + Some(size) => size, + None => const_panic!("`field` cannot be appended without the total size overflowing `usize`"), + }; + SizeInfo::Sized { size } + } + SizeInfo::SliceDst(TrailingSliceLayout { + offset: trailing_offset, + elem_size, + }) => { + // If the trailing field is dynamically sized, so too + // will the resulting layout. The offset of the trailing + // slice component is the sum of the offset of the + // trailing field and the trailing slice offset within + // that field. + // + // This will not panic (and is proven with Kani to not + // panic) if the layout components can correspond to a + // leading layout fragment of a valid Rust type, but may + // panic otherwise (e.g., combining or aligning the + // components would create a size exceeding + // `usize::MAX`). + let offset = match offset.checked_add(trailing_offset) { + Some(offset) => offset, + None => const_panic!("`field` cannot be appended without the total size overflowing `usize`"), + }; + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) + } + }, + ) + } + }; + + let statically_shallow_unpadded = self.statically_shallow_unpadded + && field.statically_shallow_unpadded + && interfield_padding == 0; + + DstLayout { align, size_info, statically_shallow_unpadded } + } + + /// Like `Layout::pad_to_align`, this routine rounds the size of this layout + /// up to the nearest multiple of this type's alignment or `repr_packed` + /// (whichever is less). This method leaves DST layouts unchanged, since the + /// trailing padding of DSTs is computed at runtime. + /// + /// The accompanying boolean is `true` if the resulting composition of + /// fields necessitated static (as opposed to dynamic) padding; otherwise + /// `false`. + /// + /// In order to match the layout of a `#[repr(C)]` struct, this method + /// should be invoked after the invocations of [`DstLayout::extend`]. If + /// `self` corresponds to a type marked with `repr(packed(N))`, then + /// `repr_packed` should be set to `Some(N)`, otherwise `None`. + /// + /// This method cannot be used to match the layout of a record with the + /// default representation, as that representation is mostly unspecified. + /// + /// # Safety + /// + /// If a (potentially hypothetical) valid `repr(C)` type begins with fields + /// whose layout are `self` followed only by zero or more bytes of trailing + /// padding (not included in `self`), then unsafe code may rely on + /// `self.pad_to_align(repr_packed)` producing a layout that correctly + /// encapsulates the layout of that type. + /// + /// We make no guarantees to the behavior of this method if `self` cannot + /// appear in a valid Rust type (e.g., because the addition of trailing + /// padding would lead to a size larger than `isize::MAX`). + #[doc(hidden)] + #[must_use] + #[inline] + pub const fn pad_to_align(self) -> Self { + use util::padding_needed_for; + + let (static_padding, size_info) = match self.size_info { + // For sized layouts, we add the minimum amount of trailing padding + // needed to satisfy alignment. + SizeInfo::Sized { size: unpadded_size } => { + let padding = padding_needed_for(unpadded_size, self.align); + let size = match unpadded_size.checked_add(padding) { + Some(size) => size, + None => const_panic!("Adding padding caused size to overflow `usize`."), + }; + (padding, SizeInfo::Sized { size }) + } + // For DST layouts, trailing padding depends on the length of the + // trailing DST and is computed at runtime. This does not alter the + // offset or element size of the layout, so we leave `size_info` + // unchanged. + size_info @ SizeInfo::SliceDst(_) => (0, size_info), + }; + + let statically_shallow_unpadded = self.statically_shallow_unpadded && static_padding == 0; + + DstLayout { align: self.align, size_info, statically_shallow_unpadded } + } + + /// Produces `true` if `self` requires static padding; otherwise `false`. + #[must_use] + #[inline(always)] + pub const fn requires_static_padding(self) -> bool { + !self.statically_shallow_unpadded + } + + /// Produces `true` if there exists any metadata for which a type of layout + /// `self` would require dynamic trailing padding; otherwise `false`. + #[must_use] + #[inline(always)] + pub const fn requires_dynamic_padding(self) -> bool { + // A `% self.align.get()` cannot panic, since `align` is non-zero. + #[allow(clippy::arithmetic_side_effects)] + match self.size_info { + SizeInfo::Sized { .. } => false, + SizeInfo::SliceDst(trailing_slice_layout) => { + // SAFETY: This predicate is formally proved sound by + // `proofs::prove_requires_dynamic_padding`. + trailing_slice_layout.offset % self.align.get() != 0 + || trailing_slice_layout.elem_size % self.align.get() != 0 + } + } + } + + /// Validates that a cast is sound from a layout perspective. + /// + /// Validates that the size and alignment requirements of a type with the + /// layout described in `self` would not be violated by performing a + /// `cast_type` cast from a pointer with address `addr` which refers to a + /// memory region of size `bytes_len`. + /// + /// If the cast is valid, `validate_cast_and_convert_metadata` returns + /// `(elems, split_at)`. If `self` describes a dynamically-sized type, then + /// `elems` is the maximum number of trailing slice elements for which a + /// cast would be valid (for sized types, `elem` is meaningless and should + /// be ignored). `split_at` is the index at which to split the memory region + /// in order for the prefix (suffix) to contain the result of the cast, and + /// in order for the remaining suffix (prefix) to contain the leftover + /// bytes. + /// + /// There are three conditions under which a cast can fail: + /// - The smallest possible value for the type is larger than the provided + /// memory region + /// - A prefix cast is requested, and `addr` does not satisfy `self`'s + /// alignment requirement + /// - A suffix cast is requested, and `addr + bytes_len` does not satisfy + /// `self`'s alignment requirement (as a consequence, since all instances + /// of the type are a multiple of its alignment, no size for the type will + /// result in a starting address which is properly aligned) + /// + /// # Safety + /// + /// The caller may assume that this implementation is correct, and may rely + /// on that assumption for the soundness of their code. In particular, the + /// caller may assume that, if `validate_cast_and_convert_metadata` returns + /// `Some((elems, split_at))`, then: + /// - A pointer to the type (for dynamically sized types, this includes + /// `elems` as its pointer metadata) describes an object of size `size <= + /// bytes_len` + /// - If this is a prefix cast: + /// - `addr` satisfies `self`'s alignment + /// - `size == split_at` + /// - If this is a suffix cast: + /// - `split_at == bytes_len - size` + /// - `addr + split_at` satisfies `self`'s alignment + /// + /// Note that this method does *not* ensure that a pointer constructed from + /// its return values will be a valid pointer. In particular, this method + /// does not reason about `isize` overflow, which is a requirement of many + /// Rust pointer APIs, and may at some point be determined to be a validity + /// invariant of pointer types themselves. This should never be a problem so + /// long as the arguments to this method are derived from a known-valid + /// pointer (e.g., one derived from a safe Rust reference), but it is + /// nonetheless the caller's responsibility to justify that pointer + /// arithmetic will not overflow based on a safety argument *other than* the + /// mere fact that this method returned successfully. + /// + /// # Panics + /// + /// `validate_cast_and_convert_metadata` will panic if `self` describes a + /// DST whose trailing slice element is zero-sized. + /// + /// If `addr + bytes_len` overflows `usize`, + /// `validate_cast_and_convert_metadata` may panic, or it may return + /// incorrect results. No guarantees are made about when + /// `validate_cast_and_convert_metadata` will panic. The caller should not + /// rely on `validate_cast_and_convert_metadata` panicking in any particular + /// condition, even if `debug_assertions` are enabled. + #[allow(unused)] + #[inline(always)] + pub(crate) const fn validate_cast_and_convert_metadata( + &self, + addr: usize, + bytes_len: usize, + cast_type: CastType, + ) -> Result<(usize, usize), MetadataCastError> { + // `debug_assert!`, but with `#[allow(clippy::arithmetic_side_effects)]`. + macro_rules! __const_debug_assert { + ($e:expr $(, $msg:expr)?) => { + const_debug_assert!({ + #[allow(clippy::arithmetic_side_effects)] + let e = $e; + e + } $(, $msg)?); + }; + } + + // Note that, in practice, `self` is always a compile-time constant. We + // do this check earlier than needed to ensure that we always panic as a + // result of bugs in the program (such as calling this function on an + // invalid type) instead of allowing this panic to be hidden if the cast + // would have failed anyway for runtime reasons (such as a too-small + // memory region). + // + // FIXME(#67): Once our MSRV is 1.65, use let-else: + // https://blog.rust-lang.org/2022/11/03/Rust-1.65.0.html#let-else-statements + let size_info = match self.size_info.try_to_nonzero_elem_size() { + Some(size_info) => size_info, + None => const_panic!("attempted to cast to slice type with zero-sized element"), + }; + + // Precondition + __const_debug_assert!( + addr.checked_add(bytes_len).is_some(), + "`addr` + `bytes_len` > usize::MAX" + ); + + // Alignment checks go in their own block to avoid introducing variables + // into the top-level scope. + { + // We check alignment for `addr` (for prefix casts) or `addr + + // bytes_len` (for suffix casts). For a prefix cast, the correctness + // of this check is trivial - `addr` is the address the object will + // live at. + // + // For a suffix cast, we know that all valid sizes for the type are + // a multiple of the alignment (and by safety precondition, we know + // `DstLayout` may only describe valid Rust types). Thus, a + // validly-sized instance which lives at a validly-aligned address + // must also end at a validly-aligned address. Thus, if the end + // address for a suffix cast (`addr + bytes_len`) is not aligned, + // then no valid start address will be aligned either. + let offset = match cast_type { + CastType::Prefix => 0, + CastType::Suffix => bytes_len, + }; + + // Addition is guaranteed not to overflow because `offset <= + // bytes_len`, and `addr + bytes_len <= usize::MAX` is a + // precondition of this method. Modulus is guaranteed not to divide + // by 0 because `align` is non-zero. + #[allow(clippy::arithmetic_side_effects)] + if (addr + offset) % self.align.get() != 0 { + return Err(MetadataCastError::Alignment); + } + } + + let (elems, self_bytes) = match size_info { + SizeInfo::Sized { size } => { + if size > bytes_len { + return Err(MetadataCastError::Size); + } + (0, size) + } + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + // Calculate the maximum number of bytes that could be consumed + // - any number of bytes larger than this will either not be a + // multiple of the alignment, or will be larger than + // `bytes_len`. + let max_total_bytes = + util::round_down_to_next_multiple_of_alignment(bytes_len, self.align); + // Calculate the maximum number of bytes that could be consumed + // by the trailing slice. + // + // FIXME(#67): Once our MSRV is 1.65, use let-else: + // https://blog.rust-lang.org/2022/11/03/Rust-1.65.0.html#let-else-statements + let max_slice_and_padding_bytes = match max_total_bytes.checked_sub(offset) { + Some(max) => max, + // `bytes_len` too small even for 0 trailing slice elements. + None => return Err(MetadataCastError::Size), + }; + + // Calculate the number of elements that fit in + // `max_slice_and_padding_bytes`; any remaining bytes will be + // considered padding. + // + // Guaranteed not to divide by zero: `elem_size` is non-zero. + #[allow(clippy::arithmetic_side_effects)] + let elems = max_slice_and_padding_bytes / elem_size.get(); + // Guaranteed not to overflow on multiplication: `usize::MAX >= + // max_slice_and_padding_bytes >= (max_slice_and_padding_bytes / + // elem_size) * elem_size`. + // + // Guaranteed not to overflow on addition: + // - max_slice_and_padding_bytes == max_total_bytes - offset + // - elems * elem_size <= max_slice_and_padding_bytes == max_total_bytes - offset + // - elems * elem_size + offset <= max_total_bytes <= usize::MAX + #[allow(clippy::arithmetic_side_effects)] + let without_padding = offset + elems * elem_size.get(); + // `self_bytes` is equal to the offset bytes plus the bytes + // consumed by the trailing slice plus any padding bytes + // required to satisfy the alignment. Note that we have computed + // the maximum number of trailing slice elements that could fit + // in `self_bytes`, so any padding is guaranteed to be less than + // the size of an extra element. + // + // Guaranteed not to overflow: + // - By previous comment: without_padding == elems * elem_size + + // offset <= max_total_bytes + // - By construction, `max_total_bytes` is a multiple of + // `self.align`. + // - At most, adding padding needed to round `without_padding` + // up to the next multiple of the alignment will bring + // `self_bytes` up to `max_total_bytes`. + #[allow(clippy::arithmetic_side_effects)] + let self_bytes = + without_padding + util::padding_needed_for(without_padding, self.align); + (elems, self_bytes) + } + }; + + __const_debug_assert!(self_bytes <= bytes_len); + + let split_at = match cast_type { + CastType::Prefix => self_bytes, + // Guaranteed not to underflow: + // - In the `Sized` branch, only returns `size` if `size <= + // bytes_len`. + // - In the `SliceDst` branch, calculates `self_bytes <= + // max_toatl_bytes`, which is upper-bounded by `bytes_len`. + #[allow(clippy::arithmetic_side_effects)] + CastType::Suffix => bytes_len - self_bytes, + }; + + Ok((elems, split_at)) + } +} + +pub(crate) use cast_from::CastFrom; +mod cast_from { + use crate::*; + + pub(crate) struct CastFrom<Dst: ?Sized> { + _never: core::convert::Infallible, + _marker: PhantomData<Dst>, + } + + // SAFETY: The implementation of `Project::project` preserves the address + // of the referent – it only modifies pointer metadata. + unsafe impl<Src, Dst> crate::pointer::cast::Cast<Src, Dst> for CastFrom<Dst> + where + Src: KnownLayout + ?Sized, + Dst: KnownLayout + ?Sized, + { + } + + // SAFETY: The implementation of `Project::project` preserves the size of + // the referent (see inline comments for a more detailed proof of this). + unsafe impl<Src, Dst> crate::pointer::cast::CastExact<Src, Dst> for CastFrom<Dst> + where + Src: KnownLayout + ?Sized, + Dst: KnownLayout + ?Sized, + { + } + + // SAFETY: `project` produces a pointer which refers to the same referent + // bytes as its input, or to a subset of them (see inline comments for a + // more detailed proof of this). It does this using provenance-preserving + // operations. + unsafe impl<Src, Dst> crate::pointer::cast::Project<Src, Dst> for CastFrom<Dst> + where + Src: KnownLayout + ?Sized, + Dst: KnownLayout + ?Sized, + { + /// # PME + /// + /// Generates a post-monomorphization error if it is not possible to + /// implement soundly. + // + // FIXME(#1817): Support Sized->Unsized and Unsized->Sized casts + fn project(src: PtrInner<'_, Src>) -> *mut Dst { + /// The parameters required in order to perform a pointer cast from + /// `Src` to `Dst`. + /// + /// These are a compile-time function of the layouts of `Src` + /// and `Dst`. + /// + /// # Safety + /// + /// `Src`'s alignment must not be smaller than `Dst`'s alignment. + struct CastParams<Src: ?Sized, Dst: ?Sized> { + inner: CastParamsInner, + _src: PhantomData<Src>, + _dst: PhantomData<Dst>, + } + + #[derive(Copy, Clone)] + enum CastParamsInner { + // At compile time (specifically, post-monomorphization time), + // we need to compute two things: + // - Whether, given *any* `*Src`, it is possible to construct a + // `*Dst` which addresses the same number of bytes (ie, + // whether, for any `Src` pointer metadata, there exists `Dst` + // pointer metadata that addresses the same number of bytes) + // - If this is possible, any information necessary to perform + // the `Src`->`Dst` metadata conversion at runtime. + // + // Assume that `Src` and `Dst` are slice DSTs, and define: + // - `S_OFF = Src::LAYOUT.size_info.offset` + // - `S_ELEM = Src::LAYOUT.size_info.elem_size` + // - `D_OFF = Dst::LAYOUT.size_info.offset` + // - `D_ELEM = Dst::LAYOUT.size_info.elem_size` + // + // We are trying to solve the following equation: + // + // D_OFF + d_meta * D_ELEM = S_OFF + s_meta * S_ELEM + // + // At runtime, we will be attempting to compute `d_meta`, given + // `s_meta` (a runtime value) and all other parameters (which + // are compile-time values). We can solve like so: + // + // D_OFF + d_meta * D_ELEM = S_OFF + s_meta * S_ELEM + // + // d_meta * D_ELEM = S_OFF - D_OFF + s_meta * S_ELEM + // + // d_meta = (S_OFF - D_OFF + s_meta * S_ELEM)/D_ELEM + // + // Since `d_meta` will be a `usize`, we need the right-hand side + // to be an integer, and this needs to hold for *any* value of + // `s_meta` (in order for our conversion to be infallible - ie, + // to not have to reject certain values of `s_meta` at runtime). + // This means that: + // + // - `s_meta * S_ELEM` must be a multiple of `D_ELEM` + // - Since this must hold for any value of `s_meta`, `S_ELEM` + // must be a multiple of `D_ELEM` + // - `S_OFF - D_OFF` must be a multiple of `D_ELEM` + // + // Thus, let `OFFSET_DELTA_ELEMS = (S_OFF - D_OFF)/D_ELEM` and + // `ELEM_MULTIPLE = S_ELEM/D_ELEM`. We can rewrite the above + // expression as: + // + // d_meta = (S_OFF - D_OFF + s_meta * S_ELEM)/D_ELEM + // + // d_meta = OFFSET_DELTA_ELEMS + s_meta * ELEM_MULTIPLE + // + // Thus, we just need to compute the following and confirm that + // they have integer solutions in order to both a) determine + // whether infallible `Src` -> `Dst` casts are possible and, b) + // pre-compute the parameters necessary to perform those casts + // at runtime. These parameters are encapsulated in + // `CastParams`, which acts as a witness that such infallible + // casts are possible. + /// The parameters required in order to perform an + /// unsized-to-unsized pointer cast from `Src` to `Dst` as + /// described above. + /// + /// # Safety + /// + /// `Src` and `Dst` must both be slice DSTs. + /// + /// `offset_delta_elems` and `elem_multiple` must be valid as + /// described above. + UnsizedToUnsized { offset_delta_elems: usize, elem_multiple: usize }, + + /// The metadata of a `Dst` which has the same size as `Src: + /// Sized`. + /// + /// # Safety + /// + /// `Src: Sized` and `Dst` must be a slice DST. + /// + /// A raw `Dst` pointer with metadata `dst_meta` must address + /// `size_of::<Src>()` bytes. + SizedToUnsized { dst_meta: usize }, + + /// The metadata of a `Dst` which has the same size as `Src: + /// Sized`. + /// + /// # Safety + /// + /// `Src` and `Dst` must both be `Sized` and `size_of::<Src>() + /// == size_of::<Dst>()`. + SizedToSized, + } + + impl<Src: ?Sized, Dst: ?Sized> Copy for CastParams<Src, Dst> {} + impl<Src: ?Sized, Dst: ?Sized> Clone for CastParams<Src, Dst> { + fn clone(&self) -> Self { + *self + } + } + + impl<Src: ?Sized, Dst: ?Sized> CastParams<Src, Dst> { + const fn try_compute( + src: &DstLayout, + dst: &DstLayout, + ) -> Option<CastParams<Src, Dst>> { + if src.align.get() < dst.align.get() { + return None; + } + + let inner = match (src.size_info, dst.size_info) { + ( + SizeInfo::Sized { size: src_size }, + SizeInfo::Sized { size: dst_size }, + ) => { + if src_size != dst_size { + return None; + } + + // SAFETY: We checked above that `src_size == + // dst_size`. + CastParamsInner::SizedToSized + } + (SizeInfo::Sized { size: src_size }, SizeInfo::SliceDst(dst)) => { + let offset_delta = if let Some(od) = src_size.checked_sub(dst.offset) { + od + } else { + return None; + }; + + let dst_elem_size = if let Some(e) = NonZeroUsize::new(dst.elem_size) { + e + } else { + return None; + }; + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let delta_mod_other_elem = offset_delta % dst_elem_size.get(); + + if delta_mod_other_elem != 0 { + return None; + } + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let dst_meta = offset_delta / dst_elem_size.get(); + + // SAFETY: The preceding math ensures that a `Dst` + // with `dst_meta` addresses `src_size` bytes. + CastParamsInner::SizedToUnsized { dst_meta } + } + (SizeInfo::SliceDst(src), SizeInfo::SliceDst(dst)) => { + let offset_delta = if let Some(od) = src.offset.checked_sub(dst.offset) + { + od + } else { + return None; + }; + + let dst_elem_size = if let Some(e) = NonZeroUsize::new(dst.elem_size) { + e + } else { + return None; + }; + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let delta_mod_other_elem = offset_delta % dst_elem_size.get(); + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let elem_remainder = src.elem_size % dst_elem_size.get(); + + if delta_mod_other_elem != 0 + || src.elem_size < dst.elem_size + || elem_remainder != 0 + { + return None; + } + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let offset_delta_elems = offset_delta / dst_elem_size.get(); + + // PANICS: `dst_elem_size: NonZeroUsize`, so this won't + // divide by zero. + #[allow(clippy::arithmetic_side_effects)] + let elem_multiple = src.elem_size / dst_elem_size.get(); + + CastParamsInner::UnsizedToUnsized { + // SAFETY: We checked above that this is an exact ratio. + offset_delta_elems, + // SAFETY: We checked above that this is an exact ratio. + elem_multiple, + } + } + _ => return None, + }; + + // SAFETY: We checked above that `src.align >= dst.align`. + Some(CastParams { inner, _src: PhantomData, _dst: PhantomData }) + } + } + + impl<Src: KnownLayout + ?Sized, Dst: KnownLayout + ?Sized> CastParams<Src, Dst> { + /// # Safety + /// + /// `src_meta` describes a `Src` whose size is no larger than + /// `isize::MAX`. + /// + /// The returned metadata describes a `Dst` of the same size as + /// the original `Src`. + #[inline(always)] + unsafe fn cast_metadata( + self, + src_meta: Src::PointerMetadata, + ) -> Dst::PointerMetadata { + #[allow(unused)] + use crate::util::polyfills::*; + + let dst_meta = match self.inner { + CastParamsInner::UnsizedToUnsized { offset_delta_elems, elem_multiple } => { + let src_meta = src_meta.to_elem_count(); + #[allow( + unstable_name_collisions, + clippy::multiple_unsafe_ops_per_block + )] + // SAFETY: `self` is a witness that the following + // equation holds: + // + // D_OFF + d_meta * D_ELEM = S_OFF + s_meta * S_ELEM + // + // Since the caller promises that `src_meta` is + // valid `Src` metadata, this math will not + // overflow, and the returned value will describe a + // `Dst` of the same size. + unsafe { + offset_delta_elems + .unchecked_add(src_meta.unchecked_mul(elem_multiple)) + } + } + CastParamsInner::SizedToUnsized { dst_meta } => dst_meta, + CastParamsInner::SizedToSized => 0, + }; + Dst::PointerMetadata::from_elem_count(dst_meta) + } + } + + trait Params<Src: ?Sized> { + const CAST_PARAMS: CastParams<Src, Self>; + } + + impl<Src, Dst> Params<Src> for Dst + where + Src: KnownLayout + ?Sized, + Dst: KnownLayout + ?Sized, + { + const CAST_PARAMS: CastParams<Src, Dst> = + match CastParams::try_compute(&Src::LAYOUT, &Dst::LAYOUT) { + Some(params) => params, + None => const_panic!( + "cannot `transmute_ref!` or `transmute_mut!` between incompatible types" + ), + }; + } + + let src_meta = <Src as KnownLayout>::pointer_to_metadata(src.as_ptr()); + let params = <Dst as Params<Src>>::CAST_PARAMS; + + // SAFETY: `src: PtrInner` guarantees that `src`'s referent is zero + // bytes or lives in a single allocation, which means that it is no + // larger than `isize::MAX` bytes [1]. + // + // [1] https://doc.rust-lang.org/1.92.0/std/ptr/index.html#allocation + let dst_meta = unsafe { params.cast_metadata(src_meta) }; + + <Dst as KnownLayout>::raw_from_ptr_len(src.as_non_null().cast(), dst_meta).as_ptr() + } + } +} + +// FIXME(#67): For some reason, on our MSRV toolchain, this `allow` isn't +// enforced despite having `#![allow(unknown_lints)]` at the crate root, but +// putting it here works. Once our MSRV is high enough that this bug has been +// fixed, remove this `allow`. +#[allow(unknown_lints)] +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_dst_layout_for_slice() { + let layout = DstLayout::for_slice::<u32>(); + match layout.size_info { + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + assert_eq!(offset, 0); + assert_eq!(elem_size, 4); + } + _ => panic!("Expected SliceDst"), + } + assert_eq!(layout.align.get(), 4); + } + + /// Tests of when a sized `DstLayout` is extended with a sized field. + #[allow(clippy::decimal_literal_representation)] + #[test] + fn test_dst_layout_extend_sized_with_sized() { + // This macro constructs a layout corresponding to a `u8` and extends it + // with a zero-sized trailing field of given alignment `n`. The macro + // tests that the resulting layout has both size and alignment `min(n, + // P)` for all valid values of `repr(packed(P))`. + macro_rules! test_align_is_size { + ($n:expr) => { + let base = DstLayout::for_type::<u8>(); + let trailing_field = DstLayout::for_type::<elain::Align<$n>>(); + + let packs = + core::iter::once(None).chain((0..29).map(|p| NonZeroUsize::new(2usize.pow(p)))); + + for pack in packs { + let composite = base.extend(trailing_field, pack); + let max_align = pack.unwrap_or(DstLayout::CURRENT_MAX_ALIGN); + let align = $n.min(max_align.get()); + assert_eq!( + composite, + DstLayout { + align: NonZeroUsize::new(align).unwrap(), + size_info: SizeInfo::Sized { size: align }, + statically_shallow_unpadded: false, + } + ) + } + }; + } + + test_align_is_size!(1); + test_align_is_size!(2); + test_align_is_size!(4); + test_align_is_size!(8); + test_align_is_size!(16); + test_align_is_size!(32); + test_align_is_size!(64); + test_align_is_size!(128); + test_align_is_size!(256); + test_align_is_size!(512); + test_align_is_size!(1024); + test_align_is_size!(2048); + test_align_is_size!(4096); + test_align_is_size!(8192); + test_align_is_size!(16384); + test_align_is_size!(32768); + test_align_is_size!(65536); + test_align_is_size!(131072); + test_align_is_size!(262144); + test_align_is_size!(524288); + test_align_is_size!(1048576); + test_align_is_size!(2097152); + test_align_is_size!(4194304); + test_align_is_size!(8388608); + test_align_is_size!(16777216); + test_align_is_size!(33554432); + test_align_is_size!(67108864); + test_align_is_size!(33554432); + test_align_is_size!(134217728); + test_align_is_size!(268435456); + } + + /// Tests of when a sized `DstLayout` is extended with a DST field. + #[test] + fn test_dst_layout_extend_sized_with_dst() { + // Test that for all combinations of real-world alignments and + // `repr_packed` values, that the extension of a sized `DstLayout`` with + // a DST field correctly computes the trailing offset in the composite + // layout. + + let aligns = (0..29).map(|p| NonZeroUsize::new(2usize.pow(p)).unwrap()); + let packs = core::iter::once(None).chain(aligns.clone().map(Some)); + + for align in aligns { + for pack in packs.clone() { + let base = DstLayout::for_type::<u8>(); + let elem_size = 42; + let trailing_field_offset = 11; + + let trailing_field = DstLayout { + align, + size_info: SizeInfo::SliceDst(TrailingSliceLayout { elem_size, offset: 11 }), + statically_shallow_unpadded: false, + }; + + let composite = base.extend(trailing_field, pack); + + let max_align = pack.unwrap_or(DstLayout::CURRENT_MAX_ALIGN).get(); + + let align = align.get().min(max_align); + + assert_eq!( + composite, + DstLayout { + align: NonZeroUsize::new(align).unwrap(), + size_info: SizeInfo::SliceDst(TrailingSliceLayout { + elem_size, + offset: align + trailing_field_offset, + }), + statically_shallow_unpadded: false, + } + ) + } + } + } + + /// Tests that calling `pad_to_align` on a sized `DstLayout` adds the + /// expected amount of trailing padding. + #[test] + fn test_dst_layout_pad_to_align_with_sized() { + // For all valid alignments `align`, construct a one-byte layout aligned + // to `align`, call `pad_to_align`, and assert that the size of the + // resulting layout is equal to `align`. + for align in (0..29).map(|p| NonZeroUsize::new(2usize.pow(p)).unwrap()) { + let layout = DstLayout { + align, + size_info: SizeInfo::Sized { size: 1 }, + statically_shallow_unpadded: true, + }; + + assert_eq!( + layout.pad_to_align(), + DstLayout { + align, + size_info: SizeInfo::Sized { size: align.get() }, + statically_shallow_unpadded: align.get() == 1 + } + ); + } + + // Test explicitly-provided combinations of unpadded and padded + // counterparts. + + macro_rules! test { + (unpadded { size: $unpadded_size:expr, align: $unpadded_align:expr } + => padded { size: $padded_size:expr, align: $padded_align:expr }) => { + let unpadded = DstLayout { + align: NonZeroUsize::new($unpadded_align).unwrap(), + size_info: SizeInfo::Sized { size: $unpadded_size }, + statically_shallow_unpadded: false, + }; + let padded = unpadded.pad_to_align(); + + assert_eq!( + padded, + DstLayout { + align: NonZeroUsize::new($padded_align).unwrap(), + size_info: SizeInfo::Sized { size: $padded_size }, + statically_shallow_unpadded: false, + } + ); + }; + } + + test!(unpadded { size: 0, align: 4 } => padded { size: 0, align: 4 }); + test!(unpadded { size: 1, align: 4 } => padded { size: 4, align: 4 }); + test!(unpadded { size: 2, align: 4 } => padded { size: 4, align: 4 }); + test!(unpadded { size: 3, align: 4 } => padded { size: 4, align: 4 }); + test!(unpadded { size: 4, align: 4 } => padded { size: 4, align: 4 }); + test!(unpadded { size: 5, align: 4 } => padded { size: 8, align: 4 }); + test!(unpadded { size: 6, align: 4 } => padded { size: 8, align: 4 }); + test!(unpadded { size: 7, align: 4 } => padded { size: 8, align: 4 }); + test!(unpadded { size: 8, align: 4 } => padded { size: 8, align: 4 }); + + let current_max_align = DstLayout::CURRENT_MAX_ALIGN.get(); + + test!(unpadded { size: 1, align: current_max_align } + => padded { size: current_max_align, align: current_max_align }); + + test!(unpadded { size: current_max_align + 1, align: current_max_align } + => padded { size: current_max_align * 2, align: current_max_align }); + } + + /// Tests that calling `pad_to_align` on a DST `DstLayout` is a no-op. + #[test] + fn test_dst_layout_pad_to_align_with_dst() { + for align in (0..29).map(|p| NonZeroUsize::new(2usize.pow(p)).unwrap()) { + for offset in 0..10 { + for elem_size in 0..10 { + let layout = DstLayout { + align, + size_info: SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }), + statically_shallow_unpadded: false, + }; + assert_eq!(layout.pad_to_align(), layout); + } + } + } + } + + // This test takes a long time when running under Miri, so we skip it in + // that case. This is acceptable because this is a logic test that doesn't + // attempt to expose UB. + #[test] + #[cfg_attr(miri, ignore)] + fn test_validate_cast_and_convert_metadata() { + #[allow(non_local_definitions)] + impl From<usize> for SizeInfo { + fn from(size: usize) -> SizeInfo { + SizeInfo::Sized { size } + } + } + + #[allow(non_local_definitions)] + impl From<(usize, usize)> for SizeInfo { + fn from((offset, elem_size): (usize, usize)) -> SizeInfo { + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) + } + } + + fn layout<S: Into<SizeInfo>>(s: S, align: usize) -> DstLayout { + DstLayout { + size_info: s.into(), + align: NonZeroUsize::new(align).unwrap(), + statically_shallow_unpadded: false, + } + } + + /// This macro accepts arguments in the form of: + /// + /// layout(_, _).validate(_, _, _), Ok(Some((_, _))) + /// | | | | | | | + /// size ---------+ | | | | | | + /// align -----------+ | | | | | + /// addr ------------------------+ | | | | + /// bytes_len ----------------------+ | | | + /// cast_type -------------------------+ | | + /// elems ------------------------------------------+ | + /// split_at ------------------------------------------+ + /// + /// `.validate` is shorthand for `.validate_cast_and_convert_metadata` + /// for brevity. + /// + /// Each argument can either be an iterator or a wildcard. Each + /// wildcarded variable is implicitly replaced by an iterator over a + /// representative sample of values for that variable. Each `test!` + /// invocation iterates over every combination of values provided by + /// each variable's iterator (ie, the cartesian product) and validates + /// that the results are expected. + /// + /// The final argument uses the same syntax, but it has a different + /// meaning: + /// - If it is `Ok(pat)`, then the pattern `pat` is supplied to + /// a matching assert to validate the computed result for each + /// combination of input values. + /// - If it is `Err(Some(msg) | None)`, then `test!` validates that the + /// call to `validate_cast_and_convert_metadata` panics with the given + /// panic message or, if the current Rust toolchain version is too + /// early to support panicking in `const fn`s, panics with *some* + /// message. In the latter case, the `const_panic!` macro is used, + /// which emits code which causes a non-panicking error at const eval + /// time, but which does panic when invoked at runtime. Thus, it is + /// merely difficult to predict the *value* of this panic. We deem + /// that testing against the real panic strings on stable and nightly + /// toolchains is enough to ensure correctness. + /// + /// Note that the meta-variables that match these variables have the + /// `tt` type, and some valid expressions are not valid `tt`s (such as + /// `a..b`). In this case, wrap the expression in parentheses, and it + /// will become valid `tt`. + macro_rules! test { + ( + layout($size:tt, $align:tt) + .validate($addr:tt, $bytes_len:tt, $cast_type:tt), $expect:pat $(,)? + ) => { + itertools::iproduct!( + test!(@generate_size $size), + test!(@generate_align $align), + test!(@generate_usize $addr), + test!(@generate_usize $bytes_len), + test!(@generate_cast_type $cast_type) + ).for_each(|(size_info, align, addr, bytes_len, cast_type)| { + // Temporarily disable the panic hook installed by the test + // harness. If we don't do this, all panic messages will be + // kept in an internal log. On its own, this isn't a + // problem, but if a non-caught panic ever happens (ie, in + // code later in this test not in this macro), all of the + // previously-buffered messages will be dumped, hiding the + // real culprit. + let previous_hook = std::panic::take_hook(); + // I don't understand why, but this seems to be required in + // addition to the previous line. + std::panic::set_hook(Box::new(|_| {})); + let actual = std::panic::catch_unwind(|| { + layout(size_info, align).validate_cast_and_convert_metadata(addr, bytes_len, cast_type) + }).map_err(|d| { + let msg = d.downcast::<&'static str>().ok().map(|s| *s.as_ref()); + assert!(msg.is_some() || cfg!(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0), "non-string panic messages are not permitted when usage of panic in const fn is enabled"); + msg + }); + std::panic::set_hook(previous_hook); + + assert!( + matches!(actual, $expect), + "layout({:?}, {}).validate_cast_and_convert_metadata({}, {}, {:?})" ,size_info, align, addr, bytes_len, cast_type + ); + }); + }; + (@generate_usize _) => { 0..8 }; + // Generate sizes for both Sized and !Sized types. + (@generate_size _) => { + test!(@generate_size (_)).chain(test!(@generate_size (_, _))) + }; + // Generate sizes for both Sized and !Sized types by chaining + // specified iterators for each. + (@generate_size ($sized_sizes:tt | $unsized_sizes:tt)) => { + test!(@generate_size ($sized_sizes)).chain(test!(@generate_size $unsized_sizes)) + }; + // Generate sizes for Sized types. + (@generate_size (_)) => { test!(@generate_size (0..8)) }; + (@generate_size ($sizes:expr)) => { $sizes.into_iter().map(Into::<SizeInfo>::into) }; + // Generate sizes for !Sized types. + (@generate_size ($min_sizes:tt, $elem_sizes:tt)) => { + itertools::iproduct!( + test!(@generate_min_size $min_sizes), + test!(@generate_elem_size $elem_sizes) + ).map(Into::<SizeInfo>::into) + }; + (@generate_fixed_size _) => { (0..8).into_iter().map(Into::<SizeInfo>::into) }; + (@generate_min_size _) => { 0..8 }; + (@generate_elem_size _) => { 1..8 }; + (@generate_align _) => { [1, 2, 4, 8, 16] }; + (@generate_opt_usize _) => { [None].into_iter().chain((0..8).map(Some).into_iter()) }; + (@generate_cast_type _) => { [CastType::Prefix, CastType::Suffix] }; + (@generate_cast_type $variant:ident) => { [CastType::$variant] }; + // Some expressions need to be wrapped in parentheses in order to be + // valid `tt`s (required by the top match pattern). See the comment + // below for more details. This arm removes these parentheses to + // avoid generating an `unused_parens` warning. + (@$_:ident ($vals:expr)) => { $vals }; + (@$_:ident $vals:expr) => { $vals }; + } + + const EVENS: [usize; 8] = [0, 2, 4, 6, 8, 10, 12, 14]; + const ODDS: [usize; 8] = [1, 3, 5, 7, 9, 11, 13, 15]; + + // base_size is too big for the memory region. + test!( + layout(((1..8) | ((1..8), (1..8))), _).validate([0], [0], _), + Ok(Err(MetadataCastError::Size)) + ); + test!( + layout(((2..8) | ((2..8), (2..8))), _).validate([0], [1], Prefix), + Ok(Err(MetadataCastError::Size)) + ); + test!( + layout(((2..8) | ((2..8), (2..8))), _).validate([0x1000_0000 - 1], [1], Suffix), + Ok(Err(MetadataCastError::Size)) + ); + + // addr is unaligned for prefix cast + test!(layout(_, [2]).validate(ODDS, _, Prefix), Ok(Err(MetadataCastError::Alignment))); + test!(layout(_, [2]).validate(ODDS, _, Prefix), Ok(Err(MetadataCastError::Alignment))); + + // addr is aligned, but end of buffer is unaligned for suffix cast + test!(layout(_, [2]).validate(EVENS, ODDS, Suffix), Ok(Err(MetadataCastError::Alignment))); + test!(layout(_, [2]).validate(EVENS, ODDS, Suffix), Ok(Err(MetadataCastError::Alignment))); + + // Unfortunately, these constants cannot easily be used in the + // implementation of `validate_cast_and_convert_metadata`, since + // `panic!` consumes a string literal, not an expression. + // + // It's important that these messages be in a separate module. If they + // were at the function's top level, we'd pass them to `test!` as, e.g., + // `Err(TRAILING)`, which would run into a subtle Rust footgun - the + // `TRAILING` identifier would be treated as a pattern to match rather + // than a value to check for equality. + mod msgs { + pub(super) const TRAILING: &str = + "attempted to cast to slice type with zero-sized element"; + pub(super) const OVERFLOW: &str = "`addr` + `bytes_len` > usize::MAX"; + } + + // casts with ZST trailing element types are unsupported + test!(layout((_, [0]), _).validate(_, _, _), Err(Some(msgs::TRAILING) | None),); + + // addr + bytes_len must not overflow usize + test!(layout(_, _).validate([usize::MAX], (1..100), _), Err(Some(msgs::OVERFLOW) | None)); + test!(layout(_, _).validate((1..100), [usize::MAX], _), Err(Some(msgs::OVERFLOW) | None)); + test!( + layout(_, _).validate( + [usize::MAX / 2 + 1, usize::MAX], + [usize::MAX / 2 + 1, usize::MAX], + _ + ), + Err(Some(msgs::OVERFLOW) | None) + ); + + // Validates that `validate_cast_and_convert_metadata` satisfies its own + // documented safety postconditions, and also a few other properties + // that aren't documented but we want to guarantee anyway. + fn validate_behavior( + (layout, addr, bytes_len, cast_type): (DstLayout, usize, usize, CastType), + ) { + if let Ok((elems, split_at)) = + layout.validate_cast_and_convert_metadata(addr, bytes_len, cast_type) + { + let (size_info, align) = (layout.size_info, layout.align); + let debug_str = format!( + "layout({:?}, {}).validate_cast_and_convert_metadata({}, {}, {:?}) => ({}, {})", + size_info, align, addr, bytes_len, cast_type, elems, split_at + ); + + // If this is a sized type (no trailing slice), then `elems` is + // meaningless, but in practice we set it to 0. Callers are not + // allowed to rely on this, but a lot of math is nicer if + // they're able to, and some callers might accidentally do that. + let sized = matches!(layout.size_info, SizeInfo::Sized { .. }); + assert!(!(sized && elems != 0), "{}", debug_str); + + let resulting_size = match layout.size_info { + SizeInfo::Sized { size } => size, + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + let padded_size = |elems| { + let without_padding = offset + elems * elem_size; + without_padding + util::padding_needed_for(without_padding, align) + }; + + let resulting_size = padded_size(elems); + // Test that `validate_cast_and_convert_metadata` + // computed the largest possible value that fits in the + // given range. + assert!(padded_size(elems + 1) > bytes_len, "{}", debug_str); + resulting_size + } + }; + + // Test safety postconditions guaranteed by + // `validate_cast_and_convert_metadata`. + assert!(resulting_size <= bytes_len, "{}", debug_str); + match cast_type { + CastType::Prefix => { + assert_eq!(addr % align, 0, "{}", debug_str); + assert_eq!(resulting_size, split_at, "{}", debug_str); + } + CastType::Suffix => { + assert_eq!(split_at, bytes_len - resulting_size, "{}", debug_str); + assert_eq!((addr + split_at) % align, 0, "{}", debug_str); + } + } + } else { + let min_size = match layout.size_info { + SizeInfo::Sized { size } => size, + SizeInfo::SliceDst(TrailingSliceLayout { offset, .. }) => { + offset + util::padding_needed_for(offset, layout.align) + } + }; + + // If a cast is invalid, it is either because... + // 1. there are insufficient bytes at the given region for type: + let insufficient_bytes = bytes_len < min_size; + // 2. performing the cast would misalign type: + let base = match cast_type { + CastType::Prefix => 0, + CastType::Suffix => bytes_len, + }; + let misaligned = (base + addr) % layout.align != 0; + + assert!(insufficient_bytes || misaligned); + } + } + + let sizes = 0..8; + let elem_sizes = 1..8; + let size_infos = sizes + .clone() + .map(Into::<SizeInfo>::into) + .chain(itertools::iproduct!(sizes, elem_sizes).map(Into::<SizeInfo>::into)); + let layouts = itertools::iproduct!(size_infos, [1, 2, 4, 8, 16, 32]) + .filter(|(size_info, align)| !matches!(size_info, SizeInfo::Sized { size } if size % align != 0)) + .map(|(size_info, align)| layout(size_info, align)); + itertools::iproduct!(layouts, 0..8, 0..8, [CastType::Prefix, CastType::Suffix]) + .for_each(validate_behavior); + } + + #[test] + #[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] + fn test_validate_rust_layout() { + use core::{ + convert::TryInto as _, + ptr::{self, NonNull}, + }; + + use crate::util::testutil::*; + + // This test synthesizes pointers with various metadata and uses Rust's + // built-in APIs to confirm that Rust makes decisions about type layout + // which are consistent with what we believe is guaranteed by the + // language. If this test fails, it doesn't just mean our code is wrong + // - it means we're misunderstanding the language's guarantees. + + #[derive(Debug)] + struct MacroArgs { + offset: usize, + align: NonZeroUsize, + elem_size: Option<usize>, + } + + /// # Safety + /// + /// `test` promises to only call `addr_of_slice_field` on a `NonNull<T>` + /// which points to a valid `T`. + /// + /// `with_elems` must produce a pointer which points to a valid `T`. + fn test<T: ?Sized, W: Fn(usize) -> NonNull<T>>( + args: MacroArgs, + with_elems: W, + addr_of_slice_field: Option<fn(NonNull<T>) -> NonNull<u8>>, + ) { + let dst = args.elem_size.is_some(); + let layout = { + let size_info = match args.elem_size { + Some(elem_size) => { + SizeInfo::SliceDst(TrailingSliceLayout { offset: args.offset, elem_size }) + } + None => SizeInfo::Sized { + // Rust only supports types whose sizes are a multiple + // of their alignment. If the macro created a type like + // this: + // + // #[repr(C, align(2))] + // struct Foo([u8; 1]); + // + // ...then Rust will automatically round the type's size + // up to 2. + size: args.offset + util::padding_needed_for(args.offset, args.align), + }, + }; + DstLayout { size_info, align: args.align, statically_shallow_unpadded: false } + }; + + for elems in 0..128 { + let ptr = with_elems(elems); + + if let Some(addr_of_slice_field) = addr_of_slice_field { + let slc_field_ptr = addr_of_slice_field(ptr).as_ptr(); + // SAFETY: Both `slc_field_ptr` and `ptr` are pointers to + // the same valid Rust object. + // Work around https://github.com/rust-lang/rust-clippy/issues/12280 + let offset: usize = + unsafe { slc_field_ptr.byte_offset_from(ptr.as_ptr()).try_into().unwrap() }; + assert_eq!(offset, args.offset); + } + + // SAFETY: `ptr` points to a valid `T`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + let (size, align) = unsafe { + (mem::size_of_val_raw(ptr.as_ptr()), mem::align_of_val_raw(ptr.as_ptr())) + }; + + // Avoid expensive allocation when running under Miri. + let assert_msg = if !cfg!(miri) { + format!("\n{:?}\nsize:{}, align:{}", args, size, align) + } else { + String::new() + }; + + let without_padding = + args.offset + args.elem_size.map(|elem_size| elems * elem_size).unwrap_or(0); + assert!(size >= without_padding, "{}", assert_msg); + assert_eq!(align, args.align.get(), "{}", assert_msg); + + // This encodes the most important part of the test: our + // understanding of how Rust determines the layout of repr(C) + // types. Sized repr(C) types are trivial, but DST types have + // some subtlety. Note that: + // - For sized types, `without_padding` is just the size of the + // type that we constructed for `Foo`. Since we may have + // requested a larger alignment, `Foo` may actually be larger + // than this, hence `padding_needed_for`. + // - For unsized types, `without_padding` is dynamically + // computed from the offset, the element size, and element + // count. We expect that the size of the object should be + // `offset + elem_size * elems` rounded up to the next + // alignment. + let expected_size = + without_padding + util::padding_needed_for(without_padding, args.align); + assert_eq!(expected_size, size, "{}", assert_msg); + + // For zero-sized element types, + // `validate_cast_and_convert_metadata` just panics, so we skip + // testing those types. + if args.elem_size.map(|elem_size| elem_size > 0).unwrap_or(true) { + let addr = ptr.addr().get(); + let (got_elems, got_split_at) = layout + .validate_cast_and_convert_metadata(addr, size, CastType::Prefix) + .unwrap(); + // Avoid expensive allocation when running under Miri. + let assert_msg = if !cfg!(miri) { + format!( + "{}\nvalidate_cast_and_convert_metadata({}, {})", + assert_msg, addr, size, + ) + } else { + String::new() + }; + assert_eq!(got_split_at, size, "{}", assert_msg); + if dst { + assert!(got_elems >= elems, "{}", assert_msg); + if got_elems != elems { + // If `validate_cast_and_convert_metadata` + // returned more elements than `elems`, that + // means that `elems` is not the maximum number + // of elements that can fit in `size` - in other + // words, there is enough padding at the end of + // the value to fit at least one more element. + // If we use this metadata to synthesize a + // pointer, despite having a different element + // count, we still expect it to have the same + // size. + let got_ptr = with_elems(got_elems); + // SAFETY: `got_ptr` is a pointer to a valid `T`. + let size_of_got_ptr = unsafe { mem::size_of_val_raw(got_ptr.as_ptr()) }; + assert_eq!(size_of_got_ptr, size, "{}", assert_msg); + } + } else { + // For sized casts, the returned element value is + // technically meaningless, and we don't guarantee any + // particular value. In practice, it's always zero. + assert_eq!(got_elems, 0, "{}", assert_msg) + } + } + } + } + + macro_rules! validate_against_rust { + ($offset:literal, $align:literal $(, $elem_size:literal)?) => {{ + #[repr(C, align($align))] + struct Foo([u8; $offset]$(, [[u8; $elem_size]])?); + + let args = MacroArgs { + offset: $offset, + align: $align.try_into().unwrap(), + elem_size: { + #[allow(unused)] + let ret = None::<usize>; + $(let ret = Some($elem_size);)? + ret + } + }; + + #[repr(C, align($align))] + struct FooAlign; + // Create an aligned buffer to use in order to synthesize + // pointers to `Foo`. We don't ever load values from these + // pointers - we just do arithmetic on them - so having a "real" + // block of memory as opposed to a validly-aligned-but-dangling + // pointer is only necessary to make Miri happy since we run it + // with "strict provenance" checking enabled. + let aligned_buf = Align::<_, FooAlign>::new([0u8; 1024]); + let with_elems = |elems| { + let slc = NonNull::slice_from_raw_parts(NonNull::from(&aligned_buf.t), elems); + #[allow(clippy::as_conversions)] + NonNull::new(slc.as_ptr() as *mut Foo).unwrap() + }; + let addr_of_slice_field = { + #[allow(unused)] + let f = None::<fn(NonNull<Foo>) -> NonNull<u8>>; + $( + // SAFETY: `test` promises to only call `f` with a `ptr` + // to a valid `Foo`. + let f: Option<fn(NonNull<Foo>) -> NonNull<u8>> = Some(|ptr: NonNull<Foo>| unsafe { + NonNull::new(ptr::addr_of_mut!((*ptr.as_ptr()).1)).unwrap().cast::<u8>() + }); + let _ = $elem_size; + )? + f + }; + + test::<Foo, _>(args, with_elems, addr_of_slice_field); + }}; + } + + // Every permutation of: + // - offset in [0, 4] + // - align in [1, 16] + // - elem_size in [0, 4] (plus no elem_size) + validate_against_rust!(0, 1); + validate_against_rust!(0, 1, 0); + validate_against_rust!(0, 1, 1); + validate_against_rust!(0, 1, 2); + validate_against_rust!(0, 1, 3); + validate_against_rust!(0, 1, 4); + validate_against_rust!(0, 2); + validate_against_rust!(0, 2, 0); + validate_against_rust!(0, 2, 1); + validate_against_rust!(0, 2, 2); + validate_against_rust!(0, 2, 3); + validate_against_rust!(0, 2, 4); + validate_against_rust!(0, 4); + validate_against_rust!(0, 4, 0); + validate_against_rust!(0, 4, 1); + validate_against_rust!(0, 4, 2); + validate_against_rust!(0, 4, 3); + validate_against_rust!(0, 4, 4); + validate_against_rust!(0, 8); + validate_against_rust!(0, 8, 0); + validate_against_rust!(0, 8, 1); + validate_against_rust!(0, 8, 2); + validate_against_rust!(0, 8, 3); + validate_against_rust!(0, 8, 4); + validate_against_rust!(0, 16); + validate_against_rust!(0, 16, 0); + validate_against_rust!(0, 16, 1); + validate_against_rust!(0, 16, 2); + validate_against_rust!(0, 16, 3); + validate_against_rust!(0, 16, 4); + validate_against_rust!(1, 1); + validate_against_rust!(1, 1, 0); + validate_against_rust!(1, 1, 1); + validate_against_rust!(1, 1, 2); + validate_against_rust!(1, 1, 3); + validate_against_rust!(1, 1, 4); + validate_against_rust!(1, 2); + validate_against_rust!(1, 2, 0); + validate_against_rust!(1, 2, 1); + validate_against_rust!(1, 2, 2); + validate_against_rust!(1, 2, 3); + validate_against_rust!(1, 2, 4); + validate_against_rust!(1, 4); + validate_against_rust!(1, 4, 0); + validate_against_rust!(1, 4, 1); + validate_against_rust!(1, 4, 2); + validate_against_rust!(1, 4, 3); + validate_against_rust!(1, 4, 4); + validate_against_rust!(1, 8); + validate_against_rust!(1, 8, 0); + validate_against_rust!(1, 8, 1); + validate_against_rust!(1, 8, 2); + validate_against_rust!(1, 8, 3); + validate_against_rust!(1, 8, 4); + validate_against_rust!(1, 16); + validate_against_rust!(1, 16, 0); + validate_against_rust!(1, 16, 1); + validate_against_rust!(1, 16, 2); + validate_against_rust!(1, 16, 3); + validate_against_rust!(1, 16, 4); + validate_against_rust!(2, 1); + validate_against_rust!(2, 1, 0); + validate_against_rust!(2, 1, 1); + validate_against_rust!(2, 1, 2); + validate_against_rust!(2, 1, 3); + validate_against_rust!(2, 1, 4); + validate_against_rust!(2, 2); + validate_against_rust!(2, 2, 0); + validate_against_rust!(2, 2, 1); + validate_against_rust!(2, 2, 2); + validate_against_rust!(2, 2, 3); + validate_against_rust!(2, 2, 4); + validate_against_rust!(2, 4); + validate_against_rust!(2, 4, 0); + validate_against_rust!(2, 4, 1); + validate_against_rust!(2, 4, 2); + validate_against_rust!(2, 4, 3); + validate_against_rust!(2, 4, 4); + validate_against_rust!(2, 8); + validate_against_rust!(2, 8, 0); + validate_against_rust!(2, 8, 1); + validate_against_rust!(2, 8, 2); + validate_against_rust!(2, 8, 3); + validate_against_rust!(2, 8, 4); + validate_against_rust!(2, 16); + validate_against_rust!(2, 16, 0); + validate_against_rust!(2, 16, 1); + validate_against_rust!(2, 16, 2); + validate_against_rust!(2, 16, 3); + validate_against_rust!(2, 16, 4); + validate_against_rust!(3, 1); + validate_against_rust!(3, 1, 0); + validate_against_rust!(3, 1, 1); + validate_against_rust!(3, 1, 2); + validate_against_rust!(3, 1, 3); + validate_against_rust!(3, 1, 4); + validate_against_rust!(3, 2); + validate_against_rust!(3, 2, 0); + validate_against_rust!(3, 2, 1); + validate_against_rust!(3, 2, 2); + validate_against_rust!(3, 2, 3); + validate_against_rust!(3, 2, 4); + validate_against_rust!(3, 4); + validate_against_rust!(3, 4, 0); + validate_against_rust!(3, 4, 1); + validate_against_rust!(3, 4, 2); + validate_against_rust!(3, 4, 3); + validate_against_rust!(3, 4, 4); + validate_against_rust!(3, 8); + validate_against_rust!(3, 8, 0); + validate_against_rust!(3, 8, 1); + validate_against_rust!(3, 8, 2); + validate_against_rust!(3, 8, 3); + validate_against_rust!(3, 8, 4); + validate_against_rust!(3, 16); + validate_against_rust!(3, 16, 0); + validate_against_rust!(3, 16, 1); + validate_against_rust!(3, 16, 2); + validate_against_rust!(3, 16, 3); + validate_against_rust!(3, 16, 4); + validate_against_rust!(4, 1); + validate_against_rust!(4, 1, 0); + validate_against_rust!(4, 1, 1); + validate_against_rust!(4, 1, 2); + validate_against_rust!(4, 1, 3); + validate_against_rust!(4, 1, 4); + validate_against_rust!(4, 2); + validate_against_rust!(4, 2, 0); + validate_against_rust!(4, 2, 1); + validate_against_rust!(4, 2, 2); + validate_against_rust!(4, 2, 3); + validate_against_rust!(4, 2, 4); + validate_against_rust!(4, 4); + validate_against_rust!(4, 4, 0); + validate_against_rust!(4, 4, 1); + validate_against_rust!(4, 4, 2); + validate_against_rust!(4, 4, 3); + validate_against_rust!(4, 4, 4); + validate_against_rust!(4, 8); + validate_against_rust!(4, 8, 0); + validate_against_rust!(4, 8, 1); + validate_against_rust!(4, 8, 2); + validate_against_rust!(4, 8, 3); + validate_against_rust!(4, 8, 4); + validate_against_rust!(4, 16); + validate_against_rust!(4, 16, 0); + validate_against_rust!(4, 16, 1); + validate_against_rust!(4, 16, 2); + validate_against_rust!(4, 16, 3); + validate_against_rust!(4, 16, 4); + } +} + +#[cfg(kani)] +mod proofs { + use core::alloc::Layout; + + use super::*; + + impl kani::Arbitrary for DstLayout { + fn any() -> Self { + let align: NonZeroUsize = kani::any(); + let size_info: SizeInfo = kani::any(); + + kani::assume(align.is_power_of_two()); + kani::assume(align < DstLayout::THEORETICAL_MAX_ALIGN); + + // For testing purposes, we most care about instantiations of + // `DstLayout` that can correspond to actual Rust types. We use + // `Layout` to verify that our `DstLayout` satisfies the validity + // conditions of Rust layouts. + kani::assume( + match size_info { + SizeInfo::Sized { size } => Layout::from_size_align(size, align.get()), + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size: _ }) => { + // `SliceDst` cannot encode an exact size, but we know + // it is at least `offset` bytes. + Layout::from_size_align(offset, align.get()) + } + } + .is_ok(), + ); + + Self { align: align, size_info: size_info, statically_shallow_unpadded: kani::any() } + } + } + + impl kani::Arbitrary for SizeInfo { + fn any() -> Self { + let is_sized: bool = kani::any(); + + match is_sized { + true => { + let size: usize = kani::any(); + + kani::assume(size <= DstLayout::MAX_SIZE); + + SizeInfo::Sized { size } + } + false => SizeInfo::SliceDst(kani::any()), + } + } + } + + impl kani::Arbitrary for TrailingSliceLayout { + fn any() -> Self { + let elem_size: usize = kani::any(); + let offset: usize = kani::any(); + + kani::assume(elem_size < DstLayout::MAX_SIZE); + kani::assume(offset < DstLayout::MAX_SIZE); + + TrailingSliceLayout { elem_size, offset } + } + } + + #[kani::proof] + fn prove_requires_dynamic_padding() { + let layout: DstLayout = kani::any(); + + let SizeInfo::SliceDst(size_info) = layout.size_info else { + kani::assume(false); + loop {} + }; + + let meta: usize = kani::any(); + + let Some(trailing_slice_size) = size_info.elem_size.checked_mul(meta) else { + // The `trailing_slice_size` exceeds `usize::MAX`; `meta` is invalid. + kani::assume(false); + loop {} + }; + + let Some(unpadded_size) = size_info.offset.checked_add(trailing_slice_size) else { + // The `unpadded_size` exceeds `usize::MAX`; `meta`` is invalid. + kani::assume(false); + loop {} + }; + + if unpadded_size >= DstLayout::MAX_SIZE { + // The `unpadded_size` exceeds `isize::MAX`; `meta` is invalid. + kani::assume(false); + loop {} + } + + let trailing_padding = util::padding_needed_for(unpadded_size, layout.align); + + if !layout.requires_dynamic_padding() { + assert!(trailing_padding == 0); + } + } + + #[kani::proof] + fn prove_dst_layout_extend() { + use crate::util::{max, min, padding_needed_for}; + + let base: DstLayout = kani::any(); + let field: DstLayout = kani::any(); + let packed: Option<NonZeroUsize> = kani::any(); + + if let Some(max_align) = packed { + kani::assume(max_align.is_power_of_two()); + kani::assume(base.align <= max_align); + } + + // The base can only be extended if it's sized. + kani::assume(matches!(base.size_info, SizeInfo::Sized { .. })); + let base_size = if let SizeInfo::Sized { size } = base.size_info { + size + } else { + unreachable!(); + }; + + // Under the above conditions, `DstLayout::extend` will not panic. + let composite = base.extend(field, packed); + + // The field's alignment is clamped by `max_align` (i.e., the + // `packed` attribute, if any) [1]. + // + // [1] Per https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers: + // + // The alignments of each field, for the purpose of positioning + // fields, is the smaller of the specified alignment and the + // alignment of the field's type. + let field_align = min(field.align, packed.unwrap_or(DstLayout::THEORETICAL_MAX_ALIGN)); + + // The struct's alignment is the maximum of its previous alignment and + // `field_align`. + assert_eq!(composite.align, max(base.align, field_align)); + + // Compute the minimum amount of inter-field padding needed to + // satisfy the field's alignment, and offset of the trailing field. + // [1] + // + // [1] Per https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers: + // + // Inter-field padding is guaranteed to be the minimum required in + // order to satisfy each field's (possibly altered) alignment. + let padding = padding_needed_for(base_size, field_align); + let offset = base_size + padding; + + // For testing purposes, we'll also construct `alloc::Layout` + // stand-ins for `DstLayout`, and show that `extend` behaves + // comparably on both types. + let base_analog = Layout::from_size_align(base_size, base.align.get()).unwrap(); + + match field.size_info { + SizeInfo::Sized { size: field_size } => { + if let SizeInfo::Sized { size: composite_size } = composite.size_info { + // If the trailing field is sized, the resulting layout will + // be sized. Its size will be the sum of the preceding + // layout, the size of the new field, and the size of + // inter-field padding between the two. + assert_eq!(composite_size, offset + field_size); + + let field_analog = + Layout::from_size_align(field_size, field_align.get()).unwrap(); + + if let Ok((actual_composite, actual_offset)) = base_analog.extend(field_analog) + { + assert_eq!(actual_offset, offset); + assert_eq!(actual_composite.size(), composite_size); + assert_eq!(actual_composite.align(), composite.align.get()); + } else { + // An error here reflects that composite of `base` + // and `field` cannot correspond to a real Rust type + // fragment, because such a fragment would violate + // the basic invariants of a valid Rust layout. At + // the time of writing, `DstLayout` is a little more + // permissive than `Layout`, so we don't assert + // anything in this branch (e.g., unreachability). + } + } else { + panic!("The composite of two sized layouts must be sized.") + } + } + SizeInfo::SliceDst(TrailingSliceLayout { + offset: field_offset, + elem_size: field_elem_size, + }) => { + if let SizeInfo::SliceDst(TrailingSliceLayout { + offset: composite_offset, + elem_size: composite_elem_size, + }) = composite.size_info + { + // The offset of the trailing slice component is the sum + // of the offset of the trailing field and the trailing + // slice offset within that field. + assert_eq!(composite_offset, offset + field_offset); + // The elem size is unchanged. + assert_eq!(composite_elem_size, field_elem_size); + + let field_analog = + Layout::from_size_align(field_offset, field_align.get()).unwrap(); + + if let Ok((actual_composite, actual_offset)) = base_analog.extend(field_analog) + { + assert_eq!(actual_offset, offset); + assert_eq!(actual_composite.size(), composite_offset); + assert_eq!(actual_composite.align(), composite.align.get()); + } else { + // An error here reflects that composite of `base` + // and `field` cannot correspond to a real Rust type + // fragment, because such a fragment would violate + // the basic invariants of a valid Rust layout. At + // the time of writing, `DstLayout` is a little more + // permissive than `Layout`, so we don't assert + // anything in this branch (e.g., unreachability). + } + } else { + panic!("The extension of a layout with a DST must result in a DST.") + } + } + } + } + + #[kani::proof] + #[kani::should_panic] + fn prove_dst_layout_extend_dst_panics() { + let base: DstLayout = kani::any(); + let field: DstLayout = kani::any(); + let packed: Option<NonZeroUsize> = kani::any(); + + if let Some(max_align) = packed { + kani::assume(max_align.is_power_of_two()); + kani::assume(base.align <= max_align); + } + + kani::assume(matches!(base.size_info, SizeInfo::SliceDst(..))); + + let _ = base.extend(field, packed); + } + + #[kani::proof] + fn prove_dst_layout_pad_to_align() { + use crate::util::padding_needed_for; + + let layout: DstLayout = kani::any(); + + let padded = layout.pad_to_align(); + + // Calling `pad_to_align` does not alter the `DstLayout`'s alignment. + assert_eq!(padded.align, layout.align); + + if let SizeInfo::Sized { size: unpadded_size } = layout.size_info { + if let SizeInfo::Sized { size: padded_size } = padded.size_info { + // If the layout is sized, it will remain sized after padding is + // added. Its sum will be its unpadded size and the size of the + // trailing padding needed to satisfy its alignment + // requirements. + let padding = padding_needed_for(unpadded_size, layout.align); + assert_eq!(padded_size, unpadded_size + padding); + + // Prove that calling `DstLayout::pad_to_align` behaves + // identically to `Layout::pad_to_align`. + let layout_analog = + Layout::from_size_align(unpadded_size, layout.align.get()).unwrap(); + let padded_analog = layout_analog.pad_to_align(); + assert_eq!(padded_analog.align(), layout.align.get()); + assert_eq!(padded_analog.size(), padded_size); + } else { + panic!("The padding of a sized layout must result in a sized layout.") + } + } else { + // If the layout is a DST, padding cannot be statically added. + assert_eq!(padded.size_info, layout.size_info); + } + } +} diff --git a/rust/zerocopy/src/lib.rs b/rust/zerocopy/src/lib.rs new file mode 100644 index 000000000000..572f0563fe2f --- /dev/null +++ b/rust/zerocopy/src/lib.rs @@ -0,0 +1,7614 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2018 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License <LICENSE-BSD or +// https://opensource.org/license/bsd-2-clause>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +// After updating the following doc comment, make sure to run the following +// command to update `README.md` based on its contents: +// +// (cd .. && cargo -q run --manifest-path tools/Cargo.toml -p generate-readme) > README.md + +//! ***<span style="font-size: 140%">Fast, safe, <span +//! style="color:red;">compile error</span>. Pick two.</span>*** +//! +//! Zerocopy makes zero-cost memory manipulation effortless. We write `unsafe` +//! so you don't have to. +//! +//! *For an overview of what's changed from zerocopy 0.7, check out our [release +//! notes][release-notes], which include a step-by-step upgrading guide.* +//! +//! *Have questions? Need more out of zerocopy? Submit a [customer request +//! issue][customer-request-issue] or ask the maintainers on +//! [GitHub][github-q-a] or [Discord][discord]!* +//! +//! [customer-request-issue]: https://github.com/google/zerocopy/issues/new/choose +//! [release-notes]: https://github.com/google/zerocopy/discussions/1680 +//! [github-q-a]: https://github.com/google/zerocopy/discussions/categories/q-a +//! [discord]: https://discord.gg/MAvWH2R6zk +//! +//! # Overview +//! +//! ##### Conversion Traits +//! +//! Zerocopy provides four derivable traits for zero-cost conversions: +//! - [`TryFromBytes`] indicates that a type may safely be converted from +//! certain byte sequences (conditional on runtime checks) +//! - [`FromZeros`] indicates that a sequence of zero bytes represents a valid +//! instance of a type +//! - [`FromBytes`] indicates that a type may safely be converted from an +//! arbitrary byte sequence +//! - [`IntoBytes`] indicates that a type may safely be converted *to* a byte +//! sequence +//! +//! These traits support sized types, slices, and [slice DSTs][slice-dsts]. +//! +//! [slice-dsts]: KnownLayout#dynamically-sized-types +//! +//! ##### Marker Traits +//! +//! Zerocopy provides three derivable marker traits that do not provide any +//! functionality themselves, but are required to call certain methods provided +//! by the conversion traits: +//! - [`KnownLayout`] indicates that zerocopy can reason about certain layout +//! qualities of a type +//! - [`Immutable`] indicates that a type is free from interior mutability, +//! except by ownership or an exclusive (`&mut`) borrow +//! - [`Unaligned`] indicates that a type's alignment requirement is 1 +//! +//! You should generally derive these marker traits whenever possible. +//! +//! ##### Conversion Macros +//! +//! Zerocopy provides six macros for safe casting between types: +//! +//! - ([`try_`][try_transmute])[`transmute`] (conditionally) converts a value of +//! one type to a value of another type of the same size +//! - ([`try_`][try_transmute_mut])[`transmute_mut`] (conditionally) converts a +//! mutable reference of one type to a mutable reference of another type of +//! the same size +//! - ([`try_`][try_transmute_ref])[`transmute_ref`] (conditionally) converts a +//! mutable or immutable reference of one type to an immutable reference of +//! another type of the same size +//! +//! These macros perform *compile-time* size and alignment checks, meaning that +//! unconditional casts have zero cost at runtime. Conditional casts do not need +//! to validate size or alignment runtime, but do need to validate contents. +//! +//! These macros cannot be used in generic contexts. For generic conversions, +//! use the methods defined by the [conversion traits](#conversion-traits). +//! +//! ##### Byteorder-Aware Numerics +//! +//! Zerocopy provides byte-order aware integer types that support these +//! conversions; see the [`byteorder`] module. These types are especially useful +//! for network parsing. +//! +//! # Cargo Features +//! +//! - **`alloc`** +//! By default, `zerocopy` is `no_std`. When the `alloc` feature is enabled, +//! the `alloc` crate is added as a dependency, and some allocation-related +//! functionality is added. +//! +//! - **`std`** +//! By default, `zerocopy` is `no_std`. When the `std` feature is enabled, the +//! `std` crate is added as a dependency (ie, `no_std` is disabled), and +//! support for some `std` types is added. `std` implies `alloc`. +//! +//! - **`derive`** +//! Provides derives for the core marker traits via the `zerocopy-derive` +//! crate. These derives are re-exported from `zerocopy`, so it is not +//! necessary to depend on `zerocopy-derive` directly. +//! +//! However, you may experience better compile times if you instead directly +//! depend on both `zerocopy` and `zerocopy-derive` in your `Cargo.toml`, +//! since doing so will allow Rust to compile these crates in parallel. To do +//! so, do *not* enable the `derive` feature, and list both dependencies in +//! your `Cargo.toml` with the same leading non-zero version number; e.g: +//! +//! ```toml +//! [dependencies] +//! zerocopy = "0.X" +//! zerocopy-derive = "0.X" +//! ``` +//! +//! To avoid the risk of [duplicate import errors][duplicate-import-errors] if +//! one of your dependencies enables zerocopy's `derive` feature, import +//! derives as `use zerocopy_derive::*` rather than by name (e.g., `use +//! zerocopy_derive::FromBytes`). +//! +//! - **`simd`** +//! When the `simd` feature is enabled, `FromZeros`, `FromBytes`, and +//! `IntoBytes` impls are emitted for all stable SIMD types which exist on the +//! target platform. Note that the layout of SIMD types is not yet stabilized, +//! so these impls may be removed in the future if layout changes make them +//! invalid. For more information, see the Unsafe Code Guidelines Reference +//! page on the [layout of packed SIMD vectors][simd-layout]. +//! +//! - **`simd-nightly`** +//! Enables the `simd` feature and adds support for SIMD types which are only +//! available on nightly. Since these types are unstable, support for any type +//! may be removed at any point in the future. +//! +//! - **`float-nightly`** +//! Adds support for the unstable `f16` and `f128` types. These types are +//! not yet fully implemented and may not be supported on all platforms. +//! +//! [duplicate-import-errors]: https://github.com/google/zerocopy/issues/1587 +//! [simd-layout]: https://rust-lang.github.io/unsafe-code-guidelines/layout/packed-simd-vectors.html +//! +//! # Build Tuning +//! +//! ## `--cfg zerocopy_inline_always` +//! +//! Upgrades `#[inline]` to `#[inline(always)]` on many of zerocopy's public +//! functions and methods. This provides a narrowly-scoped alternative that +//! *may* improve the optimization of hot paths using zerocopy without the broad +//! compile-time penalties of configuring `codegen-units=1`. +//! +//! # Security Ethos +//! +//! Zerocopy is expressly designed for use in security-critical contexts. We +//! strive to ensure that that zerocopy code is sound under Rust's current +//! memory model, and *any future memory model*. We ensure this by: +//! - **...not 'guessing' about Rust's semantics.** +//! We annotate `unsafe` code with a precise rationale for its soundness that +//! cites a relevant section of Rust's official documentation. When Rust's +//! documented semantics are unclear, we work with the Rust Operational +//! Semantics Team to clarify Rust's documentation. +//! - **...rigorously testing our implementation.** +//! We run tests using [Miri], ensuring that zerocopy is sound across a wide +//! array of supported target platforms of varying endianness and pointer +//! width, and across both current and experimental memory models of Rust. +//! - **...formally proving the correctness of our implementation.** +//! We apply formal verification tools like [Kani][kani] to prove zerocopy's +//! correctness. +//! +//! For more information, see our full [soundness policy]. +//! +//! [Miri]: https://github.com/rust-lang/miri +//! [Kani]: https://github.com/model-checking/kani +//! [soundness policy]: https://github.com/google/zerocopy/blob/main/zerocopy/POLICIES.md#soundness +//! +//! # Relationship to Project Safe Transmute +//! +//! [Project Safe Transmute] is an official initiative of the Rust Project to +//! develop language-level support for safer transmutation. The Project consults +//! with crates like zerocopy to identify aspects of safer transmutation that +//! would benefit from compiler support, and has developed an [experimental, +//! compiler-supported analysis][mcp-transmutability] which determines whether, +//! for a given type, any value of that type may be soundly transmuted into +//! another type. Once this functionality is sufficiently mature, zerocopy +//! intends to replace its internal transmutability analysis (implemented by our +//! custom derives) with the compiler-supported one. This change will likely be +//! an implementation detail that is invisible to zerocopy's users. +//! +//! Project Safe Transmute will not replace the need for most of zerocopy's +//! higher-level abstractions. The experimental compiler analysis is a tool for +//! checking the soundness of `unsafe` code, not a tool to avoid writing +//! `unsafe` code altogether. For the foreseeable future, crates like zerocopy +//! will still be required in order to provide higher-level abstractions on top +//! of the building block provided by Project Safe Transmute. +//! +//! [Project Safe Transmute]: https://rust-lang.github.io/rfcs/2835-project-safe-transmute.html +//! [mcp-transmutability]: https://github.com/rust-lang/compiler-team/issues/411 +//! +//! # MSRV +//! +//! See our [MSRV policy]. +//! +//! [MSRV policy]: https://github.com/google/zerocopy/blob/main/zerocopy/POLICIES.md#msrv +//! +//! # Changelog +//! +//! Zerocopy uses [GitHub Releases]. +//! +//! [GitHub Releases]: https://github.com/google/zerocopy/releases +//! +//! # Thanks +//! +//! Zerocopy is maintained by engineers at Google with help from [many wonderful +//! contributors][contributors]. Thank you to everyone who has lent a hand in +//! making Rust a little more secure! +//! +//! [contributors]: https://github.com/google/zerocopy/graphs/contributors + +// Sometimes we want to use lints which were added after our MSRV. +// `unknown_lints` is `warn` by default and we deny warnings in CI, so without +// this attribute, any unknown lint would cause a CI failure when testing with +// our MSRV. +#![allow(unknown_lints, non_local_definitions, unreachable_patterns)] +#![deny(renamed_and_removed_lints)] +#![deny( + anonymous_parameters, + deprecated_in_future, + late_bound_lifetime_arguments, + missing_copy_implementations, + missing_debug_implementations, + missing_docs, + path_statements, + patterns_in_fns_without_body, + rust_2018_idioms, + trivial_numeric_casts, + unreachable_pub, + unsafe_op_in_unsafe_fn, + unused_extern_crates, + // We intentionally choose not to deny `unused_qualifications`. When items + // are added to the prelude (e.g., `core::mem::size_of`), this has the + // consequence of making some uses trigger this lint on the latest toolchain + // (e.g., `mem::size_of`), but fixing it (e.g. by replacing with `size_of`) + // does not work on older toolchains. + // + // We tested a more complicated fix in #1413, but ultimately decided that, + // since this lint is just a minor style lint, the complexity isn't worth it + // - it's fine to occasionally have unused qualifications slip through, + // especially since these do not affect our user-facing API in any way. + variant_size_differences +)] +#![cfg_attr( + __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, + deny(fuzzy_provenance_casts, lossy_provenance_casts) +)] +#![deny( + clippy::all, + clippy::alloc_instead_of_core, + clippy::arithmetic_side_effects, + clippy::as_underscore, + clippy::assertions_on_result_states, + clippy::as_conversions, + clippy::correctness, + clippy::dbg_macro, + clippy::decimal_literal_representation, + clippy::double_must_use, + clippy::get_unwrap, + clippy::indexing_slicing, + clippy::missing_inline_in_public_items, + clippy::missing_safety_doc, + clippy::multiple_unsafe_ops_per_block, + clippy::must_use_candidate, + clippy::must_use_unit, + clippy::obfuscated_if_else, + clippy::perf, + clippy::print_stdout, + clippy::return_self_not_must_use, + clippy::std_instead_of_core, + clippy::style, + clippy::suspicious, + clippy::todo, + clippy::undocumented_unsafe_blocks, + clippy::unimplemented, + clippy::unnested_or_patterns, + clippy::unwrap_used, + clippy::use_debug +)] +// `clippy::incompatible_msrv` (implied by `clippy::suspicious`): This sometimes +// has false positives, and we test on our MSRV in CI, so it doesn't help us +// anyway. +#![allow(clippy::needless_lifetimes, clippy::type_complexity, clippy::incompatible_msrv)] +#![deny( + rustdoc::bare_urls, + rustdoc::broken_intra_doc_links, + rustdoc::invalid_codeblock_attributes, + rustdoc::invalid_html_tags, + rustdoc::invalid_rust_codeblocks, + rustdoc::missing_crate_level_docs, + rustdoc::private_intra_doc_links +)] +// In test code, it makes sense to weight more heavily towards concise, readable +// code over correct or debuggable code. +#![cfg_attr(any(test, kani), allow( + // In tests, you get line numbers and have access to source code, so panic + // messages are less important. You also often unwrap a lot, which would + // make expect'ing instead very verbose. + clippy::unwrap_used, + // In tests, there's no harm to "panic risks" - the worst that can happen is + // that your test will fail, and you'll fix it. By contrast, panic risks in + // production code introduce the possibly of code panicking unexpectedly "in + // the field". + clippy::arithmetic_side_effects, + clippy::indexing_slicing, +))] +#![cfg_attr(not(any(test, kani, feature = "std")), no_std)] +#![cfg_attr( + all(feature = "simd-nightly", target_arch = "arm"), + feature(stdarch_arm_neon_intrinsics) +)] +#![cfg_attr( + all(feature = "simd-nightly", any(target_arch = "powerpc", target_arch = "powerpc64")), + feature(stdarch_powerpc) +)] +#![cfg_attr(feature = "float-nightly", feature(f16, f128))] +#![cfg_attr(doc_cfg, feature(doc_cfg))] +#![cfg_attr(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, feature(coverage_attribute))] +#![cfg_attr( + any(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, miri), + feature(layout_for_ptr) +)] +#![cfg_attr(all(test, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), feature(test))] + +// This is a hack to allow zerocopy-derive derives to work in this crate. They +// assume that zerocopy is linked as an extern crate, so they access items from +// it as `zerocopy::Xxx`. This makes that still work. +#[cfg(any(feature = "derive", test))] +extern crate self as zerocopy; + +#[cfg(all(test, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS))] +extern crate test; + +#[doc(hidden)] +#[macro_use] +pub mod util; + +pub mod byte_slice; +pub mod byteorder; +mod deprecated; + +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_DEV_MODE)] +pub mod doctests; + +// This module is `pub` so that zerocopy's error types and error handling +// documentation is grouped together in a cohesive module. In practice, we +// expect most users to use the re-export of `error`'s items to avoid identifier +// stuttering. +pub mod error; +mod impls; +#[doc(hidden)] +pub mod layout; +mod macros; +#[cfg_attr(not(zerocopy_unstable_ptr), doc(hidden))] +#[cfg_attr(doc_cfg, doc(cfg(zerocopy_unstable_ptr)))] +pub mod pointer; +mod r#ref; +mod split_at; +// FIXME(#252): If we make this pub, come up with a better name. +mod wrappers; + +use core::{ + cell::{Cell, UnsafeCell}, + cmp::Ordering, + fmt::{self, Debug, Display, Formatter}, + hash::Hasher, + marker::PhantomData, + mem::{self, ManuallyDrop, MaybeUninit as CoreMaybeUninit}, + num::{ + NonZeroI128, NonZeroI16, NonZeroI32, NonZeroI64, NonZeroI8, NonZeroIsize, NonZeroU128, + NonZeroU16, NonZeroU32, NonZeroU64, NonZeroU8, NonZeroUsize, Wrapping, + }, + ops::{Deref, DerefMut}, + ptr::{self, NonNull}, + slice, +}; +#[cfg(feature = "std")] +use std::io; + +#[doc(hidden)] +pub use crate::pointer::{ + invariant::{self, BecauseExclusive}, + PtrInner, +}; +pub use crate::{ + byte_slice::*, + byteorder::*, + error::*, + r#ref::*, + split_at::{Split, SplitAt}, + wrappers::*, +}; + +#[cfg(any(feature = "alloc", test, kani))] +extern crate alloc; +#[cfg(any(feature = "alloc", test))] +use alloc::{boxed::Box, vec::Vec}; +#[cfg(any(feature = "alloc", test))] +use core::alloc::Layout; + +// Used by `KnownLayout`. +#[doc(hidden)] +pub use crate::layout::*; +// Used by `TryFromBytes::is_bit_valid`. +#[doc(hidden)] +pub use crate::pointer::{invariant::BecauseImmutable, Maybe, Ptr}; +// For each trait polyfill, as soon as the corresponding feature is stable, the +// polyfill import will be unused because method/function resolution will prefer +// the inherent method/function over a trait method/function. Thus, we suppress +// the `unused_imports` warning. +// +// See the documentation on `util::polyfills` for more information. +#[allow(unused_imports)] +use crate::util::polyfills::{self, NonNullExt as _, NumExt as _}; +#[cfg_attr(not(zerocopy_unstable_ptr), doc(hidden))] +#[cfg_attr(doc_cfg, doc(cfg(zerocopy_unstable_ptr)))] +pub use crate::util::MetadataOf; + +#[cfg(all(test, not(__ZEROCOPY_INTERNAL_USE_ONLY_DEV_MODE)))] +const _: () = { + #[deprecated = "Development of zerocopy using cargo is not supported. Please use `cargo.sh` or `win-cargo.bat` instead."] + #[allow(unused)] + const WARNING: () = (); + #[warn(deprecated)] + WARNING +}; + +#[cfg(all(any(feature = "derive", test), zerocopy_unstable_linux))] +pub use zerocopy_derive::most_traits; +/// Implements [`KnownLayout`]. +/// +/// This derive analyzes various aspects of a type's layout that are needed for +/// some of zerocopy's APIs. It can be applied to structs, enums, and unions; +/// e.g.: +/// +/// ``` +/// # use zerocopy_derive::KnownLayout; +/// #[derive(KnownLayout)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(KnownLayout)] +/// enum MyEnum { +/// # V00, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(KnownLayout)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// # Limitations +/// +/// This derive cannot currently be applied to unsized structs without an +/// explicit `repr` attribute. +/// +/// Some invocations of this derive run afoul of a [known bug] in Rust's type +/// privacy checker. For example, this code: +/// +/// ```compile_fail,E0446 +/// use zerocopy::*; +/// # use zerocopy_derive::*; +/// +/// #[derive(KnownLayout)] +/// #[repr(C)] +/// pub struct PublicType { +/// leading: Foo, +/// trailing: Bar, +/// } +/// +/// #[derive(KnownLayout)] +/// struct Foo; +/// +/// #[derive(KnownLayout)] +/// struct Bar; +/// ``` +/// +/// ...results in a compilation error: +/// +/// ```text +/// error[E0446]: private type `Bar` in public interface +/// --> examples/bug.rs:3:10 +/// | +/// 3 | #[derive(KnownLayout)] +/// | ^^^^^^^^^^^ can't leak private type +/// ... +/// 14 | struct Bar; +/// | ---------- `Bar` declared as private +/// | +/// = note: this error originates in the derive macro `KnownLayout` (in Nightly builds, run with -Z macro-backtrace for more info) +/// ``` +/// +/// This issue arises when `#[derive(KnownLayout)]` is applied to `repr(C)` +/// structs whose trailing field type is less public than the enclosing struct. +/// +/// To work around this, mark the trailing field type `pub` and annotate it with +/// `#[doc(hidden)]`; e.g.: +/// +/// ```no_run +/// use zerocopy::*; +/// # use zerocopy_derive::*; +/// +/// #[derive(KnownLayout)] +/// #[repr(C)] +/// pub struct PublicType { +/// leading: Foo, +/// trailing: Bar, +/// } +/// +/// #[derive(KnownLayout)] +/// struct Foo; +/// +/// #[doc(hidden)] +/// #[derive(KnownLayout)] +/// pub struct Bar; // <- `Bar` is now also `pub` +/// ``` +/// +/// [known bug]: https://github.com/rust-lang/rust/issues/45713 +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::KnownLayout; +// These exist so that code which was written against the old names will get +// less confusing error messages when they upgrade to a more recent version of +// zerocopy. On our MSRV toolchain, the error messages read, for example: +// +// error[E0603]: trait `FromZeroes` is private +// --> examples/deprecated.rs:1:15 +// | +// 1 | use zerocopy::FromZeroes; +// | ^^^^^^^^^^ private trait +// | +// note: the trait `FromZeroes` is defined here +// --> /Users/josh/workspace/zerocopy/src/lib.rs:1845:5 +// | +// 1845 | use FromZeros as FromZeroes; +// | ^^^^^^^^^^^^^^^^^^^^^^^ +// +// The "note" provides enough context to make it easy to figure out how to fix +// the error. +#[allow(unused)] +use {FromZeros as FromZeroes, IntoBytes as AsBytes, Ref as LayoutVerified}; + +/// Indicates that zerocopy can reason about certain aspects of a type's layout. +/// +/// This trait is required by many of zerocopy's APIs. It supports sized types, +/// slices, and [slice DSTs](#dynamically-sized-types). +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(KnownLayout)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::KnownLayout; +/// #[derive(KnownLayout)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(KnownLayout)] +/// enum MyEnum { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(KnownLayout)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated analysis to deduce the layout +/// characteristics of types. You **must** implement this trait via the derive. +/// +/// # Dynamically-sized types +/// +/// `KnownLayout` supports slice-based dynamically sized types ("slice DSTs"). +/// +/// A slice DST is a type whose trailing field is either a slice or another +/// slice DST, rather than a type with fixed size. For example: +/// +/// ``` +/// #[repr(C)] +/// struct PacketHeader { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[repr(C)] +/// struct Packet { +/// header: PacketHeader, +/// body: [u8], +/// } +/// ``` +/// +/// It can be useful to think of slice DSTs as a generalization of slices - in +/// other words, a normal slice is just the special case of a slice DST with +/// zero leading fields. In particular: +/// - Like slices, slice DSTs can have different lengths at runtime +/// - Like slices, slice DSTs cannot be passed by-value, but only by reference +/// or via other indirection such as `Box` +/// - Like slices, a reference (or `Box`, or other pointer type) to a slice DST +/// encodes the number of elements in the trailing slice field +/// +/// ## Slice DST layout +/// +/// Just like other composite Rust types, the layout of a slice DST is not +/// well-defined unless it is specified using an explicit `#[repr(...)]` +/// attribute such as `#[repr(C)]`. [Other representations are +/// supported][reprs], but in this section, we'll use `#[repr(C)]` as our +/// example. +/// +/// A `#[repr(C)]` slice DST is laid out [just like sized `#[repr(C)]` +/// types][repr-c-structs], but the presence of a variable-length field +/// introduces the possibility of *dynamic padding*. In particular, it may be +/// necessary to add trailing padding *after* the trailing slice field in order +/// to satisfy the outer type's alignment, and the amount of padding required +/// may be a function of the length of the trailing slice field. This is just a +/// natural consequence of the normal `#[repr(C)]` rules applied to slice DSTs, +/// but it can result in surprising behavior. For example, consider the +/// following type: +/// +/// ``` +/// #[repr(C)] +/// struct Foo { +/// a: u32, +/// b: u8, +/// z: [u16], +/// } +/// ``` +/// +/// Assuming that `u32` has alignment 4 (this is not true on all platforms), +/// then `Foo` has alignment 4 as well. Here is the smallest possible value for +/// `Foo`: +/// +/// ```text +/// byte offset | 01234567 +/// field | aaaab--- +/// >< +/// ``` +/// +/// In this value, `z` has length 0. Abiding by `#[repr(C)]`, the lowest offset +/// that we can place `z` at is 5, but since `z` has alignment 2, we need to +/// round up to offset 6. This means that there is one byte of padding between +/// `b` and `z`, then 0 bytes of `z` itself (denoted `><` in this diagram), and +/// then two bytes of padding after `z` in order to satisfy the overall +/// alignment of `Foo`. The size of this instance is 8 bytes. +/// +/// What about if `z` has length 1? +/// +/// ```text +/// byte offset | 01234567 +/// field | aaaab-zz +/// ``` +/// +/// In this instance, `z` has length 1, and thus takes up 2 bytes. That means +/// that we no longer need padding after `z` in order to satisfy `Foo`'s +/// alignment. We've now seen two different values of `Foo` with two different +/// lengths of `z`, but they both have the same size - 8 bytes. +/// +/// What about if `z` has length 2? +/// +/// ```text +/// byte offset | 012345678901 +/// field | aaaab-zzzz-- +/// ``` +/// +/// Now `z` has length 2, and thus takes up 4 bytes. This brings our un-padded +/// size to 10, and so we now need another 2 bytes of padding after `z` to +/// satisfy `Foo`'s alignment. +/// +/// Again, all of this is just a logical consequence of the `#[repr(C)]` rules +/// applied to slice DSTs, but it can be surprising that the amount of trailing +/// padding becomes a function of the trailing slice field's length, and thus +/// can only be computed at runtime. +/// +/// [reprs]: https://doc.rust-lang.org/reference/type-layout.html#representations +/// [repr-c-structs]: https://doc.rust-lang.org/reference/type-layout.html#reprc-structs +/// +/// ## What is a valid size? +/// +/// There are two places in zerocopy's API that we refer to "a valid size" of a +/// type. In normal casts or conversions, where the source is a byte slice, we +/// need to know whether the source byte slice is a valid size of the +/// destination type. In prefix or suffix casts, we need to know whether *there +/// exists* a valid size of the destination type which fits in the source byte +/// slice and, if so, what the largest such size is. +/// +/// As outlined above, a slice DST's size is defined by the number of elements +/// in its trailing slice field. However, there is not necessarily a 1-to-1 +/// mapping between trailing slice field length and overall size. As we saw in +/// the previous section with the type `Foo`, instances with both 0 and 1 +/// elements in the trailing `z` field result in a `Foo` whose size is 8 bytes. +/// +/// When we say "x is a valid size of `T`", we mean one of two things: +/// - If `T: Sized`, then we mean that `x == size_of::<T>()` +/// - If `T` is a slice DST, then we mean that there exists a `len` such that the instance of +/// `T` with `len` trailing slice elements has size `x` +/// +/// When we say "largest possible size of `T` that fits in a byte slice", we +/// mean one of two things: +/// - If `T: Sized`, then we mean `size_of::<T>()` if the byte slice is at least +/// `size_of::<T>()` bytes long +/// - If `T` is a slice DST, then we mean to consider all values, `len`, such +/// that the instance of `T` with `len` trailing slice elements fits in the +/// byte slice, and to choose the largest such `len`, if any +/// +/// +/// # Safety +/// +/// This trait does not convey any safety guarantees to code outside this crate. +/// +/// You must not rely on the `#[doc(hidden)]` internals of `KnownLayout`. Future +/// releases of zerocopy may make backwards-breaking changes to these items, +/// including changes that only affect soundness, which may cause code which +/// uses those items to silently become unsound. +/// +#[cfg_attr(feature = "derive", doc = "[derive]: zerocopy_derive::KnownLayout")] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.KnownLayout.html"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(KnownLayout)]` to `{Self}`") +)] +pub unsafe trait KnownLayout { + // The `Self: Sized` bound makes it so that `KnownLayout` can still be + // object safe. It's not currently object safe thanks to `const LAYOUT`, and + // it likely won't be in the future, but there's no reason not to be + // forwards-compatible with object safety. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// The type of metadata stored in a pointer to `Self`. + /// + /// This is `()` for sized types and [`usize`] for slice DSTs. + type PointerMetadata: PointerMetadata; + + /// A maybe-uninitialized analog of `Self` + /// + /// # Safety + /// + /// `Self::LAYOUT` and `Self::MaybeUninit::LAYOUT` are identical. + /// `Self::MaybeUninit` admits uninitialized bytes in all positions. + #[doc(hidden)] + type MaybeUninit: ?Sized + KnownLayout<PointerMetadata = Self::PointerMetadata>; + + /// The layout of `Self`. + /// + /// # Safety + /// + /// Callers may assume that `LAYOUT` accurately reflects the layout of + /// `Self`. In particular: + /// - `LAYOUT.align` is equal to `Self`'s alignment + /// - If `Self: Sized`, then `LAYOUT.size_info == SizeInfo::Sized { size }` + /// where `size == size_of::<Self>()` + /// - If `Self` is a slice DST, then `LAYOUT.size_info == + /// SizeInfo::SliceDst(slice_layout)` where: + /// - The size, `size`, of an instance of `Self` with `elems` trailing + /// slice elements is equal to `slice_layout.offset + + /// slice_layout.elem_size * elems` rounded up to the nearest multiple + /// of `LAYOUT.align` + /// - For such an instance, any bytes in the range `[slice_layout.offset + + /// slice_layout.elem_size * elems, size)` are padding and must not be + /// assumed to be initialized + #[doc(hidden)] + const LAYOUT: DstLayout; + + /// SAFETY: The returned pointer has the same address and provenance as + /// `bytes`. If `Self` is a DST, the returned pointer's referent has `elems` + /// elements in its trailing slice. + #[doc(hidden)] + fn raw_from_ptr_len(bytes: NonNull<u8>, meta: Self::PointerMetadata) -> NonNull<Self>; + + /// Extracts the metadata from a pointer to `Self`. + /// + /// # Safety + /// + /// `pointer_to_metadata` always returns the correct metadata stored in + /// `ptr`. + #[doc(hidden)] + fn pointer_to_metadata(ptr: *mut Self) -> Self::PointerMetadata; + + /// Computes the length of the byte range addressed by `ptr`. + /// + /// Returns `None` if the resulting length would not fit in an `usize`. + /// + /// # Safety + /// + /// Callers may assume that `size_of_val_raw` always returns the correct + /// size. + /// + /// Callers may assume that, if `ptr` addresses a byte range whose length + /// fits in an `usize`, this will return `Some`. + #[doc(hidden)] + #[must_use] + #[inline(always)] + fn size_of_val_raw(ptr: NonNull<Self>) -> Option<usize> { + let meta = Self::pointer_to_metadata(ptr.as_ptr()); + // SAFETY: `size_for_metadata` promises to only return `None` if the + // resulting size would not fit in a `usize`. + Self::size_for_metadata(meta) + } + + #[doc(hidden)] + #[must_use] + #[inline(always)] + fn raw_dangling() -> NonNull<Self> { + let meta = Self::PointerMetadata::from_elem_count(0); + Self::raw_from_ptr_len(NonNull::dangling(), meta) + } + + /// Computes the size of an object of type `Self` with the given pointer + /// metadata. + /// + /// # Safety + /// + /// `size_for_metadata` promises to return `None` if and only if the + /// resulting size would not fit in a [`usize`]. Note that the returned size + /// could exceed the actual maximum valid size of an allocated object, + /// [`isize::MAX`]. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::KnownLayout; + /// + /// assert_eq!(u8::size_for_metadata(()), Some(1)); + /// assert_eq!(u16::size_for_metadata(()), Some(2)); + /// assert_eq!(<[u8]>::size_for_metadata(42), Some(42)); + /// assert_eq!(<[u16]>::size_for_metadata(42), Some(84)); + /// + /// // This size exceeds the maximum valid object size (`isize::MAX`): + /// assert_eq!(<[u8]>::size_for_metadata(usize::MAX), Some(usize::MAX)); + /// + /// // This size, if computed, would exceed `usize::MAX`: + /// assert_eq!(<[u16]>::size_for_metadata(usize::MAX), None); + /// ``` + #[inline(always)] + fn size_for_metadata(meta: Self::PointerMetadata) -> Option<usize> { + meta.size_for_metadata(Self::LAYOUT) + } + + /// Computes whether `meta` can describe a valid allocation of `Self`. + /// + /// # Safety + /// + /// `is_valid_metadata` promises to return `true` if and only if the size of + /// an allocation of `Self` with `meta` would not overflow an + /// [`isize::MAX`]. + #[doc(hidden)] + #[inline(always)] + fn is_valid_metadata(meta: Self::PointerMetadata) -> bool { + meta.to_elem_count() <= maximum_trailing_slice_len::<Self>().to_elem_count() + } +} + +/// Efficiently produces the [`TrailingSliceLayout`] of `T`. +#[inline(always)] +pub(crate) fn trailing_slice_layout<T>() -> TrailingSliceLayout +where + T: ?Sized + KnownLayout<PointerMetadata = usize>, +{ + trait LayoutFacts { + const SIZE_INFO: TrailingSliceLayout; + } + + impl<T: ?Sized> LayoutFacts for T + where + T: KnownLayout<PointerMetadata = usize>, + { + const SIZE_INFO: TrailingSliceLayout = match T::LAYOUT.size_info { + crate::SizeInfo::Sized { .. } => const_panic!("unreachable"), + crate::SizeInfo::SliceDst(info) => info, + }; + } + + T::SIZE_INFO +} + +/// Efficiently produces the maximum trailing slice length `T`. +#[inline(always)] +pub(crate) fn maximum_trailing_slice_len<T>() -> usize +where + T: ?Sized + KnownLayout, +{ + trait LayoutFacts { + const MAX_LEN: usize; + } + + impl<T: ?Sized> LayoutFacts for T + where + T: KnownLayout, + { + const MAX_LEN: usize = match T::LAYOUT.size_info { + SizeInfo::SliceDst(TrailingSliceLayout { elem_size: 0, .. }) => usize::MAX, + _ => match T::LAYOUT.validate_cast_and_convert_metadata( + T::LAYOUT.align.get(), + DstLayout::MAX_SIZE, + CastType::Prefix, + ) { + Ok((elems, _)) => elems, + Err(_) => const_panic!("unreachable"), + }, + }; + } + + T::MAX_LEN +} + +/// The metadata associated with a [`KnownLayout`] type. +#[doc(hidden)] +pub trait PointerMetadata: Copy + Eq + Debug + Ord { + /// Constructs a `Self` from an element count. + /// + /// If `Self = ()`, this returns `()`. If `Self = usize`, this returns + /// `elems`. No other types are currently supported. + fn from_elem_count(elems: usize) -> Self; + + /// Converts `self` to an element count. + /// + /// If `Self = ()`, this returns `0`. If `Self = usize`, this returns + /// `self`. No other types are currently supported. + fn to_elem_count(self) -> usize; + + /// Computes the size of the object with the given layout and pointer + /// metadata. + /// + /// # Panics + /// + /// If `Self = ()`, `layout` must describe a sized type. If `Self = usize`, + /// `layout` must describe a slice DST. Otherwise, `size_for_metadata` may + /// panic. + /// + /// # Safety + /// + /// `size_for_metadata` promises to only return `None` if the resulting size + /// would not fit in a `usize`. + fn size_for_metadata(self, layout: DstLayout) -> Option<usize>; +} + +impl PointerMetadata for () { + #[inline] + #[allow(clippy::unused_unit)] + fn from_elem_count(_elems: usize) -> () {} + + #[inline] + fn to_elem_count(self) -> usize { + 0 + } + + #[inline] + fn size_for_metadata(self, layout: DstLayout) -> Option<usize> { + match layout.size_info { + SizeInfo::Sized { size } => Some(size), + // NOTE: This branch is unreachable, but we return `None` rather + // than `unreachable!()` to avoid generating panic paths. + SizeInfo::SliceDst(_) => None, + } + } +} + +impl PointerMetadata for usize { + #[inline] + fn from_elem_count(elems: usize) -> usize { + elems + } + + #[inline] + fn to_elem_count(self) -> usize { + self + } + + #[inline] + fn size_for_metadata(self, layout: DstLayout) -> Option<usize> { + match layout.size_info { + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + let slice_len = elem_size.checked_mul(self)?; + let without_padding = offset.checked_add(slice_len)?; + without_padding.checked_add(util::padding_needed_for(without_padding, layout.align)) + } + // NOTE: This branch is unreachable, but we return `None` rather + // than `unreachable!()` to avoid generating panic paths. + SizeInfo::Sized { .. } => None, + } + } +} + +// SAFETY: Delegates safety to `DstLayout::for_slice`. +unsafe impl<T> KnownLayout for [T] { + #[allow(clippy::missing_inline_in_public_items, dead_code)] + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type PointerMetadata = usize; + + // SAFETY: `CoreMaybeUninit<T>::LAYOUT` and `T::LAYOUT` are identical + // because `CoreMaybeUninit<T>` has the same size and alignment as `T` [1]. + // Consequently, `[CoreMaybeUninit<T>]::LAYOUT` and `[T]::LAYOUT` are + // identical, because they both lack a fixed-sized prefix and because they + // inherit the alignments of their inner element type (which are identical) + // [2][3]. + // + // `[CoreMaybeUninit<T>]` admits uninitialized bytes at all positions + // because `CoreMaybeUninit<T>` admits uninitialized bytes at all positions + // and because the inner elements of `[CoreMaybeUninit<T>]` are laid out + // back-to-back [2][3]. + // + // [1] Per https://doc.rust-lang.org/1.81.0/std/mem/union.MaybeUninit.html#layout-1: + // + // `MaybeUninit<T>` is guaranteed to have the same size, alignment, and ABI as + // `T` + // + // [2] Per https://doc.rust-lang.org/1.82.0/reference/type-layout.html#slice-layout: + // + // Slices have the same layout as the section of the array they slice. + // + // [3] Per https://doc.rust-lang.org/1.82.0/reference/type-layout.html#array-layout: + // + // An array of `[T; N]` has a size of `size_of::<T>() * N` and the same + // alignment of `T`. Arrays are laid out so that the zero-based `nth` + // element of the array is offset from the start of the array by `n * + // size_of::<T>()` bytes. + type MaybeUninit = [CoreMaybeUninit<T>]; + + const LAYOUT: DstLayout = DstLayout::for_slice::<T>(); + + // SAFETY: `.cast` preserves address and provenance. The returned pointer + // refers to an object with `elems` elements by construction. + #[inline(always)] + fn raw_from_ptr_len(data: NonNull<u8>, elems: usize) -> NonNull<Self> { + // FIXME(#67): Remove this allow. See NonNullExt for more details. + #[allow(unstable_name_collisions)] + NonNull::slice_from_raw_parts(data.cast::<T>(), elems) + } + + #[inline(always)] + fn pointer_to_metadata(ptr: *mut [T]) -> usize { + #[allow(clippy::as_conversions)] + let slc = ptr as *const [()]; + + // SAFETY: + // - `()` has alignment 1, so `slc` is trivially aligned. + // - `slc` was derived from a non-null pointer. + // - The size is 0 regardless of the length, so it is sound to + // materialize a reference regardless of location. + // - By invariant, `self.ptr` has valid provenance. + let slc = unsafe { &*slc }; + + // This is correct because the preceding `as` cast preserves the number + // of slice elements. [1] + // + // [1] Per https://doc.rust-lang.org/reference/expressions/operator-expr.html#pointer-to-pointer-cast: + // + // For slice types like `[T]` and `[U]`, the raw pointer types `*const + // [T]`, `*mut [T]`, `*const [U]`, and `*mut [U]` encode the number of + // elements in this slice. Casts between these raw pointer types + // preserve the number of elements. ... The same holds for `str` and + // any compound type whose unsized tail is a slice type, such as + // struct `Foo(i32, [u8])` or `(u64, Foo)`. + slc.len() + } +} + +#[rustfmt::skip] +impl_known_layout!( + (), + u8, i8, u16, i16, u32, i32, u64, i64, u128, i128, usize, isize, f32, f64, + bool, char, + NonZeroU8, NonZeroI8, NonZeroU16, NonZeroI16, NonZeroU32, NonZeroI32, + NonZeroU64, NonZeroI64, NonZeroU128, NonZeroI128, NonZeroUsize, NonZeroIsize +); +#[rustfmt::skip] +#[cfg(feature = "float-nightly")] +impl_known_layout!( + #[cfg_attr(doc_cfg, doc(cfg(feature = "float-nightly")))] + f16, + #[cfg_attr(doc_cfg, doc(cfg(feature = "float-nightly")))] + f128 +); +#[rustfmt::skip] +impl_known_layout!( + T => Option<T>, + T: ?Sized => PhantomData<T>, + T => Wrapping<T>, + T => CoreMaybeUninit<T>, + T: ?Sized => *const T, + T: ?Sized => *mut T, + T: ?Sized => &'_ T, + T: ?Sized => &'_ mut T, +); +impl_known_layout!(const N: usize, T => [T; N]); + +// SAFETY: `str` has the same representation as `[u8]`. `ManuallyDrop<T>` [1], +// `UnsafeCell<T>` [2], and `Cell<T>` [3] have the same representation as `T`. +// +// [1] Per https://doc.rust-lang.org/1.85.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop<T>` is guaranteed to have the same layout and bit validity as +// `T` +// +// [2] Per https://doc.rust-lang.org/1.85.0/core/cell/struct.UnsafeCell.html#memory-layout: +// +// `UnsafeCell<T>` has the same in-memory representation as its inner type +// `T`. +// +// [3] Per https://doc.rust-lang.org/1.85.0/core/cell/struct.Cell.html#memory-layout: +// +// `Cell<T>` has the same in-memory representation as `T`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + unsafe_impl_known_layout!( + #[repr([u8])] + str + ); + unsafe_impl_known_layout!(T: ?Sized + KnownLayout => #[repr(T)] ManuallyDrop<T>); + unsafe_impl_known_layout!(T: ?Sized + KnownLayout => #[repr(T)] UnsafeCell<T>); + unsafe_impl_known_layout!(T: ?Sized + KnownLayout => #[repr(T)] Cell<T>); +}; + +// SAFETY: +// - By consequence of the invariant on `T::MaybeUninit` that `T::LAYOUT` and +// `T::MaybeUninit::LAYOUT` are equal, `T` and `T::MaybeUninit` have the same: +// - Fixed prefix size +// - Alignment +// - (For DSTs) trailing slice element size +// - By consequence of the above, referents `T::MaybeUninit` and `T` have the +// require the same kind of pointer metadata, and thus it is valid to perform +// an `as` cast from `*mut T` and `*mut T::MaybeUninit`, and this operation +// preserves referent size (ie, `size_of_val_raw`). +const _: () = unsafe { + unsafe_impl_known_layout!(T: ?Sized + KnownLayout => #[repr(T::MaybeUninit)] MaybeUninit<T>) +}; + +// FIXME(#196, #2856): Eventually, we'll want to support enums variants and +// union fields being treated uniformly since they behave similarly to each +// other in terms of projecting validity – specifically, for a type `T` with +// validity `V`, if `T` is a struct type, then its fields straightforwardly also +// have validity `V`. By contrast, if `T` is an enum or union type, then +// validity is not straightforwardly recursive in this way. +#[doc(hidden)] +pub const STRUCT_VARIANT_ID: i128 = -1; +#[doc(hidden)] +pub const UNION_VARIANT_ID: i128 = -2; +#[doc(hidden)] +pub const REPR_C_UNION_VARIANT_ID: i128 = -3; + +/// # Safety +/// +/// `Self::ProjectToTag` must satisfy its safety invariant. +#[doc(hidden)] +pub unsafe trait HasTag { + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// The type's enum tag, or `()` for non-enum types. + type Tag: Immutable; + + /// A pointer projection from `Self` to its tag. + /// + /// # Safety + /// + /// It must be the case that, for all `slf: Ptr<'_, Self, I>`, it is sound + /// to project from `slf` to `Ptr<'_, Self::Tag, I>` using this projection. + type ProjectToTag: pointer::cast::Project<Self, Self::Tag>; +} + +/// Projects a given field from `Self`. +/// +/// All implementations of `HasField` for a particular field `f` in `Self` +/// should use the same `Field` type; this ensures that `Field` is inferable +/// given an explicit `VARIANT_ID` and `FIELD_ID`. +/// +/// # Safety +/// +/// A field `f` is `HasField` for `Self` if and only if: +/// +/// - If `Self` has the layout of a struct or union type, then `VARIANT_ID` is +/// `STRUCT_VARIANT_ID` or `UNION_VARIANT_ID` respectively; otherwise, if +/// `Self` has the layout of an enum type, `VARIANT_ID` is the numerical index +/// of the enum variant in which `f` appears. Note that `Self` does not need +/// to actually *be* such a type – it just needs to have the same layout as +/// such a type. For example, a `#[repr(transparent)]` wrapper around an enum +/// has the same layout as that enum. +/// - If `f` has name `n`, `FIELD_ID` is `zerocopy::ident_id!(n)`; otherwise, +/// if `f` is at index `i`, `FIELD_ID` is `zerocopy::ident_id!(i)`. +/// - `Field` is a type with the same visibility as `f`. +/// - `Type` has the same type as `f`. +/// +/// The caller must **not** assume that a pointer's referent being aligned +/// implies that calling `project` on that pointer will result in a pointer to +/// an aligned referent. For example, `HasField` may be implemented for +/// `#[repr(packed)]` structs. +/// +/// The implementation of `project` must satisfy its safety post-condition. +#[doc(hidden)] +pub unsafe trait HasField<Field, const VARIANT_ID: i128, const FIELD_ID: i128>: + HasTag +{ + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// The type of the field. + type Type: ?Sized; + + /// Projects from `slf` to the field. + /// + /// Users should generally not call `project` directly, and instead should + /// use high-level APIs like [`PtrInner::project`] or [`Ptr::project`]. + /// + /// # Safety + /// + /// The returned pointer refers to a non-strict subset of the bytes of + /// `slf`'s referent, and has the same provenance as `slf`. + #[must_use] + fn project(slf: PtrInner<'_, Self>) -> *mut Self::Type; +} + +/// Projects a given field from `Self`. +/// +/// Implementations of this trait encode the conditions under which a field can +/// be projected from a `Ptr<'_, Self, I>`, and how the invariants of that +/// [`Ptr`] (`I`) determine the invariants of pointers projected from it. In +/// other words, it is a type-level function over invariants; `I` goes in, +/// `Self::Invariants` comes out. +/// +/// # Safety +/// +/// `T: ProjectField<Field, I, VARIANT_ID, FIELD_ID>` if, for a +/// `ptr: Ptr<'_, T, I>` such that `T::is_projectable(ptr).is_ok()`, +/// `<T as HasField<Field, VARIANT_ID, FIELD_ID>>::project(ptr.as_inner())` +/// conforms to `T::Invariants`. +#[doc(hidden)] +pub unsafe trait ProjectField<Field, I, const VARIANT_ID: i128, const FIELD_ID: i128>: + HasField<Field, VARIANT_ID, FIELD_ID> +where + I: invariant::Invariants, +{ + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// The invariants of the projected field pointer, with respect to the + /// invariants, `I`, of the containing pointer. The aliasing dimension of + /// the invariants is guaranteed to remain unchanged. + type Invariants: invariant::Invariants<Aliasing = I::Aliasing>; + + /// The failure mode of projection. `()` if the projection is fallible, + /// otherwise [`core::convert::Infallible`]. + type Error; + + /// Is the given field projectable from `ptr`? + /// + /// If a field with [`Self::Invariants`] is projectable from the referent, + /// this function produces an `Ok(ptr)` from which the projection can be + /// made; otherwise `Err`. + /// + /// This method must be overriden if the field's projectability depends on + /// the value of the bytes in `ptr`. + #[inline(always)] + fn is_projectable<'a>(_ptr: Ptr<'a, Self::Tag, I>) -> Result<(), Self::Error> { + trait IsInfallible { + const IS_INFALLIBLE: bool; + } + + struct Projection<T, Field, I, const VARIANT_ID: i128, const FIELD_ID: i128>( + PhantomData<(Field, I, T)>, + ) + where + T: ?Sized + HasField<Field, VARIANT_ID, FIELD_ID>, + I: invariant::Invariants; + + impl<T, Field, I, const VARIANT_ID: i128, const FIELD_ID: i128> IsInfallible + for Projection<T, Field, I, VARIANT_ID, FIELD_ID> + where + T: ?Sized + HasField<Field, VARIANT_ID, FIELD_ID>, + I: invariant::Invariants, + { + const IS_INFALLIBLE: bool = { + let is_infallible = match VARIANT_ID { + // For nondestructive projections of struct and union + // fields, the projected field's satisfaction of + // `Invariants` does not depend on the value of the + // referent. This default implementation of `is_projectable` + // is non-destructive, as it does not overwrite any part of + // the referent. + crate::STRUCT_VARIANT_ID | crate::UNION_VARIANT_ID => true, + _enum_variant => { + use crate::invariant::{Validity, ValidityKind}; + match I::Validity::KIND { + // The `Uninit` and `Initialized` validity + // invariants do not depend on the enum's tag. In + // particular, we don't actually care about what + // variant is present – we can treat *any* range of + // uninitialized or initialized memory as containing + // an uninitialized or initialized instance of *any* + // type – the type itself is irrelevant. + ValidityKind::Uninit | ValidityKind::Initialized => true, + // The projectability of an enum field from an + // `AsInitialized` or `Valid` state is a dynamic + // property of its tag. + ValidityKind::AsInitialized | ValidityKind::Valid => false, + } + } + }; + const_assert!(is_infallible); + is_infallible + }; + } + + const_assert!( + <Projection<Self, Field, I, VARIANT_ID, FIELD_ID> as IsInfallible>::IS_INFALLIBLE + ); + + Ok(()) + } +} + +/// Analyzes whether a type is [`FromZeros`]. +/// +/// This derive analyzes, at compile time, whether the annotated type satisfies +/// the [safety conditions] of `FromZeros` and implements `FromZeros` and its +/// supertraits if it is sound to do so. This derive can be applied to structs, +/// enums, and unions; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{FromZeros, Immutable}; +/// #[derive(FromZeros)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromZeros)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromZeros, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// [safety conditions]: trait@FromZeros#safety +/// +/// # Analysis +/// +/// *This section describes, roughly, the analysis performed by this derive to +/// determine whether it is sound to implement `FromZeros` for a given type. +/// Unless you are modifying the implementation of this derive, or attempting to +/// manually implement `FromZeros` for a type yourself, you don't need to read +/// this section.* +/// +/// If a type has the following properties, then this derive can implement +/// `FromZeros` for that type: +/// +/// - If the type is a struct, all of its fields must be `FromZeros`. +/// - If the type is an enum: +/// - It must have a defined representation (`repr`s `C`, `u8`, `u16`, `u32`, +/// `u64`, `usize`, `i8`, `i16`, `i32`, `i64`, or `isize`). +/// - It must have a variant with a discriminant/tag of `0`, and its fields +/// must be `FromZeros`. See [the reference] for a description of +/// discriminant values are specified. +/// - The fields of that variant must be `FromZeros`. +/// +/// This analysis is subject to change. Unsafe code may *only* rely on the +/// documented [safety conditions] of `FromZeros`, and must *not* rely on the +/// implementation details of this derive. +/// +/// [the reference]: https://doc.rust-lang.org/reference/items/enumerations.html#custom-discriminant-values-for-fieldless-enumerations +/// +/// ## Why isn't an explicit representation required for structs? +/// +/// Neither this derive, nor the [safety conditions] of `FromZeros`, requires +/// that structs are marked with `#[repr(C)]`. +/// +/// Per the [Rust reference](reference), +/// +/// > The representation of a type can change the padding between fields, but +/// > does not change the layout of the fields themselves. +/// +/// [reference]: https://doc.rust-lang.org/reference/type-layout.html#representations +/// +/// Since the layout of structs only consists of padding bytes and field bytes, +/// a struct is soundly `FromZeros` if: +/// 1. its padding is soundly `FromZeros`, and +/// 2. its fields are soundly `FromZeros`. +/// +/// The answer to the first question is always yes: padding bytes do not have +/// any validity constraints. A [discussion] of this question in the Unsafe Code +/// Guidelines Working Group concluded that it would be virtually unimaginable +/// for future versions of rustc to add validity constraints to padding bytes. +/// +/// [discussion]: https://github.com/rust-lang/unsafe-code-guidelines/issues/174 +/// +/// Whether a struct is soundly `FromZeros` therefore solely depends on whether +/// its fields are `FromZeros`. +// FIXME(#146): Document why we don't require an enum to have an explicit `repr` +// attribute. +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::FromZeros; +/// Analyzes whether a type is [`Immutable`]. +/// +/// This derive analyzes, at compile time, whether the annotated type satisfies +/// the [safety conditions] of `Immutable` and implements `Immutable` if it is +/// sound to do so. This derive can be applied to structs, enums, and unions; +/// e.g.: +/// +/// ``` +/// # use zerocopy_derive::Immutable; +/// #[derive(Immutable)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Immutable)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// # Analysis +/// +/// *This section describes, roughly, the analysis performed by this derive to +/// determine whether it is sound to implement `Immutable` for a given type. +/// Unless you are modifying the implementation of this derive, you don't need +/// to read this section.* +/// +/// If a type has the following properties, then this derive can implement +/// `Immutable` for that type: +/// +/// - All fields must be `Immutable`. +/// +/// This analysis is subject to change. Unsafe code may *only* rely on the +/// documented [safety conditions] of `Immutable`, and must *not* rely on the +/// implementation details of this derive. +/// +/// [safety conditions]: trait@Immutable#safety +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::Immutable; + +/// Types which are free from interior mutability. +/// +/// `T: Immutable` indicates that `T` does not permit interior mutation, except +/// by ownership or an exclusive (`&mut`) borrow. +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(Immutable)]`][derive] (requires the `derive` Cargo feature); +/// e.g.: +/// +/// ``` +/// # use zerocopy_derive::Immutable; +/// #[derive(Immutable)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Immutable)] +/// enum MyEnum { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `Immutable`. +/// +/// # Safety +/// +/// Unsafe code outside of this crate must not make any assumptions about `T` +/// based on `T: Immutable`. We reserve the right to relax the requirements for +/// `Immutable` in the future, and if unsafe code outside of this crate makes +/// assumptions based on `T: Immutable`, future relaxations may cause that code +/// to become unsound. +/// +// # Safety (Internal) +// +// If `T: Immutable`, unsafe code *inside of this crate* may assume that, given +// `t: &T`, `t` does not permit interior mutation of its referent. Because +// [`UnsafeCell`] is the only type which permits interior mutation, it is +// sufficient (though not necessary) to guarantee that `T` contains no +// `UnsafeCell`s. +// +// [`UnsafeCell`]: core::cell::UnsafeCell +#[cfg_attr( + feature = "derive", + doc = "[derive]: zerocopy_derive::Immutable", + doc = "[derive-analysis]: zerocopy_derive::Immutable#analysis" +)] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.Immutable.html"), + doc = concat!("[derive-analysis]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.Immutable.html#analysis"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(Immutable)]` to `{Self}`") +)] +pub unsafe trait Immutable { + // The `Self: Sized` bound makes it so that `Immutable` is still object + // safe. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; +} + +/// Implements [`TryFromBytes`]. +/// +/// This derive synthesizes the runtime checks required to check whether a +/// sequence of initialized bytes corresponds to a valid instance of a type. +/// This derive can be applied to structs, enums, and unions; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{TryFromBytes, Immutable}; +/// #[derive(TryFromBytes)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(TryFromBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # V00, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(TryFromBytes, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// # Portability +/// +/// To ensure consistent endianness for enums with multi-byte representations, +/// explicitly specify and convert each discriminant using `.to_le()` or +/// `.to_be()`; e.g.: +/// +/// ``` +/// # use zerocopy_derive::TryFromBytes; +/// // `DataStoreVersion` is encoded in little-endian. +/// #[derive(TryFromBytes)] +/// #[repr(u32)] +/// pub enum DataStoreVersion { +/// /// Version 1 of the data store. +/// V1 = 9u32.to_le(), +/// +/// /// Version 2 of the data store. +/// V2 = 10u32.to_le(), +/// } +/// ``` +/// +/// [safety conditions]: trait@TryFromBytes#safety +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::TryFromBytes; + +/// Types for which some bit patterns are valid. +/// +/// A memory region of the appropriate length which contains initialized bytes +/// can be viewed as a `TryFromBytes` type so long as the runtime value of those +/// bytes corresponds to a [*valid instance*] of that type. For example, +/// [`bool`] is `TryFromBytes`, so zerocopy can transmute a [`u8`] into a +/// [`bool`] so long as it first checks that the value of the [`u8`] is `0` or +/// `1`. +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(TryFromBytes)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{TryFromBytes, Immutable}; +/// #[derive(TryFromBytes)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(TryFromBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # V00, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(TryFromBytes, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive ensures that the runtime check of whether bytes correspond to a +/// valid instance is sound. You **must** implement this trait via the derive. +/// +/// # What is a "valid instance"? +/// +/// In Rust, each type has *bit validity*, which refers to the set of bit +/// patterns which may appear in an instance of that type. It is impossible for +/// safe Rust code to produce values which violate bit validity (ie, values +/// outside of the "valid" set of bit patterns). If `unsafe` code produces an +/// invalid value, this is considered [undefined behavior]. +/// +/// Rust's bit validity rules are currently being decided, which means that some +/// types have three classes of bit patterns: those which are definitely valid, +/// and whose validity is documented in the language; those which may or may not +/// be considered valid at some point in the future; and those which are +/// definitely invalid. +/// +/// Zerocopy takes a conservative approach, and only considers a bit pattern to +/// be valid if its validity is a documented guarantee provided by the +/// language. +/// +/// For most use cases, Rust's current guarantees align with programmers' +/// intuitions about what ought to be valid. As a result, zerocopy's +/// conservatism should not affect most users. +/// +/// If you are negatively affected by lack of support for a particular type, +/// we encourage you to let us know by [filing an issue][github-repo]. +/// +/// # `TryFromBytes` is not symmetrical with [`IntoBytes`] +/// +/// There are some types which implement both `TryFromBytes` and [`IntoBytes`], +/// but for which `TryFromBytes` is not guaranteed to accept all byte sequences +/// produced by `IntoBytes`. In other words, for some `T: TryFromBytes + +/// IntoBytes`, there exist values of `t: T` such that +/// `TryFromBytes::try_ref_from_bytes(t.as_bytes()) == None`. Code should not +/// generally assume that values produced by `IntoBytes` will necessarily be +/// accepted as valid by `TryFromBytes`. +/// +/// # Safety +/// +/// On its own, `T: TryFromBytes` does not make any guarantees about the layout +/// or representation of `T`. It merely provides the ability to perform a +/// validity check at runtime via methods like [`try_ref_from_bytes`]. +/// +/// You must not rely on the `#[doc(hidden)]` internals of `TryFromBytes`. +/// Future releases of zerocopy may make backwards-breaking changes to these +/// items, including changes that only affect soundness, which may cause code +/// which uses those items to silently become unsound. +/// +/// [undefined behavior]: https://raphlinus.github.io/programming/rust/2018/08/17/undefined-behavior.html +/// [github-repo]: https://github.com/google/zerocopy +/// [`try_ref_from_bytes`]: TryFromBytes::try_ref_from_bytes +/// [*valid instance*]: #what-is-a-valid-instance +#[cfg_attr(feature = "derive", doc = "[derive]: zerocopy_derive::TryFromBytes")] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.TryFromBytes.html"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(TryFromBytes)]` to `{Self}`") +)] +pub unsafe trait TryFromBytes { + // The `Self: Sized` bound makes it so that `TryFromBytes` is still object + // safe. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// Does a given memory range contain a valid instance of `Self`? + /// + /// # Safety + /// + /// Unsafe code may assume that, if `is_bit_valid(candidate)` returns true, + /// `*candidate` contains a valid `Self`. + /// + /// # Panics + /// + /// `is_bit_valid` may panic. Callers are responsible for ensuring that any + /// `unsafe` code remains sound even in the face of `is_bit_valid` + /// panicking. (We support user-defined validation routines; so long as + /// these routines are not required to be `unsafe`, there is no way to + /// ensure that these do not generate panics.) + /// + /// Besides user-defined validation routines panicking, `is_bit_valid` will + /// either panic or fail to compile if called on a pointer with [`Shared`] + /// aliasing when `Self: !Immutable`. + /// + /// [`UnsafeCell`]: core::cell::UnsafeCell + /// [`Shared`]: invariant::Shared + #[doc(hidden)] + fn is_bit_valid<A>(candidate: Maybe<'_, Self, A>) -> bool + where + A: invariant::Alignment; + + /// Attempts to interpret the given `source` as a `&Self`. + /// + /// If the bytes of `source` are a valid instance of `Self`, this method + /// returns a reference to those bytes interpreted as a `Self`. If the + /// length of `source` is not a [valid size of `Self`][valid-size], or if + /// `source` is not appropriately aligned, or if `source` is not a valid + /// instance of `Self`, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::try_ref_from_bytes(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the byte sequence `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &[0xC0, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5][..]; + /// + /// let packet = Packet::try_ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[0, 1], [2, 3], [4, 5]]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0x10, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5][..]; + /// assert!(Packet::try_ref_from_bytes(bytes).is_err()); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_bytes", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_bytes(source: &[u8]) -> Result<&Self, TryCastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + static_assert_dst_is_not_zst!(Self); + match Ptr::from_ref(source).try_cast_into_no_leftover::<Self, BecauseImmutable>(None) { + Ok(source) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match source.try_into_valid() { + Ok(valid) => Ok(valid.as_ref()), + Err(e) => { + Err(e.map_src(|src| src.as_bytes::<BecauseImmutable>().as_ref()).into()) + } + } + } + Err(e) => Err(e.map_src(Ptr::as_ref).into()), + } + } + + /// Attempts to interpret the prefix of the given `source` as a `&Self`. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the leading bytes of `source`. If that prefix is a valid + /// instance of `Self`, this method returns a reference to those bytes + /// interpreted as `Self`, and a reference to the remaining bytes. If there + /// are insufficient bytes, or if `source` is not appropriately aligned, or + /// if those bytes are not a valid instance of `Self`, this returns `Err`. + /// If [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::try_ref_from_prefix(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &[0xC0, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// + /// let (packet, suffix) = Packet::try_ref_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[0, 1], [2, 3], [4, 5]]); + /// assert_eq!(suffix, &[6u8][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0x10, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// assert!(Packet::try_ref_from_prefix(bytes).is_err()); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_prefix", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_prefix(source: &[u8]) -> Result<(&Self, &[u8]), TryCastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + static_assert_dst_is_not_zst!(Self); + try_ref_from_prefix_suffix(source, CastType::Prefix, None) + } + + /// Attempts to interpret the suffix of the given `source` as a `&Self`. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the trailing bytes of `source`. If that suffix is a + /// valid instance of `Self`, this method returns a reference to those bytes + /// interpreted as `Self`, and a reference to the preceding bytes. If there + /// are insufficient bytes, or if the suffix of `source` would not be + /// appropriately aligned, or if the suffix is not a valid instance of + /// `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], you + /// can [infallibly discard the alignment error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::try_ref_from_suffix(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &[0, 0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let (prefix, packet) = Packet::try_ref_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(prefix, &[0u8][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 77, 240, 0xC0, 0x10][..]; + /// assert!(Packet::try_ref_from_suffix(bytes).is_err()); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_suffix", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_suffix(source: &[u8]) -> Result<(&[u8], &Self), TryCastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + static_assert_dst_is_not_zst!(Self); + try_ref_from_prefix_suffix(source, CastType::Suffix, None).map(swap) + } + + /// Attempts to interpret the given `source` as a `&mut Self` without + /// copying. + /// + /// If the bytes of `source` are a valid instance of `Self`, this method + /// returns a reference to those bytes interpreted as a `Self`. If the + /// length of `source` is not a [valid size of `Self`][valid-size], or if + /// `source` is not appropriately aligned, or if `source` is not a valid + /// instance of `Self`, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::try_mut_from_bytes(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &mut [0xC0, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5][..]; + /// + /// let packet = Packet::try_mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[0, 1], [2, 3], [4, 5]]); + /// + /// packet.temperature = 111; + /// + /// assert_eq!(bytes, [0xC0, 0xC0, 240, 111, 0, 1, 2, 3, 4, 5]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0x10, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// assert!(Packet::try_mut_from_bytes(bytes).is_err()); + /// ``` + /// + #[doc = codegen_header!("h5", "try_mut_from_bytes")] + /// + /// See [`TryFromBytes::try_ref_from_bytes`](#method.try_ref_from_bytes.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_bytes(bytes: &mut [u8]) -> Result<&mut Self, TryCastError<&mut [u8], Self>> + where + Self: KnownLayout + IntoBytes, + { + static_assert_dst_is_not_zst!(Self); + match Ptr::from_mut(bytes).try_cast_into_no_leftover::<Self, BecauseExclusive>(None) { + Ok(source) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match source.try_into_valid() { + Ok(source) => Ok(source.as_mut()), + Err(e) => Err(e.map_src(|src| src.as_bytes().as_mut()).into()), + } + } + Err(e) => Err(e.map_src(Ptr::as_mut).into()), + } + } + + /// Attempts to interpret the prefix of the given `source` as a `&mut + /// Self`. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the leading bytes of `source`. If that prefix is a valid + /// instance of `Self`, this method returns a reference to those bytes + /// interpreted as `Self`, and a reference to the remaining bytes. If there + /// are insufficient bytes, or if `source` is not appropriately aligned, or + /// if the bytes are not a valid instance of `Self`, this returns `Err`. If + /// [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::try_mut_from_prefix(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &mut [0xC0, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// + /// let (packet, suffix) = Packet::try_mut_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[0, 1], [2, 3], [4, 5]]); + /// assert_eq!(suffix, &[6u8][..]); + /// + /// packet.temperature = 111; + /// suffix[0] = 222; + /// + /// assert_eq!(bytes, [0xC0, 0xC0, 240, 111, 0, 1, 2, 3, 4, 5, 222]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0x10, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// assert!(Packet::try_mut_from_prefix(bytes).is_err()); + /// ``` + /// + #[doc = codegen_header!("h5", "try_mut_from_prefix")] + /// + /// See [`TryFromBytes::try_ref_from_prefix`](#method.try_ref_from_prefix.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_prefix( + source: &mut [u8], + ) -> Result<(&mut Self, &mut [u8]), TryCastError<&mut [u8], Self>> + where + Self: KnownLayout + IntoBytes, + { + static_assert_dst_is_not_zst!(Self); + try_mut_from_prefix_suffix(source, CastType::Prefix, None) + } + + /// Attempts to interpret the suffix of the given `source` as a `&mut + /// Self`. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the trailing bytes of `source`. If that suffix is a + /// valid instance of `Self`, this method returns a reference to those bytes + /// interpreted as `Self`, and a reference to the preceding bytes. If there + /// are insufficient bytes, or if the suffix of `source` would not be + /// appropriately aligned, or if the suffix is not a valid instance of + /// `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], you + /// can [infallibly discard the alignment error][ConvertError::from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::try_mut_from_suffix(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &mut [0, 0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let (prefix, packet) = Packet::try_mut_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(prefix, &[0u8][..]); + /// + /// prefix[0] = 111; + /// packet.temperature = 222; + /// + /// assert_eq!(bytes, [111, 0xC0, 0xC0, 240, 222, 2, 3, 4, 5, 6, 7]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 77, 240, 0xC0, 0x10][..]; + /// assert!(Packet::try_mut_from_suffix(bytes).is_err()); + /// ``` + /// + #[doc = codegen_header!("h5", "try_mut_from_suffix")] + /// + /// See [`TryFromBytes::try_ref_from_suffix`](#method.try_ref_from_suffix.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_suffix( + source: &mut [u8], + ) -> Result<(&mut [u8], &mut Self), TryCastError<&mut [u8], Self>> + where + Self: KnownLayout + IntoBytes, + { + static_assert_dst_is_not_zst!(Self); + try_mut_from_prefix_suffix(source, CastType::Suffix, None).map(swap) + } + + /// Attempts to interpret the given `source` as a `&Self` with a DST length + /// equal to `count`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self` with `count` trailing elements. If the length of `source` is not + /// equal to the size of `Self` with `count` elements, if `source` is not + /// appropriately aligned, or if `source` does not contain a valid instance + /// of `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], + /// you can [infallibly discard the alignment error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &[0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let packet = Packet::try_ref_from_bytes_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_ref_from_bytes_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_ref_from_bytes`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let src = 0xCAFEu16.as_bytes(); + /// let zsty = ZSTy::try_ref_from_bytes_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_ref_from_bytes`]: TryFromBytes::try_ref_from_bytes + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_bytes_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_bytes_with_elems( + source: &[u8], + count: usize, + ) -> Result<&Self, TryCastError<&[u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + Immutable, + { + match Ptr::from_ref(source).try_cast_into_no_leftover::<Self, BecauseImmutable>(Some(count)) + { + Ok(source) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match source.try_into_valid() { + Ok(source) => Ok(source.as_ref()), + Err(e) => { + Err(e.map_src(|src| src.as_bytes::<BecauseImmutable>().as_ref()).into()) + } + } + } + Err(e) => Err(e.map_src(Ptr::as_ref).into()), + } + } + + /// Attempts to interpret the prefix of the given `source` as a `&Self` with + /// a DST length equal to `count`. + /// + /// This method attempts to return a reference to the prefix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the remaining bytes. If the length of `source` is less than the size + /// of `Self` with `count` elements, if `source` is not appropriately + /// aligned, or if the prefix of `source` does not contain a valid instance + /// of `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], + /// you can [infallibly discard the alignment error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &[0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7, 8][..]; + /// + /// let (packet, suffix) = Packet::try_ref_from_prefix_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(suffix, &[8u8][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_ref_from_prefix_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_ref_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let src = 0xCAFEu16.as_bytes(); + /// let (zsty, _) = ZSTy::try_ref_from_prefix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_ref_from_prefix`]: TryFromBytes::try_ref_from_prefix + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_prefix_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_prefix_with_elems( + source: &[u8], + count: usize, + ) -> Result<(&Self, &[u8]), TryCastError<&[u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + Immutable, + { + try_ref_from_prefix_suffix(source, CastType::Prefix, Some(count)) + } + + /// Attempts to interpret the suffix of the given `source` as a `&Self` with + /// a DST length equal to `count`. + /// + /// This method attempts to return a reference to the suffix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the preceding bytes. If the length of `source` is less than the size + /// of `Self` with `count` elements, if the suffix of `source` is not + /// appropriately aligned, or if the suffix of `source` does not contain a + /// valid instance of `Self`, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &[123, 0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let (prefix, packet) = Packet::try_ref_from_suffix_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(prefix, &[123u8][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_ref_from_suffix_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_ref_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let src = 0xCAFEu16.as_bytes(); + /// let (_, zsty) = ZSTy::try_ref_from_suffix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_ref_from_prefix`]: TryFromBytes::try_ref_from_prefix + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_ref_from_suffix_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_ref_from_suffix_with_elems( + source: &[u8], + count: usize, + ) -> Result<(&[u8], &Self), TryCastError<&[u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + Immutable, + { + try_ref_from_prefix_suffix(source, CastType::Suffix, Some(count)).map(swap) + } + + /// Attempts to interpret the given `source` as a `&mut Self` with a DST + /// length equal to `count`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self` with `count` trailing elements. If the length of `source` is not + /// equal to the size of `Self` with `count` elements, if `source` is not + /// appropriately aligned, or if `source` does not contain a valid instance + /// of `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], + /// you can [infallibly discard the alignment error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &mut [0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let packet = Packet::try_mut_from_bytes_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// + /// packet.temperature = 111; + /// + /// assert_eq!(bytes, [0xC0, 0xC0, 240, 111, 2, 3, 4, 5, 6, 7]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_mut_from_bytes_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_mut_from_bytes`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let mut src = 0xCAFEu16; + /// let src = src.as_mut_bytes(); + /// let zsty = ZSTy::try_mut_from_bytes_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_mut_from_bytes`]: TryFromBytes::try_mut_from_bytes + /// + #[doc = codegen_header!("h5", "try_mut_from_bytes_with_elems")] + /// + /// See [`TryFromBytes::try_ref_from_bytes_with_elems`](#method.try_ref_from_bytes_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_bytes_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<&mut Self, TryCastError<&mut [u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + IntoBytes, + { + match Ptr::from_mut(source).try_cast_into_no_leftover::<Self, BecauseExclusive>(Some(count)) + { + Ok(source) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match source.try_into_valid() { + Ok(source) => Ok(source.as_mut()), + Err(e) => Err(e.map_src(|src| src.as_bytes().as_mut()).into()), + } + } + Err(e) => Err(e.map_src(Ptr::as_mut).into()), + } + } + + /// Attempts to interpret the prefix of the given `source` as a `&mut Self` + /// with a DST length equal to `count`. + /// + /// This method attempts to return a reference to the prefix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the remaining bytes. If the length of `source` is less than the size + /// of `Self` with `count` elements, if `source` is not appropriately + /// aligned, or if the prefix of `source` does not contain a valid instance + /// of `Self`, this returns `Err`. If [`Self: Unaligned`][self-unaligned], + /// you can [infallibly discard the alignment error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &mut [0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7, 8][..]; + /// + /// let (packet, suffix) = Packet::try_mut_from_prefix_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(suffix, &[8u8][..]); + /// + /// packet.temperature = 111; + /// suffix[0] = 222; + /// + /// assert_eq!(bytes, [0xC0, 0xC0, 240, 111, 2, 3, 4, 5, 6, 7, 222]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_mut_from_prefix_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_mut_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let mut src = 0xCAFEu16; + /// let src = src.as_mut_bytes(); + /// let (zsty, _) = ZSTy::try_mut_from_prefix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_mut_from_prefix`]: TryFromBytes::try_mut_from_prefix + /// + #[doc = codegen_header!("h5", "try_mut_from_prefix_with_elems")] + /// + /// See [`TryFromBytes::try_ref_from_prefix_with_elems`](#method.try_ref_from_prefix_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_prefix_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<(&mut Self, &mut [u8]), TryCastError<&mut [u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + IntoBytes, + { + try_mut_from_prefix_suffix(source, CastType::Prefix, Some(count)) + } + + /// Attempts to interpret the suffix of the given `source` as a `&mut Self` + /// with a DST length equal to `count`. + /// + /// This method attempts to return a reference to the suffix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the preceding bytes. If the length of `source` is less than the size + /// of `Self` with `count` elements, if the suffix of `source` is not + /// appropriately aligned, or if the suffix of `source` does not contain a + /// valid instance of `Self`, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][ConvertError::from]. + /// + /// [self-unaligned]: Unaligned + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// marshmallows: [[u8; 2]], + /// } + /// + /// let bytes = &mut [123, 0xC0, 0xC0, 240, 77, 2, 3, 4, 5, 6, 7][..]; + /// + /// let (prefix, packet) = Packet::try_mut_from_suffix_with_elems(bytes, 3).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(packet.marshmallows, [[2, 3], [4, 5], [6, 7]]); + /// assert_eq!(prefix, &[123u8][..]); + /// + /// prefix[0] = 111; + /// packet.temperature = 222; + /// + /// assert_eq!(bytes, [111, 0xC0, 0xC0, 240, 222, 2, 3, 4, 5, 6, 7]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 77, 240, 0xC0, 0xC0][..]; + /// assert!(Packet::try_mut_from_suffix_with_elems(bytes, 3).is_err()); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`try_mut_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use core::num::NonZeroU16; + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(TryFromBytes, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: NonZeroU16, + /// trailing_dst: [()], + /// } + /// + /// let mut src = 0xCAFEu16; + /// let src = src.as_mut_bytes(); + /// let (_, zsty) = ZSTy::try_mut_from_suffix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`try_mut_from_prefix`]: TryFromBytes::try_mut_from_prefix + /// + #[doc = codegen_header!("h5", "try_mut_from_suffix_with_elems")] + /// + /// See [`TryFromBytes::try_ref_from_suffix_with_elems`](#method.try_ref_from_suffix_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_mut_from_suffix_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<(&mut [u8], &mut Self), TryCastError<&mut [u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + IntoBytes, + { + try_mut_from_prefix_suffix(source, CastType::Suffix, Some(count)).map(swap) + } + + /// Attempts to read the given `source` as a `Self`. + /// + /// If `source.len() != size_of::<Self>()` or the bytes are not a valid + /// instance of `Self`, this returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// } + /// + /// let bytes = &[0xC0, 0xC0, 240, 77][..]; + /// + /// let packet = Packet::try_read_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &mut [0x10, 0xC0, 240, 77][..]; + /// assert!(Packet::try_read_from_bytes(bytes).is_err()); + /// ``` + /// + /// # Performance Considerations + /// + /// In this version of zerocopy, this method reads the `source` into a + /// well-aligned stack allocation and *then* validates that the allocation + /// is a valid `Self`. This ensures that validation can be performed using + /// aligned reads (which carry a performance advantage over unaligned reads + /// on many platforms) at the cost of an unconditional copy. + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_read_from_bytes", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_read_from_bytes(source: &[u8]) -> Result<Self, TryReadError<&[u8], Self>> + where + Self: Sized, + { + // FIXME(#2981): If `align_of::<Self>() == 1`, validate `source` in-place. + + let candidate = match CoreMaybeUninit::<Self>::read_from_bytes(source) { + Ok(candidate) => candidate, + Err(e) => { + return Err(TryReadError::Size(e.with_dst())); + } + }; + // SAFETY: `candidate` was copied from from `source: &[u8]`, so all of + // its bytes are initialized. + unsafe { try_read_from(source, candidate) } + } + + /// Attempts to read a `Self` from the prefix of the given `source`. + /// + /// This attempts to read a `Self` from the first `size_of::<Self>()` bytes + /// of `source`, returning that `Self` and any remaining bytes. If + /// `source.len() < size_of::<Self>()` or the bytes are not a valid instance + /// of `Self`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &[0xC0, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// + /// let (packet, suffix) = Packet::try_read_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(suffix, &[0u8, 1, 2, 3, 4, 5, 6][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0x10, 0xC0, 240, 77, 0, 1, 2, 3, 4, 5, 6][..]; + /// assert!(Packet::try_read_from_prefix(bytes).is_err()); + /// ``` + /// + /// # Performance Considerations + /// + /// In this version of zerocopy, this method reads the `source` into a + /// well-aligned stack allocation and *then* validates that the allocation + /// is a valid `Self`. This ensures that validation can be performed using + /// aligned reads (which carry a performance advantage over unaligned reads + /// on many platforms) at the cost of an unconditional copy. + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_read_from_prefix", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_read_from_prefix(source: &[u8]) -> Result<(Self, &[u8]), TryReadError<&[u8], Self>> + where + Self: Sized, + { + // FIXME(#2981): If `align_of::<Self>() == 1`, validate `source` in-place. + + let (candidate, suffix) = match CoreMaybeUninit::<Self>::read_from_prefix(source) { + Ok(candidate) => candidate, + Err(e) => { + return Err(TryReadError::Size(e.with_dst())); + } + }; + // SAFETY: `candidate` was copied from from `source: &[u8]`, so all of + // its bytes are initialized. + unsafe { try_read_from(source, candidate).map(|slf| (slf, suffix)) } + } + + /// Attempts to read a `Self` from the suffix of the given `source`. + /// + /// This attempts to read a `Self` from the last `size_of::<Self>()` bytes + /// of `source`, returning that `Self` and any preceding bytes. If + /// `source.len() < size_of::<Self>()` or the bytes are not a valid instance + /// of `Self`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// # #![allow(non_camel_case_types)] // For C0::xC0 + /// use zerocopy::TryFromBytes; + /// # use zerocopy_derive::*; + /// + /// // The only valid value of this type is the byte `0xC0` + /// #[derive(TryFromBytes)] + /// #[repr(u8)] + /// enum C0 { xC0 = 0xC0 } + /// + /// // The only valid value of this type is the bytes `0xC0C0`. + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct C0C0(C0, C0); + /// + /// #[derive(TryFromBytes)] + /// #[repr(C)] + /// struct Packet { + /// magic_number: C0C0, + /// mug_size: u8, + /// temperature: u8, + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 0xC0, 0xC0, 240, 77][..]; + /// + /// let (prefix, packet) = Packet::try_read_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(packet.mug_size, 240); + /// assert_eq!(packet.temperature, 77); + /// assert_eq!(prefix, &[0u8, 1, 2, 3, 4, 5][..]); + /// + /// // These bytes are not valid instance of `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 0x10, 0xC0, 240, 77][..]; + /// assert!(Packet::try_read_from_suffix(bytes).is_err()); + /// ``` + /// + /// # Performance Considerations + /// + /// In this version of zerocopy, this method reads the `source` into a + /// well-aligned stack allocation and *then* validates that the allocation + /// is a valid `Self`. This ensures that validation can be performed using + /// aligned reads (which carry a performance advantage over unaligned reads + /// on many platforms) at the cost of an unconditional copy. + /// + #[doc = codegen_section!( + header = "h5", + bench = "try_read_from_suffix", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn try_read_from_suffix(source: &[u8]) -> Result<(&[u8], Self), TryReadError<&[u8], Self>> + where + Self: Sized, + { + // FIXME(#2981): If `align_of::<Self>() == 1`, validate `source` in-place. + + let (prefix, candidate) = match CoreMaybeUninit::<Self>::read_from_suffix(source) { + Ok(candidate) => candidate, + Err(e) => { + return Err(TryReadError::Size(e.with_dst())); + } + }; + // SAFETY: `candidate` was copied from from `source: &[u8]`, so all of + // its bytes are initialized. + unsafe { try_read_from(source, candidate).map(|slf| (prefix, slf)) } + } +} + +#[inline(always)] +fn try_ref_from_prefix_suffix<T: TryFromBytes + KnownLayout + Immutable + ?Sized>( + source: &[u8], + cast_type: CastType, + meta: Option<T::PointerMetadata>, +) -> Result<(&T, &[u8]), TryCastError<&[u8], T>> { + match Ptr::from_ref(source).try_cast_into::<T, BecauseImmutable>(cast_type, meta) { + Ok((source, prefix_suffix)) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match source.try_into_valid() { + Ok(valid) => Ok((valid.as_ref(), prefix_suffix.as_ref())), + Err(e) => Err(e.map_src(|src| src.as_bytes::<BecauseImmutable>().as_ref()).into()), + } + } + Err(e) => Err(e.map_src(Ptr::as_ref).into()), + } +} + +#[inline(always)] +fn try_mut_from_prefix_suffix<T: IntoBytes + TryFromBytes + KnownLayout + ?Sized>( + candidate: &mut [u8], + cast_type: CastType, + meta: Option<T::PointerMetadata>, +) -> Result<(&mut T, &mut [u8]), TryCastError<&mut [u8], T>> { + match Ptr::from_mut(candidate).try_cast_into::<T, BecauseExclusive>(cast_type, meta) { + Ok((candidate, prefix_suffix)) => { + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to + // fix before returning. + match candidate.try_into_valid() { + Ok(valid) => Ok((valid.as_mut(), prefix_suffix.as_mut())), + Err(e) => Err(e.map_src(|src| src.as_bytes().as_mut()).into()), + } + } + Err(e) => Err(e.map_src(Ptr::as_mut).into()), + } +} + +#[inline(always)] +fn swap<T, U>((t, u): (T, U)) -> (U, T) { + (u, t) +} + +/// # Safety +/// +/// All bytes of `candidate` must be initialized. +#[inline(always)] +unsafe fn try_read_from<S, T: TryFromBytes>( + source: S, + mut candidate: CoreMaybeUninit<T>, +) -> Result<T, TryReadError<S, T>> { + // We use `from_mut` despite not mutating via `c_ptr` so that we don't need + // to add a `T: Immutable` bound. + let c_ptr = Ptr::from_mut(&mut candidate); + // SAFETY: `c_ptr` has no uninitialized sub-ranges because it derived from + // `candidate`, which the caller promises is entirely initialized. Since + // `candidate` is a `MaybeUninit`, it has no validity requirements, and so + // no values written to an `Initialized` `c_ptr` can violate its validity. + // Since `c_ptr` has `Exclusive` aliasing, no mutations may happen except + // via `c_ptr` so long as it is live, so we don't need to worry about the + // fact that `c_ptr` may have more restricted validity than `candidate`. + let c_ptr = unsafe { c_ptr.assume_validity::<invariant::Initialized>() }; + let mut c_ptr = c_ptr.cast::<_, crate::pointer::cast::CastSized, _>(); + + // Since we don't have `T: KnownLayout`, we hack around that by using + // `Wrapping<T>`, which implements `KnownLayout` even if `T` doesn't. + // + // This call may panic. If that happens, it doesn't cause any soundness + // issues, as we have not generated any invalid state which we need to fix + // before returning. + if !Wrapping::<T>::is_bit_valid(c_ptr.reborrow_shared().forget_aligned()) { + return Err(ValidityError::new(source).into()); + } + + fn _assert_same_size_and_validity<T>() + where + Wrapping<T>: pointer::TransmuteFrom<T, invariant::Valid, invariant::Valid>, + T: pointer::TransmuteFrom<Wrapping<T>, invariant::Valid, invariant::Valid>, + { + } + + _assert_same_size_and_validity::<T>(); + + // SAFETY: We just validated that `candidate` contains a valid + // `Wrapping<T>`, which has the same size and bit validity as `T`, as + // guaranteed by the preceding type assertion. + Ok(unsafe { candidate.assume_init() }) +} + +/// Types for which a sequence of `0` bytes is a valid instance. +/// +/// Any memory region of the appropriate length which is guaranteed to contain +/// only zero bytes can be viewed as any `FromZeros` type with no runtime +/// overhead. This is useful whenever memory is known to be in a zeroed state, +/// such memory returned from some allocation routines. +/// +/// # Warning: Padding bytes +/// +/// Note that, when a value is moved or copied, only the non-padding bytes of +/// that value are guaranteed to be preserved. It is unsound to assume that +/// values written to padding bytes are preserved after a move or copy. For more +/// details, see the [`FromBytes` docs][frombytes-warning-padding-bytes]. +/// +/// [frombytes-warning-padding-bytes]: FromBytes#warning-padding-bytes +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(FromZeros)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{FromZeros, Immutable}; +/// #[derive(FromZeros)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromZeros)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromZeros, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `FromZeros`. +/// +/// # Safety +/// +/// *This section describes what is required in order for `T: FromZeros`, and +/// what unsafe code may assume of such types. If you don't plan on implementing +/// `FromZeros` manually, and you don't plan on writing unsafe code that +/// operates on `FromZeros` types, then you don't need to read this section.* +/// +/// If `T: FromZeros`, then unsafe code may assume that it is sound to produce a +/// `T` whose bytes are all initialized to zero. If a type is marked as +/// `FromZeros` which violates this contract, it may cause undefined behavior. +/// +/// `#[derive(FromZeros)]` only permits [types which satisfy these +/// requirements][derive-analysis]. +/// +#[cfg_attr( + feature = "derive", + doc = "[derive]: zerocopy_derive::FromZeros", + doc = "[derive-analysis]: zerocopy_derive::FromZeros#analysis" +)] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.FromZeros.html"), + doc = concat!("[derive-analysis]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.FromZeros.html#analysis"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(FromZeros)]` to `{Self}`") +)] +pub unsafe trait FromZeros: TryFromBytes { + // The `Self: Sized` bound makes it so that `FromZeros` is still object + // safe. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// Overwrites `self` with zeros. + /// + /// Sets every byte in `self` to 0. While this is similar to doing `*self = + /// Self::new_zeroed()`, it differs in that `zero` does not semantically + /// drop the current value and replace it with a new one — it simply + /// modifies the bytes of the existing value. + /// + /// # Examples + /// + /// ``` + /// # use zerocopy::FromZeros; + /// # use zerocopy_derive::*; + /// # + /// #[derive(FromZeros)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let mut header = PacketHeader { + /// src_port: 100u16.to_be_bytes(), + /// dst_port: 200u16.to_be_bytes(), + /// length: 300u16.to_be_bytes(), + /// checksum: 400u16.to_be_bytes(), + /// }; + /// + /// header.zero(); + /// + /// assert_eq!(header.src_port, [0, 0]); + /// assert_eq!(header.dst_port, [0, 0]); + /// assert_eq!(header.length, [0, 0]); + /// assert_eq!(header.checksum, [0, 0]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "zero", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[inline(always)] + fn zero(&mut self) { + let slf: *mut Self = self; + let len = mem::size_of_val(self); + // SAFETY: + // - `self` is guaranteed by the type system to be valid for writes of + // size `size_of_val(self)`. + // - `u8`'s alignment is 1, and thus `self` is guaranteed to be aligned + // as required by `u8`. + // - Since `Self: FromZeros`, the all-zeros instance is a valid instance + // of `Self.` + // + // FIXME(#429): Add references to docs and quotes. + unsafe { ptr::write_bytes(slf.cast::<u8>(), 0, len) }; + } + + /// Creates an instance of `Self` from zeroed bytes. + /// + /// # Examples + /// + /// ``` + /// # use zerocopy::FromZeros; + /// # use zerocopy_derive::*; + /// # + /// #[derive(FromZeros)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let header: PacketHeader = FromZeros::new_zeroed(); + /// + /// assert_eq!(header.src_port, [0, 0]); + /// assert_eq!(header.dst_port, [0, 0]); + /// assert_eq!(header.length, [0, 0]); + /// assert_eq!(header.checksum, [0, 0]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "new_zeroed", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[inline(always)] + fn new_zeroed() -> Self + where + Self: Sized, + { + // SAFETY: `FromZeros` says that the all-zeros bit pattern is legal. + unsafe { mem::zeroed() } + } + + /// Creates a `Box<Self>` from zeroed bytes. + /// + /// This function is useful for allocating large values on the heap and + /// zero-initializing them, without ever creating a temporary instance of + /// `Self` on the stack. For example, `<[u8; 1048576]>::new_box_zeroed()` + /// will allocate `[u8; 1048576]` directly on the heap; it does not require + /// storing `[u8; 1048576]` in a temporary variable on the stack. + /// + /// On systems that use a heap implementation that supports allocating from + /// pre-zeroed memory, using `new_box_zeroed` (or related functions) may + /// have performance benefits. + /// + /// # Errors + /// + /// Returns an error on allocation failure. Allocation failure is guaranteed + /// never to cause a panic or an abort. + /// + #[doc = codegen_section!( + header = "h5", + bench = "new_box_zeroed", + format = "coco_static_size", + )] + #[must_use = "has no side effects (other than allocation)"] + #[cfg(any(feature = "alloc", test))] + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + #[inline] + fn new_box_zeroed() -> Result<Box<Self>, AllocError> + where + Self: Sized, + { + // If `T` is a ZST, then return a proper boxed instance of it. There is + // no allocation, but `Box` does require a correct dangling pointer. + let layout = Layout::new::<Self>(); + if layout.size() == 0 { + // Construct the `Box` from a dangling pointer to avoid calling + // `Self::new_zeroed`. This ensures that stack space is never + // allocated for `Self` even on lower opt-levels where this branch + // might not get optimized out. + + // SAFETY: Per [1], when `T` is a ZST, `Box<T>`'s only validity + // requirements are that the pointer is non-null and sufficiently + // aligned. Per [2], `NonNull::dangling` produces a pointer which + // is sufficiently aligned. Since the produced pointer is a + // `NonNull`, it is non-null. + // + // [1] Per https://doc.rust-lang.org/1.81.0/std/boxed/index.html#memory-layout: + // + // For zero-sized values, the `Box` pointer has to be non-null and sufficiently aligned. + // + // [2] Per https://doc.rust-lang.org/std/ptr/struct.NonNull.html#method.dangling: + // + // Creates a new `NonNull` that is dangling, but well-aligned. + return Ok(unsafe { Box::from_raw(NonNull::dangling().as_ptr()) }); + } + + // FIXME(#429): Add a "SAFETY" comment and remove this `allow`. + #[allow(clippy::undocumented_unsafe_blocks)] + let ptr = unsafe { alloc::alloc::alloc_zeroed(layout).cast::<Self>() }; + if ptr.is_null() { + return Err(AllocError); + } + // FIXME(#429): Add a "SAFETY" comment and remove this `allow`. + #[allow(clippy::undocumented_unsafe_blocks)] + Ok(unsafe { Box::from_raw(ptr) }) + } + + /// Creates a `Box<[Self]>` (a boxed slice) from zeroed bytes. + /// + /// This function is useful for allocating large values of `[Self]` on the + /// heap and zero-initializing them, without ever creating a temporary + /// instance of `[Self; _]` on the stack. For example, + /// `u8::new_box_slice_zeroed(1048576)` will allocate the slice directly on + /// the heap; it does not require storing the slice on the stack. + /// + /// On systems that use a heap implementation that supports allocating from + /// pre-zeroed memory, using `new_box_slice_zeroed` may have performance + /// benefits. + /// + /// If `Self` is a zero-sized type, then this function will return a + /// `Box<[Self]>` that has the correct `len`. Such a box cannot contain any + /// actual information, but its `len()` property will report the correct + /// value. + /// + /// # Errors + /// + /// Returns an error on allocation failure. Allocation failure is + /// guaranteed never to cause a panic or an abort. + /// + #[doc = codegen_section!( + header = "h5", + bench = "new_box_zeroed_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects (other than allocation)"] + #[cfg(feature = "alloc")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + #[inline] + fn new_box_zeroed_with_elems(count: usize) -> Result<Box<Self>, AllocError> + where + Self: KnownLayout<PointerMetadata = usize>, + { + // SAFETY: `alloc::alloc::alloc_zeroed` is a valid argument of + // `new_box`. The referent of the pointer returned by `alloc_zeroed` + // (and, consequently, the `Box` derived from it) is a valid instance of + // `Self`, because `Self` is `FromZeros`. + unsafe { crate::util::new_box(count, alloc::alloc::alloc_zeroed) } + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromZeros::new_box_zeroed_with_elems`")] + #[doc(hidden)] + #[cfg(feature = "alloc")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + #[must_use = "has no side effects (other than allocation)"] + #[inline(always)] + fn new_box_slice_zeroed(len: usize) -> Result<Box<[Self]>, AllocError> + where + Self: Sized, + { + <[Self]>::new_box_zeroed_with_elems(len) + } + + /// Creates a `Vec<Self>` from zeroed bytes. + /// + /// This function is useful for allocating large values of `Vec`s and + /// zero-initializing them, without ever creating a temporary instance of + /// `[Self; _]` (or many temporary instances of `Self`) on the stack. For + /// example, `u8::new_vec_zeroed(1048576)` will allocate directly on the + /// heap; it does not require storing intermediate values on the stack. + /// + /// On systems that use a heap implementation that supports allocating from + /// pre-zeroed memory, using `new_vec_zeroed` may have performance benefits. + /// + /// If `Self` is a zero-sized type, then this function will return a + /// `Vec<Self>` that has the correct `len`. Such a `Vec` cannot contain any + /// actual information, but its `len()` property will report the correct + /// value. + /// + /// # Errors + /// + /// Returns an error on allocation failure. Allocation failure is + /// guaranteed never to cause a panic or an abort. + /// + #[doc = codegen_section!( + header = "h5", + bench = "new_vec_zeroed", + format = "coco_static_size", + )] + #[must_use = "has no side effects (other than allocation)"] + #[cfg(feature = "alloc")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] + #[inline(always)] + fn new_vec_zeroed(len: usize) -> Result<Vec<Self>, AllocError> + where + Self: Sized, + { + <[Self]>::new_box_zeroed_with_elems(len).map(Into::into) + } + + /// Extends a `Vec<Self>` by pushing `additional` new items onto the end of + /// the vector. The new items are initialized with zeros. + /// + #[doc = codegen_section!( + header = "h5", + bench = "extend_vec_zeroed", + format = "coco_static_size", + )] + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[cfg(feature = "alloc")] + #[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.57.0", feature = "alloc"))))] + #[inline(always)] + fn extend_vec_zeroed(v: &mut Vec<Self>, additional: usize) -> Result<(), AllocError> + where + Self: Sized, + { + // PANICS: We pass `v.len()` for `position`, so the `position > v.len()` + // panic condition is not satisfied. + <Self as FromZeros>::insert_vec_zeroed(v, v.len(), additional) + } + + /// Inserts `additional` new items into `Vec<Self>` at `position`. The new + /// items are initialized with zeros. + /// + /// # Panics + /// + /// Panics if `position > v.len()`. + /// + #[doc = codegen_section!( + header = "h5", + bench = "insert_vec_zeroed", + format = "coco_static_size", + )] + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[cfg(feature = "alloc")] + #[cfg_attr(doc_cfg, doc(cfg(all(rust = "1.57.0", feature = "alloc"))))] + #[inline] + fn insert_vec_zeroed( + v: &mut Vec<Self>, + position: usize, + additional: usize, + ) -> Result<(), AllocError> + where + Self: Sized, + { + assert!(position <= v.len()); + // We only conditionally compile on versions on which `try_reserve` is + // stable; the Clippy lint is a false positive. + v.try_reserve(additional).map_err(|_| AllocError)?; + // SAFETY: The `try_reserve` call guarantees that these cannot overflow: + // * `ptr.add(position)` + // * `position + additional` + // * `v.len() + additional` + // + // `v.len() - position` cannot overflow because we asserted that + // `position <= v.len()`. + #[allow(clippy::multiple_unsafe_ops_per_block)] + unsafe { + // This is a potentially overlapping copy. + let ptr = v.as_mut_ptr(); + #[allow(clippy::arithmetic_side_effects)] + ptr.add(position).copy_to(ptr.add(position + additional), v.len() - position); + ptr.add(position).write_bytes(0, additional); + #[allow(clippy::arithmetic_side_effects)] + v.set_len(v.len() + additional); + } + + Ok(()) + } +} + +/// Analyzes whether a type is [`FromBytes`]. +/// +/// This derive analyzes, at compile time, whether the annotated type satisfies +/// the [safety conditions] of `FromBytes` and implements `FromBytes` and its +/// supertraits if it is sound to do so. This derive can be applied to structs, +/// enums, and unions; +/// e.g.: +/// +/// ``` +/// # use zerocopy_derive::{FromBytes, FromZeros, Immutable}; +/// #[derive(FromBytes)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # V00, V01, V02, V03, V04, V05, V06, V07, V08, V09, V0A, V0B, V0C, V0D, V0E, +/// # V0F, V10, V11, V12, V13, V14, V15, V16, V17, V18, V19, V1A, V1B, V1C, V1D, +/// # V1E, V1F, V20, V21, V22, V23, V24, V25, V26, V27, V28, V29, V2A, V2B, V2C, +/// # V2D, V2E, V2F, V30, V31, V32, V33, V34, V35, V36, V37, V38, V39, V3A, V3B, +/// # V3C, V3D, V3E, V3F, V40, V41, V42, V43, V44, V45, V46, V47, V48, V49, V4A, +/// # V4B, V4C, V4D, V4E, V4F, V50, V51, V52, V53, V54, V55, V56, V57, V58, V59, +/// # V5A, V5B, V5C, V5D, V5E, V5F, V60, V61, V62, V63, V64, V65, V66, V67, V68, +/// # V69, V6A, V6B, V6C, V6D, V6E, V6F, V70, V71, V72, V73, V74, V75, V76, V77, +/// # V78, V79, V7A, V7B, V7C, V7D, V7E, V7F, V80, V81, V82, V83, V84, V85, V86, +/// # V87, V88, V89, V8A, V8B, V8C, V8D, V8E, V8F, V90, V91, V92, V93, V94, V95, +/// # V96, V97, V98, V99, V9A, V9B, V9C, V9D, V9E, V9F, VA0, VA1, VA2, VA3, VA4, +/// # VA5, VA6, VA7, VA8, VA9, VAA, VAB, VAC, VAD, VAE, VAF, VB0, VB1, VB2, VB3, +/// # VB4, VB5, VB6, VB7, VB8, VB9, VBA, VBB, VBC, VBD, VBE, VBF, VC0, VC1, VC2, +/// # VC3, VC4, VC5, VC6, VC7, VC8, VC9, VCA, VCB, VCC, VCD, VCE, VCF, VD0, VD1, +/// # VD2, VD3, VD4, VD5, VD6, VD7, VD8, VD9, VDA, VDB, VDC, VDD, VDE, VDF, VE0, +/// # VE1, VE2, VE3, VE4, VE5, VE6, VE7, VE8, VE9, VEA, VEB, VEC, VED, VEE, VEF, +/// # VF0, VF1, VF2, VF3, VF4, VF5, VF6, VF7, VF8, VF9, VFA, VFB, VFC, VFD, VFE, +/// # VFF, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromBytes, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// [safety conditions]: trait@FromBytes#safety +/// +/// # Analysis +/// +/// *This section describes, roughly, the analysis performed by this derive to +/// determine whether it is sound to implement `FromBytes` for a given type. +/// Unless you are modifying the implementation of this derive, or attempting to +/// manually implement `FromBytes` for a type yourself, you don't need to read +/// this section.* +/// +/// If a type has the following properties, then this derive can implement +/// `FromBytes` for that type: +/// +/// - If the type is a struct, all of its fields must be `FromBytes`. +/// - If the type is an enum: +/// - It must have a defined representation which is one of `u8`, `u16`, `i8`, +/// or `i16`. +/// - The maximum number of discriminants must be used (so that every possible +/// bit pattern is a valid one). +/// - Its fields must be `FromBytes`. +/// +/// This analysis is subject to change. Unsafe code may *only* rely on the +/// documented [safety conditions] of `FromBytes`, and must *not* rely on the +/// implementation details of this derive. +/// +/// ## Why isn't an explicit representation required for structs? +/// +/// Neither this derive, nor the [safety conditions] of `FromBytes`, requires +/// that structs are marked with `#[repr(C)]`. +/// +/// Per the [Rust reference](reference), +/// +/// > The representation of a type can change the padding between fields, but +/// > does not change the layout of the fields themselves. +/// +/// [reference]: https://doc.rust-lang.org/reference/type-layout.html#representations +/// +/// Since the layout of structs only consists of padding bytes and field bytes, +/// a struct is soundly `FromBytes` if: +/// 1. its padding is soundly `FromBytes`, and +/// 2. its fields are soundly `FromBytes`. +/// +/// The answer to the first question is always yes: padding bytes do not have +/// any validity constraints. A [discussion] of this question in the Unsafe Code +/// Guidelines Working Group concluded that it would be virtually unimaginable +/// for future versions of rustc to add validity constraints to padding bytes. +/// +/// [discussion]: https://github.com/rust-lang/unsafe-code-guidelines/issues/174 +/// +/// Whether a struct is soundly `FromBytes` therefore solely depends on whether +/// its fields are `FromBytes`. +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::FromBytes; + +/// Types for which any bit pattern is valid. +/// +/// Any memory region of the appropriate length which contains initialized bytes +/// can be viewed as any `FromBytes` type with no runtime overhead. This is +/// useful for efficiently parsing bytes as structured data. +/// +/// # Warning: Padding bytes +/// +/// Note that, when a value is moved or copied, only the non-padding bytes of +/// that value are guaranteed to be preserved. It is unsound to assume that +/// values written to padding bytes are preserved after a move or copy. For +/// example, the following is unsound: +/// +/// ```rust,no_run +/// use core::mem::{size_of, transmute}; +/// use zerocopy::FromZeros; +/// # use zerocopy_derive::*; +/// +/// // Assume `Foo` is a type with padding bytes. +/// #[derive(FromZeros, Default)] +/// struct Foo { +/// # /* +/// ... +/// # */ +/// } +/// +/// let mut foo: Foo = Foo::default(); +/// FromZeros::zero(&mut foo); +/// // UNSOUND: Although `FromZeros::zero` writes zeros to all bytes of `foo`, +/// // those writes are not guaranteed to be preserved in padding bytes when +/// // `foo` is moved, so this may expose padding bytes as `u8`s. +/// let foo_bytes: [u8; size_of::<Foo>()] = unsafe { transmute(foo) }; +/// ``` +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(FromBytes)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{FromBytes, Immutable}; +/// #[derive(FromBytes)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # V00, V01, V02, V03, V04, V05, V06, V07, V08, V09, V0A, V0B, V0C, V0D, V0E, +/// # V0F, V10, V11, V12, V13, V14, V15, V16, V17, V18, V19, V1A, V1B, V1C, V1D, +/// # V1E, V1F, V20, V21, V22, V23, V24, V25, V26, V27, V28, V29, V2A, V2B, V2C, +/// # V2D, V2E, V2F, V30, V31, V32, V33, V34, V35, V36, V37, V38, V39, V3A, V3B, +/// # V3C, V3D, V3E, V3F, V40, V41, V42, V43, V44, V45, V46, V47, V48, V49, V4A, +/// # V4B, V4C, V4D, V4E, V4F, V50, V51, V52, V53, V54, V55, V56, V57, V58, V59, +/// # V5A, V5B, V5C, V5D, V5E, V5F, V60, V61, V62, V63, V64, V65, V66, V67, V68, +/// # V69, V6A, V6B, V6C, V6D, V6E, V6F, V70, V71, V72, V73, V74, V75, V76, V77, +/// # V78, V79, V7A, V7B, V7C, V7D, V7E, V7F, V80, V81, V82, V83, V84, V85, V86, +/// # V87, V88, V89, V8A, V8B, V8C, V8D, V8E, V8F, V90, V91, V92, V93, V94, V95, +/// # V96, V97, V98, V99, V9A, V9B, V9C, V9D, V9E, V9F, VA0, VA1, VA2, VA3, VA4, +/// # VA5, VA6, VA7, VA8, VA9, VAA, VAB, VAC, VAD, VAE, VAF, VB0, VB1, VB2, VB3, +/// # VB4, VB5, VB6, VB7, VB8, VB9, VBA, VBB, VBC, VBD, VBE, VBF, VC0, VC1, VC2, +/// # VC3, VC4, VC5, VC6, VC7, VC8, VC9, VCA, VCB, VCC, VCD, VCE, VCF, VD0, VD1, +/// # VD2, VD3, VD4, VD5, VD6, VD7, VD8, VD9, VDA, VDB, VDC, VDD, VDE, VDF, VE0, +/// # VE1, VE2, VE3, VE4, VE5, VE6, VE7, VE8, VE9, VEA, VEB, VEC, VED, VEE, VEF, +/// # VF0, VF1, VF2, VF3, VF4, VF5, VF6, VF7, VF8, VF9, VFA, VFB, VFC, VFD, VFE, +/// # VFF, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(FromBytes, Immutable)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `FromBytes`. +/// +/// # Safety +/// +/// *This section describes what is required in order for `T: FromBytes`, and +/// what unsafe code may assume of such types. If you don't plan on implementing +/// `FromBytes` manually, and you don't plan on writing unsafe code that +/// operates on `FromBytes` types, then you don't need to read this section.* +/// +/// If `T: FromBytes`, then unsafe code may assume that it is sound to produce a +/// `T` whose bytes are initialized to any sequence of valid `u8`s (in other +/// words, any byte value which is not uninitialized). If a type is marked as +/// `FromBytes` which violates this contract, it may cause undefined behavior. +/// +/// `#[derive(FromBytes)]` only permits [types which satisfy these +/// requirements][derive-analysis]. +/// +#[cfg_attr( + feature = "derive", + doc = "[derive]: zerocopy_derive::FromBytes", + doc = "[derive-analysis]: zerocopy_derive::FromBytes#analysis" +)] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.FromBytes.html"), + doc = concat!("[derive-analysis]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.FromBytes.html#analysis"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(FromBytes)]` to `{Self}`") +)] +pub unsafe trait FromBytes: FromZeros { + // The `Self: Sized` bound makes it so that `FromBytes` is still object + // safe. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// Interprets the given `source` as a `&Self`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self`. If the length of `source` is not a [valid size of + /// `Self`][valid-size], or if `source` is not appropriately aligned, this + /// returns `Err`. If [`Self: Unaligned`][self-unaligned], you can + /// [infallibly discard the alignment error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::ref_from_bytes(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// #[derive(FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// header: PacketHeader, + /// body: [u8], + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11][..]; + /// + /// let packet = Packet::ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.header.src_port, [0, 1]); + /// assert_eq!(packet.header.dst_port, [2, 3]); + /// assert_eq!(packet.header.length, [4, 5]); + /// assert_eq!(packet.header.checksum, [6, 7]); + /// assert_eq!(packet.body, [8, 9, 10, 11]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_bytes", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_bytes(source: &[u8]) -> Result<&Self, CastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + static_assert_dst_is_not_zst!(Self); + match Ptr::from_ref(source).try_cast_into_no_leftover::<_, BecauseImmutable>(None) { + Ok(ptr) => Ok(ptr.recall_validity().as_ref()), + Err(err) => Err(err.map_src(|src| src.as_ref())), + } + } + + /// Interprets the prefix of the given `source` as a `&Self` without + /// copying. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the leading bytes of `source`, then attempts to return + /// both a reference to those bytes interpreted as a `Self`, and a reference + /// to the remaining bytes. If there are insufficient bytes, or if `source` + /// is not appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. See [`ref_from_prefix_with_elems`], which does + /// support such types. Attempting to use this method on such types results + /// in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::ref_from_prefix(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// [`ref_from_prefix_with_elems`]: FromBytes::ref_from_prefix_with_elems + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// #[derive(FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// header: PacketHeader, + /// body: [[u8; 2]], + /// } + /// + /// // These are more bytes than are needed to encode a `Packet`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14][..]; + /// + /// let (packet, suffix) = Packet::ref_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(packet.header.src_port, [0, 1]); + /// assert_eq!(packet.header.dst_port, [2, 3]); + /// assert_eq!(packet.header.length, [4, 5]); + /// assert_eq!(packet.header.checksum, [6, 7]); + /// assert_eq!(packet.body, [[8, 9], [10, 11], [12, 13]]); + /// assert_eq!(suffix, &[14u8][..]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_prefix", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_prefix(source: &[u8]) -> Result<(&Self, &[u8]), CastError<&[u8], Self>> + where + Self: KnownLayout + Immutable, + { + static_assert_dst_is_not_zst!(Self); + ref_from_prefix_suffix(source, None, CastType::Prefix) + } + + /// Interprets the suffix of the given bytes as a `&Self`. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the trailing bytes of `source`, then attempts to return + /// both a reference to those bytes interpreted as a `Self`, and a reference + /// to the preceding bytes. If there are insufficient bytes, or if that + /// suffix of `source` is not appropriately aligned, this returns `Err`. If + /// [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. See [`ref_from_suffix_with_elems`], which does + /// support such types. Attempting to use this method on such types results + /// in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = ZSTy::ref_from_suffix(0u16.as_bytes()); // ⚠ Compile Error! + /// ``` + /// + /// [`ref_from_suffix_with_elems`]: FromBytes::ref_from_suffix_with_elems + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct PacketTrailer { + /// frame_check_sequence: [u8; 4], + /// } + /// + /// // These are more bytes than are needed to encode a `PacketTrailer`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (prefix, trailer) = PacketTrailer::ref_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(prefix, &[0, 1, 2, 3, 4, 5][..]); + /// assert_eq!(trailer.frame_check_sequence, [6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_suffix", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_suffix(source: &[u8]) -> Result<(&[u8], &Self), CastError<&[u8], Self>> + where + Self: Immutable + KnownLayout, + { + static_assert_dst_is_not_zst!(Self); + ref_from_prefix_suffix(source, None, CastType::Suffix).map(swap) + } + + /// Interprets the given `source` as a `&mut Self`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self`. If the length of `source` is not a [valid size of + /// `Self`][valid-size], or if `source` is not appropriately aligned, this + /// returns `Err`. If [`Self: Unaligned`][self-unaligned], you can + /// [infallibly discard the alignment error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. See [`mut_from_prefix_with_elems`], which does + /// support such types. Attempting to use this method on such types results + /// in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::mut_from_bytes(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// [`mut_from_prefix_with_elems`]: FromBytes::mut_from_prefix_with_elems + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// // These bytes encode a `PacketHeader`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7][..]; + /// + /// let header = PacketHeader::mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(header.src_port, [0, 1]); + /// assert_eq!(header.dst_port, [2, 3]); + /// assert_eq!(header.length, [4, 5]); + /// assert_eq!(header.checksum, [6, 7]); + /// + /// header.checksum = [0, 0]; + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 0, 0]); + /// + /// ``` + /// + #[doc = codegen_header!("h5", "mut_from_bytes")] + /// + /// See [`FromBytes::ref_from_bytes`](#method.ref_from_bytes.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_bytes(source: &mut [u8]) -> Result<&mut Self, CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout, + { + static_assert_dst_is_not_zst!(Self); + match Ptr::from_mut(source).try_cast_into_no_leftover::<_, BecauseExclusive>(None) { + Ok(ptr) => Ok(ptr.recall_validity::<_, (_, (_, _))>().as_mut()), + Err(err) => Err(err.map_src(|src| src.as_mut())), + } + } + + /// Interprets the prefix of the given `source` as a `&mut Self` without + /// copying. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the leading bytes of `source`, then attempts to return + /// both a reference to those bytes interpreted as a `Self`, and a reference + /// to the remaining bytes. If there are insufficient bytes, or if `source` + /// is not appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. See [`mut_from_suffix_with_elems`], which does + /// support such types. Attempting to use this method on such types results + /// in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::mut_from_prefix(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// [`mut_from_suffix_with_elems`]: FromBytes::mut_from_suffix_with_elems + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// // These are more bytes than are needed to encode a `PacketHeader`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (header, body) = PacketHeader::mut_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(header.src_port, [0, 1]); + /// assert_eq!(header.dst_port, [2, 3]); + /// assert_eq!(header.length, [4, 5]); + /// assert_eq!(header.checksum, [6, 7]); + /// assert_eq!(body, &[8, 9][..]); + /// + /// header.checksum = [0, 0]; + /// body.fill(1); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 0, 0, 1, 1]); + /// ``` + /// + #[doc = codegen_header!("h5", "mut_from_prefix")] + /// + /// See [`FromBytes::ref_from_prefix`](#method.ref_from_prefix.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_prefix( + source: &mut [u8], + ) -> Result<(&mut Self, &mut [u8]), CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout, + { + static_assert_dst_is_not_zst!(Self); + mut_from_prefix_suffix(source, None, CastType::Prefix) + } + + /// Interprets the suffix of the given `source` as a `&mut Self` without + /// copying. + /// + /// This method computes the [largest possible size of `Self`][valid-size] + /// that can fit in the trailing bytes of `source`, then attempts to return + /// both a reference to those bytes interpreted as a `Self`, and a reference + /// to the preceding bytes. If there are insufficient bytes, or if that + /// suffix of `source` is not appropriately aligned, this returns `Err`. If + /// [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][size-error-from]. + /// + /// `Self` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, IntoBytes, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let mut source = [85, 85]; + /// let _ = ZSTy::mut_from_suffix(&mut source[..]); // ⚠ Compile Error! + /// ``` + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct PacketTrailer { + /// frame_check_sequence: [u8; 4], + /// } + /// + /// // These are more bytes than are needed to encode a `PacketTrailer`. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (prefix, trailer) = PacketTrailer::mut_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(prefix, &[0u8, 1, 2, 3, 4, 5][..]); + /// assert_eq!(trailer.frame_check_sequence, [6, 7, 8, 9]); + /// + /// prefix.fill(0); + /// trailer.frame_check_sequence.fill(1); + /// + /// assert_eq!(bytes, [0, 0, 0, 0, 0, 0, 1, 1, 1, 1]); + /// ``` + /// + #[doc = codegen_header!("h5", "mut_from_suffix")] + /// + /// See [`FromBytes::ref_from_suffix`](#method.ref_from_suffix.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_suffix( + source: &mut [u8], + ) -> Result<(&mut [u8], &mut Self), CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout, + { + static_assert_dst_is_not_zst!(Self); + mut_from_prefix_suffix(source, None, CastType::Suffix).map(swap) + } + + /// Interprets the given `source` as a `&Self` with a DST length equal to + /// `count`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self` with `count` trailing elements. If the length of `source` is not + /// equal to the size of `Self` with `count` elements, or if `source` is not + /// appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(FromBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7][..]; + /// + /// let pixels = <[Pixel]>::ref_from_bytes_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 0, g: 1, b: 2, a: 3 }, + /// Pixel { r: 4, g: 5, b: 6, a: 7 }, + /// ]); + /// + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`ref_from_bytes`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &[85, 85][..]; + /// let zsty = ZSTy::ref_from_bytes_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`ref_from_bytes`]: FromBytes::ref_from_bytes + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_bytes_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_bytes_with_elems( + source: &[u8], + count: usize, + ) -> Result<&Self, CastError<&[u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + Immutable, + { + let source = Ptr::from_ref(source); + let maybe_slf = source.try_cast_into_no_leftover::<_, BecauseImmutable>(Some(count)); + match maybe_slf { + Ok(slf) => Ok(slf.recall_validity().as_ref()), + Err(err) => Err(err.map_src(|s| s.as_ref())), + } + } + + /// Interprets the prefix of the given `source` as a DST `&Self` with length + /// equal to `count`. + /// + /// This method attempts to return a reference to the prefix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the remaining bytes. If there are insufficient bytes, or if `source` + /// is not appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(FromBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// // These are more bytes than are needed to encode two `Pixel`s. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (pixels, suffix) = <[Pixel]>::ref_from_prefix_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 0, g: 1, b: 2, a: 3 }, + /// Pixel { r: 4, g: 5, b: 6, a: 7 }, + /// ]); + /// + /// assert_eq!(suffix, &[8, 9]); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`ref_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &[85, 85][..]; + /// let (zsty, _) = ZSTy::ref_from_prefix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`ref_from_prefix`]: FromBytes::ref_from_prefix + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_prefix_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_prefix_with_elems( + source: &[u8], + count: usize, + ) -> Result<(&Self, &[u8]), CastError<&[u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + Immutable, + { + ref_from_prefix_suffix(source, Some(count), CastType::Prefix) + } + + /// Interprets the suffix of the given `source` as a DST `&Self` with length + /// equal to `count`. + /// + /// This method attempts to return a reference to the suffix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the preceding bytes. If there are insufficient bytes, or if that + /// suffix of `source` is not appropriately aligned, this returns `Err`. If + /// [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(FromBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// // These are more bytes than are needed to encode two `Pixel`s. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (prefix, pixels) = <[Pixel]>::ref_from_suffix_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(prefix, &[0, 1]); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 2, g: 3, b: 4, a: 5 }, + /// Pixel { r: 6, g: 7, b: 8, a: 9 }, + /// ]); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`ref_from_suffix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &[85, 85][..]; + /// let (_, zsty) = ZSTy::ref_from_suffix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`ref_from_suffix`]: FromBytes::ref_from_suffix + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_from_suffix_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn ref_from_suffix_with_elems( + source: &[u8], + count: usize, + ) -> Result<(&[u8], &Self), CastError<&[u8], Self>> + where + Self: KnownLayout<PointerMetadata = usize> + Immutable, + { + ref_from_prefix_suffix(source, Some(count), CastType::Suffix).map(swap) + } + + /// Interprets the given `source` as a `&mut Self` with a DST length equal + /// to `count`. + /// + /// This method attempts to return a reference to `source` interpreted as a + /// `Self` with `count` trailing elements. If the length of `source` is not + /// equal to the size of `Self` with `count` elements, or if `source` is not + /// appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7][..]; + /// + /// let pixels = <[Pixel]>::mut_from_bytes_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 0, g: 1, b: 2, a: 3 }, + /// Pixel { r: 4, g: 5, b: 6, a: 7 }, + /// ]); + /// + /// pixels[1] = Pixel { r: 0, g: 0, b: 0, a: 0 }; + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 0, 0, 0, 0]); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`mut_from_bytes`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, Immutable, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &mut [85, 85][..]; + /// let zsty = ZSTy::mut_from_bytes_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`mut_from_bytes`]: FromBytes::mut_from_bytes + /// + #[doc = codegen_header!("h5", "mut_from_bytes_with_elems")] + /// + /// See [`TryFromBytes::ref_from_bytes_with_elems`](#method.ref_from_bytes_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_bytes_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<&mut Self, CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout<PointerMetadata = usize> + Immutable, + { + let source = Ptr::from_mut(source); + let maybe_slf = source.try_cast_into_no_leftover::<_, BecauseImmutable>(Some(count)); + match maybe_slf { + Ok(slf) => Ok(slf.recall_validity::<_, (_, (_, BecauseExclusive))>().as_mut()), + Err(err) => Err(err.map_src(|s| s.as_mut())), + } + } + + /// Interprets the prefix of the given `source` as a `&mut Self` with DST + /// length equal to `count`. + /// + /// This method attempts to return a reference to the prefix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the preceding bytes. If there are insufficient bytes, or if `source` + /// is not appropriately aligned, this returns `Err`. If [`Self: + /// Unaligned`][self-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// // These are more bytes than are needed to encode two `Pixel`s. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (pixels, suffix) = <[Pixel]>::mut_from_prefix_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 0, g: 1, b: 2, a: 3 }, + /// Pixel { r: 4, g: 5, b: 6, a: 7 }, + /// ]); + /// + /// assert_eq!(suffix, &[8, 9]); + /// + /// pixels[1] = Pixel { r: 0, g: 0, b: 0, a: 0 }; + /// suffix.fill(1); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 0, 0, 0, 0, 1, 1]); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`mut_from_prefix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, Immutable, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &mut [85, 85][..]; + /// let (zsty, _) = ZSTy::mut_from_prefix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`mut_from_prefix`]: FromBytes::mut_from_prefix + /// + #[doc = codegen_header!("h5", "mut_from_prefix_with_elems")] + /// + /// See [`TryFromBytes::ref_from_prefix_with_elems`](#method.ref_from_prefix_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_prefix_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<(&mut Self, &mut [u8]), CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout<PointerMetadata = usize>, + { + mut_from_prefix_suffix(source, Some(count), CastType::Prefix) + } + + /// Interprets the suffix of the given `source` as a `&mut Self` with DST + /// length equal to `count`. + /// + /// This method attempts to return a reference to the suffix of `source` + /// interpreted as a `Self` with `count` trailing elements, and a reference + /// to the remaining bytes. If there are insufficient bytes, or if that + /// suffix of `source` is not appropriately aligned, this returns `Err`. If + /// [`Self: Unaligned`][self-unaligned], you can [infallibly discard the + /// alignment error][size-error-from]. + /// + /// [self-unaligned]: Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Debug, PartialEq, Eq)] + /// #[derive(FromBytes, IntoBytes, Immutable)] + /// #[repr(C)] + /// struct Pixel { + /// r: u8, + /// g: u8, + /// b: u8, + /// a: u8, + /// } + /// + /// // These are more bytes than are needed to encode two `Pixel`s. + /// let bytes = &mut [0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (prefix, pixels) = <[Pixel]>::mut_from_suffix_with_elems(bytes, 2).unwrap(); + /// + /// assert_eq!(prefix, &[0, 1]); + /// + /// assert_eq!(pixels, &[ + /// Pixel { r: 2, g: 3, b: 4, a: 5 }, + /// Pixel { r: 6, g: 7, b: 8, a: 9 }, + /// ]); + /// + /// prefix.fill(9); + /// pixels[1] = Pixel { r: 0, g: 0, b: 0, a: 0 }; + /// + /// assert_eq!(bytes, [9, 9, 2, 3, 4, 5, 0, 0, 0, 0]); + /// ``` + /// + /// Since an explicit `count` is provided, this method supports types with + /// zero-sized trailing slice elements. Methods such as [`mut_from_suffix`] + /// which do not take an explicit count do not support such types. + /// + /// ``` + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, Immutable, KnownLayout)] + /// #[repr(C, packed)] + /// struct ZSTy { + /// leading_sized: [u8; 2], + /// trailing_dst: [()], + /// } + /// + /// let src = &mut [85, 85][..]; + /// let (_, zsty) = ZSTy::mut_from_suffix_with_elems(src, 42).unwrap(); + /// assert_eq!(zsty.trailing_dst.len(), 42); + /// ``` + /// + /// [`mut_from_suffix`]: FromBytes::mut_from_suffix + /// + #[doc = codegen_header!("h5", "mut_from_suffix_with_elems")] + /// + /// See [`TryFromBytes::ref_from_suffix_with_elems`](#method.ref_from_suffix_with_elems.codegen). + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn mut_from_suffix_with_elems( + source: &mut [u8], + count: usize, + ) -> Result<(&mut [u8], &mut Self), CastError<&mut [u8], Self>> + where + Self: IntoBytes + KnownLayout<PointerMetadata = usize>, + { + mut_from_prefix_suffix(source, Some(count), CastType::Suffix).map(swap) + } + + /// Reads a copy of `Self` from the given `source`. + /// + /// If `source.len() != size_of::<Self>()`, `read_from_bytes` returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// // These bytes encode a `PacketHeader`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7][..]; + /// + /// let header = PacketHeader::read_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(header.src_port, [0, 1]); + /// assert_eq!(header.dst_port, [2, 3]); + /// assert_eq!(header.length, [4, 5]); + /// assert_eq!(header.checksum, [6, 7]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "read_from_bytes", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn read_from_bytes(source: &[u8]) -> Result<Self, SizeError<&[u8], Self>> + where + Self: Sized, + { + match Ref::<_, Unalign<Self>>::sized_from(source) { + Ok(r) => Ok(Ref::read(&r).into_inner()), + Err(CastError::Size(e)) => Err(e.with_dst()), + Err(CastError::Alignment(_)) => { + // SAFETY: `Unalign<Self>` is trivially aligned, so + // `Ref::sized_from` cannot fail due to unmet alignment + // requirements. + unsafe { core::hint::unreachable_unchecked() } + } + Err(CastError::Validity(i)) => match i {}, + } + } + + /// Reads a copy of `Self` from the prefix of the given `source`. + /// + /// This attempts to read a `Self` from the first `size_of::<Self>()` bytes + /// of `source`, returning that `Self` and any remaining bytes. If + /// `source.len() < size_of::<Self>()`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// // These are more bytes than are needed to encode a `PacketHeader`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (header, body) = PacketHeader::read_from_prefix(bytes).unwrap(); + /// + /// assert_eq!(header.src_port, [0, 1]); + /// assert_eq!(header.dst_port, [2, 3]); + /// assert_eq!(header.length, [4, 5]); + /// assert_eq!(header.checksum, [6, 7]); + /// assert_eq!(body, [8, 9]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "read_from_prefix", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn read_from_prefix(source: &[u8]) -> Result<(Self, &[u8]), SizeError<&[u8], Self>> + where + Self: Sized, + { + match Ref::<_, Unalign<Self>>::sized_from_prefix(source) { + Ok((r, suffix)) => Ok((Ref::read(&r).into_inner(), suffix)), + Err(CastError::Size(e)) => Err(e.with_dst()), + Err(CastError::Alignment(_)) => { + // SAFETY: `Unalign<Self>` is trivially aligned, so + // `Ref::sized_from_prefix` cannot fail due to unmet alignment + // requirements. + unsafe { core::hint::unreachable_unchecked() } + } + Err(CastError::Validity(i)) => match i {}, + } + } + + /// Reads a copy of `Self` from the suffix of the given `source`. + /// + /// This attempts to read a `Self` from the last `size_of::<Self>()` bytes + /// of `source`, returning that `Self` and any preceding bytes. If + /// `source.len() < size_of::<Self>()`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::FromBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes)] + /// #[repr(C)] + /// struct PacketTrailer { + /// frame_check_sequence: [u8; 4], + /// } + /// + /// // These are more bytes than are needed to encode a `PacketTrailer`. + /// let bytes = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let (prefix, trailer) = PacketTrailer::read_from_suffix(bytes).unwrap(); + /// + /// assert_eq!(prefix, [0, 1, 2, 3, 4, 5]); + /// assert_eq!(trailer.frame_check_sequence, [6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "read_from_suffix", + format = "coco_static_size", + )] + #[must_use = "has no side effects"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + fn read_from_suffix(source: &[u8]) -> Result<(&[u8], Self), SizeError<&[u8], Self>> + where + Self: Sized, + { + match Ref::<_, Unalign<Self>>::sized_from_suffix(source) { + Ok((prefix, r)) => Ok((prefix, Ref::read(&r).into_inner())), + Err(CastError::Size(e)) => Err(e.with_dst()), + Err(CastError::Alignment(_)) => { + // SAFETY: `Unalign<Self>` is trivially aligned, so + // `Ref::sized_from_suffix` cannot fail due to unmet alignment + // requirements. + unsafe { core::hint::unreachable_unchecked() } + } + Err(CastError::Validity(i)) => match i {}, + } + } + + /// Reads a copy of `self` from an `io::Read`. + /// + /// This is useful for interfacing with operating system byte sinks (files, + /// sockets, etc.). + /// + /// # Examples + /// + /// ```no_run + /// use zerocopy::{byteorder::big_endian::*, FromBytes}; + /// use std::fs::File; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes)] + /// #[repr(C)] + /// struct BitmapFileHeader { + /// signature: [u8; 2], + /// size: U32, + /// reserved: U64, + /// offset: U64, + /// } + /// + /// let mut file = File::open("image.bin").unwrap(); + /// let header = BitmapFileHeader::read_from_io(&mut file).unwrap(); + /// ``` + #[cfg(feature = "std")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "std")))] + #[inline(always)] + fn read_from_io<R>(mut src: R) -> io::Result<Self> + where + Self: Sized, + R: io::Read, + { + // NOTE(#2319, #2320): We do `buf.zero()` separately rather than + // constructing `let buf = CoreMaybeUninit::zeroed()` because, if `Self` + // contains padding bytes, then a typed copy of `CoreMaybeUninit<Self>` + // will not necessarily preserve zeros written to those padding byte + // locations, and so `buf` could contain uninitialized bytes. + let mut buf = CoreMaybeUninit::<Self>::uninit(); + buf.zero(); + + let ptr = Ptr::from_mut(&mut buf); + // SAFETY: After `buf.zero()`, `buf` consists entirely of initialized, + // zeroed bytes. Since `MaybeUninit` has no validity requirements, `ptr` + // cannot be used to write values which will violate `buf`'s bit + // validity. Since `ptr` has `Exclusive` aliasing, nothing other than + // `ptr` may be used to mutate `ptr`'s referent, and so its bit validity + // cannot be violated even though `buf` may have more permissive bit + // validity than `ptr`. + let ptr = unsafe { ptr.assume_validity::<invariant::Initialized>() }; + let ptr = ptr.as_bytes(); + src.read_exact(ptr.as_mut())?; + // SAFETY: `buf` entirely consists of initialized bytes, and `Self` is + // `FromBytes`. + Ok(unsafe { buf.assume_init() }) + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::ref_from_bytes`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn ref_from(source: &[u8]) -> Option<&Self> + where + Self: KnownLayout + Immutable, + { + Self::ref_from_bytes(source).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::mut_from_bytes`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn mut_from(source: &mut [u8]) -> Option<&mut Self> + where + Self: KnownLayout + IntoBytes, + { + Self::mut_from_bytes(source).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::ref_from_prefix_with_elems`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn slice_from_prefix(source: &[u8], count: usize) -> Option<(&[Self], &[u8])> + where + Self: Sized + Immutable, + { + <[Self]>::ref_from_prefix_with_elems(source, count).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::ref_from_suffix_with_elems`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn slice_from_suffix(source: &[u8], count: usize) -> Option<(&[u8], &[Self])> + where + Self: Sized + Immutable, + { + <[Self]>::ref_from_suffix_with_elems(source, count).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::mut_from_prefix_with_elems`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn mut_slice_from_prefix(source: &mut [u8], count: usize) -> Option<(&mut [Self], &mut [u8])> + where + Self: Sized + IntoBytes, + { + <[Self]>::mut_from_prefix_with_elems(source, count).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::mut_from_suffix_with_elems`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn mut_slice_from_suffix(source: &mut [u8], count: usize) -> Option<(&mut [u8], &mut [Self])> + where + Self: Sized + IntoBytes, + { + <[Self]>::mut_from_suffix_with_elems(source, count).ok() + } + + #[deprecated(since = "0.8.0", note = "renamed to `FromBytes::read_from_bytes`")] + #[doc(hidden)] + #[must_use = "has no side effects"] + #[inline(always)] + fn read_from(source: &[u8]) -> Option<Self> + where + Self: Sized, + { + Self::read_from_bytes(source).ok() + } +} + +/// Interprets the given affix of the given bytes as a `&Self`. +/// +/// This method computes the largest possible size of `Self` that can fit in the +/// prefix or suffix bytes of `source`, then attempts to return both a reference +/// to those bytes interpreted as a `Self`, and a reference to the excess bytes. +/// If there are insufficient bytes, or if that affix of `source` is not +/// appropriately aligned, this returns `Err`. +#[inline(always)] +fn ref_from_prefix_suffix<T: FromBytes + KnownLayout + Immutable + ?Sized>( + source: &[u8], + meta: Option<T::PointerMetadata>, + cast_type: CastType, +) -> Result<(&T, &[u8]), CastError<&[u8], T>> { + let (slf, prefix_suffix) = Ptr::from_ref(source) + .try_cast_into::<_, BecauseImmutable>(cast_type, meta) + .map_err(|err| err.map_src(|s| s.as_ref()))?; + Ok((slf.recall_validity().as_ref(), prefix_suffix.as_ref())) +} + +/// Interprets the given affix of the given bytes as a `&mut Self` without +/// copying. +/// +/// This method computes the largest possible size of `Self` that can fit in the +/// prefix or suffix bytes of `source`, then attempts to return both a reference +/// to those bytes interpreted as a `Self`, and a reference to the excess bytes. +/// If there are insufficient bytes, or if that affix of `source` is not +/// appropriately aligned, this returns `Err`. +#[inline(always)] +fn mut_from_prefix_suffix<T: FromBytes + IntoBytes + KnownLayout + ?Sized>( + source: &mut [u8], + meta: Option<T::PointerMetadata>, + cast_type: CastType, +) -> Result<(&mut T, &mut [u8]), CastError<&mut [u8], T>> { + let (slf, prefix_suffix) = Ptr::from_mut(source) + .try_cast_into::<_, BecauseExclusive>(cast_type, meta) + .map_err(|err| err.map_src(|s| s.as_mut()))?; + Ok((slf.recall_validity::<_, (_, (_, _))>().as_mut(), prefix_suffix.as_mut())) +} + +/// Analyzes whether a type is [`IntoBytes`]. +/// +/// This derive analyzes, at compile time, whether the annotated type satisfies +/// the [safety conditions] of `IntoBytes` and implements `IntoBytes` if it is +/// sound to do so. This derive can be applied to structs and enums (see below +/// for union support); e.g.: +/// +/// ``` +/// # use zerocopy_derive::{IntoBytes}; +/// #[derive(IntoBytes)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(IntoBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// [safety conditions]: trait@IntoBytes#safety +/// +/// # Error Messages +/// +/// On Rust toolchains prior to 1.78.0, due to the way that the custom derive +/// for `IntoBytes` is implemented, you may get an error like this: +/// +/// ```text +/// error[E0277]: the trait bound `(): PaddingFree<Foo, true>` is not satisfied +/// --> lib.rs:23:10 +/// | +/// 1 | #[derive(IntoBytes)] +/// | ^^^^^^^^^ the trait `PaddingFree<Foo, true>` is not implemented for `()` +/// | +/// = help: the following implementations were found: +/// <() as PaddingFree<T, false>> +/// ``` +/// +/// This error indicates that the type being annotated has padding bytes, which +/// is illegal for `IntoBytes` types. Consider reducing the alignment of some +/// fields by using types in the [`byteorder`] module, wrapping field types in +/// [`Unalign`], adding explicit struct fields where those padding bytes would +/// be, or using `#[repr(packed)]`. See the Rust Reference's page on [type +/// layout] for more information about type layout and padding. +/// +/// [type layout]: https://doc.rust-lang.org/reference/type-layout.html +/// +/// # Unions +/// +/// Currently, union bit validity is [up in the air][union-validity], and so +/// zerocopy does not support `#[derive(IntoBytes)]` on unions by default. +/// However, implementing `IntoBytes` on a union type is likely sound on all +/// existing Rust toolchains - it's just that it may become unsound in the +/// future. You can opt-in to `#[derive(IntoBytes)]` support on unions by +/// passing the unstable `zerocopy_derive_union_into_bytes` cfg: +/// +/// ```shell +/// $ RUSTFLAGS='--cfg zerocopy_derive_union_into_bytes' cargo build +/// ``` +/// +/// However, it is your responsibility to ensure that this derive is sound on +/// the specific versions of the Rust toolchain you are using! We make no +/// stability or soundness guarantees regarding this cfg, and may remove it at +/// any point. +/// +/// We are actively working with Rust to stabilize the necessary language +/// guarantees to support this in a forwards-compatible way, which will enable +/// us to remove the cfg gate. As part of this effort, we need to know how much +/// demand there is for this feature. If you would like to use `IntoBytes` on +/// unions, [please let us know][discussion]. +/// +/// [union-validity]: https://github.com/rust-lang/unsafe-code-guidelines/issues/438 +/// [discussion]: https://github.com/google/zerocopy/discussions/1802 +/// +/// # Analysis +/// +/// *This section describes, roughly, the analysis performed by this derive to +/// determine whether it is sound to implement `IntoBytes` for a given type. +/// Unless you are modifying the implementation of this derive, or attempting to +/// manually implement `IntoBytes` for a type yourself, you don't need to read +/// this section.* +/// +/// If a type has the following properties, then this derive can implement +/// `IntoBytes` for that type: +/// +/// - If the type is a struct, its fields must be [`IntoBytes`]. Additionally: +/// - if the type is `repr(transparent)` or `repr(packed)`, it is +/// [`IntoBytes`] if its fields are [`IntoBytes`]; else, +/// - if the type is `repr(C)` with at most one field, it is [`IntoBytes`] +/// if its field is [`IntoBytes`]; else, +/// - if the type has no generic parameters, it is [`IntoBytes`] if the type +/// is sized and has no padding bytes; else, +/// - if the type is `repr(C)`, its fields must be [`Unaligned`]. +/// - If the type is an enum: +/// - It must have a defined representation (`repr`s `C`, `u8`, `u16`, `u32`, +/// `u64`, `usize`, `i8`, `i16`, `i32`, `i64`, or `isize`). +/// - It must have no padding bytes. +/// - Its fields must be [`IntoBytes`]. +/// +/// This analysis is subject to change. Unsafe code may *only* rely on the +/// documented [safety conditions] of `FromBytes`, and must *not* rely on the +/// implementation details of this derive. +/// +/// [Rust Reference]: https://doc.rust-lang.org/reference/type-layout.html +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::IntoBytes; + +/// Types that can be converted to an immutable slice of initialized bytes. +/// +/// Any `IntoBytes` type can be converted to a slice of initialized bytes of the +/// same size. This is useful for efficiently serializing structured data as raw +/// bytes. +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(IntoBytes)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::IntoBytes; +/// #[derive(IntoBytes)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(IntoBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `IntoBytes`. See the [derive +/// documentation][derive] for guidance on how to interpret error messages +/// produced by the derive's analysis. +/// +/// # Safety +/// +/// *This section describes what is required in order for `T: IntoBytes`, and +/// what unsafe code may assume of such types. If you don't plan on implementing +/// `IntoBytes` manually, and you don't plan on writing unsafe code that +/// operates on `IntoBytes` types, then you don't need to read this section.* +/// +/// If `T: IntoBytes`, then unsafe code may assume that it is sound to treat any +/// `t: T` as an immutable `[u8]` of length `size_of_val(t)`. If a type is +/// marked as `IntoBytes` which violates this contract, it may cause undefined +/// behavior. +/// +/// `#[derive(IntoBytes)]` only permits [types which satisfy these +/// requirements][derive-analysis]. +/// +#[cfg_attr( + feature = "derive", + doc = "[derive]: zerocopy_derive::IntoBytes", + doc = "[derive-analysis]: zerocopy_derive::IntoBytes#analysis" +)] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.IntoBytes.html"), + doc = concat!("[derive-analysis]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.IntoBytes.html#analysis"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(IntoBytes)]` to `{Self}`") +)] +pub unsafe trait IntoBytes { + // The `Self: Sized` bound makes it so that this function doesn't prevent + // `IntoBytes` from being object safe. Note that other `IntoBytes` methods + // prevent object safety, but those provide a benefit in exchange for object + // safety. If at some point we remove those methods, change their type + // signatures, or move them out of this trait so that `IntoBytes` is object + // safe again, it's important that this function not prevent object safety. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// Gets the bytes of this value. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::IntoBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(IntoBytes, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let header = PacketHeader { + /// src_port: [0, 1], + /// dst_port: [2, 3], + /// length: [4, 5], + /// checksum: [6, 7], + /// }; + /// + /// let bytes = header.as_bytes(); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 6, 7]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "as_bytes", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] + )] + #[must_use = "has no side effects"] + #[inline(always)] + fn as_bytes(&self) -> &[u8] + where + Self: Immutable, + { + // Note that this method does not have a `Self: Sized` bound; + // `size_of_val` works for unsized values too. + let len = mem::size_of_val(self); + let slf: *const Self = self; + + // SAFETY: + // - `slf.cast::<u8>()` is valid for reads for `len * size_of::<u8>()` + // many bytes because... + // - `slf` is the same pointer as `self`, and `self` is a reference + // which points to an object whose size is `len`. Thus... + // - The entire region of `len` bytes starting at `slf` is contained + // within a single allocation. + // - `slf` is non-null. + // - `slf` is trivially aligned to `align_of::<u8>() == 1`. + // - `Self: IntoBytes` ensures that all of the bytes of `slf` are + // initialized. + // - Since `slf` is derived from `self`, and `self` is an immutable + // reference, the only other references to this memory region that + // could exist are other immutable references, which by `Self: + // Immutable` don't permit mutation. + // - The total size of the resulting slice is no larger than + // `isize::MAX` because no allocation produced by safe code can be + // larger than `isize::MAX`. + // + // FIXME(#429): Add references to docs and quotes. + unsafe { slice::from_raw_parts(slf.cast::<u8>(), len) } + } + + /// Gets the bytes of this value mutably. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::IntoBytes; + /// # use zerocopy_derive::*; + /// + /// # #[derive(Eq, PartialEq, Debug)] + /// #[derive(FromBytes, IntoBytes, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let mut header = PacketHeader { + /// src_port: [0, 1], + /// dst_port: [2, 3], + /// length: [4, 5], + /// checksum: [6, 7], + /// }; + /// + /// let bytes = header.as_mut_bytes(); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 6, 7]); + /// + /// bytes.reverse(); + /// + /// assert_eq!(header, PacketHeader { + /// src_port: [7, 6], + /// dst_port: [5, 4], + /// length: [3, 2], + /// checksum: [1, 0], + /// }); + /// ``` + /// + #[doc = codegen_header!("h5", "as_mut_bytes")] + /// + /// See [`IntoBytes::as_bytes`](#method.as_bytes.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + fn as_mut_bytes(&mut self) -> &mut [u8] + where + Self: FromBytes, + { + // Note that this method does not have a `Self: Sized` bound; + // `size_of_val` works for unsized values too. + let len = mem::size_of_val(self); + let slf: *mut Self = self; + + // SAFETY: + // - `slf.cast::<u8>()` is valid for reads and writes for `len * + // size_of::<u8>()` many bytes because... + // - `slf` is the same pointer as `self`, and `self` is a reference + // which points to an object whose size is `len`. Thus... + // - The entire region of `len` bytes starting at `slf` is contained + // within a single allocation. + // - `slf` is non-null. + // - `slf` is trivially aligned to `align_of::<u8>() == 1`. + // - `Self: IntoBytes` ensures that all of the bytes of `slf` are + // initialized. + // - `Self: FromBytes` ensures that no write to this memory region + // could result in it containing an invalid `Self`. + // - Since `slf` is derived from `self`, and `self` is a mutable + // reference, no other references to this memory region can exist. + // - The total size of the resulting slice is no larger than + // `isize::MAX` because no allocation produced by safe code can be + // larger than `isize::MAX`. + // + // FIXME(#429): Add references to docs and quotes. + unsafe { slice::from_raw_parts_mut(slf.cast::<u8>(), len) } + } + + /// Writes a copy of `self` to `dst`. + /// + /// If `dst.len() != size_of_val(self)`, `write_to` returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::IntoBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(IntoBytes, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let header = PacketHeader { + /// src_port: [0, 1], + /// dst_port: [2, 3], + /// length: [4, 5], + /// checksum: [6, 7], + /// }; + /// + /// let mut bytes = [0, 0, 0, 0, 0, 0, 0, 0]; + /// + /// header.write_to(&mut bytes[..]); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 6, 7]); + /// ``` + /// + /// If too many or too few target bytes are provided, `write_to` returns + /// `Err` and leaves the target bytes unmodified: + /// + /// ``` + /// # use zerocopy::IntoBytes; + /// # let header = u128::MAX; + /// let mut excessive_bytes = &mut [0u8; 128][..]; + /// + /// let write_result = header.write_to(excessive_bytes); + /// + /// assert!(write_result.is_err()); + /// assert_eq!(excessive_bytes, [0u8; 128]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "write_to", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] + )] + #[must_use = "callers should check the return value to see if the operation succeeded"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + #[allow(clippy::mut_from_ref)] // False positive: `&self -> &mut [u8]` + fn write_to(&self, dst: &mut [u8]) -> Result<(), SizeError<&Self, &mut [u8]>> + where + Self: Immutable, + { + let src = self.as_bytes(); + if dst.len() == src.len() { + // SAFETY: Within this branch of the conditional, we have ensured + // that `dst.len()` is equal to `src.len()`. Neither the size of the + // source nor the size of the destination change between the above + // size check and the invocation of `copy_unchecked`. + unsafe { util::copy_unchecked(src, dst) } + Ok(()) + } else { + Err(SizeError::new(self)) + } + } + + /// Writes a copy of `self` to the prefix of `dst`. + /// + /// `write_to_prefix` writes `self` to the first `size_of_val(self)` bytes + /// of `dst`. If `dst.len() < size_of_val(self)`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::IntoBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(IntoBytes, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let header = PacketHeader { + /// src_port: [0, 1], + /// dst_port: [2, 3], + /// length: [4, 5], + /// checksum: [6, 7], + /// }; + /// + /// let mut bytes = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + /// + /// header.write_to_prefix(&mut bytes[..]); + /// + /// assert_eq!(bytes, [0, 1, 2, 3, 4, 5, 6, 7, 0, 0]); + /// ``` + /// + /// If insufficient target bytes are provided, `write_to_prefix` returns + /// `Err` and leaves the target bytes unmodified: + /// + /// ``` + /// # use zerocopy::IntoBytes; + /// # let header = u128::MAX; + /// let mut insufficient_bytes = &mut [0, 0][..]; + /// + /// let write_result = header.write_to_suffix(insufficient_bytes); + /// + /// assert!(write_result.is_err()); + /// assert_eq!(insufficient_bytes, [0, 0]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "write_to_prefix", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] + )] + #[must_use = "callers should check the return value to see if the operation succeeded"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + #[allow(clippy::mut_from_ref)] // False positive: `&self -> &mut [u8]` + fn write_to_prefix(&self, dst: &mut [u8]) -> Result<(), SizeError<&Self, &mut [u8]>> + where + Self: Immutable, + { + let src = self.as_bytes(); + match dst.get_mut(..src.len()) { + Some(dst) => { + // SAFETY: Within this branch of the `match`, we have ensured + // through fallible subslicing that `dst.len()` is equal to + // `src.len()`. Neither the size of the source nor the size of + // the destination change between the above subslicing operation + // and the invocation of `copy_unchecked`. + unsafe { util::copy_unchecked(src, dst) } + Ok(()) + } + None => Err(SizeError::new(self)), + } + } + + /// Writes a copy of `self` to the suffix of `dst`. + /// + /// `write_to_suffix` writes `self` to the last `size_of_val(self)` bytes of + /// `dst`. If `dst.len() < size_of_val(self)`, it returns `Err`. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::IntoBytes; + /// # use zerocopy_derive::*; + /// + /// #[derive(IntoBytes, Immutable)] + /// #[repr(C)] + /// struct PacketHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// let header = PacketHeader { + /// src_port: [0, 1], + /// dst_port: [2, 3], + /// length: [4, 5], + /// checksum: [6, 7], + /// }; + /// + /// let mut bytes = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + /// + /// header.write_to_suffix(&mut bytes[..]); + /// + /// assert_eq!(bytes, [0, 0, 0, 1, 2, 3, 4, 5, 6, 7]); + /// + /// let mut insufficient_bytes = &mut [0, 0][..]; + /// + /// let write_result = header.write_to_suffix(insufficient_bytes); + /// + /// assert!(write_result.is_err()); + /// assert_eq!(insufficient_bytes, [0, 0]); + /// ``` + /// + /// If insufficient target bytes are provided, `write_to_suffix` returns + /// `Err` and leaves the target bytes unmodified: + /// + /// ``` + /// # use zerocopy::IntoBytes; + /// # let header = u128::MAX; + /// let mut insufficient_bytes = &mut [0, 0][..]; + /// + /// let write_result = header.write_to_suffix(insufficient_bytes); + /// + /// assert!(write_result.is_err()); + /// assert_eq!(insufficient_bytes, [0, 0]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "write_to_suffix", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] + )] + #[must_use = "callers should check the return value to see if the operation succeeded"] + #[cfg_attr(zerocopy_inline_always, inline(always))] + #[cfg_attr(not(zerocopy_inline_always), inline)] + #[allow(clippy::mut_from_ref)] // False positive: `&self -> &mut [u8]` + fn write_to_suffix(&self, dst: &mut [u8]) -> Result<(), SizeError<&Self, &mut [u8]>> + where + Self: Immutable, + { + let src = self.as_bytes(); + let start = if let Some(start) = dst.len().checked_sub(src.len()) { + start + } else { + return Err(SizeError::new(self)); + }; + let dst = if let Some(dst) = dst.get_mut(start..) { + dst + } else { + // get_mut() should never return None here. We return a `SizeError` + // rather than .unwrap() because in the event the branch is not + // optimized away, returning a value is generally lighter-weight + // than panicking. + return Err(SizeError::new(self)); + }; + // SAFETY: Through fallible subslicing of `dst`, we have ensured that + // `dst.len()` is equal to `src.len()`. Neither the size of the source + // nor the size of the destination change between the above subslicing + // operation and the invocation of `copy_unchecked`. + unsafe { + util::copy_unchecked(src, dst); + } + Ok(()) + } + + /// Writes a copy of `self` to an `io::Write`. + /// + /// This is a shorthand for `dst.write_all(self.as_bytes())`, and is useful + /// for interfacing with operating system byte sinks (files, sockets, etc.). + /// + /// # Examples + /// + /// ```no_run + /// use zerocopy::{byteorder::big_endian::U16, FromBytes, IntoBytes}; + /// use std::fs::File; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, Immutable, KnownLayout)] + /// #[repr(C, packed)] + /// struct GrayscaleImage { + /// height: U16, + /// width: U16, + /// pixels: [U16], + /// } + /// + /// let image = GrayscaleImage::ref_from_bytes(&[0, 0, 0, 0][..]).unwrap(); + /// let mut file = File::create("image.bin").unwrap(); + /// image.write_to_io(&mut file).unwrap(); + /// ``` + /// + /// If the write fails, `write_to_io` returns `Err` and a partial write may + /// have occurred; e.g.: + /// + /// ``` + /// # use zerocopy::IntoBytes; + /// + /// let src = u128::MAX; + /// let mut dst = [0u8; 2]; + /// + /// let write_result = src.write_to_io(&mut dst[..]); + /// + /// assert!(write_result.is_err()); + /// assert_eq!(dst, [255, 255]); + /// ``` + #[cfg(feature = "std")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "std")))] + #[inline(always)] + fn write_to_io<W>(&self, mut dst: W) -> io::Result<()> + where + Self: Immutable, + W: io::Write, + { + dst.write_all(self.as_bytes()) + } + + #[deprecated(since = "0.8.0", note = "`IntoBytes::as_bytes_mut` was renamed to `as_mut_bytes`")] + #[doc(hidden)] + #[inline] + fn as_bytes_mut(&mut self) -> &mut [u8] + where + Self: FromBytes, + { + self.as_mut_bytes() + } +} + +/// Analyzes whether a type is [`Unaligned`]. +/// +/// This derive analyzes, at compile time, whether the annotated type satisfies +/// the [safety conditions] of `Unaligned` and implements `Unaligned` if it is +/// sound to do so. This derive can be applied to structs, enums, and unions; +/// e.g.: +/// +/// ``` +/// # use zerocopy_derive::Unaligned; +/// #[derive(Unaligned)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Unaligned)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Unaligned)] +/// #[repr(packed)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// # Analysis +/// +/// *This section describes, roughly, the analysis performed by this derive to +/// determine whether it is sound to implement `Unaligned` for a given type. +/// Unless you are modifying the implementation of this derive, or attempting to +/// manually implement `Unaligned` for a type yourself, you don't need to read +/// this section.* +/// +/// If a type has the following properties, then this derive can implement +/// `Unaligned` for that type: +/// +/// - If the type is a struct or union: +/// - If `repr(align(N))` is provided, `N` must equal 1. +/// - If the type is `repr(C)` or `repr(transparent)`, all fields must be +/// [`Unaligned`]. +/// - If the type is not `repr(C)` or `repr(transparent)`, it must be +/// `repr(packed)` or `repr(packed(1))`. +/// - If the type is an enum: +/// - If `repr(align(N))` is provided, `N` must equal 1. +/// - It must be a field-less enum (meaning that all variants have no fields). +/// - It must be `repr(i8)` or `repr(u8)`. +/// +/// [safety conditions]: trait@Unaligned#safety +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::Unaligned; + +/// Types with no alignment requirement. +/// +/// If `T: Unaligned`, then `align_of::<T>() == 1`. +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(Unaligned)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::Unaligned; +/// #[derive(Unaligned)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Unaligned)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant0, +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(Unaligned)] +/// #[repr(packed)] +/// union MyUnion { +/// # variant: u8, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `Unaligned`. +/// +/// # Safety +/// +/// *This section describes what is required in order for `T: Unaligned`, and +/// what unsafe code may assume of such types. If you don't plan on implementing +/// `Unaligned` manually, and you don't plan on writing unsafe code that +/// operates on `Unaligned` types, then you don't need to read this section.* +/// +/// If `T: Unaligned`, then unsafe code may assume that it is sound to produce a +/// reference to `T` at any memory location regardless of alignment. If a type +/// is marked as `Unaligned` which violates this contract, it may cause +/// undefined behavior. +/// +/// `#[derive(Unaligned)]` only permits [types which satisfy these +/// requirements][derive-analysis]. +/// +#[cfg_attr( + feature = "derive", + doc = "[derive]: zerocopy_derive::Unaligned", + doc = "[derive-analysis]: zerocopy_derive::Unaligned#analysis" +)] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.Unaligned.html"), + doc = concat!("[derive-analysis]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.Unaligned.html#analysis"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(Unaligned)]` to `{Self}`") +)] +pub unsafe trait Unaligned { + // The `Self: Sized` bound makes it so that `Unaligned` is still object + // safe. + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; +} + +/// Derives optimized [`PartialEq`] and [`Eq`] implementations. +/// +/// This derive can be applied to structs and enums implementing both +/// [`Immutable`] and [`IntoBytes`]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{ByteEq, Immutable, IntoBytes}; +/// #[derive(ByteEq, Immutable, IntoBytes)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(ByteEq, Immutable, IntoBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The standard library's [`derive(Eq, PartialEq)`][derive@PartialEq] computes +/// equality by individually comparing each field. Instead, the implementation +/// of [`PartialEq::eq`] emitted by `derive(ByteHash)` converts the entirety of +/// `self` and `other` to byte slices and compares those slices for equality. +/// This may have performance advantages. +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::ByteEq; +/// Derives an optimized [`Hash`] implementation. +/// +/// This derive can be applied to structs and enums implementing both +/// [`Immutable`] and [`IntoBytes`]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{ByteHash, Immutable, IntoBytes}; +/// #[derive(ByteHash, Immutable, IntoBytes)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(ByteHash, Immutable, IntoBytes)] +/// #[repr(u8)] +/// enum MyEnum { +/// # Variant, +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The standard library's [`derive(Hash)`][derive@Hash] produces hashes by +/// individually hashing each field and combining the results. Instead, the +/// implementations of [`Hash::hash()`] and [`Hash::hash_slice()`] generated by +/// `derive(ByteHash)` convert the entirety of `self` to a byte slice and hashes +/// it in a single call to [`Hasher::write()`]. This may have performance +/// advantages. +/// +/// [`Hash`]: core::hash::Hash +/// [`Hash::hash()`]: core::hash::Hash::hash() +/// [`Hash::hash_slice()`]: core::hash::Hash::hash_slice() +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::ByteHash; +/// Implements [`SplitAt`]. +/// +/// This derive can be applied to structs; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{ByteEq, Immutable, IntoBytes}; +/// #[derive(ByteEq, Immutable, IntoBytes)] +/// #[repr(C)] +/// struct MyStruct { +/// # /* +/// ... +/// # */ +/// } +/// ``` +#[cfg(any(feature = "derive", test))] +#[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] +pub use zerocopy_derive::SplitAt; + +#[cfg(feature = "alloc")] +#[cfg_attr(doc_cfg, doc(cfg(feature = "alloc")))] +#[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] +mod alloc_support { + use super::*; + + /// Extends a `Vec<T>` by pushing `additional` new items onto the end of the + /// vector. The new items are initialized with zeros. + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[doc(hidden)] + #[deprecated(since = "0.8.0", note = "moved to `FromZeros`")] + #[inline(always)] + pub fn extend_vec_zeroed<T: FromZeros>( + v: &mut Vec<T>, + additional: usize, + ) -> Result<(), AllocError> { + <T as FromZeros>::extend_vec_zeroed(v, additional) + } + + /// Inserts `additional` new items into `Vec<T>` at `position`. The new + /// items are initialized with zeros. + /// + /// # Panics + /// + /// Panics if `position > v.len()`. + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[doc(hidden)] + #[deprecated(since = "0.8.0", note = "moved to `FromZeros`")] + #[inline(always)] + pub fn insert_vec_zeroed<T: FromZeros>( + v: &mut Vec<T>, + position: usize, + additional: usize, + ) -> Result<(), AllocError> { + <T as FromZeros>::insert_vec_zeroed(v, position, additional) + } +} + +#[cfg(feature = "alloc")] +#[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] +#[doc(hidden)] +pub use alloc_support::*; + +#[cfg(test)] +#[allow(clippy::assertions_on_result_states, clippy::unreadable_literal)] +mod tests { + use static_assertions::assert_impl_all; + + use super::*; + use crate::util::testutil::*; + + // An unsized type. + // + // This is used to test the custom derives of our traits. The `[u8]` type + // gets a hand-rolled impl, so it doesn't exercise our custom derives. + #[derive(Debug, Eq, PartialEq, FromBytes, IntoBytes, Unaligned, Immutable)] + #[repr(transparent)] + struct Unsized([u8]); + + impl Unsized { + fn from_mut_slice(slc: &mut [u8]) -> &mut Unsized { + // SAFETY: This *probably* sound - since the layouts of `[u8]` and + // `Unsized` are the same, so are the layouts of `&mut [u8]` and + // `&mut Unsized`. [1] Even if it turns out that this isn't actually + // guaranteed by the language spec, we can just change this since + // it's in test code. + // + // [1] https://github.com/rust-lang/unsafe-code-guidelines/issues/375 + unsafe { mem::transmute(slc) } + } + } + + #[test] + fn test_known_layout() { + // Test that `$ty` and `ManuallyDrop<$ty>` have the expected layout. + // Test that `PhantomData<$ty>` has the same layout as `()` regardless + // of `$ty`. + macro_rules! test { + ($ty:ty, $expect:expr) => { + let expect = $expect; + assert_eq!(<$ty as KnownLayout>::LAYOUT, expect); + assert_eq!(<ManuallyDrop<$ty> as KnownLayout>::LAYOUT, expect); + assert_eq!(<PhantomData<$ty> as KnownLayout>::LAYOUT, <() as KnownLayout>::LAYOUT); + }; + } + + let layout = + |offset, align, trailing_slice_elem_size, statically_shallow_unpadded| DstLayout { + align: NonZeroUsize::new(align).unwrap(), + size_info: match trailing_slice_elem_size { + None => SizeInfo::Sized { size: offset }, + Some(elem_size) => { + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) + } + }, + statically_shallow_unpadded, + }; + + test!((), layout(0, 1, None, false)); + test!(u8, layout(1, 1, None, false)); + // Use `align_of` because `u64` alignment may be smaller than 8 on some + // platforms. + test!(u64, layout(8, mem::align_of::<u64>(), None, false)); + test!(AU64, layout(8, 8, None, false)); + + test!(Option<&'static ()>, usize::LAYOUT); + + test!([()], layout(0, 1, Some(0), true)); + test!([u8], layout(0, 1, Some(1), true)); + test!(str, layout(0, 1, Some(1), true)); + } + + #[cfg(feature = "derive")] + #[test] + fn test_known_layout_derive() { + // In this and other files (`late_compile_pass.rs`, + // `mid_compile_pass.rs`, and `struct.rs`), we test success and failure + // modes of `derive(KnownLayout)` for the following combination of + // properties: + // + // +------------+--------------------------------------+-----------+ + // | | trailing field properties | | + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // |------------+----------+----------------+----------+-----------| + // | N | N | N | N | KL00 | + // | N | N | N | Y | KL01 | + // | N | N | Y | N | KL02 | + // | N | N | Y | Y | KL03 | + // | N | Y | N | N | KL04 | + // | N | Y | N | Y | KL05 | + // | N | Y | Y | N | KL06 | + // | N | Y | Y | Y | KL07 | + // | Y | N | N | N | KL08 | + // | Y | N | N | Y | KL09 | + // | Y | N | Y | N | KL10 | + // | Y | N | Y | Y | KL11 | + // | Y | Y | N | N | KL12 | + // | Y | Y | N | Y | KL13 | + // | Y | Y | Y | N | KL14 | + // | Y | Y | Y | Y | KL15 | + // +------------+----------+----------------+----------+-----------+ + + struct NotKnownLayout<T = ()> { + _t: T, + } + + #[derive(KnownLayout)] + #[repr(C)] + struct AlignSize<const ALIGN: usize, const SIZE: usize> + where + elain::Align<ALIGN>: elain::Alignment, + { + _align: elain::Align<ALIGN>, + size: [u8; SIZE], + } + + type AU16 = AlignSize<2, 2>; + type AU32 = AlignSize<4, 4>; + + fn _assert_kl<T: ?Sized + KnownLayout>(_: &T) {} + + let sized_layout = |align, size| DstLayout { + align: NonZeroUsize::new(align).unwrap(), + size_info: SizeInfo::Sized { size }, + statically_shallow_unpadded: false, + }; + + let unsized_layout = |align, elem_size, offset, statically_shallow_unpadded| DstLayout { + align: NonZeroUsize::new(align).unwrap(), + size_info: SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }), + statically_shallow_unpadded, + }; + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | N | N | N | Y | KL01 | + #[allow(dead_code)] + #[derive(KnownLayout)] + struct KL01(NotKnownLayout<AU32>, NotKnownLayout<AU16>); + + let expected = DstLayout::for_type::<KL01>(); + + assert_eq!(<KL01 as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL01 as KnownLayout>::LAYOUT, sized_layout(4, 8)); + + // ...with `align(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(align(64))] + struct KL01Align(NotKnownLayout<AU32>, NotKnownLayout<AU16>); + + let expected = DstLayout::for_type::<KL01Align>(); + + assert_eq!(<KL01Align as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL01Align as KnownLayout>::LAYOUT, sized_layout(64, 64)); + + // ...with `packed`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(packed)] + struct KL01Packed(NotKnownLayout<AU32>, NotKnownLayout<AU16>); + + let expected = DstLayout::for_type::<KL01Packed>(); + + assert_eq!(<KL01Packed as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL01Packed as KnownLayout>::LAYOUT, sized_layout(1, 6)); + + // ...with `packed(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(packed(2))] + struct KL01PackedN(NotKnownLayout<AU32>, NotKnownLayout<AU16>); + + assert_impl_all!(KL01PackedN: KnownLayout); + + let expected = DstLayout::for_type::<KL01PackedN>(); + + assert_eq!(<KL01PackedN as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL01PackedN as KnownLayout>::LAYOUT, sized_layout(2, 6)); + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | N | N | Y | Y | KL03 | + #[allow(dead_code)] + #[derive(KnownLayout)] + struct KL03(NotKnownLayout, u8); + + let expected = DstLayout::for_type::<KL03>(); + + assert_eq!(<KL03 as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL03 as KnownLayout>::LAYOUT, sized_layout(1, 1)); + + // ... with `align(N)` + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(align(64))] + struct KL03Align(NotKnownLayout<AU32>, u8); + + let expected = DstLayout::for_type::<KL03Align>(); + + assert_eq!(<KL03Align as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL03Align as KnownLayout>::LAYOUT, sized_layout(64, 64)); + + // ... with `packed`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(packed)] + struct KL03Packed(NotKnownLayout<AU32>, u8); + + let expected = DstLayout::for_type::<KL03Packed>(); + + assert_eq!(<KL03Packed as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL03Packed as KnownLayout>::LAYOUT, sized_layout(1, 5)); + + // ... with `packed(N)` + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(packed(2))] + struct KL03PackedN(NotKnownLayout<AU32>, u8); + + assert_impl_all!(KL03PackedN: KnownLayout); + + let expected = DstLayout::for_type::<KL03PackedN>(); + + assert_eq!(<KL03PackedN as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL03PackedN as KnownLayout>::LAYOUT, sized_layout(2, 6)); + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | N | Y | N | Y | KL05 | + #[allow(dead_code)] + #[derive(KnownLayout)] + struct KL05<T>(u8, T); + + fn _test_kl05<T>(t: T) -> impl KnownLayout { + KL05(0u8, t) + } + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | N | Y | Y | Y | KL07 | + #[allow(dead_code)] + #[derive(KnownLayout)] + struct KL07<T: KnownLayout>(u8, T); + + fn _test_kl07<T: KnownLayout>(t: T) -> impl KnownLayout { + let _ = KL07(0u8, t); + } + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | Y | N | Y | N | KL10 | + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C)] + struct KL10(NotKnownLayout<AU32>, [u8]); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::<NotKnownLayout<AU32>>(), None) + .extend(<[u8] as KnownLayout>::LAYOUT, None) + .pad_to_align(); + + assert_eq!(<KL10 as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL10 as KnownLayout>::LAYOUT, unsized_layout(4, 1, 4, false)); + + // ...with `align(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, align(64))] + struct KL10Align(NotKnownLayout<AU32>, [u8]); + + let repr_align = NonZeroUsize::new(64); + + let expected = DstLayout::new_zst(repr_align) + .extend(DstLayout::for_type::<NotKnownLayout<AU32>>(), None) + .extend(<[u8] as KnownLayout>::LAYOUT, None) + .pad_to_align(); + + assert_eq!(<KL10Align as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL10Align as KnownLayout>::LAYOUT, unsized_layout(64, 1, 4, false)); + + // ...with `packed`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, packed)] + struct KL10Packed(NotKnownLayout<AU32>, [u8]); + + let repr_packed = NonZeroUsize::new(1); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::<NotKnownLayout<AU32>>(), repr_packed) + .extend(<[u8] as KnownLayout>::LAYOUT, repr_packed) + .pad_to_align(); + + assert_eq!(<KL10Packed as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL10Packed as KnownLayout>::LAYOUT, unsized_layout(1, 1, 4, false)); + + // ...with `packed(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, packed(2))] + struct KL10PackedN(NotKnownLayout<AU32>, [u8]); + + let repr_packed = NonZeroUsize::new(2); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::<NotKnownLayout<AU32>>(), repr_packed) + .extend(<[u8] as KnownLayout>::LAYOUT, repr_packed) + .pad_to_align(); + + assert_eq!(<KL10PackedN as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL10PackedN as KnownLayout>::LAYOUT, unsized_layout(2, 1, 4, false)); + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | Y | N | Y | Y | KL11 | + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C)] + struct KL11(NotKnownLayout<AU64>, u8); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::<NotKnownLayout<AU64>>(), None) + .extend(<u8 as KnownLayout>::LAYOUT, None) + .pad_to_align(); + + assert_eq!(<KL11 as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL11 as KnownLayout>::LAYOUT, sized_layout(8, 16)); + + // ...with `align(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, align(64))] + struct KL11Align(NotKnownLayout<AU64>, u8); + + let repr_align = NonZeroUsize::new(64); + + let expected = DstLayout::new_zst(repr_align) + .extend(DstLayout::for_type::<NotKnownLayout<AU64>>(), None) + .extend(<u8 as KnownLayout>::LAYOUT, None) + .pad_to_align(); + + assert_eq!(<KL11Align as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL11Align as KnownLayout>::LAYOUT, sized_layout(64, 64)); + + // ...with `packed`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, packed)] + struct KL11Packed(NotKnownLayout<AU64>, u8); + + let repr_packed = NonZeroUsize::new(1); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::<NotKnownLayout<AU64>>(), repr_packed) + .extend(<u8 as KnownLayout>::LAYOUT, repr_packed) + .pad_to_align(); + + assert_eq!(<KL11Packed as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL11Packed as KnownLayout>::LAYOUT, sized_layout(1, 9)); + + // ...with `packed(N)`: + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C, packed(2))] + struct KL11PackedN(NotKnownLayout<AU64>, u8); + + let repr_packed = NonZeroUsize::new(2); + + let expected = DstLayout::new_zst(None) + .extend(DstLayout::for_type::<NotKnownLayout<AU64>>(), repr_packed) + .extend(<u8 as KnownLayout>::LAYOUT, repr_packed) + .pad_to_align(); + + assert_eq!(<KL11PackedN as KnownLayout>::LAYOUT, expected); + assert_eq!(<KL11PackedN as KnownLayout>::LAYOUT, sized_layout(2, 10)); + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | Y | Y | Y | N | KL14 | + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C)] + struct KL14<T: ?Sized + KnownLayout>(u8, T); + + fn _test_kl14<T: ?Sized + KnownLayout>(kl: &KL14<T>) { + _assert_kl(kl) + } + + // | `repr(C)`? | generic? | `KnownLayout`? | `Sized`? | Type Name | + // | Y | Y | Y | Y | KL15 | + #[allow(dead_code)] + #[derive(KnownLayout)] + #[repr(C)] + struct KL15<T: KnownLayout>(u8, T); + + fn _test_kl15<T: KnownLayout>(t: T) -> impl KnownLayout { + let _ = KL15(0u8, t); + } + + // Test a variety of combinations of field types: + // - () + // - u8 + // - AU16 + // - [()] + // - [u8] + // - [AU16] + + #[allow(clippy::upper_case_acronyms, dead_code)] + #[derive(KnownLayout)] + #[repr(C)] + struct KLTU<T, U: ?Sized>(T, U); + + assert_eq!(<KLTU<(), ()> as KnownLayout>::LAYOUT, sized_layout(1, 0)); + + assert_eq!(<KLTU<(), u8> as KnownLayout>::LAYOUT, sized_layout(1, 1)); + + assert_eq!(<KLTU<(), AU16> as KnownLayout>::LAYOUT, sized_layout(2, 2)); + + assert_eq!(<KLTU<(), [()]> as KnownLayout>::LAYOUT, unsized_layout(1, 0, 0, false)); + + assert_eq!(<KLTU<(), [u8]> as KnownLayout>::LAYOUT, unsized_layout(1, 1, 0, false)); + + assert_eq!(<KLTU<(), [AU16]> as KnownLayout>::LAYOUT, unsized_layout(2, 2, 0, false)); + + assert_eq!(<KLTU<u8, ()> as KnownLayout>::LAYOUT, sized_layout(1, 1)); + + assert_eq!(<KLTU<u8, u8> as KnownLayout>::LAYOUT, sized_layout(1, 2)); + + assert_eq!(<KLTU<u8, AU16> as KnownLayout>::LAYOUT, sized_layout(2, 4)); + + assert_eq!(<KLTU<u8, [()]> as KnownLayout>::LAYOUT, unsized_layout(1, 0, 1, false)); + + assert_eq!(<KLTU<u8, [u8]> as KnownLayout>::LAYOUT, unsized_layout(1, 1, 1, false)); + + assert_eq!(<KLTU<u8, [AU16]> as KnownLayout>::LAYOUT, unsized_layout(2, 2, 2, false)); + + assert_eq!(<KLTU<AU16, ()> as KnownLayout>::LAYOUT, sized_layout(2, 2)); + + assert_eq!(<KLTU<AU16, u8> as KnownLayout>::LAYOUT, sized_layout(2, 4)); + + assert_eq!(<KLTU<AU16, AU16> as KnownLayout>::LAYOUT, sized_layout(2, 4)); + + assert_eq!(<KLTU<AU16, [()]> as KnownLayout>::LAYOUT, unsized_layout(2, 0, 2, false)); + + assert_eq!(<KLTU<AU16, [u8]> as KnownLayout>::LAYOUT, unsized_layout(2, 1, 2, false)); + + assert_eq!(<KLTU<AU16, [AU16]> as KnownLayout>::LAYOUT, unsized_layout(2, 2, 2, false)); + + // Test a variety of field counts. + + #[derive(KnownLayout)] + #[repr(C)] + struct KLF0; + + assert_eq!(<KLF0 as KnownLayout>::LAYOUT, sized_layout(1, 0)); + + #[derive(KnownLayout)] + #[repr(C)] + struct KLF1([u8]); + + assert_eq!(<KLF1 as KnownLayout>::LAYOUT, unsized_layout(1, 1, 0, true)); + + #[derive(KnownLayout)] + #[repr(C)] + struct KLF2(NotKnownLayout<u8>, [u8]); + + assert_eq!(<KLF2 as KnownLayout>::LAYOUT, unsized_layout(1, 1, 1, false)); + + #[derive(KnownLayout)] + #[repr(C)] + struct KLF3(NotKnownLayout<u8>, NotKnownLayout<AU16>, [u8]); + + assert_eq!(<KLF3 as KnownLayout>::LAYOUT, unsized_layout(2, 1, 4, false)); + + #[derive(KnownLayout)] + #[repr(C)] + struct KLF4(NotKnownLayout<u8>, NotKnownLayout<AU16>, NotKnownLayout<AU32>, [u8]); + + assert_eq!(<KLF4 as KnownLayout>::LAYOUT, unsized_layout(4, 1, 8, false)); + } + + #[test] + fn test_object_safety() { + fn _takes_immutable(_: &dyn Immutable) {} + fn _takes_unaligned(_: &dyn Unaligned) {} + } + + #[test] + fn test_from_zeros_only() { + // Test types that implement `FromZeros` but not `FromBytes`. + + assert!(!bool::new_zeroed()); + assert_eq!(char::new_zeroed(), '\0'); + + #[cfg(feature = "alloc")] + { + assert_eq!(bool::new_box_zeroed(), Ok(Box::new(false))); + assert_eq!(char::new_box_zeroed(), Ok(Box::new('\0'))); + + assert_eq!( + <[bool]>::new_box_zeroed_with_elems(3).unwrap().as_ref(), + [false, false, false] + ); + assert_eq!( + <[char]>::new_box_zeroed_with_elems(3).unwrap().as_ref(), + ['\0', '\0', '\0'] + ); + + assert_eq!(bool::new_vec_zeroed(3).unwrap().as_ref(), [false, false, false]); + assert_eq!(char::new_vec_zeroed(3).unwrap().as_ref(), ['\0', '\0', '\0']); + } + + let mut string = "hello".to_string(); + let s: &mut str = string.as_mut(); + assert_eq!(s, "hello"); + s.zero(); + assert_eq!(s, "\0\0\0\0\0"); + } + + #[test] + fn test_zst_count_preserved() { + // Test that, when an explicit count is provided to for a type with a + // ZST trailing slice element, that count is preserved. This is + // important since, for such types, all element counts result in objects + // of the same size, and so the correct behavior is ambiguous. However, + // preserving the count as requested by the user is the behavior that we + // document publicly. + + // FromZeros methods + #[cfg(feature = "alloc")] + assert_eq!(<[()]>::new_box_zeroed_with_elems(3).unwrap().len(), 3); + #[cfg(feature = "alloc")] + assert_eq!(<()>::new_vec_zeroed(3).unwrap().len(), 3); + + // FromBytes methods + assert_eq!(<[()]>::ref_from_bytes_with_elems(&[][..], 3).unwrap().len(), 3); + assert_eq!(<[()]>::ref_from_prefix_with_elems(&[][..], 3).unwrap().0.len(), 3); + assert_eq!(<[()]>::ref_from_suffix_with_elems(&[][..], 3).unwrap().1.len(), 3); + assert_eq!(<[()]>::mut_from_bytes_with_elems(&mut [][..], 3).unwrap().len(), 3); + assert_eq!(<[()]>::mut_from_prefix_with_elems(&mut [][..], 3).unwrap().0.len(), 3); + assert_eq!(<[()]>::mut_from_suffix_with_elems(&mut [][..], 3).unwrap().1.len(), 3); + } + + #[test] + fn test_read_write() { + const VAL: u64 = 0x12345678; + #[cfg(target_endian = "big")] + const VAL_BYTES: [u8; 8] = VAL.to_be_bytes(); + #[cfg(target_endian = "little")] + const VAL_BYTES: [u8; 8] = VAL.to_le_bytes(); + const ZEROS: [u8; 8] = [0u8; 8]; + + // Test `FromBytes::{read_from, read_from_prefix, read_from_suffix}`. + + assert_eq!(u64::read_from_bytes(&VAL_BYTES[..]), Ok(VAL)); + // The first 8 bytes are from `VAL_BYTES` and the second 8 bytes are all + // zeros. + let bytes_with_prefix: [u8; 16] = transmute!([VAL_BYTES, [0; 8]]); + assert_eq!(u64::read_from_prefix(&bytes_with_prefix[..]), Ok((VAL, &ZEROS[..]))); + assert_eq!(u64::read_from_suffix(&bytes_with_prefix[..]), Ok((&VAL_BYTES[..], 0))); + // The first 8 bytes are all zeros and the second 8 bytes are from + // `VAL_BYTES` + let bytes_with_suffix: [u8; 16] = transmute!([[0; 8], VAL_BYTES]); + assert_eq!(u64::read_from_prefix(&bytes_with_suffix[..]), Ok((0, &VAL_BYTES[..]))); + assert_eq!(u64::read_from_suffix(&bytes_with_suffix[..]), Ok((&ZEROS[..], VAL))); + + // Test `IntoBytes::{write_to, write_to_prefix, write_to_suffix}`. + + let mut bytes = [0u8; 8]; + assert_eq!(VAL.write_to(&mut bytes[..]), Ok(())); + assert_eq!(bytes, VAL_BYTES); + let mut bytes = [0u8; 16]; + assert_eq!(VAL.write_to_prefix(&mut bytes[..]), Ok(())); + let want: [u8; 16] = transmute!([VAL_BYTES, [0; 8]]); + assert_eq!(bytes, want); + let mut bytes = [0u8; 16]; + assert_eq!(VAL.write_to_suffix(&mut bytes[..]), Ok(())); + let want: [u8; 16] = transmute!([[0; 8], VAL_BYTES]); + assert_eq!(bytes, want); + } + + #[test] + #[cfg(feature = "std")] + fn test_read_io_with_padding_soundness() { + // This test is designed to exhibit potential UB in + // `FromBytes::read_from_io`. (see #2319, #2320). + + // On most platforms (where `align_of::<u16>() == 2`), `WithPadding` + // will have inter-field padding between `x` and `y`. + #[derive(FromBytes)] + #[repr(C)] + struct WithPadding { + x: u8, + y: u16, + } + struct ReadsInRead; + impl std::io::Read for ReadsInRead { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> { + // This body branches on every byte of `buf`, ensuring that it + // exhibits UB if any byte of `buf` is uninitialized. + if buf.iter().all(|&x| x == 0) { + Ok(buf.len()) + } else { + buf.iter_mut().for_each(|x| *x = 0); + Ok(buf.len()) + } + } + } + assert!(matches!(WithPadding::read_from_io(ReadsInRead), Ok(WithPadding { x: 0, y: 0 }))); + } + + #[test] + #[cfg(feature = "std")] + fn test_read_write_io() { + let mut long_buffer = [0, 0, 0, 0]; + assert!(matches!(u16::MAX.write_to_io(&mut long_buffer[..]), Ok(()))); + assert_eq!(long_buffer, [255, 255, 0, 0]); + assert!(matches!(u16::read_from_io(&long_buffer[..]), Ok(u16::MAX))); + + let mut short_buffer = [0, 0]; + assert!(u32::MAX.write_to_io(&mut short_buffer[..]).is_err()); + assert_eq!(short_buffer, [255, 255]); + assert!(u32::read_from_io(&short_buffer[..]).is_err()); + } + + #[test] + fn test_try_from_bytes_try_read_from() { + assert_eq!(<bool as TryFromBytes>::try_read_from_bytes(&[0]), Ok(false)); + assert_eq!(<bool as TryFromBytes>::try_read_from_bytes(&[1]), Ok(true)); + + assert_eq!(<bool as TryFromBytes>::try_read_from_prefix(&[0, 2]), Ok((false, &[2][..]))); + assert_eq!(<bool as TryFromBytes>::try_read_from_prefix(&[1, 2]), Ok((true, &[2][..]))); + + assert_eq!(<bool as TryFromBytes>::try_read_from_suffix(&[2, 0]), Ok((&[2][..], false))); + assert_eq!(<bool as TryFromBytes>::try_read_from_suffix(&[2, 1]), Ok((&[2][..], true))); + + // If we don't pass enough bytes, it fails. + assert!(matches!( + <u8 as TryFromBytes>::try_read_from_bytes(&[]), + Err(TryReadError::Size(_)) + )); + assert!(matches!( + <u8 as TryFromBytes>::try_read_from_prefix(&[]), + Err(TryReadError::Size(_)) + )); + assert!(matches!( + <u8 as TryFromBytes>::try_read_from_suffix(&[]), + Err(TryReadError::Size(_)) + )); + + // If we pass too many bytes, it fails. + assert!(matches!( + <u8 as TryFromBytes>::try_read_from_bytes(&[0, 0]), + Err(TryReadError::Size(_)) + )); + + // If we pass an invalid value, it fails. + assert!(matches!( + <bool as TryFromBytes>::try_read_from_bytes(&[2]), + Err(TryReadError::Validity(_)) + )); + assert!(matches!( + <bool as TryFromBytes>::try_read_from_prefix(&[2, 0]), + Err(TryReadError::Validity(_)) + )); + assert!(matches!( + <bool as TryFromBytes>::try_read_from_suffix(&[0, 2]), + Err(TryReadError::Validity(_)) + )); + + // Reading from a misaligned buffer should still succeed. Since `AU64`'s + // alignment is 8, and since we read from two adjacent addresses one + // byte apart, it is guaranteed that at least one of them (though + // possibly both) will be misaligned. + let bytes: [u8; 9] = [0, 0, 0, 0, 0, 0, 0, 0, 0]; + assert_eq!(<AU64 as TryFromBytes>::try_read_from_bytes(&bytes[..8]), Ok(AU64(0))); + assert_eq!(<AU64 as TryFromBytes>::try_read_from_bytes(&bytes[1..9]), Ok(AU64(0))); + + assert_eq!( + <AU64 as TryFromBytes>::try_read_from_prefix(&bytes[..8]), + Ok((AU64(0), &[][..])) + ); + assert_eq!( + <AU64 as TryFromBytes>::try_read_from_prefix(&bytes[1..9]), + Ok((AU64(0), &[][..])) + ); + + assert_eq!( + <AU64 as TryFromBytes>::try_read_from_suffix(&bytes[..8]), + Ok((&[][..], AU64(0))) + ); + assert_eq!( + <AU64 as TryFromBytes>::try_read_from_suffix(&bytes[1..9]), + Ok((&[][..], AU64(0))) + ); + } + + #[test] + fn test_ref_from_mut_from_bytes() { + // Test `FromBytes::{ref_from_bytes, mut_from_bytes}{,_prefix,Suffix}` + // success cases. Exhaustive coverage for these methods is covered by + // the `Ref` tests above, which these helper methods defer to. + + let mut buf = + Align::<[u8; 16], AU64>::new([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + + assert_eq!( + AU64::ref_from_bytes(&buf.t[8..]).unwrap().0.to_ne_bytes(), + [8, 9, 10, 11, 12, 13, 14, 15] + ); + let suffix = AU64::mut_from_bytes(&mut buf.t[8..]).unwrap(); + suffix.0 = 0x0101010101010101; + // The `[u8:9]` is a non-half size of the full buffer, which would catch + // `from_prefix` having the same implementation as `from_suffix` (issues #506, #511). + assert_eq!( + <[u8; 9]>::ref_from_suffix(&buf.t[..]).unwrap(), + (&[0, 1, 2, 3, 4, 5, 6][..], &[7u8, 1, 1, 1, 1, 1, 1, 1, 1]) + ); + let (prefix, suffix) = AU64::mut_from_suffix(&mut buf.t[1..]).unwrap(); + assert_eq!(prefix, &mut [1u8, 2, 3, 4, 5, 6, 7][..]); + suffix.0 = 0x0202020202020202; + let (prefix, suffix) = <[u8; 10]>::mut_from_suffix(&mut buf.t[..]).unwrap(); + assert_eq!(prefix, &mut [0u8, 1, 2, 3, 4, 5][..]); + suffix[0] = 42; + assert_eq!( + <[u8; 9]>::ref_from_prefix(&buf.t[..]).unwrap(), + (&[0u8, 1, 2, 3, 4, 5, 42, 7, 2], &[2u8, 2, 2, 2, 2, 2, 2][..]) + ); + <[u8; 2]>::mut_from_prefix(&mut buf.t[..]).unwrap().0[1] = 30; + assert_eq!(buf.t, [0, 30, 2, 3, 4, 5, 42, 7, 2, 2, 2, 2, 2, 2, 2, 2]); + } + + #[test] + fn test_ref_from_mut_from_bytes_error() { + // Test `FromBytes::{ref_from_bytes, mut_from_bytes}{,_prefix,Suffix}` + // error cases. + + // Fail because the buffer is too large. + let mut buf = Align::<[u8; 16], AU64>::default(); + // `buf.t` should be aligned to 8, so only the length check should fail. + assert!(AU64::ref_from_bytes(&buf.t[..]).is_err()); + assert!(AU64::mut_from_bytes(&mut buf.t[..]).is_err()); + assert!(<[u8; 8]>::ref_from_bytes(&buf.t[..]).is_err()); + assert!(<[u8; 8]>::mut_from_bytes(&mut buf.t[..]).is_err()); + + // Fail because the buffer is too small. + let mut buf = Align::<[u8; 4], AU64>::default(); + assert!(AU64::ref_from_bytes(&buf.t[..]).is_err()); + assert!(AU64::mut_from_bytes(&mut buf.t[..]).is_err()); + assert!(<[u8; 8]>::ref_from_bytes(&buf.t[..]).is_err()); + assert!(<[u8; 8]>::mut_from_bytes(&mut buf.t[..]).is_err()); + assert!(AU64::ref_from_prefix(&buf.t[..]).is_err()); + assert!(AU64::mut_from_prefix(&mut buf.t[..]).is_err()); + assert!(AU64::ref_from_suffix(&buf.t[..]).is_err()); + assert!(AU64::mut_from_suffix(&mut buf.t[..]).is_err()); + assert!(<[u8; 8]>::ref_from_prefix(&buf.t[..]).is_err()); + assert!(<[u8; 8]>::mut_from_prefix(&mut buf.t[..]).is_err()); + assert!(<[u8; 8]>::ref_from_suffix(&buf.t[..]).is_err()); + assert!(<[u8; 8]>::mut_from_suffix(&mut buf.t[..]).is_err()); + + // Fail because the alignment is insufficient. + let mut buf = Align::<[u8; 13], AU64>::default(); + assert!(AU64::ref_from_bytes(&buf.t[1..]).is_err()); + assert!(AU64::mut_from_bytes(&mut buf.t[1..]).is_err()); + assert!(AU64::ref_from_bytes(&buf.t[1..]).is_err()); + assert!(AU64::mut_from_bytes(&mut buf.t[1..]).is_err()); + assert!(AU64::ref_from_prefix(&buf.t[1..]).is_err()); + assert!(AU64::mut_from_prefix(&mut buf.t[1..]).is_err()); + assert!(AU64::ref_from_suffix(&buf.t[..]).is_err()); + assert!(AU64::mut_from_suffix(&mut buf.t[..]).is_err()); + } + + #[test] + fn test_to_methods() { + /// Run a series of tests by calling `IntoBytes` methods on `t`. + /// + /// `bytes` is the expected byte sequence returned from `t.as_bytes()` + /// before `t` has been modified. `post_mutation` is the expected + /// sequence returned from `t.as_bytes()` after `t.as_mut_bytes()[0]` + /// has had its bits flipped (by applying `^= 0xFF`). + /// + /// `N` is the size of `t` in bytes. + fn test<T: FromBytes + IntoBytes + Immutable + Debug + Eq + ?Sized, const N: usize>( + t: &mut T, + bytes: &[u8], + post_mutation: &T, + ) { + // Test that we can access the underlying bytes, and that we get the + // right bytes and the right number of bytes. + assert_eq!(t.as_bytes(), bytes); + + // Test that changes to the underlying byte slices are reflected in + // the original object. + t.as_mut_bytes()[0] ^= 0xFF; + assert_eq!(t, post_mutation); + t.as_mut_bytes()[0] ^= 0xFF; + + // `write_to` rejects slices that are too small or too large. + assert!(t.write_to(&mut vec![0; N - 1][..]).is_err()); + assert!(t.write_to(&mut vec![0; N + 1][..]).is_err()); + + // `write_to` works as expected. + let mut bytes = [0; N]; + assert_eq!(t.write_to(&mut bytes[..]), Ok(())); + assert_eq!(bytes, t.as_bytes()); + + // `write_to_prefix` rejects slices that are too small. + assert!(t.write_to_prefix(&mut vec![0; N - 1][..]).is_err()); + + // `write_to_prefix` works with exact-sized slices. + let mut bytes = [0; N]; + assert_eq!(t.write_to_prefix(&mut bytes[..]), Ok(())); + assert_eq!(bytes, t.as_bytes()); + + // `write_to_prefix` works with too-large slices, and any bytes past + // the prefix aren't modified. + let mut too_many_bytes = vec![0; N + 1]; + too_many_bytes[N] = 123; + assert_eq!(t.write_to_prefix(&mut too_many_bytes[..]), Ok(())); + assert_eq!(&too_many_bytes[..N], t.as_bytes()); + assert_eq!(too_many_bytes[N], 123); + + // `write_to_suffix` rejects slices that are too small. + assert!(t.write_to_suffix(&mut vec![0; N - 1][..]).is_err()); + + // `write_to_suffix` works with exact-sized slices. + let mut bytes = [0; N]; + assert_eq!(t.write_to_suffix(&mut bytes[..]), Ok(())); + assert_eq!(bytes, t.as_bytes()); + + // `write_to_suffix` works with too-large slices, and any bytes + // before the suffix aren't modified. + let mut too_many_bytes = vec![0; N + 1]; + too_many_bytes[0] = 123; + assert_eq!(t.write_to_suffix(&mut too_many_bytes[..]), Ok(())); + assert_eq!(&too_many_bytes[1..], t.as_bytes()); + assert_eq!(too_many_bytes[0], 123); + } + + #[derive(Debug, Eq, PartialEq, FromBytes, IntoBytes, Immutable)] + #[repr(C)] + struct Foo { + a: u32, + b: Wrapping<u32>, + c: Option<NonZeroU32>, + } + + let expected_bytes: Vec<u8> = if cfg!(target_endian = "little") { + vec![1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] + } else { + vec![0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0] + }; + let post_mutation_expected_a = + if cfg!(target_endian = "little") { 0x00_00_00_FE } else { 0xFF_00_00_01 }; + test::<_, 12>( + &mut Foo { a: 1, b: Wrapping(2), c: None }, + expected_bytes.as_bytes(), + &Foo { a: post_mutation_expected_a, b: Wrapping(2), c: None }, + ); + test::<_, 3>( + Unsized::from_mut_slice(&mut [1, 2, 3]), + &[1, 2, 3], + Unsized::from_mut_slice(&mut [0xFE, 2, 3]), + ); + } + + #[test] + fn test_array() { + #[derive(FromBytes, IntoBytes, Immutable)] + #[repr(C)] + struct Foo { + a: [u16; 33], + } + + let foo = Foo { a: [0xFFFF; 33] }; + let expected = [0xFFu8; 66]; + assert_eq!(foo.as_bytes(), &expected[..]); + } + + #[test] + fn test_new_zeroed() { + assert!(!bool::new_zeroed()); + assert_eq!(u64::new_zeroed(), 0); + // This test exists in order to exercise unsafe code, especially when + // running under Miri. + #[allow(clippy::unit_cmp)] + { + assert_eq!(<()>::new_zeroed(), ()); + } + } + + #[test] + fn test_transparent_packed_generic_struct() { + #[derive(IntoBytes, FromBytes, Unaligned)] + #[repr(transparent)] + #[allow(dead_code)] // We never construct this type + struct Foo<T> { + _t: T, + _phantom: PhantomData<()>, + } + + assert_impl_all!(Foo<u32>: FromZeros, FromBytes, IntoBytes); + assert_impl_all!(Foo<u8>: Unaligned); + + #[derive(IntoBytes, FromBytes, Unaligned)] + #[repr(C, packed)] + #[allow(dead_code)] // We never construct this type + struct Bar<T, U> { + _t: T, + _u: U, + } + + assert_impl_all!(Bar<u8, AU64>: FromZeros, FromBytes, IntoBytes, Unaligned); + } + + #[cfg(feature = "alloc")] + mod alloc { + use super::*; + + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[test] + fn test_extend_vec_zeroed() { + // Test extending when there is an existing allocation. + let mut v = vec![100u16, 200, 300]; + FromZeros::extend_vec_zeroed(&mut v, 3).unwrap(); + assert_eq!(v.len(), 6); + assert_eq!(&*v, &[100, 200, 300, 0, 0, 0]); + drop(v); + + // Test extending when there is no existing allocation. + let mut v: Vec<u64> = Vec::new(); + FromZeros::extend_vec_zeroed(&mut v, 3).unwrap(); + assert_eq!(v.len(), 3); + assert_eq!(&*v, &[0, 0, 0]); + drop(v); + } + + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[test] + fn test_extend_vec_zeroed_zst() { + // Test extending when there is an existing (fake) allocation. + let mut v = vec![(), (), ()]; + <()>::extend_vec_zeroed(&mut v, 3).unwrap(); + assert_eq!(v.len(), 6); + assert_eq!(&*v, &[(), (), (), (), (), ()]); + drop(v); + + // Test extending when there is no existing (fake) allocation. + let mut v: Vec<()> = Vec::new(); + <()>::extend_vec_zeroed(&mut v, 3).unwrap(); + assert_eq!(&*v, &[(), (), ()]); + drop(v); + } + + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[test] + fn test_insert_vec_zeroed() { + // Insert at start (no existing allocation). + let mut v: Vec<u64> = Vec::new(); + u64::insert_vec_zeroed(&mut v, 0, 2).unwrap(); + assert_eq!(v.len(), 2); + assert_eq!(&*v, &[0, 0]); + drop(v); + + // Insert at start. + let mut v = vec![100u64, 200, 300]; + u64::insert_vec_zeroed(&mut v, 0, 2).unwrap(); + assert_eq!(v.len(), 5); + assert_eq!(&*v, &[0, 0, 100, 200, 300]); + drop(v); + + // Insert at middle. + let mut v = vec![100u64, 200, 300]; + u64::insert_vec_zeroed(&mut v, 1, 1).unwrap(); + assert_eq!(v.len(), 4); + assert_eq!(&*v, &[100, 0, 200, 300]); + drop(v); + + // Insert at end. + let mut v = vec![100u64, 200, 300]; + u64::insert_vec_zeroed(&mut v, 3, 1).unwrap(); + assert_eq!(v.len(), 4); + assert_eq!(&*v, &[100, 200, 300, 0]); + drop(v); + } + + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + #[test] + fn test_insert_vec_zeroed_zst() { + // Insert at start (no existing fake allocation). + let mut v: Vec<()> = Vec::new(); + <()>::insert_vec_zeroed(&mut v, 0, 2).unwrap(); + assert_eq!(v.len(), 2); + assert_eq!(&*v, &[(), ()]); + drop(v); + + // Insert at start. + let mut v = vec![(), (), ()]; + <()>::insert_vec_zeroed(&mut v, 0, 2).unwrap(); + assert_eq!(v.len(), 5); + assert_eq!(&*v, &[(), (), (), (), ()]); + drop(v); + + // Insert at middle. + let mut v = vec![(), (), ()]; + <()>::insert_vec_zeroed(&mut v, 1, 1).unwrap(); + assert_eq!(v.len(), 4); + assert_eq!(&*v, &[(), (), (), ()]); + drop(v); + + // Insert at end. + let mut v = vec![(), (), ()]; + <()>::insert_vec_zeroed(&mut v, 3, 1).unwrap(); + assert_eq!(v.len(), 4); + assert_eq!(&*v, &[(), (), (), ()]); + drop(v); + } + + #[test] + fn test_new_box_zeroed() { + assert_eq!(u64::new_box_zeroed(), Ok(Box::new(0))); + } + + #[test] + fn test_new_box_zeroed_array() { + drop(<[u32; 0x1000]>::new_box_zeroed()); + } + + #[test] + fn test_new_box_zeroed_zst() { + // This test exists in order to exercise unsafe code, especially + // when running under Miri. + #[allow(clippy::unit_cmp)] + { + assert_eq!(<()>::new_box_zeroed(), Ok(Box::new(()))); + } + } + + #[test] + fn test_new_box_zeroed_with_elems() { + let mut s: Box<[u64]> = <[u64]>::new_box_zeroed_with_elems(3).unwrap(); + assert_eq!(s.len(), 3); + assert_eq!(&*s, &[0, 0, 0]); + s[1] = 3; + assert_eq!(&*s, &[0, 3, 0]); + } + + #[test] + fn test_new_box_zeroed_with_elems_empty() { + let s: Box<[u64]> = <[u64]>::new_box_zeroed_with_elems(0).unwrap(); + assert_eq!(s.len(), 0); + } + + #[test] + fn test_new_box_zeroed_with_elems_zst() { + let mut s: Box<[()]> = <[()]>::new_box_zeroed_with_elems(3).unwrap(); + assert_eq!(s.len(), 3); + assert!(s.get(10).is_none()); + // This test exists in order to exercise unsafe code, especially + // when running under Miri. + #[allow(clippy::unit_cmp)] + { + assert_eq!(s[1], ()); + } + s[2] = (); + } + + #[test] + fn test_new_box_zeroed_with_elems_zst_empty() { + let s: Box<[()]> = <[()]>::new_box_zeroed_with_elems(0).unwrap(); + assert_eq!(s.len(), 0); + } + + #[test] + fn new_box_zeroed_with_elems_errors() { + assert_eq!(<[u16]>::new_box_zeroed_with_elems(usize::MAX), Err(AllocError)); + + let max = <usize as core::convert::TryFrom<_>>::try_from(isize::MAX).unwrap(); + assert_eq!( + <[u16]>::new_box_zeroed_with_elems((max / mem::size_of::<u16>()) + 1), + Err(AllocError) + ); + } + } + + #[test] + #[allow(deprecated)] + fn test_deprecated_from_bytes() { + let val = 0u32; + let bytes = val.as_bytes(); + + assert!(u32::ref_from(bytes).is_some()); + // mut_from needs mut bytes + let mut val = 0u32; + let mut_bytes = val.as_mut_bytes(); + assert!(u32::mut_from(mut_bytes).is_some()); + + assert!(u32::read_from(bytes).is_some()); + + let (slc, rest) = <u32>::slice_from_prefix(bytes, 0).unwrap(); + assert!(slc.is_empty()); + assert_eq!(rest.len(), 4); + + let (rest, slc) = <u32>::slice_from_suffix(bytes, 0).unwrap(); + assert!(slc.is_empty()); + assert_eq!(rest.len(), 4); + + let (slc, rest) = <u32>::mut_slice_from_prefix(mut_bytes, 0).unwrap(); + assert!(slc.is_empty()); + assert_eq!(rest.len(), 4); + + let (rest, slc) = <u32>::mut_slice_from_suffix(mut_bytes, 0).unwrap(); + assert!(slc.is_empty()); + assert_eq!(rest.len(), 4); + } + + #[test] + fn test_try_ref_from_prefix_suffix() { + use crate::util::testutil::Align; + let bytes = &Align::<[u8; 4], u32>::new([0u8; 4]).t[..]; + let (r, rest): (&u32, &[u8]) = u32::try_ref_from_prefix(bytes).unwrap(); + assert_eq!(*r, 0); + assert_eq!(rest.len(), 0); + + let (rest, r): (&[u8], &u32) = u32::try_ref_from_suffix(bytes).unwrap(); + assert_eq!(*r, 0); + assert_eq!(rest.len(), 0); + } + + #[test] + fn test_raw_dangling() { + use crate::util::AsAddress; + let ptr: NonNull<u32> = u32::raw_dangling(); + assert_eq!(AsAddress::addr(ptr), 1); + + let ptr: NonNull<[u32]> = <[u32]>::raw_dangling(); + assert_eq!(AsAddress::addr(ptr), 1); + } + + #[test] + fn test_try_ref_from_prefix_with_elems() { + use crate::util::testutil::Align; + let bytes = &Align::<[u8; 8], u32>::new([0u8; 8]).t[..]; + let (r, rest): (&[u32], &[u8]) = <[u32]>::try_ref_from_prefix_with_elems(bytes, 2).unwrap(); + assert_eq!(r.len(), 2); + assert_eq!(rest.len(), 0); + } + + #[test] + fn test_try_ref_from_suffix_with_elems() { + use crate::util::testutil::Align; + let bytes = &Align::<[u8; 8], u32>::new([0u8; 8]).t[..]; + let (rest, r): (&[u8], &[u32]) = <[u32]>::try_ref_from_suffix_with_elems(bytes, 2).unwrap(); + assert_eq!(r.len(), 2); + assert_eq!(rest.len(), 0); + } +} diff --git a/rust/zerocopy/src/macros.rs b/rust/zerocopy/src/macros.rs new file mode 100644 index 000000000000..ec67c03a44fc --- /dev/null +++ b/rust/zerocopy/src/macros.rs @@ -0,0 +1,1825 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License <LICENSE-BSD or +// https://opensource.org/license/bsd-2-clause>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +/// Safely transmutes a value of one type to a value of another type of the same +/// size. +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// const fn transmute<Src, Dst>(src: Src) -> Dst +/// where +/// Src: IntoBytes, +/// Dst: FromBytes, +/// size_of::<Src>() == size_of::<Dst>(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// However, unlike a function, this macro can only be invoked when the types of +/// `Src` and `Dst` are completely concrete. The types `Src` and `Dst` are +/// inferred from the calling context; they cannot be explicitly specified in +/// the macro invocation. +/// +/// Note that the `Src` produced by the expression `$e` will *not* be dropped. +/// Semantically, its bits will be copied into a new value of type `Dst`, the +/// original `Src` will be forgotten, and the value of type `Dst` will be +/// returned. +/// +/// # `#![allow(shrink)]` +/// +/// If `#![allow(shrink)]` is provided, `transmute!` additionally supports +/// transmutations that shrink the size of the value; e.g.: +/// +/// ``` +/// # use zerocopy::transmute; +/// let u: u32 = transmute!(#![allow(shrink)] 0u64); +/// assert_eq!(u, 0u32); +/// ``` +/// +/// # Examples +/// +/// ``` +/// # use zerocopy::transmute; +/// let one_dimensional: [u8; 8] = [0, 1, 2, 3, 4, 5, 6, 7]; +/// +/// let two_dimensional: [[u8; 4]; 2] = transmute!(one_dimensional); +/// +/// assert_eq!(two_dimensional, [[0, 1, 2, 3], [4, 5, 6, 7]]); +/// ``` +/// +/// # Use in `const` contexts +/// +/// This macro can be invoked in `const` contexts. +/// +#[doc = codegen_section!( + header = "h2", + bench = "transmute", + format = "coco_static_size", +)] +#[macro_export] +macro_rules! transmute { + // NOTE: This must be a macro (rather than a function with trait bounds) + // because there's no way, in a generic context, to enforce that two types + // have the same size. `core::mem::transmute` uses compiler magic to enforce + // this so long as the types are concrete. + (#![allow(shrink)] $e:expr) => {{ + let mut e = $e; + if false { + // This branch, though never taken, ensures that the type of `e` is + // `IntoBytes` and that the type of the outer macro invocation + // expression is `FromBytes`. + + fn transmute<Src, Dst>(src: Src) -> Dst + where + Src: $crate::IntoBytes, + Dst: $crate::FromBytes, + { + let _ = src; + loop {} + } + loop {} + #[allow(unreachable_code)] + transmute(e) + } else { + use $crate::util::macro_util::core_reexport::mem::ManuallyDrop; + + // NOTE: `repr(packed)` is important! It ensures that the size of + // `Transmute` won't be rounded up to accommodate `Src`'s or `Dst`'s + // alignment, which would break the size comparison logic below. + // + // As an example of why this is problematic, consider `Src = [u8; + // 5]`, `Dst = u32`. The total size of `Transmute<Src, Dst>` would + // be 8, and so we would reject a `[u8; 5]` to `u32` transmute as + // being size-increasing, which it isn't. + #[repr(C, packed)] + union Transmute<Src, Dst> { + src: ManuallyDrop<Src>, + dst: ManuallyDrop<Dst>, + } + + // SAFETY: `Transmute` is a `repr(C)` union whose `src` field has + // type `ManuallyDrop<Src>`. Thus, the `src` field starts at byte + // offset 0 within `Transmute` [1]. `ManuallyDrop<T>` has the same + // layout and bit validity as `T`, so it is sound to transmute `Src` + // to `Transmute`. + // + // [1] https://doc.rust-lang.org/1.85.0/reference/type-layout.html#reprc-unions + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/mem/struct.ManuallyDrop.html: + // + // `ManuallyDrop<T>` is guaranteed to have the same layout and bit + // validity as `T` + let u: Transmute<_, _> = unsafe { + // Clippy: We can't annotate the types; this macro is designed + // to infer the types from the calling context. + #[allow(clippy::missing_transmute_annotations)] + $crate::util::macro_util::core_reexport::mem::transmute(e) + }; + + if false { + // SAFETY: This code is never executed. + e = ManuallyDrop::into_inner(unsafe { u.src }); + // Suppress the `unused_assignments` lint on the previous line. + let _ = e; + loop {} + } else { + // SAFETY: Per the safety comment on `let u` above, the `dst` + // field in `Transmute` starts at byte offset 0, and has the + // same layout and bit validity as `Dst`. + // + // Transmuting `Src` to `Transmute<Src, Dst>` above using + // `core::mem::transmute` ensures that `size_of::<Src>() == + // size_of::<Transmute<Src, Dst>>()`. A `#[repr(C, packed)]` + // union has the maximum size of all of its fields [1], so this + // is equivalent to `size_of::<Src>() >= size_of::<Dst>()`. + // + // The outer `if`'s `false` branch ensures that `Src: IntoBytes` + // and `Dst: FromBytes`. This, combined with the size bound, + // ensures that this transmute is sound. + // + // [1] Per https://doc.rust-lang.org/1.85.0/reference/type-layout.html#reprc-unions: + // + // The union will have a size of the maximum size of all of + // its fields rounded to its alignment + let dst = unsafe { u.dst }; + $crate::util::macro_util::must_use(ManuallyDrop::into_inner(dst)) + } + } + }}; + ($e:expr) => {{ + let e = $e; + if false { + // This branch, though never taken, ensures that the type of `e` is + // `IntoBytes` and that the type of the outer macro invocation + // expression is `FromBytes`. + + fn transmute<Src, Dst>(src: Src) -> Dst + where + Src: $crate::IntoBytes, + Dst: $crate::FromBytes, + { + let _ = src; + loop {} + } + loop {} + #[allow(unreachable_code)] + transmute(e) + } else { + // SAFETY: `core::mem::transmute` ensures that the type of `e` and + // the type of this macro invocation expression have the same size. + // We know this transmute is safe thanks to the `IntoBytes` and + // `FromBytes` bounds enforced by the `false` branch. + let u = unsafe { + // Clippy: We can't annotate the types; this macro is designed + // to infer the types from the calling context. + #[allow(clippy::missing_transmute_annotations, unnecessary_transmutes)] + $crate::util::macro_util::core_reexport::mem::transmute(e) + }; + $crate::util::macro_util::must_use(u) + } + }}; +} + +/// Safely transmutes a mutable or immutable reference of one type to an +/// immutable reference of another type of the same size and compatible +/// alignment. +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// fn transmute_ref<'src, 'dst, Src, Dst>(src: &'src Src) -> &'dst Dst +/// where +/// 'src: 'dst, +/// Src: IntoBytes + Immutable + ?Sized, +/// Dst: FromBytes + Immutable + ?Sized, +/// align_of::<Src>() >= align_of::<Dst>(), +/// size_compatible::<Src, Dst>(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The types `Src` and `Dst` are inferred from the calling context; they cannot +/// be explicitly specified in the macro invocation. +/// +/// # Size compatibility +/// +/// `transmute_ref!` supports transmuting between `Sized` types, between unsized +/// (i.e., `?Sized`) types, and from a `Sized` type to an unsized type. It +/// supports any transmutation that preserves the number of bytes of the +/// referent, even if doing so requires updating the metadata stored in an +/// unsized "fat" reference: +/// +/// ``` +/// # use zerocopy::transmute_ref; +/// # use core::mem::size_of_val; // Not in the prelude on our MSRV +/// let src: &[[u8; 2]] = &[[0, 1], [2, 3]][..]; +/// let dst: &[u8] = transmute_ref!(src); +/// +/// assert_eq!(src.len(), 2); +/// assert_eq!(dst.len(), 4); +/// assert_eq!(dst, [0, 1, 2, 3]); +/// assert_eq!(size_of_val(src), size_of_val(dst)); +/// ``` +/// +/// # Errors +/// +/// Violations of the alignment and size compatibility checks are detected +/// *after* the compiler performs monomorphization. This has two important +/// consequences. +/// +/// First, it means that generic code will *never* fail these conditions: +/// +/// ``` +/// # use zerocopy::{transmute_ref, FromBytes, IntoBytes, Immutable}; +/// fn transmute_ref<Src, Dst>(src: &Src) -> &Dst +/// where +/// Src: IntoBytes + Immutable, +/// Dst: FromBytes + Immutable, +/// { +/// transmute_ref!(src) +/// } +/// ``` +/// +/// Instead, failures will only be detected once generic code is instantiated +/// with concrete types: +/// +/// ```compile_fail,E0080 +/// # use zerocopy::{transmute_ref, FromBytes, IntoBytes, Immutable}; +/// # +/// # fn transmute_ref<Src, Dst>(src: &Src) -> &Dst +/// # where +/// # Src: IntoBytes + Immutable, +/// # Dst: FromBytes + Immutable, +/// # { +/// # transmute_ref!(src) +/// # } +/// let src: &u16 = &0; +/// let dst: &u8 = transmute_ref(src); +/// ``` +/// +/// Second, the fact that violations are detected after monomorphization means +/// that `cargo check` will usually not detect errors, even when types are +/// concrete. Instead, `cargo build` must be used to detect such errors. +/// +/// # Examples +/// +/// Transmuting between `Sized` types: +/// +/// ``` +/// # use zerocopy::transmute_ref; +/// let one_dimensional: [u8; 8] = [0, 1, 2, 3, 4, 5, 6, 7]; +/// +/// let two_dimensional: &[[u8; 4]; 2] = transmute_ref!(&one_dimensional); +/// +/// assert_eq!(two_dimensional, &[[0, 1, 2, 3], [4, 5, 6, 7]]); +/// ``` +/// +/// Transmuting between unsized types: +/// +/// ``` +/// # use {zerocopy::*, zerocopy_derive::*}; +/// # type u16 = zerocopy::byteorder::native_endian::U16; +/// # type u32 = zerocopy::byteorder::native_endian::U32; +/// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] +/// #[repr(C)] +/// struct SliceDst<T, U> { +/// t: T, +/// u: [U], +/// } +/// +/// type Src = SliceDst<u32, u16>; +/// type Dst = SliceDst<u16, u8>; +/// +/// let src = Src::ref_from_bytes(&[0, 1, 2, 3, 4, 5, 6, 7]).unwrap(); +/// let dst: &Dst = transmute_ref!(src); +/// +/// assert_eq!(src.t.as_bytes(), [0, 1, 2, 3]); +/// assert_eq!(src.u.len(), 2); +/// assert_eq!(src.u.as_bytes(), [4, 5, 6, 7]); +/// +/// assert_eq!(dst.t.as_bytes(), [0, 1]); +/// assert_eq!(dst.u, [2, 3, 4, 5, 6, 7]); +/// ``` +/// +/// # Use in `const` contexts +/// +/// This macro can be invoked in `const` contexts only when `Src: Sized` and +/// `Dst: Sized`. +/// +#[doc = codegen_section!( + header = "h2", + bench = "transmute_ref", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] +)] +#[macro_export] +macro_rules! transmute_ref { + ($e:expr) => {{ + // NOTE: This must be a macro (rather than a function with trait bounds) + // because there's no way, in a generic context, to enforce that two + // types have the same size or alignment. + + // Ensure that the source type is a reference or a mutable reference + // (note that mutable references are implicitly reborrowed here). + let e: &_ = $e; + + #[allow(unused, clippy::diverging_sub_expression)] + if false { + // This branch, though never taken, ensures that the type of `e` is + // `&T` where `T: IntoBytes + Immutable`, and that the type of this + // macro expression is `&U` where `U: FromBytes + Immutable`. + + struct AssertSrcIsIntoBytes<'a, T: ?::core::marker::Sized + $crate::IntoBytes>(&'a T); + struct AssertSrcIsImmutable<'a, T: ?::core::marker::Sized + $crate::Immutable>(&'a T); + struct AssertDstIsFromBytes<'a, U: ?::core::marker::Sized + $crate::FromBytes>(&'a U); + struct AssertDstIsImmutable<'a, T: ?::core::marker::Sized + $crate::Immutable>(&'a T); + + let _ = AssertSrcIsIntoBytes(e); + let _ = AssertSrcIsImmutable(e); + + if true { + #[allow(unused, unreachable_code)] + let u = AssertDstIsFromBytes(loop {}); + u.0 + } else { + #[allow(unused, unreachable_code)] + let u = AssertDstIsImmutable(loop {}); + u.0 + } + } else { + use $crate::util::macro_util::TransmuteRefDst; + let t = $crate::util::macro_util::Wrap::new(e); + + if false { + // This branch exists solely to force the compiler to infer the + // type of `Dst` *before* it attempts to resolve the method call + // to `transmute_ref` in the `else` branch. + // + // Without this, if `Src` is `Sized` but `Dst` is `!Sized`, the + // compiler will eagerly select the inherent impl of + // `transmute_ref` (which requires `Dst: Sized`) because inherent + // methods take priority over trait methods. It does this before + // it realizes `Dst` is `!Sized`, leading to a compile error when + // it checks the bounds later. + // + // By calling this helper (which returns `&Dst`), we force `Dst` + // to be fully resolved. By the time it gets to the `else` + // branch, the compiler knows `Dst` is `!Sized`, properly + // disqualifies the inherent method, and falls back to the trait + // implementation. + t.transmute_ref_inference_helper() + } else { + // SAFETY: The outer `if false` branch ensures that: + // - `Src: IntoBytes + Immutable` + // - `Dst: FromBytes + Immutable` + unsafe { + t.transmute_ref() + } + } + } + }} +} + +/// Safely transmutes a mutable reference of one type to a mutable reference of +/// another type of the same size and compatible alignment. +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// const fn transmute_mut<'src, 'dst, Src, Dst>(src: &'src mut Src) -> &'dst mut Dst +/// where +/// 'src: 'dst, +/// Src: FromBytes + IntoBytes + ?Sized, +/// Dst: FromBytes + IntoBytes + ?Sized, +/// align_of::<Src>() >= align_of::<Dst>(), +/// size_compatible::<Src, Dst>(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The types `Src` and `Dst` are inferred from the calling context; they cannot +/// be explicitly specified in the macro invocation. +/// +/// # Size compatibility +/// +/// `transmute_mut!` supports transmuting between `Sized` types, between unsized +/// (i.e., `?Sized`) types, and from a `Sized` type to an unsized type. It +/// supports any transmutation that preserves the number of bytes of the +/// referent, even if doing so requires updating the metadata stored in an +/// unsized "fat" reference: +/// +/// ``` +/// # use zerocopy::transmute_mut; +/// # use core::mem::size_of_val; // Not in the prelude on our MSRV +/// let src: &mut [[u8; 2]] = &mut [[0, 1], [2, 3]][..]; +/// let dst: &mut [u8] = transmute_mut!(src); +/// +/// assert_eq!(dst.len(), 4); +/// assert_eq!(dst, [0, 1, 2, 3]); +/// let dst_size = size_of_val(dst); +/// assert_eq!(src.len(), 2); +/// assert_eq!(size_of_val(src), dst_size); +/// ``` +/// +/// # Errors +/// +/// Violations of the alignment and size compatibility checks are detected +/// *after* the compiler performs monomorphization. This has two important +/// consequences. +/// +/// First, it means that generic code will *never* fail these conditions: +/// +/// ``` +/// # use zerocopy::{transmute_mut, FromBytes, IntoBytes, Immutable}; +/// fn transmute_mut<Src, Dst>(src: &mut Src) -> &mut Dst +/// where +/// Src: FromBytes + IntoBytes, +/// Dst: FromBytes + IntoBytes, +/// { +/// transmute_mut!(src) +/// } +/// ``` +/// +/// Instead, failures will only be detected once generic code is instantiated +/// with concrete types: +/// +/// ```compile_fail,E0080 +/// # use zerocopy::{transmute_mut, FromBytes, IntoBytes, Immutable}; +/// # +/// # fn transmute_mut<Src, Dst>(src: &mut Src) -> &mut Dst +/// # where +/// # Src: FromBytes + IntoBytes, +/// # Dst: FromBytes + IntoBytes, +/// # { +/// # transmute_mut!(src) +/// # } +/// let src: &mut u16 = &mut 0; +/// let dst: &mut u8 = transmute_mut(src); +/// ``` +/// +/// Second, the fact that violations are detected after monomorphization means +/// that `cargo check` will usually not detect errors, even when types are +/// concrete. Instead, `cargo build` must be used to detect such errors. +/// +/// +/// # Examples +/// +/// Transmuting between `Sized` types: +/// +/// ``` +/// # use zerocopy::transmute_mut; +/// let mut one_dimensional: [u8; 8] = [0, 1, 2, 3, 4, 5, 6, 7]; +/// +/// let two_dimensional: &mut [[u8; 4]; 2] = transmute_mut!(&mut one_dimensional); +/// +/// assert_eq!(two_dimensional, &[[0, 1, 2, 3], [4, 5, 6, 7]]); +/// +/// two_dimensional.reverse(); +/// +/// assert_eq!(one_dimensional, [4, 5, 6, 7, 0, 1, 2, 3]); +/// ``` +/// +/// Transmuting between unsized types: +/// +/// ``` +/// # use {zerocopy::*, zerocopy_derive::*}; +/// # type u16 = zerocopy::byteorder::native_endian::U16; +/// # type u32 = zerocopy::byteorder::native_endian::U32; +/// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] +/// #[repr(C)] +/// struct SliceDst<T, U> { +/// t: T, +/// u: [U], +/// } +/// +/// type Src = SliceDst<u32, u16>; +/// type Dst = SliceDst<u16, u8>; +/// +/// let mut bytes = [0, 1, 2, 3, 4, 5, 6, 7]; +/// let src = Src::mut_from_bytes(&mut bytes[..]).unwrap(); +/// let dst: &mut Dst = transmute_mut!(src); +/// +/// assert_eq!(dst.t.as_bytes(), [0, 1]); +/// assert_eq!(dst.u, [2, 3, 4, 5, 6, 7]); +/// +/// assert_eq!(src.t.as_bytes(), [0, 1, 2, 3]); +/// assert_eq!(src.u.len(), 2); +/// assert_eq!(src.u.as_bytes(), [4, 5, 6, 7]); +/// ``` +#[macro_export] +macro_rules! transmute_mut { + ($e:expr) => {{ + // NOTE: This must be a macro (rather than a function with trait bounds) + // because, for backwards-compatibility on v0.8.x, we use the autoref + // specialization trick to dispatch to different `transmute_mut` + // implementations: one which doesn't require `Src: KnownLayout + Dst: + // KnownLayout` when `Src: Sized + Dst: Sized`, and one which requires + // `KnownLayout` bounds otherwise. + + // Ensure that the source type is a mutable reference. + let e: &mut _ = $e; + + #[allow(unused)] + use $crate::util::macro_util::TransmuteMutDst as _; + let t = $crate::util::macro_util::Wrap::new(e); + if false { + // This branch exists solely to force the compiler to infer the type + // of `Dst` *before* it attempts to resolve the method call to + // `transmute_mut` in the `else` branch. + // + // Without this, if `Src` is `Sized` but `Dst` is `!Sized`, the + // compiler will eagerly select the inherent impl of `transmute_mut` + // (which requires `Dst: Sized`) because inherent methods take + // priority over trait methods. It does this before it realizes + // `Dst` is `!Sized`, leading to a compile error when it checks the + // bounds later. + // + // By calling this helper (which returns `&mut Dst`), we force `Dst` + // to be fully resolved. By the time it gets to the `else` branch, + // the compiler knows `Dst` is `!Sized`, properly disqualifies the + // inherent method, and falls back to the trait implementation. + t.transmute_mut_inference_helper() + } else { + t.transmute_mut() + } + }} +} + +/// Conditionally transmutes a value of one type to a value of another type of +/// the same size. +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// fn try_transmute<Src, Dst>(src: Src) -> Result<Dst, ValidityError<Src, Dst>> +/// where +/// Src: IntoBytes, +/// Dst: TryFromBytes, +/// size_of::<Src>() == size_of::<Dst>(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// However, unlike a function, this macro can only be invoked when the types of +/// `Src` and `Dst` are completely concrete. The types `Src` and `Dst` are +/// inferred from the calling context; they cannot be explicitly specified in +/// the macro invocation. +/// +/// Note that the `Src` produced by the expression `$e` will *not* be dropped. +/// Semantically, its bits will be copied into a new value of type `Dst`, the +/// original `Src` will be forgotten, and the value of type `Dst` will be +/// returned. +/// +/// # Examples +/// +/// ``` +/// # use zerocopy::*; +/// // 0u8 → bool = false +/// assert_eq!(try_transmute!(0u8), Ok(false)); +/// +/// // 1u8 → bool = true +/// assert_eq!(try_transmute!(1u8), Ok(true)); +/// +/// // 2u8 → bool = error +/// assert!(matches!( +/// try_transmute!(2u8), +/// Result::<bool, _>::Err(ValidityError { .. }) +/// )); +/// ``` +/// +#[doc = codegen_section!( + header = "h2", + bench = "try_transmute", + format = "coco_static_size", +)] +#[macro_export] +macro_rules! try_transmute { + ($e:expr) => {{ + // NOTE: This must be a macro (rather than a function with trait bounds) + // because there's no way, in a generic context, to enforce that two + // types have the same size. `core::mem::transmute` uses compiler magic + // to enforce this so long as the types are concrete. + + let e = $e; + if false { + // Check that the sizes of the source and destination types are + // equal. + + // SAFETY: This code is never executed. + Ok(unsafe { + // Clippy: We can't annotate the types; this macro is designed + // to infer the types from the calling context. + #[allow(clippy::missing_transmute_annotations)] + $crate::util::macro_util::core_reexport::mem::transmute(e) + }) + } else { + $crate::util::macro_util::try_transmute::<_, _>(e) + } + }} +} + +/// Conditionally transmutes a mutable or immutable reference of one type to an +/// immutable reference of another type of the same size and compatible +/// alignment. +/// +/// *Note that while the **value** of the referent is checked for validity at +/// runtime, the **size** and **alignment** are checked at compile time. For +/// conversions which are fallible with respect to size and alignment, see the +/// methods on [`TryFromBytes`].* +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// fn try_transmute_ref<Src, Dst>(src: &Src) -> Result<&Dst, ValidityError<&Src, Dst>> +/// where +/// Src: IntoBytes + Immutable + ?Sized, +/// Dst: TryFromBytes + Immutable + ?Sized, +/// align_of::<Src>() >= align_of::<Dst>(), +/// size_compatible::<Src, Dst>(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The types `Src` and `Dst` are inferred from the calling context; they cannot +/// be explicitly specified in the macro invocation. +/// +/// [`TryFromBytes`]: crate::TryFromBytes +/// +/// # Size compatibility +/// +/// `try_transmute_ref!` supports transmuting between `Sized` types, between +/// unsized (i.e., `?Sized`) types, and from a `Sized` type to an unsized type. +/// It supports any transmutation that preserves the number of bytes of the +/// referent, even if doing so requires updating the metadata stored in an +/// unsized "fat" reference: +/// +/// ``` +/// # use zerocopy::try_transmute_ref; +/// # use core::mem::size_of_val; // Not in the prelude on our MSRV +/// let src: &[[u8; 2]] = &[[0, 1], [2, 3]][..]; +/// let dst: &[u8] = try_transmute_ref!(src).unwrap(); +/// +/// assert_eq!(src.len(), 2); +/// assert_eq!(dst.len(), 4); +/// assert_eq!(dst, [0, 1, 2, 3]); +/// assert_eq!(size_of_val(src), size_of_val(dst)); +/// ``` +/// +/// # Examples +/// +/// Transmuting between `Sized` types: +/// +/// ``` +/// # use zerocopy::*; +/// // 0u8 → bool = false +/// assert_eq!(try_transmute_ref!(&0u8), Ok(&false)); +/// +/// // 1u8 → bool = true +/// assert_eq!(try_transmute_ref!(&1u8), Ok(&true)); +/// +/// // 2u8 → bool = error +/// assert!(matches!( +/// try_transmute_ref!(&2u8), +/// Result::<&bool, _>::Err(ValidityError { .. }) +/// )); +/// ``` +/// +/// Transmuting between unsized types: +/// +/// ``` +/// # use {zerocopy::*, zerocopy_derive::*}; +/// # type u16 = zerocopy::byteorder::native_endian::U16; +/// # type u32 = zerocopy::byteorder::native_endian::U32; +/// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] +/// #[repr(C)] +/// struct SliceDst<T, U> { +/// t: T, +/// u: [U], +/// } +/// +/// type Src = SliceDst<u32, u16>; +/// type Dst = SliceDst<u16, bool>; +/// +/// let src = Src::ref_from_bytes(&[0, 1, 0, 1, 0, 1, 0, 1]).unwrap(); +/// let dst: &Dst = try_transmute_ref!(src).unwrap(); +/// +/// assert_eq!(src.t.as_bytes(), [0, 1, 0, 1]); +/// assert_eq!(src.u.len(), 2); +/// assert_eq!(src.u.as_bytes(), [0, 1, 0, 1]); +/// +/// assert_eq!(dst.t.as_bytes(), [0, 1]); +/// assert_eq!(dst.u, [false, true, false, true, false, true]); +/// ``` +/// +#[doc = codegen_section!( + header = "h2", + bench = "try_transmute_ref", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] +)] +#[macro_export] +macro_rules! try_transmute_ref { + ($e:expr) => {{ + // Ensure that the source type is a reference or a mutable reference + // (note that mutable references are implicitly reborrowed here). + let e: &_ = $e; + + #[allow(unused_imports)] + use $crate::util::macro_util::TryTransmuteRefDst as _; + let t = $crate::util::macro_util::Wrap::new(e); + if false { + // This branch exists solely to force the compiler to infer the type + // of `Dst` *before* it attempts to resolve the method call to + // `try_transmute_ref` in the `else` branch. + // + // Without this, if `Src` is `Sized` but `Dst` is `!Sized`, the + // compiler will eagerly select the inherent impl of + // `try_transmute_ref` (which requires `Dst: Sized`) because + // inherent methods take priority over trait methods. It does this + // before it realizes `Dst` is `!Sized`, leading to a compile error + // when it checks the bounds later. + // + // By calling this helper (which returns `&Dst`), we force `Dst` + // to be fully resolved. By the time it gets to the `else` + // branch, the compiler knows `Dst` is `!Sized`, properly + // disqualifies the inherent method, and falls back to the trait + // implementation. + Ok(t.transmute_ref_inference_helper()) + } else { + t.try_transmute_ref() + } + }} +} + +/// Conditionally transmutes a mutable reference of one type to a mutable +/// reference of another type of the same size and compatible alignment. +/// +/// *Note that while the **value** of the referent is checked for validity at +/// runtime, the **size** and **alignment** are checked at compile time. For +/// conversions which are fallible with respect to size and alignment, see the +/// methods on [`TryFromBytes`].* +/// +/// This macro behaves like an invocation of this function: +/// +/// ```ignore +/// fn try_transmute_mut<Src, Dst>(src: &mut Src) -> Result<&mut Dst, ValidityError<&mut Src, Dst>> +/// where +/// Src: FromBytes + IntoBytes + ?Sized, +/// Dst: TryFromBytes + IntoBytes + ?Sized, +/// align_of::<Src>() >= align_of::<Dst>(), +/// size_compatible::<Src, Dst>(), +/// { +/// # /* +/// ... +/// # */ +/// } +/// ``` +/// +/// The types `Src` and `Dst` are inferred from the calling context; they cannot +/// be explicitly specified in the macro invocation. +/// +/// [`TryFromBytes`]: crate::TryFromBytes +/// +/// # Size compatibility +/// +/// `try_transmute_mut!` supports transmuting between `Sized` types, between +/// unsized (i.e., `?Sized`) types, and from a `Sized` type to an unsized type. +/// It supports any transmutation that preserves the number of bytes of the +/// referent, even if doing so requires updating the metadata stored in an +/// unsized "fat" reference: +/// +/// ``` +/// # use zerocopy::try_transmute_mut; +/// # use core::mem::size_of_val; // Not in the prelude on our MSRV +/// let src: &mut [[u8; 2]] = &mut [[0, 1], [2, 3]][..]; +/// let dst: &mut [u8] = try_transmute_mut!(src).unwrap(); +/// +/// assert_eq!(dst.len(), 4); +/// assert_eq!(dst, [0, 1, 2, 3]); +/// let dst_size = size_of_val(dst); +/// assert_eq!(src.len(), 2); +/// assert_eq!(size_of_val(src), dst_size); +/// ``` +/// +/// # Examples +/// +/// Transmuting between `Sized` types: +/// +/// ``` +/// # use zerocopy::*; +/// // 0u8 → bool = false +/// let src = &mut 0u8; +/// assert_eq!(try_transmute_mut!(src), Ok(&mut false)); +/// +/// // 1u8 → bool = true +/// let src = &mut 1u8; +/// assert_eq!(try_transmute_mut!(src), Ok(&mut true)); +/// +/// // 2u8 → bool = error +/// let src = &mut 2u8; +/// assert!(matches!( +/// try_transmute_mut!(src), +/// Result::<&mut bool, _>::Err(ValidityError { .. }) +/// )); +/// ``` +/// +/// Transmuting between unsized types: +/// +/// ``` +/// # use {zerocopy::*, zerocopy_derive::*}; +/// # type u16 = zerocopy::byteorder::native_endian::U16; +/// # type u32 = zerocopy::byteorder::native_endian::U32; +/// #[derive(KnownLayout, FromBytes, IntoBytes, Immutable)] +/// #[repr(C)] +/// struct SliceDst<T, U> { +/// t: T, +/// u: [U], +/// } +/// +/// type Src = SliceDst<u32, u16>; +/// type Dst = SliceDst<u16, bool>; +/// +/// let mut bytes = [0, 1, 0, 1, 0, 1, 0, 1]; +/// let src = Src::mut_from_bytes(&mut bytes).unwrap(); +/// +/// assert_eq!(src.t.as_bytes(), [0, 1, 0, 1]); +/// assert_eq!(src.u.len(), 2); +/// assert_eq!(src.u.as_bytes(), [0, 1, 0, 1]); +/// +/// let dst: &Dst = try_transmute_mut!(src).unwrap(); +/// +/// assert_eq!(dst.t.as_bytes(), [0, 1]); +/// assert_eq!(dst.u, [false, true, false, true, false, true]); +/// ``` +#[macro_export] +macro_rules! try_transmute_mut { + ($e:expr) => {{ + // Ensure that the source type is a mutable reference. + let e: &mut _ = $e; + + #[allow(unused_imports)] + use $crate::util::macro_util::TryTransmuteMutDst as _; + let t = $crate::util::macro_util::Wrap::new(e); + if false { + // This branch exists solely to force the compiler to infer the type + // of `Dst` *before* it attempts to resolve the method call to + // `try_transmute_mut` in the `else` branch. + // + // Without this, if `Src` is `Sized` but `Dst` is `!Sized`, the + // compiler will eagerly select the inherent impl of + // `try_transmute_mut` (which requires `Dst: Sized`) because + // inherent methods take priority over trait methods. It does this + // before it realizes `Dst` is `!Sized`, leading to a compile error + // when it checks the bounds later. + // + // By calling this helper (which returns `&Dst`), we force `Dst` + // to be fully resolved. By the time it gets to the `else` + // branch, the compiler knows `Dst` is `!Sized`, properly + // disqualifies the inherent method, and falls back to the trait + // implementation. + Ok(t.transmute_mut_inference_helper()) + } else { + t.try_transmute_mut() + } + }} +} + +/// Includes a file and safely transmutes it to a value of an arbitrary type. +/// +/// The file will be included as a byte array, `[u8; N]`, which will be +/// transmuted to another type, `T`. `T` is inferred from the calling context, +/// and must implement [`FromBytes`]. +/// +/// The file is located relative to the current file (similarly to how modules +/// are found). The provided path is interpreted in a platform-specific way at +/// compile time. So, for instance, an invocation with a Windows path containing +/// backslashes `\` would not compile correctly on Unix. +/// +/// `include_value!` is ignorant of byte order. For byte order-aware types, see +/// the [`byteorder`] module. +/// +/// [`FromBytes`]: crate::FromBytes +/// [`byteorder`]: crate::byteorder +/// +/// # Examples +/// +/// Assume there are two files in the same directory with the following +/// contents: +/// +/// File `data` (no trailing newline): +/// +/// ```text +/// abcd +/// ``` +/// +/// File `main.rs`: +/// +/// ```rust +/// use zerocopy::include_value; +/// # macro_rules! include_value { +/// # ($file:expr) => { zerocopy::include_value!(concat!("../testdata/include_value/", $file)) }; +/// # } +/// +/// fn main() { +/// let as_u32: u32 = include_value!("data"); +/// assert_eq!(as_u32, u32::from_ne_bytes([b'a', b'b', b'c', b'd'])); +/// let as_i32: i32 = include_value!("data"); +/// assert_eq!(as_i32, i32::from_ne_bytes([b'a', b'b', b'c', b'd'])); +/// } +/// ``` +/// +/// # Use in `const` contexts +/// +/// This macro can be invoked in `const` contexts. +#[doc(alias("include_bytes", "include_data", "include_type"))] +#[macro_export] +macro_rules! include_value { + ($file:expr $(,)?) => { + $crate::transmute!(*::core::include_bytes!($file)) + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! cryptocorrosion_derive_traits { + ( + #[repr($repr:ident)] + $(#[$attr:meta])* + $vis:vis struct $name:ident $(<$($tyvar:ident),*>)? + $( + ( + $($tuple_field_vis:vis $tuple_field_ty:ty),* + ); + )? + + $( + { + $($field_vis:vis $field_name:ident: $field_ty:ty,)* + } + )? + ) => { + $crate::cryptocorrosion_derive_traits!(@assert_allowed_struct_repr #[repr($repr)]); + + $(#[$attr])* + #[repr($repr)] + $vis struct $name $(<$($tyvar),*>)? + $( + ( + $($tuple_field_vis $tuple_field_ty),* + ); + )? + + $( + { + $($field_vis $field_name: $field_ty,)* + } + )? + + // SAFETY: See inline. + unsafe impl $(<$($tyvar),*>)? $crate::TryFromBytes for $name$(<$($tyvar),*>)? + where + $( + $($tuple_field_ty: $crate::FromBytes,)* + )? + + $( + $($field_ty: $crate::FromBytes,)* + )? + { + #[inline(always)] + fn is_bit_valid<A>(_: $crate::Maybe<'_, Self, A>) -> bool + where + A: $crate::invariant::Alignment, + { + // SAFETY: This macro only accepts `#[repr(C)]` and + // `#[repr(transparent)]` structs, and this `impl` block + // requires all field types to be `FromBytes`. Thus, all + // initialized byte sequences constitutes valid instances of + // `Self`. + true + } + + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` and + // `#[repr(transparent)]` structs, and this `impl` block requires all + // field types to be `FromBytes`, which is a sub-trait of `FromZeros`. + unsafe impl $(<$($tyvar),*>)? $crate::FromZeros for $name$(<$($tyvar),*>)? + where + $( + $($tuple_field_ty: $crate::FromBytes,)* + )? + + $( + $($field_ty: $crate::FromBytes,)* + )? + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` and + // `#[repr(transparent)]` structs, and this `impl` block requires all + // field types to be `FromBytes`. + unsafe impl $(<$($tyvar),*>)? $crate::FromBytes for $name$(<$($tyvar),*>)? + where + $( + $($tuple_field_ty: $crate::FromBytes,)* + )? + + $( + $($field_ty: $crate::FromBytes,)* + )? + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` and + // `#[repr(transparent)]` structs, this `impl` block requires all field + // types to be `IntoBytes`, and a padding check is used to ensures that + // there are no padding bytes. + unsafe impl $(<$($tyvar),*>)? $crate::IntoBytes for $name$(<$($tyvar),*>)? + where + $( + $($tuple_field_ty: $crate::IntoBytes,)* + )? + + $( + $($field_ty: $crate::IntoBytes,)* + )? + + (): $crate::util::macro_util::PaddingFree< + Self, + { + $crate::cryptocorrosion_derive_traits!( + @struct_padding_check #[repr($repr)] + $(($($tuple_field_ty),*))? + $({$($field_ty),*})? + ) + }, + >, + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` and + // `#[repr(transparent)]` structs, and this `impl` block requires all + // field types to be `Immutable`. + unsafe impl $(<$($tyvar),*>)? $crate::Immutable for $name$(<$($tyvar),*>)? + where + $( + $($tuple_field_ty: $crate::Immutable,)* + )? + + $( + $($field_ty: $crate::Immutable,)* + )? + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + }; + (@assert_allowed_struct_repr #[repr(transparent)]) => {}; + (@assert_allowed_struct_repr #[repr(C)]) => {}; + (@assert_allowed_struct_repr #[$_attr:meta]) => { + compile_error!("repr must be `#[repr(transparent)]` or `#[repr(C)]`"); + }; + ( + @struct_padding_check #[repr(transparent)] + $(($($tuple_field_ty:ty),*))? + $({$($field_ty:ty),*})? + ) => { + // SAFETY: `#[repr(transparent)]` structs cannot have the same layout as + // their single non-zero-sized field, and so cannot have any padding + // outside of that field. + 0 + }; + ( + @struct_padding_check #[repr(C)] + $(($($tuple_field_ty:ty),*))? + $({$($field_ty:ty),*})? + ) => { + $crate::struct_padding!( + Self, + None, + None, + [ + $($($tuple_field_ty),*)? + $($($field_ty),*)? + ] + ) + }; + ( + #[repr(C)] + $(#[$attr:meta])* + $vis:vis union $name:ident { + $( + $field_name:ident: $field_ty:ty, + )* + } + ) => { + $(#[$attr])* + #[repr(C)] + $vis union $name { + $( + $field_name: $field_ty, + )* + } + + // SAFETY: See inline. + unsafe impl $crate::TryFromBytes for $name + where + $( + $field_ty: $crate::FromBytes, + )* + { + #[inline(always)] + fn is_bit_valid<A>(_: $crate::Maybe<'_, Self, A>) -> bool + where + A: $crate::invariant::Alignment, + { + // SAFETY: This macro only accepts `#[repr(C)]` unions, and this + // `impl` block requires all field types to be `FromBytes`. + // Thus, all initialized byte sequences constitutes valid + // instances of `Self`. + true + } + + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` unions, and this `impl` + // block requires all field types to be `FromBytes`, which is a + // sub-trait of `FromZeros`. + unsafe impl $crate::FromZeros for $name + where + $( + $field_ty: $crate::FromBytes, + )* + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` unions, and this `impl` + // block requires all field types to be `FromBytes`. + unsafe impl $crate::FromBytes for $name + where + $( + $field_ty: $crate::FromBytes, + )* + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` unions, this `impl` + // block requires all field types to be `IntoBytes`, and a padding check + // is used to ensures that there are no padding bytes before or after + // any field. + unsafe impl $crate::IntoBytes for $name + where + $( + $field_ty: $crate::IntoBytes, + )* + (): $crate::util::macro_util::PaddingFree< + Self, + { + $crate::union_padding!( + Self, + None::<usize>, + None::<usize>, + [$($field_ty),*] + ) + }, + >, + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + + // SAFETY: This macro only accepts `#[repr(C)]` unions, and this `impl` + // block requires all field types to be `Immutable`. + unsafe impl $crate::Immutable for $name + where + $( + $field_ty: $crate::Immutable, + )* + { + fn only_derive_is_allowed_to_implement_this_trait() {} + } + }; +} + +#[cfg(test)] +mod tests { + use crate::{ + byteorder::native_endian::{U16, U32}, + util::testutil::*, + *, + }; + + #[derive(KnownLayout, Immutable, FromBytes, IntoBytes, PartialEq, Debug)] + #[repr(C)] + struct SliceDst<T, U> { + a: T, + b: [U], + } + + #[test] + fn test_transmute() { + // Test that memory is transmuted as expected. + let array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let x: [[u8; 2]; 4] = transmute!(array_of_u8s); + assert_eq!(x, array_of_arrays); + let x: [u8; 8] = transmute!(array_of_arrays); + assert_eq!(x, array_of_u8s); + + // Test that memory is transmuted as expected when shrinking. + let x: [[u8; 2]; 3] = transmute!(#![allow(shrink)] array_of_u8s); + assert_eq!(x, [[0u8, 1], [2, 3], [4, 5]]); + + // Test that the source expression's value is forgotten rather than + // dropped. + #[derive(IntoBytes)] + #[repr(transparent)] + struct PanicOnDrop(()); + impl Drop for PanicOnDrop { + fn drop(&mut self) { + panic!("PanicOnDrop::drop"); + } + } + #[allow(clippy::let_unit_value)] + let _: () = transmute!(PanicOnDrop(())); + #[allow(clippy::let_unit_value)] + let _: () = transmute!(#![allow(shrink)] PanicOnDrop(())); + + // Test that `transmute!` is legal in a const context. + const ARRAY_OF_U8S: [u8; 8] = [0u8, 1, 2, 3, 4, 5, 6, 7]; + const ARRAY_OF_ARRAYS: [[u8; 2]; 4] = [[0, 1], [2, 3], [4, 5], [6, 7]]; + const X: [[u8; 2]; 4] = transmute!(ARRAY_OF_U8S); + assert_eq!(X, ARRAY_OF_ARRAYS); + const X_SHRINK: [[u8; 2]; 3] = transmute!(#![allow(shrink)] ARRAY_OF_U8S); + assert_eq!(X_SHRINK, [[0u8, 1], [2, 3], [4, 5]]); + + // Test that `transmute!` works with `!Immutable` types. + let x: usize = transmute!(UnsafeCell::new(1usize)); + assert_eq!(x, 1); + let x: UnsafeCell<usize> = transmute!(1usize); + assert_eq!(x.into_inner(), 1); + let x: UnsafeCell<isize> = transmute!(UnsafeCell::new(1usize)); + assert_eq!(x.into_inner(), 1); + } + + // A `Sized` type which doesn't implement `KnownLayout` (it is "not + // `KnownLayout`", or `Nkl`). + // + // This permits us to test that `transmute_ref!` and `transmute_mut!` work + // for types which are `Sized + !KnownLayout`. When we added support for + // slice DSTs in #1924, this new support relied on `KnownLayout`, but we + // need to make sure to remain backwards-compatible with code which uses + // these macros with types which are `!KnownLayout`. + #[derive(FromBytes, IntoBytes, Immutable, PartialEq, Eq, Debug)] + #[repr(transparent)] + struct Nkl<T>(T); + + #[test] + fn test_transmute_ref() { + // Test that memory is transmuted as expected. + let array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let x: &[[u8; 2]; 4] = transmute_ref!(&array_of_u8s); + assert_eq!(*x, array_of_arrays); + let x: &[u8; 8] = transmute_ref!(&array_of_arrays); + assert_eq!(*x, array_of_u8s); + + // Test that `transmute_ref!` is legal in a const context. + const ARRAY_OF_U8S: [u8; 8] = [0u8, 1, 2, 3, 4, 5, 6, 7]; + const ARRAY_OF_ARRAYS: [[u8; 2]; 4] = [[0, 1], [2, 3], [4, 5], [6, 7]]; + #[allow(clippy::redundant_static_lifetimes)] + const X: &'static [[u8; 2]; 4] = transmute_ref!(&ARRAY_OF_U8S); + assert_eq!(*X, ARRAY_OF_ARRAYS); + + // Test sized -> unsized transmutation. + let array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let slice_of_arrays = &array_of_arrays[..]; + let x: &[[u8; 2]] = transmute_ref!(&array_of_u8s); + assert_eq!(x, slice_of_arrays); + + // Before 1.61.0, we can't define the `const fn transmute_ref` function + // that we do on and after 1.61.0. + #[cfg(no_zerocopy_generic_bounds_in_const_fn_1_61_0)] + { + // Test that `transmute_ref!` supports non-`KnownLayout` `Sized` + // types. + const ARRAY_OF_NKL_U8S: Nkl<[u8; 8]> = Nkl([0u8, 1, 2, 3, 4, 5, 6, 7]); + const ARRAY_OF_NKL_ARRAYS: Nkl<[[u8; 2]; 4]> = Nkl([[0, 1], [2, 3], [4, 5], [6, 7]]); + const X_NKL: &Nkl<[[u8; 2]; 4]> = transmute_ref!(&ARRAY_OF_NKL_U8S); + assert_eq!(*X_NKL, ARRAY_OF_NKL_ARRAYS); + } + + #[cfg(not(no_zerocopy_generic_bounds_in_const_fn_1_61_0))] + { + // Call through a generic function to make sure our autoref + // specialization trick works even when types are generic. + const fn transmute_ref<T, U>(t: &T) -> &U + where + T: IntoBytes + Immutable, + U: FromBytes + Immutable, + { + transmute_ref!(t) + } + + // Test that `transmute_ref!` supports non-`KnownLayout` `Sized` + // types. + const ARRAY_OF_NKL_U8S: Nkl<[u8; 8]> = Nkl([0u8, 1, 2, 3, 4, 5, 6, 7]); + const ARRAY_OF_NKL_ARRAYS: Nkl<[[u8; 2]; 4]> = Nkl([[0, 1], [2, 3], [4, 5], [6, 7]]); + const X_NKL: &Nkl<[[u8; 2]; 4]> = transmute_ref(&ARRAY_OF_NKL_U8S); + assert_eq!(*X_NKL, ARRAY_OF_NKL_ARRAYS); + } + + // Test that `transmute_ref!` works on slice DSTs in and that memory is + // transmuted as expected. + let slice_dst_of_u8s = + SliceDst::<U16, [u8; 2]>::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + let slice_dst_of_u16s = + SliceDst::<U16, U16>::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + let x: &SliceDst<U16, U16> = transmute_ref!(slice_dst_of_u8s); + assert_eq!(x, slice_dst_of_u16s); + + let slice_dst_of_u8s = + SliceDst::<U16, u8>::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + let x: &[u8] = transmute_ref!(slice_dst_of_u8s); + assert_eq!(x, [0, 1, 2, 3, 4, 5]); + + let x: &[u8] = transmute_ref!(slice_dst_of_u16s); + assert_eq!(x, [0, 1, 2, 3, 4, 5]); + + let x: &[U16] = transmute_ref!(slice_dst_of_u16s); + let slice_of_u16s: &[U16] = <[U16]>::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + assert_eq!(x, slice_of_u16s); + + // Test that transmuting from a type with larger trailing slice offset + // and larger trailing slice element works. + let bytes = &[0, 1, 2, 3, 4, 5, 6, 7][..]; + let slice_dst_big = SliceDst::<U32, U16>::ref_from_bytes(bytes).unwrap(); + let slice_dst_small = SliceDst::<U16, u8>::ref_from_bytes(bytes).unwrap(); + let x: &SliceDst<U16, u8> = transmute_ref!(slice_dst_big); + assert_eq!(x, slice_dst_small); + + // Test that it's legal to transmute a reference while shrinking the + // lifetime (note that `X` has the lifetime `'static`). + let x: &[u8; 8] = transmute_ref!(X); + assert_eq!(*x, ARRAY_OF_U8S); + + // Test that `transmute_ref!` supports decreasing alignment. + let u = AU64(0); + let array = [0, 0, 0, 0, 0, 0, 0, 0]; + let x: &[u8; 8] = transmute_ref!(&u); + assert_eq!(*x, array); + + // Test that a mutable reference can be turned into an immutable one. + let mut x = 0u8; + #[allow(clippy::useless_transmute)] + let y: &u8 = transmute_ref!(&mut x); + assert_eq!(*y, 0); + } + + #[test] + fn test_try_transmute() { + // Test that memory is transmuted with `try_transmute` as expected. + let array_of_bools = [false, true, false, true, false, true, false, true]; + let array_of_arrays = [[0, 1], [0, 1], [0, 1], [0, 1]]; + let x: Result<[[u8; 2]; 4], _> = try_transmute!(array_of_bools); + assert_eq!(x, Ok(array_of_arrays)); + let x: Result<[bool; 8], _> = try_transmute!(array_of_arrays); + assert_eq!(x, Ok(array_of_bools)); + + // Test that `try_transmute!` works with `!Immutable` types. + let x: Result<usize, _> = try_transmute!(UnsafeCell::new(1usize)); + assert_eq!(x.unwrap(), 1); + let x: Result<UnsafeCell<usize>, _> = try_transmute!(1usize); + assert_eq!(x.unwrap().into_inner(), 1); + let x: Result<UnsafeCell<isize>, _> = try_transmute!(UnsafeCell::new(1usize)); + assert_eq!(x.unwrap().into_inner(), 1); + + #[derive(FromBytes, IntoBytes, Debug, PartialEq)] + #[repr(transparent)] + struct PanicOnDrop<T>(T); + + impl<T> Drop for PanicOnDrop<T> { + fn drop(&mut self) { + panic!("PanicOnDrop dropped"); + } + } + + // Since `try_transmute!` semantically moves its argument on failure, + // the `PanicOnDrop` is not dropped, and thus this shouldn't panic. + let x: Result<usize, _> = try_transmute!(PanicOnDrop(1usize)); + assert_eq!(x, Ok(1)); + + // Since `try_transmute!` semantically returns ownership of its argument + // on failure, the `PanicOnDrop` is returned rather than dropped, and + // thus this shouldn't panic. + let y: Result<bool, _> = try_transmute!(PanicOnDrop(2u8)); + // We have to use `map_err` instead of comparing against + // `Err(PanicOnDrop(2u8))` because the latter would create and then drop + // its `PanicOnDrop` temporary, which would cause a panic. + assert_eq!(y.as_ref().map_err(|p| &p.src.0), Err::<&bool, _>(&2u8)); + mem::forget(y); + } + + #[test] + fn test_try_transmute_ref() { + // Test that memory is transmuted with `try_transmute_ref` as expected. + let array_of_bools = &[false, true, false, true, false, true, false, true]; + let array_of_arrays = &[[0, 1], [0, 1], [0, 1], [0, 1]]; + let x: Result<&[[u8; 2]; 4], _> = try_transmute_ref!(array_of_bools); + assert_eq!(x, Ok(array_of_arrays)); + let x: Result<&[bool; 8], _> = try_transmute_ref!(array_of_arrays); + assert_eq!(x, Ok(array_of_bools)); + + // Test that it's legal to transmute a reference while shrinking the + // lifetime. + { + let x: Result<&[[u8; 2]; 4], _> = try_transmute_ref!(array_of_bools); + assert_eq!(x, Ok(array_of_arrays)); + } + + // Test that `try_transmute_ref!` supports decreasing alignment. + let u = AU64(0); + let array = [0u8, 0, 0, 0, 0, 0, 0, 0]; + let x: Result<&[u8; 8], _> = try_transmute_ref!(&u); + assert_eq!(x, Ok(&array)); + + // Test that a mutable reference can be turned into an immutable one. + let mut x = 0u8; + #[allow(clippy::useless_transmute)] + let y: Result<&u8, _> = try_transmute_ref!(&mut x); + assert_eq!(y, Ok(&0)); + + // Test that sized types work which don't implement `KnownLayout`. + let array_of_nkl_u8s = Nkl([0u8, 1, 2, 3, 4, 5, 6, 7]); + let array_of_nkl_arrays = Nkl([[0, 1], [2, 3], [4, 5], [6, 7]]); + let x: Result<&Nkl<[[u8; 2]; 4]>, _> = try_transmute_ref!(&array_of_nkl_u8s); + assert_eq!(x, Ok(&array_of_nkl_arrays)); + + // Test sized -> unsized transmutation. + let array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let slice_of_arrays = &array_of_arrays[..]; + let x: Result<&[[u8; 2]], _> = try_transmute_ref!(&array_of_u8s); + assert_eq!(x, Ok(slice_of_arrays)); + + // Test unsized -> unsized transmutation. + let slice_dst_of_u8s = + SliceDst::<U16, [u8; 2]>::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + let slice_dst_of_u16s = + SliceDst::<U16, U16>::ref_from_bytes(&[0, 1, 2, 3, 4, 5][..]).unwrap(); + let x: Result<&SliceDst<U16, U16>, _> = try_transmute_ref!(slice_dst_of_u8s); + assert_eq!(x, Ok(slice_dst_of_u16s)); + } + + #[test] + fn test_try_transmute_mut() { + // Test that memory is transmuted with `try_transmute_mut` as expected. + let array_of_u8s = &mut [0u8, 1, 0, 1, 0, 1, 0, 1]; + let array_of_arrays = &mut [[0u8, 1], [0, 1], [0, 1], [0, 1]]; + let x: Result<&mut [[u8; 2]; 4], _> = try_transmute_mut!(array_of_u8s); + assert_eq!(x, Ok(array_of_arrays)); + + let array_of_bools = &mut [false, true, false, true, false, true, false, true]; + let array_of_arrays = &mut [[0u8, 1], [0, 1], [0, 1], [0, 1]]; + let x: Result<&mut [bool; 8], _> = try_transmute_mut!(array_of_arrays); + assert_eq!(x, Ok(array_of_bools)); + + // Test that it's legal to transmute a reference while shrinking the + // lifetime. + let array_of_bools = &mut [false, true, false, true, false, true, false, true]; + let array_of_arrays = &mut [[0u8, 1], [0, 1], [0, 1], [0, 1]]; + { + let x: Result<&mut [bool; 8], _> = try_transmute_mut!(array_of_arrays); + assert_eq!(x, Ok(array_of_bools)); + } + + // Test that `try_transmute_mut!` supports decreasing alignment. + let u = &mut AU64(0); + let array = &mut [0u8, 0, 0, 0, 0, 0, 0, 0]; + let x: Result<&mut [u8; 8], _> = try_transmute_mut!(u); + assert_eq!(x, Ok(array)); + + // Test that a mutable reference can be turned into an immutable one. + let mut x = 0u8; + #[allow(clippy::useless_transmute)] + let y: Result<&mut u8, _> = try_transmute_mut!(&mut x); + assert_eq!(y, Ok(&mut 0)); + + // Test that sized types work which don't implement `KnownLayout`. + let mut array_of_nkl_u8s = Nkl([0u8, 1, 2, 3, 4, 5, 6, 7]); + let mut array_of_nkl_arrays = Nkl([[0, 1], [2, 3], [4, 5], [6, 7]]); + let x: Result<&mut Nkl<[[u8; 2]; 4]>, _> = try_transmute_mut!(&mut array_of_nkl_u8s); + assert_eq!(x, Ok(&mut array_of_nkl_arrays)); + + // Test sized -> unsized transmutation. + let mut array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let mut array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let slice_of_arrays = &mut array_of_arrays[..]; + let x: Result<&mut [[u8; 2]], _> = try_transmute_mut!(&mut array_of_u8s); + assert_eq!(x, Ok(slice_of_arrays)); + + // Test unsized -> unsized transmutation. + let mut bytes = [0, 1, 2, 3, 4, 5, 6]; + let slice_dst_of_u8s = SliceDst::<u8, [u8; 2]>::mut_from_bytes(&mut bytes[..]).unwrap(); + let mut bytes = [0, 1, 2, 3, 4, 5, 6]; + let slice_dst_of_u16s = SliceDst::<u8, U16>::mut_from_bytes(&mut bytes[..]).unwrap(); + let x: Result<&mut SliceDst<u8, U16>, _> = try_transmute_mut!(slice_dst_of_u8s); + assert_eq!(x, Ok(slice_dst_of_u16s)); + } + + #[test] + fn test_transmute_mut() { + // Test that memory is transmuted as expected. + let mut array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let mut array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let x: &mut [[u8; 2]; 4] = transmute_mut!(&mut array_of_u8s); + assert_eq!(*x, array_of_arrays); + let x: &mut [u8; 8] = transmute_mut!(&mut array_of_arrays); + assert_eq!(*x, array_of_u8s); + + { + // Test that it's legal to transmute a reference while shrinking the + // lifetime. + let x: &mut [u8; 8] = transmute_mut!(&mut array_of_arrays); + assert_eq!(*x, array_of_u8s); + } + + // Test that `transmute_mut!` supports non-`KnownLayout` types. + let mut array_of_u8s = Nkl([0u8, 1, 2, 3, 4, 5, 6, 7]); + let mut array_of_arrays = Nkl([[0, 1], [2, 3], [4, 5], [6, 7]]); + let x: &mut Nkl<[[u8; 2]; 4]> = transmute_mut!(&mut array_of_u8s); + assert_eq!(*x, array_of_arrays); + let x: &mut Nkl<[u8; 8]> = transmute_mut!(&mut array_of_arrays); + assert_eq!(*x, array_of_u8s); + + // Test that `transmute_mut!` supports decreasing alignment. + let mut u = AU64(0); + let array = [0, 0, 0, 0, 0, 0, 0, 0]; + let x: &[u8; 8] = transmute_mut!(&mut u); + assert_eq!(*x, array); + + // Test that a mutable reference can be turned into an immutable one. + let mut x = 0u8; + #[allow(clippy::useless_transmute)] + let y: &u8 = transmute_mut!(&mut x); + assert_eq!(*y, 0); + + // Test that `transmute_mut!` works on slice DSTs in and that memory is + // transmuted as expected. + let mut bytes = [0, 1, 2, 3, 4, 5, 6]; + let slice_dst_of_u8s = SliceDst::<u8, [u8; 2]>::mut_from_bytes(&mut bytes[..]).unwrap(); + let mut bytes = [0, 1, 2, 3, 4, 5, 6]; + let slice_dst_of_u16s = SliceDst::<u8, U16>::mut_from_bytes(&mut bytes[..]).unwrap(); + let x: &mut SliceDst<u8, U16> = transmute_mut!(slice_dst_of_u8s); + assert_eq!(x, slice_dst_of_u16s); + + // Test that `transmute_mut!` works on slices that memory is transmuted + // as expected. + let array_of_u16s: &mut [u16] = &mut [0u16, 1, 2]; + let array_of_i16s: &mut [i16] = &mut [0i16, 1, 2]; + let x: &mut [i16] = transmute_mut!(array_of_u16s); + assert_eq!(x, array_of_i16s); + + // Test that transmuting from a type with larger trailing slice offset + // and larger trailing slice element works. + let mut bytes = [0, 1, 2, 3, 4, 5, 6, 7]; + let slice_dst_big = SliceDst::<U32, U16>::mut_from_bytes(&mut bytes[..]).unwrap(); + let mut bytes = [0, 1, 2, 3, 4, 5, 6, 7]; + let slice_dst_small = SliceDst::<U16, u8>::mut_from_bytes(&mut bytes[..]).unwrap(); + let x: &mut SliceDst<U16, u8> = transmute_mut!(slice_dst_big); + assert_eq!(x, slice_dst_small); + + // Test sized -> unsized transmutation. + let mut array_of_u8s = [0u8, 1, 2, 3, 4, 5, 6, 7]; + let mut array_of_arrays = [[0, 1], [2, 3], [4, 5], [6, 7]]; + let slice_of_arrays = &mut array_of_arrays[..]; + let x: &mut [[u8; 2]] = transmute_mut!(&mut array_of_u8s); + assert_eq!(x, slice_of_arrays); + } + + #[test] + fn test_macros_evaluate_args_once() { + let mut ctr = 0; + #[allow(clippy::useless_transmute)] + let _: usize = transmute!({ + ctr += 1; + 0usize + }); + assert_eq!(ctr, 1); + + let mut ctr = 0; + let _: &usize = transmute_ref!({ + ctr += 1; + &0usize + }); + assert_eq!(ctr, 1); + + let mut ctr: usize = 0; + let _: &mut usize = transmute_mut!({ + ctr += 1; + &mut ctr + }); + assert_eq!(ctr, 1); + + let mut ctr = 0; + #[allow(clippy::useless_transmute)] + let _: usize = try_transmute!({ + ctr += 1; + 0usize + }) + .unwrap(); + assert_eq!(ctr, 1); + } + + #[test] + fn test_include_value() { + const AS_U32: u32 = include_value!("../testdata/include_value/data"); + assert_eq!(AS_U32, u32::from_ne_bytes([b'a', b'b', b'c', b'd'])); + const AS_I32: i32 = include_value!("../testdata/include_value/data"); + assert_eq!(AS_I32, i32::from_ne_bytes([b'a', b'b', b'c', b'd'])); + } + + #[test] + #[allow(non_camel_case_types, unreachable_pub, dead_code)] + fn test_cryptocorrosion_derive_traits() { + // Test the set of invocations added in + // https://github.com/cryptocorrosion/cryptocorrosion/pull/85 + + fn assert_impls<T: FromBytes + IntoBytes + Immutable>() {} + + cryptocorrosion_derive_traits! { + #[repr(C)] + #[derive(Clone, Copy)] + pub union vec128_storage { + d: [u32; 4], + q: [u64; 2], + } + } + + assert_impls::<vec128_storage>(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct u32x4_generic([u32; 4]); + } + + assert_impls::<u32x4_generic>(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct u64x2_generic([u64; 2]); + } + + assert_impls::<u64x2_generic>(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct u128x1_generic([u128; 1]); + } + + assert_impls::<u128x1_generic>(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone, Default)] + #[allow(non_camel_case_types)] + pub struct x2<W, G>(pub [W; 2], PhantomData<G>); + } + + enum NotZerocopy {} + assert_impls::<x2<(), NotZerocopy>>(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone, Default)] + #[allow(non_camel_case_types)] + pub struct x4<W>(pub [W; 4]); + } + + assert_impls::<x4<()>>(); + + #[cfg(feature = "simd")] + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + #[cfg(target_arch = "x86")] + use core::arch::x86::{__m128i, __m256i}; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{__m128i, __m256i}; + + cryptocorrosion_derive_traits! { + #[repr(C)] + #[derive(Copy, Clone)] + pub struct X4(__m128i, __m128i, __m128i, __m128i); + } + + assert_impls::<X4>(); + + cryptocorrosion_derive_traits! { + #[repr(C)] + /// Generic wrapper for unparameterized storage of any of the + /// possible impls. Converting into and out of this type should + /// be essentially free, although it may be more aligned than a + /// particular impl requires. + #[allow(non_camel_case_types)] + #[derive(Copy, Clone)] + pub union vec128_storage { + u32x4: [u32; 4], + u64x2: [u64; 2], + u128x1: [u128; 1], + sse2: __m128i, + } + } + + assert_impls::<vec128_storage>(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[allow(non_camel_case_types)] + #[derive(Copy, Clone)] + pub struct vec<S3, S4, NI> { + x: __m128i, + s3: PhantomData<S3>, + s4: PhantomData<S4>, + ni: PhantomData<NI>, + } + } + + assert_impls::<vec<NotZerocopy, NotZerocopy, NotZerocopy>>(); + + cryptocorrosion_derive_traits! { + #[repr(transparent)] + #[derive(Copy, Clone)] + pub struct u32x4x2_avx2<NI> { + x: __m256i, + ni: PhantomData<NI>, + } + } + + assert_impls::<u32x4x2_avx2<NotZerocopy>>(); + } + + // Make sure that our derive works for `#[repr(C)]` structs even though + // cryptocorrosion doesn't currently have any. + cryptocorrosion_derive_traits! { + #[repr(C)] + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct ReprC(u8, u8, u16); + } + } +} diff --git a/rust/zerocopy/src/pointer/inner.rs b/rust/zerocopy/src/pointer/inner.rs new file mode 100644 index 000000000000..949b60a3f83e --- /dev/null +++ b/rust/zerocopy/src/pointer/inner.rs @@ -0,0 +1,754 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2024 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use core::{marker::PhantomData, ops::Range, ptr::NonNull}; + +pub use _def::PtrInner; + +#[allow(unused_imports)] +use crate::util::polyfills::NumExt as _; +use crate::{ + layout::{CastType, MetadataCastError}, + pointer::cast, + util::AsAddress, + AlignmentError, CastError, KnownLayout, MetadataOf, SizeError, SplitAt, +}; + +mod _def { + use super::*; + /// The inner pointer stored inside a [`Ptr`][crate::Ptr]. + /// + /// `PtrInner<'a, T>` is [covariant] in `'a` and invariant in `T`. + /// + /// [covariant]: https://doc.rust-lang.org/reference/subtyping.html + #[allow(missing_debug_implementations)] + pub struct PtrInner<'a, T> + where + T: ?Sized, + { + /// # Invariants + /// + /// 0. If `ptr`'s referent is not zero sized, then `ptr` has valid + /// provenance for its referent, which is entirely contained in some + /// Rust allocation, `A`. + /// 1. If `ptr`'s referent is not zero sized, `A` is guaranteed to live + /// for at least `'a`. + /// + /// # Postconditions + /// + /// By virtue of these invariants, code may assume the following, which + /// are logical implications of the invariants: + /// - `ptr`'s referent is not larger than `isize::MAX` bytes \[1\] + /// - `ptr`'s referent does not wrap around the address space \[1\] + /// + /// \[1\] Per <https://doc.rust-lang.org/1.85.0/std/ptr/index.html#allocated-object>: + /// + /// For any allocated object with `base` address, `size`, and a set of + /// `addresses`, the following are guaranteed: + /// ... + /// - `size <= isize::MAX` + /// + /// As a consequence of these guarantees, given any address `a` within + /// the set of addresses of an allocated object: + /// ... + /// - It is guaranteed that, given `o = a - base` (i.e., the offset of + /// `a` within the allocated object), `base + o` will not wrap + /// around the address space (in other words, will not overflow + /// `usize`) + ptr: NonNull<T>, + // SAFETY: `&'a UnsafeCell<T>` is covariant in `'a` and invariant in `T` + // [1]. We use this construction rather than the equivalent `&mut T`, + // because our MSRV of 1.65 prohibits `&mut` types in const contexts. + // + // [1] https://doc.rust-lang.org/1.81.0/reference/subtyping.html#variance + _marker: PhantomData<&'a core::cell::UnsafeCell<T>>, + } + + impl<'a, T: 'a + ?Sized> Copy for PtrInner<'a, T> {} + impl<'a, T: 'a + ?Sized> Clone for PtrInner<'a, T> { + #[inline(always)] + fn clone(&self) -> PtrInner<'a, T> { + // SAFETY: None of the invariants on `ptr` are affected by having + // multiple copies of a `PtrInner`. + *self + } + } + + impl<'a, T: 'a + ?Sized> PtrInner<'a, T> { + /// Constructs a `Ptr` from a [`NonNull`]. + /// + /// # Safety + /// + /// The caller promises that: + /// + /// 0. If `ptr`'s referent is not zero sized, then `ptr` has valid + /// provenance for its referent, which is entirely contained in some + /// Rust allocation, `A`. + /// 1. If `ptr`'s referent is not zero sized, `A` is guaranteed to live + /// for at least `'a`. + #[inline(always)] + #[must_use] + pub const unsafe fn new(ptr: NonNull<T>) -> PtrInner<'a, T> { + // SAFETY: The caller has promised to satisfy all safety invariants + // of `PtrInner`. + Self { ptr, _marker: PhantomData } + } + + /// Converts this `PtrInner<T>` to a [`NonNull<T>`]. + /// + /// Note that this method does not consume `self`. The caller should + /// watch out for `unsafe` code which uses the returned `NonNull` in a + /// way that violates the safety invariants of `self`. + #[inline(always)] + #[must_use] + pub const fn as_non_null(&self) -> NonNull<T> { + self.ptr + } + + /// Converts this `PtrInner<T>` to a [`*mut T`]. + /// + /// Note that this method does not consume `self`. The caller should + /// watch out for `unsafe` code which uses the returned `*mut T` in a + /// way that violates the safety invariants of `self`. + #[inline(always)] + #[must_use] + pub const fn as_ptr(&self) -> *mut T { + self.ptr.as_ptr() + } + } +} + +impl<'a, T: ?Sized> PtrInner<'a, T> { + /// Constructs a `PtrInner` from a reference. + #[inline] + pub fn from_ref(ptr: &'a T) -> Self { + let ptr = NonNull::from(ptr); + // SAFETY: + // 0. If `ptr`'s referent is not zero sized, then `ptr`, by invariant on + // `&'a T` [1], has valid provenance for its referent, which is + // entirely contained in some Rust allocation, `A`. + // 1. If `ptr`'s referent is not zero sized, then `A`, by invariant on + // `&'a T`, is guaranteed to live for at least `'a`. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/primitive.reference.html#safety: + // + // For all types, `T: ?Sized`, and for all `t: &T` or `t: &mut T`, + // when such values cross an API boundary, the following invariants + // must generally be upheld: + // ... + // - if `size_of_val(t) > 0`, then `t` is dereferenceable for + // `size_of_val(t)` many bytes + // + // If `t` points at address `a`, being “dereferenceable” for N bytes + // means that the memory range `[a, a + N)` is all contained within a + // single allocated object. + unsafe { Self::new(ptr) } + } + + /// Constructs a `PtrInner` from a mutable reference. + #[inline] + pub fn from_mut(ptr: &'a mut T) -> Self { + let ptr = NonNull::from(ptr); + // SAFETY: + // 0. If `ptr`'s referent is not zero sized, then `ptr`, by invariant on + // `&'a mut T` [1], has valid provenance for its referent, which is + // entirely contained in some Rust allocation, `A`. + // 1. If `ptr`'s referent is not zero sized, then `A`, by invariant on + // `&'a mut T`, is guaranteed to live for at least `'a`. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/primitive.reference.html#safety: + // + // For all types, `T: ?Sized`, and for all `t: &T` or `t: &mut T`, + // when such values cross an API boundary, the following invariants + // must generally be upheld: + // ... + // - if `size_of_val(t) > 0`, then `t` is dereferenceable for + // `size_of_val(t)` many bytes + // + // If `t` points at address `a`, being “dereferenceable” for N bytes + // means that the memory range `[a, a + N)` is all contained within a + // single allocated object. + unsafe { Self::new(ptr) } + } + + /// # Safety + /// + /// The caller may assume that the resulting `PtrInner` addresses the subset + /// of the bytes of `self`'s referent addressed by `C::project(self)`. + #[must_use] + #[inline(always)] + pub fn project<U: ?Sized, C: cast::Project<T, U>>(self) -> PtrInner<'a, U> { + let projected_raw = C::project(self); + + // SAFETY: `self`'s referent lives at a `NonNull` address, and is either + // zero-sized or lives in an allocation. In either case, it does not + // wrap around the address space [1], and so none of the addresses + // contained in it or one-past-the-end of it are null. + // + // By invariant on `C: Project`, `C::project` is a provenance-preserving + // projection which preserves or shrinks the set of referent bytes, so + // `projected_raw` references a subset of `self`'s referent, and so it + // cannot be null. + // + // [1] https://doc.rust-lang.org/1.92.0/std/ptr/index.html#allocation + let projected_non_null = unsafe { NonNull::new_unchecked(projected_raw) }; + + // SAFETY: As described in the preceding safety comment, `projected_raw`, + // and thus `projected_non_null`, addresses a subset of `self`'s + // referent. Thus, `projected_non_null` either: + // - Addresses zero bytes or, + // - Addresses a subset of the referent of `self`. In this case, `self` + // has provenance for its referent, which lives in an allocation. + // Since `projected_non_null` was constructed using a sequence of + // provenance-preserving operations, it also has provenance for its + // referent and that referent lives in an allocation. By invariant on + // `self`, that allocation lives for `'a`. + unsafe { PtrInner::new(projected_non_null) } + } +} + +#[allow(clippy::needless_lifetimes)] +impl<'a, T> PtrInner<'a, T> +where + T: ?Sized + KnownLayout, +{ + /// Extracts the metadata of this `ptr`. + #[inline] + #[must_use] + pub fn meta(self) -> MetadataOf<T> { + let meta = T::pointer_to_metadata(self.as_ptr()); + // SAFETY: By invariant on `PtrInner`, `self.as_non_null()` addresses no + // more than `isize::MAX` bytes. + unsafe { MetadataOf::new_unchecked(meta) } + } + + /// Produces a `PtrInner` with the same address and provenance as `self` but + /// the given `meta`. + /// + /// # Safety + /// + /// The caller promises that if `self`'s referent is not zero sized, then + /// a pointer constructed from its address with the given `meta` metadata + /// will address a subset of the allocation pointed to by `self`. + #[inline] + #[must_use] + pub unsafe fn with_meta(self, meta: T::PointerMetadata) -> Self + where + T: KnownLayout, + { + let raw = T::raw_from_ptr_len(self.as_non_null().cast(), meta); + + // SAFETY: + // + // Lemma 0: `raw` either addresses zero bytes, or addresses a subset of + // the allocation pointed to by `self` and has the same + // provenance as `self`. Proof: `raw` is constructed using + // provenance-preserving operations, and the caller has + // promised that, if `self`'s referent is not zero-sized, the + // resulting pointer addresses a subset of the allocation + // pointed to by `self`. + // + // 0. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `ptr` is derived from some valid Rust allocation, + // `A`. + // 1. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `ptr` has valid provenance for `A`. + // 2. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `ptr` addresses a byte range which is entirely + // contained in `A`. + // 3. Per Lemma 0 and by invariant on `self`, `ptr` addresses a byte + // range whose length fits in an `isize`. + // 4. Per Lemma 0 and by invariant on `self`, `ptr` addresses a byte + // range which does not wrap around the address space. + // 5. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `A` is guaranteed to live for at least `'a`. + unsafe { PtrInner::new(raw) } + } +} + +#[allow(clippy::needless_lifetimes)] +impl<'a, T> PtrInner<'a, T> +where + T: ?Sized + KnownLayout<PointerMetadata = usize>, +{ + /// Splits `T` in two. + /// + /// # Safety + /// + /// The caller promises that: + /// - `l_len.get() <= self.meta()`. + /// + /// ## (Non-)Overlap + /// + /// Given `let (left, right) = ptr.split_at(l_len)`, it is guaranteed that + /// `left` and `right` are contiguous and non-overlapping if + /// `l_len.padding_needed_for() == 0`. This is true for all `[T]`. + /// + /// If `l_len.padding_needed_for() != 0`, then the left pointer will overlap + /// the right pointer to satisfy `T`'s padding requirements. + #[inline] + #[must_use] + pub unsafe fn split_at_unchecked( + self, + l_len: crate::util::MetadataOf<T>, + ) -> (Self, PtrInner<'a, [T::Elem]>) + where + T: SplitAt, + { + let l_len = l_len.get(); + + // SAFETY: The caller promises that `l_len.get() <= self.meta()`. + // Trivially, `0 <= l_len`. + let left = unsafe { self.with_meta(l_len) }; + + let right = self.trailing_slice(); + // SAFETY: The caller promises that `l_len <= self.meta() = slf.meta()`. + // Trivially, `slf.meta() <= slf.meta()`. + let right = unsafe { right.slice_unchecked(l_len..self.meta().get()) }; + + // SAFETY: If `l_len.padding_needed_for() == 0`, then `left` and `right` + // are non-overlapping. Proof: `left` is constructed `slf` with `l_len` + // as its (exclusive) upper bound. If `l_len.padding_needed_for() == 0`, + // then `left` requires no trailing padding following its final element. + // Since `right` is constructed from `slf`'s trailing slice with `l_len` + // as its (inclusive) lower bound, no byte is referred to by both + // pointers. + // + // Conversely, `l_len.padding_needed_for() == N`, where `N + // > 0`, `left` requires `N` bytes of trailing padding following its + // final element. Since `right` is constructed from the trailing slice + // of `slf` with `l_len` as its (inclusive) lower bound, the first `N` + // bytes of `right` are aliased by `left`. + (left, right) + } + + /// Produces the trailing slice of `self`. + #[inline] + #[must_use] + pub fn trailing_slice(self) -> PtrInner<'a, [T::Elem]> + where + T: SplitAt, + { + let offset = crate::trailing_slice_layout::<T>().offset; + + let bytes = self.as_non_null().cast::<u8>().as_ptr(); + + // SAFETY: + // - By invariant on `T: KnownLayout`, `T::LAYOUT` describes `T`'s + // layout. `offset` is the offset of the trailing slice within `T`, + // which is by definition in-bounds or one byte past the end of any + // `T`, regardless of metadata. By invariant on `PtrInner`, `self` + // (and thus `bytes`) points to a byte range of size `<= isize::MAX`, + // and so `offset <= isize::MAX`. Since `size_of::<u8>() == 1`, + // `offset * size_of::<u8>() <= isize::MAX`. + // - If `offset > 0`, then by invariant on `PtrInner`, `self` (and thus + // `bytes`) points to a byte range entirely contained within the same + // allocated object as `self`. As explained above, this offset results + // in a pointer to or one byte past the end of this allocated object. + let bytes = unsafe { bytes.add(offset) }; + + // SAFETY: By the preceding safety argument, `bytes` is within or one + // byte past the end of the same allocated object as `self`, which + // ensures that it is non-null. + let bytes = unsafe { NonNull::new_unchecked(bytes) }; + + let ptr = KnownLayout::raw_from_ptr_len(bytes, self.meta().get()); + + // SAFETY: + // 0. If `ptr`'s referent is not zero sized, then `ptr` is derived from + // some valid Rust allocation, `A`, because `ptr` is derived from + // the same allocated object as `self`. + // 1. If `ptr`'s referent is not zero sized, then `ptr` has valid + // provenance for `A` because `raw` is derived from the same + // allocated object as `self` via provenance-preserving operations. + // 2. If `ptr`'s referent is not zero sized, then `ptr` addresses a byte + // range which is entirely contained in `A`, by previous safety proof + // on `bytes`. + // 3. `ptr` addresses a byte range whose length fits in an `isize`, by + // consequence of #2. + // 4. `ptr` addresses a byte range which does not wrap around the + // address space, by consequence of #2. + // 5. If `ptr`'s referent is not zero sized, then `A` is guaranteed to + // live for at least `'a`, because `ptr` is derived from `self`. + unsafe { PtrInner::new(ptr) } + } +} + +#[allow(clippy::needless_lifetimes)] +impl<'a, T> PtrInner<'a, [T]> { + /// Creates a pointer which addresses the given `range` of self. + /// + /// # Safety + /// + /// `range` is a valid range (`start <= end`) and `end <= self.meta()`. + #[inline] + #[must_use] + pub unsafe fn slice_unchecked(self, range: Range<usize>) -> Self { + let base = self.as_non_null().cast::<T>().as_ptr(); + + // SAFETY: The caller promises that `start <= end <= self.meta()`. By + // invariant, if `self`'s referent is not zero-sized, then `self` refers + // to a byte range which is contained within a single allocation, which + // is no more than `isize::MAX` bytes long, and which does not wrap + // around the address space. Thus, this pointer arithmetic remains + // in-bounds of the same allocation, and does not wrap around the + // address space. The offset (in bytes) does not overflow `isize`. + // + // If `self`'s referent is zero-sized, then these conditions are + // trivially satisfied. + let base = unsafe { base.add(range.start) }; + + // SAFETY: The caller promises that `start <= end`, and so this will not + // underflow. + #[allow(unstable_name_collisions)] + let len = unsafe { range.end.unchecked_sub(range.start) }; + + let ptr = core::ptr::slice_from_raw_parts_mut(base, len); + + // SAFETY: By invariant, `self`'s referent is either a ZST or lives + // entirely in an allocation. `ptr` points inside of or one byte past + // the end of that referent. Thus, in either case, `ptr` is non-null. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + + // SAFETY: + // + // Lemma 0: `ptr` addresses a subset of the bytes addressed by `self`, + // and has the same provenance. Proof: The caller guarantees + // that `start <= end <= self.meta()`. Thus, `base` is + // in-bounds of `self`, and `base + (end - start)` is also + // in-bounds of self. Finally, `ptr` is constructed using + // provenance-preserving operations. + // + // 0. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `ptr` has valid provenance for its referent, + // which is entirely contained in some Rust allocation, `A`. + // 1. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is not + // zero sized, then `A` is guaranteed to live for at least `'a`. + unsafe { PtrInner::new(ptr) } + } + + /// Iteratively projects the elements `PtrInner<T>` from `PtrInner<[T]>`. + #[inline] + pub fn iter(&self) -> impl Iterator<Item = PtrInner<'a, T>> { + // FIXME(#429): Once `NonNull::cast` documents that it preserves + // provenance, cite those docs. + let base = self.as_non_null().cast::<T>().as_ptr(); + (0..self.meta().get()).map(move |i| { + // FIXME(https://github.com/rust-lang/rust/issues/74265): Use + // `NonNull::get_unchecked_mut`. + + // SAFETY: If the following conditions are not satisfied + // `pointer::cast` may induce Undefined Behavior [1]: + // + // > - The computed offset, `count * size_of::<T>()` bytes, must not + // > overflow `isize``. + // > - If the computed offset is non-zero, then `self` must be + // > derived from a pointer to some allocated object, and the + // > entire memory range between `self` and the result must be in + // > bounds of that allocated object. In particular, this range + // > must not “wrap around” the edge of the address space. + // + // [1] https://doc.rust-lang.org/std/primitive.pointer.html#method.add + // + // We satisfy both of these conditions here: + // - By invariant on `Ptr`, `self` addresses a byte range whose + // length fits in an `isize`. Since `elem` is contained in `self`, + // the computed offset of `elem` must fit within `isize.` + // - If the computed offset is non-zero, then this means that the + // referent is not zero-sized. In this case, `base` points to an + // allocated object (by invariant on `self`). Thus: + // - By contract, `self.meta()` accurately reflects the number of + // elements in the slice. `i` is in bounds of `c.meta()` by + // construction, and so the result of this addition cannot + // overflow past the end of the allocation referred to by `c`. + // - By invariant on `Ptr`, `self` addresses a byte range which + // does not wrap around the address space. Since `elem` is + // contained in `self`, the computed offset of `elem` must wrap + // around the address space. + // + // FIXME(#429): Once `pointer::add` documents that it preserves + // provenance, cite those docs. + let elem = unsafe { base.add(i) }; + + // SAFETY: `elem` must not be null. `base` is constructed from a + // `NonNull` pointer, and the addition that produces `elem` must not + // overflow or wrap around, so `elem >= base > 0`. + // + // FIXME(#429): Once `NonNull::new_unchecked` documents that it + // preserves provenance, cite those docs. + let elem = unsafe { NonNull::new_unchecked(elem) }; + + // SAFETY: The safety invariants of `Ptr::new` (see definition) are + // satisfied: + // 0. If `elem`'s referent is not zero sized, then `elem` has valid + // provenance for its referent, because it derived from `self` + // using a series of provenance-preserving operations, and + // because `self` has valid provenance for its referent. By the + // same argument, `elem`'s referent is entirely contained within + // the same allocated object as `self`'s referent. + // 1. If `elem`'s referent is not zero sized, then the allocation of + // `elem` is guaranteed to live for at least `'a`, because `elem` + // is entirely contained in `self`, which lives for at least `'a` + // by invariant on `Ptr`. + unsafe { PtrInner::new(elem) } + }) + } +} + +impl<'a, T, const N: usize> PtrInner<'a, [T; N]> { + /// Casts this pointer-to-array into a slice. + /// + /// # Safety + /// + /// Callers may assume that the returned `PtrInner` references the same + /// address and length as `self`. + #[allow(clippy::wrong_self_convention)] + #[inline] + #[must_use] + pub fn as_slice(self) -> PtrInner<'a, [T]> { + let start = self.as_non_null().cast::<T>().as_ptr(); + let slice = core::ptr::slice_from_raw_parts_mut(start, N); + // SAFETY: `slice` is not null, because it is derived from `start` + // which is non-null. + let slice = unsafe { NonNull::new_unchecked(slice) }; + // SAFETY: Lemma: In the following safety arguments, note that `slice` + // is derived from `self` in two steps: first, by casting `self: [T; N]` + // to `start: T`, then by constructing a pointer to a slice starting at + // `start` of length `N`. As a result, `slice` references exactly the + // same allocation as `self`, if any. + // + // 0. By the above lemma, if `slice`'s referent is not zero sized, then + // `slice` has the same referent as `self`. By invariant on `self`, + // this referent is entirely contained within some allocation, `A`. + // Because `slice` was constructed using provenance-preserving + // operations, it has provenance for its entire referent. + // 1. By the above lemma, if `slice`'s referent is not zero sized, then + // `A` is guaranteed to live for at least `'a`, because it is derived + // from the same allocation as `self`, which, by invariant on + // `PtrInner`, lives for at least `'a`. + unsafe { PtrInner::new(slice) } + } +} + +impl<'a> PtrInner<'a, [u8]> { + /// Attempts to cast `self` to a `U` using the given cast type. + /// + /// If `U` is a slice DST and pointer metadata (`meta`) is provided, then + /// the cast will only succeed if it would produce an object with the given + /// metadata. + /// + /// Returns `None` if the resulting `U` would be invalidly-aligned, if no + /// `U` can fit in `self`, or if the provided pointer metadata describes an + /// invalid instance of `U`. On success, returns a pointer to the + /// largest-possible `U` which fits in `self`. + /// + /// # Safety + /// + /// The caller may assume that this implementation is correct, and may rely + /// on that assumption for the soundness of their code. In particular, the + /// caller may assume that, if `try_cast_into` returns `Some((ptr, + /// remainder))`, then `ptr` and `remainder` refer to non-overlapping byte + /// ranges within `self`, and that `ptr` and `remainder` entirely cover + /// `self`. Finally: + /// - If this is a prefix cast, `ptr` has the same address as `self`. + /// - If this is a suffix cast, `remainder` has the same address as `self`. + #[inline] + pub fn try_cast_into<U>( + self, + cast_type: CastType, + meta: Option<U::PointerMetadata>, + ) -> Result<(PtrInner<'a, U>, PtrInner<'a, [u8]>), CastError<Self, U>> + where + U: 'a + ?Sized + KnownLayout, + { + // PANICS: By invariant, the byte range addressed by + // `self.as_non_null()` does not wrap around the address space. This + // implies that the sum of the address (represented as a `usize`) and + // length do not overflow `usize`, as required by + // `validate_cast_and_convert_metadata`. Thus, this call to + // `validate_cast_and_convert_metadata` will only panic if `U` is a DST + // whose trailing slice element is zero-sized. + let maybe_metadata = MetadataOf::<U>::validate_cast_and_convert_metadata( + AsAddress::addr(self.as_ptr()), + self.meta(), + cast_type, + meta, + ); + + let (elems, split_at) = match maybe_metadata { + Ok((elems, split_at)) => (elems, split_at), + Err(MetadataCastError::Alignment) => { + // SAFETY: Since `validate_cast_and_convert_metadata` returned + // an alignment error, `U` must have an alignment requirement + // greater than one. + let err = unsafe { AlignmentError::<_, U>::new_unchecked(self) }; + return Err(CastError::Alignment(err)); + } + Err(MetadataCastError::Size) => return Err(CastError::Size(SizeError::new(self))), + }; + + // SAFETY: `validate_cast_and_convert_metadata` promises to return + // `split_at <= self.meta()`. + // + // Lemma 0: `l_slice` and `r_slice` are non-overlapping. Proof: By + // contract on `PtrInner::split_at_unchecked`, the produced `PtrInner`s + // are always non-overlapping if `self` is a `[T]`; here it is a `[u8]`. + let (l_slice, r_slice) = unsafe { self.split_at_unchecked(split_at) }; + + let (target, remainder) = match cast_type { + CastType::Prefix => (l_slice, r_slice), + CastType::Suffix => (r_slice, l_slice), + }; + + let base = target.as_non_null().cast::<u8>(); + + let ptr = U::raw_from_ptr_len(base, elems.get()); + + // SAFETY: + // 0. By invariant, if `target`'s referent is not zero sized, then + // `target` has provenance valid for some Rust allocation, `A`. + // Because `ptr` is derived from `target` via provenance-preserving + // operations, `ptr` will also have provenance valid for its entire + // referent. + // 1. `validate_cast_and_convert_metadata` promises that the object + // described by `elems` and `split_at` lives at a byte range which is + // a subset of the input byte range. Thus, by invariant, if + // `target`'s referent is not zero sized, then `target` refers to an + // allocation which is guaranteed to live for at least `'a`, and thus + // so does `ptr`. + Ok((unsafe { PtrInner::new(ptr) }, remainder)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::*; + + #[test] + fn test_meta() { + let arr = [1; 16]; + let dst = <[u8]>::ref_from_bytes(&arr[..]).unwrap(); + let ptr = PtrInner::from_ref(dst); + assert_eq!(ptr.meta().get(), 16); + + // SAFETY: 8 is less than 16 + let ptr = unsafe { ptr.with_meta(8) }; + + assert_eq!(ptr.meta().get(), 8); + } + + #[test] + fn test_split_at() { + fn test_split_at<const OFFSET: usize, const BUFFER_SIZE: usize>() { + #[derive(FromBytes, KnownLayout, SplitAt, Immutable)] + #[repr(C)] + struct SliceDst<const OFFSET: usize> { + prefix: [u8; OFFSET], + trailing: [u8], + } + + let n: usize = BUFFER_SIZE - OFFSET; + let arr = [1; BUFFER_SIZE]; + let dst = SliceDst::<OFFSET>::ref_from_bytes(&arr[..]).unwrap(); + let ptr = PtrInner::from_ref(dst); + for i in 0..=n { + assert_eq!(ptr.meta().get(), n); + // SAFETY: `i` is in bounds by construction. + let i = unsafe { MetadataOf::new_unchecked(i) }; + // SAFETY: `i` is in bounds by construction. + let (l, r) = unsafe { ptr.split_at_unchecked(i) }; + // SAFETY: Points to a valid value by construction. + #[allow(clippy::undocumented_unsafe_blocks, clippy::as_conversions)] + // Clippy false positive + let l_sum: usize = l + .trailing_slice() + .iter() + .map( + #[inline(always)] + |ptr| unsafe { core::ptr::read_unaligned(ptr.as_ptr()) } as usize, + ) + .sum(); + // SAFETY: Points to a valid value by construction. + #[allow(clippy::undocumented_unsafe_blocks, clippy::as_conversions)] + // Clippy false positive + let r_sum: usize = r + .iter() + .map( + #[inline(always)] + |ptr| unsafe { core::ptr::read_unaligned(ptr.as_ptr()) } as usize, + ) + .sum(); + assert_eq!(l_sum, i.get()); + assert_eq!(r_sum, n - i.get()); + assert_eq!(l_sum + r_sum, n); + } + } + + test_split_at::<0, 16>(); + test_split_at::<1, 17>(); + test_split_at::<2, 18>(); + } + + #[test] + fn test_trailing_slice() { + fn test_trailing_slice<const OFFSET: usize, const BUFFER_SIZE: usize>() { + #[derive(FromBytes, KnownLayout, SplitAt, Immutable)] + #[repr(C)] + struct SliceDst<const OFFSET: usize> { + prefix: [u8; OFFSET], + trailing: [u8], + } + + let n: usize = BUFFER_SIZE - OFFSET; + let arr = [1; BUFFER_SIZE]; + let dst = SliceDst::<OFFSET>::ref_from_bytes(&arr[..]).unwrap(); + let ptr = PtrInner::from_ref(dst); + + assert_eq!(ptr.meta().get(), n); + let trailing = ptr.trailing_slice(); + assert_eq!(trailing.meta().get(), n); + + assert_eq!( + // SAFETY: We assume this to be sound for the sake of this test, + // which will fail, here, in miri, if the safety precondition of + // `offset_of` is not satisfied. + unsafe { + #[allow(clippy::as_conversions)] + let offset = (trailing.as_ptr() as *mut u8).offset_from(ptr.as_ptr() as *mut _); + offset + }, + isize::try_from(OFFSET).unwrap(), + ); + + // SAFETY: Points to a valid value by construction. + #[allow(clippy::undocumented_unsafe_blocks, clippy::as_conversions)] + // Clippy false positive + let trailing: usize = trailing + .iter() + .map(|ptr| unsafe { core::ptr::read_unaligned(ptr.as_ptr()) } as usize) + .sum(); + + assert_eq!(trailing, n); + } + + test_trailing_slice::<0, 16>(); + test_trailing_slice::<1, 17>(); + test_trailing_slice::<2, 18>(); + } + #[test] + fn test_ptr_inner_clone() { + let mut x = 0u8; + let p = PtrInner::from_mut(&mut x); + #[allow(clippy::clone_on_copy)] + let p2 = p.clone(); + assert_eq!(p.as_non_null(), p2.as_non_null()); + } +} diff --git a/rust/zerocopy/src/pointer/invariant.rs b/rust/zerocopy/src/pointer/invariant.rs new file mode 100644 index 000000000000..7ff0d43dad5e --- /dev/null +++ b/rust/zerocopy/src/pointer/invariant.rs @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2024 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +#![allow(missing_copy_implementations, missing_debug_implementations, missing_docs)] + +//! The parameterized invariants of a [`Ptr`][super::Ptr]. +//! +//! Invariants are encoded as ([`Aliasing`], [`Alignment`], [`Validity`]) +//! triples implementing the [`Invariants`] trait. + +/// The invariants of a [`Ptr`][super::Ptr]. +pub trait Invariants: Sealed { + type Aliasing: Aliasing; + type Alignment: Alignment; + type Validity: Validity; +} + +impl<A: Aliasing, AA: Alignment, V: Validity> Invariants for (A, AA, V) { + type Aliasing = A; + type Alignment = AA; + type Validity = V; +} + +/// The aliasing invariant of a [`Ptr`][super::Ptr]. +/// +/// All aliasing invariants must permit reading from the bytes of a pointer's +/// referent which are not covered by [`UnsafeCell`]s. +/// +/// [`UnsafeCell`]: core::cell::UnsafeCell +pub trait Aliasing: Sealed { + /// Is `Self` [`Exclusive`]? + #[doc(hidden)] + const IS_EXCLUSIVE: bool; +} + +/// The alignment invariant of a [`Ptr`][super::Ptr]. +pub trait Alignment: Sealed { + #[doc(hidden)] + #[must_use] + fn read<T, I, R>(ptr: crate::Ptr<'_, T, I>) -> T + where + T: Copy + Read<I::Aliasing, R>, + I: Invariants<Alignment = Self, Validity = Valid>, + I::Aliasing: Reference; +} + +/// The validity invariant of a [`Ptr`][super::Ptr]. +/// +/// # Safety +/// +/// In this section, we will use `Ptr<T, V>` as a shorthand for `Ptr<T, I: +/// Invariants<Validity = V>>` for brevity. +/// +/// Each `V: Validity` defines a set of bit values which may appear in the +/// referent of a `Ptr<T, V>`, denoted `S(T, V)`. Each `V: Validity`, in its +/// documentation, provides a definition of `S(T, V)` which must be valid for +/// all `T: ?Sized`. Any `V: Validity` must guarantee that this set is only a +/// function of the *bit validity* of the referent type, `T`, and not of any +/// other property of `T`. As a consequence, given `V: Validity`, `T`, and `U` +/// where `T` and `U` have the same bit validity, `S(V, T) = S(V, U)`. +/// +/// It is guaranteed that the referent of any `ptr: Ptr<T, V>` is a member of +/// `S(T, V)`. Unsafe code must ensure that this guarantee will be upheld for +/// any existing `Ptr`s or any `Ptr`s that that code creates. +/// +/// An important implication of this guarantee is that it restricts what +/// transmutes are sound, where "transmute" is used in this context to refer to +/// changing the referent type or validity invariant of a `Ptr`, as either +/// change may change the set of bit values permitted to appear in the referent. +/// In particular, the following are necessary (but not sufficient) conditions +/// in order for a transmute from `src: Ptr<T, V>` to `dst: Ptr<U, W>` to be +/// sound: +/// - If `S(T, V) = S(U, W)`, then no restrictions apply; otherwise, +/// - If `dst` permits mutation of its referent (e.g. via `Exclusive` aliasing +/// or interior mutation under `Shared` aliasing), then it must hold that +/// `S(T, V) ⊇ S(U, W)` - in other words, the transmute must not expand the +/// set of allowed referent bit patterns. A violation of this requirement +/// would permit using `dst` to write `x` where `x ∈ S(U, W)` but `x ∉ S(T, +/// V)`, which would violate the guarantee that `src`'s referent may only +/// contain values in `S(T, V)`. +/// - If the referent may be mutated without going through `dst` while `dst` is +/// live (e.g. via interior mutation on a `Shared`-aliased `Ptr` or `&` +/// reference), then it must hold that `S(T, V) ⊆ S(U, W)` - in other words, +/// the transmute must not shrink the set of allowed referent bit patterns. A +/// violation of this requirement would permit using `src` or another +/// mechanism (e.g. a `&` reference used to derive `src`) to write `x` where +/// `x ∈ S(T, V)` but `x ∉ S(U, W)`, which would violate the guarantee that +/// `dst`'s referent may only contain values in `S(U, W)`. +pub unsafe trait Validity: Sealed { + const KIND: ValidityKind; +} + +pub enum ValidityKind { + Uninit, + AsInitialized, + Initialized, + Valid, +} + +/// An [`Aliasing`] invariant which is either [`Shared`] or [`Exclusive`]. +/// +/// # Safety +/// +/// Given `A: Reference`, callers may assume that either `A = Shared` or `A = +/// Exclusive`. +pub trait Reference: Aliasing + Sealed {} + +/// The `Ptr<'a, T>` adheres to the aliasing rules of a `&'a T`. +/// +/// The referent of a shared-aliased `Ptr` may be concurrently referenced by any +/// number of shared-aliased `Ptr` or `&T` references, or by any number of +/// `Ptr<U>` or `&U` references as permitted by `T`'s library safety invariants, +/// and may not be concurrently referenced by any exclusively-aliased `Ptr`s or +/// `&mut` references. The referent must not be mutated, except via +/// [`UnsafeCell`]s, and only when permitted by `T`'s library safety invariants. +/// +/// [`UnsafeCell`]: core::cell::UnsafeCell +pub enum Shared {} +impl Aliasing for Shared { + const IS_EXCLUSIVE: bool = false; +} +impl Reference for Shared {} + +/// The `Ptr<'a, T>` adheres to the aliasing rules of a `&'a mut T`. +/// +/// The referent of an exclusively-aliased `Ptr` may not be concurrently +/// referenced by any other `Ptr`s or references, and may not be accessed (read +/// or written) other than via this `Ptr`. +pub enum Exclusive {} +impl Aliasing for Exclusive { + const IS_EXCLUSIVE: bool = true; +} +impl Reference for Exclusive {} + +/// It is unknown whether the pointer is aligned. +pub enum Unaligned {} + +impl Alignment for Unaligned { + #[inline(always)] + fn read<T, I, R>(ptr: crate::Ptr<'_, T, I>) -> T + where + T: Copy + Read<I::Aliasing, R>, + I: Invariants<Alignment = Self, Validity = Valid>, + I::Aliasing: Reference, + { + (*ptr.into_unalign().as_ref()).into_inner() + } +} + +/// The referent is aligned: for `Ptr<T>`, the referent's address is a multiple +/// of the `T`'s alignment. +pub enum Aligned {} +impl Alignment for Aligned { + #[inline(always)] + fn read<T, I, R>(ptr: crate::Ptr<'_, T, I>) -> T + where + T: Copy + Read<I::Aliasing, R>, + I: Invariants<Alignment = Self, Validity = Valid>, + I::Aliasing: Reference, + { + *ptr.as_ref() + } +} + +/// Any bit pattern is allowed in the `Ptr`'s referent, including uninitialized +/// bytes. +pub enum Uninit {} +// SAFETY: `Uninit`'s validity is well-defined for all `T: ?Sized`, and is not a +// function of any property of `T` other than its bit validity (in fact, it's +// not even a property of `T`'s bit validity, but this is more than we are +// required to uphold). +unsafe impl Validity for Uninit { + const KIND: ValidityKind = ValidityKind::Uninit; +} + +/// The byte ranges initialized in `T` are also initialized in the referent of a +/// `Ptr<T>`. +/// +/// Formally: uninitialized bytes may only be present in `Ptr<T>`'s referent +/// where they are guaranteed to be present in `T`. This is a dynamic property: +/// if, at a particular byte offset, a valid enum discriminant is set, the +/// subsequent bytes may only have uninitialized bytes as specified by the +/// corresponding enum. +/// +/// Formally, given `len = size_of_val_raw(ptr)`, at every byte offset, `b`, in +/// the range `[0, len)`: +/// - If, in any instance `t: T` of length `len`, the byte at offset `b` in `t` +/// is initialized, then the byte at offset `b` within `*ptr` must be +/// initialized. +/// - Let `c` be the contents of the byte range `[0, b)` in `*ptr`. Let `S` be +/// the subset of valid instances of `T` of length `len` which contain `c` in +/// the offset range `[0, b)`. If, in any instance of `t: T` in `S`, the byte +/// at offset `b` in `t` is initialized, then the byte at offset `b` in `*ptr` +/// must be initialized. +/// +/// Pragmatically, this means that if `*ptr` is guaranteed to contain an enum +/// type at a particular offset, and the enum discriminant stored in `*ptr` +/// corresponds to a valid variant of that enum type, then it is guaranteed +/// that the appropriate bytes of `*ptr` are initialized as defined by that +/// variant's bit validity (although note that the variant may contain another +/// enum type, in which case the same rules apply depending on the state of +/// its discriminant, and so on recursively). +pub enum AsInitialized {} +// SAFETY: `AsInitialized`'s validity is well-defined for all `T: ?Sized`, and +// is not a function of any property of `T` other than its bit validity. +unsafe impl Validity for AsInitialized { + const KIND: ValidityKind = ValidityKind::AsInitialized; +} + +/// The byte ranges in the referent are fully initialized. In other words, if +/// the referent is `N` bytes long, then it contains a bit-valid `[u8; N]`. +pub enum Initialized {} +// SAFETY: `Initialized`'s validity is well-defined for all `T: ?Sized`, and is +// not a function of any property of `T` other than its bit validity (in fact, +// it's not even a property of `T`'s bit validity, but this is more than we are +// required to uphold). +unsafe impl Validity for Initialized { + const KIND: ValidityKind = ValidityKind::Initialized; +} + +/// The referent of a `Ptr<T>` is valid for `T`, upholding bit validity and any +/// library safety invariants. +pub enum Valid {} +// SAFETY: `Valid`'s validity is well-defined for all `T: ?Sized`, and is not a +// function of any property of `T` other than its bit validity. +unsafe impl Validity for Valid { + const KIND: ValidityKind = ValidityKind::Valid; +} + +/// # Safety +/// +/// `DT: CastableFrom<ST, SV, DV>` is sound if `SV = DV = Uninit` or `SV = DV = +/// Initialized`. +pub unsafe trait CastableFrom<ST: ?Sized, SV, DV> {} + +// SAFETY: `SV = DV = Uninit`. +unsafe impl<ST: ?Sized, DT: ?Sized> CastableFrom<ST, Uninit, Uninit> for DT {} +// SAFETY: `SV = DV = Initialized`. +unsafe impl<ST: ?Sized, DT: ?Sized> CastableFrom<ST, Initialized, Initialized> for DT {} + +/// [`Ptr`](crate::Ptr) referents that permit unsynchronized read operations. +/// +/// `T: Read<A, R>` implies that a pointer to `T` with aliasing `A` permits +/// unsynchronized read operations. This can be because `A` is [`Exclusive`] or +/// because `T` does not permit interior mutation. +/// +/// # Safety +/// +/// `T: Read<A, R>` if either of the following conditions holds: +/// - `A` is [`Exclusive`] +/// - `T` implements [`Immutable`](crate::Immutable) +/// +/// As a consequence, if `T: Read<A, R>`, then any `Ptr<T, (A, ...)>` is +/// permitted to perform unsynchronized reads from its referent. +pub trait Read<A: Aliasing, R> {} + +impl<A: Aliasing, T: ?Sized + crate::Immutable> Read<A, BecauseImmutable> for T {} +impl<T: ?Sized> Read<Exclusive, BecauseExclusive> for T {} + +/// Unsynchronized reads are permitted because only one live [`Ptr`](crate::Ptr) +/// or reference may exist to the referent bytes at a time. +#[derive(Copy, Clone, Debug)] +pub enum BecauseExclusive {} + +/// Unsynchronized reads are permitted because no live [`Ptr`](crate::Ptr)s or +/// references permit interior mutation. +#[derive(Copy, Clone, Debug)] +pub enum BecauseImmutable {} + +use sealed::Sealed; +mod sealed { + use super::*; + + pub trait Sealed {} + + impl Sealed for Shared {} + impl Sealed for Exclusive {} + + impl Sealed for Unaligned {} + impl Sealed for Aligned {} + + impl Sealed for Uninit {} + impl Sealed for AsInitialized {} + impl Sealed for Initialized {} + impl Sealed for Valid {} + + impl<A: Sealed, AA: Sealed, V: Sealed> Sealed for (A, AA, V) {} + + impl Sealed for BecauseImmutable {} + impl Sealed for BecauseExclusive {} +} diff --git a/rust/zerocopy/src/pointer/mod.rs b/rust/zerocopy/src/pointer/mod.rs new file mode 100644 index 000000000000..d6eacc52febe --- /dev/null +++ b/rust/zerocopy/src/pointer/mod.rs @@ -0,0 +1,410 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Abstractions over raw pointers. + +#![allow(missing_docs)] + +mod inner; +pub mod invariant; +mod ptr; +pub mod transmute; + +pub use inner::PtrInner; +pub use invariant::{BecauseExclusive, BecauseImmutable, Read}; +pub use ptr::{Ptr, TryWithError}; +pub use transmute::*; + +use crate::wrappers::ReadOnly; + +/// A shorthand for a maybe-valid, maybe-aligned reference. Used as the argument +/// to [`TryFromBytes::is_bit_valid`]. +/// +/// [`TryFromBytes::is_bit_valid`]: crate::TryFromBytes::is_bit_valid +pub type Maybe<'a, T, Alignment = invariant::Unaligned> = + Ptr<'a, ReadOnly<T>, (invariant::Shared, Alignment, invariant::Initialized)>; + +/// Checks if the referent is zeroed. +pub(crate) fn is_zeroed<T, I>(ptr: Ptr<'_, T, I>) -> bool +where + T: crate::Immutable + crate::KnownLayout, + I: invariant::Invariants<Validity = invariant::Initialized>, + I::Aliasing: invariant::Reference, +{ + ptr.as_bytes().as_ref().iter().all( + #[inline(always)] + |&byte| byte == 0, + ) +} + +pub mod cast { + use core::{marker::PhantomData, mem}; + + use crate::{ + layout::{SizeInfo, TrailingSliceLayout}, + HasField, KnownLayout, PtrInner, + }; + + /// A pointer cast or projection. + /// + /// # Safety + /// + /// The implementation of `project` must satisfy its safety post-condition. + pub unsafe trait Project<Src: ?Sized, Dst: ?Sized> { + /// Projects a pointer from `Src` to `Dst`. + /// + /// Users should generally not call `project` directly, and instead + /// should use high-level APIs like [`PtrInner::project`] or + /// [`Ptr::project`]. + /// + /// [`Ptr::project`]: crate::pointer::Ptr::project + /// + /// # Safety + /// + /// The returned pointer refers to a non-strict subset of the bytes of + /// `src`'s referent, and has the same provenance as `src`. + fn project(src: PtrInner<'_, Src>) -> *mut Dst; + } + + /// A [`Project`] which preserves the address of the referent – a pointer + /// cast. + /// + /// # Safety + /// + /// A `Cast` projection must preserve the address of the referent. It may + /// shrink the set of referent bytes, and it may change the referent's type. + pub unsafe trait Cast<Src: ?Sized, Dst: ?Sized>: Project<Src, Dst> {} + + /// A [`Cast`] which does not shrink the set of referent bytes. + /// + /// # Safety + /// + /// A `CastExact` projection must preserve the set of referent bytes. + pub unsafe trait CastExact<Src: ?Sized, Dst: ?Sized>: Cast<Src, Dst> {} + + /// A no-op pointer cast. + #[derive(Default, Copy, Clone)] + #[allow(missing_debug_implementations)] + pub struct IdCast; + + // SAFETY: `project` returns its argument unchanged, and so it is a + // provenance-preserving projection which preserves the set of referent + // bytes. + unsafe impl<T: ?Sized> Project<T, T> for IdCast { + #[inline(always)] + fn project(src: PtrInner<'_, T>) -> *mut T { + src.as_ptr() + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl<T: ?Sized> Cast<T, T> for IdCast {} + + // SAFETY: The `Project::project` impl preserves referent size. + unsafe impl<T: ?Sized> CastExact<T, T> for IdCast {} + + /// A pointer cast which preserves or shrinks the set of referent bytes of + /// a statically-sized referent. + /// + /// # Safety + /// + /// The implementation of [`Project`] uses a compile-time assertion to + /// guarantee that `Dst` is no larger than `Src`. Thus, `CastSized` has a + /// sound implementation of [`Project`] for all `Src` and `Dst` – the caller + /// may pass any `Src` and `Dst` without being responsible for soundness. + #[allow(missing_debug_implementations, missing_copy_implementations)] + pub enum CastSized {} + + // SAFETY: By the `static_assert!`, `Dst` is no larger than `Src`, + // and so all casts preserve or shrink the set of referent bytes. All + // operations preserve provenance. + unsafe impl<Src, Dst> Project<Src, Dst> for CastSized { + #[inline(always)] + fn project(src: PtrInner<'_, Src>) -> *mut Dst { + static_assert!(Src, Dst => mem::size_of::<Src>() >= mem::size_of::<Dst>()); + src.as_ptr().cast::<Dst>() + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl<Src, Dst> Cast<Src, Dst> for CastSized {} + + /// A pointer cast which preserves the set of referent bytes of a + /// statically-sized referent. + /// + /// # Safety + /// + /// The implementation of [`Project`] uses a compile-time assertion to + /// guarantee that `Dst` has the same size as `Src`. Thus, `CastSizedExact` + /// has a sound implementation of [`Project`] for all `Src` and `Dst` – the + /// caller may pass any `Src` and `Dst` without being responsible for + /// soundness. + #[allow(missing_debug_implementations, missing_copy_implementations)] + pub enum CastSizedExact {} + + // SAFETY: By the `static_assert!`, `Dst` has the same size as `Src`, + // and so all casts preserve the set of referent bytes. All operations + // preserve provenance. + unsafe impl<Src, Dst> Project<Src, Dst> for CastSizedExact { + #[inline(always)] + fn project(src: PtrInner<'_, Src>) -> *mut Dst { + static_assert!(Src, Dst => mem::size_of::<Src>() == mem::size_of::<Dst>()); + src.as_ptr().cast::<Dst>() + } + } + + // SAFETY: The `Project::project_raw` impl preserves referent address. + unsafe impl<Src, Dst> Cast<Src, Dst> for CastSizedExact {} + + // SAFETY: By the `static_assert!`, `Project::project_raw` impl preserves + // referent size. + unsafe impl<Src, Dst> CastExact<Src, Dst> for CastSizedExact {} + + /// A pointer cast which preserves or shrinks the set of referent bytes of + /// a dynamically-sized referent. + /// + /// # Safety + /// + /// The implementation of [`Project`] uses a compile-time assertion to + /// guarantee that the cast preserves the set of referent bytes. Thus, + /// `CastUnsized` has a sound implementation of [`Project`] for all `Src` + /// and `Dst` – the caller may pass any `Src` and `Dst` without being + /// responsible for soundness. + #[allow(missing_debug_implementations, missing_copy_implementations)] + pub enum CastUnsized {} + + // SAFETY: By the `static_assert!`, `Src` and `Dst` are either: + // - Both sized and equal in size + // - Both slice DSTs with the same trailing slice offset and element size + // and with align_of::<Src>() == align_of::<Dst>(). These ensure that any + // given pointer metadata encodes the same size for both `Src` and `Dst` + // (note that the alignment is required as it affects the amount of + // trailing padding). Thus, `project` preserves the set of referent bytes. + unsafe impl<Src, Dst> Project<Src, Dst> for CastUnsized + where + Src: ?Sized + KnownLayout, + Dst: ?Sized + KnownLayout<PointerMetadata = Src::PointerMetadata>, + { + #[inline(always)] + fn project(src: PtrInner<'_, Src>) -> *mut Dst { + // FIXME: Do we want this to support shrinking casts as well? If so, + // we'll need to remove the `CastExact` impl. + static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { + let src = <Src as KnownLayout>::LAYOUT; + let dst = <Dst as KnownLayout>::LAYOUT; + match (src.size_info, dst.size_info) { + (SizeInfo::Sized { size: src_size }, SizeInfo::Sized { size: dst_size }) => src_size == dst_size, + ( + SizeInfo::SliceDst(TrailingSliceLayout { offset: src_offset, elem_size: src_elem_size }), + SizeInfo::SliceDst(TrailingSliceLayout { offset: dst_offset, elem_size: dst_elem_size }) + ) => src.align.get() == dst.align.get() && src_offset == dst_offset && src_elem_size == dst_elem_size, + _ => false, + } + }); + + let metadata = Src::pointer_to_metadata(src.as_ptr()); + Dst::raw_from_ptr_len(src.as_non_null().cast::<u8>(), metadata).as_ptr() + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl<Src, Dst> Cast<Src, Dst> for CastUnsized + where + Src: ?Sized + KnownLayout, + Dst: ?Sized + KnownLayout<PointerMetadata = Src::PointerMetadata>, + { + } + + // SAFETY: By the `static_assert!` in `Project::project`, `Src` and `Dst` + // are either: + // - Both sized and equal in size + // - Both slice DSTs with the same alignment, trailing slice offset, and + // element size. These ensure that any given pointer metadata encodes the + // same size for both `Src` and `Dst` (note that the alignment is required + // as it affects the amount of trailing padding). + unsafe impl<Src, Dst> CastExact<Src, Dst> for CastUnsized + where + Src: ?Sized + KnownLayout, + Dst: ?Sized + KnownLayout<PointerMetadata = Src::PointerMetadata>, + { + } + + /// A field projection + /// + /// A `Projection` is a [`Project`] which implements projection by + /// delegating to an implementation of [`HasField::project`]. + #[allow(missing_debug_implementations, missing_copy_implementations)] + pub struct Projection<F: ?Sized, const VARIANT_ID: i128, const FIELD_ID: i128> { + _never: core::convert::Infallible, + _phantom: PhantomData<F>, + } + + // SAFETY: `HasField::project` has the same safety post-conditions as + // `Project::project`. + unsafe impl<T: ?Sized, F, const VARIANT_ID: i128, const FIELD_ID: i128> Project<T, T::Type> + for Projection<F, VARIANT_ID, FIELD_ID> + where + T: HasField<F, VARIANT_ID, FIELD_ID>, + { + #[inline(always)] + fn project(src: PtrInner<'_, T>) -> *mut T::Type { + T::project(src) + } + } + + // SAFETY: All `repr(C)` union fields exist at offset 0 within the union [1], + // and so any union projection is actually a cast (ie, preserves address). + // + // [1] Per + // https://doc.rust-lang.org/1.92.0/reference/type-layout.html#reprc-unions, + // it's not *technically* guaranteed that non-maximally-sized fields + // are at offset 0, but it's clear that this is the intention of `repr(C)` + // unions. It says: + // + // > A union declared with `#[repr(C)]` will have the same size and + // > alignment as an equivalent C union declaration in the C language for + // > the target platform. + // + // Note that this only mentions size and alignment, not layout. However, + // C unions *do* guarantee that all fields start at offset 0. [2] + // + // This is also reinforced by + // https://doc.rust-lang.org/1.92.0/reference/items/unions.html#r-items.union.fields.offset: + // + // > Fields might have a non-zero offset (except when the C + // > representation is used); in that case the bits starting at the + // > offset of the fields are read + // + // [2] Per https://port70.net/~nsz/c/c11/n1570.html#6.7.2.1p16: + // + // > The size of a union is sufficient to contain the largest of its + // > members. The value of at most one of the members can be stored in a + // > union object at any time. A pointer to a union object, suitably + // > converted, points to each of its members (or if a member is a + // > bit-field, then to the unit in which it resides), and vice versa. + // + // FIXME(https://github.com/rust-lang/unsafe-code-guidelines/issues/595): + // Cite the documentation once it's updated. + unsafe impl<T: ?Sized, F, const FIELD_ID: i128> Cast<T, T::Type> + for Projection<F, { crate::REPR_C_UNION_VARIANT_ID }, FIELD_ID> + where + T: HasField<F, { crate::REPR_C_UNION_VARIANT_ID }, FIELD_ID>, + { + } + + /// A transitive sequence of projections. + /// + /// Given `TU: Project` and `UV: Project`, `TransitiveProject<_, TU, UV>` is + /// a [`Project`] which projects by applying `TU` followed by `UV`. + /// + /// If `TU: Cast` and `UV: Cast`, then `TransitiveProject<_, TU, UV>: Cast`. + #[allow(missing_debug_implementations)] + pub struct TransitiveProject<U: ?Sized, TU, UV> { + _never: core::convert::Infallible, + _projections: PhantomData<(TU, UV)>, + // On our MSRV (1.56), the debuginfo for a tuple containing both an + // uninhabited type and a DST causes an ICE. We split `U` from `TU` and + // `UV` to avoid this situation. + _u: PhantomData<U>, + } + + // SAFETY: Since `TU::project` and `UV::project` are each + // provenance-preserving operations which preserve or shrink the set of + // referent bytes, so is their composition. + unsafe impl<T, U, V, TU, UV> Project<T, V> for TransitiveProject<U, TU, UV> + where + T: ?Sized, + U: ?Sized, + V: ?Sized, + TU: Project<T, U>, + UV: Project<U, V>, + { + #[inline(always)] + fn project(t: PtrInner<'_, T>) -> *mut V { + t.project::<_, TU>().project::<_, UV>().as_ptr() + } + } + + // SAFETY: Since the `Project::project` impl delegates to `TU::project` and + // `UV::project`, and since `TU` and `UV` are `Cast`, the `Project::project` + // impl preserves the address of the referent. + unsafe impl<T, U, V, TU, UV> Cast<T, V> for TransitiveProject<U, TU, UV> + where + T: ?Sized, + U: ?Sized, + V: ?Sized, + TU: Cast<T, U>, + UV: Cast<U, V>, + { + } + + // SAFETY: Since the `Project::project` impl delegates to `TU::project` and + // `UV::project`, and since `TU` and `UV` are `CastExact`, the `Project::project` + // impl preserves the set of referent bytes. + unsafe impl<T, U, V, TU, UV> CastExact<T, V> for TransitiveProject<U, TU, UV> + where + T: ?Sized, + U: ?Sized, + V: ?Sized, + TU: CastExact<T, U>, + UV: CastExact<U, V>, + { + } + + /// A cast from `T` to `[u8]`. + #[allow(missing_copy_implementations, missing_debug_implementations)] + pub struct AsBytesCast; + + // SAFETY: `project` constructs a pointer with the same address as `src` + // and with a referent of the same size as `*src`. It does this using + // provenance-preserving operations. + // + // FIXME(https://github.com/rust-lang/unsafe-code-guidelines/issues/594): + // Technically, this proof assumes that `*src` is contiguous (the same is + // true of other proofs in this codebase). Is this guaranteed anywhere? + unsafe impl<T: ?Sized + KnownLayout> Project<T, [u8]> for AsBytesCast { + #[inline(always)] + fn project(src: PtrInner<'_, T>) -> *mut [u8] { + let bytes = match T::size_of_val_raw(src.as_non_null()) { + Some(bytes) => bytes, + // SAFETY: `KnownLayout::size_of_val_raw` promises to always + // return `Some` so long as the resulting size fits in a + // `usize`. By invariant on `PtrInner`, `src` refers to a range + // of bytes whose size fits in an `isize`, which implies that it + // also fits in a `usize`. + None => unsafe { core::hint::unreachable_unchecked() }, + }; + + core::ptr::slice_from_raw_parts_mut(src.as_ptr().cast::<u8>(), bytes) + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl<T: ?Sized + KnownLayout> Cast<T, [u8]> for AsBytesCast {} + + // SAFETY: The `Project::project` impl preserves the set of referent bytes. + unsafe impl<T: ?Sized + KnownLayout> CastExact<T, [u8]> for AsBytesCast {} + + /// A cast from any type to `()`. + #[allow(missing_copy_implementations, missing_debug_implementations)] + pub struct CastToUnit; + + // SAFETY: The `project` implementation projects to a subset of its + // argument's referent using provenance-preserving operations. + unsafe impl<T: ?Sized> Project<T, ()> for CastToUnit { + #[inline(always)] + fn project(src: PtrInner<'_, T>) -> *mut () { + src.as_ptr().cast::<()>() + } + } + + // SAFETY: The `project` implementation preserves referent address. + unsafe impl<T: ?Sized> Cast<T, ()> for CastToUnit {} +} diff --git a/rust/zerocopy/src/pointer/ptr.rs b/rust/zerocopy/src/pointer/ptr.rs new file mode 100644 index 000000000000..7213f6f4a04e --- /dev/null +++ b/rust/zerocopy/src/pointer/ptr.rs @@ -0,0 +1,1586 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +#![allow(missing_docs)] + +use core::{ + fmt::{Debug, Formatter}, + marker::PhantomData, +}; + +use crate::{ + pointer::{ + inner::PtrInner, + invariant::*, + transmute::{MutationCompatible, SizeEq, TransmuteFromPtr}, + }, + AlignmentError, CastError, CastType, KnownLayout, SizeError, TryFromBytes, ValidityError, +}; + +/// Module used to gate access to [`Ptr`]'s fields. +mod def { + #[cfg(doc)] + use super::super::invariant; + use super::*; + + /// A raw pointer with more restrictions. + /// + /// `Ptr<T>` is similar to [`NonNull<T>`], but it is more restrictive in the + /// following ways (note that these requirements only hold of non-zero-sized + /// referents): + /// - It must derive from a valid allocation. + /// - It must reference a byte range which is contained inside the + /// allocation from which it derives. + /// - As a consequence, the byte range it references must have a size + /// which does not overflow `isize`. + /// + /// Depending on how `Ptr` is parameterized, it may have additional + /// invariants: + /// - `ptr` conforms to the aliasing invariant of + /// [`I::Aliasing`](invariant::Aliasing). + /// - `ptr` conforms to the alignment invariant of + /// [`I::Alignment`](invariant::Alignment). + /// - `ptr` conforms to the validity invariant of + /// [`I::Validity`](invariant::Validity). + /// + /// `Ptr<'a, T>` is [covariant] in `'a` and invariant in `T`. + /// + /// [`NonNull<T>`]: core::ptr::NonNull + /// [covariant]: https://doc.rust-lang.org/reference/subtyping.html + pub struct Ptr<'a, T, I> + where + T: ?Sized, + I: Invariants, + { + /// # Invariants + /// + /// 0. `ptr` conforms to the aliasing invariant of + /// [`I::Aliasing`](invariant::Aliasing). + /// 1. `ptr` conforms to the alignment invariant of + /// [`I::Alignment`](invariant::Alignment). + /// 2. `ptr` conforms to the validity invariant of + /// [`I::Validity`](invariant::Validity). + // SAFETY: `PtrInner<'a, T>` is covariant in `'a` and invariant in `T`. + ptr: PtrInner<'a, T>, + _invariants: PhantomData<I>, + } + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + /// Constructs a new `Ptr` from a [`PtrInner`]. + /// + /// # Safety + /// + /// The caller promises that: + /// + /// 0. `ptr` conforms to the aliasing invariant of + /// [`I::Aliasing`](invariant::Aliasing). + /// 1. `ptr` conforms to the alignment invariant of + /// [`I::Alignment`](invariant::Alignment). + /// 2. `ptr` conforms to the validity invariant of + /// [`I::Validity`](invariant::Validity). + pub(crate) unsafe fn from_inner(ptr: PtrInner<'a, T>) -> Ptr<'a, T, I> { + // SAFETY: The caller has promised to satisfy all safety invariants + // of `Ptr`. + Self { ptr, _invariants: PhantomData } + } + + /// Converts this `Ptr<T>` to a [`PtrInner<T>`]. + /// + /// Note that this method does not consume `self`. The caller should + /// watch out for `unsafe` code which uses the returned value in a way + /// that violates the safety invariants of `self`. + #[inline] + #[must_use] + pub fn as_inner(&self) -> PtrInner<'a, T> { + self.ptr + } + } +} + +#[allow(unreachable_pub)] // This is a false positive on our MSRV toolchain. +pub use def::Ptr; + +/// External trait implementations on [`Ptr`]. +mod _external { + use super::*; + + /// SAFETY: Shared pointers are safely `Copy`. `Ptr`'s other invariants + /// (besides aliasing) are unaffected by the number of references that exist + /// to `Ptr`'s referent. The notable cases are: + /// - Alignment is a property of the referent type (`T`) and the address, + /// both of which are unchanged + /// - Let `S(T, V)` be the set of bit values permitted to appear in the + /// referent of a `Ptr<T, I: Invariants<Validity = V>>`. Since this copy + /// does not change `I::Validity` or `T`, `S(T, I::Validity)` is also + /// unchanged. + /// + /// We are required to guarantee that the referents of the original `Ptr` + /// and of the copy (which, of course, are actually the same since they + /// live in the same byte address range) both remain in the set `S(T, + /// I::Validity)`. Since this invariant holds on the original `Ptr`, it + /// cannot be violated by the original `Ptr`, and thus the original `Ptr` + /// cannot be used to violate this invariant on the copy. The inverse + /// holds as well. + impl<'a, T, I> Copy for Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants<Aliasing = Shared>, + { + } + + /// SAFETY: See the safety comment on `Copy`. + impl<'a, T, I> Clone for Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants<Aliasing = Shared>, + { + #[inline] + fn clone(&self) -> Self { + *self + } + } + + impl<'a, T, I> Debug for Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + self.as_inner().as_non_null().fmt(f) + } + } +} + +/// Methods for converting to and from `Ptr` and Rust's safe reference types. +mod _conversions { + use super::*; + use crate::pointer::cast::{CastExact, CastSized, IdCast}; + + /// `&'a T` → `Ptr<'a, T>` + impl<'a, T> Ptr<'a, T, (Shared, Aligned, Valid)> + where + T: 'a + ?Sized, + { + /// Constructs a `Ptr` from a shared reference. + #[inline(always)] + pub fn from_ref(ptr: &'a T) -> Self { + let inner = PtrInner::from_ref(ptr); + // SAFETY: + // 0. `ptr`, by invariant on `&'a T`, conforms to the aliasing + // invariant of `Shared`. + // 1. `ptr`, by invariant on `&'a T`, conforms to the alignment + // invariant of `Aligned`. + // 2. `ptr`'s referent, by invariant on `&'a T`, is a bit-valid `T`. + // This satisfies the requirement that a `Ptr<T, (_, _, Valid)>` + // point to a bit-valid `T`. Even if `T` permits interior + // mutation, this invariant guarantees that the returned `Ptr` + // can only ever be used to modify the referent to store + // bit-valid `T`s, which ensures that the returned `Ptr` cannot + // be used to violate the soundness of the original `ptr: &'a T` + // or of any other references that may exist to the same + // referent. + unsafe { Self::from_inner(inner) } + } + } + + /// `&'a mut T` → `Ptr<'a, T>` + impl<'a, T> Ptr<'a, T, (Exclusive, Aligned, Valid)> + where + T: 'a + ?Sized, + { + /// Constructs a `Ptr` from an exclusive reference. + #[inline(always)] + pub fn from_mut(ptr: &'a mut T) -> Self { + let inner = PtrInner::from_mut(ptr); + // SAFETY: + // 0. `ptr`, by invariant on `&'a mut T`, conforms to the aliasing + // invariant of `Exclusive`. + // 1. `ptr`, by invariant on `&'a mut T`, conforms to the alignment + // invariant of `Aligned`. + // 2. `ptr`'s referent, by invariant on `&'a mut T`, is a bit-valid + // `T`. This satisfies the requirement that a `Ptr<T, (_, _, + // Valid)>` point to a bit-valid `T`. This invariant guarantees + // that the returned `Ptr` can only ever be used to modify the + // referent to store bit-valid `T`s, which ensures that the + // returned `Ptr` cannot be used to violate the soundness of the + // original `ptr: &'a mut T`. + unsafe { Self::from_inner(inner) } + } + } + + /// `Ptr<'a, T>` → `&'a T` + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants<Alignment = Aligned, Validity = Valid>, + I::Aliasing: Reference, + { + /// Converts `self` to a shared reference. + // This consumes `self`, not `&self`, because `self` is, logically, a + // pointer. For `I::Aliasing = invariant::Shared`, `Self: Copy`, and so + // this doesn't prevent the caller from still using the pointer after + // calling `as_ref`. + #[allow(clippy::wrong_self_convention)] + #[inline] + #[must_use] + pub fn as_ref(self) -> &'a T { + let raw = self.as_inner().as_non_null(); + // SAFETY: `self` satisfies the `Aligned` invariant, so we know that + // `raw` is validly-aligned for `T`. + #[cfg(miri)] + unsafe { + crate::util::miri_promise_symbolic_alignment( + raw.as_ptr().cast(), + core::mem::align_of_val_raw(raw.as_ptr()), + ); + } + // SAFETY: This invocation of `NonNull::as_ref` satisfies its + // documented safety preconditions: + // + // 1. The pointer is properly aligned. This is ensured by-contract + // on `Ptr`, because the `I::Alignment` is `Aligned`. + // + // 2. If the pointer's referent is not zero-sized, then the pointer + // must be “dereferenceable” in the sense defined in the module + // documentation; i.e.: + // + // > The memory range of the given size starting at the pointer + // > must all be within the bounds of a single allocated object. + // > [2] + // + // This is ensured by contract on all `PtrInner`s. + // + // 3. The pointer must point to a validly-initialized instance of + // `T`. This is ensured by-contract on `Ptr`, because the + // `I::Validity` is `Valid`. + // + // 4. You must enforce Rust’s aliasing rules. This is ensured by + // contract on `Ptr`, because `I::Aliasing: Reference`. Either it + // is `Shared` or `Exclusive`. If it is `Shared`, other + // references may not mutate the referent outside of + // `UnsafeCell`s. + // + // [1]: https://doc.rust-lang.org/std/ptr/struct.NonNull.html#method.as_ref + // [2]: https://doc.rust-lang.org/std/ptr/index.html#safety + unsafe { raw.as_ref() } + } + } + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + I::Aliasing: Reference, + { + /// Reborrows `self`, producing another `Ptr`. + /// + /// Since `self` is borrowed mutably, this prevents any methods from + /// being called on `self` as long as the returned `Ptr` exists. + #[inline] + #[must_use] + #[allow(clippy::needless_lifetimes)] // Allows us to name the lifetime in the safety comment below. + pub fn reborrow<'b>(&'b mut self) -> Ptr<'b, T, I> + where + 'a: 'b, + { + // SAFETY: The following all hold by invariant on `self`, and thus + // hold of `ptr = self.as_inner()`: + // 0. SEE BELOW. + // 1. `ptr` conforms to the alignment invariant of + // [`I::Alignment`](invariant::Alignment). + // 2. `ptr` conforms to the validity invariant of + // [`I::Validity`](invariant::Validity). `self` and the returned + // `Ptr` permit the same bit values in their referents since they + // have the same referent type (`T`) and the same validity + // (`I::Validity`). Thus, regardless of what mutation is + // permitted (`Exclusive` aliasing or `Shared`-aliased interior + // mutation), neither can be used to write a value to the + // referent which violates the other's validity invariant. + // + // For aliasing (0 above), since `I::Aliasing: Reference`, + // there are two cases for `I::Aliasing`: + // - For `invariant::Shared`: `'a` outlives `'b`, and so the + // returned `Ptr` does not permit accessing the referent any + // longer than is possible via `self`. For shared aliasing, it is + // sound for multiple `Ptr`s to exist simultaneously which + // reference the same memory, so creating a new one is not + // problematic. + // - For `invariant::Exclusive`: Since `self` is `&'b mut` and we + // return a `Ptr` with lifetime `'b`, `self` is inaccessible to + // the caller for the lifetime `'b` - in other words, `self` is + // inaccessible to the caller as long as the returned `Ptr` + // exists. Since `self` is an exclusive `Ptr`, no other live + // references or `Ptr`s may exist which refer to the same memory + // while `self` is live. Thus, as long as the returned `Ptr` + // exists, no other references or `Ptr`s which refer to the same + // memory may be live. + unsafe { Ptr::from_inner(self.as_inner()) } + } + + /// Reborrows `self` as shared, producing another `Ptr` with `Shared` + /// aliasing. + /// + /// Since `self` is borrowed mutably, this prevents any methods from + /// being called on `self` as long as the returned `Ptr` exists. + #[inline] + #[must_use] + #[allow(clippy::needless_lifetimes)] // Allows us to name the lifetime in the safety comment below. + pub fn reborrow_shared<'b>(&'b mut self) -> Ptr<'b, T, (Shared, I::Alignment, I::Validity)> + where + 'a: 'b, + { + // SAFETY: The following all hold by invariant on `self`, and thus + // hold of `ptr = self.as_inner()`: + // 0. SEE BELOW. + // 1. `ptr` conforms to the alignment invariant of + // [`I::Alignment`](invariant::Alignment). + // 2. `ptr` conforms to the validity invariant of + // [`I::Validity`](invariant::Validity). `self` and the returned + // `Ptr` permit the same bit values in their referents since they + // have the same referent type (`T`) and the same validity + // (`I::Validity`). Thus, regardless of what mutation is + // permitted (`Exclusive` aliasing or `Shared`-aliased interior + // mutation), neither can be used to write a value to the + // referent which violates the other's validity invariant. + // + // For aliasing (0 above), since `I::Aliasing: Reference`, + // there are two cases for `I::Aliasing`: + // - For `invariant::Shared`: `'a` outlives `'b`, and so the + // returned `Ptr` does not permit accessing the referent any + // longer than is possible via `self`. For shared aliasing, it is + // sound for multiple `Ptr`s to exist simultaneously which + // reference the same memory, so creating a new one is not + // problematic. + // - For `invariant::Exclusive`: Since `self` is `&'b mut` and we + // return a `Ptr` with lifetime `'b`, `self` is inaccessible to + // the caller for the lifetime `'b` - in other words, `self` is + // inaccessible to the caller as long as the returned `Ptr` + // exists. Since `self` is an exclusive `Ptr`, no other live + // references or `Ptr`s may exist which refer to the same memory + // while `self` is live. Thus, as long as the returned `Ptr` + // exists, no other references or `Ptr`s which refer to the same + // memory may be live. + unsafe { Ptr::from_inner(self.as_inner()) } + } + } + + /// `Ptr<'a, T>` → `&'a mut T` + impl<'a, T> Ptr<'a, T, (Exclusive, Aligned, Valid)> + where + T: 'a + ?Sized, + { + /// Converts `self` to a mutable reference. + #[allow(clippy::wrong_self_convention)] + #[inline] + #[must_use] + pub fn as_mut(self) -> &'a mut T { + let mut raw = self.as_inner().as_non_null(); + // SAFETY: `self` satisfies the `Aligned` invariant, so we know that + // `raw` is validly-aligned for `T`. + #[cfg(miri)] + unsafe { + crate::util::miri_promise_symbolic_alignment( + raw.as_ptr().cast(), + core::mem::align_of_val_raw(raw.as_ptr()), + ); + } + // SAFETY: This invocation of `NonNull::as_mut` satisfies its + // documented safety preconditions: + // + // 1. The pointer is properly aligned. This is ensured by-contract + // on `Ptr`, because the `ALIGNMENT_INVARIANT` is `Aligned`. + // + // 2. If the pointer's referent is not zero-sized, then the pointer + // must be “dereferenceable” in the sense defined in the module + // documentation; i.e.: + // + // > The memory range of the given size starting at the pointer + // > must all be within the bounds of a single allocated object. + // > [2] + // + // This is ensured by contract on all `PtrInner`s. + // + // 3. The pointer must point to a validly-initialized instance of + // `T`. This is ensured by-contract on `Ptr`, because the + // validity invariant is `Valid`. + // + // 4. You must enforce Rust’s aliasing rules. This is ensured by + // contract on `Ptr`, because the `ALIASING_INVARIANT` is + // `Exclusive`. + // + // [1]: https://doc.rust-lang.org/std/ptr/struct.NonNull.html#method.as_mut + // [2]: https://doc.rust-lang.org/std/ptr/index.html#safety + unsafe { raw.as_mut() } + } + } + + /// `Ptr<'a, T>` → `Ptr<'a, U>` + impl<'a, T: ?Sized, I> Ptr<'a, T, I> + where + I: Invariants, + { + #[must_use] + #[inline(always)] + pub fn transmute<U, V, R>(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, V)> + where + V: Validity, + U: TransmuteFromPtr<T, I::Aliasing, I::Validity, V, <U as SizeEq<T>>::CastFrom, R> + + SizeEq<T> + + ?Sized, + { + self.transmute_with::<U, V, <U as SizeEq<T>>::CastFrom, R>() + } + + #[inline] + #[must_use] + pub fn transmute_with<U, V, C, R>(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, V)> + where + V: Validity, + U: TransmuteFromPtr<T, I::Aliasing, I::Validity, V, C, R> + ?Sized, + C: CastExact<T, U>, + { + // SAFETY: + // - By `C: CastExact`, `C` preserves referent address, and so we + // don't need to consider projections in the following safety + // arguments. + // - If aliasing is `Shared`, then by `U: TransmuteFromPtr<T>`, at + // least one of the following holds: + // - `T: Immutable` and `U: Immutable`, in which case it is + // trivially sound for shared code to operate on a `&T` and `&U` + // at the same time, as neither can perform interior mutation + // - It is directly guaranteed that it is sound for shared code to + // operate on these references simultaneously + // - By `U: TransmuteFromPtr<T, I::Aliasing, I::Validity, C, V>`, it + // is sound to perform this transmute using `C`. + unsafe { self.project_transmute_unchecked::<_, _, C>() } + } + + #[inline] + #[must_use] + pub fn recall_validity<V, R>(self) -> Ptr<'a, T, (I::Aliasing, I::Alignment, V)> + where + V: Validity, + T: TransmuteFromPtr<T, I::Aliasing, I::Validity, V, IdCast, R>, + { + let ptr = self.transmute_with::<T, V, IdCast, R>(); + // SAFETY: `self` and `ptr` have the same address and referent type. + // Therefore, if `self` satisfies `I::Alignment`, then so does + // `ptr`. + unsafe { ptr.assume_alignment::<I::Alignment>() } + } + + /// Projects and/or transmutes to a different (unsized) referent type + /// without checking interior mutability. + /// + /// Callers should prefer [`cast`] or [`project`] where possible. + /// + /// [`cast`]: Ptr::cast + /// [`project`]: Ptr::project + /// + /// # Safety + /// + /// The caller promises that: + /// - If `I::Aliasing` is [`Shared`], it must not be possible for safe + /// code, operating on a `&T` and `&U`, with the referents of `self` + /// and `self.project_transmute_unchecked()`, respectively, to cause + /// undefined behavior. + /// - It is sound to project and/or transmute a pointer of type `T` with + /// aliasing `I::Aliasing` and validity `I::Validity` to a pointer of + /// type `U` with aliasing `I::Aliasing` and validity `V`. This is a + /// subtle soundness requirement that is a function of `T`, `U`, + /// `I::Aliasing`, `I::Validity`, and `V`, and may depend upon the + /// presence, absence, or specific location of `UnsafeCell`s in `T` + /// and/or `U`, and on whether interior mutation is ever permitted via + /// those `UnsafeCell`s. See [`Validity`] for more details. + #[inline] + #[must_use] + pub unsafe fn project_transmute_unchecked<U: ?Sized, V, P>( + self, + ) -> Ptr<'a, U, (I::Aliasing, Unaligned, V)> + where + V: Validity, + P: crate::pointer::cast::Project<T, U>, + { + let ptr = self.as_inner().project::<_, P>(); + + // SAFETY: + // + // The following safety arguments rely on the fact that `P: Project` + // guarantees that `P` is a referent-preserving or -shrinking + // projection. Thus, `ptr` addresses a subset of the bytes of + // `*self`, and so certain properties that hold of `*self` also hold + // of `*ptr`. + // + // 0. `ptr` conforms to the aliasing invariant of `I::Aliasing`: + // - `Exclusive`: `self` is the only `Ptr` or reference which is + // permitted to read or modify the referent for the lifetime + // `'a`. Since we consume `self` by value, the returned pointer + // remains the only `Ptr` or reference which is permitted to + // read or modify the referent for the lifetime `'a`. + // - `Shared`: Since `self` has aliasing `Shared`, we know that + // no other code may mutate the referent during the lifetime + // `'a`, except via `UnsafeCell`s, and except as permitted by + // `T`'s library safety invariants. The caller promises that + // any safe operations which can be permitted on a `&T` and a + // `&U` simultaneously must be sound. Thus, no operations on a + // `&U` could violate `&T`'s library safety invariants, and + // vice-versa. Since any mutation via shared references outside + // of `UnsafeCell`s is unsound, this must be impossible using + // `&T` and `&U`. + // - `Inaccessible`: There are no restrictions we need to uphold. + // 1. `ptr` trivially satisfies the alignment invariant `Unaligned`. + // 2. The caller promises that the returned pointer satisfies the + // validity invariant `V` with respect to its referent type, `U`. + unsafe { Ptr::from_inner(ptr) } + } + } + + /// `Ptr<'a, T, (_, _, _)>` → `Ptr<'a, Unalign<T>, (_, Aligned, _)>` + impl<'a, T, I> Ptr<'a, T, I> + where + I: Invariants, + { + /// Converts a `Ptr` an unaligned `T` into a `Ptr` to an aligned + /// `Unalign<T>`. + #[inline] + #[must_use] + pub fn into_unalign( + self, + ) -> Ptr<'a, crate::Unalign<T>, (I::Aliasing, Aligned, I::Validity)> { + // FIXME(#1359): This should be a `transmute_with` call. + // Unfortunately, to avoid blanket impl conflicts, we only implement + // `TransmuteFrom<T>` for `Unalign<T>` (and vice versa) specifically + // for `Valid` validity, not for all validity types. + + // SAFETY: + // - By `CastSized: Cast`, `CastSized` preserves referent address, + // and so we don't need to consider projections in the following + // safety arguments. + // - Since `Unalign<T>` has the same layout as `T`, the returned + // pointer refers to `UnsafeCell`s at the same locations as + // `self`. + // - `Unalign<T>` promises to have the same bit validity as `T`. By + // invariant on `Validity`, the set of bit patterns allowed in the + // referent of a `Ptr<X, (_, _, V)>` is only a function of the + // validity of `X` and of `V`. Thus, the set of bit patterns + // allowed in the referent of a `Ptr<T, (_, _, I::Validity)>` is + // the same as the set of bit patterns allowed in the referent of + // a `Ptr<Unalign<T>, (_, _, I::Validity)>`. As a result, `self` + // and the returned `Ptr` permit the same set of bit patterns in + // their referents, and so neither can be used to violate the + // validity of the other. + let ptr = unsafe { self.project_transmute_unchecked::<_, _, CastSized>() }; + ptr.bikeshed_recall_aligned() + } + } + + impl<'a, T, I> Ptr<'a, T, I> + where + T: ?Sized, + I: Invariants<Validity = Valid>, + I::Aliasing: Reference, + { + /// Reads the referent. + #[must_use] + #[inline(always)] + pub fn read<R>(self) -> T + where + T: Copy, + T: Read<I::Aliasing, R>, + { + <I::Alignment as Alignment>::read(self) + } + + /// Views the value as an aligned reference. + /// + /// This is only available if `T` is [`Unaligned`]. + #[must_use] + #[inline] + pub fn unaligned_as_ref(self) -> &'a T + where + T: crate::Unaligned, + { + self.bikeshed_recall_aligned().as_ref() + } + } +} + +/// State transitions between invariants. +mod _transitions { + use super::*; + use crate::{ + pointer::{cast::IdCast, transmute::TryTransmuteFromPtr}, + ReadOnly, + }; + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + /// Assumes that `self` satisfies the invariants `H`. + /// + /// # Safety + /// + /// The caller promises that `self` satisfies the invariants `H`. + unsafe fn assume_invariants<H: Invariants>(self) -> Ptr<'a, T, H> { + // SAFETY: The caller has promised to satisfy all parameterized + // invariants of `Ptr`. `Ptr`'s other invariants are satisfied + // by-contract by the source `Ptr`. + unsafe { Ptr::from_inner(self.as_inner()) } + } + + /// Helps the type system unify two distinct invariant types which are + /// actually the same. + #[inline] + #[must_use] + pub fn unify_invariants< + H: Invariants<Aliasing = I::Aliasing, Alignment = I::Alignment, Validity = I::Validity>, + >( + self, + ) -> Ptr<'a, T, H> { + // SAFETY: The associated type bounds on `H` ensure that the + // invariants are unchanged. + unsafe { self.assume_invariants::<H>() } + } + + /// Assumes that `self`'s referent is validly-aligned for `T` if + /// required by `A`. + /// + /// # Safety + /// + /// The caller promises that `self`'s referent conforms to the alignment + /// invariant of `T` if required by `A`. + #[inline] + pub(crate) unsafe fn assume_alignment<A: Alignment>( + self, + ) -> Ptr<'a, T, (I::Aliasing, A, I::Validity)> { + // SAFETY: The caller promises that `self`'s referent is + // well-aligned for `T` if required by `A` . + unsafe { self.assume_invariants() } + } + + /// Checks the `self`'s alignment at runtime, returning an aligned `Ptr` + /// on success. + #[inline] + pub fn try_into_aligned( + self, + ) -> Result<Ptr<'a, T, (I::Aliasing, Aligned, I::Validity)>, AlignmentError<Self, T>> + where + T: Sized, + { + if let Err(err) = + crate::util::validate_aligned_to::<_, T>(self.as_inner().as_non_null()) + { + return Err(err.with_src(self)); + } + + // SAFETY: We just checked the alignment. + Ok(unsafe { self.assume_alignment::<Aligned>() }) + } + + /// Recalls that `self`'s referent is validly-aligned for `T`. + #[inline] + // FIXME(#859): Reconsider the name of this method before making it + // public. + #[must_use] + pub fn bikeshed_recall_aligned(self) -> Ptr<'a, T, (I::Aliasing, Aligned, I::Validity)> + where + T: crate::Unaligned, + { + // SAFETY: The bound `T: Unaligned` ensures that `T` has no + // non-trivial alignment requirement. + unsafe { self.assume_alignment::<Aligned>() } + } + + /// Assumes that `self`'s referent conforms to the validity requirement + /// of `V`. + /// + /// # Safety + /// + /// The caller promises that `self`'s referent conforms to the validity + /// requirement of `V`. + #[must_use] + #[inline] + pub unsafe fn assume_validity<V: Validity>( + self, + ) -> Ptr<'a, T, (I::Aliasing, I::Alignment, V)> { + // SAFETY: The caller promises that `self`'s referent conforms to + // the validity requirement of `V`. + unsafe { self.assume_invariants() } + } + + /// A shorthand for `self.assume_validity<invariant::Initialized>()`. + /// + /// # Safety + /// + /// The caller promises to uphold the safety preconditions of + /// `self.assume_validity<invariant::Initialized>()`. + #[must_use] + #[inline] + pub unsafe fn assume_initialized( + self, + ) -> Ptr<'a, T, (I::Aliasing, I::Alignment, Initialized)> { + // SAFETY: The caller has promised to uphold the safety + // preconditions. + unsafe { self.assume_validity::<Initialized>() } + } + + /// A shorthand for `self.assume_validity<Valid>()`. + /// + /// # Safety + /// + /// The caller promises to uphold the safety preconditions of + /// `self.assume_validity<Valid>()`. + #[must_use] + #[inline] + pub unsafe fn assume_valid(self) -> Ptr<'a, T, (I::Aliasing, I::Alignment, Valid)> { + // SAFETY: The caller has promised to uphold the safety + // preconditions. + unsafe { self.assume_validity::<Valid>() } + } + + /// Checks that `self`'s referent is validly initialized for `T`, + /// returning a `Ptr` with `Valid` on success. + /// + /// # Panics + /// + /// This method will panic if + /// [`T::is_bit_valid`][TryFromBytes::is_bit_valid] panics. + /// + /// # Safety + /// + /// On error, unsafe code may rely on this method's returned + /// `ValidityError` containing `self`. + #[inline] + pub fn try_into_valid<R, S>( + mut self, + ) -> Result<Ptr<'a, T, (I::Aliasing, I::Alignment, Valid)>, ValidityError<Self, T>> + where + T: TryFromBytes + + Read<I::Aliasing, R> + + TryTransmuteFromPtr<T, I::Aliasing, I::Validity, Valid, IdCast, S>, + ReadOnly<T>: Read<I::Aliasing, R>, + I::Aliasing: Reference, + I: Invariants<Validity = Initialized>, + { + // This call may panic. If that happens, it doesn't cause any + // soundness issues, as we have not generated any invalid state + // which we need to fix before returning. + if T::is_bit_valid(self.reborrow().transmute::<_, _, _>().reborrow_shared()) { + // SAFETY: If `T::is_bit_valid`, code may assume that `self` + // contains a bit-valid instance of `T`. By `T: + // TryTransmuteFromPtr<T, I::Aliasing, I::Validity, Valid>`, so + // long as `self`'s referent conforms to the `Valid` validity + // for `T` (which we just confirmed), then this transmute is + // sound. + Ok(unsafe { self.assume_valid() }) + } else { + Err(ValidityError::new(self)) + } + } + + /// Forgets that `self`'s referent is validly-aligned for `T`. + #[inline] + #[must_use] + pub fn forget_aligned(self) -> Ptr<'a, T, (I::Aliasing, Unaligned, I::Validity)> { + // SAFETY: `Unaligned` is less restrictive than `Aligned`. + unsafe { self.assume_invariants() } + } + } +} + +/// Casts of the referent type. +#[cfg_attr(not(zerocopy_unstable_ptr), allow(unreachable_pub))] +pub use _casts::TryWithError; +mod _casts { + use core::cell::UnsafeCell; + + use super::*; + use crate::{ + pointer::cast::{AsBytesCast, Cast}, + HasTag, ProjectField, + }; + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized, + I: Invariants, + { + /// Casts to a different referent type without checking interior + /// mutability. + /// + /// Callers should prefer [`cast`][Ptr::cast] where possible. + /// + /// # Safety + /// + /// If `I::Aliasing` is [`Shared`], it must not be possible for safe + /// code, operating on a `&T` and `&U` with the same referent + /// simultaneously, to cause undefined behavior. + #[inline] + #[must_use] + pub unsafe fn cast_unchecked<U, C: Cast<T, U>>( + self, + ) -> Ptr<'a, U, (I::Aliasing, Unaligned, I::Validity)> + where + U: 'a + CastableFrom<T, I::Validity, I::Validity> + ?Sized, + { + // SAFETY: + // - By `C: Cast`, `C` preserves the address of the referent. + // - If `I::Aliasing` is [`Shared`], the caller promises that it + // is not possible for safe code, operating on a `&T` and `&U` + // with the same referent simultaneously, to cause undefined + // behavior. + // - By `U: CastableFrom<T, I::Validity, I::Validity>`, + // `I::Validity` is either `Uninit` or `Initialized`. In both + // cases, the bit validity `I::Validity` has the same semantics + // regardless of referent type. In other words, the set of allowed + // referent values for `Ptr<T, (_, _, I::Validity)>` and `Ptr<U, + // (_, _, I::Validity)>` are identical. As a consequence, neither + // `self` nor the returned `Ptr` can be used to write values which + // are invalid for the other. + unsafe { self.project_transmute_unchecked::<_, _, C>() } + } + + /// Casts to a different referent type. + #[inline] + #[must_use] + pub fn cast<U, C, R>(self) -> Ptr<'a, U, (I::Aliasing, Unaligned, I::Validity)> + where + T: MutationCompatible<U, I::Aliasing, I::Validity, I::Validity, R>, + U: 'a + ?Sized + CastableFrom<T, I::Validity, I::Validity>, + C: Cast<T, U>, + { + // SAFETY: Because `T: MutationCompatible<U, I::Aliasing, R>`, one + // of the following holds: + // - `T: Read<I::Aliasing>` and `U: Read<I::Aliasing>`, in which + // case one of the following holds: + // - `I::Aliasing` is `Exclusive` + // - `T` and `U` are both `Immutable` + // - It is sound for safe code to operate on `&T` and `&U` with the + // same referent simultaneously. + unsafe { self.cast_unchecked::<_, C>() } + } + + #[inline(always)] + pub fn project<F, const VARIANT_ID: i128, const FIELD_ID: i128>( + mut self, + ) -> Result<Ptr<'a, T::Type, T::Invariants>, T::Error> + where + T: ProjectField<F, I, VARIANT_ID, FIELD_ID>, + I::Aliasing: Reference, + { + use crate::pointer::cast::Projection; + match T::is_projectable(self.reborrow().project_tag()) { + Ok(()) => { + let inner = self.as_inner(); + let projected = inner.project::<_, Projection<F, VARIANT_ID, FIELD_ID>>(); + // SAFETY: By `T: ProjectField<F, I, VARIANT_ID, FIELD_ID>`, + // for `self: Ptr<'_, T, I>` such that `T::is_projectable` + // (which we've verified in this match arm), + // `T::project(self.as_inner())` conforms to + // `T::Invariants`. The `projected` pointer satisfies these + // invariants because it is produced by way of an + // abstraction that is equivalent to + // `T::project(ptr.as_inner())`: by invariant on + // `PtrInner::project`, `projected` is guaranteed to address + // the subset of the bytes of `inner`'s referent addressed + // by `Projection::project(inner)`, and by invariant on + // `Projection`, `Projection::project` is implemented by + // delegating to an implementation of `HasField::project`. + Ok(unsafe { Ptr::from_inner(projected) }) + } + Err(err) => Err(err), + } + } + + #[must_use] + #[inline(always)] + pub(crate) fn project_tag(self) -> Ptr<'a, T::Tag, I> + where + T: HasTag, + { + // SAFETY: By invariant on `Self::ProjectToTag`, this is a sound + // projection. + let tag = unsafe { self.project_transmute_unchecked::<_, _, T::ProjectToTag>() }; + // SAFETY: By invariant on `Self::ProjectToTag`, the projected + // pointer has the same alignment as `ptr`. + let tag = unsafe { tag.assume_alignment() }; + tag.unify_invariants() + } + + /// Attempts to transform the pointer, restoring the original on + /// failure. + /// + /// # Safety + /// + /// If `I::Aliasing != Shared`, then if `f` returns `Err(err)`, no copy + /// of `f`'s argument must exist outside of `err`. + #[inline(always)] + pub(crate) unsafe fn try_with_unchecked<U, J, E, F>( + self, + f: F, + ) -> Result<Ptr<'a, U, J>, E::Mapped> + where + U: 'a + ?Sized, + J: Invariants<Aliasing = I::Aliasing>, + E: TryWithError<Self>, + F: FnOnce(Ptr<'a, T, I>) -> Result<Ptr<'a, U, J>, E>, + { + let old_inner = self.as_inner(); + #[rustfmt::skip] + let res = f(self).map_err(#[inline(always)] move |err: E| { + err.map(#[inline(always)] |src| { + drop(src); + + // SAFETY: + // 0. Aliasing is either `Shared` or `Exclusive`: + // - If aliasing is `Shared`, then it cannot violate + // aliasing make another copy of this pointer (in fact, + // using `I::Aliasing = Shared`, we could have just + // cloned `self`). + // - If aliasing is `Exclusive`, then `f` is not allowed + // to make another copy of `self`. In `map_err`, we are + // consuming the only value in the returned `Result`. + // By invariant on `E: TryWithError<Self>`, that `err: + // E` only contains a single `Self` and no other + // non-ZST fields which could be `Ptr`s or references + // to `self`'s referent. By the same invariant, `map` + // consumes this single `Self` and passes it to this + // closure. Since `self` was, by invariant on + // `Exclusive`, the only `Ptr` or reference live for + // `'a` with this referent, and since we `drop(src)` + // above, there are no copies left, and so we are + // creating the only copy. + // 1. `self` conforms to `I::Aliasing` by invariant on + // `Ptr`, and `old_inner` has the same address, so it + // does too. + // 2. `f` could not have violated `self`'s validity without + // itself being unsound. Assuming that `f` is sound, the + // referent of `self` is still valid for `T`. + unsafe { Ptr::from_inner(old_inner) } + }) + }); + res + } + + /// Attempts to transform the pointer, restoring the original on + /// failure. + #[inline(always)] + pub fn try_with<U, J, E, F>(self, f: F) -> Result<Ptr<'a, U, J>, E::Mapped> + where + U: 'a + ?Sized, + J: Invariants<Aliasing = I::Aliasing>, + E: TryWithError<Self>, + F: FnOnce(Ptr<'a, T, I>) -> Result<Ptr<'a, U, J>, E>, + I: Invariants<Aliasing = Shared>, + { + // SAFETY: `I::Aliasing = Shared`, so the safety condition does not + // apply. + unsafe { self.try_with_unchecked(f) } + } + } + + /// # Safety + /// + /// `Self` only contains a single `Self::Inner`, and `Self::Mapped` only + /// contains a single `MappedInner`. Other than that, `Self` and + /// `Self::Mapped` contain no non-ZST fields. + /// + /// `map` must pass ownership of `self`'s sole `Self::Inner` to `f`. + pub unsafe trait TryWithError<MappedInner> { + type Inner; + type Mapped; + fn map<F: FnOnce(Self::Inner) -> MappedInner>(self, f: F) -> Self::Mapped; + } + + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + KnownLayout + ?Sized, + I: Invariants, + { + /// Casts this pointer-to-initialized into a pointer-to-bytes. + #[allow(clippy::wrong_self_convention)] + #[must_use] + #[inline] + pub fn as_bytes<R>(self) -> Ptr<'a, [u8], (I::Aliasing, Aligned, Valid)> + where + [u8]: TransmuteFromPtr<T, I::Aliasing, I::Validity, Valid, AsBytesCast, R>, + { + self.transmute_with::<[u8], Valid, AsBytesCast, _>().bikeshed_recall_aligned() + } + } + + impl<'a, T, I, const N: usize> Ptr<'a, [T; N], I> + where + T: 'a, + I: Invariants, + { + /// Casts this pointer-to-array into a slice. + #[allow(clippy::wrong_self_convention)] + #[inline] + #[must_use] + pub fn as_slice(self) -> Ptr<'a, [T], I> { + let slice = self.as_inner().as_slice(); + // SAFETY: Note that, by post-condition on `PtrInner::as_slice`, + // `slice` refers to the same byte range as `self.as_inner()`. + // + // 0. Thus, `slice` conforms to the aliasing invariant of + // `I::Aliasing` because `self` does. + // 1. By the above lemma, `slice` conforms to the alignment + // invariant of `I::Alignment` because `self` does. + // 2. Since `[T; N]` and `[T]` have the same bit validity [1][2], + // and since `self` and the returned `Ptr` have the same validity + // invariant, neither `self` nor the returned `Ptr` can be used + // to write a value to the referent which violates the other's + // validity invariant. + // + // [1] Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#array-layout: + // + // An array of `[T; N]` has a size of `size_of::<T>() * N` and the + // same alignment of `T`. Arrays are laid out so that the + // zero-based `nth` element of the array is offset from the start + // of the array by `n * size_of::<T>()` bytes. + // + // ... + // + // Slices have the same layout as the section of the array they + // slice. + // + // [2] Per https://doc.rust-lang.org/1.81.0/reference/types/array.html#array-types: + // + // All elements of arrays are always initialized + unsafe { Ptr::from_inner(slice) } + } + } + + /// For caller convenience, these methods are generic over alignment + /// invariant. In practice, the referent is always well-aligned, because the + /// alignment of `[u8]` is 1. + impl<'a, I> Ptr<'a, [u8], I> + where + I: Invariants<Validity = Valid>, + { + /// Attempts to cast `self` to a `U` using the given cast type. + /// + /// If `U` is a slice DST and pointer metadata (`meta`) is provided, + /// then the cast will only succeed if it would produce an object with + /// the given metadata. + /// + /// Returns `None` if the resulting `U` would be invalidly-aligned, if + /// no `U` can fit in `self`, or if the provided pointer metadata + /// describes an invalid instance of `U`. On success, returns a pointer + /// to the largest-possible `U` which fits in `self`. + /// + /// # Safety + /// + /// The caller may assume that this implementation is correct, and may + /// rely on that assumption for the soundness of their code. In + /// particular, the caller may assume that, if `try_cast_into` returns + /// `Some((ptr, remainder))`, then `ptr` and `remainder` refer to + /// non-overlapping byte ranges within `self`, and that `ptr` and + /// `remainder` entirely cover `self`. Finally: + /// - If this is a prefix cast, `ptr` has the same address as `self`. + /// - If this is a suffix cast, `remainder` has the same address as + /// `self`. + #[inline(always)] + pub fn try_cast_into<U, R>( + self, + cast_type: CastType, + meta: Option<U::PointerMetadata>, + ) -> Result< + (Ptr<'a, U, (I::Aliasing, Aligned, Initialized)>, Ptr<'a, [u8], I>), + CastError<Self, U>, + > + where + I::Aliasing: Reference, + U: 'a + ?Sized + KnownLayout + Read<I::Aliasing, R>, + { + let (inner, remainder) = self.as_inner().try_cast_into(cast_type, meta).map_err( + #[inline(always)] + |err| { + err.map_src( + #[inline(always)] + |inner| + // SAFETY: `PtrInner::try_cast_into` promises to return its + // original argument on error, which was originally produced + // by `self.as_inner()`, which is guaranteed to satisfy + // `Ptr`'s invariants. + unsafe { Ptr::from_inner(inner) }, + ) + }, + )?; + + // SAFETY: + // 0. Since `U: Read<I::Aliasing, _>`, either: + // - `I::Aliasing` is `Exclusive`, in which case both `src` and + // `ptr` conform to `Exclusive` + // - `I::Aliasing` is `Shared` and `U` is `Immutable` (we already + // know that `[u8]: Immutable`). In this case, neither `U` nor + // `[u8]` permit mutation, and so `Shared` aliasing is + // satisfied. + // 1. `ptr` conforms to the alignment invariant of `Aligned` because + // it is derived from `try_cast_into`, which promises that the + // object described by `target` is validly aligned for `U`. + // 2. By trait bound, `self` - and thus `target` - is a bit-valid + // `[u8]`. `Ptr<[u8], (_, _, Valid)>` and `Ptr<_, (_, _, + // Initialized)>` have the same bit validity, and so neither + // `self` nor `res` can be used to write a value to the referent + // which violates the other's validity invariant. + let res = unsafe { Ptr::from_inner(inner) }; + + // SAFETY: + // 0. `self` and `remainder` both have the type `[u8]`. Thus, they + // have `UnsafeCell`s at the same locations. Type casting does + // not affect aliasing. + // 1. `[u8]` has no alignment requirement. + // 2. `self` has validity `Valid` and has type `[u8]`. Since + // `remainder` references a subset of `self`'s referent, it is + // also a bit-valid `[u8]`. Thus, neither `self` nor `remainder` + // can be used to write a value to the referent which violates + // the other's validity invariant. + let remainder = unsafe { Ptr::from_inner(remainder) }; + + Ok((res, remainder)) + } + + /// Attempts to cast `self` into a `U`, failing if all of the bytes of + /// `self` cannot be treated as a `U`. + /// + /// In particular, this method fails if `self` is not validly-aligned + /// for `U` or if `self`'s size is not a valid size for `U`. + /// + /// # Safety + /// + /// On success, the caller may assume that the returned pointer + /// references the same byte range as `self`. + #[allow(unused)] + #[inline(always)] + pub fn try_cast_into_no_leftover<U, R>( + self, + meta: Option<U::PointerMetadata>, + ) -> Result<Ptr<'a, U, (I::Aliasing, Aligned, Initialized)>, CastError<Self, U>> + where + I::Aliasing: Reference, + U: 'a + ?Sized + KnownLayout + Read<I::Aliasing, R>, + [u8]: Read<I::Aliasing, R>, + { + // SAFETY: The provided closure returns the only copy of `slf`. + unsafe { + self.try_with_unchecked( + #[inline(always)] + |slf| match slf.try_cast_into(CastType::Prefix, meta) { + Ok((slf, remainder)) => { + if remainder.is_empty() { + Ok(slf) + } else { + Err(CastError::Size(SizeError::<_, U>::new(()))) + } + } + Err(err) => Err(err.map_src( + #[inline(always)] + |_slf| (), + )), + }, + ) + } + } + } + + impl<'a, T, I> Ptr<'a, UnsafeCell<T>, I> + where + T: 'a + ?Sized, + I: Invariants<Aliasing = Exclusive>, + { + /// Converts this `Ptr` into a pointer to the underlying data. + /// + /// This call borrows the `UnsafeCell` mutably (at compile-time) which + /// guarantees that we possess the only reference. + /// + /// This is like [`UnsafeCell::get_mut`], but for `Ptr`. + /// + /// [`UnsafeCell::get_mut`]: core::cell::UnsafeCell::get_mut + #[must_use] + #[inline(always)] + pub fn get_mut(self) -> Ptr<'a, T, I> { + // SAFETY: As described below, `UnsafeCell<T>` has the same size + // as `T: ?Sized` (same static size or same DST layout). Thus, + // `*const UnsafeCell<T> as *const T` is a size-preserving cast. + define_cast!(unsafe { Cast<T: ?Sized> = UnsafeCell<T> => T }); + + // SAFETY: + // - Aliasing is `Exclusive`, and so we are not required to promise + // anything about the locations of `UnsafeCell`s. + // - `UnsafeCell<T>` has the same bit validity as `T` [1]. + // Technically the term "representation" doesn't guarantee this, + // but the subsequent sentence in the documentation makes it clear + // that this is the intention. + // + // By invariant on `Validity`, since `T` and `UnsafeCell<T>` have + // the same bit validity, then the set of values which may appear + // in the referent of a `Ptr<T, (_, _, V)>` is the same as the set + // which may appear in the referent of a `Ptr<UnsafeCell<T>, (_, + // _, V)>`. Thus, neither `self` nor `ptr` may be used to write a + // value to the referent which would violate the other's validity + // invariant. + // + // [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: + // + // `UnsafeCell<T>` has the same in-memory representation as its + // inner type `T`. A consequence of this guarantee is that it is + // possible to convert between `T` and `UnsafeCell<T>`. + let ptr = unsafe { self.project_transmute_unchecked::<_, _, Cast>() }; + + // SAFETY: `UnsafeCell<T>` has the same alignment as `T` [1], + // and so if `self` is guaranteed to be aligned, then so is the + // returned `Ptr`. + // + // [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: + // + // `UnsafeCell<T>` has the same in-memory representation as + // its inner type `T`. A consequence of this guarantee is that + // it is possible to convert between `T` and `UnsafeCell<T>`. + let ptr = unsafe { ptr.assume_alignment::<I::Alignment>() }; + ptr.unify_invariants() + } + } +} + +/// Projections through the referent. +mod _project { + use super::*; + + impl<'a, T, I> Ptr<'a, [T], I> + where + T: 'a, + I: Invariants, + I::Aliasing: Reference, + { + /// Iteratively projects the elements `Ptr<T>` from `Ptr<[T]>`. + #[inline] + pub fn iter(self) -> impl Iterator<Item = Ptr<'a, T, I>> { + // SAFETY: + // 0. `elem` conforms to the aliasing invariant of `I::Aliasing`: + // - `Exclusive`: `self` is consumed by value, and therefore + // cannot be used to access the slice while any yielded + // element `Ptr` is live. Each non-zero-sized element is a + // disjoint byte range within the slice, and zero-sized + // elements address no bytes, so distinct yielded element + // `Ptr`s do not alias each other. + // - `Shared`: It is sound for multiple shared `Ptr`s to exist + // simultaneously which reference the same memory. + // 1. `elem`, conditionally, conforms to the validity invariant of + // `I::Alignment`. If `elem` is projected from data well-aligned + // for `[T]`, `elem` will be valid for `T`. + // 2. `elem` conforms to the validity invariant of `I::Validity`. + // Per https://doc.rust-lang.org/1.81.0/reference/type-layout.html#array-layout: + // + // Slices have the same layout as the section of the array they + // slice. + // + // Arrays are laid out so that the zero-based `nth` element of + // the array is offset from the start of the array by `n * + // size_of::<T>()` bytes. Thus, `elem` addresses a valid `T` + // within the slice. Since `self` satisfies `I::Validity`, `elem` + // also satisfies `I::Validity`. + self.as_inner().iter().map( + #[inline(always)] + |elem| unsafe { Ptr::from_inner(elem) }, + ) + } + } + + #[allow(clippy::needless_lifetimes)] + impl<'a, T, I> Ptr<'a, T, I> + where + T: 'a + ?Sized + KnownLayout<PointerMetadata = usize>, + I: Invariants, + { + /// The number of slice elements in the object referenced by `self`. + #[inline] + #[must_use] + pub fn len(&self) -> usize { + self.as_inner().meta().get() + } + + /// Returns `true` if the slice pointer has a length of 0. + #[inline] + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + } +} + +#[cfg(test)] +mod tests { + use core::mem::{self, MaybeUninit}; + + use super::*; + #[allow(unused)] // Needed on our MSRV, but considered unused on later toolchains. + use crate::util::AsAddress; + use crate::{pointer::BecauseImmutable, util::testutil::AU64, FromBytes, Immutable}; + + mod test_ptr_try_cast_into_soundness { + use super::*; + + // This test is designed so that if `Ptr::try_cast_into_xxx` are + // buggy, it will manifest as unsoundness that Miri can detect. + + // - If `size_of::<T>() == 0`, `N == 4` + // - Else, `N == 4 * size_of::<T>()` + // + // Each test will be run for each metadata in `metas`. + fn test<T, I, const N: usize>(metas: I) + where + T: ?Sized + KnownLayout + Immutable + FromBytes, + I: IntoIterator<Item = Option<T::PointerMetadata>> + Clone, + { + let mut bytes = [MaybeUninit::<u8>::uninit(); N]; + let initialized = [MaybeUninit::new(0u8); N]; + for start in 0..=bytes.len() { + for end in start..=bytes.len() { + // Set all bytes to uninitialized other than those in + // the range we're going to pass to `try_cast_from`. + // This allows Miri to detect out-of-bounds reads + // because they read uninitialized memory. Without this, + // some out-of-bounds reads would still be in-bounds of + // `bytes`, and so might spuriously be accepted. + bytes = [MaybeUninit::<u8>::uninit(); N]; + let bytes = &mut bytes[start..end]; + // Initialize only the byte range we're going to pass to + // `try_cast_from`. + bytes.copy_from_slice(&initialized[start..end]); + + let bytes = { + let bytes: *const [MaybeUninit<u8>] = bytes; + #[allow(clippy::as_conversions)] + let bytes = bytes as *const [u8]; + // SAFETY: We just initialized these bytes to valid + // `u8`s. + unsafe { &*bytes } + }; + + // SAFETY: The bytes in `slf` must be initialized. + unsafe fn validate_and_get_len< + T: ?Sized + KnownLayout + FromBytes + Immutable, + >( + slf: Ptr<'_, T, (Shared, Aligned, Initialized)>, + ) -> usize { + let t = slf.recall_validity().as_ref(); + + let bytes = { + let len = mem::size_of_val(t); + let t: *const T = t; + // SAFETY: + // - We know `t`'s bytes are all initialized + // because we just read it from `slf`, which + // points to an initialized range of bytes. If + // there's a bug and this doesn't hold, then + // that's exactly what we're hoping Miri will + // catch! + // - Since `T: FromBytes`, `T` doesn't contain + // any `UnsafeCell`s, so it's okay for `t: T` + // and a `&[u8]` to the same memory to be + // alive concurrently. + unsafe { core::slice::from_raw_parts(t.cast::<u8>(), len) } + }; + + // This assertion ensures that `t`'s bytes are read + // and compared to another value, which in turn + // ensures that Miri gets a chance to notice if any + // of `t`'s bytes are uninitialized, which they + // shouldn't be (see the comment above). + assert_eq!(bytes, vec![0u8; bytes.len()]); + + mem::size_of_val(t) + } + + for meta in metas.clone().into_iter() { + for cast_type in [CastType::Prefix, CastType::Suffix] { + if let Ok((slf, remaining)) = Ptr::from_ref(bytes) + .try_cast_into::<T, BecauseImmutable>(cast_type, meta) + { + // SAFETY: All bytes in `bytes` have been + // initialized. + let len = unsafe { validate_and_get_len(slf) }; + assert_eq!(remaining.len(), bytes.len() - len); + #[allow(unstable_name_collisions)] + let bytes_addr = bytes.as_ptr().addr(); + #[allow(unstable_name_collisions)] + let remaining_addr = remaining.as_inner().as_ptr().addr(); + match cast_type { + CastType::Prefix => { + assert_eq!(remaining_addr, bytes_addr + len) + } + CastType::Suffix => assert_eq!(remaining_addr, bytes_addr), + } + + if let Some(want) = meta { + let got = + KnownLayout::pointer_to_metadata(slf.as_inner().as_ptr()); + assert_eq!(got, want); + } + } + } + + if let Ok(slf) = Ptr::from_ref(bytes) + .try_cast_into_no_leftover::<T, BecauseImmutable>(meta) + { + // SAFETY: All bytes in `bytes` have been + // initialized. + let len = unsafe { validate_and_get_len(slf) }; + assert_eq!(len, bytes.len()); + + if let Some(want) = meta { + let got = KnownLayout::pointer_to_metadata(slf.as_inner().as_ptr()); + assert_eq!(got, want); + } + } + } + } + } + } + + #[derive(FromBytes, KnownLayout, Immutable)] + #[repr(C)] + struct SliceDst<T> { + a: u8, + trailing: [T], + } + + // Each test case becomes its own `#[test]` function. We do this because + // this test in particular takes far, far longer to execute under Miri + // than all of our other tests combined. Previously, we had these + // execute sequentially in a single test function. We run Miri tests in + // parallel in CI, but this test being sequential meant that most of + // that parallelism was wasted, as all other tests would finish in a + // fraction of the total execution time, leaving this test to execute on + // a single thread for the remainder of the test. By putting each test + // case in its own function, we permit better use of available + // parallelism. + macro_rules! test { + ($test_name:ident: $ty:ty) => { + #[test] + #[allow(non_snake_case)] + fn $test_name() { + const S: usize = core::mem::size_of::<$ty>(); + const N: usize = if S == 0 { 4 } else { S * 4 }; + test::<$ty, _, N>([None]); + + // If `$ty` is a ZST, then we can't pass `None` as the + // pointer metadata, or else computing the correct trailing + // slice length will panic. + if S == 0 { + test::<[$ty], _, N>([Some(0), Some(1), Some(2), Some(3)]); + test::<SliceDst<$ty>, _, N>([Some(0), Some(1), Some(2), Some(3)]); + } else { + test::<[$ty], _, N>([None, Some(0), Some(1), Some(2), Some(3)]); + test::<SliceDst<$ty>, _, N>([None, Some(0), Some(1), Some(2), Some(3)]); + } + } + }; + ($ty:ident) => { + test!($ty: $ty); + }; + ($($ty:ident),*) => { $(test!($ty);)* } + } + + test!(empty_tuple: ()); + test!(u8, u16, u32, u64, usize, AU64); + test!(i8, i16, i32, i64, isize); + test!(f32, f64); + } + + #[test] + fn test_try_cast_into_explicit_count() { + macro_rules! test { + ($ty:ty, $bytes:expr, $elems:expr, $expect:expr) => {{ + let bytes = [0u8; $bytes]; + let ptr = Ptr::from_ref(&bytes[..]); + let res = + ptr.try_cast_into::<$ty, BecauseImmutable>(CastType::Prefix, Some($elems)); + if let Some(expect) = $expect { + let (ptr, _) = res.unwrap(); + assert_eq!(KnownLayout::pointer_to_metadata(ptr.as_inner().as_ptr()), expect); + } else { + let _ = res.unwrap_err(); + } + }}; + } + + #[derive(KnownLayout, Immutable)] + #[repr(C)] + struct ZstDst { + u: [u8; 8], + slc: [()], + } + + test!(ZstDst, 8, 0, Some(0)); + test!(ZstDst, 7, 0, None); + + test!(ZstDst, 8, usize::MAX, Some(usize::MAX)); + test!(ZstDst, 7, usize::MAX, None); + + #[derive(KnownLayout, Immutable)] + #[repr(C)] + struct Dst { + u: [u8; 8], + slc: [u8], + } + + test!(Dst, 8, 0, Some(0)); + test!(Dst, 7, 0, None); + + test!(Dst, 9, 1, Some(1)); + test!(Dst, 8, 1, None); + + // If we didn't properly check for overflow, this would cause the + // metadata to overflow to 0, and thus the cast would spuriously + // succeed. + test!(Dst, 8, usize::MAX - 8 + 1, None); + } + + #[test] + fn test_try_cast_into_no_leftover_restores_original_slice() { + let bytes = [0u8; 4]; + let ptr = Ptr::from_ref(&bytes[..]); + let res = ptr.try_cast_into_no_leftover::<[u8; 2], BecauseImmutable>(None); + match res { + Ok(_) => panic!("should have failed due to leftover bytes"), + Err(CastError::Size(e)) => { + assert_eq!(e.into_src().len(), 4, "Should return original slice length"); + } + Err(e) => panic!("wrong error type: {:?}", e), + } + } + + #[test] + fn test_iter_exclusive_yields_disjoint_ptrs() { + let mut arr = [0u8, 1, 2, 3]; + + { + let mut iter = Ptr::from_mut(&mut arr[..]).iter(); + let first = iter.next().unwrap().as_mut(); + let second = iter.next().unwrap().as_mut(); + + *first = 10; + *second = 20; + *first = 30; + } + + assert_eq!(arr, [30, 20, 2, 3]); + } +} diff --git a/rust/zerocopy/src/pointer/transmute.rs b/rust/zerocopy/src/pointer/transmute.rs new file mode 100644 index 000000000000..ef9836698203 --- /dev/null +++ b/rust/zerocopy/src/pointer/transmute.rs @@ -0,0 +1,522 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2025 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +#![allow(missing_docs)] + +use core::{ + cell::{Cell, UnsafeCell}, + mem::{ManuallyDrop, MaybeUninit}, + num::Wrapping, +}; + +use crate::{ + pointer::{ + cast::{self, CastExact, CastSizedExact}, + invariant::*, + }, + FromBytes, Immutable, IntoBytes, Unalign, +}; + +/// Transmutations which are sound to attempt, conditional on validating the bit +/// validity of the destination type. +/// +/// If a `Ptr` transmutation is `TryTransmuteFromPtr`, then it is sound to +/// perform that transmutation so long as some additional mechanism is used to +/// validate that the referent is bit-valid for the destination type. That +/// validation mechanism could be a type bound (such as `TransmuteFrom`) or a +/// runtime validity check. +/// +/// # Safety +/// +/// ## Post-conditions +/// +/// Given `Dst: TryTransmuteFromPtr<Src, A, SV, DV, C, _>`, callers may assume +/// the following: +/// +/// Given `src: Ptr<'a, Src, (A, _, SV)>`, if the referent of `src` is +/// `DV`-valid for `Dst`, then it is sound to transmute `src` into `dst: Ptr<'a, +/// Dst, (A, Unaligned, DV)>` using `C`. +/// +/// ## Pre-conditions +/// +/// Given `src: Ptr<Src, (A, _, SV)>` and `dst: Ptr<Dst, (A, Unaligned, DV)>`, +/// `Dst: TryTransmuteFromPtr<Src, A, SV, DV, C, _>` is sound if all of the +/// following hold: +/// - Forwards transmutation: Either of the following hold: +/// - So long as `dst` is active, no mutation of `dst`'s referent is allowed +/// except via `dst` itself +/// - The set of `DV`-valid referents of `dst` is a superset of the set of +/// `SV`-valid referents of `src` (NOTE: this condition effectively bans +/// shrinking or overwriting transmutes, which cannot satisfy this +/// condition) +/// - Reverse transmutation: Either of the following hold: +/// - `dst` does not permit mutation of its referent +/// - The set of `DV`-valid referents of `dst` is a subset of the set of +/// `SV`-valid referents of `src` (NOTE: this condition effectively bans +/// shrinking or overwriting transmutes, which cannot satisfy this +/// condition) +/// - No safe code, given access to `src` and `dst`, can cause undefined +/// behavior: Any of the following hold: +/// - `A` is `Exclusive` +/// - `Src: Immutable` and `Dst: Immutable` +/// - It is sound for shared code to operate on a `&Src` and `&Dst` which +/// reference the same byte range at the same time +/// +/// ## Proof +/// +/// Given: +/// - `src: Ptr<'a, Src, (A, _, SV)>` +/// - `src`'s referent is `DV`-valid for `Dst` +/// +/// We are trying to prove that it is sound to perform a cast from `src` to a +/// `dst: Ptr<'a, Dst, (A, Unaligned, DV)>` using `C`. We need to prove that +/// such a cast does not violate any of `src`'s invariants, and that it +/// satisfies all invariants of the destination `Ptr` type. +/// +/// First, by `C: CastExact`, `src`'s address is unchanged, so it still satisfies +/// its alignment. Since `dst`'s alignment is `Unaligned`, it trivially satisfies +/// its alignment. +/// +/// Second, aliasing is either `Exclusive` or `Shared`: +/// - If it is `Exclusive`, then both `src` and `dst` satisfy `Exclusive` +/// aliasing trivially: since `src` and `dst` have the same lifetime, `src` is +/// inaccessible so long as `dst` is alive, and no other live `Ptr`s or +/// references may reference the same referent. +/// - If it is `Shared`, then either: +/// - `Src: Immutable` and `Dst: Immutable`, and so neither `src` nor `dst` +/// permit interior mutation. +/// - It is explicitly sound for safe code to operate on a `&Src` and a `&Dst` +/// pointing to the same byte range at the same time. +/// +/// Third, `src`'s validity is satisfied. By invariant, `src`'s referent began +/// as an `SV`-valid `Src`. It is guaranteed to remain so, as either of the +/// following hold: +/// - `dst` does not permit mutation of its referent. +/// - The set of `DV`-valid referents of `dst` is a subset of the set of +/// `SV`-valid referents of `src`. Thus, any value written via `dst` is +/// guaranteed to be an `SV`-valid referent of `src`. +/// +/// Fourth, `dst`'s validity is satisfied. It is a given of this proof that the +/// referent is `DV`-valid for `Dst`. It is guaranteed to remain so, as either +/// of the following hold: +/// - So long as `dst` is active, no mutation of the referent is allowed except +/// via `dst` itself. +/// - The set of `DV`-valid referents of `dst` is a superset of the set of +/// `SV`-valid referents of `src`. Thus, any value written via `src` is +/// guaranteed to be a `DV`-valid referent of `dst`. +pub unsafe trait TryTransmuteFromPtr< + Src: ?Sized, + A: Aliasing, + SV: Validity, + DV: Validity, + C: CastExact<Src, Self>, + R, +> +{ +} + +#[allow(missing_copy_implementations, missing_debug_implementations)] +pub enum BecauseMutationCompatible {} + +// SAFETY: +// - Forwards transmutation: By `Dst: MutationCompatible<Src, A, SV, DV, _>`, we +// know that at least one of the following holds: +// - So long as `dst: Ptr<Dst>` is active, no mutation of its referent is +// allowed except via `dst` itself if either of the following hold: +// - Aliasing is `Exclusive`, in which case, so long as the `Dst` `Ptr` +// exists, no mutation is permitted except via that `Ptr` +// - Aliasing is `Shared`, `Src: Immutable`, and `Dst: Immutable`, in which +// case no mutation is possible via either `Ptr` +// - Since the underlying cast is size-preserving, `dst` addresses the same +// referent as `src`. By `Dst: TransmuteFrom<Src, SV, DV>`, the set of +// `DV`-valid referents of `dst` is a superset of the set of `SV`-valid +// referents of `src`. +// - Reverse transmutation: Since the underlying cast is size-preserving, `dst` +// addresses the same referent as `src`. By `Src: TransmuteFrom<Dst, DV, SV>`, +// the set of `DV`-valid referents of `src` is a subset of the set of +// `SV`-valid referents of `dst`. +// - No safe code, given access to `src` and `dst`, can cause undefined +// behavior: By `Dst: MutationCompatible<Src, A, SV, DV, _>`, at least one of +// the following holds: +// - `A` is `Exclusive` +// - `Src: Immutable` and `Dst: Immutable` +// - `Dst: InvariantsEq<Src>`, which guarantees that `Src` and `Dst` have the +// same invariants, and permit interior mutation on the same byte ranges +unsafe impl<Src, Dst, SV, DV, A, C, R> + TryTransmuteFromPtr<Src, A, SV, DV, C, (BecauseMutationCompatible, R)> for Dst +where + A: Aliasing, + SV: Validity, + DV: Validity, + Src: TransmuteFrom<Dst, DV, SV> + ?Sized, + Dst: MutationCompatible<Src, A, SV, DV, R> + ?Sized, + C: CastExact<Src, Dst>, +{ +} + +// SAFETY: +// - Forwards transmutation: Since aliasing is `Shared` and `Src: Immutable`, +// `src` does not permit mutation of its referent. +// - Reverse transmutation: Since aliasing is `Shared` and `Dst: Immutable`, +// `dst` does not permit mutation of its referent. +// - No safe code, given access to `src` and `dst`, can cause undefined +// behavior: `Src: Immutable` and `Dst: Immutable` +unsafe impl<Src, Dst, SV, DV, C> TryTransmuteFromPtr<Src, Shared, SV, DV, C, BecauseImmutable> + for Dst +where + SV: Validity, + DV: Validity, + Src: Immutable + ?Sized, + Dst: Immutable + ?Sized, + C: CastExact<Src, Dst>, +{ +} + +/// Denotes that `src: Ptr<Src, (A, _, SV)>` and `dst: Ptr<Self, (A, _, DV)>`, +/// referencing the same referent at the same time, cannot be used by safe code +/// to break library safety invariants of `Src` or `Self`. +/// +/// # Safety +/// +/// At least one of the following must hold: +/// - `Src: Read<A, _>` and `Self: Read<A, _>` +/// - `Self: InvariantsEq<Src>`, and, for some `V`: +/// - `Dst: TransmuteFrom<Src, V, V>` +/// - `Src: TransmuteFrom<Dst, V, V>` +pub unsafe trait MutationCompatible<Src: ?Sized, A: Aliasing, SV, DV, R> {} + +#[allow(missing_copy_implementations, missing_debug_implementations)] +pub enum BecauseRead {} + +// SAFETY: `Src: Read<A, _>` and `Dst: Read<A, _>`. +unsafe impl<Src: ?Sized, Dst: ?Sized, A: Aliasing, SV: Validity, DV: Validity, R> + MutationCompatible<Src, A, SV, DV, (BecauseRead, R)> for Dst +where + Src: Read<A, R>, + Dst: Read<A, R>, +{ +} + +/// Denotes that two types have the same invariants. +/// +/// # Safety +/// +/// It is sound for safe code to operate on a `&T` and a `&Self` pointing to the +/// same referent at the same time - no such safe code can cause undefined +/// behavior. +pub unsafe trait InvariantsEq<T: ?Sized> {} + +// SAFETY: Trivially sound to have multiple `&T` pointing to the same referent. +unsafe impl<T: ?Sized> InvariantsEq<T> for T {} + +// SAFETY: `Dst: InvariantsEq<Src> + TransmuteFrom<Src, SV, DV>`, and `Src: +// TransmuteFrom<Dst, DV, SV>`. +unsafe impl<Src: ?Sized, Dst: ?Sized, A: Aliasing, SV: Validity, DV: Validity> + MutationCompatible<Src, A, SV, DV, BecauseInvariantsEq> for Dst +where + Src: TransmuteFrom<Dst, DV, SV>, + Dst: TransmuteFrom<Src, SV, DV> + InvariantsEq<Src>, +{ +} + +#[allow(missing_debug_implementations, missing_copy_implementations)] +pub enum BecauseInvariantsEq {} + +macro_rules! unsafe_impl_invariants_eq { + ($tyvar:ident => $t:ty, $u:ty) => {{ + crate::util::macros::__unsafe(); + // SAFETY: The caller promises that this is sound. + unsafe impl<$tyvar> InvariantsEq<$t> for $u {} + // SAFETY: The caller promises that this is sound. + unsafe impl<$tyvar> InvariantsEq<$u> for $t {} + }}; +} + +impl_transitive_transmute_from!(T => MaybeUninit<T> => T => Wrapping<T>); +impl_transitive_transmute_from!(T => Wrapping<T> => T => MaybeUninit<T>); + +// SAFETY: `ManuallyDrop<T>` has the same size and bit validity as `T` [1], and +// implements `Deref<Target = T>` [2]. Thus, it is already possible for safe +// code to obtain a `&T` and a `&ManuallyDrop<T>` to the same referent at the +// same time. +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop<T>` is guaranteed to have the same layout and bit +// validity as `T` +// +// [2] https://doc.rust-lang.org/1.81.0/std/mem/struct.ManuallyDrop.html#impl-Deref-for-ManuallyDrop%3CT%3E +unsafe impl<T: ?Sized> InvariantsEq<T> for ManuallyDrop<T> {} +// SAFETY: See previous safety comment. +unsafe impl<T: ?Sized> InvariantsEq<ManuallyDrop<T>> for T {} + +/// Transmutations which are always sound. +/// +/// `TransmuteFromPtr` is a shorthand for [`TryTransmuteFromPtr`] and +/// [`TransmuteFrom`]. +/// +/// # Safety +/// +/// `Dst: TransmuteFromPtr<Src, A, SV, DV, _>` is equivalent to `Dst: +/// TryTransmuteFromPtr<Src, A, SV, DV, _> + TransmuteFrom<Src, SV, DV>`. +pub unsafe trait TransmuteFromPtr< + Src: ?Sized, + A: Aliasing, + SV: Validity, + DV: Validity, + C: CastExact<Src, Self>, + R, +>: TryTransmuteFromPtr<Src, A, SV, DV, C, R> + TransmuteFrom<Src, SV, DV> +{ +} + +// SAFETY: The `where` bounds are equivalent to the safety invariant on +// `TransmuteFromPtr`. +unsafe impl< + Src: ?Sized, + Dst: ?Sized, + A: Aliasing, + SV: Validity, + DV: Validity, + C: CastExact<Src, Dst>, + R, + > TransmuteFromPtr<Src, A, SV, DV, C, R> for Dst +where + Dst: TransmuteFrom<Src, SV, DV> + TryTransmuteFromPtr<Src, A, SV, DV, C, R>, +{ +} + +/// Denotes that any `SV`-valid `Src` may soundly be transmuted into a +/// `DV`-valid `Self`. +/// +/// # Safety +/// +/// Given `src: Ptr<Src, (_, _, SV)>` and `dst: Ptr<Dst, (_, _, DV)>`, if the +/// referents of `src` and `dst` are the same size, then the set of bit patterns +/// allowed to appear in `src`'s referent must be a subset of the set allowed to +/// appear in `dst`'s referent. +/// +/// If the referents are not the same size, then `Dst: TransmuteFrom<Src, SV, +/// DV>` conveys no safety guarantee. +pub unsafe trait TransmuteFrom<Src: ?Sized, SV, DV> {} + +/// Carries the ability to perform a size-preserving cast or conversion from a +/// raw pointer to `Src` to a raw pointer to `Self`. +/// +/// The cast/conversion is carried by the associated [`CastFrom`] type, and +/// may be a no-op cast (without updating pointer metadata) or a conversion +/// which updates pointer metadata. +/// +/// # Safety +/// +/// `SizeEq` on its own conveys no safety guarantee. Any safety guarantees come +/// from the safety invariants on the associated [`CastFrom`] type, specifically +/// the [`CastExact`] bound. +/// +/// [`CastFrom`]: SizeEq::CastFrom +/// [`CastExact`]: CastExact +pub trait SizeEq<Src: ?Sized> { + type CastFrom: CastExact<Src, Self>; +} + +impl<T: ?Sized> SizeEq<T> for T { + type CastFrom = cast::IdCast; +} + +// SAFETY: Since `Src: IntoBytes`, the set of valid `Src`'s is the set of +// initialized bit patterns, which is exactly the set allowed in the referent of +// any `Initialized` `Ptr`. +unsafe impl<Src, Dst> TransmuteFrom<Src, Valid, Initialized> for Dst +where + Src: IntoBytes + ?Sized, + Dst: ?Sized, +{ +} + +// SAFETY: Since `Dst: FromBytes`, any initialized bit pattern may appear in the +// referent of a `Ptr<Dst, (_, _, Valid)>`. This is exactly equal to the set of +// bit patterns which may appear in the referent of any `Initialized` `Ptr`. +unsafe impl<Src, Dst> TransmuteFrom<Src, Initialized, Valid> for Dst +where + Src: ?Sized, + Dst: FromBytes + ?Sized, +{ +} + +// FIXME(#2354): This seems like a smell - the soundness of this bound has +// nothing to do with `Src` or `Dst` - we're basically just saying `[u8; N]` is +// transmutable into `[u8; N]`. + +// SAFETY: The set of allowed bit patterns in the referent of any `Initialized` +// `Ptr` is the same regardless of referent type. +unsafe impl<Src, Dst> TransmuteFrom<Src, Initialized, Initialized> for Dst +where + Src: ?Sized, + Dst: ?Sized, +{ +} + +// FIXME(#2354): This seems like a smell - the soundness of this bound has +// nothing to do with `Dst` - we're basically just saying that any type is +// transmutable into `MaybeUninit<[u8; N]>`. + +// SAFETY: A `Dst` with validity `Uninit` permits any byte sequence, and +// therefore can be transmuted from any value. +unsafe impl<Src, Dst, V> TransmuteFrom<Src, V, Uninit> for Dst +where + Src: ?Sized, + Dst: ?Sized, + V: Validity, +{ +} + +// SAFETY: +// - `ManuallyDrop<T>` has the same size as `T` [1] +// - `ManuallyDrop<T>` has the same validity as `T` [1] +// +// [1] Per https://doc.rust-lang.org/1.81.0/std/mem/struct.ManuallyDrop.html: +// +// `ManuallyDrop<T>` is guaranteed to have the same layout and bit validity as +// `T` +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T: ?Sized => ManuallyDrop<T>) }; + +// SAFETY: +// - `Unalign<T>` promises to have the same size as `T`. +// - `Unalign<T>` promises to have the same validity as `T`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T => Unalign<T>) }; +// SAFETY: `Unalign<T>` promises to have the same size and validity as `T`. +// Given `u: &Unalign<T>`, it is already possible to obtain `let t = +// u.try_deref().unwrap()`. Because `Unalign<T>` has the same size as `T`, the +// returned `&T` must point to the same referent as `u`, and thus it must be +// sound for these two references to exist at the same time since it's already +// possible for safe code to get into this state. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_invariants_eq!(T => T, Unalign<T>) }; + +// SAFETY: +// - `Wrapping<T>` has the same size as `T` [1]. +// - `Wrapping<T>` has only one field, which is `pub` [2]. We are also +// guaranteed per that `Wrapping<T>` has the same layout as `T` [1]. The only +// way for both of these to be true simultaneously is for `Wrapping<T>` to +// have the same bit validity as `T`. In particular, in order to change the +// bit validity, one of the following would need to happen: +// - `Wrapping` could change its `repr`, but this would violate the layout +// guarantee. +// - `Wrapping` could add or change its fields, but this would be a +// stability-breaking change. +// +// [1] Per https://doc.rust-lang.org/1.85.0/core/num/struct.Wrapping.html#layout-1: +// +// `Wrapping<T>` is guaranteed to have the same layout and ABI as `T`. +// +// [2] Definition from https://doc.rust-lang.org/1.85.0/core/num/struct.Wrapping.html: +// +// ``` +// #[repr(transparent)] +// pub struct Wrapping<T>(pub T); +// ``` +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T => Wrapping<T>) }; + +// SAFETY: By the preceding safety proof, `Wrapping<T>` and `T` have the same +// layout and bit validity. Since a `Wrapping<T>`'s `T` field is `pub`, given +// `w: &Wrapping<T>`, it's possible to do `let t = &w.t`, which means that it's +// already possible for safe code to obtain a `&Wrapping<T>` and a `&T` pointing +// to the same referent at the same time. Thus, this must be sound. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_invariants_eq!(T => T, Wrapping<T>) }; + +// SAFETY: +// - `UnsafeCell<T>` has the same size as `T` [1]. +// - Per [1], `UnsafeCell<T>` has the same bit validity as `T`. Technically the +// term "representation" doesn't guarantee this, but the subsequent sentence +// in the documentation makes it clear that this is the intention. +// +// [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: +// +// `UnsafeCell<T>` has the same in-memory representation as its inner type +// `T`. A consequence of this guarantee is that it is possible to convert +// between `T` and `UnsafeCell<T>`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T: ?Sized => UnsafeCell<T>) }; + +// SAFETY: +// - `Cell<T>` has the same size as `T` [1]. +// - Per [1], `Cell<T>` has the same bit validity as `T`. Technically the term +// "representation" doesn't guarantee this, but it does promise to have the +// "same memory layout and caveats as `UnsafeCell<T>`." The `UnsafeCell` docs +// [2] make it clear that bit validity is the intention even if that phrase +// isn't used. +// +// [1] Per https://doc.rust-lang.org/1.85.0/std/cell/struct.Cell.html#memory-layout: +// +// `Cell<T>` has the same memory layout and caveats as `UnsafeCell<T>`. In +// particular, this means that `Cell<T>` has the same in-memory representation +// as its inner type `T`. +// +// [2] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout: +// +// `UnsafeCell<T>` has the same in-memory representation as its inner type +// `T`. A consequence of this guarantee is that it is possible to convert +// between `T` and `UnsafeCell<T>`. +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { unsafe_impl_for_transparent_wrapper!(pub T: ?Sized => Cell<T>) }; + +impl_transitive_transmute_from!(T: ?Sized => Cell<T> => T => UnsafeCell<T>); +impl_transitive_transmute_from!(T: ?Sized => UnsafeCell<T> => T => Cell<T>); + +// SAFETY: `MaybeUninit<T>` has no validity requirements. Currently this is not +// explicitly guaranteed, but it's obvious from `MaybeUninit`'s documentation +// that this is the intention: +// https://doc.rust-lang.org/1.85.0/core/mem/union.MaybeUninit.html +unsafe impl<T> TransmuteFrom<T, Uninit, Valid> for MaybeUninit<T> {} + +impl<T> SizeEq<T> for MaybeUninit<T> { + type CastFrom = CastSizedExact; +} + +impl<T> SizeEq<MaybeUninit<T>> for T { + type CastFrom = CastSizedExact; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pointer::cast::Project as _; + + fn test_size_eq<Src, Dst: SizeEq<Src>>(mut src: Src) { + let _: *mut Dst = + <Dst as SizeEq<Src>>::CastFrom::project(crate::pointer::PtrInner::from_mut(&mut src)); + } + + #[test] + fn test_transmute_coverage() { + // SizeEq<T> for MaybeUninit<T> + test_size_eq::<u8, MaybeUninit<u8>>(0u8); + + // SizeEq<MaybeUninit<T>> for T + test_size_eq::<MaybeUninit<u8>, u8>(MaybeUninit::<u8>::new(0)); + + // Transitive: MaybeUninit<T> -> Wrapping<T> + // T => MaybeUninit<T> => T => Wrapping<T> + test_size_eq::<u8, Wrapping<u8>>(0u8); + + // T => Wrapping<T> => T => MaybeUninit<T> + test_size_eq::<Wrapping<u8>, MaybeUninit<u8>>(Wrapping(0u8)); + + // T: ?Sized => Cell<T> => T => UnsafeCell<T> + test_size_eq::<Cell<u8>, UnsafeCell<u8>>(Cell::new(0u8)); + + // T: ?Sized => UnsafeCell<T> => T => Cell<T> + test_size_eq::<UnsafeCell<u8>, Cell<u8>>(UnsafeCell::new(0u8)); + } +} diff --git a/rust/zerocopy/src/ref.rs b/rust/zerocopy/src/ref.rs new file mode 100644 index 000000000000..e49f2a887ffa --- /dev/null +++ b/rust/zerocopy/src/ref.rs @@ -0,0 +1,1358 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2024 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License <LICENSE-BSD or +// https://opensource.org/license/bsd-2-clause>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. +use super::*; +use crate::pointer::{ + BecauseInvariantsEq, BecauseMutationCompatible, MutationCompatible, TransmuteFromPtr, +}; + +mod def { + use core::marker::PhantomData; + + use crate::{ + ByteSlice, ByteSliceMut, CloneableByteSlice, CopyableByteSlice, IntoByteSlice, + IntoByteSliceMut, + }; + + /// A typed reference derived from a byte slice. + /// + /// A `Ref<B, T>` is a reference to a `T` which is stored in a byte slice, `B`. + /// Unlike a native reference (`&T` or `&mut T`), `Ref<B, T>` has the same + /// mutability as the byte slice it was constructed from (`B`). + /// + /// # Examples + /// + /// `Ref` can be used to treat a sequence of bytes as a structured type, and + /// to read and write the fields of that type as if the byte slice reference + /// were simply a reference to that type. + /// + /// ```rust + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] + /// #[repr(C)] + /// struct UdpHeader { + /// src_port: [u8; 2], + /// dst_port: [u8; 2], + /// length: [u8; 2], + /// checksum: [u8; 2], + /// } + /// + /// #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned)] + /// #[repr(C, packed)] + /// struct UdpPacket { + /// header: UdpHeader, + /// body: [u8], + /// } + /// + /// impl UdpPacket { + /// pub fn parse<B: ByteSlice>(bytes: B) -> Option<Ref<B, UdpPacket>> { + /// Ref::from_bytes(bytes).ok() + /// } + /// } + /// ``` + pub struct Ref<B, T: ?Sized>( + // INVARIANTS: The referent (via `.deref`, `.deref_mut`, `.into`) byte + // slice is aligned to `T`'s alignment and its size corresponds to a + // valid size for `T`. + B, + PhantomData<T>, + ); + + impl<B, T: ?Sized> Ref<B, T> { + /// Constructs a new `Ref`. + /// + /// # Safety + /// + /// `bytes` dereferences (via [`deref`], [`deref_mut`], and [`into`]) to + /// a byte slice which is aligned to `T`'s alignment and whose size is a + /// valid size for `T`. + /// + /// [`deref`]: core::ops::Deref::deref + /// [`deref_mut`]: core::ops::DerefMut::deref_mut + /// [`into`]: core::convert::Into::into + pub(crate) unsafe fn new_unchecked(bytes: B) -> Ref<B, T> { + // INVARIANTS: The caller has promised that `bytes`'s referent is + // validly-aligned and has a valid size. + Ref(bytes, PhantomData) + } + } + + impl<B: ByteSlice, T: ?Sized> Ref<B, T> { + /// Access the byte slice as a [`ByteSlice`]. + /// + /// # Safety + /// + /// The caller promises not to call methods on the returned + /// [`ByteSlice`] other than `ByteSlice` methods (for example, via + /// `Any::downcast_ref`). + /// + /// `as_byte_slice` promises to return a `ByteSlice` whose referent is + /// validly-aligned for `T` and has a valid size for `T`. + pub(crate) unsafe fn as_byte_slice(&self) -> &impl ByteSlice { + // INVARIANTS: The caller promises not to call methods other than + // those on `ByteSlice`. Since `B: ByteSlice`, dereference stability + // guarantees that calling `ByteSlice` methods will not change the + // address or length of `self.0`'s referent. + // + // SAFETY: By invariant on `self.0`, the alignment and size + // post-conditions are upheld. + &self.0 + } + } + + impl<B: ByteSliceMut, T: ?Sized> Ref<B, T> { + /// Access the byte slice as a [`ByteSliceMut`]. + /// + /// # Safety + /// + /// The caller promises not to call methods on the returned + /// [`ByteSliceMut`] other than `ByteSliceMut` methods (for example, via + /// `Any::downcast_mut`). + /// + /// `as_byte_slice` promises to return a `ByteSlice` whose referent is + /// validly-aligned for `T` and has a valid size for `T`. + pub(crate) unsafe fn as_byte_slice_mut(&mut self) -> &mut impl ByteSliceMut { + // INVARIANTS: The caller promises not to call methods other than + // those on `ByteSliceMut`. Since `B: ByteSlice`, dereference + // stability guarantees that calling `ByteSlice` methods will not + // change the address or length of `self.0`'s referent. + // + // SAFETY: By invariant on `self.0`, the alignment and size + // post-conditions are upheld. + &mut self.0 + } + } + + impl<'a, B: IntoByteSlice<'a>, T: ?Sized> Ref<B, T> { + /// Access the byte slice as an [`IntoByteSlice`]. + /// + /// # Safety + /// + /// The caller promises not to call methods on the returned + /// [`IntoByteSlice`] other than `IntoByteSlice` methods (for example, + /// via `Any::downcast_ref`). + /// + /// `as_byte_slice` promises to return a `ByteSlice` whose referent is + /// validly-aligned for `T` and has a valid size for `T`. + pub(crate) unsafe fn into_byte_slice(self) -> impl IntoByteSlice<'a> { + // INVARIANTS: The caller promises not to call methods other than + // those on `IntoByteSlice`. Since `B: ByteSlice`, dereference + // stability guarantees that calling `ByteSlice` methods will not + // change the address or length of `self.0`'s referent. + // + // SAFETY: By invariant on `self.0`, the alignment and size + // post-conditions are upheld. + self.0 + } + } + + impl<'a, B: IntoByteSliceMut<'a>, T: ?Sized> Ref<B, T> { + /// Access the byte slice as an [`IntoByteSliceMut`]. + /// + /// # Safety + /// + /// The caller promises not to call methods on the returned + /// [`IntoByteSliceMut`] other than `IntoByteSliceMut` methods (for + /// example, via `Any::downcast_mut`). + /// + /// `as_byte_slice` promises to return a `ByteSlice` whose referent is + /// validly-aligned for `T` and has a valid size for `T`. + pub(crate) unsafe fn into_byte_slice_mut(self) -> impl IntoByteSliceMut<'a> { + // INVARIANTS: The caller promises not to call methods other than + // those on `IntoByteSliceMut`. Since `B: ByteSlice`, dereference + // stability guarantees that calling `ByteSlice` methods will not + // change the address or length of `self.0`'s referent. + // + // SAFETY: By invariant on `self.0`, the alignment and size + // post-conditions are upheld. + self.0 + } + } + + impl<B: CloneableByteSlice + Clone, T: ?Sized> Clone for Ref<B, T> { + #[inline] + fn clone(&self) -> Ref<B, T> { + // INVARIANTS: Since `B: CloneableByteSlice`, `self.0.clone()` has + // the same address and length as `self.0`. Since `self.0` upholds + // the field invariants, so does `self.0.clone()`. + Ref(self.0.clone(), PhantomData) + } + } + + // INVARIANTS: Since `B: CopyableByteSlice`, the copied `Ref`'s `.0` has the + // same address and length as the original `Ref`'s `.0`. Since the original + // upholds the field invariants, so does the copy. + impl<B: CopyableByteSlice + Copy, T: ?Sized> Copy for Ref<B, T> {} +} + +#[allow(unreachable_pub)] // This is a false positive on our MSRV toolchain. +pub use def::Ref; + +use crate::pointer::{ + invariant::{Aligned, BecauseExclusive, Initialized, Unaligned, Valid}, + BecauseRead, PtrInner, +}; + +impl<B, T> Ref<B, T> +where + B: ByteSlice, +{ + #[must_use = "has no side effects"] + pub(crate) fn sized_from(bytes: B) -> Result<Ref<B, T>, CastError<B, T>> { + if bytes.len() != mem::size_of::<T>() { + return Err(SizeError::new(bytes).into()); + } + if let Err(err) = util::validate_aligned_to::<_, T>(bytes.deref()) { + return Err(err.with_src(bytes).into()); + } + + // SAFETY: We just validated size and alignment. + Ok(unsafe { Ref::new_unchecked(bytes) }) + } +} + +impl<B, T> Ref<B, T> +where + B: SplitByteSlice, +{ + #[must_use = "has no side effects"] + pub(crate) fn sized_from_prefix(bytes: B) -> Result<(Ref<B, T>, B), CastError<B, T>> { + if bytes.len() < mem::size_of::<T>() { + return Err(SizeError::new(bytes).into()); + } + if let Err(err) = util::validate_aligned_to::<_, T>(bytes.deref()) { + return Err(err.with_src(bytes).into()); + } + let (bytes, suffix) = bytes.split_at(mem::size_of::<T>()).map_err( + #[inline(always)] + |b| SizeError::new(b).into(), + )?; + // SAFETY: We just validated alignment and that `bytes` is at least as + // large as `T`. `bytes.split_at(mem::size_of::<T>())?` ensures that the + // new `bytes` is exactly the size of `T`. By safety postcondition on + // `SplitByteSlice::split_at` we can rely on `split_at` to produce the + // correct `bytes` and `suffix`. + let r = unsafe { Ref::new_unchecked(bytes) }; + Ok((r, suffix)) + } + + #[must_use = "has no side effects"] + pub(crate) fn sized_from_suffix(bytes: B) -> Result<(B, Ref<B, T>), CastError<B, T>> { + let bytes_len = bytes.len(); + let split_at = if let Some(split_at) = bytes_len.checked_sub(mem::size_of::<T>()) { + split_at + } else { + return Err(SizeError::new(bytes).into()); + }; + let (prefix, bytes) = bytes.split_at(split_at).map_err(|b| SizeError::new(b).into())?; + if let Err(err) = util::validate_aligned_to::<_, T>(bytes.deref()) { + return Err(err.with_src(bytes).into()); + } + // SAFETY: Since `split_at` is defined as `bytes_len - size_of::<T>()`, + // the `bytes` which results from `let (prefix, bytes) = + // bytes.split_at(split_at)?` has length `size_of::<T>()`. After + // constructing `bytes`, we validate that it has the proper alignment. + // By safety postcondition on `SplitByteSlice::split_at` we can rely on + // `split_at` to produce the correct `prefix` and `bytes`. + let r = unsafe { Ref::new_unchecked(bytes) }; + Ok((prefix, r)) + } +} + +impl<B, T> Ref<B, T> +where + B: ByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + /// Constructs a `Ref` from a byte slice. + /// + /// If the length of `source` is not a [valid size of `T`][valid-size], or + /// if `source` is not appropriately aligned for `T`, this returns `Err`. If + /// [`T: Unaligned`][t-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// `T` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_bytes(&b"UU"[..]); // ⚠ Compile Error! + /// ``` + #[must_use = "has no side effects"] + #[inline] + pub fn from_bytes(source: B) -> Result<Ref<B, T>, CastError<B, T>> { + static_assert_dst_is_not_zst!(T); + if let Err(e) = + Ptr::from_ref(source.deref()).try_cast_into_no_leftover::<T, BecauseImmutable>(None) + { + return Err(e.with_src(()).with_src(source)); + } + // SAFETY: `try_cast_into_no_leftover` validates size and alignment. + Ok(unsafe { Ref::new_unchecked(source) }) + } +} + +impl<B, T> Ref<B, T> +where + B: SplitByteSlice, + T: KnownLayout + Immutable + ?Sized, +{ + /// Constructs a `Ref` from the prefix of a byte slice. + /// + /// This method computes the [largest possible size of `T`][valid-size] that + /// can fit in the leading bytes of `source`, then attempts to return both a + /// `Ref` to those bytes, and a reference to the remaining bytes. If there + /// are insufficient bytes, or if `source` is not appropriately aligned, + /// this returns `Err`. If [`T: Unaligned`][t-unaligned], you can + /// [infallibly discard the alignment error][size-error-from]. + /// + /// `T` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_prefix(&b"UU"[..]); // ⚠ Compile Error! + /// ``` + #[must_use = "has no side effects"] + #[inline] + pub fn from_prefix(source: B) -> Result<(Ref<B, T>, B), CastError<B, T>> { + static_assert_dst_is_not_zst!(T); + let remainder = match Ptr::from_ref(source.deref()) + .try_cast_into::<T, BecauseImmutable>(CastType::Prefix, None) + { + Ok((_, remainder)) => remainder, + Err(e) => { + return Err(e.with_src(()).with_src(source)); + } + }; + + // SAFETY: `remainder` is constructed as a subset of `source`, and so it + // cannot have a larger size than `source`. Both of their `len` methods + // measure bytes (`source` deref's to `[u8]`, and `remainder` is a + // `Ptr<[u8]>`), so `source.len() >= remainder.len()`. Thus, this cannot + // underflow. + #[allow(unstable_name_collisions)] + let split_at = unsafe { source.len().unchecked_sub(remainder.len()) }; + let (bytes, suffix) = source.split_at(split_at).map_err(|b| SizeError::new(b).into())?; + // SAFETY: `try_cast_into` validates size and alignment, and returns a + // `split_at` that indicates how many bytes of `source` correspond to a + // valid `T`. By safety postcondition on `SplitByteSlice::split_at` we + // can rely on `split_at` to produce the correct `source` and `suffix`. + let r = unsafe { Ref::new_unchecked(bytes) }; + Ok((r, suffix)) + } + + /// Constructs a `Ref` from the suffix of a byte slice. + /// + /// This method computes the [largest possible size of `T`][valid-size] that + /// can fit in the trailing bytes of `source`, then attempts to return both + /// a `Ref` to those bytes, and a reference to the preceding bytes. If there + /// are insufficient bytes, or if that suffix of `source` is not + /// appropriately aligned, this returns `Err`. If [`T: + /// Unaligned`][t-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// `T` may be a sized type, a slice, or a [slice DST][slice-dst]. + /// + /// [valid-size]: crate::KnownLayout#what-is-a-valid-size + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// [slice-dst]: KnownLayout#dynamically-sized-types + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_suffix(&b"UU"[..]); // ⚠ Compile Error! + /// ``` + #[must_use = "has no side effects"] + #[inline] + pub fn from_suffix(source: B) -> Result<(B, Ref<B, T>), CastError<B, T>> { + static_assert_dst_is_not_zst!(T); + let remainder = match Ptr::from_ref(source.deref()) + .try_cast_into::<T, BecauseImmutable>(CastType::Suffix, None) + { + Ok((_, remainder)) => remainder, + Err(e) => { + let e = e.with_src(()); + return Err(e.with_src(source)); + } + }; + + let split_at = remainder.len(); + let (prefix, bytes) = source.split_at(split_at).map_err(|b| SizeError::new(b).into())?; + // SAFETY: `try_cast_into` validates size and alignment, and returns a + // `split_at` that indicates how many bytes of `source` correspond to a + // valid `T`. By safety postcondition on `SplitByteSlice::split_at` we + // can rely on `split_at` to produce the correct `prefix` and `bytes`. + let r = unsafe { Ref::new_unchecked(bytes) }; + Ok((prefix, r)) + } +} + +impl<B, T> Ref<B, T> +where + B: ByteSlice, + T: KnownLayout<PointerMetadata = usize> + Immutable + ?Sized, +{ + /// Constructs a `Ref` from the given bytes with DST length equal to `count` + /// without copying. + /// + /// This method attempts to return a `Ref` to the prefix of `source` + /// interpreted as a `T` with `count` trailing elements, and a reference to + /// the remaining bytes. If the length of `source` is not equal to the size + /// of `Self` with `count` elements, or if `source` is not appropriately + /// aligned, this returns `Err`. If [`T: Unaligned`][t-unaligned], you can + /// [infallibly discard the alignment error][size-error-from]. + /// + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_bytes_with_elems(&b"UU"[..], 42); // ⚠ Compile Error! + /// ``` + #[inline] + pub fn from_bytes_with_elems(source: B, count: usize) -> Result<Ref<B, T>, CastError<B, T>> { + static_assert_dst_is_not_zst!(T); + let expected_len = match T::size_for_metadata(count) { + Some(len) => len, + None => return Err(SizeError::new(source).into()), + }; + if source.len() != expected_len { + return Err(SizeError::new(source).into()); + } + Self::from_bytes(source) + } +} + +impl<B, T> Ref<B, T> +where + B: SplitByteSlice, + T: KnownLayout<PointerMetadata = usize> + Immutable + ?Sized, +{ + /// Constructs a `Ref` from the prefix of the given bytes with DST + /// length equal to `count` without copying. + /// + /// This method attempts to return a `Ref` to the prefix of `source` + /// interpreted as a `T` with `count` trailing elements, and a reference to + /// the remaining bytes. If there are insufficient bytes, or if `source` is + /// not appropriately aligned, this returns `Err`. If [`T: + /// Unaligned`][t-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_prefix_with_elems(&b"UU"[..], 42); // ⚠ Compile Error! + /// ``` + #[inline] + pub fn from_prefix_with_elems( + source: B, + count: usize, + ) -> Result<(Ref<B, T>, B), CastError<B, T>> { + static_assert_dst_is_not_zst!(T); + let expected_len = match T::size_for_metadata(count) { + Some(len) => len, + None => return Err(SizeError::new(source).into()), + }; + let (prefix, bytes) = source.split_at(expected_len).map_err(SizeError::new)?; + Self::from_bytes(prefix).map(move |l| (l, bytes)) + } + + /// Constructs a `Ref` from the suffix of the given bytes with DST length + /// equal to `count` without copying. + /// + /// This method attempts to return a `Ref` to the suffix of `source` + /// interpreted as a `T` with `count` trailing elements, and a reference to + /// the preceding bytes. If there are insufficient bytes, or if that suffix + /// of `source` is not appropriately aligned, this returns `Err`. If [`T: + /// Unaligned`][t-unaligned], you can [infallibly discard the alignment + /// error][size-error-from]. + /// + /// [t-unaligned]: crate::Unaligned + /// [size-error-from]: error/struct.SizeError.html#method.from-1 + /// + /// # Compile-Time Assertions + /// + /// This method cannot yet be used on unsized types whose dynamically-sized + /// component is zero-sized. Attempting to use this method on such types + /// results in a compile-time assertion error; e.g.: + /// + /// ```compile_fail,E0080 + /// use zerocopy::*; + /// # use zerocopy_derive::*; + /// + /// #[derive(Immutable, KnownLayout)] + /// #[repr(C)] + /// struct ZSTy { + /// leading_sized: u16, + /// trailing_dst: [()], + /// } + /// + /// let _ = Ref::<_, ZSTy>::from_suffix_with_elems(&b"UU"[..], 42); // ⚠ Compile Error! + /// ``` + #[inline] + pub fn from_suffix_with_elems( + source: B, + count: usize, + ) -> Result<(B, Ref<B, T>), CastError<B, T>> { + static_assert_dst_is_not_zst!(T); + let expected_len = match T::size_for_metadata(count) { + Some(len) => len, + None => return Err(SizeError::new(source).into()), + }; + let split_at = if let Some(split_at) = source.len().checked_sub(expected_len) { + split_at + } else { + return Err(SizeError::new(source).into()); + }; + // SAFETY: The preceding `source.len().checked_sub(expected_len)` + // guarantees that `split_at` is in-bounds. + let (bytes, suffix) = unsafe { source.split_at_unchecked(split_at) }; + Self::from_bytes(suffix).map(move |l| (bytes, l)) + } +} + +impl<'a, B, T> Ref<B, T> +where + B: 'a + IntoByteSlice<'a>, + T: FromBytes + KnownLayout + Immutable + ?Sized, +{ + /// Converts this `Ref` into a reference. + /// + /// `into_ref` consumes the `Ref`, and returns a reference to `T`. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::into_ref(r)` instead of `r.into_ref()`. This is so that + /// there is no conflict with a method on the inner type. + #[must_use = "has no side effects"] + #[inline(always)] + pub fn into_ref(r: Self) -> &'a T { + // Presumably unreachable, since we've guarded each constructor of `Ref`. + static_assert_dst_is_not_zst!(T); + + // SAFETY: We don't call any methods on `b` other than those provided by + // `IntoByteSlice`. + let b = unsafe { r.into_byte_slice() }; + let b = b.into_byte_slice(); + + if let crate::layout::SizeInfo::Sized { .. } = T::LAYOUT.size_info { + let ptr = Ptr::from_ref(b); + // SAFETY: We just checked that `T: Sized`. By invariant on `r`, + // `b`'s size is equal to `size_of::<T>()`. + let ptr = unsafe { cast_for_sized::<T, _, _, _>(ptr) }; + + // SAFETY: None of the preceding transformations modifies the + // address of the pointer, and by invariant on `r`, we know that it + // is validly-aligned. + let ptr = unsafe { ptr.assume_alignment::<Aligned>() }; + return ptr.as_ref(); + } + + // PANICS: By post-condition on `into_byte_slice`, `b`'s size and + // alignment are valid for `T`. By post-condition, `b.into_byte_slice()` + // produces a byte slice with identical address and length to that + // produced by `b.deref()`. + let ptr = Ptr::from_ref(b.into_byte_slice()) + .try_cast_into_no_leftover::<T, BecauseImmutable>(None) + .expect("zerocopy internal error: into_ref should be infallible"); + let ptr = ptr.recall_validity(); + ptr.as_ref() + } +} + +impl<'a, B, T> Ref<B, T> +where + B: 'a + IntoByteSliceMut<'a>, + T: FromBytes + IntoBytes + KnownLayout + ?Sized, +{ + /// Converts this `Ref` into a mutable reference. + /// + /// `into_mut` consumes the `Ref`, and returns a mutable reference to `T`. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::into_mut(r)` instead of `r.into_mut()`. This is so that + /// there is no conflict with a method on the inner type. + #[must_use = "has no side effects"] + #[inline(always)] + pub fn into_mut(r: Self) -> &'a mut T { + // Presumably unreachable, since we've guarded each constructor of `Ref`. + static_assert_dst_is_not_zst!(T); + + // SAFETY: We don't call any methods on `b` other than those provided by + // `IntoByteSliceMut`. + let b = unsafe { r.into_byte_slice_mut() }; + let b = b.into_byte_slice_mut(); + + if let crate::layout::SizeInfo::Sized { .. } = T::LAYOUT.size_info { + let ptr = Ptr::from_mut(b); + // SAFETY: We just checked that `T: Sized`. By invariant on `r`, + // `b`'s size is equal to `size_of::<T>()`. + let ptr = unsafe { + cast_for_sized::< + T, + _, + (BecauseRead, BecauseExclusive), + (BecauseMutationCompatible, BecauseInvariantsEq), + >(ptr) + }; + + // SAFETY: None of the preceding transformations modifies the + // address of the pointer, and by invariant on `r`, we know that it + // is validly-aligned. + let ptr = unsafe { ptr.assume_alignment::<Aligned>() }; + return ptr.as_mut(); + } + + // PANICS: By post-condition on `into_byte_slice_mut`, `b`'s size and + // alignment are valid for `T`. By post-condition, + // `b.into_byte_slice_mut()` produces a byte slice with identical + // address and length to that produced by `b.deref_mut()`. + let ptr = Ptr::from_mut(b.into_byte_slice_mut()) + .try_cast_into_no_leftover::<T, BecauseExclusive>(None) + .expect("zerocopy internal error: into_ref should be infallible"); + let ptr = ptr.recall_validity::<_, (_, (_, _))>(); + ptr.as_mut() + } +} + +impl<B, T> Ref<B, T> +where + B: ByteSlice, + T: ?Sized, +{ + /// Gets the underlying bytes. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::bytes(r)` instead of `r.bytes()`. This is so that there is + /// no conflict with a method on the inner type. + #[inline] + pub fn bytes(r: &Self) -> &[u8] { + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSlice`. + unsafe { r.as_byte_slice().deref() } + } +} + +impl<B, T> Ref<B, T> +where + B: ByteSliceMut, + T: ?Sized, +{ + /// Gets the underlying bytes mutably. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::bytes_mut(r)` instead of `r.bytes_mut()`. This is so that + /// there is no conflict with a method on the inner type. + #[inline] + pub fn bytes_mut(r: &mut Self) -> &mut [u8] { + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSliceMut`. + unsafe { r.as_byte_slice_mut().deref_mut() } + } +} + +impl<B, T> Ref<B, T> +where + B: ByteSlice, + T: FromBytes, +{ + /// Reads a copy of `T`. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::read(r)` instead of `r.read()`. This is so that there is no + /// conflict with a method on the inner type. + #[must_use = "has no side effects"] + #[inline] + pub fn read(r: &Self) -> T { + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSlice`. + let b = unsafe { r.as_byte_slice() }; + + // SAFETY: By postcondition on `as_byte_slice`, we know that `b` is a + // valid size and alignment for `T`. By safety invariant on `ByteSlice`, + // we know that this is preserved via `.deref()`. Because `T: + // FromBytes`, it is sound to interpret these bytes as a `T`. + unsafe { ptr::read(b.deref().as_ptr().cast::<T>()) } + } +} + +impl<B, T> Ref<B, T> +where + B: ByteSliceMut, + T: IntoBytes, +{ + /// Writes the bytes of `t` and then forgets `t`. + /// + /// Note: this is an associated function, which means that you have to call + /// it as `Ref::write(r, t)` instead of `r.write(t)`. This is so that there + /// is no conflict with a method on the inner type. + #[inline] + pub fn write(r: &mut Self, t: T) { + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSliceMut`. + let b = unsafe { r.as_byte_slice_mut() }; + + // SAFETY: By postcondition on `as_byte_slice_mut`, we know that `b` is + // a valid size and alignment for `T`. By safety invariant on + // `ByteSlice`, we know that this is preserved via `.deref()`. Writing + // `t` to the buffer will allow all of the bytes of `t` to be accessed + // as a `[u8]`, but because `T: IntoBytes`, we know that this is sound. + unsafe { ptr::write(b.deref_mut().as_mut_ptr().cast::<T>(), t) } + } +} + +impl<B, T> Deref for Ref<B, T> +where + B: ByteSlice, + T: FromBytes + KnownLayout + Immutable + ?Sized, +{ + type Target = T; + #[inline] + fn deref(&self) -> &T { + // Presumably unreachable, since we've guarded each constructor of `Ref`. + static_assert_dst_is_not_zst!(T); + + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSlice`. + let b = unsafe { self.as_byte_slice() }; + let b = b.deref(); + + if let crate::layout::SizeInfo::Sized { .. } = T::LAYOUT.size_info { + let ptr = Ptr::from_ref(b); + // SAFETY: We just checked that `T: Sized`. By invariant on `r`, + // `b`'s size is equal to `size_of::<T>()`. + let ptr = unsafe { cast_for_sized::<T, _, _, _>(ptr) }; + + // SAFETY: None of the preceding transformations modifies the + // address of the pointer, and by invariant on `r`, we know that it + // is validly-aligned. + let ptr = unsafe { ptr.assume_alignment::<Aligned>() }; + return ptr.as_ref(); + } + + // PANICS: By postcondition on `as_byte_slice`, `b`'s size and alignment + // are valid for `T`, and by invariant on `ByteSlice`, these are + // preserved through `.deref()`, so this `unwrap` will not panic. + let ptr = Ptr::from_ref(b) + .try_cast_into_no_leftover::<T, BecauseImmutable>(None) + .expect("zerocopy internal error: Deref::deref should be infallible"); + let ptr = ptr.recall_validity(); + ptr.as_ref() + } +} + +impl<B, T> DerefMut for Ref<B, T> +where + B: ByteSliceMut, + // FIXME(#251): We can't remove `Immutable` here because it's required by + // the impl of `Deref`, which is a super-trait of `DerefMut`. Maybe we can + // add a separate inherent method for this? + T: FromBytes + IntoBytes + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn deref_mut(&mut self) -> &mut T { + // Presumably unreachable, since we've guarded each constructor of `Ref`. + static_assert_dst_is_not_zst!(T); + + // SAFETY: We don't call any methods on `b` other than those provided by + // `ByteSliceMut`. + let b = unsafe { self.as_byte_slice_mut() }; + let b = b.deref_mut(); + + if let crate::layout::SizeInfo::Sized { .. } = T::LAYOUT.size_info { + let ptr = Ptr::from_mut(b); + // SAFETY: We just checked that `T: Sized`. By invariant on `r`, + // `b`'s size is equal to `size_of::<T>()`. + let ptr = unsafe { + cast_for_sized::< + T, + _, + (BecauseRead, BecauseExclusive), + (BecauseMutationCompatible, BecauseInvariantsEq), + >(ptr) + }; + + // SAFETY: None of the preceding transformations modifies the + // address of the pointer, and by invariant on `r`, we know that it + // is validly-aligned. + let ptr = unsafe { ptr.assume_alignment::<Aligned>() }; + return ptr.as_mut(); + } + + // PANICS: By postcondition on `as_byte_slice_mut`, `b`'s size and + // alignment are valid for `T`, and by invariant on `ByteSlice`, these + // are preserved through `.deref_mut()`, so this `unwrap` will not + // panic. + let ptr = Ptr::from_mut(b) + .try_cast_into_no_leftover::<T, BecauseExclusive>(None) + .expect("zerocopy internal error: DerefMut::deref_mut should be infallible"); + let ptr = ptr.recall_validity::<_, (_, (_, BecauseExclusive))>(); + ptr.as_mut() + } +} + +impl<T, B> Display for Ref<B, T> +where + B: ByteSlice, + T: FromBytes + Display + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { + let inner: &T = self; + inner.fmt(fmt) + } +} + +impl<T, B> Debug for Ref<B, T> +where + B: ByteSlice, + T: FromBytes + Debug + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { + let inner: &T = self; + fmt.debug_tuple("Ref").field(&inner).finish() + } +} + +impl<T, B> Eq for Ref<B, T> +where + B: ByteSlice, + T: FromBytes + Eq + KnownLayout + Immutable + ?Sized, +{ +} + +impl<T, B> PartialEq for Ref<B, T> +where + B: ByteSlice, + T: FromBytes + PartialEq + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn eq(&self, other: &Self) -> bool { + self.deref().eq(other.deref()) + } +} + +impl<T, B> Ord for Ref<B, T> +where + B: ByteSlice, + T: FromBytes + Ord + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + let inner: &T = self; + let other_inner: &T = other; + inner.cmp(other_inner) + } +} + +impl<T, B> PartialOrd for Ref<B, T> +where + B: ByteSlice, + T: FromBytes + PartialOrd + KnownLayout + Immutable + ?Sized, +{ + #[inline] + fn partial_cmp(&self, other: &Self) -> Option<Ordering> { + let inner: &T = self; + let other_inner: &T = other; + inner.partial_cmp(other_inner) + } +} + +/// # Safety +/// +/// `T: Sized` and `ptr`'s referent must have size `size_of::<T>()`. +#[inline(always)] +unsafe fn cast_for_sized<'a, T, A, R, S>( + ptr: Ptr<'a, [u8], (A, Aligned, Valid)>, +) -> Ptr<'a, T, (A, Unaligned, Valid)> +where + T: FromBytes + KnownLayout + ?Sized, + A: crate::invariant::Aliasing, + [u8]: MutationCompatible<T, A, Initialized, Initialized, R>, + T: TransmuteFromPtr<T, A, Initialized, Valid, crate::pointer::cast::IdCast, S>, +{ + use crate::pointer::cast::{Cast, Project}; + + enum CastForSized {} + + // SAFETY: `CastForSized` is only used below with the input `ptr`, which the + // caller promises has size `size_of::<T>()`. Thus, the referent produced in + // this cast has the same size as `ptr`'s referent. All operations preserve + // provenance. + unsafe impl<T: ?Sized + KnownLayout> Project<[u8], T> for CastForSized { + #[inline(always)] + fn project(src: PtrInner<'_, [u8]>) -> *mut T { + T::raw_from_ptr_len( + src.as_non_null().cast(), + <T::PointerMetadata as crate::PointerMetadata>::from_elem_count(0), + ) + .as_ptr() + } + } + + // SAFETY: The `Project::project` impl preserves referent address. + unsafe impl<T: ?Sized + KnownLayout> Cast<[u8], T> for CastForSized {} + + ptr.recall_validity::<Initialized, (_, (_, _))>() + .cast::<_, CastForSized, _>() + .recall_validity::<Valid, _>() +} + +#[cfg(test)] +#[allow(clippy::assertions_on_result_states)] +mod tests { + use core::convert::TryInto as _; + + use super::*; + use crate::util::testutil::*; + + #[test] + fn test_mut_slice_into_ref() { + // Prior to #1260/#1299, calling `into_ref` on a `&mut [u8]`-backed + // `Ref` was not supported. + let mut buf = [0u8]; + let r = Ref::<&mut [u8], u8>::from_bytes(&mut buf).unwrap(); + assert_eq!(Ref::into_ref(r), &0); + } + + #[test] + fn test_address() { + // Test that the `Deref` and `DerefMut` implementations return a + // reference which points to the right region of memory. + + let buf = [0]; + let r = Ref::<_, u8>::from_bytes(&buf[..]).unwrap(); + let buf_ptr = buf.as_ptr(); + let deref_ptr: *const u8 = r.deref(); + assert_eq!(buf_ptr, deref_ptr); + + let buf = [0]; + let r = Ref::<_, [u8]>::from_bytes(&buf[..]).unwrap(); + let buf_ptr = buf.as_ptr(); + let deref_ptr = r.deref().as_ptr(); + assert_eq!(buf_ptr, deref_ptr); + } + + // Verify that values written to a `Ref` are properly shared between the + // typed and untyped representations, that reads via `deref` and `read` + // behave the same, and that writes via `deref_mut` and `write` behave the + // same. + fn test_new_helper(mut r: Ref<&mut [u8], AU64>) { + // assert that the value starts at 0 + assert_eq!(*r, AU64(0)); + assert_eq!(Ref::read(&r), AU64(0)); + + // Assert that values written to the typed value are reflected in the + // byte slice. + const VAL1: AU64 = AU64(0xFF00FF00FF00FF00); + *r = VAL1; + assert_eq!(Ref::bytes(&r), &VAL1.to_bytes()); + *r = AU64(0); + Ref::write(&mut r, VAL1); + assert_eq!(Ref::bytes(&r), &VAL1.to_bytes()); + + // Assert that values written to the byte slice are reflected in the + // typed value. + const VAL2: AU64 = AU64(!VAL1.0); // different from `VAL1` + Ref::bytes_mut(&mut r).copy_from_slice(&VAL2.to_bytes()[..]); + assert_eq!(*r, VAL2); + assert_eq!(Ref::read(&r), VAL2); + } + + // Verify that values written to a `Ref` are properly shared between the + // typed and untyped representations; pass a value with `typed_len` `AU64`s + // backed by an array of `typed_len * 8` bytes. + fn test_new_helper_slice(mut r: Ref<&mut [u8], [AU64]>, typed_len: usize) { + // Assert that the value starts out zeroed. + assert_eq!(&*r, vec![AU64(0); typed_len].as_slice()); + + // Check the backing storage is the exact same slice. + let untyped_len = typed_len * 8; + assert_eq!(Ref::bytes(&r).len(), untyped_len); + assert_eq!(Ref::bytes(&r).as_ptr(), r.as_ptr().cast::<u8>()); + + // Assert that values written to the typed value are reflected in the + // byte slice. + const VAL1: AU64 = AU64(0xFF00FF00FF00FF00); + for typed in &mut *r { + *typed = VAL1; + } + assert_eq!(Ref::bytes(&r), VAL1.0.to_ne_bytes().repeat(typed_len).as_slice()); + + // Assert that values written to the byte slice are reflected in the + // typed value. + const VAL2: AU64 = AU64(!VAL1.0); // different from VAL1 + Ref::bytes_mut(&mut r).copy_from_slice(&VAL2.0.to_ne_bytes().repeat(typed_len)); + assert!(r.iter().copied().all(|x| x == VAL2)); + } + + #[test] + fn test_new_aligned_sized() { + // Test that a properly-aligned, properly-sized buffer works for new, + // new_from_prefix, and new_from_suffix, and that new_from_prefix and + // new_from_suffix return empty slices. Test that a properly-aligned + // buffer whose length is a multiple of the element size works for + // new_slice. + + // A buffer with an alignment of 8. + let mut buf = Align::<[u8; 8], AU64>::default(); + // `buf.t` should be aligned to 8, so this should always succeed. + test_new_helper(Ref::<_, AU64>::from_bytes(&mut buf.t[..]).unwrap()); + { + // In a block so that `r` and `suffix` don't live too long. + buf.set_default(); + let (r, suffix) = Ref::<_, AU64>::from_prefix(&mut buf.t[..]).unwrap(); + assert!(suffix.is_empty()); + test_new_helper(r); + } + { + buf.set_default(); + let (prefix, r) = Ref::<_, AU64>::from_suffix(&mut buf.t[..]).unwrap(); + assert!(prefix.is_empty()); + test_new_helper(r); + } + + // A buffer with alignment 8 and length 24. We choose this length very + // intentionally: if we instead used length 16, then the prefix and + // suffix lengths would be identical. In the past, we used length 16, + // which resulted in this test failing to discover the bug uncovered in + // #506. + let mut buf = Align::<[u8; 24], AU64>::default(); + // `buf.t` should be aligned to 8 and have a length which is a multiple + // of `size_of::<AU64>()`, so this should always succeed. + test_new_helper_slice(Ref::<_, [AU64]>::from_bytes(&mut buf.t[..]).unwrap(), 3); + buf.set_default(); + let r = Ref::<_, [AU64]>::from_bytes_with_elems(&mut buf.t[..], 3).unwrap(); + test_new_helper_slice(r, 3); + + let ascending: [u8; 24] = (0..24).collect::<Vec<_>>().try_into().unwrap(); + // 16 ascending bytes followed by 8 zeros. + let mut ascending_prefix = ascending; + ascending_prefix[16..].copy_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0]); + // 8 zeros followed by 16 ascending bytes. + let mut ascending_suffix = ascending; + ascending_suffix[..8].copy_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0]); + { + buf.t = ascending_suffix; + let (r, suffix) = Ref::<_, [AU64]>::from_prefix_with_elems(&mut buf.t[..], 1).unwrap(); + assert_eq!(suffix, &ascending[8..]); + test_new_helper_slice(r, 1); + } + { + buf.t = ascending_prefix; + let (prefix, r) = Ref::<_, [AU64]>::from_suffix_with_elems(&mut buf.t[..], 1).unwrap(); + assert_eq!(prefix, &ascending[..16]); + test_new_helper_slice(r, 1); + } + } + + #[test] + fn test_new_oversized() { + // Test that a properly-aligned, overly-sized buffer works for + // `new_from_prefix` and `new_from_suffix`, and that they return the + // remainder and prefix of the slice respectively. + + let mut buf = Align::<[u8; 16], AU64>::default(); + { + // In a block so that `r` and `suffix` don't live too long. `buf.t` + // should be aligned to 8, so this should always succeed. + let (r, suffix) = Ref::<_, AU64>::from_prefix(&mut buf.t[..]).unwrap(); + assert_eq!(suffix.len(), 8); + test_new_helper(r); + } + { + buf.set_default(); + // `buf.t` should be aligned to 8, so this should always succeed. + let (prefix, r) = Ref::<_, AU64>::from_suffix(&mut buf.t[..]).unwrap(); + assert_eq!(prefix.len(), 8); + test_new_helper(r); + } + } + + #[test] + #[allow(clippy::cognitive_complexity)] + fn test_new_error() { + // Fail because the buffer is too large. + + // A buffer with an alignment of 8. + let buf = Align::<[u8; 16], AU64>::default(); + // `buf.t` should be aligned to 8, so only the length check should fail. + assert!(Ref::<_, AU64>::from_bytes(&buf.t[..]).is_err()); + + // Fail because the buffer is too small. + + // A buffer with an alignment of 8. + let buf = Align::<[u8; 4], AU64>::default(); + // `buf.t` should be aligned to 8, so only the length check should fail. + assert!(Ref::<_, AU64>::from_bytes(&buf.t[..]).is_err()); + assert!(Ref::<_, AU64>::from_prefix(&buf.t[..]).is_err()); + assert!(Ref::<_, AU64>::from_suffix(&buf.t[..]).is_err()); + + // Fail because the length is not a multiple of the element size. + + let buf = Align::<[u8; 12], AU64>::default(); + // `buf.t` has length 12, but element size is 8. + assert!(Ref::<_, [AU64]>::from_bytes(&buf.t[..]).is_err()); + + // Fail because the buffer is too short. + let buf = Align::<[u8; 12], AU64>::default(); + // `buf.t` has length 12, but the element size is 8 (and we're expecting + // two of them). For each function, we test with a length that would + // cause the size to overflow `usize`, and with a normal length that + // will fail thanks to the buffer being too short; these are different + // error paths, and while the error types are the same, the distinction + // shows up in code coverage metrics. + let n = (usize::MAX / mem::size_of::<AU64>()) + 1; + assert!(Ref::<_, [AU64]>::from_bytes_with_elems(&buf.t[..], n).is_err()); + assert!(Ref::<_, [AU64]>::from_bytes_with_elems(&buf.t[..], 2).is_err()); + assert!(Ref::<_, [AU64]>::from_prefix_with_elems(&buf.t[..], n).is_err()); + assert!(Ref::<_, [AU64]>::from_prefix_with_elems(&buf.t[..], 2).is_err()); + assert!(Ref::<_, [AU64]>::from_suffix_with_elems(&buf.t[..], n).is_err()); + assert!(Ref::<_, [AU64]>::from_suffix_with_elems(&buf.t[..], 2).is_err()); + + // Fail because the alignment is insufficient. + + // A buffer with an alignment of 8. An odd buffer size is chosen so that + // the last byte of the buffer has odd alignment. + let buf = Align::<[u8; 13], AU64>::default(); + // Slicing from 1, we get a buffer with size 12 (so the length check + // should succeed) but an alignment of only 1, which is insufficient. + assert!(Ref::<_, AU64>::from_bytes(&buf.t[1..]).is_err()); + assert!(Ref::<_, AU64>::from_prefix(&buf.t[1..]).is_err()); + assert!(Ref::<_, [AU64]>::from_bytes(&buf.t[1..]).is_err()); + assert!(Ref::<_, [AU64]>::from_bytes_with_elems(&buf.t[1..], 1).is_err()); + assert!(Ref::<_, [AU64]>::from_prefix_with_elems(&buf.t[1..], 1).is_err()); + assert!(Ref::<_, [AU64]>::from_suffix_with_elems(&buf.t[1..], 1).is_err()); + // Slicing is unnecessary here because `new_from_suffix` uses the suffix + // of the slice, which has odd alignment. + assert!(Ref::<_, AU64>::from_suffix(&buf.t[..]).is_err()); + + // Fail due to arithmetic overflow. + + let buf = Align::<[u8; 16], AU64>::default(); + let unreasonable_len = usize::MAX / mem::size_of::<AU64>() + 1; + assert!(Ref::<_, [AU64]>::from_prefix_with_elems(&buf.t[..], unreasonable_len).is_err()); + assert!(Ref::<_, [AU64]>::from_suffix_with_elems(&buf.t[..], unreasonable_len).is_err()); + } + + #[test] + #[allow(unstable_name_collisions)] + #[allow(clippy::as_conversions)] + fn test_into_ref_mut() { + #[allow(unused)] + use crate::util::AsAddress as _; + + let mut buf = Align::<[u8; 8], u64>::default(); + let r = Ref::<_, u64>::from_bytes(&buf.t[..]).unwrap(); + let rf = Ref::into_ref(r); + assert_eq!(rf, &0u64); + let buf_addr = (&buf.t as *const [u8; 8]).addr(); + assert_eq!((rf as *const u64).addr(), buf_addr); + + let r = Ref::<_, u64>::from_bytes(&mut buf.t[..]).unwrap(); + let rf = Ref::into_mut(r); + assert_eq!(rf, &mut 0u64); + assert_eq!((rf as *mut u64).addr(), buf_addr); + + *rf = u64::MAX; + assert_eq!(buf.t, [0xFF; 8]); + } + + #[test] + fn test_display_debug() { + let buf = Align::<[u8; 8], u64>::default(); + let r = Ref::<_, u64>::from_bytes(&buf.t[..]).unwrap(); + assert_eq!(format!("{}", r), "0"); + assert_eq!(format!("{:?}", r), "Ref(0)"); + + let buf = Align::<[u8; 8], u64>::default(); + let r = Ref::<_, [u64]>::from_bytes(&buf.t[..]).unwrap(); + assert_eq!(format!("{:?}", r), "Ref([0])"); + } + + #[test] + fn test_eq() { + let buf1 = 0_u64; + let r1 = Ref::<_, u64>::from_bytes(buf1.as_bytes()).unwrap(); + let buf2 = 0_u64; + let r2 = Ref::<_, u64>::from_bytes(buf2.as_bytes()).unwrap(); + assert_eq!(r1, r2); + } + + #[test] + fn test_ne() { + let buf1 = 0_u64; + let r1 = Ref::<_, u64>::from_bytes(buf1.as_bytes()).unwrap(); + let buf2 = 1_u64; + let r2 = Ref::<_, u64>::from_bytes(buf2.as_bytes()).unwrap(); + assert_ne!(r1, r2); + } + + #[test] + fn test_ord() { + let buf1 = 0_u64; + let r1 = Ref::<_, u64>::from_bytes(buf1.as_bytes()).unwrap(); + let buf2 = 1_u64; + let r2 = Ref::<_, u64>::from_bytes(buf2.as_bytes()).unwrap(); + assert!(r1 < r2); + assert_eq!(PartialOrd::partial_cmp(&r1, &r2), Some(Ordering::Less)); + assert_eq!(Ord::cmp(&r1, &r2), Ordering::Less); + } +} + +#[cfg(all(test, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS))] +mod benches { + use test::{self, Bencher}; + + use super::*; + use crate::util::testutil::*; + + #[bench] + fn bench_from_bytes_sized(b: &mut Bencher) { + let buf = Align::<[u8; 8], AU64>::default(); + // `buf.t` should be aligned to 8, so this should always succeed. + let bytes = &buf.t[..]; + b.iter(|| test::black_box(Ref::<_, AU64>::from_bytes(test::black_box(bytes)).unwrap())); + } + + #[bench] + fn bench_into_ref_sized(b: &mut Bencher) { + let buf = Align::<[u8; 8], AU64>::default(); + let bytes = &buf.t[..]; + let r = Ref::<_, AU64>::from_bytes(bytes).unwrap(); + b.iter(|| test::black_box(Ref::into_ref(test::black_box(r)))); + } + + #[bench] + fn bench_into_mut_sized(b: &mut Bencher) { + let mut buf = Align::<[u8; 8], AU64>::default(); + let buf = &mut buf.t[..]; + let _ = Ref::<_, AU64>::from_bytes(&mut *buf).unwrap(); + b.iter(move || { + // SAFETY: The preceding `from_bytes` succeeded, and so we know that + // `buf` is validly-aligned and has the correct length. + let r = unsafe { Ref::<&mut [u8], AU64>::new_unchecked(&mut *buf) }; + test::black_box(Ref::into_mut(test::black_box(r))); + }); + } + + #[bench] + fn bench_deref_sized(b: &mut Bencher) { + let buf = Align::<[u8; 8], AU64>::default(); + let bytes = &buf.t[..]; + let r = Ref::<_, AU64>::from_bytes(bytes).unwrap(); + b.iter(|| { + let temp = test::black_box(r); + test::black_box(temp.deref()); + }); + } + + #[bench] + fn bench_deref_mut_sized(b: &mut Bencher) { + let mut buf = Align::<[u8; 8], AU64>::default(); + let buf = &mut buf.t[..]; + let _ = Ref::<_, AU64>::from_bytes(&mut *buf).unwrap(); + b.iter(|| { + // SAFETY: The preceding `from_bytes` succeeded, and so we know that + // `buf` is validly-aligned and has the correct length. + let r = unsafe { Ref::<&mut [u8], AU64>::new_unchecked(&mut *buf) }; + let mut temp = test::black_box(r); + test::black_box(temp.deref_mut()); + }); + } +} diff --git a/rust/zerocopy/src/split_at.rs b/rust/zerocopy/src/split_at.rs new file mode 100644 index 000000000000..d7778425a31d --- /dev/null +++ b/rust/zerocopy/src/split_at.rs @@ -0,0 +1,1090 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2025 The Fuchsia Authors +// +// Licensed under the 2-Clause BSD License <LICENSE-BSD or +// https://opensource.org/license/bsd-2-clause>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use super::*; +use crate::pointer::invariant::{Aligned, Exclusive, Invariants, Shared, Valid}; + +/// Types that can be split in two. +/// +/// This trait generalizes Rust's existing support for splitting slices to +/// support slices and slice-based dynamically-sized types ("slice DSTs"). +/// +/// # Implementation +/// +/// **Do not implement this trait yourself!** Instead, use +/// [`#[derive(SplitAt)]`][derive]; e.g.: +/// +/// ``` +/// # use zerocopy_derive::{SplitAt, KnownLayout}; +/// #[derive(SplitAt, KnownLayout)] +/// #[repr(C)] +/// struct MyStruct<T: ?Sized> { +/// # /* +/// ..., +/// # */ +/// // `SplitAt` types must have at least one field. +/// field: T, +/// } +/// ``` +/// +/// This derive performs a sophisticated, compile-time safety analysis to +/// determine whether a type is `SplitAt`. +/// +/// # Safety +/// +/// This trait does not convey any safety guarantees to code outside this crate. +/// +/// You must not rely on the `#[doc(hidden)]` internals of `SplitAt`. Future +/// releases of zerocopy may make backwards-breaking changes to these items, +/// including changes that only affect soundness, which may cause code which +/// uses those items to silently become unsound. +/// +#[cfg_attr(feature = "derive", doc = "[derive]: zerocopy_derive::SplitAt")] +#[cfg_attr( + not(feature = "derive"), + doc = concat!("[derive]: https://docs.rs/zerocopy/", env!("CARGO_PKG_VERSION"), "/zerocopy/derive.SplitAt.html"), +)] +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented(note = "Consider adding `#[derive(SplitAt)]` to `{Self}`") +)] +// # Safety +// +// The trailing slice is well-aligned for its element type. `Self` is `[T]`, or +// a `repr(C)` or `repr(transparent)` slice DST. +pub unsafe trait SplitAt: KnownLayout<PointerMetadata = usize> { + /// The element type of the trailing slice. + type Elem; + + #[doc(hidden)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized; + + /// Unsafely splits `self` in two. + /// + /// # Safety + /// + /// The caller promises that `l_len` is not greater than the length of + /// `self`'s trailing slice. + /// + #[doc = codegen_section!( + header = "h5", + bench = "split_at_unchecked", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[inline] + #[must_use] + unsafe fn split_at_unchecked(&self, l_len: usize) -> Split<&Self> { + // SAFETY: By precondition on the caller, `l_len <= self.len()`. + unsafe { Split::<&Self>::new(self, l_len) } + } + + /// Attempts to split `self` in two. + /// + /// Returns `None` if `l_len` is greater than the length of `self`'s + /// trailing slice. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// length: u8, + /// body: [u8], + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = &[4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at(packet.length as usize).unwrap(); + /// + /// // Use the `Immutable` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_immutable(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "split_at", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[inline] + #[must_use = "has no side effects"] + fn split_at(&self, l_len: usize) -> Option<Split<&Self>> { + MetadataOf::new_in_bounds(self, l_len).map( + #[inline(always)] + |l_len| { + // SAFETY: We have ensured that `l_len <= self.len()` (by + // post-condition on `MetadataOf::new_in_bounds`) + unsafe { Split::new(self, l_len.get()) } + }, + ) + } + + /// Unsafely splits `self` in two. + /// + /// # Safety + /// + /// The caller promises that `l_len` is not greater than the length of + /// `self`'s trailing slice. + /// + #[doc = codegen_header!("h5", "split_at_mut_unchecked")] + /// + /// See [`SplitAt::split_at_unchecked`](#method.split_at_unchecked.codegen). + #[inline] + #[must_use] + unsafe fn split_at_mut_unchecked(&mut self, l_len: usize) -> Split<&mut Self> { + // SAFETY: By precondition on the caller, `l_len <= self.len()`. + unsafe { Split::<&mut Self>::new(self, l_len) } + } + + /// Attempts to split `self` in two. + /// + /// Returns `None` if `l_len` is greater than the length of `self`'s + /// trailing slice, or if the given `l_len` would result in [the trailing + /// padding](KnownLayout#slice-dst-layout) of the left portion overlapping + /// the right portion. + /// + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, IntoBytes)] + /// #[repr(C)] + /// struct Packet<B: ?Sized> { + /// length: u8, + /// body: B, + /// } + /// + /// // These bytes encode a `Packet`. + /// let mut bytes = &mut [4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::<[u8]>::mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// { + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at_mut(packet.length as usize).unwrap(); + /// + /// // Use the `IntoBytes` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_into_bytes(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// + /// rest.fill(0); + /// } + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 0, 0, 0, 0, 0]); + /// ``` + /// + #[doc = codegen_header!("h5", "split_at_mut")] + /// + /// See [`SplitAt::split_at`](#method.split_at.codegen). + #[inline] + fn split_at_mut(&mut self, l_len: usize) -> Option<Split<&mut Self>> { + MetadataOf::new_in_bounds(self, l_len).map( + #[inline(always)] + |l_len| { + // SAFETY: We have ensured that `l_len <= self.len()` (by + // post-condition on `MetadataOf::new_in_bounds`) + unsafe { Split::new(self, l_len.get()) } + }, + ) + } +} + +// SAFETY: `[T]`'s trailing slice is `[T]`, which is trivially aligned. +unsafe impl<T> SplitAt for [T] { + type Elem = T; + + #[inline] + #[allow(dead_code)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } +} + +/// A `T` that has been split into two possibly-overlapping parts. +/// +/// For some dynamically sized types, the padding that appears after the +/// trailing slice field [is a dynamic function of the trailing slice +/// length](KnownLayout#slice-dst-layout). If `T` is split at a length that +/// requires trailing padding, the trailing padding of the left part of the +/// split `T` will overlap the right part. If `T` is a mutable reference or +/// permits interior mutation, you must ensure that the left and right parts do +/// not overlap. You can do this at zero-cost using using +/// [`Self::via_immutable`], [`Self::via_into_bytes`], or +/// [`Self::via_unaligned`], or with a dynamic check by using +/// [`Self::via_runtime_check`]. +#[derive(Debug)] +pub struct Split<T> { + /// A pointer to the source slice DST. + source: T, + /// The length of the future left half of `source`. + /// + /// # Safety + /// + /// If `source` is a pointer to a slice DST, `l_len` is no greater than + /// `source`'s length. + l_len: usize, +} + +impl<T> Split<T> { + /// Produces a `Split` of `source` with `l_len`. + /// + /// # Safety + /// + /// `l_len` is no greater than `source`'s length. + #[inline(always)] + unsafe fn new(source: T, l_len: usize) -> Self { + Self { source, l_len } + } +} + +impl<'a, T> Split<&'a T> +where + T: ?Sized + SplitAt, +{ + #[inline(always)] + fn into_ptr(self) -> Split<Ptr<'a, T, (Shared, Aligned, Valid)>> { + let source = Ptr::from_ref(self.source); + // SAFETY: `Ptr::from_ref(self.source)` points to exactly `self.source` + // and thus maintains the invariants of `self` with respect to `l_len`. + unsafe { Split::new(source, self.l_len) } + } + + /// Produces the split parts of `self`, using [`Immutable`] to ensure that + /// it is sound to have concurrent references to both parts. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable)] + /// #[repr(C)] + /// struct Packet { + /// length: u8, + /// body: [u8], + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = &[4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at(packet.length as usize).unwrap(); + /// + /// // Use the `Immutable` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_immutable(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "split_via_immutable", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_immutable(self) -> (&'a T, &'a [T::Elem]) + where + T: Immutable, + { + let (l, r) = self.into_ptr().via_immutable(); + (l.as_ref(), r.as_ref()) + } + + /// Produces the split parts of `self`, using [`IntoBytes`] to ensure that + /// it is sound to have concurrent references to both parts. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable, IntoBytes)] + /// #[repr(C)] + /// struct Packet<B: ?Sized> { + /// length: u8, + /// body: B, + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = &[4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::<[u8]>::ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at(packet.length as usize).unwrap(); + /// + /// // Use the `IntoBytes` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_into_bytes(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_header!("h5", "split_via_into_bytes")] + /// + /// See [`Split::via_immutable`](#method.split_via_immutable.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_into_bytes(self) -> (&'a T, &'a [T::Elem]) + where + T: IntoBytes, + { + let (l, r) = self.into_ptr().via_into_bytes(); + (l.as_ref(), r.as_ref()) + } + + /// Produces the split parts of `self`, using [`Unaligned`] to ensure that + /// it is sound to have concurrent references to both parts. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable, Unaligned)] + /// #[repr(C)] + /// struct Packet { + /// length: u8, + /// body: [u8], + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = &[4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::ref_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at(packet.length as usize).unwrap(); + /// + /// // Use the `Unaligned` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_unaligned(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// ``` + /// + #[doc = codegen_header!("h5", "split_via_unaligned")] + /// + /// See [`Split::via_immutable`](#method.split_via_immutable.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_unaligned(self) -> (&'a T, &'a [T::Elem]) + where + T: Unaligned, + { + let (l, r) = self.into_ptr().via_unaligned(); + (l.as_ref(), r.as_ref()) + } + + /// Produces the split parts of `self`, using a dynamic check to ensure that + /// it is sound to have concurrent references to both parts. You should + /// prefer using [`Self::via_immutable`], [`Self::via_into_bytes`], or + /// [`Self::via_unaligned`], which have no runtime cost. + /// + /// Note that this check is overly conservative if `T` is [`Immutable`]; for + /// some types, this check will reject some splits which + /// [`Self::via_immutable`] will accept. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes, IntoBytes, network_endian::U16}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, Immutable, Debug)] + /// #[repr(C, align(2))] + /// struct Packet { + /// length: U16, + /// body: [u8], + /// } + /// + /// // These bytes encode a `Packet`. + /// let bytes = [ + /// 4u16.to_be(), + /// 1u16.to_be(), + /// 2u16.to_be(), + /// 3u16.to_be(), + /// 4u16.to_be() + /// ]; + /// + /// let packet = Packet::ref_from_bytes(bytes.as_bytes()).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [0, 1, 0, 2, 0, 3, 0, 4]); + /// + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at(packet.length.into()).unwrap(); + /// + /// // Use a dynamic check to prove that it's okay to return concurrent + /// // references to `packet` and `rest`. + /// let (packet, rest) = split.via_runtime_check().unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [0, 1, 0, 2]); + /// assert_eq!(rest, [0, 3, 0, 4]); + /// + /// // Attempt to split `packet` at `length - 1`. + /// let idx = packet.length.get() - 1; + /// let split = packet.split_at(idx as usize).unwrap(); + /// + /// // Attempt (and fail) to use a dynamic check to prove that it's okay + /// // to return concurrent references to `packet` and `rest`. Note that + /// // this is a case of `via_runtime_check` being overly conservative. + /// // Although the left and right parts indeed overlap, the `Immutable` + /// // bound ensures that concurrently referencing these overlapping + /// // parts is sound. + /// assert!(split.via_runtime_check().is_err()); + /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "split_via_runtime_check", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_runtime_check(self) -> Result<(&'a T, &'a [T::Elem]), Self> { + match self.into_ptr().via_runtime_check() { + Ok((l, r)) => Ok((l.as_ref(), r.as_ref())), + Err(s) => Err(s.into_ref()), + } + } + + /// Unsafely produces the split parts of `self`. + /// + /// # Safety + /// + /// If `T` permits interior mutation, the trailing padding bytes of the left + /// portion must not overlap the right portion. For some dynamically sized + /// types, the padding that appears after the trailing slice field [is a + /// dynamic function of the trailing slice + /// length](KnownLayout#slice-dst-layout). Thus, for some types, this + /// condition is dependent on the length of the left portion. + /// + #[doc = codegen_section!( + header = "h5", + bench = "split_via_unchecked", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] + #[must_use = "has no side effects"] + #[inline(always)] + pub unsafe fn via_unchecked(self) -> (&'a T, &'a [T::Elem]) { + // SAFETY: The aliasing of `self.into_ptr()` is not `Exclusive`, but the + // caller has promised that if `T` permits interior mutation then the + // left and right portions of `self` split at `l_len` do not overlap. + let (l, r) = unsafe { self.into_ptr().via_unchecked() }; + (l.as_ref(), r.as_ref()) + } +} + +impl<'a, T> Split<&'a mut T> +where + T: ?Sized + SplitAt, +{ + #[inline(always)] + fn into_ptr(self) -> Split<Ptr<'a, T, (Exclusive, Aligned, Valid)>> { + let source = Ptr::from_mut(self.source); + // SAFETY: `Ptr::from_mut(self.source)` points to exactly `self.source`, + // and thus maintains the invariants of `self` with respect to `l_len`. + unsafe { Split::new(source, self.l_len) } + } + + /// Produces the split parts of `self`, using [`IntoBytes`] to ensure that + /// it is sound to have concurrent references to both parts. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, IntoBytes)] + /// #[repr(C)] + /// struct Packet<B: ?Sized> { + /// length: u8, + /// body: B, + /// } + /// + /// // These bytes encode a `Packet`. + /// let mut bytes = &mut [4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::<[u8]>::mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// { + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at_mut(packet.length as usize).unwrap(); + /// + /// // Use the `IntoBytes` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_into_bytes(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// + /// rest.fill(0); + /// } + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 0, 0, 0, 0, 0]); + /// ``` + /// + /// # Code Generation + /// + /// See [`Split::via_immutable`](#method.split_via_immutable.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_into_bytes(self) -> (&'a mut T, &'a mut [T::Elem]) + where + T: IntoBytes, + { + let (l, r) = self.into_ptr().via_into_bytes(); + (l.as_mut(), r.as_mut()) + } + + /// Produces the split parts of `self`, using [`Unaligned`] to ensure that + /// it is sound to have concurrent references to both parts. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, IntoBytes, Unaligned)] + /// #[repr(C)] + /// struct Packet<B: ?Sized> { + /// length: u8, + /// body: B, + /// } + /// + /// // These bytes encode a `Packet`. + /// let mut bytes = &mut [4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::<[u8]>::mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// { + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at_mut(packet.length as usize).unwrap(); + /// + /// // Use the `Unaligned` bound on `Packet` to prove that it's okay to + /// // return concurrent references to `packet` and `rest`. + /// let (packet, rest) = split.via_unaligned(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// + /// rest.fill(0); + /// } + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 0, 0, 0, 0, 0]); + /// ``` + /// + /// # Code Generation + /// + /// See [`Split::via_immutable`](#method.split_via_immutable.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_unaligned(self) -> (&'a mut T, &'a mut [T::Elem]) + where + T: Unaligned, + { + let (l, r) = self.into_ptr().via_unaligned(); + (l.as_mut(), r.as_mut()) + } + + /// Produces the split parts of `self`, using a dynamic check to ensure that + /// it is sound to have concurrent references to both parts. You should + /// prefer using [`Self::via_into_bytes`] or [`Self::via_unaligned`], which + /// have no runtime cost. + /// + /// # Examples + /// + /// ``` + /// use zerocopy::{SplitAt, FromBytes}; + /// # use zerocopy_derive::*; + /// + /// #[derive(SplitAt, FromBytes, KnownLayout, IntoBytes, Debug)] + /// #[repr(C)] + /// struct Packet<B: ?Sized> { + /// length: u8, + /// body: B, + /// } + /// + /// // These bytes encode a `Packet`. + /// let mut bytes = &mut [4, 1, 2, 3, 4, 5, 6, 7, 8, 9][..]; + /// + /// let packet = Packet::<[u8]>::mut_from_bytes(bytes).unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// + /// { + /// // Attempt to split `packet` at `length`. + /// let split = packet.split_at_mut(packet.length as usize).unwrap(); + /// + /// // Use a dynamic check to prove that it's okay to return concurrent + /// // references to `packet` and `rest`. + /// let (packet, rest) = split.via_runtime_check().unwrap(); + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4]); + /// assert_eq!(rest, [5, 6, 7, 8, 9]); + /// + /// rest.fill(0); + /// } + /// + /// assert_eq!(packet.length, 4); + /// assert_eq!(packet.body, [1, 2, 3, 4, 0, 0, 0, 0, 0]); + /// ``` + /// + /// # Code Generation + /// + /// See [`Split::via_runtime_check`](#method.split_via_runtime_check.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub fn via_runtime_check(self) -> Result<(&'a mut T, &'a mut [T::Elem]), Self> { + match self.into_ptr().via_runtime_check() { + Ok((l, r)) => Ok((l.as_mut(), r.as_mut())), + Err(s) => Err(s.into_mut()), + } + } + + /// Unsafely produces the split parts of `self`. + /// + /// # Safety + /// + /// The trailing padding bytes of the left portion must not overlap the + /// right portion. For some dynamically sized types, the padding that + /// appears after the trailing slice field [is a dynamic function of the + /// trailing slice length](KnownLayout#slice-dst-layout). Thus, for some + /// types, this condition is dependent on the length of the left portion. + /// + /// # Code Generation + /// + /// See [`Split::via_unchecked`](#method.split_via_unchecked.codegen). + #[must_use = "has no side effects"] + #[inline(always)] + pub unsafe fn via_unchecked(self) -> (&'a mut T, &'a mut [T::Elem]) { + // SAFETY: The aliasing of `self.into_ptr()` is `Exclusive`, and the + // caller has promised that the left and right portions of `self` split + // at `l_len` do not overlap. + let (l, r) = unsafe { self.into_ptr().via_unchecked() }; + (l.as_mut(), r.as_mut()) + } +} + +impl<'a, T, I> Split<Ptr<'a, T, I>> +where + T: ?Sized + SplitAt, + I: Invariants<Alignment = Aligned, Validity = Valid>, +{ + fn into_ref(self) -> Split<&'a T> + where + I: Invariants<Aliasing = Shared>, + { + // SAFETY: `self.source.as_ref()` points to exactly the same referent as + // `self.source` and thus maintains the invariants of `self` with + // respect to `l_len`. + unsafe { Split::new(self.source.as_ref(), self.l_len) } + } + + fn into_mut(self) -> Split<&'a mut T> + where + I: Invariants<Aliasing = Exclusive>, + { + // SAFETY: `self.source.as_mut()` points to exactly the same referent as + // `self.source` and thus maintains the invariants of `self` with + // respect to `l_len`. + unsafe { Split::new(self.source.unify_invariants().as_mut(), self.l_len) } + } + + /// Produces the length of `self`'s left part. + #[inline(always)] + fn l_len(&self) -> MetadataOf<T> { + // SAFETY: By invariant on `Split`, `self.l_len` is not greater than the + // length of `self.source`. + unsafe { MetadataOf::<T>::new_unchecked(self.l_len) } + } + + /// Produces the split parts of `self`, using [`Immutable`] to ensure that + /// it is sound to have concurrent references to both parts. + #[inline(always)] + fn via_immutable(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) + where + T: Immutable, + I: Invariants<Aliasing = Shared>, + { + // SAFETY: `Aliasing = Shared` and `T: Immutable`. + unsafe { self.via_unchecked() } + } + + /// Produces the split parts of `self`, using [`IntoBytes`] to ensure that + /// it is sound to have concurrent references to both parts. + #[inline(always)] + fn via_into_bytes(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) + where + T: IntoBytes, + { + // SAFETY: By `T: IntoBytes`, `T` has no padding for any length. + // Consequently, `T` can be split into non-overlapping parts at any + // index. + unsafe { self.via_unchecked() } + } + + /// Produces the split parts of `self`, using [`Unaligned`] to ensure that + /// it is sound to have concurrent references to both parts. + #[inline(always)] + fn via_unaligned(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) + where + T: Unaligned, + { + // SAFETY: By `T: SplitAt + Unaligned`, `T` is either a slice or a + // `repr(C)` or `repr(transparent)` slice DST that is well-aligned at + // any address and length. If `T` is a slice DST with alignment 1, + // `repr(C)` or `repr(transparent)` ensures that no padding is placed + // after the final element of the trailing slice. Consequently, `T` can + // be split into strictly non-overlapping parts any any index. + unsafe { self.via_unchecked() } + } + + /// Produces the split parts of `self`, using a dynamic check to ensure that + /// it is sound to have concurrent references to both parts. You should + /// prefer using [`Self::via_immutable`], [`Self::via_into_bytes`], or + /// [`Self::via_unaligned`], which have no runtime cost. + #[inline(always)] + fn via_runtime_check(self) -> Result<(Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>), Self> { + let l_len = self.l_len(); + // FIXME(#1290): Once we require `KnownLayout` on all fields, add an + // `IS_IMMUTABLE` associated const, and add `T::IS_IMMUTABLE ||` to the + // below check. + if l_len.padding_needed_for() == 0 { + // SAFETY: By `T: SplitAt`, `T` is either `[T]`, or a `repr(C)` or + // `repr(transparent)` slice DST, for which the trailing padding + // needed to accommodate `l_len` trailing elements is + // `l_len.padding_needed_for()`. If no trailing padding is required, + // the left and right parts are strictly non-overlapping. + Ok(unsafe { self.via_unchecked() }) + } else { + Err(self) + } + } + + /// Unsafely produces the split parts of `self`. + /// + /// # Safety + /// + /// The caller promises that if `I::Aliasing` is [`Exclusive`] or `T` + /// permits interior mutation, then `l_len.padding_needed_for() == 0`. + #[inline(always)] + unsafe fn via_unchecked(self) -> (Ptr<'a, T, I>, Ptr<'a, [T::Elem], I>) { + let l_len = self.l_len(); + let inner = self.source.as_inner(); + + // SAFETY: By invariant on `Self::l_len`, `l_len` is not greater than + // the length of `inner`'s trailing slice. + let (left, right) = unsafe { inner.split_at_unchecked(l_len) }; + + // Lemma 0: `left` and `right` conform to the aliasing invariant + // `I::Aliasing`. Proof: If `I::Aliasing` is `Exclusive` or `T` permits + // interior mutation, the caller promises that `l_len.padding_needed_for() + // == 0`. Consequently, by post-condition on `PtrInner::split_at_unchecked`, + // there is no trailing padding after `left`'s final element that would + // overlap into `right`. If `I::Aliasing` is shared and `T` forbids interior + // mutation, then overlap between their referents is permissible. + + // SAFETY: + // 0. `left` conforms to the aliasing invariant of `I::Aliasing`, by Lemma 0. + // 1. `left` conforms to the alignment invariant of `I::Alignment, because + // the referents of `left` and `Self` have the same address and type + // (and, thus, alignment requirement). + // 2. `left` conforms to the validity invariant of `I::Validity`, neither + // the type nor bytes of `left`'s referent have been changed. + let left = unsafe { Ptr::from_inner(left) }; + + // SAFETY: + // 0. `right` conforms to the aliasing invariant of `I::Aliasing`, by Lemma + // 0. + // 1. `right` conforms to the alignment invariant of `I::Alignment, because + // if `ptr` with `I::Alignment = Aligned`, then by invariant on `T: + // SplitAt`, the trailing slice of `ptr` (from which `right` is derived) + // will also be well-aligned. + // 2. `right` conforms to the validity invariant of `I::Validity`, + // because `right: [T::Elem]` is derived from the trailing slice of + // `ptr`, which, by contract on `T: SplitAt::Elem`, has type + // `[T::Elem]`. The `left` part cannot be used to invalidate `right`, + // because the caller promises that if `I::Aliasing` is `Exclusive` + // or `T` permits interior mutation, then `l_len.padding_needed_for() + // == 0` and thus the parts will be non-overlapping. + let right = unsafe { Ptr::from_inner(right) }; + + (left, right) + } +} + +#[cfg(test)] +mod tests { + #[cfg(feature = "derive")] + #[test] + fn test_split_at() { + use crate::{FromBytes, Immutable, IntoBytes, KnownLayout, SplitAt}; + + #[derive(FromBytes, KnownLayout, SplitAt, IntoBytes, Immutable, Debug)] + #[repr(C)] + struct SliceDst<const OFFSET: usize> { + prefix: [u8; OFFSET], + trailing: [u8], + } + + #[allow(clippy::as_conversions)] + fn test_split_at<const OFFSET: usize, const BUFFER_SIZE: usize>() { + // Test `split_at` + let n: usize = BUFFER_SIZE - OFFSET; + let arr = [1; BUFFER_SIZE]; + let dst = SliceDst::<OFFSET>::ref_from_bytes(&arr[..]).unwrap(); + for i in 0..=n { + let (l, r) = dst.split_at(i).unwrap().via_runtime_check().unwrap(); + let l_sum: u8 = l.trailing.iter().sum(); + let r_sum: u8 = r.iter().sum(); + assert_eq!(l_sum, i as u8); + assert_eq!(r_sum, (n - i) as u8); + assert_eq!(l_sum + r_sum, n as u8); + } + + // Test `split_at_mut` + let n: usize = BUFFER_SIZE - OFFSET; + let mut arr = [1; BUFFER_SIZE]; + let dst = SliceDst::<OFFSET>::mut_from_bytes(&mut arr[..]).unwrap(); + for i in 0..=n { + let (l, r) = dst.split_at_mut(i).unwrap().via_runtime_check().unwrap(); + let l_sum: u8 = l.trailing.iter().sum(); + let r_sum: u8 = r.iter().sum(); + assert_eq!(l_sum, i as u8); + assert_eq!(r_sum, (n - i) as u8); + assert_eq!(l_sum + r_sum, n as u8); + } + } + + test_split_at::<0, 16>(); + test_split_at::<1, 17>(); + test_split_at::<2, 18>(); + } + + #[cfg(feature = "derive")] + #[test] + #[allow(clippy::as_conversions)] + fn test_split_at_overlapping() { + use crate::{FromBytes, Immutable, IntoBytes, KnownLayout, SplitAt}; + + #[derive(FromBytes, KnownLayout, SplitAt, Immutable)] + #[repr(C, align(2))] + struct SliceDst { + prefix: u8, + trailing: [u8], + } + + const N: usize = 16; + + let arr = [1u16; N]; + let dst = SliceDst::ref_from_bytes(arr.as_bytes()).unwrap(); + + for i in 0..N { + let split = dst.split_at(i).unwrap().via_runtime_check(); + if i % 2 == 1 { + assert!(split.is_ok()); + } else { + assert!(split.is_err()); + } + } + } + #[test] + fn test_split_at_unchecked() { + use crate::SplitAt; + let mut arr = [1, 2, 3, 4]; + let slice = &arr[..]; + // SAFETY: 2 <= arr.len() (4) + let split = unsafe { SplitAt::split_at_unchecked(slice, 2) }; + // SAFETY: SplitAt::split_at_unchecked guarantees that the split is valid. + let (l, r) = unsafe { split.via_unchecked() }; + assert_eq!(l, &[1, 2]); + assert_eq!(r, &[3, 4]); + + let slice_mut = &mut arr[..]; + // SAFETY: 2 <= arr.len() (4) + let split = unsafe { SplitAt::split_at_mut_unchecked(slice_mut, 2) }; + // SAFETY: SplitAt::split_at_mut_unchecked guarantees that the split is valid. + let (l, r) = unsafe { split.via_unchecked() }; + assert_eq!(l, &mut [1, 2]); + assert_eq!(r, &mut [3, 4]); + } + + #[test] + fn test_split_at_via_methods() { + use crate::{FromBytes, Immutable, IntoBytes, KnownLayout, SplitAt}; + #[derive(FromBytes, KnownLayout, SplitAt, IntoBytes, Immutable, Debug)] + #[repr(C)] + struct Packet { + length: u8, + body: [u8], + } + + let arr = [1, 2, 3, 4]; + let packet = Packet::ref_from_bytes(&arr[..]).unwrap(); + + let split1 = packet.split_at(2).unwrap(); + let (l, r) = split1.via_immutable(); + assert_eq!(l.length, 1); + assert_eq!(r, &[4]); + + let split2 = packet.split_at(2).unwrap(); + let (l, r) = split2.via_into_bytes(); + assert_eq!(l.length, 1); + assert_eq!(r, &[4]); + } + #[test] + fn test_split_at_via_unaligned() { + use crate::{FromBytes, Immutable, IntoBytes, KnownLayout, SplitAt, Unaligned}; + #[derive(FromBytes, KnownLayout, SplitAt, IntoBytes, Immutable, Unaligned)] + #[repr(C)] + struct Packet { + length: u8, + body: [u8], + } + + let arr = [1, 2, 3, 4]; + let packet = Packet::ref_from_bytes(&arr[..]).unwrap(); + + let split = packet.split_at(2).unwrap(); + let (l, r) = split.via_unaligned(); + assert_eq!(l.length, 1); + assert_eq!(r, &[4]); + } +} diff --git a/rust/zerocopy/src/util/macro_util.rs b/rust/zerocopy/src/util/macro_util.rs new file mode 100644 index 000000000000..ceeb80432b0b --- /dev/null +++ b/rust/zerocopy/src/util/macro_util.rs @@ -0,0 +1,1310 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2022 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +//! Utilities used by macros and by `zerocopy-derive`. +//! +//! These are defined here `zerocopy` rather than in code generated by macros or +//! by `zerocopy-derive` so that they can be compiled once rather than +//! recompiled for every invocation (e.g., if they were defined in generated +//! code, then deriving `IntoBytes` and `FromBytes` on three different types +//! would result in the code in question being emitted and compiled six +//! different times). + +#![allow(missing_debug_implementations)] + +// FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): Remove +// this `cfg` when `size_of_val_raw` is stabilized. +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[cfg(not(target_pointer_width = "16"))] +use core::ptr::{self, NonNull}; +use core::{marker::PhantomData, mem, num::Wrapping}; + +use crate::{ + pointer::{ + cast::CastSized, + invariant::{Aligned, Initialized, Valid}, + BecauseImmutable, + }, + FromBytes, Immutable, IntoBytes, KnownLayout, Ptr, ReadOnly, TryFromBytes, ValidityError, +}; + +/// Projects the type of the field at `Index` in `Self` without regard for field +/// privacy. +/// +/// The `Index` parameter is any sort of handle that identifies the field; its +/// definition is the obligation of the implementer. +/// +/// # Safety +/// +/// Unsafe code may assume that this accurately reflects the definition of +/// `Self`. +pub unsafe trait Field<Index> { + /// The type of the field at `Index`. + type Type: ?Sized; +} + +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented( + message = "`{T}` has {PADDING_BYTES} total byte(s) of padding", + label = "types with padding cannot implement `IntoBytes`", + note = "consider using `zerocopy::Unalign` to lower the alignment of individual fields", + note = "consider adding explicit fields where padding would be", + note = "consider using `#[repr(packed)]` to remove padding" + ) +)] +pub trait PaddingFree<T: ?Sized, const PADDING_BYTES: usize> {} +impl<T: ?Sized> PaddingFree<T, 0> for () {} + +// FIXME(#1112): In the slice DST case, we should delegate to *both* +// `PaddingFree` *and* `DynamicPaddingFree` (and probably rename `PaddingFree` +// to `StaticPaddingFree` or something - or introduce a third trait with that +// name) so that we can have more clear error messages. + +#[cfg_attr( + not(no_zerocopy_diagnostic_on_unimplemented_1_78_0), + diagnostic::on_unimplemented( + message = "`{T}` has one or more padding bytes", + label = "types with padding cannot implement `IntoBytes`", + note = "consider using `zerocopy::Unalign` to lower the alignment of individual fields", + note = "consider adding explicit fields where padding would be", + note = "consider using `#[repr(packed)]` to remove padding" + ) +)] +pub trait DynamicPaddingFree<T: ?Sized, const HAS_PADDING: bool> {} +impl<T: ?Sized> DynamicPaddingFree<T, false> for () {} + +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[cfg(not(target_pointer_width = "16"))] +const _64K: usize = 1 << 16; + +// FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): Remove +// this `cfg` when `size_of_val_raw` is stabilized. +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[cfg(not(target_pointer_width = "16"))] +#[repr(C, align(65536))] +struct Aligned64kAllocation([u8; _64K]); + +/// A pointer to an aligned allocation of size 2^16. +/// +/// # Safety +/// +/// `ALIGNED_64K_ALLOCATION` is guaranteed to point to the entirety of an +/// allocation with size and alignment 2^16, and to have valid provenance. +// FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): Remove +// this `cfg` when `size_of_val_raw` is stabilized. +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[cfg(not(target_pointer_width = "16"))] +pub const ALIGNED_64K_ALLOCATION: NonNull<[u8]> = { + const REF: &Aligned64kAllocation = &Aligned64kAllocation([0; _64K]); + let ptr: *const Aligned64kAllocation = REF; + let ptr: *const [u8] = ptr::slice_from_raw_parts(ptr.cast(), _64K); + // SAFETY: + // - `ptr` is derived from a Rust reference, which is guaranteed to be + // non-null. + // - `ptr` is derived from an `&Aligned64kAllocation`, which has size and + // alignment `_64K` as promised. Its length is initialized to `_64K`, + // which means that it refers to the entire allocation. + // - `ptr` is derived from a Rust reference, which is guaranteed to have + // valid provenance. + // + // FIXME(#429): Once `NonNull::new_unchecked` docs document that it + // preserves provenance, cite those docs. + // FIXME: Replace this `as` with `ptr.cast_mut()` once our MSRV >= 1.65 + #[allow(clippy::as_conversions)] + unsafe { + NonNull::new_unchecked(ptr as *mut _) + } +}; + +/// Computes the offset of the base of the field `$trailing_field_name` within +/// the type `$ty`. +/// +/// `trailing_field_offset!` produces code which is valid in a `const` context. +// FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): Remove +// this `cfg` when `size_of_val_raw` is stabilized. +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! trailing_field_offset { + ($ty:ty, $trailing_field_name:tt) => {{ + let min_size = { + let zero_elems: *const [()] = + $crate::util::macro_util::core_reexport::ptr::slice_from_raw_parts( + $crate::util::macro_util::core_reexport::ptr::NonNull::<()>::dangling() + .as_ptr() + .cast_const(), + 0, + ); + // SAFETY: + // - If `$ty` is `Sized`, `size_of_val_raw` is always safe to call. + // - Otherwise: + // - If `$ty` is not a slice DST, this pointer conversion will + // fail due to "mismatched vtable kinds", and compilation will + // fail. + // - If `$ty` is a slice DST, we have constructed `zero_elems` to + // have zero trailing slice elements. Per the `size_of_val_raw` + // docs, "For the special case where the dynamic tail length is + // 0, this function is safe to call." [1] + // + // [1] https://doc.rust-lang.org/nightly/std/mem/fn.size_of_val_raw.html + unsafe { + #[allow(clippy::as_conversions)] + $crate::util::macro_util::core_reexport::mem::size_of_val_raw( + zero_elems as *const $ty, + ) + } + }; + + assert!(min_size <= _64K); + + #[allow(clippy::as_conversions)] + let ptr = ALIGNED_64K_ALLOCATION.as_ptr() as *const $ty; + + // SAFETY: + // - Thanks to the preceding `assert!`, we know that the value with zero + // elements fits in `_64K` bytes, and thus in the allocation addressed + // by `ALIGNED_64K_ALLOCATION`. The offset of the trailing field is + // guaranteed to be no larger than this size, so this field projection + // is guaranteed to remain in-bounds of its allocation. + // - Because the minimum size is no larger than `_64K` bytes, and + // because an object's size must always be a multiple of its alignment + // [1], we know that `$ty`'s alignment is no larger than `_64K`. The + // allocation addressed by `ALIGNED_64K_ALLOCATION` is guaranteed to + // be aligned to `_64K`, so `ptr` is guaranteed to satisfy `$ty`'s + // alignment. + // - As required by `addr_of!`, we do not write through `field`. + // + // Note that, as of [2], this requirement is technically unnecessary + // for Rust versions >= 1.75.0, but no harm in guaranteeing it anyway + // until we bump our MSRV. + // + // [1] Per https://doc.rust-lang.org/reference/type-layout.html: + // + // The size of a value is always a multiple of its alignment. + // + // [2] https://github.com/rust-lang/reference/pull/1387 + let field = unsafe { + $crate::util::macro_util::core_reexport::ptr::addr_of!((*ptr).$trailing_field_name) + }; + // SAFETY: + // - Both `ptr` and `field` are derived from the same allocated object. + // - By the preceding safety comment, `field` is in bounds of that + // allocated object. + // - The distance, in bytes, between `ptr` and `field` is required to be + // a multiple of the size of `u8`, which is trivially true because + // `u8`'s size is 1. + // - The distance, in bytes, cannot overflow `isize`. This is guaranteed + // because no allocated object can have a size larger than can fit in + // `isize`. [1] + // - The distance being in-bounds cannot rely on wrapping around the + // address space. This is guaranteed because the same is guaranteed of + // allocated objects. [1] + // + // [1] FIXME(#429), FIXME(https://github.com/rust-lang/rust/pull/116675): + // Once these are guaranteed in the Reference, cite it. + let offset = unsafe { field.cast::<u8>().offset_from(ptr.cast::<u8>()) }; + // Guaranteed not to be lossy: `field` comes after `ptr`, so the offset + // from `ptr` to `field` is guaranteed to be positive. + assert!(offset >= 0); + Some( + #[allow(clippy::as_conversions)] + { + offset as usize + }, + ) + }}; +} + +/// Computes alignment of `$ty: ?Sized`. +/// +/// `align_of!` produces code which is valid in a `const` context. +// FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): Remove +// this `cfg` when `size_of_val_raw` is stabilized. +#[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! align_of { + ($ty:ty) => {{ + // SAFETY: `OffsetOfTrailingIsAlignment` is `repr(C)`, and its layout is + // guaranteed [1] to begin with the single-byte layout for `_byte`, + // followed by the padding needed to align `_trailing`, then the layout + // for `_trailing`, and finally any trailing padding bytes needed to + // correctly-align the entire struct. + // + // This macro computes the alignment of `$ty` by counting the number of + // bytes preceding `_trailing`. For instance, if the alignment of `$ty` + // is `1`, then no padding is required align `_trailing` and it will be + // located immediately after `_byte` at offset 1. If the alignment of + // `$ty` is 2, then a single padding byte is required before + // `_trailing`, and `_trailing` will be located at offset 2. + + // This correspondence between offset and alignment holds for all valid + // Rust alignments, and we confirm this exhaustively (or, at least up to + // the maximum alignment supported by `trailing_field_offset!`) in + // `test_align_of_dst`. + // + // [1]: https://doc.rust-lang.org/nomicon/other-reprs.html#reprc + + #[repr(C)] + struct OffsetOfTrailingIsAlignment { + _byte: u8, + _trailing: $ty, + } + + trailing_field_offset!(OffsetOfTrailingIsAlignment, _trailing) + }}; +} + +mod size_to_tag { + pub trait SizeToTag<const SIZE: usize> { + type Tag; + } + + impl SizeToTag<1> for () { + type Tag = u8; + } + impl SizeToTag<2> for () { + type Tag = u16; + } + impl SizeToTag<4> for () { + type Tag = u32; + } + impl SizeToTag<8> for () { + type Tag = u64; + } + impl SizeToTag<16> for () { + type Tag = u128; + } +} + +/// An alias for the unsigned integer of the given size in bytes. +#[doc(hidden)] +pub type SizeToTag<const SIZE: usize> = <() as size_to_tag::SizeToTag<SIZE>>::Tag; + +// We put `Sized` in its own module so it can have the same name as the standard +// library `Sized` without shadowing it in the parent module. +#[cfg(not(no_zerocopy_diagnostic_on_unimplemented_1_78_0))] +mod __size_of { + #[diagnostic::on_unimplemented( + message = "`{Self}` is unsized", + label = "`IntoBytes` needs all field types to be `Sized` in order to determine whether there is padding", + note = "consider using `#[repr(packed)]` to remove padding", + note = "`IntoBytes` does not require the fields of `#[repr(packed)]` types to be `Sized`" + )] + pub trait Sized: core::marker::Sized {} + impl<T: core::marker::Sized> Sized for T {} + + #[inline(always)] + #[must_use] + #[allow(clippy::needless_maybe_sized)] + pub const fn size_of<T: Sized + ?core::marker::Sized>() -> usize { + core::mem::size_of::<T>() + } +} + +#[cfg(no_zerocopy_diagnostic_on_unimplemented_1_78_0)] +pub use core::mem::size_of; + +#[cfg(not(no_zerocopy_diagnostic_on_unimplemented_1_78_0))] +pub use __size_of::size_of; + +/// How many padding bytes does the struct type `$t` have? +/// +/// `$ts` is the list of the type of every field in `$t`. `$t` must be a struct +/// type, or else `struct_padding!`'s result may be meaningless. +/// +/// Note that `struct_padding!`'s results are independent of `repcr` since they +/// only consider the size of the type and the sizes of the fields. Whatever the +/// repr, the size of the type already takes into account any padding that the +/// compiler has decided to add. Structs with well-defined representations (such +/// as `repr(C)`) can use this macro to check for padding. Note that while this +/// may yield some consistent value for some `repr(Rust)` structs, it is not +/// guaranteed across platforms or compilations. +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! struct_padding { + ($t:ty, $_align:expr, $_packed:expr, [$($ts:ty),*]) => {{ + // The `align` and `packed` directives can be ignored here. Regardless + // of if and how they are set, comparing the size of `$t` to the sum of + // its field sizes is a reliable indicator of the presence of padding. + $crate::util::macro_util::size_of::<$t>() - (0 $(+ $crate::util::macro_util::size_of::<$ts>())*) + }}; +} + +/// Does the `repr(C)` struct type `$t` have padding? +/// +/// `$ts` is the list of the type of every field in `$t`. `$t` must be a +/// `repr(C)` struct type, or else `struct_has_padding!`'s result may be +/// meaningless. +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! repr_c_struct_has_padding { + ($t:ty, $align:expr, $packed:expr, [$($ts:tt),*]) => {{ + let layout = $crate::DstLayout::for_repr_c_struct( + $align, + $packed, + &[$($crate::repr_c_struct_has_padding!(@field $ts),)*] + ); + layout.requires_static_padding() || layout.requires_dynamic_padding() + }}; + (@field ([$t:ty])) => { + <[$t] as $crate::KnownLayout>::LAYOUT + }; + (@field ($t:ty)) => { + $crate::DstLayout::for_unpadded_type::<$t>() + }; + (@field [$t:ty]) => { + <[$t] as $crate::KnownLayout>::LAYOUT + }; + (@field $t:ty) => { + $crate::DstLayout::for_unpadded_type::<$t>() + }; +} + +/// Does the union type `$t` have padding? +/// +/// `$ts` is the list of the type of every field in `$t`. `$t` must be a union +/// type, or else `union_padding!`'s result may be meaningless. +/// +/// Note that `union_padding!`'s results are independent of `repr` since they +/// only consider the size of the type and the sizes of the fields. Whatever the +/// repr, the size of the type already takes into account any padding that the +/// compiler has decided to add. Unions with well-defined representations (such +/// as `repr(C)`) can use this macro to check for padding. Note that while this +/// may yield some consistent value for some `repr(Rust)` unions, it is not +/// guaranteed across platforms or compilations. +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! union_padding { + ($t:ty, $_align:expr, $_packed:expr, [$($ts:ty),*]) => {{ + // The `align` and `packed` directives can be ignored here. Regardless + // of if and how they are set, comparing the size of `$t` to each of its + // field sizes is a reliable indicator of the presence of padding. + let mut max = 0; + $({ + let padding = $crate::util::macro_util::size_of::<$t>() - $crate::util::macro_util::size_of::<$ts>(); + if padding > max { + max = padding; + } + })* + max + }}; +} + +/// How many padding bytes does the enum type `$t` have? +/// +/// `$disc` is the type of the enum tag, and `$ts` is a list of fields in each +/// square-bracket-delimited variant. `$t` must be an enum, or else +/// `enum_padding!`'s result may be meaningless. An enum has padding if any of +/// its variant structs [1][2] contain padding, and so all of the variants of an +/// enum must be "full" in order for the enum to not have padding. +/// +/// The results of `enum_padding!` require that the enum is not `repr(Rust)`, as +/// `repr(Rust)` enums may niche the enum's tag and reduce the total number of +/// bytes required to represent the enum as a result. As long as the enum is +/// `repr(C)`, `repr(int)`, or `repr(C, int)`, this will consistently return +/// whether the enum contains any padding bytes. +/// +/// [1]: https://doc.rust-lang.org/1.81.0/reference/type-layout.html#reprc-enums-with-fields +/// [2]: https://doc.rust-lang.org/1.81.0/reference/type-layout.html#primitive-representation-of-enums-with-fields +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! enum_padding { + ($t:ty, $_align:expr, $packed:expr, $disc:ty, $([$($ts:ty),*]),*) => {{ + // The `align` and `packed` directives are irrelevant. `$align` can be + // ignored because regardless of if and how it is set, comparing the + // size of `$t` to each of its field sizes is a reliable indicator of + // the presence of padding. `$packed` is irrelevant because it is + // forbidden on enums. + #[allow(clippy::as_conversions)] + const _: [(); 1] = [(); $packed.is_none() as usize]; + let mut max = 0; + $({ + let padding = $crate::util::macro_util::size_of::<$t>() + - ( + $crate::util::macro_util::size_of::<$disc>() + $(+ $crate::util::macro_util::size_of::<$ts>())* + ); + if padding > max { + max = padding; + } + })* + max + }}; +} + +/// Unwraps an infallible `Result`. +#[doc(hidden)] +#[macro_export] +macro_rules! into_inner { + ($e:expr) => { + match $e { + $crate::util::macro_util::core_reexport::result::Result::Ok(e) => e, + $crate::util::macro_util::core_reexport::result::Result::Err(i) => match i {}, + } + }; +} + +/// Translates an identifier or tuple index into a numeric identifier. +#[doc(hidden)] // `#[macro_export]` bypasses this module's `#[doc(hidden)]`. +#[macro_export] +macro_rules! ident_id { + ($field:ident) => { + $crate::util::macro_util::hash_name(stringify!($field)) + }; + ($field:literal) => { + $field + }; +} + +/// Computes the hash of a string. +/// +/// NOTE(#2749) on hash collisions: This function's output only needs to be +/// deterministic within a particular compilation. Thus, if a user ever reports +/// a hash collision (very unlikely given the <= 16-byte special case), we can +/// strengthen the hash function at that point and publish a new version. Since +/// this is computed at compile time on small strings, we can easily use more +/// expensive and higher-quality hash functions if need be. +#[inline(always)] +#[must_use] +#[allow(clippy::as_conversions, clippy::indexing_slicing, clippy::arithmetic_side_effects)] +pub const fn hash_name(name: &str) -> i128 { + let name = name.as_bytes(); + + // We guarantee freedom from hash collisions between any two strings of + // length 16 or less by having the hashes of such strings be equal to + // their value. There is still a possibility that such strings will have + // the same value as the hash of a string of length > 16. + if name.len() <= size_of::<u128>() { + let mut bytes = [0u8; 16]; + + let mut i = 0; + while i < name.len() { + bytes[i] = name[i]; + i += 1; + } + + return i128::from_ne_bytes(bytes); + }; + + // An implementation of FxHasher, although returning a u128. Probably + // not as strong as it could be, but probably more collision resistant + // than normal 64-bit FxHasher. + let mut hash = 0u128; + let mut i = 0; + while i < name.len() { + // This is just FxHasher's `0x517cc1b727220a95` constant + // concatenated back-to-back. + const K: u128 = 0x517cc1b727220a95517cc1b727220a95; + hash = (hash.rotate_left(5) ^ (name[i] as u128)).wrapping_mul(K); + i += 1; + } + i128::from_ne_bytes(hash.to_ne_bytes()) +} + +/// Attempts to transmute `Src` into `Dst`. +/// +/// A helper for `try_transmute!`. +/// +/// # Panics +/// +/// `try_transmute` may either produce a post-monomorphization error or a panic +/// if `Dst` is bigger than `Src`. Otherwise, `try_transmute` panics under the +/// same circumstances as [`is_bit_valid`]. +/// +/// [`is_bit_valid`]: TryFromBytes::is_bit_valid +#[inline(always)] +pub fn try_transmute<Src, Dst>(src: Src) -> Result<Dst, ValidityError<Src, Dst>> +where + Src: IntoBytes, + Dst: TryFromBytes, +{ + static_assert!(Src, Dst => mem::size_of::<Dst>() == mem::size_of::<Src>()); + + let mu_src = mem::MaybeUninit::new(src); + // SAFETY: `MaybeUninit` has no validity requirements. + let mu_dst: mem::MaybeUninit<ReadOnly<Dst>> = + unsafe { crate::util::transmute_unchecked(mu_src) }; + + let ptr = Ptr::from_ref(&mu_dst); + + // SAFETY: Since `Src: IntoBytes`, and since `size_of::<Src>() == + // size_of::<Dst>()` by the preceding assertion, all of `mu_dst`'s bytes are + // initialized. `MaybeUninit` has no validity requirements, so even if + // `ptr` is used to mutate its referent (which it actually can't be - it's + // a shared `ReadOnly` pointer), that won't violate its referent's validity. + let ptr = unsafe { ptr.assume_validity::<Initialized>() }; + if Dst::is_bit_valid(ptr.cast::<_, CastSized, _>()) { + // SAFETY: Since `Dst::is_bit_valid`, we know that `ptr`'s referent is + // bit-valid for `Dst`. `ptr` points to `mu_dst`, and no intervening + // operations have mutated it, so it is a bit-valid `Dst`. + Ok(ReadOnly::into_inner(unsafe { mu_dst.assume_init() })) + } else { + // SAFETY: `MaybeUninit` has no validity requirements. + let mu_src: mem::MaybeUninit<Src> = unsafe { crate::util::transmute_unchecked(mu_dst) }; + // SAFETY: `mu_dst`/`mu_src` was constructed from `src` and never + // modified, so it is still bit-valid. + Err(ValidityError::new(unsafe { mu_src.assume_init() })) + } +} + +/// See `try_transmute_ref!` documentation. +pub trait TryTransmuteRefDst<'a> { + type Dst: ?Sized; + + /// See `try_transmute_ref!` documentation. + fn try_transmute_ref(self) -> Result<&'a Self::Dst, ValidityError<&'a Self::Src, Self::Dst>> + where + Self: TryTransmuteRefSrc<'a>, + Self::Src: IntoBytes + Immutable + KnownLayout, + Self::Dst: TryFromBytes + Immutable + KnownLayout; +} + +pub trait TryTransmuteRefSrc<'a> { + type Src: ?Sized; +} + +impl<'a, Src, Dst> TryTransmuteRefSrc<'a> for Wrap<&'a Src, &'a Dst> +where + Src: ?Sized, + Dst: ?Sized, +{ + type Src = Src; +} + +impl<'a, Src, Dst> TryTransmuteRefDst<'a> for Wrap<&'a Src, &'a Dst> +where + Src: IntoBytes + Immutable + KnownLayout + ?Sized, + Dst: TryFromBytes + Immutable + KnownLayout + ?Sized, +{ + type Dst = Dst; + + #[inline(always)] + fn try_transmute_ref( + self, + ) -> Result< + &'a Dst, + ValidityError<&'a <Wrap<&'a Src, &'a Dst> as TryTransmuteRefSrc<'a>>::Src, Dst>, + > { + let ptr = Ptr::from_ref(self.0); + #[rustfmt::skip] + let res = ptr.try_with(#[inline(always)] |ptr| { + let ptr = ptr.recall_validity::<Initialized, _>(); + let ptr = ptr.cast::<_, crate::layout::CastFrom<Dst>, _>(); + ptr.try_into_valid() + }); + match res { + Ok(ptr) => { + static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { + Src::LAYOUT.align.get() >= Dst::LAYOUT.align.get() + }, "cannot transmute reference when destination type has higher alignment than source type"); + // SAFETY: We have checked that `Dst` does not have a stricter + // alignment requirement than `Src`. + let ptr = unsafe { ptr.assume_alignment::<Aligned>() }; + Ok(ptr.as_ref()) + } + Err(err) => Err(err.map_src(Ptr::as_ref)), + } + } +} + +pub trait TryTransmuteMutDst<'a> { + type Dst: ?Sized; + + /// See `try_transmute_mut!` documentation. + fn try_transmute_mut( + self, + ) -> Result<&'a mut Self::Dst, ValidityError<&'a mut Self::Src, Self::Dst>> + where + Self: TryTransmuteMutSrc<'a>, + Self::Src: IntoBytes, + Self::Dst: TryFromBytes; +} + +pub trait TryTransmuteMutSrc<'a> { + type Src: ?Sized; +} + +impl<'a, Src, Dst> TryTransmuteMutSrc<'a> for Wrap<&'a mut Src, &'a mut Dst> +where + Src: ?Sized, + Dst: ?Sized, +{ + type Src = Src; +} + +impl<'a, Src, Dst> TryTransmuteMutDst<'a> for Wrap<&'a mut Src, &'a mut Dst> +where + Src: FromBytes + IntoBytes + KnownLayout + ?Sized, + Dst: TryFromBytes + IntoBytes + KnownLayout + ?Sized, +{ + type Dst = Dst; + + #[inline(always)] + fn try_transmute_mut( + self, + ) -> Result< + &'a mut Dst, + ValidityError<&'a mut <Wrap<&'a mut Src, &'a mut Dst> as TryTransmuteMutSrc<'a>>::Src, Dst>, + > { + let ptr = Ptr::from_mut(self.0); + // SAFETY: The provided closure returns the only copy of `ptr`. + #[rustfmt::skip] + let res = unsafe { + ptr.try_with_unchecked(#[inline(always)] |ptr| { + let ptr = ptr.recall_validity::<Initialized, (_, (_, _))>(); + let ptr = ptr.cast::<_, crate::layout::CastFrom<Dst>, _>(); + ptr.try_into_valid() + }) + }; + match res { + Ok(ptr) => { + static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { + Src::LAYOUT.align.get() >= Dst::LAYOUT.align.get() + }, "cannot transmute reference when destination type has higher alignment than source type"); + // SAFETY: We have checked that `Dst` does not have a stricter + // alignment requirement than `Src`. + let ptr = unsafe { ptr.assume_alignment::<Aligned>() }; + Ok(ptr.as_mut()) + } + Err(err) => Err(err.map_src(Ptr::as_mut)), + } + } +} + +// Used in `transmute_ref!` and friends. +// +// This permits us to use the autoref specialization trick to dispatch to +// associated functions for `transmute_ref` and `transmute_mut` when both `Src` +// and `Dst` are `Sized`, and to trait methods otherwise. The associated +// functions, unlike the trait methods, do not require a `KnownLayout` bound. +// This permits us to add support for transmuting references to unsized types +// without breaking backwards-compatibility (on v0.8.x) with the old +// implementation, which did not require a `KnownLayout` bound to transmute +// sized types. +#[derive(Copy, Clone)] +pub struct Wrap<Src, Dst>(pub Src, pub PhantomData<Dst>); + +impl<Src, Dst> Wrap<Src, Dst> { + #[inline(always)] + pub const fn new(src: Src) -> Self { + Wrap(src, PhantomData) + } +} + +impl<'a, Src, Dst> Wrap<&'a Src, &'a Dst> +where + Src: ?Sized, + Dst: ?Sized, +{ + #[allow(clippy::must_use_candidate, clippy::missing_inline_in_public_items, clippy::empty_loop)] + pub const fn transmute_ref_inference_helper(self) -> &'a Dst { + loop {} + } +} + +impl<'a, Src, Dst> Wrap<&'a Src, &'a Dst> { + /// # Safety + /// The caller must guarantee that: + /// - `Src: IntoBytes + Immutable` + /// - `Dst: FromBytes + Immutable` + /// + /// # PME + /// + /// Instantiating this method PMEs unless both: + /// - `mem::size_of::<Dst>() == mem::size_of::<Src>()` + /// - `mem::align_of::<Dst>() <= mem::align_of::<Src>()` + #[inline(always)] + #[must_use] + pub const unsafe fn transmute_ref(self) -> &'a Dst { + static_assert!(Src, Dst => mem::size_of::<Dst>() == mem::size_of::<Src>()); + static_assert!(Src, Dst => mem::align_of::<Dst>() <= mem::align_of::<Src>()); + + let src: *const Src = self.0; + let dst = src.cast::<Dst>(); + // SAFETY: + // - We know that it is sound to view the target type of the input + // reference (`Src`) as the target type of the output reference + // (`Dst`) because the caller has guaranteed that `Src: IntoBytes`, + // `Dst: FromBytes`, and `size_of::<Src>() == size_of::<Dst>()`. + // - We know that there are no `UnsafeCell`s, and thus we don't have to + // worry about `UnsafeCell` overlap, because `Src: Immutable` and + // `Dst: Immutable`. + // - The caller has guaranteed that alignment is not increased. + // - We know that the returned lifetime will not outlive the input + // lifetime thanks to the lifetime bounds on this function. + // + // FIXME(#67): Once our MSRV is 1.58, replace this `transmute` with + // `&*dst`. + #[allow(clippy::transmute_ptr_to_ref)] + unsafe { + mem::transmute(dst) + } + } + + #[inline(always)] + pub fn try_transmute_ref(self) -> Result<&'a Dst, ValidityError<&'a Src, Dst>> + where + Src: IntoBytes + Immutable, + Dst: TryFromBytes + Immutable, + { + static_assert!(Src => mem::align_of::<Src>() == mem::align_of::<Wrapping<Src>>()); + static_assert!(Dst => mem::align_of::<Dst>() == mem::align_of::<Wrapping<Dst>>()); + + // SAFETY: By the preceding assert, `Src` and `Wrapping<Src>` have the + // same alignment. + let src: &Wrapping<Src> = + unsafe { crate::util::transmute_ref::<_, _, BecauseImmutable>(self.0) }; + let src = Wrap::new(src); + <Wrap<&'a Wrapping<Src>, &'a Wrapping<Dst>> as TryTransmuteRefDst<'a>>::try_transmute_ref( + src, + ) + .map( + // SAFETY: By the preceding assert, `Dst` and `Wrapping<Dst>` have + // the same alignment. + #[inline(always)] + |dst| unsafe { crate::util::transmute_ref::<_, _, BecauseImmutable>(dst) }, + ) + .map_err( + #[inline(always)] + |err| { + // SAFETY: By the preceding assert, `Src` and `Wrapping<Src>` have the + // same alignment. + ValidityError::new(unsafe { + crate::util::transmute_ref::<_, _, BecauseImmutable>(err.into_src()) + }) + }, + ) + } +} + +impl<'a, Src, Dst> Wrap<&'a mut Src, &'a mut Dst> +where + Src: ?Sized, + Dst: ?Sized, +{ + #[allow(clippy::must_use_candidate, clippy::missing_inline_in_public_items, clippy::empty_loop)] + pub fn transmute_mut_inference_helper(self) -> &'a mut Dst { + loop {} + } +} + +impl<'a, Src, Dst> Wrap<&'a mut Src, &'a mut Dst> { + /// Transmutes a mutable reference of one type to a mutable reference of + /// another type. + /// + /// # PME + /// + /// Instantiating this method PMEs unless both: + /// - `mem::size_of::<Dst>() == mem::size_of::<Src>()` + /// - `mem::align_of::<Dst>() <= mem::align_of::<Src>()` + #[inline(always)] + #[must_use] + pub fn transmute_mut(self) -> &'a mut Dst + where + Src: FromBytes + IntoBytes, + Dst: FromBytes + IntoBytes, + { + static_assert!(Src, Dst => mem::size_of::<Dst>() == mem::size_of::<Src>()); + static_assert!(Src, Dst => mem::align_of::<Dst>() <= mem::align_of::<Src>()); + + let src: *mut Src = self.0; + let dst = src.cast::<Dst>(); + // SAFETY: + // - We know that it is sound to view the target type of the input + // reference (`Src`) as the target type of the output reference + // (`Dst`) and vice-versa because `Src: FromBytes + IntoBytes`, `Dst: + // FromBytes + IntoBytes`, and (as asserted above) `size_of::<Src>() + // == size_of::<Dst>()`. + // - We asserted above that alignment will not increase. + // - We know that the returned lifetime will not outlive the input + // lifetime thanks to the lifetime bounds on this function. + unsafe { &mut *dst } + } + + #[inline(always)] + pub fn try_transmute_mut(self) -> Result<&'a mut Dst, ValidityError<&'a mut Src, Dst>> + where + Src: FromBytes + IntoBytes, + Dst: TryFromBytes + IntoBytes, + { + static_assert!(Src => mem::align_of::<Src>() == mem::align_of::<Wrapping<Src>>()); + static_assert!(Dst => mem::align_of::<Dst>() == mem::align_of::<Wrapping<Dst>>()); + + // SAFETY: By the preceding assert, `Src` and `Wrapping<Src>` have the + // same alignment. + let src: &mut Wrapping<Src> = + unsafe { crate::util::transmute_mut::<_, _, (_, (_, _))>(self.0) }; + let src = Wrap::new(src); + <Wrap<&'a mut Wrapping<Src>, &'a mut Wrapping<Dst>> as TryTransmuteMutDst<'a>> + ::try_transmute_mut(src) + // SAFETY: By the preceding assert, `Dst` and `Wrapping<Dst>` have the + // same alignment. + .map(|dst| unsafe { crate::util::transmute_mut::<_, _, (_, (_, _))>(dst) }) + .map_err(|err| { + // SAFETY: By the preceding assert, `Src` and `Wrapping<Src>` have the + // same alignment. + ValidityError::new(unsafe { + crate::util::transmute_mut::<_, _, (_, (_, _))>(err.into_src()) + }) + }) + } +} + +pub trait TransmuteRefDst<'a> { + type Dst: ?Sized; + + #[must_use] + fn transmute_ref(self) -> &'a Self::Dst; +} + +impl<'a, Src: ?Sized, Dst: ?Sized> TransmuteRefDst<'a> for Wrap<&'a Src, &'a Dst> +where + Src: KnownLayout + IntoBytes + Immutable, + Dst: KnownLayout<PointerMetadata = usize> + FromBytes + Immutable, +{ + type Dst = Dst; + + #[inline(always)] + fn transmute_ref(self) -> &'a Dst { + let ptr = Ptr::from_ref(self.0) + .recall_validity::<Initialized, _>() + .transmute_with::<Dst, Initialized, crate::layout::CastFrom<Dst>, (crate::pointer::BecauseMutationCompatible, _)>() + .recall_validity::<Valid, _>(); + + static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { + Src::LAYOUT.align.get() >= Dst::LAYOUT.align.get() + }, "cannot transmute reference when destination type has higher alignment than source type"); + + // SAFETY: The preceding `static_assert!` ensures that + // `Src::LAYOUT.align >= Dst::LAYOUT.align`. Since `self` is + // validly-aligned for `Src`, it is also validly-aligned for `Dst`. + let ptr = unsafe { ptr.assume_alignment() }; + + ptr.as_ref() + } +} + +pub trait TransmuteMutDst<'a> { + type Dst: ?Sized; + #[must_use] + fn transmute_mut(self) -> &'a mut Self::Dst; +} + +impl<'a, Src: ?Sized, Dst: ?Sized> TransmuteMutDst<'a> for Wrap<&'a mut Src, &'a mut Dst> +where + Src: KnownLayout + FromBytes + IntoBytes, + Dst: KnownLayout<PointerMetadata = usize> + FromBytes + IntoBytes, +{ + type Dst = Dst; + + #[inline(always)] + fn transmute_mut(self) -> &'a mut Dst { + let ptr = Ptr::from_mut(self.0) + .recall_validity::<Initialized, (_, (_, _))>() + .transmute_with::<Dst, Initialized, crate::layout::CastFrom<Dst>, _>() + .recall_validity::<Valid, (_, (_, _))>(); + + static_assert!(Src: ?Sized + KnownLayout, Dst: ?Sized + KnownLayout => { + Src::LAYOUT.align.get() >= Dst::LAYOUT.align.get() + }, "cannot transmute reference when destination type has higher alignment than source type"); + + // SAFETY: The preceding `static_assert!` ensures that + // `Src::LAYOUT.align >= Dst::LAYOUT.align`. Since `self` is + // validly-aligned for `Src`, it is also validly-aligned for `Dst`. + let ptr = unsafe { ptr.assume_alignment() }; + + ptr.as_mut() + } +} + +/// A function which emits a warning if its return value is not used. +#[must_use] +#[inline(always)] +pub const fn must_use<T>(t: T) -> T { + t +} + +// NOTE: We can't change this to a `pub use core as core_reexport` until [1] is +// fixed or we update to a semver-breaking version (as of this writing, 0.8.0) +// on the `main` branch. +// +// [1] https://github.com/obi1kenobi/cargo-semver-checks/issues/573 +pub mod core_reexport { + pub use core::*; + + pub mod mem { + pub use core::mem::*; + } +} + +#[cfg(test)] +mod tests { + use core::num::NonZeroUsize; + + use crate::util::testutil::*; + + #[cfg(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS)] + mod nightly { + use super::super::*; + use crate::util::testutil::*; + + // FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): + // Remove this `cfg` when `size_of_val_raw` is stabilized. + #[allow(clippy::decimal_literal_representation)] + #[test] + fn test_trailing_field_offset() { + assert_eq!(mem::align_of::<Aligned64kAllocation>(), _64K); + + macro_rules! test { + (#[$cfg:meta] ($($ts:ty),* ; $trailing_field_ty:ty) => $expect:expr) => {{ + #[$cfg] + struct Test($(#[allow(dead_code)] $ts,)* #[allow(dead_code)] $trailing_field_ty); + assert_eq!(test!(@offset $($ts),* ; $trailing_field_ty), $expect); + }}; + (#[$cfg:meta] $(#[$cfgs:meta])* ($($ts:ty),* ; $trailing_field_ty:ty) => $expect:expr) => { + test!(#[$cfg] ($($ts),* ; $trailing_field_ty) => $expect); + test!($(#[$cfgs])* ($($ts),* ; $trailing_field_ty) => $expect); + }; + (@offset ; $_trailing:ty) => { trailing_field_offset!(Test, 0) }; + (@offset $_t:ty ; $_trailing:ty) => { trailing_field_offset!(Test, 1) }; + } + + test!(#[repr(C)] #[repr(transparent)] #[repr(packed)](; u8) => Some(0)); + test!(#[repr(C)] #[repr(transparent)] #[repr(packed)](; [u8]) => Some(0)); + test!(#[repr(C)] #[repr(C, packed)] (u8; u8) => Some(1)); + test!(#[repr(C)] (; AU64) => Some(0)); + test!(#[repr(C)] (; [AU64]) => Some(0)); + test!(#[repr(C)] (u8; AU64) => Some(8)); + test!(#[repr(C)] (u8; [AU64]) => Some(8)); + + #[derive( + Immutable, FromBytes, Eq, PartialEq, Ord, PartialOrd, Default, Debug, Copy, Clone, + )] + #[repr(C)] + pub(crate) struct Nested<T, U: ?Sized> { + _t: T, + _u: U, + } + + test!(#[repr(C)] (; Nested<u8, AU64>) => Some(0)); + test!(#[repr(C)] (; Nested<u8, [AU64]>) => Some(0)); + test!(#[repr(C)] (u8; Nested<u8, AU64>) => Some(8)); + test!(#[repr(C)] (u8; Nested<u8, [AU64]>) => Some(8)); + + // Test that `packed(N)` limits the offset of the trailing field. + test!(#[repr(C, packed( 1))] (u8; elain::Align< 2>) => Some( 1)); + test!(#[repr(C, packed( 2))] (u8; elain::Align< 4>) => Some( 2)); + test!(#[repr(C, packed( 4))] (u8; elain::Align< 8>) => Some( 4)); + test!(#[repr(C, packed( 8))] (u8; elain::Align< 16>) => Some( 8)); + test!(#[repr(C, packed( 16))] (u8; elain::Align< 32>) => Some( 16)); + test!(#[repr(C, packed( 32))] (u8; elain::Align< 64>) => Some( 32)); + test!(#[repr(C, packed( 64))] (u8; elain::Align< 128>) => Some( 64)); + test!(#[repr(C, packed( 128))] (u8; elain::Align< 256>) => Some( 128)); + test!(#[repr(C, packed( 256))] (u8; elain::Align< 512>) => Some( 256)); + test!(#[repr(C, packed( 512))] (u8; elain::Align< 1024>) => Some( 512)); + test!(#[repr(C, packed( 1024))] (u8; elain::Align< 2048>) => Some( 1024)); + test!(#[repr(C, packed( 2048))] (u8; elain::Align< 4096>) => Some( 2048)); + test!(#[repr(C, packed( 4096))] (u8; elain::Align< 8192>) => Some( 4096)); + test!(#[repr(C, packed( 8192))] (u8; elain::Align< 16384>) => Some( 8192)); + test!(#[repr(C, packed( 16384))] (u8; elain::Align< 32768>) => Some( 16384)); + test!(#[repr(C, packed( 32768))] (u8; elain::Align< 65536>) => Some( 32768)); + test!(#[repr(C, packed( 65536))] (u8; elain::Align< 131072>) => Some( 65536)); + /* Alignments above 65536 are not yet supported. + test!(#[repr(C, packed( 131072))] (u8; elain::Align< 262144>) => Some( 131072)); + test!(#[repr(C, packed( 262144))] (u8; elain::Align< 524288>) => Some( 262144)); + test!(#[repr(C, packed( 524288))] (u8; elain::Align< 1048576>) => Some( 524288)); + test!(#[repr(C, packed( 1048576))] (u8; elain::Align< 2097152>) => Some( 1048576)); + test!(#[repr(C, packed( 2097152))] (u8; elain::Align< 4194304>) => Some( 2097152)); + test!(#[repr(C, packed( 4194304))] (u8; elain::Align< 8388608>) => Some( 4194304)); + test!(#[repr(C, packed( 8388608))] (u8; elain::Align< 16777216>) => Some( 8388608)); + test!(#[repr(C, packed( 16777216))] (u8; elain::Align< 33554432>) => Some( 16777216)); + test!(#[repr(C, packed( 33554432))] (u8; elain::Align< 67108864>) => Some( 33554432)); + test!(#[repr(C, packed( 67108864))] (u8; elain::Align< 33554432>) => Some( 67108864)); + test!(#[repr(C, packed( 33554432))] (u8; elain::Align<134217728>) => Some( 33554432)); + test!(#[repr(C, packed(134217728))] (u8; elain::Align<268435456>) => Some(134217728)); + test!(#[repr(C, packed(268435456))] (u8; elain::Align<268435456>) => Some(268435456)); + */ + + // Test that `align(N)` does not limit the offset of the trailing field. + test!(#[repr(C, align( 1))] (u8; elain::Align< 2>) => Some( 2)); + test!(#[repr(C, align( 2))] (u8; elain::Align< 4>) => Some( 4)); + test!(#[repr(C, align( 4))] (u8; elain::Align< 8>) => Some( 8)); + test!(#[repr(C, align( 8))] (u8; elain::Align< 16>) => Some( 16)); + test!(#[repr(C, align( 16))] (u8; elain::Align< 32>) => Some( 32)); + test!(#[repr(C, align( 32))] (u8; elain::Align< 64>) => Some( 64)); + test!(#[repr(C, align( 64))] (u8; elain::Align< 128>) => Some( 128)); + test!(#[repr(C, align( 128))] (u8; elain::Align< 256>) => Some( 256)); + test!(#[repr(C, align( 256))] (u8; elain::Align< 512>) => Some( 512)); + test!(#[repr(C, align( 512))] (u8; elain::Align< 1024>) => Some( 1024)); + test!(#[repr(C, align( 1024))] (u8; elain::Align< 2048>) => Some( 2048)); + test!(#[repr(C, align( 2048))] (u8; elain::Align< 4096>) => Some( 4096)); + test!(#[repr(C, align( 4096))] (u8; elain::Align< 8192>) => Some( 8192)); + test!(#[repr(C, align( 8192))] (u8; elain::Align< 16384>) => Some( 16384)); + test!(#[repr(C, align( 16384))] (u8; elain::Align< 32768>) => Some( 32768)); + test!(#[repr(C, align( 32768))] (u8; elain::Align< 65536>) => Some( 65536)); + /* Alignments above 65536 are not yet supported. + test!(#[repr(C, align( 65536))] (u8; elain::Align< 131072>) => Some( 131072)); + test!(#[repr(C, align( 131072))] (u8; elain::Align< 262144>) => Some( 262144)); + test!(#[repr(C, align( 262144))] (u8; elain::Align< 524288>) => Some( 524288)); + test!(#[repr(C, align( 524288))] (u8; elain::Align< 1048576>) => Some( 1048576)); + test!(#[repr(C, align( 1048576))] (u8; elain::Align< 2097152>) => Some( 2097152)); + test!(#[repr(C, align( 2097152))] (u8; elain::Align< 4194304>) => Some( 4194304)); + test!(#[repr(C, align( 4194304))] (u8; elain::Align< 8388608>) => Some( 8388608)); + test!(#[repr(C, align( 8388608))] (u8; elain::Align< 16777216>) => Some( 16777216)); + test!(#[repr(C, align( 16777216))] (u8; elain::Align< 33554432>) => Some( 33554432)); + test!(#[repr(C, align( 33554432))] (u8; elain::Align< 67108864>) => Some( 67108864)); + test!(#[repr(C, align( 67108864))] (u8; elain::Align< 33554432>) => Some( 33554432)); + test!(#[repr(C, align( 33554432))] (u8; elain::Align<134217728>) => Some(134217728)); + test!(#[repr(C, align(134217728))] (u8; elain::Align<268435456>) => Some(268435456)); + */ + } + + // FIXME(#29), FIXME(https://github.com/rust-lang/rust/issues/69835): + // Remove this `cfg` when `size_of_val_raw` is stabilized. + #[allow(clippy::decimal_literal_representation)] + #[test] + fn test_align_of_dst() { + // Test that `align_of!` correctly computes the alignment of DSTs. + assert_eq!(align_of!([elain::Align<1>]), Some(1)); + assert_eq!(align_of!([elain::Align<2>]), Some(2)); + assert_eq!(align_of!([elain::Align<4>]), Some(4)); + assert_eq!(align_of!([elain::Align<8>]), Some(8)); + assert_eq!(align_of!([elain::Align<16>]), Some(16)); + assert_eq!(align_of!([elain::Align<32>]), Some(32)); + assert_eq!(align_of!([elain::Align<64>]), Some(64)); + assert_eq!(align_of!([elain::Align<128>]), Some(128)); + assert_eq!(align_of!([elain::Align<256>]), Some(256)); + assert_eq!(align_of!([elain::Align<512>]), Some(512)); + assert_eq!(align_of!([elain::Align<1024>]), Some(1024)); + assert_eq!(align_of!([elain::Align<2048>]), Some(2048)); + assert_eq!(align_of!([elain::Align<4096>]), Some(4096)); + assert_eq!(align_of!([elain::Align<8192>]), Some(8192)); + assert_eq!(align_of!([elain::Align<16384>]), Some(16384)); + assert_eq!(align_of!([elain::Align<32768>]), Some(32768)); + assert_eq!(align_of!([elain::Align<65536>]), Some(65536)); + /* Alignments above 65536 are not yet supported. + assert_eq!(align_of!([elain::Align<131072>]), Some(131072)); + assert_eq!(align_of!([elain::Align<262144>]), Some(262144)); + assert_eq!(align_of!([elain::Align<524288>]), Some(524288)); + assert_eq!(align_of!([elain::Align<1048576>]), Some(1048576)); + assert_eq!(align_of!([elain::Align<2097152>]), Some(2097152)); + assert_eq!(align_of!([elain::Align<4194304>]), Some(4194304)); + assert_eq!(align_of!([elain::Align<8388608>]), Some(8388608)); + assert_eq!(align_of!([elain::Align<16777216>]), Some(16777216)); + assert_eq!(align_of!([elain::Align<33554432>]), Some(33554432)); + assert_eq!(align_of!([elain::Align<67108864>]), Some(67108864)); + assert_eq!(align_of!([elain::Align<33554432>]), Some(33554432)); + assert_eq!(align_of!([elain::Align<134217728>]), Some(134217728)); + assert_eq!(align_of!([elain::Align<268435456>]), Some(268435456)); + */ + } + } + + #[test] + fn test_enum_casts() { + // Test that casting the variants of enums with signed integer reprs to + // unsigned integers obeys expected signed -> unsigned casting rules. + + #[repr(i8)] + enum ReprI8 { + MinusOne = -1, + Zero = 0, + Min = i8::MIN, + Max = i8::MAX, + } + + #[allow(clippy::as_conversions)] + let x = ReprI8::MinusOne as u8; + assert_eq!(x, u8::MAX); + + #[allow(clippy::as_conversions)] + let x = ReprI8::Zero as u8; + assert_eq!(x, 0); + + #[allow(clippy::as_conversions)] + let x = ReprI8::Min as u8; + assert_eq!(x, 128); + + #[allow(clippy::as_conversions)] + let x = ReprI8::Max as u8; + assert_eq!(x, 127); + } + + #[test] + fn test_struct_padding() { + // Test that, for each provided repr, `struct_padding!` reports the + // expected value. + macro_rules! test { + (#[$cfg:meta] ($($ts:ty),*) => $expect:expr) => {{ + #[$cfg] + #[allow(dead_code)] + struct Test($($ts),*); + assert_eq!(struct_padding!(Test, None::<NonZeroUsize>, None::<NonZeroUsize>, [$($ts),*]), $expect); + }}; + (#[$cfg:meta] $(#[$cfgs:meta])* ($($ts:ty),*) => $expect:expr) => { + test!(#[$cfg] ($($ts),*) => $expect); + test!($(#[$cfgs])* ($($ts),*) => $expect); + }; + } + + test!(#[repr(C)] #[repr(transparent)] #[repr(packed)] () => 0); + test!(#[repr(C)] #[repr(transparent)] #[repr(packed)] (u8) => 0); + test!(#[repr(C)] #[repr(transparent)] #[repr(packed)] (u8, ()) => 0); + test!(#[repr(C)] #[repr(packed)] (u8, u8) => 0); + + test!(#[repr(C)] (u8, AU64) => 7); + // Rust won't let you put `#[repr(packed)]` on a type which contains a + // `#[repr(align(n > 1))]` type (`AU64`), so we have to use `u64` here. + // It's not ideal, but it definitely has align > 1 on /some/ of our CI + // targets, and this isn't a particularly complex macro we're testing + // anyway. + test!(#[repr(packed)] (u8, u64) => 0); + } + + #[test] + fn test_repr_c_struct_padding() { + // Test that, for each provided repr, `repr_c_struct_padding!` reports + // the expected value. + macro_rules! test { + (($($ts:tt),*) => $expect:expr) => {{ + #[repr(C)] + #[allow(dead_code)] + struct Test($($ts),*); + assert_eq!(repr_c_struct_has_padding!(Test, None::<NonZeroUsize>, None::<NonZeroUsize>, [$($ts),*]), $expect); + }}; + } + + // Test static padding + test!(() => false); + test!(([u8]) => false); + test!((u8) => false); + test!((u8, [u8]) => false); + test!((u8, ()) => false); + test!((u8, (), [u8]) => false); + test!((u8, u8) => false); + test!((u8, u8, [u8]) => false); + + test!((u8, AU64) => true); + test!((u8, AU64, [u8]) => true); + + // Test dynamic padding + test!((AU64, [AU64]) => false); + test!((u8, [AU64]) => true); + + #[repr(align(4))] + struct AU32(#[allow(unused)] u32); + test!((AU64, [AU64]) => false); + test!((AU64, [AU32]) => true); + } + + #[test] + fn test_union_padding() { + // Test that, for each provided repr, `union_padding!` reports the + // expected value. + macro_rules! test { + (#[$cfg:meta] {$($fs:ident: $ts:ty),*} => $expect:expr) => {{ + #[$cfg] + #[allow(unused)] // fields are never read + union Test{ $($fs: $ts),* } + assert_eq!(union_padding!(Test, None::<NonZeroUsize>, None::<usize>, [$($ts),*]), $expect); + }}; + (#[$cfg:meta] $(#[$cfgs:meta])* {$($fs:ident: $ts:ty),*} => $expect:expr) => { + test!(#[$cfg] {$($fs: $ts),*} => $expect); + test!($(#[$cfgs])* {$($fs: $ts),*} => $expect); + }; + } + + test!(#[repr(C)] #[repr(packed)] {a: u8} => 0); + test!(#[repr(C)] #[repr(packed)] {a: u8, b: u8} => 0); + + // Rust won't let you put `#[repr(packed)]` on a type which contains a + // `#[repr(align(n > 1))]` type (`AU64`), so we have to use `u64` here. + // It's not ideal, but it definitely has align > 1 on /some/ of our CI + // targets, and this isn't a particularly complex macro we're testing + // anyway. + test!(#[repr(C)] #[repr(packed)] {a: u8, b: u64} => 7); + } + + #[test] + fn test_enum_padding() { + // Test that, for each provided repr, `enum_has_padding!` reports the + // expected value. + macro_rules! test { + (#[repr($disc:ident $(, $c:ident)?)] { $($vs:ident ($($ts:ty),*),)* } => $expect:expr) => { + test!(@case #[repr($disc $(, $c)?)] { $($vs ($($ts),*),)* } => $expect); + }; + (#[repr($disc:ident $(, $c:ident)?)] #[$cfg:meta] $(#[$cfgs:meta])* { $($vs:ident ($($ts:ty),*),)* } => $expect:expr) => { + test!(@case #[repr($disc $(, $c)?)] #[$cfg] { $($vs ($($ts),*),)* } => $expect); + test!(#[repr($disc $(, $c)?)] $(#[$cfgs])* { $($vs ($($ts),*),)* } => $expect); + }; + (@case #[repr($disc:ident $(, $c:ident)?)] $(#[$cfg:meta])? { $($vs:ident ($($ts:ty),*),)* } => $expect:expr) => {{ + #[repr($disc $(, $c)?)] + $(#[$cfg])? + #[allow(unused)] // variants and fields are never used + enum Test { + $($vs ($($ts),*),)* + } + assert_eq!( + enum_padding!(Test, None::<NonZeroUsize>, None::<NonZeroUsize>, $disc, $([$($ts),*]),*), + $expect + ); + }}; + } + + #[allow(unused)] + #[repr(align(2))] + struct U16(u16); + + #[allow(unused)] + #[repr(align(4))] + struct U32(u32); + + test!(#[repr(u8)] #[repr(C)] { + A(u8), + } => 0); + test!(#[repr(u16)] #[repr(C)] { + A(u8, u8), + B(U16), + } => 0); + test!(#[repr(u32)] #[repr(C)] { + A(u8, u8, u8, u8), + B(U16, u8, u8), + C(u8, u8, U16), + D(U16, U16), + E(U32), + } => 0); + + // `repr(int)` can pack the discriminant more efficiently + test!(#[repr(u8)] { + A(u8, U16), + } => 0); + test!(#[repr(u8)] { + A(u8, U16, U32), + } => 0); + + // `repr(C)` cannot + test!(#[repr(u8, C)] { + A(u8, U16), + } => 2); + test!(#[repr(u8, C)] { + A(u8, u8, u8, U32), + } => 4); + + // And field ordering can always cause problems + test!(#[repr(u8)] #[repr(C)] { + A(U16, u8), + } => 2); + test!(#[repr(u8)] #[repr(C)] { + A(U32, u8, u8, u8), + } => 4); + } +} diff --git a/rust/zerocopy/src/util/macros.rs b/rust/zerocopy/src/util/macros.rs new file mode 100644 index 000000000000..7e63e3a54fc4 --- /dev/null +++ b/rust/zerocopy/src/util/macros.rs @@ -0,0 +1,1067 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +/// Unsafely implements trait(s) for a type. +/// +/// # Safety +/// +/// The trait impl must be sound. +/// +/// When implementing `TryFromBytes`: +/// - If no `is_bit_valid` impl is provided, then it must be valid for +/// `is_bit_valid` to unconditionally return `true`. In other words, it must +/// be the case that any initialized sequence of bytes constitutes a valid +/// instance of `$ty`. +/// - If an `is_bit_valid` impl is provided, then the impl of `is_bit_valid` +/// must only return `true` if its argument refers to a valid `$ty`. +macro_rules! unsafe_impl { + // Implement `$trait` for `$ty` with no bounds. + ($(#[$attr:meta])* $ty:ty: $trait:ident $(; |$candidate:ident| $is_bit_valid:expr)?) => {{ + crate::util::macros::__unsafe(); + + $(#[$attr])* + // SAFETY: The caller promises that this is sound. + unsafe impl $trait for $ty { + unsafe_impl!(@method $trait $(; |$candidate| $is_bit_valid)?); + } + }}; + + // Implement all `$traits` for `$ty` with no bounds. + // + // The 2 arms under this one are there so we can apply + // N attributes for each one of M trait implementations. + // The simple solution of: + // + // ($(#[$attrs:meta])* $ty:ty: $($traits:ident),*) => { + // $( unsafe_impl!( $(#[$attrs])* $ty: $traits ) );* + // } + // + // Won't work. The macro processor sees that the outer repetition + // contains both $attrs and $traits and expects them to match the same + // amount of fragments. + // + // To solve this we must: + // 1. Pack the attributes into a single token tree fragment we can match over. + // 2. Expand the traits. + // 3. Unpack and expand the attributes. + ($(#[$attrs:meta])* $ty:ty: $($traits:ident),*) => { + unsafe_impl!(@impl_traits_with_packed_attrs { $(#[$attrs])* } $ty: $($traits),*) + }; + + (@impl_traits_with_packed_attrs $attrs:tt $ty:ty: $($traits:ident),*) => {{ + $( unsafe_impl!(@unpack_attrs $attrs $ty: $traits); )* + }}; + + (@unpack_attrs { $(#[$attrs:meta])* } $ty:ty: $traits:ident) => { + unsafe_impl!($(#[$attrs])* $ty: $traits); + }; + + // This arm is identical to the following one, except it contains a + // preceding `const`. If we attempt to handle these with a single arm, there + // is an inherent ambiguity between `const` (the keyword) and `const` (the + // ident match for `$tyvar:ident`). + // + // To explain how this works, consider the following invocation: + // + // unsafe_impl!(const N: usize, T: ?Sized + Copy => Clone for Foo<T>); + // + // In this invocation, here are the assignments to meta-variables: + // + // |---------------|------------| + // | Meta-variable | Assignment | + // |---------------|------------| + // | $constname | N | + // | $constty | usize | + // | $tyvar | T | + // | $optbound | Sized | + // | $bound | Copy | + // | $trait | Clone | + // | $ty | Foo<T> | + // |---------------|------------| + // + // The following arm has the same behavior with the exception of the lack of + // support for a leading `const` parameter. + ( + $(#[$attr:meta])* + const $constname:ident : $constty:ident $(,)? + $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* + => $trait:ident for $ty:ty $(; |$candidate:ident| $is_bit_valid:expr)? + ) => { + unsafe_impl!( + @inner + $(#[$attr])* + @const $constname: $constty, + $($tyvar $(: $(? $optbound +)* + $($bound +)*)?,)* + => $trait for $ty $(; |$candidate| $is_bit_valid)? + ); + }; + ( + $(#[$attr:meta])* + $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* + => $trait:ident for $ty:ty $(; |$candidate:ident| $is_bit_valid:expr)? + ) => {{ + unsafe_impl!( + @inner + $(#[$attr])* + $($tyvar $(: $(? $optbound +)* + $($bound +)*)?,)* + => $trait for $ty $(; |$candidate| $is_bit_valid)? + ); + }}; + ( + @inner + $(#[$attr:meta])* + $(@const $constname:ident : $constty:ident,)* + $($tyvar:ident $(: $(? $optbound:ident +)* + $($bound:ident +)* )?,)* + => $trait:ident for $ty:ty $(; |$candidate:ident| $is_bit_valid:expr)? + ) => {{ + crate::util::macros::__unsafe(); + + $(#[$attr])* + #[allow(non_local_definitions)] + // SAFETY: The caller promises that this is sound. + unsafe impl<$($tyvar $(: $(? $optbound +)* $($bound +)*)?),* $(, const $constname: $constty,)*> $trait for $ty { + unsafe_impl!(@method $trait $(; |$candidate| $is_bit_valid)?); + } + }}; + + (@method TryFromBytes ; |$candidate:ident| $is_bit_valid:expr) => { + #[allow(clippy::missing_inline_in_public_items, dead_code)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() {} + + #[inline] + fn is_bit_valid<Alignment>($candidate: Maybe<'_, Self, Alignment>) -> bool + where + Alignment: crate::invariant::Alignment, + { + $is_bit_valid + } + }; + (@method TryFromBytes) => { + #[allow(clippy::missing_inline_in_public_items)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() {} + #[inline(always)] + fn is_bit_valid<Alignment>(_candidate: Maybe<'_, Self, Alignment>) -> bool + where + Alignment: crate::invariant::Alignment, + { + true + } + }; + (@method $trait:ident) => { + #[allow(clippy::missing_inline_in_public_items, dead_code)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() {} + }; + (@method $trait:ident; |$_candidate:ident| $_is_bit_valid:expr) => { + compile_error!("Can't provide `is_bit_valid` impl for trait other than `TryFromBytes`"); + }; +} + +/// Implements `$trait` for `$ty` where `$ty: TransmuteFrom<$repr>` (and +/// vice-versa). +/// +/// Calling this macro is safe; the internals of the macro emit appropriate +/// trait bounds which ensure that the given impl is sound. +macro_rules! impl_for_transmute_from { + ( + $(#[$attr:meta])* + $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?)? + => $trait:ident for $ty:ty [$repr:ty] + ) => { + const _: () = { + $(#[$attr])* + #[allow(non_local_definitions)] + + // SAFETY: `is_trait<T, R>` (defined and used below) requires `T: + // TransmuteFrom<R>`, `R: TransmuteFrom<T>`, and `R: $trait`. It is + // called using `$ty` and `$repr`, ensuring that `$ty` and `$repr` + // have equivalent bit validity, and ensuring that `$repr: $trait`. + // The supported traits - `TryFromBytes`, `FromZeros`, `FromBytes`, + // and `IntoBytes` - are defined only in terms of the bit validity + // of a type. Therefore, `$repr: $trait` ensures that `$ty: $trait` + // is sound. + unsafe impl<$($tyvar $(: $(? $optbound +)* $($bound +)*)?)?> $trait for $ty { + #[allow(dead_code, clippy::missing_inline_in_public_items)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() { + use crate::pointer::{*, invariant::Valid}; + + impl_for_transmute_from!(@assert_is_supported_trait $trait); + + fn is_trait<T, R>() + where + T: TransmuteFrom<R, Valid, Valid> + ?Sized, + R: TransmuteFrom<T, Valid, Valid> + ?Sized, + R: $trait, + { + } + + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn f<$($tyvar $(: $(? $optbound +)* $($bound +)*)?)?>() { + is_trait::<$ty, $repr>(); + } + } + + impl_for_transmute_from!( + @is_bit_valid + $(<$tyvar $(: $(? $optbound +)* $($bound +)*)?>)? + $trait for $ty [$repr] + ); + } + }; + }; + (@assert_is_supported_trait TryFromBytes) => {}; + (@assert_is_supported_trait FromZeros) => {}; + (@assert_is_supported_trait FromBytes) => {}; + (@assert_is_supported_trait IntoBytes) => {}; + ( + @is_bit_valid + $(<$tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?>)? + TryFromBytes for $ty:ty [$repr:ty] + ) => { + #[inline(always)] + fn is_bit_valid<Alignment>(candidate: $crate::Maybe<'_, Self, Alignment>) -> bool + where + Alignment: $crate::invariant::Alignment, + { + // SAFETY: This macro ensures that `$repr` and `Self` have the same + // size and bit validity. Thus, a bit-valid instance of `$repr` is + // also a bit-valid instance of `Self`. + <$repr as TryFromBytes>::is_bit_valid(candidate.transmute::<_, _, BecauseImmutable>()) + } + }; + ( + @is_bit_valid + $(<$tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?>)? + $trait:ident for $ty:ty [$repr:ty] + ) => { + // Trait other than `TryFromBytes`; no `is_bit_valid` impl. + }; +} + +/// Implements a trait for a type, bounding on each member of the power set of +/// a set of type variables. This is useful for implementing traits for tuples +/// or `fn` types. +/// +/// The last argument is the name of a macro which will be called in every +/// `impl` block, and is expected to expand to the name of the type for which to +/// implement the trait. +/// +/// For example, the invocation: +/// ```ignore +/// unsafe_impl_for_power_set!(A, B => Foo for type!(...)) +/// ``` +/// ...expands to: +/// ```ignore +/// unsafe impl Foo for type!() { ... } +/// unsafe impl<B> Foo for type!(B) { ... } +/// unsafe impl<A, B> Foo for type!(A, B) { ... } +/// ``` +macro_rules! unsafe_impl_for_power_set { + ( + $first:ident $(, $rest:ident)* $(-> $ret:ident)? => $trait:ident for $macro:ident!(...) + $(; |$candidate:ident| $is_bit_valid:expr)? + ) => { + unsafe_impl_for_power_set!( + $($rest),* $(-> $ret)? => $trait for $macro!(...) + $(; |$candidate| $is_bit_valid)? + ); + unsafe_impl_for_power_set!( + @impl $first $(, $rest)* $(-> $ret)? => $trait for $macro!(...) + $(; |$candidate| $is_bit_valid)? + ); + }; + ( + $(-> $ret:ident)? => $trait:ident for $macro:ident!(...) + $(; |$candidate:ident| $is_bit_valid:expr)? + ) => { + unsafe_impl_for_power_set!( + @impl $(-> $ret)? => $trait for $macro!(...) + $(; |$candidate| $is_bit_valid)? + ); + }; + ( + @impl $($vars:ident),* $(-> $ret:ident)? => $trait:ident for $macro:ident!(...) + $(; |$candidate:ident| $is_bit_valid:expr)? + ) => { + unsafe_impl!( + $($vars,)* $($ret)? => $trait for $macro!($($vars),* $(-> $ret)?) + $(; |$candidate| $is_bit_valid)? + ); + }; +} + +/// Expands to an `Option<extern "C" fn>` type with the given argument types and +/// return type. Designed for use with `unsafe_impl_for_power_set`. +macro_rules! opt_extern_c_fn { + ($($args:ident),* -> $ret:ident) => { Option<extern "C" fn($($args),*) -> $ret> }; +} + +/// Expands to an `Option<unsafe extern "C" fn>` type with the given argument +/// types and return type. Designed for use with `unsafe_impl_for_power_set`. +macro_rules! opt_unsafe_extern_c_fn { + ($($args:ident),* -> $ret:ident) => { Option<unsafe extern "C" fn($($args),*) -> $ret> }; +} + +/// Expands to an `Option<fn>` type with the given argument types and return +/// type. Designed for use with `unsafe_impl_for_power_set`. +macro_rules! opt_fn { + ($($args:ident),* -> $ret:ident) => { Option<fn($($args),*) -> $ret> }; +} + +/// Expands to an `Option<unsafe fn>` type with the given argument types and +/// return type. Designed for use with `unsafe_impl_for_power_set`. +macro_rules! opt_unsafe_fn { + ($($args:ident),* -> $ret:ident) => { Option<unsafe fn($($args),*) -> $ret> }; +} + +// This `allow` is needed because, when testing, we export this macro so it can +// be used in `doctests`. +#[allow(rustdoc::private_intra_doc_links)] +/// Implements trait(s) for a type or verifies the given implementation by +/// referencing an existing (derived) implementation. +/// +/// This macro exists so that we can provide zerocopy-derive as an optional +/// dependency and still get the benefit of using its derives to validate that +/// our trait impls are sound. +/// +/// When compiling without `--cfg 'feature = "derive"` and without `--cfg test`, +/// `impl_or_verify!` emits the provided trait impl. When compiling with either +/// of those cfgs, it is expected that the type in question is deriving the +/// traits instead. In this case, `impl_or_verify!` emits code which validates +/// that the given trait impl is at least as restrictive as the the impl emitted +/// by the custom derive. This has the effect of confirming that the impl which +/// is emitted when the `derive` feature is disabled is actually sound (on the +/// assumption that the impl emitted by the custom derive is sound). +/// +/// The caller is still required to provide a safety comment (e.g. using the +/// `const _: () = unsafe` macro). The reason for this restriction is that, +/// while `impl_or_verify!` can guarantee that the provided impl is sound when +/// it is compiled with the appropriate cfgs, there is no way to guarantee that +/// it is ever compiled with those cfgs. In particular, it would be possible to +/// accidentally place an `impl_or_verify!` call in a context that is only ever +/// compiled when the `derive` feature is disabled. If that were to happen, +/// there would be nothing to prevent an unsound trait impl from being emitted. +/// Requiring a safety comment reduces the likelihood of emitting an unsound +/// impl in this case, and also provides useful documentation for readers of the +/// code. +/// +/// Finally, if a `TryFromBytes::is_bit_valid` impl is provided, it must adhere +/// to the safety preconditions of [`unsafe_impl!`]. +/// +/// ## Example +/// +/// ```rust,ignore +/// // Note that these derives are gated by `feature = "derive"` +/// #[cfg_attr(any(feature = "derive", test), derive(FromZeros, FromBytes, IntoBytes, Unaligned))] +/// #[repr(transparent)] +/// struct Wrapper<T>(T); +/// +/// const _: () = unsafe { +/// /// SAFETY: +/// /// `Wrapper<T>` is `repr(transparent)`, so it is sound to implement any +/// /// zerocopy trait if `T` implements that trait. +/// impl_or_verify!(T: FromZeros => FromZeros for Wrapper<T>); +/// impl_or_verify!(T: FromBytes => FromBytes for Wrapper<T>); +/// impl_or_verify!(T: IntoBytes => IntoBytes for Wrapper<T>); +/// impl_or_verify!(T: Unaligned => Unaligned for Wrapper<T>); +/// } +/// ``` +#[cfg_attr(__ZEROCOPY_INTERNAL_USE_ONLY_DEV_MODE, macro_export)] // Used in `doctests.rs` +#[doc(hidden)] +macro_rules! impl_or_verify { + // The following two match arms follow the same pattern as their + // counterparts in `unsafe_impl!`; see the documentation on those arms for + // more details. + ( + const $constname:ident : $constty:ident $(,)? + $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* + => $trait:ident for $ty:ty + ) => { + impl_or_verify!(@impl { unsafe_impl!( + const $constname: $constty, $($tyvar $(: $(? $optbound +)* $($bound +)*)?),* => $trait for $ty + ); }); + impl_or_verify!(@verify $trait, { + impl<const $constname: $constty, $($tyvar $(: $(? $optbound +)* $($bound +)*)?),*> Subtrait for $ty {} + }); + }; + ( + $($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* + => $trait:ident for $ty:ty $(; |$candidate:ident| $is_bit_valid:expr)? + ) => { + impl_or_verify!(@impl { unsafe_impl!( + $($tyvar $(: $(? $optbound +)* $($bound +)*)?),* => $trait for $ty + $(; |$candidate| $is_bit_valid)? + ); }); + impl_or_verify!(@verify $trait, { + impl<$($tyvar $(: $(? $optbound +)* $($bound +)*)?),*> Subtrait for $ty {} + }); + }; + (@impl $impl_block:tt) => { + #[cfg(not(any(feature = "derive", test)))] + { $impl_block }; + }; + (@verify $trait:ident, $impl_block:tt) => { + #[cfg(any(feature = "derive", test))] + { + // On some toolchains, `Subtrait` triggers the `dead_code` lint + // because it is implemented but never used. + #[allow(dead_code)] + trait Subtrait: $trait {} + $impl_block + }; + }; +} + +/// Implements `KnownLayout` for a sized type. +macro_rules! impl_known_layout { + ($(const $constvar:ident : $constty:ty, $tyvar:ident $(: ?$optbound:ident)? => $ty:ty),* $(,)?) => { + $(impl_known_layout!(@inner const $constvar: $constty, $tyvar $(: ?$optbound)? => $ty);)* + }; + ($($tyvar:ident $(: ?$optbound:ident)? => $ty:ty),* $(,)?) => { + $(impl_known_layout!(@inner , $tyvar $(: ?$optbound)? => $ty);)* + }; + ($($(#[$attrs:meta])* $ty:ty),*) => { $(impl_known_layout!(@inner , => $(#[$attrs])* $ty);)* }; + (@inner $(const $constvar:ident : $constty:ty)? , $($tyvar:ident $(: ?$optbound:ident)?)? => $(#[$attrs:meta])* $ty:ty) => { + const _: () = { + use core::ptr::NonNull; + + #[allow(non_local_definitions)] + $(#[$attrs])* + // SAFETY: Delegates safety to `DstLayout::for_type`. + unsafe impl<$($tyvar $(: ?$optbound)?)? $(, const $constvar : $constty)?> KnownLayout for $ty { + #[allow(clippy::missing_inline_in_public_items)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() where Self: Sized {} + + type PointerMetadata = (); + + // SAFETY: `CoreMaybeUninit<T>::LAYOUT` and `T::LAYOUT` are + // identical because `CoreMaybeUninit<T>` has the same size and + // alignment as `T` [1], and `CoreMaybeUninit` admits + // uninitialized bytes in all positions. + // + // [1] Per https://doc.rust-lang.org/1.81.0/std/mem/union.MaybeUninit.html#layout-1: + // + // `MaybeUninit<T>` is guaranteed to have the same size, + // alignment, and ABI as `T` + type MaybeUninit = core::mem::MaybeUninit<Self>; + + const LAYOUT: crate::DstLayout = crate::DstLayout::for_type::<$ty>(); + + // SAFETY: `.cast` preserves address and provenance. + // + // FIXME(#429): Add documentation to `.cast` that promises that + // it preserves provenance. + #[inline(always)] + fn raw_from_ptr_len(bytes: NonNull<u8>, _meta: ()) -> NonNull<Self> { + bytes.cast::<Self>() + } + + #[inline(always)] + fn pointer_to_metadata(_ptr: *mut Self) -> () { + } + } + }; + }; +} + +/// Implements `KnownLayout` for a type in terms of the implementation of +/// another type with the same representation. +/// +/// # Safety +/// +/// - `$ty` and `$repr` must have the same: +/// - Fixed prefix size +/// - Alignment +/// - (For DSTs) trailing slice element size +/// - It must be valid to perform an `as` cast from `*mut $repr` to `*mut $ty`, +/// and this operation must preserve referent size (ie, `size_of_val_raw`). +macro_rules! unsafe_impl_known_layout { + ($($tyvar:ident: ?Sized + KnownLayout =>)? #[repr($repr:ty)] $ty:ty) => {{ + use core::ptr::NonNull; + + crate::util::macros::__unsafe(); + + #[allow(non_local_definitions)] + // SAFETY: The caller promises that this is sound. + unsafe impl<$($tyvar: ?Sized + KnownLayout)?> KnownLayout for $ty { + #[allow(clippy::missing_inline_in_public_items, dead_code)] + #[cfg_attr(all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), coverage(off))] + fn only_derive_is_allowed_to_implement_this_trait() {} + + type PointerMetadata = <$repr as KnownLayout>::PointerMetadata; + type MaybeUninit = <$repr as KnownLayout>::MaybeUninit; + + const LAYOUT: DstLayout = <$repr as KnownLayout>::LAYOUT; + + // SAFETY: All operations preserve address and provenance. Caller + // has promised that the `as` cast preserves size. + // + // FIXME(#429): Add documentation to `NonNull::new_unchecked` that + // it preserves provenance. + #[inline(always)] + fn raw_from_ptr_len(bytes: NonNull<u8>, meta: <$repr as KnownLayout>::PointerMetadata) -> NonNull<Self> { + #[allow(clippy::as_conversions)] + let ptr = <$repr>::raw_from_ptr_len(bytes, meta).as_ptr() as *mut Self; + // SAFETY: `ptr` was converted from `bytes`, which is non-null. + unsafe { NonNull::new_unchecked(ptr) } + } + + #[inline(always)] + fn pointer_to_metadata(ptr: *mut Self) -> Self::PointerMetadata { + #[allow(clippy::as_conversions)] + let ptr = ptr as *mut $repr; + <$repr>::pointer_to_metadata(ptr) + } + } + }}; +} + +/// Uses `align_of` to confirm that a type or set of types have alignment 1. +/// +/// Note that `align_of<T>` requires `T: Sized`, so this macro doesn't work for +/// unsized types. +macro_rules! assert_unaligned { + ($($tys:ty),*) => { + $( + // We only compile this assertion under `cfg(test)` to avoid taking + // an extra non-dev dependency (and making this crate more expensive + // to compile for our dependents). + #[cfg(test)] + static_assertions::const_assert_eq!(core::mem::align_of::<$tys>(), 1); + )* + }; +} + +/// Emits a function definition as either `const fn` or `fn` depending on +/// whether the current toolchain version supports `const fn` with generic trait +/// bounds. +macro_rules! maybe_const_trait_bounded_fn { + // This case handles both `self` methods (where `self` is by value) and + // non-method functions. Each `$args` may optionally be followed by `: + // $arg_tys:ty`, which can be omitted for `self`. + ($(#[$attr:meta])* $vis:vis const fn $name:ident($($args:ident $(: $arg_tys:ty)?),* $(,)?) $(-> $ret_ty:ty)? $body:block) => { + #[cfg(not(no_zerocopy_generic_bounds_in_const_fn_1_61_0))] + $(#[$attr])* $vis const fn $name($($args $(: $arg_tys)?),*) $(-> $ret_ty)? $body + + #[cfg(no_zerocopy_generic_bounds_in_const_fn_1_61_0)] + $(#[$attr])* $vis fn $name($($args $(: $arg_tys)?),*) $(-> $ret_ty)? $body + }; +} + +/// Either panic (if the current Rust toolchain supports panicking in `const +/// fn`) or evaluate a constant that will cause an array indexing error whose +/// error message will include the format string. +/// +/// The type that this expression evaluates to must be `Copy`, or else the +/// non-panicking desugaring will fail to compile. +macro_rules! const_panic { + (@non_panic $($_arg:tt)+) => {{ + // This will type check to whatever type is expected based on the call + // site. + let panic: [_; 0] = []; + // This will always fail (since we're indexing into an array of size 0. + #[allow(unconditional_panic)] + panic[0] + }}; + ($($arg:tt)+) => {{ + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + panic!($($arg)+); + #[cfg(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0)] + const_panic!(@non_panic $($arg)+) + }}; +} + +/// Either assert (if the current Rust toolchain supports panicking in `const +/// fn`) or evaluate the expression and, if it evaluates to `false`, call +/// `const_panic!`. This is used in place of `assert!` in const contexts to +/// accommodate old toolchains. +macro_rules! const_assert { + ($e:expr) => {{ + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + assert!($e); + #[cfg(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0)] + { + let e = $e; + if !e { + let _: () = const_panic!(@non_panic concat!("assertion failed: ", stringify!($e))); + } + } + }}; + ($e:expr, $($args:tt)+) => {{ + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + assert!($e, $($args)+); + #[cfg(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0)] + { + let e = $e; + if !e { + let _: () = const_panic!(@non_panic concat!("assertion failed: ", stringify!($e), ": ", stringify!($arg)), $($args)*); + } + } + }}; +} + +/// Like `const_assert!`, but relative to `debug_assert!`. +macro_rules! const_debug_assert { + ($e:expr $(, $msg:expr)?) => {{ + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + debug_assert!($e $(, $msg)?); + #[cfg(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0)] + { + // Use this (rather than `#[cfg(debug_assertions)]`) to ensure that + // `$e` is always compiled even if it will never be evaluated at + // runtime. + if cfg!(debug_assertions) { + let e = $e; + if !e { + let _: () = const_panic!(@non_panic concat!("assertion failed: ", stringify!($e) $(, ": ", $msg)?)); + } + } + } + }} +} + +/// Either invoke `unreachable!()` or `loop {}` depending on whether the Rust +/// toolchain supports panicking in `const fn`. +macro_rules! const_unreachable { + () => {{ + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + unreachable!(); + + #[cfg(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0)] + loop {} + }}; +} + +/// Asserts at compile time that `$condition` is true for `Self` or the given +/// `$tyvar`s. Unlike `const_assert`, this is *strictly* a compile-time check; +/// it cannot be evaluated in a runtime context. The condition is checked after +/// monomorphization and, upon failure, emits a compile error. +macro_rules! static_assert { + (Self $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )? => $condition:expr $(, $args:tt)*) => {{ + trait StaticAssert { + const ASSERT: bool; + } + + impl<T $(: $(? $optbound +)* $($bound +)*)?> StaticAssert for T { + const ASSERT: bool = { + const_assert!($condition $(, $args)*); + $condition + }; + } + + const_assert!(<Self as StaticAssert>::ASSERT); + }}; + ($($tyvar:ident $(: $(? $optbound:ident $(+)?)* $($bound:ident $(+)?)* )?),* => $condition:expr $(, $args:tt)*) => {{ + trait StaticAssert { + const ASSERT: bool; + } + + // NOTE: We use `PhantomData` so we can support unsized types. + impl<$($tyvar $(: $(? $optbound +)* $($bound +)*)?,)*> StaticAssert for ($(core::marker::PhantomData<$tyvar>,)*) { + const ASSERT: bool = { + const_assert!($condition $(, $args)*); + $condition + }; + } + + const_assert!(<($(core::marker::PhantomData<$tyvar>,)*) as StaticAssert>::ASSERT); + }}; +} + +/// Assert at compile time that `tyvar` does not have a zero-sized DST +/// component. +macro_rules! static_assert_dst_is_not_zst { + ($tyvar:ident) => {{ + use crate::KnownLayout; + static_assert!($tyvar: ?Sized + KnownLayout => { + let dst_is_zst = match $tyvar::LAYOUT.size_info { + crate::SizeInfo::Sized { .. } => false, + crate::SizeInfo::SliceDst(TrailingSliceLayout { elem_size, .. }) => { + elem_size == 0 + } + }; + !dst_is_zst + }, "cannot call this method on a dynamically-sized type whose trailing slice element is zero-sized"); + }} +} + +/// Defines a named [`Cast`] implementation. +/// +/// # Safety +/// +/// The caller must ensure that, given `src: *mut $src`, `src as *mut $dst` is a +/// size-preserving or size-shrinking cast. +/// +/// [`Cast`]: crate::pointer::cast::Cast +#[macro_export] +#[doc(hidden)] +macro_rules! define_cast { + // We require the caller to provide an `unsafe` block as part of the input + // syntax since a call to `define_cast!` is useless inside of an `unsafe` + // block (since it would introduce a type which can't be named outside of + // the context of that block). + (unsafe { $vis:vis $name:ident $(<$tyvar:ident $(: ?$optbound:ident)?>)? = $src:ty => $dst:ty }) => { + #[allow(missing_debug_implementations, missing_copy_implementations, unreachable_pub)] + $vis enum $name {} + + // SAFETY: The caller promises that `src as *mut $src` is a size- + // preserving or size-shrinking cast. All operations preserve + // provenance. + unsafe impl $(<$tyvar $(: ?$optbound)?>)? $crate::pointer::cast::Project<$src, $dst> for $name { + fn project(src: $crate::pointer::PtrInner<'_, $src>) -> *mut $dst { + #[allow(clippy::as_conversions)] + return src.as_ptr() as *mut $dst; + } + } + + // SAFETY: The impl of `Project::project` preserves referent address. + unsafe impl $(<$tyvar $(: ?$optbound)?>)? $crate::pointer::cast::Cast<$src, $dst> for $name {} + }; +} + +/// Implements `TransmuteFrom` and `SizeEq` for `T` and `$wrapper<T>`. +/// +/// # Safety +/// +/// `T` and `$wrapper<T>` must have the same bit validity, and must have the +/// same size in the sense of `CastExact` (specifically, both a +/// `T`-to-`$wrapper<T>` cast and a `$wrapper<T>`-to-`T` cast must be +/// size-preserving). +macro_rules! unsafe_impl_for_transparent_wrapper { + ($vis:vis T $(: ?$optbound:ident)? => $wrapper:ident<T>) => {{ + crate::util::macros::__unsafe(); + + use crate::pointer::{TransmuteFrom, cast::{CastExact, TransitiveProject}, SizeEq, invariant::Valid}; + use crate::wrappers::ReadOnly; + + // SAFETY: The caller promises that `T` and `$wrapper<T>` have the same + // bit validity. + unsafe impl<T $(: ?$optbound)?> TransmuteFrom<T, Valid, Valid> for $wrapper<T> {} + // SAFETY: See previous safety comment. + unsafe impl<T $(: ?$optbound)?> TransmuteFrom<$wrapper<T>, Valid, Valid> for T {} + // SAFETY: The caller promises that a `T` to `$wrapper<T>` cast is + // size-preserving. + define_cast!(unsafe { $vis CastToWrapper<T $(: ?$optbound)? > = T => $wrapper<T> }); + // SAFETY: The caller promises that a `T` to `$wrapper<T>` cast is + // size-preserving. + unsafe impl<T $(: ?$optbound)?> CastExact<T, $wrapper<T>> for CastToWrapper {} + // SAFETY: The caller promises that a `$wrapper<T>` to `T` cast is + // size-preserving. + define_cast!(unsafe { $vis CastFromWrapper<T $(: ?$optbound)? > = $wrapper<T> => T }); + // SAFETY: The caller promises that a `$wrapper<T>` to `T` cast is + // size-preserving. + unsafe impl<T $(: ?$optbound)?> CastExact<$wrapper<T>, T> for CastFromWrapper {} + + impl<T $(: ?$optbound)?> SizeEq<T> for $wrapper<T> { + type CastFrom = CastToWrapper; + } + impl<T $(: ?$optbound)?> SizeEq<$wrapper<T>> for T { + type CastFrom = CastFromWrapper; + } + + impl<T $(: ?$optbound)?> SizeEq<ReadOnly<T>> for $wrapper<T> { + type CastFrom = TransitiveProject< + T, + <T as SizeEq<ReadOnly<T>>>::CastFrom, + CastToWrapper, + >; + } + impl<T $(: ?$optbound)?> SizeEq<$wrapper<T>> for ReadOnly<T> { + type CastFrom = TransitiveProject< + T, + CastFromWrapper, + <ReadOnly<T> as SizeEq<T>>::CastFrom, + >; + } + + impl<T $(: ?$optbound)?> SizeEq<ReadOnly<T>> for ReadOnly<$wrapper<T>> { + type CastFrom = TransitiveProject< + $wrapper<T>, + <$wrapper<T> as SizeEq<ReadOnly<T>>>::CastFrom, + <ReadOnly<$wrapper<T>> as SizeEq<$wrapper<T>>>::CastFrom, + >; + } + impl<T $(: ?$optbound)?> SizeEq<ReadOnly<$wrapper<T>>> for ReadOnly<T> { + type CastFrom = TransitiveProject< + $wrapper<T>, + <$wrapper<T> as SizeEq<ReadOnly<$wrapper<T>>>>::CastFrom, + <ReadOnly<T> as SizeEq<$wrapper<T>>>::CastFrom, + >; + } + }}; +} + +macro_rules! impl_transitive_transmute_from { + ($($tyvar:ident $(: ?$optbound:ident)?)? => $t:ty => $u:ty => $v:ty) => { + const _: () = { + use crate::pointer::{TransmuteFrom, SizeEq, invariant::Valid}; + + impl<$($tyvar $(: ?$optbound)?)?> SizeEq<$t> for $v + where + $u: SizeEq<$t>, + $v: SizeEq<$u>, + { + type CastFrom = cast::TransitiveProject< + $u, + <$u as SizeEq<$t>>::CastFrom, + <$v as SizeEq<$u>>::CastFrom + >; + } + + // SAFETY: Since `$u: TransmuteFrom<$t, Valid, Valid>`, it is sound + // to transmute a bit-valid `$t` to a bit-valid `$u`. Since `$v: + // TransmuteFrom<$u, Valid, Valid>`, it is sound to transmute that + // bit-valid `$u` to a bit-valid `$v`. + unsafe impl<$($tyvar $(: ?$optbound)?)?> TransmuteFrom<$t, Valid, Valid> for $v + where + $u: TransmuteFrom<$t, Valid, Valid>, + $v: TransmuteFrom<$u, Valid, Valid>, + {} + }; + }; +} + +/// A no-op `unsafe fn` for use in macro expansions. +/// +/// Calling this function in a macro expansion ensures that the macro's caller +/// must wrap the call in `unsafe { ... }`. +#[inline(always)] +pub(crate) const unsafe fn __unsafe() {} + +/// Extracts the contents of doc comments. +#[allow(unused)] +macro_rules! docstring { + ($(#[doc = $content:expr])*) => { + concat!($($content, "\n",)*) + } +} + +/// Generate a rustdoc-style header with `$name` as the HTML ID for the 'Code +/// Generation' section of documentation. +#[allow(unused)] +macro_rules! codegen_header { + ($level:expr, $name:expr) => { + concat!( + " +<", + $level, + " id='method.", + $name, + ".codegen'> + <a class='doc-anchor' href='#method.", + $name, + ".codegen'>§</a> + Code Generation +</", + $level, + "> +" + ) + }; +} + +/// Generates HTML tabs. +#[rustfmt::skip] +#[allow(unused)] +macro_rules! tabs { + ( + name = $name:expr, + arity = $arity:literal, + $([ + $($open:ident)? + @index $n:literal + @title $title:literal + $(#[doc = $content:expr])* + ]),* + ) => { + concat!(" +<div class='codegen-tabs' style='--arity: ", $arity ,"'>", $(concat!(" + <details name='tab-", $name,"' style='--n: ", $n ,"'", $(stringify!($open),)*"> + <summary><h6>", $title, "</h6></summary> + <div> + +", $($content, "\n",)* " +\ + </div> + </details>"),)* +"</div>") + } +} + +/// Generates the HTML for a single benchmark example. +#[allow(unused)] +macro_rules! codegen_example { + (format = $format:expr, bench = $bench:expr) => { + tabs!( + name = $bench, + arity = 4, + [ + @index 1 + @title "Format" + /// ```ignore + #[doc = include_str!(concat!("../benches/formats/", $format, ".rs"))] + /// ``` + ], + [ + @index 2 + @title "Benchmark" + /// ```ignore + #[doc = include_str!(concat!("../benches/", $bench, ".rs"))] + /// ``` + ], + [ + open + @index 3 + @title "Assembly" + /// ```plain + #[doc = include_str!(concat!("../benches/", $bench, ".x86-64"))] + /// ``` + ], + [ + @index 4 + @title "Machine Code Analysis" + /// ```plain + #[doc = include_str!(concat!("../benches/", $bench, ".x86-64.mca"))] + /// ``` + ] + ) + } +} + +/// Generate the HTML for a suite of benchmark examples. +#[allow(unused)] +macro_rules! codegen_example_suite { + ( + bench = $bench:expr, + format = $format:expr, + arity = $arity:literal, + $([ + $($open:ident)? + @index $index:literal + @title $title:literal + @variant $variant:literal + ]),* + ) => { + tabs!( + name = $bench, + arity = $arity, + $([ + $($open)* + @index $index + @title $title + #[doc = codegen_example!( + format = concat!($format, "_", $variant), + bench = concat!($bench, "_", $variant) + )] + ]),* + ) + } +} + +/// Generates the string for code generation preamble. +#[allow(unused)] +macro_rules! codegen_preamble { + () => { + docstring!( + /// + /// This abstraction is safe and cheap, but does not necessarily + /// have zero runtime cost. The codegen you experience in practice + /// will depend on optimization level, the layout of the destination + /// type, and what the compiler can prove about the source. + /// + ) + } +} + +/// Stub for rendering codegen documentation; used to break build dependency +/// between benches and zerocopy when re-blessing codegen tests. +#[allow(unused)] +#[cfg(not(doc))] +macro_rules! codegen_section { + ( + header = $level:expr, + bench = $bench:expr, + format = $format:expr, + arity = $arity:literal, + $([ + $($open:ident)? + @index $index:literal + @title $title:literal + @variant $variant:literal + ]),* + ) => { + "" + }; + ( + header = $level:expr, + bench = $bench:expr, + format = $format:expr, + ) => { + "" + }; +} + +/// Generates the HTML for code generation documentation. +#[allow(unused)] +#[cfg(doc)] +macro_rules! codegen_section { + ( + header = $level:expr, + bench = $bench:expr, + format = $format:expr, + arity = $arity:literal, + $([ + $($open:ident)? + @index $index:literal + @title $title:literal + @variant $variant:literal + ]),* + ) => { + concat!( + codegen_header!($level, $bench), + codegen_preamble!(), + docstring!( + /// + /// The below examples illustrate typical codegen for + /// increasingly complex types: + /// + ), + codegen_example_suite!( + bench = $bench, + format = $format, + arity = $arity, + $([ + $($open)* + @index $index + @title $title + @variant $variant + ]),* + ) + ) + }; + ( + header = $level:expr, + bench = $bench:expr, + format = $format:expr, + ) => { + concat!( + codegen_header!($level, $bench), + codegen_preamble!(), + codegen_example!( + format = $format, + bench = $bench + ) + ) + } +} diff --git a/rust/zerocopy/src/util/mod.rs b/rust/zerocopy/src/util/mod.rs new file mode 100644 index 000000000000..02fd4ed62741 --- /dev/null +++ b/rust/zerocopy/src/util/mod.rs @@ -0,0 +1,944 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +#[macro_use] +pub(crate) mod macros; + +#[doc(hidden)] +pub mod macro_util; + +use core::{ + marker::PhantomData, + mem::{self, ManuallyDrop}, + num::NonZeroUsize, + ptr::NonNull, +}; + +use super::*; +use crate::pointer::{ + invariant::{Exclusive, Shared, Valid}, + SizeEq, TransmuteFromPtr, +}; + +/// Like [`PhantomData`], but [`Send`] and [`Sync`] regardless of whether the +/// wrapped `T` is. +pub(crate) struct SendSyncPhantomData<T: ?Sized>(PhantomData<T>); + +// SAFETY: `SendSyncPhantomData` does not enable any behavior which isn't sound +// to be called from multiple threads. +unsafe impl<T: ?Sized> Send for SendSyncPhantomData<T> {} +// SAFETY: `SendSyncPhantomData` does not enable any behavior which isn't sound +// to be called from multiple threads. +unsafe impl<T: ?Sized> Sync for SendSyncPhantomData<T> {} + +impl<T: ?Sized> Default for SendSyncPhantomData<T> { + fn default() -> SendSyncPhantomData<T> { + SendSyncPhantomData(PhantomData) + } +} + +impl<T: ?Sized> PartialEq for SendSyncPhantomData<T> { + fn eq(&self, _other: &Self) -> bool { + true + } +} + +impl<T: ?Sized> Eq for SendSyncPhantomData<T> {} + +impl<T: ?Sized> Clone for SendSyncPhantomData<T> { + fn clone(&self) -> Self { + SendSyncPhantomData(PhantomData) + } +} + +#[cfg(miri)] +extern "Rust" { + /// Miri-provided intrinsic that marks the pointer `ptr` as aligned to + /// `align`. + /// + /// This intrinsic is used to inform Miri's symbolic alignment checker that + /// a pointer is aligned, even if Miri cannot statically deduce that fact. + /// This is often required when performing raw pointer arithmetic or casts + /// where the alignment is guaranteed by runtime checks or invariants that + /// Miri is not aware of. + pub(crate) fn miri_promise_symbolic_alignment(ptr: *const (), align: usize); +} + +pub(crate) trait AsAddress { + fn addr(self) -> usize; +} + +impl<T: ?Sized> AsAddress for &T { + #[inline(always)] + fn addr(self) -> usize { + let ptr: *const T = self; + AsAddress::addr(ptr) + } +} + +impl<T: ?Sized> AsAddress for &mut T { + #[inline(always)] + fn addr(self) -> usize { + let ptr: *const T = self; + AsAddress::addr(ptr) + } +} + +impl<T: ?Sized> AsAddress for NonNull<T> { + #[inline(always)] + fn addr(self) -> usize { + AsAddress::addr(self.as_ptr()) + } +} + +impl<T: ?Sized> AsAddress for *const T { + #[inline(always)] + fn addr(self) -> usize { + // FIXME(#181), FIXME(https://github.com/rust-lang/rust/issues/95228): + // Use `.addr()` instead of `as usize` once it's stable, and get rid of + // this `allow`. Currently, `as usize` is the only way to accomplish + // this. + #[allow(clippy::as_conversions)] + #[cfg_attr( + __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, + allow(lossy_provenance_casts) + )] + return self.cast::<()>() as usize; + } +} + +impl<T: ?Sized> AsAddress for *mut T { + #[inline(always)] + fn addr(self) -> usize { + let ptr: *const T = self; + AsAddress::addr(ptr) + } +} + +/// Validates that `t` is aligned to `align_of::<U>()`. +#[inline(always)] +pub(crate) fn validate_aligned_to<T: AsAddress, U>(t: T) -> Result<(), AlignmentError<(), U>> { + // `mem::align_of::<U>()` is guaranteed to return a non-zero value, which in + // turn guarantees that this mod operation will not panic. + #[allow(clippy::arithmetic_side_effects)] + let remainder = t.addr() % mem::align_of::<U>(); + if remainder == 0 { + Ok(()) + } else { + // SAFETY: We just confirmed that `t.addr() % align_of::<U>() != 0`. + // That's only possible if `align_of::<U>() > 1`. + Err(unsafe { AlignmentError::new_unchecked(()) }) + } +} + +/// Returns the bytes needed to pad `len` to the next multiple of `align`. +/// +/// This function assumes that align is a power of two; there are no guarantees +/// on the answer it gives if this is not the case. +#[cfg_attr( + kani, + kani::requires(len <= DstLayout::MAX_SIZE), + kani::requires(align.is_power_of_two()), + kani::ensures(|&p| (len + p) % align.get() == 0), + // Ensures that we add the minimum required padding. + kani::ensures(|&p| p < align.get()), +)] +#[cfg_attr(not(zerocopy_inline_always), inline)] +#[cfg_attr(zerocopy_inline_always, inline(always))] +pub(crate) const fn padding_needed_for(len: usize, align: NonZeroUsize) -> usize { + #[cfg(kani)] + #[kani::proof_for_contract(padding_needed_for)] + fn proof() { + padding_needed_for(kani::any(), kani::any()); + } + + // Abstractly, we want to compute: + // align - (len % align). + // Handling the case where len%align is 0. + // Because align is a power of two, len % align = len & (align-1). + // Guaranteed not to underflow as align is nonzero. + #[allow(clippy::arithmetic_side_effects)] + let mask = align.get() - 1; + + // To efficiently subtract this value from align, we can use the bitwise + // complement. + // Note that ((!len) & (align-1)) gives us a number that with (len & + // (align-1)) sums to align-1. So subtracting 1 from x before taking the + // complement subtracts `len` from `align`. Some quick inspection of + // cases shows that this also handles the case where `len % align = 0` + // correctly too: len-1 % align then equals align-1, so the complement mod + // align will be 0, as desired. + // + // The following reasoning can be verified quickly by an SMT solver + // supporting the theory of bitvectors: + // ```smtlib + // ; Naive implementation of padding + // (define-fun padding1 ( + // (len (_ BitVec 32)) + // (align (_ BitVec 32))) (_ BitVec 32) + // (ite + // (= (_ bv0 32) (bvand len (bvsub align (_ bv1 32)))) + // (_ bv0 32) + // (bvsub align (bvand len (bvsub align (_ bv1 32)))))) + // + // ; The implementation below + // (define-fun padding2 ( + // (len (_ BitVec 32)) + // (align (_ BitVec 32))) (_ BitVec 32) + // (bvand (bvnot (bvsub len (_ bv1 32))) (bvsub align (_ bv1 32)))) + // + // (define-fun is-power-of-two ((x (_ BitVec 32))) Bool + // (= (_ bv0 32) (bvand x (bvsub x (_ bv1 32))))) + // + // (declare-const len (_ BitVec 32)) + // (declare-const align (_ BitVec 32)) + // ; Search for a case where align is a power of two and padding2 disagrees + // ; with padding1 + // (assert (and (is-power-of-two align) + // (not (= (padding1 len align) (padding2 len align))))) + // (simplify (padding1 (_ bv300 32) (_ bv32 32))) ; 20 + // (simplify (padding2 (_ bv300 32) (_ bv32 32))) ; 20 + // (simplify (padding1 (_ bv322 32) (_ bv32 32))) ; 30 + // (simplify (padding2 (_ bv322 32) (_ bv32 32))) ; 30 + // (simplify (padding1 (_ bv8 32) (_ bv8 32))) ; 0 + // (simplify (padding2 (_ bv8 32) (_ bv8 32))) ; 0 + // (check-sat) ; unsat, also works for 64-bit bitvectors + // ``` + !(len.wrapping_sub(1)) & mask +} + +/// Rounds `n` down to the largest value `m` such that `m <= n` and `m % align +/// == 0`. +/// +/// # Panics +/// +/// May panic if `align` is not a power of two. Even if it doesn't panic in this +/// case, it will produce nonsense results. +#[inline(always)] +#[cfg_attr( + kani, + kani::requires(align.is_power_of_two()), + kani::ensures(|&m| m <= n && m % align.get() == 0), + // Guarantees that `m` is the *largest* value such that `m % align == 0`. + kani::ensures(|&m| { + // If this `checked_add` fails, then the next multiple would wrap + // around, which trivially satisfies the "largest value" requirement. + m.checked_add(align.get()).map(|next_mul| next_mul > n).unwrap_or(true) + }) +)] +pub(crate) const fn round_down_to_next_multiple_of_alignment( + n: usize, + align: NonZeroUsize, +) -> usize { + #[cfg(kani)] + #[kani::proof_for_contract(round_down_to_next_multiple_of_alignment)] + fn proof() { + round_down_to_next_multiple_of_alignment(kani::any(), kani::any()); + } + + let align = align.get(); + #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))] + debug_assert!(align.is_power_of_two()); + + // Subtraction can't underflow because `align.get() >= 1`. + #[allow(clippy::arithmetic_side_effects)] + let mask = !(align - 1); + n & mask +} + +#[cfg_attr(not(zerocopy_inline_always), inline)] +#[cfg_attr(zerocopy_inline_always, inline(always))] +pub(crate) const fn max(a: NonZeroUsize, b: NonZeroUsize) -> NonZeroUsize { + if a.get() < b.get() { + b + } else { + a + } +} + +#[cfg_attr(not(zerocopy_inline_always), inline)] +#[cfg_attr(zerocopy_inline_always, inline(always))] +pub(crate) const fn min(a: NonZeroUsize, b: NonZeroUsize) -> NonZeroUsize { + if a.get() > b.get() { + b + } else { + a + } +} + +/// Copies `src` into the prefix of `dst`. +/// +/// # Safety +/// +/// The caller guarantees that `src.len() <= dst.len()`. +#[inline(always)] +pub(crate) unsafe fn copy_unchecked(src: &[u8], dst: &mut [u8]) { + debug_assert!(src.len() <= dst.len()); + // SAFETY: This invocation satisfies the safety contract of + // copy_nonoverlapping [1]: + // - `src.as_ptr()` is trivially valid for reads of `src.len()` bytes + // - `dst.as_ptr()` is valid for writes of `src.len()` bytes, because the + // caller has promised that `src.len() <= dst.len()` + // - `src` and `dst` are, trivially, properly aligned + // - the region of memory beginning at `src` with a size of `src.len()` + // bytes does not overlap with the region of memory beginning at `dst` + // with the same size, because `dst` is derived from an exclusive + // reference. + unsafe { + core::ptr::copy_nonoverlapping(src.as_ptr(), dst.as_mut_ptr(), src.len()); + }; +} + +/// Unsafely transmutes the given `src` into a type `Dst`. +/// +/// # Safety +/// +/// The value `src` must be a valid instance of `Dst`. +#[inline(always)] +pub(crate) const unsafe fn transmute_unchecked<Src, Dst>(src: Src) -> Dst { + static_assert!(Src, Dst => core::mem::size_of::<Src>() == core::mem::size_of::<Dst>()); + + #[repr(C)] + union Transmute<Src, Dst> { + src: ManuallyDrop<Src>, + dst: ManuallyDrop<Dst>, + } + + // SAFETY: Since `Transmute<Src, Dst>` is `#[repr(C)]`, its `src` and `dst` + // fields both start at the same offset and the types of those fields are + // transparent wrappers around `Src` and `Dst` [1]. Consequently, + // initializing `Transmute` with with `src` and then reading out `dst` is + // equivalent to transmuting from `Src` to `Dst` [2]. Transmuting from `src` + // to `Dst` is valid because — by contract on the caller — `src` is a valid + // instance of `Dst`. + // + // [1] Per https://doc.rust-lang.org/1.82.0/std/mem/struct.ManuallyDrop.html: + // + // `ManuallyDrop<T>` is guaranteed to have the same layout and bit + // validity as `T`, and is subject to the same layout optimizations as + // `T`. + // + // [2] Per https://doc.rust-lang.org/1.82.0/reference/items/unions.html#reading-and-writing-union-fields: + // + // Effectively, writing to and then reading from a union with the C + // representation is analogous to a transmute from the type used for + // writing to the type used for reading. + unsafe { ManuallyDrop::into_inner(Transmute { src: ManuallyDrop::new(src) }.dst) } +} + +/// # Safety +/// +/// `Src` must have a greater or equal alignment to `Dst`. +pub(crate) unsafe fn transmute_ref<Src, Dst, R>(src: &Src) -> &Dst +where + Src: ?Sized, + Dst: SizeEq<Src> + + TransmuteFromPtr<Src, Shared, Valid, Valid, <Dst as SizeEq<Src>>::CastFrom, R> + + ?Sized, +{ + let dst = Ptr::from_ref(src).transmute(); + // SAFETY: The caller promises that `Src`'s alignment is at least as large + // as `Dst`'s alignment. + let dst = unsafe { dst.assume_alignment() }; + dst.as_ref() +} + +/// # Safety +/// +/// `Src` must have a greater or equal alignment to `Dst`. +pub(crate) unsafe fn transmute_mut<Src, Dst, R>(src: &mut Src) -> &mut Dst +where + Src: ?Sized, + Dst: SizeEq<Src> + + TransmuteFromPtr<Src, Exclusive, Valid, Valid, <Dst as SizeEq<Src>>::CastFrom, R> + + ?Sized, +{ + let dst = Ptr::from_mut(src).transmute(); + // SAFETY: The caller promises that `Src`'s alignment is at least as large + // as `Dst`'s alignment. + let dst = unsafe { dst.assume_alignment() }; + dst.as_mut() +} + +/// Uses `allocate` to create a `Box<T>`. +/// +/// # Errors +/// +/// Returns an error on allocation failure. Allocation failure is guaranteed +/// never to cause a panic or an abort. +/// +/// # Safety +/// +/// `allocate` must be either `alloc::alloc::alloc` or +/// `alloc::alloc::alloc_zeroed`. The referent of the box returned by `new_box` +/// has the same bit-validity as the referent of the pointer returned by the +/// given `allocate` and sufficient size to store `T` with `meta`. +#[must_use = "has no side effects (other than allocation)"] +#[cfg(feature = "alloc")] +#[inline] +pub(crate) unsafe fn new_box<T>( + meta: T::PointerMetadata, + allocate: unsafe fn(core::alloc::Layout) -> *mut u8, +) -> Result<alloc::boxed::Box<T>, AllocError> +where + T: ?Sized + crate::KnownLayout, +{ + let align = T::LAYOUT.align.get(); + if !T::is_valid_metadata(meta) { + return Err(AllocError); + } + let size = match T::size_for_metadata(meta) { + Some(size) => size, + // Thanks to the `!T::is_valid_metadata(meta)` check + // above, this branch is unreachable. Fortunately, the + // optimizer recognizes this, so replacing this branch + // with `unreachable_unchecked` produces no codegen + // improvements. + None => return Err(AllocError), + }; + let ptr = if size != 0 { + // SAFETY: + // - `align` is derived from a `NonZeroUsize` and is thus non-zero. + // - `align` is a power of two because, by invariant on + // `KnownLayout::LAYOUT` `<T as KnownLayout>::LAYOUT` accurately + // reflects the layout of `T`. + // - `size`, by invariant on `size_for_metadata` is well-aligned for + // `align` and, by the check on `T::is_valid_metadata(meta)`, is less + // than `isize::MAX`. + let layout: Layout = unsafe { Layout::from_size_align_unchecked(size, align) }; + // SAFETY: By contract on the caller, `allocate` is either + // `alloc::alloc::alloc` or `alloc::alloc::alloc_zeroed`. The above + // check ensures their shared safety precondition: that the supplied + // layout is not zero-sized type [1]. + // + // [1] Per https://doc.rust-lang.org/1.81.0/std/alloc/trait.GlobalAlloc.html#tymethod.alloc: + // + // This function is unsafe because undefined behavior can result if + // the caller does not ensure that layout has non-zero size. + let ptr = unsafe { allocate(layout) }; + match NonNull::new(ptr) { + Some(ptr) => ptr, + None => return Err(AllocError), + } + } else { + // We use `transmute` instead of an `as` cast since Miri (with strict + // provenance enabled) notices and complains that an `as` cast creates a + // pointer with no provenance. Miri isn't smart enough to realize that + // we're only executing this branch when we're constructing a zero-sized + // `Box`, which doesn't require provenance. + // + // SAFETY: any initialized bit sequence is a bit-valid `*mut u8`. All + // bits of a `usize` are initialized. + // + // `#[allow(unknown_lints)]` is for `integer_to_ptr_transmutes` + #[allow(unknown_lints)] + #[allow(clippy::useless_transmute, integer_to_ptr_transmutes)] + let dangling = unsafe { mem::transmute::<usize, *mut u8>(align) }; + // SAFETY: `dangling` is constructed from `align`, which is derived from + // a `NonZeroUsize`, which is guaranteed to be non-zero. + // + // `Box<[T]>` does not allocate when `T` is zero-sized or when `len` is + // zero, but it does require a non-null dangling pointer for its + // allocation. + // + // FIXME(https://github.com/rust-lang/rust/issues/95228): Use + // `std::ptr::without_provenance` once it's stable. That may optimize + // better. As written, Rust may assume that this consumes "exposed" + // provenance, and thus Rust may have to assume that this may consume + // provenance from any pointer whose provenance has been exposed. + unsafe { NonNull::new_unchecked(dangling) } + }; + + let ptr = T::raw_from_ptr_len(ptr, meta); + + // FIXME(#429): Add a "SAFETY" comment and remove this `allow`. Make sure to + // include a justification that `ptr.as_ptr()` is validly-aligned in the ZST + // case (in which we manually construct a dangling pointer) and to justify + // why `Box` is safe to drop (it's because `allocate` uses the system + // allocator). + #[allow(clippy::undocumented_unsafe_blocks)] + Ok(unsafe { alloc::boxed::Box::from_raw(ptr.as_ptr()) }) +} + +mod len_of { + use super::*; + + /// A witness type for metadata of a valid instance of `&T`. + pub struct MetadataOf<T: ?Sized + KnownLayout> { + /// # Safety + /// + /// The size of an instance of `&T` with the given metadata is not + /// larger than `isize::MAX`. + meta: T::PointerMetadata, + _p: PhantomData<T>, + } + + impl<T: ?Sized + KnownLayout> Copy for MetadataOf<T> {} + impl<T: ?Sized + KnownLayout> Clone for MetadataOf<T> { + #[inline] + fn clone(&self) -> Self { + *self + } + } + + impl<T: ?Sized + KnownLayout> core::fmt::Debug for MetadataOf<T> + where + T::PointerMetadata: core::fmt::Debug, + { + #[inline] + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("MetadataOf").field("meta", &self.meta).finish() + } + } + + impl<T: ?Sized> MetadataOf<T> + where + T: KnownLayout, + { + /// Returns `None` if `meta` is greater than `t`'s metadata. + #[inline(always)] + pub(crate) fn new_in_bounds(t: &T, meta: usize) -> Option<Self> + where + T: KnownLayout<PointerMetadata = usize>, + { + if meta <= Ptr::from_ref(t).len() { + // SAFETY: We have checked that `meta` is not greater than `t`'s + // metadata, which, by invariant on `&T`, addresses no more than + // `isize::MAX` bytes [1][2]. + // + // [1] Per https://doc.rust-lang.org/1.85.0/std/primitive.reference.html#safety: + // + // For all types, `T: ?Sized`, and for all `t: &T` or `t: + // &mut T`, when such values cross an API boundary, the + // following invariants must generally be upheld: + // + // * `t` is non-null + // * `t` is aligned to `align_of_val(t)` + // * if `size_of_val(t) > 0`, then `t` is dereferenceable for + // `size_of_val(t)` many bytes + // + // If `t` points at address `a`, being "dereferenceable" for + // N bytes means that the memory range `[a, a + N)` is all + // contained within a single allocated object. + // + // [2] Per https://doc.rust-lang.org/1.85.0/std/ptr/index.html#allocated-object: + // + // For any allocated object with `base` address, `size`, and + // a set of `addresses`, the following are guaranteed: + // - For all addresses `a` in `addresses`, `a` is in the + // range `base .. (base + size)` (note that this requires + // `a < base + size`, not `a <= base + size`) + // - `base` is not equal to [`null()`] (i.e., the address + // with the numerical value 0) + // - `base + size <= usize::MAX` + // - `size <= isize::MAX` + Some(unsafe { Self::new_unchecked(meta) }) + } else { + None + } + } + + /// # Safety + /// + /// The size of an instance of `&T` with the given metadata is not + /// larger than `isize::MAX`. + pub(crate) unsafe fn new_unchecked(meta: T::PointerMetadata) -> Self { + // SAFETY: The caller has promised that the size of an instance of + // `&T` with the given metadata is not larger than `isize::MAX`. + Self { meta, _p: PhantomData } + } + + pub(crate) fn get(&self) -> T::PointerMetadata + where + T::PointerMetadata: Copy, + { + self.meta + } + + #[inline] + pub(crate) fn padding_needed_for(&self) -> usize + where + T: KnownLayout<PointerMetadata = usize>, + { + let trailing_slice_layout = crate::trailing_slice_layout::<T>(); + + // FIXME(#67): Remove this allow. See NumExt for more details. + #[allow( + unstable_name_collisions, + clippy::incompatible_msrv, + clippy::multiple_unsafe_ops_per_block + )] + // SAFETY: By invariant on `self`, a `&T` with metadata `self.meta` + // describes an object of size `<= isize::MAX`. This computes the + // size of such a `&T` without any trailing padding, and so neither + // the multiplication nor the addition will overflow. + let unpadded_size = unsafe { + let trailing_size = self.meta.unchecked_mul(trailing_slice_layout.elem_size); + trailing_size.unchecked_add(trailing_slice_layout.offset) + }; + + util::padding_needed_for(unpadded_size, T::LAYOUT.align) + } + + #[inline(always)] + pub(crate) fn validate_cast_and_convert_metadata( + addr: usize, + bytes_len: MetadataOf<[u8]>, + cast_type: CastType, + meta: Option<T::PointerMetadata>, + ) -> Result<(MetadataOf<T>, MetadataOf<[u8]>), MetadataCastError> { + let layout = match meta { + None => T::LAYOUT, + // This can return `Err(MetadataCastError::Size)` if the + // metadata describes an object which can't fit in an `isize`. + Some(meta) => { + if !T::is_valid_metadata(meta) { + return Err(MetadataCastError::Size); + } + let size = match T::size_for_metadata(meta) { + Some(size) => size, + // Thanks to the `!T::is_valid_metadata(meta)` check + // above, this branch is unreachable. Fortunately, the + // optimizer recognizes this, so replacing this branch + // with `unreachable_unchecked` produces no codegen + // improvements. + None => return Err(MetadataCastError::Size), + }; + DstLayout { + align: T::LAYOUT.align, + size_info: crate::SizeInfo::Sized { size }, + statically_shallow_unpadded: false, + } + } + }; + // Lemma 0: By contract on `validate_cast_and_convert_metadata`, if + // the result is `Ok(..)`, then a `&T` with `elems` trailing slice + // elements is no larger in size than `bytes_len.get()`. + let (elems, split_at) = + layout.validate_cast_and_convert_metadata(addr, bytes_len.get(), cast_type)?; + let elems = T::PointerMetadata::from_elem_count(elems); + + // For a slice DST type, if `meta` is `Some(elems)`, then we + // synthesize `layout` to describe a sized type whose size is equal + // to the size of the instance that we are asked to cast. For sized + // types, `validate_cast_and_convert_metadata` returns `elems == 0`. + // Thus, in this case, we need to use the `elems` passed by the + // caller, not the one returned by + // `validate_cast_and_convert_metadata`. + // + // Lemma 1: A `&T` with `elems` trailing slice elements is no larger + // in size than `bytes_len.get()`. Proof: + // - If `meta` is `None`, then `elems` satisfies this condition by + // Lemma 0. + // - If `meta` is `Some(meta)`, then `layout` describes an object + // whose size is equal to the size of an `&T` with `meta` + // metadata. By Lemma 0, that size is not larger than + // `bytes_len.get()`. + // + // Lemma 2: A `&T` with `elems` trailing slice elements is no larger + // than `isize::MAX` bytes. Proof: By Lemma 1, a `&T` with metadata + // `elems` is not larger in size than `bytes_len.get()`. By + // invariant on `MetadataOf<[u8]>`, a `&[u8]` with metadata + // `bytes_len` is not larger than `isize::MAX`. Because + // `size_of::<u8>()` is `1`, a `&[u8]` with metadata `bytes_len` has + // size `bytes_len.get()` bytes. Therefore, a `&T` with metadata + // `elems` has size not larger than `isize::MAX`. + let elems = meta.unwrap_or(elems); + + // SAFETY: See Lemma 2. + let elems = unsafe { MetadataOf::new_unchecked(elems) }; + + // SAFETY: Let `size` be the size of a `&T` with metadata `elems`. + // By post-condition on `validate_cast_and_convert_metadata`, one of + // the following conditions holds: + // - `split_at == size`, in which case, by Lemma 2, `split_at <= + // isize::MAX`. Since `size_of::<u8>() == 1`, a `[u8]` with + // `split_at` elems has size not larger than `isize::MAX`. + // - `split_at == bytes_len - size`. Since `bytes_len: + // MetadataOf<u8>`, and since `size` is non-negative, `split_at` + // addresses no more bytes than `bytes_len` does. Since + // `bytes_len: MetadataOf<u8>`, `bytes_len` describes a `[u8]` + // which has no more than `isize::MAX` bytes, and thus so does + // `split_at`. + let split_at = unsafe { MetadataOf::<[u8]>::new_unchecked(split_at) }; + Ok((elems, split_at)) + } + } +} + +pub use len_of::MetadataOf; + +/// Since we support multiple versions of Rust, there are often features which +/// have been stabilized in the most recent stable release which do not yet +/// exist (stably) on our MSRV. This module provides polyfills for those +/// features so that we can write more "modern" code, and just remove the +/// polyfill once our MSRV supports the corresponding feature. Without this, +/// we'd have to write worse/more verbose code and leave FIXME comments +/// sprinkled throughout the codebase to update to the new pattern once it's +/// stabilized. +/// +/// Each trait is imported as `_` at the crate root; each polyfill should "just +/// work" at usage sites. +pub(crate) mod polyfills { + use core::ptr::{self, NonNull}; + + // A polyfill for `NonNull::slice_from_raw_parts` that we can use before our + // MSRV is 1.70, when that function was stabilized. + // + // The `#[allow(unused)]` is necessary because, on sufficiently recent + // toolchain versions, `ptr.slice_from_raw_parts()` resolves to the inherent + // method rather than to this trait, and so this trait is considered unused. + // + // FIXME(#67): Once our MSRV is 1.70, remove this. + #[allow(unused)] + pub(crate) trait NonNullExt<T> { + fn slice_from_raw_parts(data: Self, len: usize) -> NonNull<[T]>; + } + + impl<T> NonNullExt<T> for NonNull<T> { + // NOTE on coverage: this will never be tested in nightly since it's a + // polyfill for a feature which has been stabilized on our nightly + // toolchain. + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + #[inline(always)] + fn slice_from_raw_parts(data: Self, len: usize) -> NonNull<[T]> { + let ptr = ptr::slice_from_raw_parts_mut(data.as_ptr(), len); + // SAFETY: `ptr` is converted from `data`, which is non-null. + unsafe { NonNull::new_unchecked(ptr) } + } + } + + // A polyfill for `Self::unchecked_sub` that we can use until methods like + // `usize::unchecked_sub` is stabilized. + // + // The `#[allow(unused)]` is necessary because, on sufficiently recent + // toolchain versions, `ptr.slice_from_raw_parts()` resolves to the inherent + // method rather than to this trait, and so this trait is considered unused. + // + // FIXME(#67): Once our MSRV is high enough, remove this. + #[allow(unused)] + pub(crate) trait NumExt { + /// Add without checking for overflow. + /// + /// # Safety + /// + /// The caller promises that the addition will not overflow. + unsafe fn unchecked_add(self, rhs: Self) -> Self; + + /// Subtract without checking for underflow. + /// + /// # Safety + /// + /// The caller promises that the subtraction will not underflow. + unsafe fn unchecked_sub(self, rhs: Self) -> Self; + + /// Multiply without checking for overflow. + /// + /// # Safety + /// + /// The caller promises that the multiplication will not overflow. + unsafe fn unchecked_mul(self, rhs: Self) -> Self; + } + + // NOTE on coverage: these will never be tested in nightly since they're + // polyfills for a feature which has been stabilized on our nightly + // toolchain. + impl NumExt for usize { + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + #[inline(always)] + unsafe fn unchecked_add(self, rhs: usize) -> usize { + match self.checked_add(rhs) { + Some(x) => x, + None => { + // SAFETY: The caller promises that the addition will not + // underflow. + unsafe { core::hint::unreachable_unchecked() } + } + } + } + + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + #[inline(always)] + unsafe fn unchecked_sub(self, rhs: usize) -> usize { + match self.checked_sub(rhs) { + Some(x) => x, + None => { + // SAFETY: The caller promises that the subtraction will not + // underflow. + unsafe { core::hint::unreachable_unchecked() } + } + } + } + + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + #[inline(always)] + unsafe fn unchecked_mul(self, rhs: usize) -> usize { + match self.checked_mul(rhs) { + Some(x) => x, + None => { + // SAFETY: The caller promises that the multiplication will + // not overflow. + unsafe { core::hint::unreachable_unchecked() } + } + } + } + } +} + +#[cfg(test)] +pub(crate) mod testutil { + use crate::*; + + /// A `T` which is aligned to at least `align_of::<A>()`. + #[derive(Default)] + pub(crate) struct Align<T, A> { + pub(crate) t: T, + _a: [A; 0], + } + + impl<T: Default, A> Align<T, A> { + pub(crate) fn set_default(&mut self) { + self.t = T::default(); + } + } + + impl<T, A> Align<T, A> { + pub(crate) const fn new(t: T) -> Align<T, A> { + Align { t, _a: [] } + } + } + + /// A `T` which is guaranteed not to satisfy `align_of::<A>()`. + /// + /// It must be the case that `align_of::<T>() < align_of::<A>()` in order + /// for this type to work properly. + #[repr(C)] + pub(crate) struct ForceUnalign<T: Unaligned, A> { + // The outer struct is aligned to `A`, and, thanks to `repr(C)`, `t` is + // placed at the minimum offset that guarantees its alignment. If + // `align_of::<T>() < align_of::<A>()`, then that offset will be + // guaranteed *not* to satisfy `align_of::<A>()`. + // + // Note that we need `T: Unaligned` in order to guarantee that there is + // no padding between `_u` and `t`. + _u: u8, + pub(crate) t: T, + _a: [A; 0], + } + + impl<T: Unaligned, A> ForceUnalign<T, A> { + pub(crate) fn new(t: T) -> ForceUnalign<T, A> { + ForceUnalign { _u: 0, t, _a: [] } + } + } + // A `u64` with alignment 8. + // + // Though `u64` has alignment 8 on some platforms, it's not guaranteed. By + // contrast, `AU64` is guaranteed to have alignment 8 on all platforms. + #[derive( + KnownLayout, + Immutable, + FromBytes, + IntoBytes, + Eq, + PartialEq, + Ord, + PartialOrd, + Default, + Debug, + Copy, + Clone, + )] + #[repr(C, align(8))] + pub(crate) struct AU64(pub(crate) u64); + + impl AU64 { + // Converts this `AU64` to bytes using this platform's endianness. + pub(crate) fn to_bytes(self) -> [u8; 8] { + crate::transmute!(self) + } + } + + impl Display for AU64 { + #[cfg_attr( + all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + coverage(off) + )] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + Display::fmt(&self.0, f) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_round_down_to_next_multiple_of_alignment() { + fn alt_impl(n: usize, align: NonZeroUsize) -> usize { + let mul = n / align.get(); + mul * align.get() + } + + for align in [1, 2, 4, 8, 16] { + for n in 0..256 { + let align = NonZeroUsize::new(align).unwrap(); + let want = alt_impl(n, align); + let got = round_down_to_next_multiple_of_alignment(n, align); + assert_eq!(got, want, "round_down_to_next_multiple_of_alignment({}, {})", n, align); + } + } + } + + #[rustversion::since(1.57.0)] + #[test] + #[should_panic] + fn test_round_down_to_next_multiple_of_alignment_zerocopy_panic_in_const_and_vec_try_reserve() { + round_down_to_next_multiple_of_alignment(0, NonZeroUsize::new(3).unwrap()); + } + #[test] + fn test_send_sync_phantom_data() { + let x = SendSyncPhantomData::<u8>::default(); + let y = x.clone(); + assert!(x == y); + assert!(x == SendSyncPhantomData::<u8>::default()); + } + + #[test] + #[allow(clippy::as_conversions)] + fn test_as_address() { + let x = 0u8; + let r = &x; + let mut x_mut = 0u8; + let rm = &mut x_mut; + let p = r as *const u8; + let pm = rm as *mut u8; + let nn = NonNull::new(p as *mut u8).unwrap(); + + assert_eq!(AsAddress::addr(r), p as usize); + assert_eq!(AsAddress::addr(rm), pm as usize); + assert_eq!(AsAddress::addr(p), p as usize); + assert_eq!(AsAddress::addr(pm), pm as usize); + assert_eq!(AsAddress::addr(nn), p as usize); + } +} diff --git a/rust/zerocopy/src/wrappers.rs b/rust/zerocopy/src/wrappers.rs new file mode 100644 index 000000000000..1a8cf2b41d55 --- /dev/null +++ b/rust/zerocopy/src/wrappers.rs @@ -0,0 +1,1034 @@ +// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT +// +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 +// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT +// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +use core::{fmt, hash::Hash}; + +use super::*; +use crate::pointer::{invariant::Valid, SizeEq, TransmuteFrom}; + +/// A type with no alignment requirement. +/// +/// An `Unalign` wraps a `T`, removing any alignment requirement. `Unalign<T>` +/// has the same size and bit validity as `T`, but not necessarily the same +/// alignment [or ABI]. This is useful if a type with an alignment requirement +/// needs to be read from a chunk of memory which provides no alignment +/// guarantees. +/// +/// Since `Unalign` has no alignment requirement, the inner `T` may not be +/// properly aligned in memory. There are five ways to access the inner `T`: +/// - by value, using [`get`] or [`into_inner`] +/// - by reference inside of a callback, using [`update`] +/// - fallibly by reference, using [`try_deref`] or [`try_deref_mut`]; these can +/// fail if the `Unalign` does not satisfy `T`'s alignment requirement at +/// runtime +/// - unsafely by reference, using [`deref_unchecked`] or +/// [`deref_mut_unchecked`]; it is the caller's responsibility to ensure that +/// the `Unalign` satisfies `T`'s alignment requirement +/// - (where `T: Unaligned`) infallibly by reference, using [`Deref::deref`] or +/// [`DerefMut::deref_mut`] +/// +/// [or ABI]: https://github.com/google/zerocopy/issues/164 +/// [`get`]: Unalign::get +/// [`into_inner`]: Unalign::into_inner +/// [`update`]: Unalign::update +/// [`try_deref`]: Unalign::try_deref +/// [`try_deref_mut`]: Unalign::try_deref_mut +/// [`deref_unchecked`]: Unalign::deref_unchecked +/// [`deref_mut_unchecked`]: Unalign::deref_mut_unchecked +/// +/// # Example +/// +/// In this example, we need `EthernetFrame` to have no alignment requirement - +/// and thus implement [`Unaligned`]. `EtherType` is `#[repr(u16)]` and so +/// cannot implement `Unaligned`. We use `Unalign` to relax `EtherType`'s +/// alignment requirement so that `EthernetFrame` has no alignment requirement +/// and can implement `Unaligned`. +/// +/// ```rust +/// use zerocopy::*; +/// # use zerocopy_derive::*; +/// # #[derive(FromBytes, KnownLayout, Immutable, Unaligned)] #[repr(C)] struct Mac([u8; 6]); +/// +/// # #[derive(PartialEq, Copy, Clone, Debug)] +/// #[derive(TryFromBytes, KnownLayout, Immutable)] +/// #[repr(u16)] +/// enum EtherType { +/// Ipv4 = 0x0800u16.to_be(), +/// Arp = 0x0806u16.to_be(), +/// Ipv6 = 0x86DDu16.to_be(), +/// # /* +/// ... +/// # */ +/// } +/// +/// #[derive(TryFromBytes, KnownLayout, Immutable, Unaligned)] +/// #[repr(C)] +/// struct EthernetFrame { +/// src: Mac, +/// dst: Mac, +/// ethertype: Unalign<EtherType>, +/// payload: [u8], +/// } +/// +/// let bytes = &[ +/// # 0, 1, 2, 3, 4, 5, +/// # 6, 7, 8, 9, 10, 11, +/// # /* +/// ... +/// # */ +/// 0x86, 0xDD, // EtherType +/// 0xDE, 0xAD, 0xBE, 0xEF // Payload +/// ][..]; +/// +/// // PANICS: Guaranteed not to panic because `bytes` is of the right +/// // length, has the right contents, and `EthernetFrame` has no +/// // alignment requirement. +/// let packet = EthernetFrame::try_ref_from_bytes(&bytes).unwrap(); +/// +/// assert_eq!(packet.ethertype.get(), EtherType::Ipv6); +/// assert_eq!(packet.payload, [0xDE, 0xAD, 0xBE, 0xEF]); +/// ``` +/// +/// # Safety +/// +/// `Unalign<T>` is guaranteed to have the same size and bit validity as `T`, +/// and to have [`UnsafeCell`]s covering the same byte ranges as `T`. +/// `Unalign<T>` is guaranteed to have alignment 1. +// NOTE: This type is sound to use with types that need to be dropped. The +// reason is that the compiler-generated drop code automatically moves all +// values to aligned memory slots before dropping them in-place. This is not +// well-documented, but it's hinted at in places like [1] and [2]. However, this +// also means that `T` must be `Sized`; unless something changes, we can never +// support unsized `T`. [3] +// +// [1] https://github.com/rust-lang/rust/issues/54148#issuecomment-420529646 +// [2] https://github.com/google/zerocopy/pull/126#discussion_r1018512323 +// [3] https://github.com/google/zerocopy/issues/209 +#[allow(missing_debug_implementations)] +#[derive(Default, Copy)] +#[cfg_attr(any(feature = "derive", test), derive(Immutable, FromBytes, IntoBytes, Unaligned))] +#[repr(C, packed)] +pub struct Unalign<T>(T); + +// We do not use `derive(KnownLayout)` on `Unalign`, because the derive is not +// smart enough to realize that `Unalign<T>` is always sized and thus emits a +// `KnownLayout` impl bounded on `T: KnownLayout.` This is overly restrictive. +impl_known_layout!(T => Unalign<T>); + +// FIXME(https://github.com/rust-lang/rust-clippy/issues/16087): Move these +// attributes below the comment once this Clippy bug is fixed. +#[cfg_attr( + all(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, any(feature = "derive", test)), + expect(unused_unsafe) +)] +#[cfg_attr( + all( + not(__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), + any(feature = "derive", test) + ), + allow(unused_unsafe) +)] +// SAFETY: +// - `Unalign<T>` promises to have alignment 1, and so we don't require that `T: +// Unaligned`. +// - `Unalign<T>` has the same bit validity as `T`, and so it is `FromZeros`, +// `FromBytes`, or `IntoBytes` exactly when `T` is as well. +// - `Immutable`: `Unalign<T>` has the same fields as `T`, so it permits +// interior mutation exactly when `T` does. +// - `TryFromBytes`: `Unalign<T>` has the same the same bit validity as `T`, so +// `T::is_bit_valid` is a sound implementation of `is_bit_valid`. +// +#[allow(clippy::multiple_unsafe_ops_per_block)] +const _: () = unsafe { + impl_or_verify!(T => Unaligned for Unalign<T>); + impl_or_verify!(T: Immutable => Immutable for Unalign<T>); + impl_or_verify!( + T: TryFromBytes => TryFromBytes for Unalign<T>; + |c| T::is_bit_valid(c.transmute::<_, _, BecauseImmutable>()) + ); + impl_or_verify!(T: FromZeros => FromZeros for Unalign<T>); + impl_or_verify!(T: FromBytes => FromBytes for Unalign<T>); + impl_or_verify!(T: IntoBytes => IntoBytes for Unalign<T>); +}; + +// Note that `Unalign: Clone` only if `T: Copy`. Since the inner `T` may not be +// aligned, there's no way to safely call `T::clone`, and so a `T: Clone` bound +// is not sufficient to implement `Clone` for `Unalign`. +impl<T: Copy> Clone for Unalign<T> { + #[inline(always)] + fn clone(&self) -> Unalign<T> { + *self + } +} + +impl<T> Unalign<T> { + /// Constructs a new `Unalign`. + #[inline(always)] + pub const fn new(val: T) -> Unalign<T> { + Unalign(val) + } + + /// Consumes `self`, returning the inner `T`. + #[inline(always)] + pub const fn into_inner(self) -> T { + // SAFETY: Since `Unalign` is `#[repr(C, packed)]`, it has the same size + // and bit validity as `T`. + // + // We do this instead of just destructuring in order to prevent + // `Unalign`'s `Drop::drop` from being run, since dropping is not + // supported in `const fn`s. + // + // FIXME(https://github.com/rust-lang/rust/issues/73255): Destructure + // instead of using unsafe. + unsafe { crate::util::transmute_unchecked(self) } + } + + /// Attempts to return a reference to the wrapped `T`, failing if `self` is + /// not properly aligned. + /// + /// If `self` does not satisfy `align_of::<T>()`, then `try_deref` returns + /// `Err`. + /// + /// If `T: Unaligned`, then `Unalign<T>` implements [`Deref`], and callers + /// may prefer [`Deref::deref`], which is infallible. + #[inline(always)] + pub fn try_deref(&self) -> Result<&T, AlignmentError<&Self, T>> { + let inner = Ptr::from_ref(self).transmute(); + match inner.try_into_aligned() { + Ok(aligned) => Ok(aligned.as_ref()), + Err(err) => Err(err.map_src( + #[inline(always)] + |src| src.into_unalign().as_ref(), + )), + } + } + + /// Attempts to return a mutable reference to the wrapped `T`, failing if + /// `self` is not properly aligned. + /// + /// If `self` does not satisfy `align_of::<T>()`, then `try_deref` returns + /// `Err`. + /// + /// If `T: Unaligned`, then `Unalign<T>` implements [`DerefMut`], and + /// callers may prefer [`DerefMut::deref_mut`], which is infallible. + #[inline(always)] + pub fn try_deref_mut(&mut self) -> Result<&mut T, AlignmentError<&mut Self, T>> { + let inner = Ptr::from_mut(self).transmute::<_, _, (_, (_, _))>(); + match inner.try_into_aligned() { + Ok(aligned) => Ok(aligned.as_mut()), + Err(err) => Err(err.map_src(|src| src.into_unalign().as_mut())), + } + } + + /// Returns a reference to the wrapped `T` without checking alignment. + /// + /// If `T: Unaligned`, then `Unalign<T>` implements[ `Deref`], and callers + /// may prefer [`Deref::deref`], which is safe. + /// + /// # Safety + /// + /// The caller must guarantee that `self` satisfies `align_of::<T>()`. + #[inline(always)] + pub const unsafe fn deref_unchecked(&self) -> &T { + // SAFETY: `Unalign<T>` is `repr(transparent)`, so there is a valid `T` + // at the same memory location as `self`. It has no alignment guarantee, + // but the caller has promised that `self` is properly aligned, so we + // know that it is sound to create a reference to `T` at this memory + // location. + // + // We use `mem::transmute` instead of `&*self.get_ptr()` because + // dereferencing pointers is not stable in `const` on our current MSRV + // (1.56 as of this writing). + unsafe { mem::transmute(self) } + } + + /// Returns a mutable reference to the wrapped `T` without checking + /// alignment. + /// + /// If `T: Unaligned`, then `Unalign<T>` implements[ `DerefMut`], and + /// callers may prefer [`DerefMut::deref_mut`], which is safe. + /// + /// # Safety + /// + /// The caller must guarantee that `self` satisfies `align_of::<T>()`. + #[inline(always)] + pub unsafe fn deref_mut_unchecked(&mut self) -> &mut T { + // SAFETY: `self.get_mut_ptr()` returns a raw pointer to a valid `T` at + // the same memory location as `self`. It has no alignment guarantee, + // but the caller has promised that `self` is properly aligned, so we + // know that the pointer itself is aligned, and thus that it is sound to + // create a reference to a `T` at this memory location. + unsafe { &mut *self.get_mut_ptr() } + } + + /// Gets an unaligned raw pointer to the inner `T`. + /// + /// # Safety + /// + /// The returned raw pointer is not necessarily aligned to + /// `align_of::<T>()`. Most functions which operate on raw pointers require + /// those pointers to be aligned, so calling those functions with the result + /// of `get_ptr` will result in undefined behavior if alignment is not + /// guaranteed using some out-of-band mechanism. In general, the only + /// functions which are safe to call with this pointer are those which are + /// explicitly documented as being sound to use with an unaligned pointer, + /// such as [`read_unaligned`]. + /// + /// Even if the caller is permitted to mutate `self` (e.g. they have + /// ownership or a mutable borrow), it is not guaranteed to be sound to + /// write through the returned pointer. If writing is required, prefer + /// [`get_mut_ptr`] instead. + /// + /// [`read_unaligned`]: core::ptr::read_unaligned + /// [`get_mut_ptr`]: Unalign::get_mut_ptr + #[inline(always)] + pub const fn get_ptr(&self) -> *const T { + ptr::addr_of!(self.0) + } + + /// Gets an unaligned mutable raw pointer to the inner `T`. + /// + /// # Safety + /// + /// The returned raw pointer is not necessarily aligned to + /// `align_of::<T>()`. Most functions which operate on raw pointers require + /// those pointers to be aligned, so calling those functions with the result + /// of `get_ptr` will result in undefined behavior if alignment is not + /// guaranteed using some out-of-band mechanism. In general, the only + /// functions which are safe to call with this pointer are those which are + /// explicitly documented as being sound to use with an unaligned pointer, + /// such as [`read_unaligned`]. + /// + /// [`read_unaligned`]: core::ptr::read_unaligned + // FIXME(https://github.com/rust-lang/rust/issues/57349): Make this `const`. + #[inline(always)] + pub fn get_mut_ptr(&mut self) -> *mut T { + ptr::addr_of_mut!(self.0) + } + + /// Sets the inner `T`, dropping the previous value. + // FIXME(https://github.com/rust-lang/rust/issues/57349): Make this `const`. + #[inline(always)] + pub fn set(&mut self, t: T) { + *self = Unalign::new(t); + } + + /// Updates the inner `T` by calling a function on it. + /// + /// If [`T: Unaligned`], then `Unalign<T>` implements [`DerefMut`], and that + /// impl should be preferred over this method when performing updates, as it + /// will usually be faster and more ergonomic. + /// + /// For large types, this method may be expensive, as it requires copying + /// `2 * size_of::<T>()` bytes. \[1\] + /// + /// \[1\] Since the inner `T` may not be aligned, it would not be sound to + /// invoke `f` on it directly. Instead, `update` moves it into a + /// properly-aligned location in the local stack frame, calls `f` on it, and + /// then moves it back to its original location in `self`. + /// + /// [`T: Unaligned`]: Unaligned + #[inline] + pub fn update<O, F: FnOnce(&mut T) -> O>(&mut self, f: F) -> O { + if mem::align_of::<T>() == 1 { + // While we advise callers to use `DerefMut` when `T: Unaligned`, + // not all callers will be able to guarantee `T: Unaligned` in all + // cases. In particular, callers who are themselves providing an API + // which is generic over `T` may sometimes be called by *their* + // callers with `T` such that `align_of::<T>() == 1`, but cannot + // guarantee this in the general case. Thus, this optimization may + // sometimes be helpful. + + // SAFETY: Since `T`'s alignment is 1, `self` satisfies its + // alignment by definition. + let t = unsafe { self.deref_mut_unchecked() }; + return f(t); + } + + // On drop, this moves `copy` out of itself and uses `ptr::write` to + // overwrite `slf`. + struct WriteBackOnDrop<T> { + copy: ManuallyDrop<T>, + slf: *mut Unalign<T>, + } + + impl<T> Drop for WriteBackOnDrop<T> { + fn drop(&mut self) { + // SAFETY: We never use `copy` again as required by + // `ManuallyDrop::take`. + let copy = unsafe { ManuallyDrop::take(&mut self.copy) }; + // SAFETY: `slf` is the raw pointer value of `self`. We know it + // is valid for writes and properly aligned because `self` is a + // mutable reference, which guarantees both of these properties. + unsafe { ptr::write(self.slf, Unalign::new(copy)) }; + } + } + + // SAFETY: We know that `self` is valid for reads, properly aligned, and + // points to an initialized `Unalign<T>` because it is a mutable + // reference, which guarantees all of these properties. + // + // Since `T: !Copy`, it would be unsound in the general case to allow + // both the original `Unalign<T>` and the copy to be used by safe code. + // We guarantee that the copy is used to overwrite the original in the + // `Drop::drop` impl of `WriteBackOnDrop`. So long as this `drop` is + // called before any other safe code executes, soundness is upheld. + // While this method can terminate in two ways (by returning normally or + // by unwinding due to a panic in `f`), in both cases, `write_back` is + // dropped - and its `drop` called - before any other safe code can + // execute. + let copy = unsafe { ptr::read(self) }.into_inner(); + let mut write_back = WriteBackOnDrop { copy: ManuallyDrop::new(copy), slf: self }; + + let ret = f(&mut write_back.copy); + + drop(write_back); + ret + } +} + +impl<T: Copy> Unalign<T> { + /// Gets a copy of the inner `T`. + // FIXME(https://github.com/rust-lang/rust/issues/57349): Make this `const`. + #[inline(always)] + pub fn get(&self) -> T { + let Unalign(val) = *self; + val + } +} + +impl<T: Unaligned> Deref for Unalign<T> { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &T { + Ptr::from_ref(self).transmute().bikeshed_recall_aligned().as_ref() + } +} + +impl<T: Unaligned> DerefMut for Unalign<T> { + #[inline(always)] + fn deref_mut(&mut self) -> &mut T { + Ptr::from_mut(self).transmute::<_, _, (_, (_, _))>().bikeshed_recall_aligned().as_mut() + } +} + +impl<T: Unaligned + PartialOrd> PartialOrd<Unalign<T>> for Unalign<T> { + #[inline(always)] + fn partial_cmp(&self, other: &Unalign<T>) -> Option<Ordering> { + PartialOrd::partial_cmp(self.deref(), other.deref()) + } +} + +impl<T: Unaligned + Ord> Ord for Unalign<T> { + #[inline(always)] + fn cmp(&self, other: &Unalign<T>) -> Ordering { + Ord::cmp(self.deref(), other.deref()) + } +} + +impl<T: Unaligned + PartialEq> PartialEq<Unalign<T>> for Unalign<T> { + #[inline(always)] + fn eq(&self, other: &Unalign<T>) -> bool { + PartialEq::eq(self.deref(), other.deref()) + } +} + +impl<T: Unaligned + Eq> Eq for Unalign<T> {} + +impl<T: Unaligned + Hash> Hash for Unalign<T> { + #[inline(always)] + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.deref().hash(state); + } +} + +impl<T: Unaligned + Debug> Debug for Unalign<T> { + #[inline(always)] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + Debug::fmt(self.deref(), f) + } +} + +impl<T: Unaligned + Display> Display for Unalign<T> { + #[inline(always)] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + Display::fmt(self.deref(), f) + } +} + +/// A wrapper type to construct uninitialized instances of `T`. +/// +/// `MaybeUninit` is identical to the [standard library +/// `MaybeUninit`][core-maybe-uninit] type except that it supports unsized +/// types. +/// +/// # Layout +/// +/// The same layout guarantees and caveats apply to `MaybeUninit<T>` as apply to +/// the [standard library `MaybeUninit`][core-maybe-uninit] with one exception: +/// for `T: !Sized`, there is no single value for `T`'s size. Instead, for such +/// types, the following are guaranteed: +/// - Every [valid size][valid-size] for `T` is a valid size for +/// `MaybeUninit<T>` and vice versa +/// - Given `t: *const T` and `m: *const MaybeUninit<T>` with identical fat +/// pointer metadata, `t` and `m` address the same number of bytes (and +/// likewise for `*mut`) +/// +/// [core-maybe-uninit]: core::mem::MaybeUninit +/// [valid-size]: crate::KnownLayout#what-is-a-valid-size +#[repr(transparent)] +#[doc(hidden)] +pub struct MaybeUninit<T: ?Sized + KnownLayout>( + // SAFETY: `MaybeUninit<T>` has the same size as `T`, because (by invariant + // on `T::MaybeUninit`) `T::MaybeUninit` has `T::LAYOUT` identical to `T`, + // and because (invariant on `T::LAYOUT`) we can trust that `LAYOUT` + // accurately reflects the layout of `T`. By invariant on `T::MaybeUninit`, + // it admits uninitialized bytes in all positions. Because `MaybeUninit` is + // marked `repr(transparent)`, these properties additionally hold true for + // `Self`. + T::MaybeUninit, +); + +#[doc(hidden)] +impl<T: ?Sized + KnownLayout> MaybeUninit<T> { + /// Constructs a `MaybeUninit<T>` initialized with the given value. + #[inline(always)] + pub fn new(val: T) -> Self + where + T: Sized, + Self: Sized, + { + // SAFETY: It is valid to transmute `val` to `MaybeUninit<T>` because it + // is both valid to transmute `val` to `T::MaybeUninit`, and it is valid + // to transmute from `T::MaybeUninit` to `MaybeUninit<T>`. + // + // First, it is valid to transmute `val` to `T::MaybeUninit` because, by + // invariant on `T::MaybeUninit`: + // - For `T: Sized`, `T` and `T::MaybeUninit` have the same size. + // - All byte sequences of the correct size are valid values of + // `T::MaybeUninit`. + // + // Second, it is additionally valid to transmute from `T::MaybeUninit` + // to `MaybeUninit<T>`, because `MaybeUninit<T>` is a + // `repr(transparent)` wrapper around `T::MaybeUninit`. + // + // These two transmutes are collapsed into one so we don't need to add a + // `T::MaybeUninit: Sized` bound to this function's `where` clause. + unsafe { crate::util::transmute_unchecked(val) } + } + + /// Constructs an uninitialized `MaybeUninit<T>`. + #[must_use] + #[inline(always)] + pub fn uninit() -> Self + where + T: Sized, + Self: Sized, + { + let uninit = CoreMaybeUninit::<T>::uninit(); + // SAFETY: It is valid to transmute from `CoreMaybeUninit<T>` to + // `MaybeUninit<T>` since they both admit uninitialized bytes in all + // positions, and they have the same size (i.e., that of `T`). + // + // `MaybeUninit<T>` has the same size as `T`, because (by invariant on + // `T::MaybeUninit`) `T::MaybeUninit` has `T::LAYOUT` identical to `T`, + // and because (invariant on `T::LAYOUT`) we can trust that `LAYOUT` + // accurately reflects the layout of `T`. + // + // `CoreMaybeUninit<T>` has the same size as `T` [1] and admits + // uninitialized bytes in all positions. + // + // [1] Per https://doc.rust-lang.org/1.81.0/std/mem/union.MaybeUninit.html#layout-1: + // + // `MaybeUninit<T>` is guaranteed to have the same size, alignment, + // and ABI as `T` + unsafe { crate::util::transmute_unchecked(uninit) } + } + + /// Creates a `Box<MaybeUninit<T>>`. + /// + /// This function is useful for allocating large, uninit values on the heap + /// without ever creating a temporary instance of `Self` on the stack. + /// + /// # Errors + /// + /// Returns an error on allocation failure. Allocation failure is guaranteed + /// never to cause a panic or an abort. + #[cfg(feature = "alloc")] + #[inline] + pub fn new_boxed_uninit(meta: T::PointerMetadata) -> Result<Box<Self>, AllocError> { + // SAFETY: `alloc::alloc::alloc_zeroed` is a valid argument of + // `new_box`. The referent of the pointer returned by `alloc` (and, + // consequently, the `Box` derived from it) is a valid instance of + // `Self`, because `Self` is `MaybeUninit` and thus admits arbitrary + // (un)initialized bytes. + unsafe { crate::util::new_box(meta, alloc::alloc::alloc) } + } + + /// Extracts the value from the `MaybeUninit<T>` container. + /// + /// # Safety + /// + /// The caller must ensure that `self` is in an bit-valid state. Depending + /// on subsequent use, it may also need to be in a library-valid state. + #[inline(always)] + pub unsafe fn assume_init(self) -> T + where + T: Sized, + Self: Sized, + { + // SAFETY: The caller guarantees that `self` is in an bit-valid state. + unsafe { crate::util::transmute_unchecked(self) } + } +} + +impl<T: ?Sized + KnownLayout> fmt::Debug for MaybeUninit<T> { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad(core::any::type_name::<Self>()) + } +} + +#[allow(unreachable_pub)] // False positive on MSRV +#[doc(hidden)] +pub use read_only_def::*; +mod read_only_def { + /// A read-only wrapper. + /// + /// A `ReadOnly<T>` disables any interior mutability in `T`, ensuring that + /// a `&ReadOnly<T>` is genuinely read-only. Thus, `ReadOnly<T>` is + /// [`Immutable`] regardless of whether `T` is. + /// + /// Note that `&mut ReadOnly<T>` still permits mutation – the read-only + /// property only applies to shared references. + /// + /// [`Immutable`]: crate::Immutable + #[repr(transparent)] + pub struct ReadOnly<T: ?Sized> { + // INVARIANT: `inner` is never mutated through a `&ReadOnly<T>` + // reference. + inner: T, + } + + impl<T> ReadOnly<T> { + /// Creates a new `ReadOnly`. + #[must_use] + #[inline(always)] + pub const fn new(t: T) -> ReadOnly<T> { + ReadOnly { inner: t } + } + + /// Returns the inner value. + #[must_use] + #[inline(always)] + pub fn into_inner(r: ReadOnly<T>) -> T { + r.inner + } + } + + impl<T: ?Sized> ReadOnly<T> { + #[inline(always)] + pub(crate) fn as_mut(r: &mut ReadOnly<T>) -> &mut T { + // SAFETY: `r: &mut ReadOnly`, so this doesn't violate the invariant + // that `inner` is never mutated through a `&ReadOnly<T>` reference. + &mut r.inner + } + + /// # Safety + /// + /// The caller promises not to mutate the referent (i.e., via interior + /// mutation). + pub(crate) const unsafe fn as_ref_unchecked(r: &ReadOnly<T>) -> &T { + // SAFETY: The caller promises not to mutate the referent. + &r.inner + } + } +} + +// SAFETY: `ReadOnly<T>` is a `#[repr(transparent)` wrapper around `T`. +const _: () = unsafe { + unsafe_impl_known_layout!(T: ?Sized + KnownLayout => #[repr(T)] ReadOnly<T>); +}; + +#[allow(clippy::multiple_unsafe_ops_per_block)] +// SAFETY: +// - `ReadOnly<T>` has the same alignment as `T`, and so it is `Unaligned` +// exactly when `T` is as well. +// - `ReadOnly<T>` has the same bit validity as `T`, and so this `is_bit_valid` +// implementation is correct, and thus the `TryFromBytes` impl is sound. +// - `ReadOnly<T>` has the same bit validity as `T`, and so it is `FromZeros`, +// `FromBytes`, and `IntoBytes` exactly when `T` is as well. +const _: () = unsafe { + unsafe_impl!(T: ?Sized + Unaligned => Unaligned for ReadOnly<T>); + unsafe_impl!( + T: ?Sized + TryFromBytes => TryFromBytes for ReadOnly<T>; + |c| T::is_bit_valid(c.cast::<_, <ReadOnly<T> as SizeEq<ReadOnly<ReadOnly<T>>>>::CastFrom, _>()) + ); + unsafe_impl!(T: ?Sized + FromZeros => FromZeros for ReadOnly<T>); + unsafe_impl!(T: ?Sized + FromBytes => FromBytes for ReadOnly<T>); + unsafe_impl!(T: ?Sized + IntoBytes => IntoBytes for ReadOnly<T>); +}; + +// SAFETY: By invariant, `inner` is never mutated through a `&ReadOnly<T>` +// reference. +const _: () = unsafe { + unsafe_impl!(T: ?Sized => Immutable for ReadOnly<T>); +}; + +const _: () = { + use crate::pointer::cast::CastExact; + + // SAFETY: `ReadOnly<T>` has the same layout as `T`. + define_cast!(unsafe { pub CastFromReadOnly<T: ?Sized> = ReadOnly<T> => T}); + // SAFETY: `ReadOnly<T>` has the same layout as `T`. + unsafe impl<T: ?Sized> CastExact<ReadOnly<T>, T> for CastFromReadOnly {} + // SAFETY: `ReadOnly<T>` has the same layout as `T`. + define_cast!(unsafe { pub CastToReadOnly<T: ?Sized> = T => ReadOnly<T>}); + // SAFETY: `ReadOnly<T>` has the same layout as `T`. + unsafe impl<T: ?Sized> CastExact<T, ReadOnly<T>> for CastToReadOnly {} + + impl<T: ?Sized> SizeEq<ReadOnly<T>> for T { + type CastFrom = CastFromReadOnly; + } + + impl<T: ?Sized> SizeEq<T> for ReadOnly<T> { + type CastFrom = CastToReadOnly; + } +}; + +// SAFETY: `ReadOnly<T>` is a `#[repr(transparent)]` wrapper around `T`, and so +// it has the same bit validity as `T`. +unsafe impl<T: ?Sized> TransmuteFrom<T, Valid, Valid> for ReadOnly<T> {} + +// SAFETY: `ReadOnly<T>` is a `#[repr(transparent)]` wrapper around `T`, and so +// it has the same bit validity as `T`. +unsafe impl<T: ?Sized> TransmuteFrom<ReadOnly<T>, Valid, Valid> for T {} + +impl<'a, T: ?Sized + Immutable> From<&'a T> for &'a ReadOnly<T> { + #[inline(always)] + fn from(t: &'a T) -> &'a ReadOnly<T> { + let ro = Ptr::from_ref(t).transmute::<_, _, (_, _)>(); + // SAFETY: `ReadOnly<T>` has the same alignment as `T`, and + // `Ptr::from_ref` produces an aligned `Ptr`. + let ro = unsafe { ro.assume_alignment() }; + ro.as_ref() + } +} + +impl<T: ?Sized + Immutable> Deref for ReadOnly<T> { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &Self::Target { + // SAFETY: By `T: Immutable`, `&T` doesn't permit interior mutation. + unsafe { ReadOnly::as_ref_unchecked(self) } + } +} + +impl<T: ?Sized + Immutable> DerefMut for ReadOnly<T> { + #[inline(always)] + fn deref_mut(&mut self) -> &mut Self::Target { + ReadOnly::as_mut(self) + } +} + +impl<T: ?Sized + Immutable + Debug> Debug for ReadOnly<T> { + #[inline(always)] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + self.deref().fmt(f) + } +} + +// SAFETY: See safety comment on `ProjectToTag`. +unsafe impl<T: HasTag + ?Sized> HasTag for ReadOnly<T> { + #[allow(clippy::missing_inline_in_public_items)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type Tag = T::Tag; + + // SAFETY: `<T as SizeEq<ReadOnly<T>>>::CastFrom` is a no-op projection that + // produces a pointer with the same referent. By invariant, for any `Ptr<'_, + // T, I>` it is sound to use `T::ProjectToTag` to project to a `Ptr<'_, + // T::Tag, I>`. Since `ReadOnly<T>` has the same layout and validity as `T`, + // the same is true of projecting from a `Ptr<'_, ReadOnly<T>, I>`. + type ProjectToTag = crate::pointer::cast::TransitiveProject< + T, + <T as SizeEq<ReadOnly<T>>>::CastFrom, + T::ProjectToTag, + >; +} + +// SAFETY: `ReadOnly<T>` is a `#[repr(transparent)]` wrapper around `T`, and so +// has the same fields at the same offsets. Thus, it satisfies the safety +// invariants of `HasField<Field, VARIANT_ID, FIELD_ID>` for field `f` exactly +// when `T` does, as guaranteed by the `T: HasField` bound: +// - If `VARIANT_ID` is `STRUCT_VARIANT_ID` or `UNION_VARIANT_ID`, then `T` has +// the layout of a struct or union type. Since `ReadOnly<T>` is a transparent +// wrapper around `T`, it does too. Otherwise, if `VARIANT_ID` is an enum +// variant index, then `T` has the layout of an enum type, and `ReadOnly<T>` +// does too. +// - By `T: HasField<_, _, FIELD_ID>`: +// - `T` has a field `f` with name `n` such that +// `FIELD_ID = zerocopy::ident_id!(n)` or at index `i` such that +// `FIELD_ID = zerocopy::ident_id!(i)`. +// - `Field` has the same visibility as `f`. +// - `T::Type` has the same type as `f`. Thus, `ReadOnly<T::Type>` has the +// same type as `f`, wrapped in `ReadOnly`. +// +// `project` satisfies its post-condition – namely, that the returned pointer +// refers to a non-strict subset of the bytes of `slf`'s referent, and has the +// same provenance as `slf` – because all intermediate operations satisfy those +// same conditions. +unsafe impl<T, Field, const VARIANT_ID: i128, const FIELD_ID: i128> + HasField<Field, VARIANT_ID, FIELD_ID> for ReadOnly<T> +where + T: HasField<Field, VARIANT_ID, FIELD_ID> + ?Sized, +{ + #[allow(clippy::missing_inline_in_public_items)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type Type = ReadOnly<T::Type>; + + #[inline(always)] + fn project(slf: PtrInner<'_, Self>) -> *mut ReadOnly<T::Type> { + slf.project::<_, <T as SizeEq<ReadOnly<T>>>::CastFrom>() + .project::<_, crate::pointer::cast::Projection<Field, VARIANT_ID, FIELD_ID>>() + .project::<_, <ReadOnly<T::Type> as SizeEq<T::Type>>::CastFrom>() + .as_non_null() + .as_ptr() + } +} + +// SAFETY: `ReadOnly<T>` is a `#[repr(transparent)]` wrapper around `T`, and so +// has the same fields at the same offsets. `is_projectable` simply delegates to +// `T::is_projectable`, which is sound because a `Ptr<'_, ReadOnly<T>, I>` will +// be projectable exactly when a `Ptr<'_, T, I>` referent is. +unsafe impl<T, Field, I, const VARIANT_ID: i128, const FIELD_ID: i128> + ProjectField<Field, I, VARIANT_ID, FIELD_ID> for ReadOnly<T> +where + T: ProjectField<Field, I, VARIANT_ID, FIELD_ID> + ?Sized, + I: invariant::Invariants, +{ + #[allow(clippy::missing_inline_in_public_items)] + fn only_derive_is_allowed_to_implement_this_trait() + where + Self: Sized, + { + } + + type Invariants = T::Invariants; + + type Error = T::Error; + + #[inline(always)] + fn is_projectable<'a>(ptr: Ptr<'a, Self::Tag, I>) -> Result<(), Self::Error> { + T::is_projectable(ptr) + } +} + +#[cfg(test)] +mod tests { + use core::panic::AssertUnwindSafe; + + use super::*; + use crate::util::testutil::*; + + #[test] + fn test_unalign() { + // Test methods that don't depend on alignment. + let mut u = Unalign::new(AU64(123)); + assert_eq!(u.get(), AU64(123)); + assert_eq!(u.into_inner(), AU64(123)); + assert_eq!(u.get_ptr(), <*const _>::cast::<AU64>(&u)); + assert_eq!(u.get_mut_ptr(), <*mut _>::cast::<AU64>(&mut u)); + u.set(AU64(321)); + assert_eq!(u.get(), AU64(321)); + + // Test methods that depend on alignment (when alignment is satisfied). + let mut u: Align<_, AU64> = Align::new(Unalign::new(AU64(123))); + assert_eq!(u.t.try_deref().unwrap(), &AU64(123)); + assert_eq!(u.t.try_deref_mut().unwrap(), &mut AU64(123)); + // SAFETY: The `Align<_, AU64>` guarantees proper alignment. + assert_eq!(unsafe { u.t.deref_unchecked() }, &AU64(123)); + // SAFETY: The `Align<_, AU64>` guarantees proper alignment. + assert_eq!(unsafe { u.t.deref_mut_unchecked() }, &mut AU64(123)); + *u.t.try_deref_mut().unwrap() = AU64(321); + assert_eq!(u.t.get(), AU64(321)); + + // Test methods that depend on alignment (when alignment is not + // satisfied). + let mut u: ForceUnalign<_, AU64> = ForceUnalign::new(Unalign::new(AU64(123))); + assert!(matches!(u.t.try_deref(), Err(AlignmentError { .. }))); + assert!(matches!(u.t.try_deref_mut(), Err(AlignmentError { .. }))); + + // Test methods that depend on `T: Unaligned`. + let mut u = Unalign::new(123u8); + assert_eq!(u.try_deref(), Ok(&123)); + assert_eq!(u.try_deref_mut(), Ok(&mut 123)); + assert_eq!(u.deref(), &123); + assert_eq!(u.deref_mut(), &mut 123); + *u = 21; + assert_eq!(u.get(), 21); + + // Test that some `Unalign` functions and methods are `const`. + const _UNALIGN: Unalign<u64> = Unalign::new(0); + const _UNALIGN_PTR: *const u64 = _UNALIGN.get_ptr(); + const _U64: u64 = _UNALIGN.into_inner(); + // Make sure all code is considered "used". + // + // FIXME(https://github.com/rust-lang/rust/issues/104084): Remove this + // attribute. + #[allow(dead_code)] + const _: () = { + let x: Align<_, AU64> = Align::new(Unalign::new(AU64(123))); + // Make sure that `deref_unchecked` is `const`. + // + // SAFETY: The `Align<_, AU64>` guarantees proper alignment. + let au64 = unsafe { x.t.deref_unchecked() }; + match au64 { + AU64(123) => {} + _ => const_unreachable!(), + } + }; + } + + #[test] + fn test_unalign_update() { + let mut u = Unalign::new(AU64(123)); + u.update(|a| a.0 += 1); + assert_eq!(u.get(), AU64(124)); + + // Test that, even if the callback panics, the original is still + // correctly overwritten. Use a `Box` so that Miri is more likely to + // catch any unsoundness (which would likely result in two `Box`es for + // the same heap object, which is the sort of thing that Miri would + // probably catch). + let mut u = Unalign::new(Box::new(AU64(123))); + let res = std::panic::catch_unwind(AssertUnwindSafe(|| { + u.update(|a| { + a.0 += 1; + panic!(); + }) + })); + assert!(res.is_err()); + assert_eq!(u.into_inner(), Box::new(AU64(124))); + + // Test the align_of::<T>() == 1 optimization. + let mut u = Unalign::new([0u8, 1]); + u.update(|a| a[0] += 1); + assert_eq!(u.get(), [1u8, 1]); + } + + #[test] + fn test_unalign_copy_clone() { + // Test that `Copy` and `Clone` do not cause soundness issues. This test + // is mainly meant to exercise UB that would be caught by Miri. + + // `u.t` is definitely not validly-aligned for `AU64`'s alignment of 8. + let u = ForceUnalign::<_, AU64>::new(Unalign::new(AU64(123))); + #[allow(clippy::clone_on_copy)] + let v = u.t.clone(); + let w = u.t; + assert_eq!(u.t.get(), v.get()); + assert_eq!(u.t.get(), w.get()); + assert_eq!(v.get(), w.get()); + } + + #[test] + fn test_unalign_trait_impls() { + let zero = Unalign::new(0u8); + let one = Unalign::new(1u8); + + assert!(zero < one); + assert_eq!(PartialOrd::partial_cmp(&zero, &one), Some(Ordering::Less)); + assert_eq!(Ord::cmp(&zero, &one), Ordering::Less); + + assert_ne!(zero, one); + assert_eq!(zero, zero); + assert!(!PartialEq::eq(&zero, &one)); + assert!(PartialEq::eq(&zero, &zero)); + + fn hash<T: Hash>(t: &T) -> u64 { + let mut h = std::collections::hash_map::DefaultHasher::new(); + t.hash(&mut h); + h.finish() + } + + assert_eq!(hash(&zero), hash(&0u8)); + assert_eq!(hash(&one), hash(&1u8)); + + assert_eq!(format!("{:?}", zero), format!("{:?}", 0u8)); + assert_eq!(format!("{:?}", one), format!("{:?}", 1u8)); + assert_eq!(format!("{}", zero), format!("{}", 0u8)); + assert_eq!(format!("{}", one), format!("{}", 1u8)); + } + + #[test] + #[allow(clippy::as_conversions)] + fn test_maybe_uninit() { + // int + { + let input = 42; + let uninit = MaybeUninit::new(input); + // SAFETY: `uninit` is in an initialized state + let output = unsafe { uninit.assume_init() }; + assert_eq!(input, output); + } + + // thin ref + { + let input = 42; + let uninit = MaybeUninit::new(&input); + // SAFETY: `uninit` is in an initialized state + let output = unsafe { uninit.assume_init() }; + assert_eq!(&input as *const _, output as *const _); + assert_eq!(input, *output); + } + + // wide ref + { + let input = [1, 2, 3, 4]; + let uninit = MaybeUninit::new(&input[..]); + // SAFETY: `uninit` is in an initialized state + let output = unsafe { uninit.assume_init() }; + assert_eq!(&input[..] as *const _, output as *const _); + assert_eq!(input, *output); + } + } + #[test] + fn test_maybe_uninit_uninit() { + let _uninit = MaybeUninit::<u8>::uninit(); + // Cannot check value, but can check it compiles and runs + } + + #[test] + #[cfg(feature = "alloc")] + fn test_maybe_uninit_new_boxed_uninit() { + let _boxed = MaybeUninit::<u8>::new_boxed_uninit(()).unwrap(); + } + + #[test] + fn test_maybe_uninit_debug() { + let uninit = MaybeUninit::<u8>::uninit(); + assert!(format!("{:?}", uninit).contains("MaybeUninit")); + } +} |