From b7a124928b5875675539b6a6f708f56911f271f7 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Thu, 22 Jan 2026 16:43:02 -0500 Subject: rust/drm: Fixup import styles This is to match https://docs.kernel.org/rust/coding-guidelines.html#imports There should be no functional changes in this patch. Signed-off-by: Lyude Paul Link: https://patch.msgid.link/20260122214316.3281257-1-lyude@redhat.com [ Move trailing `//` at the end. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/drm/device.rs | 31 ++++++++++++++++++++++++------- rust/kernel/drm/driver.rs | 12 +++++++++--- rust/kernel/drm/file.rs | 14 +++++++++++--- rust/kernel/drm/gem/mod.rs | 25 ++++++++++++++++++++----- 4 files changed, 64 insertions(+), 18 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 3ce8f62a0056..ae123ffece79 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -6,15 +6,32 @@ use crate::{ alloc::allocator::Kmalloc, - bindings, device, drm, - drm::driver::AllocImpl, - error::from_err_ptr, - error::Result, + bindings, + device, + drm::{ + self, + driver::AllocImpl, // + }, + error::{ + from_err_ptr, + Result, // + }, prelude::*, - sync::aref::{ARef, AlwaysRefCounted}, - types::Opaque, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, // +}; +use core::{ + alloc::Layout, + mem, + ops::Deref, + ptr::{ + self, + NonNull, // + }, }; -use core::{alloc::Layout, mem, ops::Deref, ptr, ptr::NonNull}; #[cfg(CONFIG_DRM_LEGACY)] macro_rules! drm_legacy_fields { diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index e09f977b5b51..6bd097f16ce7 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -5,10 +5,16 @@ //! C header: [`include/drm/drm_drv.h`](srctree/include/drm/drm_drv.h) use crate::{ - bindings, device, devres, drm, - error::{to_result, Result}, + bindings, + device, + devres, + drm, + error::{ + to_result, + Result, // + }, prelude::*, - sync::aref::ARef, + sync::aref::ARef, // }; use macros::vtable; diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 8c46f8d51951..7dade6dfa1ba 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -4,9 +4,17 @@ //! //! C header: [`include/drm/drm_file.h`](srctree/include/drm/drm_file.h) -use crate::{bindings, drm, error::Result, prelude::*, types::Opaque}; -use core::marker::PhantomData; -use core::pin::Pin; +use crate::{ + bindings, + drm, + error::Result, + prelude::*, + types::Opaque, // +}; +use core::{ + marker::PhantomData, + pin::Pin, // +}; /// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance). pub trait DriverFile { diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index d49a9ba02635..56b7641b1405 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -6,14 +6,29 @@ use crate::{ alloc::flags::*, - bindings, drm, - drm::driver::{AllocImpl, AllocOps}, - error::{to_result, Result}, + bindings, + drm::{ + self, + driver::{ + AllocImpl, + AllocOps, // + }, + }, + error::{ + to_result, + Result, // + }, prelude::*, - sync::aref::{ARef, AlwaysRefCounted}, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, types::Opaque, }; -use core::{ops::Deref, ptr::NonNull}; +use core::{ + ops::Deref, + ptr::NonNull, // +}; /// A type alias for retrieving a [`Driver`]s [`DriverFile`] implementation from its /// [`DriverObject`] implementation. -- cgit v1.2.3 From 6ad005ce6994599e2ae338007a54dd21063aae15 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Thu, 22 Jan 2026 16:43:03 -0500 Subject: rust/drm: Remove imports covered by prelude::* This just removes any explicit imports of items in files that are already being pulled in by `use prelude::*;`. There should be no functional changes in this patch. Signed-off-by: Lyude Paul Link: https://patch.msgid.link/20260122214316.3281257-2-lyude@redhat.com Signed-off-by: Danilo Krummrich --- rust/kernel/drm/device.rs | 5 +---- rust/kernel/drm/driver.rs | 6 +----- rust/kernel/drm/file.rs | 6 +----- rust/kernel/drm/gem/mod.rs | 6 +----- 4 files changed, 4 insertions(+), 19 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index ae123ffece79..629ef0bd1188 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -12,10 +12,7 @@ use crate::{ self, driver::AllocImpl, // }, - error::{ - from_err_ptr, - Result, // - }, + error::from_err_ptr, prelude::*, sync::aref::{ ARef, diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 6bd097f16ce7..5233bdebc9fc 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -9,14 +9,10 @@ use crate::{ device, devres, drm, - error::{ - to_result, - Result, // - }, + error::to_result, prelude::*, sync::aref::ARef, // }; -use macros::vtable; /// Driver use the GEM memory manager. This should be set for all modern drivers. pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 7dade6dfa1ba..10160601ce5a 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -7,14 +7,10 @@ use crate::{ bindings, drm, - error::Result, prelude::*, types::Opaque, // }; -use core::{ - marker::PhantomData, - pin::Pin, // -}; +use core::marker::PhantomData; /// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance). pub trait DriverFile { diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 56b7641b1405..b4199945db37 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -5,7 +5,6 @@ //! C header: [`include/drm/drm_gem.h`](srctree/include/drm/drm_gem.h) use crate::{ - alloc::flags::*, bindings, drm::{ self, @@ -14,10 +13,7 @@ use crate::{ AllocOps, // }, }, - error::{ - to_result, - Result, // - }, + error::to_result, prelude::*, sync::aref::{ ARef, -- cgit v1.2.3 From 59f237a0d17fb95f60de15386a6ab968af4bd4d4 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Fri, 6 Mar 2026 16:22:00 +0900 Subject: rust: add EMSGSIZE error code Add the EMSGSIZE error code, which indicates that a message is too long. Tested-by: Zhi Wang Acked-by: Miguel Ojeda Reviewed-by: Gary Guo Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260306-cmdq-continuation-v6-3-cc7b629200ee@nvidia.com Signed-off-by: Alexandre Courbot --- rust/kernel/error.rs | 1 + 1 file changed, 1 insertion(+) (limited to 'rust/kernel') diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 258b12afdcba..10fcf1f0404d 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -67,6 +67,7 @@ pub mod code { declare_err!(EDOM, "Math argument out of domain of func."); declare_err!(ERANGE, "Math result not representable."); declare_err!(EOVERFLOW, "Value too large for defined data type."); + declare_err!(EMSGSIZE, "Message too long."); declare_err!(ETIMEDOUT, "Connection timed out."); declare_err!(ERESTARTSYS, "Restart the system call."); declare_err!(ERESTARTNOINTR, "System call was interrupted by a signal and will be restarted."); -- cgit v1.2.3 From e2d599021c843d97ee38ba351cb0117eb984e038 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Fri, 6 Feb 2026 15:00:15 +0900 Subject: rust: io: turn IoCapable into a functional trait `IoCapable` is currently used as a marker trait to signal that the methods of the `Io` trait corresponding to `T` have been overridden by the implementor (the default implementations triggering a build-time error). This goes against the DRY principle and separates the signaling of the capability from its implementation, making it possible to forget a step while implementing a new `Io`. Another undesirable side-effect is that it makes the implementation of I/O backends boilerplate-y and convoluted: currently this is done using two levels of imbricated macros that generate unsafe code. Fix these issues by turning `IoCapable` into a functional trait that includes the raw implementation of the I/O access for `T` using unsafe methods that work with an arbitrary address. This allows us to turn the default methods of `Io` into regular methods that check the passed offset, turn it into an address, and call into the corresponding `IoCapable` functions, removing the need to overload them at all. `IoCapable` must still be implemented for all supported primitive types, which is still done more concisely using a macro, but this macro becomes much simpler and does not require calling into another one. Reviewed-by: Daniel Almeida Acked-by: Alice Ryhl Signed-off-by: Alexandre Courbot Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260206-io-v2-1-71dea20a06e6@nvidia.com Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 169 ++++++++++++++++++++++++++++++++++++++------------ rust/kernel/pci/io.rs | 37 ++++++++++- 2 files changed, 163 insertions(+), 43 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index e5fba6bf6db0..ec78c614c959 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -320,14 +320,29 @@ const fn offset_valid(offset: usize, size: usize) -> bool { } } -/// Marker trait indicating that an I/O backend supports operations of a certain type. +/// Trait indicating that an I/O backend supports operations of a certain type and providing an +/// implementation for these operations. /// /// Different I/O backends can implement this trait to expose only the operations they support. /// /// For example, a PCI configuration space may implement `IoCapable`, `IoCapable`, /// and `IoCapable`, but not `IoCapable`, while an MMIO region on a 64-bit /// system might implement all four. -pub trait IoCapable {} +pub trait IoCapable { + /// Performs an I/O read of type `T` at `address` and returns the result. + /// + /// # Safety + /// + /// The range `[address..address + size_of::()]` must be within the bounds of `Self`. + unsafe fn io_read(&self, address: usize) -> T; + + /// Performs an I/O write of `value` at `address`. + /// + /// # Safety + /// + /// The range `[address..address + size_of::()]` must be within the bounds of `Self`. + unsafe fn io_write(&self, value: T, address: usize); +} /// Types implementing this trait (e.g. MMIO BARs or PCI config regions) /// can perform I/O operations on regions of memory. @@ -369,146 +384,198 @@ pub trait Io { /// Fallible 8-bit read with runtime bounds check. #[inline(always)] - fn try_read8(&self, _offset: usize) -> Result + fn try_read8(&self, offset: usize) -> Result where Self: IoCapable, { - build_error!("Backend does not support fallible 8-bit read") + let address = self.io_addr::(offset)?; + + // SAFETY: `address` has been validated by `io_addr`. + Ok(unsafe { self.io_read(address) }) } /// Fallible 16-bit read with runtime bounds check. #[inline(always)] - fn try_read16(&self, _offset: usize) -> Result + fn try_read16(&self, offset: usize) -> Result where Self: IoCapable, { - build_error!("Backend does not support fallible 16-bit read") + let address = self.io_addr::(offset)?; + + // SAFETY: `address` has been validated by `io_addr`. + Ok(unsafe { self.io_read(address) }) } /// Fallible 32-bit read with runtime bounds check. #[inline(always)] - fn try_read32(&self, _offset: usize) -> Result + fn try_read32(&self, offset: usize) -> Result where Self: IoCapable, { - build_error!("Backend does not support fallible 32-bit read") + let address = self.io_addr::(offset)?; + + // SAFETY: `address` has been validated by `io_addr`. + Ok(unsafe { self.io_read(address) }) } /// Fallible 64-bit read with runtime bounds check. #[inline(always)] - fn try_read64(&self, _offset: usize) -> Result + fn try_read64(&self, offset: usize) -> Result where Self: IoCapable, { - build_error!("Backend does not support fallible 64-bit read") + let address = self.io_addr::(offset)?; + + // SAFETY: `address` has been validated by `io_addr`. + Ok(unsafe { self.io_read(address) }) } /// Fallible 8-bit write with runtime bounds check. #[inline(always)] - fn try_write8(&self, _value: u8, _offset: usize) -> Result + fn try_write8(&self, value: u8, offset: usize) -> Result where Self: IoCapable, { - build_error!("Backend does not support fallible 8-bit write") + let address = self.io_addr::(offset)?; + + // SAFETY: `address` has been validated by `io_addr`. + unsafe { self.io_write(value, address) }; + Ok(()) } /// Fallible 16-bit write with runtime bounds check. #[inline(always)] - fn try_write16(&self, _value: u16, _offset: usize) -> Result + fn try_write16(&self, value: u16, offset: usize) -> Result where Self: IoCapable, { - build_error!("Backend does not support fallible 16-bit write") + let address = self.io_addr::(offset)?; + + // SAFETY: `address` has been validated by `io_addr`. + unsafe { self.io_write(value, address) }; + Ok(()) } /// Fallible 32-bit write with runtime bounds check. #[inline(always)] - fn try_write32(&self, _value: u32, _offset: usize) -> Result + fn try_write32(&self, value: u32, offset: usize) -> Result where Self: IoCapable, { - build_error!("Backend does not support fallible 32-bit write") + let address = self.io_addr::(offset)?; + + // SAFETY: `address` has been validated by `io_addr`. + unsafe { self.io_write(value, address) }; + Ok(()) } /// Fallible 64-bit write with runtime bounds check. #[inline(always)] - fn try_write64(&self, _value: u64, _offset: usize) -> Result + fn try_write64(&self, value: u64, offset: usize) -> Result where Self: IoCapable, { - build_error!("Backend does not support fallible 64-bit write") + let address = self.io_addr::(offset)?; + + // SAFETY: `address` has been validated by `io_addr`. + unsafe { self.io_write(value, address) }; + Ok(()) } /// Infallible 8-bit read with compile-time bounds check. #[inline(always)] - fn read8(&self, _offset: usize) -> u8 + fn read8(&self, offset: usize) -> u8 where Self: IoKnownSize + IoCapable, { - build_error!("Backend does not support infallible 8-bit read") + let address = self.io_addr_assert::(offset); + + // SAFETY: `address` has been validated by `io_addr_assert`. + unsafe { self.io_read(address) } } /// Infallible 16-bit read with compile-time bounds check. #[inline(always)] - fn read16(&self, _offset: usize) -> u16 + fn read16(&self, offset: usize) -> u16 where Self: IoKnownSize + IoCapable, { - build_error!("Backend does not support infallible 16-bit read") + let address = self.io_addr_assert::(offset); + + // SAFETY: `address` has been validated by `io_addr_assert`. + unsafe { self.io_read(address) } } /// Infallible 32-bit read with compile-time bounds check. #[inline(always)] - fn read32(&self, _offset: usize) -> u32 + fn read32(&self, offset: usize) -> u32 where Self: IoKnownSize + IoCapable, { - build_error!("Backend does not support infallible 32-bit read") + let address = self.io_addr_assert::(offset); + + // SAFETY: `address` has been validated by `io_addr_assert`. + unsafe { self.io_read(address) } } /// Infallible 64-bit read with compile-time bounds check. #[inline(always)] - fn read64(&self, _offset: usize) -> u64 + fn read64(&self, offset: usize) -> u64 where Self: IoKnownSize + IoCapable, { - build_error!("Backend does not support infallible 64-bit read") + let address = self.io_addr_assert::(offset); + + // SAFETY: `address` has been validated by `io_addr_assert`. + unsafe { self.io_read(address) } } /// Infallible 8-bit write with compile-time bounds check. #[inline(always)] - fn write8(&self, _value: u8, _offset: usize) + fn write8(&self, value: u8, offset: usize) where Self: IoKnownSize + IoCapable, { - build_error!("Backend does not support infallible 8-bit write") + let address = self.io_addr_assert::(offset); + + // SAFETY: `address` has been validated by `io_addr_assert`. + unsafe { self.io_write(value, address) } } /// Infallible 16-bit write with compile-time bounds check. #[inline(always)] - fn write16(&self, _value: u16, _offset: usize) + fn write16(&self, value: u16, offset: usize) where Self: IoKnownSize + IoCapable, { - build_error!("Backend does not support infallible 16-bit write") + let address = self.io_addr_assert::(offset); + + // SAFETY: `address` has been validated by `io_addr_assert`. + unsafe { self.io_write(value, address) } } /// Infallible 32-bit write with compile-time bounds check. #[inline(always)] - fn write32(&self, _value: u32, _offset: usize) + fn write32(&self, value: u32, offset: usize) where Self: IoKnownSize + IoCapable, { - build_error!("Backend does not support infallible 32-bit write") + let address = self.io_addr_assert::(offset); + + // SAFETY: `address` has been validated by `io_addr_assert`. + unsafe { self.io_write(value, address) } } /// Infallible 64-bit write with compile-time bounds check. #[inline(always)] - fn write64(&self, _value: u64, _offset: usize) + fn write64(&self, value: u64, offset: usize) where Self: IoKnownSize + IoCapable, { - build_error!("Backend does not support infallible 64-bit write") + let address = self.io_addr_assert::(offset); + + // SAFETY: `address` has been validated by `io_addr_assert`. + unsafe { self.io_write(value, address) } } } @@ -534,14 +601,36 @@ pub trait IoKnownSize: Io { } } -// MMIO regions support 8, 16, and 32-bit accesses. -impl IoCapable for Mmio {} -impl IoCapable for Mmio {} -impl IoCapable for Mmio {} +/// Implements [`IoCapable`] on `$mmio` for `$ty` using `$read_fn` and `$write_fn`. +macro_rules! impl_mmio_io_capable { + ($mmio:ident, $(#[$attr:meta])* $ty:ty, $read_fn:ident, $write_fn:ident) => { + $(#[$attr])* + impl IoCapable<$ty> for $mmio { + unsafe fn io_read(&self, address: usize) -> $ty { + // SAFETY: By the trait invariant `address` is a valid address for MMIO operations. + unsafe { bindings::$read_fn(address as *const c_void) } + } + + unsafe fn io_write(&self, value: $ty, address: usize) { + // SAFETY: By the trait invariant `address` is a valid address for MMIO operations. + unsafe { bindings::$write_fn(value, address as *mut c_void) } + } + } + }; +} +// MMIO regions support 8, 16, and 32-bit accesses. +impl_mmio_io_capable!(Mmio, u8, readb, writeb); +impl_mmio_io_capable!(Mmio, u16, readw, writew); +impl_mmio_io_capable!(Mmio, u32, readl, writel); // MMIO regions on 64-bit systems also support 64-bit accesses. -#[cfg(CONFIG_64BIT)] -impl IoCapable for Mmio {} +impl_mmio_io_capable!( + Mmio, + #[cfg(CONFIG_64BIT)] + u64, + readq, + writeq +); impl Io for Mmio { /// Returns the base address of this mapping. diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index fb6edab2aea7..4feca8033eb4 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -142,10 +142,41 @@ macro_rules! call_config_write { }; } +/// Implements [`IoCapable`] on [`ConfigSpace`] for `$ty` using `$read_fn` and `$write_fn`. +macro_rules! impl_config_space_io_capable { + ($ty:ty, $read_fn:ident, $write_fn:ident) => { + impl<'a, S: ConfigSpaceKind> IoCapable<$ty> for ConfigSpace<'a, S> { + unsafe fn io_read(&self, address: usize) -> $ty { + let mut val: $ty = 0; + + // Return value from C function is ignored in infallible accessors. + let _ret = + // SAFETY: By the type invariant `self.pdev` is a valid address. + // CAST: The offset is cast to `i32` because the C functions expect a 32-bit + // signed offset parameter. PCI configuration space size is at most 4096 bytes, + // so the value always fits within `i32` without truncation or sign change. + unsafe { bindings::$read_fn(self.pdev.as_raw(), address as i32, &mut val) }; + + val + } + + unsafe fn io_write(&self, value: $ty, address: usize) { + // Return value from C function is ignored in infallible accessors. + let _ret = + // SAFETY: By the type invariant `self.pdev` is a valid address. + // CAST: The offset is cast to `i32` because the C functions expect a 32-bit + // signed offset parameter. PCI configuration space size is at most 4096 bytes, + // so the value always fits within `i32` without truncation or sign change. + unsafe { bindings::$write_fn(self.pdev.as_raw(), address as i32, value) }; + } + } + }; +} + // PCI configuration space supports 8, 16, and 32-bit accesses. -impl<'a, S: ConfigSpaceKind> IoCapable for ConfigSpace<'a, S> {} -impl<'a, S: ConfigSpaceKind> IoCapable for ConfigSpace<'a, S> {} -impl<'a, S: ConfigSpaceKind> IoCapable for ConfigSpace<'a, S> {} +impl_config_space_io_capable!(u8, pci_read_config_byte, pci_write_config_byte); +impl_config_space_io_capable!(u16, pci_read_config_word, pci_write_config_word); +impl_config_space_io_capable!(u32, pci_read_config_dword, pci_write_config_dword); impl<'a, S: ConfigSpaceKind> Io for ConfigSpace<'a, S> { /// Returns the base address of the I/O region. It is always 0 for configuration space. -- cgit v1.2.3 From 19103d4f93673c804ef82dd797cd2b935d0bf70f Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Fri, 6 Feb 2026 15:00:16 +0900 Subject: rust: io: mem: use non-relaxed I/O ops in examples The `_relaxed` I/O variant methods are about to be replaced by a wrapper type exposing this access pattern with the regular methods of the `Io` trait. Thus replace the examples to use the regular I/O methods. Since these are examples, we want them to use the most standard ops anyway, and the relaxed variants were but an addition that was MMIO-specific. Reviewed-by: Daniel Almeida Acked-by: Alice Ryhl Signed-off-by: Alexandre Courbot Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260206-io-v2-2-71dea20a06e6@nvidia.com Signed-off-by: Danilo Krummrich --- rust/kernel/io/mem.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index 620022cff401..7dc78d547f7a 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -54,6 +54,7 @@ impl<'a> IoRequest<'a> { /// use kernel::{ /// bindings, /// device::Core, + /// io::Io, /// of, /// platform, /// }; @@ -78,9 +79,9 @@ impl<'a> IoRequest<'a> { /// let io = iomem.access(pdev.as_ref())?; /// /// // Read and write a 32-bit value at `offset`. - /// let data = io.read32_relaxed(offset); + /// let data = io.read32(offset); /// - /// io.write32_relaxed(data, offset); + /// io.write32(data, offset); /// /// # Ok(SampleDriver) /// } @@ -117,6 +118,7 @@ impl<'a> IoRequest<'a> { /// use kernel::{ /// bindings, /// device::Core, + /// io::Io, /// of, /// platform, /// }; @@ -141,9 +143,9 @@ impl<'a> IoRequest<'a> { /// /// let io = iomem.access(pdev.as_ref())?; /// - /// let data = io.try_read32_relaxed(offset)?; + /// let data = io.try_read32(offset)?; /// - /// io.try_write32_relaxed(data, offset)?; + /// io.try_write32(data, offset)?; /// /// # Ok(SampleDriver) /// } -- cgit v1.2.3 From 1d1c5c73d7e8f166b6b55ae06a3c509561b854cd Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Fri, 6 Feb 2026 15:00:17 +0900 Subject: rust: io: provide Mmio relaxed ops through a wrapper type Relaxed I/O accessors for `Mmio` are currently implemented as an extra set of methods that mirror the ones defined in `Io`, but with the `_relaxed` suffix. This makes these methods impossible to use with generic code, which is a highly plausible proposition now that we have the `Io` trait. Address this by adding a new `RelaxedMmio` wrapper type for `Mmio` that provides its own `IoCapable` implementations relying on the relaxed C accessors. This makes it possible to use relaxed operations on a `Mmio` simply by wrapping it, and to use `RelaxedMmio` in code generic against `Io`. Acked-by: Alice Ryhl Signed-off-by: Alexandre Courbot Reviewed-by: Daniel Almeida Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260206-io-v2-3-71dea20a06e6@nvidia.com [ Use kernel import style in examples. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) (limited to 'rust/kernel') diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index ec78c614c959..8a0e35070153 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -742,3 +742,69 @@ impl Mmio { call_mmio_write(writeq_relaxed) <- u64 ); } + +/// [`Mmio`] wrapper using relaxed accessors. +/// +/// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO operands instead of +/// the regular ones. +/// +/// See [`Mmio::relaxed`] for a usage example. +#[repr(transparent)] +pub struct RelaxedMmio(Mmio); + +impl Io for RelaxedMmio { + #[inline] + fn addr(&self) -> usize { + self.0.addr() + } + + #[inline] + fn maxsize(&self) -> usize { + self.0.maxsize() + } +} + +impl IoKnownSize for RelaxedMmio { + const MIN_SIZE: usize = SIZE; +} + +impl Mmio { + /// Returns a [`RelaxedMmio`] reference that performs relaxed I/O operations. + /// + /// Relaxed accessors do not provide ordering guarantees with respect to DMA or memory accesses + /// and can be used when such ordering is not required. + /// + /// # Examples + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// RelaxedMmio, + /// }; + /// + /// fn do_io(io: &Mmio<0x100>) { + /// // The access is performed using `readl_relaxed` instead of `readl`. + /// let v = io.relaxed().read32(0x10); + /// } + /// + /// ``` + pub fn relaxed(&self) -> &RelaxedMmio { + // SAFETY: `RelaxedMmio` is `#[repr(transparent)]` over `Mmio`, so `Mmio` and + // `RelaxedMmio` have identical layout. + unsafe { core::mem::transmute(self) } + } +} + +// MMIO regions support 8, 16, and 32-bit accesses. +impl_mmio_io_capable!(RelaxedMmio, u8, readb_relaxed, writeb_relaxed); +impl_mmio_io_capable!(RelaxedMmio, u16, readw_relaxed, writew_relaxed); +impl_mmio_io_capable!(RelaxedMmio, u32, readl_relaxed, writel_relaxed); +// MMIO regions on 64-bit systems also support 64-bit accesses. +impl_mmio_io_capable!( + RelaxedMmio, + #[cfg(CONFIG_64BIT)] + u64, + readq_relaxed, + writeq_relaxed +); -- cgit v1.2.3 From e385eb0d1c2c4d2dbc48d1bcbc44fd43cbb154a4 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Fri, 6 Feb 2026 15:00:18 +0900 Subject: rust: io: remove legacy relaxed accessors of Mmio The relaxed access functionality is now provided by the `RelaxedMmio` wrapper type, and we don't have any user of the legacy methods left. Remove them. Reviewed-by: Daniel Almeida Acked-by: Alice Ryhl Signed-off-by: Alexandre Courbot Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260206-io-v2-4-71dea20a06e6@nvidia.com Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 40 ---------------------------------------- 1 file changed, 40 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 8a0e35070153..0d946e0c5d4f 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -701,46 +701,6 @@ impl Mmio { // SAFETY: `Mmio` is a transparent wrapper around `MmioRaw`. unsafe { &*core::ptr::from_ref(raw).cast() } } - - io_define_read!(infallible, pub read8_relaxed, call_mmio_read(readb_relaxed) -> u8); - io_define_read!(infallible, pub read16_relaxed, call_mmio_read(readw_relaxed) -> u16); - io_define_read!(infallible, pub read32_relaxed, call_mmio_read(readl_relaxed) -> u32); - io_define_read!( - infallible, - #[cfg(CONFIG_64BIT)] - pub read64_relaxed, - call_mmio_read(readq_relaxed) -> u64 - ); - - io_define_read!(fallible, pub try_read8_relaxed, call_mmio_read(readb_relaxed) -> u8); - io_define_read!(fallible, pub try_read16_relaxed, call_mmio_read(readw_relaxed) -> u16); - io_define_read!(fallible, pub try_read32_relaxed, call_mmio_read(readl_relaxed) -> u32); - io_define_read!( - fallible, - #[cfg(CONFIG_64BIT)] - pub try_read64_relaxed, - call_mmio_read(readq_relaxed) -> u64 - ); - - io_define_write!(infallible, pub write8_relaxed, call_mmio_write(writeb_relaxed) <- u8); - io_define_write!(infallible, pub write16_relaxed, call_mmio_write(writew_relaxed) <- u16); - io_define_write!(infallible, pub write32_relaxed, call_mmio_write(writel_relaxed) <- u32); - io_define_write!( - infallible, - #[cfg(CONFIG_64BIT)] - pub write64_relaxed, - call_mmio_write(writeq_relaxed) <- u64 - ); - - io_define_write!(fallible, pub try_write8_relaxed, call_mmio_write(writeb_relaxed) <- u8); - io_define_write!(fallible, pub try_write16_relaxed, call_mmio_write(writew_relaxed) <- u16); - io_define_write!(fallible, pub try_write32_relaxed, call_mmio_write(writel_relaxed) <- u32); - io_define_write!( - fallible, - #[cfg(CONFIG_64BIT)] - pub try_write64_relaxed, - call_mmio_write(writeq_relaxed) <- u64 - ); } /// [`Mmio`] wrapper using relaxed accessors. -- cgit v1.2.3 From 50aad5510fbbf8dd8f5f63380e1a1e7ae73216c4 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Fri, 6 Feb 2026 15:00:19 +0900 Subject: rust: pci: io: remove overloaded Io methods of ConfigSpace Since `ConfigSpace` now has the relevant implementations of `IoCapable`, the default methods of `Io` can be used in place of the overloaded ones. Remove them as well as the macros generating them. Reviewed-by: Daniel Almeida Acked-by: Alice Ryhl Signed-off-by: Alexandre Courbot Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260206-io-v2-5-71dea20a06e6@nvidia.com Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 4 --- rust/kernel/pci/io.rs | 70 --------------------------------------------------- 2 files changed, 74 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 0d946e0c5d4f..2ae2420be344 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -215,7 +215,6 @@ macro_rules! call_mmio_write { /// * `$c_fn:ident` - The backend-specific C function or identifier to be passed into the /// `$call_macro`. /// * `$type_name:ty` - The Rust type of the value being read (e.g., `u8`, `u32`). -#[macro_export] macro_rules! io_define_read { (infallible, $(#[$attr:meta])* $vis:vis $name:ident, $call_macro:ident($c_fn:ident) -> $type_name:ty) => { @@ -249,7 +248,6 @@ macro_rules! io_define_read { } }; } -pub use io_define_read; /// Generates an accessor method for writing to an I/O backend. /// @@ -274,7 +272,6 @@ pub use io_define_read; /// `$call_macro`. /// * `$type_name:ty` - The Rust type of the value being written (e.g., `u8`, `u32`). Note the use /// of `<-` before the type to denote a write operation. -#[macro_export] macro_rules! io_define_write { (infallible, $(#[$attr:meta])* $vis:vis $name:ident, $call_macro:ident($c_fn:ident) <- $type_name:ty) => { @@ -306,7 +303,6 @@ macro_rules! io_define_write { } }; } -pub use io_define_write; /// Checks whether an access of type `U` at the given `offset` /// is valid within this region. diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index 4feca8033eb4..ae78676c927f 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -8,8 +8,6 @@ use crate::{ device, devres::Devres, io::{ - io_define_read, - io_define_write, Io, IoCapable, IoKnownSize, @@ -85,63 +83,6 @@ pub struct ConfigSpace<'a, S: ConfigSpaceKind = Extended> { _marker: PhantomData, } -/// Internal helper macros used to invoke C PCI configuration space read functions. -/// -/// This macro is intended to be used by higher-level PCI configuration space access macros -/// (io_define_read) and provides a unified expansion for infallible vs. fallible read semantics. It -/// emits a direct call into the corresponding C helper and performs the required cast to the Rust -/// return type. -/// -/// # Parameters -/// -/// * `$c_fn` – The C function performing the PCI configuration space write. -/// * `$self` – The I/O backend object. -/// * `$ty` – The type of the value to read. -/// * `$addr` – The PCI configuration space offset to read. -/// -/// This macro does not perform any validation; all invariants must be upheld by the higher-level -/// abstraction invoking it. -macro_rules! call_config_read { - (infallible, $c_fn:ident, $self:ident, $ty:ty, $addr:expr) => {{ - let mut val: $ty = 0; - // SAFETY: By the type invariant `$self.pdev` is a valid address. - // CAST: The offset is cast to `i32` because the C functions expect a 32-bit signed offset - // parameter. PCI configuration space size is at most 4096 bytes, so the value always fits - // within `i32` without truncation or sign change. - // Return value from C function is ignored in infallible accessors. - let _ret = unsafe { bindings::$c_fn($self.pdev.as_raw(), $addr as i32, &mut val) }; - val - }}; -} - -/// Internal helper macros used to invoke C PCI configuration space write functions. -/// -/// This macro is intended to be used by higher-level PCI configuration space access macros -/// (io_define_write) and provides a unified expansion for infallible vs. fallible read semantics. -/// It emits a direct call into the corresponding C helper and performs the required cast to the -/// Rust return type. -/// -/// # Parameters -/// -/// * `$c_fn` – The C function performing the PCI configuration space write. -/// * `$self` – The I/O backend object. -/// * `$ty` – The type of the written value. -/// * `$addr` – The configuration space offset to write. -/// * `$value` – The value to write. -/// -/// This macro does not perform any validation; all invariants must be upheld by the higher-level -/// abstraction invoking it. -macro_rules! call_config_write { - (infallible, $c_fn:ident, $self:ident, $ty:ty, $addr:expr, $value:expr) => { - // SAFETY: By the type invariant `$self.pdev` is a valid address. - // CAST: The offset is cast to `i32` because the C functions expect a 32-bit signed offset - // parameter. PCI configuration space size is at most 4096 bytes, so the value always fits - // within `i32` without truncation or sign change. - // Return value from C function is ignored in infallible accessors. - let _ret = unsafe { bindings::$c_fn($self.pdev.as_raw(), $addr as i32, $value) }; - }; -} - /// Implements [`IoCapable`] on [`ConfigSpace`] for `$ty` using `$read_fn` and `$write_fn`. macro_rules! impl_config_space_io_capable { ($ty:ty, $read_fn:ident, $write_fn:ident) => { @@ -190,17 +131,6 @@ impl<'a, S: ConfigSpaceKind> Io for ConfigSpace<'a, S> { fn maxsize(&self) -> usize { self.pdev.cfg_size().into_raw() } - - // PCI configuration space does not support fallible operations. - // The default implementations from the Io trait are not used. - - io_define_read!(infallible, read8, call_config_read(pci_read_config_byte) -> u8); - io_define_read!(infallible, read16, call_config_read(pci_read_config_word) -> u16); - io_define_read!(infallible, read32, call_config_read(pci_read_config_dword) -> u32); - - io_define_write!(infallible, write8, call_config_write(pci_write_config_byte) <- u8); - io_define_write!(infallible, write16, call_config_write(pci_write_config_word) <- u16); - io_define_write!(infallible, write32, call_config_write(pci_write_config_dword) <- u32); } impl<'a, S: ConfigSpaceKind> IoKnownSize for ConfigSpace<'a, S> { -- cgit v1.2.3 From 6291ee23da4224a7584ece2d292104e872b9b5fc Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Fri, 6 Feb 2026 15:00:20 +0900 Subject: rust: io: remove overloaded Io methods of Mmio Since `Mmio` now has the relevant implementations of `IoCapable`, the default methods of `Io` can be used in place of the overloaded ones. Remove them as well as the macros generating them. Reviewed-by: Daniel Almeida Acked-by: Alice Ryhl Signed-off-by: Alexandre Courbot Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260206-io-v2-6-71dea20a06e6@nvidia.com Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 207 ------------------------------------------------------ 1 file changed, 207 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 2ae2420be344..947eb378d297 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -137,173 +137,6 @@ impl MmioRaw { #[repr(transparent)] pub struct Mmio(MmioRaw); -/// Internal helper macros used to invoke C MMIO read functions. -/// -/// This macro is intended to be used by higher-level MMIO access macros (io_define_read) and -/// provides a unified expansion for infallible vs. fallible read semantics. It emits a direct call -/// into the corresponding C helper and performs the required cast to the Rust return type. -/// -/// # Parameters -/// -/// * `$c_fn` – The C function performing the MMIO read. -/// * `$self` – The I/O backend object. -/// * `$ty` – The type of the value to be read. -/// * `$addr` – The MMIO address to read. -/// -/// This macro does not perform any validation; all invariants must be upheld by the higher-level -/// abstraction invoking it. -macro_rules! call_mmio_read { - (infallible, $c_fn:ident, $self:ident, $type:ty, $addr:expr) => { - // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. - unsafe { bindings::$c_fn($addr as *const c_void) as $type } - }; - - (fallible, $c_fn:ident, $self:ident, $type:ty, $addr:expr) => {{ - // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. - Ok(unsafe { bindings::$c_fn($addr as *const c_void) as $type }) - }}; -} - -/// Internal helper macros used to invoke C MMIO write functions. -/// -/// This macro is intended to be used by higher-level MMIO access macros (io_define_write) and -/// provides a unified expansion for infallible vs. fallible write semantics. It emits a direct call -/// into the corresponding C helper and performs the required cast to the Rust return type. -/// -/// # Parameters -/// -/// * `$c_fn` – The C function performing the MMIO write. -/// * `$self` – The I/O backend object. -/// * `$ty` – The type of the written value. -/// * `$addr` – The MMIO address to write. -/// * `$value` – The value to write. -/// -/// This macro does not perform any validation; all invariants must be upheld by the higher-level -/// abstraction invoking it. -macro_rules! call_mmio_write { - (infallible, $c_fn:ident, $self:ident, $ty:ty, $addr:expr, $value:expr) => { - // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. - unsafe { bindings::$c_fn($value, $addr as *mut c_void) } - }; - - (fallible, $c_fn:ident, $self:ident, $ty:ty, $addr:expr, $value:expr) => {{ - // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. - unsafe { bindings::$c_fn($value, $addr as *mut c_void) }; - Ok(()) - }}; -} - -/// Generates an accessor method for reading from an I/O backend. -/// -/// This macro reduces boilerplate by automatically generating either compile-time bounds-checked -/// (infallible) or runtime bounds-checked (fallible) read methods. It abstracts the address -/// calculation and bounds checking, and delegates the actual I/O read operation to a specified -/// helper macro, making it generic over different I/O backends. -/// -/// # Parameters -/// -/// * `infallible` / `fallible` - Determines the bounds-checking strategy. `infallible` relies on -/// `IoKnownSize` for compile-time checks and returns the value directly. `fallible` performs -/// runtime checks against `maxsize()` and returns a `Result`. -/// * `$(#[$attr:meta])*` - Optional attributes to apply to the generated method (e.g., -/// `#[cfg(CONFIG_64BIT)]` or inline directives). -/// * `$vis:vis` - The visibility of the generated method (e.g., `pub`). -/// * `$name:ident` / `$try_name:ident` - The name of the generated method (e.g., `read32`, -/// `try_read8`). -/// * `$call_macro:ident` - The backend-specific helper macro used to emit the actual I/O call -/// (e.g., `call_mmio_read`). -/// * `$c_fn:ident` - The backend-specific C function or identifier to be passed into the -/// `$call_macro`. -/// * `$type_name:ty` - The Rust type of the value being read (e.g., `u8`, `u32`). -macro_rules! io_define_read { - (infallible, $(#[$attr:meta])* $vis:vis $name:ident, $call_macro:ident($c_fn:ident) -> - $type_name:ty) => { - /// Read IO data from a given offset known at compile time. - /// - /// Bound checks are performed on compile time, hence if the offset is not known at compile - /// time, the build will fail. - $(#[$attr])* - // Always inline to optimize out error path of `io_addr_assert`. - #[inline(always)] - $vis fn $name(&self, offset: usize) -> $type_name { - let addr = self.io_addr_assert::<$type_name>(offset); - - // SAFETY: By the type invariant `addr` is a valid address for IO operations. - $call_macro!(infallible, $c_fn, self, $type_name, addr) - } - }; - - (fallible, $(#[$attr:meta])* $vis:vis $try_name:ident, $call_macro:ident($c_fn:ident) -> - $type_name:ty) => { - /// Read IO data from a given offset. - /// - /// Bound checks are performed on runtime, it fails if the offset (plus the type size) is - /// out of bounds. - $(#[$attr])* - $vis fn $try_name(&self, offset: usize) -> Result<$type_name> { - let addr = self.io_addr::<$type_name>(offset)?; - - // SAFETY: By the type invariant `addr` is a valid address for IO operations. - $call_macro!(fallible, $c_fn, self, $type_name, addr) - } - }; -} - -/// Generates an accessor method for writing to an I/O backend. -/// -/// This macro reduces boilerplate by automatically generating either compile-time bounds-checked -/// (infallible) or runtime bounds-checked (fallible) write methods. It abstracts the address -/// calculation and bounds checking, and delegates the actual I/O write operation to a specified -/// helper macro, making it generic over different I/O backends. -/// -/// # Parameters -/// -/// * `infallible` / `fallible` - Determines the bounds-checking strategy. `infallible` relies on -/// `IoKnownSize` for compile-time checks and returns `()`. `fallible` performs runtime checks -/// against `maxsize()` and returns a `Result`. -/// * `$(#[$attr:meta])*` - Optional attributes to apply to the generated method (e.g., -/// `#[cfg(CONFIG_64BIT)]` or inline directives). -/// * `$vis:vis` - The visibility of the generated method (e.g., `pub`). -/// * `$name:ident` / `$try_name:ident` - The name of the generated method (e.g., `write32`, -/// `try_write8`). -/// * `$call_macro:ident` - The backend-specific helper macro used to emit the actual I/O call -/// (e.g., `call_mmio_write`). -/// * `$c_fn:ident` - The backend-specific C function or identifier to be passed into the -/// `$call_macro`. -/// * `$type_name:ty` - The Rust type of the value being written (e.g., `u8`, `u32`). Note the use -/// of `<-` before the type to denote a write operation. -macro_rules! io_define_write { - (infallible, $(#[$attr:meta])* $vis:vis $name:ident, $call_macro:ident($c_fn:ident) <- - $type_name:ty) => { - /// Write IO data from a given offset known at compile time. - /// - /// Bound checks are performed on compile time, hence if the offset is not known at compile - /// time, the build will fail. - $(#[$attr])* - // Always inline to optimize out error path of `io_addr_assert`. - #[inline(always)] - $vis fn $name(&self, value: $type_name, offset: usize) { - let addr = self.io_addr_assert::<$type_name>(offset); - - $call_macro!(infallible, $c_fn, self, $type_name, addr, value); - } - }; - - (fallible, $(#[$attr:meta])* $vis:vis $try_name:ident, $call_macro:ident($c_fn:ident) <- - $type_name:ty) => { - /// Write IO data from a given offset. - /// - /// Bound checks are performed on runtime, it fails if the offset (plus the type size) is - /// out of bounds. - $(#[$attr])* - $vis fn $try_name(&self, value: $type_name, offset: usize) -> Result { - let addr = self.io_addr::<$type_name>(offset)?; - - $call_macro!(fallible, $c_fn, self, $type_name, addr, value) - } - }; -} - /// Checks whether an access of type `U` at the given `offset` /// is valid within this region. #[inline] @@ -640,46 +473,6 @@ impl Io for Mmio { fn maxsize(&self) -> usize { self.0.maxsize() } - - io_define_read!(fallible, try_read8, call_mmio_read(readb) -> u8); - io_define_read!(fallible, try_read16, call_mmio_read(readw) -> u16); - io_define_read!(fallible, try_read32, call_mmio_read(readl) -> u32); - io_define_read!( - fallible, - #[cfg(CONFIG_64BIT)] - try_read64, - call_mmio_read(readq) -> u64 - ); - - io_define_write!(fallible, try_write8, call_mmio_write(writeb) <- u8); - io_define_write!(fallible, try_write16, call_mmio_write(writew) <- u16); - io_define_write!(fallible, try_write32, call_mmio_write(writel) <- u32); - io_define_write!( - fallible, - #[cfg(CONFIG_64BIT)] - try_write64, - call_mmio_write(writeq) <- u64 - ); - - io_define_read!(infallible, read8, call_mmio_read(readb) -> u8); - io_define_read!(infallible, read16, call_mmio_read(readw) -> u16); - io_define_read!(infallible, read32, call_mmio_read(readl) -> u32); - io_define_read!( - infallible, - #[cfg(CONFIG_64BIT)] - read64, - call_mmio_read(readq) -> u64 - ); - - io_define_write!(infallible, write8, call_mmio_write(writeb) <- u8); - io_define_write!(infallible, write16, call_mmio_write(writew) <- u16); - io_define_write!(infallible, write32, call_mmio_write(writel) <- u32); - io_define_write!( - infallible, - #[cfg(CONFIG_64BIT)] - write64, - call_mmio_write(writeq) <- u64 - ); } impl IoKnownSize for Mmio { -- cgit v1.2.3 From 3cc319d5f433a4d560cc944ecfb1fe50b866cd66 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Sat, 14 Mar 2026 10:06:11 +0900 Subject: rust: enable the `generic_arg_infer` feature This feature is stable since 1.89, and used in subsequent patches. Reviewed-by: Gary Guo Tested-by: Dirk Behme Acked-by: Miguel Ojeda Signed-off-by: Alexandre Courbot Link: https://patch.msgid.link/20260314-register-v9-1-86805b2f7e9d@nvidia.com [ Resolve merge conflict. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/lib.rs | 3 +++ 1 file changed, 3 insertions(+) (limited to 'rust/kernel') diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index d93292d47420..34b924819288 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -41,6 +41,9 @@ // Stable since Rust 1.84.0. #![feature(strict_provenance)] // +// Stable since Rust 1.89.0. +#![feature(generic_arg_infer)] +// // Expected to become stable. #![feature(arbitrary_self_types)] // -- cgit v1.2.3 From c59a2d14cd248c77457b821b15c72e6a6a268553 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Sat, 14 Mar 2026 10:06:12 +0900 Subject: rust: num: add `shr` and `shl` methods to `Bounded` Shifting a `Bounded` left or right changes the number of bits required to represent the value. Add methods that perform the shift and return a `Bounded` with the appropriately adjusted bit width. These methods are particularly useful for bitfield extraction. Suggested-by: Alice Ryhl Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Reviewed-by: Daniel Almeida Tested-by: Dirk Behme Acked-by: Miguel Ojeda Acked-by: Yury Norov Signed-off-by: Alexandre Courbot Link: https://patch.msgid.link/20260314-register-v9-2-86805b2f7e9d@nvidia.com Signed-off-by: Danilo Krummrich --- rust/kernel/num/bounded.rs | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) (limited to 'rust/kernel') diff --git a/rust/kernel/num/bounded.rs b/rust/kernel/num/bounded.rs index fa81acbdc8c2..2f5f13ecd3d6 100644 --- a/rust/kernel/num/bounded.rs +++ b/rust/kernel/num/bounded.rs @@ -473,6 +473,48 @@ where // `N` bits, and with the same signedness. unsafe { Bounded::__new(value) } } + + /// Right-shifts `self` by `SHIFT` and returns the result as a `Bounded<_, RES>`, where `RES >= + /// N - SHIFT`. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<0xff00>(); + /// let v_shifted: Bounded:: = v.shr::<8, _>(); + /// + /// assert_eq!(v_shifted.get(), 0xff); + /// ``` + pub fn shr(self) -> Bounded { + const { assert!(RES + SHIFT >= N) } + + // SAFETY: We shift the value right by `SHIFT`, reducing the number of bits needed to + // represent the shifted value by as much, and just asserted that `RES >= N - SHIFT`. + unsafe { Bounded::__new(self.0 >> SHIFT) } + } + + /// Left-shifts `self` by `SHIFT` and returns the result as a `Bounded<_, RES>`, where `RES >= + /// N + SHIFT`. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::::new::<0xff>(); + /// let v_shifted: Bounded:: = v.shl::<8, _>(); + /// + /// assert_eq!(v_shifted.get(), 0xff00); + /// ``` + pub fn shl(self) -> Bounded { + const { assert!(RES >= N + SHIFT) } + + // SAFETY: We shift the value left by `SHIFT`, augmenting the number of bits needed to + // represent the shifted value by as much, and just asserted that `RES >= N + SHIFT`. + unsafe { Bounded::__new(self.0 << SHIFT) } + } } impl Deref for Bounded -- cgit v1.2.3 From 164f8634bfd8eef7b90c429156c59706635cfb88 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Sat, 14 Mar 2026 10:06:13 +0900 Subject: rust: num: add `into_bool` method to `Bounded` Single-bit numbers are typically treated as booleans. There is an `Into` implementation for those, but invoking it from contexts that lack type expectations is not always convenient. Add an `into_bool` method as a simpler shortcut. Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Reviewed-by: Daniel Almeida Reviewed-by: Yury Norov Tested-by: Dirk Behme Acked-by: Miguel Ojeda Signed-off-by: Alexandre Courbot Link: https://patch.msgid.link/20260314-register-v9-3-86805b2f7e9d@nvidia.com Signed-off-by: Danilo Krummrich --- rust/kernel/num/bounded.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'rust/kernel') diff --git a/rust/kernel/num/bounded.rs b/rust/kernel/num/bounded.rs index 2f5f13ecd3d6..d28d118abd8e 100644 --- a/rust/kernel/num/bounded.rs +++ b/rust/kernel/num/bounded.rs @@ -1101,3 +1101,24 @@ where unsafe { Self::__new(T::from(value)) } } } + +impl Bounded +where + T: Integer + Zeroable, +{ + /// Converts this [`Bounded`] into a [`bool`]. + /// + /// This is a shorter way of writing `bool::from(self)`. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// assert_eq!(Bounded::::new::<0>().into_bool(), false); + /// assert_eq!(Bounded::::new::<1>().into_bool(), true); + /// ``` + pub fn into_bool(self) -> bool { + self.into() + } +} -- cgit v1.2.3 From 7836ec76ec5cd8d45759a6a360b1fda4829d2734 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Sat, 14 Mar 2026 10:06:14 +0900 Subject: rust: num: make Bounded::get const There is a need to access the inner value of a `Bounded` in const context, notably for bitfields and registers. Remove the invariant check of `Bounded::get`, which allows us to make it const. Reviewed-by: Gary Guo Signed-off-by: Alexandre Courbot Link: https://patch.msgid.link/20260314-register-v9-4-86805b2f7e9d@nvidia.com Signed-off-by: Danilo Krummrich --- rust/kernel/num/bounded.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/num/bounded.rs b/rust/kernel/num/bounded.rs index d28d118abd8e..bbab6bbcb315 100644 --- a/rust/kernel/num/bounded.rs +++ b/rust/kernel/num/bounded.rs @@ -379,6 +379,9 @@ where /// Returns the wrapped value as the backing type. /// + /// This is similar to the [`Deref`] implementation, but doesn't enforce the size invariant of + /// the [`Bounded`], which might produce slightly less optimal code. + /// /// # Examples /// /// ``` @@ -387,8 +390,8 @@ where /// let v = Bounded::::new::<7>(); /// assert_eq!(v.get(), 7u32); /// ``` - pub fn get(self) -> T { - *self.deref() + pub const fn get(self) -> T { + self.0 } /// Increases the number of bits usable for `self`. -- cgit v1.2.3 From 498823541be1e2d9f947b37a10cc98e681da9828 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Sat, 14 Mar 2026 10:06:15 +0900 Subject: rust: io: add IoLoc type and generic I/O accessors I/O accesses are defined by the following properties: - An I/O location, which consists of a start address, a width, and a type to interpret the read value as, - A value, which is returned for reads or provided for writes. Introduce the `IoLoc` trait, which allows implementing types to fully specify an I/O location. This allows I/O operations to be made generic through the new `read` and `write` methods. This design will allow us to factorize the I/O code working with primitives, and to introduce ways to perform I/O with a higher degree of control through register types. Co-developed-by: Gary Guo Signed-off-by: Gary Guo Signed-off-by: Alexandre Courbot Link: https://patch.msgid.link/20260314-register-v9-5-86805b2f7e9d@nvidia.com [ Fix incorrect reference to io_addr_assert() in try_update(). - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 124 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) (limited to 'rust/kernel') diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 947eb378d297..e1e9802bb603 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -173,6 +173,30 @@ pub trait IoCapable { unsafe fn io_write(&self, value: T, address: usize); } +/// Describes a given I/O location: its offset, width, and type to convert the raw value from and +/// into. +/// +/// This trait is the key abstraction allowing [`Io::read`], [`Io::write`], and [`Io::update`] (and +/// their fallible [`try_read`](Io::try_read), [`try_write`](Io::try_write) and +/// [`try_update`](Io::try_update) counterparts) to work uniformly with both raw [`usize`] offsets +/// (for primitive types like [`u32`]) and typed ones. +/// +/// An `IoLoc` carries three pieces of information: +/// +/// - The offset to access (returned by [`IoLoc::offset`]), +/// - The width of the access (determined by [`IoLoc::IoType`]), +/// - The type `T` in which the raw data is returned or provided. +/// +/// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type +/// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`). +pub trait IoLoc { + /// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset). + type IoType: Into + From; + + /// Consumes `self` and returns the offset of this location. + fn offset(self) -> usize; +} + /// Types implementing this trait (e.g. MMIO BARs or PCI config regions) /// can perform I/O operations on regions of memory. /// @@ -406,6 +430,106 @@ pub trait Io { // SAFETY: `address` has been validated by `io_addr_assert`. unsafe { self.io_write(value, address) } } + + /// Generic fallible read with runtime bounds check. + #[inline(always)] + fn try_read(&self, location: L) -> Result + where + L: IoLoc, + Self: IoCapable, + { + let address = self.io_addr::(location.offset())?; + + // SAFETY: `address` has been validated by `io_addr`. + Ok(unsafe { self.io_read(address) }.into()) + } + + /// Generic fallible write with runtime bounds check. + #[inline(always)] + fn try_write(&self, location: L, value: T) -> Result + where + L: IoLoc, + Self: IoCapable, + { + let address = self.io_addr::(location.offset())?; + let io_value = value.into(); + + // SAFETY: `address` has been validated by `io_addr`. + unsafe { self.io_write(io_value, address) } + + Ok(()) + } + + /// Generic fallible update with runtime bounds check. + /// + /// Note: this does not perform any synchronization. The caller is responsible for ensuring + /// exclusive access if required. + #[inline(always)] + fn try_update(&self, location: L, f: F) -> Result + where + L: IoLoc, + Self: IoCapable, + F: FnOnce(T) -> T, + { + let address = self.io_addr::(location.offset())?; + + // SAFETY: `address` has been validated by `io_addr`. + let value: T = unsafe { self.io_read(address) }.into(); + let io_value = f(value).into(); + + // SAFETY: `address` has been validated by `io_addr`. + unsafe { self.io_write(io_value, address) } + + Ok(()) + } + + /// Generic infallible read with compile-time bounds check. + #[inline(always)] + fn read(&self, location: L) -> T + where + L: IoLoc, + Self: IoKnownSize + IoCapable, + { + let address = self.io_addr_assert::(location.offset()); + + // SAFETY: `address` has been validated by `io_addr_assert`. + unsafe { self.io_read(address) }.into() + } + + /// Generic infallible write with compile-time bounds check. + #[inline(always)] + fn write(&self, location: L, value: T) + where + L: IoLoc, + Self: IoKnownSize + IoCapable, + { + let address = self.io_addr_assert::(location.offset()); + let io_value = value.into(); + + // SAFETY: `address` has been validated by `io_addr_assert`. + unsafe { self.io_write(io_value, address) } + } + + /// Generic infallible update with compile-time bounds check. + /// + /// Note: this does not perform any synchronization. The caller is responsible for ensuring + /// exclusive access if required. + #[inline(always)] + fn update(&self, location: L, f: F) + where + L: IoLoc, + Self: IoKnownSize + IoCapable + Sized, + F: FnOnce(T) -> T, + { + let address = self.io_addr_assert::(location.offset()); + + // SAFETY: `address` has been validated by `io_addr_assert`. + let value: T = unsafe { self.io_read(address) }.into(); + let io_value = f(value).into(); + + // SAFETY: `address` has been validated by `io_addr_assert`. + unsafe { self.io_write(io_value, address) } + } } /// Trait for types with a known size at compile time. -- cgit v1.2.3 From 147b41ba23d63f43ed34e6940417a5506e323370 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Sat, 14 Mar 2026 10:06:16 +0900 Subject: rust: io: use generic read/write accessors for primitive accesses By providing the required `IoLoc` implementations on `usize`, we can leverage the generic accessors and reduce the number of unsafe blocks in the module. This also allows us to directly call the generic `read/write/update` methods with primitive types, so add examples illustrating this. Signed-off-by: Alexandre Courbot Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260314-register-v9-6-86805b2f7e9d@nvidia.com [ Slightly improve wording in doc-comment. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 218 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 150 insertions(+), 68 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index e1e9802bb603..c51a87b9169b 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -197,6 +197,26 @@ pub trait IoLoc { fn offset(self) -> usize; } +/// Implements [`IoLoc<$ty>`] for [`usize`], allowing [`usize`] to be used as a parameter of +/// [`Io::read`] and [`Io::write`]. +macro_rules! impl_usize_ioloc { + ($($ty:ty),*) => { + $( + impl IoLoc<$ty> for usize { + type IoType = $ty; + + #[inline(always)] + fn offset(self) -> usize { + self + } + } + )* + } +} + +// Provide the ability to read any primitive type from a [`usize`]. +impl_usize_ioloc!(u8, u16, u32, u64); + /// Types implementing this trait (e.g. MMIO BARs or PCI config regions) /// can perform I/O operations on regions of memory. /// @@ -241,10 +261,7 @@ pub trait Io { where Self: IoCapable, { - let address = self.io_addr::(offset)?; - - // SAFETY: `address` has been validated by `io_addr`. - Ok(unsafe { self.io_read(address) }) + self.try_read(offset) } /// Fallible 16-bit read with runtime bounds check. @@ -253,10 +270,7 @@ pub trait Io { where Self: IoCapable, { - let address = self.io_addr::(offset)?; - - // SAFETY: `address` has been validated by `io_addr`. - Ok(unsafe { self.io_read(address) }) + self.try_read(offset) } /// Fallible 32-bit read with runtime bounds check. @@ -265,10 +279,7 @@ pub trait Io { where Self: IoCapable, { - let address = self.io_addr::(offset)?; - - // SAFETY: `address` has been validated by `io_addr`. - Ok(unsafe { self.io_read(address) }) + self.try_read(offset) } /// Fallible 64-bit read with runtime bounds check. @@ -277,10 +288,7 @@ pub trait Io { where Self: IoCapable, { - let address = self.io_addr::(offset)?; - - // SAFETY: `address` has been validated by `io_addr`. - Ok(unsafe { self.io_read(address) }) + self.try_read(offset) } /// Fallible 8-bit write with runtime bounds check. @@ -289,11 +297,7 @@ pub trait Io { where Self: IoCapable, { - let address = self.io_addr::(offset)?; - - // SAFETY: `address` has been validated by `io_addr`. - unsafe { self.io_write(value, address) }; - Ok(()) + self.try_write(offset, value) } /// Fallible 16-bit write with runtime bounds check. @@ -302,11 +306,7 @@ pub trait Io { where Self: IoCapable, { - let address = self.io_addr::(offset)?; - - // SAFETY: `address` has been validated by `io_addr`. - unsafe { self.io_write(value, address) }; - Ok(()) + self.try_write(offset, value) } /// Fallible 32-bit write with runtime bounds check. @@ -315,11 +315,7 @@ pub trait Io { where Self: IoCapable, { - let address = self.io_addr::(offset)?; - - // SAFETY: `address` has been validated by `io_addr`. - unsafe { self.io_write(value, address) }; - Ok(()) + self.try_write(offset, value) } /// Fallible 64-bit write with runtime bounds check. @@ -328,11 +324,7 @@ pub trait Io { where Self: IoCapable, { - let address = self.io_addr::(offset)?; - - // SAFETY: `address` has been validated by `io_addr`. - unsafe { self.io_write(value, address) }; - Ok(()) + self.try_write(offset, value) } /// Infallible 8-bit read with compile-time bounds check. @@ -341,10 +333,7 @@ pub trait Io { where Self: IoKnownSize + IoCapable, { - let address = self.io_addr_assert::(offset); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_read(address) } + self.read(offset) } /// Infallible 16-bit read with compile-time bounds check. @@ -353,10 +342,7 @@ pub trait Io { where Self: IoKnownSize + IoCapable, { - let address = self.io_addr_assert::(offset); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_read(address) } + self.read(offset) } /// Infallible 32-bit read with compile-time bounds check. @@ -365,10 +351,7 @@ pub trait Io { where Self: IoKnownSize + IoCapable, { - let address = self.io_addr_assert::(offset); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_read(address) } + self.read(offset) } /// Infallible 64-bit read with compile-time bounds check. @@ -377,10 +360,7 @@ pub trait Io { where Self: IoKnownSize + IoCapable, { - let address = self.io_addr_assert::(offset); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_read(address) } + self.read(offset) } /// Infallible 8-bit write with compile-time bounds check. @@ -389,10 +369,7 @@ pub trait Io { where Self: IoKnownSize + IoCapable, { - let address = self.io_addr_assert::(offset); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_write(value, address) } + self.write(offset, value) } /// Infallible 16-bit write with compile-time bounds check. @@ -401,10 +378,7 @@ pub trait Io { where Self: IoKnownSize + IoCapable, { - let address = self.io_addr_assert::(offset); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_write(value, address) } + self.write(offset, value) } /// Infallible 32-bit write with compile-time bounds check. @@ -413,10 +387,7 @@ pub trait Io { where Self: IoKnownSize + IoCapable, { - let address = self.io_addr_assert::(offset); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_write(value, address) } + self.write(offset, value) } /// Infallible 64-bit write with compile-time bounds check. @@ -425,13 +396,31 @@ pub trait Io { where Self: IoKnownSize + IoCapable, { - let address = self.io_addr_assert::(offset); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_write(value, address) } + self.write(offset, value) } /// Generic fallible read with runtime bounds check. + /// + /// # Examples + /// + /// Read a primitive type from an I/O address: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// }; + /// + /// fn do_reads(io: &Mmio) -> Result { + /// // 32-bit read from address `0x10`. + /// let v: u32 = io.try_read(0x10)?; + /// + /// // 8-bit read from address `0xfff`. + /// let v: u8 = io.try_read(0xfff)?; + /// + /// Ok(()) + /// } + /// ``` #[inline(always)] fn try_read(&self, location: L) -> Result where @@ -445,6 +434,27 @@ pub trait Io { } /// Generic fallible write with runtime bounds check. + /// + /// # Examples + /// + /// Write a primitive type to an I/O address: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// }; + /// + /// fn do_writes(io: &Mmio) -> Result { + /// // 32-bit write of value `1` at address `0x10`. + /// io.try_write(0x10, 1u32)?; + /// + /// // 8-bit write of value `0xff` at address `0xfff`. + /// io.try_write(0xfff, 0xffu8)?; + /// + /// Ok(()) + /// } + /// ``` #[inline(always)] fn try_write(&self, location: L, value: T) -> Result where @@ -464,6 +474,23 @@ pub trait Io { /// /// Note: this does not perform any synchronization. The caller is responsible for ensuring /// exclusive access if required. + /// + /// # Examples + /// + /// Read the u32 value at address `0x10`, increment it, and store the updated value back: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// }; + /// + /// fn do_update(io: &Mmio<0x1000>) -> Result { + /// io.try_update(0x10, |v: u32| { + /// v + 1 + /// }) + /// } + /// ``` #[inline(always)] fn try_update(&self, location: L, f: F) -> Result where @@ -484,6 +511,25 @@ pub trait Io { } /// Generic infallible read with compile-time bounds check. + /// + /// # Examples + /// + /// Read a primitive type from an I/O address: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// }; + /// + /// fn do_reads(io: &Mmio<0x1000>) { + /// // 32-bit read from address `0x10`. + /// let v: u32 = io.read(0x10); + /// + /// // 8-bit read from the top of the I/O space. + /// let v: u8 = io.read(0xfff); + /// } + /// ``` #[inline(always)] fn read(&self, location: L) -> T where @@ -497,6 +543,25 @@ pub trait Io { } /// Generic infallible write with compile-time bounds check. + /// + /// # Examples + /// + /// Write a primitive type to an I/O address: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// }; + /// + /// fn do_writes(io: &Mmio<0x1000>) { + /// // 32-bit write of value `1` at address `0x10`. + /// io.write(0x10, 1u32); + /// + /// // 8-bit write of value `0xff` at the top of the I/O space. + /// io.write(0xfff, 0xffu8); + /// } + /// ``` #[inline(always)] fn write(&self, location: L, value: T) where @@ -514,6 +579,23 @@ pub trait Io { /// /// Note: this does not perform any synchronization. The caller is responsible for ensuring /// exclusive access if required. + /// + /// # Examples + /// + /// Read the u32 value at address `0x10`, increment it, and store the updated value back: + /// + /// ```no_run + /// use kernel::io::{ + /// Io, + /// Mmio, + /// }; + /// + /// fn do_update(io: &Mmio<0x1000>) { + /// io.update(0x10, |v: u32| { + /// v + 1 + /// }) + /// } + /// ``` #[inline(always)] fn update(&self, location: L, f: F) where -- cgit v1.2.3 From 20ba6a1dbcb957152f6d858015b3a3311dd6da49 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Sat, 14 Mar 2026 10:06:17 +0900 Subject: rust: io: add `register!` macro Add a macro for defining hardware register types with I/O accessors. Each register field is represented as a `Bounded` of the appropriate bit width, ensuring field values are never silently truncated. Fields can optionally be converted to/from custom types, either fallibly or infallibly. The address of registers can be direct, relative, or indexed, supporting most of the patterns in which registers are arranged. Suggested-by: Danilo Krummrich Link: https://lore.kernel.org/all/20250306222336.23482-6-dakr@kernel.org/ Co-developed-by: Gary Guo Signed-off-by: Gary Guo Signed-off-by: Alexandre Courbot Link: https://patch.msgid.link/20260314-register-v9-7-86805b2f7e9d@nvidia.com [ * Improve wording and formatting of doc-comments, * Import build_assert!(), * Add missing inline annotations, * Call static_assert!() with absolute path, * Use expect instead of allow. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 5 +- rust/kernel/io/register.rs | 1229 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1233 insertions(+), 1 deletion(-) create mode 100644 rust/kernel/io/register.rs (limited to 'rust/kernel') diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index c51a87b9169b..4950cecf30ca 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -11,8 +11,10 @@ use crate::{ pub mod mem; pub mod poll; +pub mod register; pub mod resource; +pub use crate::register; pub use resource::Resource; /// Physical address type. @@ -179,7 +181,8 @@ pub trait IoCapable { /// This trait is the key abstraction allowing [`Io::read`], [`Io::write`], and [`Io::update`] (and /// their fallible [`try_read`](Io::try_read), [`try_write`](Io::try_write) and /// [`try_update`](Io::try_update) counterparts) to work uniformly with both raw [`usize`] offsets -/// (for primitive types like [`u32`]) and typed ones. +/// (for primitive types like [`u32`]) and typed ones (like those generated by the [`register!`] +/// macro). /// /// An `IoLoc` carries three pieces of information: /// diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs new file mode 100644 index 000000000000..dbd458aaa761 --- /dev/null +++ b/rust/kernel/io/register.rs @@ -0,0 +1,1229 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Macro to define register layout and accessors. +//! +//! The [`register!`](kernel::io::register!) macro provides an intuitive and readable syntax for +//! defining a dedicated type for each register and accessing it using [`Io`](super::Io). Each such +//! type comes with its own field accessors that can return an error if a field's value is invalid. +//! +//! Note: most of the items in this module are public so they can be referenced by the macro, but +//! most are not to be used directly by users. Outside of the `register!` macro itself, the only +//! items you might want to import from this module are [`WithBase`] and [`Array`]. +//! +//! # Simple example +//! +//! ```no_run +//! use kernel::io::register; +//! +//! register! { +//! /// Basic information about the chip. +//! pub BOOT_0(u32) @ 0x00000100 { +//! /// Vendor ID. +//! 15:8 vendor_id; +//! /// Major revision of the chip. +//! 7:4 major_revision; +//! /// Minor revision of the chip. +//! 3:0 minor_revision; +//! } +//! } +//! ``` +//! +//! This defines a 32-bit `BOOT_0` type which can be read from or written to offset `0x100` of an +//! `Io` region, with the described bitfields. For instance, `minor_revision` consists of the 4 +//! least significant bits of the type. +//! +//! Fields are instances of [`Bounded`](kernel::num::Bounded) and can be read by calling their +//! getter method, which is named after them. They also have setter methods prefixed with `with_` +//! for runtime values and `with_const_` for constant values. All setters return the updated +//! register value. +//! +//! Fields can also be transparently converted from/to an arbitrary type by using the `=>` and +//! `?=>` syntaxes. +//! +//! If present, doc comments above register or fields definitions are added to the relevant item +//! they document (the register type itself, or the field's setter and getter methods). +//! +//! Note that multiple registers can be defined in a single `register!` invocation. This can be +//! useful to group related registers together. +//! +//! Here is how the register defined above can be used in code: +//! +//! +//! ```no_run +//! use kernel::{ +//! io::{ +//! register, +//! Io, +//! IoLoc, +//! }, +//! num::Bounded, +//! }; +//! # use kernel::io::Mmio; +//! # register! { +//! # pub BOOT_0(u32) @ 0x00000100 { +//! # 15:8 vendor_id; +//! # 7:4 major_revision; +//! # 3:0 minor_revision; +//! # } +//! # } +//! # fn test(io: &Mmio<0x1000>) { +//! # fn obtain_vendor_id() -> u8 { 0xff } +//! +//! // Read from the register's defined offset (0x100). +//! let boot0 = io.read(BOOT_0); +//! pr_info!("chip revision: {}.{}", boot0.major_revision().get(), boot0.minor_revision().get()); +//! +//! // Update some fields and write the new value back. +//! let new_boot0 = boot0 +//! // Constant values. +//! .with_const_major_revision::<3>() +//! .with_const_minor_revision::<10>() +//! // Runtime value. +//! .with_vendor_id(obtain_vendor_id()); +//! io.write((), new_boot0); +//! +//! // Or, build a new value from zero and write it: +//! io.write((), BOOT_0::zeroed() +//! .with_const_major_revision::<3>() +//! .with_const_minor_revision::<10>() +//! .with_vendor_id(obtain_vendor_id()) +//! ); +//! +//! // Or, read and update the register in a single step. +//! io.update(BOOT_0, |r| r +//! .with_const_major_revision::<3>() +//! .with_const_minor_revision::<10>() +//! .with_vendor_id(obtain_vendor_id()) +//! ); +//! +//! // Constant values can also be built using the const setters. +//! const V: BOOT_0 = pin_init::zeroed::() +//! .with_const_major_revision::<3>() +//! .with_const_minor_revision::<10>(); +//! # } +//! ``` +//! +//! For more extensive documentation about how to define registers, see the +//! [`register!`](kernel::io::register!) macro. + +use core::marker::PhantomData; + +use crate::io::IoLoc; + +use kernel::build_assert; + +/// Trait implemented by all registers. +pub trait Register: Sized { + /// Backing primitive type of the register. + type Storage: Into + From; + + /// Start offset of the register. + /// + /// The interpretation of this offset depends on the type of the register. + const OFFSET: usize; +} + +/// Trait implemented by registers with a fixed offset. +pub trait FixedRegister: Register {} + +/// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when +/// passing a [`FixedRegister`] value. +impl IoLoc for () +where + T: FixedRegister, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + T::OFFSET + } +} + +/// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used +/// as an [`IoLoc`]. +impl IoLoc for T +where + T: FixedRegister, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + T::OFFSET + } +} + +/// Location of a fixed register. +pub struct FixedRegisterLoc(PhantomData); + +impl FixedRegisterLoc { + /// Returns the location of `T`. + #[inline(always)] + // We do not implement `Default` so we can be const. + #[expect(clippy::new_without_default)] + pub const fn new() -> Self { + Self(PhantomData) + } +} + +impl IoLoc for FixedRegisterLoc +where + T: FixedRegister, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + T::OFFSET + } +} + +/// Trait providing a base address to be added to the offset of a relative register to obtain +/// its actual offset. +/// +/// The `T` generic argument is used to distinguish which base to use, in case a type provides +/// several bases. It is given to the `register!` macro to restrict the use of the register to +/// implementors of this particular variant. +pub trait RegisterBase { + /// Base address to which register offsets are added. + const BASE: usize; +} + +/// Trait implemented by all registers that are relative to a base. +pub trait WithBase { + /// Family of bases applicable to this register. + type BaseFamily; + + /// Returns the absolute location of this type when using `B` as its base. + #[inline(always)] + fn of>() -> RelativeRegisterLoc + where + Self: Register, + { + RelativeRegisterLoc::new() + } +} + +/// Trait implemented by relative registers. +pub trait RelativeRegister: Register + WithBase {} + +/// Location of a relative register. +/// +/// This can either be an immediately accessible regular [`RelativeRegister`], or a +/// [`RelativeRegisterArray`] that needs one additional resolution through +/// [`RelativeRegisterLoc::at`]. +pub struct RelativeRegisterLoc(PhantomData, PhantomData); + +impl RelativeRegisterLoc +where + T: Register + WithBase, + B: RegisterBase + ?Sized, +{ + /// Returns the location of a relative register or register array. + #[inline(always)] + // We do not implement `Default` so we can be const. + #[expect(clippy::new_without_default)] + pub const fn new() -> Self { + Self(PhantomData, PhantomData) + } + + // Returns the absolute offset of the relative register using base `B`. + // + // This is implemented as a private const method so it can be reused by the [`IoLoc`] + // implementations of both [`RelativeRegisterLoc`] and [`RelativeRegisterArrayLoc`]. + #[inline] + const fn offset(self) -> usize { + B::BASE + T::OFFSET + } +} + +impl IoLoc for RelativeRegisterLoc +where + T: RelativeRegister, + B: RegisterBase + ?Sized, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + RelativeRegisterLoc::offset(self) + } +} + +/// Trait implemented by arrays of registers. +pub trait RegisterArray: Register { + /// Number of elements in the registers array. + const SIZE: usize; + /// Number of bytes between the start of elements in the registers array. + const STRIDE: usize; +} + +/// Location of an array register. +pub struct RegisterArrayLoc(usize, PhantomData); + +impl RegisterArrayLoc { + /// Returns the location of register `T` at position `idx`, with build-time validation. + #[inline(always)] + pub fn new(idx: usize) -> Self { + build_assert!(idx < T::SIZE); + + Self(idx, PhantomData) + } + + /// Attempts to return the location of register `T` at position `idx`, with runtime validation. + #[inline(always)] + pub fn try_new(idx: usize) -> Option { + if idx < T::SIZE { + Some(Self(idx, PhantomData)) + } else { + None + } + } +} + +impl IoLoc for RegisterArrayLoc +where + T: RegisterArray, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + T::OFFSET + self.0 * T::STRIDE + } +} + +/// Trait providing location builders for [`RegisterArray`]s. +pub trait Array { + /// Returns the location of the register at position `idx`, with build-time validation. + #[inline(always)] + fn at(idx: usize) -> RegisterArrayLoc + where + Self: RegisterArray, + { + RegisterArrayLoc::new(idx) + } + + /// Returns the location of the register at position `idx`, with runtime validation. + #[inline(always)] + fn try_at(idx: usize) -> Option> + where + Self: RegisterArray, + { + RegisterArrayLoc::try_new(idx) + } +} + +/// Trait implemented by arrays of relative registers. +pub trait RelativeRegisterArray: RegisterArray + WithBase {} + +/// Location of a relative array register. +pub struct RelativeRegisterArrayLoc< + T: RelativeRegisterArray, + B: RegisterBase + ?Sized, +>(RelativeRegisterLoc, usize); + +impl RelativeRegisterArrayLoc +where + T: RelativeRegisterArray, + B: RegisterBase + ?Sized, +{ + /// Returns the location of register `T` from the base `B` at index `idx`, with build-time + /// validation. + #[inline(always)] + pub fn new(idx: usize) -> Self { + build_assert!(idx < T::SIZE); + + Self(RelativeRegisterLoc::new(), idx) + } + + /// Attempts to return the location of register `T` from the base `B` at index `idx`, with + /// runtime validation. + #[inline(always)] + pub fn try_new(idx: usize) -> Option { + if idx < T::SIZE { + Some(Self(RelativeRegisterLoc::new(), idx)) + } else { + None + } + } +} + +/// Methods exclusive to [`RelativeRegisterLoc`]s created with a [`RelativeRegisterArray`]. +impl RelativeRegisterLoc +where + T: RelativeRegisterArray, + B: RegisterBase + ?Sized, +{ + /// Returns the location of the register at position `idx`, with build-time validation. + #[inline(always)] + pub fn at(self, idx: usize) -> RelativeRegisterArrayLoc { + RelativeRegisterArrayLoc::new(idx) + } + + /// Returns the location of the register at position `idx`, with runtime validation. + #[inline(always)] + pub fn try_at(self, idx: usize) -> Option> { + RelativeRegisterArrayLoc::try_new(idx) + } +} + +impl IoLoc for RelativeRegisterArrayLoc +where + T: RelativeRegisterArray, + B: RegisterBase + ?Sized, +{ + type IoType = T::Storage; + + #[inline(always)] + fn offset(self) -> usize { + self.0.offset() + self.1 * T::STRIDE + } +} + +/// Defines a dedicated type for a register, including getter and setter methods for its fields and +/// methods to read and write it from an [`Io`](kernel::io::Io) region. +/// +/// This documentation focuses on how to declare registers. See the [module-level +/// documentation](mod@kernel::io::register) for examples of how to access them. +/// +/// There are 4 possible kinds of registers: fixed offset registers, relative registers, arrays of +/// registers, and relative arrays of registers. +/// +/// ## Fixed offset registers +/// +/// These are the simplest kind of registers. Their location is simply an offset inside the I/O +/// region. For instance: +/// +/// ```ignore +/// register! { +/// pub FIXED_REG(u16) @ 0x80 { +/// ... +/// } +/// } +/// ``` +/// +/// This creates a 16-bit register named `FIXED_REG` located at offset `0x80` of an I/O region. +/// +/// These registers' location can be built simply by referencing their name: +/// +/// ```no_run +/// use kernel::{ +/// io::{ +/// register, +/// Io, +/// }, +/// }; +/// # use kernel::io::Mmio; +/// +/// register! { +/// FIXED_REG(u32) @ 0x100 { +/// 16:8 high_byte; +/// 7:0 low_byte; +/// } +/// } +/// +/// # fn test(io: &Mmio<0x1000>) { +/// let val = io.read(FIXED_REG); +/// +/// // Write from an already-existing value. +/// io.write(FIXED_REG, val.with_low_byte(0xff)); +/// +/// // Create a register value from scratch. +/// let val2 = FIXED_REG::zeroed().with_high_byte(0x80); +/// +/// // The location of fixed offset registers is already contained in their type. Thus, the +/// // `location` argument of `Io::write` is technically redundant and can be replaced by `()`. +/// io.write((), val2); +/// # } +/// +/// ``` +/// +/// It is possible to create an alias of an existing register with new field definitions by using +/// the `=> ALIAS` syntax. This is useful for cases where a register's interpretation depends on +/// the context: +/// +/// ```no_run +/// use kernel::io::register; +/// +/// register! { +/// /// Scratch register. +/// pub SCRATCH(u32) @ 0x00000200 { +/// 31:0 value; +/// } +/// +/// /// Boot status of the firmware. +/// pub SCRATCH_BOOT_STATUS(u32) => SCRATCH { +/// 0:0 completed; +/// } +/// } +/// ``` +/// +/// In this example, `SCRATCH_BOOT_STATUS` uses the same I/O address as `SCRATCH`, while providing +/// its own `completed` field. +/// +/// ## Relative registers +/// +/// Relative registers can be instantiated several times at a relative offset of a group of bases. +/// For instance, imagine the following I/O space: +/// +/// ```text +/// +-----------------------------+ +/// | ... | +/// | | +/// 0x100--->+------------CPU0-------------+ +/// | | +/// 0x110--->+-----------------------------+ +/// | CPU_CTL | +/// +-----------------------------+ +/// | ... | +/// | | +/// | | +/// 0x200--->+------------CPU1-------------+ +/// | | +/// 0x210--->+-----------------------------+ +/// | CPU_CTL | +/// +-----------------------------+ +/// | ... | +/// +-----------------------------+ +/// ``` +/// +/// `CPU0` and `CPU1` both have a `CPU_CTL` register that starts at offset `0x10` of their I/O +/// space segment. Since both instances of `CPU_CTL` share the same layout, we don't want to define +/// them twice and would prefer a way to select which one to use from a single definition. +/// +/// This can be done using the `Base + Offset` syntax when specifying the register's address: +/// +/// ```ignore +/// register! { +/// pub RELATIVE_REG(u32) @ Base + 0x80 { +/// ... +/// } +/// } +/// ``` +/// +/// This creates a register with an offset of `0x80` from a given base. +/// +/// `Base` is an arbitrary type (typically a ZST) to be used as a generic parameter of the +/// [`RegisterBase`] trait to provide the base as a constant, i.e. each type providing a base for +/// this register needs to implement `RegisterBase`. +/// +/// The location of relative registers can be built using the [`WithBase::of`] method to specify +/// its base. All relative registers implement [`WithBase`]. +/// +/// Here is the above layout translated into code: +/// +/// ```no_run +/// use kernel::{ +/// io::{ +/// register, +/// register::{ +/// RegisterBase, +/// WithBase, +/// }, +/// Io, +/// }, +/// }; +/// # use kernel::io::Mmio; +/// +/// // Type used to identify the base. +/// pub struct CpuCtlBase; +/// +/// // ZST describing `CPU0`. +/// struct Cpu0; +/// impl RegisterBase for Cpu0 { +/// const BASE: usize = 0x100; +/// } +/// +/// // ZST describing `CPU1`. +/// struct Cpu1; +/// impl RegisterBase for Cpu1 { +/// const BASE: usize = 0x200; +/// } +/// +/// // This makes `CPU_CTL` accessible from all implementors of `RegisterBase`. +/// register! { +/// /// CPU core control. +/// pub CPU_CTL(u32) @ CpuCtlBase + 0x10 { +/// 0:0 start; +/// } +/// } +/// +/// # fn test(io: Mmio<0x1000>) { +/// // Read the status of `Cpu0`. +/// let cpu0_started = io.read(CPU_CTL::of::()); +/// +/// // Stop `Cpu0`. +/// io.write(WithBase::of::(), CPU_CTL::zeroed()); +/// # } +/// +/// // Aliases can also be defined for relative register. +/// register! { +/// /// Alias to CPU core control. +/// pub CPU_CTL_ALIAS(u32) => CpuCtlBase + CPU_CTL { +/// /// Start the aliased CPU core. +/// 1:1 alias_start; +/// } +/// } +/// +/// # fn test2(io: Mmio<0x1000>) { +/// // Start the aliased `CPU0`, leaving its other fields untouched. +/// io.update(CPU_CTL_ALIAS::of::(), |r| r.with_alias_start(true)); +/// # } +/// ``` +/// +/// ## Arrays of registers +/// +/// Some I/O areas contain consecutive registers that share the same field layout. These areas can +/// be defined as an array of identical registers, allowing them to be accessed by index with +/// compile-time or runtime bound checking: +/// +/// ```ignore +/// register! { +/// pub REGISTER_ARRAY(u8)[10, stride = 4] @ 0x100 { +/// ... +/// } +/// } +/// ``` +/// +/// This defines `REGISTER_ARRAY`, an array of 10 byte registers starting at offset `0x100`. Each +/// register is separated from its neighbor by 4 bytes. +/// +/// The `stride` parameter is optional; if unspecified, the registers are placed consecutively from +/// each other. +/// +/// A location for a register in a register array is built using the [`Array::at`] trait method. +/// All arrays of registers implement [`Array`]. +/// +/// ```no_run +/// use kernel::{ +/// io::{ +/// register, +/// register::Array, +/// Io, +/// }, +/// }; +/// # use kernel::io::Mmio; +/// # fn get_scratch_idx() -> usize { +/// # 0x15 +/// # } +/// +/// // Array of 64 consecutive registers with the same layout starting at offset `0x80`. +/// register! { +/// /// Scratch registers. +/// pub SCRATCH(u32)[64] @ 0x00000080 { +/// 31:0 value; +/// } +/// } +/// +/// # fn test(io: &Mmio<0x1000>) +/// # -> Result<(), Error>{ +/// // Read scratch register 0, i.e. I/O address `0x80`. +/// let scratch_0 = io.read(SCRATCH::at(0)).value(); +/// +/// // Write scratch register 15, i.e. I/O address `0x80 + (15 * 4)`. +/// io.write(Array::at(15), SCRATCH::from(0xffeeaabb)); +/// +/// // This is out of bounds and won't build. +/// // let scratch_128 = io.read(SCRATCH::at(128)).value(); +/// +/// // Runtime-obtained array index. +/// let idx = get_scratch_idx(); +/// // Access on a runtime index returns an error if it is out-of-bounds. +/// let some_scratch = io.read(SCRATCH::try_at(idx).ok_or(EINVAL)?).value(); +/// +/// // Alias to a specific register in an array. +/// // Here `SCRATCH[8]` is used to convey the firmware exit code. +/// register! { +/// /// Firmware exit status code. +/// pub FIRMWARE_STATUS(u32) => SCRATCH[8] { +/// 7:0 status; +/// } +/// } +/// +/// let status = io.read(FIRMWARE_STATUS).status(); +/// +/// // Non-contiguous register arrays can be defined by adding a stride parameter. +/// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the +/// // registers of the two declarations below are interleaved. +/// register! { +/// /// Scratch registers bank 0. +/// pub SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ 0x000000c0 { +/// 31:0 value; +/// } +/// +/// /// Scratch registers bank 1. +/// pub SCRATCH_INTERLEAVED_1(u32)[16, stride = 8] @ 0x000000c4 { +/// 31:0 value; +/// } +/// } +/// # Ok(()) +/// # } +/// ``` +/// +/// ## Relative arrays of registers +/// +/// Combining the two features described in the sections above, arrays of registers accessible from +/// a base can also be defined: +/// +/// ```ignore +/// register! { +/// pub RELATIVE_REGISTER_ARRAY(u8)[10, stride = 4] @ Base + 0x100 { +/// ... +/// } +/// } +/// ``` +/// +/// Like relative registers, they implement the [`WithBase`] trait. However the return value of +/// [`WithBase::of`] cannot be used directly as a location and must be further specified using the +/// [`at`](RelativeRegisterLoc::at) method. +/// +/// ```no_run +/// use kernel::{ +/// io::{ +/// register, +/// register::{ +/// RegisterBase, +/// WithBase, +/// }, +/// Io, +/// }, +/// }; +/// # use kernel::io::Mmio; +/// # fn get_scratch_idx() -> usize { +/// # 0x15 +/// # } +/// +/// // Type used as parameter of `RegisterBase` to specify the base. +/// pub struct CpuCtlBase; +/// +/// // ZST describing `CPU0`. +/// struct Cpu0; +/// impl RegisterBase for Cpu0 { +/// const BASE: usize = 0x100; +/// } +/// +/// // ZST describing `CPU1`. +/// struct Cpu1; +/// impl RegisterBase for Cpu1 { +/// const BASE: usize = 0x200; +/// } +/// +/// // 64 per-cpu scratch registers, arranged as a contiguous array. +/// register! { +/// /// Per-CPU scratch registers. +/// pub CPU_SCRATCH(u32)[64] @ CpuCtlBase + 0x00000080 { +/// 31:0 value; +/// } +/// } +/// +/// # fn test(io: &Mmio<0x1000>) -> Result<(), Error> { +/// // Read scratch register 0 of CPU0. +/// let scratch = io.read(CPU_SCRATCH::of::().at(0)); +/// +/// // Write the retrieved value into scratch register 15 of CPU1. +/// io.write(WithBase::of::().at(15), scratch); +/// +/// // This won't build. +/// // let cpu0_scratch_128 = io.read(CPU_SCRATCH::of::().at(128)).value(); +/// +/// // Runtime-obtained array index. +/// let scratch_idx = get_scratch_idx(); +/// // Access on a runtime index returns an error if it is out-of-bounds. +/// let cpu0_scratch = io.read( +/// CPU_SCRATCH::of::().try_at(scratch_idx).ok_or(EINVAL)? +/// ).value(); +/// # Ok(()) +/// # } +/// +/// // Alias to `SCRATCH[8]` used to convey the firmware exit code. +/// register! { +/// /// Per-CPU firmware exit status code. +/// pub CPU_FIRMWARE_STATUS(u32) => CpuCtlBase + CPU_SCRATCH[8] { +/// 7:0 status; +/// } +/// } +/// +/// // Non-contiguous relative register arrays can be defined by adding a stride parameter. +/// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the +/// // registers of the two declarations below are interleaved. +/// register! { +/// /// Scratch registers bank 0. +/// pub CPU_SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d00 { +/// 31:0 value; +/// } +/// +/// /// Scratch registers bank 1. +/// pub CPU_SCRATCH_INTERLEAVED_1(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d04 { +/// 31:0 value; +/// } +/// } +/// +/// # fn test2(io: &Mmio<0x1000>) -> Result<(), Error> { +/// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::()).status(); +/// # Ok(()) +/// # } +/// ``` +#[macro_export] +macro_rules! register { + // Entry point for the macro, allowing multiple registers to be defined in one call. + // It matches all possible register declaration patterns to dispatch them to corresponding + // `@reg` rule that defines a single register. + ( + $( + $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) + $([ $size:expr $(, stride = $stride:expr)? ])? + $(@ $($base:ident +)? $offset:literal)? + $(=> $alias:ident $(+ $alias_offset:ident)? $([$alias_idx:expr])? )? + { $($fields:tt)* } + )* + ) => { + $( + $crate::register!( + @reg $(#[$attr])* $vis $name ($storage) $([$size $(, stride = $stride)?])? + $(@ $($base +)? $offset)? + $(=> $alias $(+ $alias_offset)? $([$alias_idx])? )? + { $($fields)* } + ); + )* + }; + + // All the rules below are private helpers. + + // Creates a register at a fixed offset of the MMIO space. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $offset:literal + { $($fields:tt)* } + ) => { + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!(@io_base $name($storage) @ $offset); + $crate::register!(@io_fixed $(#[$attr])* $vis $name($storage)); + }; + + // Creates an alias register of fixed offset register `alias` with its own fields. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:ident + { $($fields:tt)* } + ) => { + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!( + @io_base $name($storage) @ + <$alias as $crate::io::register::Register>::OFFSET + ); + $crate::register!(@io_fixed $(#[$attr])* $vis $name($storage)); + }; + + // Creates a register at a relative offset from a base address provider. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $base:ident + $offset:literal + { $($fields:tt)* } + ) => { + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!(@io_base $name($storage) @ $offset); + $crate::register!(@io_relative $vis $name($storage) @ $base); + }; + + // Creates an alias register of relative offset register `alias` with its own fields. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $base:ident + $alias:ident + { $($fields:tt)* } + ) => { + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!( + @io_base $name($storage) @ <$alias as $crate::io::register::Register>::OFFSET + ); + $crate::register!(@io_relative $vis $name($storage) @ $base); + }; + + // Creates an array of registers at a fixed offset of the MMIO space. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) + [ $size:expr, stride = $stride:expr ] @ $offset:literal { $($fields:tt)* } + ) => { + ::kernel::static_assert!(::core::mem::size_of::<$storage>() <= $stride); + + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!(@io_base $name($storage) @ $offset); + $crate::register!(@io_array $vis $name($storage) [ $size, stride = $stride ]); + }; + + // Shortcut for contiguous array of registers (stride == size of element). + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] @ $offset:literal + { $($fields:tt)* } + ) => { + $crate::register!( + $(#[$attr])* $vis $name($storage) [ $size, stride = ::core::mem::size_of::<$storage>() ] + @ $offset { $($fields)* } + ); + }; + + // Creates an alias of register `idx` of array of registers `alias` with its own fields. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:ident [ $idx:expr ] + { $($fields:tt)* } + ) => { + ::kernel::static_assert!($idx < <$alias as $crate::io::register::RegisterArray>::SIZE); + + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!( + @io_base $name($storage) @ + <$alias as $crate::io::register::Register>::OFFSET + + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE + ); + $crate::register!(@io_fixed $(#[$attr])* $vis $name($storage)); + }; + + // Creates an array of registers at a relative offset from a base address provider. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) + [ $size:expr, stride = $stride:expr ] + @ $base:ident + $offset:literal { $($fields:tt)* } + ) => { + ::kernel::static_assert!(::core::mem::size_of::<$storage>() <= $stride); + + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!(@io_base $name($storage) @ $offset); + $crate::register!( + @io_relative_array $vis $name($storage) [ $size, stride = $stride ] @ $base + $offset + ); + }; + + // Shortcut for contiguous array of relative registers (stride == size of element). + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] + @ $base:ident + $offset:literal { $($fields:tt)* } + ) => { + $crate::register!( + $(#[$attr])* $vis $name($storage) [ $size, stride = ::core::mem::size_of::<$storage>() ] + @ $base + $offset { $($fields)* } + ); + }; + + // Creates an alias of register `idx` of relative array of registers `alias` with its own + // fields. + ( + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) + => $base:ident + $alias:ident [ $idx:expr ] { $($fields:tt)* } + ) => { + ::kernel::static_assert!($idx < <$alias as $crate::io::register::RegisterArray>::SIZE); + + $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); + $crate::register!( + @io_base $name($storage) @ + <$alias as $crate::io::register::Register>::OFFSET + + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE + ); + $crate::register!(@io_relative $vis $name($storage) @ $base); + }; + + // Generates the bitfield for the register. + // + // `#[allow(non_camel_case_types)]` is added since register names typically use + // `SCREAMING_CASE`. + ( + @bitfield $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* } + ) => { + $crate::register!(@bitfield_core + #[allow(non_camel_case_types)] + $(#[$attr])* $vis $name $storage + ); + $crate::register!(@bitfield_fields $vis $name $storage { $($fields)* }); + }; + + // Implementations shared by all registers types. + (@io_base $name:ident($storage:ty) @ $offset:expr) => { + impl $crate::io::register::Register for $name { + type Storage = $storage; + + const OFFSET: usize = $offset; + } + }; + + // Implementations of fixed registers. + (@io_fixed $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty)) => { + impl $crate::io::register::FixedRegister for $name {} + + $(#[$attr])* + $vis const $name: $crate::io::register::FixedRegisterLoc<$name> = + $crate::io::register::FixedRegisterLoc::<$name>::new(); + }; + + // Implementations of relative registers. + (@io_relative $vis:vis $name:ident ($storage:ty) @ $base:ident) => { + impl $crate::io::register::WithBase for $name { + type BaseFamily = $base; + } + + impl $crate::io::register::RelativeRegister for $name {} + }; + + // Implementations of register arrays. + (@io_array $vis:vis $name:ident ($storage:ty) [ $size:expr, stride = $stride:expr ]) => { + impl $crate::io::register::Array for $name {} + + impl $crate::io::register::RegisterArray for $name { + const SIZE: usize = $size; + const STRIDE: usize = $stride; + } + }; + + // Implementations of relative array registers. + ( + @io_relative_array $vis:vis $name:ident ($storage:ty) [ $size:expr, stride = $stride:expr ] + @ $base:ident + $offset:literal + ) => { + impl $crate::io::register::WithBase for $name { + type BaseFamily = $base; + } + + impl $crate::io::register::RegisterArray for $name { + const SIZE: usize = $size; + const STRIDE: usize = $stride; + } + + impl $crate::io::register::RelativeRegisterArray for $name {} + }; + + // Defines the wrapper `$name` type and its conversions from/to the storage type. + (@bitfield_core $(#[$attr:meta])* $vis:vis $name:ident $storage:ty) => { + $(#[$attr])* + #[repr(transparent)] + #[derive(Clone, Copy, PartialEq, Eq)] + $vis struct $name { + inner: $storage, + } + + #[allow(dead_code)] + impl $name { + /// Creates a bitfield from a raw value. + #[inline(always)] + $vis const fn from_raw(value: $storage) -> Self { + Self{ inner: value } + } + + /// Turns this bitfield into its raw value. + /// + /// This is similar to the [`From`] implementation, but is shorter to invoke in + /// most cases. + #[inline(always)] + $vis const fn into_raw(self) -> $storage { + self.inner + } + } + + // SAFETY: `$storage` is `Zeroable` and `$name` is transparent. + unsafe impl ::pin_init::Zeroable for $name {} + + impl ::core::convert::From<$name> for $storage { + #[inline(always)] + fn from(val: $name) -> $storage { + val.into_raw() + } + } + + impl ::core::convert::From<$storage> for $name { + #[inline(always)] + fn from(val: $storage) -> $name { + Self::from_raw(val) + } + } + }; + + // Definitions requiring knowledge of individual fields: private and public field accessors, + // and `Debug` implementation. + (@bitfield_fields $vis:vis $name:ident $storage:ty { + $($(#[doc = $doc:expr])* $hi:literal:$lo:literal $field:ident + $(?=> $try_into_type:ty)? + $(=> $into_type:ty)? + ; + )* + } + ) => { + #[allow(dead_code)] + impl $name { + $( + $crate::register!(@private_field_accessors $vis $name $storage : $hi:$lo $field); + $crate::register!( + @public_field_accessors $(#[doc = $doc])* $vis $name $storage : $hi:$lo $field + $(?=> $try_into_type)? + $(=> $into_type)? + ); + )* + } + + $crate::register!(@debug $name { $($field;)* }); + }; + + // Private field accessors working with the exact `Bounded` type for the field. + ( + @private_field_accessors $vis:vis $name:ident $storage:ty : $hi:tt:$lo:tt $field:ident + ) => { + ::kernel::macros::paste!( + $vis const [<$field:upper _RANGE>]: ::core::ops::RangeInclusive = $lo..=$hi; + $vis const [<$field:upper _MASK>]: $storage = + ((((1 << $hi) - 1) << 1) + 1) - ((1 << $lo) - 1); + $vis const [<$field:upper _SHIFT>]: u32 = $lo; + ); + + ::kernel::macros::paste!( + fn [<__ $field>](self) -> + ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> { + // Left shift to align the field's MSB with the storage MSB. + const ALIGN_TOP: u32 = $storage::BITS - ($hi + 1); + // Right shift to move the top-aligned field to bit 0 of the storage. + const ALIGN_BOTTOM: u32 = ALIGN_TOP + $lo; + + // Extract the field using two shifts. `Bounded::shr` produces the correctly-sized + // output type. + let val = ::kernel::num::Bounded::<$storage, { $storage::BITS }>::from( + self.inner << ALIGN_TOP + ); + val.shr::() + } + + const fn [<__with_ $field>]( + mut self, + value: ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }>, + ) -> Self + { + const MASK: $storage = <$name>::[<$field:upper _MASK>]; + const SHIFT: u32 = <$name>::[<$field:upper _SHIFT>]; + + let value = value.get() << SHIFT; + self.inner = (self.inner & !MASK) | value; + + self + } + ); + }; + + // Public accessors for fields infallibly (`=>`) converted to a type. + ( + @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : + $hi:literal:$lo:literal $field:ident => $into_type:ty + ) => { + ::kernel::macros::paste!( + + $(#[doc = $doc])* + #[doc = "Returns the value of this field."] + #[inline(always)] + $vis fn $field(self) -> $into_type + { + self.[<__ $field>]().into() + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the given `value`."] + #[inline(always)] + $vis fn [](self, value: $into_type) -> Self + { + self.[<__with_ $field>](value.into()) + } + + ); + }; + + // Public accessors for fields fallibly (`?=>`) converted to a type. + ( + @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : + $hi:tt:$lo:tt $field:ident ?=> $try_into_type:ty + ) => { + ::kernel::macros::paste!( + + $(#[doc = $doc])* + #[doc = "Returns the value of this field."] + #[inline(always)] + $vis fn $field(self) -> + Result< + $try_into_type, + <$try_into_type as ::core::convert::TryFrom< + ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> + >>::Error + > + { + self.[<__ $field>]().try_into() + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the given `value`."] + #[inline(always)] + $vis fn [](self, value: $try_into_type) -> Self + { + self.[<__with_ $field>](value.into()) + } + + ); + }; + + // Public accessors for fields not converted to a type. + ( + @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : + $hi:tt:$lo:tt $field:ident + ) => { + ::kernel::macros::paste!( + + $(#[doc = $doc])* + #[doc = "Returns the value of this field."] + #[inline(always)] + $vis fn $field(self) -> + ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> + { + self.[<__ $field>]() + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the compile-time constant `VALUE`."] + #[inline(always)] + $vis const fn [](self) -> Self { + self.[<__with_ $field>]( + ::kernel::num::Bounded::<$storage, { $hi + 1 - $lo }>::new::() + ) + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the given `value`."] + #[inline(always)] + $vis fn []( + self, + value: T, + ) -> Self + where T: Into<::kernel::num::Bounded<$storage, { $hi + 1 - $lo }>>, + { + self.[<__with_ $field>](value.into()) + } + + $(#[doc = $doc])* + #[doc = "Tries to set this field to `value`, returning an error if it is out of range."] + #[inline(always)] + $vis fn []( + self, + value: T, + ) -> ::kernel::error::Result + where T: ::kernel::num::TryIntoBounded<$storage, { $hi + 1 - $lo }>, + { + Ok( + self.[<__with_ $field>]( + value.try_into_bounded().ok_or(::kernel::error::code::EOVERFLOW)? + ) + ) + } + + ); + }; + + // `Debug` implementation. + (@debug $name:ident { $($field:ident;)* }) => { + impl ::kernel::fmt::Debug for $name { + fn fmt(&self, f: &mut ::kernel::fmt::Formatter<'_>) -> ::kernel::fmt::Result { + f.debug_struct(stringify!($name)) + .field("", &::kernel::prelude::fmt!("{:#x}", self.inner)) + $( + .field(stringify!($field), &self.$field()) + )* + .finish() + } + } + }; +} -- cgit v1.2.3 From 9a52a8f5ed97d47c9641248874f4c6a78e136d97 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Sat, 14 Mar 2026 10:06:18 +0900 Subject: rust: io: introduce `write_reg` and `LocatedRegister` Some I/O types, like fixed address registers, carry their location alongside their values. For these types, the regular `Io::write` method can lead into repeating the location information twice: once to provide the location itself, another time to build the value. We are also considering supporting making all register values carry their full location information for convenience and safety. Add a new `Io::write_reg` method that takes a single argument implementing `LocatedRegister`, a trait that decomposes implementors into a `(location, value)` tuple. This allows write operations on fixed offset registers to be done while specifying their name only once. Suggested-by: Danilo Krummrich Link: https://lore.kernel.org/all/DH0XBLXZD81K.22SWIZ1ZAOW1@kernel.org/ Signed-off-by: Alexandre Courbot Link: https://patch.msgid.link/20260314-register-v9-8-86805b2f7e9d@nvidia.com [ Replace FIFO with VERSION register in the examples. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 87 ++++++++++++++++++++++++++++++++++++++++++++++ rust/kernel/io/register.rs | 35 +++++++++++++++++-- 2 files changed, 120 insertions(+), 2 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 4950cecf30ca..fcc7678fd9e3 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -17,6 +17,8 @@ pub mod resource; pub use crate::register; pub use resource::Resource; +use register::LocatedRegister; + /// Physical address type. /// /// This is a type alias to either `u32` or `u64` depending on the config option @@ -473,6 +475,49 @@ pub trait Io { Ok(()) } + /// Generic fallible write of a fully-located register value. + /// + /// # Examples + /// + /// Tuples carrying a location and a value can be used with this method: + /// + /// ```no_run + /// use kernel::io::{ + /// register, + /// Io, + /// Mmio, + /// }; + /// + /// register! { + /// VERSION(u32) @ 0x100 { + /// 15:8 major; + /// 7:0 minor; + /// } + /// } + /// + /// impl VERSION { + /// fn new(major: u8, minor: u8) -> Self { + /// VERSION::zeroed().with_major(major).with_minor(minor) + /// } + /// } + /// + /// fn do_write_reg(io: &Mmio) -> Result { + /// + /// io.try_write_reg(VERSION::new(1, 0)) + /// } + /// ``` + #[inline(always)] + fn try_write_reg(&self, value: V) -> Result + where + L: IoLoc, + V: LocatedRegister, + Self: IoCapable, + { + let (location, value) = value.into_io_op(); + + self.try_write(location, value) + } + /// Generic fallible update with runtime bounds check. /// /// Note: this does not perform any synchronization. The caller is responsible for ensuring @@ -578,6 +623,48 @@ pub trait Io { unsafe { self.io_write(io_value, address) } } + /// Generic infallible write of a fully-located register value. + /// + /// # Examples + /// + /// Tuples carrying a location and a value can be used with this method: + /// + /// ```no_run + /// use kernel::io::{ + /// register, + /// Io, + /// Mmio, + /// }; + /// + /// register! { + /// VERSION(u32) @ 0x100 { + /// 15:8 major; + /// 7:0 minor; + /// } + /// } + /// + /// impl VERSION { + /// fn new(major: u8, minor: u8) -> Self { + /// VERSION::zeroed().with_major(major).with_minor(minor) + /// } + /// } + /// + /// fn do_write_reg(io: &Mmio<0x1000>) { + /// io.write_reg(VERSION::new(1, 0)); + /// } + /// ``` + #[inline(always)] + fn write_reg(&self, value: V) + where + L: IoLoc, + V: LocatedRegister, + Self: IoKnownSize + IoCapable, + { + let (location, value) = value.into_io_op(); + + self.write(location, value) + } + /// Generic infallible update with compile-time bounds check. /// /// Note: this does not perform any synchronization. The caller is responsible for ensuring diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index dbd458aaa761..abc49926abfe 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -80,10 +80,10 @@ //! .with_const_minor_revision::<10>() //! // Runtime value. //! .with_vendor_id(obtain_vendor_id()); -//! io.write((), new_boot0); +//! io.write_reg(new_boot0); //! //! // Or, build a new value from zero and write it: -//! io.write((), BOOT_0::zeroed() +//! io.write_reg(BOOT_0::zeroed() //! .with_const_major_revision::<3>() //! .with_const_minor_revision::<10>() //! .with_vendor_id(obtain_vendor_id()) @@ -382,6 +382,34 @@ where } } +/// Trait implemented by items that contain both a register value and the absolute I/O location at +/// which to write it. +/// +/// Implementors can be used with [`Io::write_reg`](super::Io::write_reg). +pub trait LocatedRegister { + /// Register value to write. + type Value: Register; + /// Full location information at which to write the value. + type Location: IoLoc; + + /// Consumes `self` and returns a `(location, value)` tuple describing a valid I/O write + /// operation. + fn into_io_op(self) -> (Self::Location, Self::Value); +} + +impl LocatedRegister for T +where + T: FixedRegister, +{ + type Location = FixedRegisterLoc; + type Value = T; + + #[inline(always)] + fn into_io_op(self) -> (FixedRegisterLoc, T) { + (FixedRegisterLoc::new(), self) + } +} + /// Defines a dedicated type for a register, including getter and setter methods for its fields and /// methods to read and write it from an [`Io`](kernel::io::Io) region. /// @@ -436,6 +464,9 @@ where /// // The location of fixed offset registers is already contained in their type. Thus, the /// // `location` argument of `Io::write` is technically redundant and can be replaced by `()`. /// io.write((), val2); +/// +/// // Or, the single-argument `Io::write_reg` can be used. +/// io.write_reg(val2); /// # } /// /// ``` -- cgit v1.2.3 From f9f0b4a1f35d39a1a2a2f8ec46eb7b81efc70a63 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 19 Mar 2026 17:07:22 -0400 Subject: rust: interop: Add list module for C linked list interface Add a new module `kernel::interop::list` for working with C's doubly circular linked lists. Provide low-level iteration over list nodes. Typed iteration over actual items is provided with a `clist_create` macro to assist in creation of the `CList` type. Cc: Nikola Djukic Reviewed-by: Daniel Almeida Reviewed-by: Alexandre Courbot Reviewed-by: Alice Ryhl Acked-by: Alexandre Courbot Acked-by: Gary Guo Acked-by: Miguel Ojeda Signed-off-by: Joel Fernandes Link: https://patch.msgid.link/20260319210722.1543776-1-joelagnelf@nvidia.com [ * Remove stray empty comment and double blank line in doctest, * Improve wording and fix a few typos, * Use markdown emphasis instead of caps, * Move interop/mod.rs to interop.rs. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/interop.rs | 9 ++ rust/kernel/interop/list.rs | 339 ++++++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 2 + 3 files changed, 350 insertions(+) create mode 100644 rust/kernel/interop.rs create mode 100644 rust/kernel/interop/list.rs (limited to 'rust/kernel') diff --git a/rust/kernel/interop.rs b/rust/kernel/interop.rs new file mode 100644 index 000000000000..3b371d782a59 --- /dev/null +++ b/rust/kernel/interop.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Infrastructure for interfacing Rust code with C kernel subsystems. +//! +//! This module is intended for low-level, unsafe Rust infrastructure code +//! that interoperates between Rust and C. It is *not* for use directly in +//! Rust drivers. + +pub mod list; diff --git a/rust/kernel/interop/list.rs b/rust/kernel/interop/list.rs new file mode 100644 index 000000000000..54265ea036bb --- /dev/null +++ b/rust/kernel/interop/list.rs @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust interface for C doubly circular intrusive linked lists. +//! +//! This module provides Rust abstractions for iterating over C `list_head`-based +//! linked lists. It should only be used for cases where C and Rust code share +//! direct access to the same linked list through a C interop interface. +//! +//! Note: This *must not* be used by Rust components that just need a linked list +//! primitive. Use [`kernel::list::List`] instead. +//! +//! # Examples +//! +//! ``` +//! use kernel::{ +//! bindings, +//! interop::list::clist_create, +//! types::Opaque, +//! }; +//! # // Create test list with values (0, 10, 20) - normally done by C code but it is +//! # // emulated here for doctests using the C bindings. +//! # use core::mem::MaybeUninit; +//! # +//! # /// C struct with embedded `list_head` (typically will be allocated by C code). +//! # #[repr(C)] +//! # pub struct SampleItemC { +//! # pub value: i32, +//! # pub link: bindings::list_head, +//! # } +//! # +//! # let mut head = MaybeUninit::::uninit(); +//! # +//! # let head = head.as_mut_ptr(); +//! # // SAFETY: `head` and all the items are test objects allocated in this scope. +//! # unsafe { bindings::INIT_LIST_HEAD(head) }; +//! # +//! # let mut items = [ +//! # MaybeUninit::::uninit(), +//! # MaybeUninit::::uninit(), +//! # MaybeUninit::::uninit(), +//! # ]; +//! # +//! # for (i, item) in items.iter_mut().enumerate() { +//! # let ptr = item.as_mut_ptr(); +//! # // SAFETY: `ptr` points to a valid `MaybeUninit`. +//! # unsafe { (*ptr).value = i as i32 * 10 }; +//! # // SAFETY: `&raw mut` creates a pointer valid for `INIT_LIST_HEAD`. +//! # unsafe { bindings::INIT_LIST_HEAD(&raw mut (*ptr).link) }; +//! # // SAFETY: `link` was just initialized and `head` is a valid list head. +//! # unsafe { bindings::list_add_tail(&mut (*ptr).link, head) }; +//! # } +//! +//! /// Rust wrapper for the C struct. +//! /// +//! /// The list item struct in this example is defined in C code as: +//! /// +//! /// ```c +//! /// struct SampleItemC { +//! /// int value; +//! /// struct list_head link; +//! /// }; +//! /// ``` +//! #[repr(transparent)] +//! pub struct Item(Opaque); +//! +//! impl Item { +//! pub fn value(&self) -> i32 { +//! // SAFETY: `Item` has the same layout as `SampleItemC`. +//! unsafe { (*self.0.get()).value } +//! } +//! } +//! +//! // Create typed [`CList`] from sentinel head. +//! // SAFETY: `head` is valid and initialized, items are `SampleItemC` with +//! // embedded `link` field, and `Item` is `#[repr(transparent)]` over `SampleItemC`. +//! let list = unsafe { clist_create!(head, Item, SampleItemC, link) }; +//! +//! // Iterate directly over typed items. +//! let mut found_0 = false; +//! let mut found_10 = false; +//! let mut found_20 = false; +//! +//! for item in list.iter() { +//! let val = item.value(); +//! if val == 0 { found_0 = true; } +//! if val == 10 { found_10 = true; } +//! if val == 20 { found_20 = true; } +//! } +//! +//! assert!(found_0 && found_10 && found_20); +//! ``` + +use core::{ + iter::FusedIterator, + marker::PhantomData, // +}; + +use crate::{ + bindings, + types::Opaque, // +}; + +use pin_init::{ + pin_data, + pin_init, + PinInit, // +}; + +/// FFI wrapper for a C `list_head` object used in intrusive linked lists. +/// +/// # Invariants +/// +/// - The underlying `list_head` is initialized with valid non-`NULL` `next`/`prev` pointers. +#[pin_data] +#[repr(transparent)] +pub struct CListHead { + #[pin] + inner: Opaque, +} + +impl CListHead { + /// Create a `&CListHead` reference from a raw `list_head` pointer. + /// + /// # Safety + /// + /// - `ptr` must be a valid pointer to an initialized `list_head` (e.g. via + /// `INIT_LIST_HEAD()`), with valid non-`NULL` `next`/`prev` pointers. + /// - `ptr` must remain valid for the lifetime `'a`. + /// - The list and all linked `list_head` nodes must not be modified from + /// anywhere for the lifetime `'a`, unless done so via any [`CListHead`] APIs. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::list_head) -> &'a Self { + // SAFETY: + // - `CListHead` has the same layout as `list_head`. + // - `ptr` is valid and unmodified for `'a` per caller guarantees. + unsafe { &*ptr.cast() } + } + + /// Get the raw `list_head` pointer. + #[inline] + pub fn as_raw(&self) -> *mut bindings::list_head { + self.inner.get() + } + + /// Get the next [`CListHead`] in the list. + #[inline] + pub fn next(&self) -> &Self { + let raw = self.as_raw(); + // SAFETY: + // - `self.as_raw()` is valid and initialized per type invariants. + // - The `next` pointer is valid and non-`NULL` per type invariants + // (initialized via `INIT_LIST_HEAD()` or equivalent). + unsafe { Self::from_raw((*raw).next) } + } + + /// Check if this node is linked in a list (not isolated). + #[inline] + pub fn is_linked(&self) -> bool { + let raw = self.as_raw(); + // SAFETY: `self.as_raw()` is valid per type invariants. + unsafe { (*raw).next != raw && (*raw).prev != raw } + } + + /// Returns a pin-initializer for the list head. + pub fn new() -> impl PinInit { + pin_init!(Self { + // SAFETY: `INIT_LIST_HEAD` initializes `slot` to a valid empty list. + inner <- Opaque::ffi_init(|slot| unsafe { bindings::INIT_LIST_HEAD(slot) }), + }) + } +} + +// SAFETY: `list_head` contains no thread-bound state; it only holds +// `next`/`prev` pointers. +unsafe impl Send for CListHead {} + +// SAFETY: `CListHead` can be shared among threads as modifications are +// not allowed at the moment. +unsafe impl Sync for CListHead {} + +impl PartialEq for CListHead { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self, other) + } +} + +impl Eq for CListHead {} + +/// Low-level iterator over `list_head` nodes. +/// +/// An iterator used to iterate over a C intrusive linked list (`list_head`). The caller has to +/// perform conversion of returned [`CListHead`] to an item (using [`container_of`] or similar). +/// +/// # Invariants +/// +/// `current` and `sentinel` are valid references into an initialized linked list. +struct CListHeadIter<'a> { + /// Current position in the list. + current: &'a CListHead, + /// The sentinel head (used to detect end of iteration). + sentinel: &'a CListHead, +} + +impl<'a> Iterator for CListHeadIter<'a> { + type Item = &'a CListHead; + + #[inline] + fn next(&mut self) -> Option { + // Check if we've reached the sentinel (end of list). + if self.current == self.sentinel { + return None; + } + + let item = self.current; + self.current = item.next(); + Some(item) + } +} + +impl<'a> FusedIterator for CListHeadIter<'a> {} + +/// A typed C linked list with a sentinel head intended for FFI use-cases where +/// a C subsystem manages a linked list that Rust code needs to read. Generally +/// required only for special cases. +/// +/// A sentinel head [`CListHead`] represents the entire linked list and can be used +/// for iteration over items of type `T`; it is not associated with a specific item. +/// +/// The const generic `OFFSET` specifies the byte offset of the `list_head` field within +/// the struct that `T` wraps. +/// +/// # Invariants +/// +/// - The sentinel [`CListHead`] has valid non-`NULL` `next`/`prev` pointers. +/// - `OFFSET` is the byte offset of the `list_head` field within the struct that `T` wraps. +/// - All the list's `list_head` nodes have valid non-`NULL` `next`/`prev` pointers. +#[repr(transparent)] +pub struct CList(CListHead, PhantomData); + +impl CList { + /// Create a typed [`CList`] reference from a raw sentinel `list_head` pointer. + /// + /// # Safety + /// + /// - `ptr` must be a valid pointer to an initialized sentinel `list_head` (e.g. via + /// `INIT_LIST_HEAD()`), with valid non-`NULL` `next`/`prev` pointers. + /// - `ptr` must remain valid for the lifetime `'a`. + /// - The list and all linked nodes must not be concurrently modified for the lifetime `'a`. + /// - The list must contain items where the `list_head` field is at byte offset `OFFSET`. + /// - `T` must be `#[repr(transparent)]` over the C struct. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::list_head) -> &'a Self { + // SAFETY: + // - `CList` has the same layout as `CListHead` due to `#[repr(transparent)]`. + // - Caller guarantees `ptr` is a valid, sentinel `list_head` object. + unsafe { &*ptr.cast() } + } + + /// Check if the list is empty. + #[inline] + pub fn is_empty(&self) -> bool { + !self.0.is_linked() + } + + /// Create an iterator over typed items. + #[inline] + pub fn iter(&self) -> CListIter<'_, T, OFFSET> { + let head = &self.0; + CListIter { + head_iter: CListHeadIter { + current: head.next(), + sentinel: head, + }, + _phantom: PhantomData, + } + } +} + +/// High-level iterator over typed list items. +pub struct CListIter<'a, T, const OFFSET: usize> { + head_iter: CListHeadIter<'a>, + _phantom: PhantomData<&'a T>, +} + +impl<'a, T, const OFFSET: usize> Iterator for CListIter<'a, T, OFFSET> { + type Item = &'a T; + + #[inline] + fn next(&mut self) -> Option { + let head = self.head_iter.next()?; + + // Convert to item using `OFFSET`. + // + // SAFETY: The pointer calculation is valid because `OFFSET` is derived + // from `offset_of!` per type invariants. + Some(unsafe { &*head.as_raw().byte_sub(OFFSET).cast::() }) + } +} + +impl<'a, T, const OFFSET: usize> FusedIterator for CListIter<'a, T, OFFSET> {} + +/// Create a C doubly-circular linked list interface [`CList`] from a raw `list_head` pointer. +/// +/// This macro creates a `CList` that can iterate over items of type `$rust_type` +/// linked via the `$field` field in the underlying C struct `$c_type`. +/// +/// # Arguments +/// +/// - `$head`: Raw pointer to the sentinel `list_head` object (`*mut bindings::list_head`). +/// - `$rust_type`: Each item's Rust wrapper type. +/// - `$c_type`: Each item's C struct type that contains the embedded `list_head`. +/// - `$field`: The name of the `list_head` field within the C struct. +/// +/// # Safety +/// +/// The caller must ensure: +/// +/// - `$head` is a valid, initialized sentinel `list_head` (e.g. via `INIT_LIST_HEAD()`) +/// pointing to a list that is not concurrently modified for the lifetime of the [`CList`]. +/// - The list contains items of type `$c_type` linked via an embedded `$field`. +/// - `$rust_type` is `#[repr(transparent)]` over `$c_type` or has compatible layout. +/// +/// # Examples +/// +/// Refer to the examples in the [`crate::interop::list`] module documentation. +#[macro_export] +macro_rules! clist_create { + ($head:expr, $rust_type:ty, $c_type:ty, $($field:tt).+) => {{ + // Compile-time check that field path is a `list_head`. + let _: fn(*const $c_type) -> *const $crate::bindings::list_head = + |p| &raw const (*p).$($field).+; + + // Calculate offset and create `CList`. + const OFFSET: usize = ::core::mem::offset_of!($c_type, $($field).+); + $crate::interop::list::CList::<$rust_type, OFFSET>::from_raw($head) + }}; +} +pub use clist_create; diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 34b924819288..b8eb0d217f85 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -29,6 +29,7 @@ #![feature(lint_reasons)] // // Stable since Rust 1.82.0. +#![feature(offset_of_nested)] #![feature(raw_ref_op)] // // Stable since Rust 1.83.0. @@ -110,6 +111,7 @@ pub mod id_pool; #[doc(hidden)] pub mod impl_flags; pub mod init; +pub mod interop; pub mod io; pub mod ioctl; pub mod iommu; -- cgit v1.2.3 From b9616d9721bf8a56d5038e85d2ebbe0ec9d56a94 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Fri, 20 Mar 2026 00:57:10 -0400 Subject: rust: gpu: Add GPU buddy allocator bindings Add safe Rust abstractions over the Linux kernel's GPU buddy allocator for physical memory management. The GPU buddy allocator implements a binary buddy system useful for GPU physical memory allocation. nova-core will use it for physical memory allocation. Cc: Nikola Djukic Signed-off-by: Joel Fernandes Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260320045711.43494-2-joelagnelf@nvidia.com [ * Use doc-comments for GpuBuddyAllocMode methods and GpuBuddyGuard, * Fix comma splice in GpuBuddyParams::chunk_size doc-comment, * Remove redundant summary in GpuBuddy::new doc-comment, * Drop Rust helper for gpu_buddy_block_size(). - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/gpu.rs | 6 + rust/kernel/gpu/buddy.rs | 614 +++++++++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 2 + 3 files changed, 622 insertions(+) create mode 100644 rust/kernel/gpu.rs create mode 100644 rust/kernel/gpu/buddy.rs (limited to 'rust/kernel') diff --git a/rust/kernel/gpu.rs b/rust/kernel/gpu.rs new file mode 100644 index 000000000000..1dc5d0c8c09d --- /dev/null +++ b/rust/kernel/gpu.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! GPU subsystem abstractions. + +#[cfg(CONFIG_GPU_BUDDY = "y")] +pub mod buddy; diff --git a/rust/kernel/gpu/buddy.rs b/rust/kernel/gpu/buddy.rs new file mode 100644 index 000000000000..d502ada6ebbd --- /dev/null +++ b/rust/kernel/gpu/buddy.rs @@ -0,0 +1,614 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! GPU buddy allocator bindings. +//! +//! C header: [`include/linux/gpu_buddy.h`](srctree/include/linux/gpu_buddy.h) +//! +//! This module provides Rust abstractions over the Linux kernel's GPU buddy +//! allocator, which implements a binary buddy memory allocator. +//! +//! The buddy allocator manages a contiguous address space and allocates blocks +//! in power-of-two sizes, useful for GPU physical memory management. +//! +//! # Examples +//! +//! Create a buddy allocator and perform a basic range allocation: +//! +//! ``` +//! use kernel::{ +//! gpu::buddy::{ +//! GpuBuddy, +//! GpuBuddyAllocFlags, +//! GpuBuddyAllocMode, +//! GpuBuddyParams, // +//! }, +//! prelude::*, +//! ptr::Alignment, +//! sizes::*, // +//! }; +//! +//! // Create a 1GB buddy allocator with 4KB minimum chunk size. +//! let buddy = GpuBuddy::new(GpuBuddyParams { +//! base_offset: 0, +//! size: SZ_1G as u64, +//! chunk_size: Alignment::new::(), +//! })?; +//! +//! assert_eq!(buddy.size(), SZ_1G as u64); +//! assert_eq!(buddy.chunk_size(), Alignment::new::()); +//! let initial_free = buddy.avail(); +//! +//! // Allocate 16MB. Block lands at the top of the address range. +//! let allocated = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Simple, +//! SZ_16M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_16M as u64); +//! +//! let block = allocated.iter().next().expect("expected one block"); +//! assert_eq!(block.offset(), (SZ_1G - SZ_16M) as u64); +//! assert_eq!(block.order(), 12); // 2^12 pages = 16MB +//! assert_eq!(block.size(), SZ_16M as u64); +//! assert_eq!(allocated.iter().count(), 1); +//! +//! // Dropping the allocation returns the range to the buddy allocator. +//! drop(allocated); +//! assert_eq!(buddy.avail(), initial_free); +//! # Ok::<(), Error>(()) +//! ``` +//! +//! Top-down allocation allocates from the highest addresses: +//! +//! ``` +//! # use kernel::{ +//! # gpu::buddy::{GpuBuddy, GpuBuddyAllocMode, GpuBuddyAllocFlags, GpuBuddyParams}, +//! # prelude::*, +//! # ptr::Alignment, +//! # sizes::*, // +//! # }; +//! # let buddy = GpuBuddy::new(GpuBuddyParams { +//! # base_offset: 0, +//! # size: SZ_1G as u64, +//! # chunk_size: Alignment::new::(), +//! # })?; +//! # let initial_free = buddy.avail(); +//! let topdown = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::TopDown, +//! SZ_16M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_16M as u64); +//! +//! let block = topdown.iter().next().expect("expected one block"); +//! assert_eq!(block.offset(), (SZ_1G - SZ_16M) as u64); +//! assert_eq!(block.order(), 12); +//! assert_eq!(block.size(), SZ_16M as u64); +//! +//! // Dropping the allocation returns the range to the buddy allocator. +//! drop(topdown); +//! assert_eq!(buddy.avail(), initial_free); +//! # Ok::<(), Error>(()) +//! ``` +//! +//! Non-contiguous allocation can fill fragmented memory by returning multiple +//! blocks: +//! +//! ``` +//! # use kernel::{ +//! # gpu::buddy::{ +//! # GpuBuddy, GpuBuddyAllocFlags, GpuBuddyAllocMode, GpuBuddyParams, +//! # }, +//! # prelude::*, +//! # ptr::Alignment, +//! # sizes::*, // +//! # }; +//! # let buddy = GpuBuddy::new(GpuBuddyParams { +//! # base_offset: 0, +//! # size: SZ_1G as u64, +//! # chunk_size: Alignment::new::(), +//! # })?; +//! # let initial_free = buddy.avail(); +//! // Create fragmentation by allocating 4MB blocks at [0,4M) and [8M,12M). +//! let frag1 = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Range(0..SZ_4M as u64), +//! SZ_4M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_4M as u64); +//! +//! let frag2 = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Range(SZ_8M as u64..(SZ_8M + SZ_4M) as u64), +//! SZ_4M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_8M as u64); +//! +//! // Allocate 8MB, this returns 2 blocks from the holes. +//! let fragmented = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Range(0..SZ_16M as u64), +//! SZ_8M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_16M as u64); +//! +//! let (mut count, mut total) = (0u32, 0u64); +//! for block in fragmented.iter() { +//! assert_eq!(block.size(), SZ_4M as u64); +//! total += block.size(); +//! count += 1; +//! } +//! assert_eq!(total, SZ_8M as u64); +//! assert_eq!(count, 2); +//! # Ok::<(), Error>(()) +//! ``` +//! +//! Contiguous allocation fails when only fragmented space is available: +//! +//! ``` +//! # use kernel::{ +//! # gpu::buddy::{ +//! # GpuBuddy, GpuBuddyAllocFlag, GpuBuddyAllocFlags, GpuBuddyAllocMode, GpuBuddyParams, +//! # }, +//! # prelude::*, +//! # ptr::Alignment, +//! # sizes::*, // +//! # }; +//! // Create a small 16MB buddy allocator with fragmented memory. +//! let small = GpuBuddy::new(GpuBuddyParams { +//! base_offset: 0, +//! size: SZ_16M as u64, +//! chunk_size: Alignment::new::(), +//! })?; +//! +//! let _hole1 = KBox::pin_init( +//! small.alloc_blocks( +//! GpuBuddyAllocMode::Range(0..SZ_4M as u64), +//! SZ_4M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! +//! let _hole2 = KBox::pin_init( +//! small.alloc_blocks( +//! GpuBuddyAllocMode::Range(SZ_8M as u64..(SZ_8M + SZ_4M) as u64), +//! SZ_4M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! +//! // 8MB contiguous should fail, only two non-contiguous 4MB holes exist. +//! let result = KBox::pin_init( +//! small.alloc_blocks( +//! GpuBuddyAllocMode::Simple, +//! SZ_8M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlag::Contiguous, +//! ), +//! GFP_KERNEL, +//! ); +//! assert!(result.is_err()); +//! # Ok::<(), Error>(()) +//! ``` + +use core::ops::Range; + +use crate::{ + bindings, + clist_create, + error::to_result, + interop::list::CListHead, + new_mutex, + prelude::*, + ptr::Alignment, + sync::{ + lock::mutex::MutexGuard, + Arc, + Mutex, // + }, + types::Opaque, // +}; + +/// Allocation mode for the GPU buddy allocator. +/// +/// The mode determines the primary allocation strategy. Modes are mutually +/// exclusive: an allocation is either simple, range-constrained, or top-down. +/// +/// Orthogonal modifier flags (e.g., contiguous, clear) are specified separately +/// via [`GpuBuddyAllocFlags`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum GpuBuddyAllocMode { + /// Simple allocation without constraints. + Simple, + /// Range-based allocation within the given address range. + Range(Range), + /// Allocate from top of address space downward. + TopDown, +} + +impl GpuBuddyAllocMode { + /// Returns the C flags corresponding to the allocation mode. + fn as_flags(&self) -> usize { + match self { + Self::Simple => 0, + Self::Range(_) => bindings::GPU_BUDDY_RANGE_ALLOCATION, + Self::TopDown => bindings::GPU_BUDDY_TOPDOWN_ALLOCATION, + } + } + + /// Extracts the range start/end, defaulting to `(0, 0)` for non-range modes. + fn range(&self) -> (u64, u64) { + match self { + Self::Range(range) => (range.start, range.end), + _ => (0, 0), + } + } +} + +crate::impl_flags!( + /// Modifier flags for GPU buddy allocation. + /// + /// These flags can be combined with any [`GpuBuddyAllocMode`] to control + /// additional allocation behavior. + #[derive(Clone, Copy, Default, PartialEq, Eq)] + pub struct GpuBuddyAllocFlags(usize); + + /// Individual modifier flag for GPU buddy allocation. + #[derive(Clone, Copy, PartialEq, Eq)] + pub enum GpuBuddyAllocFlag { + /// Allocate physically contiguous blocks. + Contiguous = bindings::GPU_BUDDY_CONTIGUOUS_ALLOCATION, + + /// Request allocation from cleared (zeroed) memory. + Clear = bindings::GPU_BUDDY_CLEAR_ALLOCATION, + + /// Disable trimming of partially used blocks. + TrimDisable = bindings::GPU_BUDDY_TRIM_DISABLE, + } +); + +/// Parameters for creating a GPU buddy allocator. +pub struct GpuBuddyParams { + /// Base offset (in bytes) where the managed memory region starts. + /// Allocations will be offset by this value. + pub base_offset: u64, + /// Total size (in bytes) of the address space managed by the allocator. + pub size: u64, + /// Minimum allocation unit / chunk size; must be >= 4KB. + pub chunk_size: Alignment, +} + +/// Inner structure holding the actual buddy allocator. +/// +/// # Synchronization +/// +/// The C `gpu_buddy` API requires synchronization (see `include/linux/gpu_buddy.h`). +/// Internal locking ensures all allocator and free operations are properly +/// synchronized, preventing races between concurrent allocations and the +/// freeing that occurs when [`AllocatedBlocks`] is dropped. +/// +/// # Invariants +/// +/// The inner [`Opaque`] contains an initialized buddy allocator. +#[pin_data(PinnedDrop)] +struct GpuBuddyInner { + #[pin] + inner: Opaque, + + // TODO: Replace `Mutex<()>` with `Mutex>` once `Mutex::new()` + // accepts `impl PinInit`. + #[pin] + lock: Mutex<()>, + /// Cached creation parameters (do not change after init). + params: GpuBuddyParams, +} + +impl GpuBuddyInner { + /// Create a pin-initializer for the buddy allocator. + fn new(params: GpuBuddyParams) -> impl PinInit { + let size = params.size; + let chunk_size = params.chunk_size; + + // INVARIANT: `gpu_buddy_init` returns 0 on success, at which point the + // `gpu_buddy` structure is initialized and ready for use with all + // `gpu_buddy_*` APIs. `try_pin_init!` only completes if all fields succeed, + // so the invariant holds when construction finishes. + try_pin_init!(Self { + inner <- Opaque::try_ffi_init(|ptr| { + // SAFETY: `ptr` points to valid uninitialized memory from the pin-init + // infrastructure. `gpu_buddy_init` will initialize the structure. + to_result(unsafe { + bindings::gpu_buddy_init(ptr, size, chunk_size.as_usize() as u64) + }) + }), + lock <- new_mutex!(()), + params, + }) + } + + /// Lock the mutex and return a guard for accessing the allocator. + fn lock(&self) -> GpuBuddyGuard<'_> { + GpuBuddyGuard { + inner: self, + _guard: self.lock.lock(), + } + } +} + +#[pinned_drop] +impl PinnedDrop for GpuBuddyInner { + fn drop(self: Pin<&mut Self>) { + let guard = self.lock(); + + // SAFETY: Per the type invariant, `inner` contains an initialized + // allocator. `guard` provides exclusive access. + unsafe { bindings::gpu_buddy_fini(guard.as_raw()) }; + } +} + +// SAFETY: `GpuBuddyInner` can be sent between threads. +unsafe impl Send for GpuBuddyInner {} + +// SAFETY: `GpuBuddyInner` is `Sync` because `GpuBuddyInner::lock` +// serializes all access to the C allocator, preventing data races. +unsafe impl Sync for GpuBuddyInner {} + +/// Guard that proves the lock is held, enabling access to the allocator. +/// +/// The `_guard` holds the lock for the duration of this guard's lifetime. +struct GpuBuddyGuard<'a> { + inner: &'a GpuBuddyInner, + _guard: MutexGuard<'a, ()>, +} + +impl GpuBuddyGuard<'_> { + /// Get a raw pointer to the underlying C `gpu_buddy` structure. + fn as_raw(&self) -> *mut bindings::gpu_buddy { + self.inner.inner.get() + } +} + +/// GPU buddy allocator instance. +/// +/// This structure wraps the C `gpu_buddy` allocator using reference counting. +/// The allocator is automatically cleaned up when all references are dropped. +/// +/// Refer to the module-level documentation for usage examples. +pub struct GpuBuddy(Arc); + +impl GpuBuddy { + /// Create a new buddy allocator. + /// + /// The allocator manages a contiguous address space of the given size, with the + /// specified minimum allocation unit (chunk_size must be at least 4KB). + pub fn new(params: GpuBuddyParams) -> Result { + Arc::pin_init(GpuBuddyInner::new(params), GFP_KERNEL).map(Self) + } + + /// Get the base offset for allocations. + pub fn base_offset(&self) -> u64 { + self.0.params.base_offset + } + + /// Get the chunk size (minimum allocation unit). + pub fn chunk_size(&self) -> Alignment { + self.0.params.chunk_size + } + + /// Get the total managed size. + pub fn size(&self) -> u64 { + self.0.params.size + } + + /// Get the available (free) memory in bytes. + pub fn avail(&self) -> u64 { + let guard = self.0.lock(); + + // SAFETY: Per the type invariant, `inner` contains an initialized allocator. + // `guard` provides exclusive access. + unsafe { (*guard.as_raw()).avail } + } + + /// Allocate blocks from the buddy allocator. + /// + /// Returns a pin-initializer for [`AllocatedBlocks`]. + pub fn alloc_blocks( + &self, + mode: GpuBuddyAllocMode, + size: u64, + min_block_size: Alignment, + flags: impl Into, + ) -> impl PinInit { + let buddy_arc = Arc::clone(&self.0); + let (start, end) = mode.range(); + let mode_flags = mode.as_flags(); + let modifier_flags = flags.into(); + + // Create pin-initializer that initializes list and allocates blocks. + try_pin_init!(AllocatedBlocks { + buddy: buddy_arc, + list <- CListHead::new(), + _: { + // Reject zero-sized or inverted ranges. + if let GpuBuddyAllocMode::Range(range) = &mode { + if range.is_empty() { + Err::<(), Error>(EINVAL)?; + } + } + + // Lock while allocating to serialize with concurrent frees. + let guard = buddy.lock(); + + // SAFETY: Per the type invariant, `inner` contains an initialized + // allocator. `guard` provides exclusive access. + to_result(unsafe { + bindings::gpu_buddy_alloc_blocks( + guard.as_raw(), + start, + end, + size, + min_block_size.as_usize() as u64, + list.as_raw(), + mode_flags | usize::from(modifier_flags), + ) + })? + } + }) + } +} + +/// Allocated blocks from the buddy allocator with automatic cleanup. +/// +/// This structure owns a list of allocated blocks and ensures they are +/// automatically freed when dropped. Use `iter()` to iterate over all +/// allocated blocks. +/// +/// # Invariants +/// +/// - `list` is an initialized, valid list head containing allocated blocks. +#[pin_data(PinnedDrop)] +pub struct AllocatedBlocks { + #[pin] + list: CListHead, + buddy: Arc, +} + +impl AllocatedBlocks { + /// Check if the block list is empty. + pub fn is_empty(&self) -> bool { + // An empty list head points to itself. + !self.list.is_linked() + } + + /// Iterate over allocated blocks. + /// + /// Returns an iterator yielding [`AllocatedBlock`] values. Each [`AllocatedBlock`] + /// borrows `self` and is only valid for the duration of that borrow. + pub fn iter(&self) -> impl Iterator> + '_ { + let head = self.list.as_raw(); + // SAFETY: Per the type invariant, `list` is an initialized sentinel `list_head` + // and is not concurrently modified (we hold a `&self` borrow). The list contains + // `gpu_buddy_block` items linked via `__bindgen_anon_1.link`. `Block` is + // `#[repr(transparent)]` over `gpu_buddy_block`. + let clist = unsafe { + clist_create!( + head, + Block, + bindings::gpu_buddy_block, + __bindgen_anon_1.link + ) + }; + + clist + .iter() + .map(|this| AllocatedBlock { this, blocks: self }) + } +} + +#[pinned_drop] +impl PinnedDrop for AllocatedBlocks { + fn drop(self: Pin<&mut Self>) { + let guard = self.buddy.lock(); + + // SAFETY: + // - list is valid per the type's invariants. + // - guard provides exclusive access to the allocator. + unsafe { + bindings::gpu_buddy_free_list(guard.as_raw(), self.list.as_raw(), 0); + } + } +} + +/// A GPU buddy block. +/// +/// Transparent wrapper over C `gpu_buddy_block` structure. This type is returned +/// as references during iteration over [`AllocatedBlocks`]. +/// +/// # Invariants +/// +/// The inner [`Opaque`] contains a valid, allocated `gpu_buddy_block`. +#[repr(transparent)] +struct Block(Opaque); + +impl Block { + /// Get a raw pointer to the underlying C block. + fn as_raw(&self) -> *mut bindings::gpu_buddy_block { + self.0.get() + } + + /// Get the block's raw offset in the buddy address space (without base offset). + fn offset(&self) -> u64 { + // SAFETY: `self.as_raw()` is valid per the type's invariants. + unsafe { bindings::gpu_buddy_block_offset(self.as_raw()) } + } + + /// Get the block order. + fn order(&self) -> u32 { + // SAFETY: `self.as_raw()` is valid per the type's invariants. + unsafe { bindings::gpu_buddy_block_order(self.as_raw()) } + } +} + +// SAFETY: `Block` is a wrapper around `gpu_buddy_block` which can be +// sent across threads safely. +unsafe impl Send for Block {} + +// SAFETY: `Block` is only accessed through shared references after +// allocation, and thus safe to access concurrently across threads. +unsafe impl Sync for Block {} + +/// A buddy block paired with its owning [`AllocatedBlocks`] context. +/// +/// Unlike a raw block, which only knows its offset within the buddy address +/// space, an [`AllocatedBlock`] also has access to the allocator's `base_offset` +/// and `chunk_size`, enabling it to compute absolute offsets and byte sizes. +/// +/// Returned by [`AllocatedBlocks::iter()`]. +pub struct AllocatedBlock<'a> { + this: &'a Block, + blocks: &'a AllocatedBlocks, +} + +impl AllocatedBlock<'_> { + /// Get the block's offset in the address space. + /// + /// Returns the absolute offset including the allocator's base offset. + /// This is the actual address to use for accessing the allocated memory. + pub fn offset(&self) -> u64 { + self.blocks.buddy.params.base_offset + self.this.offset() + } + + /// Get the block order (size = chunk_size << order). + pub fn order(&self) -> u32 { + self.this.order() + } + + /// Get the block's size in bytes. + pub fn size(&self) -> u64 { + (self.blocks.buddy.params.chunk_size.as_usize() as u64) << self.this.order() + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index b8eb0d217f85..e0837ffc91bf 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -105,6 +105,8 @@ pub mod faux; pub mod firmware; pub mod fmt; pub mod fs; +#[cfg(CONFIG_GPU_BUDDY = "y")] +pub mod gpu; #[cfg(CONFIG_I2C = "y")] pub mod i2c; pub mod id_pool; -- cgit v1.2.3 From 7ea1a61129b26709fe85cf5d50da5c47458deb3a Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 20 Mar 2026 20:45:36 +0100 Subject: rust: dma: use "kernel vertical" style for imports Convert all imports to use "kernel vertical" style. With this, subsequent patches neither introduce unrelated changes nor leave an inconsistent import pattern. While at it, drop unnecessary imports covered by prelude::*. Link: https://docs.kernel.org/rust/coding-guidelines.html#imports Reviewed-by: Gary Guo Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260320194626.36263-2-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/dma.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index a396f8435739..2eea7e2f8f04 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -5,12 +5,20 @@ //! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h) use crate::{ - bindings, build_assert, device, - device::{Bound, Core}, - error::{to_result, Result}, + bindings, + build_assert, + device::{ + self, + Bound, + Core, // + }, + error::to_result, prelude::*, sync::aref::ARef, - transmute::{AsBytes, FromBytes}, + transmute::{ + AsBytes, + FromBytes, // + }, // }; use core::ptr::NonNull; -- cgit v1.2.3 From d9aee73c56ee971b08173071ad93fa5ebf00a32e Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Fri, 20 Mar 2026 20:45:37 +0100 Subject: rust: dma: add generalized container for types other than slices Currently, `CoherentAllocation` is concecptually a DMA coherent container of a slice of `[T]` of runtime-checked length. Generalize it by creating `dma::Coherent` which can hold any value of `T`. `Coherent::alloc_with_attrs` is implemented but not yet exposed, as I believe we should not expose the way to obtain an uninitialized coherent region. `Coherent<[T]>` provides a `len` method instead of the previous `count()` method to be consistent with methods on slices. The existing type is re-defined as a type alias of `Coherent<[T]>` to ease transition. Methods in use are not yet removed. Signed-off-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260320194626.36263-3-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/device.rs | 4 +- rust/kernel/dma.rs | 361 +++++++++++++++++++++++++++++++------------------- 2 files changed, 226 insertions(+), 139 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 94e0548e7687..379058eca2ed 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -575,7 +575,7 @@ pub struct CoreInternal; /// The bound context indicates that for the entire duration of the lifetime of a [`Device`] /// reference, the [`Device`] is guaranteed to be bound to a driver. /// -/// Some APIs, such as [`dma::CoherentAllocation`] or [`Devres`] rely on the [`Device`] to be bound, +/// Some APIs, such as [`dma::Coherent`] or [`Devres`] rely on the [`Device`] to be bound, /// which can be proven with the [`Bound`] device context. /// /// Any abstraction that can guarantee a scope where the corresponding bus device is bound, should @@ -584,7 +584,7 @@ pub struct CoreInternal; /// /// [`Devres`]: kernel::devres::Devres /// [`Devres::access`]: kernel::devres::Devres::access -/// [`dma::CoherentAllocation`]: kernel::dma::CoherentAllocation +/// [`dma::Coherent`]: kernel::dma::Coherent pub struct Bound; mod private { diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 2eea7e2f8f04..ff3e147f1a23 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -6,7 +6,6 @@ use crate::{ bindings, - build_assert, device::{ self, Bound, @@ -14,6 +13,7 @@ use crate::{ }, error::to_result, prelude::*, + ptr::KnownSize, sync::aref::ARef, transmute::{ AsBytes, @@ -357,18 +357,17 @@ impl From for bindings::dma_data_direction { /// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map /// large coherent DMA regions. /// -/// A [`CoherentAllocation`] instance contains a pointer to the allocated region (in the +/// A [`Coherent`] instance contains a pointer to the allocated region (in the /// processor's virtual address space) and the device address which can be given to the device -/// as the DMA address base of the region. The region is released once [`CoherentAllocation`] +/// as the DMA address base of the region. The region is released once [`Coherent`] /// is dropped. /// /// # Invariants /// -/// - For the lifetime of an instance of [`CoherentAllocation`], the `cpu_addr` is a valid pointer +/// - For the lifetime of an instance of [`Coherent`], the `cpu_addr` is a valid pointer /// to an allocated region of coherent memory and `dma_handle` is the DMA address base of the /// region. -/// - The size in bytes of the allocation is equal to `size_of:: * count`. -/// - `size_of:: * count` fits into a `usize`. +/// - The size in bytes of the allocation is equal to size information via pointer. // TODO // // DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness @@ -379,124 +378,260 @@ impl From for bindings::dma_data_direction { // allocation from surviving device unbind; it would require RCU read side critical sections to // access the memory, which may require subsequent unnecessary copies. // -// Hence, find a way to revoke the device resources of a `CoherentAllocation`, but not the -// entire `CoherentAllocation` including the allocated memory itself. -pub struct CoherentAllocation { +// Hence, find a way to revoke the device resources of a `Coherent`, but not the +// entire `Coherent` including the allocated memory itself. +pub struct Coherent { dev: ARef, dma_handle: DmaAddress, - count: usize, cpu_addr: NonNull, dma_attrs: Attrs, } -impl CoherentAllocation { - /// Allocates a region of `size_of:: * count` of coherent memory. +impl Coherent { + /// Returns the size in bytes of this allocation. + #[inline] + pub fn size(&self) -> usize { + T::size(self.cpu_addr.as_ptr()) + } + + /// Returns the raw pointer to the allocated region in the CPU's virtual address space. + #[inline] + pub fn as_ptr(&self) -> *const T { + self.cpu_addr.as_ptr() + } + + /// Returns the raw pointer to the allocated region in the CPU's virtual address space as + /// a mutable pointer. + #[inline] + pub fn as_mut_ptr(&self) -> *mut T { + self.cpu_addr.as_ptr() + } + + /// Returns a DMA handle which may be given to the device as the DMA address base of + /// the region. + #[inline] + pub fn dma_handle(&self) -> DmaAddress { + self.dma_handle + } + + /// Returns a reference to the data in the region. /// - /// # Examples + /// # Safety /// - /// ``` - /// # use kernel::device::{Bound, Device}; - /// use kernel::dma::{attrs::*, CoherentAllocation}; + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// slice is live. + /// * Callers must ensure that this call does not race with a write to the same region while + /// the returned slice is live. + #[inline] + pub unsafe fn as_ref(&self) -> &T { + // SAFETY: per safety requirement. + unsafe { &*self.as_ptr() } + } + + /// Returns a mutable reference to the data in the region. /// - /// # fn test(dev: &Device) -> Result { - /// let c: CoherentAllocation = - /// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; - /// # Ok::<(), Error>(()) } - /// ``` - pub fn alloc_attrs( + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// slice is live. + /// * Callers must ensure that this call does not race with a read or write to the same region + /// while the returned slice is live. + #[expect(clippy::mut_from_ref, reason = "unsafe to use API")] + #[inline] + pub unsafe fn as_mut(&self) -> &mut T { + // SAFETY: per safety requirement. + unsafe { &mut *self.as_mut_ptr() } + } + + /// Reads the value of `field` and ensures that its type is [`FromBytes`]. + /// + /// # Safety + /// + /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is + /// validated beforehand. + /// + /// Public but hidden since it should only be used from [`dma_read`] macro. + #[doc(hidden)] + pub unsafe fn field_read(&self, field: *const F) -> F { + // SAFETY: + // - By the safety requirements field is valid. + // - Using read_volatile() here is not sound as per the usual rules, the usage here is + // a special exception with the following notes in place. When dealing with a potential + // race from a hardware or code outside kernel (e.g. user-space program), we need that + // read on a valid memory is not UB. Currently read_volatile() is used for this, and the + // rationale behind is that it should generate the same code as READ_ONCE() which the + // kernel already relies on to avoid UB on data races. Note that the usage of + // read_volatile() is limited to this particular case, it cannot be used to prevent + // the UB caused by racing between two kernel functions nor do they provide atomicity. + unsafe { field.read_volatile() } + } + + /// Writes a value to `field` and ensures that its type is [`AsBytes`]. + /// + /// # Safety + /// + /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is + /// validated beforehand. + /// + /// Public but hidden since it should only be used from [`dma_write`] macro. + #[doc(hidden)] + pub unsafe fn field_write(&self, field: *mut F, val: F) { + // SAFETY: + // - By the safety requirements field is valid. + // - Using write_volatile() here is not sound as per the usual rules, the usage here is + // a special exception with the following notes in place. When dealing with a potential + // race from a hardware or code outside kernel (e.g. user-space program), we need that + // write on a valid memory is not UB. Currently write_volatile() is used for this, and the + // rationale behind is that it should generate the same code as WRITE_ONCE() which the + // kernel already relies on to avoid UB on data races. Note that the usage of + // write_volatile() is limited to this particular case, it cannot be used to prevent + // the UB caused by racing between two kernel functions nor do they provide atomicity. + unsafe { field.write_volatile(val) } + } +} + +impl Coherent { + /// Allocates a region of `T` of coherent memory. + #[expect(unused)] + fn alloc_with_attrs( dev: &device::Device, - count: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, - ) -> Result> { - build_assert!( - core::mem::size_of::() > 0, - "It doesn't make sense for the allocated type to be a ZST" - ); + ) -> Result { + const { + assert!( + core::mem::size_of::() > 0, + "It doesn't make sense for the allocated type to be a ZST" + ); + } - let size = count - .checked_mul(core::mem::size_of::()) - .ok_or(EOVERFLOW)?; let mut dma_handle = 0; // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. let addr = unsafe { bindings::dma_alloc_attrs( dev.as_raw(), - size, + core::mem::size_of::(), &mut dma_handle, gfp_flags.as_raw(), dma_attrs.as_raw(), ) }; - let addr = NonNull::new(addr).ok_or(ENOMEM)?; + let cpu_addr = NonNull::new(addr.cast()).ok_or(ENOMEM)?; // INVARIANT: - // - We just successfully allocated a coherent region which is accessible for - // `count` elements, hence the cpu address is valid. We also hold a refcounted reference - // to the device. - // - The allocated `size` is equal to `size_of:: * count`. - // - The allocated `size` fits into a `usize`. + // - We just successfully allocated a coherent region which is adequately sized for `T`, + // hence the cpu address is valid. + // - We also hold a refcounted reference to the device. Ok(Self { dev: dev.into(), dma_handle, - count, - cpu_addr: addr.cast(), + cpu_addr, dma_attrs, }) } - /// Performs the same functionality as [`CoherentAllocation::alloc_attrs`], except the - /// `dma_attrs` is 0 by default. - pub fn alloc_coherent( + /// Allocates a region of `[T; len]` of coherent memory. + fn alloc_slice_with_attrs( dev: &device::Device, - count: usize, + len: usize, gfp_flags: kernel::alloc::Flags, - ) -> Result> { - CoherentAllocation::alloc_attrs(dev, count, gfp_flags, Attrs(0)) + dma_attrs: Attrs, + ) -> Result> { + const { + assert!( + core::mem::size_of::() > 0, + "It doesn't make sense for the allocated type to be a ZST" + ); + } + + // `dma_alloc_attrs` cannot handle zero-length allocation, bail early. + if len == 0 { + Err(EINVAL)?; + } + + let size = core::mem::size_of::().checked_mul(len).ok_or(ENOMEM)?; + let mut dma_handle = 0; + // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. + let addr = unsafe { + bindings::dma_alloc_attrs( + dev.as_raw(), + size, + &mut dma_handle, + gfp_flags.as_raw(), + dma_attrs.as_raw(), + ) + }; + let cpu_addr = NonNull::slice_from_raw_parts(NonNull::new(addr.cast()).ok_or(ENOMEM)?, len); + // INVARIANT: + // - We just successfully allocated a coherent region which is adequately sized for + // `[T; len]`, hence the cpu address is valid. + // - We also hold a refcounted reference to the device. + Ok(Coherent { + dev: dev.into(), + dma_handle, + cpu_addr, + dma_attrs, + }) } +} +impl Coherent<[T]> { /// Returns the number of elements `T` in this allocation. /// /// Note that this is not the size of the allocation in bytes, which is provided by /// [`Self::size`]. - pub fn count(&self) -> usize { - self.count + #[inline] + #[expect(clippy::len_without_is_empty, reason = "Coherent slice is never empty")] + pub fn len(&self) -> usize { + self.cpu_addr.len() } +} - /// Returns the size in bytes of this allocation. - pub fn size(&self) -> usize { - // INVARIANT: The type invariant of `Self` guarantees that `size_of:: * count` fits into - // a `usize`. - self.count * core::mem::size_of::() - } +// Type alias for compatibility. +#[doc(hidden)] +pub type CoherentAllocation = Coherent<[T]>; - /// Returns the raw pointer to the allocated region in the CPU's virtual address space. - #[inline] - pub fn as_ptr(&self) -> *const [T] { - core::ptr::slice_from_raw_parts(self.cpu_addr.as_ptr(), self.count) +impl CoherentAllocation { + /// Allocates a region of `size_of:: * count` of coherent memory. + /// + /// # Examples + /// + /// ``` + /// # use kernel::device::{Bound, Device}; + /// use kernel::dma::{attrs::*, CoherentAllocation}; + /// + /// # fn test(dev: &Device) -> Result { + /// let c: CoherentAllocation = + /// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// # Ok::<(), Error>(()) } + /// ``` + pub fn alloc_attrs( + dev: &device::Device, + count: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result> { + Coherent::alloc_slice_with_attrs(dev, count, gfp_flags, dma_attrs) } - /// Returns the raw pointer to the allocated region in the CPU's virtual address space as - /// a mutable pointer. - #[inline] - pub fn as_mut_ptr(&self) -> *mut [T] { - core::ptr::slice_from_raw_parts_mut(self.cpu_addr.as_ptr(), self.count) + /// Performs the same functionality as [`CoherentAllocation::alloc_attrs`], except the + /// `dma_attrs` is 0 by default. + pub fn alloc_coherent( + dev: &device::Device, + count: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result> { + CoherentAllocation::alloc_attrs(dev, count, gfp_flags, Attrs(0)) } /// Returns the base address to the allocated region in the CPU's virtual address space. pub fn start_ptr(&self) -> *const T { - self.cpu_addr.as_ptr() + self.as_ptr().cast() } /// Returns the base address to the allocated region in the CPU's virtual address space as /// a mutable pointer. pub fn start_ptr_mut(&mut self) -> *mut T { - self.cpu_addr.as_ptr() - } - - /// Returns a DMA handle which may be given to the device as the DMA address base of - /// the region. - pub fn dma_handle(&self) -> DmaAddress { - self.dma_handle + self.as_mut_ptr().cast() } /// Returns a DMA handle starting at `offset` (in units of `T`) which may be given to the @@ -504,11 +639,9 @@ impl CoherentAllocation { /// /// Returns `EINVAL` if `offset` is not within the bounds of the allocation. pub fn dma_handle_with_offset(&self, offset: usize) -> Result { - if offset >= self.count { + if offset >= self.len() { Err(EINVAL) } else { - // INVARIANT: The type invariant of `Self` guarantees that `size_of:: * count` fits - // into a `usize`, and `offset` is inferior to `count`. Ok(self.dma_handle + (offset * core::mem::size_of::()) as DmaAddress) } } @@ -516,7 +649,7 @@ impl CoherentAllocation { /// Common helper to validate a range applied from the allocated region in the CPU's virtual /// address space. fn validate_range(&self, offset: usize, count: usize) -> Result { - if offset.checked_add(count).ok_or(EOVERFLOW)? > self.count { + if offset.checked_add(count).ok_or(EOVERFLOW)? > self.len() { return Err(EINVAL); } Ok(()) @@ -601,66 +734,20 @@ impl CoherentAllocation { }; Ok(()) } - - /// Reads the value of `field` and ensures that its type is [`FromBytes`]. - /// - /// # Safety - /// - /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is - /// validated beforehand. - /// - /// Public but hidden since it should only be used from [`dma_read`] macro. - #[doc(hidden)] - pub unsafe fn field_read(&self, field: *const F) -> F { - // SAFETY: - // - By the safety requirements field is valid. - // - Using read_volatile() here is not sound as per the usual rules, the usage here is - // a special exception with the following notes in place. When dealing with a potential - // race from a hardware or code outside kernel (e.g. user-space program), we need that - // read on a valid memory is not UB. Currently read_volatile() is used for this, and the - // rationale behind is that it should generate the same code as READ_ONCE() which the - // kernel already relies on to avoid UB on data races. Note that the usage of - // read_volatile() is limited to this particular case, it cannot be used to prevent - // the UB caused by racing between two kernel functions nor do they provide atomicity. - unsafe { field.read_volatile() } - } - - /// Writes a value to `field` and ensures that its type is [`AsBytes`]. - /// - /// # Safety - /// - /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is - /// validated beforehand. - /// - /// Public but hidden since it should only be used from [`dma_write`] macro. - #[doc(hidden)] - pub unsafe fn field_write(&self, field: *mut F, val: F) { - // SAFETY: - // - By the safety requirements field is valid. - // - Using write_volatile() here is not sound as per the usual rules, the usage here is - // a special exception with the following notes in place. When dealing with a potential - // race from a hardware or code outside kernel (e.g. user-space program), we need that - // write on a valid memory is not UB. Currently write_volatile() is used for this, and the - // rationale behind is that it should generate the same code as WRITE_ONCE() which the - // kernel already relies on to avoid UB on data races. Note that the usage of - // write_volatile() is limited to this particular case, it cannot be used to prevent - // the UB caused by racing between two kernel functions nor do they provide atomicity. - unsafe { field.write_volatile(val) } - } } /// Note that the device configured to do DMA must be halted before this object is dropped. -impl Drop for CoherentAllocation { +impl Drop for Coherent { fn drop(&mut self) { - let size = self.count * core::mem::size_of::(); + let size = T::size(self.cpu_addr.as_ptr()); // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. // The cpu address, and the dma handle are valid due to the type invariants on - // `CoherentAllocation`. + // `Coherent`. unsafe { bindings::dma_free_attrs( self.dev.as_raw(), size, - self.start_ptr_mut().cast(), + self.cpu_addr.as_ptr().cast(), self.dma_handle, self.dma_attrs.as_raw(), ) @@ -668,20 +755,20 @@ impl Drop for CoherentAllocation { } } -// SAFETY: It is safe to send a `CoherentAllocation` to another thread if `T` +// SAFETY: It is safe to send a `Coherent` to another thread if `T` // can be sent to another thread. -unsafe impl Send for CoherentAllocation {} +unsafe impl Send for Coherent {} /// Reads a field of an item from an allocated region of structs. /// /// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating -/// to a [`CoherentAllocation`] and `proj` is a [projection specification](kernel::ptr::project!). +/// to a [`Coherent`] and `proj` is a [projection specification](kernel::ptr::project!). /// /// # Examples /// /// ``` /// use kernel::device::Device; -/// use kernel::dma::{attrs::*, CoherentAllocation}; +/// use kernel::dma::{attrs::*, Coherent}; /// /// struct MyStruct { field: u32, } /// @@ -690,7 +777,7 @@ unsafe impl Send for CoherentAllocation {} /// // SAFETY: Instances of `MyStruct` have no uninitialized portions. /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; /// -/// # fn test(alloc: &kernel::dma::CoherentAllocation) -> Result { +/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result { /// let whole = kernel::dma_read!(alloc, [2]?); /// let field = kernel::dma_read!(alloc, [1]?.field); /// # Ok::<(), Error>(()) } @@ -700,17 +787,17 @@ macro_rules! dma_read { ($dma:expr, $($proj:tt)*) => {{ let dma = &$dma; let ptr = $crate::ptr::project!( - $crate::dma::CoherentAllocation::as_ptr(dma), $($proj)* + $crate::dma::Coherent::as_ptr(dma), $($proj)* ); // SAFETY: The pointer created by the projection is within the DMA region. - unsafe { $crate::dma::CoherentAllocation::field_read(dma, ptr) } + unsafe { $crate::dma::Coherent::field_read(dma, ptr) } }}; } /// Writes to a field of an item from an allocated region of structs. /// /// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression -/// evaluating to a [`CoherentAllocation`], `proj` is a +/// evaluating to a [`Coherent`], `proj` is a /// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the /// projected location. /// @@ -718,7 +805,7 @@ macro_rules! dma_read { /// /// ``` /// use kernel::device::Device; -/// use kernel::dma::{attrs::*, CoherentAllocation}; +/// use kernel::dma::{attrs::*, Coherent}; /// /// struct MyStruct { member: u32, } /// @@ -727,7 +814,7 @@ macro_rules! dma_read { /// // SAFETY: Instances of `MyStruct` have no uninitialized portions. /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; /// -/// # fn test(alloc: &kernel::dma::CoherentAllocation) -> Result { +/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result { /// kernel::dma_write!(alloc, [2]?.member, 0xf); /// kernel::dma_write!(alloc, [1]?, MyStruct { member: 0xf }); /// # Ok::<(), Error>(()) } @@ -737,11 +824,11 @@ macro_rules! dma_write { (@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{ let dma = &$dma; let ptr = $crate::ptr::project!( - mut $crate::dma::CoherentAllocation::as_mut_ptr(dma), $($proj)* + mut $crate::dma::Coherent::as_mut_ptr(dma), $($proj)* ); let val = $val; // SAFETY: The pointer created by the projection is within the DMA region. - unsafe { $crate::dma::CoherentAllocation::field_write(dma, ptr, val) } + unsafe { $crate::dma::Coherent::field_write(dma, ptr, val) } }}; (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*]) -- cgit v1.2.3 From f84ecffa3f745572164c1269f20eec2589d432c9 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Fri, 20 Mar 2026 20:45:38 +0100 Subject: rust: dma: add zeroed constructor to `Coherent` These constructors create a coherent container of a single object instead of slice. They are named `zeroed` and `zeroed_with_attrs` to emphasis that they are created initialized zeroed. It is intended that there'll be new constructors that take `PinInit` instead of zeroing. Signed-off-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260320194626.36263-4-dakr@kernel.org [ Use kernel import style. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/dma.rs | 93 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 85 insertions(+), 8 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index ff3e147f1a23..59477c865dd3 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -47,7 +47,7 @@ pub trait Device: AsRef> { /// # Safety /// /// This method must not be called concurrently with any DMA allocation or mapping primitives, - /// such as [`CoherentAllocation::alloc_attrs`]. + /// such as [`Coherent::zeroed`]. unsafe fn dma_set_mask(&self, mask: DmaMask) -> Result { // SAFETY: // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. @@ -64,7 +64,7 @@ pub trait Device: AsRef> { /// # Safety /// /// This method must not be called concurrently with any DMA allocation or mapping primitives, - /// such as [`CoherentAllocation::alloc_attrs`]. + /// such as [`Coherent::zeroed`]. unsafe fn dma_set_coherent_mask(&self, mask: DmaMask) -> Result { // SAFETY: // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. @@ -83,7 +83,7 @@ pub trait Device: AsRef> { /// # Safety /// /// This method must not be called concurrently with any DMA allocation or mapping primitives, - /// such as [`CoherentAllocation::alloc_attrs`]. + /// such as [`Coherent::zeroed`]. unsafe fn dma_set_mask_and_coherent(&self, mask: DmaMask) -> Result { // SAFETY: // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. @@ -102,7 +102,7 @@ pub trait Device: AsRef> { /// # Safety /// /// This method must not be called concurrently with any DMA allocation or mapping primitives, - /// such as [`CoherentAllocation::alloc_attrs`]. + /// such as [`Coherent::zeroed`]. unsafe fn dma_set_max_seg_size(&self, size: u32) { // SAFETY: // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. @@ -202,12 +202,12 @@ impl DmaMask { /// /// ``` /// # use kernel::device::{Bound, Device}; -/// use kernel::dma::{attrs::*, CoherentAllocation}; +/// use kernel::dma::{attrs::*, Coherent}; /// /// # fn test(dev: &Device) -> Result { /// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN; -/// let c: CoherentAllocation = -/// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, attribs)?; +/// let c: Coherent<[u64]> = +/// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, attribs)?; /// # Ok::<(), Error>(()) } /// ``` #[derive(Clone, Copy, PartialEq)] @@ -492,7 +492,6 @@ impl Coherent { impl Coherent { /// Allocates a region of `T` of coherent memory. - #[expect(unused)] fn alloc_with_attrs( dev: &device::Device, gfp_flags: kernel::alloc::Flags, @@ -529,6 +528,41 @@ impl Coherent { }) } + /// Allocates a region of type `T` of coherent memory. + /// + /// # Examples + /// + /// ``` + /// # use kernel::device::{ + /// # Bound, + /// # Device, + /// # }; + /// use kernel::dma::{ + /// attrs::*, + /// Coherent, + /// }; + /// + /// # fn test(dev: &Device) -> Result { + /// let c: Coherent<[u64; 4]> = + /// Coherent::zeroed_with_attrs(dev, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + pub fn zeroed_with_attrs( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + Self::alloc_with_attrs(dev, gfp_flags | __GFP_ZERO, dma_attrs) + } + + /// Performs the same functionality as [`Coherent::zeroed_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn zeroed(dev: &device::Device, gfp_flags: kernel::alloc::Flags) -> Result { + Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) + } + /// Allocates a region of `[T; len]` of coherent memory. fn alloc_slice_with_attrs( dev: &device::Device, @@ -572,6 +606,49 @@ impl Coherent { dma_attrs, }) } + + /// Allocates a zeroed region of type `T` of coherent memory. + /// + /// Unlike `Coherent::<[T; N]>::zeroed_with_attrs`, `Coherent::::zeroed_slices` support + /// a runtime length. + /// + /// # Examples + /// + /// ``` + /// # use kernel::device::{ + /// # Bound, + /// # Device, + /// # }; + /// use kernel::dma::{ + /// attrs::*, + /// Coherent, + /// }; + /// + /// # fn test(dev: &Device) -> Result { + /// let c: Coherent<[u64]> = + /// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + pub fn zeroed_slice_with_attrs( + dev: &device::Device, + len: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result> { + Coherent::alloc_slice_with_attrs(dev, len, gfp_flags | __GFP_ZERO, dma_attrs) + } + + /// Performs the same functionality as [`Coherent::zeroed_slice_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn zeroed_slice( + dev: &device::Device, + len: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result> { + Self::zeroed_slice_with_attrs(dev, len, gfp_flags, Attrs(0)) + } } impl Coherent<[T]> { -- cgit v1.2.3 From 80f4a7b5138d1c427ee5626e8a796aa6b2994a95 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 20 Mar 2026 20:45:39 +0100 Subject: rust: dma: introduce dma::CoherentBox for memory initialization Currently, dma::Coherent cannot safely provide (mutable) access to its underlying memory because the memory might be concurrently accessed by a DMA device. This makes it difficult to safely initialize the memory before handing it over to the hardware. Introduce dma::CoherentBox, a type that encapsulates a dma::Coherent before its DMA address is exposed to the device. dma::CoherentBox can guarantee exclusive access to the inner dma::Coherent and implement Deref and DerefMut. Once the memory is properly initialized, dma::CoherentBox can be converted into a regular dma::Coherent. Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260320194626.36263-5-dakr@kernel.org [ Remove unnecessary trait bounds. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/dma.rs | 154 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 153 insertions(+), 1 deletion(-) (limited to 'rust/kernel') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 59477c865dd3..5b41603cee2e 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -20,7 +20,13 @@ use crate::{ FromBytes, // }, // }; -use core::ptr::NonNull; +use core::{ + ops::{ + Deref, + DerefMut, // + }, + ptr::NonNull, // +}; /// DMA address type. /// @@ -352,6 +358,152 @@ impl From for bindings::dma_data_direction { } } +/// CPU-owned DMA allocation that can be converted into a device-shared [`Coherent`] object. +/// +/// Unlike [`Coherent`], a [`CoherentBox`] is guaranteed to be fully owned by the CPU -- its DMA +/// address is not exposed and it cannot be accessed by a device. This means it can safely be used +/// like a normal boxed allocation (e.g. direct reads, writes, and mutable slices are all safe). +/// +/// A typical use is to allocate a [`CoherentBox`], populate it with normal CPU access, and then +/// convert it into a [`Coherent`] object to share it with the device. +/// +/// # Examples +/// +/// `CoherentBox`: +/// +/// ``` +/// # use kernel::device::{ +/// # Bound, +/// # Device, +/// # }; +/// use kernel::dma::{attrs::*, +/// Coherent, +/// CoherentBox, +/// }; +/// +/// # fn test(dev: &Device) -> Result { +/// let mut dmem: CoherentBox = CoherentBox::zeroed(dev, GFP_KERNEL)?; +/// *dmem = 42; +/// let dmem: Coherent = dmem.into(); +/// # Ok::<(), Error>(()) } +/// ``` +/// +/// `CoherentBox<[T]>`: +/// +/// +/// ``` +/// # use kernel::device::{ +/// # Bound, +/// # Device, +/// # }; +/// use kernel::dma::{attrs::*, +/// Coherent, +/// CoherentBox, +/// }; +/// +/// # fn test(dev: &Device) -> Result { +/// let mut dmem: CoherentBox<[u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?; +/// dmem.fill(42); +/// let dmem: Coherent<[u64]> = dmem.into(); +/// # Ok::<(), Error>(()) } +/// ``` +pub struct CoherentBox(Coherent); + +impl CoherentBox<[T]> { + /// [`CoherentBox`] variant of [`Coherent::zeroed_slice_with_attrs`]. + #[inline] + pub fn zeroed_slice_with_attrs( + dev: &device::Device, + count: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + Coherent::zeroed_slice_with_attrs(dev, count, gfp_flags, dma_attrs).map(Self) + } + + /// Same as [CoherentBox::zeroed_slice_with_attrs], but with `dma::Attrs(0)`. + #[inline] + pub fn zeroed_slice( + dev: &device::Device, + count: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result { + Self::zeroed_slice_with_attrs(dev, count, gfp_flags, Attrs(0)) + } + + /// Initializes the element at `i` using the given initializer. + /// + /// Returns `EINVAL` if `i` is out of bounds. + pub fn init_at(&mut self, i: usize, init: impl Init) -> Result + where + Error: From, + { + if i >= self.0.len() { + return Err(EINVAL); + } + + let ptr = &raw mut self[i]; + + // SAFETY: + // - `ptr` is valid, properly aligned, and within this allocation. + // - `T: AsBytes + FromBytes` guarantees all bit patterns are valid, so partial writes on + // error cannot leave the element in an invalid state. + // - The DMA address has not been exposed yet, so there is no concurrent device access. + unsafe { init.__init(ptr)? }; + + Ok(()) + } +} + +impl CoherentBox { + /// Same as [`CoherentBox::zeroed_slice_with_attrs`], but for a single element. + #[inline] + pub fn zeroed_with_attrs( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + Coherent::zeroed_with_attrs(dev, gfp_flags, dma_attrs).map(Self) + } + + /// Same as [`CoherentBox::zeroed_slice`], but for a single element. + #[inline] + pub fn zeroed(dev: &device::Device, gfp_flags: kernel::alloc::Flags) -> Result { + Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) + } +} + +impl Deref for CoherentBox { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: + // - We have not exposed the DMA address yet, so there can't be any concurrent access by a + // device. + // - We have exclusive access to `self.0`. + unsafe { self.0.as_ref() } + } +} + +impl DerefMut for CoherentBox { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: + // - We have not exposed the DMA address yet, so there can't be any concurrent access by a + // device. + // - We have exclusive access to `self.0`. + unsafe { self.0.as_mut() } + } +} + +impl From> for Coherent { + #[inline] + fn from(value: CoherentBox) -> Self { + value.0 + } +} + /// An abstraction of the `dma_alloc_coherent` API. /// /// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map -- cgit v1.2.3 From 40251bac0e00818b0b632d2d154e4f1f815849eb Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 20 Mar 2026 20:45:40 +0100 Subject: rust: dma: add Coherent:init() and Coherent::init_with_attrs() Analogous to Coherent::zeroed() and Coherent::zeroed_with_attrs(), add Coherent:init() and Coherent::init_with_attrs() which both take an impl Init argument initializing the DMA coherent memory. Compared to CoherentInit, Coherent::init() is a one-shot constructor that runs an Init closure and immediately exposes the DMA handle, whereas CoherentInit is a multi-stage initializer that provides safe &mut T access by withholding the DMA address until converted to Coherent. Reviewed-by: Gary Guo Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260320194626.36263-6-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/dma.rs | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) (limited to 'rust/kernel') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 5b41603cee2e..be44ec99af5f 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -715,6 +715,44 @@ impl Coherent { Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) } + /// Same as [`Coherent::zeroed_with_attrs`], but instead of a zero-initialization the memory is + /// initialized with `init`. + pub fn init_with_attrs( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + init: impl Init, + ) -> Result + where + Error: From, + { + let dmem = Self::alloc_with_attrs(dev, gfp_flags, dma_attrs)?; + let ptr = dmem.as_mut_ptr(); + + // SAFETY: + // - `ptr` is valid, properly aligned, and points to exclusively owned memory. + // - If `__init` fails, `self` is dropped, which safely frees the underlying `Coherent`'s + // DMA memory. `T: AsBytes + FromBytes` ensures there are no complex `Drop` requirements + // we are bypassing. + unsafe { init.__init(ptr)? }; + + Ok(dmem) + } + + /// Same as [`Coherent::zeroed`], but instead of a zero-initialization the memory is initialized + /// with `init`. + #[inline] + pub fn init( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + init: impl Init, + ) -> Result + where + Error: From, + { + Self::init_with_attrs(dev, gfp_flags, Attrs(0), init) + } + /// Allocates a region of `[T; len]` of coherent memory. fn alloc_slice_with_attrs( dev: &device::Device, -- cgit v1.2.3 From 55fd681cdd8599edc82013674cbf87c2f22d58f8 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sat, 21 Mar 2026 19:18:08 +0100 Subject: rust: dma: remove dma::CoherentAllocation Now that everything has been converted to the new dma::Coherent API, remove dma::CoherentAllocation. Suggested-by: Gary Guo Reviewed-by: Gary Guo Link: https://patch.msgid.link/DH8O47F2GM1Z.3H3E13RSKIV22@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/dma.rs | 150 ----------------------------------------------------- 1 file changed, 150 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index be44ec99af5f..bf823818a67d 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -853,156 +853,6 @@ impl Coherent<[T]> { } } -// Type alias for compatibility. -#[doc(hidden)] -pub type CoherentAllocation = Coherent<[T]>; - -impl CoherentAllocation { - /// Allocates a region of `size_of:: * count` of coherent memory. - /// - /// # Examples - /// - /// ``` - /// # use kernel::device::{Bound, Device}; - /// use kernel::dma::{attrs::*, CoherentAllocation}; - /// - /// # fn test(dev: &Device) -> Result { - /// let c: CoherentAllocation = - /// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; - /// # Ok::<(), Error>(()) } - /// ``` - pub fn alloc_attrs( - dev: &device::Device, - count: usize, - gfp_flags: kernel::alloc::Flags, - dma_attrs: Attrs, - ) -> Result> { - Coherent::alloc_slice_with_attrs(dev, count, gfp_flags, dma_attrs) - } - - /// Performs the same functionality as [`CoherentAllocation::alloc_attrs`], except the - /// `dma_attrs` is 0 by default. - pub fn alloc_coherent( - dev: &device::Device, - count: usize, - gfp_flags: kernel::alloc::Flags, - ) -> Result> { - CoherentAllocation::alloc_attrs(dev, count, gfp_flags, Attrs(0)) - } - - /// Returns the base address to the allocated region in the CPU's virtual address space. - pub fn start_ptr(&self) -> *const T { - self.as_ptr().cast() - } - - /// Returns the base address to the allocated region in the CPU's virtual address space as - /// a mutable pointer. - pub fn start_ptr_mut(&mut self) -> *mut T { - self.as_mut_ptr().cast() - } - - /// Returns a DMA handle starting at `offset` (in units of `T`) which may be given to the - /// device as the DMA address base of the region. - /// - /// Returns `EINVAL` if `offset` is not within the bounds of the allocation. - pub fn dma_handle_with_offset(&self, offset: usize) -> Result { - if offset >= self.len() { - Err(EINVAL) - } else { - Ok(self.dma_handle + (offset * core::mem::size_of::()) as DmaAddress) - } - } - - /// Common helper to validate a range applied from the allocated region in the CPU's virtual - /// address space. - fn validate_range(&self, offset: usize, count: usize) -> Result { - if offset.checked_add(count).ok_or(EOVERFLOW)? > self.len() { - return Err(EINVAL); - } - Ok(()) - } - - /// Returns the data from the region starting from `offset` as a slice. - /// `offset` and `count` are in units of `T`, not the number of bytes. - /// - /// For ringbuffer type of r/w access or use-cases where the pointer to the live data is needed, - /// [`CoherentAllocation::start_ptr`] or [`CoherentAllocation::start_ptr_mut`] could be used - /// instead. - /// - /// # Safety - /// - /// * Callers must ensure that the device does not read/write to/from memory while the returned - /// slice is live. - /// * Callers must ensure that this call does not race with a write to the same region while - /// the returned slice is live. - pub unsafe fn as_slice(&self, offset: usize, count: usize) -> Result<&[T]> { - self.validate_range(offset, count)?; - // SAFETY: - // - The pointer is valid due to type invariant on `CoherentAllocation`, - // we've just checked that the range and index is within bounds. The immutability of the - // data is also guaranteed by the safety requirements of the function. - // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked - // that `self.count` won't overflow early in the constructor. - Ok(unsafe { core::slice::from_raw_parts(self.start_ptr().add(offset), count) }) - } - - /// Performs the same functionality as [`CoherentAllocation::as_slice`], except that a mutable - /// slice is returned. - /// - /// # Safety - /// - /// * Callers must ensure that the device does not read/write to/from memory while the returned - /// slice is live. - /// * Callers must ensure that this call does not race with a read or write to the same region - /// while the returned slice is live. - pub unsafe fn as_slice_mut(&mut self, offset: usize, count: usize) -> Result<&mut [T]> { - self.validate_range(offset, count)?; - // SAFETY: - // - The pointer is valid due to type invariant on `CoherentAllocation`, - // we've just checked that the range and index is within bounds. The immutability of the - // data is also guaranteed by the safety requirements of the function. - // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked - // that `self.count` won't overflow early in the constructor. - Ok(unsafe { core::slice::from_raw_parts_mut(self.start_ptr_mut().add(offset), count) }) - } - - /// Writes data to the region starting from `offset`. `offset` is in units of `T`, not the - /// number of bytes. - /// - /// # Safety - /// - /// * Callers must ensure that this call does not race with a read or write to the same region - /// that overlaps with this write. - /// - /// # Examples - /// - /// ``` - /// # fn test(alloc: &mut kernel::dma::CoherentAllocation) -> Result { - /// let somedata: [u8; 4] = [0xf; 4]; - /// let buf: &[u8] = &somedata; - /// // SAFETY: There is no concurrent HW operation on the device and no other R/W access to the - /// // region. - /// unsafe { alloc.write(buf, 0)?; } - /// # Ok::<(), Error>(()) } - /// ``` - pub unsafe fn write(&mut self, src: &[T], offset: usize) -> Result { - self.validate_range(offset, src.len())?; - // SAFETY: - // - The pointer is valid due to type invariant on `CoherentAllocation` - // and we've just checked that the range and index is within bounds. - // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked - // that `self.count` won't overflow early in the constructor. - unsafe { - core::ptr::copy_nonoverlapping( - src.as_ptr(), - self.start_ptr_mut().add(offset), - src.len(), - ) - }; - Ok(()) - } -} - /// Note that the device configured to do DMA must be halted before this object is dropped. impl Drop for Coherent { fn drop(&mut self) { -- cgit v1.2.3 From d35ae50c5f48dfcd33cb24bf477ce912fa0af1f7 Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Thu, 19 Mar 2026 16:26:53 -0500 Subject: rust: device: add device name method Add a name() method to the `Device` type, which returns a CStr that contains the device name. Signed-off-by: Timur Tabi Reviewed-by: Alice Ryhl Reviewed-by: Alexandre Courbot Reviewed-by: Gary Guo Tested-by: John Hubbard Tested-by: Eliot Courtney Link: https://patch.msgid.link/20260319212658.2541610-2-ttabi@nvidia.com Signed-off-by: Danilo Krummrich --- rust/kernel/device.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'rust/kernel') diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 379058eca2ed..6d5396a43ebe 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -489,6 +489,17 @@ impl Device { // defined as a `#[repr(transparent)]` wrapper around `fwnode_handle`. Some(unsafe { &*fwnode_handle.cast() }) } + + /// Returns the name of the device. + /// + /// This is the kobject name of the device, or its initial name if the kobject is not yet + /// available. + #[inline] + pub fn name(&self) -> &CStr { + // SAFETY: By its type invariant `self.as_raw()` is a valid pointer to a `struct device`. + // The returned string is valid for the lifetime of the device. + unsafe { CStr::from_char_ptr(bindings::dev_name(self.as_raw())) } + } } // SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic -- cgit v1.2.3 From 69bfce0f25c8f949f7a02c3ed93e36aadad3fa57 Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Thu, 19 Mar 2026 16:26:54 -0500 Subject: rust: uaccess: add write_dma() for copying from DMA buffers to userspace Add UserSliceWriter::write_dma() to copy data from a Coherent<[u8]> to userspace. This provides a safe interface for copying DMA buffer contents to userspace without requiring callers to work with raw pointers. Because write_dma() and write_slice() have common code, factor that code out into a helper function, write_raw(). The method handles bounds checking and offset calculation internally, wrapping the unsafe copy_to_user() call. Signed-off-by: Timur Tabi Reviewed-by: Alexandre Courbot Acked-by: Miguel Ojeda Tested-by: John Hubbard Tested-by: Eliot Courtney Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20260319212658.2541610-3-ttabi@nvidia.com [ Rebase onto Coherent changes; remove unnecessary turbofish from cast(). - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/uaccess.rs | 85 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 75 insertions(+), 10 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs index f989539a31b4..e26ef90ba8ad 100644 --- a/rust/kernel/uaccess.rs +++ b/rust/kernel/uaccess.rs @@ -7,6 +7,7 @@ use crate::{ alloc::{Allocator, Flags}, bindings, + dma::Coherent, error::Result, ffi::{c_char, c_void}, fs::file, @@ -459,20 +460,19 @@ impl UserSliceWriter { self.length == 0 } - /// Writes raw data to this user pointer from a kernel buffer. + /// Low-level write from a raw pointer. /// - /// Fails with [`EFAULT`] if the write happens on a bad address, or if the write goes out of - /// bounds of this [`UserSliceWriter`]. This call may modify the associated userspace slice even - /// if it returns an error. - pub fn write_slice(&mut self, data: &[u8]) -> Result { - let len = data.len(); - let data_ptr = data.as_ptr().cast::(); + /// # Safety + /// + /// The caller must ensure that `from` is valid for reads of `len` bytes. + unsafe fn write_raw(&mut self, from: *const u8, len: usize) -> Result { if len > self.length { return Err(EFAULT); } - // SAFETY: `data_ptr` points into an immutable slice of length `len`, so we may read - // that many bytes from it. - let res = unsafe { bindings::copy_to_user(self.ptr.as_mut_ptr(), data_ptr, len) }; + + // SAFETY: Caller guarantees `from` is valid for `len` bytes (see this function's + // safety contract). + let res = unsafe { bindings::copy_to_user(self.ptr.as_mut_ptr(), from.cast(), len) }; if res != 0 { return Err(EFAULT); } @@ -481,6 +481,71 @@ impl UserSliceWriter { Ok(()) } + /// Writes raw data to this user pointer from a kernel buffer. + /// + /// Fails with [`EFAULT`] if the write happens on a bad address, or if the write goes out of + /// bounds of this [`UserSliceWriter`]. This call may modify the associated userspace slice even + /// if it returns an error. + pub fn write_slice(&mut self, data: &[u8]) -> Result { + // SAFETY: `data` is a valid slice, so `data.as_ptr()` is valid for + // reading `data.len()` bytes. + unsafe { self.write_raw(data.as_ptr(), data.len()) } + } + + /// Writes raw data to this user pointer from a DMA coherent allocation. + /// + /// Copies `count` bytes from `alloc` starting from `offset` into this userspace slice. + /// + /// # Errors + /// + /// - [`EOVERFLOW`]: `offset + count` overflows. + /// - [`ERANGE`]: `offset + count` exceeds the size of `alloc`, or `count` exceeds the + /// size of the user-space buffer. + /// - [`EFAULT`]: the write hits a bad address or goes out of bounds of this + /// [`UserSliceWriter`]. + /// + /// This call may modify the associated userspace slice even if it returns an error. + /// + /// Note: The memory may be concurrently modified by hardware (e.g., DMA). In such cases, + /// the copied data may be inconsistent, but this does not cause undefined behavior. + /// + /// # Example + /// + /// Copy the first 256 bytes of a DMA coherent allocation into a userspace buffer: + /// + /// ```no_run + /// use kernel::uaccess::UserSliceWriter; + /// use kernel::dma::Coherent; + /// + /// fn copy_dma_to_user( + /// mut writer: UserSliceWriter, + /// alloc: &Coherent<[u8]>, + /// ) -> Result { + /// writer.write_dma(alloc, 0, 256) + /// } + /// ``` + pub fn write_dma(&mut self, alloc: &Coherent<[u8]>, offset: usize, count: usize) -> Result { + let len = alloc.size(); + if offset.checked_add(count).ok_or(EOVERFLOW)? > len { + return Err(ERANGE); + } + + if count > self.len() { + return Err(ERANGE); + } + + // SAFETY: `as_ptr()` returns a valid pointer to a memory region of `count()` bytes, as + // guaranteed by the `Coherent` invariants. The check above ensures `offset + count <= len`. + let src_ptr = unsafe { alloc.as_ptr().cast::().add(offset) }; + + // Note: Use `write_raw` instead of `write_slice` because the allocation is coherent + // memory that hardware may modify (e.g., DMA); we cannot form a `&[u8]` slice over + // such volatile memory. + // + // SAFETY: `src_ptr` points into the allocation and is valid for `count` bytes (see above). + unsafe { self.write_raw(src_ptr, count) } + } + /// Writes raw data to this user pointer from a kernel buffer partially. /// /// This is the same as [`Self::write_slice`] but considers the given `offset` into `data` and -- cgit v1.2.3 From 01681851393642e10b5cc3f35eb6a1916ee4aff1 Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Thu, 19 Mar 2026 16:26:55 -0500 Subject: rust: dma: implement BinaryWriter for Coherent<[u8]> Implement the BinaryWriter trait for Coherent<[u8]>, enabling DMA coherent allocations to be exposed as readable binary files. The implementation handles offset tracking and bounds checking, copying data from the coherent allocation to userspace via write_dma(). Signed-off-by: Timur Tabi Reviewed-by: Alice Ryhl Reviewed-by: Alexandre Courbot Tested-by: John Hubbard Tested-by: Eliot Courtney Link: https://patch.msgid.link/20260319212658.2541610-4-ttabi@nvidia.com [ Rebase onto Coherent changes. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/dma.rs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'rust/kernel') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index bf823818a67d..3eef7c2396bb 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -6,12 +6,14 @@ use crate::{ bindings, + debugfs, device::{ self, Bound, Core, // }, error::to_result, + fs::file, prelude::*, ptr::KnownSize, sync::aref::ARef, @@ -19,6 +21,7 @@ use crate::{ AsBytes, FromBytes, // }, // + uaccess::UserSliceWriter, }; use core::{ ops::{ @@ -876,6 +879,37 @@ impl Drop for Coherent { // can be sent to another thread. unsafe impl Send for Coherent {} +// SAFETY: Sharing `&Coherent` across threads is safe if `T` is `Sync`, because all +// methods that access the buffer contents (`field_read`, `field_write`, `as_slice`, +// `as_slice_mut`) are `unsafe`, and callers are responsible for ensuring no data races occur. +// The safe methods only return metadata or raw pointers whose use requires `unsafe`. +unsafe impl Sync for Coherent {} + +impl debugfs::BinaryWriter for Coherent<[u8]> { + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result { + if offset.is_negative() { + return Err(EINVAL); + } + + // If the offset is too large for a usize (e.g. on 32-bit platforms), + // then consider that as past EOF and just return 0 bytes. + let Ok(offset_val) = usize::try_from(*offset) else { + return Ok(0); + }; + + let count = self.size().saturating_sub(offset_val).min(writer.len()); + + writer.write_dma(self, offset_val, count)?; + + *offset += count as i64; + Ok(count) + } +} + /// Reads a field of an item from an allocated region of structs. /// /// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating -- cgit v1.2.3 From e64b9cc293ae710c815c2de1ec9dcaa0784a8017 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Mon, 16 Mar 2026 17:16:09 -0400 Subject: rust: drm: Add gem::impl_aref_for_gem_obj! In the future we're going to be introducing more GEM object types in rust then just gem::Object. Since all types of GEM objects have refcounting, let's introduce a macro that we can use in the gem crate in order to copy this boilerplate implementation for each type: impl_aref_for_gem_obj!(). Signed-off-by: Lyude Paul Reviewed-by: Daniel Almeida Reviewed-by: Janne Grunau Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260316211646.650074-2-lyude@redhat.com [ Resolve merge conflicts. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/drm/gem/mod.rs | 51 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 15 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index b4199945db37..48ffdd7400ca 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -26,6 +26,41 @@ use core::{ ptr::NonNull, // }; +/// A macro for implementing [`AlwaysRefCounted`] for any GEM object type. +/// +/// Since all GEM objects use the same refcounting scheme. +#[macro_export] +macro_rules! impl_aref_for_gem_obj { + ( + impl $( <$( $tparam_id:ident ),+> )? for $type:ty + $( + where + $( $bind_param:path : $bind_trait:path ),+ + )? + ) => { + // SAFETY: All GEM objects are refcounted. + unsafe impl $( <$( $tparam_id ),+> )? $crate::types::AlwaysRefCounted for $type + where + Self: IntoGEMObject, + $( $( $bind_param : $bind_trait ),+ )? + { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is + // non-zero. + unsafe { bindings::drm_gem_object_get(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: core::ptr::NonNull) { + // SAFETY: `obj` is a valid pointer to an `Object`. + let obj = unsafe { obj.as_ref() }.as_raw(); + + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::drm_gem_object_put(obj) }; + } + } + }; +} + /// A type alias for retrieving a [`Driver`]s [`DriverFile`] implementation from its /// [`DriverObject`] implementation. /// @@ -263,21 +298,7 @@ impl Object { } } -// SAFETY: Instances of `Object` are always reference-counted. -unsafe impl crate::sync::aref::AlwaysRefCounted for Object { - fn inc_ref(&self) { - // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. - unsafe { bindings::drm_gem_object_get(self.as_raw()) }; - } - - unsafe fn dec_ref(obj: NonNull) { - // SAFETY: `obj` is a valid pointer to an `Object`. - let obj = unsafe { obj.as_ref() }; - - // SAFETY: The safety requirements guarantee that the refcount is non-zero. - unsafe { bindings::drm_gem_object_put(obj.as_raw()) } - } -} +impl_aref_for_gem_obj!(impl for Object where T: DriverObject); impl super::private::Sealed for Object {} -- cgit v1.2.3 From 442ba16a5a51368f5bafd011609f40782aec6d65 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Mon, 16 Mar 2026 17:16:12 -0400 Subject: rust: gem: Introduce DriverObject::Args This is an associated type that may be used in order to specify a data-type to pass to gem objects when constructing them, allowing for drivers to more easily initialize their private-data for gem objects. Signed-off-by: Lyude Paul Reviewed-by: Alice Ryhl Reviewed-by: Daniel Almeida Reviewed-by: Janne Grunau Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260316211646.650074-5-lyude@redhat.com [ Resolve merge conflicts in Tyr. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/drm/gem/mod.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 48ffdd7400ca..ed974bfdc861 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -73,8 +73,15 @@ pub trait DriverObject: Sync + Send + Sized { /// Parent `Driver` for this object. type Driver: drm::Driver; + /// The data type to use for passing arguments to [`DriverObject::new`]. + type Args; + /// Create a new driver data object for a GEM object of a given size. - fn new(dev: &drm::Device, size: usize) -> impl PinInit; + fn new( + dev: &drm::Device, + size: usize, + args: Self::Args, + ) -> impl PinInit; /// Open a new handle to an existing object, associated with a File. fn open(_obj: &::Object, _file: &DriverFile) -> Result { @@ -241,11 +248,11 @@ impl Object { }; /// Create a new GEM object. - pub fn new(dev: &drm::Device, size: usize) -> Result> { + pub fn new(dev: &drm::Device, size: usize, args: T::Args) -> Result> { let obj: Pin> = KBox::pin_init( try_pin_init!(Self { obj: Opaque::new(bindings::drm_gem_object::default()), - data <- T::new(dev, size), + data <- T::new(dev, size, args), }), GFP_KERNEL, )?; -- cgit v1.2.3 From f5e841e4966c1873b9bb2c69d07947a54284e5eb Mon Sep 17 00:00:00 2001 From: Daniel Almeida Date: Mon, 23 Mar 2026 20:26:59 -0300 Subject: rust: workqueue: add support for ARef Add support for the ARef smart pointer. This allows an instance of ARef to handle deferred work directly, which can be convenient or even necessary at times, depending on the specifics of the driver or subsystem. The implementation is similar to that of Arc, and a subsequent patch will implement support for drm::Device as the first user. This is notably important for work items that need access to the drm device, as it was not possible to enqueue work on a ARef> previously without failing the orphan rule. Reviewed-by: Alice Ryhl Acked-by: Danilo Krummrich Signed-off-by: Daniel Almeida Link: https://lore.kernel.org/r/20260323-aref-workitem-v3-1-f59729b812aa@collabora.com Signed-off-by: Alice Ryhl --- rust/kernel/workqueue.rs | 85 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 6 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 706e833e9702..6ae7f3fb3496 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -192,9 +192,9 @@ use crate::{ sync::Arc, sync::LockClassKey, time::Jiffies, - types::Opaque, + types::{ARef, AlwaysRefCounted, Opaque}, }; -use core::marker::PhantomData; +use core::{marker::PhantomData, ptr::NonNull}; /// Creates a [`Work`] initialiser with the given name and a newly-created lock class. #[macro_export] @@ -425,10 +425,11 @@ pub unsafe trait RawDelayedWorkItem: RawWorkItem {} /// Defines the method that should be called directly when a work item is executed. /// -/// This trait is implemented by `Pin>` and [`Arc`], and is mainly intended to be -/// implemented for smart pointer types. For your own structs, you would implement [`WorkItem`] -/// instead. The [`run`] method on this trait will usually just perform the appropriate -/// `container_of` translation and then call into the [`run`][WorkItem::run] method from the +/// This trait is implemented by `Pin>`, [`Arc`] and [`ARef`], and +/// is mainly intended to be implemented for smart pointer types. For your own +/// structs, you would implement [`WorkItem`] instead. The [`run`] method on +/// this trait will usually just perform the appropriate `container_of` +/// translation and then call into the [`run`][WorkItem::run] method from the /// [`WorkItem`] trait. /// /// This trait is used when the `work_struct` field is defined using the [`Work`] helper. @@ -934,6 +935,78 @@ where { } +// SAFETY: Like the `Arc` implementation, the `__enqueue` implementation for +// `ARef` obtains a `work_struct` from the `Work` field using +// `T::raw_get_work`, so the same safety reasoning applies: +// +// - `__enqueue` gets the `work_struct` from the `Work` field, using `T::raw_get_work`. +// - The only safe way to create a `Work` object is through `Work::new`. +// - `Work::new` makes sure that `T::Pointer::run` is passed to `init_work_with_key`. +// - Finally `Work` and `RawWorkItem` guarantee that the correct `Work` field +// will be used because of the ID const generic bound. This makes sure that `T::raw_get_work` +// uses the correct offset for the `Work` field, and `Work::new` picks the correct +// implementation of `WorkItemPointer` for `ARef`. +unsafe impl WorkItemPointer for ARef +where + T: AlwaysRefCounted, + T: WorkItem, + T: HasWork, +{ + unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { + // The `__enqueue` method always uses a `work_struct` stored in a `Work`. + let ptr = ptr.cast::>(); + + // SAFETY: This computes the pointer that `__enqueue` got from + // `ARef::into_raw`. + let ptr = unsafe { T::work_container_of(ptr) }; + + // SAFETY: The safety contract of `work_container_of` ensures that it + // returns a valid non-null pointer. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + + // SAFETY: This pointer comes from `ARef::into_raw` and we've been given + // back ownership. + let aref = unsafe { ARef::from_raw(ptr) }; + + T::run(aref) + } +} + +// SAFETY: The `work_struct` raw pointer is guaranteed to be valid for the duration of the call to +// the closure because we get it from an `ARef`, which means that the ref count will be at least 1, +// and we don't drop the `ARef` ourselves. If `queue_work_on` returns true, it is further guaranteed +// to be valid until a call to the function pointer in `work_struct` because we leak the memory it +// points to, and only reclaim it if the closure returns false, or in `WorkItemPointer::run`, which +// is what the function pointer in the `work_struct` must be pointing to, according to the safety +// requirements of `WorkItemPointer`. +unsafe impl RawWorkItem for ARef +where + T: AlwaysRefCounted, + T: WorkItem, + T: HasWork, +{ + type EnqueueOutput = Result<(), Self>; + + unsafe fn __enqueue(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool, + { + let ptr = ARef::into_raw(self); + + // SAFETY: Pointers from ARef::into_raw are valid and non-null. + let work_ptr = unsafe { T::raw_get_work(ptr.as_ptr()) }; + // SAFETY: `raw_get_work` returns a pointer to a valid value. + let work_ptr = unsafe { Work::raw_get(work_ptr) }; + + if queue_work_on(work_ptr) { + Ok(()) + } else { + // SAFETY: The work queue has not taken ownership of the pointer. + Err(unsafe { ARef::from_raw(ptr) }) + } + } +} + /// Returns the system work queue (`system_wq`). /// /// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are -- cgit v1.2.3 From 72a723df8decf70e04f799a6defda8bb62d41848 Mon Sep 17 00:00:00 2001 From: Daniel Almeida Date: Mon, 23 Mar 2026 20:27:00 -0300 Subject: rust: drm: dispatch work items to the private data This implementation dispatches any work enqueued on ARef> to its driver-provided handler. It does so by building upon the newly-added ARef support in workqueue.rs in order to call into the driver implementations for work_container_of and raw_get_work. This is notably important for work items that need access to the drm device, as it was not possible to enqueue work on a ARef> previously without failing the orphan rule. The current implementation needs T::Data to live inline with drm::Device in order for work_container_of to function. This restriction is already captured by the trait bounds. Drivers that need to share their ownership of T::Data may trivially get around this: // Lives inline in drm::Device struct DataWrapper { work: ..., // Heap-allocated, shared ownership. data: Arc, } Reviewed-by: Alice Ryhl Acked-by: Danilo Krummrich Signed-off-by: Daniel Almeida Link: https://lore.kernel.org/r/20260323-aref-workitem-v3-2-f59729b812aa@collabora.com Signed-off-by: Alice Ryhl --- rust/kernel/drm/device.rs | 56 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 629ef0bd1188..5db5c7e3bb7a 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -6,8 +6,7 @@ use crate::{ alloc::allocator::Kmalloc, - bindings, - device, + bindings, device, drm::{ self, driver::AllocImpl, // @@ -18,7 +17,12 @@ use crate::{ ARef, AlwaysRefCounted, // }, - types::Opaque, // + types::Opaque, + workqueue::{ + HasWork, + Work, + WorkItem, // + }, }; use core::{ alloc::Layout, @@ -241,3 +245,49 @@ unsafe impl Send for Device {} // SAFETY: A `drm::Device` can be shared among threads because all immutable methods are protected // by the synchronization in `struct drm_device`. unsafe impl Sync for Device {} + +impl WorkItem for Device +where + T: drm::Driver, + T::Data: WorkItem>>, + T::Data: HasWork, ID>, +{ + type Pointer = ARef>; + + fn run(ptr: ARef>) { + T::Data::run(ptr); + } +} + +// SAFETY: +// +// - `raw_get_work` and `work_container_of` return valid pointers by relying on +// `T::Data::raw_get_work` and `container_of`. In particular, `T::Data` is +// stored inline in `drm::Device`, so the `container_of` call is valid. +// +// - The two methods are true inverses of each other: given `ptr: *mut +// Device`, `raw_get_work` will return a `*mut Work, ID>` through +// `T::Data::raw_get_work` and given a `ptr: *mut Work, ID>`, +// `work_container_of` will return a `*mut Device` through `container_of`. +unsafe impl HasWork, ID> for Device +where + T: drm::Driver, + T::Data: HasWork, ID>, +{ + unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work, ID> { + // SAFETY: The caller promises that `ptr` points to a valid `Device`. + let data_ptr = unsafe { &raw mut (*ptr).data }; + + // SAFETY: `data_ptr` is a valid pointer to `T::Data`. + unsafe { T::Data::raw_get_work(data_ptr) } + } + + unsafe fn work_container_of(ptr: *mut Work, ID>) -> *mut Self { + // SAFETY: The caller promises that `ptr` points at a `Work` field in + // `T::Data`. + let data_ptr = unsafe { T::Data::work_container_of(ptr) }; + + // SAFETY: `T::Data` is stored as the `data` field in `Device`. + unsafe { crate::container_of!(data_ptr, Self, data) } + } +} -- cgit v1.2.3 From 332666484f06cd85ad382329f5b3165aa627e9f8 Mon Sep 17 00:00:00 2001 From: Daniel Almeida Date: Mon, 23 Mar 2026 20:27:01 -0300 Subject: rust: workqueue: add delayed work support for ARef The preceding patches added support for ARef work items. By the same token, add support for delayed work items too. The rationale is the same: it may be convenient or even necessary at times to implement HasDelayedWork directly on ARef. A follow up patch will also implement support for drm::Device as the first user. Reviewed-by: Alice Ryhl Acked-by: Danilo Krummrich Signed-off-by: Daniel Almeida Link: https://lore.kernel.org/r/20260323-aref-workitem-v3-3-f59729b812aa@collabora.com Signed-off-by: Alice Ryhl --- rust/kernel/workqueue.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'rust/kernel') diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 6ae7f3fb3496..4ee4ff567197 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -1007,6 +1007,17 @@ where } } +// SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in +// `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of +// the `delayed_work` has the same access rules as its `work` field. +unsafe impl RawDelayedWorkItem for ARef +where + T: WorkItem, + T: HasDelayedWork, + T: AlwaysRefCounted, +{ +} + /// Returns the system work queue (`system_wq`). /// /// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are -- cgit v1.2.3 From 206bada308d0d715d62ac841784af34a2ca22ff6 Mon Sep 17 00:00:00 2001 From: Daniel Almeida Date: Mon, 23 Mar 2026 20:27:02 -0300 Subject: rust: drm: dispatch delayed work items to the private data Much like the patch that dispatched (regular) work items, we also need to dispatch delayed work items in order not to trigger the orphan rule. This allows a drm::Device to dispatch the delayed work to T::Data. Reviewed-by: Alice Ryhl Acked-by: Danilo Krummrich Signed-off-by: Daniel Almeida Link: https://lore.kernel.org/r/20260323-aref-workitem-v3-4-f59729b812aa@collabora.com Signed-off-by: Alice Ryhl --- rust/kernel/drm/device.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'rust/kernel') diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 5db5c7e3bb7a..adbafe8db54d 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -19,6 +19,7 @@ use crate::{ }, types::Opaque, workqueue::{ + HasDelayedWork, HasWork, Work, WorkItem, // @@ -291,3 +292,15 @@ where unsafe { crate::container_of!(data_ptr, Self, data) } } } + +// SAFETY: Our `HasWork` implementation returns a `work_struct` that is +// stored in the `work` field of a `delayed_work` with the same access rules as +// the `work_struct` owing to the bound on `T::Data: HasDelayedWork, +// ID>`, which requires that `T::Data::raw_get_work` return a `work_struct` that +// is inside a `delayed_work`. +unsafe impl HasDelayedWork, ID> for Device +where + T: drm::Driver, + T::Data: HasDelayedWork, ID>, +{ +} -- cgit v1.2.3 From d4cf576672fbfee061d6f4f70c74b3b3d163447c Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 26 Mar 2026 15:25:36 +0000 Subject: rust: workqueue: use new sync::aref path for imports ARef and AlwaysRefCounted are being moved to sync::aref, and the re-exports under types are planned to be removed. Thus, update imports to the new path. Acked-by: Danilo Krummrich Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260326-drm-rust-next-fix-aref-v1-1-7f6f58d2828a@google.com Signed-off-by: Alice Ryhl --- rust/kernel/workqueue.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 4ee4ff567197..7e253b6f299c 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -189,10 +189,16 @@ use crate::{ alloc::{AllocError, Flags}, container_of, prelude::*, - sync::Arc, - sync::LockClassKey, + sync::{ + aref::{ + ARef, + AlwaysRefCounted, // + }, + Arc, + LockClassKey, // + }, time::Jiffies, - types::{ARef, AlwaysRefCounted, Opaque}, + types::Opaque, }; use core::{marker::PhantomData, ptr::NonNull}; -- cgit v1.2.3 From bdf6b22fd52954f5ac88689eeaf960ac9687b78c Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 26 Mar 2026 15:25:37 +0000 Subject: rust: drm: use new sync::aref path for imports ARef and AlwaysRefCounted are being moved to sync::aref, and the re-exports under types are planned to be removed. Thus, update imports to the new path. Acked-by: Danilo Krummrich Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260326-drm-rust-next-fix-aref-v1-2-7f6f58d2828a@google.com Signed-off-by: Alice Ryhl --- rust/kernel/drm/gem/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'rust/kernel') diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index ed974bfdc861..6cc441ee5b63 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -39,7 +39,7 @@ macro_rules! impl_aref_for_gem_obj { )? ) => { // SAFETY: All GEM objects are refcounted. - unsafe impl $( <$( $tparam_id ),+> )? $crate::types::AlwaysRefCounted for $type + unsafe impl $( <$( $tparam_id ),+> )? $crate::sync::aref::AlwaysRefCounted for $type where Self: IntoGEMObject, $( $( $bind_param : $bind_trait ),+ )? -- cgit v1.2.3 From 89b4964c0456d9939a9f5187891a36bb87111e58 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Mon, 16 Mar 2026 17:16:10 -0400 Subject: rust: drm: gem: Add raw_dma_resv() function For retrieving a pointer to the struct dma_resv for a given GEM object. We also introduce it in a new trait, BaseObjectPrivate, which we automatically implement for all gem objects and don't expose to users outside of the crate. Signed-off-by: Lyude Paul Reviewed-by: Janne Grunau Tested-by: Janne Grunau Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260316211646.650074-3-lyude@redhat.com [ Fix incorrect reference in safety comment. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/drm/gem/mod.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'rust/kernel') diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 6cc441ee5b63..a78b98c40d56 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -215,6 +215,18 @@ pub trait BaseObject: IntoGEMObject { impl BaseObject for T {} +/// Crate-private base operations shared by all GEM object classes. +#[expect(unused)] +pub(crate) trait BaseObjectPrivate: IntoGEMObject { + /// Return a pointer to this object's dma_resv. + fn raw_dma_resv(&self) -> *mut bindings::dma_resv { + // SAFETY: `self.as_raw()` always returns a valid pointer to the base DRM GEM object. + unsafe { (*self.as_raw()).resv } + } +} + +impl BaseObjectPrivate for T {} + /// A base GEM object. /// /// # Invariants -- cgit v1.2.3 From 80df573af9ef3aa63e1bacb6e17d57a7cd69afe2 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Mon, 16 Mar 2026 17:16:13 -0400 Subject: rust: drm: gem: shmem: Add DRM shmem helper abstraction The DRM shmem helper includes common code useful for drivers which allocate GEM objects as anonymous shmem. Add a Rust abstraction for this. Drivers can choose the raw GEM implementation or the shmem layer, depending on their needs. Signed-off-by: Asahi Lina Signed-off-by: Daniel Almeida Reviewed-by: Daniel Almeida Signed-off-by: Lyude Paul Reviewed-by: Janne Grunau Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260316211646.650074-6-lyude@redhat.com [ * DRM_GEM_SHMEM_HELPER is a tristate; when a module driver selects it, it becomes =m. The Rust kernel crate and its C helpers are always built into vmlinux and can't reference symbols from a module, causing link errors. Thus, add RUST_DRM_GEM_SHMEM_HELPER bool Kconfig that selects DRM_GEM_SHMEM_HELPER, forcing it built-in when Rust drivers need it; use cfg(CONFIG_RUST_DRM_GEM_SHMEM_HELPER) for the shmem module. * Add cfg_attr(not(CONFIG_RUST_DRM_GEM_SHMEM_HELPER), expect(unused)) on pub(crate) use impl_aref_for_gem_obj and BaseObjectPrivate, so that unused warnings are suppressed when shmem is not enabled. * Enable const_refs_to_static (stabilized in 1.83) to prevent build errors with older compilers. * Use &raw const for bindings::drm_gem_shmem_vm_ops and add #[allow(unused_unsafe, reason = "Safe since Rust 1.82.0")]. * Fix incorrect C Header path and minor spelling and formatting issues. * Drop shmem::Object::sg_table() as the current implementation is unsound. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/drm/gem/mod.rs | 7 +- rust/kernel/drm/gem/shmem.rs | 228 +++++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 3 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 rust/kernel/drm/gem/shmem.rs (limited to 'rust/kernel') diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index a78b98c40d56..75acda7ba500 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -26,6 +26,9 @@ use core::{ ptr::NonNull, // }; +#[cfg(CONFIG_RUST_DRM_GEM_SHMEM_HELPER)] +pub mod shmem; + /// A macro for implementing [`AlwaysRefCounted`] for any GEM object type. /// /// Since all GEM objects use the same refcounting scheme. @@ -60,6 +63,8 @@ macro_rules! impl_aref_for_gem_obj { } }; } +#[cfg_attr(not(CONFIG_RUST_DRM_GEM_SHMEM_HELPER), allow(unused))] +pub(crate) use impl_aref_for_gem_obj; /// A type alias for retrieving a [`Driver`]s [`DriverFile`] implementation from its /// [`DriverObject`] implementation. @@ -216,7 +221,7 @@ pub trait BaseObject: IntoGEMObject { impl BaseObject for T {} /// Crate-private base operations shared by all GEM object classes. -#[expect(unused)] +#[cfg_attr(not(CONFIG_RUST_DRM_GEM_SHMEM_HELPER), expect(unused))] pub(crate) trait BaseObjectPrivate: IntoGEMObject { /// Return a pointer to this object's dma_resv. fn raw_dma_resv(&self) -> *mut bindings::dma_resv { diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs new file mode 100644 index 000000000000..d025fb035195 --- /dev/null +++ b/rust/kernel/drm/gem/shmem.rs @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! DRM GEM shmem helper objects +//! +//! C header: [`include/linux/drm/drm_gem_shmem_helper.h`](srctree/include/drm/drm_gem_shmem_helper.h) + +// TODO: +// - There are a number of spots here that manually acquire/release the DMA reservation lock using +// dma_resv_(un)lock(). In the future we should add support for ww mutex, expose a method to +// acquire a reference to the WwMutex, and then use that directly instead of the C functions here. + +use crate::{ + container_of, + drm::{ + device, + driver, + gem, + private::Sealed, // + }, + error::to_result, + prelude::*, + types::{ + ARef, + Opaque, // + }, // +}; +use core::{ + ops::{ + Deref, + DerefMut, // + }, + ptr::NonNull, +}; +use gem::{ + BaseObjectPrivate, + DriverObject, + IntoGEMObject, // +}; + +/// A struct for controlling the creation of shmem-backed GEM objects. +/// +/// This is used with [`Object::new()`] to control various properties that can only be set when +/// initially creating a shmem-backed GEM object. +#[derive(Default)] +pub struct ObjectConfig<'a, T: DriverObject> { + /// Whether to set the write-combine map flag. + pub map_wc: bool, + + /// Reuse the DMA reservation from another GEM object. + /// + /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. + pub parent_resv_obj: Option<&'a Object>, +} + +/// A shmem-backed GEM object. +/// +/// # Invariants +/// +/// `obj` contains a valid initialized `struct drm_gem_shmem_object` for the lifetime of this +/// object. +#[repr(C)] +#[pin_data] +pub struct Object { + #[pin] + obj: Opaque, + /// Parent object that owns this object's DMA reservation object. + parent_resv_obj: Option>>, + #[pin] + inner: T, +} + +super::impl_aref_for_gem_obj!(impl for Object where T: DriverObject); + +// SAFETY: All GEM objects are thread-safe. +unsafe impl Send for Object {} + +// SAFETY: All GEM objects are thread-safe. +unsafe impl Sync for Object {} + +impl Object { + /// `drm_gem_object_funcs` vtable suitable for GEM shmem objects. + const VTABLE: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { + free: Some(Self::free_callback), + open: Some(super::open_callback::), + close: Some(super::close_callback::), + print_info: Some(bindings::drm_gem_shmem_object_print_info), + export: None, + pin: Some(bindings::drm_gem_shmem_object_pin), + unpin: Some(bindings::drm_gem_shmem_object_unpin), + get_sg_table: Some(bindings::drm_gem_shmem_object_get_sg_table), + vmap: Some(bindings::drm_gem_shmem_object_vmap), + vunmap: Some(bindings::drm_gem_shmem_object_vunmap), + mmap: Some(bindings::drm_gem_shmem_object_mmap), + status: None, + rss: None, + #[allow(unused_unsafe, reason = "Safe since Rust 1.82.0")] + // SAFETY: `drm_gem_shmem_vm_ops` is a valid, static const on the C side. + vm_ops: unsafe { &raw const bindings::drm_gem_shmem_vm_ops }, + evict: None, + }; + + /// Return a raw pointer to the embedded drm_gem_shmem_object. + fn as_raw_shmem(&self) -> *mut bindings::drm_gem_shmem_object { + self.obj.get() + } + + /// Create a new shmem-backed DRM object of the given size. + /// + /// Additional config options can be specified using `config`. + pub fn new( + dev: &device::Device, + size: usize, + config: ObjectConfig<'_, T>, + args: T::Args, + ) -> Result> { + let new: Pin> = KBox::try_pin_init( + try_pin_init!(Self { + obj <- Opaque::init_zeroed(), + parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), + inner <- T::new(dev, size, args), + }), + GFP_KERNEL, + )?; + + // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. + unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; + + // SAFETY: The arguments are all valid via the type invariants. + to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_raw_shmem(), size) })?; + + // SAFETY: We never move out of `self`. + let new = KBox::into_raw(unsafe { Pin::into_inner_unchecked(new) }); + + // SAFETY: We're taking over the owned refcount from `drm_gem_shmem_init`. + let obj = unsafe { ARef::from_raw(NonNull::new_unchecked(new)) }; + + // Start filling out values from `config` + if let Some(parent_resv) = config.parent_resv_obj { + // SAFETY: We have yet to expose the new gem object outside of this function, so it is + // safe to modify this field. + unsafe { (*obj.obj.get()).base.resv = parent_resv.raw_dma_resv() }; + } + + // SAFETY: We have yet to expose this object outside of this function, so we're guaranteed + // to have exclusive access - thus making this safe to hold a mutable reference to. + let shmem = unsafe { &mut *obj.as_raw_shmem() }; + shmem.set_map_wc(config.map_wc); + + Ok(obj) + } + + /// Returns the `Device` that owns this GEM object. + pub fn dev(&self) -> &device::Device { + // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`. + unsafe { device::Device::from_raw((*self.as_raw()).dev) } + } + + extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { + // SAFETY: + // - DRM always passes a valid gem object here + // - We used drm_gem_shmem_create() in our create_gem_object callback, so we know that + // `obj` is contained within a drm_gem_shmem_object + let this = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) }; + + // SAFETY: + // - We're in free_callback - so this function is safe to call. + // - We won't be using the gem resources on `this` after this call. + unsafe { bindings::drm_gem_shmem_release(this) }; + + // SAFETY: + // - We verified above that `obj` is valid, which makes `this` valid + // - This function is set in AllocOps, so we know that `this` is contained within a + // `Object` + let this = unsafe { container_of!(Opaque::cast_from(this), Self, obj) }.cast_mut(); + + // SAFETY: We're recovering the Kbox<> we created in gem_create_object() + let _ = unsafe { KBox::from_raw(this) }; + } +} + +impl Deref for Object { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for Object { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl Sealed for Object {} + +impl gem::IntoGEMObject for Object { + fn as_raw(&self) -> *mut bindings::drm_gem_object { + // SAFETY: + // - Our immutable reference is proof that this is safe to dereference. + // - `obj` is always a valid drm_gem_shmem_object via our type invariants. + unsafe { &raw mut (*self.obj.get()).base } + } + + unsafe fn from_raw<'a>(obj: *mut bindings::drm_gem_object) -> &'a Object { + // SAFETY: The safety contract of from_gem_obj() guarantees that `obj` is contained within + // `Self` + unsafe { + let obj = Opaque::cast_from(container_of!(obj, bindings::drm_gem_shmem_object, base)); + + &*container_of!(obj, Object, obj) + } + } +} + +impl driver::AllocImpl for Object { + type Driver = T::Driver; + + const ALLOC_OPS: driver::AllocOps = driver::AllocOps { + gem_create_object: None, + prime_handle_to_fd: None, + prime_fd_to_handle: None, + gem_prime_import: None, + gem_prime_import_sg_table: Some(bindings::drm_gem_shmem_prime_import_sg_table), + dumb_create: Some(bindings::drm_gem_shmem_dumb_create), + dumb_map_offset: None, + }; +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index e0837ffc91bf..40de00ce4f97 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -38,6 +38,7 @@ #![feature(const_option)] #![feature(const_ptr_write)] #![feature(const_refs_to_cell)] +#![feature(const_refs_to_static)] // // Stable since Rust 1.84.0. #![feature(strict_provenance)] -- cgit v1.2.3 From 25fe63db0024da172457055532c96bef924a8c78 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Wed, 25 Mar 2026 01:39:15 +0100 Subject: rust: uaccess: generalize write_dma() to accept any Coherent Generalize write_dma() from &Coherent<[u8]> to &Coherent where T: KnownSize + AsBytes + ?Sized. The function body only uses as_ptr() and size(), which work for any such T, so there is no reason to restrict it to byte slices. Acked-by: Miguel Ojeda Acked-by: Gary Guo Reviewed-by: Alexandre Courbot Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20260325003921.3420-1-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/uaccess.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'rust/kernel') diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs index e26ef90ba8ad..6c9c1cce3c63 100644 --- a/rust/kernel/uaccess.rs +++ b/rust/kernel/uaccess.rs @@ -12,6 +12,7 @@ use crate::{ ffi::{c_char, c_void}, fs::file, prelude::*, + ptr::KnownSize, transmute::{AsBytes, FromBytes}, }; use core::mem::{size_of, MaybeUninit}; @@ -524,7 +525,12 @@ impl UserSliceWriter { /// writer.write_dma(alloc, 0, 256) /// } /// ``` - pub fn write_dma(&mut self, alloc: &Coherent<[u8]>, offset: usize, count: usize) -> Result { + pub fn write_dma( + &mut self, + alloc: &Coherent, + offset: usize, + count: usize, + ) -> Result { let len = alloc.size(); if offset.checked_add(count).ok_or(EOVERFLOW)? > len { return Err(ERANGE); -- cgit v1.2.3 From d1619a433806c61d240a1eb9b4f03cb33ac40fce Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Wed, 25 Mar 2026 01:39:16 +0100 Subject: rust: dma: generalize BinaryWriter impl for Coherent Generalize the BinaryWriter implementation from Coherent<[u8]> to Coherent where T: KnownSize + AsBytes + ?Sized. The implementation only uses size() and write_dma(), neither of which depends on the inner type being a byte slice. This allows any Coherent allocation with an AsBytes inner type to be exposed as a debugfs binary file. Acked-by: Gary Guo Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260325003921.3420-2-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/dma.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'rust/kernel') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 3eef7c2396bb..7bc97f9f83fd 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -885,7 +885,7 @@ unsafe impl Send for Coherent {} // The safe methods only return metadata or raw pointers whose use requires `unsafe`. unsafe impl Sync for Coherent {} -impl debugfs::BinaryWriter for Coherent<[u8]> { +impl debugfs::BinaryWriter for Coherent { fn write_to_slice( &self, writer: &mut UserSliceWriter, -- cgit v1.2.3 From 18fb5f1f0289b8217c0c43d54d12bccc201dd640 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sat, 21 Mar 2026 18:27:46 +0100 Subject: rust: dma: remove DMA_ATTR_NO_KERNEL_MAPPING from public attrs When DMA_ATTR_NO_KERNEL_MAPPING is passed to dma_alloc_attrs(), the returned CPU address is not a pointer to the allocated memory but an opaque handle (e.g. struct page *). Coherent (or CoherentAllocation respectively) stores this value as NonNull and exposes methods that dereference it and even modify its contents. Remove the flag from the public attrs module such that drivers cannot pass it to Coherent (or CoherentAllocation respectively) in the first place. Instead DMA_ATTR_NO_KERNEL_MAPPING can be supported with an additional opaque type (e.g. CoherentHandle) which does not provide access to the allocated memory. Cc: stable@vger.kernel.org Fixes: ad2907b4e308 ("rust: add dma coherent allocator abstraction") Signed-off-by: Danilo Krummrich Reviewed-by: Alice Ryhl Reviewed-by: Alexandre Courbot Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260321172749.592387-1-dakr@kernel.org Signed-off-by: Alexandre Courbot --- rust/kernel/dma.rs | 3 --- 1 file changed, 3 deletions(-) (limited to 'rust/kernel') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 7bc97f9f83fd..beb4dea7dd84 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -267,9 +267,6 @@ pub mod attrs { /// Specifies that writes to the mapping may be buffered to improve performance. pub const DMA_ATTR_WRITE_COMBINE: Attrs = Attrs(bindings::DMA_ATTR_WRITE_COMBINE); - /// Lets the platform to avoid creating a kernel virtual mapping for the allocated buffer. - pub const DMA_ATTR_NO_KERNEL_MAPPING: Attrs = Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING); - /// Allows platform code to skip synchronization of the CPU cache for the given buffer assuming /// that it has been already transferred to 'device' domain. pub const DMA_ATTR_SKIP_CPU_SYNC: Attrs = Attrs(bindings::DMA_ATTR_SKIP_CPU_SYNC); -- cgit v1.2.3 From 6dd782af1eb6be16a80349ea7822f0f8a23bb3e8 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sat, 21 Mar 2026 18:27:47 +0100 Subject: rust: dma: add CoherentHandle for DMA allocations without kernel mapping Add CoherentHandle, an opaque DMA allocation type for buffers that are only ever accessed by hardware. Unlike Coherent, it does not provide CPU access to the allocated memory. CoherentHandle implicitly sets DMA_ATTR_NO_KERNEL_MAPPING and stores the value returned by dma_alloc_attrs() as an opaque handle (NonNull) rather than a typed pointer, since with this flag the C API returns an opaque cookie (e.g. struct page *), not a CPU pointer to the allocated memory. Only the DMA bus address is exposed to drivers; the opaque handle is used solely to free the allocation on drop. This commit is for reference only; there is currently no in-tree user. Signed-off-by: Danilo Krummrich Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260321172749.592387-2-dakr@kernel.org [acourbot: fix conflict in dma.rs.] Signed-off-by: Alexandre Courbot --- rust/kernel/dma.rs | 119 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) (limited to 'rust/kernel') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index beb4dea7dd84..7ea3c3b73367 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -907,6 +907,125 @@ impl debugfs::BinaryWriter for Coherent { } } +/// An opaque DMA allocation without a kernel virtual mapping. +/// +/// Unlike [`Coherent`], a `CoherentHandle` does not provide CPU access to the allocated memory. +/// The allocation is always performed with `DMA_ATTR_NO_KERNEL_MAPPING`, meaning no kernel +/// virtual mapping is created for the buffer. The value returned by the C API as the CPU +/// address is an opaque handle used only to free the allocation. +/// +/// This is useful for buffers that are only ever accessed by hardware. +/// +/// # Invariants +/// +/// - `cpu_handle` holds the opaque handle returned by `dma_alloc_attrs` with +/// `DMA_ATTR_NO_KERNEL_MAPPING` set, and is only valid for passing back to `dma_free_attrs`. +/// - `dma_handle` is the corresponding bus address for device DMA. +/// - `size` is the allocation size in bytes as passed to `dma_alloc_attrs`. +/// - `dma_attrs` contains the attributes used for the allocation, always including +/// `DMA_ATTR_NO_KERNEL_MAPPING`. +pub struct CoherentHandle { + dev: ARef, + dma_handle: DmaAddress, + cpu_handle: NonNull, + size: usize, + dma_attrs: Attrs, +} + +impl CoherentHandle { + /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping. + /// + /// Additional DMA attributes may be passed via `dma_attrs`; `DMA_ATTR_NO_KERNEL_MAPPING` is + /// always set implicitly. + /// + /// Returns `EINVAL` if `size` is zero, `ENOMEM` if the allocation fails. + pub fn alloc_with_attrs( + dev: &device::Device, + size: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + if size == 0 { + return Err(EINVAL); + } + + let dma_attrs = dma_attrs | Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING); + let mut dma_handle = 0; + // SAFETY: `dev.as_raw()` is valid by the type invariant on `device::Device`. + let cpu_handle = unsafe { + bindings::dma_alloc_attrs( + dev.as_raw(), + size, + &mut dma_handle, + gfp_flags.as_raw(), + dma_attrs.as_raw(), + ) + }; + + let cpu_handle = NonNull::new(cpu_handle).ok_or(ENOMEM)?; + + // INVARIANT: `cpu_handle` is the opaque handle from a successful `dma_alloc_attrs` call + // with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_handle` is the corresponding DMA address, + // and we hold a refcounted reference to the device. + Ok(Self { + dev: dev.into(), + dma_handle, + cpu_handle, + size, + dma_attrs, + }) + } + + /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping. + #[inline] + pub fn alloc( + dev: &device::Device, + size: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result { + Self::alloc_with_attrs(dev, size, gfp_flags, Attrs(0)) + } + + /// Returns the DMA handle for this allocation. + /// + /// This address can be programmed into device hardware for DMA access. + #[inline] + pub fn dma_handle(&self) -> DmaAddress { + self.dma_handle + } + + /// Returns the size in bytes of this allocation. + #[inline] + pub fn size(&self) -> usize { + self.size + } +} + +impl Drop for CoherentHandle { + fn drop(&mut self) { + // SAFETY: All values are valid by the type invariants on `CoherentHandle`. + // `cpu_handle` is the opaque handle from `dma_alloc_attrs` and is passed back unchanged. + unsafe { + bindings::dma_free_attrs( + self.dev.as_raw(), + self.size, + self.cpu_handle.as_ptr(), + self.dma_handle, + self.dma_attrs.as_raw(), + ) + } + } +} + +// SAFETY: `CoherentHandle` only holds a device reference, a DMA handle, an opaque CPU handle, +// and a size. None of these are tied to a specific thread. +unsafe impl Send for CoherentHandle {} + +// SAFETY: `CoherentHandle` provides no CPU access to the underlying allocation. The only +// operations on `&CoherentHandle` are reading the DMA handle and size, both of which are +// plain `Copy` values. +unsafe impl Sync for CoherentHandle {} + /// Reads a field of an item from an allocated region of structs. /// /// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating -- cgit v1.2.3 From 816718c611cab2164d718b91ad8204afcd0af81f Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Fri, 27 Mar 2026 00:22:07 +0900 Subject: rust: dma: add from-slice constructors for Coherent and CoherentBox A very common pattern is to create a block of coherent memory with the content of an already-existing slice of bytes (e.g. a loaded firmware blob). `CoherentBox` makes this easier, but still implies a potentially panicking operation with `copy_from_slice` that requires a `PANIC` comment. Add `from_slice_with_attrs` and `from_slice` methods to both `Coherent` and `CoherentBox` to turn this into a trivial one-step operation. Reviewed-by: Andreas Hindborg Reviewed-by: Gary Guo Reviewed-by: Danilo Krummrich Link: https://patch.msgid.link/20260327-b4-nova-dma-removal-v2-1-616e1d0b5cb3@nvidia.com Signed-off-by: Alexandre Courbot --- rust/kernel/dma.rs | 107 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) (limited to 'rust/kernel') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 7ea3c3b73367..4995ee5dc689 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -453,6 +453,66 @@ impl CoherentBox<[T]> { Ok(()) } + + /// Allocates a region of coherent memory of the same size as `data` and initializes it with a + /// copy of its contents. + /// + /// This is the [`CoherentBox`] variant of [`Coherent::from_slice_with_attrs`]. + /// + /// # Examples + /// + /// ``` + /// use core::ops::Deref; + /// + /// # use kernel::device::{Bound, Device}; + /// use kernel::dma::{ + /// attrs::*, + /// CoherentBox + /// }; + /// + /// # fn test(dev: &Device) -> Result { + /// let data = [0u8, 1u8, 2u8, 3u8]; + /// let c: CoherentBox<[u8]> = + /// CoherentBox::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// + /// assert_eq!(c.deref(), &data); + /// # Ok::<(), Error>(()) } + /// ``` + pub fn from_slice_with_attrs( + dev: &device::Device, + data: &[T], + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result + where + T: Copy, + { + let mut slice = Self(Coherent::::alloc_slice_with_attrs( + dev, + data.len(), + gfp_flags, + dma_attrs, + )?); + + // PANIC: `slice` was created with length `data.len()`. + slice.copy_from_slice(data); + + Ok(slice) + } + + /// Performs the same functionality as [`CoherentBox::from_slice_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn from_slice( + dev: &device::Device, + data: &[T], + gfp_flags: kernel::alloc::Flags, + ) -> Result + where + T: Copy, + { + Self::from_slice_with_attrs(dev, data, gfp_flags, Attrs(0)) + } } impl CoherentBox { @@ -839,6 +899,53 @@ impl Coherent { ) -> Result> { Self::zeroed_slice_with_attrs(dev, len, gfp_flags, Attrs(0)) } + + /// Allocates a region of coherent memory of the same size as `data` and initializes it with a + /// copy of its contents. + /// + /// # Examples + /// + /// ``` + /// # use kernel::device::{Bound, Device}; + /// use kernel::dma::{ + /// attrs::*, + /// Coherent + /// }; + /// + /// # fn test(dev: &Device) -> Result { + /// let data = [0u8, 1u8, 2u8, 3u8]; + /// // `c` has the same content as `data`. + /// let c: Coherent<[u8]> = + /// Coherent::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + pub fn from_slice_with_attrs( + dev: &device::Device, + data: &[T], + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result> + where + T: Copy, + { + CoherentBox::from_slice_with_attrs(dev, data, gfp_flags, dma_attrs).map(Into::into) + } + + /// Performs the same functionality as [`Coherent::from_slice_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn from_slice( + dev: &device::Device, + data: &[T], + gfp_flags: kernel::alloc::Flags, + ) -> Result> + where + T: Copy, + { + Self::from_slice_with_attrs(dev, data, gfp_flags, Attrs(0)) + } } impl Coherent<[T]> { -- cgit v1.2.3