From b7a124928b5875675539b6a6f708f56911f271f7 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Thu, 22 Jan 2026 16:43:02 -0500 Subject: rust/drm: Fixup import styles This is to match https://docs.kernel.org/rust/coding-guidelines.html#imports There should be no functional changes in this patch. Signed-off-by: Lyude Paul Link: https://patch.msgid.link/20260122214316.3281257-1-lyude@redhat.com [ Move trailing `//` at the end. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/drm/device.rs | 31 ++++++++++++++++++++++++------- rust/kernel/drm/driver.rs | 12 +++++++++--- rust/kernel/drm/file.rs | 14 +++++++++++--- rust/kernel/drm/gem/mod.rs | 25 ++++++++++++++++++++----- 4 files changed, 64 insertions(+), 18 deletions(-) (limited to 'rust') diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 3ce8f62a0056..ae123ffece79 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -6,15 +6,32 @@ use crate::{ alloc::allocator::Kmalloc, - bindings, device, drm, - drm::driver::AllocImpl, - error::from_err_ptr, - error::Result, + bindings, + device, + drm::{ + self, + driver::AllocImpl, // + }, + error::{ + from_err_ptr, + Result, // + }, prelude::*, - sync::aref::{ARef, AlwaysRefCounted}, - types::Opaque, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, // +}; +use core::{ + alloc::Layout, + mem, + ops::Deref, + ptr::{ + self, + NonNull, // + }, }; -use core::{alloc::Layout, mem, ops::Deref, ptr, ptr::NonNull}; #[cfg(CONFIG_DRM_LEGACY)] macro_rules! drm_legacy_fields { diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index e09f977b5b51..6bd097f16ce7 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -5,10 +5,16 @@ //! C header: [`include/drm/drm_drv.h`](srctree/include/drm/drm_drv.h) use crate::{ - bindings, device, devres, drm, - error::{to_result, Result}, + bindings, + device, + devres, + drm, + error::{ + to_result, + Result, // + }, prelude::*, - sync::aref::ARef, + sync::aref::ARef, // }; use macros::vtable; diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 8c46f8d51951..7dade6dfa1ba 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -4,9 +4,17 @@ //! //! C header: [`include/drm/drm_file.h`](srctree/include/drm/drm_file.h) -use crate::{bindings, drm, error::Result, prelude::*, types::Opaque}; -use core::marker::PhantomData; -use core::pin::Pin; +use crate::{ + bindings, + drm, + error::Result, + prelude::*, + types::Opaque, // +}; +use core::{ + marker::PhantomData, + pin::Pin, // +}; /// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance). pub trait DriverFile { diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index d49a9ba02635..56b7641b1405 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -6,14 +6,29 @@ use crate::{ alloc::flags::*, - bindings, drm, - drm::driver::{AllocImpl, AllocOps}, - error::{to_result, Result}, + bindings, + drm::{ + self, + driver::{ + AllocImpl, + AllocOps, // + }, + }, + error::{ + to_result, + Result, // + }, prelude::*, - sync::aref::{ARef, AlwaysRefCounted}, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, types::Opaque, }; -use core::{ops::Deref, ptr::NonNull}; +use core::{ + ops::Deref, + ptr::NonNull, // +}; /// A type alias for retrieving a [`Driver`]s [`DriverFile`] implementation from its /// [`DriverObject`] implementation. -- cgit v1.2.3 From 6ad005ce6994599e2ae338007a54dd21063aae15 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Thu, 22 Jan 2026 16:43:03 -0500 Subject: rust/drm: Remove imports covered by prelude::* This just removes any explicit imports of items in files that are already being pulled in by `use prelude::*;`. There should be no functional changes in this patch. Signed-off-by: Lyude Paul Link: https://patch.msgid.link/20260122214316.3281257-2-lyude@redhat.com Signed-off-by: Danilo Krummrich --- rust/kernel/drm/device.rs | 5 +---- rust/kernel/drm/driver.rs | 6 +----- rust/kernel/drm/file.rs | 6 +----- rust/kernel/drm/gem/mod.rs | 6 +----- 4 files changed, 4 insertions(+), 19 deletions(-) (limited to 'rust') diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index ae123ffece79..629ef0bd1188 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -12,10 +12,7 @@ use crate::{ self, driver::AllocImpl, // }, - error::{ - from_err_ptr, - Result, // - }, + error::from_err_ptr, prelude::*, sync::aref::{ ARef, diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 6bd097f16ce7..5233bdebc9fc 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -9,14 +9,10 @@ use crate::{ device, devres, drm, - error::{ - to_result, - Result, // - }, + error::to_result, prelude::*, sync::aref::ARef, // }; -use macros::vtable; /// Driver use the GEM memory manager. This should be set for all modern drivers. pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 7dade6dfa1ba..10160601ce5a 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -7,14 +7,10 @@ use crate::{ bindings, drm, - error::Result, prelude::*, types::Opaque, // }; -use core::{ - marker::PhantomData, - pin::Pin, // -}; +use core::marker::PhantomData; /// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance). pub trait DriverFile { diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 56b7641b1405..b4199945db37 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -5,7 +5,6 @@ //! C header: [`include/drm/drm_gem.h`](srctree/include/drm/drm_gem.h) use crate::{ - alloc::flags::*, bindings, drm::{ self, @@ -14,10 +13,7 @@ use crate::{ AllocOps, // }, }, - error::{ - to_result, - Result, // - }, + error::to_result, prelude::*, sync::aref::{ ARef, -- cgit v1.2.3 From 59f237a0d17fb95f60de15386a6ab968af4bd4d4 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Fri, 6 Mar 2026 16:22:00 +0900 Subject: rust: add EMSGSIZE error code Add the EMSGSIZE error code, which indicates that a message is too long. Tested-by: Zhi Wang Acked-by: Miguel Ojeda Reviewed-by: Gary Guo Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260306-cmdq-continuation-v6-3-cc7b629200ee@nvidia.com Signed-off-by: Alexandre Courbot --- rust/kernel/error.rs | 1 + 1 file changed, 1 insertion(+) (limited to 'rust') diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 258b12afdcba..10fcf1f0404d 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -67,6 +67,7 @@ pub mod code { declare_err!(EDOM, "Math argument out of domain of func."); declare_err!(ERANGE, "Math result not representable."); declare_err!(EOVERFLOW, "Value too large for defined data type."); + declare_err!(EMSGSIZE, "Message too long."); declare_err!(ETIMEDOUT, "Connection timed out."); declare_err!(ERESTARTSYS, "Restart the system call."); declare_err!(ERESTARTNOINTR, "System call was interrupted by a signal and will be restarted."); -- cgit v1.2.3 From f9f0b4a1f35d39a1a2a2f8ec46eb7b81efc70a63 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 19 Mar 2026 17:07:22 -0400 Subject: rust: interop: Add list module for C linked list interface Add a new module `kernel::interop::list` for working with C's doubly circular linked lists. Provide low-level iteration over list nodes. Typed iteration over actual items is provided with a `clist_create` macro to assist in creation of the `CList` type. Cc: Nikola Djukic Reviewed-by: Daniel Almeida Reviewed-by: Alexandre Courbot Reviewed-by: Alice Ryhl Acked-by: Alexandre Courbot Acked-by: Gary Guo Acked-by: Miguel Ojeda Signed-off-by: Joel Fernandes Link: https://patch.msgid.link/20260319210722.1543776-1-joelagnelf@nvidia.com [ * Remove stray empty comment and double blank line in doctest, * Improve wording and fix a few typos, * Use markdown emphasis instead of caps, * Move interop/mod.rs to interop.rs. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/helpers/helpers.c | 1 + rust/helpers/list.c | 17 +++ rust/kernel/interop.rs | 9 ++ rust/kernel/interop/list.rs | 339 ++++++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 2 + 5 files changed, 368 insertions(+) create mode 100644 rust/helpers/list.c create mode 100644 rust/kernel/interop.rs create mode 100644 rust/kernel/interop/list.rs (limited to 'rust') diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index a3c42e51f00a..724fcb8240ac 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -35,6 +35,7 @@ #include "io.c" #include "jump_label.c" #include "kunit.c" +#include "list.c" #include "maple_tree.c" #include "mm.c" #include "mutex.c" diff --git a/rust/helpers/list.c b/rust/helpers/list.c new file mode 100644 index 000000000000..18095a5593c5 --- /dev/null +++ b/rust/helpers/list.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Helpers for C circular doubly linked list implementation. + */ + +#include + +__rust_helper void rust_helper_INIT_LIST_HEAD(struct list_head *list) +{ + INIT_LIST_HEAD(list); +} + +__rust_helper void rust_helper_list_add_tail(struct list_head *new, struct list_head *head) +{ + list_add_tail(new, head); +} diff --git a/rust/kernel/interop.rs b/rust/kernel/interop.rs new file mode 100644 index 000000000000..3b371d782a59 --- /dev/null +++ b/rust/kernel/interop.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Infrastructure for interfacing Rust code with C kernel subsystems. +//! +//! This module is intended for low-level, unsafe Rust infrastructure code +//! that interoperates between Rust and C. It is *not* for use directly in +//! Rust drivers. + +pub mod list; diff --git a/rust/kernel/interop/list.rs b/rust/kernel/interop/list.rs new file mode 100644 index 000000000000..54265ea036bb --- /dev/null +++ b/rust/kernel/interop/list.rs @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust interface for C doubly circular intrusive linked lists. +//! +//! This module provides Rust abstractions for iterating over C `list_head`-based +//! linked lists. It should only be used for cases where C and Rust code share +//! direct access to the same linked list through a C interop interface. +//! +//! Note: This *must not* be used by Rust components that just need a linked list +//! primitive. Use [`kernel::list::List`] instead. +//! +//! # Examples +//! +//! ``` +//! use kernel::{ +//! bindings, +//! interop::list::clist_create, +//! types::Opaque, +//! }; +//! # // Create test list with values (0, 10, 20) - normally done by C code but it is +//! # // emulated here for doctests using the C bindings. +//! # use core::mem::MaybeUninit; +//! # +//! # /// C struct with embedded `list_head` (typically will be allocated by C code). +//! # #[repr(C)] +//! # pub struct SampleItemC { +//! # pub value: i32, +//! # pub link: bindings::list_head, +//! # } +//! # +//! # let mut head = MaybeUninit::::uninit(); +//! # +//! # let head = head.as_mut_ptr(); +//! # // SAFETY: `head` and all the items are test objects allocated in this scope. +//! # unsafe { bindings::INIT_LIST_HEAD(head) }; +//! # +//! # let mut items = [ +//! # MaybeUninit::::uninit(), +//! # MaybeUninit::::uninit(), +//! # MaybeUninit::::uninit(), +//! # ]; +//! # +//! # for (i, item) in items.iter_mut().enumerate() { +//! # let ptr = item.as_mut_ptr(); +//! # // SAFETY: `ptr` points to a valid `MaybeUninit`. +//! # unsafe { (*ptr).value = i as i32 * 10 }; +//! # // SAFETY: `&raw mut` creates a pointer valid for `INIT_LIST_HEAD`. +//! # unsafe { bindings::INIT_LIST_HEAD(&raw mut (*ptr).link) }; +//! # // SAFETY: `link` was just initialized and `head` is a valid list head. +//! # unsafe { bindings::list_add_tail(&mut (*ptr).link, head) }; +//! # } +//! +//! /// Rust wrapper for the C struct. +//! /// +//! /// The list item struct in this example is defined in C code as: +//! /// +//! /// ```c +//! /// struct SampleItemC { +//! /// int value; +//! /// struct list_head link; +//! /// }; +//! /// ``` +//! #[repr(transparent)] +//! pub struct Item(Opaque); +//! +//! impl Item { +//! pub fn value(&self) -> i32 { +//! // SAFETY: `Item` has the same layout as `SampleItemC`. +//! unsafe { (*self.0.get()).value } +//! } +//! } +//! +//! // Create typed [`CList`] from sentinel head. +//! // SAFETY: `head` is valid and initialized, items are `SampleItemC` with +//! // embedded `link` field, and `Item` is `#[repr(transparent)]` over `SampleItemC`. +//! let list = unsafe { clist_create!(head, Item, SampleItemC, link) }; +//! +//! // Iterate directly over typed items. +//! let mut found_0 = false; +//! let mut found_10 = false; +//! let mut found_20 = false; +//! +//! for item in list.iter() { +//! let val = item.value(); +//! if val == 0 { found_0 = true; } +//! if val == 10 { found_10 = true; } +//! if val == 20 { found_20 = true; } +//! } +//! +//! assert!(found_0 && found_10 && found_20); +//! ``` + +use core::{ + iter::FusedIterator, + marker::PhantomData, // +}; + +use crate::{ + bindings, + types::Opaque, // +}; + +use pin_init::{ + pin_data, + pin_init, + PinInit, // +}; + +/// FFI wrapper for a C `list_head` object used in intrusive linked lists. +/// +/// # Invariants +/// +/// - The underlying `list_head` is initialized with valid non-`NULL` `next`/`prev` pointers. +#[pin_data] +#[repr(transparent)] +pub struct CListHead { + #[pin] + inner: Opaque, +} + +impl CListHead { + /// Create a `&CListHead` reference from a raw `list_head` pointer. + /// + /// # Safety + /// + /// - `ptr` must be a valid pointer to an initialized `list_head` (e.g. via + /// `INIT_LIST_HEAD()`), with valid non-`NULL` `next`/`prev` pointers. + /// - `ptr` must remain valid for the lifetime `'a`. + /// - The list and all linked `list_head` nodes must not be modified from + /// anywhere for the lifetime `'a`, unless done so via any [`CListHead`] APIs. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::list_head) -> &'a Self { + // SAFETY: + // - `CListHead` has the same layout as `list_head`. + // - `ptr` is valid and unmodified for `'a` per caller guarantees. + unsafe { &*ptr.cast() } + } + + /// Get the raw `list_head` pointer. + #[inline] + pub fn as_raw(&self) -> *mut bindings::list_head { + self.inner.get() + } + + /// Get the next [`CListHead`] in the list. + #[inline] + pub fn next(&self) -> &Self { + let raw = self.as_raw(); + // SAFETY: + // - `self.as_raw()` is valid and initialized per type invariants. + // - The `next` pointer is valid and non-`NULL` per type invariants + // (initialized via `INIT_LIST_HEAD()` or equivalent). + unsafe { Self::from_raw((*raw).next) } + } + + /// Check if this node is linked in a list (not isolated). + #[inline] + pub fn is_linked(&self) -> bool { + let raw = self.as_raw(); + // SAFETY: `self.as_raw()` is valid per type invariants. + unsafe { (*raw).next != raw && (*raw).prev != raw } + } + + /// Returns a pin-initializer for the list head. + pub fn new() -> impl PinInit { + pin_init!(Self { + // SAFETY: `INIT_LIST_HEAD` initializes `slot` to a valid empty list. + inner <- Opaque::ffi_init(|slot| unsafe { bindings::INIT_LIST_HEAD(slot) }), + }) + } +} + +// SAFETY: `list_head` contains no thread-bound state; it only holds +// `next`/`prev` pointers. +unsafe impl Send for CListHead {} + +// SAFETY: `CListHead` can be shared among threads as modifications are +// not allowed at the moment. +unsafe impl Sync for CListHead {} + +impl PartialEq for CListHead { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self, other) + } +} + +impl Eq for CListHead {} + +/// Low-level iterator over `list_head` nodes. +/// +/// An iterator used to iterate over a C intrusive linked list (`list_head`). The caller has to +/// perform conversion of returned [`CListHead`] to an item (using [`container_of`] or similar). +/// +/// # Invariants +/// +/// `current` and `sentinel` are valid references into an initialized linked list. +struct CListHeadIter<'a> { + /// Current position in the list. + current: &'a CListHead, + /// The sentinel head (used to detect end of iteration). + sentinel: &'a CListHead, +} + +impl<'a> Iterator for CListHeadIter<'a> { + type Item = &'a CListHead; + + #[inline] + fn next(&mut self) -> Option { + // Check if we've reached the sentinel (end of list). + if self.current == self.sentinel { + return None; + } + + let item = self.current; + self.current = item.next(); + Some(item) + } +} + +impl<'a> FusedIterator for CListHeadIter<'a> {} + +/// A typed C linked list with a sentinel head intended for FFI use-cases where +/// a C subsystem manages a linked list that Rust code needs to read. Generally +/// required only for special cases. +/// +/// A sentinel head [`CListHead`] represents the entire linked list and can be used +/// for iteration over items of type `T`; it is not associated with a specific item. +/// +/// The const generic `OFFSET` specifies the byte offset of the `list_head` field within +/// the struct that `T` wraps. +/// +/// # Invariants +/// +/// - The sentinel [`CListHead`] has valid non-`NULL` `next`/`prev` pointers. +/// - `OFFSET` is the byte offset of the `list_head` field within the struct that `T` wraps. +/// - All the list's `list_head` nodes have valid non-`NULL` `next`/`prev` pointers. +#[repr(transparent)] +pub struct CList(CListHead, PhantomData); + +impl CList { + /// Create a typed [`CList`] reference from a raw sentinel `list_head` pointer. + /// + /// # Safety + /// + /// - `ptr` must be a valid pointer to an initialized sentinel `list_head` (e.g. via + /// `INIT_LIST_HEAD()`), with valid non-`NULL` `next`/`prev` pointers. + /// - `ptr` must remain valid for the lifetime `'a`. + /// - The list and all linked nodes must not be concurrently modified for the lifetime `'a`. + /// - The list must contain items where the `list_head` field is at byte offset `OFFSET`. + /// - `T` must be `#[repr(transparent)]` over the C struct. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::list_head) -> &'a Self { + // SAFETY: + // - `CList` has the same layout as `CListHead` due to `#[repr(transparent)]`. + // - Caller guarantees `ptr` is a valid, sentinel `list_head` object. + unsafe { &*ptr.cast() } + } + + /// Check if the list is empty. + #[inline] + pub fn is_empty(&self) -> bool { + !self.0.is_linked() + } + + /// Create an iterator over typed items. + #[inline] + pub fn iter(&self) -> CListIter<'_, T, OFFSET> { + let head = &self.0; + CListIter { + head_iter: CListHeadIter { + current: head.next(), + sentinel: head, + }, + _phantom: PhantomData, + } + } +} + +/// High-level iterator over typed list items. +pub struct CListIter<'a, T, const OFFSET: usize> { + head_iter: CListHeadIter<'a>, + _phantom: PhantomData<&'a T>, +} + +impl<'a, T, const OFFSET: usize> Iterator for CListIter<'a, T, OFFSET> { + type Item = &'a T; + + #[inline] + fn next(&mut self) -> Option { + let head = self.head_iter.next()?; + + // Convert to item using `OFFSET`. + // + // SAFETY: The pointer calculation is valid because `OFFSET` is derived + // from `offset_of!` per type invariants. + Some(unsafe { &*head.as_raw().byte_sub(OFFSET).cast::() }) + } +} + +impl<'a, T, const OFFSET: usize> FusedIterator for CListIter<'a, T, OFFSET> {} + +/// Create a C doubly-circular linked list interface [`CList`] from a raw `list_head` pointer. +/// +/// This macro creates a `CList` that can iterate over items of type `$rust_type` +/// linked via the `$field` field in the underlying C struct `$c_type`. +/// +/// # Arguments +/// +/// - `$head`: Raw pointer to the sentinel `list_head` object (`*mut bindings::list_head`). +/// - `$rust_type`: Each item's Rust wrapper type. +/// - `$c_type`: Each item's C struct type that contains the embedded `list_head`. +/// - `$field`: The name of the `list_head` field within the C struct. +/// +/// # Safety +/// +/// The caller must ensure: +/// +/// - `$head` is a valid, initialized sentinel `list_head` (e.g. via `INIT_LIST_HEAD()`) +/// pointing to a list that is not concurrently modified for the lifetime of the [`CList`]. +/// - The list contains items of type `$c_type` linked via an embedded `$field`. +/// - `$rust_type` is `#[repr(transparent)]` over `$c_type` or has compatible layout. +/// +/// # Examples +/// +/// Refer to the examples in the [`crate::interop::list`] module documentation. +#[macro_export] +macro_rules! clist_create { + ($head:expr, $rust_type:ty, $c_type:ty, $($field:tt).+) => {{ + // Compile-time check that field path is a `list_head`. + let _: fn(*const $c_type) -> *const $crate::bindings::list_head = + |p| &raw const (*p).$($field).+; + + // Calculate offset and create `CList`. + const OFFSET: usize = ::core::mem::offset_of!($c_type, $($field).+); + $crate::interop::list::CList::<$rust_type, OFFSET>::from_raw($head) + }}; +} +pub use clist_create; diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 34b924819288..b8eb0d217f85 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -29,6 +29,7 @@ #![feature(lint_reasons)] // // Stable since Rust 1.82.0. +#![feature(offset_of_nested)] #![feature(raw_ref_op)] // // Stable since Rust 1.83.0. @@ -110,6 +111,7 @@ pub mod id_pool; #[doc(hidden)] pub mod impl_flags; pub mod init; +pub mod interop; pub mod io; pub mod ioctl; pub mod iommu; -- cgit v1.2.3 From b9616d9721bf8a56d5038e85d2ebbe0ec9d56a94 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Fri, 20 Mar 2026 00:57:10 -0400 Subject: rust: gpu: Add GPU buddy allocator bindings Add safe Rust abstractions over the Linux kernel's GPU buddy allocator for physical memory management. The GPU buddy allocator implements a binary buddy system useful for GPU physical memory allocation. nova-core will use it for physical memory allocation. Cc: Nikola Djukic Signed-off-by: Joel Fernandes Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260320045711.43494-2-joelagnelf@nvidia.com [ * Use doc-comments for GpuBuddyAllocMode methods and GpuBuddyGuard, * Fix comma splice in GpuBuddyParams::chunk_size doc-comment, * Remove redundant summary in GpuBuddy::new doc-comment, * Drop Rust helper for gpu_buddy_block_size(). - Danilo ] Signed-off-by: Danilo Krummrich --- rust/bindings/bindings_helper.h | 11 + rust/helpers/gpu.c | 17 ++ rust/helpers/helpers.c | 1 + rust/kernel/gpu.rs | 6 + rust/kernel/gpu/buddy.rs | 614 ++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 2 + 6 files changed, 651 insertions(+) create mode 100644 rust/helpers/gpu.c create mode 100644 rust/kernel/gpu.rs create mode 100644 rust/kernel/gpu/buddy.rs (limited to 'rust') diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 083cc44aa952..dbb765a9fdbd 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -146,6 +147,16 @@ const vm_flags_t RUST_CONST_HELPER_VM_MIXEDMAP = VM_MIXEDMAP; const vm_flags_t RUST_CONST_HELPER_VM_HUGEPAGE = VM_HUGEPAGE; const vm_flags_t RUST_CONST_HELPER_VM_NOHUGEPAGE = VM_NOHUGEPAGE; +#if IS_ENABLED(CONFIG_GPU_BUDDY) +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_RANGE_ALLOCATION = GPU_BUDDY_RANGE_ALLOCATION; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_TOPDOWN_ALLOCATION = GPU_BUDDY_TOPDOWN_ALLOCATION; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CONTIGUOUS_ALLOCATION = + GPU_BUDDY_CONTIGUOUS_ALLOCATION; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CLEAR_ALLOCATION = GPU_BUDDY_CLEAR_ALLOCATION; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_CLEARED = GPU_BUDDY_CLEARED; +const unsigned long RUST_CONST_HELPER_GPU_BUDDY_TRIM_DISABLE = GPU_BUDDY_TRIM_DISABLE; +#endif + #if IS_ENABLED(CONFIG_ANDROID_BINDER_IPC_RUST) #include "../../drivers/android/binder/rust_binder.h" #include "../../drivers/android/binder/rust_binder_events.h" diff --git a/rust/helpers/gpu.c b/rust/helpers/gpu.c new file mode 100644 index 000000000000..a25448d54d72 --- /dev/null +++ b/rust/helpers/gpu.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#ifdef CONFIG_GPU_BUDDY + +__rust_helper u64 rust_helper_gpu_buddy_block_offset(const struct gpu_buddy_block *block) +{ + return gpu_buddy_block_offset(block); +} + +__rust_helper unsigned int rust_helper_gpu_buddy_block_order(struct gpu_buddy_block *block) +{ + return gpu_buddy_block_order(block); +} + +#endif /* CONFIG_GPU_BUDDY */ diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 724fcb8240ac..a53929ce52a3 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -32,6 +32,7 @@ #include "err.c" #include "irq.c" #include "fs.c" +#include "gpu.c" #include "io.c" #include "jump_label.c" #include "kunit.c" diff --git a/rust/kernel/gpu.rs b/rust/kernel/gpu.rs new file mode 100644 index 000000000000..1dc5d0c8c09d --- /dev/null +++ b/rust/kernel/gpu.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! GPU subsystem abstractions. + +#[cfg(CONFIG_GPU_BUDDY = "y")] +pub mod buddy; diff --git a/rust/kernel/gpu/buddy.rs b/rust/kernel/gpu/buddy.rs new file mode 100644 index 000000000000..d502ada6ebbd --- /dev/null +++ b/rust/kernel/gpu/buddy.rs @@ -0,0 +1,614 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! GPU buddy allocator bindings. +//! +//! C header: [`include/linux/gpu_buddy.h`](srctree/include/linux/gpu_buddy.h) +//! +//! This module provides Rust abstractions over the Linux kernel's GPU buddy +//! allocator, which implements a binary buddy memory allocator. +//! +//! The buddy allocator manages a contiguous address space and allocates blocks +//! in power-of-two sizes, useful for GPU physical memory management. +//! +//! # Examples +//! +//! Create a buddy allocator and perform a basic range allocation: +//! +//! ``` +//! use kernel::{ +//! gpu::buddy::{ +//! GpuBuddy, +//! GpuBuddyAllocFlags, +//! GpuBuddyAllocMode, +//! GpuBuddyParams, // +//! }, +//! prelude::*, +//! ptr::Alignment, +//! sizes::*, // +//! }; +//! +//! // Create a 1GB buddy allocator with 4KB minimum chunk size. +//! let buddy = GpuBuddy::new(GpuBuddyParams { +//! base_offset: 0, +//! size: SZ_1G as u64, +//! chunk_size: Alignment::new::(), +//! })?; +//! +//! assert_eq!(buddy.size(), SZ_1G as u64); +//! assert_eq!(buddy.chunk_size(), Alignment::new::()); +//! let initial_free = buddy.avail(); +//! +//! // Allocate 16MB. Block lands at the top of the address range. +//! let allocated = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Simple, +//! SZ_16M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_16M as u64); +//! +//! let block = allocated.iter().next().expect("expected one block"); +//! assert_eq!(block.offset(), (SZ_1G - SZ_16M) as u64); +//! assert_eq!(block.order(), 12); // 2^12 pages = 16MB +//! assert_eq!(block.size(), SZ_16M as u64); +//! assert_eq!(allocated.iter().count(), 1); +//! +//! // Dropping the allocation returns the range to the buddy allocator. +//! drop(allocated); +//! assert_eq!(buddy.avail(), initial_free); +//! # Ok::<(), Error>(()) +//! ``` +//! +//! Top-down allocation allocates from the highest addresses: +//! +//! ``` +//! # use kernel::{ +//! # gpu::buddy::{GpuBuddy, GpuBuddyAllocMode, GpuBuddyAllocFlags, GpuBuddyParams}, +//! # prelude::*, +//! # ptr::Alignment, +//! # sizes::*, // +//! # }; +//! # let buddy = GpuBuddy::new(GpuBuddyParams { +//! # base_offset: 0, +//! # size: SZ_1G as u64, +//! # chunk_size: Alignment::new::(), +//! # })?; +//! # let initial_free = buddy.avail(); +//! let topdown = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::TopDown, +//! SZ_16M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_16M as u64); +//! +//! let block = topdown.iter().next().expect("expected one block"); +//! assert_eq!(block.offset(), (SZ_1G - SZ_16M) as u64); +//! assert_eq!(block.order(), 12); +//! assert_eq!(block.size(), SZ_16M as u64); +//! +//! // Dropping the allocation returns the range to the buddy allocator. +//! drop(topdown); +//! assert_eq!(buddy.avail(), initial_free); +//! # Ok::<(), Error>(()) +//! ``` +//! +//! Non-contiguous allocation can fill fragmented memory by returning multiple +//! blocks: +//! +//! ``` +//! # use kernel::{ +//! # gpu::buddy::{ +//! # GpuBuddy, GpuBuddyAllocFlags, GpuBuddyAllocMode, GpuBuddyParams, +//! # }, +//! # prelude::*, +//! # ptr::Alignment, +//! # sizes::*, // +//! # }; +//! # let buddy = GpuBuddy::new(GpuBuddyParams { +//! # base_offset: 0, +//! # size: SZ_1G as u64, +//! # chunk_size: Alignment::new::(), +//! # })?; +//! # let initial_free = buddy.avail(); +//! // Create fragmentation by allocating 4MB blocks at [0,4M) and [8M,12M). +//! let frag1 = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Range(0..SZ_4M as u64), +//! SZ_4M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_4M as u64); +//! +//! let frag2 = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Range(SZ_8M as u64..(SZ_8M + SZ_4M) as u64), +//! SZ_4M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_8M as u64); +//! +//! // Allocate 8MB, this returns 2 blocks from the holes. +//! let fragmented = KBox::pin_init( +//! buddy.alloc_blocks( +//! GpuBuddyAllocMode::Range(0..SZ_16M as u64), +//! SZ_8M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! assert_eq!(buddy.avail(), initial_free - SZ_16M as u64); +//! +//! let (mut count, mut total) = (0u32, 0u64); +//! for block in fragmented.iter() { +//! assert_eq!(block.size(), SZ_4M as u64); +//! total += block.size(); +//! count += 1; +//! } +//! assert_eq!(total, SZ_8M as u64); +//! assert_eq!(count, 2); +//! # Ok::<(), Error>(()) +//! ``` +//! +//! Contiguous allocation fails when only fragmented space is available: +//! +//! ``` +//! # use kernel::{ +//! # gpu::buddy::{ +//! # GpuBuddy, GpuBuddyAllocFlag, GpuBuddyAllocFlags, GpuBuddyAllocMode, GpuBuddyParams, +//! # }, +//! # prelude::*, +//! # ptr::Alignment, +//! # sizes::*, // +//! # }; +//! // Create a small 16MB buddy allocator with fragmented memory. +//! let small = GpuBuddy::new(GpuBuddyParams { +//! base_offset: 0, +//! size: SZ_16M as u64, +//! chunk_size: Alignment::new::(), +//! })?; +//! +//! let _hole1 = KBox::pin_init( +//! small.alloc_blocks( +//! GpuBuddyAllocMode::Range(0..SZ_4M as u64), +//! SZ_4M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! +//! let _hole2 = KBox::pin_init( +//! small.alloc_blocks( +//! GpuBuddyAllocMode::Range(SZ_8M as u64..(SZ_8M + SZ_4M) as u64), +//! SZ_4M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlags::default(), +//! ), +//! GFP_KERNEL, +//! )?; +//! +//! // 8MB contiguous should fail, only two non-contiguous 4MB holes exist. +//! let result = KBox::pin_init( +//! small.alloc_blocks( +//! GpuBuddyAllocMode::Simple, +//! SZ_8M as u64, +//! Alignment::new::(), +//! GpuBuddyAllocFlag::Contiguous, +//! ), +//! GFP_KERNEL, +//! ); +//! assert!(result.is_err()); +//! # Ok::<(), Error>(()) +//! ``` + +use core::ops::Range; + +use crate::{ + bindings, + clist_create, + error::to_result, + interop::list::CListHead, + new_mutex, + prelude::*, + ptr::Alignment, + sync::{ + lock::mutex::MutexGuard, + Arc, + Mutex, // + }, + types::Opaque, // +}; + +/// Allocation mode for the GPU buddy allocator. +/// +/// The mode determines the primary allocation strategy. Modes are mutually +/// exclusive: an allocation is either simple, range-constrained, or top-down. +/// +/// Orthogonal modifier flags (e.g., contiguous, clear) are specified separately +/// via [`GpuBuddyAllocFlags`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum GpuBuddyAllocMode { + /// Simple allocation without constraints. + Simple, + /// Range-based allocation within the given address range. + Range(Range), + /// Allocate from top of address space downward. + TopDown, +} + +impl GpuBuddyAllocMode { + /// Returns the C flags corresponding to the allocation mode. + fn as_flags(&self) -> usize { + match self { + Self::Simple => 0, + Self::Range(_) => bindings::GPU_BUDDY_RANGE_ALLOCATION, + Self::TopDown => bindings::GPU_BUDDY_TOPDOWN_ALLOCATION, + } + } + + /// Extracts the range start/end, defaulting to `(0, 0)` for non-range modes. + fn range(&self) -> (u64, u64) { + match self { + Self::Range(range) => (range.start, range.end), + _ => (0, 0), + } + } +} + +crate::impl_flags!( + /// Modifier flags for GPU buddy allocation. + /// + /// These flags can be combined with any [`GpuBuddyAllocMode`] to control + /// additional allocation behavior. + #[derive(Clone, Copy, Default, PartialEq, Eq)] + pub struct GpuBuddyAllocFlags(usize); + + /// Individual modifier flag for GPU buddy allocation. + #[derive(Clone, Copy, PartialEq, Eq)] + pub enum GpuBuddyAllocFlag { + /// Allocate physically contiguous blocks. + Contiguous = bindings::GPU_BUDDY_CONTIGUOUS_ALLOCATION, + + /// Request allocation from cleared (zeroed) memory. + Clear = bindings::GPU_BUDDY_CLEAR_ALLOCATION, + + /// Disable trimming of partially used blocks. + TrimDisable = bindings::GPU_BUDDY_TRIM_DISABLE, + } +); + +/// Parameters for creating a GPU buddy allocator. +pub struct GpuBuddyParams { + /// Base offset (in bytes) where the managed memory region starts. + /// Allocations will be offset by this value. + pub base_offset: u64, + /// Total size (in bytes) of the address space managed by the allocator. + pub size: u64, + /// Minimum allocation unit / chunk size; must be >= 4KB. + pub chunk_size: Alignment, +} + +/// Inner structure holding the actual buddy allocator. +/// +/// # Synchronization +/// +/// The C `gpu_buddy` API requires synchronization (see `include/linux/gpu_buddy.h`). +/// Internal locking ensures all allocator and free operations are properly +/// synchronized, preventing races between concurrent allocations and the +/// freeing that occurs when [`AllocatedBlocks`] is dropped. +/// +/// # Invariants +/// +/// The inner [`Opaque`] contains an initialized buddy allocator. +#[pin_data(PinnedDrop)] +struct GpuBuddyInner { + #[pin] + inner: Opaque, + + // TODO: Replace `Mutex<()>` with `Mutex>` once `Mutex::new()` + // accepts `impl PinInit`. + #[pin] + lock: Mutex<()>, + /// Cached creation parameters (do not change after init). + params: GpuBuddyParams, +} + +impl GpuBuddyInner { + /// Create a pin-initializer for the buddy allocator. + fn new(params: GpuBuddyParams) -> impl PinInit { + let size = params.size; + let chunk_size = params.chunk_size; + + // INVARIANT: `gpu_buddy_init` returns 0 on success, at which point the + // `gpu_buddy` structure is initialized and ready for use with all + // `gpu_buddy_*` APIs. `try_pin_init!` only completes if all fields succeed, + // so the invariant holds when construction finishes. + try_pin_init!(Self { + inner <- Opaque::try_ffi_init(|ptr| { + // SAFETY: `ptr` points to valid uninitialized memory from the pin-init + // infrastructure. `gpu_buddy_init` will initialize the structure. + to_result(unsafe { + bindings::gpu_buddy_init(ptr, size, chunk_size.as_usize() as u64) + }) + }), + lock <- new_mutex!(()), + params, + }) + } + + /// Lock the mutex and return a guard for accessing the allocator. + fn lock(&self) -> GpuBuddyGuard<'_> { + GpuBuddyGuard { + inner: self, + _guard: self.lock.lock(), + } + } +} + +#[pinned_drop] +impl PinnedDrop for GpuBuddyInner { + fn drop(self: Pin<&mut Self>) { + let guard = self.lock(); + + // SAFETY: Per the type invariant, `inner` contains an initialized + // allocator. `guard` provides exclusive access. + unsafe { bindings::gpu_buddy_fini(guard.as_raw()) }; + } +} + +// SAFETY: `GpuBuddyInner` can be sent between threads. +unsafe impl Send for GpuBuddyInner {} + +// SAFETY: `GpuBuddyInner` is `Sync` because `GpuBuddyInner::lock` +// serializes all access to the C allocator, preventing data races. +unsafe impl Sync for GpuBuddyInner {} + +/// Guard that proves the lock is held, enabling access to the allocator. +/// +/// The `_guard` holds the lock for the duration of this guard's lifetime. +struct GpuBuddyGuard<'a> { + inner: &'a GpuBuddyInner, + _guard: MutexGuard<'a, ()>, +} + +impl GpuBuddyGuard<'_> { + /// Get a raw pointer to the underlying C `gpu_buddy` structure. + fn as_raw(&self) -> *mut bindings::gpu_buddy { + self.inner.inner.get() + } +} + +/// GPU buddy allocator instance. +/// +/// This structure wraps the C `gpu_buddy` allocator using reference counting. +/// The allocator is automatically cleaned up when all references are dropped. +/// +/// Refer to the module-level documentation for usage examples. +pub struct GpuBuddy(Arc); + +impl GpuBuddy { + /// Create a new buddy allocator. + /// + /// The allocator manages a contiguous address space of the given size, with the + /// specified minimum allocation unit (chunk_size must be at least 4KB). + pub fn new(params: GpuBuddyParams) -> Result { + Arc::pin_init(GpuBuddyInner::new(params), GFP_KERNEL).map(Self) + } + + /// Get the base offset for allocations. + pub fn base_offset(&self) -> u64 { + self.0.params.base_offset + } + + /// Get the chunk size (minimum allocation unit). + pub fn chunk_size(&self) -> Alignment { + self.0.params.chunk_size + } + + /// Get the total managed size. + pub fn size(&self) -> u64 { + self.0.params.size + } + + /// Get the available (free) memory in bytes. + pub fn avail(&self) -> u64 { + let guard = self.0.lock(); + + // SAFETY: Per the type invariant, `inner` contains an initialized allocator. + // `guard` provides exclusive access. + unsafe { (*guard.as_raw()).avail } + } + + /// Allocate blocks from the buddy allocator. + /// + /// Returns a pin-initializer for [`AllocatedBlocks`]. + pub fn alloc_blocks( + &self, + mode: GpuBuddyAllocMode, + size: u64, + min_block_size: Alignment, + flags: impl Into, + ) -> impl PinInit { + let buddy_arc = Arc::clone(&self.0); + let (start, end) = mode.range(); + let mode_flags = mode.as_flags(); + let modifier_flags = flags.into(); + + // Create pin-initializer that initializes list and allocates blocks. + try_pin_init!(AllocatedBlocks { + buddy: buddy_arc, + list <- CListHead::new(), + _: { + // Reject zero-sized or inverted ranges. + if let GpuBuddyAllocMode::Range(range) = &mode { + if range.is_empty() { + Err::<(), Error>(EINVAL)?; + } + } + + // Lock while allocating to serialize with concurrent frees. + let guard = buddy.lock(); + + // SAFETY: Per the type invariant, `inner` contains an initialized + // allocator. `guard` provides exclusive access. + to_result(unsafe { + bindings::gpu_buddy_alloc_blocks( + guard.as_raw(), + start, + end, + size, + min_block_size.as_usize() as u64, + list.as_raw(), + mode_flags | usize::from(modifier_flags), + ) + })? + } + }) + } +} + +/// Allocated blocks from the buddy allocator with automatic cleanup. +/// +/// This structure owns a list of allocated blocks and ensures they are +/// automatically freed when dropped. Use `iter()` to iterate over all +/// allocated blocks. +/// +/// # Invariants +/// +/// - `list` is an initialized, valid list head containing allocated blocks. +#[pin_data(PinnedDrop)] +pub struct AllocatedBlocks { + #[pin] + list: CListHead, + buddy: Arc, +} + +impl AllocatedBlocks { + /// Check if the block list is empty. + pub fn is_empty(&self) -> bool { + // An empty list head points to itself. + !self.list.is_linked() + } + + /// Iterate over allocated blocks. + /// + /// Returns an iterator yielding [`AllocatedBlock`] values. Each [`AllocatedBlock`] + /// borrows `self` and is only valid for the duration of that borrow. + pub fn iter(&self) -> impl Iterator> + '_ { + let head = self.list.as_raw(); + // SAFETY: Per the type invariant, `list` is an initialized sentinel `list_head` + // and is not concurrently modified (we hold a `&self` borrow). The list contains + // `gpu_buddy_block` items linked via `__bindgen_anon_1.link`. `Block` is + // `#[repr(transparent)]` over `gpu_buddy_block`. + let clist = unsafe { + clist_create!( + head, + Block, + bindings::gpu_buddy_block, + __bindgen_anon_1.link + ) + }; + + clist + .iter() + .map(|this| AllocatedBlock { this, blocks: self }) + } +} + +#[pinned_drop] +impl PinnedDrop for AllocatedBlocks { + fn drop(self: Pin<&mut Self>) { + let guard = self.buddy.lock(); + + // SAFETY: + // - list is valid per the type's invariants. + // - guard provides exclusive access to the allocator. + unsafe { + bindings::gpu_buddy_free_list(guard.as_raw(), self.list.as_raw(), 0); + } + } +} + +/// A GPU buddy block. +/// +/// Transparent wrapper over C `gpu_buddy_block` structure. This type is returned +/// as references during iteration over [`AllocatedBlocks`]. +/// +/// # Invariants +/// +/// The inner [`Opaque`] contains a valid, allocated `gpu_buddy_block`. +#[repr(transparent)] +struct Block(Opaque); + +impl Block { + /// Get a raw pointer to the underlying C block. + fn as_raw(&self) -> *mut bindings::gpu_buddy_block { + self.0.get() + } + + /// Get the block's raw offset in the buddy address space (without base offset). + fn offset(&self) -> u64 { + // SAFETY: `self.as_raw()` is valid per the type's invariants. + unsafe { bindings::gpu_buddy_block_offset(self.as_raw()) } + } + + /// Get the block order. + fn order(&self) -> u32 { + // SAFETY: `self.as_raw()` is valid per the type's invariants. + unsafe { bindings::gpu_buddy_block_order(self.as_raw()) } + } +} + +// SAFETY: `Block` is a wrapper around `gpu_buddy_block` which can be +// sent across threads safely. +unsafe impl Send for Block {} + +// SAFETY: `Block` is only accessed through shared references after +// allocation, and thus safe to access concurrently across threads. +unsafe impl Sync for Block {} + +/// A buddy block paired with its owning [`AllocatedBlocks`] context. +/// +/// Unlike a raw block, which only knows its offset within the buddy address +/// space, an [`AllocatedBlock`] also has access to the allocator's `base_offset` +/// and `chunk_size`, enabling it to compute absolute offsets and byte sizes. +/// +/// Returned by [`AllocatedBlocks::iter()`]. +pub struct AllocatedBlock<'a> { + this: &'a Block, + blocks: &'a AllocatedBlocks, +} + +impl AllocatedBlock<'_> { + /// Get the block's offset in the address space. + /// + /// Returns the absolute offset including the allocator's base offset. + /// This is the actual address to use for accessing the allocated memory. + pub fn offset(&self) -> u64 { + self.blocks.buddy.params.base_offset + self.this.offset() + } + + /// Get the block order (size = chunk_size << order). + pub fn order(&self) -> u32 { + self.this.order() + } + + /// Get the block's size in bytes. + pub fn size(&self) -> u64 { + (self.blocks.buddy.params.chunk_size.as_usize() as u64) << self.this.order() + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index b8eb0d217f85..e0837ffc91bf 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -105,6 +105,8 @@ pub mod faux; pub mod firmware; pub mod fmt; pub mod fs; +#[cfg(CONFIG_GPU_BUDDY = "y")] +pub mod gpu; #[cfg(CONFIG_I2C = "y")] pub mod i2c; pub mod id_pool; -- cgit v1.2.3 From 7ea1a61129b26709fe85cf5d50da5c47458deb3a Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 20 Mar 2026 20:45:36 +0100 Subject: rust: dma: use "kernel vertical" style for imports Convert all imports to use "kernel vertical" style. With this, subsequent patches neither introduce unrelated changes nor leave an inconsistent import pattern. While at it, drop unnecessary imports covered by prelude::*. Link: https://docs.kernel.org/rust/coding-guidelines.html#imports Reviewed-by: Gary Guo Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260320194626.36263-2-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/dma.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'rust') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index a396f8435739..2eea7e2f8f04 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -5,12 +5,20 @@ //! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h) use crate::{ - bindings, build_assert, device, - device::{Bound, Core}, - error::{to_result, Result}, + bindings, + build_assert, + device::{ + self, + Bound, + Core, // + }, + error::to_result, prelude::*, sync::aref::ARef, - transmute::{AsBytes, FromBytes}, + transmute::{ + AsBytes, + FromBytes, // + }, // }; use core::ptr::NonNull; -- cgit v1.2.3 From d9aee73c56ee971b08173071ad93fa5ebf00a32e Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Fri, 20 Mar 2026 20:45:37 +0100 Subject: rust: dma: add generalized container for types other than slices Currently, `CoherentAllocation` is concecptually a DMA coherent container of a slice of `[T]` of runtime-checked length. Generalize it by creating `dma::Coherent` which can hold any value of `T`. `Coherent::alloc_with_attrs` is implemented but not yet exposed, as I believe we should not expose the way to obtain an uninitialized coherent region. `Coherent<[T]>` provides a `len` method instead of the previous `count()` method to be consistent with methods on slices. The existing type is re-defined as a type alias of `Coherent<[T]>` to ease transition. Methods in use are not yet removed. Signed-off-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260320194626.36263-3-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/device.rs | 4 +- rust/kernel/dma.rs | 361 +++++++++++++++++++++++++++++++------------------- 2 files changed, 226 insertions(+), 139 deletions(-) (limited to 'rust') diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 94e0548e7687..379058eca2ed 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -575,7 +575,7 @@ pub struct CoreInternal; /// The bound context indicates that for the entire duration of the lifetime of a [`Device`] /// reference, the [`Device`] is guaranteed to be bound to a driver. /// -/// Some APIs, such as [`dma::CoherentAllocation`] or [`Devres`] rely on the [`Device`] to be bound, +/// Some APIs, such as [`dma::Coherent`] or [`Devres`] rely on the [`Device`] to be bound, /// which can be proven with the [`Bound`] device context. /// /// Any abstraction that can guarantee a scope where the corresponding bus device is bound, should @@ -584,7 +584,7 @@ pub struct CoreInternal; /// /// [`Devres`]: kernel::devres::Devres /// [`Devres::access`]: kernel::devres::Devres::access -/// [`dma::CoherentAllocation`]: kernel::dma::CoherentAllocation +/// [`dma::Coherent`]: kernel::dma::Coherent pub struct Bound; mod private { diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 2eea7e2f8f04..ff3e147f1a23 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -6,7 +6,6 @@ use crate::{ bindings, - build_assert, device::{ self, Bound, @@ -14,6 +13,7 @@ use crate::{ }, error::to_result, prelude::*, + ptr::KnownSize, sync::aref::ARef, transmute::{ AsBytes, @@ -357,18 +357,17 @@ impl From for bindings::dma_data_direction { /// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map /// large coherent DMA regions. /// -/// A [`CoherentAllocation`] instance contains a pointer to the allocated region (in the +/// A [`Coherent`] instance contains a pointer to the allocated region (in the /// processor's virtual address space) and the device address which can be given to the device -/// as the DMA address base of the region. The region is released once [`CoherentAllocation`] +/// as the DMA address base of the region. The region is released once [`Coherent`] /// is dropped. /// /// # Invariants /// -/// - For the lifetime of an instance of [`CoherentAllocation`], the `cpu_addr` is a valid pointer +/// - For the lifetime of an instance of [`Coherent`], the `cpu_addr` is a valid pointer /// to an allocated region of coherent memory and `dma_handle` is the DMA address base of the /// region. -/// - The size in bytes of the allocation is equal to `size_of:: * count`. -/// - `size_of:: * count` fits into a `usize`. +/// - The size in bytes of the allocation is equal to size information via pointer. // TODO // // DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness @@ -379,124 +378,260 @@ impl From for bindings::dma_data_direction { // allocation from surviving device unbind; it would require RCU read side critical sections to // access the memory, which may require subsequent unnecessary copies. // -// Hence, find a way to revoke the device resources of a `CoherentAllocation`, but not the -// entire `CoherentAllocation` including the allocated memory itself. -pub struct CoherentAllocation { +// Hence, find a way to revoke the device resources of a `Coherent`, but not the +// entire `Coherent` including the allocated memory itself. +pub struct Coherent { dev: ARef, dma_handle: DmaAddress, - count: usize, cpu_addr: NonNull, dma_attrs: Attrs, } -impl CoherentAllocation { - /// Allocates a region of `size_of:: * count` of coherent memory. +impl Coherent { + /// Returns the size in bytes of this allocation. + #[inline] + pub fn size(&self) -> usize { + T::size(self.cpu_addr.as_ptr()) + } + + /// Returns the raw pointer to the allocated region in the CPU's virtual address space. + #[inline] + pub fn as_ptr(&self) -> *const T { + self.cpu_addr.as_ptr() + } + + /// Returns the raw pointer to the allocated region in the CPU's virtual address space as + /// a mutable pointer. + #[inline] + pub fn as_mut_ptr(&self) -> *mut T { + self.cpu_addr.as_ptr() + } + + /// Returns a DMA handle which may be given to the device as the DMA address base of + /// the region. + #[inline] + pub fn dma_handle(&self) -> DmaAddress { + self.dma_handle + } + + /// Returns a reference to the data in the region. /// - /// # Examples + /// # Safety /// - /// ``` - /// # use kernel::device::{Bound, Device}; - /// use kernel::dma::{attrs::*, CoherentAllocation}; + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// slice is live. + /// * Callers must ensure that this call does not race with a write to the same region while + /// the returned slice is live. + #[inline] + pub unsafe fn as_ref(&self) -> &T { + // SAFETY: per safety requirement. + unsafe { &*self.as_ptr() } + } + + /// Returns a mutable reference to the data in the region. /// - /// # fn test(dev: &Device) -> Result { - /// let c: CoherentAllocation = - /// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; - /// # Ok::<(), Error>(()) } - /// ``` - pub fn alloc_attrs( + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// slice is live. + /// * Callers must ensure that this call does not race with a read or write to the same region + /// while the returned slice is live. + #[expect(clippy::mut_from_ref, reason = "unsafe to use API")] + #[inline] + pub unsafe fn as_mut(&self) -> &mut T { + // SAFETY: per safety requirement. + unsafe { &mut *self.as_mut_ptr() } + } + + /// Reads the value of `field` and ensures that its type is [`FromBytes`]. + /// + /// # Safety + /// + /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is + /// validated beforehand. + /// + /// Public but hidden since it should only be used from [`dma_read`] macro. + #[doc(hidden)] + pub unsafe fn field_read(&self, field: *const F) -> F { + // SAFETY: + // - By the safety requirements field is valid. + // - Using read_volatile() here is not sound as per the usual rules, the usage here is + // a special exception with the following notes in place. When dealing with a potential + // race from a hardware or code outside kernel (e.g. user-space program), we need that + // read on a valid memory is not UB. Currently read_volatile() is used for this, and the + // rationale behind is that it should generate the same code as READ_ONCE() which the + // kernel already relies on to avoid UB on data races. Note that the usage of + // read_volatile() is limited to this particular case, it cannot be used to prevent + // the UB caused by racing between two kernel functions nor do they provide atomicity. + unsafe { field.read_volatile() } + } + + /// Writes a value to `field` and ensures that its type is [`AsBytes`]. + /// + /// # Safety + /// + /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is + /// validated beforehand. + /// + /// Public but hidden since it should only be used from [`dma_write`] macro. + #[doc(hidden)] + pub unsafe fn field_write(&self, field: *mut F, val: F) { + // SAFETY: + // - By the safety requirements field is valid. + // - Using write_volatile() here is not sound as per the usual rules, the usage here is + // a special exception with the following notes in place. When dealing with a potential + // race from a hardware or code outside kernel (e.g. user-space program), we need that + // write on a valid memory is not UB. Currently write_volatile() is used for this, and the + // rationale behind is that it should generate the same code as WRITE_ONCE() which the + // kernel already relies on to avoid UB on data races. Note that the usage of + // write_volatile() is limited to this particular case, it cannot be used to prevent + // the UB caused by racing between two kernel functions nor do they provide atomicity. + unsafe { field.write_volatile(val) } + } +} + +impl Coherent { + /// Allocates a region of `T` of coherent memory. + #[expect(unused)] + fn alloc_with_attrs( dev: &device::Device, - count: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, - ) -> Result> { - build_assert!( - core::mem::size_of::() > 0, - "It doesn't make sense for the allocated type to be a ZST" - ); + ) -> Result { + const { + assert!( + core::mem::size_of::() > 0, + "It doesn't make sense for the allocated type to be a ZST" + ); + } - let size = count - .checked_mul(core::mem::size_of::()) - .ok_or(EOVERFLOW)?; let mut dma_handle = 0; // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. let addr = unsafe { bindings::dma_alloc_attrs( dev.as_raw(), - size, + core::mem::size_of::(), &mut dma_handle, gfp_flags.as_raw(), dma_attrs.as_raw(), ) }; - let addr = NonNull::new(addr).ok_or(ENOMEM)?; + let cpu_addr = NonNull::new(addr.cast()).ok_or(ENOMEM)?; // INVARIANT: - // - We just successfully allocated a coherent region which is accessible for - // `count` elements, hence the cpu address is valid. We also hold a refcounted reference - // to the device. - // - The allocated `size` is equal to `size_of:: * count`. - // - The allocated `size` fits into a `usize`. + // - We just successfully allocated a coherent region which is adequately sized for `T`, + // hence the cpu address is valid. + // - We also hold a refcounted reference to the device. Ok(Self { dev: dev.into(), dma_handle, - count, - cpu_addr: addr.cast(), + cpu_addr, dma_attrs, }) } - /// Performs the same functionality as [`CoherentAllocation::alloc_attrs`], except the - /// `dma_attrs` is 0 by default. - pub fn alloc_coherent( + /// Allocates a region of `[T; len]` of coherent memory. + fn alloc_slice_with_attrs( dev: &device::Device, - count: usize, + len: usize, gfp_flags: kernel::alloc::Flags, - ) -> Result> { - CoherentAllocation::alloc_attrs(dev, count, gfp_flags, Attrs(0)) + dma_attrs: Attrs, + ) -> Result> { + const { + assert!( + core::mem::size_of::() > 0, + "It doesn't make sense for the allocated type to be a ZST" + ); + } + + // `dma_alloc_attrs` cannot handle zero-length allocation, bail early. + if len == 0 { + Err(EINVAL)?; + } + + let size = core::mem::size_of::().checked_mul(len).ok_or(ENOMEM)?; + let mut dma_handle = 0; + // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. + let addr = unsafe { + bindings::dma_alloc_attrs( + dev.as_raw(), + size, + &mut dma_handle, + gfp_flags.as_raw(), + dma_attrs.as_raw(), + ) + }; + let cpu_addr = NonNull::slice_from_raw_parts(NonNull::new(addr.cast()).ok_or(ENOMEM)?, len); + // INVARIANT: + // - We just successfully allocated a coherent region which is adequately sized for + // `[T; len]`, hence the cpu address is valid. + // - We also hold a refcounted reference to the device. + Ok(Coherent { + dev: dev.into(), + dma_handle, + cpu_addr, + dma_attrs, + }) } +} +impl Coherent<[T]> { /// Returns the number of elements `T` in this allocation. /// /// Note that this is not the size of the allocation in bytes, which is provided by /// [`Self::size`]. - pub fn count(&self) -> usize { - self.count + #[inline] + #[expect(clippy::len_without_is_empty, reason = "Coherent slice is never empty")] + pub fn len(&self) -> usize { + self.cpu_addr.len() } +} - /// Returns the size in bytes of this allocation. - pub fn size(&self) -> usize { - // INVARIANT: The type invariant of `Self` guarantees that `size_of:: * count` fits into - // a `usize`. - self.count * core::mem::size_of::() - } +// Type alias for compatibility. +#[doc(hidden)] +pub type CoherentAllocation = Coherent<[T]>; - /// Returns the raw pointer to the allocated region in the CPU's virtual address space. - #[inline] - pub fn as_ptr(&self) -> *const [T] { - core::ptr::slice_from_raw_parts(self.cpu_addr.as_ptr(), self.count) +impl CoherentAllocation { + /// Allocates a region of `size_of:: * count` of coherent memory. + /// + /// # Examples + /// + /// ``` + /// # use kernel::device::{Bound, Device}; + /// use kernel::dma::{attrs::*, CoherentAllocation}; + /// + /// # fn test(dev: &Device) -> Result { + /// let c: CoherentAllocation = + /// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// # Ok::<(), Error>(()) } + /// ``` + pub fn alloc_attrs( + dev: &device::Device, + count: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result> { + Coherent::alloc_slice_with_attrs(dev, count, gfp_flags, dma_attrs) } - /// Returns the raw pointer to the allocated region in the CPU's virtual address space as - /// a mutable pointer. - #[inline] - pub fn as_mut_ptr(&self) -> *mut [T] { - core::ptr::slice_from_raw_parts_mut(self.cpu_addr.as_ptr(), self.count) + /// Performs the same functionality as [`CoherentAllocation::alloc_attrs`], except the + /// `dma_attrs` is 0 by default. + pub fn alloc_coherent( + dev: &device::Device, + count: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result> { + CoherentAllocation::alloc_attrs(dev, count, gfp_flags, Attrs(0)) } /// Returns the base address to the allocated region in the CPU's virtual address space. pub fn start_ptr(&self) -> *const T { - self.cpu_addr.as_ptr() + self.as_ptr().cast() } /// Returns the base address to the allocated region in the CPU's virtual address space as /// a mutable pointer. pub fn start_ptr_mut(&mut self) -> *mut T { - self.cpu_addr.as_ptr() - } - - /// Returns a DMA handle which may be given to the device as the DMA address base of - /// the region. - pub fn dma_handle(&self) -> DmaAddress { - self.dma_handle + self.as_mut_ptr().cast() } /// Returns a DMA handle starting at `offset` (in units of `T`) which may be given to the @@ -504,11 +639,9 @@ impl CoherentAllocation { /// /// Returns `EINVAL` if `offset` is not within the bounds of the allocation. pub fn dma_handle_with_offset(&self, offset: usize) -> Result { - if offset >= self.count { + if offset >= self.len() { Err(EINVAL) } else { - // INVARIANT: The type invariant of `Self` guarantees that `size_of:: * count` fits - // into a `usize`, and `offset` is inferior to `count`. Ok(self.dma_handle + (offset * core::mem::size_of::()) as DmaAddress) } } @@ -516,7 +649,7 @@ impl CoherentAllocation { /// Common helper to validate a range applied from the allocated region in the CPU's virtual /// address space. fn validate_range(&self, offset: usize, count: usize) -> Result { - if offset.checked_add(count).ok_or(EOVERFLOW)? > self.count { + if offset.checked_add(count).ok_or(EOVERFLOW)? > self.len() { return Err(EINVAL); } Ok(()) @@ -601,66 +734,20 @@ impl CoherentAllocation { }; Ok(()) } - - /// Reads the value of `field` and ensures that its type is [`FromBytes`]. - /// - /// # Safety - /// - /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is - /// validated beforehand. - /// - /// Public but hidden since it should only be used from [`dma_read`] macro. - #[doc(hidden)] - pub unsafe fn field_read(&self, field: *const F) -> F { - // SAFETY: - // - By the safety requirements field is valid. - // - Using read_volatile() here is not sound as per the usual rules, the usage here is - // a special exception with the following notes in place. When dealing with a potential - // race from a hardware or code outside kernel (e.g. user-space program), we need that - // read on a valid memory is not UB. Currently read_volatile() is used for this, and the - // rationale behind is that it should generate the same code as READ_ONCE() which the - // kernel already relies on to avoid UB on data races. Note that the usage of - // read_volatile() is limited to this particular case, it cannot be used to prevent - // the UB caused by racing between two kernel functions nor do they provide atomicity. - unsafe { field.read_volatile() } - } - - /// Writes a value to `field` and ensures that its type is [`AsBytes`]. - /// - /// # Safety - /// - /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is - /// validated beforehand. - /// - /// Public but hidden since it should only be used from [`dma_write`] macro. - #[doc(hidden)] - pub unsafe fn field_write(&self, field: *mut F, val: F) { - // SAFETY: - // - By the safety requirements field is valid. - // - Using write_volatile() here is not sound as per the usual rules, the usage here is - // a special exception with the following notes in place. When dealing with a potential - // race from a hardware or code outside kernel (e.g. user-space program), we need that - // write on a valid memory is not UB. Currently write_volatile() is used for this, and the - // rationale behind is that it should generate the same code as WRITE_ONCE() which the - // kernel already relies on to avoid UB on data races. Note that the usage of - // write_volatile() is limited to this particular case, it cannot be used to prevent - // the UB caused by racing between two kernel functions nor do they provide atomicity. - unsafe { field.write_volatile(val) } - } } /// Note that the device configured to do DMA must be halted before this object is dropped. -impl Drop for CoherentAllocation { +impl Drop for Coherent { fn drop(&mut self) { - let size = self.count * core::mem::size_of::(); + let size = T::size(self.cpu_addr.as_ptr()); // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. // The cpu address, and the dma handle are valid due to the type invariants on - // `CoherentAllocation`. + // `Coherent`. unsafe { bindings::dma_free_attrs( self.dev.as_raw(), size, - self.start_ptr_mut().cast(), + self.cpu_addr.as_ptr().cast(), self.dma_handle, self.dma_attrs.as_raw(), ) @@ -668,20 +755,20 @@ impl Drop for CoherentAllocation { } } -// SAFETY: It is safe to send a `CoherentAllocation` to another thread if `T` +// SAFETY: It is safe to send a `Coherent` to another thread if `T` // can be sent to another thread. -unsafe impl Send for CoherentAllocation {} +unsafe impl Send for Coherent {} /// Reads a field of an item from an allocated region of structs. /// /// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating -/// to a [`CoherentAllocation`] and `proj` is a [projection specification](kernel::ptr::project!). +/// to a [`Coherent`] and `proj` is a [projection specification](kernel::ptr::project!). /// /// # Examples /// /// ``` /// use kernel::device::Device; -/// use kernel::dma::{attrs::*, CoherentAllocation}; +/// use kernel::dma::{attrs::*, Coherent}; /// /// struct MyStruct { field: u32, } /// @@ -690,7 +777,7 @@ unsafe impl Send for CoherentAllocation {} /// // SAFETY: Instances of `MyStruct` have no uninitialized portions. /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; /// -/// # fn test(alloc: &kernel::dma::CoherentAllocation) -> Result { +/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result { /// let whole = kernel::dma_read!(alloc, [2]?); /// let field = kernel::dma_read!(alloc, [1]?.field); /// # Ok::<(), Error>(()) } @@ -700,17 +787,17 @@ macro_rules! dma_read { ($dma:expr, $($proj:tt)*) => {{ let dma = &$dma; let ptr = $crate::ptr::project!( - $crate::dma::CoherentAllocation::as_ptr(dma), $($proj)* + $crate::dma::Coherent::as_ptr(dma), $($proj)* ); // SAFETY: The pointer created by the projection is within the DMA region. - unsafe { $crate::dma::CoherentAllocation::field_read(dma, ptr) } + unsafe { $crate::dma::Coherent::field_read(dma, ptr) } }}; } /// Writes to a field of an item from an allocated region of structs. /// /// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression -/// evaluating to a [`CoherentAllocation`], `proj` is a +/// evaluating to a [`Coherent`], `proj` is a /// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the /// projected location. /// @@ -718,7 +805,7 @@ macro_rules! dma_read { /// /// ``` /// use kernel::device::Device; -/// use kernel::dma::{attrs::*, CoherentAllocation}; +/// use kernel::dma::{attrs::*, Coherent}; /// /// struct MyStruct { member: u32, } /// @@ -727,7 +814,7 @@ macro_rules! dma_read { /// // SAFETY: Instances of `MyStruct` have no uninitialized portions. /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; /// -/// # fn test(alloc: &kernel::dma::CoherentAllocation) -> Result { +/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result { /// kernel::dma_write!(alloc, [2]?.member, 0xf); /// kernel::dma_write!(alloc, [1]?, MyStruct { member: 0xf }); /// # Ok::<(), Error>(()) } @@ -737,11 +824,11 @@ macro_rules! dma_write { (@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{ let dma = &$dma; let ptr = $crate::ptr::project!( - mut $crate::dma::CoherentAllocation::as_mut_ptr(dma), $($proj)* + mut $crate::dma::Coherent::as_mut_ptr(dma), $($proj)* ); let val = $val; // SAFETY: The pointer created by the projection is within the DMA region. - unsafe { $crate::dma::CoherentAllocation::field_write(dma, ptr, val) } + unsafe { $crate::dma::Coherent::field_write(dma, ptr, val) } }}; (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*]) -- cgit v1.2.3 From f84ecffa3f745572164c1269f20eec2589d432c9 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Fri, 20 Mar 2026 20:45:38 +0100 Subject: rust: dma: add zeroed constructor to `Coherent` These constructors create a coherent container of a single object instead of slice. They are named `zeroed` and `zeroed_with_attrs` to emphasis that they are created initialized zeroed. It is intended that there'll be new constructors that take `PinInit` instead of zeroing. Signed-off-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260320194626.36263-4-dakr@kernel.org [ Use kernel import style. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/dma.rs | 93 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 85 insertions(+), 8 deletions(-) (limited to 'rust') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index ff3e147f1a23..59477c865dd3 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -47,7 +47,7 @@ pub trait Device: AsRef> { /// # Safety /// /// This method must not be called concurrently with any DMA allocation or mapping primitives, - /// such as [`CoherentAllocation::alloc_attrs`]. + /// such as [`Coherent::zeroed`]. unsafe fn dma_set_mask(&self, mask: DmaMask) -> Result { // SAFETY: // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. @@ -64,7 +64,7 @@ pub trait Device: AsRef> { /// # Safety /// /// This method must not be called concurrently with any DMA allocation or mapping primitives, - /// such as [`CoherentAllocation::alloc_attrs`]. + /// such as [`Coherent::zeroed`]. unsafe fn dma_set_coherent_mask(&self, mask: DmaMask) -> Result { // SAFETY: // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. @@ -83,7 +83,7 @@ pub trait Device: AsRef> { /// # Safety /// /// This method must not be called concurrently with any DMA allocation or mapping primitives, - /// such as [`CoherentAllocation::alloc_attrs`]. + /// such as [`Coherent::zeroed`]. unsafe fn dma_set_mask_and_coherent(&self, mask: DmaMask) -> Result { // SAFETY: // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. @@ -102,7 +102,7 @@ pub trait Device: AsRef> { /// # Safety /// /// This method must not be called concurrently with any DMA allocation or mapping primitives, - /// such as [`CoherentAllocation::alloc_attrs`]. + /// such as [`Coherent::zeroed`]. unsafe fn dma_set_max_seg_size(&self, size: u32) { // SAFETY: // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. @@ -202,12 +202,12 @@ impl DmaMask { /// /// ``` /// # use kernel::device::{Bound, Device}; -/// use kernel::dma::{attrs::*, CoherentAllocation}; +/// use kernel::dma::{attrs::*, Coherent}; /// /// # fn test(dev: &Device) -> Result { /// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN; -/// let c: CoherentAllocation = -/// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, attribs)?; +/// let c: Coherent<[u64]> = +/// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, attribs)?; /// # Ok::<(), Error>(()) } /// ``` #[derive(Clone, Copy, PartialEq)] @@ -492,7 +492,6 @@ impl Coherent { impl Coherent { /// Allocates a region of `T` of coherent memory. - #[expect(unused)] fn alloc_with_attrs( dev: &device::Device, gfp_flags: kernel::alloc::Flags, @@ -529,6 +528,41 @@ impl Coherent { }) } + /// Allocates a region of type `T` of coherent memory. + /// + /// # Examples + /// + /// ``` + /// # use kernel::device::{ + /// # Bound, + /// # Device, + /// # }; + /// use kernel::dma::{ + /// attrs::*, + /// Coherent, + /// }; + /// + /// # fn test(dev: &Device) -> Result { + /// let c: Coherent<[u64; 4]> = + /// Coherent::zeroed_with_attrs(dev, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + pub fn zeroed_with_attrs( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + Self::alloc_with_attrs(dev, gfp_flags | __GFP_ZERO, dma_attrs) + } + + /// Performs the same functionality as [`Coherent::zeroed_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn zeroed(dev: &device::Device, gfp_flags: kernel::alloc::Flags) -> Result { + Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) + } + /// Allocates a region of `[T; len]` of coherent memory. fn alloc_slice_with_attrs( dev: &device::Device, @@ -572,6 +606,49 @@ impl Coherent { dma_attrs, }) } + + /// Allocates a zeroed region of type `T` of coherent memory. + /// + /// Unlike `Coherent::<[T; N]>::zeroed_with_attrs`, `Coherent::::zeroed_slices` support + /// a runtime length. + /// + /// # Examples + /// + /// ``` + /// # use kernel::device::{ + /// # Bound, + /// # Device, + /// # }; + /// use kernel::dma::{ + /// attrs::*, + /// Coherent, + /// }; + /// + /// # fn test(dev: &Device) -> Result { + /// let c: Coherent<[u64]> = + /// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + pub fn zeroed_slice_with_attrs( + dev: &device::Device, + len: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result> { + Coherent::alloc_slice_with_attrs(dev, len, gfp_flags | __GFP_ZERO, dma_attrs) + } + + /// Performs the same functionality as [`Coherent::zeroed_slice_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn zeroed_slice( + dev: &device::Device, + len: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result> { + Self::zeroed_slice_with_attrs(dev, len, gfp_flags, Attrs(0)) + } } impl Coherent<[T]> { -- cgit v1.2.3 From 80f4a7b5138d1c427ee5626e8a796aa6b2994a95 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 20 Mar 2026 20:45:39 +0100 Subject: rust: dma: introduce dma::CoherentBox for memory initialization Currently, dma::Coherent cannot safely provide (mutable) access to its underlying memory because the memory might be concurrently accessed by a DMA device. This makes it difficult to safely initialize the memory before handing it over to the hardware. Introduce dma::CoherentBox, a type that encapsulates a dma::Coherent before its DMA address is exposed to the device. dma::CoherentBox can guarantee exclusive access to the inner dma::Coherent and implement Deref and DerefMut. Once the memory is properly initialized, dma::CoherentBox can be converted into a regular dma::Coherent. Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260320194626.36263-5-dakr@kernel.org [ Remove unnecessary trait bounds. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/dma.rs | 154 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 153 insertions(+), 1 deletion(-) (limited to 'rust') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 59477c865dd3..5b41603cee2e 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -20,7 +20,13 @@ use crate::{ FromBytes, // }, // }; -use core::ptr::NonNull; +use core::{ + ops::{ + Deref, + DerefMut, // + }, + ptr::NonNull, // +}; /// DMA address type. /// @@ -352,6 +358,152 @@ impl From for bindings::dma_data_direction { } } +/// CPU-owned DMA allocation that can be converted into a device-shared [`Coherent`] object. +/// +/// Unlike [`Coherent`], a [`CoherentBox`] is guaranteed to be fully owned by the CPU -- its DMA +/// address is not exposed and it cannot be accessed by a device. This means it can safely be used +/// like a normal boxed allocation (e.g. direct reads, writes, and mutable slices are all safe). +/// +/// A typical use is to allocate a [`CoherentBox`], populate it with normal CPU access, and then +/// convert it into a [`Coherent`] object to share it with the device. +/// +/// # Examples +/// +/// `CoherentBox`: +/// +/// ``` +/// # use kernel::device::{ +/// # Bound, +/// # Device, +/// # }; +/// use kernel::dma::{attrs::*, +/// Coherent, +/// CoherentBox, +/// }; +/// +/// # fn test(dev: &Device) -> Result { +/// let mut dmem: CoherentBox = CoherentBox::zeroed(dev, GFP_KERNEL)?; +/// *dmem = 42; +/// let dmem: Coherent = dmem.into(); +/// # Ok::<(), Error>(()) } +/// ``` +/// +/// `CoherentBox<[T]>`: +/// +/// +/// ``` +/// # use kernel::device::{ +/// # Bound, +/// # Device, +/// # }; +/// use kernel::dma::{attrs::*, +/// Coherent, +/// CoherentBox, +/// }; +/// +/// # fn test(dev: &Device) -> Result { +/// let mut dmem: CoherentBox<[u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?; +/// dmem.fill(42); +/// let dmem: Coherent<[u64]> = dmem.into(); +/// # Ok::<(), Error>(()) } +/// ``` +pub struct CoherentBox(Coherent); + +impl CoherentBox<[T]> { + /// [`CoherentBox`] variant of [`Coherent::zeroed_slice_with_attrs`]. + #[inline] + pub fn zeroed_slice_with_attrs( + dev: &device::Device, + count: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + Coherent::zeroed_slice_with_attrs(dev, count, gfp_flags, dma_attrs).map(Self) + } + + /// Same as [CoherentBox::zeroed_slice_with_attrs], but with `dma::Attrs(0)`. + #[inline] + pub fn zeroed_slice( + dev: &device::Device, + count: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result { + Self::zeroed_slice_with_attrs(dev, count, gfp_flags, Attrs(0)) + } + + /// Initializes the element at `i` using the given initializer. + /// + /// Returns `EINVAL` if `i` is out of bounds. + pub fn init_at(&mut self, i: usize, init: impl Init) -> Result + where + Error: From, + { + if i >= self.0.len() { + return Err(EINVAL); + } + + let ptr = &raw mut self[i]; + + // SAFETY: + // - `ptr` is valid, properly aligned, and within this allocation. + // - `T: AsBytes + FromBytes` guarantees all bit patterns are valid, so partial writes on + // error cannot leave the element in an invalid state. + // - The DMA address has not been exposed yet, so there is no concurrent device access. + unsafe { init.__init(ptr)? }; + + Ok(()) + } +} + +impl CoherentBox { + /// Same as [`CoherentBox::zeroed_slice_with_attrs`], but for a single element. + #[inline] + pub fn zeroed_with_attrs( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + Coherent::zeroed_with_attrs(dev, gfp_flags, dma_attrs).map(Self) + } + + /// Same as [`CoherentBox::zeroed_slice`], but for a single element. + #[inline] + pub fn zeroed(dev: &device::Device, gfp_flags: kernel::alloc::Flags) -> Result { + Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) + } +} + +impl Deref for CoherentBox { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: + // - We have not exposed the DMA address yet, so there can't be any concurrent access by a + // device. + // - We have exclusive access to `self.0`. + unsafe { self.0.as_ref() } + } +} + +impl DerefMut for CoherentBox { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: + // - We have not exposed the DMA address yet, so there can't be any concurrent access by a + // device. + // - We have exclusive access to `self.0`. + unsafe { self.0.as_mut() } + } +} + +impl From> for Coherent { + #[inline] + fn from(value: CoherentBox) -> Self { + value.0 + } +} + /// An abstraction of the `dma_alloc_coherent` API. /// /// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map -- cgit v1.2.3 From 40251bac0e00818b0b632d2d154e4f1f815849eb Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 20 Mar 2026 20:45:40 +0100 Subject: rust: dma: add Coherent:init() and Coherent::init_with_attrs() Analogous to Coherent::zeroed() and Coherent::zeroed_with_attrs(), add Coherent:init() and Coherent::init_with_attrs() which both take an impl Init argument initializing the DMA coherent memory. Compared to CoherentInit, Coherent::init() is a one-shot constructor that runs an Init closure and immediately exposes the DMA handle, whereas CoherentInit is a multi-stage initializer that provides safe &mut T access by withholding the DMA address until converted to Coherent. Reviewed-by: Gary Guo Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260320194626.36263-6-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/dma.rs | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) (limited to 'rust') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 5b41603cee2e..be44ec99af5f 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -715,6 +715,44 @@ impl Coherent { Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) } + /// Same as [`Coherent::zeroed_with_attrs`], but instead of a zero-initialization the memory is + /// initialized with `init`. + pub fn init_with_attrs( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + init: impl Init, + ) -> Result + where + Error: From, + { + let dmem = Self::alloc_with_attrs(dev, gfp_flags, dma_attrs)?; + let ptr = dmem.as_mut_ptr(); + + // SAFETY: + // - `ptr` is valid, properly aligned, and points to exclusively owned memory. + // - If `__init` fails, `self` is dropped, which safely frees the underlying `Coherent`'s + // DMA memory. `T: AsBytes + FromBytes` ensures there are no complex `Drop` requirements + // we are bypassing. + unsafe { init.__init(ptr)? }; + + Ok(dmem) + } + + /// Same as [`Coherent::zeroed`], but instead of a zero-initialization the memory is initialized + /// with `init`. + #[inline] + pub fn init( + dev: &device::Device, + gfp_flags: kernel::alloc::Flags, + init: impl Init, + ) -> Result + where + Error: From, + { + Self::init_with_attrs(dev, gfp_flags, Attrs(0), init) + } + /// Allocates a region of `[T; len]` of coherent memory. fn alloc_slice_with_attrs( dev: &device::Device, -- cgit v1.2.3 From 55fd681cdd8599edc82013674cbf87c2f22d58f8 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sat, 21 Mar 2026 19:18:08 +0100 Subject: rust: dma: remove dma::CoherentAllocation Now that everything has been converted to the new dma::Coherent API, remove dma::CoherentAllocation. Suggested-by: Gary Guo Reviewed-by: Gary Guo Link: https://patch.msgid.link/DH8O47F2GM1Z.3H3E13RSKIV22@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/dma.rs | 150 ----------------------------------------------------- 1 file changed, 150 deletions(-) (limited to 'rust') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index be44ec99af5f..bf823818a67d 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -853,156 +853,6 @@ impl Coherent<[T]> { } } -// Type alias for compatibility. -#[doc(hidden)] -pub type CoherentAllocation = Coherent<[T]>; - -impl CoherentAllocation { - /// Allocates a region of `size_of:: * count` of coherent memory. - /// - /// # Examples - /// - /// ``` - /// # use kernel::device::{Bound, Device}; - /// use kernel::dma::{attrs::*, CoherentAllocation}; - /// - /// # fn test(dev: &Device) -> Result { - /// let c: CoherentAllocation = - /// CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; - /// # Ok::<(), Error>(()) } - /// ``` - pub fn alloc_attrs( - dev: &device::Device, - count: usize, - gfp_flags: kernel::alloc::Flags, - dma_attrs: Attrs, - ) -> Result> { - Coherent::alloc_slice_with_attrs(dev, count, gfp_flags, dma_attrs) - } - - /// Performs the same functionality as [`CoherentAllocation::alloc_attrs`], except the - /// `dma_attrs` is 0 by default. - pub fn alloc_coherent( - dev: &device::Device, - count: usize, - gfp_flags: kernel::alloc::Flags, - ) -> Result> { - CoherentAllocation::alloc_attrs(dev, count, gfp_flags, Attrs(0)) - } - - /// Returns the base address to the allocated region in the CPU's virtual address space. - pub fn start_ptr(&self) -> *const T { - self.as_ptr().cast() - } - - /// Returns the base address to the allocated region in the CPU's virtual address space as - /// a mutable pointer. - pub fn start_ptr_mut(&mut self) -> *mut T { - self.as_mut_ptr().cast() - } - - /// Returns a DMA handle starting at `offset` (in units of `T`) which may be given to the - /// device as the DMA address base of the region. - /// - /// Returns `EINVAL` if `offset` is not within the bounds of the allocation. - pub fn dma_handle_with_offset(&self, offset: usize) -> Result { - if offset >= self.len() { - Err(EINVAL) - } else { - Ok(self.dma_handle + (offset * core::mem::size_of::()) as DmaAddress) - } - } - - /// Common helper to validate a range applied from the allocated region in the CPU's virtual - /// address space. - fn validate_range(&self, offset: usize, count: usize) -> Result { - if offset.checked_add(count).ok_or(EOVERFLOW)? > self.len() { - return Err(EINVAL); - } - Ok(()) - } - - /// Returns the data from the region starting from `offset` as a slice. - /// `offset` and `count` are in units of `T`, not the number of bytes. - /// - /// For ringbuffer type of r/w access or use-cases where the pointer to the live data is needed, - /// [`CoherentAllocation::start_ptr`] or [`CoherentAllocation::start_ptr_mut`] could be used - /// instead. - /// - /// # Safety - /// - /// * Callers must ensure that the device does not read/write to/from memory while the returned - /// slice is live. - /// * Callers must ensure that this call does not race with a write to the same region while - /// the returned slice is live. - pub unsafe fn as_slice(&self, offset: usize, count: usize) -> Result<&[T]> { - self.validate_range(offset, count)?; - // SAFETY: - // - The pointer is valid due to type invariant on `CoherentAllocation`, - // we've just checked that the range and index is within bounds. The immutability of the - // data is also guaranteed by the safety requirements of the function. - // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked - // that `self.count` won't overflow early in the constructor. - Ok(unsafe { core::slice::from_raw_parts(self.start_ptr().add(offset), count) }) - } - - /// Performs the same functionality as [`CoherentAllocation::as_slice`], except that a mutable - /// slice is returned. - /// - /// # Safety - /// - /// * Callers must ensure that the device does not read/write to/from memory while the returned - /// slice is live. - /// * Callers must ensure that this call does not race with a read or write to the same region - /// while the returned slice is live. - pub unsafe fn as_slice_mut(&mut self, offset: usize, count: usize) -> Result<&mut [T]> { - self.validate_range(offset, count)?; - // SAFETY: - // - The pointer is valid due to type invariant on `CoherentAllocation`, - // we've just checked that the range and index is within bounds. The immutability of the - // data is also guaranteed by the safety requirements of the function. - // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked - // that `self.count` won't overflow early in the constructor. - Ok(unsafe { core::slice::from_raw_parts_mut(self.start_ptr_mut().add(offset), count) }) - } - - /// Writes data to the region starting from `offset`. `offset` is in units of `T`, not the - /// number of bytes. - /// - /// # Safety - /// - /// * Callers must ensure that this call does not race with a read or write to the same region - /// that overlaps with this write. - /// - /// # Examples - /// - /// ``` - /// # fn test(alloc: &mut kernel::dma::CoherentAllocation) -> Result { - /// let somedata: [u8; 4] = [0xf; 4]; - /// let buf: &[u8] = &somedata; - /// // SAFETY: There is no concurrent HW operation on the device and no other R/W access to the - /// // region. - /// unsafe { alloc.write(buf, 0)?; } - /// # Ok::<(), Error>(()) } - /// ``` - pub unsafe fn write(&mut self, src: &[T], offset: usize) -> Result { - self.validate_range(offset, src.len())?; - // SAFETY: - // - The pointer is valid due to type invariant on `CoherentAllocation` - // and we've just checked that the range and index is within bounds. - // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked - // that `self.count` won't overflow early in the constructor. - unsafe { - core::ptr::copy_nonoverlapping( - src.as_ptr(), - self.start_ptr_mut().add(offset), - src.len(), - ) - }; - Ok(()) - } -} - /// Note that the device configured to do DMA must be halted before this object is dropped. impl Drop for Coherent { fn drop(&mut self) { -- cgit v1.2.3 From d35ae50c5f48dfcd33cb24bf477ce912fa0af1f7 Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Thu, 19 Mar 2026 16:26:53 -0500 Subject: rust: device: add device name method Add a name() method to the `Device` type, which returns a CStr that contains the device name. Signed-off-by: Timur Tabi Reviewed-by: Alice Ryhl Reviewed-by: Alexandre Courbot Reviewed-by: Gary Guo Tested-by: John Hubbard Tested-by: Eliot Courtney Link: https://patch.msgid.link/20260319212658.2541610-2-ttabi@nvidia.com Signed-off-by: Danilo Krummrich --- rust/helpers/device.c | 5 +++++ rust/kernel/device.rs | 11 +++++++++++ 2 files changed, 16 insertions(+) (limited to 'rust') diff --git a/rust/helpers/device.c b/rust/helpers/device.c index a8ab931a9bd1..3be4ee590784 100644 --- a/rust/helpers/device.c +++ b/rust/helpers/device.c @@ -25,3 +25,8 @@ __rust_helper void rust_helper_dev_set_drvdata(struct device *dev, void *data) { dev_set_drvdata(dev, data); } + +__rust_helper const char *rust_helper_dev_name(const struct device *dev) +{ + return dev_name(dev); +} diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 379058eca2ed..6d5396a43ebe 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -489,6 +489,17 @@ impl Device { // defined as a `#[repr(transparent)]` wrapper around `fwnode_handle`. Some(unsafe { &*fwnode_handle.cast() }) } + + /// Returns the name of the device. + /// + /// This is the kobject name of the device, or its initial name if the kobject is not yet + /// available. + #[inline] + pub fn name(&self) -> &CStr { + // SAFETY: By its type invariant `self.as_raw()` is a valid pointer to a `struct device`. + // The returned string is valid for the lifetime of the device. + unsafe { CStr::from_char_ptr(bindings::dev_name(self.as_raw())) } + } } // SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic -- cgit v1.2.3 From 69bfce0f25c8f949f7a02c3ed93e36aadad3fa57 Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Thu, 19 Mar 2026 16:26:54 -0500 Subject: rust: uaccess: add write_dma() for copying from DMA buffers to userspace Add UserSliceWriter::write_dma() to copy data from a Coherent<[u8]> to userspace. This provides a safe interface for copying DMA buffer contents to userspace without requiring callers to work with raw pointers. Because write_dma() and write_slice() have common code, factor that code out into a helper function, write_raw(). The method handles bounds checking and offset calculation internally, wrapping the unsafe copy_to_user() call. Signed-off-by: Timur Tabi Reviewed-by: Alexandre Courbot Acked-by: Miguel Ojeda Tested-by: John Hubbard Tested-by: Eliot Courtney Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20260319212658.2541610-3-ttabi@nvidia.com [ Rebase onto Coherent changes; remove unnecessary turbofish from cast(). - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/uaccess.rs | 85 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 75 insertions(+), 10 deletions(-) (limited to 'rust') diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs index f989539a31b4..e26ef90ba8ad 100644 --- a/rust/kernel/uaccess.rs +++ b/rust/kernel/uaccess.rs @@ -7,6 +7,7 @@ use crate::{ alloc::{Allocator, Flags}, bindings, + dma::Coherent, error::Result, ffi::{c_char, c_void}, fs::file, @@ -459,20 +460,19 @@ impl UserSliceWriter { self.length == 0 } - /// Writes raw data to this user pointer from a kernel buffer. + /// Low-level write from a raw pointer. /// - /// Fails with [`EFAULT`] if the write happens on a bad address, or if the write goes out of - /// bounds of this [`UserSliceWriter`]. This call may modify the associated userspace slice even - /// if it returns an error. - pub fn write_slice(&mut self, data: &[u8]) -> Result { - let len = data.len(); - let data_ptr = data.as_ptr().cast::(); + /// # Safety + /// + /// The caller must ensure that `from` is valid for reads of `len` bytes. + unsafe fn write_raw(&mut self, from: *const u8, len: usize) -> Result { if len > self.length { return Err(EFAULT); } - // SAFETY: `data_ptr` points into an immutable slice of length `len`, so we may read - // that many bytes from it. - let res = unsafe { bindings::copy_to_user(self.ptr.as_mut_ptr(), data_ptr, len) }; + + // SAFETY: Caller guarantees `from` is valid for `len` bytes (see this function's + // safety contract). + let res = unsafe { bindings::copy_to_user(self.ptr.as_mut_ptr(), from.cast(), len) }; if res != 0 { return Err(EFAULT); } @@ -481,6 +481,71 @@ impl UserSliceWriter { Ok(()) } + /// Writes raw data to this user pointer from a kernel buffer. + /// + /// Fails with [`EFAULT`] if the write happens on a bad address, or if the write goes out of + /// bounds of this [`UserSliceWriter`]. This call may modify the associated userspace slice even + /// if it returns an error. + pub fn write_slice(&mut self, data: &[u8]) -> Result { + // SAFETY: `data` is a valid slice, so `data.as_ptr()` is valid for + // reading `data.len()` bytes. + unsafe { self.write_raw(data.as_ptr(), data.len()) } + } + + /// Writes raw data to this user pointer from a DMA coherent allocation. + /// + /// Copies `count` bytes from `alloc` starting from `offset` into this userspace slice. + /// + /// # Errors + /// + /// - [`EOVERFLOW`]: `offset + count` overflows. + /// - [`ERANGE`]: `offset + count` exceeds the size of `alloc`, or `count` exceeds the + /// size of the user-space buffer. + /// - [`EFAULT`]: the write hits a bad address or goes out of bounds of this + /// [`UserSliceWriter`]. + /// + /// This call may modify the associated userspace slice even if it returns an error. + /// + /// Note: The memory may be concurrently modified by hardware (e.g., DMA). In such cases, + /// the copied data may be inconsistent, but this does not cause undefined behavior. + /// + /// # Example + /// + /// Copy the first 256 bytes of a DMA coherent allocation into a userspace buffer: + /// + /// ```no_run + /// use kernel::uaccess::UserSliceWriter; + /// use kernel::dma::Coherent; + /// + /// fn copy_dma_to_user( + /// mut writer: UserSliceWriter, + /// alloc: &Coherent<[u8]>, + /// ) -> Result { + /// writer.write_dma(alloc, 0, 256) + /// } + /// ``` + pub fn write_dma(&mut self, alloc: &Coherent<[u8]>, offset: usize, count: usize) -> Result { + let len = alloc.size(); + if offset.checked_add(count).ok_or(EOVERFLOW)? > len { + return Err(ERANGE); + } + + if count > self.len() { + return Err(ERANGE); + } + + // SAFETY: `as_ptr()` returns a valid pointer to a memory region of `count()` bytes, as + // guaranteed by the `Coherent` invariants. The check above ensures `offset + count <= len`. + let src_ptr = unsafe { alloc.as_ptr().cast::().add(offset) }; + + // Note: Use `write_raw` instead of `write_slice` because the allocation is coherent + // memory that hardware may modify (e.g., DMA); we cannot form a `&[u8]` slice over + // such volatile memory. + // + // SAFETY: `src_ptr` points into the allocation and is valid for `count` bytes (see above). + unsafe { self.write_raw(src_ptr, count) } + } + /// Writes raw data to this user pointer from a kernel buffer partially. /// /// This is the same as [`Self::write_slice`] but considers the given `offset` into `data` and -- cgit v1.2.3 From 01681851393642e10b5cc3f35eb6a1916ee4aff1 Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Thu, 19 Mar 2026 16:26:55 -0500 Subject: rust: dma: implement BinaryWriter for Coherent<[u8]> Implement the BinaryWriter trait for Coherent<[u8]>, enabling DMA coherent allocations to be exposed as readable binary files. The implementation handles offset tracking and bounds checking, copying data from the coherent allocation to userspace via write_dma(). Signed-off-by: Timur Tabi Reviewed-by: Alice Ryhl Reviewed-by: Alexandre Courbot Tested-by: John Hubbard Tested-by: Eliot Courtney Link: https://patch.msgid.link/20260319212658.2541610-4-ttabi@nvidia.com [ Rebase onto Coherent changes. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/dma.rs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'rust') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index bf823818a67d..3eef7c2396bb 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -6,12 +6,14 @@ use crate::{ bindings, + debugfs, device::{ self, Bound, Core, // }, error::to_result, + fs::file, prelude::*, ptr::KnownSize, sync::aref::ARef, @@ -19,6 +21,7 @@ use crate::{ AsBytes, FromBytes, // }, // + uaccess::UserSliceWriter, }; use core::{ ops::{ @@ -876,6 +879,37 @@ impl Drop for Coherent { // can be sent to another thread. unsafe impl Send for Coherent {} +// SAFETY: Sharing `&Coherent` across threads is safe if `T` is `Sync`, because all +// methods that access the buffer contents (`field_read`, `field_write`, `as_slice`, +// `as_slice_mut`) are `unsafe`, and callers are responsible for ensuring no data races occur. +// The safe methods only return metadata or raw pointers whose use requires `unsafe`. +unsafe impl Sync for Coherent {} + +impl debugfs::BinaryWriter for Coherent<[u8]> { + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result { + if offset.is_negative() { + return Err(EINVAL); + } + + // If the offset is too large for a usize (e.g. on 32-bit platforms), + // then consider that as past EOF and just return 0 bytes. + let Ok(offset_val) = usize::try_from(*offset) else { + return Ok(0); + }; + + let count = self.size().saturating_sub(offset_val).min(writer.len()); + + writer.write_dma(self, offset_val, count)?; + + *offset += count as i64; + Ok(count) + } +} + /// Reads a field of an item from an allocated region of structs. /// /// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating -- cgit v1.2.3 From e64b9cc293ae710c815c2de1ec9dcaa0784a8017 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Mon, 16 Mar 2026 17:16:09 -0400 Subject: rust: drm: Add gem::impl_aref_for_gem_obj! In the future we're going to be introducing more GEM object types in rust then just gem::Object. Since all types of GEM objects have refcounting, let's introduce a macro that we can use in the gem crate in order to copy this boilerplate implementation for each type: impl_aref_for_gem_obj!(). Signed-off-by: Lyude Paul Reviewed-by: Daniel Almeida Reviewed-by: Janne Grunau Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260316211646.650074-2-lyude@redhat.com [ Resolve merge conflicts. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/drm/gem/mod.rs | 51 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 15 deletions(-) (limited to 'rust') diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index b4199945db37..48ffdd7400ca 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -26,6 +26,41 @@ use core::{ ptr::NonNull, // }; +/// A macro for implementing [`AlwaysRefCounted`] for any GEM object type. +/// +/// Since all GEM objects use the same refcounting scheme. +#[macro_export] +macro_rules! impl_aref_for_gem_obj { + ( + impl $( <$( $tparam_id:ident ),+> )? for $type:ty + $( + where + $( $bind_param:path : $bind_trait:path ),+ + )? + ) => { + // SAFETY: All GEM objects are refcounted. + unsafe impl $( <$( $tparam_id ),+> )? $crate::types::AlwaysRefCounted for $type + where + Self: IntoGEMObject, + $( $( $bind_param : $bind_trait ),+ )? + { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is + // non-zero. + unsafe { bindings::drm_gem_object_get(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: core::ptr::NonNull) { + // SAFETY: `obj` is a valid pointer to an `Object`. + let obj = unsafe { obj.as_ref() }.as_raw(); + + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::drm_gem_object_put(obj) }; + } + } + }; +} + /// A type alias for retrieving a [`Driver`]s [`DriverFile`] implementation from its /// [`DriverObject`] implementation. /// @@ -263,21 +298,7 @@ impl Object { } } -// SAFETY: Instances of `Object` are always reference-counted. -unsafe impl crate::sync::aref::AlwaysRefCounted for Object { - fn inc_ref(&self) { - // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. - unsafe { bindings::drm_gem_object_get(self.as_raw()) }; - } - - unsafe fn dec_ref(obj: NonNull) { - // SAFETY: `obj` is a valid pointer to an `Object`. - let obj = unsafe { obj.as_ref() }; - - // SAFETY: The safety requirements guarantee that the refcount is non-zero. - unsafe { bindings::drm_gem_object_put(obj.as_raw()) } - } -} +impl_aref_for_gem_obj!(impl for Object where T: DriverObject); impl super::private::Sealed for Object {} -- cgit v1.2.3 From 442ba16a5a51368f5bafd011609f40782aec6d65 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Mon, 16 Mar 2026 17:16:12 -0400 Subject: rust: gem: Introduce DriverObject::Args This is an associated type that may be used in order to specify a data-type to pass to gem objects when constructing them, allowing for drivers to more easily initialize their private-data for gem objects. Signed-off-by: Lyude Paul Reviewed-by: Alice Ryhl Reviewed-by: Daniel Almeida Reviewed-by: Janne Grunau Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260316211646.650074-5-lyude@redhat.com [ Resolve merge conflicts in Tyr. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/drm/gem/mod.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'rust') diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 48ffdd7400ca..ed974bfdc861 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -73,8 +73,15 @@ pub trait DriverObject: Sync + Send + Sized { /// Parent `Driver` for this object. type Driver: drm::Driver; + /// The data type to use for passing arguments to [`DriverObject::new`]. + type Args; + /// Create a new driver data object for a GEM object of a given size. - fn new(dev: &drm::Device, size: usize) -> impl PinInit; + fn new( + dev: &drm::Device, + size: usize, + args: Self::Args, + ) -> impl PinInit; /// Open a new handle to an existing object, associated with a File. fn open(_obj: &::Object, _file: &DriverFile) -> Result { @@ -241,11 +248,11 @@ impl Object { }; /// Create a new GEM object. - pub fn new(dev: &drm::Device, size: usize) -> Result> { + pub fn new(dev: &drm::Device, size: usize, args: T::Args) -> Result> { let obj: Pin> = KBox::pin_init( try_pin_init!(Self { obj: Opaque::new(bindings::drm_gem_object::default()), - data <- T::new(dev, size), + data <- T::new(dev, size, args), }), GFP_KERNEL, )?; -- cgit v1.2.3 From f5e841e4966c1873b9bb2c69d07947a54284e5eb Mon Sep 17 00:00:00 2001 From: Daniel Almeida Date: Mon, 23 Mar 2026 20:26:59 -0300 Subject: rust: workqueue: add support for ARef Add support for the ARef smart pointer. This allows an instance of ARef to handle deferred work directly, which can be convenient or even necessary at times, depending on the specifics of the driver or subsystem. The implementation is similar to that of Arc, and a subsequent patch will implement support for drm::Device as the first user. This is notably important for work items that need access to the drm device, as it was not possible to enqueue work on a ARef> previously without failing the orphan rule. Reviewed-by: Alice Ryhl Acked-by: Danilo Krummrich Signed-off-by: Daniel Almeida Link: https://lore.kernel.org/r/20260323-aref-workitem-v3-1-f59729b812aa@collabora.com Signed-off-by: Alice Ryhl --- rust/kernel/workqueue.rs | 85 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 6 deletions(-) (limited to 'rust') diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 706e833e9702..6ae7f3fb3496 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -192,9 +192,9 @@ use crate::{ sync::Arc, sync::LockClassKey, time::Jiffies, - types::Opaque, + types::{ARef, AlwaysRefCounted, Opaque}, }; -use core::marker::PhantomData; +use core::{marker::PhantomData, ptr::NonNull}; /// Creates a [`Work`] initialiser with the given name and a newly-created lock class. #[macro_export] @@ -425,10 +425,11 @@ pub unsafe trait RawDelayedWorkItem: RawWorkItem {} /// Defines the method that should be called directly when a work item is executed. /// -/// This trait is implemented by `Pin>` and [`Arc`], and is mainly intended to be -/// implemented for smart pointer types. For your own structs, you would implement [`WorkItem`] -/// instead. The [`run`] method on this trait will usually just perform the appropriate -/// `container_of` translation and then call into the [`run`][WorkItem::run] method from the +/// This trait is implemented by `Pin>`, [`Arc`] and [`ARef`], and +/// is mainly intended to be implemented for smart pointer types. For your own +/// structs, you would implement [`WorkItem`] instead. The [`run`] method on +/// this trait will usually just perform the appropriate `container_of` +/// translation and then call into the [`run`][WorkItem::run] method from the /// [`WorkItem`] trait. /// /// This trait is used when the `work_struct` field is defined using the [`Work`] helper. @@ -934,6 +935,78 @@ where { } +// SAFETY: Like the `Arc` implementation, the `__enqueue` implementation for +// `ARef` obtains a `work_struct` from the `Work` field using +// `T::raw_get_work`, so the same safety reasoning applies: +// +// - `__enqueue` gets the `work_struct` from the `Work` field, using `T::raw_get_work`. +// - The only safe way to create a `Work` object is through `Work::new`. +// - `Work::new` makes sure that `T::Pointer::run` is passed to `init_work_with_key`. +// - Finally `Work` and `RawWorkItem` guarantee that the correct `Work` field +// will be used because of the ID const generic bound. This makes sure that `T::raw_get_work` +// uses the correct offset for the `Work` field, and `Work::new` picks the correct +// implementation of `WorkItemPointer` for `ARef`. +unsafe impl WorkItemPointer for ARef +where + T: AlwaysRefCounted, + T: WorkItem, + T: HasWork, +{ + unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { + // The `__enqueue` method always uses a `work_struct` stored in a `Work`. + let ptr = ptr.cast::>(); + + // SAFETY: This computes the pointer that `__enqueue` got from + // `ARef::into_raw`. + let ptr = unsafe { T::work_container_of(ptr) }; + + // SAFETY: The safety contract of `work_container_of` ensures that it + // returns a valid non-null pointer. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + + // SAFETY: This pointer comes from `ARef::into_raw` and we've been given + // back ownership. + let aref = unsafe { ARef::from_raw(ptr) }; + + T::run(aref) + } +} + +// SAFETY: The `work_struct` raw pointer is guaranteed to be valid for the duration of the call to +// the closure because we get it from an `ARef`, which means that the ref count will be at least 1, +// and we don't drop the `ARef` ourselves. If `queue_work_on` returns true, it is further guaranteed +// to be valid until a call to the function pointer in `work_struct` because we leak the memory it +// points to, and only reclaim it if the closure returns false, or in `WorkItemPointer::run`, which +// is what the function pointer in the `work_struct` must be pointing to, according to the safety +// requirements of `WorkItemPointer`. +unsafe impl RawWorkItem for ARef +where + T: AlwaysRefCounted, + T: WorkItem, + T: HasWork, +{ + type EnqueueOutput = Result<(), Self>; + + unsafe fn __enqueue(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool, + { + let ptr = ARef::into_raw(self); + + // SAFETY: Pointers from ARef::into_raw are valid and non-null. + let work_ptr = unsafe { T::raw_get_work(ptr.as_ptr()) }; + // SAFETY: `raw_get_work` returns a pointer to a valid value. + let work_ptr = unsafe { Work::raw_get(work_ptr) }; + + if queue_work_on(work_ptr) { + Ok(()) + } else { + // SAFETY: The work queue has not taken ownership of the pointer. + Err(unsafe { ARef::from_raw(ptr) }) + } + } +} + /// Returns the system work queue (`system_wq`). /// /// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are -- cgit v1.2.3 From 72a723df8decf70e04f799a6defda8bb62d41848 Mon Sep 17 00:00:00 2001 From: Daniel Almeida Date: Mon, 23 Mar 2026 20:27:00 -0300 Subject: rust: drm: dispatch work items to the private data This implementation dispatches any work enqueued on ARef> to its driver-provided handler. It does so by building upon the newly-added ARef support in workqueue.rs in order to call into the driver implementations for work_container_of and raw_get_work. This is notably important for work items that need access to the drm device, as it was not possible to enqueue work on a ARef> previously without failing the orphan rule. The current implementation needs T::Data to live inline with drm::Device in order for work_container_of to function. This restriction is already captured by the trait bounds. Drivers that need to share their ownership of T::Data may trivially get around this: // Lives inline in drm::Device struct DataWrapper { work: ..., // Heap-allocated, shared ownership. data: Arc, } Reviewed-by: Alice Ryhl Acked-by: Danilo Krummrich Signed-off-by: Daniel Almeida Link: https://lore.kernel.org/r/20260323-aref-workitem-v3-2-f59729b812aa@collabora.com Signed-off-by: Alice Ryhl --- rust/kernel/drm/device.rs | 56 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-) (limited to 'rust') diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 629ef0bd1188..5db5c7e3bb7a 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -6,8 +6,7 @@ use crate::{ alloc::allocator::Kmalloc, - bindings, - device, + bindings, device, drm::{ self, driver::AllocImpl, // @@ -18,7 +17,12 @@ use crate::{ ARef, AlwaysRefCounted, // }, - types::Opaque, // + types::Opaque, + workqueue::{ + HasWork, + Work, + WorkItem, // + }, }; use core::{ alloc::Layout, @@ -241,3 +245,49 @@ unsafe impl Send for Device {} // SAFETY: A `drm::Device` can be shared among threads because all immutable methods are protected // by the synchronization in `struct drm_device`. unsafe impl Sync for Device {} + +impl WorkItem for Device +where + T: drm::Driver, + T::Data: WorkItem>>, + T::Data: HasWork, ID>, +{ + type Pointer = ARef>; + + fn run(ptr: ARef>) { + T::Data::run(ptr); + } +} + +// SAFETY: +// +// - `raw_get_work` and `work_container_of` return valid pointers by relying on +// `T::Data::raw_get_work` and `container_of`. In particular, `T::Data` is +// stored inline in `drm::Device`, so the `container_of` call is valid. +// +// - The two methods are true inverses of each other: given `ptr: *mut +// Device`, `raw_get_work` will return a `*mut Work, ID>` through +// `T::Data::raw_get_work` and given a `ptr: *mut Work, ID>`, +// `work_container_of` will return a `*mut Device` through `container_of`. +unsafe impl HasWork, ID> for Device +where + T: drm::Driver, + T::Data: HasWork, ID>, +{ + unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work, ID> { + // SAFETY: The caller promises that `ptr` points to a valid `Device`. + let data_ptr = unsafe { &raw mut (*ptr).data }; + + // SAFETY: `data_ptr` is a valid pointer to `T::Data`. + unsafe { T::Data::raw_get_work(data_ptr) } + } + + unsafe fn work_container_of(ptr: *mut Work, ID>) -> *mut Self { + // SAFETY: The caller promises that `ptr` points at a `Work` field in + // `T::Data`. + let data_ptr = unsafe { T::Data::work_container_of(ptr) }; + + // SAFETY: `T::Data` is stored as the `data` field in `Device`. + unsafe { crate::container_of!(data_ptr, Self, data) } + } +} -- cgit v1.2.3 From 332666484f06cd85ad382329f5b3165aa627e9f8 Mon Sep 17 00:00:00 2001 From: Daniel Almeida Date: Mon, 23 Mar 2026 20:27:01 -0300 Subject: rust: workqueue: add delayed work support for ARef The preceding patches added support for ARef work items. By the same token, add support for delayed work items too. The rationale is the same: it may be convenient or even necessary at times to implement HasDelayedWork directly on ARef. A follow up patch will also implement support for drm::Device as the first user. Reviewed-by: Alice Ryhl Acked-by: Danilo Krummrich Signed-off-by: Daniel Almeida Link: https://lore.kernel.org/r/20260323-aref-workitem-v3-3-f59729b812aa@collabora.com Signed-off-by: Alice Ryhl --- rust/kernel/workqueue.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'rust') diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 6ae7f3fb3496..4ee4ff567197 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -1007,6 +1007,17 @@ where } } +// SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in +// `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of +// the `delayed_work` has the same access rules as its `work` field. +unsafe impl RawDelayedWorkItem for ARef +where + T: WorkItem, + T: HasDelayedWork, + T: AlwaysRefCounted, +{ +} + /// Returns the system work queue (`system_wq`). /// /// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are -- cgit v1.2.3 From 206bada308d0d715d62ac841784af34a2ca22ff6 Mon Sep 17 00:00:00 2001 From: Daniel Almeida Date: Mon, 23 Mar 2026 20:27:02 -0300 Subject: rust: drm: dispatch delayed work items to the private data Much like the patch that dispatched (regular) work items, we also need to dispatch delayed work items in order not to trigger the orphan rule. This allows a drm::Device to dispatch the delayed work to T::Data. Reviewed-by: Alice Ryhl Acked-by: Danilo Krummrich Signed-off-by: Daniel Almeida Link: https://lore.kernel.org/r/20260323-aref-workitem-v3-4-f59729b812aa@collabora.com Signed-off-by: Alice Ryhl --- rust/kernel/drm/device.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'rust') diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 5db5c7e3bb7a..adbafe8db54d 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -19,6 +19,7 @@ use crate::{ }, types::Opaque, workqueue::{ + HasDelayedWork, HasWork, Work, WorkItem, // @@ -291,3 +292,15 @@ where unsafe { crate::container_of!(data_ptr, Self, data) } } } + +// SAFETY: Our `HasWork` implementation returns a `work_struct` that is +// stored in the `work` field of a `delayed_work` with the same access rules as +// the `work_struct` owing to the bound on `T::Data: HasDelayedWork, +// ID>`, which requires that `T::Data::raw_get_work` return a `work_struct` that +// is inside a `delayed_work`. +unsafe impl HasDelayedWork, ID> for Device +where + T: drm::Driver, + T::Data: HasDelayedWork, ID>, +{ +} -- cgit v1.2.3 From d4cf576672fbfee061d6f4f70c74b3b3d163447c Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 26 Mar 2026 15:25:36 +0000 Subject: rust: workqueue: use new sync::aref path for imports ARef and AlwaysRefCounted are being moved to sync::aref, and the re-exports under types are planned to be removed. Thus, update imports to the new path. Acked-by: Danilo Krummrich Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260326-drm-rust-next-fix-aref-v1-1-7f6f58d2828a@google.com Signed-off-by: Alice Ryhl --- rust/kernel/workqueue.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'rust') diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 4ee4ff567197..7e253b6f299c 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -189,10 +189,16 @@ use crate::{ alloc::{AllocError, Flags}, container_of, prelude::*, - sync::Arc, - sync::LockClassKey, + sync::{ + aref::{ + ARef, + AlwaysRefCounted, // + }, + Arc, + LockClassKey, // + }, time::Jiffies, - types::{ARef, AlwaysRefCounted, Opaque}, + types::Opaque, }; use core::{marker::PhantomData, ptr::NonNull}; -- cgit v1.2.3 From bdf6b22fd52954f5ac88689eeaf960ac9687b78c Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 26 Mar 2026 15:25:37 +0000 Subject: rust: drm: use new sync::aref path for imports ARef and AlwaysRefCounted are being moved to sync::aref, and the re-exports under types are planned to be removed. Thus, update imports to the new path. Acked-by: Danilo Krummrich Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260326-drm-rust-next-fix-aref-v1-2-7f6f58d2828a@google.com Signed-off-by: Alice Ryhl --- rust/kernel/drm/gem/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'rust') diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index ed974bfdc861..6cc441ee5b63 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -39,7 +39,7 @@ macro_rules! impl_aref_for_gem_obj { )? ) => { // SAFETY: All GEM objects are refcounted. - unsafe impl $( <$( $tparam_id ),+> )? $crate::types::AlwaysRefCounted for $type + unsafe impl $( <$( $tparam_id ),+> )? $crate::sync::aref::AlwaysRefCounted for $type where Self: IntoGEMObject, $( $( $bind_param : $bind_trait ),+ )? -- cgit v1.2.3 From 9b836641d3bfa1ab096ec6263f0fa6880cb9c5ef Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 20 Mar 2026 16:08:26 +0000 Subject: rust: helpers: Add bindings/wrappers for dma_resv_lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is just for basic usage in the DRM shmem abstractions for implied locking, not intended as a full DMA Reservation abstraction yet. Cc: Sumit Semwal Cc: Christian König Signed-off-by: Asahi Lina Signed-off-by: Daniel Almeida Reviewed-by: Alice Ryhl Signed-off-by: Lyude Paul Reviewed-by: Janne Grunau Reviewed-by: Daniel Almeida Acked-by: David Airlie Signed-off-by: Alice Ryhl Link: https://patch.msgid.link/20260320-gpuvm-rust-v5-2-76fd44f17a87@google.com Signed-off-by: Danilo Krummrich --- rust/bindings/bindings_helper.h | 1 + rust/helpers/dma-resv.c | 14 ++++++++++++++ rust/helpers/helpers.c | 1 + 3 files changed, 16 insertions(+) create mode 100644 rust/helpers/dma-resv.c (limited to 'rust') diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index dbb765a9fdbd..563863d96d38 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers/dma-resv.c b/rust/helpers/dma-resv.c new file mode 100644 index 000000000000..71914d8241e2 --- /dev/null +++ b/rust/helpers/dma-resv.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper +int rust_helper_dma_resv_lock(struct dma_resv *obj, struct ww_acquire_ctx *ctx) +{ + return dma_resv_lock(obj, ctx); +} + +__rust_helper void rust_helper_dma_resv_unlock(struct dma_resv *obj) +{ + dma_resv_unlock(obj); +} diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index a53929ce52a3..b6b20ad2e0e6 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -28,6 +28,7 @@ #include "cred.c" #include "device.c" #include "dma.c" +#include "dma-resv.c" #include "drm.c" #include "err.c" #include "irq.c" -- cgit v1.2.3 From 89b4964c0456d9939a9f5187891a36bb87111e58 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Mon, 16 Mar 2026 17:16:10 -0400 Subject: rust: drm: gem: Add raw_dma_resv() function For retrieving a pointer to the struct dma_resv for a given GEM object. We also introduce it in a new trait, BaseObjectPrivate, which we automatically implement for all gem objects and don't expose to users outside of the crate. Signed-off-by: Lyude Paul Reviewed-by: Janne Grunau Tested-by: Janne Grunau Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260316211646.650074-3-lyude@redhat.com [ Fix incorrect reference in safety comment. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/drm/gem/mod.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'rust') diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 6cc441ee5b63..a78b98c40d56 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -215,6 +215,18 @@ pub trait BaseObject: IntoGEMObject { impl BaseObject for T {} +/// Crate-private base operations shared by all GEM object classes. +#[expect(unused)] +pub(crate) trait BaseObjectPrivate: IntoGEMObject { + /// Return a pointer to this object's dma_resv. + fn raw_dma_resv(&self) -> *mut bindings::dma_resv { + // SAFETY: `self.as_raw()` always returns a valid pointer to the base DRM GEM object. + unsafe { (*self.as_raw()).resv } + } +} + +impl BaseObjectPrivate for T {} + /// A base GEM object. /// /// # Invariants -- cgit v1.2.3 From 80df573af9ef3aa63e1bacb6e17d57a7cd69afe2 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Mon, 16 Mar 2026 17:16:13 -0400 Subject: rust: drm: gem: shmem: Add DRM shmem helper abstraction The DRM shmem helper includes common code useful for drivers which allocate GEM objects as anonymous shmem. Add a Rust abstraction for this. Drivers can choose the raw GEM implementation or the shmem layer, depending on their needs. Signed-off-by: Asahi Lina Signed-off-by: Daniel Almeida Reviewed-by: Daniel Almeida Signed-off-by: Lyude Paul Reviewed-by: Janne Grunau Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260316211646.650074-6-lyude@redhat.com [ * DRM_GEM_SHMEM_HELPER is a tristate; when a module driver selects it, it becomes =m. The Rust kernel crate and its C helpers are always built into vmlinux and can't reference symbols from a module, causing link errors. Thus, add RUST_DRM_GEM_SHMEM_HELPER bool Kconfig that selects DRM_GEM_SHMEM_HELPER, forcing it built-in when Rust drivers need it; use cfg(CONFIG_RUST_DRM_GEM_SHMEM_HELPER) for the shmem module. * Add cfg_attr(not(CONFIG_RUST_DRM_GEM_SHMEM_HELPER), expect(unused)) on pub(crate) use impl_aref_for_gem_obj and BaseObjectPrivate, so that unused warnings are suppressed when shmem is not enabled. * Enable const_refs_to_static (stabilized in 1.83) to prevent build errors with older compilers. * Use &raw const for bindings::drm_gem_shmem_vm_ops and add #[allow(unused_unsafe, reason = "Safe since Rust 1.82.0")]. * Fix incorrect C Header path and minor spelling and formatting issues. * Drop shmem::Object::sg_table() as the current implementation is unsound. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/bindings/bindings_helper.h | 2 + rust/helpers/drm.c | 56 +++++++++- rust/kernel/drm/gem/mod.rs | 7 +- rust/kernel/drm/gem/shmem.rs | 228 ++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 5 files changed, 292 insertions(+), 2 deletions(-) create mode 100644 rust/kernel/drm/gem/shmem.rs (limited to 'rust') diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 563863d96d38..eda8f50d3a3c 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -63,6 +64,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers/drm.c b/rust/helpers/drm.c index fe226f7b53ef..65f3f22b0e1d 100644 --- a/rust/helpers/drm.c +++ b/rust/helpers/drm.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #ifdef CONFIG_DRM @@ -21,4 +22,57 @@ rust_helper_drm_vma_node_offset_addr(struct drm_vma_offset_node *node) return drm_vma_node_offset_addr(node); } -#endif +#ifdef CONFIG_DRM_GEM_SHMEM_HELPER +__rust_helper void +rust_helper_drm_gem_shmem_object_free(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_free(obj); +} + +__rust_helper void +rust_helper_drm_gem_shmem_object_print_info(struct drm_printer *p, unsigned int indent, + const struct drm_gem_object *obj) +{ + drm_gem_shmem_object_print_info(p, indent, obj); +} + +__rust_helper int +rust_helper_drm_gem_shmem_object_pin(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_pin(obj); +} + +__rust_helper void +rust_helper_drm_gem_shmem_object_unpin(struct drm_gem_object *obj) +{ + drm_gem_shmem_object_unpin(obj); +} + +__rust_helper struct sg_table * +rust_helper_drm_gem_shmem_object_get_sg_table(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_get_sg_table(obj); +} + +__rust_helper int +rust_helper_drm_gem_shmem_object_vmap(struct drm_gem_object *obj, + struct iosys_map *map) +{ + return drm_gem_shmem_object_vmap(obj, map); +} + +__rust_helper void +rust_helper_drm_gem_shmem_object_vunmap(struct drm_gem_object *obj, + struct iosys_map *map) +{ + drm_gem_shmem_object_vunmap(obj, map); +} + +__rust_helper int +rust_helper_drm_gem_shmem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) +{ + return drm_gem_shmem_object_mmap(obj, vma); +} + +#endif /* CONFIG_DRM_GEM_SHMEM_HELPER */ +#endif /* CONFIG_DRM */ diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index a78b98c40d56..75acda7ba500 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -26,6 +26,9 @@ use core::{ ptr::NonNull, // }; +#[cfg(CONFIG_RUST_DRM_GEM_SHMEM_HELPER)] +pub mod shmem; + /// A macro for implementing [`AlwaysRefCounted`] for any GEM object type. /// /// Since all GEM objects use the same refcounting scheme. @@ -60,6 +63,8 @@ macro_rules! impl_aref_for_gem_obj { } }; } +#[cfg_attr(not(CONFIG_RUST_DRM_GEM_SHMEM_HELPER), allow(unused))] +pub(crate) use impl_aref_for_gem_obj; /// A type alias for retrieving a [`Driver`]s [`DriverFile`] implementation from its /// [`DriverObject`] implementation. @@ -216,7 +221,7 @@ pub trait BaseObject: IntoGEMObject { impl BaseObject for T {} /// Crate-private base operations shared by all GEM object classes. -#[expect(unused)] +#[cfg_attr(not(CONFIG_RUST_DRM_GEM_SHMEM_HELPER), expect(unused))] pub(crate) trait BaseObjectPrivate: IntoGEMObject { /// Return a pointer to this object's dma_resv. fn raw_dma_resv(&self) -> *mut bindings::dma_resv { diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs new file mode 100644 index 000000000000..d025fb035195 --- /dev/null +++ b/rust/kernel/drm/gem/shmem.rs @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! DRM GEM shmem helper objects +//! +//! C header: [`include/linux/drm/drm_gem_shmem_helper.h`](srctree/include/drm/drm_gem_shmem_helper.h) + +// TODO: +// - There are a number of spots here that manually acquire/release the DMA reservation lock using +// dma_resv_(un)lock(). In the future we should add support for ww mutex, expose a method to +// acquire a reference to the WwMutex, and then use that directly instead of the C functions here. + +use crate::{ + container_of, + drm::{ + device, + driver, + gem, + private::Sealed, // + }, + error::to_result, + prelude::*, + types::{ + ARef, + Opaque, // + }, // +}; +use core::{ + ops::{ + Deref, + DerefMut, // + }, + ptr::NonNull, +}; +use gem::{ + BaseObjectPrivate, + DriverObject, + IntoGEMObject, // +}; + +/// A struct for controlling the creation of shmem-backed GEM objects. +/// +/// This is used with [`Object::new()`] to control various properties that can only be set when +/// initially creating a shmem-backed GEM object. +#[derive(Default)] +pub struct ObjectConfig<'a, T: DriverObject> { + /// Whether to set the write-combine map flag. + pub map_wc: bool, + + /// Reuse the DMA reservation from another GEM object. + /// + /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. + pub parent_resv_obj: Option<&'a Object>, +} + +/// A shmem-backed GEM object. +/// +/// # Invariants +/// +/// `obj` contains a valid initialized `struct drm_gem_shmem_object` for the lifetime of this +/// object. +#[repr(C)] +#[pin_data] +pub struct Object { + #[pin] + obj: Opaque, + /// Parent object that owns this object's DMA reservation object. + parent_resv_obj: Option>>, + #[pin] + inner: T, +} + +super::impl_aref_for_gem_obj!(impl for Object where T: DriverObject); + +// SAFETY: All GEM objects are thread-safe. +unsafe impl Send for Object {} + +// SAFETY: All GEM objects are thread-safe. +unsafe impl Sync for Object {} + +impl Object { + /// `drm_gem_object_funcs` vtable suitable for GEM shmem objects. + const VTABLE: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { + free: Some(Self::free_callback), + open: Some(super::open_callback::), + close: Some(super::close_callback::), + print_info: Some(bindings::drm_gem_shmem_object_print_info), + export: None, + pin: Some(bindings::drm_gem_shmem_object_pin), + unpin: Some(bindings::drm_gem_shmem_object_unpin), + get_sg_table: Some(bindings::drm_gem_shmem_object_get_sg_table), + vmap: Some(bindings::drm_gem_shmem_object_vmap), + vunmap: Some(bindings::drm_gem_shmem_object_vunmap), + mmap: Some(bindings::drm_gem_shmem_object_mmap), + status: None, + rss: None, + #[allow(unused_unsafe, reason = "Safe since Rust 1.82.0")] + // SAFETY: `drm_gem_shmem_vm_ops` is a valid, static const on the C side. + vm_ops: unsafe { &raw const bindings::drm_gem_shmem_vm_ops }, + evict: None, + }; + + /// Return a raw pointer to the embedded drm_gem_shmem_object. + fn as_raw_shmem(&self) -> *mut bindings::drm_gem_shmem_object { + self.obj.get() + } + + /// Create a new shmem-backed DRM object of the given size. + /// + /// Additional config options can be specified using `config`. + pub fn new( + dev: &device::Device, + size: usize, + config: ObjectConfig<'_, T>, + args: T::Args, + ) -> Result> { + let new: Pin> = KBox::try_pin_init( + try_pin_init!(Self { + obj <- Opaque::init_zeroed(), + parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), + inner <- T::new(dev, size, args), + }), + GFP_KERNEL, + )?; + + // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. + unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; + + // SAFETY: The arguments are all valid via the type invariants. + to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_raw_shmem(), size) })?; + + // SAFETY: We never move out of `self`. + let new = KBox::into_raw(unsafe { Pin::into_inner_unchecked(new) }); + + // SAFETY: We're taking over the owned refcount from `drm_gem_shmem_init`. + let obj = unsafe { ARef::from_raw(NonNull::new_unchecked(new)) }; + + // Start filling out values from `config` + if let Some(parent_resv) = config.parent_resv_obj { + // SAFETY: We have yet to expose the new gem object outside of this function, so it is + // safe to modify this field. + unsafe { (*obj.obj.get()).base.resv = parent_resv.raw_dma_resv() }; + } + + // SAFETY: We have yet to expose this object outside of this function, so we're guaranteed + // to have exclusive access - thus making this safe to hold a mutable reference to. + let shmem = unsafe { &mut *obj.as_raw_shmem() }; + shmem.set_map_wc(config.map_wc); + + Ok(obj) + } + + /// Returns the `Device` that owns this GEM object. + pub fn dev(&self) -> &device::Device { + // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`. + unsafe { device::Device::from_raw((*self.as_raw()).dev) } + } + + extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { + // SAFETY: + // - DRM always passes a valid gem object here + // - We used drm_gem_shmem_create() in our create_gem_object callback, so we know that + // `obj` is contained within a drm_gem_shmem_object + let this = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) }; + + // SAFETY: + // - We're in free_callback - so this function is safe to call. + // - We won't be using the gem resources on `this` after this call. + unsafe { bindings::drm_gem_shmem_release(this) }; + + // SAFETY: + // - We verified above that `obj` is valid, which makes `this` valid + // - This function is set in AllocOps, so we know that `this` is contained within a + // `Object` + let this = unsafe { container_of!(Opaque::cast_from(this), Self, obj) }.cast_mut(); + + // SAFETY: We're recovering the Kbox<> we created in gem_create_object() + let _ = unsafe { KBox::from_raw(this) }; + } +} + +impl Deref for Object { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for Object { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl Sealed for Object {} + +impl gem::IntoGEMObject for Object { + fn as_raw(&self) -> *mut bindings::drm_gem_object { + // SAFETY: + // - Our immutable reference is proof that this is safe to dereference. + // - `obj` is always a valid drm_gem_shmem_object via our type invariants. + unsafe { &raw mut (*self.obj.get()).base } + } + + unsafe fn from_raw<'a>(obj: *mut bindings::drm_gem_object) -> &'a Object { + // SAFETY: The safety contract of from_gem_obj() guarantees that `obj` is contained within + // `Self` + unsafe { + let obj = Opaque::cast_from(container_of!(obj, bindings::drm_gem_shmem_object, base)); + + &*container_of!(obj, Object, obj) + } + } +} + +impl driver::AllocImpl for Object { + type Driver = T::Driver; + + const ALLOC_OPS: driver::AllocOps = driver::AllocOps { + gem_create_object: None, + prime_handle_to_fd: None, + prime_fd_to_handle: None, + gem_prime_import: None, + gem_prime_import_sg_table: Some(bindings::drm_gem_shmem_prime_import_sg_table), + dumb_create: Some(bindings::drm_gem_shmem_dumb_create), + dumb_map_offset: None, + }; +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index e0837ffc91bf..40de00ce4f97 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -38,6 +38,7 @@ #![feature(const_option)] #![feature(const_ptr_write)] #![feature(const_refs_to_cell)] +#![feature(const_refs_to_static)] // // Stable since Rust 1.84.0. #![feature(strict_provenance)] -- cgit v1.2.3 From 25fe63db0024da172457055532c96bef924a8c78 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Wed, 25 Mar 2026 01:39:15 +0100 Subject: rust: uaccess: generalize write_dma() to accept any Coherent Generalize write_dma() from &Coherent<[u8]> to &Coherent where T: KnownSize + AsBytes + ?Sized. The function body only uses as_ptr() and size(), which work for any such T, so there is no reason to restrict it to byte slices. Acked-by: Miguel Ojeda Acked-by: Gary Guo Reviewed-by: Alexandre Courbot Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20260325003921.3420-1-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/uaccess.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'rust') diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs index e26ef90ba8ad..6c9c1cce3c63 100644 --- a/rust/kernel/uaccess.rs +++ b/rust/kernel/uaccess.rs @@ -12,6 +12,7 @@ use crate::{ ffi::{c_char, c_void}, fs::file, prelude::*, + ptr::KnownSize, transmute::{AsBytes, FromBytes}, }; use core::mem::{size_of, MaybeUninit}; @@ -524,7 +525,12 @@ impl UserSliceWriter { /// writer.write_dma(alloc, 0, 256) /// } /// ``` - pub fn write_dma(&mut self, alloc: &Coherent<[u8]>, offset: usize, count: usize) -> Result { + pub fn write_dma( + &mut self, + alloc: &Coherent, + offset: usize, + count: usize, + ) -> Result { let len = alloc.size(); if offset.checked_add(count).ok_or(EOVERFLOW)? > len { return Err(ERANGE); -- cgit v1.2.3 From d1619a433806c61d240a1eb9b4f03cb33ac40fce Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Wed, 25 Mar 2026 01:39:16 +0100 Subject: rust: dma: generalize BinaryWriter impl for Coherent Generalize the BinaryWriter implementation from Coherent<[u8]> to Coherent where T: KnownSize + AsBytes + ?Sized. The implementation only uses size() and write_dma(), neither of which depends on the inner type being a byte slice. This allows any Coherent allocation with an AsBytes inner type to be exposed as a debugfs binary file. Acked-by: Gary Guo Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260325003921.3420-2-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/dma.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'rust') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 3eef7c2396bb..7bc97f9f83fd 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -885,7 +885,7 @@ unsafe impl Send for Coherent {} // The safe methods only return metadata or raw pointers whose use requires `unsafe`. unsafe impl Sync for Coherent {} -impl debugfs::BinaryWriter for Coherent<[u8]> { +impl debugfs::BinaryWriter for Coherent { fn write_to_slice( &self, writer: &mut UserSliceWriter, -- cgit v1.2.3 From 18fb5f1f0289b8217c0c43d54d12bccc201dd640 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sat, 21 Mar 2026 18:27:46 +0100 Subject: rust: dma: remove DMA_ATTR_NO_KERNEL_MAPPING from public attrs When DMA_ATTR_NO_KERNEL_MAPPING is passed to dma_alloc_attrs(), the returned CPU address is not a pointer to the allocated memory but an opaque handle (e.g. struct page *). Coherent (or CoherentAllocation respectively) stores this value as NonNull and exposes methods that dereference it and even modify its contents. Remove the flag from the public attrs module such that drivers cannot pass it to Coherent (or CoherentAllocation respectively) in the first place. Instead DMA_ATTR_NO_KERNEL_MAPPING can be supported with an additional opaque type (e.g. CoherentHandle) which does not provide access to the allocated memory. Cc: stable@vger.kernel.org Fixes: ad2907b4e308 ("rust: add dma coherent allocator abstraction") Signed-off-by: Danilo Krummrich Reviewed-by: Alice Ryhl Reviewed-by: Alexandre Courbot Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260321172749.592387-1-dakr@kernel.org Signed-off-by: Alexandre Courbot --- rust/kernel/dma.rs | 3 --- 1 file changed, 3 deletions(-) (limited to 'rust') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 7bc97f9f83fd..beb4dea7dd84 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -267,9 +267,6 @@ pub mod attrs { /// Specifies that writes to the mapping may be buffered to improve performance. pub const DMA_ATTR_WRITE_COMBINE: Attrs = Attrs(bindings::DMA_ATTR_WRITE_COMBINE); - /// Lets the platform to avoid creating a kernel virtual mapping for the allocated buffer. - pub const DMA_ATTR_NO_KERNEL_MAPPING: Attrs = Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING); - /// Allows platform code to skip synchronization of the CPU cache for the given buffer assuming /// that it has been already transferred to 'device' domain. pub const DMA_ATTR_SKIP_CPU_SYNC: Attrs = Attrs(bindings::DMA_ATTR_SKIP_CPU_SYNC); -- cgit v1.2.3 From 6dd782af1eb6be16a80349ea7822f0f8a23bb3e8 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sat, 21 Mar 2026 18:27:47 +0100 Subject: rust: dma: add CoherentHandle for DMA allocations without kernel mapping Add CoherentHandle, an opaque DMA allocation type for buffers that are only ever accessed by hardware. Unlike Coherent, it does not provide CPU access to the allocated memory. CoherentHandle implicitly sets DMA_ATTR_NO_KERNEL_MAPPING and stores the value returned by dma_alloc_attrs() as an opaque handle (NonNull) rather than a typed pointer, since with this flag the C API returns an opaque cookie (e.g. struct page *), not a CPU pointer to the allocated memory. Only the DMA bus address is exposed to drivers; the opaque handle is used solely to free the allocation on drop. This commit is for reference only; there is currently no in-tree user. Signed-off-by: Danilo Krummrich Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260321172749.592387-2-dakr@kernel.org [acourbot: fix conflict in dma.rs.] Signed-off-by: Alexandre Courbot --- rust/kernel/dma.rs | 119 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) (limited to 'rust') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index beb4dea7dd84..7ea3c3b73367 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -907,6 +907,125 @@ impl debugfs::BinaryWriter for Coherent { } } +/// An opaque DMA allocation without a kernel virtual mapping. +/// +/// Unlike [`Coherent`], a `CoherentHandle` does not provide CPU access to the allocated memory. +/// The allocation is always performed with `DMA_ATTR_NO_KERNEL_MAPPING`, meaning no kernel +/// virtual mapping is created for the buffer. The value returned by the C API as the CPU +/// address is an opaque handle used only to free the allocation. +/// +/// This is useful for buffers that are only ever accessed by hardware. +/// +/// # Invariants +/// +/// - `cpu_handle` holds the opaque handle returned by `dma_alloc_attrs` with +/// `DMA_ATTR_NO_KERNEL_MAPPING` set, and is only valid for passing back to `dma_free_attrs`. +/// - `dma_handle` is the corresponding bus address for device DMA. +/// - `size` is the allocation size in bytes as passed to `dma_alloc_attrs`. +/// - `dma_attrs` contains the attributes used for the allocation, always including +/// `DMA_ATTR_NO_KERNEL_MAPPING`. +pub struct CoherentHandle { + dev: ARef, + dma_handle: DmaAddress, + cpu_handle: NonNull, + size: usize, + dma_attrs: Attrs, +} + +impl CoherentHandle { + /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping. + /// + /// Additional DMA attributes may be passed via `dma_attrs`; `DMA_ATTR_NO_KERNEL_MAPPING` is + /// always set implicitly. + /// + /// Returns `EINVAL` if `size` is zero, `ENOMEM` if the allocation fails. + pub fn alloc_with_attrs( + dev: &device::Device, + size: usize, + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result { + if size == 0 { + return Err(EINVAL); + } + + let dma_attrs = dma_attrs | Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING); + let mut dma_handle = 0; + // SAFETY: `dev.as_raw()` is valid by the type invariant on `device::Device`. + let cpu_handle = unsafe { + bindings::dma_alloc_attrs( + dev.as_raw(), + size, + &mut dma_handle, + gfp_flags.as_raw(), + dma_attrs.as_raw(), + ) + }; + + let cpu_handle = NonNull::new(cpu_handle).ok_or(ENOMEM)?; + + // INVARIANT: `cpu_handle` is the opaque handle from a successful `dma_alloc_attrs` call + // with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_handle` is the corresponding DMA address, + // and we hold a refcounted reference to the device. + Ok(Self { + dev: dev.into(), + dma_handle, + cpu_handle, + size, + dma_attrs, + }) + } + + /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping. + #[inline] + pub fn alloc( + dev: &device::Device, + size: usize, + gfp_flags: kernel::alloc::Flags, + ) -> Result { + Self::alloc_with_attrs(dev, size, gfp_flags, Attrs(0)) + } + + /// Returns the DMA handle for this allocation. + /// + /// This address can be programmed into device hardware for DMA access. + #[inline] + pub fn dma_handle(&self) -> DmaAddress { + self.dma_handle + } + + /// Returns the size in bytes of this allocation. + #[inline] + pub fn size(&self) -> usize { + self.size + } +} + +impl Drop for CoherentHandle { + fn drop(&mut self) { + // SAFETY: All values are valid by the type invariants on `CoherentHandle`. + // `cpu_handle` is the opaque handle from `dma_alloc_attrs` and is passed back unchanged. + unsafe { + bindings::dma_free_attrs( + self.dev.as_raw(), + self.size, + self.cpu_handle.as_ptr(), + self.dma_handle, + self.dma_attrs.as_raw(), + ) + } + } +} + +// SAFETY: `CoherentHandle` only holds a device reference, a DMA handle, an opaque CPU handle, +// and a size. None of these are tied to a specific thread. +unsafe impl Send for CoherentHandle {} + +// SAFETY: `CoherentHandle` provides no CPU access to the underlying allocation. The only +// operations on `&CoherentHandle` are reading the DMA handle and size, both of which are +// plain `Copy` values. +unsafe impl Sync for CoherentHandle {} + /// Reads a field of an item from an allocated region of structs. /// /// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating -- cgit v1.2.3 From 816718c611cab2164d718b91ad8204afcd0af81f Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Fri, 27 Mar 2026 00:22:07 +0900 Subject: rust: dma: add from-slice constructors for Coherent and CoherentBox A very common pattern is to create a block of coherent memory with the content of an already-existing slice of bytes (e.g. a loaded firmware blob). `CoherentBox` makes this easier, but still implies a potentially panicking operation with `copy_from_slice` that requires a `PANIC` comment. Add `from_slice_with_attrs` and `from_slice` methods to both `Coherent` and `CoherentBox` to turn this into a trivial one-step operation. Reviewed-by: Andreas Hindborg Reviewed-by: Gary Guo Reviewed-by: Danilo Krummrich Link: https://patch.msgid.link/20260327-b4-nova-dma-removal-v2-1-616e1d0b5cb3@nvidia.com Signed-off-by: Alexandre Courbot --- rust/kernel/dma.rs | 107 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) (limited to 'rust') diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 7ea3c3b73367..4995ee5dc689 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -453,6 +453,66 @@ impl CoherentBox<[T]> { Ok(()) } + + /// Allocates a region of coherent memory of the same size as `data` and initializes it with a + /// copy of its contents. + /// + /// This is the [`CoherentBox`] variant of [`Coherent::from_slice_with_attrs`]. + /// + /// # Examples + /// + /// ``` + /// use core::ops::Deref; + /// + /// # use kernel::device::{Bound, Device}; + /// use kernel::dma::{ + /// attrs::*, + /// CoherentBox + /// }; + /// + /// # fn test(dev: &Device) -> Result { + /// let data = [0u8, 1u8, 2u8, 3u8]; + /// let c: CoherentBox<[u8]> = + /// CoherentBox::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// + /// assert_eq!(c.deref(), &data); + /// # Ok::<(), Error>(()) } + /// ``` + pub fn from_slice_with_attrs( + dev: &device::Device, + data: &[T], + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result + where + T: Copy, + { + let mut slice = Self(Coherent::::alloc_slice_with_attrs( + dev, + data.len(), + gfp_flags, + dma_attrs, + )?); + + // PANIC: `slice` was created with length `data.len()`. + slice.copy_from_slice(data); + + Ok(slice) + } + + /// Performs the same functionality as [`CoherentBox::from_slice_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn from_slice( + dev: &device::Device, + data: &[T], + gfp_flags: kernel::alloc::Flags, + ) -> Result + where + T: Copy, + { + Self::from_slice_with_attrs(dev, data, gfp_flags, Attrs(0)) + } } impl CoherentBox { @@ -839,6 +899,53 @@ impl Coherent { ) -> Result> { Self::zeroed_slice_with_attrs(dev, len, gfp_flags, Attrs(0)) } + + /// Allocates a region of coherent memory of the same size as `data` and initializes it with a + /// copy of its contents. + /// + /// # Examples + /// + /// ``` + /// # use kernel::device::{Bound, Device}; + /// use kernel::dma::{ + /// attrs::*, + /// Coherent + /// }; + /// + /// # fn test(dev: &Device) -> Result { + /// let data = [0u8, 1u8, 2u8, 3u8]; + /// // `c` has the same content as `data`. + /// let c: Coherent<[u8]> = + /// Coherent::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?; + /// + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + pub fn from_slice_with_attrs( + dev: &device::Device, + data: &[T], + gfp_flags: kernel::alloc::Flags, + dma_attrs: Attrs, + ) -> Result> + where + T: Copy, + { + CoherentBox::from_slice_with_attrs(dev, data, gfp_flags, dma_attrs).map(Into::into) + } + + /// Performs the same functionality as [`Coherent::from_slice_with_attrs`], except the + /// `dma_attrs` is 0 by default. + #[inline] + pub fn from_slice( + dev: &device::Device, + data: &[T], + gfp_flags: kernel::alloc::Flags, + ) -> Result> + where + T: Copy, + { + Self::from_slice_with_attrs(dev, data, gfp_flags, Attrs(0)) + } } impl Coherent<[T]> { -- cgit v1.2.3