summaryrefslogtreecommitdiff
path: root/rust
diff options
context:
space:
mode:
Diffstat (limited to 'rust')
-rw-r--r--rust/bindings/bindings_helper.h5
-rw-r--r--rust/helpers/dma_fence.c49
-rw-r--r--rust/helpers/helpers.c1
-rw-r--r--rust/helpers/pci.c6
-rw-r--r--rust/kernel/auxiliary.rs17
-rw-r--r--rust/kernel/dma_buf/dma_fence.rs1022
-rw-r--r--rust/kernel/dma_buf/mod.rs14
-rw-r--r--rust/kernel/drm/driver.rs77
-rw-r--r--rust/kernel/drm/gem/mod.rs13
-rw-r--r--rust/kernel/drm/gem/shmem.rs5
-rw-r--r--rust/kernel/io.rs39
-rw-r--r--rust/kernel/io/resource.rs8
-rw-r--r--rust/kernel/lib.rs2
-rw-r--r--rust/kernel/maple_tree.rs30
-rw-r--r--rust/kernel/pci.rs14
-rw-r--r--rust/kernel/pci/irq.rs68
-rw-r--r--rust/kernel/uaccess.rs14
17 files changed, 1274 insertions, 110 deletions
diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h
index c706dda5b5ca..3791af0aadc7 100644
--- a/rust/bindings/bindings_helper.h
+++ b/rust/bindings/bindings_helper.h
@@ -53,6 +53,7 @@
#include <linux/debugfs.h>
#include <linux/device/faux.h>
#include <linux/dma-direction.h>
+#include <linux/dma-fence.h>
#include <linux/dma-mapping.h>
#include <linux/dma-resv.h>
#include <linux/errname.h>
@@ -106,8 +107,8 @@
#include <../../drivers/base/base.h>
#if defined(CONFIG_DRM_PANIC_SCREEN_QR_CODE)
-// Used by `#[export]` in `drivers/gpu/drm/drm_panic_qr.rs`.
-#include <drm/drm_panic.h>
+// Used by `#[export]` in `drivers/gpu/drm/drm_panic_helper_qr.rs`.
+#include <drm/drm_panic_helper.h>
#endif
/* `bindgen` gets confused at certain things. */
diff --git a/rust/helpers/dma_fence.c b/rust/helpers/dma_fence.c
new file mode 100644
index 000000000000..549f6b6a7171
--- /dev/null
+++ b/rust/helpers/dma_fence.c
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/dma-fence.h>
+
+__rust_helper void rust_helper_dma_fence_get(struct dma_fence *f)
+{
+ dma_fence_get(f);
+}
+
+__rust_helper void rust_helper_dma_fence_put(struct dma_fence *f)
+{
+ dma_fence_put(f);
+}
+
+__rust_helper bool rust_helper_dma_fence_begin_signalling(void)
+{
+ return dma_fence_begin_signalling();
+}
+
+__rust_helper void rust_helper_dma_fence_end_signalling(bool cookie)
+{
+ dma_fence_end_signalling(cookie);
+}
+
+__rust_helper bool rust_helper_dma_fence_is_signaled(struct dma_fence *f)
+{
+ return dma_fence_is_signaled(f);
+}
+
+__rust_helper bool rust_helper_dma_fence_test_signaled_flag(struct dma_fence *f)
+{
+ return dma_fence_test_signaled_flag(f);
+}
+
+__rust_helper void rust_helper_dma_fence_lock_irqsave(struct dma_fence *f, unsigned long *flags)
+{
+ dma_fence_lock_irqsave(f, *flags);
+}
+
+__rust_helper void rust_helper_dma_fence_unlock_irqrestore(struct dma_fence *f,
+ unsigned long *flags)
+{
+ dma_fence_unlock_irqrestore(f, *flags);
+}
+
+__rust_helper void rust_helper_dma_fence_set_error(struct dma_fence *f, int error)
+{
+ dma_fence_set_error(f, error);
+}
diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c
index 9bb850294a2f..d62c7be1ca7b 100644
--- a/rust/helpers/helpers.c
+++ b/rust/helpers/helpers.c
@@ -58,6 +58,7 @@
#include "device.c"
#include "dma.c"
#ifdef CONFIG_DMA_SHARED_BUFFER
+#include "dma_fence.c"
#include "dma-resv.c"
#endif
#include "drm.c"
diff --git a/rust/helpers/pci.c b/rust/helpers/pci.c
index 502f56e02fcc..a8540f90b13c 100644
--- a/rust/helpers/pci.c
+++ b/rust/helpers/pci.c
@@ -19,6 +19,12 @@ __rust_helper resource_size_t rust_helper_pci_resource_len(struct pci_dev *pdev,
return pci_resource_len(pdev, bar);
}
+__rust_helper unsigned long rust_helper_pci_resource_flags(const struct pci_dev *pdev,
+ int bar)
+{
+ return pci_resource_flags(pdev, bar);
+}
+
__rust_helper bool rust_helper_dev_is_pci(const struct device *dev)
{
return dev_is_pci(dev);
diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs
index 2ace0428e45e..9d3de109b381 100644
--- a/rust/kernel/auxiliary.rs
+++ b/rust/kernel/auxiliary.rs
@@ -313,6 +313,10 @@ impl Device<device::Bound> {
/// `F` is the [`ForLt`](trait@ForLt) encoding of the data type. The closure receives a pinned
/// reference to the registration data.
///
+ /// The outer reference carries the `&self` lifetime while the inner type carries the HRTB
+ /// lifetime `'a`, implying `'a` outlives `&self`. This allows the closure to coerce covariant
+ /// sub-fields (e.g. `&'a T` to the caller's lifetime) and return them directly in `R`.
+ ///
/// For covariant types that implement [`trait@CovariantForLt`], prefer
/// [`registration_data`](Self::registration_data) which returns a direct reference.
///
@@ -322,13 +326,14 @@ impl Device<device::Bound> {
/// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was
/// registered by a C driver.
#[inline]
- pub fn registration_data_with<F: ForLt + 'static, R>(
- &self,
- f: impl for<'a> FnOnce(Pin<&'a F::Of<'a>>) -> R,
+ pub fn registration_data_with<'this, F: ForLt + 'static, R>(
+ &'this self,
+ f: impl for<'a> FnOnce(Pin<&'this F::Of<'a>>) -> R,
) -> Result<R> {
- // SAFETY: The HRTB closure prevents the caller from smuggling in references with a
- // concrete short lifetime, making the round-trip from `'static` sound regardless of
- // variance.
+ // SAFETY: The HRTB on the inner type prevents the caller from exploiting a specific
+ // choice of `'a`. Covariant sub-fields can be safely coerced to `'this`, while
+ // invariant fields cannot be coerced and thus cannot escape with an incorrect
+ // lifetime.
let pinned = unsafe { self.registration_data_pinned::<F>()? };
Ok(f(pinned))
diff --git a/rust/kernel/dma_buf/dma_fence.rs b/rust/kernel/dma_buf/dma_fence.rs
new file mode 100644
index 000000000000..18a43e1bb442
--- /dev/null
+++ b/rust/kernel/dma_buf/dma_fence.rs
@@ -0,0 +1,1022 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025-2026 Red Hat Inc.
+ * Author: Philipp Stanner <pstanner@redhat.com>
+ */
+
+//! DMA Fence support.
+//!
+//! Reference: <https://docs.kernel.org/driver-api/dma-buf.html#c.dma_fence>
+//!
+//! header: [`include/linux/dma-fence.h`](srctree/include/linux/dma-fence.h)
+
+use crate::{
+ alloc::AllocError,
+ bindings,
+ container_of,
+ error::to_result,
+ prelude::*,
+ types::ForeignOwnable,
+ types::Opaque, //
+};
+
+use core::{
+ marker::PhantomData,
+ mem::ManuallyDrop,
+ ops::Deref,
+ ptr,
+ ptr::{
+ drop_in_place,
+ NonNull, //
+ }, //
+};
+
+use kernel::{
+ str::CString,
+ sync::{
+ aref::{
+ ARef,
+ AlwaysRefCounted, //
+ },
+ atomic::{
+ Atomic,
+ Relaxed, //
+ },
+ rcu::rcu_barrier, //
+ }, //
+};
+
+/// VTable for dma_fence backend_ops callbacks.
+//
+// Mandatory dma_fence backend_ops are implemented implicitly through
+// [`FenceContext`]. Additional ones shall get implemented on this trait.
+pub trait FenceContextOps {
+ /// The generic payload data for [`DriverFence`]s created on this fctx.
+ type FenceDataType: Send + Sync;
+}
+
+/// A dma-fence context. A fence context takes care of associating related fences
+/// with each other, providing each with raising sequence numbers and a common
+/// identifier.
+#[pin_data(PinnedDrop)]
+pub struct FenceContext<T: FenceContextOps + Send + Sync> {
+ /// The fence context number.
+ nr: u64,
+ /// The sequence number for the next fence created.
+ seqno: Atomic<u64>,
+ // The name parameters can be accessed by the dma_fence backend_ops. UAF
+ // errors are prevented by the `call_rcu()` in `drop_driver_fence_data()`.
+ /// The name of the driver this FenceContext's fences belong to.
+ driver_name: CString,
+ /// The name of the timeline this FenceContext's fences belong to.
+ timeline_name: CString,
+ /// The number of all unsignaled fences on this context.
+ // Used to prevent bugs due to forgotten fences.
+ //
+ // The lifetime on `DriverFence`s should typically prevent this from
+ // happening.
+ //
+ // However, we cannot fully guarantee in Rust that `DriverFence`s will not
+ // be forgotten, e.g., through `core::mem::forget()`. This could circumvent
+ // the lifetime which intends to enforce that all fences disappear before
+ // their context.
+ nr_of_unsignaled_fences: Atomic<usize>,
+ /// The user's data.
+ #[pin]
+ data: T,
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> FenceContext<T> {
+ // This can later be extended as a vtable in case other parties need support
+ // for the more "exotic" callbacks.
+ const OPS: bindings::dma_fence_ops = bindings::dma_fence_ops {
+ get_driver_name: Some(Self::get_driver_name),
+ get_timeline_name: Some(Self::get_timeline_name),
+ enable_signaling: None,
+ signaled: None,
+ // Deprecated.
+ wait: None,
+ // Must never be implemented for these abstractions.
+ release: None,
+ set_deadline: None,
+ };
+
+ /// Create a new `FenceContext`.
+ pub fn new<E>(
+ initial_seqno: u64,
+ driver_name: &CStr,
+ timeline_name: &CStr,
+ data: impl PinInit<T, E>,
+ ) -> impl PinInit<Self, Error>
+ where
+ Error: From<E>,
+ {
+ let driver_name = CString::try_from(driver_name);
+ let timeline_name = CString::try_from(timeline_name);
+ try_pin_init!(Self {
+ // SAFETY: `dma_fence_context_alloc()` merely works on a global
+ // atomic. Parameter `1` is the number of contexts we want to
+ // allocate.
+ nr: unsafe { bindings::dma_fence_context_alloc(1) },
+ seqno: Atomic::new(initial_seqno),
+ driver_name: driver_name?,
+ timeline_name: timeline_name?,
+ nr_of_unsignaled_fences: Atomic::new(0),
+ data <- data,
+ })
+ }
+
+ fn next_seqno(&self) -> u64 {
+ self.seqno.fetch_add(1, Relaxed)
+ }
+
+ /// Allocate the memory for a [`DriverFence`] and already store `data` inside.
+ ///
+ /// This is needed because many times, creation of a [`DriverFence`] must not
+ /// fail, and allocating might deadlock in some situations.
+ ///
+ /// The `data` you pass here must not perform any operations that are illegal
+ /// in atomic context in its [`Drop`] implementation.
+ pub fn new_fence_allocation(
+ &self,
+ data: T::FenceDataType,
+ ) -> Result<DriverFenceAllocation<'_, T>> {
+ let fence_data = DriverFenceData {
+ rcu_head: Default::default(),
+ // `inner` remains uninitialized until a `DriverFence` takes over.
+ inner: Fence {
+ inner: Opaque::uninit(),
+ },
+ fctx: self,
+ data,
+ };
+
+ // In order to support the C dma_fence callbacks, it is necessary for
+ // a `Fence` and a `DriverFence` to live in the same allocation,
+ // because the C backend passes a dma_fence, from which the driver most
+ // likely wants to be able to access its `data` in `DriverFence`.
+ //
+ // Hence, we need the manage the memory manually. It will be freed by the
+ // C backend automatically once the refcount within `Fence` drops to 0.
+ let data = KBox::new(fence_data, GFP_KERNEL | __GFP_ZERO)?;
+
+ Ok(DriverFenceAllocation {
+ data,
+ ops: &Self::OPS,
+ })
+ }
+
+ extern "C" fn get_driver_name(ptr: *mut bindings::dma_fence) -> *const c_char {
+ // SAFETY: The C backend only invokes this callback with `ptr` pointing
+ // to a valid, unsignaled `bindings::dma_fence`. All fences created in
+ // this module always reside within `Fence` which always resides in a
+ // `DriverFenceData`, thus satisfying the function's safety
+ // requirements.
+ let fctx = unsafe { Self::from_raw_fence(ptr) };
+
+ fctx.driver_name.as_char_ptr()
+ }
+
+ extern "C" fn get_timeline_name(ptr: *mut bindings::dma_fence) -> *const c_char {
+ // SAFETY: The C backend only invokes this callback with `ptr` pointing
+ // to a valid, unsignaled `bindings::dma_fence`. All fences created in
+ // this module always reside within `Fence` which always resides in a
+ // `DriverFenceData`, thus satisfying the function's safety
+ // requirements.
+ let fctx = unsafe { Self::from_raw_fence(ptr) };
+
+ fctx.timeline_name.as_char_ptr()
+ }
+
+ /// Create a [`FenceContext`] from an associated [`bindings::dma_fence`].
+ ///
+ /// # Safety
+ ///
+ /// `ptr` must be a valid pointer to a [`bindings::dma_fence`] which resides
+ /// within a [`Fence`], which in turn resides in a [`DriverFenceData`].
+ unsafe fn from_raw_fence(ptr: *mut bindings::dma_fence) -> &'a Self {
+ let opaque_fence = Opaque::cast_from(ptr);
+
+ // SAFETY: Safe due to the function's overall safety requirements.
+ let fence_ptr = unsafe { container_of!(opaque_fence, Fence, inner) };
+
+ // CAST: `DriverFenceData` is `repr(C)` and a `Fence` is its first member.
+ let fence_data_ptr: *const DriverFenceData<'a, T> = fence_ptr.cast();
+
+ // SAFETY: Safe because of the comments directly above.
+ let fence_data = unsafe { &*fence_data_ptr };
+
+ fence_data.fctx
+ }
+}
+
+#[pinned_drop]
+impl<T: FenceContextOps + Send + Sync> PinnedDrop for FenceContext<T> {
+ fn drop(self: Pin<&mut Self>) {
+ // Fence ops callbacks can be called on unsignaled fences. Since these
+ // callbacks can access the fence context and its data, it needs to be
+ // guaranteed that a context only drops after all associated
+ // `DriverFence`s have been dropped. This is unlikely to occur, but
+ // would result in silent UAF. Throw a panic to prevent that.
+ //
+ // TODO:
+ // It would be better if the fence context signals all forgotten fences
+ // itself. To do so, it would keep a list of unsignaled fences. That
+ // list's members would have to be pre-allocated (see
+ // `FenceCallback::new_fence_allocation()`).
+ if self.nr_of_unsignaled_fences.load(Relaxed) != 0 {
+ panic!("Forgotten fences in FenceContext.");
+ }
+
+ // Ensure that the driver cannot unload while there are still dma_fence
+ // callbacks running. At the same time, the RCU barrier addresses the
+ // problem inherited by the C backend, in which backend ops callbacks
+ // might be accessing the fence while it is being signaled (or shortly
+ // after). This could cause UAF access on the fence context's
+ // `fctx.driver_name` and `fctx.timeline_name`.
+ //
+ // Wait for the RCU callbacks in `DriverFence::drop`.
+ rcu_barrier();
+ }
+}
+
+/// Error type for fence callback registration.
+///
+/// Generic over `T` so that `AlreadySignaled` can return the callback to the
+/// caller, allowing it to reclaim any resources owned by the callback (e.g.,
+/// a fence handle that needs to be signaled).
+#[derive(Debug)]
+pub enum CallbackError<T> {
+ /// The fence was already signaled. The callback is returned so the caller
+ /// can extract owned resources without losing them.
+ AlreadySignaled(T),
+ /// Some other error occurred during registration.
+ Other(Error),
+}
+
+impl<T> From<CallbackError<T>> for Error {
+ #[inline]
+ fn from(err: CallbackError<T>) -> Self {
+ match err {
+ CallbackError::AlreadySignaled(_) => ENOENT,
+ CallbackError::Other(e) => e,
+ }
+ }
+}
+
+impl<T> From<AllocError> for CallbackError<T> {
+ #[inline]
+ fn from(e: AllocError) -> Self {
+ CallbackError::Other(Error::from(e))
+ }
+}
+
+/// Trait for callbacks that can be registered on fences.
+///
+/// When the fence signals, the callback will be invoked.
+///
+/// # Example
+///
+/// ```rust
+/// use kernel::dma_buf::FenceCallback;
+///
+/// struct MyCallback {
+/// // Your callback state here
+/// }
+///
+/// impl FenceCallback for MyCallback {
+/// fn on_signal(&mut self) {
+/// pr_info!("Fence signaled!\n");
+/// // Handle fence completion
+/// }
+/// }
+/// ```
+pub trait FenceCallback: Send + 'static {
+ /// Called when the fence is signaled.
+ ///
+ /// This is called from the fence signaling path, which may be in interrupt
+ /// context or with locks held, which is why `self` is only borrowed, so that
+ /// it cannot drop. Implementations must not sleep or perform
+ /// long-running operations.
+ ///
+ /// An implementation likely wants to inform itself (e.g., through a work item)
+ /// within this callback that the associated [`FenceCallbackRegistration`]
+ /// can now be dropped.
+ fn on_signal(&mut self);
+}
+
+/// A callback registration on a fence.
+///
+/// When this object is dropped, the callback is automatically removed if it
+/// hasn't been called yet.
+#[pin_data(PinnedDrop)]
+pub struct FenceCallbackRegistration<T: FenceCallback + 'static> {
+ #[pin]
+ callback_foreign: Opaque<bindings::dma_fence_cb>,
+ callback: ManuallyDrop<T>,
+ fence: ARef<Fence>,
+}
+
+impl<T: FenceCallback> FenceCallbackRegistration<T> {
+ /// Create a [`PinInit`] closure for registering a callback on a fence.
+ ///
+ /// The actual attempt at registering the callback will take place once you
+ /// call an allocator's `pin_init()` function.
+ ///
+ /// On success the callback is pinned in place and will fire when the fence
+ /// signals. On `AlreadySignaled` the callback is returned to the caller so
+ /// that owned resources can be reclaimed.
+ pub fn new<'a>(fence: &'a Fence, callback: T) -> impl PinInit<Self, CallbackError<T>> + 'a
+ where
+ T: 'a,
+ {
+ try_pin_init!(Self {
+ // We need to fully initialize the fence because after
+ // `dma_fence_add_callback()` ran, the callback might immediately
+ // get invoked.
+ callback: ManuallyDrop::new(callback),
+ fence: ARef::from(fence),
+ callback_foreign <- Opaque::try_ffi_init(|ptr| {
+ // SAFETY: `fence.inner.get()` is a valid, initialized `struct
+ // dma_fence`. `ptr` points to the `struct dma_fence_cb` field
+ // within the pinned allocation, so it remains valid until
+ // `dma_fence_remove_callback()` in `PinnedDrop` or until the
+ // callback fires.
+ let ret = unsafe {
+ to_result(bindings::dma_fence_add_callback(
+ fence.inner.get(),
+ ptr,
+ Some(Self::dma_fence_callback),
+ ))
+ };
+ match ret {
+ Ok(()) => Ok(()),
+ Err(e) => {
+ // SAFETY: We could not register the callback. Thus,
+ // C will not use it. So we can just take it back
+ // and pass it to the user again.
+ let cb_back = unsafe { ManuallyDrop::take(callback) };
+ if e == ENOENT {
+ Err(CallbackError::AlreadySignaled(cb_back))
+ } else {
+ Err(CallbackError::Other(e))
+ }
+ },
+ }
+ }),
+ }? CallbackError<T>)
+ }
+
+ /// Raw dma fence callback that is called by the C code.
+ ///
+ /// # Safety
+ ///
+ /// This is only called by the dma_fence subsystem with valid pointers.
+ unsafe extern "C" fn dma_fence_callback(
+ _fence: *mut bindings::dma_fence,
+ callback_foreign: *mut bindings::dma_fence_cb,
+ ) {
+ let ptr = Opaque::cast_from(callback_foreign).cast_mut();
+
+ // SAFETY: All callbacks we can receive here have been created in such a way that they are
+ // embedded into a `FenceCallbackRegistration`.
+ let reg: *mut Self = unsafe { container_of!(ptr, Self, callback_foreign) };
+
+ // SAFETY: `reg` is a valid `Self` pointer.
+ //
+ // The backend ensures synchronisation so whoever holds the registration object cannot drop
+ // it while this code is running. See `FenceCallbackRegistration::drop`.
+ unsafe { (*reg).callback.on_signal() };
+ }
+
+ /// Returns a reference to the fence this callback is registered on.
+ #[inline]
+ pub fn fence(&self) -> &Fence {
+ &self.fence
+ }
+}
+
+#[pinned_drop]
+impl<T: FenceCallback> PinnedDrop for FenceCallbackRegistration<T> {
+ fn drop(self: Pin<&mut Self>) {
+ // Always call `dma_fence_remove_callback()`, even if the callback
+ // already ran. This is necessary for synchronization:
+ // `dma_fence_remove_callback()` acquires `fence->lock`, which ensures
+ // that any in-flight `dma_fence_signal()` (which calls our callback
+ // while holding the same lock) has completed before we free the struct.
+ //
+ // Without this, Drop can race with a concurrent signal:
+ // CPU0 (signal, lock held): take() -> on_signal(fence_ref) (in progress)
+ // CPU1 (drop): skips lock -> frees struct
+ // CPU0: accesses fence_ref -> use-after-free
+ //
+ // When the callback has already fired, the signal path detached the
+ // list node via `INIT_LIST_HEAD()`, so dma_fence_remove_callback just
+ // sees an empty node and returns false — the lock acquisition is the
+ // only thing that matters.
+ //
+ // SAFETY: The fence pointer is valid and the cb was initialized by
+ // `dma_fence_add_callback()` during construction.
+ unsafe {
+ bindings::dma_fence_remove_callback(self.fence.as_raw(), self.callback_foreign.get())
+ };
+
+ // SAFETY: This is literally the drop implementation, so no one has
+ // dropped this so far; so we can do it now.
+ unsafe { ManuallyDrop::<T>::drop(self.project().callback) };
+ }
+}
+
+// SAFETY: FenceCallbackRegistration can be sent between threads.
+unsafe impl<T: FenceCallback> Send for FenceCallbackRegistration<T> {}
+
+// SAFETY: &FenceCallbackRegistration can be shared between threads if &T can.
+unsafe impl<T: FenceCallback> Sync for FenceCallbackRegistration<T> where T: Sync {}
+
+/// The receiving counterpart of a [`DriverFence`].
+///
+/// The Rust DMA fence implementation has a dualistic design: [`DriverFence`]s
+/// are the producer-side, intended to be always owned by only one party. That
+/// party has the monopoly on signaling the fence.
+///
+/// A [`Fence`] is the counterpart for consumers. Thus, [`Fence`]s are always
+/// refcounted and can shared with an arbitrary number of parties, including
+/// userspace. A [`Fence`] can only be used for actions such as checking the
+/// fence's status or for registering callbacks on it.
+///
+/// Once the associated [`DriverFence`] signals, all
+/// [`FenceCallbackRegistration`]s registered on the [`Fence`] will be executed.
+///
+/// A [`Fence`] can arbitrarily outlive its [`DriverFence`] and the
+/// [`FenceContext`]. Signaling a [`DriverFence`] decouples it from its
+/// [`Fence`]s.
+#[repr(transparent)]
+pub struct Fence {
+ /// The actual dma_fence passed to C.
+ inner: Opaque<bindings::dma_fence>,
+}
+
+/// Guard helper for locking within this module.
+///
+/// Its only purpose for now is to avoid a number of unsafe lock-unlock cycles.
+/// It is never used outside of this module.
+// TODO: This should be made more canonical, probably by basing it on a
+// SpinLockIrqGuard once available.
+struct FenceGuard<'a> {
+ inner: &'a Fence,
+ flags: usize,
+}
+
+impl<'a> Deref for FenceGuard<'a> {
+ type Target = &'a Fence;
+
+ fn deref(&self) -> &Self::Target {
+ &self.inner
+ }
+}
+
+impl Drop for FenceGuard<'_> {
+ fn drop(&mut self) {
+ // SAFETY: `fence` is valid because `self` is valid. `flag_ptr` is
+ // merely a pointer to an integer, which lives as long as this function.
+ // When a `FenceGuard` exists, the lock has been taken by definition.
+ unsafe { bindings::dma_fence_unlock_irqrestore(self.as_raw(), &raw mut self.flags) };
+ }
+}
+
+// SAFETY: Fences are literally designed to be shared between threads.
+unsafe impl Send for Fence {}
+// SAFETY: Fences are literally designed to be shared between threads.
+unsafe impl Sync for Fence {}
+
+impl Fence {
+ /// Check whether the fence was signaled at the moment of the function call.
+ ///
+ /// Note that this can return `true` for a [`Fence`] whose [`DriverFence`]
+ /// has not yet been dropped. The reason is that the fence ops callbacks can
+ /// cause the fence to get signaled by the C backend.
+ #[inline]
+ pub fn is_signaled(&self) -> bool {
+ // We should not use `dma_fence_is_signaled_locked()` here, because
+ // according to the C backend's recommendations, that function is
+ // problematic and we should avoid calling that function with a lock
+ // held.
+
+ // SAFETY: Inner `fence` is valid because `self` is valid.
+ let ret = unsafe { bindings::dma_fence_is_signaled(self.as_raw()) };
+
+ // To be as robust as possible for the future we guarantee that an API
+ // caller can 100% rely on the signaling being completed (i.e., all
+ // fence callbacks ran), so we have to take the lock.
+ //
+ // The reason is that the C dma_fence backend currently does not
+ // carefully synchronize the `dma_fence_is_signaled()` function with the
+ // proper spinlock. This can lead to the function returning `true` while
+ // fence callbacks are still being executed. This can be mitigated by
+ // guarding the entire function with the spinlock.
+ //
+ // The fundamental reason is that the C backend currently does guard
+ // setting of the fence's signaled-bit with the fence's spinlock, but
+ // reading is done locklessly.
+ //
+ // See commit c8a5d5ea3ba6a.
+ let _ = self.lock();
+
+ ret
+ }
+
+ /// Lock the fence. A helper only to be used internally in this module.
+ fn lock(&self) -> FenceGuard<'_> {
+ let mut guard = FenceGuard {
+ inner: self,
+ flags: 0,
+ };
+
+ // SAFETY: `fence` is valid because `self` is valid. `flag_ptr` is
+ // merely a pointer to an integer, whose lifetime is tied to the guard
+ // object.
+ unsafe { bindings::dma_fence_lock_irqsave(self.as_raw(), &raw mut guard.flags) };
+
+ guard
+ }
+
+ /// Get the fence's sequence number.
+ #[inline]
+ pub fn seqno(&self) -> u64 {
+ // SAFETY: Valid because `self` is valid.
+ unsafe { (*self.as_raw()).seqno }
+ }
+
+ fn as_raw(&self) -> *mut bindings::dma_fence {
+ self.inner.get()
+ }
+
+ /// Create a [`Fence`] from a raw C [`bindings::dma_fence`].
+ ///
+ /// # Safety
+ ///
+ /// `ptr` must point to an initialized fence that is embedded into a [`Fence`].
+ #[inline]
+ pub unsafe fn from_raw<'a>(ptr: *mut bindings::dma_fence) -> &'a Self {
+ // SAFETY: Safe as per the function's overall safety requirements.
+ unsafe { &*ptr.cast() }
+ }
+}
+
+// SAFETY: These implement the C backends refcounting methods which are proven
+// to work correctly.
+unsafe impl AlwaysRefCounted for Fence {
+ fn inc_ref(&self) {
+ // SAFETY: `self.as_raw()` is a pointer to a valid `struct dma_fence`.
+ unsafe { bindings::dma_fence_get(self.as_raw()) }
+ }
+
+ unsafe fn dec_ref(ptr: NonNull<Self>) {
+ // SAFETY: `ptr` is never a NULL pointer; and when `dec_ref()` is called
+ // the fence is by definition still valid.
+ let fence = unsafe { (*ptr.as_ptr()).inner.get() };
+
+ // SAFETY: `fence` was created validly above. When `dec_ref()` is called,
+ // there is by definition still a reference alive that can be put.
+ unsafe { bindings::dma_fence_put(fence) }
+ }
+}
+
+// Necessary to guarantee that `inner` always comes first and can be freed by C.
+// Also useful for using casts instead of container_of().
+#[repr(C)]
+#[pin_data]
+struct DriverFenceData<'a, T: Send + Sync + FenceContextOps> {
+ #[pin]
+ /// The inner fence.
+ // Must always be the first member so that unsafe casting works; but also
+ // necessary so that the C backend can free the allocation (coming from our
+ // Rust code) with kfree_rcu().
+ inner: Fence,
+ /// Callback head for dropping this in a deferred manner through RCU.
+ rcu_head: bindings::callback_head,
+ /// Reference to access the FenceContext.
+ fctx: &'a FenceContext<T>,
+ /// The API user's data. It is essential that the data only performs
+ /// operations legal in atomic context in its [`Drop`] implementation.
+ #[pin]
+ data: T::FenceDataType,
+}
+
+/// A synchronization primitive mainly for GPU drivers.
+///
+/// The Rust DMA fence implementation has a dualistic design: [`DriverFence`]s
+/// are the producer-side, intended to be always owned by only one party. That
+/// party has the monopoly on signaling the fence.
+///
+/// A [`Fence`] is the counterpart for consumers. Thus, [`Fence`]s are always
+/// refcounted and can be shared with an arbitrary number of parties, including
+/// userspace. A [`Fence`] can only be used for actions such as checking the
+/// fence's status or for registering callbacks on it.
+///
+/// Once the associated [`DriverFence`] signals, all
+/// [`FenceCallbackRegistration`]s registered on a [`Fence`] will be executed.
+///
+/// A [`Fence`] can arbitrarily outlive its [`DriverFence`] and the
+/// [`FenceContext`]. Signaling a [`DriverFence`] decouples it from its
+/// [`Fence`]s.
+///
+/// It is crucial that a [`DriverFence`] always correctly represents the state
+/// of the associated job on the hardware. Especially, it is strictly necessary
+/// that the owner ensures that all [`DriverFence`]s eventually get signaled.
+/// As a last resort, a [`DriverFence`] will signal itself if it drops
+/// unsignaled and print a warning.
+///
+/// This design intends to implement the [`bindings::dma_fence_ops`] in such a
+/// way that the driver-data necessary to implement the callback's functionality
+/// resides in the [`FenceContext`]. Thus, a [`DriverFence`] contains a
+/// reference to the context, which can be accessed in the callbacks. The
+/// implementation, therefore, ensures that a [`DriverFence`] cannot outlive its
+/// [`FenceContext`]. Unfortunately, this can be circumvented under certain
+/// circumstances in Rust (e.g., usage of [`core::mem::forget`]).
+///
+/// In the unlikely case of such violations, a panic is thrown.
+///
+/// # Examples
+///
+/// ```
+/// use kernel::{
+/// dma_buf::{
+/// DriverFence,
+/// FenceContext,
+/// FenceContextOps,
+/// FenceCallback,
+/// FenceCallbackRegistration,
+/// },
+/// str::CString,
+/// sync::aref::ARef, //
+/// };
+/// use core::fmt::Display;
+///
+/// struct CallbackData { }
+///
+/// impl FenceCallback for CallbackData {
+/// fn on_signal(&mut self) {
+/// pr_info!("DmaFence callback executed.\n");
+/// }
+/// }
+///
+/// #[pin_data]
+/// struct FenceContextData {}
+///
+/// impl FenceContextData {
+/// fn new() -> impl PinInit<Self> {
+/// pin_init!(Self {})
+/// }
+/// }
+///
+/// impl FenceContextOps for FenceContextData {
+/// type FenceDataType = FenceData;
+/// }
+///
+/// let fctx_data = FenceContextData::new();
+///
+///
+/// let mut fctx = KBox::pin_init(
+/// FenceContext::new(0, c"dummy_driver", c"dummy_timeline", fctx_data),
+/// GFP_KERNEL
+/// )?;
+///
+/// struct FenceData {
+/// data: CString,
+/// }
+///
+/// let fence_data = FenceData { data: c"dummy_data".try_into()? };
+///
+/// let fence_alloc = fctx.new_fence_allocation(fence_data)?;
+/// let mut fence = fence_alloc.new_fence();
+///
+/// let cb_data = CallbackData { };
+/// let waiting_fence = ARef::from(fence.as_fence());
+/// let cb_reg = FenceCallbackRegistration::new(&waiting_fence, cb_data);
+/// let cb_reg = KBox::pin_init(cb_reg, GFP_KERNEL)?;
+///
+/// // TODO signalling guards
+/// assert_eq!(waiting_fence.is_signaled(), false);
+/// fence.signal(Ok(()));
+/// assert_eq!(waiting_fence.is_signaled(), true);
+///
+/// Ok::<(), Error>(())
+/// ```
+pub struct DriverFence<'a, T: Send + Sync + FenceContextOps> {
+ /// The actual content of the fence. Lives in a [`NonNull`] so that its
+ /// memory can be managed independently. Valid until both the [`DriverFence`]
+ /// and all associated [`Fence`]s have disappeared.
+ data: NonNull<DriverFenceData<'a, T>>,
+}
+
+/// A pre-prepared DMA fence, carrying the user's data and the memory it and the
+/// fence reside in. Only useful for creating a [`DriverFence`]. Splitting
+/// allocation and full initialization is necessary because fences cannot be
+/// allocated dynamically in some circumstances (deadlock).
+pub struct DriverFenceAllocation<'a, T: Send + Sync + FenceContextOps> {
+ /// The memory for the actual content of the fence.
+ /// Handed over to a [`DriverFence`], or deallocated once the
+ /// [`DriverFenceAllocation`] drops.
+ data: KBox<DriverFenceData<'a, T>>,
+ /// Reference for the ops for the associated [`FenceContext`]
+ ops: &'static bindings::dma_fence_ops,
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> DriverFenceAllocation<'a, T> {
+ /// Create a new [`DriverFence`], the signalable counterpart of a [`Fence`].
+ ///
+ /// This increments the sequence number in the associated [`FenceContext`].
+ pub fn new_fence(self) -> DriverFence<'a, T> {
+ // We feed the C dma_fence backend a NULL for the spinlock so that it
+ // uses per-fence locks automatically.
+ let null_ptr: *mut bindings::spinlock = ptr::null_mut();
+ let seqno = self.data.fctx.next_seqno();
+ let fence_ptr = self.as_raw();
+ // SAFETY: `fence_ptr` has been created directly above. It will live
+ // at least as long as `Self`. The same applies to `&Self::OPS`.
+ unsafe {
+ bindings::dma_fence_init(fence_ptr, self.ops, null_ptr, self.data.fctx.nr, seqno)
+ };
+
+ self.data.fctx.nr_of_unsignaled_fences.fetch_add(1, Relaxed);
+
+ // A `DriverFenceAllocation`'s purpose is to carry allocated memory, so
+ // that `DriverFence`s can always be created without allocating. In this
+ // method, ownership over that memory is transferred to the new
+ // `DriverFence` and managed through refcounting. The C dma_fence
+ // backend will ultimately free the memory once the refcount reaches 0.
+ let ptr = KBox::into_raw(self.data);
+ // SAFETY: `ptr` was just created validly directly above.
+ let ptr = unsafe { NonNull::new_unchecked(ptr) };
+
+ DriverFence { data: ptr }
+ }
+
+ fn as_raw(&self) -> *mut bindings::dma_fence {
+ self.data.inner.inner.get()
+ }
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> DriverFence<'a, T> {
+ fn as_raw(&self) -> *mut bindings::dma_fence {
+ // SAFETY: Valid because `self` is valid.
+ let fence_data = unsafe { &*self.data.as_ptr() };
+
+ fence_data.inner.inner.get()
+ }
+
+ /// Create a [`DriverFence`] from a raw pointer to a [`bindings::dma_fence`].
+ ///
+ /// # Safety
+ ///
+ /// `ptr` must be a valid pointer to a `dma_fence` that was obtained through
+ /// a [`DriverFence`] with matching generic data for both fence and associated
+ /// [`FenceContext`].
+ unsafe fn from_raw(ptr: *mut bindings::dma_fence) -> Self {
+ let opaque_fence = Opaque::cast_from(ptr);
+
+ // SAFETY: Safe due to the function's overall safety requirements.
+ let fence_ptr = unsafe { container_of!(opaque_fence, Fence, inner) };
+
+ // DriverFenceData is `repr(C)` and a Fence is its first member.
+ let fence_data_ptr = fence_ptr as *mut DriverFenceData<'a, T>;
+
+ // SAFETY: `fence_data_ptr` was created validly above.
+ let data = unsafe { NonNull::new_unchecked(fence_data_ptr) };
+
+ Self { data }
+ }
+
+ /// Return the underlying [`Fence`].
+ #[inline]
+ pub fn as_fence(&self) -> &Fence {
+ // SAFETY: `self` is by definition still valid, and it cannot drop until
+ // this new reference is gone.
+ unsafe { Fence::from_raw(self.as_raw()) }
+ }
+
+ /// Signal the fence. This will invoke all registered callbacks.
+ pub fn signal(self, res: Result) {
+ let fence = self.as_fence().lock();
+
+ // SAFETY: `fence` is valid because `self` is valid. The lock must be
+ // held, which we acquired directly above.
+ if !unsafe { bindings::dma_fence_test_signaled_flag(fence.as_raw()) } {
+ if let Err(err) = res {
+ // SAFETY: `fence` is valid because `self` is valid. The fence
+ // must not have been signaled yet, which we check directly above.
+ unsafe { bindings::dma_fence_set_error(fence.as_raw(), err.to_errno()) };
+ }
+ // SAFETY: `fence` is valid because `self` is valid. The lock must
+ // be held, which we acquired above.
+ unsafe { bindings::dma_fence_signal_locked(fence.as_raw()) };
+ }
+
+ // SAFETY: `self.data` is valid because `self` is valid.
+ let fctx = unsafe { self.data.as_ref().fctx };
+ let _ = fctx.nr_of_unsignaled_fences.fetch_sub(1, Relaxed);
+ }
+}
+
+// SAFETY: Fences are literally designed to be shared between threads.
+unsafe impl<'a, T: Send + Sync + FenceContextOps> Send for DriverFence<'a, T> {}
+// SAFETY: Fences are literally designed to be shared between threads.
+unsafe impl<'a, T: Send + Sync + FenceContextOps> Sync for DriverFence<'a, T> {}
+
+impl<'a, T: Send + Sync + FenceContextOps> Deref for DriverFence<'a, T> {
+ type Target = T::FenceDataType;
+
+ fn deref(&self) -> &Self::Target {
+ // SAFETY: Thanks to refcounting, `data` is always valid as long as `self` is.
+ let data = unsafe { &*self.data.as_ptr() };
+
+ &data.data
+ }
+}
+
+/// A borrow wrapper for [`DriverFence`]. Implements [`Deref`].
+pub struct DriverFenceBorrow<'a, T: Send + Sync + FenceContextOps> {
+ driver_fence: ManuallyDrop<DriverFence<'a, T>>,
+ _lifetime: PhantomData<&'a T>,
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> Deref for DriverFenceBorrow<'a, T> {
+ type Target = DriverFence<'a, T>;
+
+ fn deref(&self) -> &Self::Target {
+ self.driver_fence.deref()
+ }
+}
+
+// SAFETY: The Rust dma_fence abstractions are already designed around the inner
+// C `dma_fence`, which can serve safely as the identification point when being
+// owned by C. Moreover, safety is ensured by not dropping `DriverFence` and by
+// only allowing operations without side effects on the Borrowed type.
+unsafe impl<T: Send + Sync + FenceContextOps> ForeignOwnable for DriverFence<'_, T> {
+ type Borrowed<'a>
+ = DriverFenceBorrow<'a, T>
+ where
+ Self: 'a;
+ type BorrowedMut<'a>
+ = DriverFenceBorrow<'a, T>
+ where
+ Self: 'a;
+
+ const FOREIGN_ALIGN: usize = core::mem::align_of::<bindings::dma_fence>();
+
+ fn into_foreign(self) -> *mut c_void {
+ let fence = self;
+
+ let ptr = fence.as_raw();
+
+ // DriverFence must not drop.
+ let _ = ManuallyDrop::new(fence);
+
+ ptr.cast()
+ }
+
+ unsafe fn from_foreign(ptr: *mut c_void) -> Self {
+ // SAFETY: Safe because the trait implementation only invokes this with
+ // a valid `ptr`, associated to a `DriverFence` with matching generic data.
+ unsafe { Self::from_raw(ptr.cast()) }
+ }
+
+ unsafe fn borrow<'a>(ptr: *mut c_void) -> Self::Borrowed<'a>
+ where
+ Self: 'a,
+ {
+ // SAFETY: The trait implementation ensures that `ptr` always resides
+ // within a [`Fence`] within a [`DriverFenceData`].
+ let driver_fence = unsafe { Self::from_raw(ptr.cast()) };
+
+ let driver_fence = ManuallyDrop::new(driver_fence);
+
+ DriverFenceBorrow {
+ driver_fence,
+ _lifetime: PhantomData,
+ }
+ }
+
+ unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> Self::BorrowedMut<'a>
+ // FIXME: The bound below and the one above in `borrow` should actually be
+ // unnecessary since the compiler should be able to completely derive all
+ // necessary information automatically. There is currently a compiler bug
+ // preventing that, though:
+ //
+ // https://github.com/rust-lang/rust/issues/155430.
+ //
+ // (Help to) fix the compiler bug and remove the bounds afterwards.
+ where
+ Self: 'a,
+ {
+ // SAFETY: The trait implementation ensures that `ptr` always resides
+ // within a [`Fence`] within a [`DriverFenceData`].
+ let driver_fence = unsafe { Self::from_raw(ptr.cast()) };
+
+ let driver_fence = ManuallyDrop::new(driver_fence);
+
+ DriverFenceBorrow {
+ driver_fence,
+ _lifetime: PhantomData,
+ }
+ }
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> Drop for DriverFence<'a, T> {
+ fn drop(&mut self) {
+ let guard = self.as_fence().lock();
+
+ // Use dma_fence_test_signaled_flag() instead of
+ // dma_fence_is_signaled_locked() because the C backend wants to get rid
+ // of the latter.
+
+ // SAFETY: `guard` is valid until the `call_rcu()` below.
+ let signaled: bool = unsafe { bindings::dma_fence_test_signaled_flag(guard.as_raw()) };
+ if !signaled {
+ pr_err!("DriverFence drops unsignaled. Danger of memory corruption!\n");
+ // SAFETY: `guard` is valid until the `call_rcu()` below. The fence
+ // must not have been signaled yet, which we check directly above.
+ unsafe { bindings::dma_fence_set_error(guard.as_raw(), ECANCELED.to_errno()) };
+ // SAFETY: `guard` is valid until the `call_rcu()` below. The lock
+ // must be held, which we acquired above.
+ unsafe { bindings::dma_fence_signal_locked(guard.as_raw()) };
+
+ // SAFETY: `self.data` is valid because `self` is valid.
+ let fctx = unsafe { self.data.as_ref().fctx };
+ let _ = fctx.nr_of_unsignaled_fences.fetch_sub(1, Relaxed);
+ }
+ drop(guard);
+
+ // `DriverFenceData` could be accessed through some dma_fence
+ // callbacks right now. Access is being revoked in principle above by
+ // signaling the fence, but since the C backend does not guarantee
+ // perfect full synchronization, we have to wait for one grace period to
+ // ensure that all accessors of `DriverFenceData` (through the
+ // dma_fence_ops accessible through a `Fence`) are gone.
+
+ if !core::mem::needs_drop::<T::FenceDataType>() {
+ // SAFETY: Once a `DriverFence` is initialized, the inner `fence` is
+ // valid and initialized. It is valid until the refcount drops to 0,
+ // which can earliest happen once we drop the `DriverFence`'s
+ // reference here.
+ unsafe { bindings::dma_fence_put(self.as_raw()) };
+ return;
+ }
+
+ // SAFETY: Valid because `self` is valid.
+ let rcu_head_ptr = unsafe { &raw mut (*self.data.as_ptr()).rcu_head };
+
+ // SAFETY: `call_rcu()` is always safe to be called. `rcu_head_ptr` was
+ // created validly above. The module must perform a `synchronize_rcu()`
+ // or `rcu_barrier()` call to guard against module unload.
+ unsafe { bindings::call_rcu(rcu_head_ptr, Some(drop_driver_fence_data::<T>)) };
+ }
+}
+
+// TODO:
+// The entire call_rcu() mechanism in the drop above and the code below would be
+// unnecessary if C's dma_fence_signal() could be reworked in a way that after it
+// ran, the caller knows that no fence_ops callbacks can be running anymore.
+// In other words, if the dma_fence backend would use its spinlock for full
+// synchronization.
+//
+// Then we could move the drop_in_place() and dma_fence_put() upwards into the
+// drop() implementation and call it a day.
+
+/// Finally really drop this `DriverFence<T>`
+///
+/// # Safety
+///
+/// `head` references the `rcu_head` field of an `DriverFenceData<T>`. All
+/// accessors to that `DriverFenceData<T>` must be gone by now. This must be
+/// ensured by signalling the associated `DriverFence<T>` and then waiting
+/// for a grace period until calling this function here.
+unsafe extern "C" fn drop_driver_fence_data<T: Send + Sync + FenceContextOps>(
+ head: *mut bindings::callback_head,
+) {
+ // SAFETY: Caller provides a pointer to the `rcu_head` field of a `DriverFenceData<C>`.
+ let fence_data = unsafe { container_of!(head, DriverFenceData<'_, T>, rcu_head) };
+
+ // SAFETY: `fence_data` was created validly above. All the fence's data will
+ // only drop below, but the raw pointer to the raw C `dma_fence` remains
+ // valid because the reference count is only decremented at the end of the
+ // function.
+ let fence = unsafe { (*fence_data).inner.inner.get() };
+
+ // SAFETY: `fence_data` was created validly above. The user has already
+ // dropped the only conventional accessor to the user data, the `DriverFence`,
+ // one grace period ago. All accessors are gone now.
+ unsafe { drop_in_place(&raw mut (*fence_data).data) };
+
+ // The inner `Fence` explicitly does not get dropped because there may be
+ // many more users / consumers, each holding their own reference.
+
+ // SAFETY: Once a `DriverFence` is initialized, the inner `fence` is valid
+ // and initialized. It is valid until the refcount drops to 0, which can
+ // earliest happen once we drop the `DriverFence`'s reference here.
+ unsafe { bindings::dma_fence_put(fence) };
+
+ // The actual memory the data associated with a `DriverFence` lives in
+ // gets freed by the C dma_fence backend once the fence's refcount reaches 0.
+}
diff --git a/rust/kernel/dma_buf/mod.rs b/rust/kernel/dma_buf/mod.rs
new file mode 100644
index 000000000000..4764a828642e
--- /dev/null
+++ b/rust/kernel/dma_buf/mod.rs
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+//! DMA-buf subsystem abstractions.
+
+pub mod dma_fence;
+
+pub use self::dma_fence::{
+ DriverFence,
+ Fence,
+ FenceCallback,
+ FenceCallbackRegistration,
+ FenceContext,
+ FenceContextOps, //
+};
diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs
index 74f6ed690d8b..f6db91e81e79 100644
--- a/rust/kernel/drm/driver.rs
+++ b/rust/kernel/drm/driver.rs
@@ -12,7 +12,10 @@ use crate::{
prelude::*,
sync::aref::ARef, //
};
-use core::ptr::NonNull;
+use core::{
+ marker::PhantomPinned,
+ ptr::NonNull, //
+};
/// Driver use the GEM memory manager. This should be set for all modern drivers.
pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM;
@@ -143,9 +146,15 @@ pub trait Driver {
/// The registration type of a `drm::Device`.
///
/// Once the `Registration` structure is dropped, the device is unregistered.
+#[pin_data(PinnedDrop)]
pub struct Registration<'a, T: Driver> {
drm: ARef<drm::Device<T>>,
- _reg_data: Pin<KBox<T::RegistrationData<'a>>>,
+ #[pin]
+ data: T::RegistrationData<'a>,
+ /// Even if `T::RegistrationData` is `Unpin`, the registration must not be moved because a
+ /// pointer to `data` is stored in the DRM device.
+ #[pin]
+ _pin: PhantomPinned,
}
impl<'a, T: Driver> Registration<'a, T> {
@@ -159,39 +168,40 @@ impl<'a, T: Driver> Registration<'a, T> {
pub unsafe fn new<E>(
dev: &'a device::Device<device::Bound>,
drm: drm::UnregisteredDevice<T>,
- reg_data: impl PinInit<T::RegistrationData<'a>, E>,
- flags: usize,
- ) -> Result<Self>
+ data: impl PinInit<T::RegistrationData<'a>, E>,
+ ) -> impl PinInit<Self, Error>
where
Error: From<E>,
{
- let parent = drm.as_ref();
- if parent.as_ref().as_raw() != dev.as_raw() {
- return Err(EINVAL);
- }
-
- let reg_data: Pin<KBox<T::RegistrationData<'a>>> = KBox::pin_init(reg_data, GFP_KERNEL)?;
-
- // Store the registration data pointer in the device before registration, so that it is
- // visible once ioctls can be called.
- let ptr: NonNull<T::RegistrationData<'static>> =
- NonNull::from(Pin::get_ref(reg_data.as_ref())).cast();
-
- // SAFETY: No concurrent access; the device is not yet registered.
- unsafe { *drm.registration_data.get() = ptr };
-
- // SAFETY: `drm` is a valid, initialized but not yet registered DRM device.
- let ret = unsafe { bindings::drm_dev_register(drm.as_raw(), flags) };
- if let Err(e) = to_result(ret) {
- // SAFETY: `drm_dev_register()` synchronizes SRCU on failure, so no concurrent
- // access to `registration_data` is possible at this point.
- unsafe { *drm.registration_data.get() = NonNull::dangling() };
- return Err(e);
- }
-
- Ok(Self {
+ try_pin_init!(Self {
+ _: {
+ let parent = drm.as_ref();
+ if parent.as_ref().as_raw() != dev.as_raw() {
+ return Err(EINVAL);
+ }
+ },
+
+ data <- data,
drm: (&*drm).into(),
- _reg_data: reg_data,
+ _pin: PhantomPinned,
+
+ _: {
+ // Store the registration data pointer in the device before registration, so
+ // that it is visible once ioctls can be called.
+ let ptr: NonNull<T::RegistrationData<'static>> =
+ NonNull::from(data.as_ref().get_ref()).cast();
+
+ // SAFETY: No concurrent access; the device is not yet registered.
+ unsafe { *drm.registration_data.get() = ptr };
+
+ // SAFETY: `drm` is a valid, initialized but not yet registered DRM device.
+ let ret = unsafe { bindings::drm_dev_register(drm.as_raw(), 0) };
+ to_result(ret).inspect_err(|_| {
+ // SAFETY: `drm_dev_register()` synchronizes SRCU on failure, so no
+ // concurrent access to `registration_data` is possible at this point.
+ unsafe { *drm.registration_data.get() = NonNull::dangling() };
+ })?;
+ },
})
}
@@ -208,8 +218,9 @@ unsafe impl<T: Driver> Sync for Registration<'_, T> {}
// SAFETY: Registration with and unregistration from the DRM subsystem can happen from any thread.
unsafe impl<T: Driver> Send for Registration<'_, T> {}
-impl<T: Driver> Drop for Registration<'_, T> {
- fn drop(&mut self) {
+#[pinned_drop]
+impl<T: Driver> PinnedDrop for Registration<'_, T> {
+ fn drop(self: Pin<&mut Self>) {
// Use `drm_dev_unplug` rather than `drm_dev_unregister` to ensure that existing
// `drm_dev_enter()` critical sections complete before unregistration proceeds. This
// is required for the safety of `RegistrationGuard`, which relies on the SRCU barrier in
diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs
index e1ebad77ebe2..85657a39e146 100644
--- a/rust/kernel/drm/gem/mod.rs
+++ b/rust/kernel/drm/gem/mod.rs
@@ -266,18 +266,7 @@ impl<T: DriverObject, Ctx: DeviceContext> Object<T, Ctx> {
free: Some(Self::free_callback),
open: Some(open_callback::<T>),
close: Some(close_callback::<T>),
- print_info: None,
- export: None,
- pin: None,
- unpin: None,
- get_sg_table: None,
- vmap: None,
- vunmap: None,
- mmap: None,
- status: None,
- vm_ops: core::ptr::null_mut(),
- evict: None,
- rss: None,
+ ..pin_init::zeroed()
};
/// Returns the `Device` that owns this GEM object.
diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs
index a687d46d170d..24cb5343e0db 100644
--- a/rust/kernel/drm/gem/shmem.rs
+++ b/rust/kernel/drm/gem/shmem.rs
@@ -132,19 +132,16 @@ impl<T: DriverObject> Object<T> {
open: Some(super::open_callback::<T>),
close: Some(super::close_callback::<T>),
print_info: Some(bindings::drm_gem_shmem_object_print_info),
- export: None,
pin: Some(bindings::drm_gem_shmem_object_pin),
unpin: Some(bindings::drm_gem_shmem_object_unpin),
get_sg_table: Some(bindings::drm_gem_shmem_object_get_sg_table),
vmap: Some(bindings::drm_gem_shmem_object_vmap),
vunmap: Some(bindings::drm_gem_shmem_object_vunmap),
mmap: Some(bindings::drm_gem_shmem_object_mmap),
- status: None,
- rss: None,
#[allow(unused_unsafe, reason = "Safe since Rust 1.82.0")]
// SAFETY: `drm_gem_shmem_vm_ops` is a valid, static const on the C side.
vm_ops: unsafe { &raw const bindings::drm_gem_shmem_vm_ops },
- evict: None,
+ ..pin_init::zeroed()
};
/// Return a raw pointer to the embedded drm_gem_shmem_object.
diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs
index 9d52ad9a2027..a877df11a03e 100644
--- a/rust/kernel/io.rs
+++ b/rust/kernel/io.rs
@@ -511,6 +511,45 @@ pub trait Io<'a>: IoBase<'a> {
self.len() == 0
}
+ /// Convert into a different typed I/O view.
+ ///
+ /// The target type must be known (statically) to be of the same or smaller size to current
+ /// type, and the current view must be properly aligned for the target type.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// use kernel::io::{
+ /// io_project,
+ /// Mmio,
+ /// Io,
+ /// Region,
+ /// };
+ /// #[derive(FromBytes, IntoBytes)]
+ /// #[repr(C)]
+ /// struct MyStruct { field: u32, }
+ ///
+ /// # fn test(mmio: &Mmio<'_, Region<0x1000>>) {
+ /// // let mmio: Mmio<'_, Region<0x1000>>;
+ /// let whole: Mmio<'_, MyStruct> = mmio.cast();
+ /// # }
+ /// ```
+ #[inline]
+ fn cast<U>(self) -> <Self::Backend as IoBackend>::View<'a, U>
+ where
+ Self::Target: FromBytes + IntoBytes,
+ U: FromBytes + IntoBytes,
+ {
+ let view = self.as_view();
+ let ptr = Self::Backend::as_ptr(view);
+
+ const_assert!(size_of::<U>() <= Self::Target::MIN_SIZE);
+ const_assert!(align_of::<U>() <= Self::Target::MIN_ALIGN.as_usize());
+
+ // SAFETY: We have checked bounds and alignment, so this is a valid projection.
+ unsafe { Self::Backend::project_view(view, ptr.cast()) }
+ }
+
/// Try to convert into a different typed I/O view.
///
/// A runtime check is performed to ensure that the target type is of same or smaller size to
diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs
index a33a416b289a..758ce62b3cce 100644
--- a/rust/kernel/io/resource.rs
+++ b/rust/kernel/io/resource.rs
@@ -226,10 +226,18 @@ impl Flags {
/// Resource represents a memory region that must be ioremaped using `ioremap_np`.
pub const IORESOURCE_MEM_NONPOSTED: Flags = Flags::new(bindings::IORESOURCE_MEM_NONPOSTED);
+ /// Memory region uses a 64-bit address (consumes two consecutive PCI resource slots).
+ pub const IORESOURCE_MEM_64: Flags = Flags::new(bindings::IORESOURCE_MEM_64);
+
// Always inline to optimize out error path of `build_assert`.
#[inline(always)]
const fn new(value: u32) -> Self {
build_assert!(value as u64 <= c_ulong::MAX as u64);
Flags(value as c_ulong)
}
+
+ /// Wrap a raw `c_ulong` value returned by a C API into [`Flags`].
+ pub(crate) const fn from_raw(value: c_ulong) -> Self {
+ Flags(value)
+ }
}
diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
index a4747f75b163..1ab41129d557 100644
--- a/rust/kernel/lib.rs
+++ b/rust/kernel/lib.rs
@@ -67,6 +67,8 @@ pub mod device;
pub mod device_id;
pub mod devres;
pub mod dma;
+#[cfg(CONFIG_DMA_SHARED_BUFFER)]
+pub mod dma_buf;
pub mod driver;
#[cfg(CONFIG_DRM = "y")]
pub mod drm;
diff --git a/rust/kernel/maple_tree.rs b/rust/kernel/maple_tree.rs
index 265d6396a78a..7abe41228cb7 100644
--- a/rust/kernel/maple_tree.rs
+++ b/rust/kernel/maple_tree.rs
@@ -16,7 +16,11 @@ use kernel::{
alloc::Flags,
error::to_result,
prelude::*,
- types::{ForeignOwnable, Opaque},
+ types::{
+ ForeignOwnable,
+ NotThreadSafe,
+ Opaque, //
+ },
};
/// A maple tree optimized for storing non-overlapping ranges.
@@ -240,7 +244,10 @@ impl<T: ForeignOwnable> MapleTree<T> {
unsafe { bindings::spin_lock(self.ma_lock()) };
// INVARIANT: We just took the spinlock.
- MapleGuard(self)
+ MapleGuard {
+ tree: self,
+ _not_send: NotThreadSafe,
+ }
}
#[inline]
@@ -302,19 +309,30 @@ impl<T: ForeignOwnable> PinnedDrop for MapleTree<T> {
}
}
+// SAFETY: `MapleTree<T>` is `Send` if `T` is `Send` because `MapleTree` owns its elements.
+unsafe impl<T: ForeignOwnable + Send> Send for MapleTree<T> {}
+
+// SAFETY: `&MapleTree<T>` allows inserting and erasing entries from any thread, so `T: Send` is
+// required, and shared borrows of entries require `T: Sync`.
+unsafe impl<T: ForeignOwnable + Send + Sync> Sync for MapleTree<T> {}
+
/// A reference to a [`MapleTree`] that owns the inner lock.
///
/// # Invariants
///
/// This guard owns the inner spinlock.
#[must_use = "if unused, the lock will be immediately unlocked"]
-pub struct MapleGuard<'tree, T: ForeignOwnable>(&'tree MapleTree<T>);
+pub struct MapleGuard<'tree, T: ForeignOwnable> {
+ tree: &'tree MapleTree<T>,
+ // A held spinlock must be released on the same CPU that acquired it.
+ _not_send: NotThreadSafe,
+}
impl<'tree, T: ForeignOwnable> Drop for MapleGuard<'tree, T> {
#[inline]
fn drop(&mut self) {
// SAFETY: By the type invariants, we hold this spinlock.
- unsafe { bindings::spin_unlock(self.0.ma_lock()) };
+ unsafe { bindings::spin_unlock(self.tree.ma_lock()) };
}
}
@@ -323,7 +341,7 @@ impl<'tree, T: ForeignOwnable> MapleGuard<'tree, T> {
pub fn ma_state(&mut self, first: usize, end: usize) -> MaState<'_, T> {
// SAFETY: The `MaState` borrows this `MapleGuard`, so it can also borrow the `MapleGuard`s
// read/write permissions to the maple tree.
- unsafe { MaState::new_raw(self.0, first, end) }
+ unsafe { MaState::new_raw(self.tree, first, end) }
}
/// Load the value at the given index.
@@ -375,7 +393,7 @@ impl<'tree, T: ForeignOwnable> MapleGuard<'tree, T> {
#[inline]
pub fn load(&mut self, index: usize) -> Option<T::BorrowedMut<'_>> {
// SAFETY: `self.tree` contains a valid maple tree.
- let ret = unsafe { bindings::mtree_load(self.0.tree.get(), index) };
+ let ret = unsafe { bindings::mtree_load(self.tree.tree.get(), index) };
if ret.is_null() {
return None;
}
diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs
index 77c52fe59551..769d01d23429 100644
--- a/rust/kernel/pci.rs
+++ b/rust/kernel/pci.rs
@@ -17,6 +17,7 @@ use crate::{
from_result,
to_result, //
},
+ io::resource,
prelude::*,
str::CStr,
types::Opaque,
@@ -447,6 +448,19 @@ impl Device {
Ok(unsafe { bindings::pci_resource_len(self.as_raw(), bar.try_into()?) })
}
+ /// Returns the resource flags (`IORESOURCE_*`) of the given PCI BAR.
+ pub fn resource_flags(&self, bar: u32) -> Result<resource::Flags> {
+ if !Bar::index_is_valid(bar) {
+ return Err(EINVAL);
+ }
+
+ // SAFETY:
+ // - `bar` is a valid bar number, as guaranteed by the above call to `Bar::index_is_valid`,
+ // - by its type invariant `self.as_raw` is always a valid pointer to a `struct pci_dev`.
+ let raw = unsafe { bindings::pci_resource_flags(self.as_raw(), bar.try_into()?) };
+ Ok(resource::Flags::from_raw(raw))
+ }
+
/// Returns the PCI class as a `Class` struct.
#[inline]
pub fn pci_class(&self) -> Class {
diff --git a/rust/kernel/pci/irq.rs b/rust/kernel/pci/irq.rs
index 22e2cdf82a21..2ce4b46380e8 100644
--- a/rust/kernel/pci/irq.rs
+++ b/rust/kernel/pci/irq.rs
@@ -13,27 +13,26 @@ use crate::{
};
use core::num::NonZero;
-/// IRQ type flags for PCI interrupt allocation.
-#[derive(Debug, Clone, Copy)]
-pub enum IrqType {
- /// INTx interrupts.
- Intx,
- /// Message Signaled Interrupts (MSI).
- Msi,
- /// Extended Message Signaled Interrupts (MSI-X).
- MsiX,
-}
-
-impl IrqType {
- /// Convert to the corresponding kernel flags.
- const fn as_raw(self) -> u32 {
- match self {
- IrqType::Intx => bindings::PCI_IRQ_INTX,
- IrqType::Msi => bindings::PCI_IRQ_MSI,
- IrqType::MsiX => bindings::PCI_IRQ_MSIX,
- }
+crate::impl_flags!(
+ /// Set of IRQ types that can be used for PCI interrupt allocation.
+ #[derive(Debug, Clone, Copy, Default)]
+ pub struct IrqTypes(u32);
+
+ /// IRQ type flags for PCI interrupt allocation.
+ #[derive(Debug, Clone, Copy)]
+ pub enum IrqType {
+ /// INTx interrupts.
+ Intx = bindings::PCI_IRQ_INTX,
+
+ /// Message Signaled Interrupts (MSI).
+ Msi = bindings::PCI_IRQ_MSI,
+
+ /// Extended Message Signaled Interrupts (MSI-X).
+ MsiX = bindings::PCI_IRQ_MSIX,
}
+);
+impl IrqType {
/// Construct from raw value.
#[inline]
const fn from_raw(raw: u32) -> Self {
@@ -45,33 +44,10 @@ impl IrqType {
}
}
-/// Set of IRQ types that can be used for PCI interrupt allocation.
-#[derive(Debug, Clone, Copy, Default)]
-pub struct IrqTypes(u32);
-
impl IrqTypes {
/// Create a set containing all IRQ types (MSI-X, MSI, and INTx).
pub const fn all() -> Self {
- Self(bindings::PCI_IRQ_ALL_TYPES)
- }
-
- /// Build a set of IRQ types.
- ///
- /// # Examples
- ///
- /// ```ignore
- /// // Create a set with only MSI and MSI-X (no INTx interrupts).
- /// let msi_only = IrqTypes::default()
- /// .with(IrqType::Msi)
- /// .with(IrqType::MsiX);
- /// ```
- pub const fn with(self, irq_type: IrqType) -> Self {
- Self(self.0 | irq_type.as_raw())
- }
-
- /// Get the raw flags value.
- const fn as_raw(self) -> u32 {
- self.0
+ Self(Self::all_bits())
}
}
@@ -205,9 +181,7 @@ impl Device<device::Bound> {
/// let vectors = dev.alloc_irq_vectors(1, 32, pci::IrqTypes::all())?;
///
/// // Allocate MSI or MSI-X only (no INTx interrupts).
- /// let msi_only = pci::IrqTypes::default()
- /// .with(pci::IrqType::Msi)
- /// .with(pci::IrqType::MsiX);
+ /// let msi_only = pci::IrqType::Msi | pci::IrqType::MsiX;
/// let vectors = dev.alloc_irq_vectors(4, 16, msi_only)?;
/// # Ok(())
/// # }
@@ -224,7 +198,7 @@ impl Device<device::Bound> {
// - `pci_alloc_irq_vectors` internally validates all other parameters
// and returns error codes.
let ret = unsafe {
- bindings::pci_alloc_irq_vectors(self.as_raw(), min_vecs, max_vecs, irq_types.as_raw())
+ bindings::pci_alloc_irq_vectors(self.as_raw(), min_vecs, max_vecs, u32::from(irq_types))
};
to_result(ret)?;
diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs
index f09078228c53..f1aecda7a90c 100644
--- a/rust/kernel/uaccess.rs
+++ b/rust/kernel/uaccess.rs
@@ -558,6 +558,7 @@ impl UserSliceWriter {
/// truncates the write to the boundaries of `self` and `data`.
///
/// On success, returns the number of bytes written.
+ #[inline]
pub fn write_slice_partial(&mut self, data: &[u8], offset: usize) -> Result<usize> {
let end = offset.saturating_add(self.len()).min(data.len());
@@ -624,6 +625,19 @@ impl UserSliceWriter {
self.length -= len;
Ok(())
}
+
+ /// Writes as much of the provided value as fits in the remaining buffer.
+ ///
+ /// Copies `min(size_of::<T>(), self.len())` bytes to userspace. Returns the number of bytes
+ /// actually written. This is useful for versioned structs where an older userspace may provide
+ /// a smaller buffer than the current kernel struct.
+ ///
+ /// Fails with [`EFAULT`] if the write happens on a bad address. This call may modify the
+ /// associated userspace slice even if it returns an error.
+ #[inline]
+ pub fn write_truncated<T: AsBytes>(&mut self, value: &T) -> Result<usize> {
+ self.write_slice_partial(value.as_bytes(), 0)
+ }
}
/// Reads a nul-terminated string into `dst` and returns the length.
generated by cgit v1.2.3 (git 2.0.4) at 2026-11-05 19:19:59 +0000