summaryrefslogtreecommitdiff
path: root/rust/kernel/sync
diff options
context:
space:
mode:
Diffstat (limited to 'rust/kernel/sync')
-rw-r--r--rust/kernel/sync/arc.rs53
-rw-r--r--rust/kernel/sync/aref.rs79
-rw-r--r--rust/kernel/sync/atomic.rs310
-rw-r--r--rust/kernel/sync/atomic/internal.rs51
-rw-r--r--rust/kernel/sync/atomic/ordering.rs2
-rw-r--r--rust/kernel/sync/atomic/predefine.rs116
-rw-r--r--rust/kernel/sync/barrier.rs127
-rw-r--r--rust/kernel/sync/completion.rs2
-rw-r--r--rust/kernel/sync/lock/global.rs5
-rw-r--r--rust/kernel/sync/lock/spinlock.rs329
-rw-r--r--rust/kernel/sync/locked_by.rs2
-rw-r--r--rust/kernel/sync/poll.rs83
-rw-r--r--rust/kernel/sync/rcu.rs36
-rw-r--r--rust/kernel/sync/refcount.rs8
-rw-r--r--rust/kernel/sync/srcu.rs171
15 files changed, 1270 insertions, 104 deletions
diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs
index 921e19333b89..8ae0fe6f19ec 100644
--- a/rust/kernel/sync/arc.rs
+++ b/rust/kernel/sync/arc.rs
@@ -128,7 +128,7 @@ mod std_vendor;
/// # Ok::<(), Error>(())
/// ```
#[repr(transparent)]
-#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))]
+#[derive(core::marker::CoercePointee)]
pub struct Arc<T: ?Sized> {
ptr: NonNull<ArcInner<T>>,
// NB: this informs dropck that objects of type `ArcInner<T>` may be used in `<Arc<T> as
@@ -154,7 +154,7 @@ impl<T: ?Sized> ArcInner<T> {
///
/// # Safety
///
- /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must
+ /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the [`Arc`] must
/// not yet have been destroyed.
unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> {
let refcount_layout = Layout::new::<Refcount>();
@@ -182,15 +182,6 @@ impl<T: ?Sized> ArcInner<T> {
}
}
-// This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the
-// dynamically-sized type (DST) `U`.
-#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
-impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {}
-
-// This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`.
-#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
-impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {}
-
// SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because
// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs
// `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` using a
@@ -262,7 +253,7 @@ impl<T: ?Sized> Arc<T> {
/// Convert the [`Arc`] into a raw pointer.
///
- /// The raw pointer has ownership of the refcount that this Arc object owned.
+ /// The raw pointer has ownership of the refcount that this [`Arc`] object owned.
pub fn into_raw(self) -> *const T {
let ptr = self.ptr.as_ptr();
core::mem::forget(self);
@@ -270,7 +261,7 @@ impl<T: ?Sized> Arc<T> {
unsafe { core::ptr::addr_of!((*ptr).data) }
}
- /// Return a raw pointer to the data in this arc.
+ /// Return a raw pointer to the data in this [`Arc`].
pub fn as_ptr(this: &Self) -> *const T {
let ptr = this.ptr.as_ptr();
@@ -314,7 +305,7 @@ impl<T: ?Sized> Arc<T> {
/// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique.
///
- /// When this destroys the `Arc`, it does so while properly avoiding races. This means that
+ /// When this destroys the [`Arc`], it does so while properly avoiding races. This means that
/// this method will never call the destructor of the value.
///
/// # Examples
@@ -354,11 +345,11 @@ impl<T: ?Sized> Arc<T> {
// If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will
// return without further touching the `Arc`. If the refcount reaches zero, then there are
- // no other arcs, and we can create a `UniqueArc`.
+ // no other `Arc`s, and we can create a `UniqueArc`.
if refcount.dec_and_test() {
refcount.set(1);
- // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We
+ // INVARIANT: We own the only refcount to this `Arc`, so we may create a `UniqueArc`. We
// must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin
// their values.
Some(Pin::from(UniqueArc {
@@ -547,20 +538,12 @@ impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> {
/// # Ok::<(), Error>(())
/// ```
#[repr(transparent)]
-#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))]
+#[derive(core::marker::CoercePointee)]
pub struct ArcBorrow<'a, T: ?Sized + 'a> {
inner: NonNull<ArcInner<T>>,
_p: PhantomData<&'a ()>,
}
-// This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into
-// `ArcBorrow<U>`.
-#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]
-impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>>
- for ArcBorrow<'_, T>
-{
-}
-
impl<T: ?Sized> Clone for ArcBorrow<'_, T> {
fn clone(&self) -> Self {
*self
@@ -729,20 +712,22 @@ impl<T> InPlaceInit<T> for UniqueArc<T> {
impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> {
type Initialized = UniqueArc<T>;
+ #[inline]
fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
let slot = self.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
// slot is valid.
- unsafe { init.__init(slot)? };
+ unsafe { pin_init::raw_try_init(slot, init)? };
// SAFETY: All fields have been initialized.
Ok(unsafe { self.assume_init() })
}
+ #[inline]
fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
let slot = self.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
// slot is valid and will not be moved, because we pin it later.
- unsafe { init.__pinned_init(slot)? };
+ unsafe { pin_init::raw_try_init(slot, init)? };
// SAFETY: All fields have been initialized.
Ok(unsafe { self.assume_init() }.into())
}
@@ -775,6 +760,14 @@ impl<T> UniqueArc<T> {
}
}
+impl<T: ?Sized> UniqueArc<T> {
+ /// Return a raw pointer to the data in this [`UniqueArc`].
+ #[inline]
+ pub fn as_ptr(this: &Self) -> *const T {
+ Arc::as_ptr(&this.inner)
+ }
+}
+
impl<T> UniqueArc<MaybeUninit<T>> {
/// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it.
pub fn write(mut self, value: T) -> UniqueArc<T> {
@@ -799,9 +792,10 @@ impl<T> UniqueArc<MaybeUninit<T>> {
}
/// Initialize `self` using the given initializer.
+ #[inline]
pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> {
// SAFETY: The supplied pointer is valid for initialization.
- match unsafe { init.__init(self.as_mut_ptr()) } {
+ match unsafe { pin_init::raw_try_init(self.as_mut_ptr(), init) } {
// SAFETY: Initialization completed successfully.
Ok(()) => Ok(unsafe { self.assume_init() }),
Err(err) => Err(err),
@@ -809,13 +803,14 @@ impl<T> UniqueArc<MaybeUninit<T>> {
}
/// Pin-initialize `self` using the given pin-initializer.
+ #[inline]
pub fn pin_init_with<E>(
mut self,
init: impl PinInit<T, E>,
) -> core::result::Result<Pin<UniqueArc<T>>, E> {
// SAFETY: The supplied pointer is valid for initialization and we will later pin the value
// to ensure it does not move.
- match unsafe { init.__pinned_init(self.as_mut_ptr()) } {
+ match unsafe { pin_init::raw_try_init(self.as_mut_ptr(), init) } {
// SAFETY: Initialization completed successfully.
Ok(()) => Ok(unsafe { self.assume_init() }.into()),
Err(err) => Err(err),
diff --git a/rust/kernel/sync/aref.rs b/rust/kernel/sync/aref.rs
index 0616c0353c2b..9983ee085248 100644
--- a/rust/kernel/sync/aref.rs
+++ b/rust/kernel/sync/aref.rs
@@ -17,7 +17,17 @@
//! [`Arc`]: crate::sync::Arc
//! [`Arc<T>`]: crate::sync::Arc
-use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull};
+use core::{
+ marker::PhantomData,
+ mem::ManuallyDrop,
+ ops::Deref,
+ ptr::NonNull, //
+};
+
+use crate::{
+ prelude::*,
+ types::ForeignOwnable, //
+};
/// Types that are _always_ reference counted.
///
@@ -170,3 +180,70 @@ impl<T: AlwaysRefCounted> Drop for ARef<T> {
unsafe { T::dec_ref(self.ptr) };
}
}
+
+impl<T, U> PartialEq<ARef<U>> for ARef<T>
+where
+ T: AlwaysRefCounted + PartialEq<U>,
+ U: AlwaysRefCounted,
+{
+ #[inline]
+ fn eq(&self, other: &ARef<U>) -> bool {
+ T::eq(&**self, &**other)
+ }
+}
+impl<T: AlwaysRefCounted + Eq> Eq for ARef<T> {}
+
+// SAFETY: `into_foreign` returns a pointer from `NonNull::as_ptr`, so it's non-null. The
+// `ARef` invariant guarantees that `ptr` points to a valid `T`, so it's aligned to `T`.
+unsafe impl<T: AlwaysRefCounted> ForeignOwnable for ARef<T> {
+ const FOREIGN_ALIGN: usize = core::mem::align_of::<T>();
+
+ type Borrowed<'a>
+ = &'a T
+ where
+ Self: 'a;
+ type BorrowedMut<'a>
+ = &'a T
+ where
+ Self: 'a;
+
+ #[inline]
+ fn into_foreign(self) -> *mut c_void {
+ ARef::into_raw(self).as_ptr().cast()
+ }
+
+ #[inline]
+ unsafe fn from_foreign(ptr: *mut c_void) -> Self {
+ // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous
+ // call to `Self::into_foreign`.
+ let ptr = unsafe { NonNull::new_unchecked(ptr.cast()) };
+
+ // SAFETY: `ptr` came from `into_foreign`, which consumed an `ARef` without decrementing
+ // the refcount, so we can transfer the ownership to the new `ARef`.
+ unsafe { ARef::from_raw(ptr) }
+ }
+
+ #[inline]
+ unsafe fn borrow<'a>(ptr: *mut c_void) -> &'a T {
+ // SAFETY: The safety requirements of this method ensure that the object remains alive and
+ // immutable for the duration of 'a.
+ unsafe { &*ptr.cast() }
+ }
+
+ #[inline]
+ unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> &'a T {
+ // SAFETY: The safety requirements for `borrow_mut` are a superset of the safety
+ // requirements for `borrow`.
+ unsafe { <Self as ForeignOwnable>::borrow(ptr) }
+ }
+}
+
+impl<T, U> PartialEq<&'_ U> for ARef<T>
+where
+ T: AlwaysRefCounted + PartialEq<U>,
+{
+ #[inline]
+ fn eq(&self, other: &&U) -> bool {
+ T::eq(&**self, other)
+ }
+}
diff --git a/rust/kernel/sync/atomic.rs b/rust/kernel/sync/atomic.rs
index 4aebeacb961a..9cd009d57e35 100644
--- a/rust/kernel/sync/atomic.rs
+++ b/rust/kernel/sync/atomic.rs
@@ -51,6 +51,10 @@ use ordering::OrderingType;
#[repr(transparent)]
pub struct Atomic<T: AtomicType>(AtomicRepr<T::Repr>);
+// SAFETY: `Atomic<T>` is safe to transfer between execution contexts because of the safety
+// requirement of `AtomicType`.
+unsafe impl<T: AtomicType> Send for Atomic<T> {}
+
// SAFETY: `Atomic<T>` is safe to share among execution contexts because all accesses are atomic.
unsafe impl<T: AtomicType> Sync for Atomic<T> {}
@@ -68,6 +72,11 @@ unsafe impl<T: AtomicType> Sync for Atomic<T> {}
///
/// - [`Self`] must have the same size and alignment as [`Self::Repr`].
/// - [`Self`] must be [round-trip transmutable] to [`Self::Repr`].
+/// - [`Self`] must be safe to transfer between execution contexts, if it's [`Send`], this is
+/// automatically satisfied. The exception is pointer types that are even though marked as
+/// `!Send` (e.g. raw pointers and [`NonNull<T>`]) but requiring `unsafe` to do anything
+/// meaningful on them. This is because transferring pointer values between execution contexts is
+/// safe as long as the actual `unsafe` dereferencing is justified.
///
/// Note that this is more relaxed than requiring the bi-directional transmutability (i.e.
/// [`transmute()`] is always sound between `U` and `T`) because of the support for atomic
@@ -108,7 +117,8 @@ unsafe impl<T: AtomicType> Sync for Atomic<T> {}
/// [`transmute()`]: core::mem::transmute
/// [round-trip transmutable]: AtomicType#round-trip-transmutability
/// [Examples]: AtomicType#examples
-pub unsafe trait AtomicType: Sized + Send + Copy {
+/// [`NonNull<T>`]: core::ptr::NonNull
+pub unsafe trait AtomicType: Sized + Copy {
/// The backing atomic implementation type.
type Repr: AtomicImpl;
}
@@ -204,10 +214,7 @@ impl<T: AtomicType> Atomic<T> {
/// // no data race.
/// unsafe { Atomic::from_ptr(foo_a_ptr) }.store(2, Release);
/// ```
- pub unsafe fn from_ptr<'a>(ptr: *mut T) -> &'a Self
- where
- T: Sync,
- {
+ pub unsafe fn from_ptr<'a>(ptr: *mut T) -> &'a Self {
// CAST: `T` and `Atomic<T>` have the same size, alignment and bit validity.
// SAFETY: Per function safety requirement, `ptr` is a valid pointer and the object will
// live long enough. It's safe to return a `&Atomic<T>` because function safety requirement
@@ -235,6 +242,17 @@ impl<T: AtomicType> Atomic<T> {
/// Returns a mutable reference to the underlying atomic `T`.
///
/// This is safe because the mutable reference of the atomic `T` guarantees exclusive access.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::sync::atomic::{Atomic, Relaxed};
+ ///
+ /// let mut atomic_val = Atomic::new(0u32);
+ /// let val_mut = atomic_val.get_mut();
+ /// *val_mut = 101;
+ /// assert_eq!(101, atomic_val.load(Relaxed));
+ /// ```
pub fn get_mut(&mut self) -> &mut T {
// CAST: `T` and `T::Repr` has the same size and alignment per the safety requirement of
// `AtomicType`, and per the type invariants `self.0` is a valid `T`, therefore the casting
@@ -527,16 +545,14 @@ where
/// use kernel::sync::atomic::{Atomic, Acquire, Full, Relaxed};
///
/// let x = Atomic::new(42);
- ///
/// assert_eq!(42, x.load(Relaxed));
- ///
- /// assert_eq!(54, { x.fetch_add(12, Acquire); x.load(Relaxed) });
+ /// assert_eq!(42, x.fetch_add(12, Acquire));
+ /// assert_eq!(54, x.load(Relaxed));
///
/// let x = Atomic::new(42);
- ///
/// assert_eq!(42, x.load(Relaxed));
- ///
- /// assert_eq!(54, { x.fetch_add(12, Full); x.load(Relaxed) } );
+ /// assert_eq!(42, x.fetch_add(12, Full));
+ /// assert_eq!(54, x.load(Relaxed));
/// ```
#[inline(always)]
pub fn fetch_add<Rhs, Ordering: ordering::Ordering>(&self, v: Rhs, _: Ordering) -> T
@@ -559,4 +575,276 @@ where
// SAFETY: `ret` comes from reading `self.0`, which is a valid `T` per type invariants.
unsafe { from_repr(ret) }
}
+
+ /// Atomic fetch and subtract.
+ ///
+ /// Atomically updates `*self` to `(*self).wrapping_sub(v)`, and returns the value of `*self`
+ /// before the update.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::sync::atomic::{Atomic, Acquire, Full, Relaxed};
+ ///
+ /// let x = Atomic::new(42);
+ /// assert_eq!(42, x.load(Relaxed));
+ /// assert_eq!(42, x.fetch_sub(12, Acquire));
+ /// assert_eq!(30, x.load(Relaxed));
+ ///
+ /// let x = Atomic::new(42);
+ /// assert_eq!(42, x.load(Relaxed));
+ /// assert_eq!(42, x.fetch_sub(12, Full));
+ /// assert_eq!(30, x.load(Relaxed));
+ /// ```
+ #[inline(always)]
+ pub fn fetch_sub<Rhs, Ordering: ordering::Ordering>(&self, v: Rhs, _: Ordering) -> T
+ where
+ // Types that support addition also support subtraction.
+ T: AtomicAdd<Rhs>,
+ {
+ let v = T::rhs_into_delta(v);
+
+ // INVARIANT: `self.0` is a valid `T` after `atomic_fetch_sub*()` due to safety requirement
+ // of `AtomicAdd`.
+ let ret = {
+ match Ordering::TYPE {
+ OrderingType::Full => T::Repr::atomic_fetch_sub(&self.0, v),
+ OrderingType::Acquire => T::Repr::atomic_fetch_sub_acquire(&self.0, v),
+ OrderingType::Release => T::Repr::atomic_fetch_sub_release(&self.0, v),
+ OrderingType::Relaxed => T::Repr::atomic_fetch_sub_relaxed(&self.0, v),
+ }
+ };
+
+ // SAFETY: `ret` comes from reading `self.0`, which is a valid `T` per type invariants.
+ unsafe { from_repr(ret) }
+ }
+}
+
+#[cfg(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64))]
+#[repr(C)]
+#[derive(Clone, Copy)]
+struct Flag {
+ bool_field: bool,
+}
+
+/// # Invariants
+///
+/// `padding` must be all zeroes.
+#[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))]
+#[repr(C, align(4))]
+#[derive(Clone, Copy)]
+struct Flag {
+ #[cfg(target_endian = "big")]
+ padding: [u8; 3],
+ bool_field: bool,
+ #[cfg(target_endian = "little")]
+ padding: [u8; 3],
+}
+
+impl Flag {
+ #[inline(always)]
+ const fn new(b: bool) -> Self {
+ // INVARIANT: `padding` is all zeroes.
+ Self {
+ bool_field: b,
+ #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))]
+ padding: [0; 3],
+ }
+ }
+}
+
+// SAFETY: `Flag` and `Repr` have the same size and alignment, and `Flag` is round-trip
+// transmutable to the selected representation (`i8` or `i32`).
+unsafe impl AtomicType for Flag {
+ #[cfg(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64))]
+ type Repr = i8;
+ #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))]
+ type Repr = i32;
+}
+
+/// An atomic flag type intended to be backed by performance-optimal integer type.
+///
+/// The backing integer type is an implementation detail; it may vary by architecture and change
+/// in the future.
+///
+/// [`AtomicFlag`] is generally preferable to [`Atomic<bool>`] when you need read-modify-write
+/// (RMW) operations (e.g. [`Atomic::xchg()`]/[`Atomic::cmpxchg()`]) or when [`Atomic<bool>`] does
+/// not save memory due to padding. On some architectures that do not support byte-sized atomic
+/// RMW operations, RMW operations on [`Atomic<bool>`] are slower.
+///
+/// If you only use [`Atomic::load()`]/[`Atomic::store()`], [`Atomic<bool>`] is fine.
+///
+/// # Examples
+///
+/// ```
+/// use kernel::sync::atomic::{AtomicFlag, Relaxed};
+///
+/// let flag = AtomicFlag::new(false);
+/// assert_eq!(false, flag.load(Relaxed));
+/// flag.store(true, Relaxed);
+/// assert_eq!(true, flag.load(Relaxed));
+/// ```
+pub struct AtomicFlag(Atomic<Flag>);
+
+impl AtomicFlag {
+ /// Creates a new atomic flag.
+ #[inline(always)]
+ pub const fn new(b: bool) -> Self {
+ Self(Atomic::new(Flag::new(b)))
+ }
+
+ /// Returns a mutable reference to the underlying flag as a [`bool`].
+ ///
+ /// This is safe because the mutable reference of the atomic flag guarantees exclusive access.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use kernel::sync::atomic::{AtomicFlag, Relaxed};
+ ///
+ /// let mut atomic_flag = AtomicFlag::new(false);
+ /// assert_eq!(false, atomic_flag.load(Relaxed));
+ /// *atomic_flag.get_mut() = true;
+ /// assert_eq!(true, atomic_flag.load(Relaxed));
+ /// ```
+ #[inline(always)]
+ pub fn get_mut(&mut self) -> &mut bool {
+ &mut self.0.get_mut().bool_field
+ }
+
+ /// Loads the value from the atomic flag.
+ #[inline(always)]
+ pub fn load<Ordering: ordering::AcquireOrRelaxed>(&self, o: Ordering) -> bool {
+ self.0.load(o).bool_field
+ }
+
+ /// Stores a value to the atomic flag.
+ #[inline(always)]
+ pub fn store<Ordering: ordering::ReleaseOrRelaxed>(&self, v: bool, o: Ordering) {
+ self.0.store(Flag::new(v), o);
+ }
+
+ /// Stores a value to the atomic flag and returns the previous value.
+ #[inline(always)]
+ pub fn xchg<Ordering: ordering::Ordering>(&self, new: bool, o: Ordering) -> bool {
+ self.0.xchg(Flag::new(new), o).bool_field
+ }
+
+ /// Store a value to the atomic flag if the current value is equal to `old`.
+ #[inline(always)]
+ pub fn cmpxchg<Ordering: ordering::Ordering>(
+ &self,
+ old: bool,
+ new: bool,
+ o: Ordering,
+ ) -> Result<bool, bool> {
+ match self.0.cmpxchg(Flag::new(old), Flag::new(new), o) {
+ Ok(_) => Ok(old),
+ Err(f) => Err(f.bool_field),
+ }
+ }
+}
+
+/// Atomic load over raw pointers.
+///
+/// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work
+/// with C side on synchronizations:
+///
+/// - `atomic_load(.., Relaxed)` maps to `READ_ONCE()` when used for inter-thread communication.
+/// - `atomic_load(.., Acquire)` maps to `smp_load_acquire()`.
+///
+/// # Safety
+///
+/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.
+/// - If there is a concurrent store from kernel (C or Rust), it has to be atomic.
+#[doc(alias("READ_ONCE", "smp_load_acquire"))]
+#[inline(always)]
+pub unsafe fn atomic_load<T: AtomicType, Ordering: ordering::AcquireOrRelaxed>(
+ ptr: *mut T,
+ o: Ordering,
+) -> T
+where
+ T::Repr: AtomicBasicOps,
+{
+ // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to
+ // `align_of::<T>()`, and all concurrent stores from kernel are atomic, hence no data race per
+ // LKMM.
+ unsafe { Atomic::from_ptr(ptr) }.load(o)
+}
+
+/// Atomic store over raw pointers.
+///
+/// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work
+/// with C side on synchronizations:
+///
+/// - `atomic_store(.., Relaxed)` maps to `WRITE_ONCE()` when used for inter-thread communication.
+/// - `atomic_load(.., Release)` maps to `smp_store_release()`.
+///
+/// # Safety
+///
+/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.
+/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic.
+#[doc(alias("WRITE_ONCE", "smp_store_release"))]
+#[inline(always)]
+pub unsafe fn atomic_store<T: AtomicType, Ordering: ordering::ReleaseOrRelaxed>(
+ ptr: *mut T,
+ v: T,
+ o: Ordering,
+) where
+ T::Repr: AtomicBasicOps,
+{
+ // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to
+ // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race
+ // per LKMM.
+ unsafe { Atomic::from_ptr(ptr) }.store(v, o);
+}
+
+/// Atomic exchange over raw pointers.
+///
+/// This function provides a short-cut of `Atomic::from_ptr().xchg(..)`, and can be used to work
+/// with C side on synchronizations.
+///
+/// # Safety
+///
+/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.
+/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic.
+#[inline(always)]
+pub unsafe fn xchg<T: AtomicType, Ordering: ordering::Ordering>(
+ ptr: *mut T,
+ new: T,
+ o: Ordering,
+) -> T
+where
+ T::Repr: AtomicExchangeOps,
+{
+ // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to
+ // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race
+ // per LKMM.
+ unsafe { Atomic::from_ptr(ptr) }.xchg(new, o)
+}
+
+/// Atomic compare and exchange over raw pointers.
+///
+/// This function provides a short-cut of `Atomic::from_ptr().cmpxchg(..)`, and can be used to work
+/// with C side on synchronizations.
+///
+/// # Safety
+///
+/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.
+/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic.
+#[doc(alias("try_cmpxchg"))]
+#[inline(always)]
+pub unsafe fn cmpxchg<T: AtomicType, Ordering: ordering::Ordering>(
+ ptr: *mut T,
+ old: T,
+ new: T,
+ o: Ordering,
+) -> Result<T, T>
+where
+ T::Repr: AtomicExchangeOps,
+{
+ // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to
+ // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race
+ // per LKMM.
+ unsafe { Atomic::from_ptr(ptr) }.cmpxchg(old, new, o)
}
diff --git a/rust/kernel/sync/atomic/internal.rs b/rust/kernel/sync/atomic/internal.rs
index 0dac58bca2b3..9c8a7a203abd 100644
--- a/rust/kernel/sync/atomic/internal.rs
+++ b/rust/kernel/sync/atomic/internal.rs
@@ -4,9 +4,13 @@
//!
//! Provides 1:1 mapping to the C atomic operations.
-use crate::bindings;
-use crate::macros::paste;
+use crate::{
+ bindings,
+ build_assert::static_assert,
+ macros::paste, //
+};
use core::cell::UnsafeCell;
+use ffi::c_void;
mod private {
/// Sealed trait marker to disable customized impls on atomic implementation traits.
@@ -14,10 +18,11 @@ mod private {
}
// The C side supports atomic primitives only for `i32` and `i64` (`atomic_t` and `atomic64_t`),
-// while the Rust side also layers provides atomic support for `i8` and `i16`
-// on top of lower-level C primitives.
+// while the Rust side also provides atomic support for `i8`, `i16` and `*const c_void` on top of
+// lower-level C primitives.
impl private::Sealed for i8 {}
impl private::Sealed for i16 {}
+impl private::Sealed for *const c_void {}
impl private::Sealed for i32 {}
impl private::Sealed for i64 {}
@@ -26,10 +31,10 @@ impl private::Sealed for i64 {}
/// This trait is sealed, and only types that map directly to the C side atomics
/// or can be implemented with lower-level C primitives are allowed to implement this:
///
-/// - `i8` and `i16` are implemented with lower-level C primitives.
+/// - `i8`, `i16` and `*const c_void` are implemented with lower-level C primitives.
/// - `i32` map to `atomic_t`
/// - `i64` map to `atomic64_t`
-pub trait AtomicImpl: Sized + Send + Copy + private::Sealed {
+pub trait AtomicImpl: Sized + Copy + private::Sealed {
/// The type of the delta in arithmetic or logical operations.
///
/// For example, in `atomic_add(ptr, v)`, it's the type of `v`. Usually it's the same type of
@@ -37,20 +42,31 @@ pub trait AtomicImpl: Sized + Send + Copy + private::Sealed {
type Delta;
}
-// The current helpers of load/store uses `{WRITE,READ}_ONCE()` hence the atomicity is only
-// guaranteed against read-modify-write operations if the architecture supports native atomic RmW.
-#[cfg(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW)]
+// The current helpers of load/store of atomic `i8`, `i16` and pointers use `{WRITE,READ}_ONCE()`
+// hence the atomicity is only guaranteed against read-modify-write operations if the architecture
+// supports native atomic RmW.
+//
+// In the future when a CONFIG_ARCH_SUPPORTS_ATOMIC_RMW=n architecture plans to support Rust, the
+// load/store helpers that guarantee atomicity against RmW operations (usually via a lock) need to
+// be added.
+static_assert!(
+ cfg!(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW),
+ "The current implementation of atomic i8/i16/ptr relies on the architecure being \
+ ARCH_SUPPORTS_ATOMIC_RMW"
+);
+
impl AtomicImpl for i8 {
type Delta = Self;
}
-// The current helpers of load/store uses `{WRITE,READ}_ONCE()` hence the atomicity is only
-// guaranteed against read-modify-write operations if the architecture supports native atomic RmW.
-#[cfg(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW)]
impl AtomicImpl for i16 {
type Delta = Self;
}
+impl AtomicImpl for *const c_void {
+ type Delta = isize;
+}
+
// `atomic_t` implements atomic operations on `i32`.
impl AtomicImpl for i32 {
type Delta = Self;
@@ -262,7 +278,7 @@ macro_rules! declare_and_impl_atomic_methods {
}
declare_and_impl_atomic_methods!(
- [ i8 => atomic_i8, i16 => atomic_i16, i32 => atomic, i64 => atomic64 ]
+ [ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ]
/// Basic atomic operations
pub trait AtomicBasicOps {
/// Atomic read (load).
@@ -280,7 +296,7 @@ declare_and_impl_atomic_methods!(
);
declare_and_impl_atomic_methods!(
- [ i8 => atomic_i8, i16 => atomic_i16, i32 => atomic, i64 => atomic64 ]
+ [ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ]
/// Exchange and compare-and-exchange atomic operations
pub trait AtomicExchangeOps {
/// Atomic exchange.
@@ -324,7 +340,12 @@ declare_and_impl_atomic_methods!(
/// Atomically updates `*a` to `(*a).wrapping_add(v)`, and returns the value of `*a`
/// before the update.
fn fetch_add[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self::Delta) -> Self {
- // SAFETY: `a.as_ptr()` is valid and properly aligned.
+ // SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned.
+ unsafe { bindings::#call(v, a.as_ptr().cast()) }
+ }
+
+ fn fetch_sub[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self::Delta) -> Self {
+ // SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned.
unsafe { bindings::#call(v, a.as_ptr().cast()) }
}
}
diff --git a/rust/kernel/sync/atomic/ordering.rs b/rust/kernel/sync/atomic/ordering.rs
index 3f103aa8db99..c4e732e7212f 100644
--- a/rust/kernel/sync/atomic/ordering.rs
+++ b/rust/kernel/sync/atomic/ordering.rs
@@ -15,7 +15,7 @@
//! - It provides ordering between the annotated operation and all the following memory accesses.
//! - It provides ordering between all the preceding memory accesses and all the following memory
//! accesses.
-//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb()`).
+//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb(Full)`).
//! - [`Relaxed`] provides no ordering except the dependency orderings. Dependency orderings are
//! described in "DEPENDENCY RELATIONS" in [`LKMM`]'s [`explanation`].
//!
diff --git a/rust/kernel/sync/atomic/predefine.rs b/rust/kernel/sync/atomic/predefine.rs
index 67a0406d3ea4..3d63f40791fa 100644
--- a/rust/kernel/sync/atomic/predefine.rs
+++ b/rust/kernel/sync/atomic/predefine.rs
@@ -2,8 +2,7 @@
//! Pre-defined atomic types
-use crate::static_assert;
-use core::mem::{align_of, size_of};
+use crate::prelude::*;
// Ensure size and alignment requirements are checked.
static_assert!(size_of::<bool>() == size_of::<i8>());
@@ -28,6 +27,26 @@ unsafe impl super::AtomicType for i16 {
type Repr = i16;
}
+// SAFETY:
+//
+// - `*mut T` has the same size and alignment with `*const c_void`, and is round-trip
+// transmutable to `*const c_void`.
+// - `*mut T` is safe to transfer between execution contexts. See the safety requirement of
+// [`AtomicType`].
+unsafe impl<T: Sized> super::AtomicType for *mut T {
+ type Repr = *const c_void;
+}
+
+// SAFETY:
+//
+// - `*const T` has the same size and alignment with `*const c_void`, and is round-trip
+// transmutable to `*const c_void`.
+// - `*const T` is safe to transfer between execution contexts. See the safety requirement of
+// [`AtomicType`].
+unsafe impl<T: Sized> super::AtomicType for *const T {
+ type Repr = *const c_void;
+}
+
// SAFETY: `i32` has the same size and alignment with itself, and is round-trip transmutable to
// itself.
unsafe impl super::AtomicType for i32 {
@@ -133,9 +152,8 @@ unsafe impl super::AtomicAdd<usize> for usize {
}
}
-use crate::macros::kunit_tests;
-
-#[kunit_tests(rust_atomics)]
+#[cfg(CONFIG_RUST_ATOMICS_KUNIT_TEST)]
+#[macros::kunit_tests(rust_atomics)]
mod tests {
use super::super::*;
@@ -157,6 +175,14 @@ mod tests {
assert_eq!(v, x.load(Relaxed));
});
+
+ for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
+ let x = Atomic::new(v);
+ let ptr = x.as_ptr();
+
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ assert_eq!(v, unsafe { atomic_load(ptr, Relaxed) });
+ });
}
#[test]
@@ -167,6 +193,17 @@ mod tests {
x.store(v, Release);
assert_eq!(v, x.load(Acquire));
});
+
+ for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
+ let x = Atomic::new(0);
+ let ptr = x.as_ptr();
+
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ unsafe { atomic_store(ptr, v, Release) };
+
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ assert_eq!(v, unsafe { atomic_load(ptr, Acquire) });
+ });
}
#[test]
@@ -180,6 +217,18 @@ mod tests {
assert_eq!(old, x.xchg(new, Full));
assert_eq!(new, x.load(Relaxed));
});
+
+ for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
+ let x = Atomic::new(v);
+ let ptr = x.as_ptr();
+
+ let old = v;
+ let new = v + 1;
+
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ assert_eq!(old, unsafe { xchg(ptr, new, Full) });
+ assert_eq!(new, x.load(Relaxed));
+ });
}
#[test]
@@ -195,6 +244,21 @@ mod tests {
assert_eq!(Ok(old), x.cmpxchg(old, new, Relaxed));
assert_eq!(new, x.load(Relaxed));
});
+
+ for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {
+ let x = Atomic::new(v);
+ let ptr = x.as_ptr();
+
+ let old = v;
+ let new = v + 1;
+
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ assert_eq!(Err(old), unsafe { cmpxchg(ptr, new, new, Full) });
+ assert_eq!(old, x.load(Relaxed));
+ // SAFETY: `ptr` is a valid pointer and no concurrent access.
+ assert_eq!(Ok(old), unsafe { cmpxchg(ptr, old, new, Relaxed) });
+ assert_eq!(new, x.load(Relaxed));
+ });
}
#[test]
@@ -226,4 +290,46 @@ mod tests {
assert_eq!(false, x.load(Relaxed));
assert_eq!(Ok(false), x.cmpxchg(false, true, Full));
}
+
+ #[test]
+ fn atomic_ptr_tests() {
+ let mut v = 42;
+ let mut u = 43;
+ let x = Atomic::new(&raw mut v);
+
+ assert_eq!(x.load(Acquire), &raw mut v);
+ assert_eq!(x.cmpxchg(&raw mut u, &raw mut u, Relaxed), Err(&raw mut v));
+ assert_eq!(x.cmpxchg(&raw mut v, &raw mut u, Relaxed), Ok(&raw mut v));
+ assert_eq!(x.load(Relaxed), &raw mut u);
+
+ let x = Atomic::new(&raw const v);
+
+ assert_eq!(x.load(Acquire), &raw const v);
+ assert_eq!(
+ x.cmpxchg(&raw const u, &raw const u, Relaxed),
+ Err(&raw const v)
+ );
+ assert_eq!(
+ x.cmpxchg(&raw const v, &raw const u, Relaxed),
+ Ok(&raw const v)
+ );
+ assert_eq!(x.load(Relaxed), &raw const u);
+ }
+
+ #[test]
+ fn atomic_flag_tests() {
+ let mut flag = AtomicFlag::new(false);
+
+ assert_eq!(false, flag.load(Relaxed));
+
+ *flag.get_mut() = true;
+ assert_eq!(true, flag.load(Relaxed));
+
+ assert_eq!(true, flag.xchg(false, Relaxed));
+ assert_eq!(false, flag.load(Relaxed));
+
+ *flag.get_mut() = true;
+ assert_eq!(Ok(true), flag.cmpxchg(true, false, Full));
+ assert_eq!(false, flag.load(Relaxed));
+ }
}
diff --git a/rust/kernel/sync/barrier.rs b/rust/kernel/sync/barrier.rs
index 8f2d435fcd94..1180695d533a 100644
--- a/rust/kernel/sync/barrier.rs
+++ b/rust/kernel/sync/barrier.rs
@@ -7,6 +7,38 @@
//!
//! [`LKMM`]: srctree/tools/memory-model/
+#![expect(private_bounds, reason = "sealed implementation")]
+
+/// Memory barrier orderings.
+///
+/// The semantics of these orderings follows the [`LKMM`] definitions and rules.
+///
+/// - [`Read`] provides ordering between preceding load operations and succeeding load operations.
+/// - [`Write`] provides ordering between preceding store operations and succeeding store
+/// operations.
+/// - [`Full`] provides ordering between all the preceding memory accesses and succeeding memory
+/// accesses.
+///
+/// [`LKMM`]: srctree/tools/memory-model/
+pub mod ordering {
+ pub use crate::sync::atomic::ordering::Full;
+
+ /// The annotation type for read-read barrier ordering.
+ pub struct Read;
+
+ /// The annotation type for write-write barrier ordering.
+ pub struct Write;
+}
+
+pub use ordering::{
+ Full,
+ Read,
+ Write, //
+};
+
+struct Smp;
+struct Dma;
+
/// A compiler barrier.
///
/// A barrier that prevents compiler from reordering memory accesses across the barrier.
@@ -19,43 +51,82 @@ pub(crate) fn barrier() {
unsafe { core::arch::asm!("") };
}
-/// A full memory barrier.
+trait MemoryBarrier<Flavour = ()> {
+ fn run();
+}
+
+macro_rules! define_barrier {
+ ($([$flavour:ident])? $ordering:ident, $binding:ident) => {
+ impl MemoryBarrier$(<$flavour>)? for $ordering {
+ #[inline]
+ fn run() {
+ // SAFETY: barrier methods are safe to call.
+ unsafe { bindings::$binding() };
+ }
+ }
+ };
+}
+
+define_barrier!(Full, mb);
+define_barrier!(Read, rmb);
+define_barrier!(Write, wmb);
+define_barrier!([Dma] Full, dma_mb);
+define_barrier!([Dma] Read, dma_rmb);
+define_barrier!([Dma] Write, dma_wmb);
+define_barrier!([Smp] Full, smp_mb);
+define_barrier!([Smp] Read, smp_rmb);
+define_barrier!([Smp] Write, smp_wmb);
+
+/// Memory barrier.
///
/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
-#[inline(always)]
-pub fn smp_mb() {
- if cfg!(CONFIG_SMP) {
- // SAFETY: `smp_mb()` is safe to call.
- unsafe { bindings::smp_mb() };
- } else {
- barrier();
- }
+///
+/// The specific forms of reordering can be specified using the parameter.
+/// - `mb(Read)` provides a read-read barrier.
+/// - `mb(Write)` provides a write-write barrier.
+/// - `mb(Full)` provides a full barrier.
+///
+/// # Examples
+///
+/// ```
+/// # use kernel::sync::barrier::*;
+/// mb(Read);
+/// mb(Write);
+/// mb(Full);
+/// ```
+#[inline]
+#[doc(alias = "rmb")]
+#[doc(alias = "wmb")]
+pub fn mb<T: MemoryBarrier>(_: T) {
+ T::run()
}
-/// A write-write memory barrier.
+/// Memory barrier between CPUs.
///
-/// A barrier that prevents compiler and CPU from reordering memory write accesses across the
-/// barrier.
-#[inline(always)]
-pub fn smp_wmb() {
+/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
+/// Does not prevent re-ordering with respect to other bus-mastering devices.
+///
+/// See [`mb`] for usage.
+#[inline]
+#[doc(alias = "smp_rmb")]
+#[doc(alias = "smp_wmb")]
+pub fn smp_mb<T: MemoryBarrier<Smp>>(_: T) {
if cfg!(CONFIG_SMP) {
- // SAFETY: `smp_wmb()` is safe to call.
- unsafe { bindings::smp_wmb() };
+ T::run()
} else {
- barrier();
+ barrier()
}
}
-/// A read-read memory barrier.
+/// Memory barrier between local CPU and bus-mastering devices.
///
-/// A barrier that prevents compiler and CPU from reordering memory read accesses across the
-/// barrier.
-#[inline(always)]
-pub fn smp_rmb() {
- if cfg!(CONFIG_SMP) {
- // SAFETY: `smp_rmb()` is safe to call.
- unsafe { bindings::smp_rmb() };
- } else {
- barrier();
- }
+/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
+/// Does not prevent re-ordering with respect to other CPUs.
+///
+/// See [`mb`] for usage.
+#[inline]
+#[doc(alias = "dma_rmb")]
+#[doc(alias = "dma_wmb")]
+pub fn dma_mb<T: MemoryBarrier<Dma>>(_: T) {
+ T::run()
}
diff --git a/rust/kernel/sync/completion.rs b/rust/kernel/sync/completion.rs
index c50012a940a3..35ff049ff078 100644
--- a/rust/kernel/sync/completion.rs
+++ b/rust/kernel/sync/completion.rs
@@ -94,6 +94,7 @@ impl Completion {
///
/// This method wakes up all tasks waiting on this completion; after this operation the
/// completion is permanently done, i.e. signals all current and future waiters.
+ #[inline]
pub fn complete_all(&self) {
// SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`.
unsafe { bindings::complete_all(self.as_raw()) };
@@ -105,6 +106,7 @@ impl Completion {
/// timeout.
///
/// See also [`Completion::complete_all`].
+ #[inline]
pub fn wait_for_completion(&self) {
// SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`.
unsafe { bindings::wait_for_completion(self.as_raw()) };
diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global.rs
index aecbdc34738f..ebb10521d8bd 100644
--- a/rust/kernel/sync/lock/global.rs
+++ b/rust/kernel/sync/lock/global.rs
@@ -85,6 +85,7 @@ impl<B: GlobalLockBackend> GlobalLock<B> {
}
/// Try to lock this global lock.
+ #[must_use = "if unused, the lock will be immediately unlocked"]
#[inline]
pub fn try_lock(&'static self) -> Option<GlobalGuard<B>> {
Some(GlobalGuard {
@@ -96,6 +97,7 @@ impl<B: GlobalLockBackend> GlobalLock<B> {
/// A guard for a [`GlobalLock`].
///
/// See [`global_lock!`] for examples.
+#[must_use = "the lock unlocks immediately when the guard is unused"]
pub struct GlobalGuard<B: GlobalLockBackend> {
inner: Guard<'static, B::Item, B::Backend>,
}
@@ -304,4 +306,7 @@ macro_rules! global_lock_inner {
(backend SpinLock) => {
$crate::sync::lock::spinlock::SpinLockBackend
};
+ (backend SpinLockIrq) => {
+ $crate::sync::lock::spinlock::SpinLockIrqBackend
+ };
}
diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs
index ef76fa07ca3a..aafc80125f59 100644
--- a/rust/kernel/sync/lock/spinlock.rs
+++ b/rust/kernel/sync/lock/spinlock.rs
@@ -3,6 +3,11 @@
//! A kernel spinlock.
//!
//! This module allows Rust code to use the kernel's `spinlock_t`.
+use super::*;
+use crate::{
+ interrupt::LocalInterruptDisabled,
+ prelude::*, //
+};
/// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class.
///
@@ -82,7 +87,7 @@ pub use new_spinlock;
/// ```
///
/// [`spinlock_t`]: srctree/include/linux/spinlock.h
-pub type SpinLock<T> = super::Lock<T, SpinLockBackend>;
+pub type SpinLock<T> = Lock<T, SpinLockBackend>;
/// A kernel `spinlock_t` lock backend.
pub struct SpinLockBackend;
@@ -91,13 +96,11 @@ pub struct SpinLockBackend;
///
/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLock`]. It will unlock
/// the [`SpinLock`] upon being dropped.
-///
-/// [`Guard`]: super::Guard
-pub type SpinLockGuard<'a, T> = super::Guard<'a, T, SpinLockBackend>;
+pub type SpinLockGuard<'a, T> = Guard<'a, T, SpinLockBackend>;
// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the
// default implementation that always calls the same locking method.
-unsafe impl super::Backend for SpinLockBackend {
+unsafe impl Backend for SpinLockBackend {
type State = bindings::spinlock_t;
type GuardState = ();
@@ -144,3 +147,319 @@ unsafe impl super::Backend for SpinLockBackend {
unsafe { bindings::spin_assert_is_held(ptr) }
}
}
+
+/// Creates a [`SpinLockIrq`] initialiser with the given name and a newly-created lock class.
+///
+/// It uses the name if one is given, otherwise it generates one based on the file name and line
+/// number.
+#[macro_export]
+macro_rules! new_spinlock_irq {
+ ($inner:expr $(, $name:literal)? $(,)?) => {
+ $crate::sync::SpinLockIrq::new(
+ $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!())
+ };
+}
+pub use new_spinlock_irq;
+
+/// A variant of `SpinLock` that ensures interrupts are disabled in the critical section.
+///
+/// This lock can be acquired in two ways:
+///
+/// - Using [`lock()`] like any other type of lock, in which case the bindings will modify the
+/// interrupt state to ensure that local processor interrupts remain disabled for at least as
+/// long as the [`SpinLockIrqGuard`] exists.
+/// - Using [`lock_with()`] in contexts where a [`LocalInterruptDisabled`] token is present and
+/// local processor interrupts are already known to be disabled, in which case the local
+/// interrupt state will not be touched. This method should be preferred if a
+/// [`LocalInterruptDisabled`] token is present in the scope.
+///
+/// For more info on spinlocks, see [`SpinLock`]. For more information on interrupts,
+/// [see the interrupt module](kernel::interrupt).
+///
+/// # Examples
+///
+/// The following example shows how to declare, allocate initialise and access a struct (`Example`)
+/// that contains an inner struct (`Inner`) that is protected by a spinlock that requires local
+/// processor interrupts to be disabled.
+///
+/// ```
+/// use kernel::sync::{new_spinlock_irq, SpinLockIrq};
+///
+/// struct Inner {
+/// a: u32,
+/// b: u32,
+/// }
+///
+/// #[pin_data]
+/// struct Example {
+/// #[pin]
+/// c: SpinLockIrq<Inner>,
+/// #[pin]
+/// d: SpinLockIrq<Inner>,
+/// }
+///
+/// impl Example {
+/// fn new() -> impl PinInit<Self> {
+/// pin_init!(Self {
+/// c <- new_spinlock_irq!(Inner { a: 0, b: 10 }),
+/// d <- new_spinlock_irq!(Inner { a: 20, b: 30 }),
+/// })
+/// }
+/// }
+///
+/// // Allocate a boxed `Example`
+/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?;
+///
+/// // Accessing an `Example` from a context where interrupts may not be disabled already.
+/// let c_guard = e.c.lock(); // interrupts are disabled now, +1 interrupt disable refcount
+/// let d_guard = e.d.lock(); // no interrupt state change, +1 interrupt disable refcount
+///
+/// assert_eq!(c_guard.a, 0);
+/// assert_eq!(c_guard.b, 10);
+/// assert_eq!(d_guard.a, 20);
+/// assert_eq!(d_guard.b, 30);
+///
+/// drop(c_guard); // Dropping c_guard will not re-enable interrupts just yet, since d_guard is
+/// // still in scope.
+/// drop(d_guard); // Last interrupt disable reference dropped here, so interrupts are re-enabled
+/// // now
+/// # Ok::<(), Error>(())
+/// ```
+///
+/// The next example demonstrates locking a [`SpinLockIrq`] using [`lock_with()`] in a function
+/// which can only be called when local processor interrupts are already disabled.
+///
+/// ```
+/// use kernel::sync::{new_spinlock_irq, SpinLockIrq};
+/// use kernel::interrupt::*;
+///
+/// struct Inner {
+/// a: u32,
+/// }
+///
+/// #[pin_data]
+/// struct Example {
+/// #[pin]
+/// inner: SpinLockIrq<Inner>,
+/// }
+///
+/// impl Example {
+/// fn new() -> impl PinInit<Self> {
+/// pin_init!(Self {
+/// inner <- new_spinlock_irq!(Inner { a: 20 }),
+/// })
+/// }
+/// }
+///
+/// // Accessing an `Example` from a function that can only be called in no-interrupt contexts.
+/// fn noirq_work(e: &Example, interrupt_disabled: &LocalInterruptDisabled) {
+/// // Because we know interrupts are disabled from interrupt_disable, we can skip toggling
+/// // interrupt state using lock_with() and the provided token
+/// assert_eq!(e.inner.lock_with(interrupt_disabled).a, 20);
+/// }
+///
+/// # let e = KBox::pin_init(Example::new(), GFP_KERNEL)?;
+/// # let interrupt_guard = local_interrupt_disable();
+/// # noirq_work(&e, &interrupt_guard);
+/// #
+/// # Ok::<(), Error>(())
+/// ```
+///
+/// [`lock()`]: SpinLockIrq::lock
+/// [`lock_with()`]: SpinLockIrq::lock_with
+pub type SpinLockIrq<T> = super::Lock<T, SpinLockIrqBackend>;
+
+/// A kernel `spinlock_t` lock backend that can only be acquired in interrupt disabled contexts.
+pub struct SpinLockIrqBackend;
+
+/// A [`Guard`] acquired from locking a [`SpinLockIrq`] using [`lock()`].
+///
+/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLockIrq`] using
+/// [`lock()`]. It will unlock the [`SpinLockIrq`] and decrement the local processor's interrupt
+/// disablement refcount upon being dropped.
+///
+/// [`lock()`]: SpinLockIrq::lock
+pub type SpinLockIrqGuard<'a, T> = Guard<'a, T, SpinLockIrqBackend>;
+
+// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the
+// default implementation that always calls the same locking method.
+unsafe impl Backend for SpinLockIrqBackend {
+ type State = bindings::spinlock_t;
+ type GuardState = ();
+
+ #[inline]
+ unsafe fn init(
+ ptr: *mut Self::State,
+ name: *const crate::ffi::c_char,
+ key: *mut bindings::lock_class_key,
+ ) {
+ // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and
+ // `key` are valid for read indefinitely.
+ unsafe { bindings::__spin_lock_init(ptr, name, key) }
+ }
+
+ #[inline]
+ unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState {
+ // SAFETY: The safety requirements of this function ensure that `ptr` points to valid
+ // memory, and that it has been initialised before.
+ unsafe { bindings::spin_lock_irq_disable(ptr) }
+ }
+
+ #[inline]
+ unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) {
+ // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the
+ // caller is the owner of the spinlock.
+ unsafe { bindings::spin_unlock_irq_enable(ptr) }
+ }
+
+ #[inline]
+ unsafe fn try_lock(ptr: *mut Self::State) -> Option<Self::GuardState> {
+ // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
+ let result = unsafe { bindings::spin_trylock_irq_disable(ptr) };
+
+ if result != 0 {
+ Some(())
+ } else {
+ None
+ }
+ }
+
+ #[inline]
+ unsafe fn assert_is_held(ptr: *mut Self::State) {
+ // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
+ unsafe { bindings::spin_assert_is_held(ptr) }
+ }
+}
+
+impl<T: ?Sized> Lock<T, SpinLockIrqBackend> {
+ /// Casts the lock as a `Lock<T, SpinLockBackend>`.
+ #[inline]
+ fn as_lock_in_interrupt<'a>(&'a self, _context: &'a LocalInterruptDisabled) -> &'a SpinLock<T> {
+ // SAFETY:
+ // - `Lock<T, SpinLockBackend>` and `Lock<T, SpinLockIrqBackend>` both have identical data
+ // layouts.
+ // - As long as local interrupts are disabled (which is proven to be true by _context), it
+ // is safe to treat a lock with SpinLockIrqBackend as a SpinLockBackend lock.
+ unsafe { core::mem::transmute(self) }
+ }
+
+ /// Acquires the lock without modifying local interrupt state.
+ ///
+ /// This function should be used in place of the more expensive [`Lock::lock()`] function when
+ /// possible for [`SpinLockIrq`] locks.
+ #[inline]
+ pub fn lock_with<'a>(&'a self, context: &'a LocalInterruptDisabled) -> SpinLockGuard<'a, T> {
+ self.as_lock_in_interrupt(context).lock()
+ }
+
+ /// Tries to acquire the lock without modifying local interrupt state.
+ ///
+ /// This function should be used in place of the more expensive [`Lock::try_lock()`] function
+ /// when possible for [`SpinLockIrq`] locks.
+ ///
+ /// Returns a guard that can be used to access the data protected by the lock if successful.
+ #[must_use = "if unused, the lock will be immediately unlocked"]
+ #[inline]
+ pub fn try_lock_with<'a>(
+ &'a self,
+ context: &'a LocalInterruptDisabled,
+ ) -> Option<SpinLockGuard<'a, T>> {
+ self.as_lock_in_interrupt(context).try_lock()
+ }
+}
+
+#[kunit_tests(rust_spinlock_irq_condvar)]
+mod tests {
+ use super::*;
+ use crate::{
+ sync::*,
+ workqueue::{
+ self,
+ impl_has_work,
+ new_work,
+ Work,
+ WorkItem, //
+ },
+ };
+
+ struct TestState {
+ value: u32,
+ waiter_ready: bool,
+ }
+
+ #[pin_data]
+ struct Test {
+ #[pin]
+ state: SpinLockIrq<TestState>,
+
+ #[pin]
+ state_changed: CondVar,
+
+ #[pin]
+ waiter_state_changed: CondVar,
+
+ #[pin]
+ wait_work: Work<Self>,
+ }
+
+ impl_has_work! {
+ impl HasWork<Self> for Test { self.wait_work }
+ }
+
+ impl Test {
+ pub(crate) fn new() -> Result<Arc<Self>> {
+ Arc::try_pin_init(
+ try_pin_init!(
+ Self {
+ state <- new_spinlock_irq!(TestState {
+ value: 1,
+ waiter_ready: false
+ }),
+ state_changed <- new_condvar!(),
+ waiter_state_changed <- new_condvar!(),
+ wait_work <- new_work!("IrqCondvarTest::wait_work")
+ }
+ ),
+ GFP_KERNEL,
+ )
+ }
+ }
+
+ impl WorkItem for Test {
+ type Pointer = Arc<Self>;
+
+ fn run(this: Arc<Self>) {
+ // Wait for the test to be ready to wait for us
+ let mut state = this.state.lock();
+
+ // Make sure the interrupts actually turned off
+ // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()`
+ unsafe { bindings::lockdep_assert_irqs_disabled() };
+
+ while !state.waiter_ready {
+ this.waiter_state_changed.wait(&mut state);
+ }
+
+ // Deliver the exciting value update our test has been waiting for
+ state.value += 1;
+ this.state_changed.notify_sync();
+ }
+ }
+
+ #[test]
+ fn spinlock_irq_condvar() -> Result {
+ let testdata = Test::new()?;
+
+ let _ = workqueue::system().enqueue(testdata.clone());
+
+ // Let the updater know when we're ready to wait
+ let mut state = testdata.state.lock();
+ state.waiter_ready = true;
+ testdata.waiter_state_changed.notify_sync();
+
+ // Wait for the exciting value update
+ testdata.state_changed.wait(&mut state);
+ assert_eq!(state.value, 2);
+ Ok(())
+ }
+}
diff --git a/rust/kernel/sync/locked_by.rs b/rust/kernel/sync/locked_by.rs
index 61f100a45b35..fb4a1430b3b4 100644
--- a/rust/kernel/sync/locked_by.rs
+++ b/rust/kernel/sync/locked_by.rs
@@ -3,7 +3,7 @@
//! A wrapper for data protected by a lock that does not wrap it.
use super::{lock::Backend, lock::Lock};
-use crate::build_assert;
+use crate::build_assert::build_assert;
use core::{cell::UnsafeCell, mem::size_of, ptr};
/// Allows access to some data to be serialised by a lock that does not wrap it.
diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs
index 0ec985d560c8..dc40bfaa57e6 100644
--- a/rust/kernel/sync/poll.rs
+++ b/rust/kernel/sync/poll.rs
@@ -5,12 +5,22 @@
//! Utilities for working with `struct poll_table`.
use crate::{
+ alloc::AllocError,
bindings,
fs::File,
prelude::*,
- sync::{CondVar, LockClassKey},
+ sync::{
+ rcu::synchronize_rcu,
+ CondVar,
+ LockClassKey, //
+ }, //
+ types::Opaque, //
+};
+use core::{
+ marker::PhantomData,
+ mem::ManuallyDrop,
+ ops::Deref, //
};
-use core::{marker::PhantomData, ops::Deref};
/// Creates a [`PollCondVar`] initialiser with the given name and a newly-created lock class.
#[macro_export]
@@ -66,6 +76,7 @@ impl<'a> PollTable<'a> {
///
/// [`CondVar`]: crate::sync::CondVar
#[pin_data(PinnedDrop)]
+#[repr(transparent)]
pub struct PollCondVar {
#[pin]
inner: CondVar,
@@ -99,8 +110,70 @@ impl PinnedDrop for PollCondVar {
unsafe { bindings::__wake_up_pollfree(self.inner.wait_queue_head.get()) };
// Wait for epoll items to be properly removed.
- //
- // SAFETY: Just an FFI call.
- unsafe { bindings::synchronize_rcu() };
+ synchronize_rcu();
+ }
+}
+
+/// A [`KBox<PollCondVar>`] that uses `kfree_rcu`.
+///
+/// [`KBox<PollCondVar>`]: PollCondVar
+pub struct PollCondVarBox {
+ inner: ManuallyDrop<Pin<KBox<PollCondVarBoxInner>>>,
+}
+
+#[pin_data]
+#[repr(C)]
+struct PollCondVarBoxInner {
+ #[pin]
+ inner: PollCondVar,
+ rcu: Opaque<bindings::kvfree_rcu_head>,
+}
+
+// SAFETY: PollCondVar is Send
+unsafe impl Send for PollCondVarBoxInner {}
+// SAFETY: PollCondVar is Sync
+unsafe impl Sync for PollCondVarBoxInner {}
+
+impl PollCondVarBox {
+ /// Constructs a new boxed [`PollCondVar`].
+ pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> Result<Self, AllocError> {
+ let b = KBox::pin_init(
+ pin_init!(PollCondVarBoxInner {
+ inner <- PollCondVar::new(name, key),
+ rcu: Opaque::uninit(),
+ }),
+ GFP_KERNEL,
+ )
+ .map_err(|_| AllocError)?;
+
+ Ok(PollCondVarBox {
+ inner: ManuallyDrop::new(b),
+ })
+ }
+}
+
+impl Deref for PollCondVarBox {
+ type Target = PollCondVar;
+ fn deref(&self) -> &PollCondVar {
+ &self.inner.inner
+ }
+}
+
+impl Drop for PollCondVarBox {
+ #[inline]
+ fn drop(&mut self) {
+ // SAFETY: ManuallyDrop::take ok because not already taken.
+ let boxed = unsafe { ManuallyDrop::take(&mut self.inner) };
+
+ // SAFETY: The code below frees the box without calling the actual destructor of the type,
+ // but it's okay because it re-implements the destructor using `kfree_rcu()` in place of
+ // `synchronize_rcu()`.
+ let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(boxed) });
+
+ // SAFETY: The pointer points at a valid `wait_queue_head`.
+ unsafe { bindings::__wake_up_pollfree((*ptr).inner.inner.wait_queue_head.get()) };
+
+ // SAFETY: This was allocated using `KBox::pin_init`, so it can be freed with `kvfree`.
+ unsafe { bindings::kvfree_call_rcu((*ptr).rcu.get(), ptr.cast::<ffi::c_void>()) };
}
}
diff --git a/rust/kernel/sync/rcu.rs b/rust/kernel/sync/rcu.rs
index a32bef6e490b..0daa1ac87d81 100644
--- a/rust/kernel/sync/rcu.rs
+++ b/rust/kernel/sync/rcu.rs
@@ -50,3 +50,39 @@ impl Drop for Guard {
pub fn read_lock() -> Guard {
Guard::new()
}
+
+/// Wait until all in-flight `call_rcu()` callbacks complete.
+///
+/// Note that this primitive does not necessarily wait for an RCU grace period
+/// to complete. For example, if there are no RCU callbacks queued anywhere
+/// in the system, then [`rcu_barrier()`] is within its rights to return
+/// immediately, without waiting for anything, much less an RCU grace period.
+/// In fact, [`rcu_barrier()`] will normally not result in any RCU grace periods
+/// beyond those that were already destined to be executed.
+///
+/// In kernels built with `CONFIG_RCU_LAZY=y`, this function also hurries all
+/// pending lazy RCU callbacks.
+///
+/// Note that this is one of the RCU primitives which must not be called in
+/// atomic context.
+#[inline]
+pub fn rcu_barrier() {
+ // SAFETY: `rcu_barrier()` is always safe to be called. It just might wait for a grace period.
+ unsafe { bindings::rcu_barrier() };
+}
+
+/// Wait for one RCU grace period.
+///
+/// Waits for all RCU read-side critical sections (such as those established by
+/// a [`Guard`]) at the moment of the function call to finish.
+///
+/// Does not prevent new read-side critical sections from starting, which may
+/// begin and run while this call is blocking.
+///
+/// Note that this is one of the RCU primitives which must not be called in
+/// atomic context.
+#[inline]
+pub fn synchronize_rcu() {
+ // SAFETY: `synchronize_rcu()` is always safe to be called from process context.
+ unsafe { bindings::synchronize_rcu() };
+}
diff --git a/rust/kernel/sync/refcount.rs b/rust/kernel/sync/refcount.rs
index 6c7ae8b05a0b..23a5d201f343 100644
--- a/rust/kernel/sync/refcount.rs
+++ b/rust/kernel/sync/refcount.rs
@@ -4,9 +4,11 @@
//!
//! C header: [`include/linux/refcount.h`](srctree/include/linux/refcount.h)
-use crate::build_assert;
-use crate::sync::atomic::Atomic;
-use crate::types::Opaque;
+use crate::{
+ build_assert::build_assert,
+ sync::atomic::Atomic,
+ types::Opaque, //
+};
/// Atomic reference counter.
///
diff --git a/rust/kernel/sync/srcu.rs b/rust/kernel/sync/srcu.rs
new file mode 100644
index 000000000000..723e5e277fd6
--- /dev/null
+++ b/rust/kernel/sync/srcu.rs
@@ -0,0 +1,171 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Sleepable read-copy update (SRCU) support.
+//!
+//! C header: [`include/linux/srcu.h`](srctree/include/linux/srcu.h)
+
+use crate::{
+ bindings,
+ error::to_result,
+ prelude::*,
+ sync::LockClassKey,
+ types::{
+ NotThreadSafe,
+ Opaque, //
+ },
+};
+
+use pin_init::pin_data;
+
+/// Creates an [`Srcu`] initialiser with the given name and a newly-created lock class.
+#[doc(hidden)]
+#[macro_export]
+macro_rules! new_srcu {
+ ($($name:literal)?) => {
+ $crate::sync::Srcu::new($crate::optional_name!($($name)?), $crate::static_lock_class!())
+ };
+}
+pub use new_srcu;
+
+/// Sleepable read-copy update primitive.
+///
+/// SRCU readers may sleep while holding the read-side guard.
+///
+/// The destructor waits for active readers and callbacks, so it may sleep.
+/// If a read-side guard has been leaked, dropping an [`Srcu`] may never return.
+///
+/// # Invariants
+///
+/// This represents a valid `struct srcu_struct` initialized by the C SRCU API
+/// and it remains pinned and valid until the pinned destructor runs.
+#[repr(transparent)]
+#[pin_data(PinnedDrop)]
+pub struct Srcu {
+ #[pin]
+ inner: Opaque<bindings::srcu_struct>,
+}
+
+impl Srcu {
+ /// Creates a new SRCU instance.
+ #[inline]
+ pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self, Error> {
+ try_pin_init!(Self {
+ // INVARIANT: On success, the C initializer creates a valid `srcu_struct` and
+ // it remains pinned until `PinnedDrop` runs.
+ inner <- Opaque::try_ffi_init(|ptr: *mut bindings::srcu_struct| {
+ // SAFETY: `ptr` points to valid uninitialised memory for a `srcu_struct`.
+ to_result(unsafe {
+ bindings::init_srcu_struct_with_key(ptr, name.as_char_ptr(), key.as_ptr())
+ })
+ }),
+ })
+ }
+
+ /// Enters an SRCU read-side critical section.
+ ///
+ /// Leaking the returned [`Guard`] leaves the SRCU read-side critical
+ /// section active and makes `drop` sleep forever.
+ #[inline]
+ pub fn read_lock(&self) -> Guard<'_> {
+ // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`.
+ let idx = unsafe { bindings::srcu_read_lock(self.inner.get()) };
+
+ // INVARIANT: `idx` was returned by `srcu_read_lock()` for this `Srcu`.
+ Guard {
+ srcu: self,
+ idx,
+ _not_send: NotThreadSafe,
+ }
+ }
+
+ /// Waits until all pre-existing SRCU readers have completed.
+ #[inline]
+ pub fn synchronize(&self) {
+ // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`.
+ unsafe { bindings::synchronize_srcu(self.inner.get()) };
+ }
+
+ /// Waits until all pre-existing SRCU readers have completed, expedited.
+ ///
+ /// This requests a lower-latency grace period than [`Srcu::synchronize`] typically
+ /// at the cost of higher system-wide overhead. Prefer [`Srcu::synchronize`] by default
+ /// and use this variant only when reducing reset or teardown latency is more important
+ /// than the extra cost.
+ #[inline]
+ pub fn synchronize_expedited(&self) {
+ // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`.
+ unsafe { bindings::synchronize_srcu_expedited(self.inner.get()) };
+ }
+}
+
+#[pinned_drop]
+impl PinnedDrop for Srcu {
+ fn drop(self: Pin<&mut Self>) {
+ let ptr = self.inner.get();
+
+ if crate::warn_on!(
+ // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`
+ // and `srcu_readers_active()` only checks the active reader count.
+ unsafe { bindings::srcu_readers_active(ptr) }
+ ) {
+ // `cleanup_srcu_struct()` may return early if there are still active readers.
+ // This should only happen if a guard was leaked with `mem::forget`, which is
+ // "WRONG" code and may cause a UAF because Rust will free the `srcu_struct`
+ // while it is still referenced from the C side (e.g. by `call_srcu()` callbacks).
+ //
+ // Another consequence of leaking guards is that `call_srcu()` callbacks will
+ // never run because the grace period can never complete due to permanently
+ // active readers (i.e. leaked guards).
+ //
+ // If this ever happens, that means the guard was leaked by mistake and the
+ // caller must fix the bug. Sleeping here is intentional and less harmful
+ // than risking a UAF.
+ //
+ // SAFETY: By the type invariants, `self` contains a valid and pinned
+ // `struct srcu_struct`.
+ unsafe { bindings::synchronize_srcu(ptr) };
+ }
+
+ // Ensure all SRCU callbacks have been finished before freeing.
+ // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`.
+ unsafe { bindings::srcu_barrier(ptr) };
+
+ // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`.
+ unsafe { bindings::cleanup_srcu_struct(ptr) };
+ }
+}
+
+// SAFETY: `srcu_struct` may be shared and used across threads.
+unsafe impl Send for Srcu {}
+// SAFETY: `srcu_struct` may be shared and used concurrently.
+unsafe impl Sync for Srcu {}
+
+/// Guard for an active SRCU read-side critical section on a particular [`Srcu`].
+///
+/// Leaking this guard with [`core::mem::forget`] leaves the SRCU read-side
+/// critical section active and makes dropping the associated [`Srcu`] sleep forever.
+///
+/// # Invariants
+///
+/// `idx` is the index returned by `srcu_read_lock()` for `srcu`.
+#[must_use = "if unused, the lock will be immediately unlocked"]
+pub struct Guard<'a> {
+ srcu: &'a Srcu,
+ idx: i32,
+ _not_send: NotThreadSafe,
+}
+
+impl Guard<'_> {
+ /// Explicitly releases the SRCU read-side critical section.
+ #[inline]
+ pub fn unlock(self) {}
+}
+
+impl Drop for Guard<'_> {
+ #[inline]
+ fn drop(&mut self) {
+ // SAFETY: `Guard` is only constructible through `Srcu::read_lock()`,
+ // which returns a valid index for the SRCU instance.
+ unsafe { bindings::srcu_read_unlock(self.srcu.inner.get(), self.idx) };
+ }
+}
generated by cgit v1.2.3 (git 2.0.4) at 2026-09-21 21:42:36 +0000