summaryrefslogtreecommitdiff
path: root/rust
diff options
context:
space:
mode:
Diffstat (limited to 'rust')
-rw-r--r--rust/helpers/barrier.c30
-rw-r--r--rust/helpers/helpers.c1
-rw-r--r--rust/helpers/interrupt.c13
-rw-r--r--rust/helpers/spinlock.c15
-rw-r--r--rust/helpers/sync.c5
-rw-r--r--rust/kernel/interrupt.rs89
-rw-r--r--rust/kernel/lib.rs1
-rw-r--r--rust/kernel/revocable.rs28
-rw-r--r--rust/kernel/sync.rs9
-rw-r--r--rust/kernel/sync/atomic/ordering.rs2
-rw-r--r--rust/kernel/sync/barrier.rs127
-rw-r--r--rust/kernel/sync/lock/global.rs3
-rw-r--r--rust/kernel/sync/lock/spinlock.rs329
-rw-r--r--rust/kernel/sync/poll.rs10
-rw-r--r--rust/kernel/sync/rcu.rs16
15 files changed, 629 insertions, 49 deletions
diff --git a/rust/helpers/barrier.c b/rust/helpers/barrier.c
index fed8853745c8..dbc7a3017c78 100644
--- a/rust/helpers/barrier.c
+++ b/rust/helpers/barrier.c
@@ -2,6 +2,36 @@
#include <asm/barrier.h>
+__rust_helper void rust_helper_mb(void)
+{
+ mb();
+}
+
+__rust_helper void rust_helper_rmb(void)
+{
+ rmb();
+}
+
+__rust_helper void rust_helper_wmb(void)
+{
+ wmb();
+}
+
+__rust_helper void rust_helper_dma_mb(void)
+{
+ dma_mb();
+}
+
+__rust_helper void rust_helper_dma_rmb(void)
+{
+ dma_rmb();
+}
+
+__rust_helper void rust_helper_dma_wmb(void)
+{
+ dma_wmb();
+}
+
__rust_helper void rust_helper_smp_mb(void)
{
smp_mb();
diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c
index 4b90a1390ad5..d60af162a078 100644
--- a/rust/helpers/helpers.c
+++ b/rust/helpers/helpers.c
@@ -67,6 +67,7 @@
#include "irq.c"
#include "fs.c"
#include "gpu.c"
+#include "interrupt.c"
#include "io.c"
#include "jump_label.c"
#include "kunit.c"
diff --git a/rust/helpers/interrupt.c b/rust/helpers/interrupt.c
new file mode 100644
index 000000000000..69595498620f
--- /dev/null
+++ b/rust/helpers/interrupt.c
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/spinlock.h>
+
+__rust_helper void rust_helper_local_interrupt_disable(void)
+{
+ local_interrupt_disable();
+}
+
+__rust_helper void rust_helper_local_interrupt_enable(void)
+{
+ local_interrupt_enable();
+}
diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c
index 4d13062cf253..d53400c15022 100644
--- a/rust/helpers/spinlock.c
+++ b/rust/helpers/spinlock.c
@@ -36,3 +36,18 @@ __rust_helper void rust_helper_spin_assert_is_held(spinlock_t *lock)
{
lockdep_assert_held(lock);
}
+
+__rust_helper void rust_helper_spin_lock_irq_disable(spinlock_t *lock)
+{
+ spin_lock_irq_disable(lock);
+}
+
+__rust_helper void rust_helper_spin_unlock_irq_enable(spinlock_t *lock)
+{
+ spin_unlock_irq_enable(lock);
+}
+
+__rust_helper int rust_helper_spin_trylock_irq_disable(spinlock_t *lock)
+{
+ return spin_trylock_irq_disable(lock);
+}
diff --git a/rust/helpers/sync.c b/rust/helpers/sync.c
index 82d6aff73b04..4f474fe847c4 100644
--- a/rust/helpers/sync.c
+++ b/rust/helpers/sync.c
@@ -11,3 +11,8 @@ __rust_helper void rust_helper_lockdep_unregister_key(struct lock_class_key *k)
{
lockdep_unregister_key(k);
}
+
+__rust_helper void rust_helper_lockdep_assert_irqs_disabled(void)
+{
+ lockdep_assert_irqs_disabled();
+}
diff --git a/rust/kernel/interrupt.rs b/rust/kernel/interrupt.rs
new file mode 100644
index 000000000000..a880ec3b8538
--- /dev/null
+++ b/rust/kernel/interrupt.rs
@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Interrupt controls
+//!
+//! This module allows Rust code to annotate areas of code where local processor interrupts should
+//! be disabled, along with actually disabling local processor interrupts.
+//!
+//! # ⚠️ Warning! ⚠️
+//!
+//! The usage of this module can be more complicated than meets the eye, especially surrounding
+//! [preemptible kernels]. It's recommended to take care when using the functions and types defined
+//! here and familiarize yourself with the various documentation we have before using them, along
+//! with the various documents we link to here.
+//!
+//! # Reading material
+//!
+//! - [Software interrupts and realtime (LWN)](https://lwn.net/Articles/520076)
+//!
+//! [preemptible kernels]: https://www.kernel.org/doc/html/latest/locking/preempt-locking.html
+
+use crate::types::NotThreadSafe;
+
+/// A guard that represents local processor interrupt disablement on preemptible kernels.
+///
+/// [`LocalInterruptDisabled`] is a guard type that represents that local processor interrupts have
+/// been disabled on a preemptible kernel.
+///
+/// Certain functions take an immutable reference of [`LocalInterruptDisabled`] in order to require
+/// that they may only be run in local-interrupt-disabled contexts on preemptible kernels.
+///
+/// This is a marker type; it has no size, and is simply used as a compile-time guarantee that local
+/// processor interrupts are disabled on preemptible kernels. Note that no guarantees about the
+/// state of interrupts are made by this type on non-preemptible kernels.
+///
+/// # Invariants
+///
+/// Local processor interrupts are disabled on preemptible kernels for as long as an object of this
+/// type exists.
+pub struct LocalInterruptDisabled(NotThreadSafe);
+
+/// Disable local processor interrupts on a preemptible kernel.
+///
+/// This function disables local processor interrupts on a preemptible kernel, and returns a
+/// [`LocalInterruptDisabled`] token as proof of this. On non-preemptible kernels, this function is
+/// a no-op.
+///
+/// **Usage of this function is discouraged** unless you are absolutely sure you know what you are
+/// doing, as kernel interfaces for Rust that deal with interrupt state will typically handle local
+/// processor interrupt state management on their own and managing this by hand is quite error
+/// prone.
+#[inline]
+pub fn local_interrupt_disable() -> LocalInterruptDisabled {
+ // SAFETY: It's always safe to call `local_interrupt_disable()`.
+ unsafe { bindings::local_interrupt_disable() };
+
+ LocalInterruptDisabled(NotThreadSafe)
+}
+
+impl Drop for LocalInterruptDisabled {
+ #[inline]
+ fn drop(&mut self) {
+ // SAFETY: Per type invariants, a `local_interrupt_disable()` must be called to create this
+ // object, hence calling the corresponding `local_interrupt_enable()` is safe.
+ unsafe { bindings::local_interrupt_enable() };
+ }
+}
+
+impl LocalInterruptDisabled {
+ /// Assume that local processor interrupts are disabled on preemptible kernels.
+ ///
+ /// This can be used for annotating code that is known to be run in contexts where local
+ /// processor interrupts are disabled on preemptible kernels. It makes no changes to the local
+ /// interrupt state on its own.
+ ///
+ /// # Safety
+ ///
+ /// For the whole life `'a`, local interrupts must be disabled on preemptible kernels. This
+ /// could be a context like, for example, an interrupt handler.
+ #[inline]
+ pub unsafe fn assume_disabled<'a>() -> &'a LocalInterruptDisabled {
+ const ASSUME_DISABLED: &LocalInterruptDisabled = &LocalInterruptDisabled(NotThreadSafe);
+
+ // Confirm they're actually disabled if lockdep is available
+ // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()`.
+ unsafe { bindings::lockdep_assert_irqs_disabled() };
+
+ ASSUME_DISABLED
+ }
+}
diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
index 59144e1e3d36..73fd046ee8ed 100644
--- a/rust/kernel/lib.rs
+++ b/rust/kernel/lib.rs
@@ -82,6 +82,7 @@ pub mod id_pool;
pub mod impl_flags;
pub mod init;
pub mod interop;
+pub mod interrupt;
pub mod io;
pub mod ioctl;
pub mod iommu;
diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs
index 0f4ae673256d..0e55e2a0fb37 100644
--- a/rust/kernel/revocable.rs
+++ b/rust/kernel/revocable.rs
@@ -7,12 +7,21 @@
use pin_init::Wrapper;
-use crate::{bindings, prelude::*, sync::rcu, types::Opaque};
+use crate::{
+ prelude::*,
+ sync::{
+ atomic::{
+ AtomicFlag,
+ Relaxed, //
+ },
+ rcu, //
+ },
+ types::Opaque, //
+};
use core::{
marker::PhantomData,
ops::Deref,
- ptr::drop_in_place,
- sync::atomic::{AtomicBool, Ordering},
+ ptr::drop_in_place, //
};
/// An object that can become inaccessible at runtime.
@@ -65,7 +74,7 @@ use core::{
/// ```
#[pin_data(PinnedDrop)]
pub struct Revocable<T> {
- is_available: AtomicBool,
+ is_available: AtomicFlag,
#[pin]
data: Opaque<T>,
}
@@ -84,7 +93,7 @@ impl<T> Revocable<T> {
/// Creates a new revocable instance of the given data.
pub fn new<E>(data: impl PinInit<T, E>) -> impl PinInit<Self, E> {
try_pin_init!(Self {
- is_available: AtomicBool::new(true),
+ is_available: AtomicFlag::new(true),
data <- Opaque::pin_init(data),
}? E)
}
@@ -98,7 +107,7 @@ impl<T> Revocable<T> {
/// because another CPU may be waiting to complete the revocation of this object.
pub fn try_access(&self) -> Option<RevocableGuard<'_, T>> {
let guard = rcu::read_lock();
- if self.is_available.load(Ordering::Relaxed) {
+ if self.is_available.load(Relaxed) {
// Since `self.is_available` is true, data is initialised and has to remain valid
// because the RCU read side lock prevents it from being dropped.
Some(RevocableGuard::new(self.data.get(), guard))
@@ -116,7 +125,7 @@ impl<T> Revocable<T> {
/// allowed to sleep because another CPU may be waiting to complete the revocation of this
/// object.
pub fn try_access_with_guard<'a>(&'a self, _guard: &'a rcu::Guard) -> Option<&'a T> {
- if self.is_available.load(Ordering::Relaxed) {
+ if self.is_available.load(Relaxed) {
// SAFETY: Since `self.is_available` is true, data is initialised and has to remain
// valid because the RCU read side lock prevents it from being dropped.
Some(unsafe { &*self.data.get() })
@@ -157,12 +166,11 @@ impl<T> Revocable<T> {
///
/// Callers must ensure that there are no more concurrent users of the revocable object.
unsafe fn revoke_internal<const SYNC: bool>(&self) -> bool {
- let revoke = self.is_available.swap(false, Ordering::Relaxed);
+ let revoke = self.is_available.xchg(false, Relaxed);
if revoke {
if SYNC {
- // SAFETY: Just an FFI call, there are no further requirements.
- unsafe { bindings::synchronize_rcu() };
+ rcu::synchronize_rcu();
}
// SAFETY: We know `self.data` is valid because only one CPU can succeed the
diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs
index 993dbf2caa0e..df4f2604ff9b 100644
--- a/rust/kernel/sync.rs
+++ b/rust/kernel/sync.rs
@@ -27,7 +27,14 @@ pub use completion::Completion;
pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult};
pub use lock::global::{global_lock, GlobalGuard, GlobalLock, GlobalLockBackend, GlobalLockedBy};
pub use lock::mutex::{new_mutex, Mutex, MutexGuard};
-pub use lock::spinlock::{new_spinlock, SpinLock, SpinLockGuard};
+pub use lock::spinlock::{
+ new_spinlock,
+ new_spinlock_irq,
+ SpinLock,
+ SpinLockGuard,
+ SpinLockIrq,
+ SpinLockIrqGuard, //
+};
pub use locked_by::LockedBy;
pub use refcount::Refcount;
pub use set_once::SetOnce;
diff --git a/rust/kernel/sync/atomic/ordering.rs b/rust/kernel/sync/atomic/ordering.rs
index 3f103aa8db99..c4e732e7212f 100644
--- a/rust/kernel/sync/atomic/ordering.rs
+++ b/rust/kernel/sync/atomic/ordering.rs
@@ -15,7 +15,7 @@
//! - It provides ordering between the annotated operation and all the following memory accesses.
//! - It provides ordering between all the preceding memory accesses and all the following memory
//! accesses.
-//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb()`).
+//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb(Full)`).
//! - [`Relaxed`] provides no ordering except the dependency orderings. Dependency orderings are
//! described in "DEPENDENCY RELATIONS" in [`LKMM`]'s [`explanation`].
//!
diff --git a/rust/kernel/sync/barrier.rs b/rust/kernel/sync/barrier.rs
index 8f2d435fcd94..1180695d533a 100644
--- a/rust/kernel/sync/barrier.rs
+++ b/rust/kernel/sync/barrier.rs
@@ -7,6 +7,38 @@
//!
//! [`LKMM`]: srctree/tools/memory-model/
+#![expect(private_bounds, reason = "sealed implementation")]
+
+/// Memory barrier orderings.
+///
+/// The semantics of these orderings follows the [`LKMM`] definitions and rules.
+///
+/// - [`Read`] provides ordering between preceding load operations and succeeding load operations.
+/// - [`Write`] provides ordering between preceding store operations and succeeding store
+/// operations.
+/// - [`Full`] provides ordering between all the preceding memory accesses and succeeding memory
+/// accesses.
+///
+/// [`LKMM`]: srctree/tools/memory-model/
+pub mod ordering {
+ pub use crate::sync::atomic::ordering::Full;
+
+ /// The annotation type for read-read barrier ordering.
+ pub struct Read;
+
+ /// The annotation type for write-write barrier ordering.
+ pub struct Write;
+}
+
+pub use ordering::{
+ Full,
+ Read,
+ Write, //
+};
+
+struct Smp;
+struct Dma;
+
/// A compiler barrier.
///
/// A barrier that prevents compiler from reordering memory accesses across the barrier.
@@ -19,43 +51,82 @@ pub(crate) fn barrier() {
unsafe { core::arch::asm!("") };
}
-/// A full memory barrier.
+trait MemoryBarrier<Flavour = ()> {
+ fn run();
+}
+
+macro_rules! define_barrier {
+ ($([$flavour:ident])? $ordering:ident, $binding:ident) => {
+ impl MemoryBarrier$(<$flavour>)? for $ordering {
+ #[inline]
+ fn run() {
+ // SAFETY: barrier methods are safe to call.
+ unsafe { bindings::$binding() };
+ }
+ }
+ };
+}
+
+define_barrier!(Full, mb);
+define_barrier!(Read, rmb);
+define_barrier!(Write, wmb);
+define_barrier!([Dma] Full, dma_mb);
+define_barrier!([Dma] Read, dma_rmb);
+define_barrier!([Dma] Write, dma_wmb);
+define_barrier!([Smp] Full, smp_mb);
+define_barrier!([Smp] Read, smp_rmb);
+define_barrier!([Smp] Write, smp_wmb);
+
+/// Memory barrier.
///
/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
-#[inline(always)]
-pub fn smp_mb() {
- if cfg!(CONFIG_SMP) {
- // SAFETY: `smp_mb()` is safe to call.
- unsafe { bindings::smp_mb() };
- } else {
- barrier();
- }
+///
+/// The specific forms of reordering can be specified using the parameter.
+/// - `mb(Read)` provides a read-read barrier.
+/// - `mb(Write)` provides a write-write barrier.
+/// - `mb(Full)` provides a full barrier.
+///
+/// # Examples
+///
+/// ```
+/// # use kernel::sync::barrier::*;
+/// mb(Read);
+/// mb(Write);
+/// mb(Full);
+/// ```
+#[inline]
+#[doc(alias = "rmb")]
+#[doc(alias = "wmb")]
+pub fn mb<T: MemoryBarrier>(_: T) {
+ T::run()
}
-/// A write-write memory barrier.
+/// Memory barrier between CPUs.
///
-/// A barrier that prevents compiler and CPU from reordering memory write accesses across the
-/// barrier.
-#[inline(always)]
-pub fn smp_wmb() {
+/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
+/// Does not prevent re-ordering with respect to other bus-mastering devices.
+///
+/// See [`mb`] for usage.
+#[inline]
+#[doc(alias = "smp_rmb")]
+#[doc(alias = "smp_wmb")]
+pub fn smp_mb<T: MemoryBarrier<Smp>>(_: T) {
if cfg!(CONFIG_SMP) {
- // SAFETY: `smp_wmb()` is safe to call.
- unsafe { bindings::smp_wmb() };
+ T::run()
} else {
- barrier();
+ barrier()
}
}
-/// A read-read memory barrier.
+/// Memory barrier between local CPU and bus-mastering devices.
///
-/// A barrier that prevents compiler and CPU from reordering memory read accesses across the
-/// barrier.
-#[inline(always)]
-pub fn smp_rmb() {
- if cfg!(CONFIG_SMP) {
- // SAFETY: `smp_rmb()` is safe to call.
- unsafe { bindings::smp_rmb() };
- } else {
- barrier();
- }
+/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
+/// Does not prevent re-ordering with respect to other CPUs.
+///
+/// See [`mb`] for usage.
+#[inline]
+#[doc(alias = "dma_rmb")]
+#[doc(alias = "dma_wmb")]
+pub fn dma_mb<T: MemoryBarrier<Dma>>(_: T) {
+ T::run()
}
diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global.rs
index ec2dd84316fc..ebb10521d8bd 100644
--- a/rust/kernel/sync/lock/global.rs
+++ b/rust/kernel/sync/lock/global.rs
@@ -306,4 +306,7 @@ macro_rules! global_lock_inner {
(backend SpinLock) => {
$crate::sync::lock::spinlock::SpinLockBackend
};
+ (backend SpinLockIrq) => {
+ $crate::sync::lock::spinlock::SpinLockIrqBackend
+ };
}
diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs
index ef76fa07ca3a..aafc80125f59 100644
--- a/rust/kernel/sync/lock/spinlock.rs
+++ b/rust/kernel/sync/lock/spinlock.rs
@@ -3,6 +3,11 @@
//! A kernel spinlock.
//!
//! This module allows Rust code to use the kernel's `spinlock_t`.
+use super::*;
+use crate::{
+ interrupt::LocalInterruptDisabled,
+ prelude::*, //
+};
/// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class.
///
@@ -82,7 +87,7 @@ pub use new_spinlock;
/// ```
///
/// [`spinlock_t`]: srctree/include/linux/spinlock.h
-pub type SpinLock<T> = super::Lock<T, SpinLockBackend>;
+pub type SpinLock<T> = Lock<T, SpinLockBackend>;
/// A kernel `spinlock_t` lock backend.
pub struct SpinLockBackend;
@@ -91,13 +96,11 @@ pub struct SpinLockBackend;
///
/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLock`]. It will unlock
/// the [`SpinLock`] upon being dropped.
-///
-/// [`Guard`]: super::Guard
-pub type SpinLockGuard<'a, T> = super::Guard<'a, T, SpinLockBackend>;
+pub type SpinLockGuard<'a, T> = Guard<'a, T, SpinLockBackend>;
// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the
// default implementation that always calls the same locking method.
-unsafe impl super::Backend for SpinLockBackend {
+unsafe impl Backend for SpinLockBackend {
type State = bindings::spinlock_t;
type GuardState = ();
@@ -144,3 +147,319 @@ unsafe impl super::Backend for SpinLockBackend {
unsafe { bindings::spin_assert_is_held(ptr) }
}
}
+
+/// Creates a [`SpinLockIrq`] initialiser with the given name and a newly-created lock class.
+///
+/// It uses the name if one is given, otherwise it generates one based on the file name and line
+/// number.
+#[macro_export]
+macro_rules! new_spinlock_irq {
+ ($inner:expr $(, $name:literal)? $(,)?) => {
+ $crate::sync::SpinLockIrq::new(
+ $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!())
+ };
+}
+pub use new_spinlock_irq;
+
+/// A variant of `SpinLock` that ensures interrupts are disabled in the critical section.
+///
+/// This lock can be acquired in two ways:
+///
+/// - Using [`lock()`] like any other type of lock, in which case the bindings will modify the
+/// interrupt state to ensure that local processor interrupts remain disabled for at least as
+/// long as the [`SpinLockIrqGuard`] exists.
+/// - Using [`lock_with()`] in contexts where a [`LocalInterruptDisabled`] token is present and
+/// local processor interrupts are already known to be disabled, in which case the local
+/// interrupt state will not be touched. This method should be preferred if a
+/// [`LocalInterruptDisabled`] token is present in the scope.
+///
+/// For more info on spinlocks, see [`SpinLock`]. For more information on interrupts,
+/// [see the interrupt module](kernel::interrupt).
+///
+/// # Examples
+///
+/// The following example shows how to declare, allocate initialise and access a struct (`Example`)
+/// that contains an inner struct (`Inner`) that is protected by a spinlock that requires local
+/// processor interrupts to be disabled.
+///
+/// ```
+/// use kernel::sync::{new_spinlock_irq, SpinLockIrq};
+///
+/// struct Inner {
+/// a: u32,
+/// b: u32,
+/// }
+///
+/// #[pin_data]
+/// struct Example {
+/// #[pin]
+/// c: SpinLockIrq<Inner>,
+/// #[pin]
+/// d: SpinLockIrq<Inner>,
+/// }
+///
+/// impl Example {
+/// fn new() -> impl PinInit<Self> {
+/// pin_init!(Self {
+/// c <- new_spinlock_irq!(Inner { a: 0, b: 10 }),
+/// d <- new_spinlock_irq!(Inner { a: 20, b: 30 }),
+/// })
+/// }
+/// }
+///
+/// // Allocate a boxed `Example`
+/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?;
+///
+/// // Accessing an `Example` from a context where interrupts may not be disabled already.
+/// let c_guard = e.c.lock(); // interrupts are disabled now, +1 interrupt disable refcount
+/// let d_guard = e.d.lock(); // no interrupt state change, +1 interrupt disable refcount
+///
+/// assert_eq!(c_guard.a, 0);
+/// assert_eq!(c_guard.b, 10);
+/// assert_eq!(d_guard.a, 20);
+/// assert_eq!(d_guard.b, 30);
+///
+/// drop(c_guard); // Dropping c_guard will not re-enable interrupts just yet, since d_guard is
+/// // still in scope.
+/// drop(d_guard); // Last interrupt disable reference dropped here, so interrupts are re-enabled
+/// // now
+/// # Ok::<(), Error>(())
+/// ```
+///
+/// The next example demonstrates locking a [`SpinLockIrq`] using [`lock_with()`] in a function
+/// which can only be called when local processor interrupts are already disabled.
+///
+/// ```
+/// use kernel::sync::{new_spinlock_irq, SpinLockIrq};
+/// use kernel::interrupt::*;
+///
+/// struct Inner {
+/// a: u32,
+/// }
+///
+/// #[pin_data]
+/// struct Example {
+/// #[pin]
+/// inner: SpinLockIrq<Inner>,
+/// }
+///
+/// impl Example {
+/// fn new() -> impl PinInit<Self> {
+/// pin_init!(Self {
+/// inner <- new_spinlock_irq!(Inner { a: 20 }),
+/// })
+/// }
+/// }
+///
+/// // Accessing an `Example` from a function that can only be called in no-interrupt contexts.
+/// fn noirq_work(e: &Example, interrupt_disabled: &LocalInterruptDisabled) {
+/// // Because we know interrupts are disabled from interrupt_disable, we can skip toggling
+/// // interrupt state using lock_with() and the provided token
+/// assert_eq!(e.inner.lock_with(interrupt_disabled).a, 20);
+/// }
+///
+/// # let e = KBox::pin_init(Example::new(), GFP_KERNEL)?;
+/// # let interrupt_guard = local_interrupt_disable();
+/// # noirq_work(&e, &interrupt_guard);
+/// #
+/// # Ok::<(), Error>(())
+/// ```
+///
+/// [`lock()`]: SpinLockIrq::lock
+/// [`lock_with()`]: SpinLockIrq::lock_with
+pub type SpinLockIrq<T> = super::Lock<T, SpinLockIrqBackend>;
+
+/// A kernel `spinlock_t` lock backend that can only be acquired in interrupt disabled contexts.
+pub struct SpinLockIrqBackend;
+
+/// A [`Guard`] acquired from locking a [`SpinLockIrq`] using [`lock()`].
+///
+/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLockIrq`] using
+/// [`lock()`]. It will unlock the [`SpinLockIrq`] and decrement the local processor's interrupt
+/// disablement refcount upon being dropped.
+///
+/// [`lock()`]: SpinLockIrq::lock
+pub type SpinLockIrqGuard<'a, T> = Guard<'a, T, SpinLockIrqBackend>;
+
+// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the
+// default implementation that always calls the same locking method.
+unsafe impl Backend for SpinLockIrqBackend {
+ type State = bindings::spinlock_t;
+ type GuardState = ();
+
+ #[inline]
+ unsafe fn init(
+ ptr: *mut Self::State,
+ name: *const crate::ffi::c_char,
+ key: *mut bindings::lock_class_key,
+ ) {
+ // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and
+ // `key` are valid for read indefinitely.
+ unsafe { bindings::__spin_lock_init(ptr, name, key) }
+ }
+
+ #[inline]
+ unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState {
+ // SAFETY: The safety requirements of this function ensure that `ptr` points to valid
+ // memory, and that it has been initialised before.
+ unsafe { bindings::spin_lock_irq_disable(ptr) }
+ }
+
+ #[inline]
+ unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) {
+ // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the
+ // caller is the owner of the spinlock.
+ unsafe { bindings::spin_unlock_irq_enable(ptr) }
+ }
+
+ #[inline]
+ unsafe fn try_lock(ptr: *mut Self::State) -> Option<Self::GuardState> {
+ // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
+ let result = unsafe { bindings::spin_trylock_irq_disable(ptr) };
+
+ if result != 0 {
+ Some(())
+ } else {
+ None
+ }
+ }
+
+ #[inline]
+ unsafe fn assert_is_held(ptr: *mut Self::State) {
+ // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
+ unsafe { bindings::spin_assert_is_held(ptr) }
+ }
+}
+
+impl<T: ?Sized> Lock<T, SpinLockIrqBackend> {
+ /// Casts the lock as a `Lock<T, SpinLockBackend>`.
+ #[inline]
+ fn as_lock_in_interrupt<'a>(&'a self, _context: &'a LocalInterruptDisabled) -> &'a SpinLock<T> {
+ // SAFETY:
+ // - `Lock<T, SpinLockBackend>` and `Lock<T, SpinLockIrqBackend>` both have identical data
+ // layouts.
+ // - As long as local interrupts are disabled (which is proven to be true by _context), it
+ // is safe to treat a lock with SpinLockIrqBackend as a SpinLockBackend lock.
+ unsafe { core::mem::transmute(self) }
+ }
+
+ /// Acquires the lock without modifying local interrupt state.
+ ///
+ /// This function should be used in place of the more expensive [`Lock::lock()`] function when
+ /// possible for [`SpinLockIrq`] locks.
+ #[inline]
+ pub fn lock_with<'a>(&'a self, context: &'a LocalInterruptDisabled) -> SpinLockGuard<'a, T> {
+ self.as_lock_in_interrupt(context).lock()
+ }
+
+ /// Tries to acquire the lock without modifying local interrupt state.
+ ///
+ /// This function should be used in place of the more expensive [`Lock::try_lock()`] function
+ /// when possible for [`SpinLockIrq`] locks.
+ ///
+ /// Returns a guard that can be used to access the data protected by the lock if successful.
+ #[must_use = "if unused, the lock will be immediately unlocked"]
+ #[inline]
+ pub fn try_lock_with<'a>(
+ &'a self,
+ context: &'a LocalInterruptDisabled,
+ ) -> Option<SpinLockGuard<'a, T>> {
+ self.as_lock_in_interrupt(context).try_lock()
+ }
+}
+
+#[kunit_tests(rust_spinlock_irq_condvar)]
+mod tests {
+ use super::*;
+ use crate::{
+ sync::*,
+ workqueue::{
+ self,
+ impl_has_work,
+ new_work,
+ Work,
+ WorkItem, //
+ },
+ };
+
+ struct TestState {
+ value: u32,
+ waiter_ready: bool,
+ }
+
+ #[pin_data]
+ struct Test {
+ #[pin]
+ state: SpinLockIrq<TestState>,
+
+ #[pin]
+ state_changed: CondVar,
+
+ #[pin]
+ waiter_state_changed: CondVar,
+
+ #[pin]
+ wait_work: Work<Self>,
+ }
+
+ impl_has_work! {
+ impl HasWork<Self> for Test { self.wait_work }
+ }
+
+ impl Test {
+ pub(crate) fn new() -> Result<Arc<Self>> {
+ Arc::try_pin_init(
+ try_pin_init!(
+ Self {
+ state <- new_spinlock_irq!(TestState {
+ value: 1,
+ waiter_ready: false
+ }),
+ state_changed <- new_condvar!(),
+ waiter_state_changed <- new_condvar!(),
+ wait_work <- new_work!("IrqCondvarTest::wait_work")
+ }
+ ),
+ GFP_KERNEL,
+ )
+ }
+ }
+
+ impl WorkItem for Test {
+ type Pointer = Arc<Self>;
+
+ fn run(this: Arc<Self>) {
+ // Wait for the test to be ready to wait for us
+ let mut state = this.state.lock();
+
+ // Make sure the interrupts actually turned off
+ // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()`
+ unsafe { bindings::lockdep_assert_irqs_disabled() };
+
+ while !state.waiter_ready {
+ this.waiter_state_changed.wait(&mut state);
+ }
+
+ // Deliver the exciting value update our test has been waiting for
+ state.value += 1;
+ this.state_changed.notify_sync();
+ }
+ }
+
+ #[test]
+ fn spinlock_irq_condvar() -> Result {
+ let testdata = Test::new()?;
+
+ let _ = workqueue::system().enqueue(testdata.clone());
+
+ // Let the updater know when we're ready to wait
+ let mut state = testdata.state.lock();
+ state.waiter_ready = true;
+ testdata.waiter_state_changed.notify_sync();
+
+ // Wait for the exciting value update
+ testdata.state_changed.wait(&mut state);
+ assert_eq!(state.value, 2);
+ Ok(())
+ }
+}
diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs
index 0ec985d560c8..5aa0ce9ba01b 100644
--- a/rust/kernel/sync/poll.rs
+++ b/rust/kernel/sync/poll.rs
@@ -8,7 +8,11 @@ use crate::{
bindings,
fs::File,
prelude::*,
- sync::{CondVar, LockClassKey},
+ sync::{
+ rcu::synchronize_rcu,
+ CondVar,
+ LockClassKey, //
+ }, //
};
use core::{marker::PhantomData, ops::Deref};
@@ -99,8 +103,6 @@ impl PinnedDrop for PollCondVar {
unsafe { bindings::__wake_up_pollfree(self.inner.wait_queue_head.get()) };
// Wait for epoll items to be properly removed.
- //
- // SAFETY: Just an FFI call.
- unsafe { bindings::synchronize_rcu() };
+ synchronize_rcu();
}
}
diff --git a/rust/kernel/sync/rcu.rs b/rust/kernel/sync/rcu.rs
index 42d1b26a2143..0daa1ac87d81 100644
--- a/rust/kernel/sync/rcu.rs
+++ b/rust/kernel/sync/rcu.rs
@@ -70,3 +70,19 @@ pub fn rcu_barrier() {
// SAFETY: `rcu_barrier()` is always safe to be called. It just might wait for a grace period.
unsafe { bindings::rcu_barrier() };
}
+
+/// Wait for one RCU grace period.
+///
+/// Waits for all RCU read-side critical sections (such as those established by
+/// a [`Guard`]) at the moment of the function call to finish.
+///
+/// Does not prevent new read-side critical sections from starting, which may
+/// begin and run while this call is blocking.
+///
+/// Note that this is one of the RCU primitives which must not be called in
+/// atomic context.
+#[inline]
+pub fn synchronize_rcu() {
+ // SAFETY: `synchronize_rcu()` is always safe to be called from process context.
+ unsafe { bindings::synchronize_rcu() };
+}
generated by cgit v1.2.3 (git 2.0.4) at 2026-08-20 23:31:00 +0000