diff options
Diffstat (limited to 'rust/kernel')
62 files changed, 4002 insertions, 1438 deletions
diff --git a/rust/kernel/Kconfig.test b/rust/kernel/Kconfig.test index e6a5c7a795f0..3bf4194f28c7 100644 --- a/rust/kernel/Kconfig.test +++ b/rust/kernel/Kconfig.test @@ -83,4 +83,25 @@ config RUST_BITFIELD_KUNIT_TEST If unsure, say N. +config RUST_BUG_KUNIT_TEST + bool "KUnit tests for the Rust BUG/WARN functionality" if !KUNIT_ALL_TESTS + depends on BUG + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust BUG/WARN functionality. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_TIME_KUNIT_TEST + bool "KUnit tests for the Rust time API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust time API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + endif diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 21067bde6860..68d80f3fa6c8 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -130,6 +130,12 @@ impl NumaNode { } Ok(Self(node)) } + + /// Returns the raw NUMA node identifier as an `i32`. + #[inline] + pub fn id(&self) -> i32 { + self.0 + } } /// Specify necessary constant to pass the information to Allocator that the caller doesn't care diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index c63d6acdbb6f..2c929eb1c3d6 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -345,7 +345,7 @@ where /// } /// /// // Allocate a boxed slice of 10 `Example`s. - /// let s = KBox::pin_slice( + /// let s = KBox::pin_slice::<_, _, Error, _>( /// | _i | Example::new(), /// 10, /// GFP_KERNEL @@ -355,15 +355,16 @@ where /// assert_eq!(s[3].d.lock().a, 20); /// # Ok::<(), Error>(()) /// ``` - pub fn pin_slice<Func, Item, E>( + pub fn pin_slice<Func, Item, E, E2>( mut init: Func, len: usize, flags: Flags, ) -> Result<Pin<Box<[T], A>>, E> where Func: FnMut(usize) -> Item, - Item: PinInit<T, E>, + Item: PinInit<T, E2>, E: From<AllocError>, + E: From<E2>, { let mut buffer = super::Vec::<T, A>::with_capacity(len, flags)?; for i in 0..len { diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs index 60dfbec8f330..9d3de109b381 100644 --- a/rust/kernel/auxiliary.rs +++ b/rust/kernel/auxiliary.rs @@ -137,10 +137,18 @@ pub struct DeviceId(bindings::auxiliary_device_id); impl DeviceId { /// Create a new [`DeviceId`] from name. + /// + /// This is only intended to be called in const context, when constructing a + /// device ID table, where exceeding `AUXILIARY_NAME_SIZE` is a compile time error. pub const fn new(modname: &'static CStr, name: &'static CStr) -> Self { let name = name.to_bytes_with_nul(); let modname = modname.to_bytes_with_nul(); + assert!( + modname.len().saturating_add(name.len()) <= bindings::AUXILIARY_NAME_SIZE as usize, + "auxiliary device ID is too long" + ); + let mut id: bindings::auxiliary_device_id = pin_init::zeroed(); let mut i = 0; while i < modname.len() { @@ -305,6 +313,10 @@ impl Device<device::Bound> { /// `F` is the [`ForLt`](trait@ForLt) encoding of the data type. The closure receives a pinned /// reference to the registration data. /// + /// The outer reference carries the `&self` lifetime while the inner type carries the HRTB + /// lifetime `'a`, implying `'a` outlives `&self`. This allows the closure to coerce covariant + /// sub-fields (e.g. `&'a T` to the caller's lifetime) and return them directly in `R`. + /// /// For covariant types that implement [`trait@CovariantForLt`], prefer /// [`registration_data`](Self::registration_data) which returns a direct reference. /// @@ -314,13 +326,14 @@ impl Device<device::Bound> { /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was /// registered by a C driver. #[inline] - pub fn registration_data_with<F: ForLt + 'static, R>( - &self, - f: impl for<'a> FnOnce(Pin<&'a F::Of<'a>>) -> R, + pub fn registration_data_with<'this, F: ForLt + 'static, R>( + &'this self, + f: impl for<'a> FnOnce(Pin<&'this F::Of<'a>>) -> R, ) -> Result<R> { - // SAFETY: The HRTB closure prevents the caller from smuggling in references with a - // concrete short lifetime, making the round-trip from `'static` sound regardless of - // variance. + // SAFETY: The HRTB on the inner type prevents the caller from exploiting a specific + // choice of `'a`. Covariant sub-fields can be safely coerced to `'this`, while + // invariant fields cannot be coerced and thus cannot escape with an incorrect + // lifetime. let pinned = unsafe { self.registration_data_pinned::<F>()? }; Ok(f(pinned)) diff --git a/rust/kernel/bitfield.rs b/rust/kernel/bitfield.rs index a0d089423f21..8d7d78175336 100644 --- a/rust/kernel/bitfield.rs +++ b/rust/kernel/bitfield.rs @@ -295,7 +295,6 @@ macro_rules! bitfield { $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* } ) => { $crate::bitfield!(@core - #[allow(non_camel_case_types)] $(#[$attr])* $vis $name $storage ); $crate::bitfield!(@fields $vis $name $storage { $($fields)* }); @@ -346,6 +345,15 @@ macro_rules! bitfield { Self::from_raw(val) } } + + // SAFETY: `$name` is transparent over `$storage` and `$storage` has no interior mutability. + unsafe impl $crate::mem::AsRepr for $name { + // Normalize `$storage` to the canonical repr type in case it is signed. + type Repr = <$storage as $crate::mem::AsRepr>::Repr; + } + + // SAFETY: `$name` is transparent over `$storage`. + unsafe impl $crate::mem::AsReprMut for $name {} }; // Definitions requiring knowledge of individual fields: private and public field accessors, diff --git a/rust/kernel/bitmap.rs b/rust/kernel/bitmap.rs index b27e0ec80d64..610d565535cc 100644 --- a/rust/kernel/bitmap.rs +++ b/rust/kernel/bitmap.rs @@ -10,31 +10,70 @@ use crate::alloc::{AllocError, Flags}; use crate::bindings; #[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] use crate::pr_err; -use core::ptr::NonNull; +use crate::ptr::Alignment; +use core::{ + num::NonZero, + ptr::NonNull, // +}; /// Represents a C bitmap. Wraps underlying C bitmap API. /// /// # Invariants /// /// Must reference a `[c_ulong]` long enough to fit `data.len()` bits. +/// Must not be longer than [`i32::MAX`] bits, so offsets and lengths used with +/// [`Bitmap`] functions fit in the `int` and `unsigned int` arguments of the C bitmap API. +/// This also matches [`BitmapVec::MAX_LEN`]. #[cfg_attr(CONFIG_64BIT, repr(align(8)))] #[cfg_attr(not(CONFIG_64BIT), repr(align(4)))] pub struct Bitmap { data: [()], } +macro_rules! bitmap_assert { + ($cond:expr, $($arg:tt)+) => { + #[cfg(CONFIG_RUST_BITMAP_HARDENED)] + assert!($cond, $($arg)*); + } +} + +macro_rules! bitmap_assert_return { + ($cond:expr, $($arg:tt)+) => { + #[cfg(CONFIG_RUST_BITMAP_HARDENED)] + assert!($cond, $($arg)*); + + #[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] + if !($cond) { + pr_err!($($arg)*); + return + } + } +} + impl Bitmap { /// Borrows a C bitmap. /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `nbits` exceeds [`i32::MAX`]. + /// /// # Safety /// /// * `ptr` holds a non-null address of an initialized array of `unsigned long` /// that is large enough to hold `nbits` bits. + /// * `nbits` must not exceed [`i32::MAX`]. /// * the array must not be freed for the lifetime of this [`Bitmap`] /// * concurrent access only happens through atomic operations pub unsafe fn from_raw<'a>(ptr: *const usize, nbits: usize) -> &'a Bitmap { + bitmap_assert!( + nbits <= i32::MAX as usize, + "`nbits` must be <= {}, was {}", + i32::MAX, + nbits + ); let data: *const [()] = core::ptr::slice_from_raw_parts(ptr.cast(), nbits); // INVARIANT: `data` references an initialized array that can hold `nbits` bits. + // INVARIANT: The caller guarantees that `nbits` does not exceed `i32::MAX`. // SAFETY: // The caller guarantees that `data` (derived from `ptr` and `nbits`) // points to a valid, initialized, and appropriately sized memory region @@ -51,15 +90,27 @@ impl Bitmap { /// Borrows a C bitmap exclusively. /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `nbits` exceeds [`i32::MAX`]. + /// /// # Safety /// /// * `ptr` holds a non-null address of an initialized array of `unsigned long` /// that is large enough to hold `nbits` bits. + /// * `nbits` must not exceed [`i32::MAX`]. /// * the array must not be freed for the lifetime of this [`Bitmap`] /// * no concurrent access may happen. pub unsafe fn from_raw_mut<'a>(ptr: *mut usize, nbits: usize) -> &'a mut Bitmap { + bitmap_assert!( + nbits <= i32::MAX as usize, + "`nbits` must be <= {}, was {}", + i32::MAX, + nbits + ); let data: *mut [()] = core::ptr::slice_from_raw_parts_mut(ptr.cast(), nbits); // INVARIANT: `data` references an initialized array that can hold `nbits` bits. + // INVARIANT: The caller guarantees that `nbits` does not exceed `i32::MAX`. // SAFETY: // The caller guarantees that `data` (derived from `ptr` and `nbits`) // points to a valid, initialized, and appropriately sized memory region @@ -96,26 +147,6 @@ union BitmapRepr { ptr: NonNull<usize>, } -macro_rules! bitmap_assert { - ($cond:expr, $($arg:tt)+) => { - #[cfg(CONFIG_RUST_BITMAP_HARDENED)] - assert!($cond, $($arg)*); - } -} - -macro_rules! bitmap_assert_return { - ($cond:expr, $($arg:tt)+) => { - #[cfg(CONFIG_RUST_BITMAP_HARDENED)] - assert!($cond, $($arg)*); - - #[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] - if !($cond) { - pr_err!($($arg)*); - return - } - } -} - /// Represents an owned bitmap. /// /// Wraps underlying C bitmap API. See [`Bitmap`] for available @@ -415,7 +446,8 @@ impl Bitmap { #[inline] pub fn copy_and_extend(&mut self, src: &Bitmap) { let len = core::cmp::min(src.len(), self.len()); - // SAFETY: access to `self` and `src` is within bounds. + // SAFETY: Access to `self` and `src` is within bounds. Both lengths fit in `u32` + // because a `Bitmap` is at most `i32::MAX` bits, so the casts are lossless. unsafe { bindings::bitmap_copy_and_extend( self.as_mut_ptr(), @@ -459,7 +491,13 @@ impl Bitmap { /// Finds next set bit, starting from `start`. /// - /// Returns `None` if `start` is greater or equal to `self.nbits`. + /// Returns `None` if no bits are set on or after the given index. The + /// index `start` must be in bounds. + /// + /// # Panics + /// + /// Panics if `CONFIG_RUST_BITMAP_HARDENED` is enabled and `start` is + /// out of bounds. #[inline] pub fn next_bit(&self, start: usize) -> Option<usize> { bitmap_assert!( @@ -479,7 +517,14 @@ impl Bitmap { } /// Finds next zero bit, starting from `start`. - /// Returns `None` if `start` is greater than or equal to `self.len()`. + /// + /// Returns `None` if no bits are zero on or after the given index. The + /// index `start` must be in bounds. + /// + /// # Panics + /// + /// Panics if `CONFIG_RUST_BITMAP_HARDENED` is enabled and `start` is + /// out of bounds. #[inline] pub fn next_zero_bit(&self, start: usize) -> Option<usize> { bitmap_assert!( @@ -497,13 +542,159 @@ impl Bitmap { Some(index) } } + + /// Finds a contiguous area of `nbits` zero bits at or after `start`, where the area plus + /// `align_offset` is aligned to `align`. + /// + /// Returns the bit index of the start of the area, or [`None`] if no such area fitting in + /// the bitmap exists. + /// + /// The returned index plus `align_offset` is a multiple of `align`. + /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `start` is out of bounds. + #[inline] + pub fn next_zero_area_off( + &self, + start: usize, + nbits: NonZero<usize>, + align: Alignment, + align_offset: usize, + ) -> Option<usize> { + bitmap_assert!( + start < self.len(), + "`start` must be < {}, was {}", + self.len(), + start + ); + + let nr = u32::try_from(nbits.get()).ok()?; + let align_mask = align.as_usize() - 1; + + // The C alignment and end arithmetic must not overflow, or it can read out of bounds. + // Overflow is only possible on 32-bit. + #[cfg(not(CONFIG_64BIT))] + align_mask + .checked_add(self.len())? + .checked_add(nbits.get())?; + + // SAFETY: `bitmap_find_next_zero_area_off` is safe to use with an out of bounds `start` + // value and, given the overflow check above, never reads beyond `self.len()` bits. + let index = unsafe { + bindings::bitmap_find_next_zero_area_off( + self.as_ptr().cast_mut(), + self.len(), + start, + nr, + align_mask, + align_offset, + ) + }; + + (index < self.len()).then_some(index) + } + + /// Finds a contiguous area of `nbits` zero bits at or after `start`, aligned to `align`. + /// + /// Returns the bit index of the start of the area, or [`None`] if no such area fitting in + /// the bitmap exists. + /// + /// The returned index is a multiple of `align`. + /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and `start` is out of bounds. + /// + /// # Examples + /// + /// ``` + /// use kernel::{ + /// alloc::{AllocError, flags::GFP_KERNEL}, + /// bitmap::BitmapVec, + /// ptr::Alignment, + /// sizes::SizeConstants, // + /// }; + /// + /// let mut b = BitmapVec::new(64, GFP_KERNEL)?; + /// + /// assert_eq!(Some(0), b.next_zero_area(0, cv!(8), Alignment::SZ_1)); + /// b.set(0, cv!(5)); + /// assert_eq!(Some(5), b.next_zero_area(0, cv!(8), Alignment::SZ_1)); + /// assert_eq!(Some(8), b.next_zero_area(0, cv!(8), Alignment::SZ_8)); + /// assert_eq!(None, b.next_zero_area(0, cv!(65), Alignment::SZ_1)); + /// # Ok::<(), AllocError>(()) + /// ``` + #[inline] + pub fn next_zero_area( + &self, + start: usize, + nbits: NonZero<usize>, + align: Alignment, + ) -> Option<usize> { + self.next_zero_area_off(start, nbits, align, 0) + } + + /// Sets a contiguous area of `nbits` bits starting at `start`. + /// + /// If CONFIG_RUST_BITMAP_HARDENED is not enabled and the area `start..start + nbits` is out of + /// bounds, does nothing. + /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and the area `start..start + nbits` is out + /// of bounds. + #[inline] + pub fn set(&mut self, start: usize, nbits: NonZero<usize>) { + bitmap_assert_return!( + start + .checked_add(nbits.get()) + .is_some_and(|end| end <= self.len()), + "Area `start..start + nbits` ({}..{}) must be within bounds {}", + start, + start.saturating_add(nbits.get()), + self.len() + ); + // SAFETY: The area `start..start + nbits` is within bounds and a `Bitmap` is at most + // `i32::MAX` bits, so the casts are lossless. + unsafe { bindings::__bitmap_set(self.as_mut_ptr(), start as u32, nbits.get() as i32) }; + } + + /// Clears a contiguous area of `nbits` bits starting at `start`. + /// + /// If CONFIG_RUST_BITMAP_HARDENED is not enabled and the area `start..start + nbits` is out of + /// bounds, does nothing. + /// + /// # Panics + /// + /// Panics if CONFIG_RUST_BITMAP_HARDENED is enabled and the area `start..start + nbits` is out + /// of bounds. + #[inline] + pub fn clear(&mut self, start: usize, nbits: NonZero<usize>) { + bitmap_assert_return!( + start + .checked_add(nbits.get()) + .is_some_and(|end| end <= self.len()), + "Area `start..start + nbits` ({}..{}) must be within bounds {}", + start, + start.saturating_add(nbits.get()), + self.len() + ); + // SAFETY: The area `start..start + nbits` is within bounds and a `Bitmap` is at most + // `i32::MAX` bits, so the casts are lossless. + unsafe { bindings::__bitmap_clear(self.as_mut_ptr(), start as u32, nbits.get() as i32) }; + } } #[cfg(CONFIG_RUST_BITMAP_KUNIT_TEST)] #[macros::kunit_tests(rust_kernel_bitmap)] mod tests { use super::*; - use kernel::alloc::flags::GFP_KERNEL; + use kernel::{ + alloc::flags::GFP_KERNEL, + num::cv, + sizes::SizeConstants, // + }; #[test] fn bitmap_borrow() { @@ -572,24 +763,21 @@ mod tests { } #[test] + #[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] fn owned_bitmap_out_of_bounds() -> Result<(), AllocError> { - // TODO: Kunit #[test]s do not support `cfg` yet, - // so we add it here in the body. - #[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] - { - let mut b = BitmapVec::new(128, GFP_KERNEL)?; - b.set_bit(2048); - b.set_bit_atomic(2048); - b.clear_bit(2048); - b.clear_bit_atomic(2048); - assert_eq!(None, b.next_bit(2048)); - assert_eq!(None, b.next_zero_bit(2048)); - assert_eq!(None, b.last_bit()); - } + let mut b = BitmapVec::new(128, GFP_KERNEL)?; + + b.set_bit(2048); + b.set_bit_atomic(2048); + b.clear_bit(2048); + b.clear_bit_atomic(2048); + assert_eq!(None, b.next_bit(2048)); + assert_eq!(None, b.next_zero_bit(2048)); + assert_eq!(None, b.last_bit()); Ok(()) } - // TODO: uncomment once kunit supports [should_panic] and `cfg`. + // TODO: uncomment once kunit supports `#[should_panic]`. // #[cfg(CONFIG_RUST_BITMAP_HARDENED)] // #[test] // #[should_panic] @@ -617,4 +805,91 @@ mod tests { assert_eq!(Some(17), long_bitmap.last_bit()); Ok(()) } + + #[test] + fn bitmap_area_set_clear_find() -> Result<(), AllocError> { + let mut b = BitmapVec::new(128, GFP_KERNEL)?; + + assert_eq!(Some(0), b.next_zero_area(0, cv!(5), Alignment::SZ_1)); + b.set(0, cv!(5)); // Now contains {[0, 5)}. + + assert_eq!(Some(0), b.next_bit(0)); + assert_eq!(Some(4), b.next_bit(4)); + assert_eq!(Some(5), b.next_zero_bit(0)); + assert_eq!(Some(5), b.next_zero_area(0, cv!(5), Alignment::SZ_1)); + assert_eq!(Some(8), b.next_zero_area(0, cv!(5), Alignment::SZ_8)); + + b.set(8, cv!(8)); // Now contains {[0, 5), [8, 16)}. + assert_eq!(Some(16), b.next_zero_area(0, cv!(4), Alignment::SZ_16)); + assert_eq!(Some(16), b.next_zero_area(0, cv!(4), Alignment::SZ_1)); + + b.clear(0, cv!(5)); // Now contains {[8, 16)}. + assert_eq!(Some(0), b.next_zero_area(0, cv!(5), Alignment::SZ_1)); + assert_eq!(Some(8), b.next_bit(0)); + assert_eq!(Some(15), b.last_bit()); + + b.set(60, cv!(10)); // Now contains {[8, 16), [60, 70)}. + assert_eq!(Some(60), b.next_bit(16)); + assert_eq!(Some(69), b.last_bit()); + assert_eq!(Some(16), b.next_zero_area(9, cv!(40), Alignment::SZ_1)); + assert_eq!(Some(70), b.next_zero_area(0, cv!(45), Alignment::SZ_1)); + + b.clear(62, cv!(6)); // Now contains {[8, 16), [60, 62), [68, 70)}. + assert_eq!(Some(62), b.next_zero_area(60, cv!(6), Alignment::SZ_1)); + assert_eq!(Some(61), b.next_bit(61)); + assert_eq!(Some(69), b.last_bit()); + Ok(()) + } + + #[test] + fn bitmap_area_exhaustion() -> Result<(), AllocError> { + let mut b = BitmapVec::new(64, GFP_KERNEL)?; + + assert_eq!(None, b.next_zero_area(0, cv!(65), Alignment::SZ_1)); + assert_eq!(None, b.next_zero_area(0, cv!(usize::MAX), Alignment::SZ_1)); + assert_eq!(None, b.next_zero_area(1, cv!(usize::MAX), Alignment::SZ_1)); + + b.set_bit(0); // Now contains {[0, 1)}. + assert_eq!(None, b.next_zero_area(0, cv!(usize::MAX), Alignment::SZ_1)); + + b.set(0, cv!(61)); // Now contains {[0, 61)}. + assert_eq!(None, b.next_zero_area(0, cv!(4), Alignment::SZ_1)); + assert_eq!(Some(61), b.next_zero_area(0, cv!(3), Alignment::SZ_1)); + assert_eq!(None, b.next_zero_area(0, cv!(1), Alignment::SZ_64)); + Ok(()) + } + + #[test] + fn bitmap_area_off() -> Result<(), AllocError> { + let mut b = BitmapVec::new(64, GFP_KERNEL)?; + + b.set(0, cv!(5)); // Now contains {[0, 5)}. + + // The area plus align_offset starts at a multiple of the alignment. + assert_eq!(Some(7), b.next_zero_area_off(0, cv!(8), Alignment::SZ_8, 1)); + assert_eq!(Some(5), b.next_zero_area_off(0, cv!(8), Alignment::SZ_8, 3)); + + // A zero offset behaves like next_zero_area(). + assert_eq!( + b.next_zero_area(0, cv!(8), Alignment::SZ_8), + b.next_zero_area_off(0, cv!(8), Alignment::SZ_8, 0) + ); + Ok(()) + } + + #[test] + #[cfg(not(CONFIG_RUST_BITMAP_HARDENED))] + fn owned_bitmap_area_out_of_bounds() -> Result<(), AllocError> { + let mut b = BitmapVec::new(64, GFP_KERNEL)?; + + // Should be ignored since out of bounds. + b.set(64, cv!(4)); + b.set(62, cv!(8)); + b.set(usize::MAX, cv!(1)); + b.clear(usize::MAX, cv!(1)); + b.clear(2048, cv!(8)); + assert_eq!(None, b.next_bit(0)); + assert_eq!(None, b.next_zero_area(64, cv!(1), Alignment::SZ_1)); + Ok(()) + } } diff --git a/rust/kernel/block/mq/gen_disk.rs b/rust/kernel/block/mq/gen_disk.rs index fc97dd873974..0fe640f3aec6 100644 --- a/rust/kernel/block/mq/gen_disk.rs +++ b/rust/kernel/block/mq/gen_disk.rs @@ -7,14 +7,26 @@ use crate::{ bindings, - block::mq::{Operations, TagSet}, - error::{self, from_err_ptr, Result}, - fmt::{self, Write}, + block::mq::{ + Operations, + TagSet, // + }, + error::{ + from_err_ptr, + to_result, // + }, + fmt::{ + self, + Write, // + }, prelude::*, static_lock_class, str::NullTerminatedFormatter, sync::Arc, - types::{ForeignOwnable, ScopeGuard}, + types::{ + ForeignOwnable, + ScopeGuard, // + }, // }; /// A builder for [`GenDisk`]. @@ -54,7 +66,7 @@ impl GenDiskBuilder { /// and that it is a power of two. pub fn validate_block_size(size: u32) -> Result { if !(512..=bindings::PAGE_SIZE as u32).contains(&size) || !size.is_power_of_two() { - Err(error::code::EINVAL) + Err(EINVAL) } else { Ok(()) } @@ -63,7 +75,7 @@ impl GenDiskBuilder { /// Set the logical block size of the device to be built. /// /// This method will check that block size is a power of two and between 512 - /// and 4096. If not, an error is returned and the block size is not set. + /// and `PAGE_SIZE`. If not, an error is returned and the block size is not set. /// /// This is the smallest unit the storage device can address. It is /// typically 4096 bytes. @@ -76,7 +88,7 @@ impl GenDiskBuilder { /// Set the physical block size of the device to be built. /// /// This method will check that block size is a power of two and between 512 - /// and 4096. If not, an error is returned and the block size is not set. + /// and `PAGE_SIZE`. If not, an error is returned and the block size is not set. /// /// This is the smallest unit a physical storage device can write /// atomically. It is usually the same as the logical block size but may be @@ -125,30 +137,9 @@ impl GenDiskBuilder { ) })?; - const TABLE: bindings::block_device_operations = bindings::block_device_operations { - submit_bio: None, - open: None, - release: None, - ioctl: None, - compat_ioctl: None, - check_events: None, - unlock_native_capacity: None, - getgeo: None, - set_read_only: None, - swap_slot_free_notify: None, - report_zones: None, - devnode: None, - alternative_gpt_sector: None, - get_unique_id: None, - // TODO: Set to `THIS_MODULE`. - owner: core::ptr::null_mut(), - pr_ops: core::ptr::null_mut(), - free_disk: None, - poll_bio: None, - }; - - // SAFETY: `gendisk` is a valid pointer as we initialized it above - unsafe { (*gendisk).fops = &TABLE }; + // SAFETY: `gendisk` is a valid pointer. We have exclusive access, + // since the disk is not added to the VFS yet. + unsafe { (*gendisk).fops = &GenDisk::<T>::VTABLE }; let cleanup_failure = ScopeGuard::new_with_data((gendisk, data), |(gendisk, data)| { // SAFETY: `gendisk` came from `__blk_mq_alloc_disk()` above and @@ -177,7 +168,7 @@ impl GenDiskBuilder { // operation, so we will not race. unsafe { bindings::set_capacity(gendisk, self.capacity_sectors) }; - crate::error::to_result( + to_result( // SAFETY: `gendisk` points to a valid and initialized instance of // `struct gendisk`. unsafe { @@ -211,9 +202,22 @@ pub struct GenDisk<T: Operations> { gendisk: *mut bindings::gendisk, } +impl<T: Operations> GenDisk<T> { + const VTABLE: bindings::block_device_operations = bindings::block_device_operations { + owner: crate::module::this_module::<T::OwnerModule>().as_ptr(), + ..pin_init::zeroed() + }; +} + // SAFETY: `GenDisk` is an owned pointer to a `struct gendisk` and an `Arc` to a -// `TagSet` It is safe to send this to other threads as long as T is Send. -unsafe impl<T: Operations + Send> Send for GenDisk<T> {} +// `TagSet`. It is safe to send this to other threads as long as these two are `Send`. +unsafe impl<T> Send for GenDisk<T> +where + T: Operations, + T::QueueData: Send, + Arc<TagSet<T>>: Send, +{ +} impl<T: Operations> Drop for GenDisk<T> { fn drop(&mut self) { diff --git a/rust/kernel/block/mq/operations.rs b/rust/kernel/block/mq/operations.rs index 861903e18fbf..08111d8368eb 100644 --- a/rust/kernel/block/mq/operations.rs +++ b/rust/kernel/block/mq/operations.rs @@ -6,11 +6,17 @@ use crate::{ bindings, - block::mq::{request::RequestDataWrapper, Request}, - error::{from_result, Result}, + block::mq::{ + request::RequestDataWrapper, + Request, // + }, + error::from_result, prelude::*, - sync::{aref::ARef, Refcount}, - types::ForeignOwnable, + sync::{ + aref::ARef, + Refcount, // + }, + types::ForeignOwnable, // }; use core::marker::PhantomData; @@ -30,7 +36,7 @@ type ForeignBorrowed<'a, T> = <T as ForeignOwnable>::Borrowed<'a>; pub trait Operations: Sized { /// Data associated with the `struct request_queue` that is allocated for /// the `GenDisk` associated with this `Operations` implementation. - type QueueData: ForeignOwnable; + type QueueData: ForeignOwnable + Sync; /// Called by the kernel to queue a request with the driver. If `is_last` is /// `false`, the driver is allowed to defer committing the request. diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request.rs index ce3e30c81cb5..6115f9aec228 100644 --- a/rust/kernel/block/mq/request.rs +++ b/rust/kernel/block/mq/request.rs @@ -7,15 +7,21 @@ use crate::{ bindings, block::mq::Operations, - error::Result, + prelude::*, sync::{ - aref::{ARef, AlwaysRefCounted}, + aref::{ + ARef, + AlwaysRefCounted, // + }, atomic::Relaxed, - Refcount, + Refcount, // }, - types::Opaque, + types::Opaque, // +}; +use core::{ + marker::PhantomData, + ptr::NonNull, // }; -use core::{marker::PhantomData, ptr::NonNull}; /// A wrapper around a blk-mq [`struct request`]. This represents an IO request. /// diff --git a/rust/kernel/block/mq/tag_set.rs b/rust/kernel/block/mq/tag_set.rs index dae9df408a86..5f04008820b7 100644 --- a/rust/kernel/block/mq/tag_set.rs +++ b/rust/kernel/block/mq/tag_set.rs @@ -4,17 +4,21 @@ //! //! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h) -use core::pin::Pin; - use crate::{ bindings, - block::mq::{operations::OperationsVTable, request::RequestDataWrapper, Operations}, - error::{self, Result}, - prelude::try_pin_init, - types::Opaque, + block::mq::{ + operations::OperationsVTable, + request::RequestDataWrapper, + Operations, // + }, + error::to_result, + prelude::*, + types::Opaque, // +}; +use core::{ + convert::TryInto, + marker::PhantomData, // }; -use core::{convert::TryInto, marker::PhantomData}; -use pin_init::{pin_data, pinned_drop, PinInit}; /// A wrapper for the C `struct blk_mq_tag_set`. /// @@ -33,11 +37,7 @@ pub struct TagSet<T: Operations> { impl<T: Operations> TagSet<T> { /// Try to create a new tag set - pub fn new( - nr_hw_queues: u32, - num_tags: u32, - num_maps: u32, - ) -> impl PinInit<Self, error::Error> { + pub fn new(nr_hw_queues: u32, num_tags: u32, num_maps: u32) -> impl PinInit<Self, Error> { let tag_set: bindings::blk_mq_tag_set = pin_init::zeroed(); let tag_set: Result<_> = core::mem::size_of::<RequestDataWrapper>() .try_into() @@ -63,7 +63,7 @@ impl<T: Operations> TagSet<T> { // SAFETY: we do not move out of `tag_set`. let tag_set: &mut Opaque<_> = unsafe { Pin::get_unchecked_mut(tag_set) }; // SAFETY: `tag_set` is a reference to an initialized `blk_mq_tag_set`. - error::to_result( unsafe { bindings::blk_mq_alloc_tag_set(tag_set.get())}) + to_result( unsafe { bindings::blk_mq_alloc_tag_set(tag_set.get())}) }), _p: PhantomData, }) @@ -83,3 +83,10 @@ impl<T: Operations> PinnedDrop for TagSet<T> { unsafe { bindings::blk_mq_free_tag_set(self.inner.get()) }; } } + +// SAFETY: It is safe to transfer ownership of `TagSet` between threads. +unsafe impl<T: Operations> Send for TagSet<T> {} + +// SAFETY: It is safe to share `&TagSet` across threads. `TagSet` exposes no +// `&self` methods that mutate the contained `blk_mq_tag_set`. +unsafe impl<T: Operations> Sync for TagSet<T> {} diff --git a/rust/kernel/bug.rs b/rust/kernel/bug.rs index 3566f0234ca4..3b40b0bd37b5 100644 --- a/rust/kernel/bug.rs +++ b/rust/kernel/bug.rs @@ -85,15 +85,18 @@ macro_rules! warn_flags { ($file:expr, $flags:expr) => { if false { _ = $file; + _ = $flags; } - // SAFETY: It is always safe to call `warn_slowpath_fmt()` - // with a valid null-terminated string. + // SAFETY: + // - The file name is a null-terminated string with `'static` lifetime. + // - `TAINT_WARN` is a valid taint number. + // - `fmt` can be NULL, and NULL is passed. unsafe { $crate::bindings::warn_slowpath_fmt( $crate::str::CStrExt::as_char_ptr($crate::c_str!(::core::file!())), line!() as $crate::ffi::c_int, - $flags as $crate::ffi::c_uint, + $crate::bindings::TAINT_WARN, ::core::ptr::null(), ); } @@ -152,3 +155,33 @@ macro_rules! warn_on { cond }}; } + +#[cfg(CONFIG_RUST_BUG_KUNIT_TEST)] +#[macros::kunit_tests(rust_kernel_bug)] +mod tests { + // The counter is incremented by the kernel warning path, not by `warn_on!` + // itself. A count of one means the warning was really reported. + #[test] + fn test_warn_on() { + // SAFETY: `kunit_get_current_test()` is always safe to call (it has + // fallbacks for when no KUnit test is running). + let test = unsafe { bindings::kunit_get_current_test() }; + + // SAFETY: This function runs only as a KUnit test case, so `test` is a + // valid pointer to the running test. + let handle = unsafe { bindings::kunit_start_suppress_warning(test) }; + + assert!(!warn_on!(false)); + assert!(warn_on!(true)); + + // SAFETY: `kunit_suppressed_warning_count()` accepts any value returned by + // `kunit_start_suppress_warning()`. + let suppressed_count = unsafe { bindings::kunit_suppressed_warning_count(handle) }; + + // SAFETY: `test` is valid as above. `kunit_end_suppress_warning()` accepts any + // value returned by `kunit_start_suppress_warning()`. + unsafe { bindings::kunit_end_suppress_warning(test, handle) }; + + assert_eq!(suppressed_count, 1); + } +} diff --git a/rust/kernel/configfs.rs b/rust/kernel/configfs.rs index cd082b83e9e7..dbed72844b31 100644 --- a/rust/kernel/configfs.rs +++ b/rust/kernel/configfs.rs @@ -135,8 +135,9 @@ pub struct Subsystem<Data> { // SAFETY: We do not provide any operations on `Subsystem`. unsafe impl<Data> Sync for Subsystem<Data> {} -// SAFETY: Ownership of `Subsystem` can safely be transferred to other threads. -unsafe impl<Data> Send for Subsystem<Data> {} +// SAFETY: Ownership of `Subsystem` can safely be transferred to other threads +// if its data can be transferred as well. +unsafe impl<Data: Send> Send for Subsystem<Data> {} impl<Data> Subsystem<Data> { /// Create an initializer for a [`Subsystem`]. @@ -249,6 +250,13 @@ pub struct Group<Data> { data: Data, } +// SAFETY: We do not provide any operations on `Group`. +unsafe impl<Data> Sync for Group<Data> {} + +// SAFETY: Ownership of `Group` can safely be transferred to other threads if +// its data can be transferred as well. +unsafe impl<Data: Send> Send for Group<Data> {} + impl<Data> Group<Data> { /// Create an initializer for a new group. /// @@ -297,26 +305,39 @@ unsafe impl<Data> HasGroup<Data> for Group<Data> { /// /// `this` must be a valid pointer. /// -/// If `this` does not represent the root group of a configfs subsystem, -/// `this` must be a pointer to a `bindings::config_group` embedded in a -/// `Group<Parent>`. +/// If `this` is the `su_group` field of a `bindings::configfs_subsystem`, that +/// `configfs_subsystem` must be embedded in a `Subsystem<Parent>`. /// -/// Otherwise, `this` must be a pointer to a `bindings::config_group` that -/// is embedded in a `bindings::configfs_subsystem` that is embedded in a -/// `Subsystem<Parent>`. +/// Otherwise, `this` must be a pointer to a `bindings::config_group` embedded +/// in a `Group<Parent>`. unsafe fn get_group_data<'a, Parent>(this: *mut bindings::config_group) -> &'a Parent { // SAFETY: `this` is a valid pointer. - let is_root = unsafe { (*this).cg_subsys.is_null() }; + let subsys = unsafe { (*this).cg_subsys }; + + // `link_group()` in `fs/configfs/dir.c` assigns `cg_subsys` for every + // `config_group` attached anywhere in a registered subsystem, including + // the subsystem's own `su_group` (which gets a pointer to itself). The + // only `config_group` with a NULL `cg_subsys` is the configfs root, and + // userspace cannot trigger callbacks on it. The group is therefore the + // subsystem's `su_group` iff it equals `&cg_subsys->su_group`. + let is_root = !subsys.is_null() + && core::ptr::eq( + this.cast_const(), + // SAFETY: For every `config_group` the configfs core dispatches a + // callback on, `cg_subsys` was set by `link_group()` at registration time + // and points to a valid `configfs_subsystem` that outlives the callback. + unsafe { &raw const (*subsys).su_group }, + ); - if !is_root { - // SAFETY: By C API contact,`this` was returned from a call to - // `make_group`. The pointer is known to be embedded within a - // `Group<Parent>`. - unsafe { &(*Group::<Parent>::container_of(this)).data } - } else { - // SAFETY: By C API contract, `this` is a pointer to the - // `bindings::config_group` field within a `Subsystem<Parent>`. + if is_root { + // SAFETY: By the above, `this` is the `su_group` field of a + // `configfs_subsystem` that, by function safety requirements, is + // embedded in a `Subsystem<Parent>`. unsafe { &(*Subsystem::container_of(this)).data } + } else { + // SAFETY: By function safety requirements, `this` is a + // `config_group` embedded in a `Group<Parent>`. + unsafe { &(*Group::<Parent>::container_of(this)).data } } } @@ -325,7 +346,9 @@ struct GroupOperationsVTable<Parent, Child>(PhantomData<(Parent, Child)>); impl<Parent, Child> GroupOperationsVTable<Parent, Child> where Parent: GroupOperations<Child = Child>, - Child: 'static, + // We transfer `Arc<Group<Data>>` across a thread boundary in `make_group` + // and `drop_item`. + Arc<Group<Child>>: Send, { /// # Safety /// @@ -344,8 +367,9 @@ where this: *mut bindings::config_group, name: *const kernel::ffi::c_char, ) -> *mut bindings::config_group { - // SAFETY: By function safety requirements of this function, this call - // is safe. + // SAFETY: By function safety requirements, `this` points to a configfs + // group containing `Parent`. The `GroupOperations` bound guarantees + // that `Parent: Sync`, so it is safe to share it with this thread. let parent_data = unsafe { get_group_data(this) }; let group_init = match Parent::make_group( @@ -390,8 +414,9 @@ where this: *mut bindings::config_group, item: *mut bindings::config_item, ) { - // SAFETY: By function safety requirements of this function, this call - // is safe. + // SAFETY: By function safety requirements, `this` points to a configfs + // group containing `Parent`. The `GroupOperations` bound guarantees + // that `Parent: Sync`, so it is safe to share it with this thread. let parent_data = unsafe { get_group_data(this) }; // SAFETY: By function safety requirements, `item` is embedded in a @@ -403,7 +428,9 @@ where if Parent::HAS_DROP_ITEM { // SAFETY: We called `into_raw` to produce `r_child_group_ptr` in - // `make_group`. + // `make_group`. This function may be executing on a different + // thread than `into_raw`. As `Arc<Group<Child>>: Send` this + // ownership transfer is safe. let arc: Arc<Group<Child>> = unsafe { Arc::from_raw(r_child_group_ptr.cast_mut()) }; Parent::drop_item(parent_data, arc.as_arc_borrow()); @@ -434,11 +461,13 @@ struct ItemOperationsVTable<Container, Data>(PhantomData<(Container, Data)>); impl<Data> ItemOperationsVTable<Group<Data>, Data> where Data: 'static, + // We transfer `Arc<Group<Data>>` across a thread boundary in `release`. + Arc<Group<Data>>: Send, { /// # Safety /// - /// `this` must be a pointer to a `bindings::config_group` embedded in a - /// `Group<Parent>`. + /// `this` must be a pointer to the `cg_item` field of a `bindings::config_group` embedded in a + /// valid `Group<Data>`. /// /// This function will destroy the pointee of `this`. The pointee of `this` /// must not be accessed after the function returns. @@ -450,8 +479,9 @@ where // embedded within a `Group<Data>`. let r_group_ptr = unsafe { Group::<Data>::container_of(c_group_ptr) }; - // SAFETY: We called `into_raw` on `r_group_ptr` in - // `make_group`. + // SAFETY: We called `into_raw` on `r_group_ptr` in `make_group`. This + // function may be running on a different thread than the thread that + // called `into_raw`. As `Arc<Group<Data>>: Send`, this is safe. let pin_self: Arc<Group<Data>> = unsafe { Arc::from_raw(r_group_ptr.cast_mut()) }; drop(pin_self); } @@ -483,12 +513,12 @@ impl<Data> ItemOperationsVTable<Subsystem<Data>, Data> { /// /// Implement this trait on structs that embed a [`Subsystem`] or a [`Group`]. #[vtable] -pub trait GroupOperations { +pub trait GroupOperations: Sync { /// The child data object type. /// /// This group will create subgroups (subdirectories) backed by this kind of /// object. - type Child: 'static; + type Child: 'static + Send; /// Creates a new subgroup. /// @@ -555,8 +585,10 @@ where // `config_group`. unsafe { container_of!(item, bindings::config_group, cg_item) }; - // SAFETY: The function safety requirements for this function satisfy - // the conditions for this call. + // SAFETY: By function safety requirements, `c_group` points to a + // configfs group containing `Data`. The `AttributeOperations` bound + // guarantees that `Data: Sync`, so it is safe to share it with this + // thread. let data: &Data = unsafe { get_group_data(c_group) }; // SAFETY: By function safety requirements, `page` is writable for `PAGE_SIZE`. @@ -589,8 +621,10 @@ where // `config_group`. unsafe { container_of!(item, bindings::config_group, cg_item) }; - // SAFETY: The function safety requirements for this function satisfy - // the conditions for this call. + // SAFETY: By function safety requirements, `c_group` points to a + // configfs group containing `Data`. The `AttributeOperations` bound + // guarantees that `Data: Sync`, so it is safe to share it with this + // thread. let data: &Data = unsafe { get_group_data(c_group) }; let ret = O::store( @@ -643,7 +677,7 @@ where pub trait AttributeOperations<const ID: u64 = 0> { /// The type of the object that contains the field that is backing the /// attribute for this operation. - type Data; + type Data: Sync; /// Renders the value of an attribute. /// @@ -749,15 +783,18 @@ macro_rules! impl_item_type { ) -> Self where Data: GroupOperations<Child = Child>, - Child: 'static, + Arc<Group<Child>>: Send, + Arc<Group<Data>>: Send, { Self { item_type: Opaque::new(bindings::config_item_type { ct_owner: owner.as_ptr(), - ct_group_ops: GroupOperationsVTable::<Data, Child>::vtable_ptr().cast_mut(), - ct_item_ops: ItemOperationsVTable::<$tpe, Data>::vtable_ptr().cast_mut(), - ct_attrs: core::ptr::from_ref(attributes).cast_mut().cast(), - ct_bin_attrs: core::ptr::null_mut(), + ct_group_ops: GroupOperationsVTable::<Data, Child>::vtable_ptr(), + ct_item_ops: ItemOperationsVTable::<$tpe, Data>::vtable_ptr(), + __bindgen_anon_1: bindings::config_item_type__bindgen_ty_1 { + ct_attrs_const: core::ptr::from_ref(attributes).cast(), + }, + ct_bin_attrs: core::ptr::null(), }), _p: PhantomData, } @@ -767,14 +804,19 @@ macro_rules! impl_item_type { pub const fn new<const N: usize>( owner: &'static ThisModule, attributes: &'static AttributeList<N, Data>, - ) -> Self { + ) -> Self + where + Arc<Group<Data>>: Send, + { Self { item_type: Opaque::new(bindings::config_item_type { ct_owner: owner.as_ptr(), - ct_group_ops: core::ptr::null_mut(), - ct_item_ops: ItemOperationsVTable::<$tpe, Data>::vtable_ptr().cast_mut(), - ct_attrs: core::ptr::from_ref(attributes).cast_mut().cast(), - ct_bin_attrs: core::ptr::null_mut(), + ct_group_ops: core::ptr::null(), + ct_item_ops: ItemOperationsVTable::<$tpe, Data>::vtable_ptr(), + __bindgen_anon_1: bindings::config_item_type__bindgen_ty_1 { + ct_attrs_const: core::ptr::from_ref(attributes).cast(), + }, + ct_bin_attrs: core::ptr::null(), }), _p: PhantomData, } diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index affa2b9490ef..b75d692846a2 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -14,7 +14,13 @@ use crate::{ cpumask, device::{Bound, Device}, devres, - error::{code::*, from_err_ptr, from_result, to_result, Result, VTABLE_DEFAULT_ERROR}, + error::{ + code::*, + from_result, + to_result, + Result, + VTABLE_DEFAULT_ERROR, // + }, ffi::{c_char, c_ulong}, prelude::*, types::ForeignOwnable, @@ -29,7 +35,10 @@ use core::{ marker::PhantomData, ops::{Deref, DerefMut}, pin::Pin, - ptr, + ptr::{ + self, + NonNull, // + }, }; use macros::vtable; @@ -687,12 +696,13 @@ struct PolicyCpu<'a>(&'a mut Policy); impl<'a> PolicyCpu<'a> { fn from_cpu(cpu: CpuId) -> Result<Self> { // SAFETY: It is safe to call `cpufreq_cpu_get` for any valid CPU. - let ptr = from_err_ptr(unsafe { bindings::cpufreq_cpu_get(u32::from(cpu)) })?; + let ptr = + NonNull::new(unsafe { bindings::cpufreq_cpu_get(u32::from(cpu)) }).ok_or(ENODEV)?; Ok(Self( - // SAFETY: The `ptr` is guaranteed to be valid and remains valid for the lifetime of - // the returned reference. - unsafe { Policy::from_raw_mut(ptr) }, + // SAFETY: `ptr` is non-NULL and `cpufreq_cpu_get()` took a reference on it, so it is + // valid for writing and remains valid for the lifetime of the returned reference. + unsafe { Policy::from_raw_mut(ptr.as_ptr()) }, )) } } diff --git a/rust/kernel/debugfs.rs b/rust/kernel/debugfs.rs index d7b8014a6474..2beb55d444ca 100644 --- a/rust/kernel/debugfs.rs +++ b/rust/kernel/debugfs.rs @@ -538,7 +538,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { } } - fn create_file<T: Sync>(&self, name: &CStr, data: &'data T, vtable: &'static FileOps<T>) { + fn create_file<T: Sync>(&self, name: &CStr, data: &'data T, vtable: &FileOps<T>) { #[cfg(CONFIG_DEBUG_FS)] core::mem::forget(Entry::file(name, &self.entry, data, vtable)); } @@ -550,7 +550,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { /// This function does not produce an owning handle to the file. The created /// file is removed when the [`Scope`] that this directory belongs /// to is dropped. - pub fn read_only_file<T: Writer + Send + Sync + 'static>(&self, name: &CStr, data: &'data T) { + pub fn read_only_file<T: Writer + Send + Sync>(&self, name: &CStr, data: &'data T) { self.create_file(name, data, &T::FILE_OPS) } @@ -560,11 +560,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { /// /// This function does not produce an owning handle to the file. The created file is removed /// when the [`Scope`] that this directory belongs to is dropped. - pub fn read_binary_file<T: BinaryWriter + Send + Sync + 'static>( - &self, - name: &CStr, - data: &'data T, - ) { + pub fn read_binary_file<T: BinaryWriter + Send + Sync>(&self, name: &CStr, data: &'data T) { self.create_file(name, data, &T::FILE_OPS) } @@ -596,11 +592,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { /// This function does not produce an owning handle to the file. The created /// file is removed when the [`Scope`] that this directory belongs /// to is dropped. - pub fn read_write_file<T: Writer + Reader + Send + Sync + 'static>( - &self, - name: &CStr, - data: &'data T, - ) { + pub fn read_write_file<T: Writer + Reader + Send + Sync>(&self, name: &CStr, data: &'data T) { let vtable = &<T as ReadWriteFile<_>>::FILE_OPS; self.create_file(name, data, vtable) } @@ -612,7 +604,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { /// /// This function does not produce an owning handle to the file. The created file is removed /// when the [`Scope`] that this directory belongs to is dropped. - pub fn read_write_binary_file<T: BinaryWriter + BinaryReader + Send + Sync + 'static>( + pub fn read_write_binary_file<T: BinaryWriter + BinaryReader + Send + Sync>( &self, name: &CStr, data: &'data T, @@ -655,7 +647,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { /// This function does not produce an owning handle to the file. The created /// file is removed when the [`Scope`] that this directory belongs /// to is dropped. - pub fn write_only_file<T: Reader + Send + Sync + 'static>(&self, name: &CStr, data: &'data T) { + pub fn write_only_file<T: Reader + Send + Sync>(&self, name: &CStr, data: &'data T) { let vtable = &<T as WriteFile<_>>::FILE_OPS; self.create_file(name, data, vtable) } @@ -666,11 +658,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { /// /// This function does not produce an owning handle to the file. The created file is removed /// when the [`Scope`] that this directory belongs to is dropped. - pub fn write_binary_file<T: BinaryReader + Send + Sync + 'static>( - &self, - name: &CStr, - data: &'data T, - ) { + pub fn write_binary_file<T: BinaryReader + Send + Sync>(&self, name: &CStr, data: &'data T) { self.create_file(name, data, &T::FILE_OPS) } diff --git a/rust/kernel/debugfs/entry.rs b/rust/kernel/debugfs/entry.rs index 46aad64896ec..88a870d8c295 100644 --- a/rust/kernel/debugfs/entry.rs +++ b/rust/kernel/debugfs/entry.rs @@ -74,7 +74,7 @@ impl Entry<'static> { parent.as_ptr(), core::ptr::from_ref(data) as *mut c_void, core::ptr::null(), - &**file_ops, + file_ops.fops(), ) }; @@ -127,7 +127,7 @@ impl<'a> Entry<'a> { parent.as_ptr(), core::ptr::from_ref(data) as *mut c_void, core::ptr::null(), - &**file_ops, + file_ops.fops(), ) }; diff --git a/rust/kernel/debugfs/file_ops.rs b/rust/kernel/debugfs/file_ops.rs index f15908f71c4a..47acc4851d47 100644 --- a/rust/kernel/debugfs/file_ops.rs +++ b/rust/kernel/debugfs/file_ops.rs @@ -20,9 +20,6 @@ use crate::{ use core::marker::PhantomData; -#[cfg(CONFIG_DEBUG_FS)] -use core::ops::Deref; - /// # Invariant /// /// `FileOps<T>` will always contain an `operations` which is safe to use for a file backed @@ -30,7 +27,7 @@ use core::ops::Deref; /// into a reference. pub(super) struct FileOps<T> { #[cfg(CONFIG_DEBUG_FS)] - operations: bindings::file_operations, + operations: &'static bindings::file_operations, #[cfg(CONFIG_DEBUG_FS)] mode: u16, _phantom: PhantomData<T>, @@ -41,7 +38,7 @@ impl<T> FileOps<T> { /// /// The caller asserts that the provided `operations` is safe to use for a file whose /// inode has a pointer to `T` in its private data that is safe to convert into a reference. - const unsafe fn new(operations: bindings::file_operations, mode: u16) -> Self { + const unsafe fn new(operations: &'static bindings::file_operations, mode: u16) -> Self { Self { #[cfg(CONFIG_DEBUG_FS)] operations, @@ -65,11 +62,11 @@ impl<T: Adapter> FileOps<T> { } #[cfg(CONFIG_DEBUG_FS)] -impl<T> Deref for FileOps<T> { - type Target = bindings::file_operations; - - fn deref(&self) -> &Self::Target { - &self.operations +impl<T> FileOps<T> { + /// Returns a `'static` reference to the inner `file_operations`. + #[inline] + pub(crate) fn fops(&self) -> &'static bindings::file_operations { + self.operations } } @@ -130,7 +127,7 @@ pub(crate) trait ReadFile<T> { impl<T: Writer + Sync> ReadFile<T> for T { const FILE_OPS: FileOps<T> = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { read: Some(bindings::seq_read), llseek: Some(bindings::seq_lseek), release: Some(bindings::single_release), @@ -181,7 +178,7 @@ pub(crate) trait ReadWriteFile<T> { impl<T: Writer + Reader + Sync> ReadWriteFile<T> for T { const FILE_OPS: FileOps<T> = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { open: Some(writer_open::<T>), read: Some(bindings::seq_read), write: Some(write::<T>), @@ -238,7 +235,7 @@ pub(crate) trait WriteFile<T> { impl<T: Reader + Sync> WriteFile<T> for T { const FILE_OPS: FileOps<T> = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { open: Some(write_only_open), write: Some(write_only_write::<T>), llseek: Some(bindings::noop_llseek), @@ -290,7 +287,7 @@ pub(crate) trait BinaryReadFile<T> { impl<T: BinaryWriter + Sync> BinaryReadFile<T> for T { const FILE_OPS: FileOps<T> = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { read: Some(blob_read::<T>), llseek: Some(bindings::default_llseek), open: Some(bindings::simple_open), @@ -344,7 +341,7 @@ pub(crate) trait BinaryWriteFile<T> { impl<T: BinaryReader + Sync> BinaryWriteFile<T> for T { const FILE_OPS: FileOps<T> = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { write: Some(blob_write::<T>), llseek: Some(bindings::default_llseek), open: Some(bindings::simple_open), @@ -368,7 +365,7 @@ pub(crate) trait BinaryReadWriteFile<T> { impl<T: BinaryWriter + BinaryReader + Sync> BinaryReadWriteFile<T> for T { const FILE_OPS: FileOps<T> = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { read: Some(blob_read::<T>), write: Some(blob_write::<T>), llseek: Some(bindings::default_llseek), diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs index c81fca5b4986..f0b9cb84e58e 100644 --- a/rust/kernel/device_id.rs +++ b/rust/kernel/device_id.rs @@ -146,8 +146,7 @@ impl<T: RawDeviceId, const N: usize> IdArray<T, (), N> { /// If the device implements [`RawDeviceIdIndex`], consider using [`IdArray::new`] instead. pub const fn new_without_index(ids: [T; N]) -> Self { // SAFETY: `T` is layout-wise compatible with `T::RawType`, so is the array of them. - let raw_ids: [MaybeUninit<T::RawType>; N] = unsafe { core::mem::transmute_copy(&ids) }; - core::mem::forget(ids); + let raw_ids: [MaybeUninit<T::RawType>; N] = unsafe { crate::mem::transmute(ids) }; Self { ids: raw_ids, diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 2ce09f8e90c6..580e1c3427bc 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -24,7 +24,6 @@ use crate::{ }, prelude::*, ptr::KnownSize, - sync::aref::ARef, transmute::{ AsBytes, FromBytes, // @@ -223,7 +222,7 @@ impl DmaMask { /// /// # fn test(dev: &Device<Bound>) -> Result { /// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN; -/// let c: Coherent<[u64]> = +/// let c: Coherent<'_, [u64]> = /// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, attribs)?; /// # Ok::<(), Error>(()) } /// ``` @@ -310,25 +309,25 @@ pub enum DataDirection { /// /// This is used when the buffer can be both read from and written to by the device. /// The cache for the corresponding memory region is both flushed and invalidated. - Bidirectional = Self::const_cast(bindings::dma_data_direction_DMA_BIDIRECTIONAL), + Bidirectional = Self::const_cast(bindings::dma_data_direction::DMA_BIDIRECTIONAL), /// The DMA mapping is for data transfer from memory to the device (write). /// /// The CPU has prepared data in the buffer, and the device will read it. /// The cache for the corresponding memory region is flushed before device access. - ToDevice = Self::const_cast(bindings::dma_data_direction_DMA_TO_DEVICE), + ToDevice = Self::const_cast(bindings::dma_data_direction::DMA_TO_DEVICE), /// The DMA mapping is for data transfer from the device to memory (read). /// /// The device will write data into the buffer for the CPU to read. /// The cache for the corresponding memory region is invalidated before CPU access. - FromDevice = Self::const_cast(bindings::dma_data_direction_DMA_FROM_DEVICE), + FromDevice = Self::const_cast(bindings::dma_data_direction::DMA_FROM_DEVICE), /// The DMA mapping is not for data transfer. /// /// This is primarily for debugging purposes. With this direction, the DMA mapping API /// will not perform any cache coherency operations. - None = Self::const_cast(bindings::dma_data_direction_DMA_NONE), + None = Self::const_cast(bindings::dma_data_direction::DMA_NONE), } impl DataDirection { @@ -340,7 +339,7 @@ impl DataDirection { // CAST: The C standard allows compilers to choose different integer types for enums. // To safely check the value, we cast it to a wide signed integer type (`i128`) // which can hold any standard C integer enum type without truncation. - let wide_val = val as i128; + let wide_val = val.0 as i128; // Check if the value is outside the valid range for the target type `u32`. // CAST: `u32::MAX` is cast to `i128` to match the type of `wide_val` for the comparison. @@ -358,11 +357,12 @@ impl DataDirection { impl From<DataDirection> for bindings::dma_data_direction { /// Returns the raw representation of [`enum dma_data_direction`]. fn from(direction: DataDirection) -> Self { - // CAST: `direction as u32` gets the underlying representation of our `#[repr(u32)]` enum. - // The subsequent cast to `Self` (the bindgen type) assumes the C enum is compatible - // with the enum variants of `DataDirection`, which is a valid assumption given our - // compile-time checks. - direction as u32 as Self + match direction { + DataDirection::Bidirectional => Self::DMA_BIDIRECTIONAL, + DataDirection::ToDevice => Self::DMA_TO_DEVICE, + DataDirection::FromDevice => Self::DMA_FROM_DEVICE, + DataDirection::None => Self::DMA_NONE, + } } } @@ -390,9 +390,9 @@ impl From<DataDirection> for bindings::dma_data_direction { /// }; /// /// # fn test(dev: &Device<Bound>) -> Result { -/// let mut dmem: CoherentBox<u64> = CoherentBox::zeroed(dev, GFP_KERNEL)?; +/// let mut dmem: CoherentBox<'_, u64> = CoherentBox::zeroed(dev, GFP_KERNEL)?; /// *dmem = 42; -/// let dmem: Coherent<u64> = dmem.into(); +/// let dmem: Coherent<'_, u64> = dmem.into(); /// # Ok::<(), Error>(()) } /// ``` /// @@ -410,18 +410,18 @@ impl From<DataDirection> for bindings::dma_data_direction { /// }; /// /// # fn test(dev: &Device<Bound>) -> Result { -/// let mut dmem: CoherentBox<[u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?; +/// let mut dmem: CoherentBox<'_, [u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?; /// dmem.fill(42); -/// let dmem: Coherent<[u64]> = dmem.into(); +/// let dmem: Coherent<'_, [u64]> = dmem.into(); /// # Ok::<(), Error>(()) } /// ``` -pub struct CoherentBox<T: KnownSize + ?Sized>(Coherent<T>); +pub struct CoherentBox<'a, T: KnownSize + ?Sized>(Coherent<'a, T>); -impl<T: AsBytes + FromBytes> CoherentBox<[T]> { +impl<'a, T: AsBytes + FromBytes> CoherentBox<'a, [T]> { /// [`CoherentBox`] variant of [`Coherent::zeroed_slice_with_attrs`]. #[inline] pub fn zeroed_slice_with_attrs( - dev: &device::Device<Bound>, + dev: &'a device::Device<Bound>, count: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, @@ -432,7 +432,7 @@ impl<T: AsBytes + FromBytes> CoherentBox<[T]> { /// Same as [CoherentBox::zeroed_slice_with_attrs], but with `dma::Attrs(0)`. #[inline] pub fn zeroed_slice( - dev: &device::Device<Bound>, + dev: &'a device::Device<Bound>, count: usize, gfp_flags: kernel::alloc::Flags, ) -> Result<Self> { @@ -480,14 +480,14 @@ impl<T: AsBytes + FromBytes> CoherentBox<[T]> { /// /// # fn test(dev: &Device<Bound>) -> Result { /// let data = [0u8, 1u8, 2u8, 3u8]; - /// let c: CoherentBox<[u8]> = + /// let c: CoherentBox<'_, [u8]> = /// CoherentBox::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?; /// /// assert_eq!(c.deref(), &data); /// # Ok::<(), Error>(()) } /// ``` pub fn from_slice_with_attrs( - dev: &device::Device<Bound>, + dev: &'a device::Device<Bound>, data: &[T], gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, @@ -512,7 +512,7 @@ impl<T: AsBytes + FromBytes> CoherentBox<[T]> { /// `dma_attrs` is 0 by default. #[inline] pub fn from_slice( - dev: &device::Device<Bound>, + dev: &'a device::Device<Bound>, data: &[T], gfp_flags: kernel::alloc::Flags, ) -> Result<Self> @@ -523,11 +523,11 @@ impl<T: AsBytes + FromBytes> CoherentBox<[T]> { } } -impl<T: AsBytes + FromBytes> CoherentBox<T> { +impl<'a, T: AsBytes + FromBytes> CoherentBox<'a, T> { /// Same as [`CoherentBox::zeroed_slice_with_attrs`], but for a single element. #[inline] pub fn zeroed_with_attrs( - dev: &device::Device<Bound>, + dev: &'a device::Device<Bound>, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, ) -> Result<Self> { @@ -536,12 +536,12 @@ impl<T: AsBytes + FromBytes> CoherentBox<T> { /// Same as [`CoherentBox::zeroed_slice`], but for a single element. #[inline] - pub fn zeroed(dev: &device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> { + pub fn zeroed(dev: &'a device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> { Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) } } -impl<T: KnownSize + ?Sized> Deref for CoherentBox<T> { +impl<T: KnownSize + ?Sized> Deref for CoherentBox<'_, T> { type Target = T; #[inline] @@ -554,7 +554,7 @@ impl<T: KnownSize + ?Sized> Deref for CoherentBox<T> { } } -impl<T: AsBytes + FromBytes + KnownSize + ?Sized> DerefMut for CoherentBox<T> { +impl<T: AsBytes + FromBytes + KnownSize + ?Sized> DerefMut for CoherentBox<'_, T> { #[inline] fn deref_mut(&mut self) -> &mut Self::Target { // SAFETY: @@ -565,9 +565,9 @@ impl<T: AsBytes + FromBytes + KnownSize + ?Sized> DerefMut for CoherentBox<T> { } } -impl<T: AsBytes + FromBytes + KnownSize + ?Sized> From<CoherentBox<T>> for Coherent<T> { +impl<'a, T: AsBytes + FromBytes + KnownSize + ?Sized> From<CoherentBox<'a, T>> for Coherent<'a, T> { #[inline] - fn from(value: CoherentBox<T>) -> Self { + fn from(value: CoherentBox<'a, T>) -> Self { value.0 } } @@ -588,26 +588,20 @@ impl<T: AsBytes + FromBytes + KnownSize + ?Sized> From<CoherentBox<T>> for Coher /// to an allocated region of coherent memory and `dma_addr` is the DMA address base of the /// region. /// - The size in bytes of the allocation is equal to size information via pointer. -// TODO -// -// DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness -// reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure -// that device resources can never survive device unbind. // -// However, it is neither desirable nor necessary to protect the allocated memory of the DMA -// allocation from surviving device unbind; it would require RCU read side critical sections to -// access the memory, which may require subsequent unnecessary copies. -// -// Hence, find a way to revoke the device resources of a `Coherent`, but not the -// entire `Coherent` including the allocated memory itself. -pub struct Coherent<T: KnownSize + ?Sized> { - dev: ARef<device::Device>, +// The lifetime parameter ties DMA allocations to the device's bound scope, ensuring they are freed +// before the device is unbound under normal circumstances. However, if a `Coherent` is leaked (e.g. +// via `mem::forget`), device resources such as IOMMU mappings will not be released. Making all +// constructors `unsafe` to prevent this is considered too restrictive for the common case; this +// soundness hole is accepted for now. +pub struct Coherent<'a, T: KnownSize + ?Sized> { + dev: &'a device::Device<Bound>, dma_addr: DmaAddress, cpu_addr: NonNull<T>, dma_attrs: Attrs, } -impl<T: KnownSize + ?Sized> Coherent<T> { +impl<T: KnownSize + ?Sized> Coherent<'_, T> { /// Returns the size in bytes of this allocation. #[inline] pub fn size(&self) -> usize { @@ -663,10 +657,10 @@ impl<T: KnownSize + ?Sized> Coherent<T> { } } -impl<T: AsBytes + FromBytes> Coherent<T> { +impl<'a, T: AsBytes + FromBytes> Coherent<'a, T> { /// Allocates a region of `T` of coherent memory. fn alloc_with_attrs( - dev: &device::Device<Bound>, + dev: &'a device::Device<Bound>, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, ) -> Result<Self> { @@ -692,9 +686,9 @@ impl<T: AsBytes + FromBytes> Coherent<T> { // INVARIANT: // - We just successfully allocated a coherent region which is adequately sized for `T`, // hence the cpu address is valid. - // - We also hold a refcounted reference to the device. + // - `dev` is a valid reference to a bound device that outlives this allocation. Ok(Self { - dev: dev.into(), + dev, dma_addr, cpu_addr, dma_attrs, @@ -716,13 +710,13 @@ impl<T: AsBytes + FromBytes> Coherent<T> { /// }; /// /// # fn test(dev: &Device<Bound>) -> Result { - /// let c: Coherent<[u64; 4]> = + /// let c: Coherent<'_, [u64; 4]> = /// Coherent::zeroed_with_attrs(dev, GFP_KERNEL, DMA_ATTR_NO_WARN)?; /// # Ok::<(), Error>(()) } /// ``` #[inline] pub fn zeroed_with_attrs( - dev: &device::Device<Bound>, + dev: &'a device::Device<Bound>, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, ) -> Result<Self> { @@ -732,14 +726,14 @@ impl<T: AsBytes + FromBytes> Coherent<T> { /// Performs the same functionality as [`Coherent::zeroed_with_attrs`], except the /// `dma_attrs` is 0 by default. #[inline] - pub fn zeroed(dev: &device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> { + pub fn zeroed(dev: &'a device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> { Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) } /// Same as [`Coherent::zeroed_with_attrs`], but instead of a zero-initialization the memory is /// initialized with `init`. pub fn init_with_attrs<E>( - dev: &device::Device<Bound>, + dev: &'a device::Device<Bound>, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, init: impl Init<T, E>, @@ -764,7 +758,7 @@ impl<T: AsBytes + FromBytes> Coherent<T> { /// with `init`. #[inline] pub fn init<E>( - dev: &device::Device<Bound>, + dev: &'a device::Device<Bound>, gfp_flags: kernel::alloc::Flags, init: impl Init<T, E>, ) -> Result<Self> @@ -776,11 +770,11 @@ impl<T: AsBytes + FromBytes> Coherent<T> { /// Allocates a region of `[T; len]` of coherent memory. fn alloc_slice_with_attrs( - dev: &device::Device<Bound>, + dev: &'a device::Device<Bound>, len: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, - ) -> Result<Coherent<[T]>> { + ) -> Result<Coherent<'a, [T]>> { const { assert!( core::mem::size_of::<T>() > 0, @@ -809,9 +803,9 @@ impl<T: AsBytes + FromBytes> Coherent<T> { // INVARIANT: // - We just successfully allocated a coherent region which is adequately sized for // `[T; len]`, hence the cpu address is valid. - // - We also hold a refcounted reference to the device. + // - `dev` is a valid reference to a bound device that outlives this allocation. Ok(Coherent { - dev: dev.into(), + dev, dma_addr, cpu_addr, dma_attrs, @@ -836,17 +830,17 @@ impl<T: AsBytes + FromBytes> Coherent<T> { /// }; /// /// # fn test(dev: &Device<Bound>) -> Result { - /// let c: Coherent<[u64]> = + /// let c: Coherent<'_, [u64]> = /// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; /// # Ok::<(), Error>(()) } /// ``` #[inline] pub fn zeroed_slice_with_attrs( - dev: &device::Device<Bound>, + dev: &'a device::Device<Bound>, len: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, - ) -> Result<Coherent<[T]>> { + ) -> Result<Coherent<'a, [T]>> { Coherent::alloc_slice_with_attrs(dev, len, gfp_flags | __GFP_ZERO, dma_attrs) } @@ -854,10 +848,10 @@ impl<T: AsBytes + FromBytes> Coherent<T> { /// `dma_attrs` is 0 by default. #[inline] pub fn zeroed_slice( - dev: &device::Device<Bound>, + dev: &'a device::Device<Bound>, len: usize, gfp_flags: kernel::alloc::Flags, - ) -> Result<Coherent<[T]>> { + ) -> Result<Coherent<'a, [T]>> { Self::zeroed_slice_with_attrs(dev, len, gfp_flags, Attrs(0)) } @@ -876,18 +870,18 @@ impl<T: AsBytes + FromBytes> Coherent<T> { /// # fn test(dev: &Device<Bound>) -> Result { /// let data = [0u8, 1u8, 2u8, 3u8]; /// // `c` has the same content as `data`. - /// let c: Coherent<[u8]> = + /// let c: Coherent<'_, [u8]> = /// Coherent::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?; /// /// # Ok::<(), Error>(()) } /// ``` #[inline] pub fn from_slice_with_attrs( - dev: &device::Device<Bound>, + dev: &'a device::Device<Bound>, data: &[T], gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, - ) -> Result<Coherent<[T]>> + ) -> Result<Coherent<'a, [T]>> where T: Copy, { @@ -898,10 +892,10 @@ impl<T: AsBytes + FromBytes> Coherent<T> { /// `dma_attrs` is 0 by default. #[inline] pub fn from_slice( - dev: &device::Device<Bound>, + dev: &'a device::Device<Bound>, data: &[T], gfp_flags: kernel::alloc::Flags, - ) -> Result<Coherent<[T]>> + ) -> Result<Coherent<'a, [T]>> where T: Copy, { @@ -909,7 +903,7 @@ impl<T: AsBytes + FromBytes> Coherent<T> { } } -impl<T> Coherent<[T]> { +impl<T> Coherent<'_, [T]> { /// Returns the number of elements `T` in this allocation. /// /// Note that this is not the size of the allocation in bytes, which is provided by @@ -922,10 +916,10 @@ impl<T> Coherent<[T]> { } /// Note that the device configured to do DMA must be halted before this object is dropped. -impl<T: KnownSize + ?Sized> Drop for Coherent<T> { +impl<T: KnownSize + ?Sized> Drop for Coherent<'_, T> { fn drop(&mut self) { let size = T::size(self.cpu_addr.as_ptr()); - // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. + // SAFETY: Device pointer is guaranteed as valid by the lifetime of this `Coherent`. // The cpu address, and the dma address are valid due to the type invariants on // `Coherent`. unsafe { @@ -942,15 +936,15 @@ impl<T: KnownSize + ?Sized> Drop for Coherent<T> { // SAFETY: It is safe to send a `Coherent` to another thread if `T` // can be sent to another thread. -unsafe impl<T: KnownSize + Send + ?Sized> Send for Coherent<T> {} +unsafe impl<T: KnownSize + Send + ?Sized> Send for Coherent<'_, T> {} // SAFETY: Sharing `&Coherent` across threads is safe if `T` is `Sync`, because all // methods that access the buffer contents (`field_read`, `field_write`, `as_slice`, // `as_slice_mut`) are `unsafe`, and callers are responsible for ensuring no data races occur. // The safe methods only return metadata or raw pointers whose use requires `unsafe`. -unsafe impl<T: KnownSize + ?Sized + AsBytes + FromBytes + Sync> Sync for Coherent<T> {} +unsafe impl<T: KnownSize + ?Sized + AsBytes + FromBytes + Sync> Sync for Coherent<'_, T> {} -impl<T: KnownSize + AsBytes + ?Sized> debugfs::BinaryWriter for Coherent<T> { +impl<T: KnownSize + AsBytes + ?Sized> debugfs::BinaryWriter for Coherent<'_, T> { fn write_to_slice( &self, writer: &mut UserSliceWriter, @@ -996,15 +990,15 @@ impl<T: KnownSize + AsBytes + ?Sized> debugfs::BinaryWriter for Coherent<T> { /// - `size` is the allocation size in bytes as passed to `dma_alloc_attrs`. /// - `dma_attrs` contains the attributes used for the allocation, always including /// `DMA_ATTR_NO_KERNEL_MAPPING`. -pub struct CoherentHandle { - dev: ARef<device::Device>, +pub struct CoherentHandle<'a> { + dev: &'a device::Device<Bound>, dma_addr: DmaAddress, cpu_handle: NonNull<c_void>, size: usize, dma_attrs: Attrs, } -impl CoherentHandle { +impl<'a> CoherentHandle<'a> { /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping. /// /// Additional DMA attributes may be passed via `dma_attrs`; `DMA_ATTR_NO_KERNEL_MAPPING` is @@ -1012,7 +1006,7 @@ impl CoherentHandle { /// /// Returns `EINVAL` if `size` is zero, `ENOMEM` if the allocation fails. pub fn alloc_with_attrs( - dev: &device::Device<Bound>, + dev: &'a device::Device<Bound>, size: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, @@ -1038,9 +1032,9 @@ impl CoherentHandle { // INVARIANT: `cpu_handle` is the opaque handle from a successful `dma_alloc_attrs` call // with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_addr` is the corresponding DMA address, - // and we hold a refcounted reference to the device. + // and `dev` is a valid reference to a bound device that outlives this allocation. Ok(Self { - dev: dev.into(), + dev, dma_addr, cpu_handle, size, @@ -1051,7 +1045,7 @@ impl CoherentHandle { /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping. #[inline] pub fn alloc( - dev: &device::Device<Bound>, + dev: &'a device::Device<Bound>, size: usize, gfp_flags: kernel::alloc::Flags, ) -> Result<Self> { @@ -1073,7 +1067,7 @@ impl CoherentHandle { } } -impl Drop for CoherentHandle { +impl Drop for CoherentHandle<'_> { fn drop(&mut self) { // SAFETY: All values are valid by the type invariants on `CoherentHandle`. // `cpu_handle` is the opaque handle from `dma_alloc_attrs` and is passed back unchanged. @@ -1091,12 +1085,12 @@ impl Drop for CoherentHandle { // SAFETY: `CoherentHandle` only holds a device reference, a DMA address, an opaque CPU handle, // and a size. None of these are tied to a specific thread. -unsafe impl Send for CoherentHandle {} +unsafe impl Send for CoherentHandle<'_> {} // SAFETY: `CoherentHandle` provides no CPU access to the underlying allocation. The only // operations on `&CoherentHandle` are reading the DMA address and size, both of which are // plain `Copy` values. -unsafe impl Sync for CoherentHandle {} +unsafe impl Sync for CoherentHandle<'_> {} /// View type for `Coherent`. /// @@ -1236,7 +1230,7 @@ impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> { } } -impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<T> { +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<'_, T> { type Backend = CoherentIoBackend; type Target = T; diff --git a/rust/kernel/dma_buf/dma_fence.rs b/rust/kernel/dma_buf/dma_fence.rs new file mode 100644 index 000000000000..18a43e1bb442 --- /dev/null +++ b/rust/kernel/dma_buf/dma_fence.rs @@ -0,0 +1,1022 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2025-2026 Red Hat Inc. + * Author: Philipp Stanner <pstanner@redhat.com> + */ + +//! DMA Fence support. +//! +//! Reference: <https://docs.kernel.org/driver-api/dma-buf.html#c.dma_fence> +//! +//! header: [`include/linux/dma-fence.h`](srctree/include/linux/dma-fence.h) + +use crate::{ + alloc::AllocError, + bindings, + container_of, + error::to_result, + prelude::*, + types::ForeignOwnable, + types::Opaque, // +}; + +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, + ptr, + ptr::{ + drop_in_place, + NonNull, // + }, // +}; + +use kernel::{ + str::CString, + sync::{ + aref::{ + ARef, + AlwaysRefCounted, // + }, + atomic::{ + Atomic, + Relaxed, // + }, + rcu::rcu_barrier, // + }, // +}; + +/// VTable for dma_fence backend_ops callbacks. +// +// Mandatory dma_fence backend_ops are implemented implicitly through +// [`FenceContext`]. Additional ones shall get implemented on this trait. +pub trait FenceContextOps { + /// The generic payload data for [`DriverFence`]s created on this fctx. + type FenceDataType: Send + Sync; +} + +/// A dma-fence context. A fence context takes care of associating related fences +/// with each other, providing each with raising sequence numbers and a common +/// identifier. +#[pin_data(PinnedDrop)] +pub struct FenceContext<T: FenceContextOps + Send + Sync> { + /// The fence context number. + nr: u64, + /// The sequence number for the next fence created. + seqno: Atomic<u64>, + // The name parameters can be accessed by the dma_fence backend_ops. UAF + // errors are prevented by the `call_rcu()` in `drop_driver_fence_data()`. + /// The name of the driver this FenceContext's fences belong to. + driver_name: CString, + /// The name of the timeline this FenceContext's fences belong to. + timeline_name: CString, + /// The number of all unsignaled fences on this context. + // Used to prevent bugs due to forgotten fences. + // + // The lifetime on `DriverFence`s should typically prevent this from + // happening. + // + // However, we cannot fully guarantee in Rust that `DriverFence`s will not + // be forgotten, e.g., through `core::mem::forget()`. This could circumvent + // the lifetime which intends to enforce that all fences disappear before + // their context. + nr_of_unsignaled_fences: Atomic<usize>, + /// The user's data. + #[pin] + data: T, +} + +impl<'a, T: Send + Sync + FenceContextOps> FenceContext<T> { + // This can later be extended as a vtable in case other parties need support + // for the more "exotic" callbacks. + const OPS: bindings::dma_fence_ops = bindings::dma_fence_ops { + get_driver_name: Some(Self::get_driver_name), + get_timeline_name: Some(Self::get_timeline_name), + enable_signaling: None, + signaled: None, + // Deprecated. + wait: None, + // Must never be implemented for these abstractions. + release: None, + set_deadline: None, + }; + + /// Create a new `FenceContext`. + pub fn new<E>( + initial_seqno: u64, + driver_name: &CStr, + timeline_name: &CStr, + data: impl PinInit<T, E>, + ) -> impl PinInit<Self, Error> + where + Error: From<E>, + { + let driver_name = CString::try_from(driver_name); + let timeline_name = CString::try_from(timeline_name); + try_pin_init!(Self { + // SAFETY: `dma_fence_context_alloc()` merely works on a global + // atomic. Parameter `1` is the number of contexts we want to + // allocate. + nr: unsafe { bindings::dma_fence_context_alloc(1) }, + seqno: Atomic::new(initial_seqno), + driver_name: driver_name?, + timeline_name: timeline_name?, + nr_of_unsignaled_fences: Atomic::new(0), + data <- data, + }) + } + + fn next_seqno(&self) -> u64 { + self.seqno.fetch_add(1, Relaxed) + } + + /// Allocate the memory for a [`DriverFence`] and already store `data` inside. + /// + /// This is needed because many times, creation of a [`DriverFence`] must not + /// fail, and allocating might deadlock in some situations. + /// + /// The `data` you pass here must not perform any operations that are illegal + /// in atomic context in its [`Drop`] implementation. + pub fn new_fence_allocation( + &self, + data: T::FenceDataType, + ) -> Result<DriverFenceAllocation<'_, T>> { + let fence_data = DriverFenceData { + rcu_head: Default::default(), + // `inner` remains uninitialized until a `DriverFence` takes over. + inner: Fence { + inner: Opaque::uninit(), + }, + fctx: self, + data, + }; + + // In order to support the C dma_fence callbacks, it is necessary for + // a `Fence` and a `DriverFence` to live in the same allocation, + // because the C backend passes a dma_fence, from which the driver most + // likely wants to be able to access its `data` in `DriverFence`. + // + // Hence, we need the manage the memory manually. It will be freed by the + // C backend automatically once the refcount within `Fence` drops to 0. + let data = KBox::new(fence_data, GFP_KERNEL | __GFP_ZERO)?; + + Ok(DriverFenceAllocation { + data, + ops: &Self::OPS, + }) + } + + extern "C" fn get_driver_name(ptr: *mut bindings::dma_fence) -> *const c_char { + // SAFETY: The C backend only invokes this callback with `ptr` pointing + // to a valid, unsignaled `bindings::dma_fence`. All fences created in + // this module always reside within `Fence` which always resides in a + // `DriverFenceData`, thus satisfying the function's safety + // requirements. + let fctx = unsafe { Self::from_raw_fence(ptr) }; + + fctx.driver_name.as_char_ptr() + } + + extern "C" fn get_timeline_name(ptr: *mut bindings::dma_fence) -> *const c_char { + // SAFETY: The C backend only invokes this callback with `ptr` pointing + // to a valid, unsignaled `bindings::dma_fence`. All fences created in + // this module always reside within `Fence` which always resides in a + // `DriverFenceData`, thus satisfying the function's safety + // requirements. + let fctx = unsafe { Self::from_raw_fence(ptr) }; + + fctx.timeline_name.as_char_ptr() + } + + /// Create a [`FenceContext`] from an associated [`bindings::dma_fence`]. + /// + /// # Safety + /// + /// `ptr` must be a valid pointer to a [`bindings::dma_fence`] which resides + /// within a [`Fence`], which in turn resides in a [`DriverFenceData`]. + unsafe fn from_raw_fence(ptr: *mut bindings::dma_fence) -> &'a Self { + let opaque_fence = Opaque::cast_from(ptr); + + // SAFETY: Safe due to the function's overall safety requirements. + let fence_ptr = unsafe { container_of!(opaque_fence, Fence, inner) }; + + // CAST: `DriverFenceData` is `repr(C)` and a `Fence` is its first member. + let fence_data_ptr: *const DriverFenceData<'a, T> = fence_ptr.cast(); + + // SAFETY: Safe because of the comments directly above. + let fence_data = unsafe { &*fence_data_ptr }; + + fence_data.fctx + } +} + +#[pinned_drop] +impl<T: FenceContextOps + Send + Sync> PinnedDrop for FenceContext<T> { + fn drop(self: Pin<&mut Self>) { + // Fence ops callbacks can be called on unsignaled fences. Since these + // callbacks can access the fence context and its data, it needs to be + // guaranteed that a context only drops after all associated + // `DriverFence`s have been dropped. This is unlikely to occur, but + // would result in silent UAF. Throw a panic to prevent that. + // + // TODO: + // It would be better if the fence context signals all forgotten fences + // itself. To do so, it would keep a list of unsignaled fences. That + // list's members would have to be pre-allocated (see + // `FenceCallback::new_fence_allocation()`). + if self.nr_of_unsignaled_fences.load(Relaxed) != 0 { + panic!("Forgotten fences in FenceContext."); + } + + // Ensure that the driver cannot unload while there are still dma_fence + // callbacks running. At the same time, the RCU barrier addresses the + // problem inherited by the C backend, in which backend ops callbacks + // might be accessing the fence while it is being signaled (or shortly + // after). This could cause UAF access on the fence context's + // `fctx.driver_name` and `fctx.timeline_name`. + // + // Wait for the RCU callbacks in `DriverFence::drop`. + rcu_barrier(); + } +} + +/// Error type for fence callback registration. +/// +/// Generic over `T` so that `AlreadySignaled` can return the callback to the +/// caller, allowing it to reclaim any resources owned by the callback (e.g., +/// a fence handle that needs to be signaled). +#[derive(Debug)] +pub enum CallbackError<T> { + /// The fence was already signaled. The callback is returned so the caller + /// can extract owned resources without losing them. + AlreadySignaled(T), + /// Some other error occurred during registration. + Other(Error), +} + +impl<T> From<CallbackError<T>> for Error { + #[inline] + fn from(err: CallbackError<T>) -> Self { + match err { + CallbackError::AlreadySignaled(_) => ENOENT, + CallbackError::Other(e) => e, + } + } +} + +impl<T> From<AllocError> for CallbackError<T> { + #[inline] + fn from(e: AllocError) -> Self { + CallbackError::Other(Error::from(e)) + } +} + +/// Trait for callbacks that can be registered on fences. +/// +/// When the fence signals, the callback will be invoked. +/// +/// # Example +/// +/// ```rust +/// use kernel::dma_buf::FenceCallback; +/// +/// struct MyCallback { +/// // Your callback state here +/// } +/// +/// impl FenceCallback for MyCallback { +/// fn on_signal(&mut self) { +/// pr_info!("Fence signaled!\n"); +/// // Handle fence completion +/// } +/// } +/// ``` +pub trait FenceCallback: Send + 'static { + /// Called when the fence is signaled. + /// + /// This is called from the fence signaling path, which may be in interrupt + /// context or with locks held, which is why `self` is only borrowed, so that + /// it cannot drop. Implementations must not sleep or perform + /// long-running operations. + /// + /// An implementation likely wants to inform itself (e.g., through a work item) + /// within this callback that the associated [`FenceCallbackRegistration`] + /// can now be dropped. + fn on_signal(&mut self); +} + +/// A callback registration on a fence. +/// +/// When this object is dropped, the callback is automatically removed if it +/// hasn't been called yet. +#[pin_data(PinnedDrop)] +pub struct FenceCallbackRegistration<T: FenceCallback + 'static> { + #[pin] + callback_foreign: Opaque<bindings::dma_fence_cb>, + callback: ManuallyDrop<T>, + fence: ARef<Fence>, +} + +impl<T: FenceCallback> FenceCallbackRegistration<T> { + /// Create a [`PinInit`] closure for registering a callback on a fence. + /// + /// The actual attempt at registering the callback will take place once you + /// call an allocator's `pin_init()` function. + /// + /// On success the callback is pinned in place and will fire when the fence + /// signals. On `AlreadySignaled` the callback is returned to the caller so + /// that owned resources can be reclaimed. + pub fn new<'a>(fence: &'a Fence, callback: T) -> impl PinInit<Self, CallbackError<T>> + 'a + where + T: 'a, + { + try_pin_init!(Self { + // We need to fully initialize the fence because after + // `dma_fence_add_callback()` ran, the callback might immediately + // get invoked. + callback: ManuallyDrop::new(callback), + fence: ARef::from(fence), + callback_foreign <- Opaque::try_ffi_init(|ptr| { + // SAFETY: `fence.inner.get()` is a valid, initialized `struct + // dma_fence`. `ptr` points to the `struct dma_fence_cb` field + // within the pinned allocation, so it remains valid until + // `dma_fence_remove_callback()` in `PinnedDrop` or until the + // callback fires. + let ret = unsafe { + to_result(bindings::dma_fence_add_callback( + fence.inner.get(), + ptr, + Some(Self::dma_fence_callback), + )) + }; + match ret { + Ok(()) => Ok(()), + Err(e) => { + // SAFETY: We could not register the callback. Thus, + // C will not use it. So we can just take it back + // and pass it to the user again. + let cb_back = unsafe { ManuallyDrop::take(callback) }; + if e == ENOENT { + Err(CallbackError::AlreadySignaled(cb_back)) + } else { + Err(CallbackError::Other(e)) + } + }, + } + }), + }? CallbackError<T>) + } + + /// Raw dma fence callback that is called by the C code. + /// + /// # Safety + /// + /// This is only called by the dma_fence subsystem with valid pointers. + unsafe extern "C" fn dma_fence_callback( + _fence: *mut bindings::dma_fence, + callback_foreign: *mut bindings::dma_fence_cb, + ) { + let ptr = Opaque::cast_from(callback_foreign).cast_mut(); + + // SAFETY: All callbacks we can receive here have been created in such a way that they are + // embedded into a `FenceCallbackRegistration`. + let reg: *mut Self = unsafe { container_of!(ptr, Self, callback_foreign) }; + + // SAFETY: `reg` is a valid `Self` pointer. + // + // The backend ensures synchronisation so whoever holds the registration object cannot drop + // it while this code is running. See `FenceCallbackRegistration::drop`. + unsafe { (*reg).callback.on_signal() }; + } + + /// Returns a reference to the fence this callback is registered on. + #[inline] + pub fn fence(&self) -> &Fence { + &self.fence + } +} + +#[pinned_drop] +impl<T: FenceCallback> PinnedDrop for FenceCallbackRegistration<T> { + fn drop(self: Pin<&mut Self>) { + // Always call `dma_fence_remove_callback()`, even if the callback + // already ran. This is necessary for synchronization: + // `dma_fence_remove_callback()` acquires `fence->lock`, which ensures + // that any in-flight `dma_fence_signal()` (which calls our callback + // while holding the same lock) has completed before we free the struct. + // + // Without this, Drop can race with a concurrent signal: + // CPU0 (signal, lock held): take() -> on_signal(fence_ref) (in progress) + // CPU1 (drop): skips lock -> frees struct + // CPU0: accesses fence_ref -> use-after-free + // + // When the callback has already fired, the signal path detached the + // list node via `INIT_LIST_HEAD()`, so dma_fence_remove_callback just + // sees an empty node and returns false — the lock acquisition is the + // only thing that matters. + // + // SAFETY: The fence pointer is valid and the cb was initialized by + // `dma_fence_add_callback()` during construction. + unsafe { + bindings::dma_fence_remove_callback(self.fence.as_raw(), self.callback_foreign.get()) + }; + + // SAFETY: This is literally the drop implementation, so no one has + // dropped this so far; so we can do it now. + unsafe { ManuallyDrop::<T>::drop(self.project().callback) }; + } +} + +// SAFETY: FenceCallbackRegistration can be sent between threads. +unsafe impl<T: FenceCallback> Send for FenceCallbackRegistration<T> {} + +// SAFETY: &FenceCallbackRegistration can be shared between threads if &T can. +unsafe impl<T: FenceCallback> Sync for FenceCallbackRegistration<T> where T: Sync {} + +/// The receiving counterpart of a [`DriverFence`]. +/// +/// The Rust DMA fence implementation has a dualistic design: [`DriverFence`]s +/// are the producer-side, intended to be always owned by only one party. That +/// party has the monopoly on signaling the fence. +/// +/// A [`Fence`] is the counterpart for consumers. Thus, [`Fence`]s are always +/// refcounted and can shared with an arbitrary number of parties, including +/// userspace. A [`Fence`] can only be used for actions such as checking the +/// fence's status or for registering callbacks on it. +/// +/// Once the associated [`DriverFence`] signals, all +/// [`FenceCallbackRegistration`]s registered on the [`Fence`] will be executed. +/// +/// A [`Fence`] can arbitrarily outlive its [`DriverFence`] and the +/// [`FenceContext`]. Signaling a [`DriverFence`] decouples it from its +/// [`Fence`]s. +#[repr(transparent)] +pub struct Fence { + /// The actual dma_fence passed to C. + inner: Opaque<bindings::dma_fence>, +} + +/// Guard helper for locking within this module. +/// +/// Its only purpose for now is to avoid a number of unsafe lock-unlock cycles. +/// It is never used outside of this module. +// TODO: This should be made more canonical, probably by basing it on a +// SpinLockIrqGuard once available. +struct FenceGuard<'a> { + inner: &'a Fence, + flags: usize, +} + +impl<'a> Deref for FenceGuard<'a> { + type Target = &'a Fence; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl Drop for FenceGuard<'_> { + fn drop(&mut self) { + // SAFETY: `fence` is valid because `self` is valid. `flag_ptr` is + // merely a pointer to an integer, which lives as long as this function. + // When a `FenceGuard` exists, the lock has been taken by definition. + unsafe { bindings::dma_fence_unlock_irqrestore(self.as_raw(), &raw mut self.flags) }; + } +} + +// SAFETY: Fences are literally designed to be shared between threads. +unsafe impl Send for Fence {} +// SAFETY: Fences are literally designed to be shared between threads. +unsafe impl Sync for Fence {} + +impl Fence { + /// Check whether the fence was signaled at the moment of the function call. + /// + /// Note that this can return `true` for a [`Fence`] whose [`DriverFence`] + /// has not yet been dropped. The reason is that the fence ops callbacks can + /// cause the fence to get signaled by the C backend. + #[inline] + pub fn is_signaled(&self) -> bool { + // We should not use `dma_fence_is_signaled_locked()` here, because + // according to the C backend's recommendations, that function is + // problematic and we should avoid calling that function with a lock + // held. + + // SAFETY: Inner `fence` is valid because `self` is valid. + let ret = unsafe { bindings::dma_fence_is_signaled(self.as_raw()) }; + + // To be as robust as possible for the future we guarantee that an API + // caller can 100% rely on the signaling being completed (i.e., all + // fence callbacks ran), so we have to take the lock. + // + // The reason is that the C dma_fence backend currently does not + // carefully synchronize the `dma_fence_is_signaled()` function with the + // proper spinlock. This can lead to the function returning `true` while + // fence callbacks are still being executed. This can be mitigated by + // guarding the entire function with the spinlock. + // + // The fundamental reason is that the C backend currently does guard + // setting of the fence's signaled-bit with the fence's spinlock, but + // reading is done locklessly. + // + // See commit c8a5d5ea3ba6a. + let _ = self.lock(); + + ret + } + + /// Lock the fence. A helper only to be used internally in this module. + fn lock(&self) -> FenceGuard<'_> { + let mut guard = FenceGuard { + inner: self, + flags: 0, + }; + + // SAFETY: `fence` is valid because `self` is valid. `flag_ptr` is + // merely a pointer to an integer, whose lifetime is tied to the guard + // object. + unsafe { bindings::dma_fence_lock_irqsave(self.as_raw(), &raw mut guard.flags) }; + + guard + } + + /// Get the fence's sequence number. + #[inline] + pub fn seqno(&self) -> u64 { + // SAFETY: Valid because `self` is valid. + unsafe { (*self.as_raw()).seqno } + } + + fn as_raw(&self) -> *mut bindings::dma_fence { + self.inner.get() + } + + /// Create a [`Fence`] from a raw C [`bindings::dma_fence`]. + /// + /// # Safety + /// + /// `ptr` must point to an initialized fence that is embedded into a [`Fence`]. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::dma_fence) -> &'a Self { + // SAFETY: Safe as per the function's overall safety requirements. + unsafe { &*ptr.cast() } + } +} + +// SAFETY: These implement the C backends refcounting methods which are proven +// to work correctly. +unsafe impl AlwaysRefCounted for Fence { + fn inc_ref(&self) { + // SAFETY: `self.as_raw()` is a pointer to a valid `struct dma_fence`. + unsafe { bindings::dma_fence_get(self.as_raw()) } + } + + unsafe fn dec_ref(ptr: NonNull<Self>) { + // SAFETY: `ptr` is never a NULL pointer; and when `dec_ref()` is called + // the fence is by definition still valid. + let fence = unsafe { (*ptr.as_ptr()).inner.get() }; + + // SAFETY: `fence` was created validly above. When `dec_ref()` is called, + // there is by definition still a reference alive that can be put. + unsafe { bindings::dma_fence_put(fence) } + } +} + +// Necessary to guarantee that `inner` always comes first and can be freed by C. +// Also useful for using casts instead of container_of(). +#[repr(C)] +#[pin_data] +struct DriverFenceData<'a, T: Send + Sync + FenceContextOps> { + #[pin] + /// The inner fence. + // Must always be the first member so that unsafe casting works; but also + // necessary so that the C backend can free the allocation (coming from our + // Rust code) with kfree_rcu(). + inner: Fence, + /// Callback head for dropping this in a deferred manner through RCU. + rcu_head: bindings::callback_head, + /// Reference to access the FenceContext. + fctx: &'a FenceContext<T>, + /// The API user's data. It is essential that the data only performs + /// operations legal in atomic context in its [`Drop`] implementation. + #[pin] + data: T::FenceDataType, +} + +/// A synchronization primitive mainly for GPU drivers. +/// +/// The Rust DMA fence implementation has a dualistic design: [`DriverFence`]s +/// are the producer-side, intended to be always owned by only one party. That +/// party has the monopoly on signaling the fence. +/// +/// A [`Fence`] is the counterpart for consumers. Thus, [`Fence`]s are always +/// refcounted and can be shared with an arbitrary number of parties, including +/// userspace. A [`Fence`] can only be used for actions such as checking the +/// fence's status or for registering callbacks on it. +/// +/// Once the associated [`DriverFence`] signals, all +/// [`FenceCallbackRegistration`]s registered on a [`Fence`] will be executed. +/// +/// A [`Fence`] can arbitrarily outlive its [`DriverFence`] and the +/// [`FenceContext`]. Signaling a [`DriverFence`] decouples it from its +/// [`Fence`]s. +/// +/// It is crucial that a [`DriverFence`] always correctly represents the state +/// of the associated job on the hardware. Especially, it is strictly necessary +/// that the owner ensures that all [`DriverFence`]s eventually get signaled. +/// As a last resort, a [`DriverFence`] will signal itself if it drops +/// unsignaled and print a warning. +/// +/// This design intends to implement the [`bindings::dma_fence_ops`] in such a +/// way that the driver-data necessary to implement the callback's functionality +/// resides in the [`FenceContext`]. Thus, a [`DriverFence`] contains a +/// reference to the context, which can be accessed in the callbacks. The +/// implementation, therefore, ensures that a [`DriverFence`] cannot outlive its +/// [`FenceContext`]. Unfortunately, this can be circumvented under certain +/// circumstances in Rust (e.g., usage of [`core::mem::forget`]). +/// +/// In the unlikely case of such violations, a panic is thrown. +/// +/// # Examples +/// +/// ``` +/// use kernel::{ +/// dma_buf::{ +/// DriverFence, +/// FenceContext, +/// FenceContextOps, +/// FenceCallback, +/// FenceCallbackRegistration, +/// }, +/// str::CString, +/// sync::aref::ARef, // +/// }; +/// use core::fmt::Display; +/// +/// struct CallbackData { } +/// +/// impl FenceCallback for CallbackData { +/// fn on_signal(&mut self) { +/// pr_info!("DmaFence callback executed.\n"); +/// } +/// } +/// +/// #[pin_data] +/// struct FenceContextData {} +/// +/// impl FenceContextData { +/// fn new() -> impl PinInit<Self> { +/// pin_init!(Self {}) +/// } +/// } +/// +/// impl FenceContextOps for FenceContextData { +/// type FenceDataType = FenceData; +/// } +/// +/// let fctx_data = FenceContextData::new(); +/// +/// +/// let mut fctx = KBox::pin_init( +/// FenceContext::new(0, c"dummy_driver", c"dummy_timeline", fctx_data), +/// GFP_KERNEL +/// )?; +/// +/// struct FenceData { +/// data: CString, +/// } +/// +/// let fence_data = FenceData { data: c"dummy_data".try_into()? }; +/// +/// let fence_alloc = fctx.new_fence_allocation(fence_data)?; +/// let mut fence = fence_alloc.new_fence(); +/// +/// let cb_data = CallbackData { }; +/// let waiting_fence = ARef::from(fence.as_fence()); +/// let cb_reg = FenceCallbackRegistration::new(&waiting_fence, cb_data); +/// let cb_reg = KBox::pin_init(cb_reg, GFP_KERNEL)?; +/// +/// // TODO signalling guards +/// assert_eq!(waiting_fence.is_signaled(), false); +/// fence.signal(Ok(())); +/// assert_eq!(waiting_fence.is_signaled(), true); +/// +/// Ok::<(), Error>(()) +/// ``` +pub struct DriverFence<'a, T: Send + Sync + FenceContextOps> { + /// The actual content of the fence. Lives in a [`NonNull`] so that its + /// memory can be managed independently. Valid until both the [`DriverFence`] + /// and all associated [`Fence`]s have disappeared. + data: NonNull<DriverFenceData<'a, T>>, +} + +/// A pre-prepared DMA fence, carrying the user's data and the memory it and the +/// fence reside in. Only useful for creating a [`DriverFence`]. Splitting +/// allocation and full initialization is necessary because fences cannot be +/// allocated dynamically in some circumstances (deadlock). +pub struct DriverFenceAllocation<'a, T: Send + Sync + FenceContextOps> { + /// The memory for the actual content of the fence. + /// Handed over to a [`DriverFence`], or deallocated once the + /// [`DriverFenceAllocation`] drops. + data: KBox<DriverFenceData<'a, T>>, + /// Reference for the ops for the associated [`FenceContext`] + ops: &'static bindings::dma_fence_ops, +} + +impl<'a, T: Send + Sync + FenceContextOps> DriverFenceAllocation<'a, T> { + /// Create a new [`DriverFence`], the signalable counterpart of a [`Fence`]. + /// + /// This increments the sequence number in the associated [`FenceContext`]. + pub fn new_fence(self) -> DriverFence<'a, T> { + // We feed the C dma_fence backend a NULL for the spinlock so that it + // uses per-fence locks automatically. + let null_ptr: *mut bindings::spinlock = ptr::null_mut(); + let seqno = self.data.fctx.next_seqno(); + let fence_ptr = self.as_raw(); + // SAFETY: `fence_ptr` has been created directly above. It will live + // at least as long as `Self`. The same applies to `&Self::OPS`. + unsafe { + bindings::dma_fence_init(fence_ptr, self.ops, null_ptr, self.data.fctx.nr, seqno) + }; + + self.data.fctx.nr_of_unsignaled_fences.fetch_add(1, Relaxed); + + // A `DriverFenceAllocation`'s purpose is to carry allocated memory, so + // that `DriverFence`s can always be created without allocating. In this + // method, ownership over that memory is transferred to the new + // `DriverFence` and managed through refcounting. The C dma_fence + // backend will ultimately free the memory once the refcount reaches 0. + let ptr = KBox::into_raw(self.data); + // SAFETY: `ptr` was just created validly directly above. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + + DriverFence { data: ptr } + } + + fn as_raw(&self) -> *mut bindings::dma_fence { + self.data.inner.inner.get() + } +} + +impl<'a, T: Send + Sync + FenceContextOps> DriverFence<'a, T> { + fn as_raw(&self) -> *mut bindings::dma_fence { + // SAFETY: Valid because `self` is valid. + let fence_data = unsafe { &*self.data.as_ptr() }; + + fence_data.inner.inner.get() + } + + /// Create a [`DriverFence`] from a raw pointer to a [`bindings::dma_fence`]. + /// + /// # Safety + /// + /// `ptr` must be a valid pointer to a `dma_fence` that was obtained through + /// a [`DriverFence`] with matching generic data for both fence and associated + /// [`FenceContext`]. + unsafe fn from_raw(ptr: *mut bindings::dma_fence) -> Self { + let opaque_fence = Opaque::cast_from(ptr); + + // SAFETY: Safe due to the function's overall safety requirements. + let fence_ptr = unsafe { container_of!(opaque_fence, Fence, inner) }; + + // DriverFenceData is `repr(C)` and a Fence is its first member. + let fence_data_ptr = fence_ptr as *mut DriverFenceData<'a, T>; + + // SAFETY: `fence_data_ptr` was created validly above. + let data = unsafe { NonNull::new_unchecked(fence_data_ptr) }; + + Self { data } + } + + /// Return the underlying [`Fence`]. + #[inline] + pub fn as_fence(&self) -> &Fence { + // SAFETY: `self` is by definition still valid, and it cannot drop until + // this new reference is gone. + unsafe { Fence::from_raw(self.as_raw()) } + } + + /// Signal the fence. This will invoke all registered callbacks. + pub fn signal(self, res: Result) { + let fence = self.as_fence().lock(); + + // SAFETY: `fence` is valid because `self` is valid. The lock must be + // held, which we acquired directly above. + if !unsafe { bindings::dma_fence_test_signaled_flag(fence.as_raw()) } { + if let Err(err) = res { + // SAFETY: `fence` is valid because `self` is valid. The fence + // must not have been signaled yet, which we check directly above. + unsafe { bindings::dma_fence_set_error(fence.as_raw(), err.to_errno()) }; + } + // SAFETY: `fence` is valid because `self` is valid. The lock must + // be held, which we acquired above. + unsafe { bindings::dma_fence_signal_locked(fence.as_raw()) }; + } + + // SAFETY: `self.data` is valid because `self` is valid. + let fctx = unsafe { self.data.as_ref().fctx }; + let _ = fctx.nr_of_unsignaled_fences.fetch_sub(1, Relaxed); + } +} + +// SAFETY: Fences are literally designed to be shared between threads. +unsafe impl<'a, T: Send + Sync + FenceContextOps> Send for DriverFence<'a, T> {} +// SAFETY: Fences are literally designed to be shared between threads. +unsafe impl<'a, T: Send + Sync + FenceContextOps> Sync for DriverFence<'a, T> {} + +impl<'a, T: Send + Sync + FenceContextOps> Deref for DriverFence<'a, T> { + type Target = T::FenceDataType; + + fn deref(&self) -> &Self::Target { + // SAFETY: Thanks to refcounting, `data` is always valid as long as `self` is. + let data = unsafe { &*self.data.as_ptr() }; + + &data.data + } +} + +/// A borrow wrapper for [`DriverFence`]. Implements [`Deref`]. +pub struct DriverFenceBorrow<'a, T: Send + Sync + FenceContextOps> { + driver_fence: ManuallyDrop<DriverFence<'a, T>>, + _lifetime: PhantomData<&'a T>, +} + +impl<'a, T: Send + Sync + FenceContextOps> Deref for DriverFenceBorrow<'a, T> { + type Target = DriverFence<'a, T>; + + fn deref(&self) -> &Self::Target { + self.driver_fence.deref() + } +} + +// SAFETY: The Rust dma_fence abstractions are already designed around the inner +// C `dma_fence`, which can serve safely as the identification point when being +// owned by C. Moreover, safety is ensured by not dropping `DriverFence` and by +// only allowing operations without side effects on the Borrowed type. +unsafe impl<T: Send + Sync + FenceContextOps> ForeignOwnable for DriverFence<'_, T> { + type Borrowed<'a> + = DriverFenceBorrow<'a, T> + where + Self: 'a; + type BorrowedMut<'a> + = DriverFenceBorrow<'a, T> + where + Self: 'a; + + const FOREIGN_ALIGN: usize = core::mem::align_of::<bindings::dma_fence>(); + + fn into_foreign(self) -> *mut c_void { + let fence = self; + + let ptr = fence.as_raw(); + + // DriverFence must not drop. + let _ = ManuallyDrop::new(fence); + + ptr.cast() + } + + unsafe fn from_foreign(ptr: *mut c_void) -> Self { + // SAFETY: Safe because the trait implementation only invokes this with + // a valid `ptr`, associated to a `DriverFence` with matching generic data. + unsafe { Self::from_raw(ptr.cast()) } + } + + unsafe fn borrow<'a>(ptr: *mut c_void) -> Self::Borrowed<'a> + where + Self: 'a, + { + // SAFETY: The trait implementation ensures that `ptr` always resides + // within a [`Fence`] within a [`DriverFenceData`]. + let driver_fence = unsafe { Self::from_raw(ptr.cast()) }; + + let driver_fence = ManuallyDrop::new(driver_fence); + + DriverFenceBorrow { + driver_fence, + _lifetime: PhantomData, + } + } + + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> Self::BorrowedMut<'a> + // FIXME: The bound below and the one above in `borrow` should actually be + // unnecessary since the compiler should be able to completely derive all + // necessary information automatically. There is currently a compiler bug + // preventing that, though: + // + // https://github.com/rust-lang/rust/issues/155430. + // + // (Help to) fix the compiler bug and remove the bounds afterwards. + where + Self: 'a, + { + // SAFETY: The trait implementation ensures that `ptr` always resides + // within a [`Fence`] within a [`DriverFenceData`]. + let driver_fence = unsafe { Self::from_raw(ptr.cast()) }; + + let driver_fence = ManuallyDrop::new(driver_fence); + + DriverFenceBorrow { + driver_fence, + _lifetime: PhantomData, + } + } +} + +impl<'a, T: Send + Sync + FenceContextOps> Drop for DriverFence<'a, T> { + fn drop(&mut self) { + let guard = self.as_fence().lock(); + + // Use dma_fence_test_signaled_flag() instead of + // dma_fence_is_signaled_locked() because the C backend wants to get rid + // of the latter. + + // SAFETY: `guard` is valid until the `call_rcu()` below. + let signaled: bool = unsafe { bindings::dma_fence_test_signaled_flag(guard.as_raw()) }; + if !signaled { + pr_err!("DriverFence drops unsignaled. Danger of memory corruption!\n"); + // SAFETY: `guard` is valid until the `call_rcu()` below. The fence + // must not have been signaled yet, which we check directly above. + unsafe { bindings::dma_fence_set_error(guard.as_raw(), ECANCELED.to_errno()) }; + // SAFETY: `guard` is valid until the `call_rcu()` below. The lock + // must be held, which we acquired above. + unsafe { bindings::dma_fence_signal_locked(guard.as_raw()) }; + + // SAFETY: `self.data` is valid because `self` is valid. + let fctx = unsafe { self.data.as_ref().fctx }; + let _ = fctx.nr_of_unsignaled_fences.fetch_sub(1, Relaxed); + } + drop(guard); + + // `DriverFenceData` could be accessed through some dma_fence + // callbacks right now. Access is being revoked in principle above by + // signaling the fence, but since the C backend does not guarantee + // perfect full synchronization, we have to wait for one grace period to + // ensure that all accessors of `DriverFenceData` (through the + // dma_fence_ops accessible through a `Fence`) are gone. + + if !core::mem::needs_drop::<T::FenceDataType>() { + // SAFETY: Once a `DriverFence` is initialized, the inner `fence` is + // valid and initialized. It is valid until the refcount drops to 0, + // which can earliest happen once we drop the `DriverFence`'s + // reference here. + unsafe { bindings::dma_fence_put(self.as_raw()) }; + return; + } + + // SAFETY: Valid because `self` is valid. + let rcu_head_ptr = unsafe { &raw mut (*self.data.as_ptr()).rcu_head }; + + // SAFETY: `call_rcu()` is always safe to be called. `rcu_head_ptr` was + // created validly above. The module must perform a `synchronize_rcu()` + // or `rcu_barrier()` call to guard against module unload. + unsafe { bindings::call_rcu(rcu_head_ptr, Some(drop_driver_fence_data::<T>)) }; + } +} + +// TODO: +// The entire call_rcu() mechanism in the drop above and the code below would be +// unnecessary if C's dma_fence_signal() could be reworked in a way that after it +// ran, the caller knows that no fence_ops callbacks can be running anymore. +// In other words, if the dma_fence backend would use its spinlock for full +// synchronization. +// +// Then we could move the drop_in_place() and dma_fence_put() upwards into the +// drop() implementation and call it a day. + +/// Finally really drop this `DriverFence<T>` +/// +/// # Safety +/// +/// `head` references the `rcu_head` field of an `DriverFenceData<T>`. All +/// accessors to that `DriverFenceData<T>` must be gone by now. This must be +/// ensured by signalling the associated `DriverFence<T>` and then waiting +/// for a grace period until calling this function here. +unsafe extern "C" fn drop_driver_fence_data<T: Send + Sync + FenceContextOps>( + head: *mut bindings::callback_head, +) { + // SAFETY: Caller provides a pointer to the `rcu_head` field of a `DriverFenceData<C>`. + let fence_data = unsafe { container_of!(head, DriverFenceData<'_, T>, rcu_head) }; + + // SAFETY: `fence_data` was created validly above. All the fence's data will + // only drop below, but the raw pointer to the raw C `dma_fence` remains + // valid because the reference count is only decremented at the end of the + // function. + let fence = unsafe { (*fence_data).inner.inner.get() }; + + // SAFETY: `fence_data` was created validly above. The user has already + // dropped the only conventional accessor to the user data, the `DriverFence`, + // one grace period ago. All accessors are gone now. + unsafe { drop_in_place(&raw mut (*fence_data).data) }; + + // The inner `Fence` explicitly does not get dropped because there may be + // many more users / consumers, each holding their own reference. + + // SAFETY: Once a `DriverFence` is initialized, the inner `fence` is valid + // and initialized. It is valid until the refcount drops to 0, which can + // earliest happen once we drop the `DriverFence`'s reference here. + unsafe { bindings::dma_fence_put(fence) }; + + // The actual memory the data associated with a `DriverFence` lives in + // gets freed by the C dma_fence backend once the fence's refcount reaches 0. +} diff --git a/rust/kernel/dma_buf/mod.rs b/rust/kernel/dma_buf/mod.rs new file mode 100644 index 000000000000..4764a828642e --- /dev/null +++ b/rust/kernel/dma_buf/mod.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DMA-buf subsystem abstractions. + +pub mod dma_fence; + +pub use self::dma_fence::{ + DriverFence, + Fence, + FenceCallback, + FenceCallbackRegistration, + FenceContext, + FenceContextOps, // +}; diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 74f6ed690d8b..f6db91e81e79 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -12,7 +12,10 @@ use crate::{ prelude::*, sync::aref::ARef, // }; -use core::ptr::NonNull; +use core::{ + marker::PhantomPinned, + ptr::NonNull, // +}; /// Driver use the GEM memory manager. This should be set for all modern drivers. pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; @@ -143,9 +146,15 @@ pub trait Driver { /// The registration type of a `drm::Device`. /// /// Once the `Registration` structure is dropped, the device is unregistered. +#[pin_data(PinnedDrop)] pub struct Registration<'a, T: Driver> { drm: ARef<drm::Device<T>>, - _reg_data: Pin<KBox<T::RegistrationData<'a>>>, + #[pin] + data: T::RegistrationData<'a>, + /// Even if `T::RegistrationData` is `Unpin`, the registration must not be moved because a + /// pointer to `data` is stored in the DRM device. + #[pin] + _pin: PhantomPinned, } impl<'a, T: Driver> Registration<'a, T> { @@ -159,39 +168,40 @@ impl<'a, T: Driver> Registration<'a, T> { pub unsafe fn new<E>( dev: &'a device::Device<device::Bound>, drm: drm::UnregisteredDevice<T>, - reg_data: impl PinInit<T::RegistrationData<'a>, E>, - flags: usize, - ) -> Result<Self> + data: impl PinInit<T::RegistrationData<'a>, E>, + ) -> impl PinInit<Self, Error> where Error: From<E>, { - let parent = drm.as_ref(); - if parent.as_ref().as_raw() != dev.as_raw() { - return Err(EINVAL); - } - - let reg_data: Pin<KBox<T::RegistrationData<'a>>> = KBox::pin_init(reg_data, GFP_KERNEL)?; - - // Store the registration data pointer in the device before registration, so that it is - // visible once ioctls can be called. - let ptr: NonNull<T::RegistrationData<'static>> = - NonNull::from(Pin::get_ref(reg_data.as_ref())).cast(); - - // SAFETY: No concurrent access; the device is not yet registered. - unsafe { *drm.registration_data.get() = ptr }; - - // SAFETY: `drm` is a valid, initialized but not yet registered DRM device. - let ret = unsafe { bindings::drm_dev_register(drm.as_raw(), flags) }; - if let Err(e) = to_result(ret) { - // SAFETY: `drm_dev_register()` synchronizes SRCU on failure, so no concurrent - // access to `registration_data` is possible at this point. - unsafe { *drm.registration_data.get() = NonNull::dangling() }; - return Err(e); - } - - Ok(Self { + try_pin_init!(Self { + _: { + let parent = drm.as_ref(); + if parent.as_ref().as_raw() != dev.as_raw() { + return Err(EINVAL); + } + }, + + data <- data, drm: (&*drm).into(), - _reg_data: reg_data, + _pin: PhantomPinned, + + _: { + // Store the registration data pointer in the device before registration, so + // that it is visible once ioctls can be called. + let ptr: NonNull<T::RegistrationData<'static>> = + NonNull::from(data.as_ref().get_ref()).cast(); + + // SAFETY: No concurrent access; the device is not yet registered. + unsafe { *drm.registration_data.get() = ptr }; + + // SAFETY: `drm` is a valid, initialized but not yet registered DRM device. + let ret = unsafe { bindings::drm_dev_register(drm.as_raw(), 0) }; + to_result(ret).inspect_err(|_| { + // SAFETY: `drm_dev_register()` synchronizes SRCU on failure, so no + // concurrent access to `registration_data` is possible at this point. + unsafe { *drm.registration_data.get() = NonNull::dangling() }; + })?; + }, }) } @@ -208,8 +218,9 @@ unsafe impl<T: Driver> Sync for Registration<'_, T> {} // SAFETY: Registration with and unregistration from the DRM subsystem can happen from any thread. unsafe impl<T: Driver> Send for Registration<'_, T> {} -impl<T: Driver> Drop for Registration<'_, T> { - fn drop(&mut self) { +#[pinned_drop] +impl<T: Driver> PinnedDrop for Registration<'_, T> { + fn drop(self: Pin<&mut Self>) { // Use `drm_dev_unplug` rather than `drm_dev_unregister` to ensure that existing // `drm_dev_enter()` critical sections complete before unregistration proceeds. This // is required for the safety of `RegistrationGuard`, which relies on the SRCU barrier in diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index e1ebad77ebe2..85657a39e146 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -266,18 +266,7 @@ impl<T: DriverObject, Ctx: DeviceContext> Object<T, Ctx> { free: Some(Self::free_callback), open: Some(open_callback::<T>), close: Some(close_callback::<T>), - print_info: None, - export: None, - pin: None, - unpin: None, - get_sg_table: None, - vmap: None, - vunmap: None, - mmap: None, - status: None, - vm_ops: core::ptr::null_mut(), - evict: None, - rss: None, + ..pin_init::zeroed() }; /// Returns the `Device` that owns this GEM object. diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index a687d46d170d..24cb5343e0db 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -132,19 +132,16 @@ impl<T: DriverObject> Object<T> { open: Some(super::open_callback::<T>), close: Some(super::close_callback::<T>), print_info: Some(bindings::drm_gem_shmem_object_print_info), - export: None, pin: Some(bindings::drm_gem_shmem_object_pin), unpin: Some(bindings::drm_gem_shmem_object_unpin), get_sg_table: Some(bindings::drm_gem_shmem_object_get_sg_table), vmap: Some(bindings::drm_gem_shmem_object_vmap), vunmap: Some(bindings::drm_gem_shmem_object_vunmap), mmap: Some(bindings::drm_gem_shmem_object_mmap), - status: None, - rss: None, #[allow(unused_unsafe, reason = "Safe since Rust 1.82.0")] // SAFETY: `drm_gem_shmem_vm_ops` is a valid, static const on the C side. vm_ops: unsafe { &raw const bindings::drm_gem_shmem_vm_ops }, - evict: None, + ..pin_init::zeroed() }; /// Return a raw pointer to the embedded drm_gem_shmem_object. diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index e52793f77196..9003a1b86da8 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -199,8 +199,13 @@ pub mod code { /// /// The value is a valid `errno` (i.e. `>= -MAX_ERRNO && < 0`). #[derive(Clone, Copy, PartialEq, Eq)] +#[repr(transparent)] pub struct Error(NonZeroI32); +// SAFETY: `Error` is `repr(transparent)` over `NonZeroI32`, which is part of the option layout +// optimization guarantee, so all zeroes is a valid representation for `None`. +unsafe impl pin_init::ZeroableOption for Error {} + impl Error { /// Creates an [`Error`] from a kernel error code. /// diff --git a/rust/kernel/fwctl.rs b/rust/kernel/fwctl.rs index f29244fb0d1d..b7778e4d5ee0 100644 --- a/rust/kernel/fwctl.rs +++ b/rust/kernel/fwctl.rs @@ -53,13 +53,13 @@ pub enum DeviceType { #[derive(Copy, Clone, Debug, Eq, PartialEq)] pub enum RpcScope { /// Read/write access to device configuration. - Configuration = bindings::fwctl_rpc_scope_FWCTL_RPC_CONFIGURATION, + Configuration = bindings::fwctl_rpc_scope::FWCTL_RPC_CONFIGURATION.0, /// Read-only access to debug information. - DebugReadOnly = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_READ_ONLY, + DebugReadOnly = bindings::fwctl_rpc_scope::FWCTL_RPC_DEBUG_READ_ONLY.0, /// Write access to lockdown-compatible debug information. - DebugWrite = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_WRITE, + DebugWrite = bindings::fwctl_rpc_scope::FWCTL_RPC_DEBUG_WRITE.0, /// Full read/write access to all debug information (requires `CAP_SYS_RAWIO`). - DebugWriteFull = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_WRITE_FULL, + DebugWriteFull = bindings::fwctl_rpc_scope::FWCTL_RPC_DEBUG_WRITE_FULL.0, } impl TryFrom<u32> for RpcScope { @@ -539,12 +539,12 @@ impl<T: Operations> VTable<T> { /// - `out_len` must be valid for reading and writing an initialised `usize`. unsafe extern "C" fn fw_rpc_callback( uctx: *mut bindings::fwctl_uctx, - scope: u32, + scope: bindings::fwctl_rpc_scope, rpc_in: *mut ffi::c_void, in_len: usize, out_len: *mut usize, ) -> *mut ffi::c_void { - let scope = match RpcScope::try_from(scope) { + let scope = match RpcScope::try_from(scope.0) { Ok(s) => s, Err(e) => return Error::to_ptr(e), }; diff --git a/rust/kernel/i2c.rs b/rust/kernel/i2c.rs index 0487bae811fb..b4ff25c830fc 100644 --- a/rust/kernel/i2c.rs +++ b/rust/kernel/i2c.rs @@ -5,7 +5,6 @@ // I2C Driver abstractions. use crate::{ acpi, - container_of, device, device_id::{ RawDeviceId, @@ -517,26 +516,6 @@ impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for I2cClient<Ctx> { } } -impl<Ctx: device::DeviceContext> TryFrom<&device::Device<Ctx>> for &I2cClient<Ctx> { - type Error = kernel::error::Error; - - fn try_from(dev: &device::Device<Ctx>) -> Result<Self, Self::Error> { - // SAFETY: By the type invariant of `Device`, `dev.as_raw()` is a valid pointer to a - // `struct device`. - if unsafe { bindings::i2c_verify_client(dev.as_raw()).is_null() } { - return Err(EINVAL); - } - - // SAFETY: We've just verified that the type of `dev` equals to - // `bindings::i2c_client_type`, hence `dev` must be embedded in a valid - // `struct i2c_client` as guaranteed by the corresponding C code. - let idev = unsafe { container_of!(dev.as_raw(), bindings::i2c_client, dev) }; - - // SAFETY: `idev` is a valid pointer to a `struct i2c_client`. - Ok(unsafe { &*idev.cast() }) - } -} - // SAFETY: A `I2cClient` is always reference-counted and can be released from any thread. unsafe impl Send for I2cClient {} diff --git a/rust/kernel/id_pool.rs b/rust/kernel/id_pool.rs index 384753fe0e44..92eeb7caa243 100644 --- a/rust/kernel/id_pool.rs +++ b/rust/kernel/id_pool.rs @@ -4,8 +4,14 @@ //! Rust API for an ID pool backed by a [`BitmapVec`]. +use core::{ + num::NonZero, + ops::Range, // +}; + use crate::alloc::{AllocError, Flags}; use crate::bitmap::BitmapVec; +use crate::ptr::Alignment; /// Represents a dynamic ID pool backed by a [`BitmapVec`]. /// @@ -224,7 +230,13 @@ impl IdPool { /// Finds an unused ID in the bitmap. /// /// Upon success, returns its index. Otherwise, returns [`None`] - /// to indicate that a [`Self::grow_request`] is needed. + /// to indicate that a [`Self::grow_request`] is needed. `offset` must be + /// within bounds. + /// + /// # Panics + /// + /// Panics if `CONFIG_RUST_BITMAP_HARDENED` is enabled and `offset` is out + /// of bounds. #[inline] #[must_use] pub fn find_unused_id(&mut self, offset: usize) -> Option<UnusedId<'_>> { @@ -236,10 +248,43 @@ impl IdPool { } /// Releases an ID. + /// + /// The `id` must be within bounds. + /// + /// # Panics + /// + /// Panics if `CONFIG_RUST_BITMAP_HARDENED` is enabled and `id` is out of + /// bounds. #[inline] pub fn release_id(&mut self, id: usize) { self.map.clear_bit(id); } + + /// Reserves a contiguous area of `count` IDs at or after `offset`. + /// + /// The start of the returned area is a multiple of `align`. + /// + /// Returns the reserved range upon success, or [`None`] if no such area could be found. + #[inline] + #[must_use] + pub fn reserve_ids( + &mut self, + offset: usize, + count: NonZero<usize>, + align: Alignment, + ) -> Option<Range<usize>> { + let start = self.map.next_zero_area(offset, count, align)?; + self.map.set(start, count); + Some(start..start + count.get()) + } + + /// Releases a contiguous area of IDs. + #[inline] + pub fn release_ids(&mut self, range: &Range<usize>) { + if let Some(nbits) = NonZero::new(range.len()) { + self.map.clear(range.start, nbits); + } + } } /// Represents an unused id in an [`IdPool`]. diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 5ce9fd129068..a877df11a03e 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -6,11 +6,17 @@ use core::{ marker::PhantomData, - mem::MaybeUninit, // + mem::MaybeUninit, + num::TryFromIntError, // }; use crate::{ bindings, + fmt, + mem::{ + AsRepr, + AsReprMut, // + }, prelude::*, ptr::{ Alignment, @@ -35,11 +41,73 @@ use register::LocatedRegister; /// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. pub type PhysAddr = bindings::phys_addr_t; -/// Resource Size type. +/// Resource size type. /// -/// This is a type alias to either `u32` or `u64` depending on the config option -/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. -pub type ResourceSize = bindings::resource_size_t; +/// This wraps either `u32` or `u64` depending on the config option +/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a `u64` even on 32-bit architectures. +/// +/// # Examples +/// +/// ``` +/// use kernel::io::ResourceSize; +/// +/// let size = ResourceSize::from_raw(0x1000); +/// assert_eq!(size.into_raw(), 0x1000); +/// +/// // Round-trips through the raw C type. +/// let raw: kernel::bindings::resource_size_t = size.into(); +/// assert_eq!(ResourceSize::from(raw), size); +/// +/// // Fallible conversion to `usize` (can truncate on 32-bit). +/// assert_eq!(usize::try_from(size)?, 0x1000); +/// # Ok::<(), core::num::TryFromIntError>(()) +/// ``` +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct ResourceSize(bindings::resource_size_t); + +impl ResourceSize { + /// Creates a resource size from the raw C type. + #[inline] + pub const fn from_raw(value: bindings::resource_size_t) -> Self { + Self(value) + } + + /// Turns this resource size into the raw C type. + #[inline] + pub const fn into_raw(self) -> bindings::resource_size_t { + self.0 + } +} + +impl fmt::Debug for ResourceSize { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:#x}", self.0) + } +} + +impl From<bindings::resource_size_t> for ResourceSize { + #[inline] + fn from(value: bindings::resource_size_t) -> Self { + Self::from_raw(value) + } +} + +impl From<ResourceSize> for bindings::resource_size_t { + #[inline] + fn from(value: ResourceSize) -> Self { + value.into_raw() + } +} + +impl TryFrom<ResourceSize> for usize { + type Error = TryFromIntError; + + #[inline] + fn try_from(value: ResourceSize) -> Result<Self, Self::Error> { + Self::try_from(value.into_raw()) + } +} /// Untyped I/O region. /// @@ -226,6 +294,17 @@ fn io_view<'a, IO: Io<'a>, U>( Ok(unsafe { IO::Backend::project_view(view, projected_ptr) }) } +/// Returns the primitive view of a I/O view. +#[inline] +fn io_view_as_repr<'a, IO: Io<'a, Target = T>, T: AsRepr>( + this: IO, +) -> <IO::Backend as IoBackend>::View<'a, T::Repr> { + let view = this.as_view(); + + // SAFETY: `AsRepr` guarantees layout compatibility. + unsafe { IO::Backend::project_view(view, IO::Backend::as_ptr(view).cast::<T::Repr>()) } +} + /// I/O backends. /// /// This is an abstract representation to be implemented by arbitrary I/O @@ -353,15 +432,12 @@ pub trait IoCopyable: IoBackend { /// /// - The valid `Base` to operate on. For most registers, this should be [`Region`]. /// - The offset to access (returned by [`IoLoc::offset`]), -/// - The width of the access (determined by [`IoLoc::IoType`]), -/// - The type `T` in which the raw data is returned or provided. +/// - The type `T` in which the data is returned or provided. /// -/// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type -/// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`). +/// `T` is not necessarily the type for underlying I/O operation. Methods that take `IoLoc` have `T: +/// AsRepr` bound and the `<T as AsRepr>::Repr` type would be used to perform I/O and converted to +/// `T` instead. pub trait IoLoc<Base: ?Sized, T> { - /// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset). - type IoType: Into<T> + From<T>; - /// Consumes `self` and returns the offset of this location. fn offset(self) -> usize; } @@ -372,8 +448,6 @@ macro_rules! impl_usize_ioloc { ($($ty:ty),*) => { $( impl<const SIZE: usize> IoLoc<Region<SIZE>, $ty> for usize { - type IoType = $ty; - #[inline(always)] fn offset(self) -> usize { self @@ -437,6 +511,45 @@ pub trait Io<'a>: IoBase<'a> { self.len() == 0 } + /// Convert into a different typed I/O view. + /// + /// The target type must be known (statically) to be of the same or smaller size to current + /// type, and the current view must be properly aligned for the target type. + /// + /// # Examples + /// + /// ```no_run + /// use kernel::io::{ + /// io_project, + /// Mmio, + /// Io, + /// Region, + /// }; + /// #[derive(FromBytes, IntoBytes)] + /// #[repr(C)] + /// struct MyStruct { field: u32, } + /// + /// # fn test(mmio: &Mmio<'_, Region<0x1000>>) { + /// // let mmio: Mmio<'_, Region<0x1000>>; + /// let whole: Mmio<'_, MyStruct> = mmio.cast(); + /// # } + /// ``` + #[inline] + fn cast<U>(self) -> <Self::Backend as IoBackend>::View<'a, U> + where + Self::Target: FromBytes + IntoBytes, + U: FromBytes + IntoBytes, + { + let view = self.as_view(); + let ptr = Self::Backend::as_ptr(view); + + const_assert!(size_of::<U>() <= Self::Target::MIN_SIZE); + const_assert!(align_of::<U>() <= Self::Target::MIN_ALIGN.as_usize()); + + // SAFETY: We have checked bounds and alignment, so this is a valid projection. + unsafe { Self::Backend::project_view(view, ptr.cast()) } + } + /// Try to convert into a different typed I/O view. /// /// A runtime check is performed to ensure that the target type is of same or smaller size to @@ -498,10 +611,10 @@ pub trait Io<'a>: IoBase<'a> { #[inline] fn read_val(self) -> Self::Target where - Self::Backend: IoCapable<Self::Target>, - Self::Target: Sized, + Self::Target: AsReprMut, + Self::Backend: IoCapable<<Self::Target as AsRepr>::Repr>, { - Self::Backend::io_read(self.as_view()) + Self::Target::from_repr(Self::Backend::io_read(io_view_as_repr(self))) } /// Write a value to I/O. @@ -520,10 +633,10 @@ pub trait Io<'a>: IoBase<'a> { #[inline] fn write_val(self, value: Self::Target) where - Self::Backend: IoCapable<Self::Target>, - Self::Target: Sized, + Self::Target: AsRepr, + Self::Backend: IoCapable<<Self::Target as AsRepr>::Repr>, { - Self::Backend::io_write(self.as_view(), value) + Self::Backend::io_write(io_view_as_repr(self), Self::Target::into_repr(value)) } /// Copy-read from I/O memory. @@ -645,7 +758,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_read8(self, offset: usize) -> Result<u8> where - usize: IoLoc<Self::Target, u8, IoType = u8>, + usize: IoLoc<Self::Target, u8>, Self::Backend: IoCapable<u8>, { self.try_read(offset) @@ -655,7 +768,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_read16(self, offset: usize) -> Result<u16> where - usize: IoLoc<Self::Target, u16, IoType = u16>, + usize: IoLoc<Self::Target, u16>, Self::Backend: IoCapable<u16>, { self.try_read(offset) @@ -665,7 +778,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_read32(self, offset: usize) -> Result<u32> where - usize: IoLoc<Self::Target, u32, IoType = u32>, + usize: IoLoc<Self::Target, u32>, Self::Backend: IoCapable<u32>, { self.try_read(offset) @@ -675,7 +788,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_read64(self, offset: usize) -> Result<u64> where - usize: IoLoc<Self::Target, u64, IoType = u64>, + usize: IoLoc<Self::Target, u64>, Self::Backend: IoCapable<u64>, { self.try_read(offset) @@ -685,7 +798,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_write8(self, value: u8, offset: usize) -> Result where - usize: IoLoc<Self::Target, u8, IoType = u8>, + usize: IoLoc<Self::Target, u8>, Self::Backend: IoCapable<u8>, { self.try_write(offset, value) @@ -695,7 +808,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_write16(self, value: u16, offset: usize) -> Result where - usize: IoLoc<Self::Target, u16, IoType = u16>, + usize: IoLoc<Self::Target, u16>, Self::Backend: IoCapable<u16>, { self.try_write(offset, value) @@ -705,7 +818,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_write32(self, value: u32, offset: usize) -> Result where - usize: IoLoc<Self::Target, u32, IoType = u32>, + usize: IoLoc<Self::Target, u32>, Self::Backend: IoCapable<u32>, { self.try_write(offset, value) @@ -715,7 +828,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_write64(self, value: u64, offset: usize) -> Result where - usize: IoLoc<Self::Target, u64, IoType = u64>, + usize: IoLoc<Self::Target, u64>, Self::Backend: IoCapable<u64>, { self.try_write(offset, value) @@ -727,7 +840,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn read8(self, offset: usize) -> u8 where - usize: IoLoc<Self::Target, u8, IoType = u8>, + usize: IoLoc<Self::Target, u8>, Self::Backend: IoCapable<u8>, { self.read(offset) @@ -739,7 +852,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn read16(self, offset: usize) -> u16 where - usize: IoLoc<Self::Target, u16, IoType = u16>, + usize: IoLoc<Self::Target, u16>, Self::Backend: IoCapable<u16>, { self.read(offset) @@ -751,7 +864,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn read32(self, offset: usize) -> u32 where - usize: IoLoc<Self::Target, u32, IoType = u32>, + usize: IoLoc<Self::Target, u32>, Self::Backend: IoCapable<u32>, { self.read(offset) @@ -763,7 +876,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn read64(self, offset: usize) -> u64 where - usize: IoLoc<Self::Target, u64, IoType = u64>, + usize: IoLoc<Self::Target, u64>, Self::Backend: IoCapable<u64>, { self.read(offset) @@ -775,7 +888,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn write8(self, value: u8, offset: usize) where - usize: IoLoc<Self::Target, u8, IoType = u8>, + usize: IoLoc<Self::Target, u8>, Self::Backend: IoCapable<u8>, { self.write(offset, value) @@ -787,7 +900,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn write16(self, value: u16, offset: usize) where - usize: IoLoc<Self::Target, u16, IoType = u16>, + usize: IoLoc<Self::Target, u16>, Self::Backend: IoCapable<u16>, { self.write(offset, value) @@ -799,7 +912,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn write32(self, value: u32, offset: usize) where - usize: IoLoc<Self::Target, u32, IoType = u32>, + usize: IoLoc<Self::Target, u32>, Self::Backend: IoCapable<u32>, { self.write(offset, value) @@ -811,7 +924,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn write64(self, value: u64, offset: usize) where - usize: IoLoc<Self::Target, u64, IoType = u64>, + usize: IoLoc<Self::Target, u64>, Self::Backend: IoCapable<u64>, { self.write(offset, value) @@ -843,11 +956,11 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_read<T, L>(self, location: L) -> Result<T> where + T: AsReprMut, L: IoLoc<Self::Target, T>, - Self::Backend: IoCapable<L::IoType>, + Self::Backend: IoCapable<<T as AsRepr>::Repr>, { - let view = io_view::<Self, L::IoType>(self, location.offset())?; - Ok(Self::Backend::io_read(view).into()) + Ok(io_read!(self, try: location)) } /// Generic fallible write with runtime bounds check. @@ -876,12 +989,11 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_write<T, L>(self, location: L, value: T) -> Result where + T: AsRepr, L: IoLoc<Self::Target, T>, - Self::Backend: IoCapable<L::IoType>, + Self::Backend: IoCapable<<T as AsRepr>::Repr>, { - let view = io_view::<Self, L::IoType>(self, location.offset())?; - let io_value = value.into(); - Self::Backend::io_write(view, io_value); + io_write!(self, try: location, value); Ok(()) } @@ -900,6 +1012,8 @@ pub trait Io<'a>: IoBase<'a> { /// }; /// /// register! { + /// base: Region; + /// /// VERSION(u32) @ 0x100 { /// 15:8 major; /// 7:0 minor; @@ -920,9 +1034,10 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_write_reg<T, L, V>(self, value: V) -> Result where + T: AsRepr, L: IoLoc<Self::Target, T>, V: LocatedRegister<Self::Target, Location = L, Value = T>, - Self::Backend: IoCapable<L::IoType>, + Self::Backend: IoCapable<<T as AsRepr>::Repr>, { let (location, value) = value.into_io_op(); @@ -954,16 +1069,13 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_update<T, L, F>(self, location: L, f: F) -> Result where + T: AsReprMut, L: IoLoc<Self::Target, T>, - Self::Backend: IoCapable<L::IoType>, + Self::Backend: IoCapable<<T as AsRepr>::Repr>, F: FnOnce(T) -> T, { - let view = io_view::<Self, L::IoType>(self, location.offset())?; - - let value: T = Self::Backend::io_read(view).into(); - let io_value = f(value).into(); - Self::Backend::io_write(view, io_value); - + let view = io_project!(self, try: location); + view.write_val(f(view.read_val())); Ok(()) } @@ -991,11 +1103,11 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn read<T, L>(self, location: L) -> T where + T: AsReprMut, L: IoLoc<Self::Target, T>, - Self::Backend: IoCapable<L::IoType>, + Self::Backend: IoCapable<<T as AsRepr>::Repr>, { - let view = io_view_assert::<Self, L::IoType>(self, location.offset()); - Self::Backend::io_read(view).into() + io_read!(self, build: location) } /// Generic infallible write with compile-time bounds check. @@ -1022,12 +1134,11 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn write<T, L>(self, location: L, value: T) where + T: AsRepr, L: IoLoc<Self::Target, T>, - Self::Backend: IoCapable<L::IoType>, + Self::Backend: IoCapable<<T as AsRepr>::Repr>, { - let view = io_view_assert::<Self, L::IoType>(self, location.offset()); - let io_value = value.into(); - Self::Backend::io_write(view, io_value); + io_write!(self, build: location, value); } /// Generic infallible write of a fully-located register value. @@ -1045,6 +1156,8 @@ pub trait Io<'a>: IoBase<'a> { /// }; /// /// register! { + /// base: Region<0x1000>; + /// /// VERSION(u32) @ 0x100 { /// 15:8 major; /// 7:0 minor; @@ -1064,9 +1177,10 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn write_reg<T, L, V>(self, value: V) where + T: AsRepr, L: IoLoc<Self::Target, T>, V: LocatedRegister<Self::Target, Location = L, Value = T>, - Self::Backend: IoCapable<L::IoType>, + Self::Backend: IoCapable<<T as AsRepr>::Repr>, { let (location, value) = value.into_io_op(); @@ -1098,14 +1212,13 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn update<T, L, F>(self, location: L, f: F) where + T: AsReprMut, L: IoLoc<Self::Target, T>, - Self::Backend: IoCapable<L::IoType>, + Self::Backend: IoCapable<<T as AsRepr>::Repr>, F: FnOnce(T) -> T, { - let view = io_view_assert::<Self, L::IoType>(self, location.offset()); - let value: T = Self::Backend::io_read(view).into(); - let io_value = f(value).into(); - Self::Backend::io_write(view, io_value); + let view = io_project!(self, build: location); + view.write_val(f(view.read_val())); } } @@ -1649,6 +1762,25 @@ where // SAFETY: Per safety requirement. unsafe { T::Backend::project_view::<T::Target, _>(self.0, ptr) } } + + #[inline(always)] + pub fn try_project_loc<U, L>( + self, + location: L, + ) -> Result<<T::Backend as IoBackend>::View<'a, U>> + where + L: IoLoc<T::Target, U>, + { + io_view::<_, U>(self.0, location.offset()) + } + + #[inline(always)] + pub fn project_loc<U, L>(self, location: L) -> <T::Backend as IoBackend>::View<'a, U> + where + L: IoLoc<T::Target, U>, + { + io_view_assert::<_, U>(self.0, location.offset()) + } } /// Project an I/O type to a subview of it. @@ -1656,26 +1788,54 @@ where /// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that /// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). /// +/// `io_project!` can also project to a subview of registers defined with [`register!`] macro. +/// Register projection has syntax `io_project!(io, try: REGISTER)` for fallible projection and +/// `io_project!(io, build: REGISTER)` for infallible projection. +/// /// # Examples /// /// ``` /// use kernel::io::{ /// io_project, +/// register, /// Mmio, /// }; /// #[repr(C)] /// struct MyStruct { field: u32, } /// +/// register! { +/// base: MyStruct; +/// FIELD(u32) @ 0 { +/// 31:0 val; +/// } +/// } +/// /// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result { /// // let mmio: Mmio<[MyStruct]>; /// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field); /// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]); /// let nested: Mmio<'_, u32> = io_project!(whole, .field); +/// let reg: Mmio<'_, FIELD> = io_project!(whole, build: FIELD); /// # Ok::<(), Error>(()) } /// ``` #[macro_export] #[doc(hidden)] macro_rules! io_project { + // Register projection + ($io:expr, try: $ioloc:expr) => {{ + #[allow(unused)] + use $crate::io::IoBase as _; + let view = $crate::io::ProjectHelper($io.as_view()); + view.try_project_loc($ioloc)? + }}; + ($io:expr, build: $ioloc:expr) => {{ + #[allow(unused)] + use $crate::io::IoBase as _; + let view = $crate::io::ProjectHelper($io.as_view()); + view.project_loc($ioloc) + }}; + + // Field or index projection ($io:expr, $($proj:tt)*) => {{ #[allow(unused)] use $crate::io::IoBase as _; @@ -1746,6 +1906,12 @@ macro_rules! io_write { (@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => { $crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*]) }; + (@parse [$io:expr] [] [try: $ioloc:expr, $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [try: $ioloc] [, $($rest)*]) + }; + (@parse [$io:expr] [] [build: $ioloc:expr, $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [build: $ioloc] [, $($rest)*]) + }; ($io:expr, $($rest:tt)*) => { $crate::io_write!(@parse [$io] [] [$($rest)*]) }; diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 03dfd2ff48c7..b6513fa0f412 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -8,14 +8,19 @@ //! //! Note: most of the items in this module are public so they can be referenced by the macro, but //! most are not to be used directly by users. Outside of the `register!` macro itself, the only -//! items you might want to import from this module are [`WithBase`] and [`Array`]. +//! item you might want to import from this module is [`Array`]. //! //! # Simple example //! //! ```no_run -//! use kernel::io::register; +//! use kernel::io::{ +//! register, +//! Region, +//! }; //! //! register! { +//! base: Region<0x1000>; +//! //! /// Basic information about the chip. //! pub BOOT_0(u32) @ 0x00000100 { //! /// Vendor ID. @@ -55,11 +60,14 @@ //! register, //! Io, //! IoLoc, +//! Region, //! }, //! num::Bounded, //! }; -//! # use kernel::io::{Mmio, Region}; +//! # use kernel::io::Mmio; //! # register! { +//! # base: Region<0x1000>; +//! # //! # pub BOOT_0(u32) @ 0x00000100 { //! # 15:8 vendor_id; //! # 7:4 major_revision; @@ -113,149 +121,50 @@ use crate::{ io::IoLoc, // }; -use super::Region; - -/// Trait implemented by all registers. -pub trait Register: Sized { - /// Backing primitive type of the register. - type Storage: Into<Self> + From<Self>; - - /// Start offset of the register. - /// - /// The interpretation of this offset depends on the type of the register. - const OFFSET: usize; -} - -/// Trait implemented by registers with a fixed offset. -pub trait FixedRegister: Register {} - /// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when -/// passing a [`FixedRegister`] value. -impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for () +/// passing a [`FixedIoLoc`] value. +impl<Base: ?Sized, T> IoLoc<Base, T> for () where - T: FixedRegister, + T: FixedIoLoc<Base>, { - type IoType = T::Storage; - #[inline(always)] fn offset(self) -> usize { - T::OFFSET + T::LOCATION.offset() } } -/// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used -/// as an [`IoLoc`]. -impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for T -where - T: FixedRegister, -{ - type IoType = T::Storage; - - #[inline(always)] - fn offset(self) -> usize { - T::OFFSET - } -} - -/// Location of a fixed register. -pub struct FixedRegisterLoc<T: FixedRegister>(PhantomData<T>); - -impl<T: FixedRegister> FixedRegisterLoc<T> { - /// Returns the location of `T`. - #[inline(always)] - // We do not implement `Default` so we can be const. - #[expect(clippy::new_without_default)] - pub const fn new() -> Self { - Self(PhantomData) - } -} - -impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for FixedRegisterLoc<T> -where - T: FixedRegister, -{ - type IoType = T::Storage; - - #[inline(always)] - fn offset(self) -> usize { - T::OFFSET - } -} - -/// Trait providing a base address to be added to the offset of a relative register to obtain -/// its actual offset. -/// -/// The `T` generic argument is used to distinguish which base to use, in case a type provides -/// several bases. It is given to the `register!` macro to restrict the use of the register to -/// implementors of this particular variant. -pub trait RegisterBase<T> { - /// Base address to which register offsets are added. - const BASE: usize; -} - -/// Trait implemented by all registers that are relative to a base. -pub trait WithBase { - /// Family of bases applicable to this register. - type BaseFamily; +// Provides a `IoLoc` impl that for a fixed offset. +#[doc(hidden)] +pub struct OffsetLoc<Base: ?Sized, T>(usize, PhantomData<(T, Base)>); - /// Returns the absolute location of this type when using `B` as its base. - #[inline(always)] - fn of<B: RegisterBase<Self::BaseFamily>>() -> RelativeRegisterLoc<Self, B> - where - Self: Register, - { - RelativeRegisterLoc::new() - } -} - -/// Trait implemented by relative registers. -pub trait RelativeRegister: Register + WithBase {} - -/// Location of a relative register. -/// -/// This can either be an immediately accessible regular [`RelativeRegister`], or a -/// [`RelativeRegisterArray`] that needs one additional resolution through -/// [`RelativeRegisterLoc::at`]. -pub struct RelativeRegisterLoc<T: WithBase, B: ?Sized>(PhantomData<T>, PhantomData<B>); - -impl<T, B> RelativeRegisterLoc<T, B> -where - T: Register + WithBase, - B: RegisterBase<T::BaseFamily> + ?Sized, -{ - /// Returns the location of a relative register or register array. - #[inline(always)] - // We do not implement `Default` so we can be const. - #[expect(clippy::new_without_default)] - pub const fn new() -> Self { - Self(PhantomData, PhantomData) +impl<Base: ?Sized, T> OffsetLoc<Base, T> { + #[inline] + pub const fn new(offset: usize) -> Self { + Self(offset, PhantomData) } - // Returns the absolute offset of the relative register using base `B`. - // - // This is implemented as a private const method so it can be reused by the [`IoLoc`] - // implementations of both [`RelativeRegisterLoc`] and [`RelativeRegisterArrayLoc`]. #[inline] - const fn offset(self) -> usize { - B::BASE + T::OFFSET + pub const fn const_offset(self) -> usize { + self.0 } } -impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterLoc<T, B> -where - T: RelativeRegister, - B: RegisterBase<T::BaseFamily> + ?Sized, -{ - type IoType = T::Storage; - +impl<Base: ?Sized, T> IoLoc<Base, T> for OffsetLoc<Base, T> { #[inline(always)] fn offset(self) -> usize { - RelativeRegisterLoc::offset(self) + self.0 } } /// Trait implemented by arrays of registers. -pub trait RegisterArray: Register { +pub trait RegisterArray: Sized { + /// Base type for this register. + type Base: ?Sized; + + /// Start offset of the register. + /// + /// The interpretation of this offset depends on the type of the register. + const OFFSET: usize; /// Number of elements in the registers array. const SIZE: usize; /// Number of bytes between the start of elements in the registers array. @@ -285,12 +194,10 @@ impl<T: RegisterArray> RegisterArrayLoc<T> { } } -impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for RegisterArrayLoc<T> +impl<Base: ?Sized, T> IoLoc<Base, T> for RegisterArrayLoc<T> where - T: RegisterArray, + T: RegisterArray<Base = Base>, { - type IoType = T::Storage; - #[inline(always)] fn offset(self) -> usize { T::OFFSET + self.0 * T::STRIDE @@ -318,71 +225,15 @@ pub trait Array { } } -/// Trait implemented by arrays of relative registers. -pub trait RelativeRegisterArray: RegisterArray + WithBase {} - -/// Location of a relative array register. -pub struct RelativeRegisterArrayLoc< - T: RelativeRegisterArray, - B: RegisterBase<T::BaseFamily> + ?Sized, ->(RelativeRegisterLoc<T, B>, usize); - -impl<T, B> RelativeRegisterArrayLoc<T, B> -where - T: RelativeRegisterArray, - B: RegisterBase<T::BaseFamily> + ?Sized, -{ - /// Returns the location of register `T` from the base `B` at index `idx`, with build-time - /// validation. - #[inline(always)] - pub fn new(idx: usize) -> Self { - build_assert!(idx < T::SIZE); - - Self(RelativeRegisterLoc::new(), idx) - } - - /// Attempts to return the location of register `T` from the base `B` at index `idx`, with - /// runtime validation. - #[inline(always)] - pub fn try_new(idx: usize) -> Option<Self> { - if idx < T::SIZE { - Some(Self(RelativeRegisterLoc::new(), idx)) - } else { - None - } - } -} - -/// Methods exclusive to [`RelativeRegisterLoc`]s created with a [`RelativeRegisterArray`]. -impl<T, B> RelativeRegisterLoc<T, B> -where - T: RelativeRegisterArray, - B: RegisterBase<T::BaseFamily> + ?Sized, -{ - /// Returns the location of the register at position `idx`, with build-time validation. - #[inline(always)] - pub fn at(self, idx: usize) -> RelativeRegisterArrayLoc<T, B> { - RelativeRegisterArrayLoc::new(idx) - } - - /// Returns the location of the register at position `idx`, with runtime validation. - #[inline(always)] - pub fn try_at(self, idx: usize) -> Option<RelativeRegisterArrayLoc<T, B>> { - RelativeRegisterArrayLoc::try_new(idx) - } -} - -impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterArrayLoc<T, B> -where - T: RelativeRegisterArray, - B: RegisterBase<T::BaseFamily> + ?Sized, -{ - type IoType = T::Storage; +/// Trait implemented by types that indicate there is a fixed I/O location for this given type. +/// +/// Implementors can be used with [`Io::write_reg`](super::Io::write_reg). +pub trait FixedIoLoc<Base: ?Sized>: Sized { + /// Type of [`FixedIoLoc::LOCATION`]. + type Location: IoLoc<Base, Self>; - #[inline(always)] - fn offset(self) -> usize { - self.0.offset() + self.1 * T::STRIDE - } + /// Location of this type within given base. + const LOCATION: Self::Location; } /// Trait implemented by items that contain both a register value and the absolute I/O location at @@ -390,8 +241,8 @@ where /// /// Implementors can be used with [`Io::write_reg`](super::Io::write_reg). pub trait LocatedRegister<Base: ?Sized> { - /// Register value to write. - type Value: Register; + /// Value to write. + type Value; /// Full location information at which to write the value. type Location: IoLoc<Base, Self::Value>; @@ -400,27 +251,38 @@ pub trait LocatedRegister<Base: ?Sized> { fn into_io_op(self) -> (Self::Location, Self::Value); } -impl<const SIZE: usize, T> LocatedRegister<Region<SIZE>> for T +impl<Base: ?Sized, T> LocatedRegister<Base> for T where - T: FixedRegister, + T: FixedIoLoc<Base>, { - type Location = FixedRegisterLoc<Self::Value>; + type Location = T::Location; type Value = T; #[inline(always)] - fn into_io_op(self) -> (FixedRegisterLoc<T>, T) { - (FixedRegisterLoc::new(), self) + fn into_io_op(self) -> (T::Location, T) { + (T::LOCATION, self) } } +/// Helper function for register element alias implementation. +/// +/// This is used to enforce base matching and provide bounds checking. +#[doc(hidden)] +#[inline(always)] // for const eval only +pub const fn element_alias_offset<Base: ?Sized, Alias: RegisterArray<Base = Base>>( + idx: usize, +) -> usize { + assert!(idx < Alias::SIZE); + Alias::OFFSET + idx * Alias::STRIDE +} + /// Defines a dedicated type for a register, including getter and setter methods for its fields and /// methods to read and write it from an [`Io`](kernel::io::Io) region. /// /// This documentation focuses on how to declare registers. See the [module-level /// documentation](mod@kernel::io::register) for examples of how to access them. /// -/// There are 4 possible kinds of registers: fixed offset registers, relative registers, arrays of -/// registers, and relative arrays of registers. +/// Registers can either be fixed offset registers or arrays of registers. /// /// ## Fixed offset registers /// @@ -444,11 +306,14 @@ where /// io::{ /// register, /// Io, +/// Region, /// }, /// }; -/// # use kernel::io::{Mmio, Region}; +/// # use kernel::io::Mmio; /// /// register! { +/// base: Region<0x1000>; +/// /// FIXED_REG(u32) @ 0x100 { /// 15:8 high_byte; /// 7:0 low_byte; @@ -479,9 +344,14 @@ where /// the context: /// /// ```no_run -/// use kernel::io::register; +/// use kernel::io::{ +/// register, +/// Region, +/// }; /// /// register! { +/// base: Region<0x1000>; +/// /// /// Scratch register. /// pub SCRATCH(u32) @ 0x00000200 { /// 31:0 value; @@ -497,113 +367,45 @@ where /// In this example, `SCRATCH_BOOT_STATUS` uses the same I/O address as `SCRATCH`, while providing /// its own `completed` field. /// -/// ## Relative registers -/// -/// Relative registers can be instantiated several times at a relative offset of a group of bases. -/// For instance, imagine the following I/O space: +/// If you do not wish to have a bitfield defined, you can also create a register using an existing +/// type. /// -/// ```text -/// +-----------------------------+ -/// | ... | -/// | | -/// 0x100--->+------------CPU0-------------+ -/// | | -/// 0x110--->+-----------------------------+ -/// | CPU_CTL | -/// +-----------------------------+ -/// | ... | -/// | | -/// | | -/// 0x200--->+------------CPU1-------------+ -/// | | -/// 0x210--->+-----------------------------+ -/// | CPU_CTL | -/// +-----------------------------+ -/// | ... | -/// +-----------------------------+ -/// ``` -/// -/// `CPU0` and `CPU1` both have a `CPU_CTL` register that starts at offset `0x10` of their I/O -/// space segment. Since both instances of `CPU_CTL` share the same layout, we don't want to define -/// them twice and would prefer a way to select which one to use from a single definition. -/// -/// This can be done using the `Base + Offset` syntax when specifying the register's address: -/// -/// ```ignore +/// ```no_run +/// # use kernel::io::*; /// register! { -/// pub RELATIVE_REG(u32) @ Base + 0x80 { -/// ... -/// } +/// base: Region<0x1000>; +/// +/// /// UART RX register. +/// pub UART_RX: u8 @ 0x100; /// } /// ``` /// -/// This creates a register with an offset of `0x80` from a given base. +/// In case there is a fixed register associated with a specific type in the base, you can apply +/// `#[unique]` attribute which enables `write_reg` shorthand. This is automatically applied to +/// bitfields instantiated via the `register!` macro. /// -/// `Base` is an arbitrary type (typically a ZST) to be used as a generic parameter of the -/// [`RegisterBase`] trait to provide the base as a constant, i.e. each type providing a base for -/// this register needs to implement `RegisterBase<Base>`. -/// -/// The location of relative registers can be built using the [`WithBase::of`] method to specify -/// its base. All relative registers implement [`WithBase`]. -/// -/// Here is the above layout translated into code: +/// This should only be used when types meaningfully represent a register. For example, in the +/// previous `UART_RX` example, even if only a single register is defined with `u8` type, it is a +/// bad idea to annotate it with `#[unique]`. /// /// ```no_run -/// use kernel::{ -/// io::{ -/// register, -/// register::{ -/// RegisterBase, -/// WithBase, -/// }, -/// Io, -/// }, -/// }; -/// # use kernel::io::{Mmio, Region}; -/// -/// // Type used to identify the base. -/// pub struct CpuCtlBase; -/// -/// // ZST describing `CPU0`. -/// struct Cpu0; -/// impl RegisterBase<CpuCtlBase> for Cpu0 { -/// const BASE: usize = 0x100; -/// } -/// -/// // ZST describing `CPU1`. -/// struct Cpu1; -/// impl RegisterBase<CpuCtlBase> for Cpu1 { -/// const BASE: usize = 0x200; -/// } +/// # use kernel::{bitfield, io::*}; /// -/// // This makes `CPU_CTL` accessible from all implementors of `RegisterBase<CpuCtlBase>`. -/// register! { -/// /// CPU core control. -/// pub CPU_CTL(u32) @ CpuCtlBase + 0x10 { -/// 0:0 start; +/// bitfield! { +/// pub struct Reset(u32) { +/// 0:0 reset; /// } /// } /// -/// # fn test(io: Mmio<'_, Region<0x1000>>) { -/// // Read the status of `Cpu0`. -/// let cpu0_started = io.read(CPU_CTL::of::<Cpu0>()); -/// -/// // Stop `Cpu0`. -/// io.write(WithBase::of::<Cpu0>(), CPU_CTL::zeroed()); -/// # } -/// -/// // Aliases can also be defined for relative register. /// register! { -/// /// Alias to CPU core control. -/// pub CPU_CTL_ALIAS(u32) => CpuCtlBase + CPU_CTL { -/// /// Start the aliased CPU core. -/// 1:1 alias_start; -/// } +/// base: Region<0x1000>; +/// +/// pub RESET: #[unique] Reset @ 0x100; /// } /// -/// # fn test2(io: Mmio<'_, Region<0x1000>>) { -/// // Start the aliased `CPU0`, leaving its other fields untouched. -/// io.update(CPU_CTL_ALIAS::of::<Cpu0>(), |r| r.with_alias_start(true)); +/// # fn test(mmio: Mmio<'_, Region<0x1000>>) { +/// // let mmio: Mmio<'_, Region<0x1000>>; +/// mmio.write_reg(Reset::zeroed().with_const_reset::<1>()); /// # } /// ``` /// @@ -636,15 +438,18 @@ where /// register, /// register::Array, /// Io, +/// Region, /// }, /// }; -/// # use kernel::io::{Mmio, Region}; +/// # use kernel::io::Mmio; /// # fn get_scratch_idx() -> usize { /// # 0x15 /// # } /// /// // Array of 64 consecutive registers with the same layout starting at offset `0x80`. /// register! { +/// base: Region<0x1000>; +/// /// /// Scratch registers. /// pub SCRATCH(u32)[64] @ 0x00000080 { /// 31:0 value; @@ -670,6 +475,8 @@ where /// // Alias to a specific register in an array. /// // Here `SCRATCH[8]` is used to convey the firmware exit code. /// register! { +/// base: Region<0x1000>; +/// /// /// Firmware exit status code. /// pub FIRMWARE_STATUS(u32) => SCRATCH[8] { /// 7:0 status; @@ -682,6 +489,8 @@ where /// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the /// // registers of the two declarations below are interleaved. /// register! { +/// base: Region<0x1000>; +/// /// /// Scratch registers bank 0. /// pub SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ 0x000000c0 { /// 31:0 value; @@ -696,332 +505,88 @@ where /// # } /// ``` /// -/// ## Relative arrays of registers +/// ## Relative registers /// -/// Combining the two features described in the sections above, arrays of registers accessible from -/// a base can also be defined: +/// There are cases where a register region is subdivided into small subregions, and you may wish to +/// have your register definition be relative to these subregions. This may be needed, for example, +/// if these subregions are instantiated several times, or you just want it for encapsulation +/// purpose. /// -/// ```ignore -/// register! { -/// pub RELATIVE_REGISTER_ARRAY(u8)[10, stride = 4] @ Base + 0x100 { -/// ... -/// } -/// } +/// For instance, imagine the following I/O space: +/// +/// ```text +/// +-----------------------------+ +/// | ... | +/// | | +/// 0x100--->+------------CPU0-------------+ +/// | | +/// 0x110--->+-----------------------------+ +/// | CPU_CTL | +/// +-----------------------------+ +/// | ... | +/// | | +/// | | +/// 0x200--->+------------CPU1-------------+ +/// | | +/// 0x210--->+-----------------------------+ +/// | CPU_CTL | +/// +-----------------------------+ +/// | ... | +/// +-----------------------------+ /// ``` /// -/// Like relative registers, they implement the [`WithBase`] trait. However the return value of -/// [`WithBase::of`] cannot be used directly as a location and must be further specified using the -/// [`at`](RelativeRegisterLoc::at) method. +/// `CPU0` and `CPU1` both have a `CPU_CTL` register that starts at offset `0x10` of their I/O +/// space segment. Since both instances of `CPU_CTL` share the same layout, we don't want to define +/// them twice and would prefer a way to select which one to use from a single definition. +/// +/// This can be done by defining a new type for the subregion, and then defining registers that use +/// the new type as the base: /// /// ```no_run /// use kernel::{ /// io::{ +/// io_project, /// register, -/// register::{ -/// RegisterBase, -/// WithBase, -/// }, /// Io, +/// Region, /// }, /// }; -/// # use kernel::io::{Mmio, Region}; -/// # fn get_scratch_idx() -> usize { -/// # 0x15 -/// # } +/// # use kernel::io::Mmio; /// -/// // Type used as parameter of `RegisterBase` to specify the base. -/// pub struct CpuCtlBase; -/// -/// // ZST describing `CPU0`. -/// struct Cpu0; -/// impl RegisterBase<CpuCtlBase> for Cpu0 { -/// const BASE: usize = 0x100; -/// } -/// -/// // ZST describing `CPU1`. -/// struct Cpu1; -/// impl RegisterBase<CpuCtlBase> for Cpu1 { -/// const BASE: usize = 0x200; -/// } +/// // Subregion type. Make sure it has adequate size and alignment. +/// #[repr(align(4))] +/// #[derive(FromBytes, IntoBytes)] +/// pub struct CpuCtl([u8; 0x100]); /// -/// // 64 per-cpu scratch registers, arranged as a contiguous array. /// register! { -/// /// Per-CPU scratch registers. -/// pub CPU_SCRATCH(u32)[64] @ CpuCtlBase + 0x00000080 { -/// 31:0 value; -/// } -/// } -/// -/// # fn test(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { -/// // Read scratch register 0 of CPU0. -/// let scratch = io.read(CPU_SCRATCH::of::<Cpu0>().at(0)); +/// base: Region<0x1000>; /// -/// // Write the retrieved value into scratch register 15 of CPU1. -/// io.write(WithBase::of::<Cpu1>().at(15), scratch); -/// -/// // This won't build. -/// // let cpu0_scratch_128 = io.read(CPU_SCRATCH::of::<Cpu0>().at(128)).value(); -/// -/// // Runtime-obtained array index. -/// let scratch_idx = get_scratch_idx(); -/// // Access on a runtime index returns an error if it is out-of-bounds. -/// let cpu0_scratch = io.read( -/// CPU_SCRATCH::of::<Cpu0>().try_at(scratch_idx).ok_or(EINVAL)? -/// ).value(); -/// # Ok(()) -/// # } -/// -/// // Alias to `SCRATCH[8]` used to convey the firmware exit code. -/// register! { -/// /// Per-CPU firmware exit status code. -/// pub CPU_FIRMWARE_STATUS(u32) => CpuCtlBase + CPU_SCRATCH[8] { -/// 7:0 status; -/// } +/// // Subregions can just be defined like normal registers. +/// CPU0: CpuCtl @ 0x100; +/// CPU1: CpuCtl @ 0x200; /// } /// -/// // Non-contiguous relative register arrays can be defined by adding a stride parameter. -/// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the -/// // registers of the two declarations below are interleaved. +/// // Then you can define new registers on the subregion. /// register! { -/// /// Scratch registers bank 0. -/// pub CPU_SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d00 { -/// 31:0 value; -/// } +/// base: CpuCtl; /// -/// /// Scratch registers bank 1. -/// pub CPU_SCRATCH_INTERLEAVED_1(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d04 { -/// 31:0 value; +/// /// CPU core control. +/// pub CPU_CTL(u32) @ 0x10 { +/// 0:0 start; /// } /// } /// -/// # fn test2(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { -/// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::<Cpu0>()).status(); -/// # Ok(()) +/// # fn test(io: Mmio<'_, Region<0x1000>>) { +/// // Read the status of `Cpu0`. +/// let cpu0_started = io_project!(io, build: CPU0).read(CPU_CTL); +/// +/// // Stop `Cpu0`. +/// io_project!(io, build: CPU0).write_reg(CPU_CTL::zeroed()); /// # } /// ``` #[macro_export] macro_rules! register { - // Entry point for the macro, allowing multiple registers to be defined in one call. - // It matches all possible register declaration patterns to dispatch them to corresponding - // `@reg` rule that defines a single register. - // - // TODO: change `alias:ident` to `alias:path` once relative registers are replaced by I/O - // projections. - ( - $( - $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) - $([ $size:expr $(, stride = $stride:expr)? ])? - $(@ $($base:ident +)? $offset:literal)? - $(=> $alias:ident $(+ $alias_offset:ident)? $([$alias_idx:expr])? )? - { $($fields:tt)* } - )* - ) => { - $( - $crate::register!( - @reg $(#[$attr])* $vis $name ($storage) $([$size $(, stride = $stride)?])? - $(@ $($base +)? $offset)? - $(=> $alias $(+ $alias_offset)? $([$alias_idx])? )? - { $($fields)* } - ); - )* - }; - - // All the rules below are private helpers. - - // Creates a register at a fixed offset of the MMIO space. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $offset:literal - { $($fields:tt)* } - ) => { - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_fixed $(#[$attr])* $vis $name); - }; - - // Creates an alias register of fixed offset register `alias` with its own fields. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path - { $($fields:tt)* } - ) => { - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!( - @io_base $name($storage) @ - <$alias as $crate::io::register::Register>::OFFSET - ); - $crate::register!(@io_fixed $(#[$attr])* $vis $name); - }; - - // Creates a register at a relative offset from a base address provider. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $base:ident + $offset:literal - { $($fields:tt)* } - ) => { - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_relative $name @ $base); - }; - - // Creates an alias register of relative offset register `alias` with its own fields. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $base:ident + $alias:ident - { $($fields:tt)* } - ) => { - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!( - @io_base $name($storage) @ <$alias as $crate::io::register::Register>::OFFSET - ); - $crate::register!(@io_relative $name @ $base); - }; - - // Creates an array of registers at a fixed offset of the MMIO space. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) - [ $size:expr, stride = $stride:expr ] @ $offset:literal { $($fields:tt)* } - ) => { - $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride); - - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_array $name [ $size, stride = $stride ]); - }; - - // Shortcut for contiguous array of registers (stride == size of element). - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] @ $offset:literal - { $($fields:tt)* } - ) => { - $crate::register!( - @reg $(#[$attr])* $vis $name($storage) - [ $size, stride = ::core::mem::size_of::<$storage>() ] - @ $offset { $($fields)* } - ); - }; - - // Creates an alias of register `idx` of array of registers `alias` with its own fields. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path [ $idx:expr ] - { $($fields:tt)* } - ) => { - $crate::build_assert::static_assert!( - $idx < <$alias as $crate::io::register::RegisterArray>::SIZE - ); - - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!( - @io_base $name($storage) @ - <$alias as $crate::io::register::Register>::OFFSET - + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE - ); - $crate::register!(@io_fixed $(#[$attr])* $vis $name); - }; - - // Creates an array of registers at a relative offset from a base address provider. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) - [ $size:expr, stride = $stride:expr ] - @ $base:ident + $offset:literal { $($fields:tt)* } - ) => { - $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride); - - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_relative_array $name [ $size, stride = $stride ] @ $base); - }; - - // Shortcut for contiguous array of relative registers (stride == size of element). - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] - @ $base:ident + $offset:literal { $($fields:tt)* } - ) => { - $crate::register!( - @reg $(#[$attr])* $vis $name($storage) - [ $size, stride = ::core::mem::size_of::<$storage>() ] - @ $base + $offset { $($fields)* } - ); - }; - - // Creates an alias of register `idx` of relative array of registers `alias` with its own - // fields. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) - => $base:ident + $alias:ident [ $idx:expr ] { $($fields:tt)* } - ) => { - $crate::build_assert::static_assert!( - $idx < <$alias as $crate::io::register::RegisterArray>::SIZE - ); - - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!( - @io_base $name($storage) @ - <$alias as $crate::io::register::Register>::OFFSET + - $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE - ); - $crate::register!(@io_relative $name @ $base); - }; - - // Generates the bitfield for the register. - // - // `#[allow(non_camel_case_types)]` is added since register names typically use - // `SCREAMING_CASE`. - ( - @bitfield $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* } - ) => { - $crate::bitfield!( - #[allow(non_camel_case_types)] - $(#[$attr])* $vis struct $name($storage) { $($fields)* } - ); - }; - - // Implementations shared by all registers types. - (@io_base $name:ident($storage:ty) @ $offset:expr) => { - impl $crate::io::register::Register for $name { - type Storage = $storage; - - const OFFSET: usize = $offset; - } - }; - - // Implementations of fixed registers. - (@io_fixed $(#[$attr:meta])* $vis:vis $name:ident) => { - impl $crate::io::register::FixedRegister for $name {} - - $(#[$attr])* - $vis const $name: $crate::io::register::FixedRegisterLoc<$name> = - $crate::io::register::FixedRegisterLoc::<$name>::new(); - }; - - // Implementations of relative registers. - (@io_relative $name:ident @ $base:ident) => { - impl $crate::io::register::WithBase for $name { - type BaseFamily = $base; - } - - impl $crate::io::register::RelativeRegister for $name {} - }; - - // Implementations of register arrays. - (@io_array $name:ident [ $size:expr, stride = $stride:expr ]) => { - impl $crate::io::register::Array for $name {} - - impl $crate::io::register::RegisterArray for $name { - const SIZE: usize = $size; - const STRIDE: usize = $stride; - } - }; - - // Implementations of relative array registers. - ( - @io_relative_array $name:ident [ $size:expr, stride = $stride:expr ] @ $base:ident - ) => { - impl $crate::io::register::WithBase for $name { - type BaseFamily = $base; - } - - impl $crate::io::register::RegisterArray for $name { - const SIZE: usize = $size; - const STRIDE: usize = $stride; - } - - impl $crate::io::register::RelativeRegisterArray for $name {} + ($($tt:tt)*) => { + $crate::macros::register!($($tt)*); }; } diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs index 17b0c174cfc5..758ce62b3cce 100644 --- a/rust/kernel/io/resource.rs +++ b/rust/kernel/io/resource.rs @@ -58,7 +58,7 @@ impl Drop for Region { }; // SAFETY: Safe as per the invariant of `Region`. - unsafe { release_fn(start, size) }; + unsafe { release_fn(start, size.into_raw()) }; } } @@ -114,7 +114,7 @@ impl Resource { bindings::__request_region( self.0.get(), start, - size, + size.into_raw(), name.as_char_ptr(), flags.0 as c_int, ) @@ -130,7 +130,7 @@ impl Resource { pub fn size(&self) -> ResourceSize { let inner = self.0.get(); // SAFETY: Safe as per the invariants of `Resource`. - unsafe { bindings::resource_size(inner) } + ResourceSize::from_raw(unsafe { bindings::resource_size(inner) }) } /// Returns the start address of the resource. @@ -226,10 +226,18 @@ impl Flags { /// Resource represents a memory region that must be ioremaped using `ioremap_np`. pub const IORESOURCE_MEM_NONPOSTED: Flags = Flags::new(bindings::IORESOURCE_MEM_NONPOSTED); + /// Memory region uses a 64-bit address (consumes two consecutive PCI resource slots). + pub const IORESOURCE_MEM_64: Flags = Flags::new(bindings::IORESOURCE_MEM_64); + // Always inline to optimize out error path of `build_assert`. #[inline(always)] const fn new(value: u32) -> Self { build_assert!(value as u64 <= c_ulong::MAX as u64); Flags(value as c_ulong) } + + /// Wrap a raw `c_ulong` value returned by a C API into [`Flags`]. + pub(crate) const fn from_raw(value: c_ulong) -> Self { + Flags(value) + } } diff --git a/rust/kernel/irq/request.rs b/rust/kernel/irq/request.rs index c1c6525a676a..1afa9fcaad1f 100644 --- a/rust/kernel/irq/request.rs +++ b/rust/kernel/irq/request.rs @@ -25,10 +25,10 @@ use crate::{ #[repr(u32)] pub enum IrqReturn { /// The interrupt was not from this device or was not handled. - None = bindings::irqreturn_IRQ_NONE, + None = bindings::irqreturn::IRQ_NONE.0, /// The interrupt was handled by this device. - Handled = bindings::irqreturn_IRQ_HANDLED, + Handled = bindings::irqreturn::IRQ_HANDLED.0, } /// Callbacks for an IRQ handler. @@ -240,25 +240,28 @@ impl<T: Handler> PinnedDrop for Registration<'_, T> { /// # Safety /// /// This function should be only used as the callback in `request_irq`. -unsafe extern "C" fn handle_irq_callback<T: Handler>(_irq: i32, ptr: *mut c_void) -> c_uint { +unsafe extern "C" fn handle_irq_callback<T: Handler>( + _irq: i32, + ptr: *mut c_void, +) -> bindings::irqreturn_t { let ptr = ptr.cast_const().cast::<Registration<'_, T>>(); // SAFETY: `ptr` is a pointer to `Registration<'_, T>` set in `Registration::new()`. let registration = unsafe { &*ptr }; - T::handle(®istration.handler) as c_uint + bindings::irqreturn_t(T::handle(®istration.handler) as c_uint) } /// The value that can be returned from [`ThreadedHandler::handle`]. #[repr(u32)] pub enum ThreadedIrqReturn { /// The interrupt was not from this device or was not handled. - None = bindings::irqreturn_IRQ_NONE, + None = bindings::irqreturn::IRQ_NONE.0, /// The interrupt was handled by this device. - Handled = bindings::irqreturn_IRQ_HANDLED, + Handled = bindings::irqreturn::IRQ_HANDLED.0, /// The handler wants the handler thread to wake up. - WakeThread = bindings::irqreturn_IRQ_WAKE_THREAD, + WakeThread = bindings::irqreturn::IRQ_WAKE_THREAD.0, } /// Callbacks for a threaded IRQ handler. @@ -461,23 +464,26 @@ impl<T: ThreadedHandler> PinnedDrop for ThreadedRegistration<'_, T> { unsafe extern "C" fn handle_threaded_irq_callback<T: ThreadedHandler>( _irq: i32, ptr: *mut c_void, -) -> c_uint { +) -> bindings::irqreturn_t { let ptr = ptr.cast_const().cast::<ThreadedRegistration<'_, T>>(); // SAFETY: `ptr` is a pointer to `ThreadedRegistration<'_, T>` set in // `ThreadedRegistration::new()`. let registration = unsafe { &*ptr }; - T::handle(®istration.handler) as c_uint + bindings::irqreturn_t(T::handle(®istration.handler) as c_uint) } /// # Safety /// /// This function should be only used as the callback in `request_threaded_irq`. -unsafe extern "C" fn thread_fn_callback<T: ThreadedHandler>(_irq: i32, ptr: *mut c_void) -> c_uint { +unsafe extern "C" fn thread_fn_callback<T: ThreadedHandler>( + _irq: i32, + ptr: *mut c_void, +) -> bindings::irqreturn_t { let ptr = ptr.cast_const().cast::<ThreadedRegistration<'_, T>>(); // SAFETY: `ptr` is a pointer to `ThreadedRegistration<'_, T>` set in // `ThreadedRegistration::new()`. let registration = unsafe { &*ptr }; - T::handle_threaded(®istration.handler) as c_uint + bindings::irqreturn_t(T::handle_threaded(®istration.handler) as c_uint) } diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs index 91eaff8c186a..25ff272d29b2 100644 --- a/rust/kernel/kunit.rs +++ b/rust/kernel/kunit.rs @@ -58,15 +58,17 @@ pub fn info(args: fmt::Arguments<'_>) { #[doc(hidden)] #[macro_export] macro_rules! kunit_assert { - ($name:literal, $file:literal, $diff:expr, $condition:expr $(,)?) => { + ($name:literal, $condition:expr $(,)?) => { 'out: { // Do nothing if the condition is `true`. if $condition { break 'out; } - static FILE: &'static $crate::str::CStr = $file; - static LINE: i32 = ::core::line!() as i32 - $diff; + // Use `file!()` instead of `::core::file!()` here so it can be overridden. + static FILE: &'static $crate::str::CStr = $crate::c_str!(file!()); + // Use `line!()` instead of `::core::line!()` here so it can be overridden. + static LINE: i32 = line!() as i32; static CONDITION: &'static $crate::str::CStr = $crate::c_str!(stringify!($condition)); // SAFETY: FFI call without safety requirements. @@ -164,10 +166,10 @@ macro_rules! kunit_assert { #[doc(hidden)] #[macro_export] macro_rules! kunit_assert_eq { - ($name:literal, $file:literal, $diff:expr, $left:expr, $right:expr $(,)?) => {{ + ($name:literal, $left:expr, $right:expr $(,)?) => {{ // For the moment, we just forward to the expression assert because, for binary asserts, // KUnit supports only a few types (e.g. integers). - $crate::kunit_assert!($name, $file, $diff, $left == $right); + $crate::kunit_assert!($name, $left == $right); }}; } diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 4d5c96ddc49c..1ab41129d557 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -67,6 +67,8 @@ pub mod device; pub mod device_id; pub mod devres; pub mod dma; +#[cfg(CONFIG_DMA_SHARED_BUFFER)] +pub mod dma_buf; pub mod driver; #[cfg(CONFIG_DRM = "y")] pub mod drm; @@ -98,6 +100,7 @@ pub mod jump_label; pub mod kunit; pub mod list; pub mod maple_tree; +pub mod mem; pub mod miscdevice; pub mod mm; pub mod module; @@ -119,6 +122,7 @@ pub mod processor; pub mod ptr; #[cfg(CONFIG_RUST_PWM_ABSTRACTIONS)] pub mod pwm; +pub mod ratelimit; pub mod rbtree; pub mod regulator; pub mod revocable; diff --git a/rust/kernel/maple_tree.rs b/rust/kernel/maple_tree.rs index 265d6396a78a..7abe41228cb7 100644 --- a/rust/kernel/maple_tree.rs +++ b/rust/kernel/maple_tree.rs @@ -16,7 +16,11 @@ use kernel::{ alloc::Flags, error::to_result, prelude::*, - types::{ForeignOwnable, Opaque}, + types::{ + ForeignOwnable, + NotThreadSafe, + Opaque, // + }, }; /// A maple tree optimized for storing non-overlapping ranges. @@ -240,7 +244,10 @@ impl<T: ForeignOwnable> MapleTree<T> { unsafe { bindings::spin_lock(self.ma_lock()) }; // INVARIANT: We just took the spinlock. - MapleGuard(self) + MapleGuard { + tree: self, + _not_send: NotThreadSafe, + } } #[inline] @@ -302,19 +309,30 @@ impl<T: ForeignOwnable> PinnedDrop for MapleTree<T> { } } +// SAFETY: `MapleTree<T>` is `Send` if `T` is `Send` because `MapleTree` owns its elements. +unsafe impl<T: ForeignOwnable + Send> Send for MapleTree<T> {} + +// SAFETY: `&MapleTree<T>` allows inserting and erasing entries from any thread, so `T: Send` is +// required, and shared borrows of entries require `T: Sync`. +unsafe impl<T: ForeignOwnable + Send + Sync> Sync for MapleTree<T> {} + /// A reference to a [`MapleTree`] that owns the inner lock. /// /// # Invariants /// /// This guard owns the inner spinlock. #[must_use = "if unused, the lock will be immediately unlocked"] -pub struct MapleGuard<'tree, T: ForeignOwnable>(&'tree MapleTree<T>); +pub struct MapleGuard<'tree, T: ForeignOwnable> { + tree: &'tree MapleTree<T>, + // A held spinlock must be released on the same CPU that acquired it. + _not_send: NotThreadSafe, +} impl<'tree, T: ForeignOwnable> Drop for MapleGuard<'tree, T> { #[inline] fn drop(&mut self) { // SAFETY: By the type invariants, we hold this spinlock. - unsafe { bindings::spin_unlock(self.0.ma_lock()) }; + unsafe { bindings::spin_unlock(self.tree.ma_lock()) }; } } @@ -323,7 +341,7 @@ impl<'tree, T: ForeignOwnable> MapleGuard<'tree, T> { pub fn ma_state(&mut self, first: usize, end: usize) -> MaState<'_, T> { // SAFETY: The `MaState` borrows this `MapleGuard`, so it can also borrow the `MapleGuard`s // read/write permissions to the maple tree. - unsafe { MaState::new_raw(self.0, first, end) } + unsafe { MaState::new_raw(self.tree, first, end) } } /// Load the value at the given index. @@ -375,7 +393,7 @@ impl<'tree, T: ForeignOwnable> MapleGuard<'tree, T> { #[inline] pub fn load(&mut self, index: usize) -> Option<T::BorrowedMut<'_>> { // SAFETY: `self.tree` contains a valid maple tree. - let ret = unsafe { bindings::mtree_load(self.0.tree.get(), index) }; + let ret = unsafe { bindings::mtree_load(self.tree.tree.get(), index) }; if ret.is_null() { return None; } diff --git a/rust/kernel/mem.rs b/rust/kernel/mem.rs new file mode 100644 index 000000000000..cd00289ded72 --- /dev/null +++ b/rust/kernel/mem.rs @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Basic utilities for dealing with memory, values, and types. + +use crate::prelude::*; + +// TODO: `Alignment` should be moved here as well, Rust moved it upstream in 1.96. +pub use crate::ptr::Alignment; + +/// Trait indicating that [`Align<N>`] is a valid alignment. +pub trait ValidAlign { + #[doc(hidden)] + type Repr: Copy; +} + +/// Zero-sized type that provides the alignment specified via the generic parameter. +/// +/// # Examples +/// +/// ``` +/// # use kernel::mem::Align; +/// const ALIGN: usize = 128; +/// struct MyStruct { +/// _align: Align<ALIGN>, +/// } +/// assert_eq!(align_of::<MyStruct>(), ALIGN); +/// ``` +#[repr(transparent)] +#[derive(Default, Clone, Copy)] +pub struct Align<const N: usize>([<Self as ValidAlign>::Repr; 0]) +where + Self: ValidAlign; + +impl<const N: usize> Align<N> +where + Self: ValidAlign, +{ + /// Create a new [`Align<N>`]. + #[inline] + pub const fn new() -> Self { + Align([]) + } +} + +macro_rules! impl_align { + () => {}; + ($a:literal $($rest:literal)*) => { + const _: () = { + #[repr(align($a))] + #[derive(Clone, Copy)] + pub struct Repr; + + impl ValidAlign for Align<$a> { + type Repr = Repr; + } + }; + + impl_align!($($rest)*); + } +} + +impl_align!(1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536); + +impl<const N: usize> core::fmt::Debug for Align<N> +where + Self: ValidAlign, +{ + #[inline] + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "Align<{N}>") + } +} + +/// Transmute between two types. +/// +/// Use this instead of [`core::mem::transmute`] when it is known that sizes are identical but this +/// cannot be proven by the compiler. +/// +/// This is equivalent to Rust's `transmute_unchecked` intrinsics. +/// +/// # Safety +/// +/// All safety requirements of [`core::mem::transmute`] apply, plus that the size `Src` and `Dst` +/// must match. +/// +/// # Examples +/// +/// This can be used when types are known to have the same size, but only at runtime. +/// +/// ```no_run +/// # use core::any::TypeId; +/// fn to_u32<T: 'static>(v: T) -> Option<u32> { +/// if TypeId::of::<T>() != TypeId::of::<u32>() { +/// return None; +/// } +/// +/// // `core::mem::transmute` won't work here. +/// // SAFETY: We've checked that `T` is `u32`! +/// Some(unsafe { kernel::mem::transmute_unchecked(v) }) +/// } +/// +/// to_u32(1u32); +/// ``` +#[inline(always)] +pub const unsafe fn transmute_unchecked<Src, Dst>(val: Src) -> Dst { + // SAFETY: This is identical to `transmute` except that we bypassed the size check; which is + // true per safety requirement. + unsafe { core::mem::transmute_copy(&core::mem::ManuallyDrop::new(val)) } +} + +/// Version of `transmute` that performs size check at monomorphization-time. +/// +/// Use this instead of [`core::mem::transmute`] when it is known that sizes are identical but this +/// cannot be proven by the compiler during type checking and can be proven during monomorphization. +/// +/// The signature is equivalent to Rust standard library's unstable `transmute_neo` and that of +/// [RFC 3844](https://github.com/rust-lang/rfcs/pull/3844). +/// +/// # Safety +/// +/// Same as [`core::mem::transmute`]. +/// +/// # Examples +/// +/// This is typically used in generic code where it's known that type will have the same size, but +/// the compiler cannot prove it generically. +/// +/// ```no_run +/// trait IsU32 {} +/// impl IsU32 for u32 {} +/// +/// fn to_u32<T: IsU32>(v: T) -> u32 { +/// // `core::mem::transmute` won't work here. +/// // SAFETY: We know that `v` is u32! +/// unsafe { kernel::mem::transmute(v) } +/// } +/// +/// to_u32(1u32); +/// ``` +#[inline(always)] +pub const unsafe fn transmute<Src, Dst>(val: Src) -> Dst { + const_assert!(size_of::<Src>() == size_of::<Dst>()); + + // SAFETY: Size is checked above. Other safety requirements follow those of the function. + unsafe { transmute_unchecked(val) } +} + +/// Safely transmutes a value of one type to a value of another type of the same size. +/// +/// The sizes are checked during monomorphization. +/// +/// This can be considered as generic version of [`zerocopy::transmute!`] macro that defers the size +/// check and thus can be used in more cases. +/// +/// # Examples +/// +/// ```no_run +/// fn to_u32<T: FromBytes + IntoBytes>(v: T) -> u32 { +/// // `zerocopy::transmute!` won't work here. +/// kernel::mem::safe_transmute(v) +/// } +/// +/// to_u32(1i32); +/// ``` +#[inline(always)] +pub const fn safe_transmute<Src: IntoBytes, Dst: FromBytes>(val: Src) -> Dst { + // SAFETY: `transmute` is safe with `IntoBytes` and `FromBytes` bounds. + unsafe { transmute(val) } +} + +/// Type that is layout-compatible with a primitive representation. +/// +/// # Safety +/// +/// - [`Self`] must have the same size and alignment as [`Self::Repr`]. +/// - [`Self`] must be [transmutable] to [`Self::Repr`]. +/// - Neither [`Self`] nor [`Self::Repr`] contains interior mutability. +/// +/// The above basically says that `&Self` can be transmuted to `&Self::Repr`. +/// +/// [transmutable]: core::mem::transmute +pub unsafe trait AsRepr: Sized { + /// Primitive representation of this type. + type Repr; + + /// Convert from [`&Self`](Self) to [`&Self::Repr`](AsRepr::Repr). + #[inline(always)] + fn as_repr(this: &Self) -> &Self::Repr { + // SAFETY: Per safety requirement of the trait. + unsafe { core::mem::transmute(this) } + } + + /// Convert from [`Self`] to [`Self::Repr`]. + #[inline(always)] + fn into_repr(this: Self) -> Self::Repr { + // SAFETY: Per safety requirement of the trait. + unsafe { transmute(this) } + } + + /// Convert from [`Self::Repr`] to [`Self`]. + /// + /// # Safety + /// + /// `repr` must be a valid bit pattern of [`Self`] and satisfy type-specific invariants of it. + /// + /// Alternatively, if `repr` is previously obtained using [`Self::into_repr`], and each + /// `from_repr_unchecked` should correspond to a unique `into_repr` call, then it is safe to + /// call as well (this means that we're undoing a `into_repr` call getting the exact bytes + /// back). + /// + /// No guarantee is made if the result of a `into_repr` is passed to multiple + /// `from_repr_unchecked` (i.e. copies are made), to allow for cases where `Repr` is a pointer + /// and the user of the API wants ownership transfer. Users that want the ability to call + /// `from_repr_unchecked` after copying can require `Copy` bound explicitly. + #[inline(always)] + unsafe fn from_repr_unchecked(repr: Self::Repr) -> Self { + // SAFETY: Per safety requirement, `repr` is valid repr of `Self`, or it is previously from + // `into_repr`, in which case we're undoing the transmute so it is also safe. + unsafe { transmute(repr) } + } +} + +/// Type that is bi-directionally transmutable with a primitive representation. +/// +/// # Safety +/// +/// - [`Self`] must be [transmutable] from [`Self::Repr`]. +/// +/// [transmutable]: core::mem::transmute +/// [`Self::Repr`]: AsRepr::Repr +pub unsafe trait AsReprMut: AsRepr { + /// Convert from `&mut Self` to [`&mut Self::Repr`](AsRepr::Repr). + #[inline(always)] + fn as_repr_mut(this: &mut Self) -> &mut Self::Repr { + // SAFETY: Per safety requirement of the trait. + unsafe { core::mem::transmute(this) } + } + + /// Convert from [`Self::Repr`](AsRepr::Repr) to `Self`. + #[inline(always)] + fn from_repr(repr: Self::Repr) -> Self { + // SAFETY: Per safety requirement of the trait. + unsafe { transmute(repr) } + } +} + +// SAFETY: `bool` has the same size and alignment as `u8`, and Rust guarantees that `bool` has +// only two valid bit patterns: 0 (`false`) and 1 (`true`). Thus `bool` can be transmuted to `u8`. +// Neither types contain interior mutability. +unsafe impl AsRepr for bool { + type Repr = u8; +} + +// SAFETY: `*mut T` has the same size and alignment with `*const c_void`, and thus `*mut T` is +// transmutable to `*const c_void`. Neither types contain interior mutability. +unsafe impl<T> AsRepr for *mut T { + type Repr = *const c_void; +} + +// SAFETY: `*mut T` is transmutable from `*const c_void`. +unsafe impl<T> AsReprMut for *mut T {} + +// SAFETY: `*const T` has the same size and alignment with `*const c_void`, and is transmutable to +// `*const c_void`. Neither types contain interior mutability. +unsafe impl<T> AsRepr for *const T { + type Repr = *const c_void; +} + +// SAFETY: `*const T` is transmutable from `*const c_void`. +unsafe impl<T> AsReprMut for *const T {} + +macro_rules! int_impl { + ($($unsigned:ident $signed:ident ,)*) => {$( + // SAFETY: `$unsigned` has the same size and alignment with itself, and is transmutable to + // itself. It does not contain interior mutability. + unsafe impl AsRepr for $unsigned { + type Repr = $unsigned; + } + + // SAFETY: `$unsigned` is transmutable from itself. + unsafe impl AsReprMut for $unsigned {} + + // SAFETY: `$signed` has the same size and alignment with `$unsigned`, and is transmutable + // to it Neither types contain interior mutability. + unsafe impl AsRepr for $signed { + type Repr = $unsigned; + } + + // SAFETY: `$signed` is transmutable from `$unsigned`. + unsafe impl AsReprMut for $signed {} + )*}; +} + +int_impl! { + u8 i8, + u16 i16, + u32 i32, + u64 i64, + // `usize` is not normalized to particular integer for portability. + usize isize, +} diff --git a/rust/kernel/miscdevice.rs b/rust/kernel/miscdevice.rs index 8d4b719bd83f..900676ea6bbc 100644 --- a/rust/kernel/miscdevice.rs +++ b/rust/kernel/miscdevice.rs @@ -10,11 +10,9 @@ use crate::{ bindings, + container_of, device::Device, - error::{ - to_result, - VTABLE_DEFAULT_ERROR, // - }, + error::to_result, fs::{ File, Kiocb, // @@ -32,7 +30,10 @@ use crate::{ Opaque, // }, // }; -use core::marker::PhantomData; +use core::{ + marker::PhantomData, + ops::Deref, // +}; /// Options for creating a misc device. #[derive(Copy, Clone)] @@ -63,37 +64,56 @@ impl MiscDeviceOptions { /// - Deregistration occurs exactly once in [`Drop`] via `misc_deregister()`. /// - `inner` wraps a valid, pinned `miscdevice` created using /// [`MiscDeviceOptions::into_raw`]. -#[repr(transparent)] +#[repr(C)] #[pin_data(PinnedDrop)] pub struct MiscDeviceRegistration<T> { #[pin] inner: Opaque<bindings::miscdevice>, - _t: PhantomData<T>, + #[pin] + data: T, } // SAFETY: It is allowed to call `misc_deregister` on a different thread from where you called -// `misc_register`. -unsafe impl<T> Send for MiscDeviceRegistration<T> {} +// `misc_register`, and `T` is `Send`. +unsafe impl<T: Send> Send for MiscDeviceRegistration<T> {} // SAFETY: All `&self` methods on this type are written to ensure that it is safe to call them in -// parallel. -unsafe impl<T> Sync for MiscDeviceRegistration<T> {} +// parallel, and `T` is `Sync`. +unsafe impl<T: Sync> Sync for MiscDeviceRegistration<T> {} impl<T: MiscDevice> MiscDeviceRegistration<T> { /// Register a misc device. - pub fn register(opts: MiscDeviceOptions) -> impl PinInit<Self, Error> { + /// + /// # Safety + /// + /// The caller must not `mem::forget()` the returned [`MiscDeviceRegistration`] or otherwise + /// prevent its [`Drop`] implementation from running. + pub unsafe fn register<'a>( + opts: MiscDeviceOptions, + data: impl PinInit<T, Error> + 'a, + ) -> impl PinInit<Self, Error> + 'a + where + T: 'a, + { try_pin_init!(Self { + data <- data, inner <- Opaque::try_ffi_init(move |slot: *mut bindings::miscdevice| { // SAFETY: The initializer can write to the provided `slot`. unsafe { slot.write(opts.into_raw::<T>()) }; // SAFETY: We just wrote the misc device options to the slot. The miscdevice will // get unregistered before `slot` is deallocated because the memory is pinned and - // the destructor of this type deallocates the memory. + // the caller guarantees that the destructor of this type will run. + // + // The `data` field is already initialized and is only dropped after + // `misc_deregister` in `PinnedDrop::drop`. Since `T: Sync` and the caller + // guarantees that `Drop` will run before `T` becomes invalid, it is sound for the + // `open` callback to access `&MiscDeviceRegistration<T>` until `misc_deregister` + // returns. + // // INVARIANT: If this returns `Ok(())`, then the `slot` will contain a registered // misc device. to_result(unsafe { bindings::misc_register(slot) }) }), - _t: PhantomData, }) } @@ -113,6 +133,14 @@ impl<T: MiscDevice> MiscDeviceRegistration<T> { } } +impl<T> Deref for MiscDeviceRegistration<T> { + type Target = T; + #[inline] + fn deref(&self) -> &T { + &self.data + } +} + #[pinned_drop] impl<T> PinnedDrop for MiscDeviceRegistration<T> { fn drop(self: Pin<&mut Self>) { @@ -121,19 +149,23 @@ impl<T> PinnedDrop for MiscDeviceRegistration<T> { } } -/// Trait implemented by the private data of an open misc device. +/// Trait implemented by the data stored in a [`MiscDeviceRegistration`]. #[vtable] -pub trait MiscDevice: Sized { - /// What kind of pointer should `Self` be wrapped in. - type Ptr: ForeignOwnable + Send + Sync; +pub trait MiscDevice: Sized + Sync { + /// The type of the data stored in each file for this miscdevice. + /// + /// This type must be pointer-sized, so it will usually be a [`KBox`] or [`Arc`]. + /// + /// [`Arc`]: crate::sync::Arc + type File: ForeignOwnable + Send + Sync + 'static; /// Called when the misc device is opened. /// /// The returned pointer will be stored as the private data for the file. - fn open(_file: &File, _misc: &MiscDeviceRegistration<Self>) -> Result<Self::Ptr>; + fn open(_file: &File, _misc: &MiscDeviceRegistration<Self>) -> Result<Self::File>; /// Called when the misc device is released. - fn release(device: Self::Ptr, _file: &File) { + fn release(device: Self::File, _file: &File) { drop(device); } @@ -145,37 +177,33 @@ pub trait MiscDevice: Sized { /// the VMA initialization by calling methods of `vma`. If the function does not return an /// error, the kernel will complete initialization of the VMA according to the properties of /// `vma`. + #[optional] fn mmap( - _device: <Self::Ptr as ForeignOwnable>::Borrowed<'_>, + _device: <Self::File as ForeignOwnable>::Borrowed<'_>, _file: &File, _vma: &VmaNew, - ) -> Result { - build_error!(VTABLE_DEFAULT_ERROR) - } + ) -> Result; /// Read from this miscdevice. - fn read_iter(_kiocb: Kiocb<'_, Self::Ptr>, _iov: &mut IovIterDest<'_>) -> Result<usize> { - build_error!(VTABLE_DEFAULT_ERROR) - } + #[optional] + fn read_iter(_kiocb: Kiocb<'_, Self::File>, _iov: &mut IovIterDest<'_>) -> Result<usize>; /// Write to this miscdevice. - fn write_iter(_kiocb: Kiocb<'_, Self::Ptr>, _iov: &mut IovIterSource<'_>) -> Result<usize> { - build_error!(VTABLE_DEFAULT_ERROR) - } + #[optional] + fn write_iter(_kiocb: Kiocb<'_, Self::File>, _iov: &mut IovIterSource<'_>) -> Result<usize>; /// Handler for ioctls. /// /// The `cmd` argument is usually manipulated using the utilities in [`kernel::ioctl`]. /// /// [`kernel::ioctl`]: mod@crate::ioctl + #[optional] fn ioctl( - _device: <Self::Ptr as ForeignOwnable>::Borrowed<'_>, + _device: <Self::File as ForeignOwnable>::Borrowed<'_>, _file: &File, _cmd: u32, _arg: usize, - ) -> Result<isize> { - build_error!(VTABLE_DEFAULT_ERROR) - } + ) -> Result<isize>; /// Handler for ioctls. /// @@ -185,23 +213,21 @@ pub trait MiscDevice: Sized { /// that have different layout on 32-bit and 64-bit userspace. If no implementation is /// provided, then `compat_ptr_ioctl` will be used instead. #[cfg(CONFIG_COMPAT)] + #[optional] fn compat_ioctl( - _device: <Self::Ptr as ForeignOwnable>::Borrowed<'_>, + _device: <Self::File as ForeignOwnable>::Borrowed<'_>, _file: &File, _cmd: u32, _arg: usize, - ) -> Result<isize> { - build_error!(VTABLE_DEFAULT_ERROR) - } + ) -> Result<isize>; /// Show info for this fd. + #[optional] fn show_fdinfo( - _device: <Self::Ptr as ForeignOwnable>::Borrowed<'_>, + _device: <Self::File as ForeignOwnable>::Borrowed<'_>, _m: &SeqFile, _file: &File, - ) { - build_error!(VTABLE_DEFAULT_ERROR) - } + ); } /// A vtable for the file operations of a Rust miscdevice. @@ -220,13 +246,14 @@ impl<T: MiscDevice> MiscdeviceVTable<T> { } // SAFETY: The open call of a file can access the private data. - let misc_ptr = unsafe { (*raw_file).private_data }; + let misc_ptr = Opaque::cast_from(unsafe { (*raw_file).private_data }.cast()); // SAFETY: This is a miscdevice, so `misc_open()` set the private data to a pointer to the - // associated `struct miscdevice` before calling into this method. Furthermore, - // `misc_open()` ensures that the miscdevice can't be unregistered and freed during this - // call to `fops_open`. - let misc = unsafe { &*misc_ptr.cast::<MiscDeviceRegistration<T>>() }; + // associated `struct miscdevice` (which is the `inner` field of a valid + // `MiscDeviceRegistration<T>`) before calling into this method. Furthermore, `misc_open()` + // ensures that the miscdevice can't be unregistered and freed during this call to + // `fops_open`, and `T: Sync` allows sharing `&MiscDeviceRegistration<T>` across threads. + let misc = unsafe { &*container_of!(misc_ptr, MiscDeviceRegistration<T>, inner) }; // SAFETY: // * This underlying file is valid for (much longer than) the duration of `T::open`. @@ -257,7 +284,7 @@ impl<T: MiscDevice> MiscdeviceVTable<T> { // SAFETY: The release call of a file owns the private data. let private = unsafe { (*file).private_data }; // SAFETY: The release call of a file owns the private data. - let ptr = unsafe { <T::Ptr as ForeignOwnable>::from_foreign(private) }; + let ptr = unsafe { <T::File as ForeignOwnable>::from_foreign(private) }; // SAFETY: // * The file is valid for the duration of this call. @@ -320,7 +347,7 @@ impl<T: MiscDevice> MiscdeviceVTable<T> { // SAFETY: This is a Rust Miscdevice, so we call `into_foreign` in `open` and // `from_foreign` in `release`, and `fops_mmap` is guaranteed to be called between those // two operations. - let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private.cast()) }; + let device = unsafe { <T::File as ForeignOwnable>::borrow(private) }; // SAFETY: The caller provides a vma that is undergoing initial VMA setup. let area = unsafe { VmaNew::from_raw(vma) }; // SAFETY: @@ -341,7 +368,7 @@ impl<T: MiscDevice> MiscdeviceVTable<T> { // SAFETY: The ioctl call of a file can access the private data. let private = unsafe { (*file).private_data }; // SAFETY: Ioctl calls can borrow the private data of the file. - let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) }; + let device = unsafe { <T::File as ForeignOwnable>::borrow(private) }; // SAFETY: // * The file is valid for the duration of this call. @@ -366,7 +393,7 @@ impl<T: MiscDevice> MiscdeviceVTable<T> { // SAFETY: The compat ioctl call of a file can access the private data. let private = unsafe { (*file).private_data }; // SAFETY: Ioctl calls can borrow the private data of the file. - let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) }; + let device = unsafe { <T::File as ForeignOwnable>::borrow(private) }; // SAFETY: // * The file is valid for the duration of this call. @@ -387,7 +414,7 @@ impl<T: MiscDevice> MiscdeviceVTable<T> { // SAFETY: The release call of a file owns the private data. let private = unsafe { (*file).private_data }; // SAFETY: Ioctl calls can borrow the private data of the file. - let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) }; + let device = unsafe { <T::File as ForeignOwnable>::borrow(private) }; // SAFETY: // * The file is valid for the duration of this call. // * There is no active fdget_pos region on the file on this thread. diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs index 4764d7b68f2a..58bc1793fdaf 100644 --- a/rust/kernel/mm.rs +++ b/rust/kernel/mm.rs @@ -170,30 +170,48 @@ impl MmWithUser { /// /// This is an optimistic trylock operation, so it may fail if there is contention. In that /// case, you should fall back to taking the mmap read lock. - /// - /// When per-vma locks are disabled, this always returns `None`. #[inline] pub fn lock_vma_under_rcu(&self, vma_addr: usize) -> Option<VmaReadGuard<'_>> { - #[cfg(CONFIG_PER_VMA_LOCK)] - { - // SAFETY: Calling `bindings::lock_vma_under_rcu` is always okay given an mm where - // `mm_users` is non-zero. - let vma = unsafe { bindings::lock_vma_under_rcu(self.as_raw(), vma_addr) }; - if !vma.is_null() { - return Some(VmaReadGuard { - // SAFETY: If `lock_vma_under_rcu` returns a non-null ptr, then it points at a - // valid vma. The vma is stable for as long as the vma read lock is held. - vma: unsafe { VmaRef::from_raw(vma) }, - _nts: NotThreadSafe, - }); - } + // SAFETY: Calling `bindings::lock_vma_under_rcu` is always okay given an mm where + // `mm_users` is non-zero. + let vma = unsafe { bindings::lock_vma_under_rcu(self.as_raw(), vma_addr) }; + if vma.is_null() { + return None; } + Some(VmaReadGuard { + // SAFETY: If `lock_vma_under_rcu` returns a non-null ptr, then it points at a + // valid vma. The vma is stable for as long as the vma read lock is held. + vma: unsafe { VmaRef::from_raw(vma) }, + _nts: NotThreadSafe, + }) + } - // Silence warnings about unused variables. - #[cfg(not(CONFIG_PER_VMA_LOCK))] - let _ = vma_addr; - - None + /// Find the VMA covering 'address' and read-lock it. + /// + /// The fast path does not take mmap_lock. Waits for writers to finish if the + /// VMA is being modified by taking mmap_lock. + /// Use when mmap_lock is not held, otherwise use vma_start_read_locked(). + /// Nothing prevents VMAs being unmapped/mapped before or after the VMA is + /// looked up, if a stronger guarantee is required, take an mmap_lock. + /// + /// Return: If a VMA exists which spans @address, return that VMA, read-locked. + /// If no VMA is mapped there or, very unlikely, a reference count overflow + /// occurred, return NULL. + #[inline] + pub fn vma_start_read_unlocked(&self, vma_addr: usize) -> Option<VmaReadGuard<'_>> { + // SAFETY: We may invoke `vma_start_read_unlocked` because we know this `mm` has non-zero + // `mm_users`. + let vma = unsafe { bindings::vma_start_read_unlocked(self.as_raw(), vma_addr) }; + if vma.is_null() { + return None; + } + // INVARIANT: We just acquired the VMA read lock. + Some(VmaReadGuard { + // SAFETY: If `vma_start_read_unlocked` returns a non-null ptr, then it points at a + // valid vma. The vma is stable for as long as the vma read lock is held. + vma: unsafe { VmaRef::from_raw(vma) }, + _nts: NotThreadSafe, + }) } /// Lock the mmap read lock. diff --git a/rust/kernel/module_param.rs b/rust/kernel/module_param.rs index f9a14765a926..1a2fcbf4a5b3 100644 --- a/rust/kernel/module_param.rs +++ b/rust/kernel/module_param.rs @@ -74,8 +74,13 @@ where crate::error::from_result(|| { let new_value = T::try_from_param_arg(arg)?; - // SAFETY: By function safety requirements, this access is safe. - let container = unsafe { &*((*param).__bindgen_anon_1.arg.cast::<SetOnce<T>>()) }; + // SAFETY: By function safety requirements, `param` is valid for read. + let arg = unsafe { (*param).__bindgen_anon_1.arg }; + let param_access_ptr = arg.cast::<ModuleParamAccess<T>>(); + // SAFETY: The `arg` field is initialized with a pointer to a `static ModuleAccessParam` by + // the rust module macro. Thus, the pointer is valid for use as a reference. + let param_access = unsafe { &*(param_access_ptr) }; + let container = ¶m_access.value; container .populate(new_value) diff --git a/rust/kernel/net/netlink.rs b/rust/kernel/net/netlink.rs index 22ef3dde36fa..36f2e39c3ab7 100644 --- a/rust/kernel/net/netlink.rs +++ b/rust/kernel/net/netlink.rs @@ -11,8 +11,8 @@ use kernel::{ alloc::{self, AllocError}, error::to_result, + num::casts::u16_as_usize, prelude::*, - transmute::AsBytes, types::Opaque, ThisModule, }; @@ -22,6 +22,11 @@ use core::{ ptr::NonNull, // }; +use zerocopy::{ + Immutable, + IntoBytes, // +}; + /// The default netlink message size. pub const GENLMSG_DEFAULT_SIZE: usize = bindings::GENLMSG_DEFAULT_SIZE; @@ -84,11 +89,19 @@ impl GenlMsg { #[inline] fn put<T>(&mut self, attrtype: c_int, value: &T) -> Result where - T: ?Sized + AsBytes, + T: ?Sized + IntoBytes + Immutable, { + // `nla_len` is a 16-bit field that encodes the total attribute length + // (header + payload). Subtracting the header size from `u16::MAX` gives + // the largest payload that still fits within that field. + const MAX_PAYLOAD_LEN: usize = u16_as_usize(u16::MAX) - size_of::<bindings::nlattr>(); + let skb = self.skb.skb.as_ptr(); let len = size_of_val(value); let ptr = core::ptr::from_ref(value).cast::<c_void>(); + if len > MAX_PAYLOAD_LEN { + return Err(EMSGSIZE); + } // SAFETY: `skb` is valid by `NetlinkSkBuff` type invariants, and the provided value is // readable and initialized for its `size_of` bytes. to_result(unsafe { bindings::nla_put(skb, attrtype, len as c_int, ptr) }) diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs index 956cda573ddb..c4e7b1d6c6f4 100644 --- a/rust/kernel/net/phy.rs +++ b/rust/kernel/net/phy.rs @@ -123,39 +123,37 @@ impl Device { /// Gets the current link state. /// /// It returns true if the link is up. + #[inline] pub fn is_link_up(&self) -> bool { - const LINK_IS_UP: u64 = 1; - // TODO: the code to access to the bit field will be replaced with automatically - // generated code by bindgen when it becomes possible. - // SAFETY: The struct invariant ensures that we may access - // this field without additional synchronization. - let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; - bit_field.get(14, 1) == LINK_IS_UP + let phydev = self.0.get().cast_const(); + // SAFETY: By the type invariant of `Device`, `phydev` points to a valid + // `struct phy_device`, and there is no concurrent write to this field. + let link = unsafe { bindings::phy_device::link_raw(phydev) }; + link == 1 } /// Gets the current auto-negotiation configuration. /// /// It returns true if auto-negotiation is enabled. + #[inline] pub fn is_autoneg_enabled(&self) -> bool { - // TODO: the code to access to the bit field will be replaced with automatically - // generated code by bindgen when it becomes possible. - // SAFETY: The struct invariant ensures that we may access - // this field without additional synchronization. - let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; - bit_field.get(13, 1) == u64::from(bindings::AUTONEG_ENABLE) + let phydev = self.0.get().cast_const(); + // SAFETY: By the type invariant of `Device`, `phydev` points to a valid + // `struct phy_device`, and there is no concurrent write to this field. + let autoneg = unsafe { bindings::phy_device::autoneg_raw(phydev) }; + autoneg == bindings::AUTONEG_ENABLE } /// Gets the current auto-negotiation state. /// /// It returns true if auto-negotiation is completed. + #[inline] pub fn is_autoneg_completed(&self) -> bool { - const AUTONEG_COMPLETED: u64 = 1; - // TODO: the code to access to the bit field will be replaced with automatically - // generated code by bindgen when it becomes possible. - // SAFETY: The struct invariant ensures that we may access - // this field without additional synchronization. - let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; - bit_field.get(15, 1) == AUTONEG_COMPLETED + let phydev = self.0.get().cast_const(); + // SAFETY: By the type invariant of `Device`, `phydev` points to a valid + // `struct phy_device`, and there is no concurrent write to this field. + let completed = unsafe { bindings::phy_device::autoneg_complete_raw(phydev) }; + completed == 1 } /// Sets the speed of the PHY. diff --git a/rust/kernel/num.rs b/rust/kernel/num.rs index dbe848e30efe..f81f0c1250d4 100644 --- a/rust/kernel/num.rs +++ b/rust/kernel/num.rs @@ -15,9 +15,14 @@ pub enum Unsigned {} /// Designates signed primitive types. pub enum Signed {} +mod private { + pub trait Sealed {} +} + /// Describes core properties of integer types. pub trait Integer: - Sized + private::Sealed + + Sized + Copy + Clone + PartialEq @@ -56,6 +61,8 @@ pub trait Integer: macro_rules! impl_integer { ($($type:ty: $signedness:ty), *) => { $( + impl private::Sealed for $type {} + impl Integer for $type { type Signedness = $signedness; @@ -79,3 +86,152 @@ impl_integer!( i128: Signed, isize: Signed ); + +/// Creates a constant value ("cv") from an integer constant expression, with validity checked at +/// compile time. +/// +/// This works for any type that implements [`FromConst`], with the target type inferred from +/// the context, or named explicitly with `cv!(value => Type)`. It can also be used in const +/// contexts. +/// +/// # Pitfalls +/// +/// Only the target type is inferred from the context. The expression passed to [`cv!`] is typed by +/// itself. So, unsuffixed integer literal expressions default to [`i32`]. For example +/// `cv!(2 << 31)` does not evaluate to the same value as `cv!(2i64 << 31)`, since `2 << 31` is +/// treated as an [`i32`] before being widened to [`i128`] internally. Literals and arithmetic +/// expressions that overflow [`i32`] are a compile error, it's just left shifts that require care. +/// +/// # Examples +/// +/// ``` +/// use core::num::NonZero; +/// +/// use kernel::{ +/// num::Bounded, +/// ptr::Alignment, // +/// }; +/// +/// let v: NonZero<usize> = cv!(8); +/// assert_eq!(v.get(), 8); +/// +/// // Any integer constant expression works, not only literals. +/// let m: NonZero<usize> = cv!(usize::MAX); +/// assert_eq!(m.get(), usize::MAX); +/// +/// let b: Bounded<u32, 4> = cv!(15); +/// assert_eq!(b.get(), 15); +/// +/// let a: Alignment = cv!(4096); +/// assert_eq!(a.as_usize(), 4096); +/// +/// // Checked narrowing of integer constants, including in `const` items. +/// const SMALL: u8 = cv!(200u32); +/// assert_eq!(SMALL, 200); +/// +/// const N: NonZero<u8> = cv!(5); +/// assert_eq!(N.get(), 5); +/// +/// // The target type can be given explicitly. +/// let e = cv!(200u32 => u8); +/// assert_eq!(e, 200); +/// +/// // With an explicit primitive target, the expression can use generic parameters. +/// const fn as_u64<const KEY: u16>() -> u64 { +/// cv!(KEY => u64) +/// } +/// assert_eq!(as_u64::<0x40>(), 0x40); +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! cv { + (@cast $v:expr => $t:ty) => { + const { + #[allow(unused_comparisons, unused_assignments, clippy::as_underscore)] + { + let v = $v; + let r = v as $t; + // Pin `back` to `v`'s type so `as _` casts back to the source type. + let mut back = v; + back = r as _; + + ::core::assert!( + back == v && (v < 0) == (r < 0), + "value does not fit into the target type" + ); + + r + } + } + }; + + // Using `FromConst<V>::VALUE` means const generic expressions can't be used, since it requires + // the `generic_const_exprs` feature. Provide a special path for each primitive type that allows + // const generic expressions. + ($v:expr => u8) => { $crate::cv!(@cast $v => u8) }; + ($v:expr => u16) => { $crate::cv!(@cast $v => u16) }; + ($v:expr => u32) => { $crate::cv!(@cast $v => u32) }; + ($v:expr => u64) => { $crate::cv!(@cast $v => u64) }; + ($v:expr => u128) => { $crate::cv!(@cast $v => u128) }; + ($v:expr => usize) => { $crate::cv!(@cast $v => usize) }; + ($v:expr => i8) => { $crate::cv!(@cast $v => i8) }; + ($v:expr => i16) => { $crate::cv!(@cast $v => i16) }; + ($v:expr => i32) => { $crate::cv!(@cast $v => i32) }; + ($v:expr => i64) => { $crate::cv!(@cast $v => i64) }; + ($v:expr => i128) => { $crate::cv!(@cast $v => i128) }; + ($v:expr => isize) => { $crate::cv!(@cast $v => isize) }; + ($v:expr => $t:ty) => { + <$t as $crate::num::FromConst<{ $crate::cv!(@cast $v => i128) }>>::VALUE + }; + ($v:expr) => { + <_ as $crate::num::FromConst<{ $crate::cv!(@cast $v => i128) }>>::VALUE + }; +} +#[doc(inline)] +pub use cv; + +/// Types that can be created from an integer constant expression validated at compile time. +/// +/// Implement this trait to make a type usable with [`cv!`]. Use the [`cv!`] macro, not this trait +/// directly, for creating values. +#[diagnostic::on_unimplemented(message = "`{Self}` cannot be converted from a constant")] +pub trait FromConst<const V: i128>: Sized { + /// The value that corresponds to the constant `V`. + /// + /// Fails the build if `V` is not a valid value for [`Self`]. + const VALUE: Self; +} + +/// Implements [`FromConst`] for primitive integer types and their [`NonZero`](core::num::NonZero) +/// versions. +macro_rules! impl_from_const { + ($($type:ty)*) => { + $( + impl<const V: i128> FromConst<V> for $type { + const VALUE: Self = { + // CAST: The macro is only used for types up to 64 bits wide, so `MIN` and `MAX` + // widen to `i128` losslessly. + assert!( + V >= <$type>::MIN as i128 && V <= <$type>::MAX as i128, + "constant cannot be represented by the target type" + ); + + // CAST: The assert above confirmed that `V` fits in `$type`. + V as $type + }; + } + + impl<const V: i128> FromConst<V> for core::num::NonZero<$type> { + const VALUE: Self = match core::num::NonZero::new(<$type as FromConst<V>>::VALUE) { + Some(value) => value, + None => panic!("constant cannot be zero"), + }; + } + )* + }; +} + +impl_from_const!( + u8 u16 u32 u64 usize + i8 i16 i32 i64 isize +); diff --git a/rust/kernel/num/bounded.rs b/rust/kernel/num/bounded.rs index 2a2b0a4bca5e..0b32f4343922 100644 --- a/rust/kernel/num/bounded.rs +++ b/rust/kernel/num/bounded.rs @@ -14,6 +14,7 @@ use core::{ use kernel::{ num::{ + FromConst, Integer, Unsigned, // }, @@ -272,6 +273,21 @@ macro_rules! impl_const_new { unsafe { Self::__new(VALUE) } } } + + impl<const N: u32, const V: i128> FromConst<V> for Bounded<$type, N> { + const VALUE: Self = { + let value = <$type as FromConst<V>>::VALUE; + // Statically assert that `value` fits within the set number of bits. + assert!( + fits_within!(value, $type, N), + "constant cannot be represented within the given number of bits" + ); + + // SAFETY: The assert above confirmed that `value` can be represented within `N` + // bits. + unsafe { Self::__new(value) } + }; + } )* }; } diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index 3ec897709e89..769d01d23429 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -17,6 +17,7 @@ use crate::{ from_result, to_result, // }, + io::resource, prelude::*, str::CStr, types::Opaque, @@ -32,10 +33,18 @@ use core::{ }, }; +mod cap; mod id; mod io; mod irq; +pub use self::cap::{ + ExtCapId, + ExtCapability, + ExtSriovCapability, + ExtSriovRegs, + ExtSriovVfBar, // +}; pub use self::id::{ Class, ClassMask, @@ -439,6 +448,19 @@ impl Device { Ok(unsafe { bindings::pci_resource_len(self.as_raw(), bar.try_into()?) }) } + /// Returns the resource flags (`IORESOURCE_*`) of the given PCI BAR. + pub fn resource_flags(&self, bar: u32) -> Result<resource::Flags> { + if !Bar::index_is_valid(bar) { + return Err(EINVAL); + } + + // SAFETY: + // - `bar` is a valid bar number, as guaranteed by the above call to `Bar::index_is_valid`, + // - by its type invariant `self.as_raw` is always a valid pointer to a `struct pci_dev`. + let raw = unsafe { bindings::pci_resource_flags(self.as_raw(), bar.try_into()?) }; + Ok(resource::Flags::from_raw(raw)) + } + /// Returns the PCI class as a `Class` struct. #[inline] pub fn pci_class(&self) -> Class { diff --git a/rust/kernel/pci/cap.rs b/rust/kernel/pci/cap.rs new file mode 100644 index 000000000000..62f263ada1b3 --- /dev/null +++ b/rust/kernel/pci/cap.rs @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! PCI extended capability support. + +use super::{ + io::ConfigSpaceBackend, + ConfigSpace, + Extended, // +}; +use crate::{ + bindings, + io::{ + Io, + IoBackend, + Region, // + }, + num::{casts, Bounded}, + prelude::*, +}; + +/// Number of VF BAR register slots in an SR-IOV capability. +const NUM_VF_BARS: usize = casts::u32_as_usize(bindings::PCI_SRIOV_NUM_BARS); + +/// PCI extended capability IDs. +#[repr(transparent)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ExtCapId(u16); + +impl ExtCapId { + /// Single Root I/O Virtualization. + pub const SRIOV: Self = Self(casts::u32_into_u16::<{ bindings::PCI_EXT_CAP_ID_SRIOV }>()); + + /// Returns the raw PCIe extended capability ID. + #[inline] + const fn as_raw(self) -> u16 { + self.0 + } +} + +/// A typed PCI extended capability register layout. +/// +/// Implementors describe the register layout of one extended capability. The layout must start at +/// the extended capability header, and [`Self::ID`] must identify that layout. +pub trait ExtCapability: FromBytes + IntoBytes { + /// PCI extended capability ID for this register layout. + const ID: ExtCapId; +} + +impl<'a> ConfigSpace<'a, Extended> { + /// Finds and projects an extended capability into its typed register layout. + /// + /// Returns [`None`] if the device does not implement the capability. + /// + /// Returns an error if the capability is present but its register span is too small or + /// insufficiently aligned for `C`. + /// + /// # Examples + /// + /// ```no_run + /// use kernel::{ + /// device::Bound, + /// pci, + /// prelude::*, + /// }; + /// + /// fn probe_sriov(pdev: &pci::Device<Bound>) -> Result { + /// let Some(sriov) = pdev + /// .config_space_extended()? + /// .find_ext_capability::<pci::ExtSriovRegs>()? + /// else { + /// return Ok(()); + /// }; + /// + /// let first_vf_offset = sriov.first_vf_offset(); + /// let mut vf_bars = sriov.vf_bars()?; + /// let bar0 = vf_bars.next().ok_or(EINVAL)?; + /// let bar1 = vf_bars.next().ok_or(EINVAL)?; + /// let bar2 = vf_bars.next().ok_or(EINVAL)?; + /// + /// Ok(()) + /// } + /// ``` + pub fn find_ext_capability<C: ExtCapability>(&self) -> Result<Option<ConfigSpace<'a, C>>> { + let offset = usize::from( + // SAFETY: `self.pdev` is valid by the type invariant of `ConfigSpace`. + unsafe { + bindings::pci_find_ext_capability(self.pdev.as_raw(), i32::from(C::ID.as_raw())) + }, + ); + + if offset == 0 { + return Ok(None); + } + + let size = self.calculate_ext_cap_size(offset)?; + + let base = ConfigSpaceBackend::as_ptr(*self) + .cast::<u8>() + .wrapping_add(offset); + let ptr = Region::<0>::ptr_try_from_raw_parts_mut(base, size)?; + + // SAFETY: `offset` was returned by `pci_find_ext_capability`, and + // `calculate_ext_cap_size` bounds `ptr` at the next capability or the end of the extended + // configuration space. `ptr_try_from_raw_parts_mut` verified the region layout. + let capability = unsafe { ConfigSpaceBackend::project_view(*self, ptr) }; + + capability.try_cast::<C>().map(Some) + } + + /// Calculates the size of the extended capability at `offset`. + /// + /// The capability extends to the next extended capability, or to the end of the extended + /// configuration space if it is the last one. `offset` must be a DWORD-aligned offset within + /// the extended configuration space returned by `pci_find_ext_capability`. Returns an error if + /// the capability header is outside the extended configuration space. + fn calculate_ext_cap_size(&self, offset: usize) -> Result<usize> { + let header = self.try_read32(offset)?; + // SAFETY: Pure bit manipulation, no preconditions. + let next = casts::u32_as_usize(unsafe { bindings::pci_ext_cap_next(header) }); + + Ok(if next > offset { + next - offset + } else { + self.size() - offset + }) + } +} + +/// SR-IOV register layout per PCIe spec (64 bytes starting at cap offset). +/// +/// The raw registers are private because PCI core owns SR-IOV state management. Drivers should use +/// PCI core APIs for operations such as enabling VFs and querying their topology. +#[repr(C)] +#[derive(FromBytes, IntoBytes)] +pub struct ExtSriovRegs { + /// Extended capability header. + _header: u32, + /// SR-IOV capabilities. + _cap: u32, + /// SR-IOV control. + _ctrl: u16, + /// SR-IOV status. + _status: u16, + /// Initial VFs. + _initial_vfs: u16, + /// Total VFs. + _total_vfs: u16, + /// Number of VFs. + _num_vfs: u16, + /// Function dependency link. + _func_dep_link: u8, + _reserved_0: u8, + /// First VF offset. + vf_offset: u16, + /// VF stride. + _vf_stride: u16, + _reserved_1: u16, + /// VF device ID. + _vf_device_id: u16, + /// Supported page sizes. + _supported_page_sizes: u32, + /// System page size. + _system_page_size: u32, + /// VF BARs (BAR0–BAR5). + vf_bar: [u32; NUM_VF_BARS], + /// VF migration state array offset. + _migration_state: u32, +} + +impl ExtCapability for ExtSriovRegs { + const ID: ExtCapId = ExtCapId::SRIOV; +} + +/// A typed view of an SR-IOV extended capability. +pub type ExtSriovCapability<'a> = ConfigSpace<'a, ExtSriovRegs>; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BarMemoryType { + Bits32, + Bits64, +} + +impl TryFrom<Bounded<u32, 2>> for BarMemoryType { + type Error = Error; + + fn try_from(value: Bounded<u32, 2>) -> Result<Self> { + match value.get() { + 0b00 => Ok(Self::Bits32), + 0b10 => Ok(Self::Bits64), + _ => Err(EINVAL), + } + } +} + +impl From<BarMemoryType> for Bounded<u32, 2> { + fn from(value: BarMemoryType) -> Self { + match value { + BarMemoryType::Bits32 => Self::new::<0b00>(), + BarMemoryType::Bits64 => Self::new::<0b10>(), + } + } +} + +// TODO: Move the common BAR encoding definitions into the PCI core abstraction once it provides +// BAR register decoding. +crate::bitfield! { + /// Low DWORD of an SR-IOV VF BAR. + struct BarLow(u32) { + /// Base address bits 31:4. + 31:4 address; + /// Whether the address range is prefetchable. + 3:3 prefetchable => bool; + /// Memory BAR type. + 2:1 memory_type ?=> BarMemoryType; + /// Whether this is an I/O-space BAR. + 0:0 io_space => bool; + } +} + +/// A decoded VF BAR register encoding. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ExtSriovVfBar { + /// The PCI bus address encoded by the BAR, without PCI attribute bits. + /// + /// This is not necessarily the CPU resource address. + pub address: u64, + + /// Whether the BAR is 64-bit. + pub is_64bit: bool, +} + +impl ExtSriovCapability<'_> { + /// Returns the current PCIe First VF Offset in Routing ID space. + /// + /// This value may change when NumVFs changes. Drivers should use PCI core helpers when they + /// need to calculate a VF BDF; this accessor is intended for interfaces that require the First + /// VF Offset itself. + #[inline] + pub fn first_vf_offset(&self) -> u16 { + crate::io_read!(*self, .vf_offset) + } + + /// Returns an iterator over decoded VF BAR register encodings. + /// + /// All six raw VF BAR register slots are read and decoded up front. A 32-bit encoding yields + /// one entry; a 64-bit encoding combines two slots into one entry. + /// + /// A zero-valued low DWORD is yielded as a 32-bit BAR at address zero; this method does not + /// probe whether a BAR is implemented. + /// + /// Returns [`EINVAL`] and logs an error if a BAR low DWORD does not encode a 32-bit or 64-bit + /// memory BAR, or if a 64-bit encoding has no upper DWORD. + pub fn vf_bars(&self) -> Result<impl Iterator<Item = ExtSriovVfBar>> { + let slots: [u32; NUM_VF_BARS] = + core::array::from_fn(|slot| crate::io_read!(*self, .vf_bar[panic: slot])); + let mut slots = slots.into_iter(); + let mut bars = [None; NUM_VF_BARS]; + let mut count = 0; + + let mut decode = || { + while let Some(low) = slots.next().map(BarLow::from) { + // SR-IOV VF BARs describe memory-space apertures; an I/O-space encoding is not + // valid for these registers. + if low.io_space() { + return Err(EINVAL); + } + + let low_address = u64::from(low.address()) << BarLow::ADDRESS_SHIFT; + let bar = match low.memory_type()? { + BarMemoryType::Bits64 => ExtSriovVfBar { + address: (u64::from(slots.next().ok_or(EINVAL)?) << 32) | low_address, + is_64bit: true, + }, + BarMemoryType::Bits32 => ExtSriovVfBar { + address: low_address, + is_64bit: false, + }, + }; + + bars[count] = Some(bar); + count += 1; + } + + Ok(()) + }; + + decode().inspect_err(|_| { + dev_err!(self.pdev, "invalid VF BAR encoding in SR-IOV capability\n"); + })?; + Ok(bars.into_iter().flatten()) + } +} diff --git a/rust/kernel/pci/irq.rs b/rust/kernel/pci/irq.rs index 6741046ec1c0..2ce4b46380e8 100644 --- a/rust/kernel/pci/irq.rs +++ b/rust/kernel/pci/irq.rs @@ -13,27 +13,26 @@ use crate::{ }; use core::num::NonZero; -/// IRQ type flags for PCI interrupt allocation. -#[derive(Debug, Clone, Copy)] -pub enum IrqType { - /// INTx interrupts. - Intx, - /// Message Signaled Interrupts (MSI). - Msi, - /// Extended Message Signaled Interrupts (MSI-X). - MsiX, -} - -impl IrqType { - /// Convert to the corresponding kernel flags. - const fn as_raw(self) -> u32 { - match self { - IrqType::Intx => bindings::PCI_IRQ_INTX, - IrqType::Msi => bindings::PCI_IRQ_MSI, - IrqType::MsiX => bindings::PCI_IRQ_MSIX, - } +crate::impl_flags!( + /// Set of IRQ types that can be used for PCI interrupt allocation. + #[derive(Debug, Clone, Copy, Default)] + pub struct IrqTypes(u32); + + /// IRQ type flags for PCI interrupt allocation. + #[derive(Debug, Clone, Copy)] + pub enum IrqType { + /// INTx interrupts. + Intx = bindings::PCI_IRQ_INTX, + + /// Message Signaled Interrupts (MSI). + Msi = bindings::PCI_IRQ_MSI, + + /// Extended Message Signaled Interrupts (MSI-X). + MsiX = bindings::PCI_IRQ_MSIX, } +); +impl IrqType { /// Construct from raw value. #[inline] const fn from_raw(raw: u32) -> Self { @@ -45,33 +44,10 @@ impl IrqType { } } -/// Set of IRQ types that can be used for PCI interrupt allocation. -#[derive(Debug, Clone, Copy, Default)] -pub struct IrqTypes(u32); - impl IrqTypes { /// Create a set containing all IRQ types (MSI-X, MSI, and INTx). pub const fn all() -> Self { - Self(bindings::PCI_IRQ_ALL_TYPES) - } - - /// Build a set of IRQ types. - /// - /// # Examples - /// - /// ```ignore - /// // Create a set with only MSI and MSI-X (no INTx interrupts). - /// let msi_only = IrqTypes::default() - /// .with(IrqType::Msi) - /// .with(IrqType::MsiX); - /// ``` - pub const fn with(self, irq_type: IrqType) -> Self { - Self(self.0 | irq_type.as_raw()) - } - - /// Get the raw flags value. - const fn as_raw(self) -> u32 { - self.0 + Self(Self::all_bits()) } } @@ -151,8 +127,10 @@ impl<'a> IrqVectorRegistration<'a> { /// [`Self::len()`]. #[inline] pub fn index(&self, index: usize) -> Result<IrqVector<'_>> { + let index = u32::try_from(index)?; + // SAFETY: `self.dev.as_raw()` is a valid pointer to a `struct pci_dev`. - let irq = unsafe { bindings::pci_irq_vector(self.dev.as_raw(), index as u32) }; + let irq = unsafe { bindings::pci_irq_vector(self.dev.as_raw(), index) }; if irq < 0 { return Err(Error::from_errno(irq)); } @@ -203,9 +181,7 @@ impl Device<device::Bound> { /// let vectors = dev.alloc_irq_vectors(1, 32, pci::IrqTypes::all())?; /// /// // Allocate MSI or MSI-X only (no INTx interrupts). - /// let msi_only = pci::IrqTypes::default() - /// .with(pci::IrqType::Msi) - /// .with(pci::IrqType::MsiX); + /// let msi_only = pci::IrqType::Msi | pci::IrqType::MsiX; /// let vectors = dev.alloc_irq_vectors(4, 16, msi_only)?; /// # Ok(()) /// # } @@ -222,7 +198,7 @@ impl Device<device::Bound> { // - `pci_alloc_irq_vectors` internally validates all other parameters // and returns error codes. let ret = unsafe { - bindings::pci_alloc_irq_vectors(self.as_raw(), min_vecs, max_vecs, irq_types.as_raw()) + bindings::pci_alloc_irq_vectors(self.as_raw(), min_vecs, max_vecs, u32::from(irq_types)) }; to_result(ret)?; diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index ca396f1f78a6..edaf1e4c174b 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -106,14 +106,23 @@ pub use super::{ Result, // }, init::InPlaceInit, + num::cv, pr_alert, + pr_alert_ratelimited, pr_crit, + pr_crit_ratelimited, pr_debug, + pr_debug_ratelimited, pr_emerg, + pr_emerg_ratelimited, pr_err, + pr_err_ratelimited, pr_info, + pr_info_ratelimited, pr_notice, + pr_notice_ratelimited, pr_warn, + pr_warn_ratelimited, str::CStrExt as _, try_init, try_pin_init, diff --git a/rust/kernel/ptr.rs b/rust/kernel/ptr.rs index 82acb531b17b..2c12e4fd4572 100644 --- a/rust/kernel/ptr.rs +++ b/rust/kernel/ptr.rs @@ -11,7 +11,10 @@ use core::mem::{ }; use core::num::NonZero; -use crate::const_assert; +use crate::{ + const_assert, + num::FromConst, // +}; /// Type representing an alignment, which is always a power of two. /// @@ -166,6 +169,13 @@ impl Alignment { } } +impl<const V: i128> FromConst<V> for Alignment { + const VALUE: Self = match Alignment::new_checked(<usize as FromConst<V>>::VALUE) { + Some(alignment) => alignment, + None => panic!("constant is not a power of two"), + }; +} + /// Trait for items that can be aligned against an [`Alignment`]. pub trait Alignable: Sized { /// Aligns `self` down to `alignment`. diff --git a/rust/kernel/ratelimit.rs b/rust/kernel/ratelimit.rs new file mode 100644 index 000000000000..2c54099cf11e --- /dev/null +++ b/rust/kernel/ratelimit.rs @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rate limiting support. +//! +//! C header: [`include/linux/ratelimit.h`](srctree/include/linux/ratelimit.h) + +use crate::{ + bindings, + prelude::*, + types::Opaque, // +}; + +/// Defines a `static` containing a [`Ratelimit`]. +#[macro_export] +macro_rules! ratelimit_state_init { + ($name:ident, $interval:expr, $burst:expr $(,)?) => { + #[allow(non_upper_case_globals)] + static $name: $crate::ratelimit::Ratelimit = { + let name = $crate::c_str!(::core::stringify!($name)); + let interval = $interval; + let burst = $burst; + // SAFETY: This will be stored in static memory. + unsafe { $crate::ratelimit::Ratelimit::new_static(name, interval, burst) } + }; + }; +} +pub use ratelimit_state_init; + +/// Rate limiter state. +/// +/// # Invariants +/// +/// The `inner` field contains an initialized `struct ratelimit_state`. +#[pin_data(PinnedDrop)] +#[repr(transparent)] +pub struct Ratelimit { + #[pin] + inner: Opaque<bindings::ratelimit_state>, +} + +// SAFETY: `Ratelimit` is safe to be sent to any task. +unsafe impl Send for Ratelimit {} + +// SAFETY: `Ratelimit` is safe to be accessed concurrently as it is protected by an internal +// spinlock. +unsafe impl Sync for Ratelimit {} + +impl Ratelimit { + /// Constructs a [`Ratelimit`] with the specified configuration. + /// + /// If `interval` is zero, then no rate limit is applied. + #[inline] + pub fn new(interval: i32, burst: i32) -> impl PinInit<Self> { + // INVARIANT: This creates a `Ratelimit` containing an initialized `struct ratelimit_state` + pin_init!(Self { + inner <- Opaque::ffi_init(|slot: *mut bindings::ratelimit_state| { + // SAFETY: `slot` is a valid pointer to an uninitialized `struct ratelimit_state`. + // The memory is pinned so it remains valid until `ratelimit_state_exit` is called. + unsafe { bindings::ratelimit_state_init(slot, interval, burst) }; + }), + }) + } + + /// Constructs a [`Ratelimit`] with the default configuration. + #[inline] + pub fn new_default() -> impl PinInit<Self> { + Ratelimit::new(Ratelimit::DEFAULT_INTERVAL, Ratelimit::DEFAULT_BURST) + } + + /// Constructs a [`Ratelimit`] with the specified configuration. + /// + /// The name will be used for the lockdep name of the internal spinlock. See [`Self::new`] for + /// the meaning of `interval` and `burst`. + /// + /// # Safety + /// + /// The resulting value must be stored in static memory. + #[inline] + pub const unsafe fn new_static(name: &'static CStr, interval: i32, burst: i32) -> Self { + // INVARIANT: This creates a `Ratelimit` containing an initialized `struct ratelimit_state`. + Self { + inner: Opaque::new(bindings::ratelimit_state { + lock: kernel::sync::lock::spinlock::raw_spin_lock_unlocked(name), + interval, + burst, + ..pin_init::zeroed() + }), + } + } + + /// The default interval used for rate limiting. + pub const DEFAULT_INTERVAL: i32 = bindings::DEFAULT_RATELIMIT_INTERVAL as i32; + + /// The default burst size. + pub const DEFAULT_BURST: i32 = bindings::DEFAULT_RATELIMIT_BURST as i32; + + /// Check if an action should be rate-limited. + /// + /// Returns [`true`] if the action is allowed, and [`false`] if it should be suppressed. + #[inline] + pub fn ratelimit(&self, func: &CStr) -> bool { + // SAFETY: `self.inner.get()` is a valid pointer to a `struct ratelimit_state`. + // The lifetime of `func` ensures the pointer remains valid for the duration of the call. + // The C function `___ratelimit` handles its own internal locking, so it is safe to call + // concurrently. + unsafe { bindings::___ratelimit(self.inner.get(), func.as_char_ptr()) != 0 } + } +} + +#[pinned_drop] +impl PinnedDrop for Ratelimit { + #[inline] + fn drop(self: Pin<&mut Self>) { + // SAFETY: By the type invariants, this struct contains an initialized `struct + // ratelimit_state`. + unsafe { bindings::ratelimit_state_exit(self.inner.get()) }; + } +} + +/// Helper macro to implement ratelimited printing. +#[macro_export] +#[doc(hidden)] +macro_rules! print_ratelimited { + ($print_macro:ident, $($arg:tt)*) => {{ + $crate::ratelimit::ratelimit_state_init!( + _rs, + $crate::ratelimit::Ratelimit::DEFAULT_INTERVAL, + $crate::ratelimit::Ratelimit::DEFAULT_BURST, + ); + let loc = $crate::c_str!(::core::concat!(::core::file!(), ":", ::core::line!())); + if $crate::ratelimit::Ratelimit::ratelimit(&_rs, loc) { + $crate::$print_macro!($($arg)*); + } + }}; +} + +/// Prints an emergency-level message (level 0) if allowed by a rate limiter. +#[macro_export] +macro_rules! pr_emerg_ratelimited ( + ($($arg:tt)*) => ( + $crate::print_ratelimited!(pr_emerg, $($arg)*) + ) +); + +/// Prints an alert-level message (level 1) if allowed by a rate limiter. +#[macro_export] +macro_rules! pr_alert_ratelimited ( + ($($arg:tt)*) => ( + $crate::print_ratelimited!(pr_alert, $($arg)*) + ) +); + +/// Prints a critical-level message (level 2) if allowed by a rate limiter. +#[macro_export] +macro_rules! pr_crit_ratelimited ( + ($($arg:tt)*) => ( + $crate::print_ratelimited!(pr_crit, $($arg)*) + ) +); + +/// Prints an error-level message (level 3) if allowed by a rate limiter. +#[macro_export] +macro_rules! pr_err_ratelimited ( + ($($arg:tt)*) => ( + $crate::print_ratelimited!(pr_err, $($arg)*) + ) +); + +/// Prints a warning-level message (level 4) if allowed by a rate limiter. +#[macro_export] +macro_rules! pr_warn_ratelimited ( + ($($arg:tt)*) => ( + $crate::print_ratelimited!(pr_warn, $($arg)*) + ) +); + +/// Prints a notice-level message (level 5) if allowed by a rate limiter. +#[macro_export] +macro_rules! pr_notice_ratelimited ( + ($($arg:tt)*) => ( + $crate::print_ratelimited!(pr_notice, $($arg)*) + ) +); + +/// Prints an info-level message (level 6) if allowed by a rate limiter. +#[macro_export] +macro_rules! pr_info_ratelimited ( + ($($arg:tt)*) => ( + $crate::print_ratelimited!(pr_info, $($arg)*) + ) +); + +/// Prints a debug-level message (level 7) if allowed by a rate limiter. +#[macro_export] +macro_rules! pr_debug_ratelimited ( + ($($arg:tt)*) => ( + if cfg!(debug_assertions) { + $crate::print_ratelimited!(pr_debug, $($arg)*) + } + ) +); diff --git a/rust/kernel/scatterlist.rs b/rust/kernel/scatterlist.rs index b83c468b5c63..3a5250f85000 100644 --- a/rust/kernel/scatterlist.rs +++ b/rust/kernel/scatterlist.rs @@ -34,12 +34,12 @@ use crate::{ bindings, device::{Bound, Device}, devres::Devres, - dma, error, - io::ResourceSize, + dma, + error, page, prelude::*, sync::aref::ARef, - types::Opaque, + types::Opaque, // }; use core::{ops::Deref, ptr::NonNull}; @@ -90,10 +90,9 @@ impl SGEntry { /// Returns the length of this SG entry in bytes. #[inline] - pub fn dma_len(&self) -> ResourceSize { - #[allow(clippy::useless_conversion)] + pub fn dma_len(&self) -> u32 { // SAFETY: `self.as_raw()` is a valid pointer to a `struct scatterlist`. - unsafe { bindings::sg_dma_len(self.as_raw()) }.into() + unsafe { bindings::sg_dma_len(self.as_raw()) } } } @@ -350,15 +349,23 @@ where page_vec.push(page.as_ptr(), flags)?; } + // Cap segments at both the DMA mapping-path limit and the device's declared + // max segment size. + // // `dma_max_mapping_size` returns `size_t`, but `sg_alloc_table_from_pages_segment()` takes // an `unsigned int`. // // SAFETY: `dev.as_raw()` is a valid pointer to a `struct device`. - let max_segment = match unsafe { bindings::dma_max_mapping_size(dev.as_raw()) } { + let max_mapping_size = match unsafe { bindings::dma_max_mapping_size(dev.as_raw()) } { 0 => u32::MAX, - max_segment => u32::try_from(max_segment).unwrap_or(u32::MAX), + max_mapping_size => u32::try_from(max_mapping_size).unwrap_or(u32::MAX), }; + // SAFETY: `dev.as_raw()` is a valid pointer to a `struct device`. + let max_seg_size = unsafe { bindings::dma_get_max_seg_size(dev.as_raw()) }; + + let max_segment = max_mapping_size.min(max_seg_size); + Ok(try_pin_init!(&this in Self { // SAFETY: // - `page_vec` is a `KVec` of valid `struct page *` obtained from `pages`. diff --git a/rust/kernel/security.rs b/rust/kernel/security.rs index 9d271695265f..4dc3eba6ce84 100644 --- a/rust/kernel/security.rs +++ b/rust/kernel/security.rs @@ -62,8 +62,7 @@ impl SecurityCtx { /// Get the security context given its id. #[inline] pub fn from_secid(secid: u32) -> Result<Self> { - // SAFETY: `struct lsm_context` can be initialized to all zeros. - let mut ctx: bindings::lsm_context = unsafe { core::mem::zeroed() }; + let mut ctx: bindings::lsm_context = pin_init::zeroed(); // SAFETY: Just a C FFI call. The pointer is valid for writes. to_result(unsafe { bindings::security_secid_to_secctx(secid, &mut ctx) })?; diff --git a/rust/kernel/sizes.rs b/rust/kernel/sizes.rs index 521b2b38bfe7..188e00c2b8b4 100644 --- a/rust/kernel/sizes.rs +++ b/rust/kernel/sizes.rs @@ -13,6 +13,11 @@ //! these constants as [`u64`] (or [`u32`]) rather than [`usize`], because //! device address spaces are sized independently of the CPU pointer width. //! +//! The trait is also implemented for [`Alignment`], providing each size as a +//! compile-time validated alignment. +//! +//! [`Alignment`]: crate::ptr::Alignment +//! //! # Examples //! //! ``` @@ -35,6 +40,26 @@ macro_rules! define_sizes { ($($type:ty),* $(,)?) => { define_sizes!(@internal [$($type),*] + /// `0x0000_0001`. + SZ_1, + /// `0x0000_0002`. + SZ_2, + /// `0x0000_0004`. + SZ_4, + /// `0x0000_0008`. + SZ_8, + /// `0x0000_0010`. + SZ_16, + /// `0x0000_0020`. + SZ_32, + /// `0x0000_0040`. + SZ_64, + /// `0x0000_0080`. + SZ_128, + /// `0x0000_0100`. + SZ_256, + /// `0x0000_0200`. + SZ_512, /// `0x0000_0400`. SZ_1K, /// `0x0000_0800`. @@ -85,6 +110,7 @@ macro_rules! define_sizes { (@internal [$($type:ty),*] $($names_and_metas:tt)*) => { define_sizes!(@consts_and_trait $($names_and_metas)*); define_sizes!(@impls [$($type),*] $($names_and_metas)*); + define_sizes!(@impl_alignment $($names_and_metas)*); }; (@consts_and_trait $($(#[$meta:meta])* $name:ident,)*) => { @@ -99,13 +125,22 @@ macro_rules! define_sizes { /// choose the width that matches their hardware. All `SZ_*` values fit /// in a [`u32`], so all implementations are lossless. /// + /// Also implemented for [`Alignment`], providing each size as a + /// compile-time validated alignment. + /// + /// [`Alignment`]: crate::ptr::Alignment + /// /// # Examples /// /// ``` - /// use kernel::sizes::SizeConstants; + /// use kernel::{ + /// ptr::Alignment, + /// sizes::SizeConstants, // + /// }; /// /// let gpu_heap = 14 * u64::SZ_1M; /// let mmio_window = u32::SZ_16M; + /// let page_align = Alignment::SZ_4K; /// ``` pub trait SizeConstants { $( @@ -117,6 +152,16 @@ macro_rules! define_sizes { (@impls [] $($(#[$meta:meta])* $name:ident,)*) => {}; + (@impl_alignment $($(#[$meta:meta])* $name:ident,)*) => { + impl SizeConstants for crate::ptr::Alignment { + $( + $(#[$meta])* + // A non-power-of-two constant will fail the build here if used. + const $name: Self = crate::ptr::Alignment::new::<{ self::$name }>(); + )* + } + }; + (@impls [$first:ty $(, $rest:ty)*] $($(#[$meta:meta])* $name:ident,)*) => { impl SizeConstants for $first { $( diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index b3caa9a1c898..644b4279a116 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -411,7 +411,7 @@ impl AsRef<BStr> for CStr { /// const BAD: &CStr = c_str!("literal"); /// /// // `c_str!` is still needed for static non-literal C strings. -/// const GOOD: &CStr = c_str!(concat!(file!(), ":", line!(), ": My CStr!")); +/// const GOOD: &CStr = c_str!(concat!(file!(), ":", ::core::line!(), ": My CStr!")); /// ``` #[macro_export] macro_rules! c_str { diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 3e55de3b1636..ee372c048a8a 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -5,10 +5,6 @@ //! This module contains the kernel APIs related to synchronisation that have been ported or //! wrapped for usage by Rust code in the kernel. -use crate::prelude::*; -use crate::types::Opaque; -use pin_init; - mod arc; pub mod aref; pub mod atomic; @@ -16,6 +12,7 @@ pub mod barrier; pub mod completion; mod condvar; pub mod lock; +pub mod lockdep; mod locked_by; pub mod poll; pub mod rcu; @@ -36,136 +33,11 @@ pub use lock::spinlock::{ SpinLockIrq, SpinLockIrqGuard, // }; +pub use lockdep::{ + static_lock_class, + LockClassKey, // +}; pub use locked_by::LockedBy; pub use refcount::Refcount; pub use set_once::SetOnce; pub use srcu::Srcu; - -/// Represents a lockdep class. -/// -/// Wraps the kernel's `struct lock_class_key`. -#[repr(transparent)] -#[pin_data(PinnedDrop)] -pub struct LockClassKey { - #[pin] - inner: Opaque<bindings::lock_class_key>, -} - -// SAFETY: Unregistering a lock class key from a different thread than where it was registered is -// allowed. -unsafe impl Send for LockClassKey {} - -// SAFETY: `bindings::lock_class_key` is designed to be used concurrently from multiple threads and -// provides its own synchronization. -unsafe impl Sync for LockClassKey {} - -impl LockClassKey { - /// Initializes a statically allocated lock class key. - /// - /// This is usually used indirectly through the [`static_lock_class!`] macro. See its - /// documentation for more information. - /// - /// # Safety - /// - /// * Before using the returned value, it must be pinned in a static memory location. - /// * The destructor must never run on the returned `LockClassKey`. - pub const unsafe fn new_static() -> Self { - LockClassKey { - inner: Opaque::uninit(), - } - } - - /// Initializes a dynamically allocated lock class key. - /// - /// In the common case of using a statically allocated lock class key, the - /// [`static_lock_class!`] macro should be used instead. - /// - /// # Examples - /// - /// ``` - /// use kernel::alloc::KBox; - /// use kernel::types::ForeignOwnable; - /// use kernel::sync::{LockClassKey, SpinLock}; - /// use pin_init::stack_pin_init; - /// - /// let key = KBox::pin_init(LockClassKey::new_dynamic(), GFP_KERNEL)?; - /// let key_ptr = key.into_foreign(); - /// - /// { - /// stack_pin_init!(let num: SpinLock<u32> = SpinLock::new( - /// 0, - /// c"my_spinlock", - /// // SAFETY: `key_ptr` is returned by the above `into_foreign()`, whose - /// // `from_foreign()` has not yet been called. - /// unsafe { <Pin<KBox<LockClassKey>> as ForeignOwnable>::borrow(key_ptr) } - /// )); - /// } - /// - /// // SAFETY: We dropped `num`, the only use of the key, so the result of the previous - /// // `borrow` has also been dropped. Thus, it's safe to use from_foreign. - /// unsafe { drop(<Pin<KBox<LockClassKey>> as ForeignOwnable>::from_foreign(key_ptr)) }; - /// # Ok::<(), Error>(()) - /// ``` - pub fn new_dynamic() -> impl PinInit<Self> { - pin_init!(Self { - // SAFETY: lockdep_register_key expects an uninitialized block of memory - inner <- Opaque::ffi_init(|slot| unsafe { bindings::lockdep_register_key(slot) }) - }) - } - - /// Returns a raw pointer to the inner C struct. - /// - /// It is up to the caller to use the raw pointer correctly. - pub fn as_ptr(&self) -> *mut bindings::lock_class_key { - self.inner.get() - } -} - -#[pinned_drop] -impl PinnedDrop for LockClassKey { - fn drop(self: Pin<&mut Self>) { - // SAFETY: `self.as_ptr()` was registered with lockdep and `self` is pinned, so the address - // hasn't changed. Thus, it's safe to pass it to unregister. - unsafe { bindings::lockdep_unregister_key(self.as_ptr()) } - } -} - -/// Defines a new static lock class and returns a pointer to it. -/// -/// # Examples -/// -/// ``` -/// use kernel::sync::{static_lock_class, Arc, SpinLock}; -/// -/// fn new_locked_int() -> Result<Arc<SpinLock<u32>>> { -/// Arc::pin_init(SpinLock::new( -/// 42, -/// c"new_locked_int", -/// static_lock_class!(), -/// ), GFP_KERNEL) -/// } -/// ``` -#[macro_export] -macro_rules! static_lock_class { - () => {{ - static CLASS: $crate::sync::LockClassKey = - // SAFETY: The returned `LockClassKey` is stored in static memory and we pin it. Drop - // never runs on a static global. - unsafe { $crate::sync::LockClassKey::new_static() }; - $crate::prelude::Pin::static_ref(&CLASS) - }}; -} -pub use static_lock_class; - -/// Returns the given string, if one is provided, otherwise generates one based on the source code -/// location. -#[doc(hidden)] -#[macro_export] -macro_rules! optional_name { - () => { - $crate::c_str!(::core::concat!(::core::file!(), ":", ::core::line!())) - }; - ($name:literal) => { - $crate::c_str!($name) - }; -} diff --git a/rust/kernel/sync/atomic.rs b/rust/kernel/sync/atomic.rs index 9cd009d57e35..6d27898add42 100644 --- a/rust/kernel/sync/atomic.rs +++ b/rust/kernel/sync/atomic.rs @@ -140,7 +140,7 @@ pub unsafe trait AtomicAdd<Rhs = Self>: AtomicType { const fn into_repr<T: AtomicType>(v: T) -> T::Repr { // SAFETY: Per the safety requirement of `AtomicType`, `T` is round-trip transmutable to // `T::Repr`, therefore the transmute operation is sound. - unsafe { core::mem::transmute_copy(&v) } + unsafe { crate::mem::transmute(v) } } /// # Safety @@ -149,7 +149,7 @@ const fn into_repr<T: AtomicType>(v: T) -> T::Repr { #[inline(always)] const unsafe fn from_repr<T: AtomicType>(r: T::Repr) -> T { // SAFETY: Per the safety requirement of the function, the transmute operation is sound. - unsafe { core::mem::transmute_copy(&r) } + unsafe { crate::mem::transmute(r) } } impl<T: AtomicType> Atomic<T> { diff --git a/rust/kernel/sync/completion.rs b/rust/kernel/sync/completion.rs index 35ff049ff078..1771bfc0ade2 100644 --- a/rust/kernel/sync/completion.rs +++ b/rust/kernel/sync/completion.rs @@ -38,7 +38,7 @@ use crate::{bindings, prelude::*, types::Opaque}; /// done <- Completion::new(), /// }), GFP_KERNEL)?; /// -/// let _ = workqueue::system().enqueue(this.clone()); +/// let _ = workqueue::system_dfl().enqueue(this.clone()); /// /// Ok(this) /// } diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index aafc80125f59..110f91753b54 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -450,7 +450,7 @@ mod tests { fn spinlock_irq_condvar() -> Result { let testdata = Test::new()?; - let _ = workqueue::system().enqueue(testdata.clone()); + let _ = workqueue::system_dfl().enqueue(testdata.clone()); // Let the updater know when we're ready to wait let mut state = testdata.state.lock(); @@ -463,3 +463,31 @@ mod tests { Ok(()) } } + +/// Helper for creating a raw unlocked `bindings::raw_spinlock_t`. +/// +/// For use in statics containing raw spinlocks. +#[doc(alias("__SPIN_LOCK_UNLOCKED", "DEFINE_SPINLOCK"))] +pub(crate) const fn raw_spin_lock_unlocked(name: &'static CStr) -> bindings::raw_spinlock_t { + // Silence unused variable warnings. + #[cfg(not(CONFIG_DEBUG_LOCK_ALLOC))] + let _ = name; + + bindings::raw_spinlock_t { + raw_lock: bindings::__ARCH_SPIN_LOCK_UNLOCKED, + + #[cfg(CONFIG_DEBUG_SPINLOCK)] + magic: bindings::SPINLOCK_MAGIC, + #[cfg(CONFIG_DEBUG_SPINLOCK)] + owner_cpu: u32::MAX, + #[cfg(CONFIG_DEBUG_SPINLOCK)] + owner: usize::MAX as *mut c_void, + + #[cfg(CONFIG_DEBUG_LOCK_ALLOC)] + dep_map: kernel::sync::lockdep::raw_lockdep_map( + name, + kernel::sync::lockdep::LD_WAIT_SPIN, + kernel::sync::lockdep::LD_WAIT_INV, + ), + } +} diff --git a/rust/kernel/sync/lockdep.rs b/rust/kernel/sync/lockdep.rs new file mode 100644 index 000000000000..d9222be0fb29 --- /dev/null +++ b/rust/kernel/sync/lockdep.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Utilities related to lockdep. +//! +//! C headers: [`include/linux/lockdep.h`](srctree/include/linux/lockdep.h) + +use crate::{ + prelude::*, + types::Opaque, // +}; + +/// Represents a lockdep class. +/// +/// Wraps the kernel's `struct lock_class_key`. +#[repr(transparent)] +#[pin_data(PinnedDrop)] +pub struct LockClassKey { + #[pin] + inner: Opaque<bindings::lock_class_key>, +} + +// SAFETY: Unregistering a lock class key from a different thread than where it was registered is +// allowed. +unsafe impl Send for LockClassKey {} + +// SAFETY: `bindings::lock_class_key` is designed to be used concurrently from multiple threads and +// provides its own synchronization. +unsafe impl Sync for LockClassKey {} + +impl LockClassKey { + /// Initializes a statically allocated lock class key. + /// + /// This is usually used indirectly through the [`static_lock_class!`] macro. See its + /// documentation for more information. + /// + /// # Safety + /// + /// * Before using the returned value, it must be pinned in a static memory location. + /// * The destructor must never run on the returned `LockClassKey`. + pub const unsafe fn new_static() -> Self { + LockClassKey { + inner: Opaque::uninit(), + } + } + + /// Initializes a dynamically allocated lock class key. + /// + /// In the common case of using a statically allocated lock class key, the + /// [`static_lock_class!`] macro should be used instead. + /// + /// # Examples + /// + /// ``` + /// use kernel::alloc::KBox; + /// use kernel::types::ForeignOwnable; + /// use kernel::sync::{LockClassKey, SpinLock}; + /// use pin_init::stack_pin_init; + /// + /// let key = KBox::pin_init(LockClassKey::new_dynamic(), GFP_KERNEL)?; + /// let key_ptr = key.into_foreign(); + /// + /// { + /// stack_pin_init!(let num: SpinLock<u32> = SpinLock::new( + /// 0, + /// c"my_spinlock", + /// // SAFETY: `key_ptr` is returned by the above `into_foreign()`, whose + /// // `from_foreign()` has not yet been called. + /// unsafe { <Pin<KBox<LockClassKey>> as ForeignOwnable>::borrow(key_ptr) } + /// )); + /// } + /// + /// // SAFETY: We dropped `num`, the only use of the key, so the result of the previous + /// // `borrow` has also been dropped. Thus, it's safe to use from_foreign. + /// unsafe { drop(<Pin<KBox<LockClassKey>> as ForeignOwnable>::from_foreign(key_ptr)) }; + /// # Ok::<(), Error>(()) + /// ``` + pub fn new_dynamic() -> impl PinInit<Self> { + pin_init!(Self { + // SAFETY: lockdep_register_key expects an uninitialized block of memory + inner <- Opaque::ffi_init(|slot| unsafe { bindings::lockdep_register_key(slot) }) + }) + } + + /// Returns a raw pointer to the inner C struct. + /// + /// It is up to the caller to use the raw pointer correctly. + pub fn as_ptr(&self) -> *mut bindings::lock_class_key { + self.inner.get() + } +} + +#[pinned_drop] +impl PinnedDrop for LockClassKey { + fn drop(self: Pin<&mut Self>) { + // SAFETY: `self.as_ptr()` was registered with lockdep and `self` is pinned, so the address + // hasn't changed. Thus, it's safe to pass it to unregister. + unsafe { bindings::lockdep_unregister_key(self.as_ptr()) } + } +} + +/// Defines a new static lock class and returns a pointer to it. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::{static_lock_class, Arc, SpinLock}; +/// +/// fn new_locked_int() -> Result<Arc<SpinLock<u32>>> { +/// Arc::pin_init(SpinLock::new( +/// 42, +/// c"new_locked_int", +/// static_lock_class!(), +/// ), GFP_KERNEL) +/// } +/// ``` +#[macro_export] +macro_rules! static_lock_class { + () => {{ + static CLASS: $crate::sync::LockClassKey = + // SAFETY: The returned `LockClassKey` is stored in static memory and we pin it. Drop + // never runs on a static global. + unsafe { $crate::sync::LockClassKey::new_static() }; + $crate::prelude::Pin::static_ref(&CLASS) + }}; +} +pub use static_lock_class; + +/// Returns the given string, if one is provided, otherwise generates one based on the source code +/// location. +#[doc(hidden)] +#[macro_export] +macro_rules! optional_name { + () => { + $crate::c_str!(::core::concat!(::core::file!(), ":", ::core::line!())) + }; + ($name:literal) => { + $crate::c_str!($name) + }; +} + +/// Not checked, catch all. +pub const LD_WAIT_INV: u8 = bindings::lockdep_wait_type_LD_WAIT_INV as u8; +/// Spin loops, `raw_spinlock_t` etc +pub const LD_WAIT_SPIN: u8 = bindings::lockdep_wait_type_LD_WAIT_SPIN as u8; + +/// Helper for declaring a raw `struct lockdep_map` for locks in statics. +/// +/// It's up to the caller to use the returned `struct lockdep_map` correctly. +#[cfg(CONFIG_DEBUG_LOCK_ALLOC)] +pub(crate) const fn raw_lockdep_map( + name: &'static CStr, + wait_type_inner: u8, + wait_type_outer: u8, +) -> bindings::lockdep_map { + // SAFETY: All zeros is valid for this type. + let mut map: bindings::lockdep_map = unsafe { core::mem::zeroed() }; + map.name = kernel::str::as_char_ptr_in_const_context(name); + map.wait_type_inner = wait_type_inner; + map.wait_type_outer = wait_type_outer; + map +} diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs index 6c0a5e8090d0..b6a2aab12a50 100644 --- a/rust/kernel/time.rs +++ b/rust/kernel/time.rs @@ -39,6 +39,10 @@ pub const NSEC_PER_MSEC: i64 = bindings::NSEC_PER_MSEC as i64; /// The number of nanoseconds per second. pub const NSEC_PER_SEC: i64 = bindings::NSEC_PER_SEC as i64; +/// The C side `MAX_JIFFY_OFFSET`, i.e. `((LONG_MAX >> 1) - 1)`. It is the upper +/// bound the kernel uses for a jiffies span, not a wait-forever value. +const MAX_JIFFY_OFFSET: isize = (isize::MAX >> 1) - 1; + /// The time unit of Linux kernel. One jiffy equals (1/HZ) second. pub type Jiffies = crate::ffi::c_ulong; @@ -53,9 +57,9 @@ pub fn msecs_to_jiffies(msecs: Msecs) -> Jiffies { unsafe { bindings::__msecs_to_jiffies(msecs) } } -/// Trait for clock sources. +/// Trait for kernel clock identifiers. /// -/// Selection of the clock source depends on the use case. In some cases the usage of a +/// Selection of the clock depends on the use case. In some cases the usage of a /// particular clock is mandatory, e.g. in network protocols, filesystems. In other /// cases the user of the clock has to decide which clock is best suited for the /// purpose. In most scenarios clock [`Monotonic`] is the best choice as it @@ -66,13 +70,13 @@ pub fn msecs_to_jiffies(msecs: Msecs) -> Jiffies { /// Implementers must ensure that `ktime_get()` returns a value in the inclusive range /// `0..=KTIME_MAX` (i.e., greater than or equal to 0 and less than or equal to /// `KTIME_MAX`, where `KTIME_MAX` equals `i64::MAX`). -pub unsafe trait ClockSource { - /// The kernel clock ID associated with this clock source. +pub unsafe trait ClockId { + /// The kernel clock ID associated with this clock. /// /// This constant corresponds to the C side `clockid_t` value. const ID: bindings::clockid_t; - /// Get the current time from the clock source. + /// Get the current time from the clock. /// /// The function must return a value in the range `0..=KTIME_MAX`. fn ktime_get() -> bindings::ktime_t; @@ -93,7 +97,7 @@ pub struct Monotonic; // SAFETY: The kernel's `ktime_get()` is guaranteed to return a value // in `0..=KTIME_MAX`. -unsafe impl ClockSource for Monotonic { +unsafe impl ClockId for Monotonic { const ID: bindings::clockid_t = bindings::CLOCK_MONOTONIC as bindings::clockid_t; fn ktime_get() -> bindings::ktime_t { @@ -120,7 +124,7 @@ pub struct RealTime; // SAFETY: The kernel's `ktime_get_real()` is guaranteed to return a value // in `0..=KTIME_MAX`. -unsafe impl ClockSource for RealTime { +unsafe impl ClockId for RealTime { const ID: bindings::clockid_t = bindings::CLOCK_REALTIME as bindings::clockid_t; fn ktime_get() -> bindings::ktime_t { @@ -140,7 +144,7 @@ pub struct BootTime; // SAFETY: The kernel's `ktime_get_boottime()` is guaranteed to return a value // in `0..=KTIME_MAX`. -unsafe impl ClockSource for BootTime { +unsafe impl ClockId for BootTime { const ID: bindings::clockid_t = bindings::CLOCK_BOOTTIME as bindings::clockid_t; fn ktime_get() -> bindings::ktime_t { @@ -164,7 +168,7 @@ pub struct Tai; // SAFETY: The kernel's `ktime_get_clocktai()` is guaranteed to return a value // in `0..=KTIME_MAX`. -unsafe impl ClockSource for Tai { +unsafe impl ClockId for Tai { const ID: bindings::clockid_t = bindings::CLOCK_TAI as bindings::clockid_t; fn ktime_get() -> bindings::ktime_t { @@ -180,24 +184,24 @@ unsafe impl ClockSource for Tai { /// The `inner` value is in the range from 0 to `KTIME_MAX`. #[repr(transparent)] #[derive(PartialEq, PartialOrd, Eq, Ord)] -pub struct Instant<C: ClockSource> { +pub struct Instant<C: ClockId> { inner: bindings::ktime_t, _c: PhantomData<C>, } -impl<C: ClockSource> Clone for Instant<C> { +impl<C: ClockId> Clone for Instant<C> { fn clone(&self) -> Self { *self } } -impl<C: ClockSource> Copy for Instant<C> {} +impl<C: ClockId> Copy for Instant<C> {} -impl<C: ClockSource> Instant<C> { +impl<C: ClockId> Instant<C> { /// Get the current time from the clock source. #[inline] pub fn now() -> Self { - // INVARIANT: The `ClockSource::ktime_get()` function returns a value in the range + // INVARIANT: The `ClockId::ktime_get()` function returns a value in the range // from 0 to `KTIME_MAX`. Self { inner: C::ktime_get(), @@ -239,7 +243,7 @@ impl<C: ClockSource> Instant<C> { } } -impl<C: ClockSource> ops::Sub for Instant<C> { +impl<C: ClockId> ops::Sub for Instant<C> { type Output = Delta; // By the type invariant, it never overflows. @@ -251,7 +255,7 @@ impl<C: ClockSource> ops::Sub for Instant<C> { } } -impl<T: ClockSource> ops::Add<Delta> for Instant<T> { +impl<T: ClockId> ops::Add<Delta> for Instant<T> { type Output = Self; #[inline] @@ -271,7 +275,7 @@ impl<T: ClockSource> ops::Add<Delta> for Instant<T> { } } -impl<T: ClockSource> ops::Sub<Delta> for Instant<T> { +impl<T: ClockId> ops::Sub<Delta> for Instant<T> { type Output = Self; #[inline] @@ -302,6 +306,9 @@ mod private { pub trait TimeUnit: private::Sealed { /// The underlying representation of the time unit. type Repr: Copy + Clone + PartialEq + PartialOrd + Eq + Ord + core::fmt::Debug; + + /// The zero value of [`Self::Repr`]. + const ZERO: Self::Repr; } /// A time unit of nanoseconds. @@ -313,6 +320,7 @@ pub enum Nsec {} impl TimeUnit for Nsec { type Repr = i64; + const ZERO: Self::Repr = 0; } /// A time unit of jiffies. @@ -324,6 +332,7 @@ pub enum Jiffy {} impl TimeUnit for Jiffy { type Repr = isize; + const ZERO: Self::Repr = 0; } /// A span of time. @@ -337,6 +346,11 @@ pub struct Delta<U: TimeUnit = Nsec> { value: U::Repr, } +impl<U: TimeUnit> Delta<U> { + /// A span of time equal to zero. + pub const ZERO: Self = Self { value: U::ZERO }; +} + impl Delta<Jiffy> { /// Create a new [`Delta`] from a number of jiffies. #[inline] @@ -349,6 +363,27 @@ impl Delta<Jiffy> { pub const fn as_jiffies(self) -> isize { self.value } + + /// Return the number of jiffies in the [`Delta`] as an unsigned value. + /// + /// If the number of jiffies is negative, zero is returned. + /// + /// # Examples + /// + /// ``` + /// use kernel::time::Delta; + /// + /// // A negative span returns zero. + /// assert_eq!(Delta::from_jiffies(-1).as_jiffies_unsigned(), 0); + /// + /// // A positive span returns its value. + /// assert_eq!(Delta::from_jiffies(1).as_jiffies_unsigned(), 1); + /// ``` + #[inline] + pub fn as_jiffies_unsigned(self) -> usize { + // CAST: `isize::max()` makes the value non-negative, so the cast keeps it. + isize::max(self.value, 0) as usize + } } impl ops::Add for Delta { @@ -424,9 +459,6 @@ impl ops::Div for Delta { } impl Delta { - /// A span of time equal to zero. - pub const ZERO: Self = Self { value: 0 }; - /// Create a new [`Delta`] from a number of nanoseconds. #[inline] pub const fn from_nanos(nanos: i64) -> Self { @@ -554,6 +586,62 @@ impl Delta { } } + /// Convert this span to a [`Delta<Jiffy>`] suitable for use as a timeout. + /// + /// Unless the result saturates, the value is rounded up to the next whole + /// jiffy, so the resulting timeout is never shorter than `self`. + /// + /// A negative span saturates at zero jiffies, i.e. an immediate timeout. + /// + /// A span that does not fit saturates at the kernel's [`MAX_JIFFY_OFFSET`], + /// the upper bound for a jiffies span. That is a finite timeout, so a + /// saturated result can be shorter than the requested span. It is derived + /// from `long`, so only 32 bit can reach it, at about 12 days with `HZ=1000`. + /// + /// # Examples + /// + /// ``` + /// use kernel::time::Delta; + /// + /// // A negative span is an immediate timeout. + /// assert_eq!(Delta::from_millis(-1).to_jiffies_timeout().as_jiffies(), 0); + /// + /// // A span shorter than a jiffy still waits, i.e. the timeout is never + /// // shorter than the span. + /// assert!(Delta::from_nanos(1).to_jiffies_timeout().as_jiffies() >= 1); + /// ``` + /// + /// [`MAX_JIFFY_OFFSET`]: srctree/include/linux/jiffies.h + #[inline] + pub fn to_jiffies_timeout(self) -> Delta<Jiffy> { + const HZ: u64 = bindings::HZ as u64; + + // The quotient `(nsecs * HZ + NSEC_PER_SEC - 1) / NSEC_PER_SEC` has to fit in + // `u64`; `nsecs * HZ` does not. With `HZ <= NSEC_PER_SEC` the numerator is at + // most `(nsecs + 1) * NSEC_PER_SEC - 1`, so the quotient is at most `nsecs`. + crate::static_assert!(HZ <= NSEC_PER_SEC as u64); + + // CAST: `i64::max()` makes the value non-negative, so the cast keeps it. + let nsecs = i64::max(self.as_nanos(), 0) as u64; + + // SAFETY: `mul_u64_add_u64_div_u64()` must not be called with a zero divisor, + // and its result must fit in `u64`. `NSEC_PER_SEC` is a non-zero constant, and + // the assertion above bounds the quotient by `nsecs`. + let jiffies = unsafe { + bindings::mul_u64_add_u64_div_u64( + nsecs, + HZ, + (NSEC_PER_SEC - 1) as u64, + NSEC_PER_SEC as u64, + ) + }; + + // CAST: `jiffies` is clamped to `MAX_JIFFY_OFFSET`, which is `<= isize::MAX`. + let jiffies = u64::min(jiffies, MAX_JIFFY_OFFSET as u64) as isize; + + Delta::<Jiffy>::from_jiffies(jiffies) + } + /// Return `self % dividend` where `dividend` is in nanoseconds. /// /// The kernel doesn't have any emulation for `s64 % s64` on 32 bit platforms, so this is @@ -580,3 +668,48 @@ impl Delta { } } } + +#[cfg(CONFIG_RUST_TIME_KUNIT_TEST)] +#[macros::kunit_tests(rust_kernel_time)] +mod tests { + use super::*; + + #[test] + fn to_jiffies_timeout_converts() { + const HZ: isize = bindings::HZ as isize; + + // One second is exactly `HZ` jiffies, and the round-up must not add one. + assert_eq!(Delta::from_secs(1).to_jiffies_timeout().as_jiffies(), HZ); + + // One nanosecond more has to round up to the next whole jiffy. + assert_eq!( + Delta::from_nanos(NSEC_PER_SEC + 1) + .to_jiffies_timeout() + .as_jiffies(), + HZ + 1 + ); + } + + #[test] + fn to_jiffies_timeout_saturates() { + // The result never exceeds `MAX_JIFFY_OFFSET`. On 32 bit with `HZ=1000` this + // span is 2147483647 jiffies, so the clamp is what keeps it in range; on 64 + // bit it fits and the check holds for every possible return value. + let clamped = Delta::from_millis(i64::from(i32::MAX)).to_jiffies_timeout(); + assert!(clamped.as_jiffies() <= MAX_JIFFY_OFFSET); + + // `MAX_JIFFY_OFFSET` is derived from `long`, so only 32 bit can reach it. On + // 64 bit `i64::MAX` nanoseconds is about 292 years, which is 9223372036855 + // jiffies with `HZ=1000`, far below the limit. + #[cfg(not(CONFIG_64BIT))] + { + // An overlong span is clamped to `MAX_JIFFY_OFFSET`. + let overlong = Delta::from_nanos(i64::MAX).to_jiffies_timeout(); + assert_eq!(overlong.as_jiffies(), MAX_JIFFY_OFFSET); + } + + // A negative span is an immediate timeout, however long it is. + let negative = Delta::from_nanos(i64::MIN).to_jiffies_timeout(); + assert_eq!(negative.as_jiffies(), 0); + } +} diff --git a/rust/kernel/time/delay.rs b/rust/kernel/time/delay.rs index b5b1b42797a0..4debe23731bf 100644 --- a/rust/kernel/time/delay.rs +++ b/rust/kernel/time/delay.rs @@ -8,7 +8,10 @@ //! C header: [`include/linux/delay.h`](srctree/include/linux/delay.h). use super::Delta; -use crate::prelude::*; +use crate::{ + pr_warn_once, + prelude::*, // +}; /// Sleeps for a given duration at least. /// @@ -18,8 +21,7 @@ use crate::prelude::*; /// `delta` must be within `[0, i32::MAX]` microseconds; /// otherwise, it is erroneous behavior. That is, it is considered a bug /// to call this function with an out-of-range value, in which case the function -/// will sleep for at least the maximum value in the range and may warn -/// in the future. +/// will sleep for at least the maximum value in the range and warns once. /// /// The behavior above differs from the C side [`fsleep()`] for which out-of-range /// values mean "infinite timeout" instead. @@ -32,10 +34,13 @@ pub fn fsleep(delta: Delta) { // overflow inside fsleep, which could lead to unintentional infinite sleep. const MAX_DELTA: Delta = Delta::from_micros(i32::MAX as i64); + debug_assert!(delta.as_nanos() >= 0); + debug_assert!(delta <= MAX_DELTA); + let delta = if (Delta::ZERO..=MAX_DELTA).contains(&delta) { delta } else { - // TODO: Add WARN_ONCE() when it's supported. + pr_warn_once!("attempted to fsleep() with out of range delta\n"); MAX_DELTA }; @@ -54,7 +59,8 @@ pub fn fsleep(delta: Delta) { /// /// `delta` must be within `[0, MAX_UDELAY_MS]` in milliseconds; /// otherwise, it is erroneous behavior. That is, it is considered a bug to -/// call this function with an out-of-range value. +/// call this function with an out-of-range value, in which case the function +/// will delay for at least the maximum value in the range and warns once. /// /// The behavior above differs from the C side [`udelay()`] for which out-of-range /// values could lead to an overflow and unexpected behavior. @@ -69,6 +75,7 @@ pub fn udelay(delta: Delta) { let delta = if (Delta::ZERO..=MAX_UDELAY_DELTA).contains(&delta) { delta } else { + pr_warn_once!("attempted to udelay() with out of range delta\n"); MAX_UDELAY_DELTA }; diff --git a/rust/kernel/time/hrtimer.rs b/rust/kernel/time/hrtimer.rs index 2d7f1131a813..a53f7f1341be 100644 --- a/rust/kernel/time/hrtimer.rs +++ b/rust/kernel/time/hrtimer.rs @@ -402,15 +402,26 @@ //! ``` //! //! [`Arc`]: kernel::sync::Arc - -use super::{ClockSource, Delta, Instant}; +// Implementation details +// +// The reasoning for adopting a handle based approach: +// - If we explicitly drop the target of a timer callback in the timer callback, we +// may get a dangling reference. +// - If the callback owns the last reference to the target, target may be dropped +// in non-sleepable context when the callback is finished. +// - When dropping an object that is the target of an armed timer, we may drop +// fields accessed by the timer callback before we cancel the timer (drop order). +// +// By using a handle, we can make the handle own the callback target and avoid these problems. + +use super::{ClockId, Delta, Instant}; use crate::{prelude::*, types::Opaque}; use core::{marker::PhantomData, ptr::NonNull}; use pin_init::PinInit; /// A type-alias to refer to the [`Instant<C>`] for a given `T` from [`HrTimer<T>`]. /// -/// Where `C` is the [`ClockSource`] of the [`HrTimer`]. +/// Where `C` is the [`ClockId`] of the [`HrTimer`]. pub type HrTimerInstant<T> = Instant<<<T as HasHrTimer<T>>::TimerMode as HrTimerMode>::Clock>; /// A timer backed by a C `struct hrtimer`. @@ -834,14 +845,14 @@ pub unsafe trait HasHrTimer<T> { #[repr(u32)] pub enum HrTimerRestart { /// Timer should not be restarted. - NoRestart = bindings::hrtimer_restart_HRTIMER_NORESTART, + NoRestart = bindings::hrtimer_restart::HRTIMER_NORESTART.0, /// Timer should be restarted. - Restart = bindings::hrtimer_restart_HRTIMER_RESTART, + Restart = bindings::hrtimer_restart::HRTIMER_RESTART.0, } impl HrTimerRestart { fn into_c(self) -> bindings::hrtimer_restart { - self as bindings::hrtimer_restart + bindings::hrtimer_restart(self as c_uint) } } @@ -855,7 +866,7 @@ pub trait HrTimerExpires { fn as_nanos(&self) -> i64; } -impl<C: ClockSource> HrTimerExpires for Instant<C> { +impl<C: ClockId> HrTimerExpires for Instant<C> { #[inline] fn as_nanos(&self) -> i64 { Instant::<C>::as_nanos(self) @@ -870,22 +881,22 @@ impl HrTimerExpires for Delta { } mod private { - use crate::time::ClockSource; + use crate::time::ClockId; pub trait Sealed {} - impl<C: ClockSource> Sealed for super::AbsoluteMode<C> {} - impl<C: ClockSource> Sealed for super::RelativeMode<C> {} - impl<C: ClockSource> Sealed for super::AbsolutePinnedMode<C> {} - impl<C: ClockSource> Sealed for super::RelativePinnedMode<C> {} - impl<C: ClockSource> Sealed for super::AbsoluteSoftMode<C> {} - impl<C: ClockSource> Sealed for super::RelativeSoftMode<C> {} - impl<C: ClockSource> Sealed for super::AbsolutePinnedSoftMode<C> {} - impl<C: ClockSource> Sealed for super::RelativePinnedSoftMode<C> {} - impl<C: ClockSource> Sealed for super::AbsoluteHardMode<C> {} - impl<C: ClockSource> Sealed for super::RelativeHardMode<C> {} - impl<C: ClockSource> Sealed for super::AbsolutePinnedHardMode<C> {} - impl<C: ClockSource> Sealed for super::RelativePinnedHardMode<C> {} + impl<C: ClockId> Sealed for super::AbsoluteMode<C> {} + impl<C: ClockId> Sealed for super::RelativeMode<C> {} + impl<C: ClockId> Sealed for super::AbsolutePinnedMode<C> {} + impl<C: ClockId> Sealed for super::RelativePinnedMode<C> {} + impl<C: ClockId> Sealed for super::AbsoluteSoftMode<C> {} + impl<C: ClockId> Sealed for super::RelativeSoftMode<C> {} + impl<C: ClockId> Sealed for super::AbsolutePinnedSoftMode<C> {} + impl<C: ClockId> Sealed for super::RelativePinnedSoftMode<C> {} + impl<C: ClockId> Sealed for super::AbsoluteHardMode<C> {} + impl<C: ClockId> Sealed for super::RelativeHardMode<C> {} + impl<C: ClockId> Sealed for super::AbsolutePinnedHardMode<C> {} + impl<C: ClockId> Sealed for super::RelativePinnedHardMode<C> {} } /// Operational mode of [`HrTimer`]. @@ -894,16 +905,16 @@ pub trait HrTimerMode: private::Sealed { const C_MODE: bindings::hrtimer_mode; /// Type representing the clock source. - type Clock: ClockSource; + type Clock: ClockId; /// Type representing the expiration specification (absolute or relative time). type Expires: HrTimerExpires; } /// Timer that expires at a fixed point in time. -pub struct AbsoluteMode<C: ClockSource>(PhantomData<C>); +pub struct AbsoluteMode<C: ClockId>(PhantomData<C>); -impl<C: ClockSource> HrTimerMode for AbsoluteMode<C> { +impl<C: ClockId> HrTimerMode for AbsoluteMode<C> { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS; type Clock = C; @@ -911,9 +922,9 @@ impl<C: ClockSource> HrTimerMode for AbsoluteMode<C> { } /// Timer that expires after a delay from now. -pub struct RelativeMode<C: ClockSource>(PhantomData<C>); +pub struct RelativeMode<C: ClockId>(PhantomData<C>); -impl<C: ClockSource> HrTimerMode for RelativeMode<C> { +impl<C: ClockId> HrTimerMode for RelativeMode<C> { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL; type Clock = C; @@ -921,8 +932,8 @@ impl<C: ClockSource> HrTimerMode for RelativeMode<C> { } /// Timer with absolute expiration time, pinned to its current CPU. -pub struct AbsolutePinnedMode<C: ClockSource>(PhantomData<C>); -impl<C: ClockSource> HrTimerMode for AbsolutePinnedMode<C> { +pub struct AbsolutePinnedMode<C: ClockId>(PhantomData<C>); +impl<C: ClockId> HrTimerMode for AbsolutePinnedMode<C> { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED; type Clock = C; @@ -930,8 +941,8 @@ impl<C: ClockSource> HrTimerMode for AbsolutePinnedMode<C> { } /// Timer with relative expiration time, pinned to its current CPU. -pub struct RelativePinnedMode<C: ClockSource>(PhantomData<C>); -impl<C: ClockSource> HrTimerMode for RelativePinnedMode<C> { +pub struct RelativePinnedMode<C: ClockId>(PhantomData<C>); +impl<C: ClockId> HrTimerMode for RelativePinnedMode<C> { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED; type Clock = C; @@ -939,8 +950,8 @@ impl<C: ClockSource> HrTimerMode for RelativePinnedMode<C> { } /// Timer with absolute expiration, handled in soft irq context. -pub struct AbsoluteSoftMode<C: ClockSource>(PhantomData<C>); -impl<C: ClockSource> HrTimerMode for AbsoluteSoftMode<C> { +pub struct AbsoluteSoftMode<C: ClockId>(PhantomData<C>); +impl<C: ClockId> HrTimerMode for AbsoluteSoftMode<C> { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_SOFT; type Clock = C; @@ -948,8 +959,8 @@ impl<C: ClockSource> HrTimerMode for AbsoluteSoftMode<C> { } /// Timer with relative expiration, handled in soft irq context. -pub struct RelativeSoftMode<C: ClockSource>(PhantomData<C>); -impl<C: ClockSource> HrTimerMode for RelativeSoftMode<C> { +pub struct RelativeSoftMode<C: ClockId>(PhantomData<C>); +impl<C: ClockId> HrTimerMode for RelativeSoftMode<C> { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_SOFT; type Clock = C; @@ -957,8 +968,8 @@ impl<C: ClockSource> HrTimerMode for RelativeSoftMode<C> { } /// Timer with absolute expiration, pinned to CPU and handled in soft irq context. -pub struct AbsolutePinnedSoftMode<C: ClockSource>(PhantomData<C>); -impl<C: ClockSource> HrTimerMode for AbsolutePinnedSoftMode<C> { +pub struct AbsolutePinnedSoftMode<C: ClockId>(PhantomData<C>); +impl<C: ClockId> HrTimerMode for AbsolutePinnedSoftMode<C> { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED_SOFT; type Clock = C; @@ -966,8 +977,8 @@ impl<C: ClockSource> HrTimerMode for AbsolutePinnedSoftMode<C> { } /// Timer with absolute expiration, pinned to CPU and handled in soft irq context. -pub struct RelativePinnedSoftMode<C: ClockSource>(PhantomData<C>); -impl<C: ClockSource> HrTimerMode for RelativePinnedSoftMode<C> { +pub struct RelativePinnedSoftMode<C: ClockId>(PhantomData<C>); +impl<C: ClockId> HrTimerMode for RelativePinnedSoftMode<C> { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED_SOFT; type Clock = C; @@ -975,8 +986,8 @@ impl<C: ClockSource> HrTimerMode for RelativePinnedSoftMode<C> { } /// Timer with absolute expiration, handled in hard irq context. -pub struct AbsoluteHardMode<C: ClockSource>(PhantomData<C>); -impl<C: ClockSource> HrTimerMode for AbsoluteHardMode<C> { +pub struct AbsoluteHardMode<C: ClockId>(PhantomData<C>); +impl<C: ClockId> HrTimerMode for AbsoluteHardMode<C> { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_HARD; type Clock = C; @@ -984,8 +995,8 @@ impl<C: ClockSource> HrTimerMode for AbsoluteHardMode<C> { } /// Timer with relative expiration, handled in hard irq context. -pub struct RelativeHardMode<C: ClockSource>(PhantomData<C>); -impl<C: ClockSource> HrTimerMode for RelativeHardMode<C> { +pub struct RelativeHardMode<C: ClockId>(PhantomData<C>); +impl<C: ClockId> HrTimerMode for RelativeHardMode<C> { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_HARD; type Clock = C; @@ -993,8 +1004,8 @@ impl<C: ClockSource> HrTimerMode for RelativeHardMode<C> { } /// Timer with absolute expiration, pinned to CPU and handled in hard irq context. -pub struct AbsolutePinnedHardMode<C: ClockSource>(PhantomData<C>); -impl<C: ClockSource> HrTimerMode for AbsolutePinnedHardMode<C> { +pub struct AbsolutePinnedHardMode<C: ClockId>(PhantomData<C>); +impl<C: ClockId> HrTimerMode for AbsolutePinnedHardMode<C> { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED_HARD; type Clock = C; @@ -1002,8 +1013,8 @@ impl<C: ClockSource> HrTimerMode for AbsolutePinnedHardMode<C> { } /// Timer with relative expiration, pinned to CPU and handled in hard irq context. -pub struct RelativePinnedHardMode<C: ClockSource>(PhantomData<C>); -impl<C: ClockSource> HrTimerMode for RelativePinnedHardMode<C> { +pub struct RelativePinnedHardMode<C: ClockId>(PhantomData<C>); +impl<C: ClockId> HrTimerMode for RelativePinnedHardMode<C> { const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED_HARD; type Clock = C; diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs index 5f6c4d7a1a51..f1aecda7a90c 100644 --- a/rust/kernel/uaccess.rs +++ b/rust/kernel/uaccess.rs @@ -520,14 +520,14 @@ impl UserSliceWriter { /// /// fn copy_dma_to_user( /// mut writer: UserSliceWriter, - /// alloc: &Coherent<[u8]>, + /// alloc: &Coherent<'_, [u8]>, /// ) -> Result { /// writer.write_dma(alloc, 0, 256) /// } /// ``` pub fn write_dma<T: KnownSize + AsBytes + ?Sized>( &mut self, - alloc: &Coherent<T>, + alloc: &Coherent<'_, T>, offset: usize, count: usize, ) -> Result { @@ -558,6 +558,7 @@ impl UserSliceWriter { /// truncates the write to the boundaries of `self` and `data`. /// /// On success, returns the number of bytes written. + #[inline] pub fn write_slice_partial(&mut self, data: &[u8], offset: usize) -> Result<usize> { let end = offset.saturating_add(self.len()).min(data.len()); @@ -624,6 +625,19 @@ impl UserSliceWriter { self.length -= len; Ok(()) } + + /// Writes as much of the provided value as fits in the remaining buffer. + /// + /// Copies `min(size_of::<T>(), self.len())` bytes to userspace. Returns the number of bytes + /// actually written. This is useful for versioned structs where an older userspace may provide + /// a smaller buffer than the current kernel struct. + /// + /// Fails with [`EFAULT`] if the write happens on a bad address. This call may modify the + /// associated userspace slice even if it returns an error. + #[inline] + pub fn write_truncated<T: AsBytes>(&mut self, value: &T) -> Result<usize> { + self.write_slice_partial(value.as_bytes(), 0) + } } /// Reads a nul-terminated string into `dst` and returns the length. diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index 7e253b6f299c..a872f9fffb90 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -67,7 +67,7 @@ //! /// This method will enqueue the struct for execution on the system workqueue, where its value //! /// will be printed. //! fn print_later(val: Arc<MyStruct>) { -//! let _ = workqueue::system().enqueue(val); +//! let _ = workqueue::system_dfl().enqueue(val); //! } //! # print_later(MyStruct::new(42).unwrap()); //! ``` @@ -121,11 +121,11 @@ //! } //! //! fn print_1_later(val: Arc<MyStruct>) { -//! let _ = workqueue::system().enqueue::<Arc<MyStruct>, 1>(val); +//! let _ = workqueue::system_dfl().enqueue::<Arc<MyStruct>, 1>(val); //! } //! //! fn print_2_later(val: Arc<MyStruct>) { -//! let _ = workqueue::system().enqueue::<Arc<MyStruct>, 2>(val); +//! let _ = workqueue::system_dfl().enqueue::<Arc<MyStruct>, 2>(val); //! } //! # print_1_later(MyStruct::new(24, 25).unwrap()); //! # print_2_later(MyStruct::new(41, 42).unwrap()); @@ -171,13 +171,13 @@ //! /// This method will enqueue the struct for execution on the system workqueue, where its value //! /// will be printed 12 jiffies later. //! fn print_later(val: Arc<MyStruct>) { -//! let _ = workqueue::system().enqueue_delayed(val, 12); +//! let _ = workqueue::system_dfl().enqueue_delayed(val, 12); //! } //! //! /// It is also possible to use the ordinary `enqueue` method together with `DelayedWork`. This //! /// is equivalent to calling `enqueue_delayed` with a delay of zero. //! fn print_now(val: Arc<MyStruct>) { -//! let _ = workqueue::system().enqueue(val); +//! let _ = workqueue::system_dfl().enqueue(val); //! } //! # print_later(MyStruct::new(42).unwrap()); //! # print_now(MyStruct::new(42).unwrap()); @@ -1024,20 +1024,32 @@ where { } -/// Returns the system work queue (`system_wq`). +/// Returns the system per-cpu work queue (`system_percpu_wq`). /// /// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are /// users which expect relatively short queue flush time. /// /// Callers shouldn't queue work items which can run for too long. -pub fn system() -> &'static Queue { - // SAFETY: `system_wq` is a C global, always available. - unsafe { Queue::from_raw(bindings::system_wq) } +#[inline] +pub fn system_percpu() -> &'static Queue { + // SAFETY: `system_percpu_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_percpu_wq) } +} + +/// Returns the system default (unbound) work queue (`system_dfl_wq`). +/// +/// Workers are not bound to any specific CPU, not concurrency managed, and all queued work items +/// are executed immediately as long as `max_active` limit is not reached and resources are +/// available. +#[inline] +pub fn system_dfl() -> &'static Queue { + // SAFETY: `system_dfl_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_dfl_wq) } } /// Returns the system high-priority work queue (`system_highpri_wq`). /// -/// It is similar to the one returned by [`system`] but for work items which require higher +/// It is similar to the one returned by [`system_percpu`] but for work items which require higher /// scheduling priority. pub fn system_highpri() -> &'static Queue { // SAFETY: `system_highpri_wq` is a C global, always available. @@ -1046,26 +1058,16 @@ pub fn system_highpri() -> &'static Queue { /// Returns the system work queue for potentially long-running work items (`system_long_wq`). /// -/// It is similar to the one returned by [`system`] but may host long running work items. Queue -/// flushing might take relatively long. +/// It is similar to the one returned by [`system_percpu`] but may host long running work items. +/// Queue flushing might take relatively long. pub fn system_long() -> &'static Queue { // SAFETY: `system_long_wq` is a C global, always available. unsafe { Queue::from_raw(bindings::system_long_wq) } } -/// Returns the system unbound work queue (`system_unbound_wq`). -/// -/// Workers are not bound to any specific CPU, not concurrency managed, and all queued work items -/// are executed immediately as long as `max_active` limit is not reached and resources are -/// available. -pub fn system_unbound() -> &'static Queue { - // SAFETY: `system_unbound_wq` is a C global, always available. - unsafe { Queue::from_raw(bindings::system_unbound_wq) } -} - /// Returns the system freezable work queue (`system_freezable_wq`). /// -/// It is equivalent to the one returned by [`system`] except that it's freezable. +/// It is equivalent to the one returned by [`system_percpu`] except that it's freezable. /// /// A freezable workqueue participates in the freeze phase of the system suspend operations. Work /// items on the workqueue are drained and no new work item starts execution until thawed. @@ -1078,7 +1080,7 @@ pub fn system_freezable() -> &'static Queue { /// /// It is inclined towards saving power and is converted to "unbound" variants if the /// `workqueue.power_efficient` kernel parameter is specified; otherwise, it is similar to the one -/// returned by [`system`]. +/// returned by [`system_percpu`]. pub fn system_power_efficient() -> &'static Queue { // SAFETY: `system_power_efficient_wq` is a C global, always available. unsafe { Queue::from_raw(bindings::system_power_efficient_wq) } @@ -1097,7 +1099,7 @@ pub fn system_freezable_power_efficient() -> &'static Queue { /// Returns the system bottom halves work queue (`system_bh_wq`). /// -/// It is similar to the one returned by [`system`] but for work items which +/// It is similar to the one returned by [`system_percpu`] but for work items which /// need to run from a softirq context. pub fn system_bh() -> &'static Queue { // SAFETY: `system_bh_wq` is a C global, always available. |