summaryrefslogtreecommitdiff
path: root/rust/pin-init/src
diff options
context:
space:
mode:
Diffstat (limited to 'rust/pin-init/src')
-rw-r--r--rust/pin-init/src/__internal.rs13
-rw-r--r--rust/pin-init/src/alloc.rs10
-rw-r--r--rust/pin-init/src/lib.rs333
3 files changed, 209 insertions, 147 deletions
diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs
index 56dc655e323e..8e9fd18b993f 100644
--- a/rust/pin-init/src/__internal.rs
+++ b/rust/pin-init/src/__internal.rs
@@ -105,6 +105,7 @@ pub unsafe trait HasInitData {
pub struct AllData<T: ?Sized>(PhantomInvariant<T>);
impl<T: ?Sized> Clone for AllData<T> {
+ #[inline]
fn clone(&self) -> Self {
*self
}
@@ -127,6 +128,7 @@ impl<T: ?Sized> AllData<T> {
unsafe impl<T: ?Sized> HasInitData for T {
type InitData = AllData<T>;
+ #[inline]
unsafe fn __init_data() -> Self::InitData {
AllData(PhantomInvariant::new())
}
@@ -181,7 +183,7 @@ impl<T> StackInit<T> {
unsafe { this.value.assume_init_drop() };
}
// SAFETY: The memory slot is valid and this type ensures that it will stay pinned.
- unsafe { init.__pinned_init(this.value.as_mut_ptr())? };
+ unsafe { init.__init(this.value.as_mut_ptr())? };
// INVARIANT: `this.value` is initialized above.
this.is_init = true;
// SAFETY: The slot is now pinned, since we will never give access to `&mut T`.
@@ -289,7 +291,7 @@ impl<T: ?Sized> Slot<Pinned, T> {
// - when `Err` is returned, we also propagate the error without touching `ptr`;
// also `self` is consumed so it cannot be touched further.
// - the drop guard will not hand out `&mut` (only `Pin<&mut T>`).
- unsafe { init.__pinned_init(self.ptr)? };
+ unsafe { init.__init(self.ptr)? };
// SAFETY:
// - `self.ptr` is valid, properly aligned and pinned per type invariant.
@@ -385,20 +387,23 @@ pub struct AlwaysFail<T: ?Sized> {
impl<T: ?Sized> AlwaysFail<T> {
/// Creates a new initializer that always fails.
+ #[inline]
pub fn new() -> Self {
Self { _t: PhantomData }
}
}
impl<T: ?Sized> Default for AlwaysFail<T> {
+ #[inline]
fn default() -> Self {
Self::new()
}
}
-// SAFETY: `__pinned_init` always fails, which is always okay.
+// SAFETY: `__init` always fails, which is always okay.
unsafe impl<T: ?Sized> PinInit<T, ()> for AlwaysFail<T> {
- unsafe fn __pinned_init(self, _slot: *mut T) -> Result<(), ()> {
+ #[inline]
+ unsafe fn __init(self, _slot: *mut T) -> Result<(), ()> {
Err(())
}
}
diff --git a/rust/pin-init/src/alloc.rs b/rust/pin-init/src/alloc.rs
index 5017f57442d8..471652e8663a 100644
--- a/rust/pin-init/src/alloc.rs
+++ b/rust/pin-init/src/alloc.rs
@@ -35,10 +35,11 @@ pub trait InPlaceInit<T>: Sized {
/// type.
///
/// If `T: !Unpin` it will not be able to move afterwards.
+ #[inline]
fn pin_init(init: impl PinInit<T>) -> Result<Pin<Self>, AllocError> {
// SAFETY: We delegate to `init` and only change the error type.
let init = unsafe {
- pin_init_from_closure(|slot| match init.__pinned_init(slot) {
+ pin_init_from_closure(|slot| match init.__init(slot) {
Ok(()) => Ok(()),
Err(i) => match i {},
})
@@ -52,6 +53,7 @@ pub trait InPlaceInit<T>: Sized {
E: From<AllocError>;
/// Use the given initializer to in-place initialize a `T`.
+ #[inline]
fn init(init: impl Init<T>) -> Result<Self, AllocError> {
// SAFETY: We delegate to `init` and only change the error type.
let init = unsafe {
@@ -109,7 +111,7 @@ impl<T> InPlaceInit<T> for Arc<T> {
let slot = slot.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
// slot is valid and will not be moved, because we pin it later.
- unsafe { init.__pinned_init(slot)? };
+ unsafe { init.__init(slot)? };
// SAFETY: All fields have been initialized and this is the only `Arc` to that data.
Ok(unsafe { Pin::new_unchecked(this.assume_init()) })
}
@@ -136,6 +138,7 @@ impl<T> InPlaceInit<T> for Arc<T> {
impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> {
type Initialized = Box<T>;
+ #[inline]
fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
let slot = self.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
@@ -145,11 +148,12 @@ impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> {
Ok(unsafe { self.assume_init() })
}
+ #[inline]
fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
let slot = self.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
// slot is valid and will not be moved, because we pin it later.
- unsafe { init.__pinned_init(slot)? };
+ unsafe { init.__init(slot)? };
// SAFETY: All fields have been initialized.
Ok(unsafe { self.assume_init() }.into())
}
diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs
index fd40c8f244a1..7600cdbbbf98 100644
--- a/rust/pin-init/src/lib.rs
+++ b/rust/pin-init/src/lib.rs
@@ -70,7 +70,6 @@
//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
//!
//! ```rust
-//! # #![expect(clippy::disallowed_names)]
//! # #![feature(allocator_api)]
//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
//! # use core::pin::Pin;
@@ -94,7 +93,6 @@
//! (or just the stack) to actually initialize a `Foo`:
//!
//! ```rust
-//! # #![expect(clippy::disallowed_names)]
//! # #![feature(allocator_api)]
//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
//! # use core::{alloc::AllocError, pin::Pin};
@@ -456,7 +454,6 @@ pub use ::pin_init_internal::MaybeZeroable;
/// # Examples
///
/// ```rust
-/// # #![expect(clippy::disallowed_names)]
/// # #![feature(allocator_api)]
/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
/// # use pin_init::*;
@@ -508,7 +505,6 @@ macro_rules! stack_pin_init {
/// # Examples
///
/// ```rust
-/// # #![expect(clippy::disallowed_names)]
/// # #![feature(allocator_api)]
/// # #[path = "../examples/error.rs"] mod error; use error::Error;
/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
@@ -535,7 +531,6 @@ macro_rules! stack_pin_init {
/// ```
///
/// ```rust
-/// # #![expect(clippy::disallowed_names)]
/// # #![feature(allocator_api)]
/// # #[path = "../examples/error.rs"] mod error; use error::Error;
/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
@@ -658,7 +653,6 @@ macro_rules! stack_try_pin_init {
/// Users of `Foo` can now create it like this:
///
/// ```rust
-/// # #![expect(clippy::disallowed_names)]
/// # use pin_init::*;
/// # use core::pin::Pin;
/// # #[pin_data]
@@ -895,7 +889,7 @@ macro_rules! assert_pinned {
/// When implementing this trait you will need to take great care. Also there are probably very few
/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
///
-/// The [`PinInit::__pinned_init`] function:
+/// The [`PinInit::__init`] function:
/// - returns `Ok(())` if it initialized every field of `slot`,
/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
/// - `slot` can be deallocated without UB occurring,
@@ -915,15 +909,33 @@ macro_rules! assert_pinned {
#[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")]
#[must_use = "An initializer must be used in order to create its value."]
pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
+ /// Alias of [`PinInit::__init`].
+ ///
+ /// New code should use `__init` instead.
+ ///
+ /// # Safety
+ ///
+ /// Same as `__init`.
+ #[inline(always)]
+ #[cfg(not(kernel))]
+ #[deprecated = "use `raw_try_init` instead"]
+ unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
+ // SAFETY: Per safety requirement.
+ unsafe { self.__init(slot) }
+ }
+
/// Initializes `slot`.
///
+ /// It is not recommended to call this directly. Use [`raw_init`] or [`raw_try_init`].
+ ///
/// # Safety
///
/// - `slot` is a valid pointer to uninitialized memory.
/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
/// deallocate.
/// - `slot` will not move until it is dropped, i.e. it will be pinned.
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
+ /// If `Self: Init<T, E>`, this requirement is cancelled and it may be moved.
+ unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
/// First initializes the value using `self` then calls the function `f` with the initialized
/// value.
@@ -943,6 +955,7 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
/// Ok(())
/// });
/// ```
+ #[inline]
fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E>
where
F: FnOnce(Pin<&mut T>) -> Result<(), E>,
@@ -951,10 +964,38 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
}
}
+/// Initializes `slot` with an initializer.
+///
+/// # Safety
+///
+/// - `slot` is a valid pointer to uninitialized memory.
+/// - `slot` will not move until it is dropped, i.e. it will be pinned.
+/// If `init` implements `Init<T, E>`, this requirement is cancelled and it may be moved.
+#[inline(always)]
+pub unsafe fn raw_init<T>(slot: *mut T, init: impl PinInit<T>) {
+ // SAFETY: Per safety requirement.
+ unsafe { init.__init(slot).unwrap_or_else(|e| match e {}) }
+}
+
+/// Fallibly initializes `slot` with an initializer.
+///
+/// # Safety
+///
+/// - `slot` is a valid pointer to uninitialized memory.
+/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
+/// deallocate.
+/// - `slot` will not move until it is dropped, i.e. it will be pinned.
+/// If `init` implements `Init<T, E>`, this requirement is cancelled and it may be moved.
+#[inline(always)]
+pub unsafe fn raw_try_init<T, E>(slot: *mut T, init: impl PinInit<T, E>) -> Result<(), E> {
+ // SAFETY: Per safety requirement.
+ unsafe { init.__init(slot) }
+}
+
/// An initializer returned by [`PinInit::pin_chain`].
pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::PhantomInvariant<(E, T)>);
-// SAFETY: The `__pinned_init` function is implemented such that it
+// SAFETY: The `__init` function is implemented such that it
// - returns `Ok(())` on successful initialization,
// - returns `Err(err)` on error and in this case `slot` will be dropped.
// - considers `slot` pinned.
@@ -963,15 +1004,14 @@ where
I: PinInit<T, E>,
F: FnOnce(Pin<&mut T>) -> Result<(), E>,
{
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: All requirements fulfilled since this function is `__pinned_init`.
- unsafe { self.0.__pinned_init(slot)? };
- // SAFETY: The above call initialized `slot` and we still have unique access.
- let val = unsafe { &mut *slot };
- // SAFETY: `slot` is considered pinned.
- let val = unsafe { Pin::new_unchecked(val) };
- // SAFETY: `slot` was initialized above.
- (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) })
+ #[inline]
+ unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
+ // SAFETY: All requirements fulfilled since this function is `__init`.
+ let slot = unsafe { __internal::Slot::<__internal::Pinned, _>::new(slot) };
+ let mut guard = slot.init(self.0)?;
+ (self.1)(guard.let_binding())?;
+ core::mem::forget(guard);
+ Ok(())
}
}
@@ -988,19 +1028,8 @@ where
/// When implementing this trait you will need to take great care. Also there are probably very few
/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
///
-/// The [`Init::__init`] function:
-/// - returns `Ok(())` if it initialized every field of `slot`,
-/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
-/// - `slot` can be deallocated without UB occurring,
-/// - `slot` does not need to be dropped,
-/// - `slot` is not partially initialized.
-/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
-///
-/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
-/// code as `__init`.
-///
-/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to
-/// move the pointee after initialization.
+/// The [`PinInit::__init`] function must work without the pinning requirement; the caller is
+/// allowed to move the pointee after initialization.
///
#[cfg_attr(
kernel,
@@ -1014,15 +1043,6 @@ where
#[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")]
#[must_use = "An initializer must be used in order to create its value."]
pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
- /// Initializes `slot`.
- ///
- /// # Safety
- ///
- /// - `slot` is a valid pointer to uninitialized memory.
- /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
- /// deallocate.
- unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
-
/// First initializes the value using `self` then calls the function `f` with the initialized
/// value.
///
@@ -1031,7 +1051,6 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
/// # Examples
///
/// ```rust
- /// # #![expect(clippy::disallowed_names)]
/// use pin_init::{init, init_zeroed, Init};
///
/// struct Foo {
@@ -1051,6 +1070,7 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
/// Ok(())
/// });
/// ```
+ #[inline]
fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E>
where
F: FnOnce(&mut T) -> Result<(), E>,
@@ -1062,62 +1082,55 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
/// An initializer returned by [`Init::chain`].
pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::PhantomInvariant<(E, T)>);
-// SAFETY: The `__init` function is implemented such that it
-// - returns `Ok(())` on successful initialization,
-// - returns `Err(err)` on error and in this case `slot` will be dropped.
+// SAFETY: The `__init` function does not rely on the pinning requirement.
unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E>
where
I: Init<T, E>,
F: FnOnce(&mut T) -> Result<(), E>,
{
- unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: All requirements fulfilled since this function is `__init`.
- unsafe { self.0.__pinned_init(slot)? };
- // SAFETY: The above call initialized `slot` and we still have unique access.
- (self.1)(unsafe { &mut *slot }).inspect_err(|_|
- // SAFETY: `slot` was initialized above.
- unsafe { core::ptr::drop_in_place(slot) })
- }
}
-// SAFETY: `__pinned_init` behaves exactly the same as `__init`.
+// SAFETY: The `__init` function is implemented such that it
+// - returns `Ok(())` on successful initialization,
+// - returns `Err(err)` on error and in this case `slot` will be dropped.
unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E>
where
I: Init<T, E>,
F: FnOnce(&mut T) -> Result<(), E>,
{
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: `__init` has less strict requirements compared to `__pinned_init`.
- unsafe { self.__init(slot) }
+ #[inline]
+ unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
+ // SAFETY: All requirements fulfilled since this function is `__init`.
+ let slot = unsafe { __internal::Slot::<__internal::Unpinned, _>::new(slot) };
+ let mut guard = slot.init(self.0)?;
+ (self.1)(guard.let_binding())?;
+ core::mem::forget(guard);
+ Ok(())
}
}
/// Implement `PinInit` and `Init` for closures.
///
/// It is unsafe to create this type, since the closure needs to fulfill the same safety
-/// requirement as the `__pinned_init`/`__init` functions.
+/// requirement as the `__init` functions.
struct InitClosure<F, T: ?Sized>(F, __internal::PhantomInvariant<T>);
-// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
-// `__init` invariants.
-unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T>
-where
- F: FnOnce(*mut T) -> Result<(), E>,
+// SAFETY: When constructing via `init_from_closure`, the `__init` function does not rely on the
+// pinning requirement. When constructing via `pin_init_from_closure`, the opaque type prevents this
+// implementation from being visible.
+unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T> where
+ F: FnOnce(*mut T) -> Result<(), E>
{
- #[inline]
- unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- (self.0)(slot)
- }
}
// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
-// `__pinned_init` invariants.
+// `__init` invariants.
unsafe impl<T: ?Sized, F, E> PinInit<T, E> for InitClosure<F, T>
where
F: FnOnce(*mut T) -> Result<(), E>,
{
#[inline]
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
+ unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
(self.0)(slot)
}
}
@@ -1166,10 +1179,11 @@ pub const unsafe fn init_from_closure<T: ?Sized, E>(
///
/// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a
/// pointer must result in a valid `U`.
+#[inline]
pub const unsafe fn cast_pin_init<T, U, E>(init: impl PinInit<T, E>) -> impl PinInit<U, E> {
// SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety
// requirements.
- unsafe { pin_init_from_closure(|ptr: *mut U| init.__pinned_init(ptr.cast::<T>())) }
+ unsafe { pin_init_from_closure(|ptr: *mut U| init.__init(ptr.cast::<T>())) }
}
/// Changes the to be initialized type.
@@ -1178,6 +1192,7 @@ pub const unsafe fn cast_pin_init<T, U, E>(init: impl PinInit<T, E>) -> impl Pin
///
/// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a
/// pointer must result in a valid `U`.
+#[inline]
pub const unsafe fn cast_init<T, U, E>(init: impl Init<T, E>) -> impl Init<U, E> {
// SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety
// requirements.
@@ -1193,6 +1208,77 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
unsafe { init_from_closure(|_| Ok(())) }
}
+/// Array initializer from element initializer.
+struct ArrayInit<T: ?Sized, F>(F, __internal::PhantomInvariant<T>);
+
+// SAFETY: On success, all `N` elements of the array have been initialized. On error or panic, the
+// elements that have been initialized so far are dropped, thus leaving the array uninitialized and
+// ready to deallocate.
+unsafe impl<T, F, I, E, const N: usize> PinInit<[T; N], E> for ArrayInit<T, F>
+where
+ F: FnMut(usize) -> I,
+ I: PinInit<T, E>,
+{
+ unsafe fn __init(mut self, slot: *mut [T; N]) -> Result<(), E> {
+ /// # Invariants
+ ///
+ /// - `ptr[..num_init]` contains initialized elements of type `T`
+ /// - `ptr[num_init..N]` (where N is the size of the array) contains uninitialized memory
+ struct ArrayInitGuard<T> {
+ /// A pointer to the first element of the array.
+ ptr: *mut T,
+ /// The number of initialized elements in the array.
+ num_init: usize,
+ }
+
+ impl<T> Drop for ArrayInitGuard<T> {
+ #[inline]
+ fn drop(&mut self) {
+ // SAFETY: Per type invariant, `self.ptr[..self.num_init]` are initialized.
+ unsafe {
+ core::ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut(
+ self.ptr,
+ self.num_init,
+ ))
+ };
+ }
+ }
+
+ // INVARIANT: nothing is initialized yet.
+ let mut guard = ArrayInitGuard {
+ ptr: slot.cast::<T>(),
+ num_init: 0,
+ };
+
+ for i in 0..N {
+ // INVARIANT: Elements `self.ptr[..self.num_init]` have been initialized
+ // thus far. This holds true for every `self.num_init = i`.
+ guard.num_init = i;
+
+ let init = (self.0)(i);
+ // SAFETY:
+ // - The subslot is derived from `slot` with a valid offset.
+ // - If `Err` is touched, the subslot is not touched further, the guard will drop
+ // previously initialized elements only.
+ // - `slot` is pinned so is the subslot.
+ unsafe { init.__init(&raw mut (*slot)[i]) }?;
+ }
+
+ // Dismiss the drop guard now that all elements are initialized.
+ core::mem::forget(guard);
+ Ok(())
+ }
+}
+
+// SAFETY: `I: Init` cancels out the pinning requirement on subslots, which is the only place in the
+// `__init` function that relies on `slot` being pinned.
+unsafe impl<T, F, I, E, const N: usize> Init<[T; N], E> for ArrayInit<T, F>
+where
+ F: FnMut(usize) -> I,
+ I: Init<T, E>,
+{
+}
+
/// Initializes an array by initializing each element via the provided initializer.
///
/// # Examples
@@ -1203,32 +1289,14 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
/// let array: Box<[usize; 1_000]> = Box::init(init_array_from_fn(|i| i)).unwrap();
/// assert_eq!(array.len(), 1_000);
/// ```
+#[inline]
pub fn init_array_from_fn<I, const N: usize, T, E>(
- mut make_init: impl FnMut(usize) -> I,
+ make_init: impl FnMut(usize) -> I,
) -> impl Init<[T; N], E>
where
I: Init<T, E>,
{
- let init = move |slot: *mut [T; N]| {
- let slot = slot.cast::<T>();
- for i in 0..N {
- let init = make_init(i);
- // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
- let ptr = unsafe { slot.add(i) };
- // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
- // requirements.
- if let Err(e) = unsafe { init.__init(ptr) } {
- // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return
- // `Err` below, `slot` will be considered uninitialized memory.
- unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
- return Err(e);
- }
- }
- Ok(())
- };
- // SAFETY: The initializer above initializes every element of the array. On failure it drops
- // any initialized elements and returns `Err`.
- unsafe { init_from_closure(init) }
+ ArrayInit(make_init, __internal::PhantomInvariant::new())
}
/// Initializes an array by initializing each element via the provided initializer.
@@ -1246,32 +1314,14 @@ where
/// Arc::pin_init(pin_init_array_from_fn(|i| CMutex::new(i))).unwrap();
/// assert_eq!(array.len(), 1_000);
/// ```
+#[inline]
pub fn pin_init_array_from_fn<I, const N: usize, T, E>(
- mut make_init: impl FnMut(usize) -> I,
+ make_init: impl FnMut(usize) -> I,
) -> impl PinInit<[T; N], E>
where
I: PinInit<T, E>,
{
- let init = move |slot: *mut [T; N]| {
- let slot = slot.cast::<T>();
- for i in 0..N {
- let init = make_init(i);
- // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
- let ptr = unsafe { slot.add(i) };
- // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
- // requirements.
- if let Err(e) = unsafe { init.__pinned_init(ptr) } {
- // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return
- // `Err` below, `slot` will be considered uninitialized memory.
- unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
- return Err(e);
- }
- }
- Ok(())
- };
- // SAFETY: The initializer above initializes every element of the array. On failure it drops
- // any initialized elements and returns `Err`.
- unsafe { pin_init_from_closure(init) }
+ ArrayInit(make_init, __internal::PhantomInvariant::new())
}
/// Construct an initializer in a closure and run it.
@@ -1300,6 +1350,7 @@ where
/// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the
/// initializer itself will fail with that error. If it returned `Ok`, then it will run the
/// initializer returned by the [`pin_init!`] invocation.
+#[inline]
pub fn pin_init_scope<T, E, F, I>(make_init: F) -> impl PinInit<T, E>
where
F: FnOnce() -> Result<I, E>,
@@ -1307,13 +1358,13 @@ where
{
// SAFETY:
// - If `make_init` returns `Err`, `Err` is returned and `slot` is completely uninitialized,
- // - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__pinned_init`.
- // - The safety requirements of `init.__pinned_init` are fulfilled, since it's being called
- // from an initializer.
+ // - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__init`.
+ // - The safety requirements of `init.__init` are fulfilled, since it's being called from an
+ // initializer.
unsafe {
pin_init_from_closure(move |slot: *mut T| -> Result<(), E> {
let init = make_init()?;
- init.__pinned_init(slot)
+ init.__init(slot)
})
}
}
@@ -1343,6 +1394,7 @@ where
/// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the
/// initializer itself will fail with that error. If it returned `Ok`, then it will run the
/// initializer returned by the [`init!`] invocation.
+#[inline]
pub fn init_scope<T, E, F, I>(make_init: F) -> impl Init<T, E>
where
F: FnOnce() -> Result<I, E>,
@@ -1361,41 +1413,29 @@ where
}
}
-// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of `slot`.
-unsafe impl<T> Init<T> for T {
- unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> {
- // SAFETY: `slot` is valid for writes by the safety requirements of this function.
- unsafe { slot.write(self) };
- Ok(())
- }
-}
+// SAFETY: The `__init` function does not rely on slot being pinned after it returns.
+unsafe impl<T> Init<T> for T {}
-// SAFETY: the `__pinned_init` function always returns `Ok(())` and initializes every field of
+// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of
// `slot`. Additionally, all pinning invariants of `T` are upheld.
unsafe impl<T> PinInit<T> for T {
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), Infallible> {
+ #[inline]
+ unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> {
// SAFETY: `slot` is valid for writes by the safety requirements of this function.
unsafe { slot.write(self) };
Ok(())
}
}
-// SAFETY: when the `__init` function returns with
-// - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld.
-// - `Err(err)`, slot was not written to.
-unsafe impl<T, E> Init<T, E> for Result<T, E> {
- unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: `slot` is valid for writes by the safety requirements of this function.
- unsafe { slot.write(self?) };
- Ok(())
- }
-}
+// SAFETY: The `__init` function does not rely on slot being pinned after it returns.
+unsafe impl<T, E> Init<T, E> for Result<T, E> {}
-// SAFETY: when the `__pinned_init` function returns with
+// SAFETY: when the `__init` function returns with
// - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld.
// - `Err(err)`, slot was not written to.
unsafe impl<T, E> PinInit<T, E> for Result<T, E> {
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
+ #[inline]
+ unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
// SAFETY: `slot` is valid for writes by the safety requirements of this function.
unsafe { slot.write(self?) };
Ok(())
@@ -1421,6 +1461,7 @@ pub trait InPlaceWrite<T> {
impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> {
type Initialized = &'static mut T;
+ #[inline]
fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
let slot = self.as_mut_ptr();
@@ -1431,6 +1472,7 @@ impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> {
unsafe { Ok(self.assume_init_mut()) }
}
+ #[inline]
fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
let slot = self.as_mut_ptr();
@@ -1438,7 +1480,7 @@ impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> {
//
// The `'static` borrow guarantees the data will not be
// moved/invalidated until it gets dropped (which is never).
- unsafe { init.__pinned_init(slot)? };
+ unsafe { init.__init(slot)? };
// SAFETY: The above call initialized the memory.
Ok(Pin::static_mut(unsafe { self.assume_init_mut() }))
@@ -1510,10 +1552,13 @@ pub unsafe trait Zeroable {
/// Whenever a type implements [`Zeroable`], this function should be preferred over
/// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`.
///
+ /// As const traits are not yet stable, [`pin_init::zeroed()`] can be used instead
+ /// when initialization is required in a `const` context.
+ ///
/// # Examples
///
/// ```
- /// use pin_init::{Zeroable, zeroed};
+ /// use pin_init::Zeroable;
///
/// #[derive(Zeroable)]
/// struct Point {
@@ -1521,10 +1566,11 @@ pub unsafe trait Zeroable {
/// y: u32,
/// }
///
- /// let point: Point = zeroed();
+ /// let point: Point = Zeroable::zeroed();
/// assert_eq!(point.x, 0);
/// assert_eq!(point.y, 0);
/// ```
+ #[inline]
fn zeroed() -> Self
where
Self: Sized,
@@ -1553,6 +1599,9 @@ pub fn init_zeroed<T: Zeroable>() -> impl Init<T> {
/// Whenever a type implements [`Zeroable`], this function should be preferred over
/// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`.
///
+/// While const traits remain unstable, this function serves as the `const` version of
+/// [`Zeroable::zeroed()`].
+///
/// # Examples
///
/// ```
@@ -1568,6 +1617,7 @@ pub fn init_zeroed<T: Zeroable>() -> impl Init<T> {
/// assert_eq!(point.x, 0);
/// assert_eq!(point.y, 0);
/// ```
+#[inline]
pub const fn zeroed<T: Zeroable>() -> T {
// SAFETY:By the type invariants of `Zeroable`, all zeroes is a valid bit pattern for `T`.
unsafe { core::mem::zeroed() }
@@ -1728,6 +1778,7 @@ pub trait Wrapper<T> {
}
impl<T> Wrapper<T> for UnsafeCell<T> {
+ #[inline]
fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> {
// SAFETY: `UnsafeCell<T>` has a compatible layout to `T`.
unsafe { cast_pin_init(value_init) }
@@ -1735,6 +1786,7 @@ impl<T> Wrapper<T> for UnsafeCell<T> {
}
impl<T> Wrapper<T> for MaybeUninit<T> {
+ #[inline]
fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> {
// SAFETY: `MaybeUninit<T>` has a compatible layout to `T`.
unsafe { cast_pin_init(value_init) }
@@ -1743,6 +1795,7 @@ impl<T> Wrapper<T> for MaybeUninit<T> {
#[cfg(all(feature = "unsafe-pinned", CONFIG_RUSTC_HAS_UNSAFE_PINNED))]
impl<T> Wrapper<T> for core::pin::UnsafePinned<T> {
+ #[inline]
fn pin_init<E>(init: impl PinInit<T, E>) -> impl PinInit<Self, E> {
// SAFETY: `UnsafePinned<T>` has a compatible layout to `T`.
unsafe { cast_pin_init(init) }
generated by cgit v1.2.3 (git 2.0.4) at 2026-08-20 22:37:24 +0000