summaryrefslogtreecommitdiff
path: root/rust/pin-init
diff options
context:
space:
mode:
Diffstat (limited to 'rust/pin-init')
-rw-r--r--rust/pin-init/examples/mutex.rs11
-rw-r--r--rust/pin-init/examples/static_init.rs10
-rw-r--r--rust/pin-init/internal/src/init.rs8
-rw-r--r--rust/pin-init/internal/src/pin_data.rs94
-rw-r--r--rust/pin-init/src/__internal.rs13
-rw-r--r--rust/pin-init/src/alloc.rs10
-rw-r--r--rust/pin-init/src/lib.rs333
7 files changed, 291 insertions, 188 deletions
diff --git a/rust/pin-init/examples/mutex.rs b/rust/pin-init/examples/mutex.rs
index 35ecb5f68dc3..e8d4dbb664fe 100644
--- a/rust/pin-init/examples/mutex.rs
+++ b/rust/pin-init/examples/mutex.rs
@@ -79,11 +79,7 @@ impl<T> CMutex<T> {
wait_list <- ListHead::new(),
spin_lock: SpinLock::new(),
locked: Cell::new(false),
- data <- unsafe {
- pin_init_from_closure(|slot: *mut UnsafeCell<T>| {
- val.__pinned_init(slot.cast::<T>())
- })
- },
+ data <- UnsafeCell::pin_init(val),
})
}
@@ -91,7 +87,7 @@ impl<T> CMutex<T> {
pub fn lock(&self) -> Pin<CMutexGuard<'_, T>> {
let mut sguard = self.spin_lock.acquire();
if self.locked.get() {
- stack_pin_init!(let wait_entry = WaitEntry::insert_new(&self.wait_list));
+ stack_pin_init!(let _wait_entry = WaitEntry::insert_new(&self.wait_list));
// println!("wait list length: {}", self.wait_list.size());
while self.locked.get() {
drop(sguard);
@@ -99,9 +95,6 @@ impl<T> CMutex<T> {
thread::park();
sguard = self.spin_lock.acquire();
}
- // This does have an effect, as the ListHead inside wait_entry implements Drop!
- #[expect(clippy::drop_non_drop)]
- drop(wait_entry);
}
self.locked.set(true);
unsafe {
diff --git a/rust/pin-init/examples/static_init.rs b/rust/pin-init/examples/static_init.rs
index 58cd4241b78c..8dd52313c1b8 100644
--- a/rust/pin-init/examples/static_init.rs
+++ b/rust/pin-init/examples/static_init.rs
@@ -59,7 +59,7 @@ impl<T, I: PinInit<T>> ops::Deref for StaticInit<T, I> {
println!("doing init");
let ptr = self.cell.get().cast::<T>();
match self.init.take() {
- Some(f) => unsafe { f.__pinned_init(ptr).unwrap() },
+ Some(f) => unsafe { pin_init::raw_init(ptr, f) },
None => unsafe { core::hint::unreachable_unchecked() },
}
self.present.set(true);
@@ -71,13 +71,11 @@ impl<T, I: PinInit<T>> ops::Deref for StaticInit<T, I> {
pub struct CountInit;
unsafe impl PinInit<CMutex<usize>> for CountInit {
- unsafe fn __pinned_init(
- self,
- slot: *mut CMutex<usize>,
- ) -> Result<(), core::convert::Infallible> {
+ unsafe fn __init(self, slot: *mut CMutex<usize>) -> Result<(), core::convert::Infallible> {
let init = CMutex::new(0);
std::thread::sleep(std::time::Duration::from_millis(1000));
- unsafe { init.__pinned_init(slot) }
+ unsafe { pin_init::raw_init(slot, init) };
+ Ok(())
}
}
diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs
index 28d30805d06b..fd0b5ea4a0a3 100644
--- a/rust/pin-init/internal/src/init.rs
+++ b/rust/pin-init/internal/src/init.rs
@@ -233,10 +233,12 @@ fn init_fields(
InitializerKind::Value { ident, .. } => ident,
InitializerKind::Init { ident, .. } => ident,
InitializerKind::Code { block, .. } => {
+ let stmt = &block.stmts;
res.extend(quote! {
#(#attrs)*
- #[allow(unused_braces)]
- #block
+ {
+ #(#stmt)*
+ }
});
continue;
}
@@ -334,7 +336,7 @@ fn make_field_check(
}),
};
quote! {
- #[allow(unreachable_code, clippy::diverging_sub_expression)]
+ #[allow(unreachable_code)]
// We use unreachable code to perform field checks. They're still checked by the compiler.
// SAFETY: this code is never executed.
let _ = || unsafe {
diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs
index 9fbbd25bcaac..ff194d27565e 100644
--- a/rust/pin-init/internal/src/pin_data.rs
+++ b/rust/pin-init/internal/src/pin_data.rs
@@ -1,13 +1,13 @@
// SPDX-License-Identifier: Apache-2.0 OR MIT
use proc_macro2::TokenStream;
-use quote::{format_ident, quote};
+use quote::{format_ident, quote, ToTokens};
use syn::{
parse::{End, Nothing, Parse},
parse_quote, parse_quote_spanned,
spanned::Spanned,
visit_mut::VisitMut,
- Attribute, Field, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause,
+ Field, Fields, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause,
};
use crate::diagnostics::{DiagCtxt, ErrorGuaranteed};
@@ -35,10 +35,18 @@ impl Parse for Args {
}
}
+impl ToTokens for Args {
+ fn to_tokens(&self, tokens: &mut TokenStream) {
+ match self {
+ Self::Nothing(_) => (),
+ Self::PinnedDrop(kw) => kw.to_tokens(tokens),
+ }
+ }
+}
+
struct FieldInfo<'a> {
field: &'a Field,
pinned: bool,
- cfg_attrs: Vec<&'a Attribute>,
}
pub(crate) fn pin_data(
@@ -68,6 +76,55 @@ pub(crate) fn pin_data(
}
};
+ // Handling cfg can gets very complicated, especially for tuple structs. Therefore, resolve all
+ // field cfgs first before continuing.
+ //
+ // We need to perform this after parsing so we can reliably detect field cfgs.
+ for (field_idx, field) in struct_.fields.iter_mut().enumerate() {
+ let cfg: Vec<_> = field
+ .attrs
+ .iter()
+ .filter(|a| a.path().is_ident("cfg"))
+ .map(|a| {
+ a.parse_args::<TokenStream>()
+ .expect("parse as token stream cannot fail")
+ })
+ .collect();
+
+ if cfg.is_empty() {
+ continue;
+ }
+
+ field.attrs.retain(|a| !a.path().is_ident("cfg"));
+ let cfg_true_struct = quote!(#struct_);
+
+ let punctuated = match &mut struct_.fields {
+ Fields::Named(fields) => &mut fields.named,
+ Fields::Unnamed(fields) => &mut fields.unnamed,
+ Fields::Unit => unreachable!(),
+ };
+ *punctuated = std::mem::take(punctuated)
+ .into_pairs()
+ .enumerate()
+ .filter(|&(i, _)| i != field_idx)
+ .map(|(_, p)| p)
+ .collect();
+ let cfg_false_struct = quote!(#struct_);
+
+ // Resolve one field at a time until we've got no more field cfgs.
+ //
+ // This is linear time because macro invocations with false cfg will not be expanded.
+ return Ok(quote!(
+ #[cfg(all(#(#cfg,)*))]
+ #[::pin_init::pin_data(#args)]
+ #cfg_true_struct
+
+ #[cfg(not(all(#(#cfg,)*)))]
+ #[::pin_init::pin_data(#args)]
+ #cfg_false_struct
+ ));
+ }
+
// The generics might contain the `Self` type. Since this macro will define a new type with the
// same generics and bounds, this poses a problem: `Self` will refer to the new type as opposed
// to this struct definition. Therefore we have to replace `Self` with the concrete name.
@@ -85,18 +142,19 @@ pub(crate) fn pin_data(
.map(|field| {
let len = field.attrs.len();
field.attrs.retain(|a| !a.path().is_ident("pin"));
- let pinned = len != field.attrs.len();
+ let pinned_count = len - field.attrs.len();
+ if pinned_count > 1 {
+ dcx.error(&field, "#[pin] attribute specified more than once");
+ }
- let cfg_attrs = field
- .attrs
- .iter()
- .filter(|a| a.path().is_ident("cfg"))
- .collect();
+ assert!(
+ !field.attrs.iter().any(|a| a.path().is_ident("cfg")),
+ "cfgs should be all resolved at this point"
+ );
FieldInfo {
field: &*field,
- pinned,
- cfg_attrs,
+ pinned: pinned_count != 0,
}
})
.collect();
@@ -182,9 +240,7 @@ fn generate_unpin_impl(
let pinned_fields = fields.iter().filter(|f| f.pinned).map(|f| {
let ident = f.field.ident.as_ref().unwrap();
let ty = &f.field.ty;
- let cfg_attrs = &f.cfg_attrs;
quote!(
- #(#cfg_attrs)*
#ident: #ty
)
});
@@ -242,7 +298,6 @@ fn generate_drop_impl(ident: &Ident, generics: &Generics, args: Args) -> TokenSt
// `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
// if it also implements `Drop`
trait MustNotImplDrop {}
- #[expect(drop_bounds)]
impl<T: ::core::ops::Drop + ?::core::marker::Sized> MustNotImplDrop for T {}
impl #impl_generics MustNotImplDrop for #ident #ty_generics
#whr
@@ -250,7 +305,6 @@ fn generate_drop_impl(ident: &Ident, generics: &Generics, args: Args) -> TokenSt
// We also take care to prevent users from writing a useless `PinnedDrop` implementation.
// They might implement `PinnedDrop` correctly for the struct, but forget to give
// `PinnedDrop` as the parameter to `#[pin_data]`.
- #[expect(non_camel_case_types)]
trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
impl<T: ::pin_init::PinnedDrop + ?::core::marker::Sized>
UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
@@ -279,7 +333,6 @@ fn generate_projections(
.iter()
.map(|field| {
let Field { vis, ident, ty, .. } = &field.field;
- let cfg_attrs = &field.cfg_attrs;
let ident = ident
.as_ref()
@@ -287,11 +340,9 @@ fn generate_projections(
if field.pinned {
(
quote!(
- #(#cfg_attrs)*
#vis #ident: ::core::pin::Pin<&'__pin mut #ty>,
),
quote!(
- #(#cfg_attrs)*
// SAFETY: this field is structurally pinned.
#ident: unsafe { ::core::pin::Pin::new_unchecked(&mut #this.#ident) },
),
@@ -299,11 +350,9 @@ fn generate_projections(
} else {
(
quote!(
- #(#cfg_attrs)*
#vis #ident: &'__pin mut #ty,
),
quote!(
- #(#cfg_attrs)*
#ident: &mut #this.#ident,
),
)
@@ -373,7 +422,6 @@ fn generate_the_pin_data(
.iter()
.map(|f| {
let Field { vis, ident, ty, .. } = f.field;
- let cfg_attrs = &f.cfg_attrs;
let field_name = ident
.as_ref()
@@ -390,7 +438,6 @@ fn generate_the_pin_data(
/// - `(*slot).#field_name` is properly aligned.
/// - `(*slot).#field_name` points to uninitialized and exclusively accessed
/// memory.
- #(#cfg_attrs)*
// Allow `non_snake_case` since the same warning will be emitted on
// the struct definition.
#[allow(non_snake_case)]
@@ -421,6 +468,7 @@ fn generate_the_pin_data(
impl #impl_generics ::core::clone::Clone for __ThePinData #ty_generics
#whr
{
+ #[inline]
fn clone(&self) -> Self { *self }
}
@@ -429,7 +477,6 @@ fn generate_the_pin_data(
{}
#[allow(dead_code)] // Some functions might never be used and private.
- #[expect(clippy::missing_safety_doc)]
impl #impl_generics __ThePinData #ty_generics
#whr
{
@@ -453,6 +500,7 @@ fn generate_the_pin_data(
{
type PinData = __ThePinData #ty_generics;
+ #[inline]
unsafe fn __pin_data() -> Self::PinData {
__ThePinData { __phantom: ::pin_init::__internal::PhantomInvariant::new() }
}
diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs
index 56dc655e323e..8e9fd18b993f 100644
--- a/rust/pin-init/src/__internal.rs
+++ b/rust/pin-init/src/__internal.rs
@@ -105,6 +105,7 @@ pub unsafe trait HasInitData {
pub struct AllData<T: ?Sized>(PhantomInvariant<T>);
impl<T: ?Sized> Clone for AllData<T> {
+ #[inline]
fn clone(&self) -> Self {
*self
}
@@ -127,6 +128,7 @@ impl<T: ?Sized> AllData<T> {
unsafe impl<T: ?Sized> HasInitData for T {
type InitData = AllData<T>;
+ #[inline]
unsafe fn __init_data() -> Self::InitData {
AllData(PhantomInvariant::new())
}
@@ -181,7 +183,7 @@ impl<T> StackInit<T> {
unsafe { this.value.assume_init_drop() };
}
// SAFETY: The memory slot is valid and this type ensures that it will stay pinned.
- unsafe { init.__pinned_init(this.value.as_mut_ptr())? };
+ unsafe { init.__init(this.value.as_mut_ptr())? };
// INVARIANT: `this.value` is initialized above.
this.is_init = true;
// SAFETY: The slot is now pinned, since we will never give access to `&mut T`.
@@ -289,7 +291,7 @@ impl<T: ?Sized> Slot<Pinned, T> {
// - when `Err` is returned, we also propagate the error without touching `ptr`;
// also `self` is consumed so it cannot be touched further.
// - the drop guard will not hand out `&mut` (only `Pin<&mut T>`).
- unsafe { init.__pinned_init(self.ptr)? };
+ unsafe { init.__init(self.ptr)? };
// SAFETY:
// - `self.ptr` is valid, properly aligned and pinned per type invariant.
@@ -385,20 +387,23 @@ pub struct AlwaysFail<T: ?Sized> {
impl<T: ?Sized> AlwaysFail<T> {
/// Creates a new initializer that always fails.
+ #[inline]
pub fn new() -> Self {
Self { _t: PhantomData }
}
}
impl<T: ?Sized> Default for AlwaysFail<T> {
+ #[inline]
fn default() -> Self {
Self::new()
}
}
-// SAFETY: `__pinned_init` always fails, which is always okay.
+// SAFETY: `__init` always fails, which is always okay.
unsafe impl<T: ?Sized> PinInit<T, ()> for AlwaysFail<T> {
- unsafe fn __pinned_init(self, _slot: *mut T) -> Result<(), ()> {
+ #[inline]
+ unsafe fn __init(self, _slot: *mut T) -> Result<(), ()> {
Err(())
}
}
diff --git a/rust/pin-init/src/alloc.rs b/rust/pin-init/src/alloc.rs
index 5017f57442d8..471652e8663a 100644
--- a/rust/pin-init/src/alloc.rs
+++ b/rust/pin-init/src/alloc.rs
@@ -35,10 +35,11 @@ pub trait InPlaceInit<T>: Sized {
/// type.
///
/// If `T: !Unpin` it will not be able to move afterwards.
+ #[inline]
fn pin_init(init: impl PinInit<T>) -> Result<Pin<Self>, AllocError> {
// SAFETY: We delegate to `init` and only change the error type.
let init = unsafe {
- pin_init_from_closure(|slot| match init.__pinned_init(slot) {
+ pin_init_from_closure(|slot| match init.__init(slot) {
Ok(()) => Ok(()),
Err(i) => match i {},
})
@@ -52,6 +53,7 @@ pub trait InPlaceInit<T>: Sized {
E: From<AllocError>;
/// Use the given initializer to in-place initialize a `T`.
+ #[inline]
fn init(init: impl Init<T>) -> Result<Self, AllocError> {
// SAFETY: We delegate to `init` and only change the error type.
let init = unsafe {
@@ -109,7 +111,7 @@ impl<T> InPlaceInit<T> for Arc<T> {
let slot = slot.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
// slot is valid and will not be moved, because we pin it later.
- unsafe { init.__pinned_init(slot)? };
+ unsafe { init.__init(slot)? };
// SAFETY: All fields have been initialized and this is the only `Arc` to that data.
Ok(unsafe { Pin::new_unchecked(this.assume_init()) })
}
@@ -136,6 +138,7 @@ impl<T> InPlaceInit<T> for Arc<T> {
impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> {
type Initialized = Box<T>;
+ #[inline]
fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
let slot = self.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
@@ -145,11 +148,12 @@ impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> {
Ok(unsafe { self.assume_init() })
}
+ #[inline]
fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
let slot = self.as_mut_ptr();
// SAFETY: When init errors/panics, slot will get deallocated but not dropped,
// slot is valid and will not be moved, because we pin it later.
- unsafe { init.__pinned_init(slot)? };
+ unsafe { init.__init(slot)? };
// SAFETY: All fields have been initialized.
Ok(unsafe { self.assume_init() }.into())
}
diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs
index fd40c8f244a1..7600cdbbbf98 100644
--- a/rust/pin-init/src/lib.rs
+++ b/rust/pin-init/src/lib.rs
@@ -70,7 +70,6 @@
//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
//!
//! ```rust
-//! # #![expect(clippy::disallowed_names)]
//! # #![feature(allocator_api)]
//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
//! # use core::pin::Pin;
@@ -94,7 +93,6 @@
//! (or just the stack) to actually initialize a `Foo`:
//!
//! ```rust
-//! # #![expect(clippy::disallowed_names)]
//! # #![feature(allocator_api)]
//! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
//! # use core::{alloc::AllocError, pin::Pin};
@@ -456,7 +454,6 @@ pub use ::pin_init_internal::MaybeZeroable;
/// # Examples
///
/// ```rust
-/// # #![expect(clippy::disallowed_names)]
/// # #![feature(allocator_api)]
/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
/// # use pin_init::*;
@@ -508,7 +505,6 @@ macro_rules! stack_pin_init {
/// # Examples
///
/// ```rust
-/// # #![expect(clippy::disallowed_names)]
/// # #![feature(allocator_api)]
/// # #[path = "../examples/error.rs"] mod error; use error::Error;
/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
@@ -535,7 +531,6 @@ macro_rules! stack_pin_init {
/// ```
///
/// ```rust
-/// # #![expect(clippy::disallowed_names)]
/// # #![feature(allocator_api)]
/// # #[path = "../examples/error.rs"] mod error; use error::Error;
/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
@@ -658,7 +653,6 @@ macro_rules! stack_try_pin_init {
/// Users of `Foo` can now create it like this:
///
/// ```rust
-/// # #![expect(clippy::disallowed_names)]
/// # use pin_init::*;
/// # use core::pin::Pin;
/// # #[pin_data]
@@ -895,7 +889,7 @@ macro_rules! assert_pinned {
/// When implementing this trait you will need to take great care. Also there are probably very few
/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
///
-/// The [`PinInit::__pinned_init`] function:
+/// The [`PinInit::__init`] function:
/// - returns `Ok(())` if it initialized every field of `slot`,
/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
/// - `slot` can be deallocated without UB occurring,
@@ -915,15 +909,33 @@ macro_rules! assert_pinned {
#[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")]
#[must_use = "An initializer must be used in order to create its value."]
pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
+ /// Alias of [`PinInit::__init`].
+ ///
+ /// New code should use `__init` instead.
+ ///
+ /// # Safety
+ ///
+ /// Same as `__init`.
+ #[inline(always)]
+ #[cfg(not(kernel))]
+ #[deprecated = "use `raw_try_init` instead"]
+ unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
+ // SAFETY: Per safety requirement.
+ unsafe { self.__init(slot) }
+ }
+
/// Initializes `slot`.
///
+ /// It is not recommended to call this directly. Use [`raw_init`] or [`raw_try_init`].
+ ///
/// # Safety
///
/// - `slot` is a valid pointer to uninitialized memory.
/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
/// deallocate.
/// - `slot` will not move until it is dropped, i.e. it will be pinned.
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
+ /// If `Self: Init<T, E>`, this requirement is cancelled and it may be moved.
+ unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
/// First initializes the value using `self` then calls the function `f` with the initialized
/// value.
@@ -943,6 +955,7 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
/// Ok(())
/// });
/// ```
+ #[inline]
fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E>
where
F: FnOnce(Pin<&mut T>) -> Result<(), E>,
@@ -951,10 +964,38 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
}
}
+/// Initializes `slot` with an initializer.
+///
+/// # Safety
+///
+/// - `slot` is a valid pointer to uninitialized memory.
+/// - `slot` will not move until it is dropped, i.e. it will be pinned.
+/// If `init` implements `Init<T, E>`, this requirement is cancelled and it may be moved.
+#[inline(always)]
+pub unsafe fn raw_init<T>(slot: *mut T, init: impl PinInit<T>) {
+ // SAFETY: Per safety requirement.
+ unsafe { init.__init(slot).unwrap_or_else(|e| match e {}) }
+}
+
+/// Fallibly initializes `slot` with an initializer.
+///
+/// # Safety
+///
+/// - `slot` is a valid pointer to uninitialized memory.
+/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
+/// deallocate.
+/// - `slot` will not move until it is dropped, i.e. it will be pinned.
+/// If `init` implements `Init<T, E>`, this requirement is cancelled and it may be moved.
+#[inline(always)]
+pub unsafe fn raw_try_init<T, E>(slot: *mut T, init: impl PinInit<T, E>) -> Result<(), E> {
+ // SAFETY: Per safety requirement.
+ unsafe { init.__init(slot) }
+}
+
/// An initializer returned by [`PinInit::pin_chain`].
pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::PhantomInvariant<(E, T)>);
-// SAFETY: The `__pinned_init` function is implemented such that it
+// SAFETY: The `__init` function is implemented such that it
// - returns `Ok(())` on successful initialization,
// - returns `Err(err)` on error and in this case `slot` will be dropped.
// - considers `slot` pinned.
@@ -963,15 +1004,14 @@ where
I: PinInit<T, E>,
F: FnOnce(Pin<&mut T>) -> Result<(), E>,
{
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: All requirements fulfilled since this function is `__pinned_init`.
- unsafe { self.0.__pinned_init(slot)? };
- // SAFETY: The above call initialized `slot` and we still have unique access.
- let val = unsafe { &mut *slot };
- // SAFETY: `slot` is considered pinned.
- let val = unsafe { Pin::new_unchecked(val) };
- // SAFETY: `slot` was initialized above.
- (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) })
+ #[inline]
+ unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
+ // SAFETY: All requirements fulfilled since this function is `__init`.
+ let slot = unsafe { __internal::Slot::<__internal::Pinned, _>::new(slot) };
+ let mut guard = slot.init(self.0)?;
+ (self.1)(guard.let_binding())?;
+ core::mem::forget(guard);
+ Ok(())
}
}
@@ -988,19 +1028,8 @@ where
/// When implementing this trait you will need to take great care. Also there are probably very few
/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
///
-/// The [`Init::__init`] function:
-/// - returns `Ok(())` if it initialized every field of `slot`,
-/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
-/// - `slot` can be deallocated without UB occurring,
-/// - `slot` does not need to be dropped,
-/// - `slot` is not partially initialized.
-/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
-///
-/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
-/// code as `__init`.
-///
-/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to
-/// move the pointee after initialization.
+/// The [`PinInit::__init`] function must work without the pinning requirement; the caller is
+/// allowed to move the pointee after initialization.
///
#[cfg_attr(
kernel,
@@ -1014,15 +1043,6 @@ where
#[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")]
#[must_use = "An initializer must be used in order to create its value."]
pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
- /// Initializes `slot`.
- ///
- /// # Safety
- ///
- /// - `slot` is a valid pointer to uninitialized memory.
- /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
- /// deallocate.
- unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
-
/// First initializes the value using `self` then calls the function `f` with the initialized
/// value.
///
@@ -1031,7 +1051,6 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
/// # Examples
///
/// ```rust
- /// # #![expect(clippy::disallowed_names)]
/// use pin_init::{init, init_zeroed, Init};
///
/// struct Foo {
@@ -1051,6 +1070,7 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
/// Ok(())
/// });
/// ```
+ #[inline]
fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E>
where
F: FnOnce(&mut T) -> Result<(), E>,
@@ -1062,62 +1082,55 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
/// An initializer returned by [`Init::chain`].
pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::PhantomInvariant<(E, T)>);
-// SAFETY: The `__init` function is implemented such that it
-// - returns `Ok(())` on successful initialization,
-// - returns `Err(err)` on error and in this case `slot` will be dropped.
+// SAFETY: The `__init` function does not rely on the pinning requirement.
unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E>
where
I: Init<T, E>,
F: FnOnce(&mut T) -> Result<(), E>,
{
- unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: All requirements fulfilled since this function is `__init`.
- unsafe { self.0.__pinned_init(slot)? };
- // SAFETY: The above call initialized `slot` and we still have unique access.
- (self.1)(unsafe { &mut *slot }).inspect_err(|_|
- // SAFETY: `slot` was initialized above.
- unsafe { core::ptr::drop_in_place(slot) })
- }
}
-// SAFETY: `__pinned_init` behaves exactly the same as `__init`.
+// SAFETY: The `__init` function is implemented such that it
+// - returns `Ok(())` on successful initialization,
+// - returns `Err(err)` on error and in this case `slot` will be dropped.
unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E>
where
I: Init<T, E>,
F: FnOnce(&mut T) -> Result<(), E>,
{
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: `__init` has less strict requirements compared to `__pinned_init`.
- unsafe { self.__init(slot) }
+ #[inline]
+ unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
+ // SAFETY: All requirements fulfilled since this function is `__init`.
+ let slot = unsafe { __internal::Slot::<__internal::Unpinned, _>::new(slot) };
+ let mut guard = slot.init(self.0)?;
+ (self.1)(guard.let_binding())?;
+ core::mem::forget(guard);
+ Ok(())
}
}
/// Implement `PinInit` and `Init` for closures.
///
/// It is unsafe to create this type, since the closure needs to fulfill the same safety
-/// requirement as the `__pinned_init`/`__init` functions.
+/// requirement as the `__init` functions.
struct InitClosure<F, T: ?Sized>(F, __internal::PhantomInvariant<T>);
-// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
-// `__init` invariants.
-unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T>
-where
- F: FnOnce(*mut T) -> Result<(), E>,
+// SAFETY: When constructing via `init_from_closure`, the `__init` function does not rely on the
+// pinning requirement. When constructing via `pin_init_from_closure`, the opaque type prevents this
+// implementation from being visible.
+unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T> where
+ F: FnOnce(*mut T) -> Result<(), E>
{
- #[inline]
- unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- (self.0)(slot)
- }
}
// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
-// `__pinned_init` invariants.
+// `__init` invariants.
unsafe impl<T: ?Sized, F, E> PinInit<T, E> for InitClosure<F, T>
where
F: FnOnce(*mut T) -> Result<(), E>,
{
#[inline]
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
+ unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
(self.0)(slot)
}
}
@@ -1166,10 +1179,11 @@ pub const unsafe fn init_from_closure<T: ?Sized, E>(
///
/// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a
/// pointer must result in a valid `U`.
+#[inline]
pub const unsafe fn cast_pin_init<T, U, E>(init: impl PinInit<T, E>) -> impl PinInit<U, E> {
// SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety
// requirements.
- unsafe { pin_init_from_closure(|ptr: *mut U| init.__pinned_init(ptr.cast::<T>())) }
+ unsafe { pin_init_from_closure(|ptr: *mut U| init.__init(ptr.cast::<T>())) }
}
/// Changes the to be initialized type.
@@ -1178,6 +1192,7 @@ pub const unsafe fn cast_pin_init<T, U, E>(init: impl PinInit<T, E>) -> impl Pin
///
/// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a
/// pointer must result in a valid `U`.
+#[inline]
pub const unsafe fn cast_init<T, U, E>(init: impl Init<T, E>) -> impl Init<U, E> {
// SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety
// requirements.
@@ -1193,6 +1208,77 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
unsafe { init_from_closure(|_| Ok(())) }
}
+/// Array initializer from element initializer.
+struct ArrayInit<T: ?Sized, F>(F, __internal::PhantomInvariant<T>);
+
+// SAFETY: On success, all `N` elements of the array have been initialized. On error or panic, the
+// elements that have been initialized so far are dropped, thus leaving the array uninitialized and
+// ready to deallocate.
+unsafe impl<T, F, I, E, const N: usize> PinInit<[T; N], E> for ArrayInit<T, F>
+where
+ F: FnMut(usize) -> I,
+ I: PinInit<T, E>,
+{
+ unsafe fn __init(mut self, slot: *mut [T; N]) -> Result<(), E> {
+ /// # Invariants
+ ///
+ /// - `ptr[..num_init]` contains initialized elements of type `T`
+ /// - `ptr[num_init..N]` (where N is the size of the array) contains uninitialized memory
+ struct ArrayInitGuard<T> {
+ /// A pointer to the first element of the array.
+ ptr: *mut T,
+ /// The number of initialized elements in the array.
+ num_init: usize,
+ }
+
+ impl<T> Drop for ArrayInitGuard<T> {
+ #[inline]
+ fn drop(&mut self) {
+ // SAFETY: Per type invariant, `self.ptr[..self.num_init]` are initialized.
+ unsafe {
+ core::ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut(
+ self.ptr,
+ self.num_init,
+ ))
+ };
+ }
+ }
+
+ // INVARIANT: nothing is initialized yet.
+ let mut guard = ArrayInitGuard {
+ ptr: slot.cast::<T>(),
+ num_init: 0,
+ };
+
+ for i in 0..N {
+ // INVARIANT: Elements `self.ptr[..self.num_init]` have been initialized
+ // thus far. This holds true for every `self.num_init = i`.
+ guard.num_init = i;
+
+ let init = (self.0)(i);
+ // SAFETY:
+ // - The subslot is derived from `slot` with a valid offset.
+ // - If `Err` is touched, the subslot is not touched further, the guard will drop
+ // previously initialized elements only.
+ // - `slot` is pinned so is the subslot.
+ unsafe { init.__init(&raw mut (*slot)[i]) }?;
+ }
+
+ // Dismiss the drop guard now that all elements are initialized.
+ core::mem::forget(guard);
+ Ok(())
+ }
+}
+
+// SAFETY: `I: Init` cancels out the pinning requirement on subslots, which is the only place in the
+// `__init` function that relies on `slot` being pinned.
+unsafe impl<T, F, I, E, const N: usize> Init<[T; N], E> for ArrayInit<T, F>
+where
+ F: FnMut(usize) -> I,
+ I: Init<T, E>,
+{
+}
+
/// Initializes an array by initializing each element via the provided initializer.
///
/// # Examples
@@ -1203,32 +1289,14 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
/// let array: Box<[usize; 1_000]> = Box::init(init_array_from_fn(|i| i)).unwrap();
/// assert_eq!(array.len(), 1_000);
/// ```
+#[inline]
pub fn init_array_from_fn<I, const N: usize, T, E>(
- mut make_init: impl FnMut(usize) -> I,
+ make_init: impl FnMut(usize) -> I,
) -> impl Init<[T; N], E>
where
I: Init<T, E>,
{
- let init = move |slot: *mut [T; N]| {
- let slot = slot.cast::<T>();
- for i in 0..N {
- let init = make_init(i);
- // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
- let ptr = unsafe { slot.add(i) };
- // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
- // requirements.
- if let Err(e) = unsafe { init.__init(ptr) } {
- // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return
- // `Err` below, `slot` will be considered uninitialized memory.
- unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
- return Err(e);
- }
- }
- Ok(())
- };
- // SAFETY: The initializer above initializes every element of the array. On failure it drops
- // any initialized elements and returns `Err`.
- unsafe { init_from_closure(init) }
+ ArrayInit(make_init, __internal::PhantomInvariant::new())
}
/// Initializes an array by initializing each element via the provided initializer.
@@ -1246,32 +1314,14 @@ where
/// Arc::pin_init(pin_init_array_from_fn(|i| CMutex::new(i))).unwrap();
/// assert_eq!(array.len(), 1_000);
/// ```
+#[inline]
pub fn pin_init_array_from_fn<I, const N: usize, T, E>(
- mut make_init: impl FnMut(usize) -> I,
+ make_init: impl FnMut(usize) -> I,
) -> impl PinInit<[T; N], E>
where
I: PinInit<T, E>,
{
- let init = move |slot: *mut [T; N]| {
- let slot = slot.cast::<T>();
- for i in 0..N {
- let init = make_init(i);
- // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
- let ptr = unsafe { slot.add(i) };
- // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
- // requirements.
- if let Err(e) = unsafe { init.__pinned_init(ptr) } {
- // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return
- // `Err` below, `slot` will be considered uninitialized memory.
- unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
- return Err(e);
- }
- }
- Ok(())
- };
- // SAFETY: The initializer above initializes every element of the array. On failure it drops
- // any initialized elements and returns `Err`.
- unsafe { pin_init_from_closure(init) }
+ ArrayInit(make_init, __internal::PhantomInvariant::new())
}
/// Construct an initializer in a closure and run it.
@@ -1300,6 +1350,7 @@ where
/// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the
/// initializer itself will fail with that error. If it returned `Ok`, then it will run the
/// initializer returned by the [`pin_init!`] invocation.
+#[inline]
pub fn pin_init_scope<T, E, F, I>(make_init: F) -> impl PinInit<T, E>
where
F: FnOnce() -> Result<I, E>,
@@ -1307,13 +1358,13 @@ where
{
// SAFETY:
// - If `make_init` returns `Err`, `Err` is returned and `slot` is completely uninitialized,
- // - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__pinned_init`.
- // - The safety requirements of `init.__pinned_init` are fulfilled, since it's being called
- // from an initializer.
+ // - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__init`.
+ // - The safety requirements of `init.__init` are fulfilled, since it's being called from an
+ // initializer.
unsafe {
pin_init_from_closure(move |slot: *mut T| -> Result<(), E> {
let init = make_init()?;
- init.__pinned_init(slot)
+ init.__init(slot)
})
}
}
@@ -1343,6 +1394,7 @@ where
/// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the
/// initializer itself will fail with that error. If it returned `Ok`, then it will run the
/// initializer returned by the [`init!`] invocation.
+#[inline]
pub fn init_scope<T, E, F, I>(make_init: F) -> impl Init<T, E>
where
F: FnOnce() -> Result<I, E>,
@@ -1361,41 +1413,29 @@ where
}
}
-// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of `slot`.
-unsafe impl<T> Init<T> for T {
- unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> {
- // SAFETY: `slot` is valid for writes by the safety requirements of this function.
- unsafe { slot.write(self) };
- Ok(())
- }
-}
+// SAFETY: The `__init` function does not rely on slot being pinned after it returns.
+unsafe impl<T> Init<T> for T {}
-// SAFETY: the `__pinned_init` function always returns `Ok(())` and initializes every field of
+// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of
// `slot`. Additionally, all pinning invariants of `T` are upheld.
unsafe impl<T> PinInit<T> for T {
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), Infallible> {
+ #[inline]
+ unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> {
// SAFETY: `slot` is valid for writes by the safety requirements of this function.
unsafe { slot.write(self) };
Ok(())
}
}
-// SAFETY: when the `__init` function returns with
-// - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld.
-// - `Err(err)`, slot was not written to.
-unsafe impl<T, E> Init<T, E> for Result<T, E> {
- unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: `slot` is valid for writes by the safety requirements of this function.
- unsafe { slot.write(self?) };
- Ok(())
- }
-}
+// SAFETY: The `__init` function does not rely on slot being pinned after it returns.
+unsafe impl<T, E> Init<T, E> for Result<T, E> {}
-// SAFETY: when the `__pinned_init` function returns with
+// SAFETY: when the `__init` function returns with
// - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld.
// - `Err(err)`, slot was not written to.
unsafe impl<T, E> PinInit<T, E> for Result<T, E> {
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
+ #[inline]
+ unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
// SAFETY: `slot` is valid for writes by the safety requirements of this function.
unsafe { slot.write(self?) };
Ok(())
@@ -1421,6 +1461,7 @@ pub trait InPlaceWrite<T> {
impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> {
type Initialized = &'static mut T;
+ #[inline]
fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
let slot = self.as_mut_ptr();
@@ -1431,6 +1472,7 @@ impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> {
unsafe { Ok(self.assume_init_mut()) }
}
+ #[inline]
fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
let slot = self.as_mut_ptr();
@@ -1438,7 +1480,7 @@ impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> {
//
// The `'static` borrow guarantees the data will not be
// moved/invalidated until it gets dropped (which is never).
- unsafe { init.__pinned_init(slot)? };
+ unsafe { init.__init(slot)? };
// SAFETY: The above call initialized the memory.
Ok(Pin::static_mut(unsafe { self.assume_init_mut() }))
@@ -1510,10 +1552,13 @@ pub unsafe trait Zeroable {
/// Whenever a type implements [`Zeroable`], this function should be preferred over
/// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`.
///
+ /// As const traits are not yet stable, [`pin_init::zeroed()`] can be used instead
+ /// when initialization is required in a `const` context.
+ ///
/// # Examples
///
/// ```
- /// use pin_init::{Zeroable, zeroed};
+ /// use pin_init::Zeroable;
///
/// #[derive(Zeroable)]
/// struct Point {
@@ -1521,10 +1566,11 @@ pub unsafe trait Zeroable {
/// y: u32,
/// }
///
- /// let point: Point = zeroed();
+ /// let point: Point = Zeroable::zeroed();
/// assert_eq!(point.x, 0);
/// assert_eq!(point.y, 0);
/// ```
+ #[inline]
fn zeroed() -> Self
where
Self: Sized,
@@ -1553,6 +1599,9 @@ pub fn init_zeroed<T: Zeroable>() -> impl Init<T> {
/// Whenever a type implements [`Zeroable`], this function should be preferred over
/// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`.
///
+/// While const traits remain unstable, this function serves as the `const` version of
+/// [`Zeroable::zeroed()`].
+///
/// # Examples
///
/// ```
@@ -1568,6 +1617,7 @@ pub fn init_zeroed<T: Zeroable>() -> impl Init<T> {
/// assert_eq!(point.x, 0);
/// assert_eq!(point.y, 0);
/// ```
+#[inline]
pub const fn zeroed<T: Zeroable>() -> T {
// SAFETY:By the type invariants of `Zeroable`, all zeroes is a valid bit pattern for `T`.
unsafe { core::mem::zeroed() }
@@ -1728,6 +1778,7 @@ pub trait Wrapper<T> {
}
impl<T> Wrapper<T> for UnsafeCell<T> {
+ #[inline]
fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> {
// SAFETY: `UnsafeCell<T>` has a compatible layout to `T`.
unsafe { cast_pin_init(value_init) }
@@ -1735,6 +1786,7 @@ impl<T> Wrapper<T> for UnsafeCell<T> {
}
impl<T> Wrapper<T> for MaybeUninit<T> {
+ #[inline]
fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> {
// SAFETY: `MaybeUninit<T>` has a compatible layout to `T`.
unsafe { cast_pin_init(value_init) }
@@ -1743,6 +1795,7 @@ impl<T> Wrapper<T> for MaybeUninit<T> {
#[cfg(all(feature = "unsafe-pinned", CONFIG_RUSTC_HAS_UNSAFE_PINNED))]
impl<T> Wrapper<T> for core::pin::UnsafePinned<T> {
+ #[inline]
fn pin_init<E>(init: impl PinInit<T, E>) -> impl PinInit<Self, E> {
// SAFETY: `UnsafePinned<T>` has a compatible layout to `T`.
unsafe { cast_pin_init(init) }
generated by cgit v1.2.3 (git 2.0.4) at 2026-08-20 22:34:55 +0000