diff options
Diffstat (limited to 'rust/pin-init')
| -rw-r--r-- | rust/pin-init/README.md | 5 | ||||
| -rw-r--r-- | rust/pin-init/examples/error.rs | 2 | ||||
| -rw-r--r-- | rust/pin-init/examples/linked_list.rs | 1 | ||||
| -rw-r--r-- | rust/pin-init/examples/mutex.rs | 14 | ||||
| -rw-r--r-- | rust/pin-init/examples/pthread_mutex.rs | 3 | ||||
| -rw-r--r-- | rust/pin-init/examples/static_init.rs | 13 | ||||
| -rw-r--r-- | rust/pin-init/internal/src/diagnostics.rs | 14 | ||||
| -rw-r--r-- | rust/pin-init/internal/src/init.rs | 259 | ||||
| -rw-r--r-- | rust/pin-init/internal/src/lib.rs | 1 | ||||
| -rw-r--r-- | rust/pin-init/internal/src/pin_data.rs | 324 | ||||
| -rw-r--r-- | rust/pin-init/internal/src/zeroable.rs | 2 | ||||
| -rw-r--r-- | rust/pin-init/src/__internal.rs | 270 | ||||
| -rw-r--r-- | rust/pin-init/src/alloc.rs | 10 | ||||
| -rw-r--r-- | rust/pin-init/src/lib.rs | 477 |
14 files changed, 780 insertions, 615 deletions
diff --git a/rust/pin-init/README.md b/rust/pin-init/README.md index 6cee6ab1eb57..2312c9e75f8c 100644 --- a/rust/pin-init/README.md +++ b/rust/pin-init/README.md @@ -3,7 +3,7 @@ [](https://deps.rs/repo/github/Rust-for-Linux/pin-init)  [](#nightly-only) - + # `pin-init` > [!NOTE] @@ -160,7 +160,6 @@ actually does the initialization in the correct way. Here are the things to look ```rust use pin_init::{pin_data, pinned_drop, PinInit, PinnedDrop, pin_init_from_closure}; use core::{ - ptr::addr_of_mut, marker::PhantomPinned, cell::UnsafeCell, pin::Pin, @@ -199,7 +198,7 @@ impl RawFoo { unsafe { pin_init_from_closure(move |slot: *mut Self| { // `slot` contains uninit memory, avoid creating a reference. - let foo = addr_of_mut!((*slot).foo); + let foo = &raw mut (*slot).foo; let foo = UnsafeCell::raw_get(foo).cast::<bindings::foo>(); // Initialize the `foo` diff --git a/rust/pin-init/examples/error.rs b/rust/pin-init/examples/error.rs index 8f4e135eb8ba..96f095398e8d 100644 --- a/rust/pin-init/examples/error.rs +++ b/rust/pin-init/examples/error.rs @@ -11,6 +11,7 @@ use std::alloc::AllocError; pub struct Error; impl From<Infallible> for Error { + #[inline] fn from(e: Infallible) -> Self { match e {} } @@ -18,6 +19,7 @@ impl From<Infallible> for Error { #[cfg(feature = "alloc")] impl From<AllocError> for Error { + #[inline] fn from(_: AllocError) -> Self { Self } diff --git a/rust/pin-init/examples/linked_list.rs b/rust/pin-init/examples/linked_list.rs index 8445a5890cb7..424585fe226d 100644 --- a/rust/pin-init/examples/linked_list.rs +++ b/rust/pin-init/examples/linked_list.rs @@ -2,7 +2,6 @@ #![allow(clippy::undocumented_unsafe_blocks)] #![cfg_attr(feature = "alloc", feature(allocator_api))] -#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] use core::{ cell::Cell, diff --git a/rust/pin-init/examples/mutex.rs b/rust/pin-init/examples/mutex.rs index 9f295226cd64..e8d4dbb664fe 100644 --- a/rust/pin-init/examples/mutex.rs +++ b/rust/pin-init/examples/mutex.rs @@ -2,7 +2,6 @@ #![allow(clippy::undocumented_unsafe_blocks)] #![cfg_attr(feature = "alloc", feature(allocator_api))] -#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] #![allow(clippy::missing_safety_doc)] use core::{ @@ -80,11 +79,7 @@ impl<T> CMutex<T> { wait_list <- ListHead::new(), spin_lock: SpinLock::new(), locked: Cell::new(false), - data <- unsafe { - pin_init_from_closure(|slot: *mut UnsafeCell<T>| { - val.__pinned_init(slot.cast::<T>()) - }) - }, + data <- UnsafeCell::pin_init(val), }) } @@ -92,7 +87,7 @@ impl<T> CMutex<T> { pub fn lock(&self) -> Pin<CMutexGuard<'_, T>> { let mut sguard = self.spin_lock.acquire(); if self.locked.get() { - stack_pin_init!(let wait_entry = WaitEntry::insert_new(&self.wait_list)); + stack_pin_init!(let _wait_entry = WaitEntry::insert_new(&self.wait_list)); // println!("wait list length: {}", self.wait_list.size()); while self.locked.get() { drop(sguard); @@ -100,9 +95,6 @@ impl<T> CMutex<T> { thread::park(); sguard = self.spin_lock.acquire(); } - // This does have an effect, as the ListHead inside wait_entry implements Drop! - #[expect(clippy::drop_non_drop)] - drop(wait_entry); } self.locked.set(true); unsafe { @@ -219,7 +211,7 @@ fn main() { for h in handles { h.join().expect("thread panicked"); } - println!("{:?}", &*mtx.lock()); + println!("{:?}", *mtx.lock()); assert_eq!(*mtx.lock(), workload * thread_count * 2); } } diff --git a/rust/pin-init/examples/pthread_mutex.rs b/rust/pin-init/examples/pthread_mutex.rs index 4e082ec7d5de..00f457e68827 100644 --- a/rust/pin-init/examples/pthread_mutex.rs +++ b/rust/pin-init/examples/pthread_mutex.rs @@ -3,7 +3,6 @@ // inspired by <https://github.com/nbdd0121/pin-init/blob/trunk/examples/pthread_mutex.rs> #![allow(clippy::undocumented_unsafe_blocks)] #![cfg_attr(feature = "alloc", feature(allocator_api))] -#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] #[cfg(not(windows))] mod pthread_mtx { @@ -178,7 +177,7 @@ fn main() { for h in handles { h.join().expect("thread panicked"); } - println!("{:?}", &*mtx.lock()); + println!("{:?}", *mtx.lock()); assert_eq!(*mtx.lock(), workload * thread_count * 2); } } diff --git a/rust/pin-init/examples/static_init.rs b/rust/pin-init/examples/static_init.rs index 0e165daa9798..8dd52313c1b8 100644 --- a/rust/pin-init/examples/static_init.rs +++ b/rust/pin-init/examples/static_init.rs @@ -2,7 +2,6 @@ #![allow(clippy::undocumented_unsafe_blocks)] #![cfg_attr(feature = "alloc", feature(allocator_api))] -#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] #![allow(unused_imports)] use core::{ @@ -60,7 +59,7 @@ impl<T, I: PinInit<T>> ops::Deref for StaticInit<T, I> { println!("doing init"); let ptr = self.cell.get().cast::<T>(); match self.init.take() { - Some(f) => unsafe { f.__pinned_init(ptr).unwrap() }, + Some(f) => unsafe { pin_init::raw_init(ptr, f) }, None => unsafe { core::hint::unreachable_unchecked() }, } self.present.set(true); @@ -72,13 +71,11 @@ impl<T, I: PinInit<T>> ops::Deref for StaticInit<T, I> { pub struct CountInit; unsafe impl PinInit<CMutex<usize>> for CountInit { - unsafe fn __pinned_init( - self, - slot: *mut CMutex<usize>, - ) -> Result<(), core::convert::Infallible> { + unsafe fn __init(self, slot: *mut CMutex<usize>) -> Result<(), core::convert::Infallible> { let init = CMutex::new(0); std::thread::sleep(std::time::Duration::from_millis(1000)); - unsafe { init.__pinned_init(slot) } + unsafe { pin_init::raw_init(slot, init) }; + Ok(()) } } @@ -118,7 +115,7 @@ fn main() { for h in handles { h.join().expect("thread panicked"); } - println!("{:?}, {:?}", &*mtx.lock(), &*COUNT.lock()); + println!("{:?}, {:?}", *mtx.lock(), *COUNT.lock()); assert_eq!(*mtx.lock(), workload * thread_count * 2); } } diff --git a/rust/pin-init/internal/src/diagnostics.rs b/rust/pin-init/internal/src/diagnostics.rs index 3bdb477c2f2b..c7d9b3e624fc 100644 --- a/rust/pin-init/internal/src/diagnostics.rs +++ b/rust/pin-init/internal/src/diagnostics.rs @@ -3,6 +3,7 @@ use std::fmt::Display; use proc_macro2::TokenStream; +use quote::quote_spanned; use syn::{spanned::Spanned, Error}; pub(crate) struct DiagCtxt(TokenStream); @@ -15,6 +16,19 @@ impl DiagCtxt { ErrorGuaranteed(()) } + pub(crate) fn warn(&mut self, span: impl Spanned, msg: impl Display) { + // Have the message start on a new line for visual clarity. + let msg = format!("\n{}", msg); + self.0.extend(quote_spanned!(span.span() => + // Approximate using deprecated warning while `proc_macro_diagnostic` is unstable. + const _: () = { + #[deprecated = #msg] + const fn warn() {} + warn(); + }; + )); + } + pub(crate) fn with( fun: impl FnOnce(&mut DiagCtxt) -> Result<TokenStream, ErrorGuaranteed>, ) -> TokenStream { diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index 2fe918f4d82a..fd0b5ea4a0a3 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use proc_macro2::{Span, TokenStream}; -use quote::{format_ident, quote, quote_spanned}; +use quote::{format_ident, quote}; use syn::{ braced, parse::{End, Parse}, @@ -103,17 +103,15 @@ pub(crate) fn expand( |(_, err)| Box::new(err), ); let slot = format_ident!("slot"); - let (has_data_trait, data_trait, get_data, init_from_closure) = if pinned { + let (has_data_trait, get_data, init_from_closure) = if pinned { ( format_ident!("HasPinData"), - format_ident!("PinData"), format_ident!("__pin_data"), format_ident!("pin_init_from_closure"), ) } else { ( format_ident!("HasInitData"), - format_ident!("InitData"), format_ident!("__init_data"), format_ident!("init_from_closure"), ) @@ -157,8 +155,7 @@ pub(crate) fn expand( #path::#get_data() }; // Ensure that `#data` really is of type `#data` and help with type inference: - let init = ::pin_init::__internal::#data_trait::make_closure::<_, #error>( - #data, + let init = #data.__make_closure::<_, #error>( move |slot| { #zeroable_check #this @@ -172,8 +169,7 @@ pub(crate) fn expand( init(slot).map(|__InitOk| ()) }; // SAFETY: TODO - let init = unsafe { ::pin_init::#init_from_closure::<_, #error>(init) }; - init + unsafe { ::pin_init::#init_from_closure::<_, #error>(init) } }}) } @@ -232,123 +228,84 @@ fn init_fields( cfgs.retain(|attr| attr.path().is_ident("cfg")); cfgs }; + + let ident = match kind { + InitializerKind::Value { ident, .. } => ident, + InitializerKind::Init { ident, .. } => ident, + InitializerKind::Code { block, .. } => { + let stmt = &block.stmts; + res.extend(quote! { + #(#attrs)* + { + #(#stmt)* + } + }); + continue; + } + }; + + let slot = if pinned { + quote! { + // SAFETY: + // - `slot` is valid and properly aligned. + // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned. + // - `make_field_check` prevents `#ident` from being used twice, therefore + // `(*slot).#ident` is exclusively accessed and has not been initialized. + (unsafe { #data.#ident(#slot) }) + } + } else { + quote! { + // For `init!()` macro, everything is unpinned. + // SAFETY: + // - `&raw mut (*slot).#ident` is valid. + // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned. + // - `make_field_check` prevents `#ident` from being used twice, therefore + // `(*slot).#ident` is exclusively accessed and has not been initialized. + (unsafe { + ::pin_init::__internal::Slot::<::pin_init::__internal::Unpinned, _>::new( + &raw mut (*#slot).#ident + ) + }) + } + }; + + // `mixed_site` ensures that the guard is not accessible to the user-controlled code. + let guard = format_ident!("__{ident}_guard", span = Span::mixed_site()); + let init = match kind { InitializerKind::Value { ident, value } => { - let mut value_ident = ident.clone(); - let value_prep = value.as_ref().map(|value| &value.1).map(|value| { - // Setting the span of `value_ident` to `value`'s span improves error messages - // when the type of `value` is wrong. - value_ident.set_span(value.span()); - quote!(let #value_ident = #value;) - }); - // Again span for better diagnostics - let write = quote_spanned!(ident.span()=> ::core::ptr::write); - // NOTE: the field accessor ensures that the initialized field is properly aligned. - // Unaligned fields will cause the compiler to emit E0793. We do not support - // unaligned fields since `Init::__init` requires an aligned pointer; the call to - // `ptr::write` below has the same requirement. - let accessor = if pinned { - let project_ident = format_ident!("__project_{ident}"); - quote! { - // SAFETY: TODO - unsafe { #data.#project_ident(&mut (*#slot).#ident) } - } - } else { - quote! { - // SAFETY: TODO - unsafe { &mut (*#slot).#ident } - } - }; + let value = value + .as_ref() + .map(|(_, value)| quote!(#value)) + .unwrap_or_else(|| quote!(#ident)); + quote! { #(#attrs)* - { - #value_prep - // SAFETY: TODO - unsafe { #write(::core::ptr::addr_of_mut!((*#slot).#ident), #value_ident) }; - } - #(#cfgs)* - #[allow(unused_variables)] - let #ident = #accessor; + let mut #guard = #slot.write(#value); + } } - InitializerKind::Init { ident, value, .. } => { - // Again span for better diagnostics - let init = format_ident!("init", span = value.span()); - // NOTE: the field accessor ensures that the initialized field is properly aligned. - // Unaligned fields will cause the compiler to emit E0793. We do not support - // unaligned fields since `Init::__init` requires an aligned pointer; the call to - // `ptr::write` below has the same requirement. - let (value_init, accessor) = if pinned { - let project_ident = format_ident!("__project_{ident}"); - ( - quote! { - // SAFETY: - // - `slot` is valid, because we are inside of an initializer closure, we - // return when an error/panic occurs. - // - We also use `#data` to require the correct trait (`Init` or `PinInit`) - // for `#ident`. - unsafe { #data.#ident(::core::ptr::addr_of_mut!((*#slot).#ident), #init)? }; - }, - quote! { - // SAFETY: TODO - unsafe { #data.#project_ident(&mut (*#slot).#ident) } - }, - ) - } else { - ( - quote! { - // SAFETY: `slot` is valid, because we are inside of an initializer - // closure, we return when an error/panic occurs. - unsafe { - ::pin_init::Init::__init( - #init, - ::core::ptr::addr_of_mut!((*#slot).#ident), - )? - }; - }, - quote! { - // SAFETY: TODO - unsafe { &mut (*#slot).#ident } - }, - ) - }; + InitializerKind::Init { value, .. } => { quote! { #(#attrs)* - { - let #init = #value; - #value_init - } - #(#cfgs)* - #[allow(unused_variables)] - let #ident = #accessor; + let mut #guard = #slot.init(#value)?; } } - InitializerKind::Code { block: value, .. } => quote! { - #(#attrs)* - #[allow(unused_braces)] - #value - }, + InitializerKind::Code { .. } => unreachable!(), }; - res.extend(init); - if let Some(ident) = kind.ident() { - // `mixed_site` ensures that the guard is not accessible to the user-controlled code. - let guard = format_ident!("__{ident}_guard", span = Span::mixed_site()); - res.extend(quote! { - #(#cfgs)* - // Create the drop guard: - // - // We rely on macro hygiene to make it impossible for users to access this local - // variable. - // SAFETY: We forget the guard later when initialization has succeeded. - let #guard = unsafe { - ::pin_init::__internal::DropGuard::new( - ::core::ptr::addr_of_mut!((*slot).#ident) - ) - }; - }); - guards.push(guard); - guard_attrs.push(cfgs); - } + + res.extend(quote! { + #init + + #(#cfgs)* + // Allow `non_snake_case` since the same warning is going to be reported for the struct + // field. + #[allow(unused_variables, non_snake_case)] + let #ident = #guard.let_binding(); + }); + + guards.push(guard); + guard_attrs.push(cfgs); } quote! { #res @@ -361,49 +318,49 @@ fn init_fields( } } -/// Generate the check for ensuring that every field has been initialized. +/// Generate the check for ensuring that every field has been initialized and aligned. fn make_field_check( fields: &Punctuated<InitializerField, Token![,]>, init_kind: InitKind, path: &Path, ) -> TokenStream { - let field_attrs = fields + let field_attrs: Vec<_> = fields .iter() - .filter_map(|f| f.kind.ident().map(|_| &f.attrs)); - let field_name = fields.iter().filter_map(|f| f.kind.ident()); - match init_kind { - InitKind::Normal => quote! { - // We use unreachable code to ensure that all fields have been mentioned exactly once, - // this struct initializer will still be type-checked and complain with a very natural - // error message if a field is forgotten/mentioned more than once. - #[allow(unreachable_code, clippy::diverging_sub_expression)] - // SAFETY: this code is never executed. - let _ = || unsafe { - ::core::ptr::write(slot, #path { - #( - #(#field_attrs)* - #field_name: ::core::panic!(), - )* - }) - }; - }, - InitKind::Zeroing => quote! { - // We use unreachable code to ensure that all fields have been mentioned at most once. - // Since the user specified `..Zeroable::zeroed()` at the end, all missing fields will - // be zeroed. This struct initializer will still be type-checked and complain with a - // very natural error message if a field is mentioned more than once, or doesn't exist. - #[allow(unreachable_code, clippy::diverging_sub_expression, unused_assignments)] - // SAFETY: this code is never executed. - let _ = || unsafe { - ::core::ptr::write(slot, #path { - #( - #(#field_attrs)* - #field_name: ::core::panic!(), - )* - ..::core::mem::zeroed() - }) - }; - }, + .filter_map(|f| f.kind.ident().map(|_| &f.attrs)) + .collect(); + let field_name: Vec<_> = fields.iter().filter_map(|f| f.kind.ident()).collect(); + let zeroing_trailer = match init_kind { + InitKind::Normal => None, + InitKind::Zeroing => Some(quote! { + ..::core::mem::zeroed() + }), + }; + quote! { + #[allow(unreachable_code)] + // We use unreachable code to perform field checks. They're still checked by the compiler. + // SAFETY: this code is never executed. + let _ = || unsafe { + // Create references to ensure that the initialized field is properly aligned. + // Unaligned fields will cause the compiler to emit E0793. We do not support + // unaligned fields since `Init::__init` requires an aligned pointer; the call to + // `ptr::write` for value-initialization case has the same requirement. + #( + #(#field_attrs)* + let _ = &(*slot).#field_name; + )* + + // If the zeroing trailer is not present, this checks that all fields have been + // mentioned exactly once. If the zeroing trailer is present, all missing fields will be + // zeroed, so this checks that all fields have been mentioned at most once. The use of + // struct initializer will still generate very natural error messages for any misuse. + ::core::ptr::write(slot, #path { + #( + #(#field_attrs)* + #field_name: loop {}, + )* + #zeroing_trailer + }) + }; } } diff --git a/rust/pin-init/internal/src/lib.rs b/rust/pin-init/internal/src/lib.rs index 08372c8f65f0..60d5093f3128 100644 --- a/rust/pin-init/internal/src/lib.rs +++ b/rust/pin-init/internal/src/lib.rs @@ -6,7 +6,6 @@ //! `pin-init` proc macros. -#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] // Documentation is done in the pin-init crate instead. #![allow(missing_docs)] diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 7d871236b49c..ff194d27565e 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -1,13 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use proc_macro2::TokenStream; -use quote::{format_ident, quote}; +use quote::{format_ident, quote, ToTokens}; use syn::{ parse::{End, Nothing, Parse}, parse_quote, parse_quote_spanned, spanned::Spanned, visit_mut::VisitMut, - Field, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause, + Field, Fields, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause, }; use crate::diagnostics::{DiagCtxt, ErrorGuaranteed}; @@ -35,6 +35,20 @@ impl Parse for Args { } } +impl ToTokens for Args { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Self::Nothing(_) => (), + Self::PinnedDrop(kw) => kw.to_tokens(tokens), + } + } +} + +struct FieldInfo<'a> { + field: &'a Field, + pinned: bool, +} + pub(crate) fn pin_data( args: Args, input: Item, @@ -62,6 +76,55 @@ pub(crate) fn pin_data( } }; + // Handling cfg can gets very complicated, especially for tuple structs. Therefore, resolve all + // field cfgs first before continuing. + // + // We need to perform this after parsing so we can reliably detect field cfgs. + for (field_idx, field) in struct_.fields.iter_mut().enumerate() { + let cfg: Vec<_> = field + .attrs + .iter() + .filter(|a| a.path().is_ident("cfg")) + .map(|a| { + a.parse_args::<TokenStream>() + .expect("parse as token stream cannot fail") + }) + .collect(); + + if cfg.is_empty() { + continue; + } + + field.attrs.retain(|a| !a.path().is_ident("cfg")); + let cfg_true_struct = quote!(#struct_); + + let punctuated = match &mut struct_.fields { + Fields::Named(fields) => &mut fields.named, + Fields::Unnamed(fields) => &mut fields.unnamed, + Fields::Unit => unreachable!(), + }; + *punctuated = std::mem::take(punctuated) + .into_pairs() + .enumerate() + .filter(|&(i, _)| i != field_idx) + .map(|(_, p)| p) + .collect(); + let cfg_false_struct = quote!(#struct_); + + // Resolve one field at a time until we've got no more field cfgs. + // + // This is linear time because macro invocations with false cfg will not be expanded. + return Ok(quote!( + #[cfg(all(#(#cfg,)*))] + #[::pin_init::pin_data(#args)] + #cfg_true_struct + + #[cfg(not(all(#(#cfg,)*)))] + #[::pin_init::pin_data(#args)] + #cfg_false_struct + )); + } + // The generics might contain the `Self` type. Since this macro will define a new type with the // same generics and bounds, this poses a problem: `Self` will refer to the new type as opposed // to this struct definition. Therefore we have to replace `Self` with the concrete name. @@ -73,24 +136,38 @@ pub(crate) fn pin_data( replacer.visit_generics_mut(&mut struct_.generics); replacer.visit_fields_mut(&mut struct_.fields); - let fields: Vec<(bool, &Field)> = struct_ + let fields: Vec<FieldInfo<'_>> = struct_ .fields .iter_mut() .map(|field| { let len = field.attrs.len(); field.attrs.retain(|a| !a.path().is_ident("pin")); - (len != field.attrs.len(), &*field) + let pinned_count = len - field.attrs.len(); + if pinned_count > 1 { + dcx.error(&field, "#[pin] attribute specified more than once"); + } + + assert!( + !field.attrs.iter().any(|a| a.path().is_ident("cfg")), + "cfgs should be all resolved at this point" + ); + + FieldInfo { + field: &*field, + pinned: pinned_count != 0, + } }) .collect(); - for (pinned, field) in &fields { - if !pinned && is_phantom_pinned(&field.ty) { - dcx.error( - field, + for field in &fields { + let ident = field.field.ident.as_ref().unwrap(); + + if !field.pinned && is_phantom_pinned(&field.field.ty) { + dcx.warn( + field.field, format!( - "The field `{}` of type `PhantomPinned` only has an effect \ + "The field `{ident}` of type `PhantomPinned` only has an effect \ if it has the `#[pin]` attribute", - field.ident.as_ref().unwrap(), ), ); } @@ -143,7 +220,7 @@ fn is_phantom_pinned(ty: &Type) -> bool { fn generate_unpin_impl( ident: &Ident, generics: &Generics, - fields: &[(bool, &Field)], + fields: &[FieldInfo<'_>], ) -> TokenStream { let (_, ty_generics, _) = generics.split_for_impl(); let mut generics_with_pin_lt = generics.clone(); @@ -160,19 +237,26 @@ fn generate_unpin_impl( else { unreachable!() }; - let pinned_fields = fields.iter().filter_map(|(b, f)| b.then_some(f)); + let pinned_fields = fields.iter().filter(|f| f.pinned).map(|f| { + let ident = f.field.ident.as_ref().unwrap(); + let ty = &f.field.ty; + quote!( + #ident: #ty + ) + }); quote! { // This struct will be used for the unpin analysis. It is needed, because only structurally // pinned fields are relevant whether the struct should implement `Unpin`. - #[allow(dead_code)] // The fields below are never used. + #[allow( + dead_code, // The fields below are never used. + non_snake_case // The warning will be emitted on the struct definition. + )] struct __Unpin #generics_with_pin_lt #where_token #predicates { - __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, - __phantom: ::core::marker::PhantomData< - fn(#ident #ty_generics) -> #ident #ty_generics - >, + __phantom_pin: ::pin_init::__internal::PhantomInvariantLifetime<'__pin>, + __phantom: ::pin_init::__internal::PhantomInvariant<#ident #ty_generics>, #(#pinned_fields),* } @@ -214,7 +298,6 @@ fn generate_drop_impl(ident: &Ident, generics: &Generics, args: Args) -> TokenSt // `Drop`. Additionally we will implement this trait for the struct leading to a conflict, // if it also implements `Drop` trait MustNotImplDrop {} - #[expect(drop_bounds)] impl<T: ::core::ops::Drop + ?::core::marker::Sized> MustNotImplDrop for T {} impl #impl_generics MustNotImplDrop for #ident #ty_generics #whr @@ -222,7 +305,6 @@ fn generate_drop_impl(ident: &Ident, generics: &Generics, args: Args) -> TokenSt // We also take care to prevent users from writing a useless `PinnedDrop` implementation. // They might implement `PinnedDrop` correctly for the struct, but forget to give // `PinnedDrop` as the parameter to `#[pin_data]`. - #[expect(non_camel_case_types)] trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} impl<T: ::pin_init::PinnedDrop + ?::core::marker::Sized> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} @@ -238,7 +320,7 @@ fn generate_projections( vis: &Visibility, ident: &Ident, generics: &Generics, - fields: &[(bool, &Field)], + fields: &[FieldInfo<'_>], ) -> TokenStream { let (impl_generics, ty_generics, _) = generics.split_for_impl(); let mut generics_with_pin_lt = generics.clone(); @@ -247,32 +329,20 @@ fn generate_projections( let projection = format_ident!("{ident}Projection"); let this = format_ident!("this"); - let (fields_decl, fields_proj) = collect_tuple(fields.iter().map( - |( - pinned, - Field { - vis, - ident, - ty, - attrs, - .. - }, - )| { - let mut attrs = attrs.clone(); - attrs.retain(|a| !a.path().is_ident("pin")); - let mut no_doc_attrs = attrs.clone(); - no_doc_attrs.retain(|a| !a.path().is_ident("doc")); + let (fields_decl, fields_proj): (Vec<_>, Vec<_>) = fields + .iter() + .map(|field| { + let Field { vis, ident, ty, .. } = &field.field; + let ident = ident .as_ref() .expect("only structs with named fields are supported"); - if *pinned { + if field.pinned { ( quote!( - #(#attrs)* #vis #ident: ::core::pin::Pin<&'__pin mut #ty>, ), quote!( - #(#no_doc_attrs)* // SAFETY: this field is structurally pinned. #ident: unsafe { ::core::pin::Pin::new_unchecked(&mut #this.#ident) }, ), @@ -280,31 +350,33 @@ fn generate_projections( } else { ( quote!( - #(#attrs)* #vis #ident: &'__pin mut #ty, ), quote!( - #(#no_doc_attrs)* #ident: &mut #this.#ident, ), ) } - }, - )); + }) + .collect(); let structurally_pinned_fields_docs = fields .iter() - .filter_map(|(pinned, field)| pinned.then_some(field)) - .map(|Field { ident, .. }| format!(" - `{}`", ident.as_ref().unwrap())); + .filter(|f| f.pinned) + .map(|f| format!(" - `{}`", f.field.ident.as_ref().unwrap())); let not_structurally_pinned_fields_docs = fields .iter() - .filter_map(|(pinned, field)| (!pinned).then_some(field)) - .map(|Field { ident, .. }| format!(" - `{}`", ident.as_ref().unwrap())); + .filter(|f| !f.pinned) + .map(|f| format!(" - `{}`", f.field.ident.as_ref().unwrap())); let docs = format!(" Pin-projections of [`{ident}`]"); quote! { #[doc = #docs] - #[allow(dead_code)] + // Allow `non_snake_case` since the same warning will be emitted on + // the struct definition. + #[allow(dead_code, non_snake_case)] #[doc(hidden)] - #vis struct #projection #generics_with_pin_lt { + #vis struct #projection #generics_with_pin_lt + #whr + { #(#fields_decl)* ___pin_phantom_data: ::core::marker::PhantomData<&'__pin mut ()>, } @@ -336,91 +408,52 @@ fn generate_projections( fn generate_the_pin_data( vis: &Visibility, - ident: &Ident, + struct_name: &Ident, generics: &Generics, - fields: &[(bool, &Field)], + fields: &[FieldInfo<'_>], ) -> TokenStream { let (impl_generics, ty_generics, whr) = generics.split_for_impl(); // For every field, we create an initializing projection function according to its projection - // type. If a field is structurally pinned, then it must be initialized via `PinInit`, if it is - // not structurally pinned, then it can be initialized via `Init`. - // - // The functions are `unsafe` to prevent accidentally calling them. - fn handle_field( - Field { - vis, - ident, - ty, - attrs, - .. - }: &Field, - struct_ident: &Ident, - pinned: bool, - ) -> TokenStream { - let mut attrs = attrs.clone(); - attrs.retain(|a| !a.path().is_ident("pin")); - let ident = ident - .as_ref() - .expect("only structs with named fields are supported"); - let project_ident = format_ident!("__project_{ident}"); - let (init_ty, init_fn, project_ty, project_body, pin_safety) = if pinned { - ( - quote!(PinInit), - quote!(__pinned_init), - quote!(::core::pin::Pin<&'__slot mut #ty>), - // SAFETY: this field is structurally pinned. - quote!(unsafe { ::core::pin::Pin::new_unchecked(slot) }), - quote!( - /// - `slot` will not move until it is dropped, i.e. it will be pinned. - ), - ) - } else { - ( - quote!(Init), - quote!(__init), - quote!(&'__slot mut #ty), - quote!(slot), - quote!(), - ) - }; - let slot_safety = format!( - " `slot` points at the field `{ident}` inside of `{struct_ident}`, which is pinned.", - ); - quote! { - /// # Safety - /// - /// - `slot` is a valid pointer to uninitialized memory. - /// - the caller does not touch `slot` when `Err` is returned, they are only permitted - /// to deallocate. - #pin_safety - #(#attrs)* - #vis unsafe fn #ident<E>( - self, - slot: *mut #ty, - init: impl ::pin_init::#init_ty<#ty, E>, - ) -> ::core::result::Result<(), E> { - // SAFETY: this function has the same safety requirements as the __init function - // called below. - unsafe { ::pin_init::#init_ty::#init_fn(init, slot) } - } - - /// # Safety - /// - #[doc = #slot_safety] - #(#attrs)* - #vis unsafe fn #project_ident<'__slot>( - self, - slot: &'__slot mut #ty, - ) -> #project_ty { - #project_body - } - } - } - + // type. If a field is structurally pinned, we create a `Slot` with `Pinned` which must be + // initialized via `PinInit`; if it is not structurally pinned, then we create a `Slot` with + // `Unpinned` which allows initialization via `Init`. let field_accessors = fields .iter() - .map(|(pinned, field)| handle_field(field, ident, *pinned)) + .map(|f| { + let Field { vis, ident, ty, .. } = f.field; + + let field_name = ident + .as_ref() + .expect("only structs with named fields are supported"); + let pin_marker = if f.pinned { + quote!(Pinned) + } else { + quote!(Unpinned) + }; + quote! { + /// # Safety + /// + /// - `slot` is valid and properly aligned. + /// - `(*slot).#field_name` is properly aligned. + /// - `(*slot).#field_name` points to uninitialized and exclusively accessed + /// memory. + // Allow `non_snake_case` since the same warning will be emitted on + // the struct definition. + #[allow(non_snake_case)] + #[inline(always)] + #vis unsafe fn #field_name( + self, + slot: *mut #struct_name #ty_generics, + ) -> ::pin_init::__internal::Slot<::pin_init::__internal::#pin_marker, #ty> { + // SAFETY: + // - If `#pin_marker` is `Pinned`, the corresponding field is structurally + // pinned. + // - Other safety requirements follows the safety requirement. + unsafe { ::pin_init::__internal::Slot::new(&raw mut (*slot).#field_name) } + } + } + }) .collect::<TokenStream>(); quote! { // We declare this struct which will host all of the projection function for our type. It @@ -429,14 +462,13 @@ fn generate_the_pin_data( #vis struct __ThePinData #generics #whr { - __phantom: ::core::marker::PhantomData< - fn(#ident #ty_generics) -> #ident #ty_generics - >, + __phantom: ::pin_init::__internal::PhantomInvariant<#struct_name #ty_generics>, } impl #impl_generics ::core::clone::Clone for __ThePinData #ty_generics #whr { + #[inline] fn clone(&self) -> Self { *self } } @@ -445,31 +477,34 @@ fn generate_the_pin_data( {} #[allow(dead_code)] // Some functions might never be used and private. - #[expect(clippy::missing_safety_doc)] impl #impl_generics __ThePinData #ty_generics #whr { + /// Type inference helper function. + #[inline(always)] + #vis fn __make_closure<__F, __E>(self, f: __F) -> __F + where + __F: FnOnce(*mut #struct_name #ty_generics) -> + ::core::result::Result<::pin_init::__internal::InitOk, __E>, + { + f + } + #field_accessors } // SAFETY: We have added the correct projection functions above to `__ThePinData` and // we also use the least restrictive generics possible. - unsafe impl #impl_generics ::pin_init::__internal::HasPinData for #ident #ty_generics + unsafe impl #impl_generics ::pin_init::__internal::HasPinData for #struct_name #ty_generics #whr { type PinData = __ThePinData #ty_generics; + #[inline] unsafe fn __pin_data() -> Self::PinData { - __ThePinData { __phantom: ::core::marker::PhantomData } + __ThePinData { __phantom: ::pin_init::__internal::PhantomInvariant::new() } } } - - // SAFETY: TODO - unsafe impl #impl_generics ::pin_init::__internal::PinData for __ThePinData #ty_generics - #whr - { - type Datee = #ident #ty_generics; - } } } @@ -500,14 +535,3 @@ impl VisitMut for SelfReplacer { // Do not descend into items, since items reset/change what `Self` refers to. } } - -// replace with `.collect()` once MSRV is above 1.79 -fn collect_tuple<A, B>(iter: impl Iterator<Item = (A, B)>) -> (Vec<A>, Vec<B>) { - let mut res_a = vec![]; - let mut res_b = vec![]; - for (a, b) in iter { - res_a.push(a); - res_b.push(b); - } - (res_a, res_b) -} diff --git a/rust/pin-init/internal/src/zeroable.rs b/rust/pin-init/internal/src/zeroable.rs index 05683319b0f7..b11feaeb1ca6 100644 --- a/rust/pin-init/internal/src/zeroable.rs +++ b/rust/pin-init/internal/src/zeroable.rs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: Apache-2.0 OR MIT use proc_macro2::TokenStream; use quote::quote; diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index 90adbdc1893b..8e9fd18b993f 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -7,42 +7,54 @@ use super::*; -/// See the [nomicon] for what subtyping is. See also [this table]. +/// Zero-sized type used to mark a type as invariant. +/// +/// This is a polyfill for the [unstable type] in the standard library of the same name. /// -/// The reason for not using `PhantomData<*mut T>` is that that type never implements [`Send`] and -/// [`Sync`]. Hence `fn(*mut T) -> *mut T` is used, as that type always implements them. +/// See the [nomicon] for what subtyping is. See also [this table]. /// +/// [unstable type]: https://doc.rust-lang.org/nightly/std/marker/struct.PhantomInvariant.html /// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html /// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns -pub(crate) type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>; +#[repr(transparent)] +pub struct PhantomInvariant<T: ?Sized>(PhantomData<fn(T) -> T>); -/// Module-internal type implementing `PinInit` and `Init`. -/// -/// It is unsafe to create this type, since the closure needs to fulfill the same safety -/// requirement as the `__pinned_init`/`__init` functions. -pub(crate) struct InitClosure<F, T: ?Sized, E>(pub(crate) F, pub(crate) Invariant<(E, T)>); - -// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the -// `__init` invariants. -unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T, E> -where - F: FnOnce(*mut T) -> Result<(), E>, -{ - #[inline] - unsafe fn __init(self, slot: *mut T) -> Result<(), E> { - (self.0)(slot) +impl<T: ?Sized> Clone for PhantomInvariant<T> { + #[inline(always)] + fn clone(&self) -> Self { + *self } } -// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the -// `__pinned_init` invariants. -unsafe impl<T: ?Sized, F, E> PinInit<T, E> for InitClosure<F, T, E> -where - F: FnOnce(*mut T) -> Result<(), E>, -{ - #[inline] - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { - (self.0)(slot) +impl<T: ?Sized> Copy for PhantomInvariant<T> {} + +impl<T: ?Sized> Default for PhantomInvariant<T> { + #[inline(always)] + fn default() -> Self { + Self::new() + } +} + +impl<T: ?Sized> PhantomInvariant<T> { + #[inline(always)] + pub const fn new() -> Self { + Self(PhantomData) + } +} + +/// Zero-sized type used to mark a lifetime as invariant. +/// +/// This is a polyfill for the [unstable type] in the standard library of the same name. +/// +/// [unstable type]: https://doc.rust-lang.org/nightly/std/marker/struct.PhantomInvariantLifetime.html +#[repr(transparent)] +#[derive(Clone, Copy, Default)] +pub struct PhantomInvariantLifetime<'a>(PhantomInvariant<&'a ()>); + +impl PhantomInvariantLifetime<'_> { + #[inline(always)] + pub const fn new() -> Self { + Self(PhantomInvariant::new()) } } @@ -71,30 +83,12 @@ impl InitOk { /// /// Only the `init` module is allowed to use this trait. pub unsafe trait HasPinData { - type PinData: PinData; + type PinData; #[expect(clippy::missing_safety_doc)] unsafe fn __pin_data() -> Self::PinData; } -/// Marker trait for pinning data of structs. -/// -/// # Safety -/// -/// Only the `init` module is allowed to use this trait. -pub unsafe trait PinData: Copy { - type Datee: ?Sized + HasPinData; - - /// Type inference helper function. - #[inline(always)] - fn make_closure<F, E>(self, f: F) -> F - where - F: FnOnce(*mut Self::Datee) -> Result<InitOk, E>, - { - f - } -} - /// This trait is automatically implemented for every type. It aims to provide the same type /// inference help as `HasPinData`. /// @@ -102,33 +96,16 @@ pub unsafe trait PinData: Copy { /// /// Only the `init` module is allowed to use this trait. pub unsafe trait HasInitData { - type InitData: InitData; + type InitData; #[expect(clippy::missing_safety_doc)] unsafe fn __init_data() -> Self::InitData; } -/// Same function as `PinData`, but for arbitrary data. -/// -/// # Safety -/// -/// Only the `init` module is allowed to use this trait. -pub unsafe trait InitData: Copy { - type Datee: ?Sized + HasInitData; - - /// Type inference helper function. - #[inline(always)] - fn make_closure<F, E>(self, f: F) -> F - where - F: FnOnce(*mut Self::Datee) -> Result<InitOk, E>, - { - f - } -} - -pub struct AllData<T: ?Sized>(Invariant<T>); +pub struct AllData<T: ?Sized>(PhantomInvariant<T>); impl<T: ?Sized> Clone for AllData<T> { + #[inline] fn clone(&self) -> Self { *self } @@ -136,17 +113,24 @@ impl<T: ?Sized> Clone for AllData<T> { impl<T: ?Sized> Copy for AllData<T> {} -// SAFETY: TODO. -unsafe impl<T: ?Sized> InitData for AllData<T> { - type Datee = T; +impl<T: ?Sized> AllData<T> { + /// Type inference helper function. + #[inline(always)] + pub fn __make_closure<F, E>(self, f: F) -> F + where + F: FnOnce(*mut T) -> Result<InitOk, E>, + { + f + } } // SAFETY: TODO. unsafe impl<T: ?Sized> HasInitData for T { type InitData = AllData<T>; + #[inline] unsafe fn __init_data() -> Self::InitData { - AllData(PhantomData) + AllData(PhantomInvariant::new()) } } @@ -199,7 +183,7 @@ impl<T> StackInit<T> { unsafe { this.value.assume_init_drop() }; } // SAFETY: The memory slot is valid and this type ensures that it will stay pinned. - unsafe { init.__pinned_init(this.value.as_mut_ptr())? }; + unsafe { init.__init(this.value.as_mut_ptr())? }; // INVARIANT: `this.value` is initialized above. this.is_init = true; // SAFETY: The slot is now pinned, since we will never give access to `&mut T`. @@ -235,35 +219,144 @@ fn stack_init_reuse() { println!("{value:?}"); } +// Marker types that determines type of `DropGuard`'s let bindings. +pub struct Pinned; +pub struct Unpinned; + +/// Represent an uninitialized field. +/// +/// # Invariants +/// +/// - `ptr` is valid, properly aligned and points to uninitialized and exclusively accessed memory. +/// - If `P` is `Pinned`, then `ptr` is structurally pinned. +pub struct Slot<P, T: ?Sized> { + ptr: *mut T, + _phantom: PhantomData<P>, +} + +impl<P, T: ?Sized> Slot<P, T> { + /// # Safety + /// + /// - `ptr` is valid, properly aligned and points to uninitialized and exclusively accessed + /// memory. + /// - If `P` is `Pinned`, then `ptr` is structurally pinned. + #[inline(always)] + pub unsafe fn new(ptr: *mut T) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr, + _phantom: PhantomData, + } + } + + /// Initialize the field by value. + #[inline(always)] + pub fn write(self, value: T) -> DropGuard<P, T> + where + T: Sized, + { + // SAFETY: `self.ptr` is a valid and aligned pointer for write. + unsafe { self.ptr.write(value) } + // SAFETY: + // - `self.ptr` is valid and properly aligned per type invariant. + // - `*self.ptr` is initialized above and the ownership is transferred to the guard. + // - If `P` is `Pinned`, `self.ptr` is pinned. + unsafe { DropGuard::new(self.ptr) } + } +} + +impl<T: ?Sized> Slot<Unpinned, T> { + /// Initialize the field. + #[inline(always)] + pub fn init<E>(self, init: impl Init<T, E>) -> Result<DropGuard<Unpinned, T>, E> { + // SAFETY: + // - `self.ptr` is valid and properly aligned. + // - when `Err` is returned, we also propagate the error without touching `slot`; + // also `self` is consumed so it cannot be touched further. + unsafe { init.__init(self.ptr)? }; + + // SAFETY: + // - `self.ptr` is valid and properly aligned per type invariant. + // - `*self.ptr` is initialized above and the ownership is transferred to the guard. + Ok(unsafe { DropGuard::new(self.ptr) }) + } +} + +impl<T: ?Sized> Slot<Pinned, T> { + /// Initialize the field. + #[inline(always)] + pub fn init<E>(self, init: impl PinInit<T, E>) -> Result<DropGuard<Pinned, T>, E> { + // SAFETY: + // - `self.ptr` is valid and properly aligned. + // - when `Err` is returned, we also propagate the error without touching `ptr`; + // also `self` is consumed so it cannot be touched further. + // - the drop guard will not hand out `&mut` (only `Pin<&mut T>`). + unsafe { init.__init(self.ptr)? }; + + // SAFETY: + // - `self.ptr` is valid, properly aligned and pinned per type invariant. + // - `*self.ptr` is initialized above and the ownership is transferred to the guard. + Ok(unsafe { DropGuard::new(self.ptr) }) + } +} + /// When a value of this type is dropped, it drops a `T`. /// /// Can be forgotten to prevent the drop. -pub struct DropGuard<T: ?Sized> { +/// +/// # Invariants +/// +/// - `ptr` is valid and properly aligned. +/// - `*ptr` is initialized and owned by this guard. +/// - if `P` is `Pinned`, `ptr` is pinned. +pub struct DropGuard<P, T: ?Sized> { ptr: *mut T, + phantom: PhantomData<P>, } -impl<T: ?Sized> DropGuard<T> { - /// Creates a new [`DropGuard<T>`]. It will [`ptr::drop_in_place`] `ptr` when it gets dropped. +impl<P, T: ?Sized> DropGuard<P, T> { + /// Creates a drop guard and transfer the ownership of the pointer content. /// - /// # Safety + /// The ownership is only relinguished if the guard is forgotten via [`core::mem::forget`]. /// - /// `ptr` must be a valid pointer. + /// # Safety /// - /// It is the callers responsibility that `self` will only get dropped if the pointee of `ptr`: - /// - has not been dropped, - /// - is not accessible by any other means, - /// - will not be dropped by any other means. + /// - `ptr` is valid and properly aligned. + /// - `*ptr` is initialized, and the ownership is transferred to this guard. + /// - if `P` is `Pinned`, `ptr` is pinned. #[inline] pub unsafe fn new(ptr: *mut T) -> Self { - Self { ptr } + // INVARIANT: By safety requirement. + Self { + ptr, + phantom: PhantomData, + } } } -impl<T: ?Sized> Drop for DropGuard<T> { +impl<T: ?Sized> DropGuard<Unpinned, T> { + /// Create a let binding for accessor use. + #[inline] + pub fn let_binding(&mut self) -> &mut T { + // SAFETY: Per type invariant. + unsafe { &mut *self.ptr } + } +} + +impl<T: ?Sized> DropGuard<Pinned, T> { + /// Create a let binding for accessor use. + #[inline] + pub fn let_binding(&mut self) -> Pin<&mut T> { + // SAFETY: `self.ptr` is valid, properly aligned, initialized, exclusively accessible and + // pinned per type invariant. + unsafe { Pin::new_unchecked(&mut *self.ptr) } + } +} + +impl<P, T: ?Sized> Drop for DropGuard<P, T> { #[inline] fn drop(&mut self) { - // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function - // ensuring that this operation is safe. + // SAFETY: `self.ptr` is valid, properly aligned and `*self.ptr` is owned by this guard. unsafe { ptr::drop_in_place(self.ptr) } } } @@ -294,20 +387,23 @@ pub struct AlwaysFail<T: ?Sized> { impl<T: ?Sized> AlwaysFail<T> { /// Creates a new initializer that always fails. + #[inline] pub fn new() -> Self { Self { _t: PhantomData } } } impl<T: ?Sized> Default for AlwaysFail<T> { + #[inline] fn default() -> Self { Self::new() } } -// SAFETY: `__pinned_init` always fails, which is always okay. +// SAFETY: `__init` always fails, which is always okay. unsafe impl<T: ?Sized> PinInit<T, ()> for AlwaysFail<T> { - unsafe fn __pinned_init(self, _slot: *mut T) -> Result<(), ()> { + #[inline] + unsafe fn __init(self, _slot: *mut T) -> Result<(), ()> { Err(()) } } diff --git a/rust/pin-init/src/alloc.rs b/rust/pin-init/src/alloc.rs index 5017f57442d8..471652e8663a 100644 --- a/rust/pin-init/src/alloc.rs +++ b/rust/pin-init/src/alloc.rs @@ -35,10 +35,11 @@ pub trait InPlaceInit<T>: Sized { /// type. /// /// If `T: !Unpin` it will not be able to move afterwards. + #[inline] fn pin_init(init: impl PinInit<T>) -> Result<Pin<Self>, AllocError> { // SAFETY: We delegate to `init` and only change the error type. let init = unsafe { - pin_init_from_closure(|slot| match init.__pinned_init(slot) { + pin_init_from_closure(|slot| match init.__init(slot) { Ok(()) => Ok(()), Err(i) => match i {}, }) @@ -52,6 +53,7 @@ pub trait InPlaceInit<T>: Sized { E: From<AllocError>; /// Use the given initializer to in-place initialize a `T`. + #[inline] fn init(init: impl Init<T>) -> Result<Self, AllocError> { // SAFETY: We delegate to `init` and only change the error type. let init = unsafe { @@ -109,7 +111,7 @@ impl<T> InPlaceInit<T> for Arc<T> { let slot = slot.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; + unsafe { init.__init(slot)? }; // SAFETY: All fields have been initialized and this is the only `Arc` to that data. Ok(unsafe { Pin::new_unchecked(this.assume_init()) }) } @@ -136,6 +138,7 @@ impl<T> InPlaceInit<T> for Arc<T> { impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> { type Initialized = Box<T>; + #[inline] fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, @@ -145,11 +148,12 @@ impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> { Ok(unsafe { self.assume_init() }) } + #[inline] fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; + unsafe { init.__init(slot)? }; // SAFETY: All fields have been initialized. Ok(unsafe { self.assume_init() }.into()) } diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index fe4c85ae3f02..f1463be9479d 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -70,7 +70,6 @@ //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. //! //! ```rust -//! # #![expect(clippy::disallowed_names)] //! # #![feature(allocator_api)] //! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; //! # use core::pin::Pin; @@ -94,7 +93,6 @@ //! (or just the stack) to actually initialize a `Foo`: //! //! ```rust -//! # #![expect(clippy::disallowed_names)] //! # #![feature(allocator_api)] //! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; //! # use core::{alloc::AllocError, pin::Pin}; @@ -172,7 +170,6 @@ //! # #![feature(extern_types)] //! use pin_init::{pin_data, pinned_drop, PinInit, PinnedDrop, pin_init_from_closure}; //! use core::{ -//! ptr::addr_of_mut, //! marker::PhantomPinned, //! cell::UnsafeCell, //! pin::Pin, @@ -211,7 +208,7 @@ //! unsafe { //! pin_init_from_closure(move |slot: *mut Self| { //! // `slot` contains uninit memory, avoid creating a reference. -//! let foo = addr_of_mut!((*slot).foo); +//! let foo = &raw mut (*slot).foo; //! let foo = UnsafeCell::raw_get(foo).cast::<bindings::foo>(); //! //! // Initialize the `foo` @@ -264,14 +261,6 @@ //! [`impl Init<T, E>`]: crate::Init //! [Rust-for-Linux]: https://rust-for-linux.com/ -#![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] -#![cfg_attr( - all( - any(feature = "alloc", feature = "std"), - not(RUSTC_NEW_UNINIT_IS_STABLE) - ), - feature(new_uninit) -)] #![forbid(missing_docs, unsafe_op_in_unsafe_fn)] #![cfg_attr(not(feature = "std"), no_std)] #![cfg_attr(feature = "alloc", feature(allocator_api))] @@ -279,6 +268,8 @@ all(feature = "unsafe-pinned", CONFIG_RUSTC_HAS_UNSAFE_PINNED), feature(unsafe_pinned) )] +#![cfg_attr(all(USE_RUSTC_FEATURES, doc), allow(internal_features))] +#![cfg_attr(all(USE_RUSTC_FEATURES, doc), feature(rustdoc_internals))] use core::{ cell::UnsafeCell, @@ -438,7 +429,7 @@ pub use ::pin_init_internal::Zeroable; /// ``` /// use pin_init::MaybeZeroable; /// -/// // implmements `Zeroable` +/// // implements `Zeroable` /// #[derive(MaybeZeroable)] /// pub struct DriverData { /// pub(crate) id: i64, @@ -446,7 +437,7 @@ pub use ::pin_init_internal::Zeroable; /// len: usize, /// } /// -/// // does not implmement `Zeroable` +/// // does not implement `Zeroable` /// #[derive(MaybeZeroable)] /// pub struct DriverData2 { /// pub(crate) id: i64, @@ -463,7 +454,6 @@ pub use ::pin_init_internal::MaybeZeroable; /// # Examples /// /// ```rust -/// # #![expect(clippy::disallowed_names)] /// # #![feature(allocator_api)] /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; /// # use pin_init::*; @@ -500,13 +490,7 @@ macro_rules! stack_pin_init { (let $var:ident $(: $t:ty)? = $val:expr) => { let val = $val; let mut $var = ::core::pin::pin!($crate::__internal::StackInit$(::<$t>)?::uninit()); - let mut $var = match $crate::__internal::StackInit::init($var, val) { - Ok(res) => res, - Err(x) => { - let x: ::core::convert::Infallible = x; - match x {} - } - }; + let Ok(mut $var) = $crate::__internal::StackInit::init($var, val); }; } @@ -515,7 +499,6 @@ macro_rules! stack_pin_init { /// # Examples /// /// ```rust -/// # #![expect(clippy::disallowed_names)] /// # #![feature(allocator_api)] /// # #[path = "../examples/error.rs"] mod error; use error::Error; /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; @@ -542,7 +525,6 @@ macro_rules! stack_pin_init { /// ``` /// /// ```rust -/// # #![expect(clippy::disallowed_names)] /// # #![feature(allocator_api)] /// # #[path = "../examples/error.rs"] mod error; use error::Error; /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; @@ -665,7 +647,6 @@ macro_rules! stack_try_pin_init { /// Users of `Foo` can now create it like this: /// /// ```rust -/// # #![expect(clippy::disallowed_names)] /// # use pin_init::*; /// # use core::pin::Pin; /// # #[pin_data] @@ -755,7 +736,7 @@ macro_rules! stack_try_pin_init { /// /// ```rust /// # use pin_init::*; -/// # use core::{ptr::addr_of_mut, marker::PhantomPinned}; +/// # use core::marker::PhantomPinned; /// #[pin_data] /// #[derive(Zeroable)] /// struct Buf { @@ -769,7 +750,7 @@ macro_rules! stack_try_pin_init { /// let init = pin_init!(&this in Buf { /// buf: [0; 64], /// // SAFETY: TODO. -/// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() }, +/// ptr: unsafe { (&raw mut (*this.as_ptr()).buf).cast() }, /// pin: PhantomPinned, /// }); /// let init = pin_init!(Buf { @@ -874,12 +855,12 @@ pub use pin_init_internal::init; #[macro_export] macro_rules! assert_pinned { ($ty:ty, $field:ident, $field_ty:ty, inline) => { - let _ = move |ptr: *mut $field_ty| { - // SAFETY: This code is unreachable. - let data = unsafe { <$ty as $crate::__internal::HasPinData>::__pin_data() }; - let init = $crate::__internal::AlwaysFail::<$field_ty>::new(); - // SAFETY: This code is unreachable. - unsafe { data.$field(ptr, init) }.ok(); + // SAFETY: This code is unreachable. + let _ = move |ptr: *mut $ty| unsafe { + let data = <$ty as $crate::__internal::HasPinData>::__pin_data(); + _ = data + .$field(ptr) + .init($crate::__internal::AlwaysFail::<$field_ty>::new()); }; }; @@ -902,7 +883,7 @@ macro_rules! assert_pinned { /// When implementing this trait you will need to take great care. Also there are probably very few /// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. /// -/// The [`PinInit::__pinned_init`] function: +/// The [`PinInit::__init`] function: /// - returns `Ok(())` if it initialized every field of `slot`, /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: /// - `slot` can be deallocated without UB occurring, @@ -922,15 +903,33 @@ macro_rules! assert_pinned { #[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")] #[must_use = "An initializer must be used in order to create its value."] pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { + /// Alias of [`PinInit::__init`]. + /// + /// New code should use `__init` instead. + /// + /// # Safety + /// + /// Same as `__init`. + #[inline(always)] + #[cfg(not(kernel))] + #[deprecated = "use `raw_try_init` instead"] + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: Per safety requirement. + unsafe { self.__init(slot) } + } + /// Initializes `slot`. /// + /// It is not recommended to call this directly. Use [`raw_init`] or [`raw_try_init`]. + /// /// # Safety /// /// - `slot` is a valid pointer to uninitialized memory. /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to /// deallocate. /// - `slot` will not move until it is dropped, i.e. it will be pinned. - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>; + /// If `Self: Init<T, E>`, this requirement is cancelled and it may be moved. + unsafe fn __init(self, slot: *mut T) -> Result<(), E>; /// First initializes the value using `self` then calls the function `f` with the initialized /// value. @@ -950,18 +949,47 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { /// Ok(()) /// }); /// ``` + #[inline] fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E> where F: FnOnce(Pin<&mut T>) -> Result<(), E>, { - ChainPinInit(self, f, PhantomData) + ChainPinInit(self, f, __internal::PhantomInvariant::new()) } } +/// Initializes `slot` with an initializer. +/// +/// # Safety +/// +/// - `slot` is a valid pointer to uninitialized memory. +/// - `slot` will not move until it is dropped, i.e. it will be pinned. +/// If `init` implements `Init<T, E>`, this requirement is cancelled and it may be moved. +#[inline(always)] +pub unsafe fn raw_init<T>(slot: *mut T, init: impl PinInit<T>) { + // SAFETY: Per safety requirement. + unsafe { init.__init(slot).unwrap_or_else(|e| match e {}) } +} + +/// Fallibly initializes `slot` with an initializer. +/// +/// # Safety +/// +/// - `slot` is a valid pointer to uninitialized memory. +/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to +/// deallocate. +/// - `slot` will not move until it is dropped, i.e. it will be pinned. +/// If `init` implements `Init<T, E>`, this requirement is cancelled and it may be moved. +#[inline(always)] +pub unsafe fn raw_try_init<T, E>(slot: *mut T, init: impl PinInit<T, E>) -> Result<(), E> { + // SAFETY: Per safety requirement. + unsafe { init.__init(slot) } +} + /// An initializer returned by [`PinInit::pin_chain`]. -pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, T)>); +pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::PhantomInvariant<(E, T)>); -// SAFETY: The `__pinned_init` function is implemented such that it +// SAFETY: The `__init` function is implemented such that it // - returns `Ok(())` on successful initialization, // - returns `Err(err)` on error and in this case `slot` will be dropped. // - considers `slot` pinned. @@ -970,15 +998,14 @@ where I: PinInit<T, E>, F: FnOnce(Pin<&mut T>) -> Result<(), E>, { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: All requirements fulfilled since this function is `__pinned_init`. - unsafe { self.0.__pinned_init(slot)? }; - // SAFETY: The above call initialized `slot` and we still have unique access. - let val = unsafe { &mut *slot }; - // SAFETY: `slot` is considered pinned. - let val = unsafe { Pin::new_unchecked(val) }; - // SAFETY: `slot` was initialized above. - (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) }) + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__init`. + let slot = unsafe { __internal::Slot::<__internal::Pinned, _>::new(slot) }; + let mut guard = slot.init(self.0)?; + (self.1)(guard.let_binding())?; + core::mem::forget(guard); + Ok(()) } } @@ -995,19 +1022,8 @@ where /// When implementing this trait you will need to take great care. Also there are probably very few /// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. /// -/// The [`Init::__init`] function: -/// - returns `Ok(())` if it initialized every field of `slot`, -/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: -/// - `slot` can be deallocated without UB occurring, -/// - `slot` does not need to be dropped, -/// - `slot` is not partially initialized. -/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. -/// -/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same -/// code as `__init`. -/// -/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to -/// move the pointee after initialization. +/// The [`PinInit::__init`] function must work without the pinning requirement; the caller is +/// allowed to move the pointee after initialization. /// #[cfg_attr( kernel, @@ -1021,15 +1037,6 @@ where #[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")] #[must_use = "An initializer must be used in order to create its value."] pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { - /// Initializes `slot`. - /// - /// # Safety - /// - /// - `slot` is a valid pointer to uninitialized memory. - /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to - /// deallocate. - unsafe fn __init(self, slot: *mut T) -> Result<(), E>; - /// First initializes the value using `self` then calls the function `f` with the initialized /// value. /// @@ -1038,7 +1045,6 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { /// # Examples /// /// ```rust - /// # #![expect(clippy::disallowed_names)] /// use pin_init::{init, init_zeroed, Init}; /// /// struct Foo { @@ -1058,44 +1064,68 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { /// Ok(()) /// }); /// ``` + #[inline] fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E> where F: FnOnce(&mut T) -> Result<(), E>, { - ChainInit(self, f, PhantomData) + ChainInit(self, f, __internal::PhantomInvariant::new()) } } /// An initializer returned by [`Init::chain`]. -pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, T)>); +pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::PhantomInvariant<(E, T)>); + +// SAFETY: The `__init` function does not rely on the pinning requirement. +unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E> +where + I: Init<T, E>, + F: FnOnce(&mut T) -> Result<(), E>, +{ +} // SAFETY: The `__init` function is implemented such that it // - returns `Ok(())` on successful initialization, // - returns `Err(err)` on error and in this case `slot` will be dropped. -unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E> +unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E> where I: Init<T, E>, F: FnOnce(&mut T) -> Result<(), E>, { + #[inline] unsafe fn __init(self, slot: *mut T) -> Result<(), E> { // SAFETY: All requirements fulfilled since this function is `__init`. - unsafe { self.0.__pinned_init(slot)? }; - // SAFETY: The above call initialized `slot` and we still have unique access. - (self.1)(unsafe { &mut *slot }).inspect_err(|_| - // SAFETY: `slot` was initialized above. - unsafe { core::ptr::drop_in_place(slot) }) + let slot = unsafe { __internal::Slot::<__internal::Unpinned, _>::new(slot) }; + let mut guard = slot.init(self.0)?; + (self.1)(guard.let_binding())?; + core::mem::forget(guard); + Ok(()) } } -// SAFETY: `__pinned_init` behaves exactly the same as `__init`. -unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E> +/// Implement `PinInit` and `Init` for closures. +/// +/// It is unsafe to create this type, since the closure needs to fulfill the same safety +/// requirement as the `__init` functions. +struct InitClosure<F, T: ?Sized>(F, __internal::PhantomInvariant<T>); + +// SAFETY: When constructing via `init_from_closure`, the `__init` function does not rely on the +// pinning requirement. When constructing via `pin_init_from_closure`, the opaque type prevents this +// implementation from being visible. +unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T> where + F: FnOnce(*mut T) -> Result<(), E> +{ +} + +// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the +// `__init` invariants. +unsafe impl<T: ?Sized, F, E> PinInit<T, E> for InitClosure<F, T> where - I: Init<T, E>, - F: FnOnce(&mut T) -> Result<(), E>, + F: FnOnce(*mut T) -> Result<(), E>, { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: `__init` has less strict requirements compared to `__pinned_init`. - unsafe { self.__init(slot) } + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + (self.0)(slot) } } @@ -1115,7 +1145,7 @@ where pub const unsafe fn pin_init_from_closure<T: ?Sized, E>( f: impl FnOnce(*mut T) -> Result<(), E>, ) -> impl PinInit<T, E> { - __internal::InitClosure(f, PhantomData) + InitClosure(f, __internal::PhantomInvariant::new()) } /// Creates a new [`Init<T, E>`] from the given closure. @@ -1134,7 +1164,7 @@ pub const unsafe fn pin_init_from_closure<T: ?Sized, E>( pub const unsafe fn init_from_closure<T: ?Sized, E>( f: impl FnOnce(*mut T) -> Result<(), E>, ) -> impl Init<T, E> { - __internal::InitClosure(f, PhantomData) + InitClosure(f, __internal::PhantomInvariant::new()) } /// Changes the to be initialized type. @@ -1143,14 +1173,11 @@ pub const unsafe fn init_from_closure<T: ?Sized, E>( /// /// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a /// pointer must result in a valid `U`. +#[inline] pub const unsafe fn cast_pin_init<T, U, E>(init: impl PinInit<T, E>) -> impl PinInit<U, E> { // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety // requirements. - let res = unsafe { pin_init_from_closure(|ptr: *mut U| init.__pinned_init(ptr.cast::<T>())) }; - // FIXME: remove the let statement once the nightly-MSRV allows it (1.78 otherwise encounters a - // cycle when computing the type returned by this function) - #[allow(clippy::let_and_return)] - res + unsafe { pin_init_from_closure(|ptr: *mut U| init.__init(ptr.cast::<T>())) } } /// Changes the to be initialized type. @@ -1159,14 +1186,11 @@ pub const unsafe fn cast_pin_init<T, U, E>(init: impl PinInit<T, E>) -> impl Pin /// /// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a /// pointer must result in a valid `U`. +#[inline] pub const unsafe fn cast_init<T, U, E>(init: impl Init<T, E>) -> impl Init<U, E> { // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety // requirements. - let res = unsafe { init_from_closure(|ptr: *mut U| init.__init(ptr.cast::<T>())) }; - // FIXME: remove the let statement once the nightly-MSRV allows it (1.78 otherwise encounters a - // cycle when computing the type returned by this function) - #[allow(clippy::let_and_return)] - res + unsafe { init_from_closure(|ptr: *mut U| init.__init(ptr.cast::<T>())) } } /// An initializer that leaves the memory uninitialized. @@ -1178,6 +1202,77 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { unsafe { init_from_closure(|_| Ok(())) } } +/// Array initializer from element initializer. +struct ArrayInit<T: ?Sized, F>(F, __internal::PhantomInvariant<T>); + +// SAFETY: On success, all `N` elements of the array have been initialized. On error or panic, the +// elements that have been initialized so far are dropped, thus leaving the array uninitialized and +// ready to deallocate. +unsafe impl<T, F, I, E, const N: usize> PinInit<[T; N], E> for ArrayInit<T, F> +where + F: FnMut(usize) -> I, + I: PinInit<T, E>, +{ + unsafe fn __init(mut self, slot: *mut [T; N]) -> Result<(), E> { + /// # Invariants + /// + /// - `ptr[..num_init]` contains initialized elements of type `T` + /// - `ptr[num_init..N]` (where N is the size of the array) contains uninitialized memory + struct ArrayInitGuard<T> { + /// A pointer to the first element of the array. + ptr: *mut T, + /// The number of initialized elements in the array. + num_init: usize, + } + + impl<T> Drop for ArrayInitGuard<T> { + #[inline] + fn drop(&mut self) { + // SAFETY: Per type invariant, `self.ptr[..self.num_init]` are initialized. + unsafe { + core::ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut( + self.ptr, + self.num_init, + )) + }; + } + } + + // INVARIANT: nothing is initialized yet. + let mut guard = ArrayInitGuard { + ptr: slot.cast::<T>(), + num_init: 0, + }; + + for i in 0..N { + // INVARIANT: Elements `self.ptr[..self.num_init]` have been initialized + // thus far. This holds true for every `self.num_init = i`. + guard.num_init = i; + + let init = (self.0)(i); + // SAFETY: + // - The subslot is derived from `slot` with a valid offset. + // - If `Err` is touched, the subslot is not touched further, the guard will drop + // previously initialized elements only. + // - `slot` is pinned so is the subslot. + unsafe { init.__init(&raw mut (*slot)[i]) }?; + } + + // Dismiss the drop guard now that all elements are initialized. + core::mem::forget(guard); + Ok(()) + } +} + +// SAFETY: `I: Init` cancels out the pinning requirement on subslots, which is the only place in the +// `__init` function that relies on `slot` being pinned. +unsafe impl<T, F, I, E, const N: usize> Init<[T; N], E> for ArrayInit<T, F> +where + F: FnMut(usize) -> I, + I: Init<T, E>, +{ +} + /// Initializes an array by initializing each element via the provided initializer. /// /// # Examples @@ -1188,32 +1283,14 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { /// let array: Box<[usize; 1_000]> = Box::init(init_array_from_fn(|i| i)).unwrap(); /// assert_eq!(array.len(), 1_000); /// ``` +#[inline] pub fn init_array_from_fn<I, const N: usize, T, E>( - mut make_init: impl FnMut(usize) -> I, + make_init: impl FnMut(usize) -> I, ) -> impl Init<[T; N], E> where I: Init<T, E>, { - let init = move |slot: *mut [T; N]| { - let slot = slot.cast::<T>(); - for i in 0..N { - let init = make_init(i); - // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. - let ptr = unsafe { slot.add(i) }; - // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` - // requirements. - if let Err(e) = unsafe { init.__init(ptr) } { - // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return - // `Err` below, `slot` will be considered uninitialized memory. - unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; - return Err(e); - } - } - Ok(()) - }; - // SAFETY: The initializer above initializes every element of the array. On failure it drops - // any initialized elements and returns `Err`. - unsafe { init_from_closure(init) } + ArrayInit(make_init, __internal::PhantomInvariant::new()) } /// Initializes an array by initializing each element via the provided initializer. @@ -1231,32 +1308,14 @@ where /// Arc::pin_init(pin_init_array_from_fn(|i| CMutex::new(i))).unwrap(); /// assert_eq!(array.len(), 1_000); /// ``` +#[inline] pub fn pin_init_array_from_fn<I, const N: usize, T, E>( - mut make_init: impl FnMut(usize) -> I, + make_init: impl FnMut(usize) -> I, ) -> impl PinInit<[T; N], E> where I: PinInit<T, E>, { - let init = move |slot: *mut [T; N]| { - let slot = slot.cast::<T>(); - for i in 0..N { - let init = make_init(i); - // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. - let ptr = unsafe { slot.add(i) }; - // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` - // requirements. - if let Err(e) = unsafe { init.__pinned_init(ptr) } { - // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return - // `Err` below, `slot` will be considered uninitialized memory. - unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; - return Err(e); - } - } - Ok(()) - }; - // SAFETY: The initializer above initializes every element of the array. On failure it drops - // any initialized elements and returns `Err`. - unsafe { pin_init_from_closure(init) } + ArrayInit(make_init, __internal::PhantomInvariant::new()) } /// Construct an initializer in a closure and run it. @@ -1285,6 +1344,7 @@ where /// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the /// initializer itself will fail with that error. If it returned `Ok`, then it will run the /// initializer returned by the [`pin_init!`] invocation. +#[inline] pub fn pin_init_scope<T, E, F, I>(make_init: F) -> impl PinInit<T, E> where F: FnOnce() -> Result<I, E>, @@ -1292,13 +1352,13 @@ where { // SAFETY: // - If `make_init` returns `Err`, `Err` is returned and `slot` is completely uninitialized, - // - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__pinned_init`. - // - The safety requirements of `init.__pinned_init` are fulfilled, since it's being called - // from an initializer. + // - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__init`. + // - The safety requirements of `init.__init` are fulfilled, since it's being called from an + // initializer. unsafe { pin_init_from_closure(move |slot: *mut T| -> Result<(), E> { let init = make_init()?; - init.__pinned_init(slot) + init.__init(slot) }) } } @@ -1328,6 +1388,7 @@ where /// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the /// initializer itself will fail with that error. If it returned `Ok`, then it will run the /// initializer returned by the [`init!`] invocation. +#[inline] pub fn init_scope<T, E, F, I>(make_init: F) -> impl Init<T, E> where F: FnOnce() -> Result<I, E>, @@ -1346,41 +1407,29 @@ where } } -// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of `slot`. -unsafe impl<T> Init<T> for T { - unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> { - // SAFETY: `slot` is valid for writes by the safety requirements of this function. - unsafe { slot.write(self) }; - Ok(()) - } -} +// SAFETY: The `__init` function does not rely on slot being pinned after it returns. +unsafe impl<T> Init<T> for T {} -// SAFETY: the `__pinned_init` function always returns `Ok(())` and initializes every field of +// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of // `slot`. Additionally, all pinning invariants of `T` are upheld. unsafe impl<T> PinInit<T> for T { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), Infallible> { + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> { // SAFETY: `slot` is valid for writes by the safety requirements of this function. unsafe { slot.write(self) }; Ok(()) } } -// SAFETY: when the `__init` function returns with -// - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld. -// - `Err(err)`, slot was not written to. -unsafe impl<T, E> Init<T, E> for Result<T, E> { - unsafe fn __init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: `slot` is valid for writes by the safety requirements of this function. - unsafe { slot.write(self?) }; - Ok(()) - } -} +// SAFETY: The `__init` function does not rely on slot being pinned after it returns. +unsafe impl<T, E> Init<T, E> for Result<T, E> {} -// SAFETY: when the `__pinned_init` function returns with +// SAFETY: when the `__init` function returns with // - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld. // - `Err(err)`, slot was not written to. unsafe impl<T, E> PinInit<T, E> for Result<T, E> { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { // SAFETY: `slot` is valid for writes by the safety requirements of this function. unsafe { slot.write(self?) }; Ok(()) @@ -1406,6 +1455,7 @@ pub trait InPlaceWrite<T> { impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> { type Initialized = &'static mut T; + #[inline] fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { let slot = self.as_mut_ptr(); @@ -1416,6 +1466,7 @@ impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> { unsafe { Ok(self.assume_init_mut()) } } + #[inline] fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { let slot = self.as_mut_ptr(); @@ -1423,7 +1474,7 @@ impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> { // // The `'static` borrow guarantees the data will not be // moved/invalidated until it gets dropped (which is never). - unsafe { init.__pinned_init(slot)? }; + unsafe { init.__init(slot)? }; // SAFETY: The above call initialized the memory. Ok(Pin::static_mut(unsafe { self.assume_init_mut() })) @@ -1495,10 +1546,13 @@ pub unsafe trait Zeroable { /// Whenever a type implements [`Zeroable`], this function should be preferred over /// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`. /// + /// As const traits are not yet stable, [`pin_init::zeroed()`] can be used instead + /// when initialization is required in a `const` context. + /// /// # Examples /// /// ``` - /// use pin_init::{Zeroable, zeroed}; + /// use pin_init::Zeroable; /// /// #[derive(Zeroable)] /// struct Point { @@ -1506,10 +1560,11 @@ pub unsafe trait Zeroable { /// y: u32, /// } /// - /// let point: Point = zeroed(); + /// let point: Point = Zeroable::zeroed(); /// assert_eq!(point.x, 0); /// assert_eq!(point.y, 0); /// ``` + #[inline] fn zeroed() -> Self where Self: Sized, @@ -1518,27 +1573,6 @@ pub unsafe trait Zeroable { } } -/// Marker trait for types that allow `Option<Self>` to be set to all zeroes in order to write -/// `None` to that location. -/// -/// # Safety -/// -/// The implementer needs to ensure that `unsafe impl Zeroable for Option<Self> {}` is sound. -pub unsafe trait ZeroableOption {} - -// SAFETY: by the safety requirement of `ZeroableOption`, this is valid. -unsafe impl<T: ZeroableOption> Zeroable for Option<T> {} - -// SAFETY: `Option<&T>` is part of the option layout optimization guarantee: -// <https://doc.rust-lang.org/stable/std/option/index.html#representation>. -unsafe impl<T> ZeroableOption for &T {} -// SAFETY: `Option<&mut T>` is part of the option layout optimization guarantee: -// <https://doc.rust-lang.org/stable/std/option/index.html#representation>. -unsafe impl<T> ZeroableOption for &mut T {} -// SAFETY: `Option<NonNull<T>>` is part of the option layout optimization guarantee: -// <https://doc.rust-lang.org/stable/std/option/index.html#representation>. -unsafe impl<T> ZeroableOption for NonNull<T> {} - /// Create an initializer for a zeroed `T`. /// /// The returned initializer will write `0x00` to every byte of the given `slot`. @@ -1559,6 +1593,9 @@ pub fn init_zeroed<T: Zeroable>() -> impl Init<T> { /// Whenever a type implements [`Zeroable`], this function should be preferred over /// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`. /// +/// While const traits remain unstable, this function serves as the `const` version of +/// [`Zeroable::zeroed()`]. +/// /// # Examples /// /// ``` @@ -1574,6 +1611,7 @@ pub fn init_zeroed<T: Zeroable>() -> impl Init<T> { /// assert_eq!(point.x, 0); /// assert_eq!(point.y, 0); /// ``` +#[inline] pub const fn zeroed<T: Zeroable>() -> T { // SAFETY:By the type invariants of `Zeroable`, all zeroes is a valid bit pattern for `T`. unsafe { core::mem::zeroed() } @@ -1610,13 +1648,6 @@ impl_zeroable! { // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`. {<T: ?Sized + Zeroable>} UnsafeCell<T>, - // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee: - // <https://doc.rust-lang.org/stable/std/option/index.html#representation>). - Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>, - Option<NonZeroU128>, Option<NonZeroUsize>, - Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>, - Option<NonZeroI128>, Option<NonZeroIsize>, - // SAFETY: `null` pointer is valid. // // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be @@ -1635,8 +1666,14 @@ impl_zeroable! { } macro_rules! impl_tuple_zeroable { - ($(,)?) => {}; + ($first:ident, $(,)?) => { + #[cfg_attr(all(USE_RUSTC_FEATURES, doc), doc(fake_variadic))] + /// Implemented for tuples up to 10 items long. + // SAFETY: All elements are zeroable and padding can be zero. + unsafe impl<$first: Zeroable> Zeroable for ($first,) {} + }; ($first:ident, $($t:ident),* $(,)?) => { + #[cfg_attr(doc, doc(hidden))] // SAFETY: All elements are zeroable and padding can be zero. unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {} impl_tuple_zeroable!($($t),* ,); @@ -1645,13 +1682,33 @@ macro_rules! impl_tuple_zeroable { impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J); +/// Marker trait for types that allow `Option<Self>` to be set to all zeroes in order to write +/// `None` to that location. +/// +/// # Safety +/// +/// The implementer needs to ensure that `unsafe impl Zeroable for Option<Self> {}` is sound. +pub unsafe trait ZeroableOption {} + +// SAFETY: by the safety requirement of `ZeroableOption`, this is valid. +unsafe impl<T: ZeroableOption> Zeroable for Option<T> {} + macro_rules! impl_fn_zeroable_option { ([$($abi:literal),* $(,)?] $args:tt) => { $(impl_fn_zeroable_option!({extern $abi} $args);)* $(impl_fn_zeroable_option!({unsafe extern $abi} $args);)* }; ({$($prefix:tt)*} {$(,)?}) => {}; + ({$($prefix:tt)*} {$ret:ident, $arg:ident $(,)?}) => { + #[cfg_attr(all(USE_RUSTC_FEATURES, doc), doc(fake_variadic))] + /// Implemented for function pointers with up to 20 arity. + // SAFETY: function pointers are part of the option layout optimization: + // <https://doc.rust-lang.org/stable/std/option/index.html#representation>. + unsafe impl<$ret, $arg> ZeroableOption for $($prefix)* fn($arg) -> $ret {} + impl_fn_zeroable_option!({$($prefix)*} {$arg,}); + }; ({$($prefix:tt)*} {$ret:ident, $($rest:ident),* $(,)?}) => { + #[cfg_attr(doc, doc(hidden))] // SAFETY: function pointers are part of the option layout optimization: // <https://doc.rust-lang.org/stable/std/option/index.html#representation>. unsafe impl<$ret, $($rest),*> ZeroableOption for $($prefix)* fn($($rest),*) -> $ret {} @@ -1661,6 +1718,29 @@ macro_rules! impl_fn_zeroable_option { impl_fn_zeroable_option!(["Rust", "C"] { A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U }); +macro_rules! impl_zeroable_option { + ($($({$($generics:tt)*})? $t:ty, )*) => { + // SAFETY: Safety comments written in the macro invocation. + $(unsafe impl$($($generics)*)? ZeroableOption for $t {})* + }; +} + +impl_zeroable_option! { + // SAFETY: `Option<&T>` is part of the option layout optimization guarantee: + // <https://doc.rust-lang.org/stable/std/option/index.html#representation>. + {<T: ?Sized>} &T, + // SAFETY: `Option<&mut T>` is part of the option layout optimization guarantee: + // <https://doc.rust-lang.org/stable/std/option/index.html#representation>. + {<T: ?Sized>} &mut T, + // SAFETY: `Option<NonNull<T>>` is part of the option layout optimization guarantee: + // <https://doc.rust-lang.org/stable/std/option/index.html#representation>. + {<T: ?Sized>} NonNull<T>, + // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee: + // <https://doc.rust-lang.org/stable/std/option/index.html#representation>). + NonZero<u8>, NonZero<u16>, NonZero<u32>, NonZero<u64>, NonZero<u128>, NonZero<usize>, + NonZero<i8>, NonZero<i16>, NonZero<i32>, NonZero<i64>, NonZero<i128>, NonZero<isize>, +} + /// This trait allows creating an instance of `Self` which contains exactly one /// [structurally pinned value](https://doc.rust-lang.org/std/pin/index.html#projections-and-structural-pinning). /// @@ -1692,6 +1772,7 @@ pub trait Wrapper<T> { } impl<T> Wrapper<T> for UnsafeCell<T> { + #[inline] fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> { // SAFETY: `UnsafeCell<T>` has a compatible layout to `T`. unsafe { cast_pin_init(value_init) } @@ -1699,6 +1780,7 @@ impl<T> Wrapper<T> for UnsafeCell<T> { } impl<T> Wrapper<T> for MaybeUninit<T> { + #[inline] fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> { // SAFETY: `MaybeUninit<T>` has a compatible layout to `T`. unsafe { cast_pin_init(value_init) } @@ -1707,6 +1789,7 @@ impl<T> Wrapper<T> for MaybeUninit<T> { #[cfg(all(feature = "unsafe-pinned", CONFIG_RUSTC_HAS_UNSAFE_PINNED))] impl<T> Wrapper<T> for core::pin::UnsafePinned<T> { + #[inline] fn pin_init<E>(init: impl PinInit<T, E>) -> impl PinInit<Self, E> { // SAFETY: `UnsafePinned<T>` has a compatible layout to `T`. unsafe { cast_pin_init(init) } |