diff options
Diffstat (limited to 'drivers/gpu/nova-core')
50 files changed, 3349 insertions, 923 deletions
diff --git a/drivers/gpu/nova-core/Kconfig b/drivers/gpu/nova-core/Kconfig index a4f2380654e2..f918f69e0599 100644 --- a/drivers/gpu/nova-core/Kconfig +++ b/drivers/gpu/nova-core/Kconfig @@ -3,6 +3,7 @@ config NOVA_CORE depends on 64BIT depends on PCI depends on RUST + depends on !CPU_BIG_ENDIAN select AUXILIARY_BUS select RUST_FW_LOADER_ABSTRACTIONS default n @@ -13,4 +14,4 @@ config NOVA_CORE This driver is work in progress and may not be functional. - If M is selected, the module will be called nova_core. + If M is selected, the module will be called nova-core. diff --git a/drivers/gpu/nova-core/Makefile b/drivers/gpu/nova-core/Makefile index 2d78c50126e1..4ae544f808f4 100644 --- a/drivers/gpu/nova-core/Makefile +++ b/drivers/gpu/nova-core/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_NOVA_CORE) += nova_core.o +obj-$(CONFIG_NOVA_CORE) += nova-core.o +nova-core-y := nova_core.o diff --git a/drivers/gpu/nova-core/bitfield.rs b/drivers/gpu/nova-core/bitfield.rs index 02efdcf78d89..660c3911402d 100644 --- a/drivers/gpu/nova-core/bitfield.rs +++ b/drivers/gpu/nova-core/bitfield.rs @@ -170,7 +170,7 @@ macro_rules! bitfield { (@check_field_bounds $hi:tt:$lo:tt $field:ident as bool) => { #[allow(clippy::eq_op)] const _: () = { - ::kernel::build_assert!( + ::kernel::build_assert::build_assert!( $hi == $lo, concat!("boolean field `", stringify!($field), "` covers more than one bit") ); @@ -181,7 +181,7 @@ macro_rules! bitfield { (@check_field_bounds $hi:tt:$lo:tt $field:ident as $type:tt) => { #[allow(clippy::eq_op)] const _: () = { - ::kernel::build_assert!( + ::kernel::build_assert::build_assert!( $hi >= $lo, concat!("field `", stringify!($field), "`'s MSB is smaller than its LSB") ); diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 84b0e1703150..5738d4ac521b 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -3,9 +3,6 @@ use kernel::{ auxiliary, device::Core, - devres::Devres, - dma::Device, - dma::DmaMask, pci, pci::{ Class, @@ -14,13 +11,11 @@ use kernel::{ }, prelude::*, sizes::SZ_16M, - sync::{ - atomic::{ - Atomic, - Relaxed, // - }, - Arc, + sync::atomic::{ + Atomic, + Relaxed, // }, + types::ForLt, }; use crate::gpu::Gpu; @@ -29,29 +24,24 @@ use crate::gpu::Gpu; static AUXILIARY_ID_COUNTER: Atomic<u32> = Atomic::new(0); #[pin_data] -pub(crate) struct NovaCore { +pub(crate) struct NovaCore<'bound> { #[pin] - pub(crate) gpu: Gpu, - #[pin] - _reg: Devres<auxiliary::Registration>, + pub(crate) gpu: Gpu<'bound>, + bar: pci::Bar<'bound, BAR0_SIZE>, + #[allow(clippy::type_complexity)] + _reg: auxiliary::Registration<'bound, ForLt!(())>, } -const BAR0_SIZE: usize = SZ_16M; +pub(crate) struct NovaCoreDriver; -// For now we only support Ampere which can use up to 47-bit DMA addresses. -// -// TODO: Add an abstraction for this to support newer GPUs which may support -// larger DMA addresses. Limiting these GPUs to smaller address widths won't -// have any adverse affects, unless installed on systems which require larger -// DMA addresses. These systems should be quite rare. -const GPU_DMA_BITS: u32 = 47; +const BAR0_SIZE: usize = SZ_16M; -pub(crate) type Bar0 = pci::Bar<BAR0_SIZE>; +pub(crate) type Bar0<'a> = &'a pci::Bar<'a, BAR0_SIZE>; kernel::pci_device_table!( PCI_TABLE, MODULE_PCI_TABLE, - <NovaCore as pci::Driver>::IdInfo, + <NovaCoreDriver as pci::Driver>::IdInfo, [ // Modern NVIDIA GPUs will show up as either VGA or 3D controllers. ( @@ -73,42 +63,39 @@ kernel::pci_device_table!( ] ); -impl pci::Driver for NovaCore { +impl pci::Driver for NovaCoreDriver { type IdInfo = (); + type Data<'bound> = NovaCore<'bound>; const ID_TABLE: pci::IdTable<Self::IdInfo> = &PCI_TABLE; - fn probe(pdev: &pci::Device<Core>, _info: &Self::IdInfo) -> impl PinInit<Self, Error> { + fn probe<'bound>( + pdev: &'bound pci::Device<Core<'_>>, + _info: &'bound Self::IdInfo, + ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound { pin_init::pin_init_scope(move || { dev_dbg!(pdev, "Probe Nova Core GPU driver.\n"); pdev.enable_device_mem()?; pdev.set_master(); - // SAFETY: No concurrent DMA allocations or mappings can be made because - // the device is still being probed and therefore isn't being used by - // other threads of execution. - unsafe { pdev.dma_set_mask_and_coherent(DmaMask::new::<GPU_DMA_BITS>())? }; - - let bar = Arc::pin_init( - pdev.iomap_region_sized::<BAR0_SIZE>(0, c"nova-core/bar0"), - GFP_KERNEL, - )?; - - Ok(try_pin_init!(Self { - gpu <- Gpu::new(pdev, bar.clone(), bar.access(pdev.as_ref())?), - _reg <- auxiliary::Registration::new( + Ok(try_pin_init!(NovaCore { + bar: pdev.iomap_region_sized::<BAR0_SIZE>(0, c"nova-core/bar0")?, + // TODO: Use `&bar` self-referential pin-init syntax once available. + // + // SAFETY: `bar` is initialized before this expression is evaluated + // (`try_pin_init!()` initializes fields in declaration order), lives at a pinned + // stable address, and is dropped after `gpu` (struct field drop order). + gpu <- Gpu::new(pdev, unsafe { &*core::ptr::from_ref(bar) }), + _reg: auxiliary::Registration::new( pdev.as_ref(), c"nova-drm", // TODO[XARR]: Use XArray or perhaps IDA for proper ID allocation/recycling. For // now, use a simple atomic counter that never recycles IDs. AUXILIARY_ID_COUNTER.fetch_add(1, Relaxed), - crate::MODULE_NAME - ), + crate::MODULE_NAME, + (), + )?, })) }) } - - fn unbind(pdev: &pci::Device<Core>, this: Pin<&Self>) { - this.gpu.unbind(pdev.as_ref()); - } } diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 33927af4134c..94c7696a6493 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -40,6 +40,7 @@ use crate::{ regs, }; +pub(crate) mod fsp; pub(crate) mod gsp; mod hal; pub(crate) mod sec2; @@ -372,7 +373,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { } /// Resets DMA-related registers. - pub(crate) fn dma_reset(&self, bar: &Bar0) { + pub(crate) fn dma_reset(&self, bar: Bar0<'_>) { bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| { v.with_allow_phys_no_ctx(true) }); @@ -384,7 +385,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { } /// Reset the controller, select the falcon core, and wait for memory scrubbing to complete. - pub(crate) fn reset(&self, bar: &Bar0) -> Result { + pub(crate) fn reset(&self, bar: Bar0<'_>) -> Result { self.hal.reset_eng(bar)?; self.hal.select_core(self, bar)?; self.hal.reset_wait_mem_scrubbing(bar)?; @@ -403,7 +404,11 @@ impl<E: FalconEngine + 'static> Falcon<E> { /// Write a slice to Falcon IMEM memory using programmed I/O (PIO). /// /// Returns `EINVAL` if `img.len()` is not a multiple of 4. - fn pio_wr_imem_slice(&self, bar: &Bar0, load_offsets: FalconPioImemLoadTarget<'_>) -> Result { + fn pio_wr_imem_slice( + &self, + bar: Bar0<'_>, + load_offsets: FalconPioImemLoadTarget<'_>, + ) -> Result { // Rejecting misaligned images here allows us to avoid checking // inside the loops. if load_offsets.data.len() % 4 != 0 { @@ -440,7 +445,11 @@ impl<E: FalconEngine + 'static> Falcon<E> { /// Write a slice to Falcon DMEM memory using programmed I/O (PIO). /// /// Returns `EINVAL` if `img.len()` is not a multiple of 4. - fn pio_wr_dmem_slice(&self, bar: &Bar0, load_offsets: FalconPioDmemLoadTarget<'_>) -> Result { + fn pio_wr_dmem_slice( + &self, + bar: Bar0<'_>, + load_offsets: FalconPioDmemLoadTarget<'_>, + ) -> Result { // Rejecting misaligned images here allows us to avoid checking // inside the loops. if load_offsets.data.len() % 4 != 0 { @@ -468,7 +477,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { /// Perform a PIO copy into `IMEM` and `DMEM` of `fw`, and prepare the falcon to run it. pub(crate) fn pio_load<F: FalconFirmware<Target = E> + FalconPioLoadable>( &self, - bar: &Bar0, + bar: Bar0<'_>, fw: &F, ) -> Result { bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| { @@ -488,7 +497,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { } self.pio_wr_dmem_slice(bar, fw.dmem_load_params())?; - self.hal.program_brom(self, bar, &fw.brom_params())?; + self.hal.program_brom(self, bar, &fw.brom_params()); bar.write( WithBase::of::<E>(), @@ -504,7 +513,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { /// `sec` is set if the loaded firmware is expected to run in secure mode. fn dma_wr( &self, - bar: &Bar0, + bar: Bar0<'_>, dma_obj: &Coherent<[u8]>, target_mem: FalconMem, load_offsets: FalconDmaLoadTarget, @@ -611,7 +620,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { fn dma_load<F: FalconFirmware<Target = E> + FalconDmaLoadable>( &self, dev: &Device<device::Bound>, - bar: &Bar0, + bar: Bar0<'_>, fw: &F, ) -> Result { // DMA object with firmware content as the source of the DMA engine. @@ -647,7 +656,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { )?; self.dma_wr(bar, &dma_obj, FalconMem::Dmem, fw.dmem_load_params())?; - self.hal.program_brom(self, bar, &fw.brom_params())?; + self.hal.program_brom(self, bar, &fw.brom_params()); // Set `BootVec` to start of non-secure code. bar.write( @@ -659,7 +668,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { } /// Wait until the falcon CPU is halted. - pub(crate) fn wait_till_halted(&self, bar: &Bar0) -> Result<()> { + pub(crate) fn wait_till_halted(&self, bar: Bar0<'_>) -> Result<()> { // TIMEOUT: arbitrarily large value, firmwares should complete in less than 2 seconds. read_poll_timeout( || Ok(bar.read(regs::NV_PFALCON_FALCON_CPUCTL::of::<E>())), @@ -672,7 +681,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { } /// Start the falcon CPU. - pub(crate) fn start(&self, bar: &Bar0) -> Result<()> { + pub(crate) fn start(&self, bar: Bar0<'_>) -> Result<()> { match bar .read(regs::NV_PFALCON_FALCON_CPUCTL::of::<E>()) .alias_en() @@ -691,7 +700,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { } /// Writes values to the mailbox registers if provided. - pub(crate) fn write_mailboxes(&self, bar: &Bar0, mbox0: Option<u32>, mbox1: Option<u32>) { + pub(crate) fn write_mailboxes(&self, bar: Bar0<'_>, mbox0: Option<u32>, mbox1: Option<u32>) { if let Some(mbox0) = mbox0 { bar.write( WithBase::of::<E>(), @@ -708,19 +717,19 @@ impl<E: FalconEngine + 'static> Falcon<E> { } /// Reads the value from `mbox0` register. - pub(crate) fn read_mailbox0(&self, bar: &Bar0) -> u32 { + pub(crate) fn read_mailbox0(&self, bar: Bar0<'_>) -> u32 { bar.read(regs::NV_PFALCON_FALCON_MAILBOX0::of::<E>()) .value() } /// Reads the value from `mbox1` register. - pub(crate) fn read_mailbox1(&self, bar: &Bar0) -> u32 { + pub(crate) fn read_mailbox1(&self, bar: Bar0<'_>) -> u32 { bar.read(regs::NV_PFALCON_FALCON_MAILBOX1::of::<E>()) .value() } /// Reads values from both mailbox registers. - pub(crate) fn read_mailboxes(&self, bar: &Bar0) -> (u32, u32) { + pub(crate) fn read_mailboxes(&self, bar: Bar0<'_>) -> (u32, u32) { let mbox0 = self.read_mailbox0(bar); let mbox1 = self.read_mailbox1(bar); @@ -736,7 +745,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { /// the `MBOX0` and `MBOX1` registers. pub(crate) fn boot( &self, - bar: &Bar0, + bar: Bar0<'_>, mbox0: Option<u32>, mbox1: Option<u32>, ) -> Result<(u32, u32)> { @@ -750,7 +759,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { /// falcon instance. `engine_id_mask` and `ucode_id` are obtained from the firmware header. pub(crate) fn signature_reg_fuse_version( &self, - bar: &Bar0, + bar: Bar0<'_>, engine_id_mask: u16, ucode_id: u8, ) -> Result<u32> { @@ -761,7 +770,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { /// Check if the RISC-V core is active. /// /// Returns `true` if the RISC-V core is active, `false` otherwise. - pub(crate) fn is_riscv_active(&self, bar: &Bar0) -> bool { + pub(crate) fn is_riscv_active(&self, bar: Bar0<'_>) -> bool { self.hal.is_riscv_active(bar) } @@ -770,7 +779,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { pub(crate) fn load<F: FalconFirmware<Target = E> + FalconDmaLoadable>( &self, dev: &Device<device::Bound>, - bar: &Bar0, + bar: Bar0<'_>, fw: &F, ) -> Result { match self.hal.load_method() { @@ -780,7 +789,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { } /// Write the application version to the OS register. - pub(crate) fn write_os_version(&self, bar: &Bar0, app_version: u32) { + pub(crate) fn write_os_version(&self, bar: Bar0<'_>, app_version: u32) { bar.write( WithBase::of::<E>(), regs::NV_PFALCON_FALCON_OS::zeroed().with_value(app_version), diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs new file mode 100644 index 000000000000..52cdb84ef0e8 --- /dev/null +++ b/drivers/gpu/nova-core/falcon/fsp.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! FSP (Foundation Security Processor) falcon engine for Hopper/Blackwell GPUs. +//! +//! The FSP falcon handles secure boot and Chain of Trust operations +//! on Hopper and Blackwell architectures, replacing SEC2's role. + +use kernel::{ + io::{ + poll::read_poll_timeout, + register::{ + Array, + RegisterBase, + WithBase, // + }, + Io, // + }, + prelude::*, + time::Delta, +}; + +use crate::{ + driver::Bar0, + falcon::{ + Falcon, + FalconEngine, + PFalcon2Base, + PFalconBase, // + }, + num, + regs, // +}; + +/// FSP message timeout in milliseconds. +const FSP_MSG_TIMEOUT_MS: i64 = 2000; + +/// Type specifying the `Fsp` falcon engine. Cannot be instantiated. +pub(crate) struct Fsp(()); + +impl RegisterBase<PFalconBase> for Fsp { + const BASE: usize = 0x8f2000; +} + +impl RegisterBase<PFalcon2Base> for Fsp { + const BASE: usize = 0x8f3000; +} + +impl FalconEngine for Fsp {} + +impl Falcon<Fsp> { + /// Writes `data` to FSP external memory at offset `0`. + /// + /// `data` is interpreted as little-endian 32-bit words. Returns `EINVAL` + /// if the `data` length is not 4-byte aligned. + fn write_emem(&mut self, bar: Bar0<'_>, data: &[u8]) -> Result { + if data.len() % 4 != 0 { + return Err(EINVAL); + } + + // Begin a write burst at offset `0`, auto-incrementing on each write. + bar.write( + WithBase::of::<Fsp>(), + regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincw(true), + ); + + for chunk in data.chunks_exact(4) { + let value = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); + + // Write the next 32-bit `value`; hardware advances the offset. + bar.write( + WithBase::of::<Fsp>(), + regs::NV_PFALCON_FALCON_EMEMD::zeroed().with_data(value), + ); + } + + Ok(()) + } + + /// Reads FSP external memory from offset `0` into `data`. + /// + /// `data` is stored as little-endian 32-bit words. Returns `EINVAL` if + /// the `data` length is not 4-byte aligned. + fn read_emem(&mut self, bar: Bar0<'_>, data: &mut [u8]) -> Result { + if data.len() % 4 != 0 { + return Err(EINVAL); + } + + // Begin a read burst at offset `0`, auto-incrementing on each read. + bar.write( + WithBase::of::<Fsp>(), + regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincr(true), + ); + + for chunk in data.chunks_exact_mut(4) { + // Read the next 32-bit word; hardware advances the offset. + let value = bar.read(regs::NV_PFALCON_FALCON_EMEMD::of::<Fsp>()).data(); + chunk.copy_from_slice(&value.to_le_bytes()); + } + + Ok(()) + } + + /// Poll FSP for incoming data. + /// + /// Returns the size of available data in bytes, or 0 if no data is available. + /// + /// The FSP message queue is not circular. Pointers are reset to 0 after each + /// message exchange, so `tail >= head` is always true when data is present. + fn poll_msgq(&self, bar: Bar0<'_>) -> u32 { + let head = bar.read(regs::NV_PFSP_MSGQ_HEAD::at(0)).val(); + let tail = bar.read(regs::NV_PFSP_MSGQ_TAIL::at(0)).val(); + + if head == tail { + return 0; + } + + // TAIL points at last DWORD written, so add 4 to get total size. + tail.saturating_sub(head).saturating_add(4) + } + + /// Writes `packet` to FSP EMEM and updates the queue pointers to notify FSP. + /// + /// Returns `EINVAL` if `packet` is empty or its length is not 4-byte aligned. + pub(crate) fn send_msg(&mut self, bar: Bar0<'_>, packet: &[u8]) -> Result { + if packet.is_empty() { + return Err(EINVAL); + } + + self.write_emem(bar, packet)?; + + // Update queue pointers. TAIL points at the last DWORD written. + let tail_offset = u32::try_from(packet.len() - 4).map_err(|_| EINVAL)?; + bar.write( + Array::at(0), + regs::NV_PFSP_QUEUE_TAIL::zeroed().with_address(tail_offset), + ); + bar.write( + Array::at(0), + regs::NV_PFSP_QUEUE_HEAD::zeroed().with_address(0), + ); + + Ok(()) + } + + /// Reads the next message from FSP EMEM into a newly-allocated buffer and resets the queue + /// pointers. + /// + /// Returns `ETIMEDOUT` if no message was available until timeout, or a regular error code if a + /// memory allocation error occurred. + pub(crate) fn recv_msg(&mut self, bar: Bar0<'_>) -> Result<KVec<u8>> { + let msg_size = read_poll_timeout( + || Ok(self.poll_msgq(bar)), + |&size| size > 0, + Delta::from_millis(10), + Delta::from_millis(FSP_MSG_TIMEOUT_MS), + ) + .map(num::u32_as_usize)?; + + let mut buffer = KVec::<u8>::new(); + buffer.resize(msg_size, 0, GFP_KERNEL)?; + + self.read_emem(bar, &mut buffer)?; + + // Reset message queue pointers after reading. + bar.write(Array::at(0), regs::NV_PFSP_MSGQ_TAIL::zeroed().with_val(0)); + bar.write(Array::at(0), regs::NV_PFSP_MSGQ_HEAD::zeroed().with_val(0)); + + Ok(buffer) + } +} diff --git a/drivers/gpu/nova-core/falcon/gsp.rs b/drivers/gpu/nova-core/falcon/gsp.rs index df6d5a382c7a..d1f6f7fcffff 100644 --- a/drivers/gpu/nova-core/falcon/gsp.rs +++ b/drivers/gpu/nova-core/falcon/gsp.rs @@ -24,6 +24,10 @@ use crate::{ regs, }; +/// Pattern returned by GSP register reads while the PRIV target mask still blocks CPU access. +const GSP_TARGET_MASK_LOCKED_PATTERN: u32 = 0xbadf_4100; +const GSP_TARGET_MASK_LOCKED_MASK: u32 = 0xffff_ff00; + /// Type specifying the `Gsp` falcon engine. Cannot be instantiated. pub(crate) struct Gsp(()); @@ -40,7 +44,7 @@ impl FalconEngine for Gsp {} impl Falcon<Gsp> { /// Clears the SWGEN0 bit in the Falcon's IRQ status clear register to /// allow GSP to signal CPU for processing new messages in message queue. - pub(crate) fn clear_swgen0_intr(&self, bar: &Bar0) { + pub(crate) fn clear_swgen0_intr(&self, bar: Bar0<'_>) { bar.write( WithBase::of::<Gsp>(), regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true), @@ -48,7 +52,7 @@ impl Falcon<Gsp> { } /// Checks if GSP reload/resume has completed during the boot process. - pub(crate) fn check_reload_completed(&self, bar: &Bar0, timeout: Delta) -> Result<bool> { + pub(crate) fn check_reload_completed(&self, bar: Bar0<'_>, timeout: Delta) -> Result<bool> { read_poll_timeout( || Ok(bar.read(regs::NV_PGC6_BSI_SECURE_SCRATCH_14)), |val| val.boot_stage_3_handoff(), @@ -57,4 +61,19 @@ impl Falcon<Gsp> { ) .map(|_| true) } + + /// Returns whether the RISC-V branch privilege lockdown bit is set. + pub(crate) fn riscv_branch_privilege_lockdown(&self, bar: Bar0<'_>) -> bool { + bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<Gsp>()) + .riscv_br_priv_lockdown() + } + + /// Returns whether GSP registers can be read by the CPU. + pub(crate) fn priv_target_mask_released(&self, bar: Bar0<'_>) -> bool { + let hwcfg2 = bar + .read(regs::NV_PFALCON_FALCON_HWCFG2::of::<Gsp>()) + .into_raw(); + + hwcfg2 != 0 && (hwcfg2 & GSP_TARGET_MASK_LOCKED_MASK) != GSP_TARGET_MASK_LOCKED_PATTERN + } } diff --git a/drivers/gpu/nova-core/falcon/hal.rs b/drivers/gpu/nova-core/falcon/hal.rs index a7e5ea8d0272..89b56823906b 100644 --- a/drivers/gpu/nova-core/falcon/hal.rs +++ b/drivers/gpu/nova-core/falcon/hal.rs @@ -9,7 +9,10 @@ use crate::{ FalconBromParams, FalconEngine, // }, - gpu::Chipset, + gpu::{ + Architecture, + Chipset, // + }, }; mod ga102; @@ -31,7 +34,7 @@ pub(crate) enum LoadMethod { /// registers. pub(crate) trait FalconHal<E: FalconEngine>: Send + Sync { /// Activates the Falcon core if the engine is a risvc/falcon dual engine. - fn select_core(&self, _falcon: &Falcon<E>, _bar: &Bar0) -> Result { + fn select_core(&self, _falcon: &Falcon<E>, _bar: Bar0<'_>) -> Result { Ok(()) } @@ -40,23 +43,23 @@ pub(crate) trait FalconHal<E: FalconEngine>: Send + Sync { fn signature_reg_fuse_version( &self, falcon: &Falcon<E>, - bar: &Bar0, + bar: Bar0<'_>, engine_id_mask: u16, ucode_id: u8, ) -> Result<u32>; /// Program the boot ROM registers prior to starting a secure firmware. - fn program_brom(&self, falcon: &Falcon<E>, bar: &Bar0, params: &FalconBromParams) -> Result; + fn program_brom(&self, falcon: &Falcon<E>, bar: Bar0<'_>, params: &FalconBromParams); /// Check if the RISC-V core is active. /// Returns `true` if the RISC-V core is active, `false` otherwise. - fn is_riscv_active(&self, bar: &Bar0) -> bool; + fn is_riscv_active(&self, bar: Bar0<'_>) -> bool; /// Wait for memory scrubbing to complete. - fn reset_wait_mem_scrubbing(&self, bar: &Bar0) -> Result; + fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result; /// Reset the falcon engine. - fn reset_eng(&self, bar: &Bar0) -> Result; + fn reset_eng(&self, bar: Bar0<'_>) -> Result; /// Returns the method used to load data into the falcon's memory. /// @@ -74,16 +77,21 @@ pub(crate) trait FalconHal<E: FalconEngine>: Send + Sync { pub(super) fn falcon_hal<E: FalconEngine + 'static>( chipset: Chipset, ) -> Result<KBox<dyn FalconHal<E>>> { - use Chipset::*; - - let hal = match chipset { - TU102 | TU104 | TU106 | TU116 | TU117 => { + let hal = match chipset.arch() { + Architecture::Turing => { + KBox::new(tu102::Tu102::<E>::new(), GFP_KERNEL)? as KBox<dyn FalconHal<E>> + } + // GA100 boots like Turing so use Turing HAL + Architecture::Ampere if chipset == Chipset::GA100 => { KBox::new(tu102::Tu102::<E>::new(), GFP_KERNEL)? as KBox<dyn FalconHal<E>> } - GA102 | GA103 | GA104 | GA106 | GA107 | AD102 | AD103 | AD104 | AD106 | AD107 => { + Architecture::Ampere + | Architecture::Ada + | Architecture::Hopper + | Architecture::BlackwellGB10x + | Architecture::BlackwellGB20x => { KBox::new(ga102::Ga102::<E>::new(), GFP_KERNEL)? as KBox<dyn FalconHal<E>> } - _ => return Err(ENOTSUPP), }; Ok(hal) diff --git a/drivers/gpu/nova-core/falcon/hal/ga102.rs b/drivers/gpu/nova-core/falcon/hal/ga102.rs index 8368a61ddeef..cf6ce47e6b25 100644 --- a/drivers/gpu/nova-core/falcon/hal/ga102.rs +++ b/drivers/gpu/nova-core/falcon/hal/ga102.rs @@ -31,7 +31,7 @@ use crate::{ use super::FalconHal; -fn select_core_ga102<E: FalconEngine>(bar: &Bar0) -> Result { +fn select_core_ga102<E: FalconEngine>(bar: Bar0<'_>) -> Result { let bcr_ctrl = bar.read(regs::NV_PRISCV_RISCV_BCR_CTRL::of::<E>()); if bcr_ctrl.core_select() != PeregrineCoreSelect::Falcon { bar.write( @@ -53,7 +53,7 @@ fn select_core_ga102<E: FalconEngine>(bar: &Bar0) -> Result { fn signature_reg_fuse_version_ga102( dev: &device::Device, - bar: &Bar0, + bar: Bar0<'_>, engine_id_mask: u16, ucode_id: u8, ) -> Result<u32> { @@ -86,7 +86,7 @@ fn signature_reg_fuse_version_ga102( Ok(u16::BITS - reg_fuse_version.leading_zeros()) } -fn program_brom_ga102<E: FalconEngine>(bar: &Bar0, params: &FalconBromParams) -> Result { +fn program_brom_ga102<E: FalconEngine>(bar: Bar0<'_>, params: &FalconBromParams) { bar.write( WithBase::of::<E>().at(0), regs::NV_PFALCON2_FALCON_BROM_PARAADDR::zeroed().with_value(params.pkc_data_offset), @@ -104,8 +104,6 @@ fn program_brom_ga102<E: FalconEngine>(bar: &Bar0, params: &FalconBromParams) -> WithBase::of::<E>(), regs::NV_PFALCON2_FALCON_MOD_SEL::zeroed().with_algo(FalconModSelAlgo::Rsa3k), ); - - Ok(()) } pub(super) struct Ga102<E: FalconEngine>(PhantomData<E>); @@ -117,30 +115,30 @@ impl<E: FalconEngine> Ga102<E> { } impl<E: FalconEngine> FalconHal<E> for Ga102<E> { - fn select_core(&self, _falcon: &Falcon<E>, bar: &Bar0) -> Result { + fn select_core(&self, _falcon: &Falcon<E>, bar: Bar0<'_>) -> Result { select_core_ga102::<E>(bar) } fn signature_reg_fuse_version( &self, falcon: &Falcon<E>, - bar: &Bar0, + bar: Bar0<'_>, engine_id_mask: u16, ucode_id: u8, ) -> Result<u32> { signature_reg_fuse_version_ga102(&falcon.dev, bar, engine_id_mask, ucode_id) } - fn program_brom(&self, _falcon: &Falcon<E>, bar: &Bar0, params: &FalconBromParams) -> Result { - program_brom_ga102::<E>(bar, params) + fn program_brom(&self, _falcon: &Falcon<E>, bar: Bar0<'_>, params: &FalconBromParams) { + program_brom_ga102::<E>(bar, params); } - fn is_riscv_active(&self, bar: &Bar0) -> bool { + fn is_riscv_active(&self, bar: Bar0<'_>) -> bool { bar.read(regs::NV_PRISCV_RISCV_CPUCTL::of::<E>()) .active_stat() } - fn reset_wait_mem_scrubbing(&self, bar: &Bar0) -> Result { + fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result { // TIMEOUT: memory scrubbing should complete in less than 20ms. read_poll_timeout( || Ok(bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>())), @@ -151,7 +149,7 @@ impl<E: FalconEngine> FalconHal<E> for Ga102<E> { .map(|_| ()) } - fn reset_eng(&self, bar: &Bar0) -> Result { + fn reset_eng(&self, bar: Bar0<'_>) -> Result { let _ = bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>()); // According to OpenRM's `kflcnPreResetWait_GA102` documentation, HW sometimes does not set diff --git a/drivers/gpu/nova-core/falcon/hal/tu102.rs b/drivers/gpu/nova-core/falcon/hal/tu102.rs index c7a90266cb44..3aaee3869312 100644 --- a/drivers/gpu/nova-core/falcon/hal/tu102.rs +++ b/drivers/gpu/nova-core/falcon/hal/tu102.rs @@ -34,30 +34,28 @@ impl<E: FalconEngine> Tu102<E> { } impl<E: FalconEngine> FalconHal<E> for Tu102<E> { - fn select_core(&self, _falcon: &Falcon<E>, _bar: &Bar0) -> Result { + fn select_core(&self, _falcon: &Falcon<E>, _bar: Bar0<'_>) -> Result { Ok(()) } fn signature_reg_fuse_version( &self, _falcon: &Falcon<E>, - _bar: &Bar0, + _bar: Bar0<'_>, _engine_id_mask: u16, _ucode_id: u8, ) -> Result<u32> { Ok(0) } - fn program_brom(&self, _falcon: &Falcon<E>, _bar: &Bar0, _params: &FalconBromParams) -> Result { - Ok(()) - } + fn program_brom(&self, _falcon: &Falcon<E>, _bar: Bar0<'_>, _params: &FalconBromParams) {} - fn is_riscv_active(&self, bar: &Bar0) -> bool { + fn is_riscv_active(&self, bar: Bar0<'_>) -> bool { bar.read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS::of::<E>()) .active_stat() } - fn reset_wait_mem_scrubbing(&self, bar: &Bar0) -> Result { + fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result { // TIMEOUT: memory scrubbing should complete in less than 10ms. read_poll_timeout( || Ok(bar.read(regs::NV_PFALCON_FALCON_DMACTL::of::<E>())), @@ -68,7 +66,7 @@ impl<E: FalconEngine> FalconHal<E> for Tu102<E> { .map(|_| ()) } - fn reset_eng(&self, bar: &Bar0) -> Result { + fn reset_eng(&self, bar: Bar0<'_>) -> Result { regs::NV_PFALCON_FALCON_ENGINE::reset_engine::<E>(bar); self.reset_wait_mem_scrubbing(bar)?; diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index bdd5eed760e1..725e428154cf 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -15,8 +15,7 @@ use kernel::{ Alignable, Alignment, // }, - sizes::*, - sync::aref::ARef, // + sizes::*, // }; use crate::{ @@ -24,11 +23,8 @@ use crate::{ firmware::gsp::GspFirmware, gpu::Chipset, gsp, - num::{ - usize_as_u64, - FromSafeCast, // - }, - regs, + num::FromSafeCast, + regs, // }; mod hal; @@ -46,21 +42,20 @@ mod hal; /// Because of this, the sysmem flush memory page must be registered as early as possible during /// driver initialization, and before any falcon is reset. /// -/// Users are responsible for manually calling [`Self::unregister`] before dropping this object, -/// otherwise the GPU might still use it even after it has been freed. -pub(crate) struct SysmemFlush { +pub(crate) struct SysmemFlush<'sys> { /// Chipset we are operating on. chipset: Chipset, - device: ARef<device::Device>, + device: &'sys device::Device, + bar: Bar0<'sys>, /// Keep the page alive as long as we need it. page: CoherentHandle, } -impl SysmemFlush { +impl<'sys> SysmemFlush<'sys> { /// Allocate a memory page and register it as the sysmem flush page. pub(crate) fn register( - dev: &device::Device<device::Bound>, - bar: &Bar0, + dev: &'sys device::Device<device::Bound>, + bar: Bar0<'sys>, chipset: Chipset, ) -> Result<Self> { let page = CoherentHandle::alloc(dev, kernel::page::PAGE_SIZE, GFP_KERNEL)?; @@ -69,20 +64,19 @@ impl SysmemFlush { Ok(Self { chipset, - device: dev.into(), + device: dev, + bar, page, }) } +} - /// Unregister the managed sysmem flush page. - /// - /// In order to gracefully tear down the GPU, users must make sure to call this method before - /// dropping the object. - pub(crate) fn unregister(&self, bar: &Bar0) { +impl Drop for SysmemFlush<'_> { + fn drop(&mut self) { let hal = hal::fb_hal(self.chipset); - if hal.read_sysmem_flush_page(bar) == self.page.dma_handle() { - let _ = hal.write_sysmem_flush_page(bar, 0).inspect_err(|e| { + if hal.read_sysmem_flush_page(self.bar) == self.page.dma_handle() { + let _ = hal.write_sysmem_flush_page(self.bar, 0).inspect_err(|e| { dev_warn!( &self.device, "failed to unregister sysmem flush page: {:?}\n", @@ -127,8 +121,8 @@ impl fmt::Debug for FbRange { if f.alternate() { let size = self.len(); - if size < usize_as_u64(SZ_1M) { - let size_kib = size / usize_as_u64(SZ_1K); + if size < u64::SZ_1M { + let size_kib = size / u64::SZ_1K; f.write_fmt(fmt!( "{:#x}..{:#x} ({} KiB)", self.0.start, @@ -136,7 +130,7 @@ impl fmt::Debug for FbRange { size_kib )) } else { - let size_mib = size / usize_as_u64(SZ_1M); + let size_mib = size / u64::SZ_1M; f.write_fmt(fmt!( "{:#x}..{:#x} ({} MiB)", self.0.start, @@ -171,11 +165,13 @@ pub(crate) struct FbLayout { pub(crate) wpr2: FbRange, pub(crate) heap: FbRange, pub(crate) vf_partition_count: u8, + /// PMU reserved memory size, in bytes. + pub(crate) pmu_reserved_size: u32, } impl FbLayout { /// Computes the FB layout for `chipset` required to run the `gsp_fw` GSP firmware. - pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result<Self> { + pub(crate) fn new(chipset: Chipset, bar: Bar0<'_>, gsp_fw: &GspFirmware) -> Result<Self> { let hal = hal::fb_hal(chipset); let fb = { @@ -186,7 +182,7 @@ impl FbLayout { let vga_workspace = { let vga_base = { - const NV_PRAMIN_SIZE: u64 = usize_as_u64(SZ_1M); + const NV_PRAMIN_SIZE: u64 = u64::SZ_1M; let base = fb.end - NV_PRAMIN_SIZE; if hal.supports_display(bar) { @@ -196,7 +192,7 @@ impl FbLayout { { Some(addr) => { if addr < base { - const VBIOS_WORKSPACE_SIZE: u64 = usize_as_u64(SZ_128K); + const VBIOS_WORKSPACE_SIZE: u64 = u64::SZ_128K; // Point workspace address to end of framebuffer. fb.end - VBIOS_WORKSPACE_SIZE @@ -216,10 +212,10 @@ impl FbLayout { let frts = { const FRTS_DOWN_ALIGN: Alignment = Alignment::new::<SZ_128K>(); - const FRTS_SIZE: u64 = usize_as_u64(SZ_1M); - let frts_base = vga_workspace.start.align_down(FRTS_DOWN_ALIGN) - FRTS_SIZE; + let frts_size: u64 = hal.frts_size(); + let frts_base = vga_workspace.start.align_down(FRTS_DOWN_ALIGN) - frts_size; - FbRange(frts_base..frts_base + FRTS_SIZE) + FbRange(frts_base..frts_base + frts_size) }; let boot = { @@ -241,7 +237,7 @@ impl FbLayout { let wpr2_heap = { const WPR2_HEAP_DOWN_ALIGN: Alignment = Alignment::new::<SZ_1M>(); let wpr2_heap_size = - gsp::LibosParams::from_chipset(chipset).wpr_heap_size(chipset, fb.end); + gsp::LibosParams::from_chipset(chipset).wpr_heap_size(chipset, fb.end)?; let wpr2_heap_addr = (elf.start - wpr2_heap_size).align_down(WPR2_HEAP_DOWN_ALIGN); FbRange(wpr2_heap_addr..(elf.start).align_down(WPR2_HEAP_DOWN_ALIGN)) @@ -256,9 +252,8 @@ impl FbLayout { }; let heap = { - const HEAP_SIZE: u64 = usize_as_u64(SZ_1M); - - FbRange(wpr2.start - HEAP_SIZE..wpr2.start) + let heap_size = u64::from(hal.non_wpr_heap_size()); + FbRange(wpr2.start - heap_size..wpr2.start) }; Ok(Self { @@ -271,6 +266,7 @@ impl FbLayout { wpr2, heap, vf_partition_count: 0, + pmu_reserved_size: hal.pmu_reserved_size(), }) } } diff --git a/drivers/gpu/nova-core/fb/hal.rs b/drivers/gpu/nova-core/fb/hal.rs index aba0abd8ee00..714f0b51cd8f 100644 --- a/drivers/gpu/nova-core/fb/hal.rs +++ b/drivers/gpu/nova-core/fb/hal.rs @@ -1,41 +1,56 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::prelude::*; use crate::{ driver::Bar0, - gpu::Chipset, // + gpu::{ + Architecture, + Chipset, // + }, }; mod ga100; mod ga102; +mod gb100; +mod gb202; +mod gh100; mod tu102; pub(crate) trait FbHal { /// Returns the address of the currently-registered sysmem flush page. - fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64; + fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64; /// Register `addr` as the address of the sysmem flush page. /// /// This might fail if the address is too large for the receiving register. - fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result; + fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result; /// Returns `true` is display is supported. - fn supports_display(&self, bar: &Bar0) -> bool; + fn supports_display(&self, bar: Bar0<'_>) -> bool; /// Returns the VRAM size, in bytes. - fn vidmem_size(&self, bar: &Bar0) -> u64; + fn vidmem_size(&self, bar: Bar0<'_>) -> u64; + + /// Returns the amount of VRAM to reserve for the PMU. + fn pmu_reserved_size(&self) -> u32; + + /// Returns the non-WPR heap size for this chipset, in bytes. + fn non_wpr_heap_size(&self) -> u32; + + /// Returns the FRTS size, in bytes. + fn frts_size(&self) -> u64; } /// Returns the HAL corresponding to `chipset`. pub(super) fn fb_hal(chipset: Chipset) -> &'static dyn FbHal { - use Chipset::*; - - match chipset { - TU102 | TU104 | TU106 | TU117 | TU116 => tu102::TU102_HAL, - GA100 => ga100::GA100_HAL, - GA102 | GA103 | GA104 | GA106 | GA107 | AD102 | AD103 | AD104 | AD106 | AD107 => { - ga102::GA102_HAL - } + match chipset.arch() { + Architecture::Turing => tu102::TU102_HAL, + Architecture::Ampere if chipset == Chipset::GA100 => ga100::GA100_HAL, + Architecture::Ampere | Architecture::Ada => ga102::GA102_HAL, + Architecture::Hopper => gh100::GH100_HAL, + Architecture::BlackwellGB10x => gb100::GB100_HAL, + Architecture::BlackwellGB20x => gb202::GB202_HAL, } } diff --git a/drivers/gpu/nova-core/fb/hal/ga100.rs b/drivers/gpu/nova-core/fb/hal/ga100.rs index 1c03783cddef..3cc1caf361c7 100644 --- a/drivers/gpu/nova-core/fb/hal/ga100.rs +++ b/drivers/gpu/nova-core/fb/hal/ga100.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::{ io::Io, @@ -16,13 +17,13 @@ use super::tu102::FLUSH_SYSMEM_ADDR_SHIFT; struct Ga100; -pub(super) fn read_sysmem_flush_page_ga100(bar: &Bar0) -> u64 { +pub(super) fn read_sysmem_flush_page_ga100(bar: Bar0<'_>) -> u64 { u64::from(bar.read(regs::NV_PFB_NISO_FLUSH_SYSMEM_ADDR).adr_39_08()) << FLUSH_SYSMEM_ADDR_SHIFT | u64::from(bar.read(regs::NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI).adr_63_40()) << FLUSH_SYSMEM_ADDR_SHIFT_HI } -pub(super) fn write_sysmem_flush_page_ga100(bar: &Bar0, addr: u64) { +pub(super) fn write_sysmem_flush_page_ga100(bar: Bar0<'_>, addr: u64) { bar.write_reg( regs::NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr_63_40( Bounded::<u64, _>::from(addr) @@ -39,7 +40,7 @@ pub(super) fn write_sysmem_flush_page_ga100(bar: &Bar0, addr: u64) { ); } -pub(super) fn display_enabled_ga100(bar: &Bar0) -> bool { +pub(super) fn display_enabled_ga100(bar: Bar0<'_>) -> bool { !bar.read(regs::ga100::NV_FUSE_STATUS_OPT_DISPLAY) .display_disabled() } @@ -49,23 +50,37 @@ pub(super) fn display_enabled_ga100(bar: &Bar0) -> bool { const FLUSH_SYSMEM_ADDR_SHIFT_HI: u32 = 40; impl FbHal for Ga100 { - fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64 { + fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { read_sysmem_flush_page_ga100(bar) } - fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result { + fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { write_sysmem_flush_page_ga100(bar, addr); Ok(()) } - fn supports_display(&self, bar: &Bar0) -> bool { + fn supports_display(&self, bar: Bar0<'_>) -> bool { display_enabled_ga100(bar) } - fn vidmem_size(&self, bar: &Bar0) -> u64 { + fn vidmem_size(&self, bar: Bar0<'_>) -> u64 { super::tu102::vidmem_size_gp102(bar) } + + fn pmu_reserved_size(&self) -> u32 { + super::tu102::pmu_reserved_size_tu102() + } + + fn non_wpr_heap_size(&self) -> u32 { + super::tu102::non_wpr_heap_size_tu102() + } + + // GA100 is a special case where its FRTS region exists, but is empty. We + // return a size of 0 because we still need to record where the region starts. + fn frts_size(&self) -> u64 { + 0 + } } const GA100: Ga100 = Ga100; diff --git a/drivers/gpu/nova-core/fb/hal/ga102.rs b/drivers/gpu/nova-core/fb/hal/ga102.rs index 4b9f0f74d0e7..44a2cf8a00f1 100644 --- a/drivers/gpu/nova-core/fb/hal/ga102.rs +++ b/drivers/gpu/nova-core/fb/hal/ga102.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::{ io::Io, @@ -11,30 +12,42 @@ use crate::{ regs, // }; -fn vidmem_size_ga102(bar: &Bar0) -> u64 { +pub(super) fn vidmem_size_ga102(bar: Bar0<'_>) -> u64 { bar.read(regs::NV_USABLE_FB_SIZE_IN_MB).usable_fb_size() } struct Ga102; impl FbHal for Ga102 { - fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64 { + fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { super::ga100::read_sysmem_flush_page_ga100(bar) } - fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result { + fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { super::ga100::write_sysmem_flush_page_ga100(bar, addr); Ok(()) } - fn supports_display(&self, bar: &Bar0) -> bool { + fn supports_display(&self, bar: Bar0<'_>) -> bool { super::ga100::display_enabled_ga100(bar) } - fn vidmem_size(&self, bar: &Bar0) -> u64 { + fn vidmem_size(&self, bar: Bar0<'_>) -> u64 { vidmem_size_ga102(bar) } + + fn pmu_reserved_size(&self) -> u32 { + super::tu102::pmu_reserved_size_tu102() + } + + fn non_wpr_heap_size(&self) -> u32 { + super::tu102::non_wpr_heap_size_tu102() + } + + fn frts_size(&self) -> u64 { + super::tu102::frts_size_tu102() + } } const GA102: Ga102 = Ga102; diff --git a/drivers/gpu/nova-core/fb/hal/gb100.rs b/drivers/gpu/nova-core/fb/hal/gb100.rs new file mode 100644 index 000000000000..6e0eba101ca1 --- /dev/null +++ b/drivers/gpu/nova-core/fb/hal/gb100.rs @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Blackwell GB10x framebuffer HAL. + +use kernel::{ + io::{ + register::{ + RegisterBase, + WithBase, // + }, + Io, // + }, + num::Bounded, + prelude::*, + ptr::{ + const_align_up, + Alignment, // + }, + sizes::*, // +}; + +use crate::{ + driver::Bar0, + fb::hal::FbHal, + num::usize_into_u32, + regs, // +}; + +struct Gb100; + +impl RegisterBase<regs::Hshub0Base> for Gb100 { + const BASE: usize = 0x0087_0000; +} + +fn read_sysmem_flush_page_gb100(bar: Bar0<'_>) -> u64 { + let lo = u64::from( + bar.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb100>()) + .adr(), + ); + let hi = u64::from( + bar.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb100>()) + .adr(), + ); + + lo | (hi << 32) +} + +/// Write the sysmem flush page address through the GB10x HSHUB0 registers. +/// +/// Both the primary and EG (egress) register pairs must be programmed to the same address, +/// as required by hardware. +fn write_sysmem_flush_page_gb100(bar: Bar0<'_>, addr: Bounded<u64, 52>) { + // CAST: lower 32 bits. Hardware ignores bits 7:0. + let addr_lo = *addr as u32; + let addr_hi = addr.shr::<32, 20>().cast::<u32>(); + + // Write HI first. The hardware will trigger the flush on the LO write. + + // Primary HSHUB pair. + bar.write( + regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb100>(), + regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi), + ); + bar.write( + regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb100>(), + regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo), + ); + + // EG (egress) pair -- must match the primary pair. + bar.write( + regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb100>(), + regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi), + ); + bar.write( + regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb100>(), + regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo), + ); +} + +pub(super) const fn pmu_reserved_size_gb100() -> u32 { + usize_into_u32::<{ const_align_up(SZ_8M + SZ_16M + SZ_4K, Alignment::new::<SZ_128K>()).unwrap() }>( + ) +} + +impl FbHal for Gb100 { + fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { + read_sysmem_flush_page_gb100(bar) + } + + fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { + let addr = Bounded::<u64, 52>::try_new(addr).ok_or(EINVAL)?; + + write_sysmem_flush_page_gb100(bar, addr); + + Ok(()) + } + + fn supports_display(&self, bar: Bar0<'_>) -> bool { + super::ga100::display_enabled_ga100(bar) + } + + fn vidmem_size(&self, bar: Bar0<'_>) -> u64 { + super::ga102::vidmem_size_ga102(bar) + } + + fn pmu_reserved_size(&self) -> u32 { + pmu_reserved_size_gb100() + } + + fn non_wpr_heap_size(&self) -> u32 { + // Non-WPR heap for GB10x (see Open RM: kgspGetNonWprHeapSize, GB100/GB102). + u32::SZ_2M + } + + fn frts_size(&self) -> u64 { + super::tu102::frts_size_tu102() + } +} + +const GB100: Gb100 = Gb100; +pub(super) const GB100_HAL: &dyn FbHal = &GB100; diff --git a/drivers/gpu/nova-core/fb/hal/gb202.rs b/drivers/gpu/nova-core/fb/hal/gb202.rs new file mode 100644 index 000000000000..038d1278c634 --- /dev/null +++ b/drivers/gpu/nova-core/fb/hal/gb202.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Blackwell GB20x framebuffer HAL. + +use kernel::{ + io::{ + register::{ + RegisterBase, + WithBase, // + }, + Io, // + }, + num::Bounded, + prelude::*, + sizes::SizeConstants, // +}; + +use crate::{ + driver::Bar0, + fb::hal::FbHal, + regs, // +}; + +struct Gb202; + +impl RegisterBase<regs::Fbhub0Base> for Gb202 { + const BASE: usize = 0x008a_0000; +} + +fn read_sysmem_flush_page_gb202(bar: Bar0<'_>) -> u64 { + let lo = u64::from( + bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb202>()) + .adr(), + ); + let hi = u64::from( + bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb202>()) + .adr(), + ); + + lo | (hi << 32) +} + +/// Write the sysmem flush page address through the GB20x FBHUB0 registers. +fn write_sysmem_flush_page_gb202(bar: Bar0<'_>, addr: Bounded<u64, 52>) { + // Write HI first. The hardware will trigger the flush on the LO write. + bar.write( + regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb202>(), + regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed() + .with_adr(addr.shr::<32, 20>().cast::<u32>()), + ); + bar.write( + regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb202>(), + // CAST: lower 32 bits. Hardware ignores bits 7:0. + regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(*addr as u32), + ); +} + +impl FbHal for Gb202 { + fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { + read_sysmem_flush_page_gb202(bar) + } + + fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { + let addr = Bounded::<u64, 52>::try_new(addr).ok_or(EINVAL)?; + + write_sysmem_flush_page_gb202(bar, addr); + + Ok(()) + } + + fn supports_display(&self, bar: Bar0<'_>) -> bool { + super::ga100::display_enabled_ga100(bar) + } + + fn vidmem_size(&self, bar: Bar0<'_>) -> u64 { + super::ga102::vidmem_size_ga102(bar) + } + + fn pmu_reserved_size(&self) -> u32 { + super::gb100::pmu_reserved_size_gb100() + } + + fn non_wpr_heap_size(&self) -> u32 { + // Non-WPR heap for GB20x (see Open RM: kgspGetNonWprHeapSize, GB202+). + u32::SZ_2M + u32::SZ_128K + } + + fn frts_size(&self) -> u64 { + super::tu102::frts_size_tu102() + } +} + +const GB202: Gb202 = Gb202; +pub(super) const GB202_HAL: &dyn FbHal = &GB202; diff --git a/drivers/gpu/nova-core/fb/hal/gh100.rs b/drivers/gpu/nova-core/fb/hal/gh100.rs new file mode 100644 index 000000000000..5450c7254dad --- /dev/null +++ b/drivers/gpu/nova-core/fb/hal/gh100.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use kernel::{ + prelude::*, + sizes::SizeConstants, // +}; + +use crate::{ + driver::Bar0, + fb::hal::FbHal, // +}; + +struct Gh100; + +impl FbHal for Gh100 { + fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { + super::ga100::read_sysmem_flush_page_ga100(bar) + } + + fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { + super::ga100::write_sysmem_flush_page_ga100(bar, addr); + + Ok(()) + } + + fn supports_display(&self, bar: Bar0<'_>) -> bool { + super::ga100::display_enabled_ga100(bar) + } + + fn vidmem_size(&self, bar: Bar0<'_>) -> u64 { + super::ga102::vidmem_size_ga102(bar) + } + + fn pmu_reserved_size(&self) -> u32 { + super::tu102::pmu_reserved_size_tu102() + } + + fn non_wpr_heap_size(&self) -> u32 { + // Non-WPR heap for Hopper (see Open RM: kgspCalculateFbLayout_GH100). + u32::SZ_2M + } + + fn frts_size(&self) -> u64 { + super::tu102::frts_size_tu102() + } +} + +const GH100: Gh100 = Gh100; +pub(super) const GH100_HAL: &dyn FbHal = &GH100; diff --git a/drivers/gpu/nova-core/fb/hal/tu102.rs b/drivers/gpu/nova-core/fb/hal/tu102.rs index 281bb796e198..f629e8e9d5d5 100644 --- a/drivers/gpu/nova-core/fb/hal/tu102.rs +++ b/drivers/gpu/nova-core/fb/hal/tu102.rs @@ -1,8 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::{ io::Io, - prelude::*, // + prelude::*, + sizes::*, // }; use crate::{ @@ -15,11 +17,11 @@ use crate::{ /// to be used by HALs. pub(super) const FLUSH_SYSMEM_ADDR_SHIFT: u32 = 8; -pub(super) fn read_sysmem_flush_page_gm107(bar: &Bar0) -> u64 { +pub(super) fn read_sysmem_flush_page_gm107(bar: Bar0<'_>) -> u64 { u64::from(bar.read(regs::NV_PFB_NISO_FLUSH_SYSMEM_ADDR).adr_39_08()) << FLUSH_SYSMEM_ADDR_SHIFT } -pub(super) fn write_sysmem_flush_page_gm107(bar: &Bar0, addr: u64) -> Result { +pub(super) fn write_sysmem_flush_page_gm107(bar: Bar0<'_>, addr: u64) -> Result { // Check that the address doesn't overflow the receiving 32-bit register. u32::try_from(addr >> FLUSH_SYSMEM_ADDR_SHIFT) .map_err(|_| EINVAL) @@ -28,34 +30,58 @@ pub(super) fn write_sysmem_flush_page_gm107(bar: &Bar0, addr: u64) -> Result { }) } -pub(super) fn display_enabled_gm107(bar: &Bar0) -> bool { +pub(super) fn display_enabled_gm107(bar: Bar0<'_>) -> bool { !bar.read(regs::gm107::NV_FUSE_STATUS_OPT_DISPLAY) .display_disabled() } -pub(super) fn vidmem_size_gp102(bar: &Bar0) -> u64 { +pub(super) fn vidmem_size_gp102(bar: Bar0<'_>) -> u64 { bar.read(regs::NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE) .usable_fb_size() } +pub(super) const fn pmu_reserved_size_tu102() -> u32 { + 0 +} + +pub(super) const fn non_wpr_heap_size_tu102() -> u32 { + u32::SZ_1M +} + +pub(super) const fn frts_size_tu102() -> u64 { + u64::SZ_1M +} + struct Tu102; impl FbHal for Tu102 { - fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64 { + fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { read_sysmem_flush_page_gm107(bar) } - fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result { + fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { write_sysmem_flush_page_gm107(bar, addr) } - fn supports_display(&self, bar: &Bar0) -> bool { + fn supports_display(&self, bar: Bar0<'_>) -> bool { display_enabled_gm107(bar) } - fn vidmem_size(&self, bar: &Bar0) -> u64 { + fn vidmem_size(&self, bar: Bar0<'_>) -> u64 { vidmem_size_gp102(bar) } + + fn pmu_reserved_size(&self) -> u32 { + pmu_reserved_size_tu102() + } + + fn non_wpr_heap_size(&self) -> u32 { + non_wpr_heap_size_tu102() + } + + fn frts_size(&self) -> u64 { + frts_size_tu102() + } } const TU102: Tu102 = Tu102; diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index 6c2ab69cb605..1e89390209f5 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. //! Contains structures and functions dedicated to the parsing, building and patching of firmwares //! to be loaded into a given execution unit. @@ -27,6 +28,7 @@ use crate::{ }; pub(crate) mod booter; +pub(crate) mod fsp; pub(crate) mod fwsec; pub(crate) mod gsp; pub(crate) mod riscv; @@ -48,7 +50,7 @@ fn request_firmware( /// Structure used to describe some firmwares, notably FWSEC-FRTS. #[repr(C)] -#[derive(Debug, Clone)] +#[derive(Debug, Clone, FromBytes)] pub(crate) struct FalconUCodeDescV2 { /// Header defined by 'NV_BIT_FALCON_UCODE_DESC_HEADER_VDESC*' in OpenRM. hdr: u32, @@ -84,9 +86,6 @@ pub(crate) struct FalconUCodeDescV2 { pub(crate) alt_dmem_load_size: u32, } -// SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. -unsafe impl FromBytes for FalconUCodeDescV2 {} - /// Structure used to describe some firmwares, notably FWSEC-FRTS. #[repr(C)] #[derive(Debug, Clone)] @@ -388,13 +387,7 @@ impl<'a> BinFirmware<'a> { // Extract header. .and_then(BinHdr::from_bytes_copy) // Validate header. - .and_then(|hdr| { - if hdr.bin_magic == BIN_MAGIC { - Some(hdr) - } else { - None - } - }) + .filter(|hdr| hdr.bin_magic == BIN_MAGIC) .map(|hdr| Self { hdr, fw }) .ok_or(EINVAL) } @@ -436,10 +429,16 @@ impl<const N: usize> ModInfoBuilder<N> { .make_entry_file(name, "bootloader") .make_entry_file(name, "gsp"); - if chipset.needs_fwsec_bootloader() { + let this = if chipset.needs_fwsec_bootloader() { this.make_entry_file(name, "gen_bootloader") } else { this + }; + + if chipset.uses_fsp() { + this.make_entry_file(name, "fmc") + } else { + this } } @@ -473,17 +472,119 @@ mod elf { transmute::FromBytes, // }; + /// Trait to abstract over ELF header differences. + trait ElfHeader: FromBytes { + fn shnum(&self) -> u16; + fn shoff(&self) -> u64; + fn shstrndx(&self) -> u16; + } + + /// Trait to abstract over ELF section-header differences. + trait ElfSectionHeader: FromBytes { + fn name(&self) -> u32; + fn offset(&self) -> u64; + fn size(&self) -> u64; + } + + /// Trait describing a matching ELF header and section-header format. + trait ElfFormat { + type Header: ElfHeader; + type SectionHeader: ElfSectionHeader; + } + /// Newtype to provide a [`FromBytes`] implementation. #[repr(transparent)] struct Elf64Hdr(bindings::elf64_hdr); // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. unsafe impl FromBytes for Elf64Hdr {} + impl ElfHeader for Elf64Hdr { + fn shnum(&self) -> u16 { + self.0.e_shnum + } + + fn shoff(&self) -> u64 { + self.0.e_shoff + } + + fn shstrndx(&self) -> u16 { + self.0.e_shstrndx + } + } + #[repr(transparent)] struct Elf64SHdr(bindings::elf64_shdr); // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. unsafe impl FromBytes for Elf64SHdr {} + impl ElfSectionHeader for Elf64SHdr { + fn name(&self) -> u32 { + self.0.sh_name + } + + fn offset(&self) -> u64 { + self.0.sh_offset + } + + fn size(&self) -> u64 { + self.0.sh_size + } + } + + struct Elf64Format; + + impl ElfFormat for Elf64Format { + type Header = Elf64Hdr; + type SectionHeader = Elf64SHdr; + } + + /// Newtype to provide [`FromBytes`] and [`ElfHeader`] implementations for ELF32. + #[repr(transparent)] + struct Elf32Hdr(bindings::elf32_hdr); + // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. + unsafe impl FromBytes for Elf32Hdr {} + + impl ElfHeader for Elf32Hdr { + fn shnum(&self) -> u16 { + self.0.e_shnum + } + + fn shoff(&self) -> u64 { + u64::from(self.0.e_shoff) + } + + fn shstrndx(&self) -> u16 { + self.0.e_shstrndx + } + } + + /// Newtype to provide [`FromBytes`] and [`ElfSectionHeader`] implementations for ELF32. + #[repr(transparent)] + struct Elf32SHdr(bindings::elf32_shdr); + // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. + unsafe impl FromBytes for Elf32SHdr {} + + impl ElfSectionHeader for Elf32SHdr { + fn name(&self) -> u32 { + self.0.sh_name + } + + fn offset(&self) -> u64 { + u64::from(self.0.sh_offset) + } + + fn size(&self) -> u64 { + u64::from(self.0.sh_size) + } + } + + struct Elf32Format; + + impl ElfFormat for Elf32Format { + type Header = Elf32Hdr; + type SectionHeader = Elf32SHdr; + } + /// Returns a NULL-terminated string from the ELF image at `offset`. fn elf_str(elf: &[u8], offset: u64) -> Option<&str> { let idx = usize::try_from(offset).ok()?; @@ -491,47 +592,74 @@ mod elf { CStr::from_bytes_until_nul(bytes).ok()?.to_str().ok() } - /// Tries to extract section with name `name` from the ELF64 image `elf`, and returns it. - pub(super) fn elf64_section<'a, 'b>(elf: &'a [u8], name: &'b str) -> Option<&'a [u8]> { - let hdr = &elf - .get(0..size_of::<bindings::elf64_hdr>()) - .and_then(Elf64Hdr::from_bytes)? - .0; + fn elf_section_generic<'a, F>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> + where + F: ElfFormat, + { + let hdr = F::Header::from_bytes(elf.get(0..size_of::<F::Header>())?)?; - // Get all the section headers. - let mut shdr = { - let shdr_num = usize::from(hdr.e_shnum); - let shdr_start = usize::try_from(hdr.e_shoff).ok()?; - let shdr_end = shdr_num - .checked_mul(size_of::<Elf64SHdr>()) - .and_then(|v| v.checked_add(shdr_start))?; + let shdr_num = usize::from(hdr.shnum()); + let shdr_start = usize::try_from(hdr.shoff()).ok()?; + let shdr_end = shdr_num + .checked_mul(size_of::<F::SectionHeader>()) + .and_then(|v| v.checked_add(shdr_start))?; - elf.get(shdr_start..shdr_end) - .map(|slice| slice.chunks_exact(size_of::<Elf64SHdr>()))? - }; + // Get all the section headers as an iterator over byte chunks. + let shdr_bytes = elf.get(shdr_start..shdr_end)?; + let mut shdr_iter = shdr_bytes.chunks_exact(size_of::<F::SectionHeader>()); // Get the strings table. - let strhdr = shdr + let strhdr = shdr_iter .clone() - .nth(usize::from(hdr.e_shstrndx)) - .and_then(Elf64SHdr::from_bytes)?; + .nth(usize::from(hdr.shstrndx())) + .and_then(F::SectionHeader::from_bytes)?; // Find the section which name matches `name` and return it. - shdr.find_map(|sh| { - let hdr = Elf64SHdr::from_bytes(sh)?; - let name_offset = strhdr.0.sh_offset.checked_add(u64::from(hdr.0.sh_name))?; + shdr_iter.find_map(|sh_bytes| { + let sh = F::SectionHeader::from_bytes(sh_bytes)?; + let name_offset = strhdr.offset().checked_add(u64::from(sh.name()))?; let section_name = elf_str(elf, name_offset)?; if section_name != name { return None; } - let start = usize::try_from(hdr.0.sh_offset).ok()?; - let end = usize::try_from(hdr.0.sh_size) + let start = usize::try_from(sh.offset()).ok()?; + let end = usize::try_from(sh.size()) .ok() - .and_then(|sh_size| start.checked_add(sh_size))?; + .and_then(|sz| start.checked_add(sz))?; elf.get(start..end) }) } + + /// Extract the section with name `name` from the ELF64 image `elf`. + fn elf64_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { + elf_section_generic::<Elf64Format>(elf, name) + } + + /// Extract the section with name `name` from the ELF32 image `elf`. + fn elf32_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { + elf_section_generic::<Elf32Format>(elf, name) + } + + /// Automatically detects ELF32 vs ELF64 based on the ELF header. + pub(super) fn elf_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { + // ELF identification: a 4-byte magic followed by a class byte (32- vs 64-bit). + const ELFMAG: &[u8] = b"\x7fELF"; + const SELFMAG: usize = ELFMAG.len(); + const EI_CLASS: usize = 4; + const ELFCLASS32: u8 = 1; + const ELFCLASS64: u8 = 2; + + if elf.get(0..SELFMAG) != Some(ELFMAG) { + return None; + } + + match *elf.get(EI_CLASS)? { + ELFCLASS32 => elf32_section(elf, name), + ELFCLASS64 => elf64_section(elf, name), + _ => None, + } + } } diff --git a/drivers/gpu/nova-core/firmware/booter.rs b/drivers/gpu/nova-core/firmware/booter.rs index de2a4536b532..d9313ac361af 100644 --- a/drivers/gpu/nova-core/firmware/booter.rs +++ b/drivers/gpu/nova-core/firmware/booter.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. //! Support for loading and patching the `Booter` firmware. `Booter` is a Heavy Secured firmware //! running on [`Sec2`], that is used on Turing/Ampere to load the GSP firmware into the GSP falcon @@ -8,6 +9,7 @@ use core::marker::PhantomData; use kernel::{ device, + dma::Coherent, prelude::*, transmute::FromBytes, // }; @@ -280,7 +282,6 @@ impl FirmwareObject<BooterFirmware, Unsigned> { #[derive(Copy, Clone, Debug, PartialEq)] pub(crate) enum BooterKind { Loader, - #[expect(unused)] Unloader, } @@ -293,7 +294,7 @@ impl BooterFirmware { chipset: Chipset, ver: &str, falcon: &Falcon<<Self as FalconFirmware>::Target>, - bar: &Bar0, + bar: Bar0<'_>, ) -> Result<Self> { let fw_name = match kind { BooterKind::Loader => "booter_load", @@ -396,6 +397,35 @@ impl BooterFirmware { ucode: ucode_signed, }) } + + /// Load and run the booter firmware on SEC2. + /// + /// Resets SEC2, loads this firmware image, then boots with the WPR metadata + /// address passed via the SEC2 mailboxes. + pub(crate) fn run<T>( + &self, + dev: &device::Device<device::Bound>, + bar: Bar0<'_>, + sec2_falcon: &Falcon<Sec2>, + wpr_meta: &Coherent<T>, + ) -> Result { + sec2_falcon.reset(bar)?; + sec2_falcon.load(dev, bar, self)?; + let wpr_handle = wpr_meta.dma_handle(); + let (mbox0, mbox1) = sec2_falcon.boot( + bar, + Some(wpr_handle as u32), + Some((wpr_handle >> 32) as u32), + )?; + dev_dbg!(dev, "SEC2 MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1); + + if mbox0 != 0 { + dev_err!(dev, "Booter-load failed with error {:#x}\n", mbox0); + return Err(ENODEV); + } + + Ok(()) + } } impl FalconDmaLoadable for BooterFirmware { diff --git a/drivers/gpu/nova-core/firmware/fsp.rs b/drivers/gpu/nova-core/firmware/fsp.rs new file mode 100644 index 000000000000..6eaf1c684b9d --- /dev/null +++ b/drivers/gpu/nova-core/firmware/fsp.rs @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! FSP is a hardware unit that runs FMC firmware. + +use kernel::{ + device, + dma::Coherent, + firmware::Firmware, + prelude::*, // +}; + +use crate::{ + firmware::elf, + gpu::Chipset, // +}; + +/// Size of the FSP SHA-384 hash, in bytes. +const FSP_HASH_SIZE: usize = 48; +/// Maximum size of the FSP public key (RSA-3072), in bytes. +/// +/// The FMC ELF `publickey` section may be shorter, so the remaining bytes are zero-padded. +const FSP_PKEY_SIZE: usize = 384; +/// Maximum size of the FSP signature (RSA-3072), in bytes. +/// +/// The FMC ELF `signature` section may be shorter, so the remaining bytes are zero-padded. +const FSP_SIG_SIZE: usize = 384; + +/// Structure to hold FMC signatures. +/// +/// C representation is used because this type is used for communication with the FSP. +#[derive(Debug, Clone, Copy, Zeroable)] +#[repr(C)] +pub(crate) struct FmcSignatures { + pub(crate) hash384: [u8; FSP_HASH_SIZE], + pub(crate) public_key: [u8; FSP_PKEY_SIZE], + pub(crate) signature: [u8; FSP_SIG_SIZE], +} + +pub(crate) struct FspFirmware { + /// FMC firmware image data (only the "image" ELF section). + pub(crate) fmc_image: Coherent<[u8]>, + /// FMC firmware signatures. + pub(crate) fmc_sigs: KBox<FmcSignatures>, +} + +impl FspFirmware { + pub(crate) fn new( + dev: &device::Device<device::Bound>, + chipset: Chipset, + ver: &str, + ) -> Result<Self> { + let fw = super::request_firmware(dev, chipset, "fmc", ver)?; + + // FSP expects only the "image" section, not the entire ELF file. + let fmc_image_data = elf::elf_section(fw.data(), "image").ok_or_else(|| { + dev_err!(dev, "FMC ELF file missing 'image' section\n"); + EINVAL + })?; + let fmc_image = Coherent::from_slice(dev, fmc_image_data, GFP_KERNEL)?; + + Ok(Self { + fmc_image, + fmc_sigs: Self::extract_fmc_signatures(&fw, dev)?, + }) + } + + /// Extract FMC firmware signatures for Chain of Trust verification. + /// + /// Extracts real cryptographic signatures from FMC ELF32 firmware sections. + /// Returns signatures in a heap-allocated structure to prevent stack overflow. + fn extract_fmc_signatures( + fmc_fw: &Firmware, + dev: &device::Device, + ) -> Result<KBox<FmcSignatures>> { + let get_section = |name: &str, max_len: usize| { + elf::elf_section(fmc_fw.data(), name) + .ok_or(EINVAL) + .inspect_err(|_| dev_err!(dev, "FMC firmware missing '{}' section\n", name)) + .and_then(|section| { + if section.len() > max_len { + dev_err!( + dev, + "FMC {} section size {} > maximum {}\n", + name, + section.len(), + max_len + ); + Err(EINVAL) + } else { + Ok(section) + } + }) + }; + + let hash_section = get_section("hash", FSP_HASH_SIZE)?; + let pkey_section = get_section("publickey", FSP_PKEY_SIZE)?; + let sig_section = get_section("signature", FSP_SIG_SIZE)?; + + // The hash section is a SHA-384 output: it must be exactly FSP_HASH_SIZE bytes. + if hash_section.len() != FSP_HASH_SIZE { + dev_err!( + dev, + "FMC hash section size {} != expected {}\n", + hash_section.len(), + FSP_HASH_SIZE + ); + return Err(EINVAL); + } + + // Initialize the signatures in place to avoid building the large `FmcSignatures` on the + // stack, then fill each section from the firmware. + let signatures = KBox::init( + pin_init::init_zeroed::<FmcSignatures>().chain(|sigs| { + // PANIC: src and dst lengths are both FSP_HASH_SIZE (verified above). + sigs.hash384.copy_from_slice(hash_section); + // PANIC: dst is sliced to src.len(); src.len() <= FSP_PKEY_SIZE per `get_section`. + sigs.public_key[..pkey_section.len()].copy_from_slice(pkey_section); + // PANIC: dst is sliced to src.len(); src.len() <= FSP_SIG_SIZE per `get_section`. + sigs.signature[..sig_section.len()].copy_from_slice(sig_section); + Ok(()) + }), + GFP_KERNEL, + )?; + + Ok(signatures) + } +} diff --git a/drivers/gpu/nova-core/firmware/fwsec.rs b/drivers/gpu/nova-core/firmware/fwsec.rs index 8810cb49db67..199ae2adb664 100644 --- a/drivers/gpu/nova-core/firmware/fwsec.rs +++ b/drivers/gpu/nova-core/firmware/fwsec.rs @@ -144,7 +144,6 @@ pub(crate) enum FwsecCommand { /// image into it. Frts { frts_addr: u64, frts_size: u64 }, /// Asks [`FwsecFirmware`] to load pre-OS apps on the PMU. - #[expect(dead_code)] Sb, } @@ -322,7 +321,7 @@ impl FwsecFirmware { pub(crate) fn new( dev: &Device<device::Bound>, falcon: &Falcon<Gsp>, - bar: &Bar0, + bar: Bar0<'_>, bios: &Vbios, cmd: FwsecCommand, ) -> Result<Self> { @@ -395,7 +394,7 @@ impl FwsecFirmware { &self, dev: &Device<device::Bound>, falcon: &Falcon<Gsp>, - bar: &Bar0, + bar: Bar0<'_>, ) -> Result<()> { // Reset falcon, load the firmware, and run it. falcon diff --git a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs index bcb713a868e2..039920dc340b 100644 --- a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs +++ b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs @@ -280,7 +280,7 @@ impl FwsecFirmwareWithBl { &self, dev: &Device<device::Bound>, falcon: &Falcon<Gsp>, - bar: &Bar0, + bar: Bar0<'_>, ) -> Result<()> { // Reset falcon, load the firmware, and run it. falcon diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs index 2fcc255c3bc8..99a302bae567 100644 --- a/drivers/gpu/nova-core/firmware/gsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp.rs @@ -63,6 +63,21 @@ pub(crate) struct GspFirmware { } impl GspFirmware { + fn find_gsp_sigs_section(chipset: Chipset) -> &'static str { + match chipset.arch() { + Architecture::Turing if matches!(chipset, Chipset::TU116 | Chipset::TU117) => { + ".fwsignature_tu11x" + } + Architecture::Turing => ".fwsignature_tu10x", + Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_ga100", + Architecture::Ampere => ".fwsignature_ga10x", + Architecture::Ada => ".fwsignature_ad10x", + Architecture::Hopper => ".fwsignature_gh10x", + Architecture::BlackwellGB10x => ".fwsignature_gb10x", + Architecture::BlackwellGB20x => ".fwsignature_gb20x", + } + } + /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page /// tables expected by the GSP bootloader to load it. pub(crate) fn new<'a>( @@ -73,7 +88,7 @@ impl GspFirmware { pin_init::pin_init_scope(move || { let firmware = super::request_firmware(dev, chipset, "gsp", ver)?; - let fw_section = elf::elf64_section(firmware.data(), ".fwimage").ok_or(EINVAL)?; + let fw_section = elf::elf_section(firmware.data(), ".fwimage").ok_or(EINVAL)?; let size = fw_section.len(); @@ -131,20 +146,9 @@ impl GspFirmware { }, size, signatures: { - let sigs_section = match chipset.arch() { - Architecture::Turing - if matches!(chipset, Chipset::TU116 | Chipset::TU117) => - { - ".fwsignature_tu11x" - } - Architecture::Turing => ".fwsignature_tu10x", - // GA100 uses the same firmware as Turing - Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_tu10x", - Architecture::Ampere => ".fwsignature_ga10x", - Architecture::Ada => ".fwsignature_ad10x", - }; - - elf::elf64_section(firmware.data(), sigs_section) + let sigs_section = Self::find_gsp_sigs_section(chipset); + + elf::elf_section(firmware.data(), sigs_section) .ok_or(EINVAL) .and_then(|data| Coherent::from_slice(dev, data, GFP_KERNEL))? }, diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs new file mode 100644 index 000000000000..8fc243c66e35 --- /dev/null +++ b/drivers/gpu/nova-core/fsp.rs @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! FSP (Foundation Security Processor) interface for Hopper/Blackwell GPUs. +//! +//! Hopper/Blackwell use a simplified firmware boot sequence: FMC, then FSP, then GSP. +//! Unlike Turing/Ampere/Ada, there is no SEC2 (Security Engine 2) usage. +//! FSP handles secure boot directly using FMC firmware and Chain of Trust. + +use kernel::{ + device, + dma::Coherent, + io::poll::read_poll_timeout, + prelude::*, + ptr::{ + Alignable, + Alignment, // + }, + sizes::SZ_2M, + time::Delta, + transmute::{ + AsBytes, + FromBytes, // + }, +}; + +use crate::{ + driver::Bar0, + falcon::{ + fsp::Fsp as FspEngine, + Falcon, // + }, + fb::FbLayout, + firmware::fsp::{ + FmcSignatures, + FspFirmware, // + }, + gpu::Chipset, + gsp::GspFmcBootParams, + mctp::{ + MctpHeader, + NvdmHeader, + NvdmType, // + }, + num, + regs, // +}; + +mod hal; + +/// FSP command response payload (`NVDM_PAYLOAD_COMMAND_RESPONSE`). +#[repr(C, packed)] +#[derive(Clone, Copy)] +struct NvdmPayloadCommandResponse { + task_id: u32, + command_nvdm_type: u32, + error_code: u32, +} + +/// Complete FSP response structure with MCTP and NVDM headers. +#[repr(C, packed)] +#[derive(Clone, Copy)] +struct FspResponse { + mctp_header: MctpHeader, + nvdm_header: NvdmHeader, + response: NvdmPayloadCommandResponse, +} + +// SAFETY: FspResponse is a packed C struct with only integral fields. +unsafe impl FromBytes for FspResponse {} + +/// Trait implemented by types representing a message to send to FSP. +/// +/// This provides [`Fsp::send_sync_fsp`] with the information it needs to send +/// a given message, following the same pattern as GSP's `CommandToGsp`. +trait MessageToFsp: AsBytes { + /// NVDM type identifying this message to FSP. + const NVDM_TYPE: NvdmType; +} + +/// NVDM (NVIDIA Data Model) CoT (Chain of Trust) payload, the main +/// message body sent to FSP for Chain of Trust boot. +#[repr(C, packed)] +#[derive(Clone, Copy, Zeroable)] +struct NvdmPayloadCot { + version: u16, + size: u16, + gsp_fmc_sysmem_offset: u64, + frts_sysmem_offset: u64, + frts_sysmem_size: u32, + frts_vidmem_offset: u64, + frts_vidmem_size: u32, + sigs: FmcSignatures, + gsp_boot_args_sysmem_offset: u64, +} + +/// Complete FSP message structure with MCTP and NVDM headers. +#[repr(C)] +#[derive(Clone, Copy)] +struct FspMessage { + mctp_header: MctpHeader, + nvdm_header: NvdmHeader, + cot: NvdmPayloadCot, +} + +impl FspMessage { + /// Returns an in-place initializer for [`FspMessage`]. + fn new<'a>( + fb_layout: &FbLayout, + fsp_fw: &'a FspFirmware, + args: &'a FmcBootArgs, + ) -> Result<impl Init<Self> + 'a> { + // frts_offset is relative to FB end: FRTS_location = FB_END - frts_offset + let frts_vidmem_offset = if !args.resume { + let frts_reserved_size = fb_layout.heap.len() + u64::from(fb_layout.pmu_reserved_size); + + frts_reserved_size + .align_up(Alignment::new::<SZ_2M>()) + .ok_or(EINVAL)? + } else { + 0 + }; + + let frts_size: u32 = if !args.resume { + fb_layout.frts.len().try_into()? + } else { + 0 + }; + + let version = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?.cot_version(); + let size = num::usize_into_u16::<{ core::mem::size_of::<NvdmPayloadCot>() }>(); + + Ok(init!(Self { + mctp_header: MctpHeader::single_packet(), + nvdm_header: NvdmHeader::new(NvdmType::Cot), + // The payload is packed, so we cannot use `init!`. Initialize it member-by-member using + // `chain`. + cot <- pin_init::init_zeroed(), + }) + .chain(move |msg| { + msg.cot.version = version; + msg.cot.size = size; + msg.cot.gsp_fmc_sysmem_offset = fsp_fw.fmc_image.dma_handle(); + msg.cot.frts_vidmem_offset = frts_vidmem_offset; + msg.cot.frts_vidmem_size = frts_size; + // frts_sysmem_* intentionally left at zero for now, but will be needed for e.g. + // systems without VRAM. + msg.cot.gsp_boot_args_sysmem_offset = args.fmc_boot_params.dma_handle(); + msg.cot.sigs = *fsp_fw.fmc_sigs; + + Ok(()) + })) + } +} + +// SAFETY: `FspMessage` is `#[repr(C)]` with no padding, so all of its +// bytes are initialized. +unsafe impl AsBytes for FspMessage {} + +impl MessageToFsp for FspMessage { + const NVDM_TYPE: NvdmType = NvdmType::Cot; +} + +/// Bundled arguments for FMC boot via FSP Chain of Trust. +pub(crate) struct FmcBootArgs { + chipset: Chipset, + fmc_boot_params: Coherent<GspFmcBootParams>, + resume: bool, +} + +impl FmcBootArgs { + /// Builds FMC boot arguments, allocating the DMA-coherent boot parameter + /// structure that FSP will read. + pub(crate) fn new( + dev: &device::Device<device::Bound>, + chipset: Chipset, + wpr_meta_addr: u64, + libos_addr: u64, + resume: bool, + ) -> Result<Self> { + let init = GspFmcBootParams::new(wpr_meta_addr, libos_addr); + + Ok(Self { + chipset, + fmc_boot_params: Coherent::<GspFmcBootParams>::init(dev, GFP_KERNEL, init)?, + resume, + }) + } + + /// DMA address of the FMC boot parameters, needed after boot for lockdown + /// release polling. + pub(crate) fn boot_params_dma_handle(&self) -> u64 { + self.fmc_boot_params.dma_handle() + } +} + +/// FSP interface for Hopper/Blackwell GPUs. +/// +/// An `Fsp` is produced by [`Fsp::wait_secure_boot`], which only returns once FSP secure boot +/// has completed. It owns the FSP falcon and the FMC firmware, which are used for the subsequent +/// Chain of Trust boot. +pub(crate) struct Fsp { + falcon: Falcon<FspEngine>, + fsp_fw: FspFirmware, +} + +impl Fsp { + /// Waits for FSP secure boot completion, then returns the [`Fsp`] interface. + /// + /// Polls the thermal scratch register until FSP signals boot completion or the timeout + /// elapses. Returning an [`Fsp`] only on success guarantees, at the API level, that the + /// interface is not used before secure boot has completed. + pub(crate) fn wait_secure_boot( + dev: &device::Device<device::Bound>, + bar: Bar0<'_>, + chipset: Chipset, + fsp_fw: FspFirmware, + ) -> Result<Fsp> { + /// FSP secure boot completion timeout in milliseconds. + const FSP_SECURE_BOOT_TIMEOUT_MS: i64 = 5000; + + let hal = hal::fsp_hal(chipset).ok_or(ENOTSUPP)?; + let falcon = Falcon::<FspEngine>::new(dev, chipset)?; + + read_poll_timeout( + || Ok(hal.fsp_boot_status(bar)), + |&status| status == regs::NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE_STATUS_SUCCESS, + Delta::from_millis(10), + Delta::from_millis(FSP_SECURE_BOOT_TIMEOUT_MS), + ) + .inspect_err(|e| { + dev_err!(dev, "FSP secure boot completion error: {:?}\n", e); + })?; + + Ok(Fsp { falcon, fsp_fw }) + } + + /// Sends a message to FSP and waits for the response. + fn send_sync_fsp<M>(&mut self, dev: &device::Device, bar: Bar0<'_>, msg: &M) -> Result + where + M: MessageToFsp, + { + self.falcon.send_msg(bar, msg.as_bytes())?; + + let response_buf = self.falcon.recv_msg(bar).inspect_err(|e| { + dev_err!(dev, "FSP response error: {:?}\n", e); + })?; + + let (response, _) = FspResponse::from_bytes_prefix(&response_buf[..]).ok_or_else(|| { + dev_err!(dev, "FSP response too small: {}\n", response_buf.len()); + EIO + })?; + + let mctp_header = response.mctp_header; + let nvdm_header = response.nvdm_header; + let command_nvdm_type = response.response.command_nvdm_type; + let error_code = response.response.error_code; + + if !mctp_header.is_single_packet() { + dev_err!( + dev, + "Unexpected MCTP header in FSP reply: {:x?}\n", + mctp_header, + ); + return Err(EIO); + } + + if !nvdm_header.validate(NvdmType::FspResponse) { + dev_err!( + dev, + "Unexpected NVDM header in FSP reply: {:x?}\n", + nvdm_header, + ); + return Err(EIO); + } + + if command_nvdm_type != u8::from(M::NVDM_TYPE).into() { + dev_err!( + dev, + "Expected NVDM type {:?} in reply, got {:#x}\n", + M::NVDM_TYPE, + command_nvdm_type + ); + return Err(EIO); + } + + if error_code != 0 { + dev_err!( + dev, + "NVDM command {:?} failed with error {:#x}\n", + M::NVDM_TYPE, + error_code + ); + return Err(EIO); + } + + Ok(()) + } + + /// Boots GSP FMC via FSP Chain of Trust. + /// + /// Builds the CoT message from the pre-configured [`FmcBootArgs`], sends it + /// to FSP, and waits for the response. + pub(crate) fn boot_fmc( + &mut self, + dev: &device::Device<device::Bound>, + bar: Bar0<'_>, + fb_layout: &FbLayout, + args: &FmcBootArgs, + ) -> Result { + dev_dbg!(dev, "Starting FSP boot sequence for {}\n", args.chipset); + + let msg = KBox::init(FspMessage::new(fb_layout, &self.fsp_fw, args)?, GFP_KERNEL)?; + + self.send_sync_fsp(dev, bar, &*msg)?; + + dev_dbg!(dev, "FSP Chain of Trust completed successfully\n"); + Ok(()) + } +} diff --git a/drivers/gpu/nova-core/fsp/hal.rs b/drivers/gpu/nova-core/fsp/hal.rs new file mode 100644 index 000000000000..b6f2624bb13d --- /dev/null +++ b/drivers/gpu/nova-core/fsp/hal.rs @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use crate::{ + driver::Bar0, + gpu::{ + Architecture, + Chipset, // + }, +}; + +mod gb100; +mod gb202; +mod gh100; + +pub(super) trait FspHal { + /// Returns the secure boot status from the architecture-specific `NV_THERM_I2CS_SCRATCH` register. + fn fsp_boot_status(&self, bar: Bar0<'_>) -> u32; + + /// Returns the FSP Chain of Trust protocol version this chipset advertises. + fn cot_version(&self) -> u16; +} + +/// Returns the FSP HAL, or `None` if the architecture doesn't support FSP. +pub(super) fn fsp_hal(chipset: Chipset) -> Option<&'static dyn FspHal> { + match chipset.arch() { + Architecture::Turing | Architecture::Ampere | Architecture::Ada => None, + Architecture::Hopper => Some(gh100::GH100_HAL), + Architecture::BlackwellGB10x => Some(gb100::GB100_HAL), + Architecture::BlackwellGB20x => Some(gb202::GB202_HAL), + } +} diff --git a/drivers/gpu/nova-core/fsp/hal/gb100.rs b/drivers/gpu/nova-core/fsp/hal/gb100.rs new file mode 100644 index 000000000000..42f5ecfc6400 --- /dev/null +++ b/drivers/gpu/nova-core/fsp/hal/gb100.rs @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use crate::{ + driver::Bar0, + fsp::hal::FspHal, // +}; + +struct Gb100; + +impl FspHal for Gb100 { + fn fsp_boot_status(&self, bar: Bar0<'_>) -> u32 { + // GB10x shares Hopper's FSP secure boot status register. + super::gh100::fsp_boot_status_gh100(bar) + } + + fn cot_version(&self) -> u16 { + 2 + } +} + +const GB100: Gb100 = Gb100; +pub(super) const GB100_HAL: &dyn FspHal = &GB100; diff --git a/drivers/gpu/nova-core/fsp/hal/gb202.rs b/drivers/gpu/nova-core/fsp/hal/gb202.rs new file mode 100644 index 000000000000..1091b169a645 --- /dev/null +++ b/drivers/gpu/nova-core/fsp/hal/gb202.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use kernel::io::Io; + +use crate::{ + driver::Bar0, + fsp::hal::FspHal, + regs, // +}; + +struct Gb202; + +impl FspHal for Gb202 { + fn fsp_boot_status(&self, bar: Bar0<'_>) -> u32 { + bar.read(regs::gb202::NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE) + .fsp_boot_complete() + .into() + } + + fn cot_version(&self) -> u16 { + 2 + } +} + +const GB202: Gb202 = Gb202; +pub(super) const GB202_HAL: &dyn FspHal = &GB202; diff --git a/drivers/gpu/nova-core/fsp/hal/gh100.rs b/drivers/gpu/nova-core/fsp/hal/gh100.rs new file mode 100644 index 000000000000..291acaf2845a --- /dev/null +++ b/drivers/gpu/nova-core/fsp/hal/gh100.rs @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use kernel::io::Io; + +use crate::{ + driver::Bar0, + fsp::hal::FspHal, + regs, // +}; + +struct Gh100; + +/// Reads the FSP secure boot status from the Hopper/GB10x thermal scratch register. +pub(super) fn fsp_boot_status_gh100(bar: Bar0<'_>) -> u32 { + bar.read(regs::gh100::NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE) + .fsp_boot_complete() + .into() +} + +impl FspHal for Gh100 { + fn fsp_boot_status(&self, bar: Bar0<'_>) -> u32 { + fsp_boot_status_gh100(bar) + } + + fn cot_version(&self) -> u16 { + 1 + } +} + +const GH100: Gh100 = Gh100; +pub(super) const GH100_HAL: &dyn FspHal = &GH100; diff --git a/drivers/gpu/nova-core/gfw.rs b/drivers/gpu/nova-core/gfw.rs deleted file mode 100644 index fb75dd10a172..000000000000 --- a/drivers/gpu/nova-core/gfw.rs +++ /dev/null @@ -1,76 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! GPU Firmware (`GFW`) support, a.k.a `devinit`. -//! -//! Upon reset, the GPU runs some firmware code from the BIOS to setup its core parameters. Most of -//! the GPU is considered unusable until this step is completed, so we must wait on it before -//! performing driver initialization. -//! -//! A clarification about devinit terminology: devinit is a sequence of register read/writes after -//! reset that performs tasks such as: -//! 1. Programming VRAM memory controller timings. -//! 2. Power sequencing. -//! 3. Clock and PLL configuration. -//! 4. Thermal management. -//! -//! devinit itself is a 'script' which is interpreted by an interpreter program typically running -//! on the PMU microcontroller. -//! -//! Note that the devinit sequence also needs to run during suspend/resume. - -use kernel::{ - io::{ - poll::read_poll_timeout, - Io, // - }, - prelude::*, - time::Delta, // -}; - -use crate::{ - driver::Bar0, - regs, // -}; - -/// Wait for the `GFW` (GPU firmware) boot completion signal (`GFW_BOOT`), or a 4 seconds timeout. -/// -/// Upon GPU reset, several microcontrollers (such as PMU, SEC2, GSP etc) run some firmware code to -/// setup its core parameters. Most of the GPU is considered unusable until this step is completed, -/// so it must be waited on very early during driver initialization. -/// -/// The `GFW` code includes several components that need to execute before the driver loads. These -/// components are located in the VBIOS ROM and executed in a sequence on these different -/// microcontrollers. The devinit sequence typically runs on the PMU, and the FWSEC runs on the -/// GSP. -/// -/// This function waits for a signal indicating that core initialization is complete. Before this -/// signal is received, little can be done with the GPU. This signal is set by the FWSEC running on -/// the GSP in Heavy-secured mode. -pub(crate) fn wait_gfw_boot_completion(bar: &Bar0) -> Result { - // Before accessing the completion status in `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05`, we must - // first check `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK`. This is because - // `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05` becomes accessible only after the secure firmware - // (FWSEC) lowers the privilege level to allow CPU (LS/Light-secured) access. We can only - // safely read the status register from CPU (LS/Light-secured) once the mask indicates - // that the privilege level has been lowered. - // - // TIMEOUT: arbitrarily large value. GFW starts running immediately after the GPU is put out of - // reset, and should complete in less time than that. - read_poll_timeout( - || { - Ok( - // Check that FWSEC has lowered its protection level before reading the GFW_BOOT - // status. - bar.read(regs::NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK) - .read_protection_level0() - && bar - .read(regs::NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_0_GFW_BOOT) - .completed(), - ) - }, - |&gfw_booted| gfw_booted, - Delta::from_millis(1), - Delta::from_secs(4), - ) - .map(|_| ()) -} diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 0f6fe9a1b955..b3c91731db45 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -1,14 +1,15 @@ // SPDX-License-Identifier: GPL-2.0 +use core::ops::Range; + use kernel::{ device, - devres::Devres, + dma::Device, fmt, io::Io, num::Bounded, pci, - prelude::*, - sync::Arc, // + prelude::*, // }; use crate::{ @@ -20,11 +21,15 @@ use crate::{ Falcon, // }, fb::SysmemFlush, - gfw, - gsp::Gsp, + gsp::{ + self, + Gsp, // + }, regs, }; +mod hal; + macro_rules! define_chipset { ({ $($variant:ident = $value:expr),* $(,)* }) => { @@ -86,12 +91,23 @@ define_chipset!({ GA104 = 0x174, GA106 = 0x176, GA107 = 0x177, + // Hopper + GH100 = 0x180, // Ada AD102 = 0x192, AD103 = 0x193, AD104 = 0x194, AD106 = 0x196, AD107 = 0x197, + // Blackwell GB10x + GB100 = 0x1a0, + GB102 = 0x1a2, + // Blackwell GB20x + GB202 = 0x1b2, + GB203 = 0x1b3, + GB205 = 0x1b5, + GB206 = 0x1b6, + GB207 = 0x1b7, }); impl Chipset { @@ -103,9 +119,14 @@ impl Chipset { Self::GA100 | Self::GA102 | Self::GA103 | Self::GA104 | Self::GA106 | Self::GA107 => { Architecture::Ampere } + Self::GH100 => Architecture::Hopper, Self::AD102 | Self::AD103 | Self::AD104 | Self::AD106 | Self::AD107 => { Architecture::Ada } + Self::GB100 | Self::GB102 => Architecture::BlackwellGB10x, + Self::GB202 | Self::GB203 | Self::GB205 | Self::GB206 | Self::GB207 => { + Architecture::BlackwellGB20x + } } } @@ -115,6 +136,20 @@ impl Chipset { pub(crate) const fn needs_fwsec_bootloader(self) -> bool { matches!(self.arch(), Architecture::Turing) || matches!(self, Self::GA100) } + + /// Returns `true` if this chipset boots via FSP (Hopper and later), which requires the FMC + /// firmware image. + pub(crate) const fn uses_fsp(self) -> bool { + matches!( + self.arch(), + Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x + ) + } + + /// Returns the address range of the PCI config mirror space. + pub(crate) fn pci_config_mirror_range(self) -> Range<u32> { + hal::gpu_hal(self).pci_config_mirror_range() + } } // TODO @@ -137,10 +172,14 @@ bounded_enum! { pub(crate) enum Architecture with TryFrom<Bounded<u32, 6>> { Turing = 0x16, Ampere = 0x17, + Hopper = 0x18, Ada = 0x19, + BlackwellGB10x = 0x1a, + BlackwellGB20x = 0x1b, } } +#[derive(Clone, Copy)] pub(crate) struct Revision { major: Bounded<u8, 4>, minor: Bounded<u8, 4>, @@ -162,13 +201,14 @@ impl fmt::Display for Revision { } /// Structure holding a basic description of the GPU: `Chipset` and `Revision`. +#[derive(Clone, Copy)] pub(crate) struct Spec { chipset: Chipset, revision: Revision, } impl Spec { - fn new(dev: &device::Device, bar: &Bar0) -> Result<Spec> { + fn new(dev: &device::Device, bar: Bar0<'_>) -> Result<Spec> { // Some brief notes about boot0 and boot42, in chronological order: // // NV04 through NV50: @@ -223,14 +263,16 @@ impl fmt::Display for Spec { } /// Structure holding the resources required to operate the GPU. -#[pin_data] -pub(crate) struct Gpu { +#[pin_data(PinnedDrop)] +pub(crate) struct Gpu<'gpu> { + /// Device owning the GPU. + device: &'gpu device::Device<device::Bound>, spec: Spec, - /// MMIO mapping of PCI BAR 0 - bar: Arc<Devres<Bar0>>, + /// MMIO mapping of PCI BAR 0. + bar: Bar0<'gpu>, /// System memory page required for flushing all pending GPU-side memory writes done through /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation). - sysmem_flush: SysmemFlush, + sysmem_flush: SysmemFlush<'gpu>, /// GSP falcon instance, used for GSP boot up and cleanup. gsp_falcon: Falcon<GspFalcon>, /// SEC2 falcon instance, used for GSP boot up and cleanup. @@ -238,22 +280,31 @@ pub(crate) struct Gpu { /// GSP runtime data. Temporarily an empty placeholder. #[pin] gsp: Gsp, + /// GSP unload firmware bundle, if any. + unload_bundle: Option<gsp::UnloadBundle>, } -impl Gpu { - pub(crate) fn new<'a>( - pdev: &'a pci::Device<device::Bound>, - devres_bar: Arc<Devres<Bar0>>, - bar: &'a Bar0, - ) -> impl PinInit<Self, Error> + 'a { +impl<'gpu> Gpu<'gpu> { + pub(crate) fn new( + pdev: &'gpu pci::Device<device::Core<'_>>, + bar: Bar0<'gpu>, + ) -> impl PinInit<Self, Error> + 'gpu { try_pin_init!(Self { + device: pdev.as_ref(), spec: Spec::new(pdev.as_ref(), bar).inspect(|spec| { dev_info!(pdev,"NVIDIA ({})\n", spec); })?, // We must wait for GFW_BOOT completion before doing any significant setup on the GPU. _: { - gfw::wait_gfw_boot_completion(bar) + let hal = hal::gpu_hal(spec.chipset); + let dma_mask = hal.dma_mask(); + + // SAFETY: `Gpu` owns all DMA allocations for this device, and we are + // still constructing it, so no concurrent DMA allocations can exist. + unsafe { pdev.dma_set_mask_and_coherent(dma_mask)? }; + + hal.wait_gfw_boot_completion(bar) .inspect_err(|_| dev_err!(pdev, "GFW boot did not complete\n"))?; }, @@ -269,20 +320,28 @@ impl Gpu { gsp <- Gsp::new(pdev), - _: { gsp.boot(pdev, bar, spec.chipset, gsp_falcon, sec2_falcon)? }, - - bar: devres_bar, + // This member must be initialized last, so the `UnloadBundle` can never be dropped from + // outside of the constructed `Gpu`, ensuring that the unload sequence is properly run + // in case of failure. + unload_bundle: gsp.boot(pdev, bar, spec.chipset, gsp_falcon, sec2_falcon)?, + bar, }) } +} - /// Called when the corresponding [`Device`](device::Device) is unbound. - /// - /// Note: This method must only be called from `Driver::unbind`. - pub(crate) fn unbind(&self, dev: &device::Device<device::Core>) { - kernel::warn_on!(self - .bar - .access(dev) - .inspect(|bar| self.sysmem_flush.unregister(bar)) - .is_err()); +#[pinned_drop] +impl PinnedDrop for Gpu<'_> { + fn drop(self: Pin<&mut Self>) { + let this = self.project(); + let device = *this.device; + let bar = *this.bar; + let bundle = this.unload_bundle.take(); + + let _ = this + .gsp + .as_ref() + .get_ref() + .unload(device, bar, &*this.gsp_falcon, &*this.sec2_falcon, bundle) + .inspect_err(|e| dev_err!(device, "failed to unload GSP: {:?}\n", e)); } } diff --git a/drivers/gpu/nova-core/gpu/hal.rs b/drivers/gpu/nova-core/gpu/hal.rs new file mode 100644 index 000000000000..3f25882d0e56 --- /dev/null +++ b/drivers/gpu/nova-core/gpu/hal.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 + +use core::ops::Range; + +use kernel::{ + dma::DmaMask, + prelude::*, // +}; + +use crate::{ + driver::Bar0, + gpu::{ + Architecture, + Chipset, // + }, +}; + +mod gh100; +mod tu102; + +pub(crate) trait GpuHal { + /// Waits for GFW_BOOT completion if required by this hardware family. + fn wait_gfw_boot_completion(&self, bar: Bar0<'_>) -> Result; + + /// Returns the DMA mask for the current architecture. + fn dma_mask(&self) -> DmaMask; + + /// Returns the address range of the PCI config mirror space. + fn pci_config_mirror_range(&self) -> Range<u32>; +} + +pub(super) fn gpu_hal(chipset: Chipset) -> &'static dyn GpuHal { + match chipset.arch() { + Architecture::Turing | Architecture::Ampere | Architecture::Ada => tu102::TU102_HAL, + Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { + gh100::GH100_HAL + } + } +} diff --git a/drivers/gpu/nova-core/gpu/hal/gh100.rs b/drivers/gpu/nova-core/gpu/hal/gh100.rs new file mode 100644 index 000000000000..e3f8ba0fab33 --- /dev/null +++ b/drivers/gpu/nova-core/gpu/hal/gh100.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0 + +use core::ops::Range; + +use kernel::{ + dma::DmaMask, + prelude::*, // +}; + +use crate::driver::Bar0; + +use super::GpuHal; + +struct Gh100; + +impl GpuHal for Gh100 { + fn wait_gfw_boot_completion(&self, _bar: Bar0<'_>) -> Result { + Ok(()) + } + + fn dma_mask(&self) -> DmaMask { + DmaMask::new::<52>() + } + + fn pci_config_mirror_range(&self) -> Range<u32> { + const PCI_CONFIG_MIRROR_START: u32 = 0x092000; + const PCI_CONFIG_MIRROR_SIZE: u32 = 0x001000; + + PCI_CONFIG_MIRROR_START..PCI_CONFIG_MIRROR_START + PCI_CONFIG_MIRROR_SIZE + } +} + +const GH100: Gh100 = Gh100; +pub(super) const GH100_HAL: &dyn GpuHal = &GH100; diff --git a/drivers/gpu/nova-core/gpu/hal/tu102.rs b/drivers/gpu/nova-core/gpu/hal/tu102.rs new file mode 100644 index 000000000000..b0732e53edea --- /dev/null +++ b/drivers/gpu/nova-core/gpu/hal/tu102.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! GPU Firmware (`GFW`) support, a.k.a `devinit`. +//! +//! Upon reset, the GPU runs some firmware code from the BIOS to setup its core parameters. Most of +//! the GPU is considered unusable until this step is completed, so we must wait on it before +//! performing driver initialization. +//! +//! A clarification about devinit terminology: devinit is a sequence of register read/writes after +//! reset that performs tasks such as: +//! 1. Programming VRAM memory controller timings. +//! 2. Power sequencing. +//! 3. Clock and PLL configuration. +//! 4. Thermal management. +//! +//! devinit itself is a 'script' which is interpreted by an interpreter program typically running +//! on the PMU microcontroller. +//! +//! Note that the devinit sequence also needs to run during suspend/resume. + +use core::ops::Range; + +use kernel::{ + dma::DmaMask, + io::{ + poll::read_poll_timeout, + Io, // + }, + prelude::*, + time::Delta, // +}; + +use crate::{ + driver::Bar0, + regs, // +}; + +use super::GpuHal; + +struct Tu102; + +impl GpuHal for Tu102 { + /// Wait for the `GFW` (GPU firmware) boot completion signal (`GFW_BOOT`), or a 4 seconds + /// timeout. + /// + /// Upon GPU reset, several microcontrollers (such as PMU, SEC2, GSP etc) run some firmware + /// code to setup its core parameters. Most of the GPU is considered unusable until this step + /// is completed, so it must be waited on very early during driver initialization. + /// + /// The `GFW` code includes several components that need to execute before the driver loads. + /// These components are located in the VBIOS ROM and executed in a sequence on these different + /// microcontrollers. The devinit sequence typically runs on the PMU, and the FWSEC runs on the + /// GSP. + /// + /// This function waits for a signal indicating that core initialization is complete. Before + /// this signal is received, little can be done with the GPU. This signal is set by the FWSEC + /// running on the GSP in Heavy-secured mode. + fn wait_gfw_boot_completion(&self, bar: Bar0<'_>) -> Result { + // Before accessing the completion status in `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05`, we must + // first check `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK`. This is because + // `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05` becomes accessible only after the secure firmware + // (FWSEC) lowers the privilege level to allow CPU (LS/Light-secured) access. We can only + // safely read the status register from CPU (LS/Light-secured) once the mask indicates + // that the privilege level has been lowered. + // + // TIMEOUT: arbitrarily large value. GFW starts running immediately after the GPU is put + // out of reset, and should complete in less time than that. + read_poll_timeout( + || { + Ok( + // Check that FWSEC has lowered its protection level before reading the + // GFW_BOOT status. + bar.read(regs::NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK) + .read_protection_level0() + && bar + .read(regs::NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_0_GFW_BOOT) + .completed(), + ) + }, + |&gfw_booted| gfw_booted, + Delta::from_millis(1), + Delta::from_secs(4), + ) + .map(|_| ()) + } + + fn dma_mask(&self) -> DmaMask { + DmaMask::new::<47>() + } + + fn pci_config_mirror_range(&self) -> Range<u32> { + const PCI_CONFIG_MIRROR_START: u32 = 0x088000; + const PCI_CONFIG_MIRROR_SIZE: u32 = 0x001000; + + PCI_CONFIG_MIRROR_START..PCI_CONFIG_MIRROR_START + PCI_CONFIG_MIRROR_SIZE + } +} + +const TU102: Tu102 = Tu102; +pub(super) const TU102_HAL: &dyn GpuHal = &TU102; diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index ba5b7f990031..69175ca3315c 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 mod boot; +mod hal; use kernel::{ debugfs, @@ -24,6 +25,7 @@ mod fw; mod sequencer; pub(crate) use fw::{ + GspFmcBootParams, GspFwWprMeta, LibosParams, // }; @@ -184,3 +186,6 @@ impl Gsp { }) } } + +/// Opaque bundle required to unload the GSP. Created by [`Gsp::boot`], consumed by [`Gsp::unload`]. +pub(crate) struct UnloadBundle(KBox<dyn hal::UnloadBundle>); diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 18f356c9178e..8afb62d689cb 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -1,13 +1,15 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::{ + bits, device, dma::Coherent, io::poll::read_poll_timeout, - io::Io, pci, prelude::*, - time::Delta, // + time::Delta, + types::ScopeGuard, // }; use crate::{ @@ -19,192 +21,119 @@ use crate::{ }, fb::FbLayout, firmware::{ - booter::{ - BooterFirmware, - BooterKind, // - }, - fwsec::{ - bootloader::FwsecFirmwareWithBl, - FwsecCommand, - FwsecFirmware, // - }, gsp::GspFirmware, FIRMWARE_VERSION, // }, gpu::Chipset, gsp::{ + cmdq::Cmdq, commands, - sequencer::{ - GspSequencer, - GspSequencerParams, // - }, GspFwWprMeta, // }, - regs, - vbios::Vbios, }; -impl super::Gsp { - /// Helper function to load and run the FWSEC-FRTS firmware and confirm that it has properly - /// created the WPR2 region. - fn run_fwsec_frts( - dev: &device::Device<device::Bound>, - chipset: Chipset, - falcon: &Falcon<Gsp>, - bar: &Bar0, - bios: &Vbios, - fb_layout: &FbLayout, - ) -> Result<()> { - // Check that the WPR2 region does not already exists - if it does, we cannot run - // FWSEC-FRTS until the GPU is reset. - if bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound() != 0 { - dev_err!( - dev, - "WPR2 region already exists - GPU needs to be reset to proceed\n" - ); - return Err(EBUSY); - } - - // FWSEC-FRTS will create the WPR2 region. - let fwsec_frts = FwsecFirmware::new( - dev, - falcon, - bar, - bios, - FwsecCommand::Frts { - frts_addr: fb_layout.frts.start, - frts_size: fb_layout.frts.len(), - }, - )?; - - if chipset.needs_fwsec_bootloader() { - let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?; - // Load and run the bootloader, which will load FWSEC-FRTS and run it. - fwsec_frts_bl.run(dev, falcon, bar)?; - } else { - // Load and run FWSEC-FRTS directly. - fwsec_frts.run(dev, falcon, bar)?; - } - - // SCRATCH_E contains the error code for FWSEC-FRTS. - let frts_status = bar - .read(regs::NV_PBUS_SW_SCRATCH_0E_FRTS_ERR) - .frts_err_code(); - if frts_status != 0 { - dev_err!( - dev, - "FWSEC-FRTS returned with error code {:#x}\n", - frts_status - ); - - return Err(EIO); - } - - // Check that the WPR2 region has been created as we requested. - let (wpr2_lo, wpr2_hi) = ( - bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO).lower_bound(), - bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound(), - ); +/// Arguments required to call [`Gsp::unload`](super::Gsp::unload). +/// +/// Stored as their own type to avoid repeating a long and tedious list in [`BootUnloadGuard`]. +pub(super) struct BootUnloadArgs<'a> { + gsp: &'a super::Gsp, + dev: &'a device::Device<device::Bound>, + bar: Bar0<'a>, + gsp_falcon: &'a Falcon<Gsp>, + sec2_falcon: &'a Falcon<Sec2>, + unload_bundle: Option<super::UnloadBundle>, +} - match (wpr2_lo, wpr2_hi) { - (_, 0) => { - dev_err!(dev, "WPR2 region not created after running FWSEC-FRTS\n"); +/// Guard that calls [`Gsp::unload`](super::Gsp::unload) with a +/// [`UnloadBundle`](super::UnloadBundle) when dropped. +/// +/// Used to ensure the `UnloadBundle` is run during failure paths. +pub(super) struct BootUnloadGuard<'a> { + guard: ScopeGuard<BootUnloadArgs<'a>, fn(BootUnloadArgs<'a>)>, +} - Err(EIO) - } - (wpr2_lo, _) if wpr2_lo != fb_layout.frts.start => { - dev_err!( +impl<'a> BootUnloadGuard<'a> { + /// Wraps `unload_bundle` into a guard that executes it when dropped. + pub(super) fn new( + gsp: &'a super::Gsp, + dev: &'a device::Device<device::Bound>, + bar: Bar0<'a>, + gsp_falcon: &'a Falcon<Gsp>, + sec2_falcon: &'a Falcon<Sec2>, + unload_bundle: Option<super::UnloadBundle>, + ) -> Self { + Self { + guard: ScopeGuard::new_with_data( + BootUnloadArgs { + gsp, dev, - "WPR2 region created at unexpected address {:#x}; expected {:#x}\n", - wpr2_lo, - fb_layout.frts.start, - ); - - Err(EIO) - } - (wpr2_lo, wpr2_hi) => { - dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_lo, wpr2_hi); - dev_dbg!(dev, "GPU instance built\n"); - - Ok(()) - } + bar, + gsp_falcon, + sec2_falcon, + unload_bundle, + }, + |args| { + let _ = super::Gsp::unload( + args.gsp, + args.dev, + args.bar, + args.gsp_falcon, + args.sec2_falcon, + args.unload_bundle, + ); + }, + ), } } + /// Disarms the guard and returns the [`UnloadBundle`](super::UnloadBundle) it contains. + pub(super) fn dismiss(self) -> Option<super::UnloadBundle> { + self.guard.dismiss().unload_bundle + } +} + +impl super::Gsp { /// Attempt to boot the GSP. /// /// This is a GPU-dependent and complex procedure that involves loading firmware files from /// user-space, patching them with signatures, and building firmware-specific intricate data /// structures that the GSP will use at runtime. /// - /// Upon return, the GSP is up and running, and its runtime object given as return value. + /// Upon return, the GSP is up and running, and its unload bundle (to be given as argument to + /// [`Self::unload`]) returned. pub(crate) fn boot( self: Pin<&mut Self>, pdev: &pci::Device<device::Bound>, - bar: &Bar0, + bar: Bar0<'_>, chipset: Chipset, gsp_falcon: &Falcon<Gsp>, sec2_falcon: &Falcon<Sec2>, - ) -> Result { + ) -> Result<Option<super::UnloadBundle>> { let dev = pdev.as_ref(); - - let bios = Vbios::new(dev, bar)?; + let hal = super::hal::gsp_hal(chipset); let gsp_fw = KBox::pin_init(GspFirmware::new(dev, chipset, FIRMWARE_VERSION), GFP_KERNEL)?; let fb_layout = FbLayout::new(chipset, bar, &gsp_fw)?; dev_dbg!(dev, "{:#x?}\n", fb_layout); - Self::run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, &fb_layout)?; + let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::new(&gsp_fw, &fb_layout))?; - let booter_loader = BooterFirmware::new( + // Perform the chipset-specific boot sequence, and retrieve the unload bundle. + let unload_guard = hal.boot( + &self, dev, - BooterKind::Loader, + bar, chipset, - FIRMWARE_VERSION, + &fb_layout, + &wpr_meta, + gsp_falcon, sec2_falcon, - bar, )?; - let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::new(&gsp_fw, &fb_layout))?; - - self.cmdq - .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev))?; - self.cmdq - .send_command_no_wait(bar, commands::SetRegistry::new())?; - - gsp_falcon.reset(bar)?; - let libos_handle = self.libos.dma_handle(); - let (mbox0, mbox1) = gsp_falcon.boot( - bar, - Some(libos_handle as u32), - Some((libos_handle >> 32) as u32), - )?; - dev_dbg!(pdev, "GSP MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1); - - dev_dbg!( - pdev, - "Using SEC2 to load and run the booter_load firmware...\n" - ); - - sec2_falcon.reset(bar)?; - sec2_falcon.load(dev, bar, &booter_loader)?; - let wpr_handle = wpr_meta.dma_handle(); - let (mbox0, mbox1) = sec2_falcon.boot( - bar, - Some(wpr_handle as u32), - Some((wpr_handle >> 32) as u32), - )?; - dev_dbg!(pdev, "SEC2 MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1); - - if mbox0 != 0 { - dev_err!(pdev, "Booter-load failed with error {:#x}\n", mbox0); - return Err(ENODEV); - } - gsp_falcon.write_os_version(bar, gsp_fw.bootloader.app_version); - // Poll for RISC-V to become active before running sequencer + // Poll for RISC-V to become active before continuing. read_poll_timeout( || Ok(gsp_falcon.is_riscv_active(bar)), |val: &bool| *val, @@ -214,27 +143,84 @@ impl super::Gsp { dev_dbg!(pdev, "RISC-V active? {}\n", gsp_falcon.is_riscv_active(bar),); - // Create and run the GSP sequencer. - let seq_params = GspSequencerParams { - bootloader_app_version: gsp_fw.bootloader.app_version, - libos_dma_handle: libos_handle, - gsp_falcon, - sec2_falcon, - dev: pdev.as_ref().into(), - bar, - }; - GspSequencer::run(&self.cmdq, seq_params)?; + self.cmdq + .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?; + self.cmdq + .send_command_no_wait(bar, commands::SetRegistry::new())?; + + hal.post_boot(&self, dev, bar, &gsp_fw, gsp_falcon, sec2_falcon)?; // Wait until GSP is fully initialized. commands::wait_gsp_init_done(&self.cmdq)?; // Obtain and display basic GPU information. - let info = commands::get_gsp_info(&self.cmdq, bar)?; + let info = self.cmdq.send_command(bar, commands::GetGspStaticInfo)?; match info.gpu_name() { Ok(name) => dev_info!(pdev, "GPU name: {}\n", name), Err(e) => dev_warn!(pdev, "GPU name unavailable: {:?}\n", e), } - Ok(()) + Ok(unload_guard.dismiss()) + } + + /// Shut down the GSP and wait until it is offline. + fn shutdown_gsp( + cmdq: &Cmdq, + bar: Bar0<'_>, + gsp_falcon: &Falcon<Gsp>, + mode: commands::PowerStateLevel, + ) -> Result { + // Command to shut the GSP down. + cmdq.send_command(bar, commands::UnloadingGuestDriver::new(mode))?; + + // Wait until GSP signals it is suspended. + const LIBOS_INTERRUPT_PROCESSOR_SUSPENDED: u32 = bits::bit_u32(31); + read_poll_timeout( + || Ok(gsp_falcon.read_mailbox0(bar)), + |&mb0| mb0 & LIBOS_INTERRUPT_PROCESSOR_SUSPENDED != 0, + Delta::from_millis(10), + Delta::from_secs(5), + ) + .map(|_| ()) + } + + /// Attempts to unload the GSP firmware. + /// + /// This stops all activity on the GSP. + pub(crate) fn unload( + &self, + dev: &device::Device<device::Bound>, + bar: Bar0<'_>, + gsp_falcon: &Falcon<Gsp>, + sec2_falcon: &Falcon<Sec2>, + unload_bundle: Option<super::UnloadBundle>, + ) -> Result { + // Shut down the GSP. Keep going even in case of error. + let mut res = Self::shutdown_gsp( + &self.cmdq, + bar, + gsp_falcon, + commands::PowerStateLevel::Level0, + ) + .inspect_err(|e| dev_err!(dev, "GSP shutdown failed: {:?}\n", e)); + + // Run the unload bundle to reset the GSP so it can be booted again. + if let Some(unload_bundle) = unload_bundle { + res = res.and( + unload_bundle + .0 + .run(dev, bar, gsp_falcon, sec2_falcon) + .inspect_err(|e| dev_err!(dev, "Unload bundle failed: {:?}\n", e)), + ); + } else { + dev_warn!( + dev, + "Unload bundle is missing, GSP won't be properly reset.\n" + ); + + res = Err(EAGAIN); + } + + res.inspect(|()| dev_info!(dev, "GSP successfully unloaded\n")) } } diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 275da9b1ee0e..070de0731e95 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -237,7 +237,7 @@ impl DmaGspMem { let start = gsp_mem.dma_handle(); // Write values one by one to avoid an on-stack instance of `PteArray`. for i in 0..GspMem::PTE_ARRAY_SIZE { - dma_write!(gsp_mem, .ptes.0[i], PteArray::<0>::entry(start, i)?); + dma_write!(gsp_mem, .ptes.0[build: i], PteArray::<0>::entry(start, i)?); } dma_write!( @@ -260,7 +260,7 @@ impl DmaGspMem { let rx = self.gsp_read_ptr(); // Pointer to the first entry of the CPU message queue. - let data = ptr::project!(mut self.0.as_mut_ptr(), .cpuq.msgq.data[0]); + let data = ptr::project!(mut self.0.as_mut_ptr(), .cpuq.msgq.data[build: 0]); let (tail_end, wrap_end) = if rx == 0 { // The write area is non-wrapping, and stops at the second-to-last entry of the command @@ -322,7 +322,7 @@ impl DmaGspMem { let rx = self.cpu_read_ptr(); // Pointer to the first entry of the GSP message queue. - let data = ptr::project!(self.0.as_ptr(), .gspq.msgq.data[0]); + let data = ptr::project!(self.0.as_ptr(), .gspq.msgq.data[build: 0]); let (tail_end, wrap_end) = if rx <= tx { // Read area is non-wrapping and stops right before `tx`. @@ -532,7 +532,7 @@ impl Cmdq { } /// Notifies the GSP that we have updated the command queue pointers. - fn notify_gsp(bar: &Bar0) { + fn notify_gsp(bar: Bar0<'_>) { bar.write_reg(regs::NV_PGSP_QUEUE_HEAD::zeroed().with_address(0u32)); } @@ -552,7 +552,7 @@ impl Cmdq { /// written to by its [`CommandToGsp::init_variable_payload`] method. /// /// Error codes returned by the command and reply initializers are propagated as-is. - pub(crate) fn send_command<M>(&self, bar: &Bar0, command: M) -> Result<M::Reply> + pub(crate) fn send_command<M>(&self, bar: Bar0<'_>, command: M) -> Result<M::Reply> where M: CommandToGsp, M::Reply: MessageFromGsp, @@ -580,7 +580,7 @@ impl Cmdq { /// written to by its [`CommandToGsp::init_variable_payload`] method. /// /// Error codes returned by the command initializers are propagated as-is. - pub(crate) fn send_command_no_wait<M>(&self, bar: &Bar0, command: M) -> Result + pub(crate) fn send_command_no_wait<M>(&self, bar: Bar0<'_>, command: M) -> Result where M: CommandToGsp<Reply = NoReply>, Error: From<M::InitError>, @@ -624,7 +624,7 @@ impl CmdqInner { /// written to by its [`CommandToGsp::init_variable_payload`] method. /// /// Error codes returned by the command initializers are propagated as-is. - fn send_single_command<M>(&mut self, bar: &Bar0, command: M) -> Result + fn send_single_command<M>(&mut self, bar: Bar0<'_>, command: M) -> Result where M: CommandToGsp, // This allows all error types, including `Infallible`, to be used for `M::InitError`. @@ -694,7 +694,7 @@ impl CmdqInner { /// written to by its [`CommandToGsp::init_variable_payload`] method. /// /// Error codes returned by the command initializers are propagated as-is. - fn send_command<M>(&mut self, bar: &Bar0, command: M) -> Result + fn send_command<M>(&mut self, bar: Bar0<'_>, command: M) -> Result where M: CommandToGsp, Error: From<M::InitError>, diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index c89c7b57a751..f84de9f4f045 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use core::{ array, @@ -18,7 +19,7 @@ use kernel::{ }; use crate::{ - driver::Bar0, + gpu::Chipset, gsp::{ cmdq::{ Cmdq, @@ -27,7 +28,7 @@ use crate::{ NoReply, // }, fw::{ - commands::*, + self, MsgFunction, // }, }, @@ -37,23 +38,24 @@ use crate::{ /// The `GspSetSystemInfo` command. pub(crate) struct SetSystemInfo<'a> { pdev: &'a pci::Device<device::Bound>, + chipset: Chipset, } impl<'a> SetSystemInfo<'a> { /// Creates a new `GspSetSystemInfo` command using the parameters of `pdev`. - pub(crate) fn new(pdev: &'a pci::Device<device::Bound>) -> Self { - Self { pdev } + pub(crate) fn new(pdev: &'a pci::Device<device::Bound>, chipset: Chipset) -> Self { + Self { pdev, chipset } } } impl<'a> CommandToGsp for SetSystemInfo<'a> { const FUNCTION: MsgFunction = MsgFunction::GspSetSystemInfo; - type Command = GspSetSystemInfo; + type Command = fw::commands::GspSetSystemInfo; type Reply = NoReply; type InitError = Error; fn init(&self) -> impl Init<Self::Command, Self::InitError> { - GspSetSystemInfo::init(self.pdev) + Self::Command::init(self.pdev, self.chipset) } } @@ -100,12 +102,12 @@ impl SetRegistry { impl CommandToGsp for SetRegistry { const FUNCTION: MsgFunction = MsgFunction::SetRegistry; - type Command = PackedRegistryTable; + type Command = fw::commands::PackedRegistryTable; type Reply = NoReply; type InitError = Infallible; fn init(&self) -> impl Init<Self::Command, Self::InitError> { - PackedRegistryTable::init(Self::NUM_ENTRIES as u32, self.variable_payload_len() as u32) + Self::Command::init(Self::NUM_ENTRIES as u32, self.variable_payload_len() as u32) } fn variable_payload_len(&self) -> usize { @@ -113,22 +115,22 @@ impl CommandToGsp for SetRegistry { for i in 0..Self::NUM_ENTRIES { key_size += self.entries[i].key.len() + 1; // +1 for NULL terminator } - Self::NUM_ENTRIES * size_of::<PackedRegistryEntry>() + key_size + Self::NUM_ENTRIES * size_of::<fw::commands::PackedRegistryEntry>() + key_size } fn init_variable_payload( &self, dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>, ) -> Result { - let string_data_start_offset = - size_of::<PackedRegistryTable>() + Self::NUM_ENTRIES * size_of::<PackedRegistryEntry>(); + let string_data_start_offset = size_of::<Self::Command>() + + Self::NUM_ENTRIES * size_of::<fw::commands::PackedRegistryEntry>(); // Array for string data. let mut string_data = KVec::new(); for entry in self.entries.iter().take(Self::NUM_ENTRIES) { dst.write_all( - PackedRegistryEntry::new( + fw::commands::PackedRegistryEntry::new( (string_data_start_offset + string_data.len()) as u32, entry.value, ) @@ -176,16 +178,16 @@ pub(crate) fn wait_gsp_init_done(cmdq: &Cmdq) -> Result { } /// The `GetGspStaticInfo` command. -struct GetGspStaticInfo; +pub(crate) struct GetGspStaticInfo; impl CommandToGsp for GetGspStaticInfo { const FUNCTION: MsgFunction = MsgFunction::GetGspStaticInfo; - type Command = GspStaticConfigInfo; + type Command = fw::commands::GspStaticConfigInfo; type Reply = GetGspStaticInfoReply; type InitError = Infallible; fn init(&self) -> impl Init<Self::Command, Self::InitError> { - GspStaticConfigInfo::init_zeroed() + Self::Command::init_zeroed() } } @@ -196,7 +198,7 @@ pub(crate) struct GetGspStaticInfoReply { impl MessageFromGsp for GetGspStaticInfoReply { const FUNCTION: MsgFunction = MsgFunction::GetGspStaticInfo; - type Message = GspStaticConfigInfo; + type Message = fw::commands::GspStaticConfigInfo; type InitError = Infallible; fn read( @@ -233,7 +235,45 @@ impl GetGspStaticInfoReply { } } -/// Send the [`GetGspInfo`] command and awaits for its reply. -pub(crate) fn get_gsp_info(cmdq: &Cmdq, bar: &Bar0) -> Result<GetGspStaticInfoReply> { - cmdq.send_command(bar, GetGspStaticInfo) +pub(crate) use fw::commands::PowerStateLevel; + +/// The `UnloadingGuestDriver` command, used to shut down the GSP. +/// +/// Only used within the `gsp` module. +pub(super) struct UnloadingGuestDriver { + level: PowerStateLevel, +} + +impl UnloadingGuestDriver { + /// Creates a new `UnloadingGuestDriver` command for the given [`PowerStateLevel`]. + pub(super) fn new(level: PowerStateLevel) -> Self { + Self { level } + } +} + +impl CommandToGsp for UnloadingGuestDriver { + const FUNCTION: MsgFunction = MsgFunction::UnloadingGuestDriver; + type Command = fw::commands::UnloadingGuestDriver; + type Reply = UnloadingGuestDriverReply; + type InitError = Infallible; + + fn init(&self) -> impl Init<Self::Command, Self::InitError> { + fw::commands::UnloadingGuestDriver::new(self.level) + } +} + +/// The reply from the GSP to the [`UnloadingGuestDriver`] command. +pub(super) struct UnloadingGuestDriverReply; + +impl MessageFromGsp for UnloadingGuestDriverReply { + const FUNCTION: MsgFunction = MsgFunction::UnloadingGuestDriver; + type InitError = Infallible; + type Message = (); + + fn read( + _msg: &Self::Message, + _sbuffer: &mut SBufferIter<array::IntoIter<&[u8], 2>>, + ) -> Result<Self, Self::InitError> { + Ok(UnloadingGuestDriverReply) + } } diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 0c8a74f0e8ac..4db0cfa4dc4d 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. pub(crate) mod commands; mod r570_144; @@ -17,8 +18,8 @@ use kernel::{ KnownSize, // }, sizes::{ - SZ_128K, - SZ_1M, // + SizeConstants, + SZ_128K, // }, transmute::{ AsBytes, @@ -29,7 +30,10 @@ use kernel::{ use crate::{ fb::FbLayout, firmware::gsp::GspFirmware, - gpu::Chipset, + gpu::{ + Architecture, + Chipset, // + }, gsp::{ cmdq::Cmdq, // GSP_PAGE_SIZE, @@ -106,11 +110,15 @@ const GSP_HEAP_ALIGNMENT: Alignment = Alignment::new::<{ 1 << 20 }>(); impl GspFwHeapParams { /// Returns the amount of GSP-RM heap memory used during GSP-RM boot and initialization (up to /// and including the first client subdevice allocation). - fn base_rm_size(_chipset: Chipset) -> u64 { - // TODO: this needs to be updated to return the correct value for Hopper+ once support for - // them is added: - // u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_GH100) - u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_TU10X) + fn base_rm_size(chipset: Chipset) -> u64 { + match chipset.arch() { + Architecture::Turing | Architecture::Ampere | Architecture::Ada => { + u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_TU10X) + } + Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { + u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_GH100) + } + } } /// Returns the amount of heap memory required to support a single channel allocation. @@ -122,13 +130,14 @@ impl GspFwHeapParams { /// Returns the amount of memory to reserve for management purposes for a framebuffer of size /// `fb_size`. - fn management_overhead(fb_size: u64) -> u64 { - let fb_size_gb = fb_size.div_ceil(u64::from_safe_cast(kernel::sizes::SZ_1G)); + fn management_overhead(fb_size: u64) -> Result<u64> { + let fb_size_gb = fb_size.div_ceil(u64::SZ_1G); u64::from(bindings::GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB) - .saturating_mul(fb_size_gb) + .checked_mul(fb_size_gb) + .ok_or(EINVAL)? .align_up(GSP_HEAP_ALIGNMENT) - .unwrap_or(u64::MAX) + .ok_or(EINVAL) } } @@ -145,9 +154,8 @@ impl LibosParams { const LIBOS2: LibosParams = LibosParams { carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS2), allowed_heap_size: num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MIN_MB) - * num::usize_as_u64(SZ_1M) - ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MAX_MB) - * num::usize_as_u64(SZ_1M), + * u64::SZ_1M + ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MAX_MB) * u64::SZ_1M, }; /// Version 3 of the GSP LIBOS (GA102+) @@ -155,9 +163,9 @@ impl LibosParams { carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS3_BAREMETAL), allowed_heap_size: num::u32_as_u64( bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MIN_MB, - ) * num::usize_as_u64(SZ_1M) + ) * u64::SZ_1M ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MAX_MB) - * num::usize_as_u64(SZ_1M), + * u64::SZ_1M, }; /// Returns the libos parameters corresponding to `chipset`. @@ -171,18 +179,19 @@ impl LibosParams { /// Returns the amount of memory (in bytes) to allocate for the WPR heap for a framebuffer size /// of `fb_size` (in bytes) for `chipset`. - pub(crate) fn wpr_heap_size(&self, chipset: Chipset, fb_size: u64) -> u64 { + pub(crate) fn wpr_heap_size(&self, chipset: Chipset, fb_size: u64) -> Result<u64> { // The WPR heap will contain the following: // LIBOS carveout, - self.carveout_size + Ok(self + .carveout_size // RM boot working memory, .saturating_add(GspFwHeapParams::base_rm_size(chipset)) // One RM client, .saturating_add(GspFwHeapParams::client_alloc_size()) // Overhead for memory management. - .saturating_add(GspFwHeapParams::management_overhead(fb_size)) + .saturating_add(GspFwHeapParams::management_overhead(fb_size)?) // Clamp to the supported heap sizes. - .clamp(self.allowed_heap_size.start, self.allowed_heap_size.end - 1) + .clamp(self.allowed_heap_size.start, self.allowed_heap_size.end - 1)) } } @@ -246,6 +255,7 @@ impl GspFwWprMeta { fbSize: fb_layout.fb.end - fb_layout.fb.start, vgaWorkspaceOffset: fb_layout.vga_workspace.start, vgaWorkspaceSize: fb_layout.vga_workspace.end - fb_layout.vga_workspace.start, + pmuReservedSize: fb_layout.pmu_reserved_size, ..Zeroable::init_zeroed() }); @@ -278,6 +288,7 @@ pub(crate) enum MsgFunction { Nop = bindings::NV_VGPU_MSG_FUNCTION_NOP, SetGuestSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO, SetRegistry = bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY, + UnloadingGuestDriver = bindings::NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER, // Event codes GspInitDone = bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE, @@ -322,6 +333,9 @@ impl TryFrom<u32> for MsgFunction { Ok(MsgFunction::SetGuestSystemInfo) } bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY => Ok(MsgFunction::SetRegistry), + bindings::NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER => { + Ok(MsgFunction::UnloadingGuestDriver) + } // Event codes bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE => Ok(MsgFunction::GspInitDone), @@ -920,3 +934,68 @@ impl MessageQueueInitArguments { }) } } + +#[repr(u32)] +pub(crate) enum GspDmaTarget { + #[expect(dead_code)] + LocalFb = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_LOCAL_FB, + CoherentSystem = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_COHERENT_SYSTEM, + NoncoherentSystem = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_NONCOHERENT_SYSTEM, +} + +type GspAcrBootGspRmParams = bindings::GSP_ACR_BOOT_GSP_RM_PARAMS; + +impl GspAcrBootGspRmParams { + fn new(target: GspDmaTarget, wpr_meta_addr: u64) -> impl Init<Self> { + #[allow(non_snake_case)] + let params = init!(Self { + target: target as u32, + gspRmDescSize: num::usize_into_u32::<{ size_of::<GspFwWprMeta>() }>(), + gspRmDescOffset: wpr_meta_addr, + bIsGspRmBoot: 1, + wprCarveoutOffset: 0, + wprCarveoutSize: 0, + __bindgen_padding_0: Default::default(), + }); + + params + } +} + +type GspRmParams = bindings::GSP_RM_PARAMS; + +impl GspRmParams { + fn new(target: GspDmaTarget, libos_addr: u64) -> impl Init<Self> { + #[allow(non_snake_case)] + let params = init!(Self { + target: target as u32, + bootArgsOffset: libos_addr, + __bindgen_padding_0: Default::default(), + }); + + params + } +} + +pub(crate) type GspFmcBootParams = bindings::GSP_FMC_BOOT_PARAMS; + +// SAFETY: Padding is explicit and will not contain uninitialized data. +unsafe impl AsBytes for GspFmcBootParams {} +// SAFETY: This struct only contains integer types for which all bit patterns are valid. +unsafe impl FromBytes for GspFmcBootParams {} + +impl GspFmcBootParams { + pub(crate) fn new(wpr_meta_addr: u64, libos_addr: u64) -> impl Init<Self> { + #[allow(non_snake_case)] + let init = init!(Self { + // Blackwell FSP obtains WPR info from other sources, so + // wprCarveoutOffset and wprCarveoutSize are left zero. + bootGspRmParams <- GspAcrBootGspRmParams::new(GspDmaTarget::CoherentSystem, + wpr_meta_addr), + gspRmParams <- GspRmParams::new(GspDmaTarget::NoncoherentSystem, libos_addr), + ..Zeroable::init_zeroed() + }); + + init + } +} diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs index db46276430be..7bcc41fc7fa0 100644 --- a/drivers/gpu/nova-core/gsp/fw/commands.rs +++ b/drivers/gpu/nova-core/gsp/fw/commands.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::{ device, @@ -10,7 +11,10 @@ use kernel::{ }, // }; -use crate::gsp::GSP_PAGE_SIZE; +use crate::{ + gpu::Chipset, + gsp::GSP_PAGE_SIZE, // +}; use super::bindings; @@ -24,8 +28,12 @@ static_assert!(size_of::<GspSetSystemInfo>() < GSP_PAGE_SIZE); impl GspSetSystemInfo { /// Returns an in-place initializer for the `GspSetSystemInfo` command. #[allow(non_snake_case)] - pub(crate) fn init<'a>(dev: &'a pci::Device<device::Bound>) -> impl Init<Self, Error> + 'a { + pub(crate) fn init<'a>( + dev: &'a pci::Device<device::Bound>, + chipset: Chipset, + ) -> impl Init<Self, Error> + 'a { type InnerGspSystemInfo = bindings::GspSystemInfo; + let pci_config_mirror_range = chipset.pci_config_mirror_range(); let init_inner = try_init!(InnerGspSystemInfo { gpuPhysAddr: dev.resource_start(0)?, gpuPhysFbAddr: dev.resource_start(1)?, @@ -35,8 +43,8 @@ impl GspSetSystemInfo { // Using TASK_SIZE in r535_gsp_rpc_set_system_info() seems wrong because // TASK_SIZE is per-task. That's probably a design issue in GSP-RM though. maxUserVa: (1 << 47) - 4096, - pciConfigMirrorBase: 0x088000, - pciConfigMirrorSize: 0x001000, + pciConfigMirrorBase: pci_config_mirror_range.start, + pciConfigMirrorSize: pci_config_mirror_range.end - pci_config_mirror_range.start, PCIDeviceID: (u32::from(dev.device_id()) << 16) | u32::from(dev.vendor_id().as_raw()), PCISubDeviceID: (u32::from(dev.subsystem_device_id()) << 16) @@ -129,3 +137,47 @@ unsafe impl AsBytes for GspStaticConfigInfo {} // SAFETY: This struct only contains integer types for which all bit patterns // are valid. unsafe impl FromBytes for GspStaticConfigInfo {} + +/// Power level requested to the [`UnloadingGuestDriver`] command. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u32)] +#[expect(unused)] +pub(crate) enum PowerStateLevel { + /// Full unload. + Level0 = bindings::NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0, + /// S3 (suspend to RAM). + Level3 = bindings::NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3, + /// Hibernate (suspend to disk). + Level7 = bindings::NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_7, +} + +impl PowerStateLevel { + /// Returns `true` if this state represents a power management transition, i.e. some GPU state + /// must survive it (as opposed to a full unload). + pub(crate) fn is_power_transition(self) -> bool { + self != PowerStateLevel::Level0 + } +} + +/// Payload of the `UnloadingGuestDriver` command and message. +#[repr(transparent)] +#[derive(Clone, Copy, Debug, Zeroable)] +pub(crate) struct UnloadingGuestDriver(bindings::rpc_unloading_guest_driver_v1F_07); + +impl UnloadingGuestDriver { + pub(crate) fn new(level: PowerStateLevel) -> Self { + Self(bindings::rpc_unloading_guest_driver_v1F_07 { + bInPMTransition: u8::from(level.is_power_transition()), + bGc6Entering: 0, + newLevel: level as u32, + ..Zeroable::zeroed() + }) + } +} + +// SAFETY: Padding is explicit and will not contain uninitialized data. +unsafe impl AsBytes for UnloadingGuestDriver {} + +// SAFETY: This struct only contains integer types for which all bit patterns +// are valid. +unsafe impl FromBytes for UnloadingGuestDriver {} diff --git a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs index 334e8be5fde8..ea350f9b2cc4 100644 --- a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs +++ b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs @@ -30,10 +30,14 @@ impl<T> ::core::fmt::Debug for __IncompleteArrayField<T> { fmt.write_str("__IncompleteArrayField") } } +pub const NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0: u32 = 0; +pub const NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3: u32 = 3; +pub const NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_7: u32 = 7; pub const NV_VGPU_MSG_SIGNATURE_VALID: u32 = 1129337430; pub const GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS2: u32 = 0; pub const GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS3_BAREMETAL: u32 = 23068672; pub const GSP_FW_HEAP_PARAM_BASE_RM_SIZE_TU10X: u32 = 8388608; +pub const GSP_FW_HEAP_PARAM_BASE_RM_SIZE_GH100: u32 = 14680064; pub const GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB: u32 = 98304; pub const GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE: u32 = 100663296; pub const GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MIN_MB: u32 = 64; @@ -879,6 +883,96 @@ impl Default for GSP_MSG_QUEUE_ELEMENT { } } } +pub const GSP_DMA_TARGET_GSP_DMA_TARGET_LOCAL_FB: GSP_DMA_TARGET = 0; +pub const GSP_DMA_TARGET_GSP_DMA_TARGET_COHERENT_SYSTEM: GSP_DMA_TARGET = 1; +pub const GSP_DMA_TARGET_GSP_DMA_TARGET_NONCOHERENT_SYSTEM: GSP_DMA_TARGET = 2; +pub const GSP_DMA_TARGET_GSP_DMA_TARGET_COUNT: GSP_DMA_TARGET = 3; +pub type GSP_DMA_TARGET = ffi::c_uint; +#[repr(C)] +#[derive(Debug, Default, Copy, Clone, MaybeZeroable)] +pub struct GSP_FMC_INIT_PARAMS { + pub regkeys: u32_, +} +#[repr(C)] +#[derive(Debug, Copy, Clone, MaybeZeroable)] +pub struct GSP_ACR_BOOT_GSP_RM_PARAMS { + pub target: GSP_DMA_TARGET, + pub gspRmDescSize: u32_, + pub gspRmDescOffset: u64_, + pub wprCarveoutOffset: u64_, + pub wprCarveoutSize: u32_, + pub bIsGspRmBoot: u8_, + pub __bindgen_padding_0: [u8; 3usize], +} +impl Default for GSP_ACR_BOOT_GSP_RM_PARAMS { + fn default() -> Self { + let mut s = ::core::mem::MaybeUninit::<Self>::uninit(); + unsafe { + ::core::ptr::write_bytes(s.as_mut_ptr(), 0, 1); + s.assume_init() + } + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone, MaybeZeroable)] +pub struct GSP_RM_PARAMS { + pub target: GSP_DMA_TARGET, + pub __bindgen_padding_0: [u8; 4usize], + pub bootArgsOffset: u64_, +} +impl Default for GSP_RM_PARAMS { + fn default() -> Self { + let mut s = ::core::mem::MaybeUninit::<Self>::uninit(); + unsafe { + ::core::ptr::write_bytes(s.as_mut_ptr(), 0, 1); + s.assume_init() + } + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone, MaybeZeroable)] +pub struct GSP_SPDM_PARAMS { + pub target: GSP_DMA_TARGET, + pub __bindgen_padding_0: [u8; 4usize], + pub payloadBufferOffset: u64_, + pub payloadBufferSize: u32_, + pub __bindgen_padding_1: [u8; 4usize], +} +impl Default for GSP_SPDM_PARAMS { + fn default() -> Self { + let mut s = ::core::mem::MaybeUninit::<Self>::uninit(); + unsafe { + ::core::ptr::write_bytes(s.as_mut_ptr(), 0, 1); + s.assume_init() + } + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone, MaybeZeroable)] +pub struct GSP_FMC_BOOT_PARAMS { + pub initParams: GSP_FMC_INIT_PARAMS, + pub __bindgen_padding_0: [u8; 4usize], + pub bootGspRmParams: GSP_ACR_BOOT_GSP_RM_PARAMS, + pub gspRmParams: GSP_RM_PARAMS, + pub gspSpdmParams: GSP_SPDM_PARAMS, +} +impl Default for GSP_FMC_BOOT_PARAMS { + fn default() -> Self { + let mut s = ::core::mem::MaybeUninit::<Self>::uninit(); + unsafe { + ::core::ptr::write_bytes(s.as_mut_ptr(), 0, 1); + s.assume_init() + } + } +} +#[repr(C)] +#[derive(Debug, Default, Copy, Clone, MaybeZeroable)] +pub struct rpc_unloading_guest_driver_v1F_07 { + pub bInPMTransition: u8_, + pub bGc6Entering: u8_, + pub __bindgen_padding_0: [u8; 2usize], + pub newLevel: u32_, +} #[repr(C)] #[derive(Debug, Default, MaybeZeroable)] pub struct rpc_run_cpu_sequencer_v17_00 { diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs new file mode 100644 index 000000000000..04f004856c60 --- /dev/null +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +mod gh100; +mod tu102; + +use kernel::prelude::*; + +use kernel::{ + device, + dma::Coherent, // +}; + +use crate::{ + driver::Bar0, + falcon::{ + gsp::Gsp as GspEngine, + sec2::Sec2, + Falcon, // + }, + fb::FbLayout, + firmware::gsp::GspFirmware, + gpu::{ + Architecture, + Chipset, // + }, + gsp::{ + boot::BootUnloadGuard, + Gsp, + GspFwWprMeta, // + }, +}; + +/// Trait for types containing the resources and code required to fully reset the GSP. +/// +/// The GSP unload code might run in a situation where we cannot load firmware dynamically (e.g. +/// because we are in shutdown and the file system is not accessible anymore). Thus, the firmware +/// required for unloading is prepared at load time, and stored here until it needs to be run. +pub(super) trait UnloadBundle: Send { + /// Performs the steps required to properly reset the GSP after it has been stopped. + fn run( + &self, + dev: &device::Device<device::Bound>, + bar: Bar0<'_>, + gsp_falcon: &Falcon<GspEngine>, + sec2_falcon: &Falcon<Sec2>, + ) -> Result; +} + +/// Trait implemented by GSP HALs. +pub(super) trait GspHal: Send { + /// Performs the GSP boot process, loading and running the required firmwares as needed. + /// + /// Upon success, returns a guard that runs the GSP unload sequence if GSP boot does not + /// complete. + #[allow(clippy::too_many_arguments)] + fn boot<'a>( + &self, + gsp: &'a Gsp, + dev: &'a device::Device<device::Bound>, + bar: Bar0<'a>, + chipset: Chipset, + fb_layout: &FbLayout, + wpr_meta: &Coherent<GspFwWprMeta>, + gsp_falcon: &'a Falcon<GspEngine>, + sec2_falcon: &'a Falcon<Sec2>, + ) -> Result<BootUnloadGuard<'a>>; + + /// Performs HAL-specific post-GSP boot tasks. + /// + /// This method is called by the GSP boot code after the GSP is confirmed to be running, and + /// after the initialization commands have been pushed onto its queue. + fn post_boot( + &self, + _gsp: &Gsp, + _dev: &device::Device<device::Bound>, + _bar: Bar0<'_>, + _gsp_fw: &GspFirmware, + _gsp_falcon: &Falcon<GspEngine>, + _sec2_falcon: &Falcon<Sec2>, + ) -> Result { + Ok(()) + } +} + +/// Returns the GSP HAL to be used for `chipset`. +pub(super) fn gsp_hal(chipset: Chipset) -> &'static dyn GspHal { + match chipset.arch() { + Architecture::Turing | Architecture::Ampere | Architecture::Ada => tu102::TU102_HAL, + Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { + gh100::GH100_HAL + } + } +} diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs new file mode 100644 index 000000000000..98f5ce197d13 --- /dev/null +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use kernel::prelude::*; + +use kernel::{ + device, + dma::Coherent, + io::poll::read_poll_timeout, + time::Delta, // +}; + +use crate::{ + driver::Bar0, + falcon::{ + gsp::Gsp as GspEngine, + sec2::Sec2, + Falcon, // + }, + fb::FbLayout, + firmware::{ + fsp::FspFirmware, + FIRMWARE_VERSION, // + }, + fsp::{ + FmcBootArgs, + Fsp, // + }, + gpu::Chipset, + gsp::{ + boot::BootUnloadGuard, + hal::{ + GspHal, + UnloadBundle, // + }, + Gsp, + GspFwWprMeta, // + }, +}; + +/// GSP falcon mailbox state, used to track lockdown release status. +struct GspMbox { + mbox0: u32, + mbox1: u32, +} + +impl GspMbox { + /// Reads both mailboxes from the GSP falcon. + fn read(gsp_falcon: &Falcon<GspEngine>, bar: Bar0<'_>) -> Self { + Self { + mbox0: gsp_falcon.read_mailbox0(bar), + mbox1: gsp_falcon.read_mailbox1(bar), + } + } + + /// Combines mailbox0 and mailbox1 into a 64-bit address. + fn combined_addr(&self) -> u64 { + (u64::from(self.mbox1) << 32) | u64::from(self.mbox0) + } + + /// Returns `true` if GSP lockdown has been released or a GSP-FMC error happened. + /// + /// Returns `true` both on successful lockdown release and on GSP-FMC-reported errors, since + /// either condition should stop the poll loop. + fn lockdown_released_or_error( + &self, + gsp_falcon: &Falcon<GspEngine>, + bar: Bar0<'_>, + fmc_boot_params_addr: u64, + ) -> bool { + // GSP-FMC normally clears the boot parameters address from the mailboxes early during + // boot. If the address is still there, keep polling rather than treating it as an error. + // Any other non-zero mailbox0 value is a GSP-FMC error code. + if self.mbox0 != 0 { + return self.combined_addr() != fmc_boot_params_addr; + } + + !gsp_falcon.riscv_branch_privilege_lockdown(bar) + } +} + +/// Waits for GSP lockdown to be released after FSP Chain of Trust. +fn wait_for_gsp_lockdown_release( + dev: &device::Device<device::Bound>, + bar: Bar0<'_>, + gsp_falcon: &Falcon<GspEngine>, + fmc_boot_params_addr: u64, +) -> Result { + dev_dbg!(dev, "Waiting for GSP lockdown release\n"); + + let mbox = read_poll_timeout( + || { + // While the PRIV target mask is still locked to FSP, GSP register and mailbox reads + // are not meaningful. Wait until HWCFG2 says the CPU can read them. + Ok(match gsp_falcon.priv_target_mask_released(bar) { + false => None, + true => Some(GspMbox::read(gsp_falcon, bar)), + }) + }, + |mbox| match mbox { + None => false, + Some(mbox) => mbox.lockdown_released_or_error(gsp_falcon, bar, fmc_boot_params_addr), + }, + Delta::from_millis(10), + Delta::from_secs(30), + ) + .inspect_err(|_| { + dev_err!(dev, "GSP lockdown release timeout\n"); + })? + .ok_or(EIO)?; + + // If polling stopped with a non-zero mailbox0, it was not the boot parameters address + // anymore and therefore represents a GSP-FMC error code. + if mbox.mbox0 != 0 { + dev_err!(dev, "GSP-FMC boot failed (mbox: {:#x})\n", mbox.mbox0); + return Err(EIO); + } + + dev_dbg!(dev, "GSP lockdown released\n"); + Ok(()) +} + +struct FspUnloadBundle; + +impl UnloadBundle for FspUnloadBundle { + fn run( + &self, + dev: &device::Device<device::Bound>, + bar: Bar0<'_>, + gsp_falcon: &Falcon<GspEngine>, + _sec2_falcon: &Falcon<Sec2>, + ) -> Result { + // GSP falcon does most of the work of resetting, so just wait for it to finish. + read_poll_timeout( + || Ok(gsp_falcon.is_riscv_active(bar)), + |&active| !active, + Delta::from_millis(10), + Delta::from_secs(5), + ) + .map(|_| ()) + .inspect_err(|_| dev_err!(dev, "GSP falcon failed to halt\n")) + } +} + +struct Gh100; + +impl GspHal for Gh100 { + /// Boot GSP via FSP Chain of Trust (Hopper/Blackwell+ path). + /// + /// This path uses FSP to establish a chain of trust and boot GSP-FMC. FSP handles + /// the GSP boot internally - no manual GSP reset/boot is needed. + fn boot<'a>( + &self, + gsp: &'a Gsp, + dev: &'a device::Device<device::Bound>, + bar: Bar0<'a>, + chipset: Chipset, + fb_layout: &FbLayout, + wpr_meta: &Coherent<GspFwWprMeta>, + gsp_falcon: &'a Falcon<GspEngine>, + sec2_falcon: &'a Falcon<Sec2>, + ) -> Result<BootUnloadGuard<'a>> { + let fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?; + + let unload_bundle = crate::gsp::UnloadBundle( + KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox<dyn UnloadBundle> + ); + + // Wrap the unload bundle into a drop guard so it is automatically run upon failure. + let unload_guard = + BootUnloadGuard::new(gsp, dev, bar, gsp_falcon, sec2_falcon, Some(unload_bundle)); + + let mut fsp = Fsp::wait_secure_boot(dev, bar, chipset, fsp_fw)?; + + let args = FmcBootArgs::new( + dev, + chipset, + wpr_meta.dma_handle(), + gsp.libos.dma_handle(), + false, + )?; + + fsp.boot_fmc(dev, bar, fb_layout, &args)?; + + wait_for_gsp_lockdown_release(dev, bar, gsp_falcon, args.boot_params_dma_handle())?; + + Ok(unload_guard) + } +} + +const GH100: Gh100 = Gh100; +pub(super) const GH100_HAL: &dyn GspHal = &GH100; diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs new file mode 100644 index 000000000000..2f6301af7113 --- /dev/null +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use kernel::prelude::*; + +use kernel::{ + device, + dma::Coherent, + io::Io, // +}; + +use crate::{ + driver::Bar0, + falcon::{ + gsp::Gsp as GspEngine, + sec2::Sec2, + Falcon, // + }, + fb::FbLayout, + firmware::{ + booter::{ + BooterFirmware, + BooterKind, // + }, + fwsec::{ + bootloader::FwsecFirmwareWithBl, + FwsecCommand, + FwsecFirmware, // + }, + gsp::GspFirmware, + FIRMWARE_VERSION, // + }, + gpu::Chipset, + gsp::{ + boot::BootUnloadGuard, + hal::{ + GspHal, + UnloadBundle, // + }, + sequencer::{ + GspSequencer, + GspSequencerParams, // + }, + Gsp, + GspFwWprMeta, // + }, + regs, + vbios::Vbios, // +}; + +// A ready-to-run FWSEC unload firmware. +// +// Since there are two variants of the prepared firmware (with and without a bootloader), this type +// abstracts the difference. +enum FwsecUnloadFirmware { + WithoutBl(FwsecFirmware), + WithBl(FwsecFirmwareWithBl), +} + +impl FwsecUnloadFirmware { + /// Loads the FWSEC SB firmware, as well as its bootloader if `chipset` requires it. + fn new( + dev: &device::Device<device::Bound>, + bar: Bar0<'_>, + chipset: Chipset, + bios: &Vbios, + gsp_falcon: &Falcon<GspEngine>, + ) -> Result<Self> { + let fwsec_sb = FwsecFirmware::new(dev, gsp_falcon, bar, bios, FwsecCommand::Sb)?; + + Ok(if chipset.needs_fwsec_bootloader() { + Self::WithBl(FwsecFirmwareWithBl::new(fwsec_sb, dev, chipset)?) + } else { + Self::WithoutBl(fwsec_sb) + }) + } + + /// Runs the FWSEC SB firmware. + fn run( + &self, + dev: &device::Device<device::Bound>, + bar: Bar0<'_>, + gsp_falcon: &Falcon<GspEngine>, + ) -> Result { + match self { + Self::WithoutBl(fw) => fw.run(dev, gsp_falcon, bar), + Self::WithBl(fw) => fw.run(dev, gsp_falcon, bar), + } + } +} + +// Contains the firmware required to fully reset GSP on chipsets where the GSP is started using +// FWSEC/Booter. +struct Sec2UnloadBundle { + fwsec_sb: FwsecUnloadFirmware, + booter_unloader: BooterFirmware, +} + +impl Sec2UnloadBundle { + /// Load and prepare the resources required to properly reset the GSP after it has been stopped. + fn build( + dev: &device::Device<device::Bound>, + bar: Bar0<'_>, + chipset: Chipset, + bios: &Vbios, + gsp_falcon: &Falcon<GspEngine>, + sec2_falcon: &Falcon<Sec2>, + ) -> Result<KBox<dyn UnloadBundle>> { + KBox::new( + Self { + fwsec_sb: FwsecUnloadFirmware::new(dev, bar, chipset, bios, gsp_falcon)?, + booter_unloader: BooterFirmware::new( + dev, + BooterKind::Unloader, + chipset, + FIRMWARE_VERSION, + sec2_falcon, + bar, + )?, + }, + GFP_KERNEL, + ) + .map(|b| b as KBox<dyn UnloadBundle>) + .map_err(Into::into) + } +} + +impl UnloadBundle for Sec2UnloadBundle { + fn run( + &self, + dev: &device::Device<device::Bound>, + bar: Bar0<'_>, + gsp_falcon: &Falcon<GspEngine>, + sec2_falcon: &Falcon<Sec2>, + ) -> Result { + // Run FWSEC-SB to reset the GSP falcon to its pre-libos state. + self.fwsec_sb.run(dev, bar, gsp_falcon)?; + + // Remove WPR2 region if set. + let wpr2_hi = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI); + if wpr2_hi.is_wpr2_set() { + sec2_falcon.reset(bar)?; + sec2_falcon.load(dev, bar, &self.booter_unloader)?; + + // Sentinel value to confirm that Booter Unloader has run. + const MAILBOX_SENTINEL: u32 = 0xff; + let (mbox0, _) = + sec2_falcon.boot(bar, Some(MAILBOX_SENTINEL), Some(MAILBOX_SENTINEL))?; + if mbox0 != 0 { + dev_err!(dev, "Booter Unloader returned error 0x{:x}\n", mbox0); + return Err(EINVAL); + } + + // Confirm that the WPR2 region has been removed. + let wpr2_hi = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI); + if wpr2_hi.is_wpr2_set() { + dev_err!( + dev, + "WPR2 region still set after Booter Unloader returned\n" + ); + return Err(EBUSY); + } + } + + Ok(()) + } +} + +/// Helper function to load and run the FWSEC-FRTS firmware and confirm that it has properly +/// created the WPR2 region. +fn run_fwsec_frts( + dev: &device::Device<device::Bound>, + chipset: Chipset, + falcon: &Falcon<GspEngine>, + bar: Bar0<'_>, + bios: &Vbios, + fb_layout: &FbLayout, +) -> Result { + // Check that the WPR2 region does not already exist - if it does, we cannot run + // FWSEC-FRTS until the GPU is reset. + if bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound() != 0 { + dev_err!( + dev, + "WPR2 region already exists - GPU needs to be reset to proceed\n" + ); + return Err(EBUSY); + } + + // FWSEC-FRTS will create the WPR2 region. + let fwsec_frts = FwsecFirmware::new( + dev, + falcon, + bar, + bios, + FwsecCommand::Frts { + frts_addr: fb_layout.frts.start, + frts_size: fb_layout.frts.len(), + }, + )?; + + if chipset.needs_fwsec_bootloader() { + let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?; + // Load and run the bootloader, which will load FWSEC-FRTS and run it. + fwsec_frts_bl.run(dev, falcon, bar)?; + } else { + // Load and run FWSEC-FRTS directly. + fwsec_frts.run(dev, falcon, bar)?; + } + + // SCRATCH_E contains the error code for FWSEC-FRTS. + let frts_status = bar + .read(regs::NV_PBUS_SW_SCRATCH_0E_FRTS_ERR) + .frts_err_code(); + if frts_status != 0 { + dev_err!( + dev, + "FWSEC-FRTS returned with error code {:#x}\n", + frts_status + ); + + return Err(EIO); + } + + // Check that the WPR2 region has been created as we requested. + let (wpr2_lo, wpr2_hi) = ( + bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO).lower_bound(), + bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound(), + ); + + match (wpr2_lo, wpr2_hi) { + (_, 0) => { + dev_err!(dev, "WPR2 region not created after running FWSEC-FRTS\n"); + + Err(EIO) + } + (wpr2_lo, _) if wpr2_lo != fb_layout.frts.start => { + dev_err!( + dev, + "WPR2 region created at unexpected address {:#x}; expected {:#x}\n", + wpr2_lo, + fb_layout.frts.start, + ); + + Err(EIO) + } + (wpr2_lo, wpr2_hi) => { + dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_lo, wpr2_hi); + dev_dbg!(dev, "GPU instance built\n"); + + Ok(()) + } + } +} + +struct Tu102; + +impl GspHal for Tu102 { + fn boot<'a>( + &self, + gsp: &'a Gsp, + dev: &'a device::Device<device::Bound>, + bar: Bar0<'a>, + chipset: Chipset, + fb_layout: &FbLayout, + wpr_meta: &Coherent<GspFwWprMeta>, + gsp_falcon: &'a Falcon<GspEngine>, + sec2_falcon: &'a Falcon<Sec2>, + ) -> Result<BootUnloadGuard<'a>> { + let bios = Vbios::new(dev, bar)?; + + // Try and prepare the unload bundle. + // + // If the unload bundle creation fails, the GPU will need to be reset before the driver can + // be probed again. + let unload_bundle = + Sec2UnloadBundle::build(dev, bar, chipset, &bios, gsp_falcon, sec2_falcon) + .inspect_err(|e| { + dev_warn!(dev, "Failed to prepare unload firmware: {:?}\n", e); + dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n"); + dev_warn!( + dev, + "The GPU will need to be reset before the driver can bind again.\n" + ); + }) + .ok() + .map(crate::gsp::UnloadBundle); + + // Wrap the unload bundle into a drop guard so it is automatically run upon failure. + let unload_guard = + BootUnloadGuard::new(gsp, dev, bar, gsp_falcon, sec2_falcon, unload_bundle); + + // FWSEC-FRTS is not executed on chips where the FRTS region size is 0 (e.g. GA100). + if !fb_layout.frts.is_empty() { + run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, fb_layout)?; + } + + gsp_falcon.reset(bar)?; + let libos_handle = gsp.libos.dma_handle(); + let (mbox0, mbox1) = gsp_falcon.boot( + bar, + Some(libos_handle as u32), + Some((libos_handle >> 32) as u32), + )?; + dev_dbg!(dev, "GSP MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1); + + dev_dbg!( + dev, + "Using SEC2 to load and run the booter_load firmware...\n" + ); + + BooterFirmware::new( + dev, + BooterKind::Loader, + chipset, + FIRMWARE_VERSION, + sec2_falcon, + bar, + )? + .run(dev, bar, sec2_falcon, wpr_meta)?; + + Ok(unload_guard) + } + + fn post_boot( + &self, + gsp: &Gsp, + dev: &device::Device<device::Bound>, + bar: Bar0<'_>, + gsp_fw: &GspFirmware, + gsp_falcon: &Falcon<GspEngine>, + sec2_falcon: &Falcon<Sec2>, + ) -> Result { + // Create and run the GSP sequencer. + let seq_params = GspSequencerParams { + bootloader_app_version: gsp_fw.bootloader.app_version, + libos_dma_handle: gsp.libos.dma_handle(), + gsp_falcon, + sec2_falcon, + dev, + bar, + }; + GspSequencer::run(&gsp.cmdq, seq_params)?; + + Ok(()) + } +} + +const TU102: Tu102 = Tu102; +pub(super) const TU102_HAL: &dyn GspHal = &TU102; diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs index 474e4c8021db..e0850d21adca 100644 --- a/drivers/gpu/nova-core/gsp/sequencer.rs +++ b/drivers/gpu/nova-core/gsp/sequencer.rs @@ -11,7 +11,6 @@ use kernel::{ Io, // }, prelude::*, - sync::aref::ARef, time::{ delay::fsleep, Delta, // @@ -132,7 +131,7 @@ pub(crate) struct GspSequencer<'a> { /// Sequencer information with command data. seq_info: GspSequence, /// `Bar0` for register access. - bar: &'a Bar0, + bar: Bar0<'a>, /// SEC2 falcon for core operations. sec2_falcon: &'a Falcon<Sec2>, /// GSP falcon for core operations. @@ -142,7 +141,7 @@ pub(crate) struct GspSequencer<'a> { /// Bootloader application version. bootloader_app_version: u32, /// Device for logging. - dev: ARef<device::Device>, + dev: &'a device::Device, } impl fw::RegWritePayload { @@ -281,7 +280,7 @@ pub(crate) struct GspSeqIter<'a> { /// Number of commands processed so far. cmds_processed: u32, /// Device for logging. - dev: ARef<device::Device>, + dev: &'a device::Device, } impl<'a> Iterator for GspSeqIter<'a> { @@ -309,7 +308,7 @@ impl<'a> Iterator for GspSeqIter<'a> { self.cmd_data.len() - offset }; buffer[..copy_len].copy_from_slice(&self.cmd_data[offset..offset + copy_len]); - let cmd_result = GspSeqCmd::new(&buffer, &self.dev); + let cmd_result = GspSeqCmd::new(&buffer, self.dev); cmd_result.map_or_else( |_err| { @@ -334,7 +333,7 @@ impl<'a> GspSequencer<'a> { current_offset: 0, total_cmds: self.seq_info.cmd_index, cmds_processed: 0, - dev: self.dev.clone(), + dev: self.dev, } } } @@ -350,9 +349,9 @@ pub(crate) struct GspSequencerParams<'a> { /// SEC2 falcon for core operations. pub(crate) sec2_falcon: &'a Falcon<Sec2>, /// Device for logging. - pub(crate) dev: ARef<device::Device>, + pub(crate) dev: &'a device::Device, /// BAR0 for register access. - pub(crate) bar: &'a Bar0, + pub(crate) bar: Bar0<'a>, } impl<'a> GspSequencer<'a> { diff --git a/drivers/gpu/nova-core/mctp.rs b/drivers/gpu/nova-core/mctp.rs new file mode 100644 index 000000000000..482786e07bc7 --- /dev/null +++ b/drivers/gpu/nova-core/mctp.rs @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! MCTP/NVDM protocol types for NVIDIA GPU firmware communication. +//! +//! MCTP (Management Component Transport Protocol) carries NVDM (NVIDIA +//! Data Model) messages between the kernel driver and GPU firmware processors +//! such as FSP and GSP. + +use kernel::pci::Vendor; + +/// NVDM message type identifiers carried over MCTP. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[repr(u8)] +pub(crate) enum NvdmType { + #[default] + /// Chain of Trust boot message. + Cot = 0x14, + /// FSP command response. + FspResponse = 0x15, +} + +impl TryFrom<u8> for NvdmType { + type Error = u8; + + fn try_from(value: u8) -> Result<Self, Self::Error> { + match value { + x if x == u8::from(Self::Cot) => Ok(Self::Cot), + x if x == u8::from(Self::FspResponse) => Ok(Self::FspResponse), + _ => Err(value), + } + } +} + +impl From<NvdmType> for u8 { + fn from(value: NvdmType) -> Self { + value as u8 + } +} + +bitfield! { + pub(crate) struct MctpHeader(u32), "MCTP transport header for NVIDIA firmware messages." { + 31:31 som as bool, "Start-of-message bit."; + 30:30 eom as bool, "End-of-message bit."; + 29:28 seq as u8, "Packet sequence number."; + 23:16 seid as u8, "Source endpoint ID."; + } +} + +impl MctpHeader { + /// Builds a single-packet MCTP header (`SOM=1`, `EOM=1`, `SEQ=0`, `SEID=0`). + pub(crate) fn single_packet() -> Self { + Self::default().set_som(true).set_eom(true) + } + + /// Returns whether this is a complete single-packet message (`SOM=1` and `EOM=1`). + pub(crate) fn is_single_packet(self) -> bool { + self.som() && self.eom() + } +} + +/// MCTP message type for PCI vendor-defined messages. +const MSG_TYPE_VENDOR_PCI: u8 = 0x7e; + +bitfield! { + pub(crate) struct NvdmHeader(u32), "NVIDIA Vendor-Defined Message header over MCTP." { + 31:24 nvdm_type as u8 ?=> NvdmType, "NVDM message type."; + 23:8 vendor_id as u16, "PCI vendor ID."; + 6:0 msg_type as u8, "MCTP vendor-defined message type."; + } +} + +impl NvdmHeader { + /// Builds an NVDM header for the given message type. + pub(crate) fn new(nvdm_type: NvdmType) -> Self { + Self::default() + .set_msg_type(MSG_TYPE_VENDOR_PCI) + .set_vendor_id(Vendor::NVIDIA.as_raw()) + .set_nvdm_type(nvdm_type) + } + + /// Validates this header against the expected NVIDIA NVDM format and type. + pub(crate) fn validate(self, expected_type: NvdmType) -> bool { + self.msg_type() == MSG_TYPE_VENDOR_PCI + && self.vendor_id() == Vendor::NVIDIA.as_raw() + && matches!(self.nvdm_type(), Ok(nvdm_type) if nvdm_type == expected_type) + } +} diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 04a1fa6b25f8..9f0199f7b38c 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -17,9 +17,10 @@ mod driver; mod falcon; mod fb; mod firmware; -mod gfw; +mod fsp; mod gpu; mod gsp; +mod mctp; #[macro_use] mod num; mod regs; @@ -47,13 +48,13 @@ struct NovaCoreModule { // Fields are dropped in declaration order, so `_driver` is dropped first, // then `_debugfs_guard` clears `DEBUGFS_ROOT`. #[pin] - _driver: Registration<pci::Adapter<driver::NovaCore>>, + _driver: Registration<pci::Adapter<driver::NovaCoreDriver>>, _debugfs_guard: DebugfsRootGuard, } impl InPlaceModule for NovaCoreModule { fn init(module: &'static kernel::ThisModule) -> impl PinInit<Self, Error> { - let dir = debugfs::Dir::new(kernel::c_str!("nova_core")); + let dir = debugfs::Dir::new(kernel::c_str!("nova-core")); // SAFETY: We are the only driver code running during init, so there // cannot be any concurrent access to `DEBUGFS_ROOT`. @@ -68,7 +69,7 @@ impl InPlaceModule for NovaCoreModule { module! { type: NovaCoreModule, - name: "NovaCore", + name: "nova-core", authors: ["Danilo Krummrich"], description: "Nova Core GPU driver", license: "GPL v2", diff --git a/drivers/gpu/nova-core/num.rs b/drivers/gpu/nova-core/num.rs index 6c824b8d7b97..6eb174d136ab 100644 --- a/drivers/gpu/nova-core/num.rs +++ b/drivers/gpu/nova-core/num.rs @@ -49,7 +49,7 @@ macro_rules! impl_safe_as { #[allow(unused)] #[inline(always)] pub(crate) const fn [<$from _as_ $into>](value: $from) -> $into { - kernel::static_assert!(size_of::<$into>() >= size_of::<$from>()); + ::kernel::build_assert::static_assert!(size_of::<$into>() >= size_of::<$from>()); value as $into } diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 2f171a4ff9ba..0f49c1ab83ad 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::{ io::{ @@ -7,6 +8,7 @@ use kernel::{ Io, // }, prelude::*, + sizes::SizeConstants, time, // }; @@ -30,7 +32,6 @@ use crate::{ Architecture, Chipset, // }, - num::FromSafeCast, }; // PMC @@ -147,11 +148,54 @@ register! { } } +/// Base of the GB10x HSHUB0 register window (`NV_HSHUB0_PRIV_BASE` in Open RM). +/// +/// The base is provided by the GB10x framebuffer HAL. +pub(crate) struct Hshub0Base(()); + +/// Base of the GB20x FBHUB0 register window (`NV_FBHUB0_PRI_BASE` in Open RM). +/// +/// The base is provided by the GB20x framebuffer HAL. +pub(crate) struct Fbhub0Base(()); + +register! { + // GB10x sysmem flush registers, relative to the HSHUB0 base. GB10x routes sysmembar + // through a primary and an EG (egress) pair that must both be programmed to the same + // address. Hardware ignores bits 7:0 of each LO register. The boot path uses a fixed + // HSHUB0 base, so the multiple runtime-discovered HSHUB bases are not needed here. + pub(crate) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x00000e50 { + 31:0 adr => u32; + } + + pub(crate) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x00000e54 { + 19:0 adr; + } + + pub(crate) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x000006c0 { + 31:0 adr => u32; + } + + pub(crate) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x000006c4 { + 19:0 adr; + } + + // GB20x sysmem flush registers, relative to the FBHUB0 base. Unlike the older + // NV_PFB_NISO_FLUSH_SYSMEM_ADDR registers which encode the address with an 8-bit + // right-shift, these take the raw address split into lower and upper halves. Hardware + // ignores bits 7:0 of the LO register. + pub(crate) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Fbhub0Base + 0x00001d58 { + 31:0 adr => u32; + } + + pub(crate) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Fbhub0Base + 0x00001d5c { + 19:0 adr; + } +} + impl NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE { /// Returns the usable framebuffer size, in bytes. pub(crate) fn usable_fb_size(self) -> u64 { - let size = (u64::from(self.lower_mag()) << u64::from(self.lower_scale())) - * u64::from_safe_cast(kernel::sizes::SZ_1M); + let size = (u64::from(self.lower_mag()) << u64::from(self.lower_scale())) * u64::SZ_1M; if self.ecc_mode_enabled() { // Remove the amount of memory reserved for ECC (one per 16 units). @@ -176,6 +220,11 @@ impl NV_PFB_PRI_MMU_WPR2_ADDR_HI { pub(crate) fn higher_bound(self) -> u64 { u64::from(self.hi_val()) << 12 } + + /// Returns whether the WPR2 region is currently set. + pub(crate) fn is_wpr2_set(self) -> bool { + self.hi_val() != 0 + } } // PGSP @@ -241,7 +290,7 @@ impl NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_0_GFW_BOOT { impl NV_USABLE_FB_SIZE_IN_MB { /// Returns the usable framebuffer size, in bytes. pub(crate) fn usable_fb_size(self) -> u64 { - u64::from(self.value()) * u64::from_safe_cast(kernel::sizes::SZ_1M) + u64::from(self.value()) * u64::SZ_1M } } @@ -314,6 +363,8 @@ register! { pub(crate) NV_PFALCON_FALCON_HWCFG2(u32) @ PFalconBase + 0x000000f4 { /// Signal indicating that reset is completed (GA102+). 31:31 reset_ready => bool; + /// RISC-V branch privilege lockdown bit. + 13:13 riscv_br_priv_lockdown => bool; /// Set to 0 after memory scrubbing is completed. 12:12 mem_scrubbing => bool; 10:10 riscv => bool; @@ -426,6 +477,24 @@ register! { pub(crate) NV_PFALCON_FBIF_CTL(u32) @ PFalconBase + 0x00000624 { 7:7 allow_phys_no_ctx => bool; } + + // Falcon EMEM PIO registers (used by FSP on Hopper/Blackwell). + // These provide the falcon external memory communication interface. + + pub(crate) NV_PFALCON_FALCON_EMEMC(u32) @ PFalconBase + 0x00000ac0 { + /// EMEM byte offset (4-byte aligned) within the block. + 7:2 offs; + /// EMEM block to access. + 15:8 blk; + /// Auto-increment the offset after each write. + 24:24 aincw => bool; + /// Auto-increment the offset after each read. + 25:25 aincr => bool; + } + + pub(crate) NV_PFALCON_FALCON_EMEMD(u32) @ PFalconBase + 0x00000ac4 { + 31:0 data => u32; + } } impl NV_PFALCON_FALCON_DMACTL { @@ -449,7 +518,7 @@ impl NV_PFALCON_FALCON_DMATRFCMD { impl NV_PFALCON_FALCON_ENGINE { /// Resets the falcon - pub(crate) fn reset_engine<E: FalconEngine>(bar: &Bar0) { + pub(crate) fn reset_engine<E: FalconEngine>(bar: Bar0<'_>) { bar.update(Self::of::<E>(), |r| r.with_reset(true)); // TIMEOUT: falcon engine should not take more than 10us to reset. @@ -512,6 +581,27 @@ register! { } } +// FSP (Foundation Security Processor) queue registers for Hopper/Blackwell Chain of Trust. +// These registers manage falcon EMEM communication queues. + +register! { + pub(crate) NV_PFSP_QUEUE_HEAD(u32)[8] @ 0x008f2c00 { + 31:0 address => u32; + } + + pub(crate) NV_PFSP_QUEUE_TAIL(u32)[8] @ 0x008f2c04 { + 31:0 address => u32; + } + + pub(crate) NV_PFSP_MSGQ_HEAD(u32)[8] @ 0x008f2c80 { + 31:0 val => u32; + } + + pub(crate) NV_PFSP_MSGQ_TAIL(u32)[8] @ 0x008f2c84 { + 31:0 val => u32; + } +} + // The modules below provide registers that are not identical on all supported chips. They should // only be used in HAL modules. @@ -538,3 +628,39 @@ pub(crate) mod ga100 { } } } + +pub(crate) const NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE_STATUS_SUCCESS: u32 = 0xff; + +pub(crate) mod gh100 { + use kernel::io::register; + + // PTHERM + + register! { + pub(crate) NV_THERM_I2CS_SCRATCH(u32) @ 0x000200bc { + 31:0 data; + } + + // Alias to `NV_THERM_I2CS_SCRATCH` when used to check for FSP boot completion. + pub(crate) NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE(u32) => NV_THERM_I2CS_SCRATCH { + 31:0 fsp_boot_complete; + } + } +} + +pub(crate) mod gb202 { + use kernel::io::register; + + // PTHERM + + register! { + pub(crate) NV_THERM_I2CS_SCRATCH(u32) @ 0x00ad00bc { + 31:0 data; + } + + // Alias to `NV_THERM_I2CS_SCRATCH` when used to check for FSP boot completion. + pub(crate) NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE(u32) => NV_THERM_I2CS_SCRATCH { + 31:0 fsp_boot_complete; + } + } +} diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index ebda28e596c5..c6e6bfcd6a1f 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -2,8 +2,6 @@ //! VBIOS extraction and parsing. -use core::convert::TryFrom; - use kernel::{ device, io::Io, @@ -12,10 +10,14 @@ use kernel::{ Alignable, Alignment, // }, + register, + sizes::SZ_4K, sync::aref::ARef, transmute::FromBytes, }; +use zerocopy::FromBytes as _; + use crate::{ driver::Bar0, firmware::{ @@ -27,16 +29,6 @@ use crate::{ num::FromSafeCast, }; -/// The offset of the VBIOS ROM in the BAR0 space. -const ROM_OFFSET: usize = 0x300000; -/// The maximum length of the VBIOS ROM to scan into. -const BIOS_MAX_SCAN_LEN: usize = 0x100000; -/// The size to read ahead when parsing initial BIOS image headers. -const BIOS_READ_AHEAD_SIZE: usize = 1024; -/// The bit in the last image indicator byte for the PCI Data Structure that -/// indicates the last image. Bit 0-6 are reserved, bit 7 is last image bit. -const LAST_IMAGE_BIT_MASK: u8 = 0x80; - /// BIOS Image Type from PCI Data Structure code_type field. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[repr(u8)] @@ -65,24 +57,16 @@ impl TryFrom<u8> for BiosImageType { } } -// PMU lookup table entry types. Used to locate PMU table entries -// in the Fwsec image, corresponding to falcon ucodes. -#[expect(dead_code)] -const FALCON_UCODE_ENTRY_APPID_FIRMWARE_SEC_LIC: u8 = 0x05; -#[expect(dead_code)] -const FALCON_UCODE_ENTRY_APPID_FWSEC_DBG: u8 = 0x45; -const FALCON_UCODE_ENTRY_APPID_FWSEC_PROD: u8 = 0x85; - /// Vbios Reader for constructing the VBIOS data. struct VbiosIterator<'a> { dev: &'a device::Device, - bar0: &'a Bar0, + bar0: Bar0<'a>, /// VBIOS data vector: As BIOS images are scanned, they are added to this vector for reference /// or copying into other data structures. It is the entire scanned contents of the VBIOS which /// progressively extends. It is used so that we do not re-read any contents that are already /// read as we use the cumulative length read so far, and re-read any gaps as we extend the /// length. - data: KVec<u8>, + data: KVVec<u8>, /// Current offset of the [`Iterator`]. current_offset: usize, /// Indicate whether the last image has been found. @@ -90,20 +74,111 @@ struct VbiosIterator<'a> { } impl<'a> VbiosIterator<'a> { - fn new(dev: &'a device::Device, bar0: &'a Bar0) -> Result<Self> { + /// The offset of the VBIOS ROM in the BAR0 space. + const ROM_OFFSET: usize = 0x300000; + /// The maximum length of the VBIOS ROM to scan into. + const BIOS_MAX_SCAN_LEN: usize = 0x100000; + /// The size to read ahead when parsing initial BIOS image headers. + const BIOS_READ_AHEAD_SIZE: usize = 1024; + + /// Return the byte offset where the PCI Expansion ROM images begin in the GPU's ROM. + /// + /// The GPU's ROM may begin with an Init-from-ROM (IFR) header that precedes the PCI Expansion + /// ROM images (VBIOS). When present, the PROM shadow method must parse this header to determine + /// the offset where the PCI ROM images actually begin, and adjust all subsequent reads + /// accordingly. + /// + /// On most GPUs this is not needed because the IFR microcode has already applied the ROM offset + /// so that PROM reads transparently skip the header. On GA100, for some reason, the IFR offset + /// is not applied to PROM reads. Therefore, the search for the PCI expansion must skip the IFR + /// header, if found. + fn rom_offset(dev: &device::Device, bar0: Bar0<'_>) -> Result<usize> { + // IFR Header in VBIOS. + register! { + NV_PBUS_IFR_FMT_FIXED0(u32) @ 0x300000 { + 31:0 signature; + } + } + + register! { + NV_PBUS_IFR_FMT_FIXED1(u32) @ 0x300004 { + 30:16 fixed_data_size; + 15:8 version => u8; + } + } + + register! { + NV_PBUS_IFR_FMT_FIXED2(u32) @ 0x300008 { + 19:0 total_data_size; + } + } + + /// IFR signature. + const NV_PBUS_IFR_FMT_FIXED0_SIGNATURE_VALUE: u32 = u32::from_le_bytes(*b"NVGI"); + /// ROM directory signature. + const NV_ROM_DIRECTORY_IDENTIFIER: u32 = u32::from_le_bytes(*b"RFRD"); + /// Offset of the NV_PMGR_ROM_ADDR_OFFSET register in IFR Extended section. + const IFR_SW_EXT_ROM_ADDR_OFFSET: usize = 4; + /// Size of Redundant Firmware Flash Status section. + const RFW_FLASH_STATUS_SIZE: usize = SZ_4K; + /// Offset in the ROM Directory of the PCI Option ROM offset. + const PCI_OPTION_ROM_OFFSET: usize = 8; + + let signature = bar0.read(NV_PBUS_IFR_FMT_FIXED0).signature(); + + if signature == NV_PBUS_IFR_FMT_FIXED0_SIGNATURE_VALUE { + let fixed1 = bar0.read(NV_PBUS_IFR_FMT_FIXED1); + + match fixed1.version() { + 1 | 2 => { + let fixed_data_size = usize::from(fixed1.fixed_data_size()); + let pmgr_rom_addr_offset = fixed_data_size + IFR_SW_EXT_ROM_ADDR_OFFSET; + bar0.try_read32(Self::ROM_OFFSET + pmgr_rom_addr_offset) + .map(usize::from_safe_cast) + } + 3 => { + let fixed2 = bar0.read(NV_PBUS_IFR_FMT_FIXED2); + let total_data_size = usize::from(fixed2.total_data_size()); + let flash_status_offset = + usize::from_safe_cast(bar0.try_read32(Self::ROM_OFFSET + total_data_size)?); + let dir_offset = flash_status_offset + RFW_FLASH_STATUS_SIZE; + let dir_sig = bar0.try_read32(Self::ROM_OFFSET + dir_offset)?; + if dir_sig != NV_ROM_DIRECTORY_IDENTIFIER { + dev_err!(dev, "could not find IFR ROM directory\n"); + return Err(EINVAL); + } + bar0.try_read32(Self::ROM_OFFSET + dir_offset + PCI_OPTION_ROM_OFFSET) + .map(usize::from_safe_cast) + } + _ => { + dev_err!(dev, "unsupported IFR header version {}\n", fixed1.version()); + Err(EINVAL) + } + } + } else { + Ok(0) + } + } + + fn new(dev: &'a device::Device, bar0: Bar0<'a>) -> Result<Self> { Ok(Self { dev, bar0, - data: KVec::new(), - current_offset: 0, + data: KVVec::new(), + current_offset: Self::rom_offset(dev, bar0)?, last_found: false, }) } /// Read bytes from the ROM at the current end of the data vector. fn read_more(&mut self, len: usize) -> Result { - let current_len = self.data.len(); - let start = ROM_OFFSET + current_len; + let start = self.data.len(); + let end = start + len; + + if end > Self::BIOS_MAX_SCAN_LEN { + dev_err!(self.dev, "Error: exceeded BIOS scan limit.\n"); + return Err(EINVAL); + } // Ensure length is a multiple of 4 for 32-bit reads if len % core::mem::size_of::<u32>() != 0 { @@ -117,9 +192,9 @@ impl<'a> VbiosIterator<'a> { self.data.reserve(len, GFP_KERNEL)?; // Read ROM data bytes and push directly to `data`. - for addr in (start..start + len).step_by(core::mem::size_of::<u32>()) { + for addr in (start..end).step_by(core::mem::size_of::<u32>()) { // Read 32-bit word from the VBIOS ROM - let word = self.bar0.try_read32(addr)?; + let word = self.bar0.try_read32(Self::ROM_OFFSET + addr)?; // Convert the `u32` to a 4 byte array and push each byte. word.to_ne_bytes() @@ -132,17 +207,9 @@ impl<'a> VbiosIterator<'a> { /// Read bytes at a specific offset, filling any gap. fn read_more_at_offset(&mut self, offset: usize, len: usize) -> Result { - if offset > BIOS_MAX_SCAN_LEN { - dev_err!(self.dev, "Error: exceeded BIOS scan limit.\n"); - return Err(EINVAL); - } + let end = offset.checked_add(len).ok_or(EINVAL)?; - // If `offset` is beyond current data size, fill the gap first. - let current_len = self.data.len(); - let gap_bytes = offset.saturating_sub(current_len); - - // Now read the requested bytes at the offset. - self.read_more(gap_bytes + len) + self.read_more(end.saturating_sub(self.data.len())) } /// Read a BIOS image at a specific offset and create a [`BiosImage`] from it. @@ -155,8 +222,8 @@ impl<'a> VbiosIterator<'a> { len: usize, context: &str, ) -> Result<BiosImage> { - let data_len = self.data.len(); - if offset + len > data_len { + let end = offset.checked_add(len).ok_or(EINVAL)?; + if end > self.data.len() { self.read_more_at_offset(offset, len).inspect_err(|e| { dev_err!( self.dev, @@ -167,7 +234,7 @@ impl<'a> VbiosIterator<'a> { })?; } - BiosImage::new(self.dev, &self.data[offset..offset + len]).inspect_err(|err| { + BiosImage::new(self.dev, &self.data[offset..end]).inspect_err(|err| { dev_err!( self.dev, "Failed to {} at offset {:#x}: {:?}\n", @@ -189,7 +256,7 @@ impl<'a> Iterator for VbiosIterator<'a> { return None; } - if self.current_offset > BIOS_MAX_SCAN_LEN { + if self.current_offset >= Self::BIOS_MAX_SCAN_LEN { dev_err!(self.dev, "Error: exceeded BIOS scan limit, stopping scan\n"); return None; } @@ -197,7 +264,7 @@ impl<'a> Iterator for VbiosIterator<'a> { // Parse image headers first to get image size. let image_size = match self.read_bios_image_at_offset( self.current_offset, - BIOS_READ_AHEAD_SIZE, + Self::BIOS_READ_AHEAD_SIZE, "parse initial BIOS image headers", ) { Ok(image) => image.image_size_bytes(), @@ -232,11 +299,10 @@ impl Vbios { /// Probe for VBIOS extraction. /// /// Once the VBIOS object is built, `bar0` is not read for [`Vbios`] purposes anymore. - pub(crate) fn new(dev: &device::Device, bar0: &Bar0) -> Result<Vbios> { + pub(crate) fn new(dev: &device::Device, bar0: Bar0<'_>) -> Result<Vbios> { // Images to extract from iteration let mut pci_at_image: Option<PciAtBiosImage> = None; - let mut first_fwsec_image: Option<FwSecBiosBuilder> = None; - let mut second_fwsec_image: Option<FwSecBiosBuilder> = None; + let mut fwsec_section: Option<KVVec<u8>> = None; // Parse all VBIOS images in the ROM for image_result in VbiosIterator::new(dev, bar0)? { @@ -250,24 +316,22 @@ impl Vbios { image.is_last() ); + // Once we have found the first FWSEC image, grab all data after that as the FWSEC + // section. This is indexed as one logical block to build the final FWSEC image. + if let Some(data) = fwsec_section.as_mut() { + data.extend_from_slice(&image.data, GFP_KERNEL)?; + continue; + } + // Convert to a specific image type match BiosImageType::try_from(image.pcir.code_type) { Ok(BiosImageType::PciAt) => { - pci_at_image = Some(PciAtBiosImage::try_from(image)?); - } - Ok(BiosImageType::FwSec) => { - let fwsec = FwSecBiosBuilder { - base: image, - falcon_data_offset: None, - pmu_lookup_table: None, - falcon_ucode_offset: None, - }; - if first_fwsec_image.is_none() { - first_fwsec_image = Some(fwsec); - } else { - second_fwsec_image = Some(fwsec); + // Silently ignore any extra PCI-AT images. + if pci_at_image.is_none() { + pci_at_image = Some(PciAtBiosImage::try_from(image)?); } } + Ok(BiosImageType::FwSec) => fwsec_section = Some(image.data), _ => { // Ignore other image types or unknown types } @@ -275,22 +339,18 @@ impl Vbios { } // Using all the images, setup the falcon data pointer in Fwsec. - if let (Some(mut second), Some(first), Some(pci_at)) = - (second_fwsec_image, first_fwsec_image, pci_at_image) - { - second - .setup_falcon_data(&pci_at, &first) - .inspect_err(|e| dev_err!(dev, "Falcon data setup failed: {:?}\n", e))?; - Ok(Vbios { - fwsec_image: second.build()?, - }) - } else { + let (Some(pci_at), Some(fwsec_section)) = (pci_at_image, fwsec_section) else { dev_err!( dev, "Missing required images for falcon data setup, skipping\n" ); - Err(EINVAL) - } + return Err(EINVAL); + }; + + let fwsec_image = FwSecBiosImage::new(dev, pci_at, fwsec_section) + .inspect_err(|e| dev_err!(dev, "Falcon data setup failed: {:?}\n", e))?; + + Ok(Vbios { fwsec_image }) } pub(crate) fn fwsec_image(&self) -> &FwSecBiosImage { @@ -332,6 +392,9 @@ struct PcirStruct { unsafe impl FromBytes for PcirStruct {} impl PcirStruct { + /// The bit in `last_image` that indicates the last image. + const LAST_IMAGE_BIT_MASK: u8 = 0x80; + fn new(dev: &device::Device, data: &[u8]) -> Result<Self> { let (pcir, _) = PcirStruct::from_bytes_copy_prefix(data).ok_or(EINVAL)?; @@ -355,7 +418,7 @@ impl PcirStruct { /// Check if this is the last image in the ROM. fn is_last(&self) -> bool { - self.last_image & LAST_IMAGE_BIT_MASK != 0 + self.last_image & Self::LAST_IMAGE_BIT_MASK != 0 } /// Calculate image size in bytes from 512-byte blocks. @@ -406,7 +469,7 @@ impl BitHeader { /// BIT Token Entry: Records in the BIT table followed by the BIT header. #[derive(Debug, Clone, Copy)] -#[expect(dead_code)] +#[repr(C)] struct BitToken { /// 00h: Token identifier id: u8, @@ -418,39 +481,38 @@ struct BitToken { data_offset: u16, } -// Define the token ID for the Falcon data -const BIT_TOKEN_ID_FALCON_DATA: u8 = 0x70; +// SAFETY: all bit patterns are valid for `BitToken`. +unsafe impl FromBytes for BitToken {} impl BitToken { + /// BIT token ID for Falcon data. + const ID_FALCON_DATA: u8 = 0x70; + /// Find a BIT token entry by BIT ID in a PciAtBiosImage fn from_id(image: &PciAtBiosImage, token_id: u8) -> Result<Self> { let header = &image.bit_header; + let entry_size = usize::from(header.token_size); // Offset to the first token entry let tokens_start = image.bit_offset + usize::from(header.header_size); for i in 0..usize::from(header.token_entries) { - let entry_offset = tokens_start + (i * usize::from(header.token_size)); - - // Make sure we don't go out of bounds - if entry_offset + usize::from(header.token_size) > image.base.data.len() { - return Err(EINVAL); - } + let entry_offset = i + .checked_mul(entry_size) + .and_then(|offset| tokens_start.checked_add(offset)) + .ok_or(EINVAL)?; + let entry = image + .base + .data + .get(entry_offset..) + .and_then(|data| data.get(..entry_size)) + .ok_or(EINVAL)?; + + let (token, _) = BitToken::from_bytes_copy_prefix(entry).ok_or(EINVAL)?; // Check if this token has the requested ID - if image.base.data[entry_offset] == token_id { - return Ok(BitToken { - id: image.base.data[entry_offset], - data_version: image.base.data[entry_offset + 1], - data_size: u16::from_le_bytes([ - image.base.data[entry_offset + 2], - image.base.data[entry_offset + 3], - ]), - data_offset: u16::from_le_bytes([ - image.base.data[entry_offset + 4], - image.base.data[entry_offset + 5], - ]), - }); + if token.id == token_id { + return Ok(token); } } @@ -461,67 +523,38 @@ impl BitToken { /// PCI ROM Expansion Header as defined in PCI Firmware Specification. /// -/// This is header is at the beginning of every image in the set of images in the ROM. It contains -/// a pointer to the PCI Data Structure which describes the image. For "NBSI" images (NoteBook -/// System Information), the ROM header deviates from the standard and contains an offset to the -/// NBSI image however we do not yet parse that in this module and keep it for future reference. +/// This header is at the beginning of every image in the set of images in the ROM. It contains a +/// pointer to the PCI Data Structure which describes the image. #[derive(Debug, Clone, Copy)] -#[expect(dead_code)] +#[repr(C)] struct PciRomHeader { /// 00h: Signature (0xAA55) signature: u16, - /// 02h: Reserved bytes for processor architecture unique data (20 bytes) - reserved: [u8; 20], - /// 16h: NBSI Data Offset (NBSI-specific, offset from header to NBSI image) - nbsi_data_offset: Option<u16>, + /// 02h: Reserved bytes for processor architecture unique data (22 bytes) + reserved: [u8; 22], /// 18h: Pointer to PCI Data Structure (offset from start of ROM image) pci_data_struct_offset: u16, - /// 1Ah: Size of block (this is NBSI-specific) - size_of_block: Option<u32>, } +// SAFETY: all bit patterns are valid for `PciRomHeader`. +unsafe impl FromBytes for PciRomHeader {} + impl PciRomHeader { fn new(dev: &device::Device, data: &[u8]) -> Result<Self> { - if data.len() < 26 { - // Need at least 26 bytes to read pciDataStrucPtr and sizeOfBlock. - return Err(EINVAL); - } - - let signature = u16::from_le_bytes([data[0], data[1]]); + let (rom_header, _) = PciRomHeader::from_bytes_copy_prefix(data) + .ok_or(EINVAL) + .inspect_err(|_| dev_err!(dev, "Not enough data for ROM header\n"))?; // Check for valid ROM signatures. - match signature { - 0xAA55 | 0xBB77 | 0x4E56 => {} + match rom_header.signature { + 0xAA55 | 0x4E56 => {} _ => { - dev_err!(dev, "ROM signature unknown {:#x}\n", signature); + dev_err!(dev, "ROM signature unknown {:#x}\n", rom_header.signature); return Err(EINVAL); } } - // Read the pointer to the PCI Data Structure at offset 0x18. - let pci_data_struct_ptr = u16::from_le_bytes([data[24], data[25]]); - - // Try to read optional fields if enough data. - let mut size_of_block = None; - let mut nbsi_data_offset = None; - - if data.len() >= 30 { - // Read size_of_block at offset 0x1A. - size_of_block = Some(u32::from_le_bytes([data[26], data[27], data[28], data[29]])); - } - - // For NBSI images, try to read the nbsiDataOffset at offset 0x16. - if data.len() >= 24 { - nbsi_data_offset = Some(u16::from_le_bytes([data[22], data[23]])); - } - - Ok(PciRomHeader { - signature, - reserved: [0u8; 20], - pci_data_struct_offset: pci_data_struct_ptr, - size_of_block, - nbsi_data_offset, - }) + Ok(rom_header) } } @@ -550,6 +583,9 @@ struct NpdeStruct { unsafe impl FromBytes for NpdeStruct {} impl NpdeStruct { + /// The bit in `last_image` that indicates the last image. + const LAST_IMAGE_BIT_MASK: u8 = 0x80; + fn new(dev: &device::Device, data: &[u8]) -> Option<Self> { let (npde, _) = NpdeStruct::from_bytes_copy_prefix(data)?; @@ -573,7 +609,7 @@ impl NpdeStruct { /// Check if this is the last image in the ROM. fn is_last(&self) -> bool { - self.last_image & LAST_IMAGE_BIT_MASK != 0 + self.last_image & Self::LAST_IMAGE_BIT_MASK != 0 } /// Calculate image size in bytes from 512-byte blocks. @@ -613,39 +649,14 @@ struct PciAtBiosImage { bit_offset: usize, } -#[expect(dead_code)] -struct EfiBiosImage { - base: BiosImage, - // EFI-specific fields can be added here in the future. -} - -#[expect(dead_code)] -struct NbsiBiosImage { - base: BiosImage, - // NBSI-specific fields can be added here in the future. -} - -struct FwSecBiosBuilder { - base: BiosImage, - /// These are temporary fields that are used during the construction of the - /// [`FwSecBiosBuilder`]. - /// - /// Once FwSecBiosBuilder is constructed, the `falcon_ucode_offset` will be copied into a new - /// [`FwSecBiosImage`]. - /// - /// The offset of the Falcon data from the start of Fwsec image. - falcon_data_offset: Option<usize>, - /// The [`PmuLookupTable`] starts at the offset of the falcon data pointer. - pmu_lookup_table: Option<PmuLookupTable>, - /// The offset of the Falcon ucode. - falcon_ucode_offset: Option<usize>, -} - /// The [`FwSecBiosImage`] structure contains the PMU table and the Falcon Ucode. /// /// The PMU table contains voltage/frequency tables as well as a pointer to the Falcon Ucode. pub(crate) struct FwSecBiosImage { - base: BiosImage, + /// Used for logging. + dev: ARef<device::Device>, + /// FWSEC data. + data: KVVec<u8>, /// The offset of the Falcon ucode. falcon_ucode_offset: usize, } @@ -653,18 +664,13 @@ pub(crate) struct FwSecBiosImage { /// BIOS Image structure containing various headers and reference fields to all BIOS images. /// /// A BiosImage struct is embedded into all image types and implements common operations. -#[expect(dead_code)] struct BiosImage { - /// Used for logging. - dev: ARef<device::Device>, - /// PCI ROM Expansion Header - rom_header: PciRomHeader, /// PCI Data Structure pcir: PcirStruct, /// NVIDIA PCI Data Extension (optional) npde: Option<NpdeStruct>, /// Image data (includes ROM header and PCIR) - data: KVec<u8>, + data: KVVec<u8>, } impl BiosImage { @@ -702,15 +708,8 @@ impl BiosImage { /// Creates a new BiosImage from raw byte data. fn new(dev: &device::Device, data: &[u8]) -> Result<Self> { - // Ensure we have enough data for the ROM header. - if data.len() < 26 { - dev_err!(dev, "Not enough data for ROM header\n"); - return Err(EINVAL); - } - // Parse the ROM header. - let rom_header = PciRomHeader::new(dev, &data[0..26]) - .inspect_err(|e| dev_err!(dev, "Failed to create PciRomHeader: {:?}\n", e))?; + let rom_header = PciRomHeader::new(dev, data)?; // Get the PCI Data Structure using the pointer from the ROM header. let pcir_offset = usize::from(rom_header.pci_data_struct_offset); @@ -737,12 +736,10 @@ impl BiosImage { let npde = NpdeStruct::find_in_data(dev, data, &rom_header, &pcir); // Create a copy of the data. - let mut data_copy = KVec::new(); + let mut data_copy = KVVec::new(); data_copy.extend_from_slice(data, GFP_KERNEL)?; Ok(BiosImage { - dev: dev.into(), - rom_header, pcir, npde, data: data_copy, @@ -773,33 +770,29 @@ impl PciAtBiosImage { BitToken::from_id(self, token_id) } - /// Find the Falcon data pointer structure in the [`PciAtBiosImage`]. + /// Find the Falcon data offset from the start of the FWSEC region. /// - /// This is just a 4 byte structure that contains a pointer to the Falcon data in the FWSEC - /// image. - fn falcon_data_ptr(&self) -> Result<u32> { - let token = self.get_bit_token(BIT_TOKEN_ID_FALCON_DATA)?; - - // Make sure we don't go out of bounds - if usize::from(token.data_offset) + 4 > self.base.data.len() { - return Err(EINVAL); - } - - // read the 4 bytes at the offset specified in the token + /// The BIT table contains a 4-byte pointer to the Falcon data. Testing shows this pointer + /// treats the PCI-AT and FWSEC images as logically contiguous even when an EFI image sits in + /// between them, so subtract the PCI-AT image size here to convert it to a FWSEC-relative + /// offset. + fn falcon_data_offset(&self, dev: &device::Device) -> Result<usize> { + let token = self.get_bit_token(BitToken::ID_FALCON_DATA)?; let offset = usize::from(token.data_offset); - let bytes: [u8; 4] = self.base.data[offset..offset + 4].try_into().map_err(|_| { - dev_err!(self.base.dev, "Failed to convert data slice to array\n"); - EINVAL - })?; - - let data_ptr = u32::from_le_bytes(bytes); - if (usize::from_safe_cast(data_ptr)) < self.base.data.len() { - dev_err!(self.base.dev, "Falcon data pointer out of bounds\n"); - return Err(EINVAL); - } + // Read the 4-byte falcon data pointer at the offset specified in the token. + let data = &self.base.data; + let (ptr, _) = data + .get(offset..) + .and_then(u32::from_bytes_copy_prefix) + .ok_or(EINVAL)?; - Ok(data_ptr) + usize::from_safe_cast(ptr) + .checked_sub(data.len()) + .ok_or(EINVAL) + .inspect_err(|_| { + dev_err!(dev, "Falcon data pointer out of bounds\n"); + }) } } @@ -828,18 +821,18 @@ struct PmuLookupTableEntry { data: u32, } -impl PmuLookupTableEntry { - fn new(data: &[u8]) -> Result<Self> { - if data.len() < core::mem::size_of::<Self>() { - return Err(EINVAL); - } +// SAFETY: all bit patterns are valid for `PmuLookupTableEntry`. +unsafe impl FromBytes for PmuLookupTableEntry {} - Ok(PmuLookupTableEntry { - application_id: data[0], - target_id: data[1], - data: u32::from_le_bytes(data[2..6].try_into().map_err(|_| EINVAL)?), - }) - } +impl PmuLookupTableEntry { + /// PMU lookup table application ID for firmware security license ucode. + #[expect(dead_code)] + const APPID_FIRMWARE_SEC_LIC: u8 = 0x05; + /// PMU lookup table application ID for debug FWSEC ucode. + #[expect(dead_code)] + const APPID_FWSEC_DBG: u8 = 0x45; + /// PMU lookup table application ID for production FWSEC ucode. + const APPID_FWSEC_PROD: u8 = 0x85; } #[repr(C)] @@ -859,8 +852,7 @@ unsafe impl FromBytes for PmuLookupTableHeader {} /// The table of entries is pointed to by the falcon data pointer in the BIT table, and is used to /// locate the Falcon Ucode. struct PmuLookupTable { - header: PmuLookupTableHeader, - table_data: KVec<u8>, + entries: KVVec<PmuLookupTableEntry>, } impl PmuLookupTable { @@ -871,148 +863,74 @@ impl PmuLookupTable { let entry_len = usize::from(header.entry_len); let entry_count = usize::from(header.entry_count); - let required_bytes = header_len + (entry_count * entry_len); - - if data.len() < required_bytes { - dev_err!(dev, "PmuLookupTable data length less than required\n"); - return Err(EINVAL); - } - - // Create a copy of only the table data - let table_data = { - let mut ret = KVec::new(); - ret.extend_from_slice(&data[header_len..required_bytes], GFP_KERNEL)?; - ret - }; - - Ok(PmuLookupTable { header, table_data }) - } + let data = data + .get(header_len..header_len + entry_count * entry_len) + .ok_or(EINVAL) + .inspect_err(|_| { + dev_err!(dev, "PmuLookupTable data length less than required\n"); + })?; - fn lookup_index(&self, idx: u8) -> Result<PmuLookupTableEntry> { - if idx >= self.header.entry_count { - return Err(EINVAL); + let mut entries = KVVec::with_capacity(entry_count, GFP_KERNEL)?; + for i in 0..entry_count { + let (entry, _) = PmuLookupTableEntry::from_bytes_copy_prefix(&data[i * entry_len..]) + .ok_or(EINVAL)?; + entries.push(entry, GFP_KERNEL)?; } - let index = (usize::from(idx)) * usize::from(self.header.entry_len); - PmuLookupTableEntry::new(&self.table_data[index..]) + Ok(PmuLookupTable { entries }) } // find entry by type value - fn find_entry_by_type(&self, entry_type: u8) -> Result<PmuLookupTableEntry> { - for i in 0..self.header.entry_count { - let entry = self.lookup_index(i)?; - if entry.application_id == entry_type { - return Ok(entry); - } - } - - Err(EINVAL) + fn find_entry_by_type(&self, entry_type: u8) -> Result<&PmuLookupTableEntry> { + self.entries + .iter() + .find(|entry| entry.application_id == entry_type) + .ok_or(EINVAL) } } -impl FwSecBiosBuilder { - fn setup_falcon_data( - &mut self, - pci_at_image: &PciAtBiosImage, - first_fwsec: &FwSecBiosBuilder, - ) -> Result { - let mut offset = usize::from_safe_cast(pci_at_image.falcon_data_ptr()?); - let mut pmu_in_first_fwsec = false; - - // The falcon data pointer assumes that the PciAt and FWSEC images - // are contiguous in memory. However, testing shows the EFI image sits in - // between them. So calculate the offset from the end of the PciAt image - // rather than the start of it. Compensate. - offset -= pci_at_image.base.data.len(); - - // The offset is now from the start of the first Fwsec image, however - // the offset points to a location in the second Fwsec image. Since - // the fwsec images are contiguous, subtract the length of the first Fwsec - // image from the offset to get the offset to the start of the second - // Fwsec image. - if offset < first_fwsec.base.data.len() { - pmu_in_first_fwsec = true; - } else { - offset -= first_fwsec.base.data.len(); - } - - self.falcon_data_offset = Some(offset); - - if pmu_in_first_fwsec { - self.pmu_lookup_table = Some(PmuLookupTable::new( - &self.base.dev, - &first_fwsec.base.data[offset..], - )?); - } else { - self.pmu_lookup_table = Some(PmuLookupTable::new( - &self.base.dev, - &self.base.data[offset..], - )?); - } - - match self - .pmu_lookup_table - .as_ref() - .ok_or(EINVAL)? - .find_entry_by_type(FALCON_UCODE_ENTRY_APPID_FWSEC_PROD) - { - Ok(entry) => { - let mut ucode_offset = usize::from_safe_cast(entry.data); - ucode_offset -= pci_at_image.base.data.len(); - if ucode_offset < first_fwsec.base.data.len() { - dev_err!(self.base.dev, "Falcon Ucode offset not in second Fwsec.\n"); - return Err(EINVAL); - } - ucode_offset -= first_fwsec.base.data.len(); - self.falcon_ucode_offset = Some(ucode_offset); - } - Err(e) => { - dev_err!( - self.base.dev, - "PmuLookupTableEntry not found, error: {:?}\n", - e - ); - return Err(EINVAL); - } - } - Ok(()) - } - - /// Build the final FwSecBiosImage from this builder - fn build(self) -> Result<FwSecBiosImage> { - let ret = FwSecBiosImage { - base: self.base, - falcon_ucode_offset: self.falcon_ucode_offset.ok_or(EINVAL)?, - }; +impl FwSecBiosImage { + /// Build the final `FwSecBiosImage` from the PCI-AT and FWSEC BIOS images. + fn new( + dev: &device::Device, + pci_at_image: PciAtBiosImage, + data: KVVec<u8>, + ) -> Result<FwSecBiosImage> { + let offset = pci_at_image.falcon_data_offset(dev)?; + + let pmu_lookup_data = data.get(offset..).ok_or(EINVAL)?; + let pmu_lookup_table = PmuLookupTable::new(dev, pmu_lookup_data)?; + + let entry = pmu_lookup_table + .find_entry_by_type(PmuLookupTableEntry::APPID_FWSEC_PROD) + .inspect_err(|e| { + dev_err!(dev, "PmuLookupTableEntry not found, error: {:?}\n", e); + })?; - if cfg!(debug_assertions) { - // Print the desc header for debugging - let desc = ret.header()?; - dev_dbg!(ret.base.dev, "PmuLookupTableEntry desc: {:#?}\n", desc); - } + let falcon_ucode_offset = usize::from_safe_cast(entry.data) + .checked_sub(pci_at_image.base.data.len()) + .ok_or(EINVAL) + .inspect_err(|_| { + dev_err!(dev, "Falcon Ucode offset not in Fwsec.\n"); + })?; - Ok(ret) + Ok(FwSecBiosImage { + dev: dev.into(), + data, + falcon_ucode_offset, + }) } -} -impl FwSecBiosImage { /// Get the FwSec header ([`FalconUCodeDesc`]). pub(crate) fn header(&self) -> Result<FalconUCodeDesc> { - // Get the falcon ucode offset that was found in setup_falcon_data. - let falcon_ucode_offset = self.falcon_ucode_offset; - - // Read the first 4 bytes to get the version. - let hdr_bytes: [u8; 4] = self.base.data[falcon_ucode_offset..falcon_ucode_offset + 4] - .try_into() - .map_err(|_| EINVAL)?; - let hdr = u32::from_le_bytes(hdr_bytes); - let ver = (hdr & 0xff00) >> 8; + let data = self.data.get(self.falcon_ucode_offset..).ok_or(EINVAL)?; - let data = self.base.data.get(falcon_ucode_offset..).ok_or(EINVAL)?; + // Read the version byte from the header. + let ver = data.get(1).copied().ok_or(EINVAL)?; match ver { 2 => { - let v2 = FalconUCodeDescV2::from_bytes_copy_prefix(data) - .ok_or(EINVAL)? + let v2 = FalconUCodeDescV2::read_from_prefix(data) + .map_err(|_| EINVAL)? .0; Ok(FalconUCodeDesc::V2(v2)) } @@ -1023,7 +941,7 @@ impl FwSecBiosImage { Ok(FalconUCodeDesc::V3(v3)) } _ => { - dev_err!(self.base.dev, "invalid fwsec firmware version: {:?}\n", ver); + dev_err!(self.dev, "invalid fwsec firmware version: {:?}\n", ver); Err(EINVAL) } } @@ -1031,20 +949,21 @@ impl FwSecBiosImage { /// Get the ucode data as a byte slice pub(crate) fn ucode(&self, desc: &FalconUCodeDesc) -> Result<&[u8]> { - let falcon_ucode_offset = self.falcon_ucode_offset; + let size = usize::from_safe_cast( + desc.imem_load_size() + .checked_add(desc.dmem_load_size()) + .ok_or(ERANGE)?, + ); // The ucode data follows the descriptor. - let ucode_data_offset = falcon_ucode_offset + desc.size(); - let size = usize::from_safe_cast(desc.imem_load_size() + desc.dmem_load_size()); - - // Get the data slice, checking bounds in a single operation. - self.base - .data - .get(ucode_data_offset..ucode_data_offset + size) + self.data + .get(self.falcon_ucode_offset..) + .and_then(|data| data.get(desc.size()..)) + .and_then(|data| data.get(..size)) .ok_or(ERANGE) .inspect_err(|_| { dev_err!( - self.base.dev, + self.dev, "fwsec ucode data not contained within BIOS bounds\n" ) }) @@ -1062,9 +981,9 @@ impl FwSecBiosImage { let sigs_size = sigs_count * core::mem::size_of::<Bcrt30Rsa3kSignature>(); // Make sure the data is within bounds. - if sigs_data_offset + sigs_size > self.base.data.len() { + if sigs_data_offset + sigs_size > self.data.len() { dev_err!( - self.base.dev, + self.dev, "fwsec signatures data not contained within BIOS bounds\n" ); return Err(ERANGE); @@ -1074,8 +993,7 @@ impl FwSecBiosImage { // sizeof::<Bcrt30Rsa3kSignature>()` is within the bounds of `data`. Ok(unsafe { core::slice::from_raw_parts( - self.base - .data + self.data .as_ptr() .add(sigs_data_offset) .cast::<Bcrt30Rsa3kSignature>(), |