diff options
Diffstat (limited to 'tools/testing/selftests/vfio/lib/drivers/igb/igb.c')
| -rw-r--r-- | tools/testing/selftests/vfio/lib/drivers/igb/igb.c | 585 |
1 files changed, 585 insertions, 0 deletions
diff --git a/tools/testing/selftests/vfio/lib/drivers/igb/igb.c b/tools/testing/selftests/vfio/lib/drivers/igb/igb.c new file mode 100644 index 000000000000..fd9e05d77ea4 --- /dev/null +++ b/tools/testing/selftests/vfio/lib/drivers/igb/igb.c @@ -0,0 +1,585 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <unistd.h> +#include <errno.h> +#include <stdint.h> +#include <linux/io.h> +#include <linux/pci_regs.h> +#include <linux/pci_ids.h> +#include <linux/kernel.h> +#include <linux/compiler.h> +#include <asm/barrier.h> +#include <linux/mii.h> +#include <libvfio/vfio_pci_device.h> + +#include "e1000_regs.h" +#include "e1000_defines.h" +#include "e1000_82575.h" + +#define PCI_DEVICE_ID_INTEL_82576 0x10C9 +#define IGB_MAX_CHUNK_SIZE 1024 +#define MSIX_VECTOR 0 +#define MSIX_VECTOR_MASK (1 << MSIX_VECTOR) +#define RING_SIZE 4096 /* Number of descriptors in ring */ + +struct igb_tx_desc { + union { + struct { + u64 buffer_addr; /* Address of descriptor's data buffer */ + u32 cmd_type_len; /* Command/Type/Length */ + u32 olinfo_status; /* Context/Buffer info */ + } read; + + struct { + u64 rsvd; /* Reserved */ + u32 nxtseq_seed; /* Next sequence seed */ + u32 status; /* Descriptor status */ + } wb; + }; +}; + +struct igb_rx_desc { + union { + struct { + u64 pkt_addr; /* Packet buffer address */ + u64 hdr_addr; /* Header buffer address */ + } read; + struct { + u16 pkt_info; /* RSS type, Packet type */ + u16 hdr_info; /* Split Head, buf len */ + u32 rss; /* RSS Hash */ + u32 status_error; /* ext status/error */ + u16 length; /* Packet length */ + u16 vlan; /* VLAN tag */ + } wb; /* writeback */ + }; +}; + +struct igb { + void *bar0; + u32 tx_tail; + u32 rx_tail; + struct igb_tx_desc tx_ring[RING_SIZE] __attribute__((aligned(128))); + struct igb_rx_desc rx_ring[RING_SIZE] __attribute__((aligned(128))); +}; + +static inline struct igb *to_igb_state(struct vfio_pci_device *device) +{ + return (struct igb *)device->driver.region.vaddr; +} + +static inline void igb_write32(struct igb *igb, u32 reg, u32 val) +{ + writel(val, igb->bar0 + reg); +} + +static inline u32 igb_read32(struct igb *igb, u32 reg) +{ + return readl(igb->bar0 + reg); +} + +static int igb_write_phy(struct igb *igb, u32 offset, u16 data) +{ + u32 mdic; + int i; + + /* + * Write a PHY register over MDIO. + * + * A production driver would hold the SW/FW semaphore (SWSM.SWESMBI + the + * SW_FW_SYNC PHY bit) across the MDIO transaction to serialize against the + * device's management firmware. The selftest owns the assigned function + * exclusively on a dedicated test device with no active manageability + * contending for the PHY, so the sync is omitted; it should be added here + * if this ever needs to run on a manageability-enabled NIC. + */ + mdic = (((u32)data) | + (offset << E1000_MDIC_REG_SHIFT) | + (1 << E1000_MDIC_PHY_SHIFT) | + E1000_MDIC_OP_WRITE); + + igb_write32(igb, E1000_MDIC, mdic); + + for (i = 0; i < 1000; i++) { + usleep(50); + mdic = igb_read32(igb, E1000_MDIC); + if (mdic & E1000_MDIC_READY) + break; + } + + if (!(mdic & E1000_MDIC_READY)) + return -1; + + if (mdic & E1000_MDIC_ERROR) + return -1; + + return 0; +} + +/* + * Configure the device for PHY internal loopback per 82576 datasheet + * section 3.5.6.3.1. Force the PHY to 1Gb/s full duplex with loopback + * enabled, then force the MAC link state to match. Internal loopback + * wraps data at the end of the PHY datapath (section 3.5.6.3), so the + * physical link state is irrelevant. + * + * Section 3.5.6.1 directs to "Use PHY Loopback instead of MAC Loopback + * on the 82576", and section 3.5.6.2 states "MAC Loopback is not used + * on this device." RCTL.LBM_MAC is still set elsewhere as a QEMU-only + * accommodation; see the RCTL programming in the caller for the + * rationale. + */ +static void igb_setup_loopback(struct igb *igb) +{ + u32 ctrl; + int ret; + + /* + * Kick the autoneg machinery solely to bring STATUS.LU up under + * QEMU's igb emulation: QEMU only updates STATUS.LU via its + * autoneg-done timer, and without LU set its receive path + * (e1000x_hw_rx_enabled) drops every loopback frame. On real + * hardware autoneg cannot complete before the next PHY write + * below clears the autoneg-enable bit, so this is effectively a + * no-op there. + */ + (void)igb_write_phy(igb, MII_BMCR, + BMCR_ANENABLE | BMCR_ANRESTART); + + /* PHY control: loopback + 1Gb/s full duplex, autoneg disabled. */ + ret = igb_write_phy(igb, MII_BMCR, + BMCR_LOOPBACK | + BMCR_SPEED1000 | + BMCR_FULLDPLX); + VFIO_ASSERT_EQ(ret, 0, "Failed to write PHY control register"); + + /* + * Brief delay before forcing the MAC, mirroring the kernel ethtool + * selftest in igb_integrated_phy_loopback(). Not specified by the + * datasheet, but empirically required by the kernel driver. + */ + usleep(50000); + + /* + * Force the MAC to 1Gb/s full duplex with link up. Without forcing + * the link state the descriptor engine does not run, since the chip + * normally waits for a real negotiated link. + */ + ctrl = igb_read32(igb, E1000_CTRL); + ctrl &= ~E1000_CTRL_SPD_SEL; + ctrl |= E1000_CTRL_FRCSPD | + E1000_CTRL_FRCDPX | + E1000_CTRL_SPD_1000 | + E1000_CTRL_FD | + E1000_CTRL_SLU; + igb_write32(igb, E1000_CTRL, ctrl); + + /* + * Settling delay matching the kernel ethtool selftest's msleep(500) + * at the tail of igb_integrated_phy_loopback(). Not specified by + * the datasheet; empirical, and inherited from the kernel driver. + */ + usleep(500000); +} + +static int igb_probe(struct vfio_pci_device *device) +{ + if (!vfio_pci_device_match(device, PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82576)) + return -EINVAL; + + return 0; +} + +static void igb_reset(struct igb *igb) +{ + int retries = 20; + + igb_write32(igb, E1000_CTRL, igb_read32(igb, E1000_CTRL) | E1000_CTRL_RST); + /* + * Must wait at least 1 millisecond after setting the reset bit before + * checking if this device is ready to be used (82576 datasheet section + * 4.2.1.6.1). The delay also ensures the reset has taken effect and + * cleared EECD.AUTO_RD before it is polled below. + */ + usleep(1000); + + /* + * Poll NVM Auto Read Done rather than CTRL.RST, matching + * igb_get_auto_rd_done() in the igb driver: AUTO_RD implies both that + * the reset completed and that the device finished re-reading its + * configuration from NVM, which is what actually makes it usable. + */ + while (retries-- > 0 && !(igb_read32(igb, E1000_EECD) & E1000_EECD_AUTO_RD)) + usleep(1000); + + /* + * QEMU's igb emulation does not set E1000_EECD_AUTO_RD. If we timed out, + * check if CTRL.RST is cleared, which is what QEMU uses to signal reset + * completion. + */ + if (retries < 0) { + VFIO_ASSERT_EQ(igb_read32(igb, E1000_CTRL) & E1000_CTRL_RST, 0, + "Device reset did not complete (CTRL.RST not cleared)"); + } + + igb_write32(igb, E1000_IMC, 0xFFFFFFFF); +} + +/* + * Program the device into a usable state. Split out of igb_init() so it + * can be reused after a device reset to re-program the registers that + * CTRL.RST clears. Expects bar0 to be mapped and MSI-X already enabled + * via VFIO. + */ +static void igb_hw_init(struct vfio_pci_device *device) +{ + struct igb *igb = to_igb_state(device); + u64 iova_tx, iova_rx; + u32 ctrl, rctl; + u16 cmd_reg; + int retries; + + iova_tx = to_iova(device, igb->tx_ring); + iova_rx = to_iova(device, igb->rx_ring); + + + + /* Signal that the driver is loaded */ + ctrl = igb_read32(igb, E1000_CTRL_EXT); + ctrl |= E1000_CTRL_EXT_DRV_LOAD; + ctrl &= ~E1000_CTRL_EXT_LINK_MODE_MASK; + igb_write32(igb, E1000_CTRL_EXT, ctrl); + + /* Enable PCI Bus Master. */ + cmd_reg = vfio_pci_config_readw(device, PCI_COMMAND); + if ((cmd_reg & (PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY)) != + (PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY)) { + cmd_reg |= (PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY); + vfio_pci_config_writew(device, PCI_COMMAND, cmd_reg); + } + + /* Configure PHY internal loopback for testing. */ + igb_setup_loopback(igb); + + /* + * Disable DMA re-send on PCIe completion timeout (82576 datasheet + * section 8.6.1, GCR.Completion_Timeout_Resend, bit 16). The + * mix_and_match test intentionally submits descriptors targeting + * unmapped IOVAs; with the default (set) value, the device keeps + * retrying the failed read indefinitely, which keeps PCIe AER and + * IOMMU error handling busy and interferes with reset recovery. + */ + ctrl = igb_read32(igb, E1000_GCR); + ctrl &= ~E1000_GCR_CMPL_TMOUT_RESEND; + igb_write32(igb, E1000_GCR, ctrl); + + /* Configure TX and RX descriptor rings */ + igb_write32(igb, E1000_TDBAL(0), (u32)iova_tx); + igb_write32(igb, E1000_TDBAH(0), (u32)(iova_tx >> 32)); + igb_write32(igb, E1000_TDLEN(0), RING_SIZE * sizeof(struct igb_tx_desc)); + igb_write32(igb, E1000_TDH(0), 0); + igb_write32(igb, E1000_TDT(0), 0); + igb_write32(igb, E1000_TXDCTL(0), E1000_TXDCTL_QUEUE_ENABLE); + + igb_write32(igb, E1000_RDBAL(0), (u32)iova_rx); + igb_write32(igb, E1000_RDBAH(0), (u32)(iova_rx >> 32)); + igb_write32(igb, E1000_RDLEN(0), RING_SIZE * sizeof(struct igb_rx_desc)); + igb_write32(igb, E1000_RDH(0), 0); + igb_write32(igb, E1000_RDT(0), 0); + + /* + * Select the advanced one-buffer descriptor format. Per 82576 + * datasheet section 7.1.5.2: "SRRCTL[n].DESCTYPE must be set to a + * value other than 000b for the 82576 to write back the special + * descriptors." struct igb_rx_desc matches the advanced one-buffer + * writeback layout (section 7.1.5.2), so polling rx.wb.status_error + * requires this format. Section 8.10.2 specifies DESCTYPE[27:25]. + * + * The direct write also zeroes SRRCTL.BSIZEPACKET, which is + * intentional: per section 7.1.3.1 a zero BSIZEPACKET falls back to + * the RCTL.BSIZE buffer size, whose reset default (00b) is 2048 + * bytes -- ample for the loopback frames here. + */ + igb_write32(igb, E1000_SRRCTL(0), E1000_SRRCTL_DESCTYPE_ADV_ONEBUF); + + igb_write32(igb, E1000_RXDCTL(0), E1000_RXDCTL_QUEUE_ENABLE); + + /* + * Enable Receiver and Transmitter. RCTL.LBM_MAC is set in addition + * to PHY loopback as a QEMU-only accommodation: QEMU's emulated igb + * does not honor PHY register 0 bit 14 (PHY internal loopback) and + * relies on RCTL.LBM_MAC to wrap TX descriptors back to the RX + * queue. Datasheet 8.10.1 (RCTL register) advises "When using the + * internal PHY, LBM should remain set to 00b", so setting LBM_MAC + * here deviates from datasheet guidance; empirically the bit has + * no observable effect on real 82576 hardware because MAC loopback + * is not implemented (datasheet 3.5.6.2). Setting both lets the + * selftest work on both real hardware and QEMU without conditional + * code paths. + */ + rctl = E1000_RCTL_EN | /* Receiver Enable */ + E1000_RCTL_UPE | /* Unicast Promiscuous (for dummy MAC) */ + E1000_RCTL_MPE | /* Multicast Promiscuous */ + E1000_RCTL_BAM | /* Broadcast Accept Mode */ + E1000_RCTL_LBM_MAC | /* MAC Loopback - for QEMU emulation only */ + E1000_RCTL_SECRC; /* Strip CRC (needed for memcmp) */ + igb_write32(igb, E1000_RCTL, rctl); + igb_write32(igb, E1000_TCTL, E1000_TCTL_EN | E1000_TCTL_PSP); + + /* + * Wait for TX and RX queues to be enabled. Per the RXDCTL/TXDCTL + * register definitions (8.10.10/8.12.13), the per-queue enable bit + * "remains zero" until the global RCTL.RXEN/TCTL.TXEN are set, so + * E1000_RCTL_EN and E1000_TCTL_EN must already be written above. + */ + retries = 2000; + while (retries-- > 0) { + if ((igb_read32(igb, E1000_TXDCTL(0)) & E1000_TXDCTL_QUEUE_ENABLE) && + (igb_read32(igb, E1000_RXDCTL(0)) & E1000_RXDCTL_QUEUE_ENABLE)) + break; + usleep(10); + } + VFIO_ASSERT_GE(retries, 0); + + /* + * Program MSI-X interrupt routing per 82576 datasheet: + * + * GPIE (section 7.3.2.11, Table 7-47): set Multiple_MSIX (bit 4) to + * route interrupt causes through IVAR mapping, and EIAME (bit 30) + * to apply EIAM on MSI-X assertion (without EIAME, EIAM only + * applies on EICR read/write). + * + * EIAC (section 8.8.5): enable auto-clear of EICR for vector 0. + * Without auto-clear the cause stays set after delivery and the + * test can see spurious interrupts on the next memcpy batch. + * + * EIAM (section 8.8.6): enable auto-mask of EIMS for vector 0 on + * MSI-X assertion (effective because EIAME is set). + * + * IVAR (section 7.3.1.2, register definition in 8.8.13): map RX + * cause 0 to MSI-X vector 0 and mark the entry valid. + */ + igb_write32(igb, E1000_GPIE, E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME); + igb_write32(igb, E1000_EIAC, MSIX_VECTOR_MASK); + igb_write32(igb, E1000_EIAM, MSIX_VECTOR_MASK); + + /* Map vector 0 to interrupt cause 0 and mark it valid */ + igb_write32(igb, E1000_IVAR0, E1000_IVAR_VALID); + + /* Enable interrupts on vector 0 */ + igb_write32(igb, E1000_EIMS, MSIX_VECTOR_MASK); + + /* Initialize driver state and capability limits */ + igb->tx_tail = 0; + igb->rx_tail = 0; + + device->driver.max_memcpy_size = IGB_MAX_CHUNK_SIZE; + device->driver.max_memcpy_count = RING_SIZE - 1; + device->driver.msi = MSIX_VECTOR; +} + +static void igb_init(struct vfio_pci_device *device) +{ + struct igb *igb = to_igb_state(device); + + VFIO_ASSERT_GE(device->driver.region.size, sizeof(struct igb)); + + igb->bar0 = device->bars[0].vaddr; + + igb_reset(igb); + + /* + * Enable MSI-X via VFIO before device-side register programming. + * vfio_pci_msix_enable() only touches the VFIO IRQ machinery and the + * PCI MSI-X capability via config space; it has no ordering + * dependency on the device-side writes performed by igb_hw_init(). + * Placing it here keeps igb_hw_init() reusable from the reset + * recovery path (which calls vfio_pci_irq_reenable() instead). + */ + vfio_pci_msix_enable(device, MSIX_VECTOR, 1); + + igb_hw_init(device); +} + +static void igb_remove(struct vfio_pci_device *device) +{ + struct igb *igb = to_igb_state(device); + + igb_write32(igb, E1000_RCTL, 0); + igb_write32(igb, E1000_TCTL, 0); + igb_reset(igb); + + vfio_pci_msix_disable(device); +} + +static void igb_irq_disable(struct igb *igb) +{ + igb_write32(igb, E1000_EIMC, MSIX_VECTOR_MASK); +} + +static void igb_irq_enable(struct igb *igb) +{ + igb_write32(igb, E1000_EIMS, MSIX_VECTOR_MASK); +} + +static void igb_irq_clear(struct igb *igb) +{ + /* + * Use write-to-clear (datasheet 7.3.4.2). In MSI-X mode with EIAC + * programmed, section 8.8.5 explicitly states "If any bits are set + * in EIAC, the EICR register should not be read", which rules out + * the read-to-clear path in 7.3.4.3. Bits not in EIAC are still + * cleared by writing 1. + */ + igb_write32(igb, E1000_EICR, 0xFFFFFFFF); +} + +static void igb_memcpy_start(struct vfio_pci_device *device, iova_t src, + iova_t dst, u64 size, u64 count) +{ + struct igb *igb = to_igb_state(device); + struct igb_rx_desc *rx; + struct igb_tx_desc *tx; + u32 i; + + VFIO_ASSERT_GE(size, 60, + "IGB driver requires memcpy size to be at least 60 bytes (Ethernet minimum payload size)"); + + igb_irq_disable(igb); + + for (i = 0; i < count; i++) { + tx = &igb->tx_ring[igb->tx_tail]; + rx = &igb->rx_ring[igb->rx_tail]; + + memset(tx, 0, sizeof(struct igb_tx_desc)); + memset(rx, 0, sizeof(struct igb_rx_desc)); + + rx->read.pkt_addr = cpu_to_le64(dst); + rx->read.hdr_addr = cpu_to_le64(0); + + tx->read.buffer_addr = cpu_to_le64(src); + /* + * Build an advanced data descriptor per 82576 datasheet + * section 7.2.2.3. DEXT marks the descriptor as advanced + * (required by hardware); DTYP=data selects the data + * descriptor; IFCS asks the MAC to append the Ethernet + * FCS (without it the frame is dropped as malformed); + * EOP marks end of packet. DTALEN is the buffer length + * in bits 15:0 of cmd_type_len. + */ + tx->read.cmd_type_len = cpu_to_le32((uint32_t)size | + E1000_ADVTXD_DTYP_DATA | + E1000_ADVTXD_DCMD_DEXT | + E1000_ADVTXD_DCMD_IFCS | + E1000_ADVTXD_DCMD_EOP); + /* + * PAYLEN (section 7.2.2.3.11) is the total payload size + * in olinfo_status[31:14]. + */ + tx->read.olinfo_status = + cpu_to_le32((uint32_t)size << E1000_ADVTXD_PAYLEN_SHIFT); + + igb->tx_tail = (igb->tx_tail + 1) % RING_SIZE; + igb->rx_tail = (igb->rx_tail + 1) % RING_SIZE; + } + + igb_write32(igb, E1000_RDT(0), igb->rx_tail); + igb_write32(igb, E1000_TDT(0), igb->tx_tail); +} + +/* + * Reset the device via VFIO_DEVICE_RESET (PCIe FLR on the 82576) and + * re-program it. VFIO_DEVICE_RESET tears down the kernel-side MSI-X + * trigger but leaves user-side eventfds intact, so re-arm the trigger + * via vfio_pci_irq_reenable() before reprogramming so any caller-cached + * eventfd remains valid. + * + * FLR clears device-side state to power-on reset values (datasheet + * 4.2.1.5.1: a PF FLR is "equivalent to a D0->D3->D0 transition"), so + * EIMS and EICR come back as 0 from their register-defined initial + * values, and igb_hw_init() resets tx_tail/rx_tail to 0. The next + * igb_memcpy_start() will memset each descriptor it touches before + * submission, so no explicit IMC/EICR writes or ring memsets are + * needed here. + */ +static void igb_error_reset_and_reinit(struct vfio_pci_device *device) +{ + vfio_pci_device_reset(device); + vfio_pci_msix_reenable(device, MSIX_VECTOR, 1); + igb_hw_init(device); +} + +static int igb_memcpy_wait(struct vfio_pci_device *device) +{ + struct igb *igb = to_igb_state(device); + struct igb_rx_desc *rx; + u32 status = 0; + u32 prev_tail; + int retries; + + prev_tail = (igb->rx_tail + RING_SIZE - 1) % RING_SIZE; + rx = &igb->rx_ring[prev_tail]; + + /* + * Real 82576 hardware processes the descriptor ring at line rate. + * max_memcpy_size = (RING_SIZE - 1) * IGB_MAX_CHUNK_SIZE ~= 4 MB, + * split into 4095 1 KB frames. At 1 Gb/s (~125 MB/s) the worst + * valid memcpy takes ~32 ms on the wire, plus per-frame preamble, + * SFD, IFG and FCS overhead (~3%) and descriptor fetch/writeback + * latency. Wait up to ~200 ms before declaring the device hung; + * ~6x the line-rate floor leaves comfortable headroom for host + * scheduling jitter while keeping the intentional invalid-DMA + * tests bounded. + */ + retries = 200; + while (retries-- > 0) { + status = le32_to_cpu(READ_ONCE(rx->wb.status_error)); + if (status & 1) + break; + usleep(1000); + } + + if (status & 1) + /* + * Ensure the test code doesn't speculatively read the DMA + * destination buffer before we have verified that the + * descriptor writeback is complete. + */ + rmb(); + + igb_irq_clear(igb); + + igb_irq_enable(igb); + + if (status & 1) + return 0; + + /* + * The descriptor never completed. On real 82576 hardware this + * typically follows a DMA-read fault from one of the intentional + * unmapped-IOVA tests; the fault leaves the descriptor engine + * unable to service subsequent valid descriptors. CTRL.RST alone + * reinitializes the queue registers but leaves the engine wedged + * for the current process, so a broader VFIO_DEVICE_RESET (FLR) + * is required. + */ + igb_error_reset_and_reinit(device); + + return -ETIMEDOUT; +} + +static void igb_send_msi(struct vfio_pci_device *device) +{ + struct igb *igb = to_igb_state(device); + + igb_write32(igb, E1000_EICS, MSIX_VECTOR_MASK); +} + +const struct vfio_pci_driver_ops igb_ops = { + .name = "igb", + .probe = igb_probe, + .init = igb_init, + .remove = igb_remove, + .memcpy_start = igb_memcpy_start, + .memcpy_wait = igb_memcpy_wait, + .send_msi = igb_send_msi, +}; |