Merge tag 'iommu-updates-v7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux

Pull IOMMU updates from Joerg Roedel:
 "Core:
   - Detect scatterlist length overflows and finalize deferred domain
     attachments before mapping MSI pages
   - Consolidate x86 MSI reserved-region handling and clean up internal
     DMA and SVA helpers

  AMD IOMMU:
   - Add IOMMU PerfOpt support, enabled by amdgpu for supported
     integrated GPUs in identity domains
   - Restrict forced identity mapping to GPUs that require it
   - Fix kdump device-table allocation and reuse, ATS configuration
     error handling, and an interrupt-remapping allocation leak
   - Clean up device probing and initialization
   - Add Vasant Hegde as an AMD IOMMU maintainer

  ARM SMMUv2:
   - Device-tree binding updates for Qualcomm Maili and Nord SoCs

  ARM SMMUv3:
   - Work around hardware integration issue on Google Tensor G5 SoCs
     where all DMA transactions appear as instruction fetches
   - Allow command queue size to be configured on the kernel
     command-line
   - Reduce queue sizes when running as a kdump kernel
   - Fix some pre-existing issues in the driver and kunit tests reported
     by Sashiko
   - Fix handling of Arm hardware errata relating to the contiguous hint
     when using SVA
   - Rework the TLB invalidation logic in preparation for enabling
     iommu-pt in future
   - Minor cleanups in preparation for improving support for kdump

  Apple DART:
   - Support device-tree DMA aperture specifications

  Broadcom:
   - Add BCM2712 IOMMU support for Raspberry Pi 5 display and multimedia
     devices, including the generic page-table format and shared TLB
     cache management

  Hyper-V:
   - Add a para-virtualized guest IOMMU driver with guest-managed
     stage-1 page tables and page-selective IOTLB invalidation

  Intel VT-d:
   - Enhanced IQE fault handling for batched multiple descriptors,
   - Optimization of page-table level calculation in
     compute_vasz_lg2_ss(),
   - Fixes for several pre-existing issues reported by Sashiko,
   - General cleanup and refinement

  RISC-V:
   - Report cache-coherency capability for coherent devices, enabling
     VFIO/iommufd use

  Samsung Exynos:
   - Support SysMMUs without BLOCK mode, fixing their enable sequence
     and TLB invalidation
   - Improve v7 fault diagnostics with transaction information"

* tag 'iommu-updates-v7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux: (69 commits)
  MAINTAINERS: Make Vasant Hegde AMD IOMMU maintainer
  iommu/hyperv: Make hv_iommu_device static
  iommu/hyperv: Add missing PCI dependency for the guest IOMMU driver
  iommu/amd: Don't allocate kdump device table from DMA32
  iommu/amd: Program device table when kdump reuse fails
  iommu/dma: Clean up redundant MSI cookie check
  iommu/dma: Catch scatterlist length overflows
  iommu/hyperv: Add page-selective IOTLB flush support
  iommu/hyperv: Add para-virtualized IOMMU support for Hyper-V guest
  iommu/x86: Add architectural MSI reserved region helper
  Drivers: hv: Add logical device ID registry for vPCI devices
  hyperv: Introduce new hypercall interfaces used by Hyper-V guest IOMMU
  iommu: Make iommu-dma helpers private
  iommu/arm-smmu-v3: Don't rb_erase() a never-inserted stream node
  iommu/arm-smmu-v3: Add Q_POS() macro
  iommu/arm-smmu-v3: Add arm_smmu_attach_release()
  iommu/arm-smmu-v3: Disable the impl before disabling the SMMU on shutdown
  iommu/arm-smmu-v3: Populate the tlbi at the top of the call chain
  iommu/arm-smmu-v3: Precompute the invalidation commands
  iommu/arm-smmu-v3: Keep track in arm_smmu_invs if range invalidation is used
  ...

3 files changed, 63 insertions, 0 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index a9c6f5d4a639..0e53a02ad1ba 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -156,6 +156,7 @@ struct amdgpu_watchdog_timer {
  * Modules parameters.
  */
 extern int amdgpu_modeset;
+extern int amdgpu_iommu_perfopt;
 extern unsigned int amdgpu_vram_limit;
 extern int amdgpu_vis_vram_limit;
 extern int amdgpu_gart_size;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index ef9ff30c0a7b..1f67ad0233d5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -33,6 +33,7 @@
 #include <linux/console.h>
 #include <linux/slab.h>
 #include <linux/iommu.h>
+#include <linux/amd-iommu.h>
 #include <linux/pci.h>
 #include <linux/pci-p2pdma.h>
 #include <linux/apple-gmux.h>
@@ -3751,6 +3752,28 @@ amdgpu_device_should_register_switcheroo(struct amdgpu_device *adev, bool px)
 		       apple_gmux_detect(NULL, NULL)));
 }
 
+static inline bool amdgpu_device_identity(struct amdgpu_device *adev)
+{
+	struct pci_dev *pdev = adev->pdev;
+	struct iommu_domain *domain = iommu_get_domain_for_dev(&pdev->dev);
+
+	if (!domain)
+		return false;
+
+	return domain->type == IOMMU_DOMAIN_IDENTITY;
+}
+
+static bool amdgpu_device_use_perfopt(struct amdgpu_device *adev)
+{
+	if (amdgpu_iommu_perfopt == 0)
+		return false;
+
+	if (!(adev->flags & AMD_IS_APU))
+		return false;
+
+	return amdgpu_device_identity(adev);
+}
+
 /**
  * amdgpu_device_init - initialize the driver
  *
@@ -3958,6 +3981,16 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	if (r)
 		return r;
 
+	if (amdgpu_device_use_perfopt(adev)) {
+		int perfopt_ret = amd_iommu_enable_perfopt(pdev);
+
+		/* Optional optimization; a failure to arm it must not abort probe. */
+		if (perfopt_ret)
+			dev_warn(adev->dev,
+				 "Failed to enable IOMMU PerfOpt (%d); continuing without it\n",
+				 perfopt_ret);
+	}
+
 	/*
 	 * No need to remove conflicting FBs for non-display class devices.
 	 * This prevents the sysfb from being freed accidently.
@@ -4327,6 +4360,9 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)
 
 	amdgpu_gart_dummy_page_fini(adev);
 
+	if (amdgpu_device_use_perfopt(adev))
+		amd_iommu_disable_perfopt(adev->pdev);
+
 	if (pci_dev_is_disconnected(adev->pdev))
 		amdgpu_device_unmap_mmio(adev);
 
@@ -4690,6 +4726,20 @@ int amdgpu_device_resume(struct drm_device *dev, bool notify_clients)
 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 		return 0;
 
+	if (amdgpu_device_use_perfopt(adev)) {
+		int perfopt_ret = amd_iommu_enable_perfopt(adev->pdev);
+
+		/*
+		 * Must not return on failure: a bare return would leak the
+		 * SR-IOV VF exclusive-mode acquisition taken above (released
+		 * via the exit: path).
+		 */
+		if (perfopt_ret)
+			dev_warn(adev->dev,
+				 "Failed to enable IOMMU PerfOpt (%d); continuing without it\n",
+				 perfopt_ret);
+	}
+
 	if (adev->in_s0ix)
 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index d9d794a7db96..d27f7677a520 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -185,6 +185,7 @@ char *amdgpu_disable_cu;
 char *amdgpu_virtual_display;
 int amdgpu_enforce_isolation = -1;
 int amdgpu_modeset = -1;
+int amdgpu_iommu_perfopt = -1;
 
 /* Specifies the default granularity for SVM, used in buffer
  * migration and restoration of backing memory when handling
@@ -393,6 +394,17 @@ MODULE_PARM_DESC(aspm, "ASPM support (1 = enable, 0 = disable, -1 = auto)");
 module_param_named(aspm, amdgpu_aspm, int, 0444);
 
 /**
+ * DOC: iommu_perfopt (int)
+ * Control the AMD IOMMU PerfOpt DMA-latency optimization
+ * (0 = disable; -1 = enable on supported devices).
+ * This arms the IOMMU PerfOpt control (IOMMU spec, MMIO Offset 016Ch, EFR PerfOptSup / PerfOptEn)
+ * Arming it disables ATS, PRI, PASID and SVA for the GPU and removes IOMMU DMA containment for it,
+ * trading isolation for lower DMA latency.
+ */
+MODULE_PARM_DESC(iommu_perfopt, "Control IOMMU PerfOpt DMA-latency optimization (-1 = enable on supported devices, 0 = disable)");
+module_param_named(iommu_perfopt, amdgpu_iommu_perfopt, int, 0444);
+
+/**
  * DOC: runpm (int)
  * Override for runtime power management control for dGPUs. The amdgpu driver can dynamically power down
  * the dGPUs when they are idle if supported. The default is -1 (auto enable).