summaryrefslogtreecommitdiff
path: root/drivers/gpu/drm/amd/amdkfd
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-11-20 20:43:06 -0800
committerLinus Torvalds <torvalds@linux-foundation.org>2026-11-20 20:43:06 -0800
commitc44104f3642da720164042623e4832836aefdd8e (patch)
tree76fc4494b40b186831ee690b8a335a0cb6b87480 /drivers/gpu/drm/amd/amdkfd
parentaa31d5f8b151b786b748c6bfc9a8c4f1855c8adc (diff)
parentce29b8f9fb606f87014c2d05483a3fef05a11f38 (diff)
Merge tag 'drm-fixes-2026-11-21' of https://gitlab.freedesktop.org/drm/kernelHEADmaster
Pull drm fixes from Dave Airlie: "Weekly fixes, around the usual size, though it looks like vmwgfx got hit with a lot of analysis and has a bunch of validation fixes. nouveau fixes for a 7.3 regression. I know of one open i915 7.3 regressions that hopefully a fix will land next week. Otherwise amdgpu and xe have the most with a scattering for others. core: - syncobj: Limit allocations - vblank: Resolve deadlock in timer amdgpu: - GPUVM fixes - SMU 8 fix - PCO quirk - ATPX lifetime fix - Unwinding fix for module loading failure - UserQ fix - HDMI fixes - DML2 fix - DC DRAM config fixes - DCN 4.2 fixes - DC pre-OS clock handling fixes - DPMS fix - Cleaner shader fixes for multi-XCC parts - Vblank fix for GPU resets - Properly handle firmware with different trust athorities - Freesync/VRR fixes - Backlight fix - Display warning fixes - PSR fixes amdkfd: - SVM fixes radeon: - Bounds checks in vbios parsing - ATPX lifetime fix xe: - hwmon error handling fixes - Treat CT-disabled TLB invalidation as cancelled - Validate BOs on export to charge the exporter - Always free requested IRQs on uninstall - Reset TLB invalidation state in submit fini - Use ordered clock reads for engine cycles cpu_delta - Treat non-fatal PCIe errors as recovered i915: - Regression fix for GitLab #16595 and #17423: 4K displays corrupt/black on CML/KBL - DSB programming fix for GOSUB imagination: - Use correct function for polling non-iomem ivpu: - Fix race condition in recovery nouveau: - Fix instmem io_mapping faults and slowdowns omap: - dss: Fix handling deferrals qaic: - Harden ioctl checks rocket: - iommu fixes - lifetime and validation fixes - clock request fix - devicetre core count fix tilcdc: - Fix drvdata in suspend/resume vc4: - async flip fixes - off by one shader record - object lifetime fixes - two dsi fixes vmwgfx: - bounds checking fixes - validation fixes" * tag 'drm-fixes-2026-11-21' of https://gitlab.freedesktop.org/drm/kernel: (111 commits) drm/vc4: Clear the slice caches before resetting a stalled GPU drm/vc4: Start shader code at varying offsets in its BO drm/vc4: Check for a validated shader before reading its thread mode drm/vc4: Fix off-by-one in the shader record BO handle check MAINTAINERS: Add Dmitry Baryshkov as bridge maintainer MAINTAINERS: Remove myself for VC4 DRM driver MAINTAINERS: Remove Andrzej Hajda from bridge maintainers MAINTAINERS: Remove Jonas Karlman from bridge reviewers MAINTAINERS: Move Tegra DRM under drm-misc MAINTAINERS: Move Keem Bay DRM under drm-misc MAINTAINERS: Move armada DRM under drm-misc drm/xe/hwmon: Propagate all hardware read errors drm/xe/hwmon: Report errors using structured xe_log API drm/xe/log: Add VRAM, PAT, and HWMON DRIVER_HARDWARE components drm/xe/guc: Treat CT-disabled TLB invalidation as cancelled drm/xe/dma-buf: Validate BOs on export to charge the exporter drm/xe/irq: Always free requested IRQs on uninstall drm/xe/guc: Reset TLB invalidation state in submit fini drm/xe/query: Use ordered clock reads for engine cycles cpu_delta drm/xe/ras: Treat non-fatal PCIe errors as recovered ...
Diffstat (limited to 'drivers/gpu/drm/amd/amdkfd')
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c5
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c2
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_priv.h7
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_svm.c67
4 files changed, 60 insertions, 21 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 61480e9119b8..4881684ed88f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -1534,9 +1534,12 @@ static int clean_process_queues_cpsch(struct device_queue_manager *dqm,
int retval = 0;
dqm_lock(dqm);
- if (qpd->evicted++ > 0) /* already evicted, do nothing */
+ if (qpd->fault_cleaned)
goto out;
+ qpd->evicted++;
+ qpd->fault_cleaned = true;
+
pdd = qpd_to_pdd(qpd);
/* The debugger creates processes that temporarily have not acquired
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
index 2ae1129fea55..1b597fab5395 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
@@ -370,7 +370,7 @@ static void gfx_9_clean_fault(uint16_t pasid)
if (!pdd)
return;
- /* To Clean the GFX on the faulting device */
+ /* To clean the GFX fault state on the faulting device */
dqm = pdd->dev->dqm;
dqm->ops.clean_process_queues_cpsch(dqm, &pdd->qpd);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 436160549386..a7b98a8b8e31 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -695,6 +695,11 @@ struct qcm_process_device {
bool is_debug;
unsigned int evicted; /* eviction counter, 0=active */
+ /* Set after fault clean runs; never cleared.
+ * Prevents redundant cleaning from pipelined interrupt workers.
+ */
+ bool fault_cleaned;
+
/* This flag tells if we should reset all wavefronts on
* process termination
*/
@@ -906,6 +911,8 @@ struct svm_range_list {
struct list_head deferred_range_list;
struct list_head criu_svm_metadata_list;
spinlock_t deferred_list_lock;
+ /* serializes XNACK-off evictions with the queue quiesce */
+ struct mutex evict_lock;
atomic_t evicted_ranges;
atomic_t drain_pagefaults;
struct delayed_work restore_work;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 7fd331612f08..2835c1d73d63 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1008,7 +1008,7 @@ svm_range_split_array(void *ppnew, void *ppold, size_t size,
kvfree(new);
return -ENOMEM;
}
- kvfree(pold);
+ /* The caller frees pold once all arrays of the range are split */
*(void **)ppold = old;
*(void **)ppnew = new;
@@ -1019,19 +1019,36 @@ static int
svm_range_split_pages(struct svm_range *new, struct svm_range *old,
uint64_t start, uint64_t last)
{
+ dma_addr_t *old_dma[MAX_GPU_INSTANCE];
uint64_t npages = last - start + 1;
bool has_dma_addr = false;
int i, r;
+ /*
+ * Split the arrays of all GPUs before replacing any of them, the old
+ * range must stay intact if one fails.
+ */
for (i = 0; i < MAX_GPU_INSTANCE; i++) {
+ old_dma[i] = old->dma_addr[i];
if (old->dma_addr[i])
has_dma_addr = true;
- r = svm_range_split_array(&new->dma_addr[i], &old->dma_addr[i],
+ r = svm_range_split_array(&new->dma_addr[i], &old_dma[i],
sizeof(*old->dma_addr[i]), old->start,
npages, new->start, new->npages,
old->actual_loc ? &new->vram_pages : NULL);
- if (r)
+ if (r) {
+ /* new->dma_addr[] goes with the new range */
+ while (i--)
+ if (old_dma[i] != old->dma_addr[i])
+ kvfree(old_dma[i]);
return r;
+ }
+ }
+ for (i = 0; i < MAX_GPU_INSTANCE; i++) {
+ if (old_dma[i] == old->dma_addr[i])
+ continue;
+ kvfree(old->dma_addr[i]);
+ old->dma_addr[i] = old_dma[i];
}
if (old->actual_loc) {
/*
@@ -1892,11 +1909,12 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
svm_range_lock(prange);
- /* Free backing memory of hmm_range if it was initialized
- * Override return value to TRY AGAIN only if prior returns
- * were successful
+ /*
+ * Retry if the pages were invalidated after they were taken.
+ * Only check that when taking them succeeded: a range can exist
+ * without a notifier, e.g. when allocating its PFN array failed.
*/
- if (range && !amdgpu_hmm_range_valid(range) && !r) {
+ if (!r && !amdgpu_hmm_range_valid(range)) {
pr_debug("hmm update the range, need validate again\n");
r = -EAGAIN;
}
@@ -2116,21 +2134,30 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm,
if (prange->start <= last && prange->last >= start)
atomic_inc(&prange->invalid);
+ /* The MMU notifier must not return before the GPU has stopped
+ * accessing the invalidated pages. Only the first eviction stops
+ * the queues, which takes milliseconds (HWS preemption on every
+ * GPU of the process). Serialize all evictions of this process so
+ * that a concurrent invalidation of another range waits until
+ * that quiesce has completed instead of returning while queues
+ * still run on stale PTEs.
+ */
+ mutex_lock(&svms->evict_lock);
evicted_ranges = atomic_inc_return(&svms->evicted_ranges);
- if (evicted_ranges != 1)
- return r;
-
- pr_debug("evicting svms 0x%p range [0x%lx 0x%lx]\n",
- prange->svms, prange->start, prange->last);
+ if (evicted_ranges == 1) {
+ pr_debug("evicting svms 0x%p range [0x%lx 0x%lx]\n",
+ prange->svms, prange->start, prange->last);
- /* First eviction, stop the queues */
- r = kgd2kfd_quiesce_mm(mm, KFD_QUEUE_EVICTION_TRIGGER_SVM);
- if (r)
- pr_debug("failed to quiesce KFD\n");
+ /* First eviction, stop the queues */
+ r = kgd2kfd_quiesce_mm(mm, KFD_QUEUE_EVICTION_TRIGGER_SVM);
+ if (r)
+ pr_debug("failed to quiesce KFD\n");
- pr_debug("schedule to restore svm %p ranges\n", svms);
- queue_delayed_work(system_freezable_wq, &svms->restore_work,
- msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS));
+ pr_debug("schedule to restore svm %p ranges\n", svms);
+ queue_delayed_work(system_freezable_wq, &svms->restore_work,
+ msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS));
+ }
+ mutex_unlock(&svms->evict_lock);
} else {
unsigned long s, l;
uint32_t trigger;
@@ -3412,6 +3439,7 @@ void svm_range_list_fini(struct kfd_process *p)
svm_range_free(prange, true);
}
+ mutex_destroy(&p->svms.evict_lock);
mutex_destroy(&p->svms.lock);
pr_debug("process pid %d svms 0x%p done\n",
@@ -3425,6 +3453,7 @@ int svm_range_list_init(struct kfd_process *p)
svms->objects = RB_ROOT_CACHED;
mutex_init(&svms->lock);
+ mutex_init(&svms->evict_lock);
INIT_LIST_HEAD(&svms->list);
atomic_set(&svms->evicted_ranges, 0);
atomic_set(&svms->drain_pagefaults, 0);