diff options
| author | Dave Airlie <airlied@redhat.com> | 2026-10-02 07:43:30 +1000 |
|---|---|---|
| committer | Dave Airlie <airlied@redhat.com> | 2026-10-02 07:44:44 +1000 |
| commit | 845c5cc3697c5520fe3d7fc8597d393d172cc5fd (patch) | |
| tree | a4e0c16d3eff7330337c13f4022023b882135b21 | |
| parent | cd9d5e4be968041fd811f5bb6d27d05921877e27 (diff) | |
| parent | 70dab9ee7d620e6018a64798aed065d14e9baed1 (diff) | |
Merge tag 'drm-xe-next-2026-10-01' of https://gitlab.freedesktop.org/drm/xe/kernel into drm-next
UAPI Changes:
- New sysfs to expose device UID (Badal)
Driver Changes:
- Nuke PTs only after unlinking contested VMAs (Auld)
- Limit sg segment size to PAGE_SIZE on Xen PV (Szymon)
- Move locking to xe_sriov_pf_reprovision_default (Michal)
- New workarounds (Tilak, Tales, Daniel)
- Keep VF LMEM BAR size low if no VFs enabled (Marcin)
- Drop ratelimiting from dev_unhold_work debug trace (Varun)
- System Controller's Maibox response improvement (Anoop)
- General clean-ups (Roper, Lin)
- Revert "drm/xe: Set mask bits for CCS_MODE register" (Roper)
- CPU binds and ULLS on migration queue (Brost)
- Re-apply an i2c patch silently removed by a merge (Bartosz)
- New PTL PCI ID (Atwood)
- Enable PXP for NVL-S (Alan)
- Let xe_guc_ct_send_locked() callers reserve G2H credit (Sujaritha)
- Implement print vfunc for page fault info logging (Brian)
- Do not dma_fence_put zero fences in err_out path (Jonathan)
- Add focused VM teardown and page fault tracing (Daniel)
- Fix ULLS chained job loss and GT reset replay (Varun)
- Refresh TTM LRU on SVM prefetch (Himal)
- Fix xe_mmio_wait32() to honor delay/sleep maximums (Alan)
Signed-off-by: Dave Airlie <airlied@redhat.com>
From: Rodrigo Vivi <rodrigo.vivi@intel.com>
Link: https://patch.msgid.link/ar5YsqAK3bS4Ta6Z@intel.com
81 files changed, 2721 insertions, 1374 deletions
diff --git a/Documentation/ABI/testing/sysfs-driver-intel-xe-gpu b/Documentation/ABI/testing/sysfs-driver-intel-xe-gpu new file mode 100644 index 000000000000..4dc4afc3961d --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-intel-xe-gpu @@ -0,0 +1,10 @@ +What: /sys/bus/pci/drivers/xe/.../device_uid +Date: October 2026 +KernelVersion: 7.4 +Contact: intel-xe@lists.freedesktop.org +Description: + RO. Unique 64-bit identifier of the GPU SoC device, exposed as + hexadecimal value. + + This sysfs file is present only on supported Intel Xe platforms. + Accessible only to users with administrative privileges. diff --git a/Documentation/gpu/xe/xe_migrate.rst b/Documentation/gpu/xe/xe_migrate.rst index f92faec0ac94..d297ee53a582 100644 --- a/Documentation/gpu/xe/xe_migrate.rst +++ b/Documentation/gpu/xe/xe_migrate.rst @@ -6,3 +6,6 @@ Migrate Layer .. kernel-doc:: drivers/gpu/drm/xe/xe_migrate_doc.h :doc: Migrate Layer + +.. kernel-doc:: drivers/gpu/drm/xe/xe_migrate.c + :doc: ULLS (Ultra Low Latency Submission) for migration jobs diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c index b6c9d3a07dc8..a37b26881b73 100644 --- a/drivers/gpu/drm/drm_gpusvm.c +++ b/drivers/gpu/drm/drm_gpusvm.c @@ -1528,7 +1528,49 @@ static bool drm_gpusvm_pages_inlinable(struct drm_gpusvm_pages *svm_pages, } /** - * drm_gpusvm_dma_map_pages() - DMA map one drm_gpusvm_pages instance + * drm_gpusvm_walk_devmem() - Invoke the devmem callback across faulted pages + * @gpusvm: Pointer to the GPU SVM structure + * @pfns: The already-faulted pfn array (size @npages) + * @npages: Number of pages in the CPU range + * @ctx: GPU SVM context, with a non-NULL &drm_gpusvm_ctx.devmem_fn + * + * Invoke &drm_gpusvm_ctx.devmem_fn once per contiguous run of @pfns backed by + * the same device-memory allocation. Must be called under the notifier lock. + */ +static void drm_gpusvm_walk_devmem(struct drm_gpusvm *gpusvm, + unsigned long *pfns, + unsigned long npages, + const struct drm_gpusvm_ctx *ctx) +{ + struct drm_pagemap_devmem *last = NULL; + unsigned int order = 0; + unsigned long i; + + lockdep_assert_held(&gpusvm->notifier_lock); + + for (i = 0; i < npages; i += 1 << order) { + struct page *page = hmm_pfn_to_page(pfns[i]); + struct drm_pagemap_devmem *devmem; + + order = drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages); + + if (!is_device_private_page(page) && + !is_device_coherent_page(page)) { + last = NULL; + continue; + } + + devmem = drm_pagemap_page_to_devmem(page); + if (devmem == last) + continue; + + last = devmem; + ctx->devmem_fn(devmem); + } +} + +/** + * drm_gpusvm_dma_map_pages() - Walk and DMA map one drm_gpusvm_pages instance * @gpusvm: Pointer to the GPU SVM structure * @svm_pages: The SVM pages instance to populate with dma-addresses * @pfns: The already-faulted pfn array (size @npages) @@ -1536,10 +1578,10 @@ static bool drm_gpusvm_pages_inlinable(struct drm_gpusvm_pages *svm_pages, * @ctx: GPU SVM context * @dma_dir: DMA data direction for the mappings * - * Map the faulted @pfns into @svm_pages for DMA access through its owning - * drm_device. Must be called under the notifier lock and only for an instance - * without a live mapping. On failure this unwinds the partial mapping of this - * instance before returning. + * Walk the faulted @pfns and map them into @svm_pages for DMA access through + * its owning drm_device. Must be called under the notifier lock and only for + * an instance without a live mapping. On failure this unwinds the partial + * mapping of this instance before returning. * * Return: 0 on success, negative error code on failure. */ @@ -1774,9 +1816,20 @@ retry: hmm_range.notifier_seq = mmu_interval_read_begin(notifier); - if (map_dma && - drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages, num_pages)) - goto set_seqno; + /* + * drm_gpusvm_pages_valid_unlocked() also resets any instance whose + * mapping was invalidated, so it must run on the map_dma path even when + * devmem_fn suppresses the fast path; otherwise a stale inline mapping + * is later mistaken for a dma_addr array by drm_gpusvm_dma_map_pages(). + */ + if (map_dma) { + bool pages_valid = drm_gpusvm_pages_valid_unlocked(gpusvm, + svm_pages, + num_pages); + + if (!ctx->devmem_fn && pages_valid) + goto set_seqno; + } pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL); if (!pfns) @@ -1829,28 +1882,30 @@ retry: goto retry; } - if (!map_dma) - goto done_mapping; + if (ctx->devmem_fn) + drm_gpusvm_walk_devmem(gpusvm, pfns, npages, ctx); - for (p = 0; p < num_pages; ++p) { - if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p])) - continue; + if (map_dma) { + for (p = 0; p < num_pages; ++p) { + if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p])) + continue; - err = drm_gpusvm_dma_map_pages(gpusvm, &svm_pages[p], pfns, - npages, ctx, dma_dir); - if (err) { - /* - * The failing instance was unwound by the helper. Keep - * the ones mapped earlier: the -EAGAIN retry reuses - * them, and the driver unmaps every instance with the - * range on the other error paths. - */ - drm_gpusvm_notifier_unlock(gpusvm); - goto err_free; + err = drm_gpusvm_dma_map_pages(gpusvm, &svm_pages[p], pfns, + npages, ctx, dma_dir); + if (err) { + /* + * The failing instance was unwound by the + * helper. Keep the ones mapped earlier: the + * -EAGAIN retry reuses them, and the driver + * unmaps every instance with the range on the + * other error paths. + */ + drm_gpusvm_notifier_unlock(gpusvm); + goto err_free; + } } } -done_mapping: drm_gpusvm_notifier_unlock(gpusvm); kvfree(pfns); set_seqno: diff --git a/drivers/gpu/drm/drm_pagemap.c b/drivers/gpu/drm/drm_pagemap.c index c66c58f85c34..bbef0c225b7e 100644 --- a/drivers/gpu/drm/drm_pagemap.c +++ b/drivers/gpu/drm/drm_pagemap.c @@ -1102,7 +1102,7 @@ static void drm_pagemap_dev_unhold_work(struct work_struct *work) struct drm_device *drm = dev_hold->drm; struct module *module = drm->driver->fops->owner; - drm_dbg_ratelimited(drm, "Releasing reference on provider device and module.\n"); + drm_dbg(drm, "Releasing reference on provider device and module.\n"); drm_dev_put(drm); module_put(module); kfree(dev_hold); @@ -1630,6 +1630,21 @@ struct drm_pagemap *drm_pagemap_page_to_dpagemap(struct page *page) EXPORT_SYMBOL_GPL(drm_pagemap_page_to_dpagemap); /** + * drm_pagemap_page_to_devmem() - Return the devmem allocation backing a page + * @page: The struct page. + * + * Return: The &drm_pagemap_devmem backing @page. Undefined if @page was not + * populated from a &drm_pagemap. + */ +struct drm_pagemap_devmem *drm_pagemap_page_to_devmem(struct page *page) +{ + struct drm_pagemap_zdd *zdd = drm_pagemap_page_zone_device_data(page); + + return zdd->devmem_allocation; +} +EXPORT_SYMBOL_GPL(drm_pagemap_page_to_devmem); + +/** * drm_pagemap_populate_mm() - Populate a virtual range with device memory pages * @dpagemap: Pointer to the drm_pagemap managing the device memory * @start: Start of the virtual range to populate. diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 67b8b5477639..0670a3f08ec8 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -35,6 +35,7 @@ $(obj)/generated/%_device_wa_oob.c $(obj)/generated/%_device_wa_oob.h: $(obj)/xe xe-y += xe_bb.o \ xe_bo.o \ xe_bo_evict.o \ + xe_cpu_bind.o \ xe_dep_scheduler.o \ xe_devcoredump.o \ xe_device.o \ diff --git a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h index 685c4ef17b73..4bcdc1a96785 100644 --- a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h +++ b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h @@ -525,6 +525,7 @@ enum xe_guc_klv_ids { GUC_WA_KLV_CLR_CS_INDIRECT_RING_STATE_IF_IDLE_AT_CTX_REG = 0x900e, GUC_WA_KLV_REMAP_RANGED_TLB_INV = 0x900f, GUC_WA_KLV_IGNORE_MMIO_READ_SEM_TOKEN_64 = 0x9010, + GUC_WA_KLV_IGNORE_MULTIQ_CTX_COMPLETE_WITH_PREEMPT_IN_CSB = 0x9011, }; /** diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h index 2e059cb36c82..650f117c3e13 100644 --- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h @@ -468,6 +468,7 @@ #define L3_SQ_DISABLE_COAMA_2WAY_COH REG_BIT(30) #define L3_SQ_DISABLE_COAMA REG_BIT(22) #define COMPMEMRD256BOVRFETCHEN REG_BIT(20) +#define NONCOMPMEMRD256BOVRFETCHEN REG_BIT(14) #define L3SQCREG3 XE_REG_MCR(0xb108) #define COMPPWOVERFETCHEN REG_BIT(28) @@ -636,7 +637,7 @@ * [4-6] RSVD * [7] Disabled */ -#define CCS_MODE XE_REG(0x14804, XE_REG_OPTION_MASKED) +#define CCS_MODE XE_REG(0x14804) #define CCS_MODE_CSLICE_0_3_MASK REG_GENMASK(11, 0) /* 3 bits per cslice */ #define CCS_MODE_CSLICE_MASK 0x7 /* CCS0-3 + rsvd */ #define CCS_MODE_CSLICE_WIDTH ilog2(CCS_MODE_CSLICE_MASK + 1) diff --git a/drivers/gpu/drm/xe/regs/xe_regs.h b/drivers/gpu/drm/xe/regs/xe_regs.h index ef4746b7b5d3..437485b5a0af 100644 --- a/drivers/gpu/drm/xe/regs/xe_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_regs.h @@ -30,6 +30,8 @@ #define XEHP_MTCFG_ADDR XE_REG(0x101800) #define TILE_COUNT REG_GENMASK(15, 8) +#define CRI_DEVICE_UID XE_REG(0x102008) + #define GGC XE_REG(0x108040) #define GMS_MASK REG_GENMASK(15, 8) #define GGMS_MASK REG_GENMASK(7, 6) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index f2ab9bf43a86..6921b6967330 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -1879,7 +1879,7 @@ static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo) xe_drm_client_remove_bo(bo); #endif - if (bo->vm && xe_bo_is_user(bo)) + if (bo->vm && (xe_bo_is_user(bo) || bo->flags & XE_BO_FLAG_PAGETABLE)) xe_vm_put(bo->vm); if (bo->parent_obj) @@ -2575,7 +2575,7 @@ __xe_bo_create_locked(struct xe_device *xe, * by having all the vm's bo refereferences released at vm close * time. */ - if (vm && xe_bo_is_user(bo)) + if (vm && (xe_bo_is_user(bo) || bo->flags & XE_BO_FLAG_PAGETABLE)) xe_vm_get(vm); bo->vm = vm; diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index 290ca624e2a7..861b1be231de 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -9,6 +9,8 @@ #include <drm/drm_prime.h> #include <drm/ttm/ttm_tt.h> +#include <xen/xen.h> + #include "xe_bo_types.h" #include "xe_ggtt.h" #include "xe_macros.h" @@ -494,6 +496,8 @@ void __xe_bo_release_dummy(struct kref *kref); * @bo: The bo to put. * @deferred: List to which to add the buffer object if we cannot put, or * NULL if the function is to put unconditionally. + * @added: BO was added to deferred list, written back to caller, can be NULL if + * writeback is not needed. Only set to true when added, never set to false. * * Since the final freeing of an object includes both sleeping and (!) * memory allocation in the dma_resv individualization, it's not ok @@ -513,7 +517,8 @@ void __xe_bo_release_dummy(struct kref *kref); * false otherwise. */ static inline bool -xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred) +xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred, + bool *added) { if (!deferred) { xe_bo_put(bo); @@ -523,6 +528,9 @@ xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred) if (!kref_put(&bo->ttm.base.refcount, __xe_bo_release_dummy)) return false; + if (added) + *added = true; + return llist_add(&bo->freed, deferred); } @@ -539,7 +547,7 @@ xe_bo_put_async(struct xe_bo *bo) { struct xe_bo_dev *bo_device = &xe_bo_device(bo)->bo_device; - if (xe_bo_put_deferred(bo, &bo_device->async_list)) + if (xe_bo_put_deferred(bo, &bo_device->async_list, NULL)) schedule_work(&bo_device->async_free); } @@ -574,6 +582,23 @@ static inline unsigned int xe_sg_segment_size(struct device *dev) struct scatterlist __maybe_unused sg; size_t max = BIT_ULL(sizeof(sg.length) * 8) - 1; + /* + * For Xen PV guests pages aren't contiguous in DMA (machine) address + * space. The DMA API takes care of that both in dma_alloc_* (by + * calling into the hypervisor to make the pages contiguous) and in + * dma_map_* (by bounce buffering). But xe (like i915, see commit + * 78a07fe777c4) ignores the coherency aspects of the DMA API and thus + * can't cope with bounce buffering actually happening, so add a hack + * here to force small allocations and mappings when running in PV + * mode on Xen. + * + * Note this will still break if bounce buffering is required for other + * reasons, like confidential computing hypervisors or PCIe root ports + * with addressing limitations. + */ + if (xen_pv_domain()) + return PAGE_SIZE; + max = min_t(size_t, max, dma_max_mapping_size(dev)); /* diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h index 0eb93052c1d5..8ec4a01a0092 100644 --- a/drivers/gpu/drm/xe/xe_bo_types.h +++ b/drivers/gpu/drm/xe/xe_bo_types.h @@ -90,8 +90,6 @@ struct xe_bo { /** @freed: List node for delayed put. */ struct llist_node freed; - /** @update_index: Update index if PT BO */ - int update_index; /** @created: Whether the bo has passed initial creation */ bool created; diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index f5c828cf7e8f..ad4bfd3c0703 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -61,6 +61,7 @@ * ├── survivability_mode * ├── gt_types_allowed * ├── engines_allowed + * ├── migrate_ulls_period_ms * ├── enable_psmi * └── disable_vram_page_offline * @@ -262,6 +263,20 @@ * * This attribute can only be set before binding to the device. * + * Migrate ULLS Period (ms) + * ------------------------ + * + * Migrate ULLS period, in milliseconds. This is the delay between entering + * migrate ULLS (a continuously running batch) and exiting it. Migrate ULLS is + * currently entered during page faults and SVM prefetch operations. Default 5, + * zero indicates ULLS is disabled. + * + * How to disable migration ULLS: + * + * # echo 0 > /sys/kernel/config/xe/0000:03:00.0/migrate_ulls_period_ms + * + * This attribute can only be set before binding to the device. + * * Remove devices * ============== * @@ -283,6 +298,7 @@ struct xe_config_group_device { struct xe_config_device { u64 gt_types_allowed; u64 engines_allowed; + u32 migrate_ulls_period_ms; struct wa_bb ctx_restore_post_bb[XE_ENGINE_CLASS_MAX]; struct wa_bb ctx_restore_mid_bb[XE_ENGINE_CLASS_MAX]; bool survivability_mode; @@ -306,6 +322,7 @@ struct xe_config_group_device { static const struct xe_config_device device_defaults = { .gt_types_allowed = U64_MAX, .engines_allowed = U64_MAX, + .migrate_ulls_period_ms = 5, .survivability_mode = false, .enable_psmi = false, .enable_multi_queue = true, @@ -631,6 +648,33 @@ static ssize_t enable_multi_queue_store(struct config_item *item, const char *pa return len; } +static ssize_t migrate_ulls_period_ms_show(struct config_item *item, char *page) +{ + struct xe_config_device *dev = to_xe_config_device(item); + + return sprintf(page, "%u\n", dev->migrate_ulls_period_ms); +} + +static ssize_t migrate_ulls_period_ms_store(struct config_item *item, + const char *page, size_t len) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item); + u32 val; + int ret; + + ret = kstrtou32(page, 0, &val); + if (ret) + return ret; + + guard(mutex)(&dev->lock); + if (is_bound(dev)) + return -EBUSY; + + dev->config.migrate_ulls_period_ms = val; + + return len; +} + static ssize_t disable_vram_page_offline_show(struct config_item *item, char *page) { struct xe_config_device *dev = to_xe_config_device(item); @@ -896,6 +940,7 @@ static ssize_t ctx_restore_post_bb_store(struct config_item *item, CONFIGFS_ATTR(, ctx_restore_mid_bb); CONFIGFS_ATTR(, ctx_restore_post_bb); CONFIGFS_ATTR(, enable_multi_queue); +CONFIGFS_ATTR(, migrate_ulls_period_ms); CONFIGFS_ATTR(, enable_psmi); CONFIGFS_ATTR(, disable_vram_page_offline); CONFIGFS_ATTR(, engines_allowed); @@ -906,6 +951,7 @@ static struct configfs_attribute *xe_config_device_attrs[] = { &attr_ctx_restore_mid_bb, &attr_ctx_restore_post_bb, &attr_enable_multi_queue, + &attr_migrate_ulls_period_ms, &attr_enable_psmi, &attr_disable_vram_page_offline, &attr_engines_allowed, @@ -1189,6 +1235,7 @@ static void dump_custom_dev_config(struct pci_dev *pdev, PRI_CUSTOM_ATTR("%llx", gt_types_allowed); PRI_CUSTOM_ATTR("%llx", engines_allowed); + PRI_CUSTOM_ATTR("%u", migrate_ulls_period_ms); PRI_CUSTOM_ATTR("%d", enable_multi_queue); PRI_CUSTOM_ATTR("%d", enable_psmi); PRI_CUSTOM_ATTR("%d", disable_vram_page_offline); @@ -1341,6 +1388,26 @@ bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev) } /** + * xe_configfs_get_migrate_ulls_period_ms - get configfs migrate_ulls_period_ms setting + * @pdev: pci device + * + * Return: Migrate ULLS period in milliseconds (zero is disabled). + */ +u32 xe_configfs_get_migrate_ulls_period_ms(struct pci_dev *pdev) +{ + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); + u32 ret; + + if (!dev) + return device_defaults.migrate_ulls_period_ms; + + ret = dev->config.migrate_ulls_period_ms; + config_group_put(&dev->group); + + return ret; +} + +/** * xe_configfs_get_disable_vram_page_offline - get configfs disable_vram_page_offline setting * @pdev: pci device * diff --git a/drivers/gpu/drm/xe/xe_configfs.h b/drivers/gpu/drm/xe/xe_configfs.h index 42cd1a491d01..10a00d6bbf8f 100644 --- a/drivers/gpu/drm/xe/xe_configfs.h +++ b/drivers/gpu/drm/xe/xe_configfs.h @@ -24,6 +24,7 @@ bool xe_configfs_media_gt_allowed(struct pci_dev *pdev); u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev); bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev); bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev); +u32 xe_configfs_get_migrate_ulls_period_ms(struct pci_dev *pdev); bool xe_configfs_get_disable_vram_page_offline(struct pci_dev *pdev); u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, enum xe_engine_class class, @@ -45,6 +46,7 @@ static inline bool xe_configfs_media_gt_allowed(struct pci_dev *pdev) { return t static inline u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) { return U64_MAX; } static inline bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev) { return false; } static inline bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev) { return true; } +static inline u32 xe_configfs_get_migrate_ulls_period_ms(struct pci_dev *pdev) { return 5; } static inline bool xe_configfs_get_disable_vram_page_offline(struct pci_dev *pdev) { return false; } static inline u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, enum xe_engine_class class, diff --git a/drivers/gpu/drm/xe/xe_cpu_bind.c b/drivers/gpu/drm/xe/xe_cpu_bind.c new file mode 100644 index 000000000000..318978298420 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_cpu_bind.c @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include <drm/drm_managed.h> +#include <linux/mutex.h> + +#include "xe_cpu_bind.h" +#include "xe_device_types.h" +#include "xe_exec_queue.h" +#include "xe_pt.h" +#include "xe_sched_job.h" +#include "xe_trace_bo.h" +#include "xe_vm.h" + +/** + * struct xe_cpu_bind - cpu_bind context. + */ +struct xe_cpu_bind { + /** @xe: Xe device */ + struct xe_device *xe; + /** @q: Default exec queue used for kernel binds */ + struct xe_exec_queue *q; + /** @job_mutex: Timeline mutex for @q. */ + struct mutex job_mutex; +}; + +static bool is_cpu_bind_queue(struct xe_cpu_bind *cpu_bind, + struct xe_exec_queue *q) +{ + return cpu_bind->q == q; +} + +static void xe_cpu_bind_fini(void *arg) +{ + struct xe_cpu_bind *cpu_bind = arg; + + mutex_destroy(&cpu_bind->job_mutex); + xe_exec_queue_put(cpu_bind->q); +} + +/** + * xe_cpu_bind_init() - Initialize a cpu_bind context + * @xe: &struct xe_device + * + * Return: 0 if successful, negative error code on failure + */ +int xe_cpu_bind_init(struct xe_device *xe) +{ + struct xe_cpu_bind *cpu_bind = + drmm_kzalloc(&xe->drm, sizeof(*cpu_bind), GFP_KERNEL); + struct xe_exec_queue *q; + + if (!cpu_bind) + return -ENOMEM; + + q = xe_exec_queue_create_bind(xe, xe_device_get_root_tile(xe), NULL, + EXEC_QUEUE_FLAG_KERNEL | + EXEC_QUEUE_FLAG_MIGRATE, 0); + if (IS_ERR(q)) + return PTR_ERR(q); + + cpu_bind->xe = xe; + cpu_bind->q = q; + xe->cpu_bind = cpu_bind; + + mutex_init(&cpu_bind->job_mutex); + + fs_reclaim_acquire(GFP_KERNEL); + might_lock(&cpu_bind->job_mutex); + fs_reclaim_release(GFP_KERNEL); + + return devm_add_action_or_reset(cpu_bind->xe->drm.dev, xe_cpu_bind_fini, + cpu_bind); +} + +/** + * xe_cpu_bind_queue() - Get the bind queue from cpu_bind context. + * @cpu_bind: The cpu bind context. + * + * Return: Pointer to bind queue. + */ +struct xe_exec_queue *xe_cpu_bind_queue(struct xe_cpu_bind *cpu_bind) +{ + return cpu_bind->q; +} + +/** + * xe_cpu_bind_update_pgtables_execute() - Update a VM's PTEs via the CPU + * @vm: The VM being updated + * @tile: The tile being updated + * @ops: The CPU bind PT update ops + * @pt_op: The VM PT update op + * @num_ops: The number of The VM PT update ops + * @force_clear: Force clear operation + * + * Execute the VM PT update ops array which results in a VM's PTEs being updated + * via the CPU. + */ +void +xe_cpu_bind_update_pgtables_execute(struct xe_vm *vm, struct xe_tile *tile, + const struct xe_cpu_bind_pt_update_ops *ops, + struct xe_vm_pgtable_update_op *pt_op, + u32 num_ops, bool force_clear) +{ + u32 j, i; + + for (j = 0; j < num_ops; ++j, ++pt_op) { + for (i = 0; i < pt_op->num_entries; i++) { + const struct xe_vm_pgtable_update *update = + &pt_op->entries[i]; + + xe_assert(vm->xe, !iosys_map_is_null(&update->pt_bo->vmap)); + + if (pt_op->bind && !force_clear) + ops->populate(tile, &update->pt_bo->vmap, + update); + else + ops->clear(vm, tile, &update->pt_bo->vmap, + update); + } + } + + trace_xe_vm_cpu_bind(vm); + xe_device_wmb(vm->xe); +} + +static struct dma_fence * +xe_cpu_bind_update_pgtables_no_job(struct xe_cpu_bind *cpu_bind, + struct xe_cpu_bind_pt_update *pt_update) +{ + const struct xe_cpu_bind_pt_update_ops *ops = pt_update->ops; + struct xe_vm *vm = pt_update->vops->vm; + struct xe_tile *tile; + int err, id; + + if (ops->pre_commit) { + pt_update->job = NULL; + err = ops->pre_commit(pt_update); + if (err) + return ERR_PTR(err); + } + + for_each_tile(tile, vm->xe, id) { + struct xe_vm_pgtable_update_ops *pt_update_ops = + &pt_update->vops->pt_update_ops[tile->id]; + + if (!pt_update_ops->pt_job_ops) + continue; + + xe_cpu_bind_update_pgtables_execute(vm, tile, ops, + pt_update_ops->pt_job_ops->ops, + pt_update_ops->pt_job_ops->current_op, + false); + } + + return dma_fence_get_stub(); +} + +static struct dma_fence * +xe_cpu_bind_update_pgtables_job(struct xe_cpu_bind *cpu_bind, + struct xe_cpu_bind_pt_update *pt_update) +{ + const struct xe_cpu_bind_pt_update_ops *ops = pt_update->ops; + struct xe_exec_queue *q = pt_update->vops->q; + struct xe_device *xe = cpu_bind->xe; + struct xe_sched_job *job; + struct dma_fence *fence; + struct xe_tile *tile; + int err, id; + bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q); + + job = xe_sched_job_create(q, NULL); + if (IS_ERR(job)) + return ERR_CAST(job); + + xe_assert(xe, job->is_pt_job); + + if (ops->pre_commit) { + pt_update->job = job; + err = ops->pre_commit(pt_update); + if (err) + goto err_job; + } + + if (is_cpu_bind) + mutex_lock(&cpu_bind->job_mutex); + + job->pt_update[0].vm = pt_update->vops->vm; + job->pt_update[0].ops = ops; + for_each_tile(tile, xe, id) { + struct xe_vm_pgtable_update_ops *pt_update_ops = + &pt_update->vops->pt_update_ops[tile->id]; + + job->pt_update[0].pt_job_ops[tile->id] = + xe_pt_job_ops_get(pt_update_ops->pt_job_ops); + } + + xe_sched_job_arm(job); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + if (is_cpu_bind) + mutex_unlock(&cpu_bind->job_mutex); + + return fence; + +err_job: + xe_sched_job_put(job); + return ERR_PTR(err); +} + +/** + * xe_cpu_bind_update_pgtables() - Pipelined page-table update + * @cpu_bind: The cpu bind context. + * @pt_update: PT update arguments + * + * Perform a pipelined page-table update. The update descriptors are typically + * built under the same lock critical section as a call to this function. If + * using the default engine for the updates, they will be performed in the + * order they grab the job_mutex. If different engines are used, external + * synchronization is needed for overlapping updates to maintain page-table + * consistency. Note that the meaning of "overlapping" is that the updates + * touch the same page-table, which might be a higher-level page-directory. + * If no pipelining is needed, then updates may be performed by the cpu. + * + * Return: A dma_fence that, when signaled, indicates the update completion. + */ +struct dma_fence * +xe_cpu_bind_update_pgtables(struct xe_cpu_bind *cpu_bind, + struct xe_cpu_bind_pt_update *pt_update) +{ + struct dma_fence *fence; + + fence = xe_cpu_bind_update_pgtables_no_job(cpu_bind, pt_update); + + /* -ETIME indicates a job is needed, anything else is legit error */ + if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME) + return fence; + + return xe_cpu_bind_update_pgtables_job(cpu_bind, pt_update); +} + +/** + * xe_cpu_bind_job_lock() - Lock cpu_bind job lock + * @cpu_bind: The cpu bind context. + * @q: Queue associated with the operation which requires a lock + * + * Lock the cpu_bind job lock if the queue is a cpu bind queue, otherwise + * assert the VM's dma-resv is held (user queue's have own locking). + */ +void xe_cpu_bind_job_lock(struct xe_cpu_bind *cpu_bind, + struct xe_exec_queue *q) +{ + bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q); + + if (is_cpu_bind) + mutex_lock(&cpu_bind->job_mutex); + else + xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */ +} + +/** + * xe_cpu_bind_job_unlock() - Unlock cpu_bind job lock + * @cpu_bind: The cpu bind context. + * @q: Queue associated with the operation which requires a lock + * + * Unlock the cpu_bind job lock if the queue is a cpu bind queue, otherwise + * assert the VM's dma-resv is held (user queue's have own locking). + */ +void xe_cpu_bind_job_unlock(struct xe_cpu_bind *cpu_bind, + struct xe_exec_queue *q) +{ + bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q); + + if (is_cpu_bind) + mutex_unlock(&cpu_bind->job_mutex); + else + xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */ +} + +#if IS_ENABLED(CONFIG_PROVE_LOCKING) +/** + * xe_cpu_bind_job_lock_assert() - Assert cpu_bind job lock held of queue + * @q: cpu bind queue + */ +void xe_cpu_bind_job_lock_assert(struct xe_exec_queue *q) +{ + struct xe_device *xe = gt_to_xe(q->gt); + struct xe_cpu_bind *cpu_bind = xe->cpu_bind; + + xe_assert(xe, q == cpu_bind->q); + lockdep_assert_held(&cpu_bind->job_mutex); +} +#endif diff --git a/drivers/gpu/drm/xe/xe_cpu_bind.h b/drivers/gpu/drm/xe/xe_cpu_bind.h new file mode 100644 index 000000000000..69aad5edf204 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_cpu_bind.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_CPU_BIND_H_ +#define _XE_CPU_BIND_H_ + +#include <linux/types.h> + +struct dma_fence; +struct iosys_map; +struct xe_cpu_bind; +struct xe_cpu_bind_pt_update; +struct xe_device; +struct xe_tlb_inval_job; +struct xe_tile; +struct xe_vm; +struct xe_vm_pgtable_update; +struct xe_vm_pgtable_update_op; +struct xe_vma_ops; + +/** + * struct xe_cpu_bind_pt_update_ops - Callbacks for the + * xe_cpu_bind_update_pgtables() function. + */ +struct xe_cpu_bind_pt_update_ops { + /** + * @populate: Populate a page-table with ptes. + * @tile: The tile for the current operation. + * @map: struct iosys_map into the memory to be populated. + * @update: Information about the PTEs to be inserted. + * + * This interface is intended to be used as a callback into the + * page-table system to populate shared page-tables with PTEs. + */ + void (*populate)(struct xe_tile *tile, struct iosys_map *map, + const struct xe_vm_pgtable_update *update); + /** + * @clear: Clear a page-table's ptes. + * @vm: VM being updated + * @tile: The tile for the current operation. + * @map: struct iosys_map into the page-table to be cleared. + * @update: Information about the PTEs to be cleared. + * + * This interface is intended to be used as a callback into the + * page-table system to clear PTEs from shared page-tables. + */ + void (*clear)(struct xe_vm *vm, struct xe_tile *tile, + struct iosys_map *map, + const struct xe_vm_pgtable_update *update); + + /** + * @pre_commit: Callback to be called just before arming the + * sched_job. + * @pt_update: Pointer to embeddable callback argument. + * + * Return: 0 on success, negative error code on error. + */ + int (*pre_commit)(struct xe_cpu_bind_pt_update *pt_update); +}; + +/** + * struct xe_cpu_bind_pt_update - Argument to the struct + * xe_cpu_bind_pt_update_ops callbacks. + * + * Intended to be subclassed to support additional arguments if necessary. + */ +struct xe_cpu_bind_pt_update { + /** @ops: Pointer to the struct xe_cpu_bind_pt_update_ops callbacks */ + const struct xe_cpu_bind_pt_update_ops *ops; + /** @vops: VMA operations */ + struct xe_vma_ops *vops; + /** @job: The bind job, or NULL if the update is issued immediatel */ + struct xe_sched_job *job; + /** + * @ijobs: The TLB invalidation jobs, individual instances can be NULL + */ +#define XE_CPU_BIND_INVAL_JOB_COUNT 4 + struct xe_tlb_inval_job *ijobs[XE_CPU_BIND_INVAL_JOB_COUNT]; +}; + +int xe_cpu_bind_init(struct xe_device *xe); + +struct xe_exec_queue *xe_cpu_bind_queue(struct xe_cpu_bind *cpu_bind); + +void +xe_cpu_bind_update_pgtables_execute(struct xe_vm *vm, struct xe_tile *tile, + const struct xe_cpu_bind_pt_update_ops *ops, + struct xe_vm_pgtable_update_op *pt_op, + u32 num_ops, bool force_clear); + +struct dma_fence * +xe_cpu_bind_update_pgtables(struct xe_cpu_bind *cpu_bind, + struct xe_cpu_bind_pt_update *pt_update); + +void xe_cpu_bind_job_lock(struct xe_cpu_bind *cpu_bind, + struct xe_exec_queue *q); + +void xe_cpu_bind_job_unlock(struct xe_cpu_bind *cpu_bind, + struct xe_exec_queue *q); + +#if IS_ENABLED(CONFIG_PROVE_LOCKING) +void xe_cpu_bind_job_lock_assert(struct xe_exec_queue *q); +#else +static inline void xe_cpu_bind_job_lock_assert(struct xe_exec_queue *q) +{ +} +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_devcoredump.c b/drivers/gpu/drm/xe/xe_devcoredump.c index b918f046ae15..ab681b148869 100644 --- a/drivers/gpu/drm/xe/xe_devcoredump.c +++ b/drivers/gpu/drm/xe/xe_devcoredump.c @@ -74,11 +74,6 @@ static struct xe_device *coredump_to_xe(const struct xe_devcoredump *coredump) return container_of(coredump, struct xe_device, devcoredump); } -static struct xe_guc *exec_queue_to_guc(struct xe_exec_queue *q) -{ - return &q->gt->uc.guc; -} - static ssize_t __xe_devcoredump_read(char *buffer, ssize_t count, ssize_t start, struct xe_devcoredump *coredump) @@ -325,40 +320,44 @@ static void xe_devcoredump_deferred_snap_work(struct work_struct *work) } static void devcoredump_snapshot(struct xe_devcoredump *coredump, + struct xe_gt *gt, struct xe_exec_queue *q, struct xe_sched_job *job) { struct xe_devcoredump_snapshot *ss = &coredump->snapshot; - struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_guc *guc = >->uc.guc; const char *process_name = "no process"; bool cookie; ss->snapshot_time = ktime_get_real(); ss->boot_time = ktime_get_boottime(); - if (q->vm && q->vm->xef) { + if (q && q->vm && q->vm->xef) { process_name = q->vm->xef->process_name; ss->pid = q->vm->xef->pid; } strscpy(ss->process_name, process_name); - ss->gt = q->gt; + ss->gt = gt; INIT_WORK(&ss->work, xe_devcoredump_deferred_snap_work); /* keep going if fw fails as we still want to save the memory and SW data */ - CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); cookie = dma_fence_begin_signalling(); ss->guc.log = xe_guc_log_snapshot_capture(&guc->log, true); ss->guc.ct = xe_guc_ct_snapshot_capture(&guc->ct); - ss->ge = xe_guc_exec_queue_snapshot_capture(q); - if (job) - ss->job = xe_sched_job_snapshot_capture(job); - ss->vm = xe_vm_snapshot_capture(q->vm); - xe_engine_snapshot_capture_for_queue(q); + if (q) { + ss->ge = xe_guc_exec_queue_snapshot_capture(q); + if (job) + ss->job = xe_sched_job_snapshot_capture(job); + ss->vm = xe_vm_snapshot_capture(q->vm); + + xe_engine_snapshot_capture_for_queue(q); + } queue_work(system_dfl_wq, &ss->work); @@ -366,19 +365,24 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump, } /** - * xe_devcoredump - Take the required snapshots and initialize coredump device. - * @q: The faulty xe_exec_queue, where the issue was detected. - * @job: The faulty xe_sched_job, where the issue was detected. + * __xe_devcoredump - Take the required snapshots and initialize coredump device. + * @gt: The GT where the issue was detected. + * @q: The faulty xe_exec_queue, where the issue was detected, may be NULL for + * hangs that are not tied to an exec queue (e.g. TLB invalidation + * timeouts); in that case only the GT-level state (GuC log and CT state) + * is captured. + * @job: The faulty xe_sched_job, where the issue was detected, may be NULL. * @fmt: Printf format + args to describe the reason for the core dump * * This function should be called at the crash time within the serialized * gt_reset. It is skipped if we still have the core dump device available * with the information of the 'first' snapshot. */ -__printf(3, 4) -void xe_devcoredump(struct xe_exec_queue *q, struct xe_sched_job *job, const char *fmt, ...) +__printf(4, 5) +void __xe_devcoredump(struct xe_gt *gt, struct xe_exec_queue *q, + struct xe_sched_job *job, const char *fmt, ...) { - struct xe_device *xe = gt_to_xe(q->gt); + struct xe_device *xe = gt_to_xe(gt); struct xe_devcoredump *coredump = &xe->devcoredump; va_list varg; @@ -396,7 +400,7 @@ void xe_devcoredump(struct xe_exec_queue *q, struct xe_sched_job *job, const cha coredump->snapshot.reason = kvasprintf(GFP_ATOMIC, fmt, varg); va_end(varg); - devcoredump_snapshot(coredump, q, job); + devcoredump_snapshot(coredump, gt, q, job); drm_info(&xe->drm, "Xe device coredump has been created\n"); drm_info(&xe->drm, "Check your /sys/class/drm/card%d/device/devcoredump/data\n", diff --git a/drivers/gpu/drm/xe/xe_devcoredump.h b/drivers/gpu/drm/xe/xe_devcoredump.h index 5391a80a4d1b..bc4800b3a3f3 100644 --- a/drivers/gpu/drm/xe/xe_devcoredump.h +++ b/drivers/gpu/drm/xe/xe_devcoredump.h @@ -11,15 +11,17 @@ struct drm_printer; struct xe_device; struct xe_exec_queue; +struct xe_gt; struct xe_sched_job; #ifdef CONFIG_DEV_COREDUMP -void xe_devcoredump(struct xe_exec_queue *q, struct xe_sched_job *job, const char *fmt, ...); +void __xe_devcoredump(struct xe_gt *gt, struct xe_exec_queue *q, + struct xe_sched_job *job, const char *fmt, ...); int xe_devcoredump_init(struct xe_device *xe); #else -static inline void xe_devcoredump(struct xe_exec_queue *q, - struct xe_sched_job *job, - const char *fmt, ...) +static inline void __xe_devcoredump(struct xe_gt *gt, struct xe_exec_queue *q, + struct xe_sched_job *job, + const char *fmt, ...) { } @@ -29,6 +31,11 @@ static inline int xe_devcoredump_init(struct xe_device *xe) } #endif +#define xe_devcoredump(_q, _job, _fmt, ...) \ + __xe_devcoredump((_q)->gt, _q, _job, _fmt, ##__VA_ARGS__) +#define xe_devcoredump_gt(_gt, _fmt, ...) \ + __xe_devcoredump(_gt, NULL, NULL, _fmt, ##__VA_ARGS__) + void xe_print_blob_ascii85(struct drm_printer *p, const char *prefix, char suffix, const void *blob, size_t offset, size_t size); diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 205cb4e7f9e8..e1979bbcf7bc 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -27,6 +27,7 @@ #include "xe_bo_evict.h" #include "xe_configfs.h" #include "xe_debugfs.h" +#include "xe_cpu_bind.h" #include "xe_defaults.h" #include "xe_devcoredump.h" #include "xe_device_sysfs.h" @@ -671,6 +672,7 @@ static void vf_update_device_info(struct xe_device *xe) xe->info.skip_guc_pc = 1; xe->info.skip_pcode = 1; xe->info.has_drm_ras = false; + xe->info.has_device_uid = false; } static int xe_device_vram_alloc(struct xe_device *xe) @@ -871,6 +873,12 @@ static int xe_debug_page_size_alloc_ctrl_init(struct xe_device *xe) } #endif +static void xe_uid_probe(struct xe_device *xe) +{ + if (xe->info.has_device_uid) + xe->device_uid = xe_mmio_read64_2x32(xe_root_tile_mmio(xe), CRI_DEVICE_UID); +} + int xe_device_probe(struct xe_device *xe) { struct xe_tile *tile; @@ -878,6 +886,8 @@ int xe_device_probe(struct xe_device *xe) int err; u8 id; + xe_uid_probe(xe); + xe_pat_init_early(xe); err = xe_sriov_init(xe); @@ -993,6 +1003,10 @@ int xe_device_probe(struct xe_device *xe) if (err) return err; + err = xe_cpu_bind_init(xe); + if (err) + return err; + err = xe_pagefault_init(xe); if (err) return err; diff --git a/drivers/gpu/drm/xe/xe_device_sysfs.c b/drivers/gpu/drm/xe/xe_device_sysfs.c index a73e0e957cb0..18783a286a83 100644 --- a/drivers/gpu/drm/xe/xe_device_sysfs.c +++ b/drivers/gpu/drm/xe/xe_device_sysfs.c @@ -8,6 +8,7 @@ #include <linux/pci.h> #include <linux/sysfs.h> +#include "regs/xe_regs.h" #include "xe_device.h" #include "xe_device_sysfs.h" #include "xe_mmio.h" @@ -264,6 +265,35 @@ static const struct attribute_group auto_link_downgrade_attr_group = { .attrs = auto_link_downgrade_attrs, }; +/** + * DOC: Device Unique ID + * + * On supported platforms, Xe driver exposes a unique 64-bit GPU SOC + * device identifier through the 'device_uid' sysfs entry. + * + * See Documentation/ABI/testing/sysfs-driver-intel-xe-gpu for the ABI + * specification. + */ + +static ssize_t +device_uid_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct xe_device *xe = pdev_to_xe_device(pdev); + + return sysfs_emit(buf, "0x%016llx\n", xe->device_uid); +} +static DEVICE_ATTR_ADMIN_RO(device_uid); + +static struct attribute *device_uid_attrs[] = { + &dev_attr_device_uid.attr, + NULL +}; + +static const struct attribute_group device_uid_attr_group = { + .attrs = device_uid_attrs, +}; + int xe_device_sysfs_init(struct xe_device *xe) { struct device *dev = xe->drm.dev; @@ -285,5 +315,11 @@ int xe_device_sysfs_init(struct xe_device *xe) return ret; } + if (xe->info.has_device_uid) { + ret = devm_device_add_group(dev, &device_uid_attr_group); + if (ret) + return ret; + } + return 0; } diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 4661bfce2f4e..4e0e1bc040c1 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -38,6 +38,7 @@ struct drm_pagemap_shrinker; struct intel_display; struct intel_dg_nvm_dev; +struct xe_cpu_bind; struct xe_ggtt; struct xe_i2c; struct xe_mmio_gem; @@ -89,9 +90,7 @@ enum xe_page_size_alloc_ctrl_mode { #define XE_VRAM_FLAGS_NEED64K BIT(0) -#define XE_GT0 0 -#define XE_GT1 1 -#define XE_MAX_TILES_PER_DEVICE (XE_GT1 + 1) +#define XE_MAX_TILES_PER_DEVICE 2 /* * Highest GT/tile count for any platform. Used only for memory allocation @@ -180,6 +179,8 @@ struct xe_device { u8 has_cached_pt:1; /** @info.has_device_atomics_on_smem: Supports device atomics on SMEM */ u8 has_device_atomics_on_smem:1; + /** @info.has_device_uid: Device supports unique 64-bit GPU SOC ID */ + u8 has_device_uid:1; /** @info.has_drm_ras: Device supports drm_ras (Reliability, Availability, Serviceability) */ u8 has_drm_ras:1; /** @info.has_fan_control: Device supports fan control */ @@ -230,6 +231,8 @@ struct xe_device { u8 has_usm:1; /** @info.has_64bit_timestamp: Device supports 64-bit timestamps */ u8 has_64bit_timestamp:1; + /** @info.has_pt_mirror: Device has PT mirroring across tiles */ + u8 has_pt_mirror:1; /** @info.is_dgfx: is discrete device */ u8 is_dgfx:1; /** @info.needs_scratch: needs scratch page for oob prefetch to work */ @@ -264,6 +267,9 @@ struct xe_device { bool oob_initialized; } wa_active; + /** @device_uid: unique 64-bit GPU SOC identifier */ + u64 device_uid; + /** @survivability: survivability information for device */ struct xe_survivability survivability; @@ -566,6 +572,9 @@ struct xe_device { /** @sc: System Controller */ struct xe_sysctrl sc; + /** @cpu_bind: CPU bind object */ + struct xe_cpu_bind *cpu_bind; + /** @atomic_svm_timeslice_ms: Atomic SVM fault timeslice MS */ u32 atomic_svm_timeslice_ms; diff --git a/drivers/gpu/drm/xe/xe_drm_client.c b/drivers/gpu/drm/xe/xe_drm_client.c index e116fb562c4c..4c424d1c6721 100644 --- a/drivers/gpu/drm/xe/xe_drm_client.c +++ b/drivers/gpu/drm/xe/xe_drm_client.c @@ -256,7 +256,7 @@ static void show_meminfo(struct drm_printer *p, struct drm_file *file) xe_assert(xef->xe, !list_empty(&bo->client_link)); } - xe_bo_put_deferred(bo, &deferred); + xe_bo_put_deferred(bo, &deferred, NULL); } spin_unlock(&client->bos_lock); diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index e63559a2f582..9a8efa2bf2bf 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -14,6 +14,7 @@ #include <uapi/drm/xe_drm.h> #include "xe_bo.h" +#include "xe_cpu_bind.h" #include "xe_dep_scheduler.h" #include "xe_device.h" #include "xe_gt.h" @@ -142,9 +143,8 @@ static void __xe_exec_queue_free(struct xe_exec_queue *q) { int i; - for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) - if (q->tlb_inval[i].dep_scheduler) - xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler); + for_each_tlb_inval(q, i) + xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler); if (xe_exec_queue_uses_pxp(q)) xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); @@ -166,31 +166,34 @@ static void __xe_exec_queue_free(struct xe_exec_queue *q) static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q) { - struct xe_tile *tile = gt_to_tile(q->gt); - int i; + struct xe_tile *tile; + int i = 0, j; + u8 id; - for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) { - struct xe_dep_scheduler *dep_scheduler; - struct xe_gt *gt; - struct workqueue_struct *wq; + for_each_tile(tile, xe, id) { + for (j = 0; j < (XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT + 1); ++j, ++i) { + struct xe_dep_scheduler *dep_scheduler; + struct xe_gt *gt; + struct workqueue_struct *wq; - if (i == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT) - gt = tile->primary_gt; - else - gt = tile->media_gt; + if (j == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT) + gt = tile->primary_gt; + else + gt = tile->media_gt; - if (!gt) - continue; + if (!gt) + continue; - wq = gt->tlb_inval.job_wq; + wq = gt->tlb_inval.job_wq; #define MAX_TLB_INVAL_JOBS 16 /* Picking a reasonable value */ - dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name, - MAX_TLB_INVAL_JOBS); - if (IS_ERR(dep_scheduler)) - return PTR_ERR(dep_scheduler); + dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name, + MAX_TLB_INVAL_JOBS); + if (IS_ERR(dep_scheduler)) + return PTR_ERR(dep_scheduler); - q->tlb_inval[i].dep_scheduler = dep_scheduler; + q->tlb_inval[i].dep_scheduler = dep_scheduler; + } } #undef MAX_TLB_INVAL_JOBS @@ -224,7 +227,6 @@ static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, q->ops = gt->exec_queue_ops; INIT_LIST_HEAD(&q->lr.link); INIT_LIST_HEAD(&q->vm_exec_queue_link); - INIT_LIST_HEAD(&q->multi_gt_link); INIT_LIST_HEAD(&q->hw_engine_group_link); INIT_LIST_HEAD(&q->pxp.link); spin_lock_init(&q->multi_queue.lock); @@ -636,7 +638,6 @@ ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO); void xe_exec_queue_destroy(struct kref *ref) { struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); - struct xe_exec_queue *eq, *next; int i; xe_assert(gt_to_xe(q->gt), atomic_read(&q->job_cnt) == 0); @@ -648,15 +649,9 @@ void xe_exec_queue_destroy(struct kref *ref) xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); xe_exec_queue_last_fence_put_unlocked(q); - for_each_tlb_inval(i) + for_each_tlb_inval(q, i) xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, i); - if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { - list_for_each_entry_safe(eq, next, &q->multi_gt_list, - multi_gt_link) - xe_exec_queue_put(eq); - } - if (q->user_vm) { xe_vm_put(q->user_vm); q->user_vm = NULL; @@ -1343,7 +1338,6 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, u64_to_user_ptr(args->instances); struct xe_hw_engine *hwe; struct xe_vm *vm; - struct xe_tile *tile; struct xe_exec_queue *q = NULL; u32 logical_mask; u32 flags = 0; @@ -1392,31 +1386,16 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, return -ENOENT; } - for_each_tile(tile, xe, id) { - struct xe_exec_queue *new; - - flags |= EXEC_QUEUE_FLAG_VM; - if (id) - flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD; - - new = xe_exec_queue_create_bind(xe, tile, vm, flags, - args->extensions); - if (IS_ERR(new)) { - up_read(&vm->lock); - xe_vm_put(vm); - err = PTR_ERR(new); - if (q) - goto put_exec_queue; - return err; - } - if (id == 0) - q = new; - else - list_add_tail(&new->multi_gt_list, - &q->multi_gt_link); - } + flags |= EXEC_QUEUE_FLAG_VM; + + q = xe_exec_queue_create_bind(xe, xe_device_get_root_tile(xe), + vm, flags, args->extensions); up_read(&vm->lock); xe_vm_put(vm); + if (IS_ERR(q)) { + err = PTR_ERR(q); + return err; + } } else { logical_mask = calc_validate_logical_mask(xe, eci, args->width, @@ -1567,6 +1546,7 @@ bool xe_exec_queue_is_lr(struct xe_exec_queue *q) /** * xe_exec_queue_is_idle() - Whether an exec_queue is idle. * @q: The exec_queue + * @extra_jobs: Extra jobs on the queue * * FIXME: Need to determine what to use as the short-lived * timeline lock for the exec_queues, so that the return value @@ -1578,22 +1558,9 @@ bool xe_exec_queue_is_lr(struct xe_exec_queue *q) * * Return: True if the exec_queue is idle, false otherwise. */ -bool xe_exec_queue_is_idle(struct xe_exec_queue *q) +bool xe_exec_queue_is_idle(struct xe_exec_queue *q, int extra_jobs) { - if (xe_exec_queue_is_parallel(q)) { - int i; - - for (i = 0; i < q->width; ++i) { - if (xe_lrc_seqno(q->lrc[i]) != - q->lrc[i]->fence_ctx.next_seqno - 1) - return false; - } - - return true; - } - - return xe_lrc_seqno(q->lrc[0]) == - q->lrc[0]->fence_ctx.next_seqno - 1; + return !(atomic_read(&q->job_cnt) - extra_jobs); } /** @@ -1651,14 +1618,6 @@ void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q) */ void xe_exec_queue_kill(struct xe_exec_queue *q) { - struct xe_exec_queue *eq = q, *next; - - list_for_each_entry_safe(eq, next, &eq->multi_gt_list, - multi_gt_link) { - q->ops->kill(eq); - xe_vm_remove_compute_exec_queue(q->vm, eq); - } - q->ops->kill(q); xe_vm_remove_compute_exec_queue(q->vm, q); } @@ -1709,7 +1668,7 @@ static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, struct xe_vm *vm) { if (q->flags & EXEC_QUEUE_FLAG_MIGRATE) { - xe_migrate_job_lock_assert(q); + xe_cpu_bind_job_lock_assert(q); } else if (q->flags & EXEC_QUEUE_FLAG_VM) { lockdep_assert_held(&vm->lock); } else { @@ -1819,42 +1778,40 @@ void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, * xe_exec_queue_tlb_inval_last_fence_put() - Drop ref to last TLB invalidation fence * @q: The exec queue * @vm: The VM the engine does a bind for - * @type: Either primary or media GT + * @idx: Index of tlb invalidation */ void xe_exec_queue_tlb_inval_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm, - unsigned int type) + unsigned int idx) { xe_exec_queue_last_fence_lockdep_assert(q, vm); - xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || - type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT); + xe_assert(vm->xe, idx < XE_EXEC_QUEUE_TLB_INVAL_COUNT); - xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, type); + xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, idx); } /** * xe_exec_queue_tlb_inval_last_fence_put_unlocked() - Drop ref to last TLB * invalidation fence unlocked * @q: The exec queue - * @type: Either primary or media GT + * @idx: Index of tlb invalidation * * Only safe to be called from xe_exec_queue_destroy(). */ void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q, - unsigned int type) + unsigned int idx) { - xe_assert(gt_to_xe(q->gt), type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || - type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT); + xe_assert(gt_to_xe(q->gt), idx < XE_EXEC_QUEUE_TLB_INVAL_COUNT); - dma_fence_put(q->tlb_inval[type].last_fence); - q->tlb_inval[type].last_fence = NULL; + dma_fence_put(q->tlb_inval[idx].last_fence); + q->tlb_inval[idx].last_fence = NULL; } /** * xe_exec_queue_tlb_inval_last_fence_get() - Get last fence for TLB invalidation * @q: The exec queue * @vm: The VM the engine does a bind for - * @type: Either primary or media GT + * @idx: Index of tlb invalidation * * Get last fence, takes a ref * @@ -1862,22 +1819,21 @@ void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q, */ struct dma_fence *xe_exec_queue_tlb_inval_last_fence_get(struct xe_exec_queue *q, struct xe_vm *vm, - unsigned int type) + unsigned int idx) { struct dma_fence *fence; xe_exec_queue_last_fence_lockdep_assert(q, vm); - xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || - type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT); + xe_assert(vm->xe, idx < XE_EXEC_QUEUE_TLB_INVAL_COUNT); xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM | EXEC_QUEUE_FLAG_MIGRATE)); - if (q->tlb_inval[type].last_fence && + if (q->tlb_inval[idx].last_fence && test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, - &q->tlb_inval[type].last_fence->flags)) - xe_exec_queue_tlb_inval_last_fence_put(q, vm, type); + &q->tlb_inval[idx].last_fence->flags)) + xe_exec_queue_tlb_inval_last_fence_put(q, vm, idx); - fence = q->tlb_inval[type].last_fence ?: dma_fence_get_stub(); + fence = q->tlb_inval[idx].last_fence ?: dma_fence_get_stub(); dma_fence_get(fence); return fence; } @@ -1887,26 +1843,25 @@ struct dma_fence *xe_exec_queue_tlb_inval_last_fence_get(struct xe_exec_queue *q * @q: The exec queue * @vm: The VM the engine does a bind for * @fence: The fence - * @type: Either primary or media GT + * @idx: Index of tlb invalidation * - * Set the last fence for the tlb invalidation type on the queue. Increases + * Set the last fence for the tlb invalidation client on the queue. Increases * reference count for fence, when closing queue * xe_exec_queue_tlb_inval_last_fence_put should be called. */ void xe_exec_queue_tlb_inval_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, struct dma_fence *fence, - unsigned int type) + unsigned int idx) { xe_exec_queue_last_fence_lockdep_assert(q, vm); - xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || - type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT); + xe_assert(vm->xe, idx < XE_EXEC_QUEUE_TLB_INVAL_COUNT); xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM | EXEC_QUEUE_FLAG_MIGRATE)); xe_assert(vm->xe, !dma_fence_is_container(fence)); - xe_exec_queue_tlb_inval_last_fence_put(q, vm, type); - q->tlb_inval[type].last_fence = dma_fence_get(fence); + xe_exec_queue_tlb_inval_last_fence_put(q, vm, idx); + q->tlb_inval[idx].last_fence = dma_fence_get(fence); } /** diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h index 0225426c57b0..e8963f85cabd 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.h +++ b/drivers/gpu/drm/xe/xe_exec_queue.h @@ -14,9 +14,9 @@ struct drm_file; struct xe_device; struct xe_file; -#define for_each_tlb_inval(__i) \ - for (__i = XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT; \ - __i <= XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT; ++__i) +#define for_each_tlb_inval(__q, __i) \ + for (__i = 0; __i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++__i) \ + for_each_if((__q)->tlb_inval[__i].dep_scheduler) struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, u32 logical_mask, u16 width, @@ -116,7 +116,7 @@ static inline struct xe_exec_queue *xe_exec_queue_multi_queue_primary(struct xe_ bool xe_exec_queue_is_lr(struct xe_exec_queue *q); -bool xe_exec_queue_is_idle(struct xe_exec_queue *q); +bool xe_exec_queue_is_idle(struct xe_exec_queue *q, int extra_jobs); void xe_exec_queue_kill(struct xe_exec_queue *q); @@ -141,19 +141,19 @@ void xe_exec_queue_last_fence_set(struct xe_exec_queue *e, struct xe_vm *vm, void xe_exec_queue_tlb_inval_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm, - unsigned int type); + unsigned int idx); void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q, - unsigned int type); + unsigned int idx); struct dma_fence *xe_exec_queue_tlb_inval_last_fence_get(struct xe_exec_queue *q, struct xe_vm *vm, - unsigned int type); + unsigned int idx); void xe_exec_queue_tlb_inval_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, struct dma_fence *fence, - unsigned int type); + unsigned int idx); void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q); diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index 836f88fc0faa..48c05398b016 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -137,16 +137,14 @@ struct xe_exec_queue { #define EXEC_QUEUE_FLAG_KERNEL BIT(0) /* for VM jobs. Caller needs to hold rpm ref when creating queue with this flag */ #define EXEC_QUEUE_FLAG_VM BIT(1) -/* child of VM queue for multi-tile VM jobs */ -#define EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD BIT(2) /* kernel exec_queue only, set priority to highest level */ -#define EXEC_QUEUE_FLAG_HIGH_PRIORITY BIT(3) +#define EXEC_QUEUE_FLAG_HIGH_PRIORITY BIT(2) /* flag to indicate low latency hint to guc */ -#define EXEC_QUEUE_FLAG_LOW_LATENCY BIT(4) +#define EXEC_QUEUE_FLAG_LOW_LATENCY BIT(3) /* for migration (kernel copy, clear, bind) jobs */ -#define EXEC_QUEUE_FLAG_MIGRATE BIT(5) +#define EXEC_QUEUE_FLAG_MIGRATE BIT(4) /* for programming COMMON_SLICE_CHICKEN3 on first submission */ -#define EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX BIT(6) +#define EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX BIT(5) /** * @flags: flags for this exec queue, should statically setup aside from ban @@ -158,13 +156,6 @@ struct xe_exec_queue { atomic_t ban_reason; union { - /** @multi_gt_list: list head for VM bind engines if multi-GT */ - struct list_head multi_gt_list; - /** @multi_gt_link: link for VM bind engines if multi-GT */ - struct list_head multi_gt_link; - }; - - union { /** @execlist: execlist backend specific state for exec queue */ struct xe_execlist_exec_queue *execlist; /** @guc: GuC backend specific state for exec queue */ @@ -230,7 +221,8 @@ struct xe_exec_queue { #define XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT 0 #define XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT 1 -#define XE_EXEC_QUEUE_TLB_INVAL_COUNT (XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT + 1) +#define XE_EXEC_QUEUE_TLB_INVAL_COUNT \ + ((XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT + 1) * 2) /** @tlb_inval: TLB invalidations exec queue state */ struct { diff --git a/drivers/gpu/drm/xe/xe_gt_ccs_mode.c b/drivers/gpu/drm/xe/xe_gt_ccs_mode.c index baee1f4a6b01..224a15675203 100644 --- a/drivers/gpu/drm/xe/xe_gt_ccs_mode.c +++ b/drivers/gpu/drm/xe/xe_gt_ccs_mode.c @@ -70,12 +70,6 @@ static void __xe_gt_apply_ccs_mode(struct xe_gt *gt, u32 num_engines) } } - /* - * Mask bits need to be set for the register. Though only Xe2+ - * platforms require setting of mask bits, it won't harm for older - * platforms as these bits are unused there. - */ - mode |= CCS_MODE_CSLICE_0_3_MASK << 16; xe_mmio_write32(>->mmio, CCS_MODE, mode); xe_gt_dbg(gt, "CCS_MODE=%x config:%08x, num_engines:%d, num_slices:%d\n", diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c index c7f8bbd4cb92..3ab4cb9e4964 100644 --- a/drivers/gpu/drm/xe/xe_guc.c +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -1469,6 +1469,22 @@ int xe_guc_suspend(struct xe_guc *guc) return 0; } +/** + * xe_guc_bo_wa_flags - Extra BO flags for memory shared with the GuC + * @gt: the &xe_gt whose GuC the buffer will be shared with + * + * Wa_22016122933: on the standalone media GT, memory shared between the + * CPU and the GuC must not be mapped cached on the CPU side, otherwise + * the CPU can read stale data written by the GuC (e.g. G2H CTB writes) + * for multiple seconds. + * + * Return: additional XE_BO_FLAG_* to use when allocating GuC-shared memory + */ +u32 xe_guc_bo_wa_flags(struct xe_gt *gt) +{ + return XE_GT_WA(gt, 22016122933) ? XE_BO_FLAG_NEEDS_UC : 0; +} + void xe_guc_notify(struct xe_guc *guc) { struct xe_gt *gt = guc_to_gt(guc); diff --git a/drivers/gpu/drm/xe/xe_guc.h b/drivers/gpu/drm/xe/xe_guc.h index 61e3ee19a59b..c4eca40d69cf 100644 --- a/drivers/gpu/drm/xe/xe_guc.h +++ b/drivers/gpu/drm/xe/xe_guc.h @@ -30,6 +30,7 @@ xe_guc_fw_version_at_least((guc), MAKE_GUC_VER_ARGS(ver)) struct drm_printer; +struct xe_gt; void xe_guc_comm_init_early(struct xe_guc *guc); int xe_guc_init_noalloc(struct xe_guc *guc); @@ -45,6 +46,7 @@ void xe_guc_runtime_suspend(struct xe_guc *guc); void xe_guc_runtime_resume(struct xe_guc *guc); int xe_guc_suspend(struct xe_guc *guc); int xe_guc_softreset(struct xe_guc *guc); +u32 xe_guc_bo_wa_flags(struct xe_gt *gt); void xe_guc_notify(struct xe_guc *guc); int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr); int xe_guc_mmio_send(struct xe_guc *guc, const u32 *request, u32 len); diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index ff8eee3831aa..43a2951a66d6 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -383,6 +383,12 @@ static void guc_waklv_init(struct xe_guc_ads *ads) guc_waklv_enable(ads, NULL, 0, &offset, &remain, GUC_WA_KLV_IGNORE_MMIO_READ_SEM_TOKEN_64); + /* GuC only applies this WA for scheduled MultiQ contexts; no KMD-side gating needed. */ + if (XE_GT_WA(gt, 16030224309) && + GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 73)) + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WA_KLV_IGNORE_MULTIQ_CTX_COMPLETE_WITH_PREEMPT_IN_CSB); + /* * On GuC firmware 70.66 and above, use the Feature KLV (shared with the * WA KLV buffer); older firmware uses GUC_CTL_DISABLE_MULTI_QUEUE in @@ -435,7 +441,8 @@ int xe_guc_ads_init(struct xe_guc_ads *ads) XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE | - XE_BO_FLAG_PINNED_NORESTORE); + XE_BO_FLAG_PINNED_NORESTORE | + xe_guc_bo_wa_flags(gt)); if (IS_ERR(bo)) return PTR_ERR(bo); @@ -864,7 +871,7 @@ static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads, } } - if (XE_GT_WA(hwe->gt, 16023105232)) + if (XE_GT_WA(hwe->gt, 16023105232) || XE_GT_WA(hwe->gt, 14025941587)) guc_mmio_regset_write_one(ads, regset_map, RING_IDLEDLY(hwe->mmio_base), count++); diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index 5c4733da385c..da5ce97484b8 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -376,7 +376,8 @@ int xe_guc_ct_init(struct xe_guc_ct *ct) XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE | - XE_BO_FLAG_PINNED_NORESTORE); + XE_BO_FLAG_PINNED_NORESTORE | + xe_guc_bo_wa_flags(gt)); if (IS_ERR(bo)) return PTR_ERR(bo); @@ -386,7 +387,8 @@ int xe_guc_ct_init(struct xe_guc_ct *ct) XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE | - XE_BO_FLAG_PINNED_NORESTORE); + XE_BO_FLAG_PINNED_NORESTORE | + xe_guc_bo_wa_flags(gt)); if (IS_ERR(bo)) return PTR_ERR(bo); @@ -1288,11 +1290,23 @@ int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, * @ct: GuC CT object * @action: payload dwords (HxG header dword is expected at @action[-1]) * @len: number of payload dwords in @action + * @g2h_len: G2H response space to reserve in dwords, or 0 + * @num_g2h: number of G2H messages expected, or 0 * @defer_flush: defer publishing/doorbell for batching * * Sends a single H2G message to the GuC CT buffer while the caller already * holds @ct->lock. * + * Callers that expect the GuC to reply with a G2H message must reserve the + * matching credit via @g2h_len / @num_g2h. The G2H receive path releases that + * credit unconditionally when the reply arrives, so failing to reserve it here + * corrupts the G2H accounting and kills the CT channel. + * + * Reserving G2H credit may wait for, and process, outstanding G2H messages, so + * this must not be called with a non-zero @g2h_len from a G2H handler. Handlers + * should use xe_guc_ct_send_g2h_handler(), with any reply credit reserved up + * front by the H2G that triggered the handler. + * * If @defer_flush is false, the function completes the submission immediately: * it makes the payload visible to the device, updates the H2G descriptor and * rings the GuC doorbell. @@ -1310,11 +1324,12 @@ int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, * Must be called with @ct->lock held. */ int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, - bool defer_flush) + u32 g2h_len, u32 num_g2h, bool defer_flush) { int ret; - ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL, defer_flush); + ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL, + defer_flush); if (ret == -EDEADLK) kick_reset(ct); diff --git a/drivers/gpu/drm/xe/xe_guc_ct.h b/drivers/gpu/drm/xe/xe_guc_ct.h index 3ddc665ab84a..e2a33249d65d 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.h +++ b/drivers/gpu/drm/xe/xe_guc_ct.h @@ -56,7 +56,7 @@ static inline void xe_guc_ct_irq_handler(struct xe_guc_ct *ct) int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, u32 g2h_len, u32 num_g2h); int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, - bool defer_flush); + u32 g2h_len, u32 num_g2h, bool defer_flush); int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, u32 *response_buffer); static inline int diff --git a/drivers/gpu/drm/xe/xe_guc_engine_activity.c b/drivers/gpu/drm/xe/xe_guc_engine_activity.c index a782be57caad..729ce8ac140b 100644 --- a/drivers/gpu/drm/xe/xe_guc_engine_activity.c +++ b/drivers/gpu/drm/xe/xe_guc_engine_activity.c @@ -97,7 +97,8 @@ static int allocate_engine_activity_buffers(struct xe_guc *guc, metadata_bo = xe_bo_create_pin_map_novm(gt_to_xe(gt), tile, PAGE_ALIGN(metadata_size), ttm_bo_type_kernel, XE_BO_FLAG_SYSTEM | - XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE, + XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE | + xe_guc_bo_wa_flags(gt), false); if (IS_ERR(metadata_bo)) @@ -105,7 +106,8 @@ static int allocate_engine_activity_buffers(struct xe_guc *guc, bo = xe_bo_create_pin_map_novm(gt_to_xe(gt), tile, PAGE_ALIGN(size), ttm_bo_type_kernel, XE_BO_FLAG_VRAM_IF_DGFX(tile) | - XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE, false); + XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE | + xe_guc_bo_wa_flags(gt), false); if (IS_ERR(bo)) { xe_bo_unpin_map_no_vm(metadata_bo); diff --git a/drivers/gpu/drm/xe/xe_guc_hwconfig.c b/drivers/gpu/drm/xe/xe_guc_hwconfig.c index b300901dbb8e..dc667a71b438 100644 --- a/drivers/gpu/drm/xe/xe_guc_hwconfig.c +++ b/drivers/gpu/drm/xe/xe_guc_hwconfig.c @@ -65,7 +65,7 @@ int xe_guc_hwconfig_init(struct xe_guc *guc) /* * All hwconfig the same across GTs so only GT0 needs to be configured */ - if (gt->info.id != XE_GT0) + if (gt != xe_root_mmio_gt(xe)) return 0; /* ADL_P, DG2+ supports hwconfig table */ diff --git a/drivers/gpu/drm/xe/xe_guc_log.c b/drivers/gpu/drm/xe/xe_guc_log.c index 538d4df0f7aa..7d006268ce91 100644 --- a/drivers/gpu/drm/xe/xe_guc_log.c +++ b/drivers/gpu/drm/xe/xe_guc_log.c @@ -17,6 +17,7 @@ #include "xe_force_wake.h" #include "xe_gt_printk.h" #include "xe_gt_types.h" +#include "xe_guc.h" #include "xe_map.h" #include "xe_mmio.h" #include "xe_module.h" @@ -624,14 +625,16 @@ void xe_guc_log_print_lfd(struct xe_guc_log *log, struct drm_printer *p) int xe_guc_log_init(struct xe_guc_log *log) { struct xe_device *xe = log_to_xe(log); - struct xe_tile *tile = gt_to_tile(log_to_gt(log)); + struct xe_gt *gt = log_to_gt(log); + struct xe_tile *tile = gt_to_tile(gt); struct xe_bo *bo; bo = xe_managed_bo_create_pin_map(xe, tile, GUC_LOG_SIZE, XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE | - XE_BO_FLAG_PINNED_NORESTORE); + XE_BO_FLAG_PINNED_NORESTORE | + xe_guc_bo_wa_flags(gt)); if (IS_ERR(bo)) return PTR_ERR(bo); diff --git a/drivers/gpu/drm/xe/xe_guc_pagefault.c b/drivers/gpu/drm/xe/xe_guc_pagefault.c index df237fd40551..8d665d1dac7f 100644 --- a/drivers/gpu/drm/xe/xe_guc_pagefault.c +++ b/drivers/gpu/drm/xe/xe_guc_pagefault.c @@ -4,9 +4,11 @@ */ #include "abi/guc_actions_abi.h" +#include "xe_gt_printk.h" #include "xe_guc.h" #include "xe_guc_ct.h" #include "xe_guc_pagefault.h" +#include "xe_hw_engine.h" #include "xe_pagefault.h" #include "xe_pagefault_types.h" @@ -55,7 +57,8 @@ static void guc_ack_fault(struct xe_pagefault *pf, int err) bool write_only = guc->pagefault_ack_counter++ & (XE_GUC_PAGEFAULT_FLUSH_PERIOD - 1); - xe_guc_ct_send_locked(&guc->ct, action, ARRAY_SIZE(action), + /* Pagefault acks are fire-and-forget, no G2H reply expected. */ + xe_guc_ct_send_locked(&guc->ct, action, ARRAY_SIZE(action), 0, 0, write_only); } @@ -68,10 +71,40 @@ static void guc_ack_fault_end(void *private) xe_guc_ct_unlock(&guc->ct); } +static void xe_guc_pagefault_print(struct xe_pagefault *pf, + const char *err_str) +{ + const u32 *msg = pf->producer.msg; + u32 engine_class = FIELD_GET(PFD_ENG_CLASS, msg[0]); + + xe_gt_info(pf->gt, "\n\tASID: %lu\n" + "\tFaulted Address: 0x%08lx%08lx\n" + "\tFaultType: %lu\n" + "\tAccessType: %lu\n" + "\tFaultLevel: %lu\n" + "\tEngineClass: %u %s\n" + "\tEngineInstance: %lu\n" + "\tSRCID: 0x%02lx\n" + "\tError: %s\n", + FIELD_GET(PFD_ASID, msg[1]), + FIELD_GET(PFD_VIRTUAL_ADDR_HI, msg[3]), + FIELD_GET(PFD_VIRTUAL_ADDR_LO, msg[2]) << + PFD_VIRTUAL_ADDR_LO_SHIFT, + FIELD_GET(PFD_FAULT_TYPE, msg[2]), + FIELD_GET(PFD_ACCESS_TYPE, msg[2]), + FIELD_GET(PFD_FAULT_LEVEL, msg[0]), + engine_class, + xe_hw_engine_class_to_str(engine_class), + FIELD_GET(PFD_ENG_INSTANCE, msg[0]), + FIELD_GET(PFD_SRC_ID, msg[0]), + err_str); +} + static const struct xe_pagefault_ops guc_pagefault_ops = { .ack_fault_begin = guc_ack_fault_begin, .ack_fault = guc_ack_fault, .ack_fault_end = guc_ack_fault_end, + .print = xe_guc_pagefault_print, }; /** diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c index 097b075bd89a..e0105222a2c7 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.c +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -1391,7 +1391,8 @@ int xe_guc_pc_init(struct xe_guc_pc *pc) XE_BO_FLAG_VRAM_IF_DGFX(tile) | XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE | - XE_BO_FLAG_PINNED_NORESTORE); + XE_BO_FLAG_PINNED_NORESTORE | + xe_guc_bo_wa_flags(gt)); if (IS_ERR(bo)) return PTR_ERR(bo); diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index f3ba8abfc228..0b231228bde8 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -19,6 +19,7 @@ #include "abi/guc_klvs_abi.h" #include "xe_assert.h" #include "xe_bo.h" +#include "xe_cpu_bind.h" #include "xe_devcoredump.h" #include "xe_device.h" #include "xe_exec_queue.h" @@ -42,6 +43,7 @@ #include "xe_mocs.h" #include "xe_module.h" #include "xe_pm.h" +#include "xe_pt.h" #include "xe_ring_ops_types.h" #include "xe_sched_job.h" #include "xe_sleep.h" @@ -1188,9 +1190,15 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job) xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); if (!job->restore_replay || job->last_replay) { + /* + * A chained ULLS job relies on its predecessor's postamble to + * publish its ring tail. After a reset the context comes back + * disabled with that postamble gone, so publish it here. + */ if (xe_exec_queue_is_parallel(q)) wq_item_append(q); - else + else if (!xe_sched_job_ulls_is_chained(job) || + !exec_queue_enabled(q)) xe_lrc_set_ring_tail(lrc, lrc->ring.tail); job->last_replay = false; } @@ -1207,6 +1215,9 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job) if (exec_queue_suspended(q)) return; + if (xe_sched_job_ulls_is_chained(job)) + xe_lrc_set_ulls_semaphore(lrc, xe_sched_job_lrc_seqno(job)); + if (!exec_queue_enabled(q)) { action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET; action[len++] = q->guc->id; @@ -1220,13 +1231,14 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job) set_exec_queue_pending_enable(q); set_exec_queue_enabled(q); trace_xe_exec_queue_scheduling_enable(q); - } else { + } else if (!xe_sched_job_ulls_is_chained(job)) { action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; action[len++] = q->guc->id; trace_xe_exec_queue_submit(q); } - xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h); + if (!xe_sched_job_ulls_is_chained(job) || num_g2h) + xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h); if (extra_submit) { len = 0; @@ -1238,21 +1250,70 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job) } } +static bool is_pt_job(struct xe_sched_job *job) +{ + return job->is_pt_job; +} + +static void run_pt_job(struct xe_device *xe, struct xe_sched_job *job, + bool force_clear) +{ + struct xe_tile *tile; + int id; + + for_each_tile(tile, xe, id) { + struct xe_pt_job_ops *pt_job_ops = + job->pt_update[0].pt_job_ops[id]; + + if (!pt_job_ops || !pt_job_ops->current_op) + continue; + + xe_cpu_bind_update_pgtables_execute(job->pt_update[0].vm, tile, + job->pt_update[0].ops, + pt_job_ops->ops, + pt_job_ops->current_op, + force_clear); + } +} + +static void put_pt_job(struct xe_device *xe, struct xe_sched_job *job) +{ + struct xe_tile *tile; + int id; + + for_each_tile(tile, xe, id) { + struct xe_pt_job_ops *pt_job_ops = + job->pt_update[0].pt_job_ops[id]; + + xe_pt_job_ops_put(pt_job_ops); + } +} + static struct dma_fence * guc_exec_queue_run_job(struct drm_sched_job *drm_job) { struct xe_sched_job *job = to_xe_sched_job(drm_job); struct xe_exec_queue *q = job->q; struct xe_guc *guc = exec_queue_to_guc(q); - bool killed_or_banned_or_wedged = - exec_queue_killed_or_banned_or_wedged(q); + bool killed_or_banned_or_wedged_or_error = + exec_queue_killed_or_banned_or_wedged(q) || + xe_sched_job_is_error(job); xe_gt_assert(guc_to_gt(guc), !(exec_queue_destroyed(q) || exec_queue_pending_disable(q)) || exec_queue_banned(q) || exec_queue_suspended(q)); trace_xe_sched_job_run(job); - if (!killed_or_banned_or_wedged && !xe_sched_job_is_error(job)) { + if (is_pt_job(job)) { + xe_gt_assert(guc_to_gt(guc), !exec_queue_registered(q)); + + run_pt_job(guc_to_xe(guc), job, + killed_or_banned_or_wedged_or_error); + put_pt_job(guc_to_xe(guc), job); + dma_fence_put(job->fence); /* Drop ref from xe_sched_job_arm */ + + return NULL; + } else if (!killed_or_banned_or_wedged_or_error) { if (xe_exec_queue_is_multi_queue_secondary(q)) { struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); @@ -2049,6 +2110,7 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) struct xe_guc_exec_queue *ge; long timeout; int err, i; + int max_jobs = (xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES); xe_gt_assert(guc_to_gt(guc), xe_device_uc_enabled(guc_to_xe(guc))); @@ -2088,8 +2150,11 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) submit_wq = primary->guc->sched.base.submit_wq; } + if (q->vm && q->vm->flags & XE_VM_FLAG_MIGRATION) + max_jobs = min(max_jobs, LRC_MIGRATION_ULLS_SEMAPHORE_COUNT - 1); + err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops, - submit_wq, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64, + submit_wq, max_jobs, 64, timeout, guc_to_gt(guc)->ordered_wq, NULL, ge->name, gt_to_xe(q->gt)->drm.dev); if (err) @@ -2993,6 +3058,15 @@ static void guc_exec_queue_start(struct xe_exec_queue *q) * increasing as jobs are written out. */ q->lrc[i]->ring.tail = job->ptrs[i].head; + /* + * A ULLS job's slot can have been consumed by + * the CS without the job running, leaving the + * saved head past it. Rewind so the re-emitted + * job is ahead of the head, not behind it. + */ + if (xe_sched_job_is_ulls(job)) + xe_lrc_set_ring_head(q->lrc[i], + job->ptrs[i].head); xe_lrc_set_ring_tail(q->lrc[i], xe_lrc_ring_head(q->lrc[i])); } @@ -3686,6 +3760,7 @@ xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q) snapshot->logical_mask = q->logical_mask; snapshot->width = q->width; snapshot->refcount = kref_read(&q->refcount); + snapshot->jobcount = atomic_read(&q->job_cnt); snapshot->sched_timeout = sched->base.timeout; snapshot->sched_props.timeslice_us = q->sched_props.timeslice_us; snapshot->sched_props.preempt_timeout_us = @@ -3758,6 +3833,7 @@ xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snaps drm_printf(p, "\tLogical mask: 0x%x\n", snapshot->logical_mask); drm_printf(p, "\tWidth: %d\n", snapshot->width); drm_printf(p, "\tRef: %d\n", snapshot->refcount); + drm_printf(p, "\tJob count: %u\n", snapshot->jobcount); drm_printf(p, "\tTimeout: %ld (ms)\n", snapshot->sched_timeout); drm_printf(p, "\tTimeslice: %u (us)\n", snapshot->sched_props.timeslice_us); diff --git a/drivers/gpu/drm/xe/xe_guc_submit_types.h b/drivers/gpu/drm/xe/xe_guc_submit_types.h index 7824f61b1290..8271702e692e 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit_types.h +++ b/drivers/gpu/drm/xe/xe_guc_submit_types.h @@ -77,6 +77,8 @@ struct xe_guc_submit_exec_queue_snapshot { u16 width; /** @refcount: ref count of this exec queue */ u32 refcount; + /** @jobcount: job count of this exec queue */ + u32 jobcount; /** * @sched_timeout: the time after which a job is removed from the * scheduler. diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c index 010499766fce..9680d2a5adec 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine.c +++ b/drivers/gpu/drm/xe/xe_hw_engine.c @@ -585,28 +585,102 @@ static void hw_engine_init_early(struct xe_gt *gt, struct xe_hw_engine *hwe, xe_reg_whitelist_process_engine(hwe); } +static u32 idledly_floor_ticks(u32 idledly_ns, u32 idledly_units_ps) +{ + return DIV_ROUND_DOWN_ULL((u64)idledly_ns * 1000, idledly_units_ps); +} + static void adjust_idledly(struct xe_hw_engine *hwe) { struct xe_gt *gt = hwe->gt; - u32 idledly, maxcnt; + u32 idledly, idledly_hw, idledly_reg_val, maxcnt; u32 idledly_units_ps = 8 * gt->info.timestamp_base; u32 maxcnt_units_ns = 640; - bool inhibit_switch = 0; + bool inhibit_switch = false; + bool wa_applied = false; + bool clamped_below_maxcnt = false; + + if ((!IS_SRIOV_VF(gt_to_xe(gt)) && XE_GT_WA(gt, 16023105232)) || + XE_GT_WA(gt, 14025941587)) { + u32 mincnt_idledly_ns = 5000; + + /* xe_gt_clock_init() warns and zeroes timestamp_base on unknown crystal clock. */ + if (!idledly_units_ps) + return; - if (!IS_SRIOV_VF(gt_to_xe(hwe->gt)) && XE_GT_WA(gt, 16023105232)) { - idledly = xe_mmio_read32(>->mmio, RING_IDLEDLY(hwe->mmio_base)); + idledly_reg_val = xe_mmio_read32(>->mmio, RING_IDLEDLY(hwe->mmio_base)); maxcnt = xe_mmio_read32(>->mmio, RING_PWRCTX_MAXCNT(hwe->mmio_base)); - inhibit_switch = idledly & INHIBIT_SWITCH_UNTIL_PREEMPTED; - idledly = REG_FIELD_GET(IDLE_DELAY, idledly); - idledly = DIV_ROUND_CLOSEST(idledly * idledly_units_ps, 1000); + inhibit_switch = idledly_reg_val & INHIBIT_SWITCH_UNTIL_PREEMPTED; + idledly = REG_FIELD_GET(IDLE_DELAY, idledly_reg_val); + idledly = DIV_ROUND_CLOSEST_ULL((u64)idledly * idledly_units_ps, 1000); + idledly_hw = idledly; maxcnt = REG_FIELD_GET(IDLE_WAIT_TIME, maxcnt); maxcnt *= maxcnt_units_ns; - if (xe_gt_WARN_ON(gt, idledly >= maxcnt || inhibit_switch)) { - idledly = DIV_ROUND_CLOSEST(((maxcnt - 1) * 1000), - idledly_units_ps); - xe_mmio_write32(>->mmio, RING_IDLEDLY(hwe->mmio_base), idledly); + /* + * Wa_14025941587 is applied before Wa_16023105232, which takes + * priority if the two ever conflict (not expected in practice). + */ + if (XE_GT_WA(gt, 14025941587) && + idledly < mincnt_idledly_ns) { + idledly = mincnt_idledly_ns; + wa_applied = true; + } + + if (XE_GT_WA(gt, 16023105232)) { + /* Clear the inhibit switch without disturbing a valid delay. */ + if (inhibit_switch) { + idledly_reg_val &= ~INHIBIT_SWITCH_UNTIL_PREEMPTED; + wa_applied = true; + } + + /* Warn only on the value read from hardware. */ + xe_gt_WARN_ON(gt, idledly_hw >= maxcnt); + + if (idledly >= maxcnt) { + /* maxcnt may be 0 if IDLE_WAIT_TIME is unprogrammed. */ + idledly = maxcnt ? maxcnt - 1 : 0; + clamped_below_maxcnt = true; + wa_applied = true; + } + } + + if (wa_applied) { + u32 idledly_ticks; + + /* + * Wa_16023105232 requires idledly < maxcnt, so floor + * that clamp; otherwise round up to guarantee the + * Wa_14025941587 minimum survives tick quantization. + */ + if (clamped_below_maxcnt) + idledly_ticks = idledly_floor_ticks(idledly, idledly_units_ps); + else + idledly_ticks = DIV_ROUND_UP_ULL((u64)idledly * 1000, + idledly_units_ps); + + /* + * Tick quantization can still push the rounded-up value + * to/above maxcnt; re-floor here so Wa_16023105232 keeps + * priority even in that case. + */ + if (!clamped_below_maxcnt && XE_GT_WA(gt, 16023105232) && + (u64)idledly_ticks * idledly_units_ps >= (u64)maxcnt * 1000) { + xe_gt_dbg(gt, "idledly %s: %u ticks would exceed maxcnt=%u, so flooring\n", + hwe->name, idledly_ticks, maxcnt); + idledly = maxcnt ? maxcnt - 1 : 0; + idledly_ticks = idledly_floor_ticks(idledly, idledly_units_ps); + } + + idledly_reg_val &= ~IDLE_DELAY; + idledly_reg_val |= REG_FIELD_PREP(IDLE_DELAY, idledly_ticks); + xe_gt_dbg(gt, "idledly %s: set %u max=%u inh=%u ts=%u\n", + hwe->name, idledly, maxcnt, + !!inhibit_switch, gt->info.timestamp_base); + xe_mmio_write32(>->mmio, + RING_IDLEDLY(hwe->mmio_base), + idledly_reg_val); } } } diff --git a/drivers/gpu/drm/xe/xe_i2c.c b/drivers/gpu/drm/xe/xe_i2c.c index 51ca9fb73b70..f4f381988289 100644 --- a/drivers/gpu/drm/xe/xe_i2c.c +++ b/drivers/gpu/drm/xe/xe_i2c.c @@ -111,14 +111,9 @@ static int xe_i2c_register_adapter(struct xe_i2c *i2c) { struct pci_dev *pci = to_pci_dev(i2c->drm_dev); struct platform_device *pdev; - struct fwnode_handle *fwnode; int ret; u32 id; - fwnode = fwnode_create_software_node(xe_i2c_adapter_properties, NULL); - if (IS_ERR(fwnode)) - return PTR_ERR(fwnode); - id = (pci_domain_nr(pci->bus) << 16) | pci_dev_id(pci); /* @@ -128,14 +123,16 @@ static int xe_i2c_register_adapter(struct xe_i2c *i2c) * platform_device_register_full() is done. */ pdev = platform_device_alloc(adapter_name, id); - if (!pdev) { - ret = -ENOMEM; - goto err_fwnode_remove; - } + if (!pdev) + return -ENOMEM; + + ret = device_create_managed_software_node(&pdev->dev, + xe_i2c_adapter_properties, + NULL); + if (ret) + goto err_pdev_put; pdev->dev.parent = i2c->drm_dev; - pdev->dev.fwnode = fwnode; - i2c->adapter_node = fwnode; i2c->pdev = pdev; ret = platform_device_add(pdev); @@ -146,8 +143,6 @@ static int xe_i2c_register_adapter(struct xe_i2c *i2c) err_pdev_put: platform_device_put(pdev); -err_fwnode_remove: - fwnode_remove_software_node(fwnode); return ret; } @@ -155,7 +150,6 @@ err_fwnode_remove: static void xe_i2c_unregister_adapter(struct xe_i2c *i2c) { platform_device_unregister(i2c->pdev); - fwnode_remove_software_node(i2c->adapter_node); } /** diff --git a/drivers/gpu/drm/xe/xe_i2c.h b/drivers/gpu/drm/xe/xe_i2c.h index d63adacfefe7..5969054a6b6b 100644 --- a/drivers/gpu/drm/xe/xe_i2c.h +++ b/drivers/gpu/drm/xe/xe_i2c.h @@ -33,7 +33,6 @@ struct xe_i2c_endpoint { }; struct xe_i2c { - struct fwnode_handle *adapter_node; struct platform_device *pdev; struct i2c_adapter *adapter; struct i2c_client *client[XE_I2C_MAX_CLIENTS]; diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index f1cf1463f1b2..bf5c2351aed3 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -706,6 +706,7 @@ u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc) #define LRC_CTX_JOB_TIMESTAMP_OFFSET 512 #define LRC_ENGINE_ID_PPHWSP_OFFSET 1024 #define LRC_PARALLEL_PPHWSP_OFFSET 2048 +#define LRC_ULLS_PPHWSP_OFFSET 2048 /* Mutually exclusive with parallel */ #define LRC_SEQNO_OFFSET 0 #define LRC_START_SEQNO_OFFSET (LRC_SEQNO_OFFSET + 8) @@ -768,6 +769,12 @@ static inline u32 __xe_lrc_engine_id_offset(struct xe_lrc *lrc) return xe_lrc_pphwsp_offset(lrc) + LRC_ENGINE_ID_PPHWSP_OFFSET; } +static u32 __xe_lrc_ulls_offset(struct xe_lrc *lrc) +{ + /* The ulls is stored in the driver-defined portion of PPHWSP */ + return xe_lrc_pphwsp_offset(lrc) + LRC_ULLS_PPHWSP_OFFSET; +} + static u32 __xe_lrc_ctx_timestamp_offset(struct xe_lrc *lrc) { return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP * sizeof(u32); @@ -835,6 +842,7 @@ DECL_MAP_ADDR_HELPERS(ctx_job_timestamp, lrc->bo) DECL_MAP_ADDR_HELPERS(ctx_timestamp, lrc->bo) DECL_MAP_ADDR_HELPERS(ctx_timestamp_udw, lrc->bo) DECL_MAP_ADDR_HELPERS(parallel, lrc->bo) +DECL_MAP_ADDR_HELPERS(ulls, lrc->bo) DECL_MAP_ADDR_HELPERS(indirect_ring, lrc->bo) DECL_MAP_ADDR_HELPERS(engine_id, lrc->bo) DECL_MAP_ADDR_HELPERS(queue_timestamp, lrc->bo) @@ -1805,6 +1813,26 @@ void xe_lrc_set_ring_tail(struct xe_lrc *lrc, u32 tail) xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, tail); } +/** + * xe_lrc_ring_tail_ggtt_addr() - Saved ring tail GGTT address + * @lrc: Pointer to the lrc. + * + * GGTT address of the ring tail as saved for this context - in the indirect + * ring state on platforms which have it, otherwise in the context image. + * This is what a context restore loads the tail register from, so a ring + * which advances the tail register itself must keep this in sync. + * + * Returns: saved ring tail GGTT address + */ +u32 xe_lrc_ring_tail_ggtt_addr(struct xe_lrc *lrc) +{ + if (xe_lrc_has_indirect_ring_state(lrc)) + return __xe_lrc_indirect_ring_ggtt_addr(lrc) + + INDIRECT_CTX_RING_TAIL * sizeof(u32); + + return __xe_lrc_regs_ggtt_addr(lrc) + CTX_RING_TAIL * sizeof(u32); +} + u32 xe_lrc_ring_tail(struct xe_lrc *lrc) { if (xe_lrc_has_indirect_ring_state(lrc)) @@ -1984,6 +2012,51 @@ static u32 xe_lrc_engine_id(struct xe_lrc *lrc) return xe_map_read32(xe, &map); } +#define semaphore_offset(seqno) \ + (sizeof(u32) * ((seqno) % LRC_MIGRATION_ULLS_SEMAPHORE_COUNT)) + +/** + * xe_lrc_ulls_semaphore_ggtt_addr() - ULLS semaphore GGTT address + * @lrc: Pointer to the lrc. + * @seqno: seqno of current job. + * + * Calculate ULLS semaphore GGTT address based on input seqno + * + * Returns: ULLS semaphore GGTT address + */ +u32 xe_lrc_ulls_semaphore_ggtt_addr(struct xe_lrc *lrc, u32 seqno) +{ + xe_assert(lrc_to_xe(lrc), semaphore_offset(seqno) < + LRC_PPHWSP_SIZE - LRC_ULLS_PPHWSP_OFFSET); + + return __xe_lrc_ulls_ggtt_addr(lrc) + semaphore_offset(seqno); +} + +/** + * xe_lrc_set_ulls_semaphore() - Set ULLS semaphore + * @lrc: Pointer to the lrc. + * @seqno: seqno of current job. + * + * Set ULLS semaphore based on input seqno + */ +void xe_lrc_set_ulls_semaphore(struct xe_lrc *lrc, u32 seqno) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map = __xe_lrc_ulls_map(lrc); + + xe_assert(xe, semaphore_offset(seqno) < + LRC_PPHWSP_SIZE - LRC_ULLS_PPHWSP_OFFSET); + + /* + * The ring contents this semaphore releases are ordered by the + * xe_device_wmb() at the end of xe_lrc_write_ring(). + */ + iosys_map_incr(&map, semaphore_offset(seqno)); + xe_map_write32(xe, &map, LRC_MIGRATION_ULLS_SEMAPHORE_SIGNAL); + + xe_device_wmb(xe); /* Flush write to hardware */ +} + static int instr_dw(u32 cmd_header) { /* GFXPIPE "SINGLE_DW" opcodes are a single dword */ diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index a8ff4e59a1f4..f2e829e62168 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -112,6 +112,7 @@ u32 xe_lrc_regs_offset(struct xe_lrc *lrc); void xe_lrc_set_ring_tail(struct xe_lrc *lrc, u32 tail); u32 xe_lrc_ring_tail(struct xe_lrc *lrc); +u32 xe_lrc_ring_tail_ggtt_addr(struct xe_lrc *lrc); void xe_lrc_set_ring_head(struct xe_lrc *lrc, u32 head); u32 xe_lrc_ring_head(struct xe_lrc *lrc); u32 xe_lrc_ring_space(struct xe_lrc *lrc); @@ -127,6 +128,9 @@ void xe_default_lrc_update_memirq_regs_with_address(struct xe_hw_engine *hwe); void xe_lrc_update_memirq_regs_with_address(struct xe_lrc *lrc, struct xe_hw_engine *hwe, u32 *regs); +u32 xe_lrc_ulls_semaphore_ggtt_addr(struct xe_lrc *lrc, u32 seqno); +void xe_lrc_set_ulls_semaphore(struct xe_lrc *lrc, u32 seqno); + u32 xe_lrc_read_ctx_reg(struct xe_lrc *lrc, int reg_nr); void xe_lrc_write_ctx_reg(struct xe_lrc *lrc, int reg_nr, u32 val); diff --git a/drivers/gpu/drm/xe/xe_lrc_types.h b/drivers/gpu/drm/xe/xe_lrc_types.h index 53ef48feebfc..b18daf1092e6 100644 --- a/drivers/gpu/drm/xe/xe_lrc_types.h +++ b/drivers/gpu/drm/xe/xe_lrc_types.h @@ -12,6 +12,10 @@ struct xe_bo; +#define LRC_MIGRATION_ULLS_SEMAPHORE_COUNT 64 /* Must be pow2 */ +#define LRC_MIGRATION_ULLS_SEMAPHORE_CLEAR 0 +#define LRC_MIGRATION_ULLS_SEMAPHORE_SIGNAL 1 + /** * struct xe_lrc - Logical ring context (LRC) and submission ring object */ diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index ff45c24d8889..fffb3eb6c76e 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -8,6 +8,7 @@ #include <linux/bitfield.h> #include <linux/sizes.h> +#include <drm/drm_drv.h> #include <drm/drm_managed.h> #include <drm/drm_pagemap.h> #include <drm/ttm/ttm_tt.h> @@ -22,6 +23,7 @@ #include "xe_assert.h" #include "xe_bb.h" #include "xe_bo.h" +#include "xe_configfs.h" #include "xe_exec_queue.h" #include "xe_ggtt.h" #include "xe_gt.h" @@ -32,6 +34,7 @@ #include "xe_mem_pool.h" #include "xe_mocs.h" #include "xe_pat.h" +#include "xe_pm.h" #include "xe_printk.h" #include "xe_pt.h" #include "xe_res_cursor.h" @@ -46,6 +49,157 @@ #include "xe_vram.h" /** + * DOC: ULLS (Ultra Low Latency Submission) for migration jobs + * + * Migration jobs issued on behalf of GPU page faults and SVM prefetches sit + * directly in the critical path of a stalled GPU workload. The dominant cost + * of such a job is not the copy or clear itself but the submission latency: + * the H2G round trip to GuC, the GuC scheduling decision, and the hardware + * context switch required to place the migration LRC on an engine. + * + * ULLS removes that cost by keeping the migration context resident and + * *running* on the hardware engine across jobs. Instead of the ring going + * empty and the context being switched out between jobs, the tail of every + * ULLS job parks the engine on a semaphore wait for the *next* job's + * semaphore, and then advances the ring tail itself. Submitting the next job + * therefore costs the CPU a single write to signal that semaphore - no H2G, + * no GuC round trip, no context switch, no MMIO. + * + * Requirements + * ------------ + * + * ULLS is only used on dGFX platforms with USM support, where a hardware + * engine is reserved exclusively for migration jobs. Because the engine + * spins on a semaphore while ULLS is active, it cannot be shared with + * user submissions. + * + * ULLS can be disabled by setting the ``migrate_ulls_period_ms`` configfs + * attribute to 0. Otherwise, the same attribute controls how long ULLS + * remains active before exiting, in milliseconds. + * + * Fixed size jobs + * --------------- + * + * A job updates the ring tail to cover its successor, but it is emitted long + * before that successor exists, so it can not know how much ring the + * successor will occupy. Every ULLS job is therefore padded out to exactly + * ULLS_JOB_SIZE_BYTES, which lets the next tail be computed arithmetically + * from where the current job started. + * + * This is why the shorter jobs still have to reach the same size: the "last" + * job skips the batch buffers and the postamble, and pads the difference with + * MI_NOOP. The "first" job is not covered by any predecessor's tail update + * and so is unconstrained, but is padded anyway to keep the arithmetic + * uniform. + * + * Leaving ULLS mode always goes through a "last" job, which emits no tail + * update, so an ordinary variable length migration job never follows a + * prediction. + * + * Semaphores + * ---------- + * + * The semaphores live in the driver-defined portion of the migration LRC's + * PPHWSP (see LRC_ULLS_PPHWSP_OFFSET, mutually exclusive with the parallel + * submission area). There are LRC_MIGRATION_ULLS_SEMAPHORE_COUNT of them and + * a job's semaphore is selected by ``seqno % COUNT``, so the semaphore ring + * wraps with the job seqnos. To guarantee a job can never overwrite the + * semaphore of a job still in flight, the GuC backend caps the migration + * queue's scheduler job count at LRC_MIGRATION_ULLS_SEMAPHORE_COUNT - 1. + * + * Ring layout of a ULLS job + * ------------------------- + * + * Emitted by emit_migration_job_gen12() in xe_ring_ops.c:: + * + * preamble: clear semaphore[seqno] (reuse for a later wrap) + * <copy timestamp, start seqno store> + * <batch buffer start(s)> (skipped on first/last job) + * <seqno write + user interrupt> + * postamble: wait on semaphore[seqno + 1] + * SDI saved ring tail = end of next job + * LRI RING_TAIL = end of next job + * (skipped on the last job) + * pad: MI_NOOP up to ULLS_JOB_SIZE_DW + * + * The preamble clears the current job's semaphore so it can be reused once + * the seqno space wraps. The postamble is what keeps the engine busy: it + * blocks on the next job's semaphore, which is only signaled when that job is + * actually submitted, and then advances the ring tail over it. It advances + * the saved tail as well as the tail register, keeping the two in step + * without any help from the CPU, so a context save and restore can not rewind + * the tail behind work which has already been published. + * + * The tail register write must be non-posted, i.e. it must not carry + * MI_LRI_FORCE_POSTED. Posted, the new tail is free to land after the command + * streamer has already drained the rest of the job, at which point the command + * streamer sees head == the old tail and parks as though the ring were empty. + * A parked context can be switched off the hardware, and the fast path below + * has no H2G with which to ask GuC to bring it back. + * + * The tail must be published after the semaphore wait, not before it. The + * command streamer fetches ring contents up to the tail while it is parked, + * so a tail published ahead of the wait lets it fetch the next job's slot + * before the CPU has written the job there. Once released it then executes + * the MI_NOOPs it fetched instead of the job, that job's fence never signals, + * and the engine drains and idles with nothing left to wake it. + * + * Submission fast path + * -------------------- + * + * In submit_exec_queue() (xe_guc_submit.c), a ULLS job that is not the first + * one reduces to:: + * + * xe_lrc_set_ulls_semaphore(lrc, seqno); release previous job + * + * The XE_GUC_ACTION_SCHED_CONTEXT H2G is suppressed, and so is the write of + * the saved ring tail: the previous job's postamble is parked on this job's + * semaphore and publishes this job's tail, both in the tail register and in + * the context image, as soon as it is released. The semaphore signal is all + * that is left, and the engine walks straight into this job. + * + * This does assume the context stays resident for as long as ULLS mode is + * active. Nothing else is scheduled on the reserved engine, so the only ways + * off the hardware are the "last" job below, or a reset - and a migration job + * failing already wedges the device. + * + * Enter / exit + * ------------ + * + * xe_migrate_ulls_enter() is called from the page fault handler and from the + * SVM prefetch path, i.e. exactly where low latency migration matters. It + * takes a PM runtime reference (the device must not suspend while the engine + * spins), then submits a "first" ULLS job. That first job carries no batch + * buffer; it exists only to get the context onto the hardware through the + * normal GuC path and to leave the engine waiting on the next semaphore, + * pipelining the GuC/HW context switch out of the critical path. + * + * No forcewake reference is required. Nothing in the fast path touches MMIO, + * and the engine keeps itself awake for as long as it is executing the ring. + * Not needing host MMIO access is also what lets ULLS run on SRIOV VFs. + * + * Keeping an engine spinning costs power, so ULLS is not left enabled + * indefinitely. Every enter and every ULLS job submission re-arms + * @xe_migrate.ulls.exit_work with a ULLS_EXIT_JIFFIES delay. When it fires + * with the queue idle, it submits a "last" ULLS job - again with no batch + * buffer and, crucially, with no postamble semaphore wait or tail update - + * which lets the ring drain so the context can be switched off the hardware. + * The PM reference is then dropped. If the queue was not idle, the worker + * simply re-arms itself. + * + * Job state + * --------- + * + * The state above is communicated to the ring ops and GuC backend via + * @xe_sched_job.ulls, set under @xe_migrate.job_mutex: + * + * - %ULLS_NONE: job submitted outside of ULLS mode + * - %ULLS_ENTER: job that enters ULLS mode + * - %ULLS_ACTIVE: job submitted while in ULLS mode + * - %ULLS_EXIT: job that exits ULLS mode + */ + +/** * struct xe_migrate - migrate context. */ struct xe_migrate { @@ -75,18 +229,23 @@ struct xe_migrate { * Protected by @job_mutex. */ struct dma_fence *fence; - /** - * @vm_update_sa: For integrated, used to suballocate page-tables - * out of the pt_bo. - */ - struct drm_suballoc_manager vm_update_sa; /** @min_chunk_size: For dgfx, Minimum chunk size */ u64 min_chunk_size; + /** @ulls: ULLS support */ + struct { + /** @ulls.exit_ms: ULLS exit period milliseconds */ + u32 exit_ms; + /** @ulls.enabled: ULLS is enabled, protected by job_mutex */ + bool enabled; + /** @ulls.exit_work: ULLS exit worker */ + struct delayed_work exit_work; + } ulls; }; +#define ULLS_EXIT_JIFFIES(_m) msecs_to_jiffies((_m)->ulls.exit_ms) + #define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */ #define MAX_CCS_LIMITED_TRANSFER SZ_4M /* XE_PAGE_SIZE * (FIELD_MAX(XE2_CCS_SIZE_MASK) + 1) */ -#define NUM_KERNEL_PDE 15 #define NUM_PT_SLOTS 48 #define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M #define MAX_NUM_PTE 512 @@ -101,9 +260,30 @@ struct xe_migrate { */ #define MAX_PTE_PER_SDI 0x1FEU +static bool xe_migrate_ulls_enabled(struct xe_migrate *m) +{ + lockdep_assert_held(&m->job_mutex); + return m->ulls.enabled; +} + +static void xe_migrate_ulls_toggle_enable(struct xe_migrate *m, bool enabled) +{ + lockdep_assert_held(&m->job_mutex); + m->ulls.enabled = enabled; +} + static void xe_migrate_fini(void *arg) { struct xe_migrate *m = arg; + struct xe_device *xe = tile_to_xe(m->tile); + + disable_delayed_work_sync(&m->ulls.exit_work); + scoped_guard(mutex, &m->job_mutex) { + if (xe_migrate_ulls_enabled(m)) { + xe_pm_runtime_put(xe); + xe_migrate_ulls_toggle_enable(m, false); + } + } xe_vm_lock(m->q->vm, false); xe_bo_unpin(m->pt_bo); @@ -111,7 +291,6 @@ static void xe_migrate_fini(void *arg) dma_fence_put(m->fence); xe_bo_put(m->pt_bo); - drm_suballoc_manager_fini(&m->vm_update_sa); mutex_destroy(&m->job_mutex); xe_vm_close_and_put(m->q->vm); xe_exec_queue_put(m->q); @@ -231,8 +410,6 @@ static int xe_migrate_pt_bo_alloc(struct xe_tile *tile, struct xe_migrate *m, BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE); /* Must be a multiple of 64K to support all platforms */ BUILD_BUG_ON(NUM_PT_SLOTS * XE_PAGE_SIZE % SZ_64K); - /* And one slot reserved for the 4KiB page table updates */ - BUILD_BUG_ON(!(NUM_KERNEL_PDE & 1)); /* Need to be sure everything fits in the first PT, or create more */ xe_tile_assert(tile, m->batch_base_ofs + xe_bo_size(batch) < SZ_2M); @@ -255,7 +432,8 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, struct xe_device *xe = tile_to_xe(tile); u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); u8 id = tile->id; - u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level; + u32 num_entries = NUM_PT_SLOTS, num_level = + xe_vm_pt_root(vm, id)->level; #define VRAM_IDENTITY_MAP_PT_COUNT 4 u32 num_setup = num_level + VRAM_IDENTITY_MAP_PT_COUNT; #undef VRAM_IDENTITY_MAP_PT_COUNT @@ -267,7 +445,7 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, u64 l1_pt_ofs = xe_bo_size(bo) - 5 * XE_PAGE_SIZE; entry = vm->pt_ops->pde_encode_bo(bo, l1_pt_ofs); - xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry); + xe_pt_write(xe, &xe_vm_pt_root(vm, id)->bo->vmap, 0, entry); map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE; @@ -388,17 +566,9 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, } } - if (ofs) - *ofs = map_ofs; -} - -static void xe_migrate_suballoc_manager_init(struct xe_migrate *m, u32 map_ofs) -{ /* * Example layout created above, with root level = 3: * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's - * [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's - * [PT9...PT40]: Userspace PT's for VM_BIND, 4 KiB PTE's * [PT41 = PDE 0] [PT44...PT47 = 4K and 2M vram identity maps] * * This makes the lowest part of the VM point to the pagetables. @@ -406,19 +576,13 @@ static void xe_migrate_suballoc_manager_init(struct xe_migrate *m, u32 map_ofs) * and flushes, other parts of the VM can be used either for copying and * clearing. * - * For performance, the kernel reserves PDE's, so about 20 are left - * for async VM updates. - * * To make it easier to work, each scratch PT is put in slot (1 + PT #) * everywhere, this allows lockless updates to scratch pages by using * the different addresses in VM. */ -#define NUM_VMUSA_UNIT_PER_PAGE 32 -#define VM_SA_UPDATE_UNIT_SIZE (XE_PAGE_SIZE / NUM_VMUSA_UNIT_PER_PAGE) -#define NUM_VMUSA_WRITES_PER_UNIT (VM_SA_UPDATE_UNIT_SIZE / sizeof(u64)) - drm_suballoc_manager_init(&m->vm_update_sa, - (size_t)(map_ofs / XE_PAGE_SIZE - NUM_KERNEL_PDE) * - NUM_VMUSA_UNIT_PER_PAGE, 0); + + if (ofs) + *ofs = map_ofs; } static bool xe_migrate_needs_ccs_emit(struct xe_device *xe) @@ -463,7 +627,6 @@ static int xe_migrate_lock_prepare_vm(struct xe_tile *tile, struct xe_migrate *m return err; xe_migrate_prepare_vm(tile, m, vm, &map_ofs); - xe_migrate_suballoc_manager_init(m, map_ofs); drm_exec_retry_on_contention(&exec); xe_validation_retry_on_oom(&ctx, &err); } @@ -471,6 +634,150 @@ static int xe_migrate_lock_prepare_vm(struct xe_tile *tile, struct xe_migrate *m return err; } +static struct dma_fence *__xe_migrate_job_push(struct xe_migrate *m, + struct xe_sched_job *job, + enum xe_ulls_state ulls) +{ + struct dma_fence *fence; + + lockdep_assert_held(&m->job_mutex); + xe_tile_assert(m->tile, m->q == job->q); + + job->ulls = ulls; + xe_sched_job_arm(job); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + return fence; +} + +/* + * Arm and push a migration job, tagging it as a ULLS job and deferring the + * ULLS exit while ULLS mode is active. + * + * Returns a reference to the job's finished fence. + */ +static struct dma_fence *xe_migrate_job_push(struct xe_migrate *m, + struct xe_sched_job *job) +{ + enum xe_ulls_state ulls = ULLS_NONE; + + lockdep_assert_held(&m->job_mutex); + + if (xe_migrate_ulls_enabled(m)) { + ulls = ULLS_ACTIVE; + mod_delayed_work(system_percpu_wq, &m->ulls.exit_work, + ULLS_EXIT_JIFFIES(m)); + } + + return __xe_migrate_job_push(m, job, ulls); +} + +/** + * xe_migrate_ulls_enter() - Enter ULLS mode + * @m: The migration context. + * + * If DGFX, enter ULLS mode bypassing GuC / HW context switches by utilizing + * semaphore and continuously running batches. + */ +void xe_migrate_ulls_enter(struct xe_migrate *m) +{ + struct xe_device *xe = tile_to_xe(m->tile); + struct xe_sched_job *job = NULL; + u64 batch_addr[2] = { 0, 0 }; + bool alloc = false; + + xe_assert(xe, xe->info.has_usm); + + if (!IS_DGFX(xe) || !m->ulls.exit_ms) + return; + +job_alloc: + if (alloc) { + /* + * Must be done outside job_mutex as that lock is tainted with + * reclaim. + */ + job = xe_sched_job_create(m->q, batch_addr); + if (WARN_ON_ONCE(IS_ERR(job))) + return; /* Not fatal */ + } + + mutex_lock(&m->job_mutex); + if (!xe_migrate_ulls_enabled(m)) { + struct dma_fence *fence; + + if (!job) { + alloc = true; + mutex_unlock(&m->job_mutex); + goto job_alloc; + } + + /* Pairs with PM put on ULLS exit */ + xe_pm_runtime_get_noresume(xe); + + xe_sched_job_get(job); + fence = __xe_migrate_job_push(m, job, ULLS_ENTER); + dma_fence_put(fence); + + xe_dbg(xe, "Migrate ULLS mode enter"); + xe_migrate_ulls_toggle_enable(m, true); + } + if (job) + xe_sched_job_put(job); + if (xe_migrate_ulls_enabled(m)) + mod_delayed_work(system_percpu_wq, &m->ulls.exit_work, + ULLS_EXIT_JIFFIES(m)); + mutex_unlock(&m->job_mutex); +} + +static void xe_migrate_ulls_exit(struct work_struct *work) +{ + struct xe_migrate *m = container_of(work, struct xe_migrate, + ulls.exit_work.work); + struct xe_device *xe = tile_to_xe(m->tile); + struct xe_sched_job *job = NULL; + struct dma_fence *fence = NULL; + u64 batch_addr[2] = { 0, 0 }; + int idx; + + xe_assert(xe, m->ulls.enabled); + + if (!drm_dev_enter(&xe->drm, &idx)) + return; + + /* + * Must be done outside job_mutex as that lock is tainted with + * reclaim and must be done holding a pm ref. + */ + job = xe_sched_job_create(m->q, batch_addr); + if (WARN_ON_ONCE(IS_ERR(job))) { + drm_dev_exit(idx); + mod_delayed_work(system_percpu_wq, &m->ulls.exit_work, + ULLS_EXIT_JIFFIES(m)); + return; /* Not fatal */ + } + + scoped_guard(mutex, &m->job_mutex) { + if (xe_exec_queue_is_idle(m->q, 1)) { + fence = __xe_migrate_job_push(m, job, ULLS_EXIT); + dma_fence_put(fence); + + xe_pm_runtime_put(xe); /* Pairs with PM get in enter */ + xe_migrate_ulls_toggle_enable(m, false); + cancel_delayed_work(&m->ulls.exit_work); + + xe_dbg(xe, "Migrate ULLS mode exit"); + } else { + xe_sched_job_put(job); + mod_delayed_work(system_percpu_wq, &m->ulls.exit_work, + ULLS_EXIT_JIFFIES(m)); + } + } + + drm_dev_exit(idx); +} + /** * xe_migrate_init() - Initialize a migrate context * @m: The migration context @@ -529,6 +836,8 @@ int xe_migrate_init(struct xe_migrate *m) might_lock(&m->job_mutex); fs_reclaim_release(GFP_KERNEL); + INIT_DELAYED_WORK(&m->ulls.exit_work, xe_migrate_ulls_exit); + err = devm_add_action_or_reset(xe->drm.dev, xe_migrate_fini, m); if (err) return err; @@ -546,6 +855,9 @@ int xe_migrate_init(struct xe_migrate *m) (unsigned long long)m->min_chunk_size); } + m->ulls.exit_ms = + xe_configfs_get_migrate_ulls_period_ms(to_pci_dev(xe->drm.dev)); + return err; err_out: @@ -1056,10 +1368,8 @@ static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m, } mutex_lock(&m->job_mutex); - xe_sched_job_arm(job); dma_fence_put(fence); - fence = dma_fence_get(&job->drm.s_fence->finished); - xe_sched_job_push(job); + fence = xe_migrate_job_push(m, job); dma_fence_put(m->fence); m->fence = dma_fence_get(fence); @@ -1146,6 +1456,9 @@ struct xe_lrc *xe_migrate_lrc(struct xe_migrate *migrate) return migrate->q->lrc[0]; } +/* XXX: With CPU binds this can be removed in a follow up */ +#define NUM_KERNEL_PDE 15 + static u64 migrate_vm_ppgtt_addr_tlb_inval(void) { /* @@ -1484,10 +1797,8 @@ struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_off DMA_RESV_USAGE_BOOKKEEP)); scoped_guard(mutex, &m->job_mutex) { - xe_sched_job_arm(job); dma_fence_put(fence); - fence = dma_fence_get(&job->drm.s_fence->finished); - xe_sched_job_push(job); + fence = xe_migrate_job_push(m, job); dma_fence_put(m->fence); m->fence = dma_fence_get(fence); @@ -1721,10 +2032,8 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m, } mutex_lock(&m->job_mutex); - xe_sched_job_arm(job); dma_fence_put(fence); - fence = dma_fence_get(&job->drm.s_fence->finished); - xe_sched_job_push(job); + fence = xe_migrate_job_push(m, job); dma_fence_put(m->fence); m->fence = dma_fence_get(fence); @@ -1754,55 +2063,6 @@ err_sync: return fence; } -static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs, - const struct xe_vm_pgtable_update_op *pt_op, - const struct xe_vm_pgtable_update *update, - struct xe_migrate_pt_update *pt_update) -{ - const struct xe_migrate_pt_update_ops *ops = pt_update->ops; - u32 chunk; - u32 ofs = update->ofs, size = update->qwords; - - /* - * If we have 512 entries (max), we would populate it ourselves, - * and update the PDE above it to the new pointer. - * The only time this can only happen if we have to update the top - * PDE. This requires a BO that is almost vm->size big. - * - * This shouldn't be possible in practice.. might change when 16K - * pages are used. Hence the assert. - */ - xe_tile_assert(tile, update->qwords < MAX_NUM_PTE); - if (!ppgtt_ofs) - ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile), - xe_bo_addr(update->pt_bo, 0, - XE_PAGE_SIZE), false); - - do { - u64 addr = ppgtt_ofs + ofs * 8; - - chunk = min(size, MAX_PTE_PER_SDI); - - /* Ensure populatefn can do memset64 by aligning bb->cs */ - if (!(bb->len & 1)) - bb->cs[bb->len++] = MI_NOOP; - - bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); - bb->cs[bb->len++] = lower_32_bits(addr); - bb->cs[bb->len++] = upper_32_bits(addr); - if (pt_op->bind) - ops->populate(pt_update, tile, NULL, bb->cs + bb->len, - ofs, chunk, update); - else - ops->clear(pt_update, tile, NULL, bb->cs + bb->len, - ofs, chunk, update); - - bb->len += chunk * 2; - ofs += chunk; - size -= chunk; - } while (size); -} - struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m) { return xe_vm_get(m->q->vm); @@ -1818,284 +2078,6 @@ struct migrate_test_params { container_of(_priv, struct migrate_test_params, base) #endif -static struct dma_fence * -xe_migrate_update_pgtables_cpu(struct xe_migrate *m, - struct xe_migrate_pt_update *pt_update) -{ - XE_TEST_DECLARE(struct migrate_test_params *test = - to_migrate_test_params - (xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));) - const struct xe_migrate_pt_update_ops *ops = pt_update->ops; - struct xe_vm *vm = pt_update->vops->vm; - struct xe_vm_pgtable_update_ops *pt_update_ops = - &pt_update->vops->pt_update_ops[pt_update->tile_id]; - int err; - u32 i, j; - - if (XE_TEST_ONLY(test && test->force_gpu)) - return ERR_PTR(-ETIME); - - if (ops->pre_commit) { - pt_update->job = NULL; - err = ops->pre_commit(pt_update); - if (err) - return ERR_PTR(err); - } - - for (i = 0; i < pt_update_ops->num_ops; ++i) { - const struct xe_vm_pgtable_update_op *pt_op = - &pt_update_ops->ops[i]; - - for (j = 0; j < pt_op->num_entries; j++) { - const struct xe_vm_pgtable_update *update = - &pt_op->entries[j]; - - if (pt_op->bind) - ops->populate(pt_update, m->tile, - &update->pt_bo->vmap, NULL, - update->ofs, update->qwords, - update); - else - ops->clear(pt_update, m->tile, - &update->pt_bo->vmap, NULL, - update->ofs, update->qwords, update); - } - } - - trace_xe_vm_cpu_bind(vm); - xe_device_wmb(vm->xe); - - return dma_fence_get_stub(); -} - -static struct dma_fence * -__xe_migrate_update_pgtables(struct xe_migrate *m, - struct xe_migrate_pt_update *pt_update, - struct xe_vm_pgtable_update_ops *pt_update_ops) -{ - const struct xe_migrate_pt_update_ops *ops = pt_update->ops; - struct xe_tile *tile = m->tile; - struct xe_gt *gt = tile->primary_gt; - struct xe_device *xe = tile_to_xe(tile); - struct xe_sched_job *job; - struct dma_fence *fence; - struct drm_suballoc *sa_bo = NULL; - struct xe_bb *bb; - u32 i, j, batch_size = 0, ppgtt_ofs, update_idx, page_ofs = 0; - u32 num_updates = 0, current_update = 0; - u64 addr; - int err = 0; - bool is_migrate = pt_update_ops->q == m->q; - bool usm = is_migrate && xe->info.has_usm; - - for (i = 0; i < pt_update_ops->num_ops; ++i) { - struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; - struct xe_vm_pgtable_update *updates = pt_op->entries; - - num_updates += pt_op->num_entries; - for (j = 0; j < pt_op->num_entries; ++j) { - u32 num_cmds = DIV_ROUND_UP(updates[j].qwords, - MAX_PTE_PER_SDI); - - /* align noop + MI_STORE_DATA_IMM cmd prefix */ - batch_size += 4 * num_cmds + updates[j].qwords * 2; - } - } - - /* fixed + PTE entries */ - if (IS_DGFX(xe)) - batch_size += 2; - else - batch_size += 6 * (num_updates / MAX_PTE_PER_SDI + 1) + - num_updates * 2; - - bb = xe_bb_new(gt, batch_size, usm); - if (IS_ERR(bb)) - return ERR_CAST(bb); - - /* For sysmem PTE's, need to map them in our hole.. */ - if (!IS_DGFX(xe)) { - u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); - u32 ptes, ofs; - - ppgtt_ofs = NUM_KERNEL_PDE - 1; - if (!is_migrate) { - u32 num_units = DIV_ROUND_UP(num_updates, - NUM_VMUSA_WRITES_PER_UNIT); - - if (num_units > m->vm_update_sa.size) { - err = -ENOBUFS; - goto err_bb; - } - sa_bo = drm_suballoc_new(&m->vm_update_sa, num_units, - GFP_KERNEL, true, 0); - if (IS_ERR(sa_bo)) { - err = PTR_ERR(sa_bo); - goto err_bb; - } - - ppgtt_ofs = NUM_KERNEL_PDE + - (drm_suballoc_soffset(sa_bo) / - NUM_VMUSA_UNIT_PER_PAGE); - page_ofs = (drm_suballoc_soffset(sa_bo) % - NUM_VMUSA_UNIT_PER_PAGE) * - VM_SA_UPDATE_UNIT_SIZE; - } - - /* Map our PT's to gtt */ - i = 0; - j = 0; - ptes = num_updates; - ofs = ppgtt_ofs * XE_PAGE_SIZE + page_ofs; - while (ptes) { - u32 chunk = min(MAX_PTE_PER_SDI, ptes); - u32 idx = 0; - - bb->cs[bb->len++] = MI_STORE_DATA_IMM | - MI_SDI_NUM_QW(chunk); - bb->cs[bb->len++] = ofs; - bb->cs[bb->len++] = 0; /* upper_32_bits */ - - for (; i < pt_update_ops->num_ops; ++i) { - struct xe_vm_pgtable_update_op *pt_op = - &pt_update_ops->ops[i]; - struct xe_vm_pgtable_update *updates = pt_op->entries; - - for (; j < pt_op->num_entries; ++j, ++current_update, ++idx) { - struct xe_vm *vm = pt_update->vops->vm; - struct xe_bo *pt_bo = updates[j].pt_bo; - - if (idx == chunk) - goto next_cmd; - - xe_tile_assert(tile, xe_bo_size(pt_bo) == SZ_4K); - - /* Map a PT at most once */ - if (pt_bo->update_index < 0) - pt_bo->update_index = current_update; - - addr = vm->pt_ops->pte_encode_bo(pt_bo, 0, - pat_index, 0); - bb->cs[bb->len++] = lower_32_bits(addr); - bb->cs[bb->len++] = upper_32_bits(addr); - } - - j = 0; - } - -next_cmd: - ptes -= chunk; - ofs += chunk * sizeof(u64); - } - - bb->cs[bb->len++] = MI_BATCH_BUFFER_END; - update_idx = bb->len; - - addr = xe_migrate_vm_addr(ppgtt_ofs, 0) + - (page_ofs / sizeof(u64)) * XE_PAGE_SIZE; - for (i = 0; i < pt_update_ops->num_ops; ++i) { - struct xe_vm_pgtable_update_op *pt_op = - &pt_update_ops->ops[i]; - struct xe_vm_pgtable_update *updates = pt_op->entries; - - for (j = 0; j < pt_op->num_entries; ++j) { - struct xe_bo *pt_bo = updates[j].pt_bo; - - write_pgtable(tile, bb, addr + - pt_bo->update_index * XE_PAGE_SIZE, - pt_op, &updates[j], pt_update); - } - } - } else { - /* phys pages, no preamble required */ - bb->cs[bb->len++] = MI_BATCH_BUFFER_END; - update_idx = bb->len; - - for (i = 0; i < pt_update_ops->num_ops; ++i) { - struct xe_vm_pgtable_update_op *pt_op = - &pt_update_ops->ops[i]; - struct xe_vm_pgtable_update *updates = pt_op->entries; - - for (j = 0; j < pt_op->num_entries; ++j) - write_pgtable(tile, bb, 0, pt_op, &updates[j], - pt_update); - } - } - - job = xe_bb_create_migration_job(pt_update_ops->q, bb, - xe_migrate_batch_base(m, usm), - update_idx); - if (IS_ERR(job)) { - err = PTR_ERR(job); - goto err_sa; - } - - xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB); - - if (ops->pre_commit) { - pt_update->job = job; - err = ops->pre_commit(pt_update); - if (err) - goto err_job; - } - if (is_migrate) - mutex_lock(&m->job_mutex); - - xe_sched_job_arm(job); - fence = dma_fence_get(&job->drm.s_fence->finished); - xe_sched_job_push(job); - - if (is_migrate) - mutex_unlock(&m->job_mutex); - - xe_bb_free(bb, fence); - drm_suballoc_free(sa_bo, fence); - - return fence; - -err_job: - xe_sched_job_put(job); -err_sa: - drm_suballoc_free(sa_bo, NULL); -err_bb: - xe_bb_free(bb, NULL); - return ERR_PTR(err); -} - -/** - * xe_migrate_update_pgtables() - Pipelined page-table update - * @m: The migrate context. - * @pt_update: PT update arguments - * - * Perform a pipelined page-table update. The update descriptors are typically - * built under the same lock critical section as a call to this function. If - * using the default engine for the updates, they will be performed in the - * order they grab the job_mutex. If different engines are used, external - * synchronization is needed for overlapping updates to maintain page-table - * consistency. Note that the meaning of "overlapping" is that the updates - * touch the same page-table, which might be a higher-level page-directory. - * If no pipelining is needed, then updates may be performed by the cpu. - * - * Return: A dma_fence that, when signaled, indicates the update completion. - */ -struct dma_fence * -xe_migrate_update_pgtables(struct xe_migrate *m, - struct xe_migrate_pt_update *pt_update) - -{ - struct xe_vm_pgtable_update_ops *pt_update_ops = - &pt_update->vops->pt_update_ops[pt_update->tile_id]; - struct dma_fence *fence; - - fence = xe_migrate_update_pgtables_cpu(m, pt_update); - - /* -ETIME indicates a job is needed, anything else is legit error */ - if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME) - return fence; - - return __xe_migrate_update_pgtables(m, pt_update, pt_update_ops); -} - /** * xe_migrate_wait() - Complete all operations using the xe_migrate context * @m: Migrate context to wait for. @@ -2327,9 +2309,7 @@ static struct dma_fence *xe_migrate_vram(struct xe_migrate *m, } mutex_lock(&m->job_mutex); - xe_sched_job_arm(job); - fence = dma_fence_get(&job->drm.s_fence->finished); - xe_sched_job_push(job); + fence = xe_migrate_job_push(m, job); dma_fence_put(m->fence); m->fence = dma_fence_get(fence); @@ -2597,56 +2577,6 @@ out_err: return IS_ERR(fence) ? PTR_ERR(fence) : 0; } -/** - * xe_migrate_job_lock() - Lock migrate job lock - * @m: The migration context. - * @q: Queue associated with the operation which requires a lock - * - * Lock the migrate job lock if the queue is a migration queue, otherwise - * assert the VM's dma-resv is held (user queue's have own locking). - */ -void xe_migrate_job_lock(struct xe_migrate *m, struct xe_exec_queue *q) -{ - bool is_migrate = q == m->q; - - if (is_migrate) - mutex_lock(&m->job_mutex); - else - xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */ -} - -/** - * xe_migrate_job_unlock() - Unlock migrate job lock - * @m: The migration context. - * @q: Queue associated with the operation which requires a lock - * - * Unlock the migrate job lock if the queue is a migration queue, otherwise - * assert the VM's dma-resv is held (user queue's have own locking). - */ -void xe_migrate_job_unlock(struct xe_migrate *m, struct xe_exec_queue *q) -{ - bool is_migrate = q == m->q; - - if (is_migrate) - mutex_unlock(&m->job_mutex); - else - xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */ -} - -#if IS_ENABLED(CONFIG_PROVE_LOCKING) -/** - * xe_migrate_job_lock_assert() - Assert migrate job lock held of queue - * @q: Migrate queue - */ -void xe_migrate_job_lock_assert(struct xe_exec_queue *q) -{ - struct xe_migrate *m = gt_to_tile(q->gt)->migrate; - - xe_gt_assert(q->gt, q == m->q); - lockdep_assert_held(&m->job_mutex); -} -#endif - #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) #include "tests/xe_migrate.c" #endif @@ -2696,10 +2626,7 @@ int xe_migrate_debug_ccs_overlap(struct xe_migrate *m, xe_sched_job_add_migrate_flush(job, MI_FLUSH_DW_CCS); mutex_lock(&m->job_mutex); - xe_sched_job_arm(job); - - fence = dma_fence_get(&job->drm.s_fence->finished); - xe_sched_job_push(job); + fence = xe_migrate_job_push(m, job); mutex_unlock(&m->job_mutex); dma_fence_wait(fence, false); diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index a9acc62f78f0..2e8be10fdb71 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -24,6 +24,7 @@ struct xe_pt; struct xe_tile; struct xe_vm; struct xe_vm_pgtable_update; +struct xe_vm_pgtable_update_op; struct xe_vma; enum xe_sriov_vf_ccs_rw_ctxs; @@ -33,83 +34,6 @@ enum xe_migrate_copy_dir { XE_MIGRATE_COPY_TO_SRAM, }; -/** - * struct xe_migrate_pt_update_ops - Callbacks for the - * xe_migrate_update_pgtables() function. - */ -struct xe_migrate_pt_update_ops { - /** - * @populate: Populate a command buffer or page-table with ptes. - * @pt_update: Embeddable callback argument. - * @tile: The tile for the current operation. - * @map: struct iosys_map into the memory to be populated. - * @pos: If @map is NULL, map into the memory to be populated. - * @ofs: qword offset into @map, unused if @map is NULL. - * @num_qwords: Number of qwords to write. - * @update: Information about the PTEs to be inserted. - * - * This interface is intended to be used as a callback into the - * page-table system to populate command buffers or shared - * page-tables with PTEs. - */ - void (*populate)(struct xe_migrate_pt_update *pt_update, - struct xe_tile *tile, struct iosys_map *map, - void *pos, u32 ofs, u32 num_qwords, - const struct xe_vm_pgtable_update *update); - /** - * @clear: Clear a command buffer or page-table with ptes. - * @pt_update: Embeddable callback argument. - * @tile: The tile for the current operation. - * @map: struct iosys_map into the memory to be populated. - * @pos: If @map is NULL, map into the memory to be populated. - * @ofs: qword offset into @map, unused if @map is NULL. - * @num_qwords: Number of qwords to write. - * @update: Information about the PTEs to be inserted. - * - * This interface is intended to be used as a callback into the - * page-table system to populate command buffers or shared - * page-tables with PTEs. - */ - void (*clear)(struct xe_migrate_pt_update *pt_update, - struct xe_tile *tile, struct iosys_map *map, - void *pos, u32 ofs, u32 num_qwords, - const struct xe_vm_pgtable_update *update); - - /** - * @pre_commit: Callback to be called just before arming the - * sched_job. - * @pt_update: Pointer to embeddable callback argument. - * - * Return: 0 on success, negative error code on error. - */ - int (*pre_commit)(struct xe_migrate_pt_update *pt_update); -}; - -/** - * struct xe_migrate_pt_update - Argument to the - * struct xe_migrate_pt_update_ops callbacks. - * - * Intended to be subclassed to support additional arguments if necessary. - */ -struct xe_migrate_pt_update { - /** @ops: Pointer to the struct xe_migrate_pt_update_ops callbacks */ - const struct xe_migrate_pt_update_ops *ops; - /** @vops: VMA operations */ - struct xe_vma_ops *vops; - /** @job: The job if a GPU page-table update. NULL otherwise */ - struct xe_sched_job *job; - /** - * @ijob: The TLB invalidation job for primary GT. NULL otherwise - */ - struct xe_tlb_inval_job *ijob; - /** - * @mjob: The TLB invalidation job for media GT. NULL otherwise - */ - struct xe_tlb_inval_job *mjob; - /** @tile_id: Tile ID of the update */ - u8 tile_id; -}; - struct xe_migrate *xe_migrate_alloc(struct xe_tile *tile); int xe_migrate_init(struct xe_migrate *m); @@ -165,27 +89,14 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m, struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m); -struct dma_fence * -xe_migrate_update_pgtables(struct xe_migrate *m, - struct xe_migrate_pt_update *pt_update); - void xe_migrate_wait(struct xe_migrate *m); -#if IS_ENABLED(CONFIG_PROVE_LOCKING) -void xe_migrate_job_lock_assert(struct xe_exec_queue *q); -#else -static inline void xe_migrate_job_lock_assert(struct xe_exec_queue *q) -{ -} -#endif - -void xe_migrate_job_lock(struct xe_migrate *m, struct xe_exec_queue *q); -void xe_migrate_job_unlock(struct xe_migrate *m, struct xe_exec_queue *q); - #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM) int xe_migrate_debug_ccs_overlap(struct xe_migrate *m, struct xe_bo *scratch_bo, bool write_to_ccs); #endif +void xe_migrate_ulls_enter(struct xe_migrate *m); + #endif diff --git a/drivers/gpu/drm/xe/xe_mmio.c b/drivers/gpu/drm/xe/xe_mmio.c index 7fa18dfcb5a2..455a07d79cd9 100644 --- a/drivers/gpu/drm/xe/xe_mmio.c +++ b/drivers/gpu/drm/xe/xe_mmio.c @@ -320,6 +320,7 @@ u64 xe_mmio_read64_2x32(struct xe_mmio *mmio, struct xe_reg reg) return (u64)udw << 32 | ldw; } +#define __XE_MMIO_WAIT_MAX_BACKOFF_100MS (100 * USEC_PER_MSEC) static int __xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us, u32 *out_val, bool atomic, bool expect_match) { @@ -350,10 +351,10 @@ static int __xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u wait = ktime_us_delta(end, cur); if (atomic) - udelay(wait); + udelay(min_t(s64, wait, (MAX_UDELAY_MS * 1000))); else usleep_range(wait, wait << 1); - wait <<= 1; + wait = min_t(s64, wait << 1, __XE_MMIO_WAIT_MAX_BACKOFF_100MS); } if (ret != 0) { diff --git a/drivers/gpu/drm/xe/xe_pagefault.c b/drivers/gpu/drm/xe/xe_pagefault.c index f9e5bd41cfd1..5247321c805e 100644 --- a/drivers/gpu/drm/xe/xe_pagefault.c +++ b/drivers/gpu/drm/xe/xe_pagefault.c @@ -13,8 +13,8 @@ #include "xe_gt_printk.h" #include "xe_gt_types.h" #include "xe_gt_stats.h" -#include "xe_hw_engine.h" #include "xe_log.h" +#include "xe_migrate.h" #include "xe_pagefault.h" #include "xe_pagefault_types.h" #include "xe_pm.h" @@ -267,6 +267,7 @@ static struct xe_vm *xe_pagefault_asid_to_vm(struct xe_pagefault *pf, u32 asid) static int xe_pagefault_service(struct xe_pagefault *pf) { struct xe_gt *gt = pf->gt; + struct xe_device *xe = gt_to_xe(gt); struct xe_vm *vm; struct xe_vma *vma = NULL; int err; @@ -278,8 +279,15 @@ static int xe_pagefault_service(struct xe_pagefault *pf) return -EFAULT; vm = xe_pagefault_asid_to_vm(pf, asid); - if (IS_ERR(vm)) - return PTR_ERR(vm); + if (IS_ERR(vm)) { + err = PTR_ERR(vm); + trace_xe_pagefault_fail(xe, NULL, NULL, asid, + xe_pagefault_addr(pf), + xe_pagefault_get_error(pf), err); + return err; + } + + xe_migrate_ulls_enter(gt_to_tile(gt)->migrate); down_read(&vm->lock); @@ -312,6 +320,11 @@ static int xe_pagefault_service(struct xe_pagefault *pf) err = xe_pagefault_handle_vma(gt, vma, pf, atomic); unlock_vm: + if (err) + trace_xe_pagefault_fail(xe, vm, vma, asid, + xe_pagefault_addr(pf), + xe_pagefault_get_error(pf), err); + up_read(&vm->lock); xe_vm_put(vm); @@ -607,36 +620,12 @@ static const char *xe_pagefault_error_to_str(enum xe_pagefault_error error) static void xe_pagefault_print(struct xe_pagefault *pf) { - u8 engine_class = FIELD_GET(XE_PAGEFAULT_ENGINE_CLASS_MASK, - pf->consumer.engine_class_instance); - u64 addr = xe_pagefault_addr(pf); - - xe_gt_info(pf->gt, "\n\tASID: %lu\n" - "\tFaulted Address: 0x%08x%08x\n" - "\tFaultType: %lu\n" - "\tAccessType: %lu\n" - "\tFaultLevel: %lu\n" - "\tEngineClass: %d %s\n" - "\tEngineInstance: %lu\n" - "\tSRCID: 0x%02lx\n" - "\tError: %s\n", - FIELD_GET(XE_PAGEFAULT_ASID_MASK, - pf->consumer.id), - upper_32_bits(addr), - lower_32_bits(addr), - FIELD_GET(XE_PAGEFAULT_TYPE_MASK, - pf->consumer.fault_type_level), - FIELD_GET(XE_PAGEFAULT_ACCESS_TYPE_MASK, - pf->consumer.access_type), - FIELD_GET(XE_PAGEFAULT_LEVEL_MASK, - pf->consumer.fault_type_level), - engine_class, - xe_hw_engine_class_to_str(engine_class), - FIELD_GET(XE_PAGEFAULT_ENGINE_INSTANCE_MASK, - pf->consumer.engine_class_instance), - FIELD_GET(XE_PAGEFAULT_SRCID_MASK, - pf->consumer.id), - xe_pagefault_error_to_str(xe_pagefault_get_error(pf))); + enum xe_pagefault_error err = xe_pagefault_get_error(pf); + + if (WARN_ON_ONCE(!pf->producer.ops->print)) + return; + + pf->producer.ops->print(pf, xe_pagefault_error_to_str(err)); } static void xe_pagefault_save_to_vm(struct xe_device *xe, struct xe_pagefault *pf) diff --git a/drivers/gpu/drm/xe/xe_pagefault_types.h b/drivers/gpu/drm/xe/xe_pagefault_types.h index 8ae6b9848bdc..ee467117afe9 100644 --- a/drivers/gpu/drm/xe/xe_pagefault_types.h +++ b/drivers/gpu/drm/xe/xe_pagefault_types.h @@ -139,6 +139,16 @@ struct xe_pagefault_ops { * Page fault producer ends acknowledgment from the consumer. */ void (*ack_fault_end)(void *private); + /** + * @print: Print pagefault info + * @pf: Page fault + * @err_str: Consumer servicing error string + * + * Print pagefault identification fields decoded from producer, + * along with consumer-specific @err_str prints. + * Must be implemented by all producers. + */ + void (*print)(struct xe_pagefault *pf, const char *err_str); }; /** diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index d66d5a0339ae..e44b74b9aa27 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -371,6 +371,7 @@ static const __maybe_unused struct xe_device_desc pvc_desc = { .has_display = false, .has_drm_ras = true, .has_gsc_nvm = 1, + .has_pt_mirror = 1, .has_heci_gscfi = 1, .max_gt_per_tile = 1, .max_remote_tiles = 1, @@ -460,6 +461,7 @@ static const struct xe_device_desc nvls_desc = { .has_flat_ccs = 1, .has_pre_prod_wa = 1, .has_sriov = true, + .has_pxp = true, .max_gt_per_tile = 2, MULTI_LRC_MASK, .va_bits = 48, @@ -471,6 +473,7 @@ static const struct xe_device_desc cri_desc = { PLATFORM(CRESCENTISLAND), .dma_mask_size = 52, .has_display = false, + .has_device_uid = true, .has_drm_ras = true, .has_flat_ccs = false, .has_gsc_nvm = 1, @@ -791,6 +794,7 @@ static int xe_info_init_early(struct xe_device *xe, xe->info.is_dgfx = desc->is_dgfx; xe->info.has_cached_pt = desc->has_cached_pt; + xe->info.has_device_uid = desc->has_device_uid; xe->info.has_drm_ras = desc->has_drm_ras; xe->info.has_fan_control = desc->has_fan_control; /* runtime fusing may force flat_ccs to disabled later */ @@ -806,6 +810,7 @@ static int xe_info_init_early(struct xe_device *xe, xe->info.has_mert = desc->has_mert; xe->info.has_page_reclaim_hw_assist = desc->has_page_reclaim_hw_assist; xe->info.has_pre_prod_wa = desc->has_pre_prod_wa; + xe->info.has_pt_mirror = desc->has_pt_mirror; xe->info.has_pxp = desc->has_pxp; xe->info.has_soc_remapper_sysctrl = desc->has_soc_remapper_sysctrl; xe->info.has_soc_remapper_telem = desc->has_soc_remapper_telem; diff --git a/drivers/gpu/drm/xe/xe_pci_sriov.c b/drivers/gpu/drm/xe/xe_pci_sriov.c index c023a2a1fc17..31abcee55211 100644 --- a/drivers/gpu/drm/xe/xe_pci_sriov.c +++ b/drivers/gpu/drm/xe/xe_pci_sriov.c @@ -177,6 +177,7 @@ static int pf_enable_vfs(struct xe_device *xe, int num_vfs) return num_vfs; failed: + pf_resize_vf_vram_bar(xe, xe->sriov.pf.device_total_vfs); xe_sriov_pf_unprovision_vfs(xe, num_vfs); xe_pm_runtime_put(xe); pf_finish_vfs_enabling(xe); @@ -204,6 +205,8 @@ static int pf_disable_vfs(struct xe_device *xe) pci_disable_sriov(pdev); + pf_resize_vf_vram_bar(xe, xe->sriov.pf.device_total_vfs); + xe_sriov_pf_reprovision_default(xe); pf_reset_vfs(xe, num_vfs); diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h index fed509ff601e..448bbe5ba1d7 100644 --- a/drivers/gpu/drm/xe/xe_pci_types.h +++ b/drivers/gpu/drm/xe/xe_pci_types.h @@ -40,6 +40,7 @@ struct xe_device_desc { u8 has_cached_pt:1; u8 has_display:1; + u8 has_device_uid:1; u8 has_drm_ras:1; u8 has_fan_control:1; u8 has_flat_ccs:1; @@ -52,8 +53,9 @@ struct xe_device_desc { u8 has_mbx_power_limits:1; u8 has_mbx_thermal_info:1; u8 has_mert:1; - u8 has_pre_prod_wa:1; u8 has_page_reclaim_hw_assist:1; + u8 has_pre_prod_wa:1; + u8 has_pt_mirror:1; u8 has_pxp:1; u8 has_soc_remapper_sysctrl:1; u8 has_soc_remapper_telem:1; diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index 4b351dbf6572..6ad4bbaa1860 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -7,12 +7,12 @@ #include "regs/xe_gtt_defs.h" #include "xe_bo.h" +#include "xe_cpu_bind.h" #include "xe_device.h" #include "xe_drm_client.h" #include "xe_exec_queue.h" #include "xe_gt.h" #include "xe_gt_stats.h" -#include "xe_migrate.h" #include "xe_page_reclaim.h" #include "xe_pat.h" #include "xe_pt_types.h" @@ -206,6 +206,7 @@ unsigned int xe_pt_shift(unsigned int level) */ void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred) { + bool added = false; int i; if (!pt) @@ -213,7 +214,9 @@ void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred) XE_WARN_ON(!list_empty(&pt->bo->ttm.base.gpuva.list)); xe_bo_unpin(pt->bo); - xe_bo_put_deferred(pt->bo, deferred); + xe_bo_put_deferred(pt->bo, deferred, &added); + if (added) + xe_assert(pt->bo->vm->xe, !kref_read(&pt->bo->ttm.base.refcount)); if (pt->level > 0 && pt->num_live) { struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt); @@ -376,7 +379,7 @@ xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent, entry->pt = parent; entry->flags = 0; entry->qwords = 0; - entry->pt_bo->update_index = -1; + entry->level = parent->level; if (alloc_entries) { entry->pt_entries = kmalloc_objs(*entry->pt_entries, XE_PDES); @@ -801,7 +804,7 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, .wupd.entries = entries, .clear_pt = clear_pt, }; - struct xe_pt *pt = vm->pt_root[tile->id]; + struct xe_pt *pt = xe_vm_pt_root(vm, tile->id); int ret; bool is_purged = false; @@ -1002,6 +1005,11 @@ static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset, return 0; } +static bool pt_mirroring_disabled_for_tile(struct xe_vm *vm, u8 tile_id) +{ + return !vm->xe->info.has_pt_mirror && tile_id; +} + static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = { .pt_entry = xe_pt_zap_ptes_entry, }; @@ -1043,6 +1051,9 @@ bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma) if (!(pt_mask & BIT(tile->id))) return false; + if (pt_mirroring_disabled_for_tile(xe_vma_vm(vma), tile->id)) + return true; + (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma), xe_vma_end(vma), &xe_walk.base); @@ -1095,6 +1106,9 @@ bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm, if (!(pt_mask & BIT(tile->id))) return false; + if (pt_mirroring_disabled_for_tile(vm, tile->id)) + return true; + (void)xe_pt_walk_shared(&pt->base, pt->level, xe_svm_range_start(range), xe_svm_range_end(range), &xe_walk.base); @@ -1102,32 +1116,18 @@ bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm, } static void -xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile, - struct iosys_map *map, void *data, - u32 qword_ofs, u32 num_qwords, +xe_vm_populate_pgtable(struct xe_tile *tile, struct iosys_map *map, const struct xe_vm_pgtable_update *update) { struct xe_pt_entry *ptes = update->pt_entries; - u64 *ptr = data; u32 i; - /* - * @qword_ofs is the absolute entry offset within the page table, while - * @ptes is indexed relative to @update->ofs (its first entry). The GPU - * path (write_pgtable) splits a single update into MAX_PTE_PER_SDI-sized - * chunks, calling this with an advancing @qword_ofs but a fresh @data - * pointer per chunk, so translate back into a @ptes index rather than - * assuming the chunk starts at ptes[0]. - */ - for (i = 0; i < num_qwords; i++) { - u32 idx = qword_ofs - update->ofs + i; + xe_assert(tile_to_xe(tile), map); + xe_assert(tile_to_xe(tile), !iosys_map_is_null(map)); - if (map) - xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * - sizeof(u64), u64, ptes[idx].pte); - else - ptr[i] = ptes[idx].pte; - } + for (i = 0; i < update->qwords; i++) + xe_map_wr(tile_to_xe(tile), map, (update->ofs + i) * + sizeof(u64), u64, ptes[i].pte); } static void xe_pt_cancel_bind(struct xe_vma *vma, @@ -1411,11 +1411,9 @@ static int op_add_deps(struct xe_vm *vm, struct xe_vma_op *op, } static int xe_pt_vm_dependencies(struct xe_sched_job *job, - struct xe_tlb_inval_job *ijob, - struct xe_tlb_inval_job *mjob, + struct xe_tlb_inval_job **ijobs, struct xe_vm *vm, struct xe_vma_ops *vops, - struct xe_vm_pgtable_update_ops *pt_update_ops, struct xe_range_fence_tree *rftree) { struct xe_range_fence *rtfence; @@ -1428,20 +1426,22 @@ static int xe_pt_vm_dependencies(struct xe_sched_job *job, if (!job && !no_in_syncs(vops->syncs, vops->num_syncs)) return -ETIME; - if (!job && !xe_exec_queue_is_idle(pt_update_ops->q)) + if (!job && !xe_exec_queue_is_idle(vops->q, 0)) return -ETIME; - if (pt_update_ops->wait_vm_bookkeep || pt_update_ops->wait_vm_kernel) { - err = job_test_add_deps(job, xe_vm_resv(vm), - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_BOOKKEEP : - DMA_RESV_USAGE_KERNEL); + if (vops->flags & (XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP | + XE_VMA_OPS_FLAG_WAIT_VM_KERNEL)) { + enum dma_resv_usage usage = DMA_RESV_USAGE_KERNEL; + + if (vops->flags & XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP) + usage = DMA_RESV_USAGE_BOOKKEEP; + + err = job_test_add_deps(job, xe_vm_resv(vm), usage); if (err) return err; } - rtfence = xe_range_fence_tree_first(rftree, pt_update_ops->start, - pt_update_ops->last); + rtfence = xe_range_fence_tree_first(rftree, vops->start, vops->last); while (rtfence) { fence = rtfence->fence; @@ -1459,9 +1459,8 @@ static int xe_pt_vm_dependencies(struct xe_sched_job *job, return err; } - rtfence = xe_range_fence_tree_next(rtfence, - pt_update_ops->start, - pt_update_ops->last); + rtfence = xe_range_fence_tree_next(rtfence, vops->start, + vops->last); } list_for_each_entry(op, &vops->list, link) { @@ -1474,14 +1473,11 @@ static int xe_pt_vm_dependencies(struct xe_sched_job *job, err = xe_sync_entry_add_deps(&vops->syncs[i], job); if (job) { - if (ijob) { - err = xe_tlb_inval_job_alloc_dep(ijob); - if (err) - return err; - } + for (i = 0; i < XE_CPU_BIND_INVAL_JOB_COUNT; ++i) { + if (!ijobs[i]) + continue; - if (mjob) { - err = xe_tlb_inval_job_alloc_dep(mjob); + err = xe_tlb_inval_job_alloc_dep(ijobs[i]); if (err) return err; } @@ -1490,17 +1486,14 @@ static int xe_pt_vm_dependencies(struct xe_sched_job *job, return err; } -static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update) +static int xe_pt_pre_commit(struct xe_cpu_bind_pt_update *pt_update) { struct xe_vma_ops *vops = pt_update->vops; struct xe_vm *vm = vops->vm; - struct xe_range_fence_tree *rftree = &vm->rftree[pt_update->tile_id]; - struct xe_vm_pgtable_update_ops *pt_update_ops = - &vops->pt_update_ops[pt_update->tile_id]; + struct xe_range_fence_tree *rftree = &vm->rftree; - return xe_pt_vm_dependencies(pt_update->job, pt_update->ijob, - pt_update->mjob, vm, pt_update->vops, - pt_update_ops, rftree); + return xe_pt_vm_dependencies(pt_update->job, pt_update->ijobs, + vm, vops, rftree); } #if IS_ENABLED(CONFIG_DRM_GPUSVM) @@ -1560,8 +1553,7 @@ static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma) #endif -static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma, - struct xe_vm_pgtable_update_ops *pt_update) +static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma) { struct xe_userptr_vma *uvma; unsigned long notifier_seq; @@ -1591,8 +1583,7 @@ static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma, return 0; } -static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op, - struct xe_vm_pgtable_update_ops *pt_update) +static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op) { int err = 0; @@ -1603,13 +1594,13 @@ static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op, if (!op->map.immediate && xe_vm_in_fault_mode(vm)) break; - err = vma_check_userptr(vm, op->map.vma, pt_update); + err = vma_check_userptr(vm, op->map.vma); break; case DRM_GPUVA_OP_REMAP: if (op->remap.prev && !op->remap.skip_prev) - err = vma_check_userptr(vm, op->remap.prev, pt_update); + err = vma_check_userptr(vm, op->remap.prev); if (!err && op->remap.next && !op->remap.skip_next) - err = vma_check_userptr(vm, op->remap.next, pt_update); + err = vma_check_userptr(vm, op->remap.next); break; case DRM_GPUVA_OP_UNMAP: break; @@ -1629,7 +1620,7 @@ static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op, } } } else { - err = vma_check_userptr(vm, gpuva_to_vma(op->base.prefetch.va), pt_update); + err = vma_check_userptr(vm, gpuva_to_vma(op->base.prefetch.va)); } break; #if IS_ENABLED(CONFIG_DRM_XE_GPUSVM) @@ -1655,12 +1646,10 @@ static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op, return err; } -static int xe_pt_svm_userptr_pre_commit(struct xe_migrate_pt_update *pt_update) +static int xe_pt_svm_userptr_pre_commit(struct xe_cpu_bind_pt_update *pt_update) { struct xe_vm *vm = pt_update->vops->vm; struct xe_vma_ops *vops = pt_update->vops; - struct xe_vm_pgtable_update_ops *pt_update_ops = - &vops->pt_update_ops[pt_update->tile_id]; struct xe_vma_op *op; int err; @@ -1671,7 +1660,7 @@ static int xe_pt_svm_userptr_pre_commit(struct xe_migrate_pt_update *pt_update) xe_pt_svm_userptr_notifier_lock(vm); list_for_each_entry(op, &vops->list, link) { - err = op_check_svm_userptr(vm, op, pt_update_ops); + err = op_check_svm_userptr(vm, op); if (err) { xe_pt_svm_userptr_notifier_unlock(vm); break; @@ -2009,7 +1998,7 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, .wupd.entries = entries, .prl = pt_update_op->prl, }; - struct xe_pt *pt = vm->pt_root[tile->id]; + struct xe_pt *pt = xe_vm_pt_root(vm, tile->id); (void)xe_pt_walk_shared(&pt->base, pt->level, start, end, &xe_walk.base); @@ -2018,24 +2007,23 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, } static void -xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update, - struct xe_tile *tile, struct iosys_map *map, - void *ptr, u32 qword_ofs, u32 num_qwords, - const struct xe_vm_pgtable_update *update) +xe_pt_clear_pgtable_callback(struct xe_vm *vm, struct xe_tile *tile, + struct iosys_map *map, + const struct xe_vm_pgtable_update *update) { - struct xe_vm *vm = pt_update->vops->vm; - u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level); + u64 empty = __xe_pt_empty_pte(tile, vm, update->level); int i; - if (map && map->is_iomem) - for (i = 0; i < num_qwords; ++i) - xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * + xe_assert(vm->xe, map); + xe_assert(vm->xe, !iosys_map_is_null(map)); + + if (map->is_iomem) + for (i = 0; i < update->qwords; ++i) + xe_map_wr(tile_to_xe(tile), map, (update->ofs + i) * sizeof(u64), u64, empty); - else if (map) - memset64(map->vaddr + qword_ofs * sizeof(u64), empty, - num_qwords); else - memset64(ptr, empty, num_qwords); + memset64(map->vaddr + update->ofs * sizeof(u64), empty, + update->qwords); } static void xe_pt_abort_unbind(struct xe_vma *vma, @@ -2090,13 +2078,40 @@ xe_pt_commit_prepare_unbind(struct xe_vma *vma, } } +static struct xe_vm_pgtable_update_op * +to_pt_op(struct xe_vm_pgtable_update_ops *pt_update_ops, u32 op_idx) +{ + return &pt_update_ops->pt_job_ops->ops[op_idx]; +} + +static u32 +get_current_op(struct xe_vm_pgtable_update_ops *pt_update_ops) +{ + if (!pt_update_ops->pt_job_ops) + return 0; + + return pt_update_ops->pt_job_ops->current_op; +} + +static struct xe_vm_pgtable_update_op * +to_current_pt_op(struct xe_vm_pgtable_update_ops *pt_update_ops) +{ + return to_pt_op(pt_update_ops, get_current_op(pt_update_ops)); +} + +static void +incr_current_op(struct xe_vm_pgtable_update_ops *pt_update_ops) +{ + ++pt_update_ops->pt_job_ops->current_op; +} + static void xe_pt_update_ops_rfence_interval(struct xe_vm_pgtable_update_ops *pt_update_ops, u64 start, u64 end) { u64 last; - u32 current_op = pt_update_ops->current_op; - struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; + struct xe_vm_pgtable_update_op *pt_op = + to_current_pt_op(pt_update_ops); int i, level = 0; for (i = 0; i < pt_op->num_entries; i++) { @@ -2131,8 +2146,8 @@ static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile, struct xe_vm_pgtable_update_ops *pt_update_ops, struct xe_vma *vma, bool invalidate_on_bind) { - u32 current_op = pt_update_ops->current_op; - struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; + struct xe_vm_pgtable_update_op *pt_op = + to_current_pt_op(pt_update_ops); int err; xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma)); @@ -2161,7 +2176,7 @@ static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile, xe_pt_update_ops_rfence_interval(pt_update_ops, xe_vma_start(vma), xe_vma_end(vma)); - ++pt_update_ops->current_op; + incr_current_op(pt_update_ops); pt_update_ops->needs_svm_lock |= xe_vma_is_userptr(vma); /* @@ -2204,8 +2219,8 @@ static int bind_range_prepare(struct xe_vm *vm, struct xe_tile *tile, struct xe_vm_pgtable_update_ops *pt_update_ops, struct xe_vma *vma, struct xe_svm_range *range) { - u32 current_op = pt_update_ops->current_op; - struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; + struct xe_vm_pgtable_update_op *pt_op = + to_current_pt_op(pt_update_ops); int err; xe_tile_assert(tile, xe_vma_is_cpu_addr_mirror(vma)); @@ -2229,7 +2244,7 @@ static int bind_range_prepare(struct xe_vm *vm, struct xe_tile *tile, xe_pt_update_ops_rfence_interval(pt_update_ops, xe_svm_range_start(range), xe_svm_range_end(range)); - ++pt_update_ops->current_op; + incr_current_op(pt_update_ops); pt_update_ops->needs_svm_lock = true; pt_op->vma = vma; @@ -2247,8 +2262,8 @@ static int unbind_op_prepare(struct xe_tile *tile, struct xe_vma *vma) { struct xe_device *xe = tile_to_xe(tile); - u32 current_op = pt_update_ops->current_op; - struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; + struct xe_vm_pgtable_update_op *pt_op = + to_current_pt_op(pt_update_ops); int err; if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id))) @@ -2287,7 +2302,7 @@ static int unbind_op_prepare(struct xe_tile *tile, pt_op->num_entries, false); xe_pt_update_ops_rfence_interval(pt_update_ops, xe_vma_start(vma), xe_vma_end(vma)); - ++pt_update_ops->current_op; + incr_current_op(pt_update_ops); pt_update_ops->needs_svm_lock |= xe_vma_is_userptr(vma); pt_update_ops->needs_invalidation = true; @@ -2327,8 +2342,8 @@ static int unbind_range_prepare(struct xe_vm *vm, struct xe_vm_pgtable_update_ops *pt_update_ops, struct xe_svm_range *range) { - u32 current_op = pt_update_ops->current_op; - struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; + struct xe_vm_pgtable_update_op *pt_op = + to_current_pt_op(pt_update_ops); if (!(range->tile_present & BIT(tile->id))) return 0; @@ -2349,7 +2364,7 @@ static int unbind_range_prepare(struct xe_vm *vm, pt_op->num_entries, false); xe_pt_update_ops_rfence_interval(pt_update_ops, xe_svm_range_start(range), xe_svm_range_end(range)); - ++pt_update_ops->current_op; + incr_current_op(pt_update_ops); pt_update_ops->needs_svm_lock = true; pt_update_ops->needs_invalidation |= xe_vm_has_scratch(vm) || xe_vm_has_valid_gpu_mapping(tile, range->tile_present, @@ -2364,6 +2379,7 @@ static int unbind_range_prepare(struct xe_vm *vm, static int op_prepare(struct xe_vm *vm, struct xe_tile *tile, + struct xe_vma_ops *vops, struct xe_vm_pgtable_update_ops *pt_update_ops, struct xe_vma_op *op) { @@ -2380,7 +2396,7 @@ static int op_prepare(struct xe_vm *vm, err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma, op->map.invalidate_on_bind); - pt_update_ops->wait_vm_kernel = true; + vops->flags |= XE_VMA_OPS_FLAG_WAIT_VM_KERNEL; break; case DRM_GPUVA_OP_REMAP: { @@ -2394,12 +2410,12 @@ static int op_prepare(struct xe_vm *vm, if (!err && op->remap.prev && !op->remap.skip_prev) { err = bind_op_prepare(vm, tile, pt_update_ops, op->remap.prev, false); - pt_update_ops->wait_vm_bookkeep = true; + vops->flags |= XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP; } if (!err && op->remap.next && !op->remap.skip_next) { err = bind_op_prepare(vm, tile, pt_update_ops, op->remap.next, false); - pt_update_ops->wait_vm_bookkeep = true; + vops->flags |= XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP; } break; } @@ -2435,7 +2451,7 @@ static int op_prepare(struct xe_vm *vm, } } else { err = bind_op_prepare(vm, tile, pt_update_ops, vma, false); - pt_update_ops->wait_vm_kernel = true; + vops->flags |= XE_VMA_OPS_FLAG_WAIT_VM_KERNEL; } break; } @@ -2461,8 +2477,7 @@ static int op_prepare(struct xe_vm *vm, static void xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops) { - init_llist_head(&pt_update_ops->deferred); - pt_update_ops->current_op = 0; + pt_update_ops->pt_job_ops->current_op = 0; pt_update_ops->start = ~0x0ull; pt_update_ops->last = 0x0ull; pt_update_ops->needs_svm_lock = false; @@ -2470,18 +2485,8 @@ xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops) xe_page_reclaim_list_init(&pt_update_ops->prl); } -/** - * xe_pt_update_ops_prepare() - Prepare PT update operations - * @tile: Tile of PT update operations - * @vops: VMA operationa - * - * Prepare PT update operations which includes updating internal PT state, - * allocate memory for page tables, populate page table being pruned in, and - * create PT update operations for leaf insertion / removal. - * - * Return: 0 on success, negative error code on error. - */ -int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops) +static int __xe_pt_update_ops_prepare(struct xe_tile *tile, + struct xe_vma_ops *vops) { struct xe_vm_pgtable_update_ops *pt_update_ops = &vops->pt_update_ops[tile->id]; @@ -2500,15 +2505,25 @@ int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops) return err; list_for_each_entry(op, &vops->list, link) { - err = op_prepare(vops->vm, tile, pt_update_ops, op); + err = op_prepare(vops->vm, tile, vops, pt_update_ops, op); if (err) return err; } - xe_tile_assert(tile, pt_update_ops->current_op <= + xe_tile_assert(tile, get_current_op(pt_update_ops) <= pt_update_ops->num_ops); + /* Propagate individual tile state up to VMA operation */ + if (pt_update_ops->start < vops->start) + vops->start = pt_update_ops->start; + if (pt_update_ops->last > vops->last) + vops->last = pt_update_ops->last; + if (pt_update_ops->needs_invalidation) + vops->flags |= XE_VMA_OPS_FLAG_NEEDS_INVALIDATION; + if (pt_update_ops->needs_svm_lock) + vops->flags |= XE_VMA_OPS_FLAG_NEEDS_SVM_LOCK; + #ifdef TEST_VM_OPS_ERROR if (vops->inject_error && vops->vm->xe->vm_inject_error_position == FORCE_OP_ERROR_PREPARE) @@ -2517,35 +2532,80 @@ int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops) return 0; } -ALLOW_ERROR_INJECTION(xe_pt_update_ops_prepare, ERRNO); -static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile, - struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma *vma, struct dma_fence *fence, - struct dma_fence *fence2, bool invalidate_on_bind) +/** + * xe_pt_update_ops_prepare() - Prepare PT update operations + * @xe: xe device. + * @vops: VMA operationa + * + * Prepare PT update operations which includes updating internal PT state, + * allocate memory for page tables, populate page table being pruned in, and + * create PT update operations for leaf insertion / removal. + * + * Return: 0 on success, negative error code on error. + */ +int xe_pt_update_ops_prepare(struct xe_device *xe, struct xe_vma_ops *vops) { - xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma)); + struct xe_tile *tile; + int id, err; + + for_each_tile(tile, xe, id) { + struct xe_vm_pgtable_update_ops *pt_update_ops = + &vops->pt_update_ops[id]; + + if (!vops->pt_update_ops[id].num_ops) + continue; + + if (pt_mirroring_disabled_for_tile(vops->vm, id)) { + struct xe_page_reclaim_list *prl = &pt_update_ops->prl; + + /* Transfer root PT update ops PRL to current */ + *prl = vops->pt_update_ops[0].prl; + xe_page_reclaim_entries_get(prl->entries); + continue; + } - if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { - dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - if (fence2) - dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); + err = __xe_pt_update_ops_prepare(tile, vops); + if (err) + return err; } + + return 0; +} +ALLOW_ERROR_INJECTION(xe_pt_update_ops_prepare, ERRNO); + +static void vma_add_fences(struct xe_vma *vma, struct dma_fence **fences, + int fence_count, enum dma_resv_usage usage) +{ + int i; + + if (xe_vma_has_no_bo(vma) || xe_vma_bo(vma)->vm) + return; + + for (i = 0; i < fence_count; ++i) + if (fences[i]) + dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, + fences[i], usage); +} + +static void bind_op_commit(struct xe_vm *vm, struct xe_vma *vma, + struct dma_fence **fences, int fence_count, + enum dma_resv_usage usage, u8 tile_mask, + bool invalidate_on_bind) +{ + xe_assert(vm->xe, !xe_vma_is_cpu_addr_mirror(vma)); + + vma_add_fences(vma, fences, fence_count, usage); + /* All WRITE_ONCE pair with READ_ONCE in xe_vm_has_valid_gpu_mapping() */ - WRITE_ONCE(vma->tile_present, vma->tile_present | BIT(tile->id)); + WRITE_ONCE(vma->tile_present, vma->tile_present | tile_mask); if (invalidate_on_bind) WRITE_ONCE(vma->tile_invalidated, - vma->tile_invalidated | BIT(tile->id)); + vma->tile_invalidated | tile_mask); else WRITE_ONCE(vma->tile_invalidated, - vma->tile_invalidated & ~BIT(tile->id)); - vma->tile_staged &= ~BIT(tile->id); + vma->tile_invalidated & ~tile_mask); + vma->tile_staged &= ~tile_mask; if (xe_vma_is_userptr(vma)) { xe_svm_assert_held_read_or_inject_write(vm); to_userptr_vma(vma)->userptr.initial_bind = true; @@ -2555,31 +2615,21 @@ static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile, * Kick rebind worker if this bind triggers preempt fences and not in * the rebind worker */ - if (pt_update_ops->wait_vm_bookkeep && + if (usage == DMA_RESV_USAGE_KERNEL && xe_vm_in_preempt_fence_mode(vm) && !current->mm) xe_vm_queue_rebind_worker(vm); } -static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile, - struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma *vma, struct dma_fence *fence, - struct dma_fence *fence2) +static void unbind_op_commit(struct xe_vm *vm, struct xe_vma *vma, + struct dma_fence **fences, int fence_count, + enum dma_resv_usage usage, u8 tile_mask) { - xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma)); + xe_assert(vm->xe, !xe_vma_is_cpu_addr_mirror(vma)); + + vma_add_fences(vma, fences, fence_count, usage); - if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { - dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - if (fence2) - dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - } - vma->tile_present &= ~BIT(tile->id); + vma->tile_present &= ~tile_mask; if (!vma->tile_present) { list_del_init(&vma->combined_links.rebind); if (xe_vma_is_userptr(vma)) { @@ -2594,21 +2644,19 @@ static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile, static void range_present_and_invalidated_tile(struct xe_vm *vm, struct xe_svm_range *range, - u8 tile_id) + u8 tile_mask) { /* All WRITE_ONCE pair with READ_ONCE in xe_vm_has_valid_gpu_mapping() */ lockdep_assert_held(&vm->svm.gpusvm.notifier_lock); - WRITE_ONCE(range->tile_present, range->tile_present | BIT(tile_id)); - WRITE_ONCE(range->tile_invalidated, range->tile_invalidated & ~BIT(tile_id)); + WRITE_ONCE(range->tile_present, range->tile_present | tile_mask); + WRITE_ONCE(range->tile_invalidated, range->tile_invalidated & ~tile_mask); } -static void op_commit(struct xe_vm *vm, - struct xe_tile *tile, - struct xe_vm_pgtable_update_ops *pt_update_ops, - struct xe_vma_op *op, struct dma_fence *fence, - struct dma_fence *fence2) +static void op_commit(struct xe_vm *vm, struct xe_vma_op *op, + struct dma_fence **fences, int fence_count, + enum dma_resv_usage usage, u8 tile_mask) { xe_vm_assert_held(vm); @@ -2618,8 +2666,8 @@ static void op_commit(struct xe_vm *vm, (op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR)) break; - bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence, - fence2, op->map.invalidate_on_bind); + bind_op_commit(vm, op->map.vma, fences, fence_count, usage, + tile_mask, op->map.invalidate_on_bind); break; case DRM_GPUVA_OP_REMAP: { @@ -2628,14 +2676,15 @@ static void op_commit(struct xe_vm *vm, if (xe_vma_is_cpu_addr_mirror(old)) break; - unbind_op_commit(vm, tile, pt_update_ops, old, fence, fence2); + unbind_op_commit(vm, old, fences, fence_count, usage, + tile_mask); if (op->remap.prev && !op->remap.skip_prev) - bind_op_commit(vm, tile, pt_update_ops, op->remap.prev, - fence, fence2, false); + bind_op_commit(vm, op->remap.prev, fences, fence_count, + usage, tile_mask, false); if (op->remap.next && !op->remap.skip_next) - bind_op_commit(vm, tile, pt_update_ops, op->remap.next, - fence, fence2, false); + bind_op_commit(vm, op->remap.next, fences, fence_count, + usage, tile_mask, false); break; } case DRM_GPUVA_OP_UNMAP: @@ -2643,8 +2692,8 @@ static void op_commit(struct xe_vm *vm, struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va); if (!xe_vma_is_cpu_addr_mirror(vma)) - unbind_op_commit(vm, tile, pt_update_ops, vma, fence, - fence2); + unbind_op_commit(vm, vma, fences, fence_count, + usage, tile_mask); break; } case DRM_GPUVA_OP_PREFETCH: @@ -2656,10 +2705,11 @@ static void op_commit(struct xe_vm *vm, unsigned long i; xa_for_each(&op->prefetch_range.range, i, range) - range_present_and_invalidated_tile(vm, range, tile->id); + range_present_and_invalidated_tile(vm, range, + tile_mask); } else { - bind_op_commit(vm, tile, pt_update_ops, vma, fence, - fence2, false); + bind_op_commit(vm, vma, fences, fence_count, usage, + tile_mask, false); } break; } @@ -2667,11 +2717,12 @@ static void op_commit(struct xe_vm *vm, { /* WRITE_ONCE pairs with READ_ONCE in xe_vm_has_valid_gpu_mapping() */ if (op->subop == XE_VMA_SUBOP_MAP_RANGE) - range_present_and_invalidated_tile(vm, op->map_range.range, tile->id); + range_present_and_invalidated_tile(vm, op->map_range.range, + tile_mask); else if (op->subop == XE_VMA_SUBOP_UNMAP_RANGE) WRITE_ONCE(op->unmap_range.range->tile_present, op->unmap_range.range->tile_present & - ~BIT(tile->id)); + ~tile_mask); break; } @@ -2680,34 +2731,25 @@ static void op_commit(struct xe_vm *vm, } } -static const struct xe_migrate_pt_update_ops migrate_ops = { +static const struct xe_cpu_bind_pt_update_ops cpu_bind_ops = { .populate = xe_vm_populate_pgtable, - .clear = xe_migrate_clear_pgtable_callback, + .clear = xe_pt_clear_pgtable_callback, .pre_commit = xe_pt_pre_commit, }; #if IS_ENABLED(CONFIG_DRM_GPUSVM) -static const struct xe_migrate_pt_update_ops svm_userptr_migrate_ops = { +static const struct xe_cpu_bind_pt_update_ops svm_userptr_cpu_bind_ops = { .populate = xe_vm_populate_pgtable, - .clear = xe_migrate_clear_pgtable_callback, + .clear = xe_pt_clear_pgtable_callback, .pre_commit = xe_pt_svm_userptr_pre_commit, }; #else -static const struct xe_migrate_pt_update_ops svm_userptr_migrate_ops; +static const struct xe_cpu_bind_pt_update_ops svm_userptr_cpu_bind_ops; #endif -static struct xe_dep_scheduler *to_dep_scheduler(struct xe_exec_queue *q, - struct xe_gt *gt) -{ - if (xe_gt_is_media_type(gt)) - return q->tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT].dep_scheduler; - - return q->tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT].dep_scheduler; -} - /** * xe_pt_update_ops_run() - Run PT update operations - * @tile: Tile of PT update operations + * @xe: xe device. * @vops: VMA operationa * * Run PT update operations which includes committing internal PT state changes, @@ -2717,80 +2759,83 @@ static struct xe_dep_scheduler *to_dep_scheduler(struct xe_exec_queue *q, * Return: fence on success, negative ERR_PTR on error. */ struct dma_fence * -xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) +xe_pt_update_ops_run(struct xe_device *xe, struct xe_vma_ops *vops) { struct xe_vm *vm = vops->vm; - struct xe_vm_pgtable_update_ops *pt_update_ops = - &vops->pt_update_ops[tile->id]; - struct xe_exec_queue *q = pt_update_ops->q; - struct dma_fence *fence, *ifence = NULL, *mfence = NULL; - struct xe_tlb_inval_job *ijob = NULL, *mjob = NULL; + struct xe_exec_queue *q = vops->q; + struct dma_fence *fence; + struct dma_fence *ifences[XE_CPU_BIND_INVAL_JOB_COUNT] = {}; struct xe_range_fence *rfence; + enum dma_resv_usage usage = DMA_RESV_USAGE_BOOKKEEP; struct xe_vma_op *op; - int err = 0, i; - struct xe_migrate_pt_update update = { - .ops = pt_update_ops->needs_svm_lock ? - &svm_userptr_migrate_ops : - &migrate_ops, + struct xe_tile *tile; + int err = 0, total_ops = 0, i, j; + u8 tile_mask = 0; + bool needs_invalidation = vops->flags & + XE_VMA_OPS_FLAG_NEEDS_INVALIDATION; + bool needs_svm_lock = vops->flags & + XE_VMA_OPS_FLAG_NEEDS_SVM_LOCK; + struct xe_cpu_bind_pt_update update = { + .ops = needs_svm_lock ? &svm_userptr_cpu_bind_ops : + &cpu_bind_ops, .vops = vops, - .tile_id = tile->id, }; lockdep_assert_held(&vm->lock); xe_vm_assert_held(vm); - if (!pt_update_ops->current_op) { - xe_tile_assert(tile, xe_vm_in_fault_mode(vm)); + for_each_tile(tile, xe, j) { + struct xe_vm_pgtable_update_ops *pt_update_ops = + &vops->pt_update_ops[j]; + total_ops += get_current_op(pt_update_ops); + } + if (!total_ops) { + xe_assert(xe, xe_vm_in_fault_mode(vm)); return dma_fence_get_stub(); } #ifdef TEST_VM_OPS_ERROR if (vops->inject_error && - vm->xe->vm_inject_error_position == FORCE_OP_ERROR_RUN) + xe->vm_inject_error_position == FORCE_OP_ERROR_RUN) return ERR_PTR(-ENOSPC); #endif - if (pt_update_ops->needs_invalidation) { - struct xe_dep_scheduler *dep_scheduler = - to_dep_scheduler(q, tile->primary_gt); - - ijob = xe_tlb_inval_job_create(q, &tile->primary_gt->tlb_inval, - dep_scheduler, vm, - pt_update_ops->start, - pt_update_ops->last, - XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT); - if (IS_ERR(ijob)) { - err = PTR_ERR(ijob); - goto kill_vm_tile1; - } - update.ijob = ijob; - /* - * Only add page reclaim for the primary GT. Media GT does not have - * any PPC to flush, so enabling the PPC flush bit for media is - * effectively a NOP and provides no performance benefit nor - * interfere with primary GT. - */ - if (xe_page_reclaim_list_valid(&pt_update_ops->prl)) { - xe_tlb_inval_job_add_page_reclaim(ijob, &pt_update_ops->prl); - /* Release ref from alloc, job will now handle it */ - xe_page_reclaim_list_invalidate(&pt_update_ops->prl); - } - - if (tile->media_gt) { - dep_scheduler = to_dep_scheduler(q, tile->media_gt); - - mjob = xe_tlb_inval_job_create(q, - &tile->media_gt->tlb_inval, - dep_scheduler, vm, - pt_update_ops->start, - pt_update_ops->last, - XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT); - if (IS_ERR(mjob)) { - err = PTR_ERR(mjob); + if (needs_invalidation) { + for_each_tlb_inval(q, i) { + struct xe_dep_scheduler *dep_scheduler = + q->tlb_inval[i].dep_scheduler; + struct xe_tile *tile = + &xe->tiles[i / XE_MAX_GT_PER_TILE]; + struct xe_vm_pgtable_update_ops *pt_update_ops = + &vops->pt_update_ops[tile->id]; + struct xe_page_reclaim_list *prl = &pt_update_ops->prl; + struct xe_tlb_inval_job *ijob; + struct xe_gt *gt = i % XE_MAX_GT_PER_TILE ? + tile->media_gt : tile->primary_gt; + + ijob = xe_tlb_inval_job_create(q, >->tlb_inval, + dep_scheduler, + vm, vops->start, + vops->last, i); + if (IS_ERR(ijob)) { + err = PTR_ERR(ijob); goto free_ijob; } - update.mjob = mjob; + + update.ijobs[i] = ijob; + + /* + * Only add page reclaim for the primary GT. Media GT + * does not have any PPC to flush, so enabling the PPC + * flush bit for media is effectively a NOP and provides + * no performance benefit nor interfere with primary GT. + */ + if (xe_page_reclaim_list_valid(prl)) { + xe_tlb_inval_job_add_page_reclaim(ijob, prl); + /* Release ref from alloc, job will now handle it */ + xe_page_reclaim_list_invalidate(prl); + } } } @@ -2800,65 +2845,67 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) goto free_ijob; } - fence = xe_migrate_update_pgtables(tile->migrate, &update); + fence = xe_cpu_bind_update_pgtables(xe->cpu_bind, &update); if (IS_ERR(fence)) { err = PTR_ERR(fence); goto free_rfence; } /* Point of no return - VM killed if failure after this */ - for (i = 0; i < pt_update_ops->current_op; ++i) { - struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; + for_each_tile(tile, xe, j) { + struct xe_vm_pgtable_update_ops *pt_update_ops = + &vops->pt_update_ops[j]; + + if (pt_mirroring_disabled_for_tile(vm, j)) { + xe_tile_assert(tile, !get_current_op(pt_update_ops)); + tile_mask |= BIT(tile->id); + continue; + } + + for (i = 0; i < get_current_op(pt_update_ops); ++i) { + struct xe_vm_pgtable_update_op *pt_op = + to_pt_op(pt_update_ops, i); - xe_pt_commit(pt_op->vma, pt_op->entries, - pt_op->num_entries, &pt_update_ops->deferred); - pt_op->vma = NULL; /* skip in xe_pt_update_ops_abort */ + xe_pt_commit(pt_op->vma, pt_op->entries, + pt_op->num_entries, + &pt_update_ops->pt_job_ops->deferred); + pt_op->vma = NULL; /* skip in xe_pt_update_ops_abort */ + tile_mask |= BIT(tile->id); + } } - if (xe_range_fence_insert(&vm->rftree[tile->id], rfence, + if (xe_range_fence_insert(&vm->rftree, rfence, &xe_range_fence_kfree_ops, - pt_update_ops->start, - pt_update_ops->last, fence)) + vops->start, vops->last, fence)) dma_fence_wait(fence, false); - if (ijob) - ifence = xe_tlb_inval_job_push(ijob, tile->migrate, fence); - if (mjob) - mfence = xe_tlb_inval_job_push(mjob, tile->migrate, fence); + if (vops->flags & XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP) + usage = DMA_RESV_USAGE_KERNEL; - if (!mjob && !ijob) { - dma_resv_add_fence(xe_vm_resv(vm), fence, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); + if (!needs_invalidation) { + dma_resv_add_fence(xe_vm_resv(vm), fence, usage); list_for_each_entry(op, &vops->list, link) - op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL); - } else if (ijob && !mjob) { - dma_resv_add_fence(xe_vm_resv(vm), ifence, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); - - list_for_each_entry(op, &vops->list, link) - op_commit(vops->vm, tile, pt_update_ops, op, ifence, NULL); + op_commit(vops->vm, op, &fence, 1, usage, tile_mask); } else { - dma_resv_add_fence(xe_vm_resv(vm), ifence, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); + for (i = 0; i < XE_CPU_BIND_INVAL_JOB_COUNT; ++i) { + if (!update.ijobs[i]) + continue; - dma_resv_add_fence(xe_vm_resv(vm), mfence, - pt_update_ops->wait_vm_bookkeep ? - DMA_RESV_USAGE_KERNEL : - DMA_RESV_USAGE_BOOKKEEP); + ifences[i] = xe_tlb_inval_job_push(update.ijobs[i], + fence); + xe_assert(xe, !IS_ERR_OR_NULL(ifences[i])); + + dma_resv_add_fence(xe_vm_resv(vm), ifences[i], usage); + } list_for_each_entry(op, &vops->list, link) - op_commit(vops->vm, tile, pt_update_ops, op, ifence, - mfence); + op_commit(vops->vm, op, ifences, + XE_CPU_BIND_INVAL_JOB_COUNT, usage, + tile_mask); } - if (pt_update_ops->needs_svm_lock) + if (needs_svm_lock) xe_pt_svm_userptr_notifier_unlock(vm); /* @@ -2868,21 +2915,18 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) if (!(q->flags & EXEC_QUEUE_FLAG_MIGRATE)) xe_exec_queue_last_fence_set(q, vm, fence); - xe_tlb_inval_job_put(mjob); - xe_tlb_inval_job_put(ijob); - dma_fence_put(ifence); - dma_fence_put(mfence); + for (i = 0; i < XE_CPU_BIND_INVAL_JOB_COUNT; ++i) { + xe_tlb_inval_job_put(update.ijobs[i]); + dma_fence_put(ifences[i]); + } return fence; free_rfence: kfree(rfence); free_ijob: - xe_tlb_inval_job_put(mjob); - xe_tlb_inval_job_put(ijob); -kill_vm_tile1: - if (err != -EAGAIN && err != -ENODATA && tile->id) - xe_vm_kill(vops->vm, false); + for (i = 0; i < XE_CPU_BIND_INVAL_JOB_COUNT; ++i) + xe_tlb_inval_job_put(update.ijobs[i]); return ERR_PTR(err); } @@ -2890,61 +2934,164 @@ ALLOW_ERROR_INJECTION(xe_pt_update_ops_run, ERRNO); /** * xe_pt_update_ops_fini() - Finish PT update operations - * @tile: Tile of PT update operations + * @xe: xe device. * @vops: VMA operations * * Finish PT update operations by committing to destroy page table memory */ -void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops) +void xe_pt_update_ops_fini(struct xe_device *xe, struct xe_vma_ops *vops) { - struct xe_vm_pgtable_update_ops *pt_update_ops = - &vops->pt_update_ops[tile->id]; - int i; - - xe_page_reclaim_entries_put(pt_update_ops->prl.entries); + struct xe_tile *tile; + int id; - lockdep_assert_held(&vops->vm->lock); - xe_vm_assert_held(vops->vm); + for_each_tile(tile, xe, id) { + struct xe_vm_pgtable_update_ops *pt_update_ops = + &vops->pt_update_ops[id]; - for (i = 0; i < pt_update_ops->current_op; ++i) { - struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; + if (!pt_update_ops->num_ops) + continue; - xe_pt_free_bind(pt_op->entries, pt_op->num_entries); + xe_page_reclaim_entries_put(pt_update_ops->prl.entries); } - xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred); } /** * xe_pt_update_ops_abort() - Abort PT update operations - * @tile: Tile of PT update operations + * @xe: xe device. * @vops: VMA operationa * * Abort PT update operations by unwinding internal PT state */ -void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops) +void xe_pt_update_ops_abort(struct xe_device *xe, struct xe_vma_ops *vops) { - struct xe_vm_pgtable_update_ops *pt_update_ops = - &vops->pt_update_ops[tile->id]; - int i; + struct xe_tile *tile; + int id; lockdep_assert_held(&vops->vm->lock); xe_vm_assert_held(vops->vm); - for (i = pt_update_ops->num_ops - 1; i >= 0; --i) { - struct xe_vm_pgtable_update_op *pt_op = - &pt_update_ops->ops[i]; + for_each_tile(tile, xe, id) { + struct xe_vm_pgtable_update_ops *pt_update_ops = + &vops->pt_update_ops[id]; + int i; - if (!pt_op->vma || i >= pt_update_ops->current_op) + if (pt_mirroring_disabled_for_tile(vops->vm, id)) continue; - if (pt_op->bind) - xe_pt_abort_bind(pt_op->vma, pt_op->entries, - pt_op->num_entries, - pt_op->rebind); - else - xe_pt_abort_unbind(pt_op->vma, pt_op->entries, - pt_op->num_entries); + for (i = pt_update_ops->num_ops - 1; i >= 0; --i) { + struct xe_vm_pgtable_update_op *pt_op = + to_pt_op(pt_update_ops, i); + + if (!pt_op->vma || i >= get_current_op(pt_update_ops)) + continue; + + if (pt_op->bind) + xe_pt_abort_bind(pt_op->vma, pt_op->entries, + pt_op->num_entries, + pt_op->rebind); + else + xe_pt_abort_unbind(pt_op->vma, pt_op->entries, + pt_op->num_entries); + } + } + + xe_pt_update_ops_fini(xe, vops); +} + +/** + * xe_pt_job_ops_alloc() - Allocate PT job ops + * @num_ops: Number of VM PT update ops + * + * Allocate PT job ops and internal array of VM PT update ops. + * + * Return: Pointer to PT job ops or NULL + */ +struct xe_pt_job_ops *xe_pt_job_ops_alloc(u32 num_ops) +{ + struct xe_pt_job_ops *pt_job_ops; + + pt_job_ops = kmalloc_obj(*pt_job_ops); + if (!pt_job_ops) + return NULL; + + pt_job_ops->ops = kvmalloc_array(num_ops, sizeof(*pt_job_ops->ops), + GFP_KERNEL); + if (!pt_job_ops->ops) { + kfree(pt_job_ops); + return NULL; } - xe_pt_update_ops_fini(tile, vops); + pt_job_ops->current_op = 0; + kref_init(&pt_job_ops->refcount); + init_llist_head(&pt_job_ops->deferred); + + return pt_job_ops; +} + +/** + * xe_pt_job_ops_get() - Get PT job ops + * @pt_job_ops: PT job ops to get + * + * Take a reference to PT job ops + * + * Return: Pointer to PT job ops or NULL + */ +struct xe_pt_job_ops *xe_pt_job_ops_get(struct xe_pt_job_ops *pt_job_ops) +{ + if (pt_job_ops) + kref_get(&pt_job_ops->refcount); + + return pt_job_ops; +} + +static void xe_pt_update_ops_free(struct xe_vm_pgtable_update_op *pt_op, + u32 num_ops) +{ + u32 i; + + for (i = 0; i < num_ops; ++i, ++pt_op) + xe_pt_free_bind(pt_op->entries, pt_op->num_entries); +} + +static void xe_pt_job_ops_destroy(struct kref *ref) +{ + struct xe_pt_job_ops *pt_job_ops = + container_of(ref, struct xe_pt_job_ops, refcount); + struct llist_node *freed; + struct xe_bo *bo, *next; + + xe_pt_update_ops_free(pt_job_ops->ops, + pt_job_ops->current_op); + + freed = llist_del_all(&pt_job_ops->deferred); + if (freed) { + llist_for_each_entry_safe(bo, next, freed, freed) { + struct xe_bo_dev *bo_device = + &xe_bo_device(bo)->bo_device; + /* + * If called from run_job, we are in the dma-fencing + * path and cannot take dma-resv locks so use an async + * put. + */ + if (llist_add(&bo->freed, &bo_device->async_list)) + schedule_work(&bo_device->async_free); + } + } + + kvfree(pt_job_ops->ops); + kfree(pt_job_ops); +} + +/** + * xe_pt_job_ops_put() - Put PT job ops + * @pt_job_ops: PT job ops to put + * + * Drop a reference to PT job ops + */ +void xe_pt_job_ops_put(struct xe_pt_job_ops *pt_job_ops) +{ + if (!pt_job_ops) + return; + + kref_put(&pt_job_ops->refcount, xe_pt_job_ops_destroy); } diff --git a/drivers/gpu/drm/xe/xe_pt.h b/drivers/gpu/drm/xe/xe_pt.h index 4daeebaab5a1..cd78141fb81c 100644 --- a/drivers/gpu/drm/xe/xe_pt.h +++ b/drivers/gpu/drm/xe/xe_pt.h @@ -39,14 +39,18 @@ void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred); void xe_pt_clear(struct xe_device *xe, struct xe_pt *pt); -int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops); -struct dma_fence *xe_pt_update_ops_run(struct xe_tile *tile, +int xe_pt_update_ops_prepare(struct xe_device *xe, struct xe_vma_ops *vops); +struct dma_fence *xe_pt_update_ops_run(struct xe_device *xe, struct xe_vma_ops *vops); -void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops); -void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops); +void xe_pt_update_ops_fini(struct xe_device *xe, struct xe_vma_ops *vops); +void xe_pt_update_ops_abort(struct xe_device *xe, struct xe_vma_ops *vops); bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma); bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm, struct xe_svm_range *range); +struct xe_pt_job_ops *xe_pt_job_ops_alloc(u32 num_ops); +struct xe_pt_job_ops *xe_pt_job_ops_get(struct xe_pt_job_ops *pt_job_ops); +void xe_pt_job_ops_put(struct xe_pt_job_ops *pt_job_ops); + #endif diff --git a/drivers/gpu/drm/xe/xe_pt_types.h b/drivers/gpu/drm/xe/xe_pt_types.h index a7d1bb708b69..0d4bac22ee6c 100644 --- a/drivers/gpu/drm/xe/xe_pt_types.h +++ b/drivers/gpu/drm/xe/xe_pt_types.h @@ -65,12 +65,18 @@ struct xe_vm_pgtable_update { /** @qwords: number of PTE's to write */ u32 qwords; - /** @pt: opaque pointer useful for the caller of xe_migrate_update_pgtables */ + /** + * @pt: opaque pointer useful for PT building in the bind IOCTL. Only + * safe to touch during the bind IOCTL (i.e., not in bind jobs). + */ struct xe_pt *pt; /** @pt_entries: Newly added pagetable entries */ struct xe_pt_entry *pt_entries; + /** @level: level of update */ + unsigned int level; + /** @flags: Target flags */ u32 flags; }; @@ -91,14 +97,29 @@ struct xe_vm_pgtable_update_op { bool rebind; }; +/** + * struct xe_pt_job_ops - Page-table update operations (dynamically allocated) + * + * This is the portion of &struct xe_vma_ops and + * &struct xe_vm_pgtable_update_ops that is dynamically allocated, as it + * must remain valid until the associated bind job completes. A reference + * count controls its lifetime. + */ +struct xe_pt_job_ops { + /** @current_op: current page-table update operation */ + u32 current_op; + /** @refcount: reference count */ + struct kref refcount; + /** @deferred: list of deferred PT entries to destroy */ + struct llist_head deferred; + /** @ops: page-table update operations */ + struct xe_vm_pgtable_update_op *ops; +}; + /** struct xe_vm_pgtable_update_ops: page table update operations */ struct xe_vm_pgtable_update_ops { - /** @ops: operations */ - struct xe_vm_pgtable_update_op *ops; - /** @deferred: deferred list to destroy PT entries */ - struct llist_head deferred; - /** @q: exec queue for PT operations */ - struct xe_exec_queue *q; + /** @pt_job_ops: PT update operations dynamic allocation*/ + struct xe_pt_job_ops *pt_job_ops; /** @prl: embedded page reclaim list */ struct xe_page_reclaim_list prl; /** @start: start address of ops */ @@ -107,24 +128,10 @@ struct xe_vm_pgtable_update_ops { u64 last; /** @num_ops: number of operations */ u32 num_ops; - /** @current_op: current operations */ - u32 current_op; /** @needs_svm_lock: Needs SVM lock */ bool needs_svm_lock; /** @needs_invalidation: Needs invalidation */ bool needs_invalidation; - /** - * @wait_vm_bookkeep: PT operations need to wait until VM is idle - * (bookkeep dma-resv slots are idle) and stage all future VM activity - * behind these operations (install PT operations into VM kernel - * dma-resv slot). - */ - bool wait_vm_bookkeep; - /** - * @wait_vm_kernel: PT operations need to wait until VM kernel dma-resv - * slots are idle. - */ - bool wait_vm_kernel; }; #endif diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index 46ab1f0f3564..ad63181938e3 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -143,9 +143,8 @@ static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i) static int emit_flush_invalidate(u32 addr, u32 val, u32 flush_flags, u32 *dw, int i) { - dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | - MI_FLUSH_IMM_DW | (flush_flags & MI_INVALIDATE_TLB) ?: 0; - + dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW | + (flush_flags & MI_INVALIDATE_TLB); dw[i++] = addr | MI_FLUSH_DW_USE_GTT; dw[i++] = 0; dw[i++] = val; @@ -505,6 +504,74 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job, xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); } +static int emit_ulls_preamble(struct xe_lrc *lrc, u32 *dw, int i, u32 seqno) +{ + u32 addr = xe_lrc_ulls_semaphore_ggtt_addr(lrc, seqno); + + return emit_store_imm_ggtt(addr, LRC_MIGRATION_ULLS_SEMAPHORE_CLEAR, + dw, i); +} + +/* + * Advance the ring tail from within the ring, so submitting the next ULLS job + * needs nothing from the CPU beyond signalling the semaphore. All ULLS jobs + * occupy exactly ULLS_JOB_SIZE_BYTES, so the tail the next job ends at is two + * job slots on from where this job started, even though that job has not been + * emitted yet. Both LRC and MMIO ring tail advanced in step. + */ +static int emit_ulls_ring_tail(struct xe_gt *gt, struct xe_lrc *lrc, u32 *dw, + int i, u32 head) +{ + u32 next_tail = (head + 2 * ULLS_JOB_SIZE_BYTES) & (lrc->ring.size - 1); + + xe_gt_assert(gt, IS_ALIGNED(next_tail, 8)); + + i = emit_store_imm_ggtt(xe_lrc_ring_tail_ggtt_addr(lrc), next_tail, + dw, i); + + dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | + MI_LRI_LRM_CS_MMIO; + dw[i++] = RING_TAIL(0).addr; + dw[i++] = next_tail; + + return i; +} + +/* + * Park the engine on the next job's semaphore, then publish its tail. + * + * The tail must not be published before the wait. The command streamer + * fetches ring contents up to the tail while parked, so it would fetch the + * next job's slot while it still holds MI_NOOPs and execute those once + * released, rather than the job the CPU writes there later. Publishing after + * the wait keeps the slot beyond the tail until the job is in place. + */ +static int emit_ulls_postamble(struct xe_gt *gt, struct xe_lrc *lrc, u32 *dw, + int i, u32 seqno, u32 head) +{ + dw[i++] = MI_SEMAPHORE_WAIT | + MI_SEMW_GGTT | + MI_SEMW_POLL | + MI_SEMW_COMPARE(SAD_EQ_SDD); + dw[i++] = LRC_MIGRATION_ULLS_SEMAPHORE_SIGNAL; + dw[i++] = xe_lrc_ulls_semaphore_ggtt_addr(lrc, seqno + 1); + dw[i++] = 0; + dw[i++] = 0; + + return emit_ulls_ring_tail(gt, lrc, dw, i, head); +} + +/* Pad out to the fixed ULLS job size */ +static int emit_ulls_pad(struct xe_gt *gt, u32 *dw, int i) +{ + xe_gt_assert(gt, i <= ULLS_JOB_SIZE_DW); + + while (i < ULLS_JOB_SIZE_DW) + dw[i++] = MI_NOOP; + + return i; +} + static void emit_migration_job_gen12(struct xe_sched_job *job, struct xe_lrc *lrc, u32 *head, u32 seqno) @@ -518,10 +585,16 @@ static void emit_migration_job_gen12(struct xe_sched_job *job, xe_gt_assert(gt, !job->ring_ops_force_reset); + if (xe_sched_job_is_ulls(job)) + i = emit_ulls_preamble(lrc, dw, i, seqno); + i = emit_copy_timestamp(xe, lrc, dw, i); i = emit_store_imm_ggtt(saddr, seqno, dw, i); + if (!xe_sched_job_ulls_has_batch(job)) + goto seqno_write; + dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */ i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i); @@ -532,12 +605,19 @@ static void emit_migration_job_gen12(struct xe_sched_job *job, i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i); +seqno_write: i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, job->migrate_flush_flags, dw, i); i = emit_user_interrupt(dw, i); + if (xe_sched_job_ulls_parks(job)) + i = emit_ulls_postamble(gt, lrc, dw, i, seqno, *head); + + if (xe_sched_job_is_ulls(job)) + i = emit_ulls_pad(gt, dw, i); + xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW); xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); diff --git a/drivers/gpu/drm/xe/xe_ring_ops_types.h b/drivers/gpu/drm/xe/xe_ring_ops_types.h index 52ff96bc4100..ea4af321dd7c 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops_types.h +++ b/drivers/gpu/drm/xe/xe_ring_ops_types.h @@ -6,6 +6,7 @@ #ifndef _XE_RING_OPS_TYPES_H_ #define _XE_RING_OPS_TYPES_H_ +#include <linux/build_bug.h> #include <linux/types.h> struct xe_gt; @@ -14,6 +15,29 @@ struct xe_sched_job; #define MAX_JOB_SIZE_DW 74 #define MAX_JOB_SIZE_BYTES (MAX_JOB_SIZE_DW * 4) +/* + * ULLS migration jobs advance the ring tail from within the ring itself, so a + * job has to know where its successor will end before that successor has been + * emitted. Every ULLS job is therefore padded to a fixed size, letting the + * next tail be derived arithmetically. + * + * Sized for the largest such job, emitted by emit_migration_job_gen12(): + * preamble (4), copy timestamp (8, its size on an SRIOV VF), start seqno + * store (4), arbitration off (1), batch buffer starts (2 * 3), pre-parser + * bracketed flush invalidate (6), seqno flush (4), user interrupt (3) and + * postamble (7 + 5). + */ +#define ULLS_JOB_SIZE_DW 48 +#define ULLS_JOB_SIZE_BYTES (ULLS_JOB_SIZE_DW * 4) + +/* + * RING_TAIL only encodes a qword aligned offset, and xe_lrc_write_ring() + * appends a NOP to anything shorter, either of which would desynchronise the + * ring from the tail a job predicts for its successor. + */ +static_assert(ULLS_JOB_SIZE_BYTES % 8 == 0); +static_assert(ULLS_JOB_SIZE_DW <= MAX_JOB_SIZE_DW); + /** * struct xe_ring_ops - Ring operations */ diff --git a/drivers/gpu/drm/xe/xe_sched_job.c b/drivers/gpu/drm/xe/xe_sched_job.c index a4fa00632a30..59f7215f27b1 100644 --- a/drivers/gpu/drm/xe/xe_sched_job.c +++ b/drivers/gpu/drm/xe/xe_sched_job.c @@ -26,19 +26,22 @@ static struct kmem_cache *xe_sched_job_parallel_slab; int __init xe_sched_job_module_init(void) { + struct xe_sched_job *job; + size_t size; + + size = struct_size(job, ptrs, 1); xe_sched_job_slab = - kmem_cache_create("xe_sched_job", - sizeof(struct xe_sched_job) + - sizeof(struct xe_job_ptrs), 0, + kmem_cache_create("xe_sched_job", size, 0, SLAB_HWCACHE_ALIGN, NULL); if (!xe_sched_job_slab) return -ENOMEM; + size = max_t(size_t, + struct_size(job, ptrs, + XE_HW_ENGINE_MAX_INSTANCE), + struct_size(job, pt_update, 1)); xe_sched_job_parallel_slab = - kmem_cache_create("xe_sched_job_parallel", - sizeof(struct xe_sched_job) + - sizeof(struct xe_job_ptrs) * - XE_HW_ENGINE_MAX_INSTANCE, 0, + kmem_cache_create("xe_sched_job_parallel", size, 0, SLAB_HWCACHE_ALIGN, NULL); if (!xe_sched_job_parallel_slab) { kmem_cache_destroy(xe_sched_job_slab); @@ -70,8 +73,9 @@ static void job_free(struct xe_sched_job *job) struct xe_exec_queue *q = job->q; bool is_migration = xe_sched_job_is_migration(q); - kmem_cache_free(xe_exec_queue_is_parallel(job->q) || is_migration ? - xe_sched_job_parallel_slab : xe_sched_job_slab, job); + kmem_cache_free(job->is_pt_job || xe_exec_queue_is_parallel(job->q) || + is_migration ? xe_sched_job_parallel_slab : + xe_sched_job_slab, job); } static struct xe_device *job_to_xe(struct xe_sched_job *job) @@ -84,6 +88,9 @@ static void xe_sched_job_free_fences(struct xe_sched_job *job) { int i; + if (job->is_pt_job) + return; + for (i = 0; i < job->q->width; ++i) { struct xe_job_ptrs *ptrs = &job->ptrs[i]; @@ -93,10 +100,23 @@ static void xe_sched_job_free_fences(struct xe_sched_job *job) } } +/** + * xe_sched_job_create() - Create a scheduler job + * @q: exec queue to create the scheduler job for + * @batch_addr: array of batch addresses for the job; must match the width of + * @q, or NULL to indicate a PT job that does not require a batch address + * + * Create a scheduler job for submission. + * + * Context: Reclaim + * + * Return: a &xe_sched_job object on success, or an ERR_PTR on failure. + */ struct xe_sched_job *xe_sched_job_create(struct xe_exec_queue *q, u64 *batch_addr) { bool is_migration = xe_sched_job_is_migration(q); + struct xe_device *xe = gt_to_xe(q->gt); struct xe_sched_job *job; int err; int i; @@ -105,10 +125,15 @@ struct xe_sched_job *xe_sched_job_create(struct xe_exec_queue *q, /* only a kernel context can submit a vm-less job */ XE_WARN_ON(!q->vm && !(q->flags & EXEC_QUEUE_FLAG_KERNEL)); - job = job_alloc(xe_exec_queue_is_parallel(q) || is_migration); + xe_assert(xe, batch_addr || + q->flags & (EXEC_QUEUE_FLAG_VM | EXEC_QUEUE_FLAG_MIGRATE)); + + job = job_alloc(!batch_addr || xe_exec_queue_is_parallel(q) || + is_migration); if (!job) return ERR_PTR(-ENOMEM); + job->is_pt_job = !batch_addr; job->q = q; job->sample_timestamp = U64_MAX; kref_init(&job->refcount); @@ -119,34 +144,38 @@ struct xe_sched_job *xe_sched_job_create(struct xe_exec_queue *q, if (err) goto err_free; - for (i = 0; i < q->width; ++i) { - struct dma_fence *fence = xe_lrc_alloc_seqno_fence(); - struct dma_fence_chain *chain; - - if (IS_ERR(fence)) { - err = PTR_ERR(fence); - goto err_sched_job; + if (!batch_addr) { + job->fence = dma_fence_get_stub(); + } else { + for (i = 0; i < q->width; ++i) { + struct dma_fence *fence = xe_lrc_alloc_seqno_fence(); + struct dma_fence_chain *chain; + + if (IS_ERR(fence)) { + err = PTR_ERR(fence); + goto err_sched_job; + } + job->ptrs[i].lrc_fence = fence; + + if (i + 1 == q->width) + continue; + + chain = dma_fence_chain_alloc(); + if (!chain) { + err = -ENOMEM; + goto err_sched_job; + } + job->ptrs[i].chain_fence = chain; } - job->ptrs[i].lrc_fence = fence; - if (i + 1 == q->width) - continue; + width = q->width; + if (is_migration) + width = 2; - chain = dma_fence_chain_alloc(); - if (!chain) { - err = -ENOMEM; - goto err_sched_job; - } - job->ptrs[i].chain_fence = chain; + for (i = 0; i < width; ++i) + job->ptrs[i].batch_addr = batch_addr[i]; } - width = q->width; - if (is_migration) - width = 2; - - for (i = 0; i < width; ++i) - job->ptrs[i].batch_addr = batch_addr[i]; - atomic_inc(&q->job_cnt); xe_pm_runtime_get_noresume(job_to_xe(job)); trace_xe_sched_job_create(job); @@ -246,7 +275,7 @@ bool xe_sched_job_completed(struct xe_sched_job *job) void xe_sched_job_arm(struct xe_sched_job *job) { struct xe_exec_queue *q = job->q; - struct dma_fence *fence, *prev; + struct dma_fence *fence = job->fence, *prev; struct xe_vm *vm = q->vm; u64 seqno = 0; int i; @@ -259,6 +288,9 @@ void xe_sched_job_arm(struct xe_sched_job *job) xe_vm_assert_held(q->vm); } + if (job->is_pt_job) + goto arm; + if (vm && !xe_sched_job_is_migration(q) && !xe_vm_in_lr_mode(vm) && (vm->batch_invalidate_tlb || vm->tlb_flush_seqno != q->tlb_flush_seqno)) { xe_vm_assert_held(vm); @@ -286,6 +318,7 @@ void xe_sched_job_arm(struct xe_sched_job *job) fence = &chain->base; } +arm: job->fence = dma_fence_get(fence); /* Pairs with put in scheduler */ drm_sched_job_arm(&job->drm); } @@ -329,7 +362,7 @@ xe_sched_job_snapshot_capture(struct xe_sched_job *job) snapshot->batch_addr_len = q->width; for (i = 0; i < q->width; i++) - snapshot->batch_addr[i] = + snapshot->batch_addr[i] = job->is_pt_job ? 0 : xe_device_uncanonicalize_addr(xe, job->ptrs[i].batch_addr); return snapshot; diff --git a/drivers/gpu/drm/xe/xe_sched_job.h b/drivers/gpu/drm/xe/xe_sched_job.h index 1c1cb44216c3..bbf5e3864ba3 100644 --- a/drivers/gpu/drm/xe/xe_sched_job.h +++ b/drivers/gpu/drm/xe/xe_sched_job.h @@ -83,6 +83,62 @@ xe_sched_job_add_migrate_flush(struct xe_sched_job *job, u32 flags) job->migrate_flush_flags = flags; } +/** + * xe_sched_job_is_ulls() - Is a ULLS job + * @job: Xe schedule job object + * + * Return: True if @job is submitted as part of a ULLS sequence, False + * otherwise. + */ +static inline bool xe_sched_job_is_ulls(struct xe_sched_job *job) +{ + return job->ulls != ULLS_NONE; +} + +/** + * xe_sched_job_ulls_has_batch() - Does a job carry batch buffers + * @job: Xe schedule job object + * + * The ULLS jobs which enter and exit ULLS mode exist only to move the + * migration context on and off the hardware, and carry no batch buffers. + * + * Return: True if @job carries batch buffers, False otherwise. + */ +static inline bool xe_sched_job_ulls_has_batch(struct xe_sched_job *job) +{ + return job->ulls == ULLS_NONE || job->ulls == ULLS_ACTIVE; +} + +/** + * xe_sched_job_ulls_parks() - Does a job park the engine for its successor + * @job: Xe schedule job object + * + * A ULLS job which is not the last one emits a postamble, parking the engine + * on its successor's semaphore and publishing that successor's ring tail. + * + * Return: True if @job emits a ULLS postamble, False otherwise. + */ +static inline bool xe_sched_job_ulls_parks(struct xe_sched_job *job) +{ + return job->ulls == ULLS_ENTER || job->ulls == ULLS_ACTIVE; +} + +/** + * xe_sched_job_ulls_is_chained() - Has a job's predecessor already published it + * @job: Xe schedule job object + * + * A ULLS job which is not the first one has had its ring tail published by its + * predecessor's postamble, which also left the engine parked on this job's + * semaphore. Submitting it is a semaphore write alone - no H2G and no ring + * tail write. + * + * Return: True if @job was published by its predecessor, False otherwise. + */ +static inline bool xe_sched_job_ulls_is_chained(struct xe_sched_job *job) +{ + return job->ulls == ULLS_ACTIVE || job->ulls == ULLS_EXIT; +} + bool xe_sched_job_is_migration(struct xe_exec_queue *q); struct xe_sched_job_snapshot *xe_sched_job_snapshot_capture(struct xe_sched_job *job); diff --git a/drivers/gpu/drm/xe/xe_sched_job_types.h b/drivers/gpu/drm/xe/xe_sched_job_types.h index 0490b1247a6e..c8cdbe68843f 100644 --- a/drivers/gpu/drm/xe/xe_sched_job_types.h +++ b/drivers/gpu/drm/xe/xe_sched_job_types.h @@ -10,10 +10,28 @@ #include <drm/gpu_scheduler.h> -struct xe_exec_queue; struct dma_fence; struct dma_fence_chain; +struct xe_exec_queue; +struct xe_cpu_bind_pt_update_ops; +struct xe_pt_job_ops; +struct xe_tile; +struct xe_vm; + +/** + * struct xe_pt_update_args - PT update arguments + */ +struct xe_pt_update_args { + /** @vm: VM which is being bound */ + struct xe_vm *vm; + /** @ops: CPU bind PT update ops */ + const struct xe_cpu_bind_pt_update_ops *ops; +#define XE_PT_UPDATE_JOB_OPS_COUNT 2 + /** @pt_job_ops: PT job ops state */ + struct xe_pt_job_ops *pt_job_ops[XE_PT_UPDATE_JOB_OPS_COUNT]; +}; + /** * struct xe_job_ptrs - Per hw engine instance data */ @@ -32,6 +50,23 @@ struct xe_job_ptrs { }; /** + * enum xe_ulls_state - ULLS state of a migration job + * + * Describes where a job sits in a ULLS (Ultra Low Latency Submission) + * sequence. See the ULLS documentation in xe_migrate.c. + */ +enum xe_ulls_state { + /** @ULLS_NONE: Not a ULLS job */ + ULLS_NONE = 0, + /** @ULLS_ENTER: Job which enters ULLS mode */ + ULLS_ENTER, + /** @ULLS_ACTIVE: Job submitted while in ULLS mode */ + ULLS_ACTIVE, + /** @ULLS_EXIT: Job which exits ULLS mode */ + ULLS_EXIT, +}; + +/** * struct xe_sched_job - Xe schedule job (batch buffer tracking) */ struct xe_sched_job { @@ -61,6 +96,8 @@ struct xe_sched_job { u32 migrate_flush_flags; /** @sample_timestamp: Sampling of job timestamp in TDR */ u64 sample_timestamp; + /** @ulls: ULLS state of this job */ + enum xe_ulls_state ulls; /** @ring_ops_flush_tlb: The ring ops need to flush TLB before payload. */ bool ring_ops_flush_tlb; /** @ring_ops_force_reset: The ring ops need to trigger a reset before payload. */ @@ -71,8 +108,14 @@ struct xe_sched_job { bool restore_replay; /** @last_replay: last job being replayed */ bool last_replay; - /** @ptrs: per instance pointers. */ - struct xe_job_ptrs ptrs[]; + /** @is_pt_job: is a PT job */ + bool is_pt_job; + union { + /** @ptrs: per instance pointers. */ + DECLARE_FLEX_ARRAY(struct xe_job_ptrs, ptrs); + /** @pt_update: PT update arguments */ + DECLARE_FLEX_ARRAY(struct xe_pt_update_args, pt_update); + }; }; struct xe_sched_job_snapshot { diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_provision.c b/drivers/gpu/drm/xe/xe_sriov_pf_provision.c index 0ec7ea83f12a..c37e422b5a5c 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_provision.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf_provision.c @@ -112,7 +112,7 @@ static int pf_reprovision_default(struct xe_device *xe) int result = 0; int err; - guard(mutex)(xe_sriov_pf_master_mutex(xe)); + lockdep_assert_held(xe_sriov_pf_master_mutex(xe)); for_each_gt(gt, xe, id) { err = xe_gt_sriov_pf_policy_set_sched_if_idle_locked(gt, false); @@ -138,6 +138,8 @@ int xe_sriov_pf_reprovision_default(struct xe_device *xe) { xe_assert(xe, IS_SRIOV_PF(xe)); + guard(mutex)(xe_sriov_pf_master_mutex(xe)); + if (!pf_auto_provisioning_mode(xe)) return 0; diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index f39e647512ad..909a602d091e 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -9,6 +9,7 @@ #include <drm/drm_managed.h> #include <drm/drm_pagemap.h> #include <drm/drm_pagemap_util.h> +#include <drm/ttm/ttm_bo.h> #include "xe_bo.h" #include "xe_exec_queue_types.h" @@ -1626,6 +1627,73 @@ int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range, } /** + * xe_svm_devmem_lru_bump() - Move a range's backing BO to the TTM LRU tail + * @devmem_allocation: The device-memory allocation backing the range's pages + * + * Intended as a &drm_gpusvm_ctx.devmem_fn, so drm_gpusvm_get_pages() calls it + * under the GPUSVM notifier lock. That lock sits in the reclaim path, so + * dma-resv cannot be acquired blocking beneath it; dma-resv is also the outer + * lock of the notifier when memory is allocated and migrated, so a blocking + * acquire here would invert that order. Hence the trylock. The BO is private + * (dedicated dma-resv), so the trylock succeeds unless the BO is being moved + * elsewhere (e.g. eviction); a contended BO is simply skipped. + * + * A shared device_private_page_owner can hand us allocations backed by a + * foreign driver; skip anything that is not an Xe BO before casting. + */ +void xe_svm_devmem_lru_bump(struct drm_pagemap_devmem *devmem_allocation) +{ +#if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) + struct xe_bo *bo; + + if (devmem_allocation->ops != &dpagemap_devmem_ops) + return; + + bo = to_xe_bo(devmem_allocation); + + if (!dma_resv_trylock(bo->ttm.base.resv)) + return; + + ttm_bo_move_to_lru_tail_unlocked(&bo->ttm); + dma_resv_unlock(bo->ttm.base.resv); +#endif +} + +/** + * xe_svm_range_prefetch_lru_bump() - Bump the TTM LRU for an already-valid range + * @vm: Pointer to the struct xe_vm + * @vma: Pointer to the VMA covering the range + * @range: Pointer to the xe SVM range structure + * @dpagemap: Target pagemap of the prefetch, or %NULL for system memory + * + * A valid range skips migration and binding, so nothing else refreshes its + * backing BOs on the LRU. Re-fault the CPU pages without touching the DMA + * mappings (@no_dma_map) and move each backing BO to the LRU tail. + * + * Prefetching is a best-effort optimization, so any failure is ignored; a + * later fault will refault the range. + */ +void xe_svm_range_prefetch_lru_bump(struct xe_vm *vm, struct xe_vma *vma, + struct xe_svm_range *range, + struct drm_pagemap *dpagemap) +{ + struct drm_gpusvm_ctx ctx = { + .read_only = xe_vma_read_only(vma), + .device_private_page_owner = + xe_svm_private_page_owner(vm, !dpagemap), + .no_dma_map = 1, + .devmem_fn = xe_svm_devmem_lru_bump, + }; + + guard(mutex)(&range->lock); + + if (xe_svm_range_is_removed(range)) + return; + + xe_svm_range_get_pages(vm, range, &ctx); +} + +/** * xe_svm_ranges_zap_ptes_in_range - clear ptes of svm ranges in input range * @vm: Pointer to the xe_vm structure * @start: Start of the input range diff --git a/drivers/gpu/drm/xe/xe_svm.h b/drivers/gpu/drm/xe/xe_svm.h index 2ef4ef026ccd..ffc1cc808102 100644 --- a/drivers/gpu/drm/xe/xe_svm.h +++ b/drivers/gpu/drm/xe/xe_svm.h @@ -128,6 +128,12 @@ struct xe_svm_range *xe_svm_range_find_or_insert(struct xe_vm *vm, u64 addr, int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range, struct drm_gpusvm_ctx *ctx); +void xe_svm_devmem_lru_bump(struct drm_pagemap_devmem *devmem_allocation); + +void xe_svm_range_prefetch_lru_bump(struct xe_vm *vm, struct xe_vma *vma, + struct xe_svm_range *range, + struct drm_pagemap *dpagemap); + bool xe_svm_range_needs_migrate_to_vram(struct xe_svm_range *range, struct xe_vma *vma, const struct drm_pagemap *dpagemap); @@ -356,6 +362,18 @@ int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range, return -EINVAL; } +static inline +void xe_svm_devmem_lru_bump(struct drm_pagemap_devmem *devmem_allocation) +{ +} + +static inline +void xe_svm_range_prefetch_lru_bump(struct xe_vm *vm, struct xe_vma *vma, + struct xe_svm_range *range, + struct drm_pagemap *dpagemap) +{ +} + static inline struct xe_svm_range *to_xe_range(struct drm_gpusvm_range *r) { return NULL; diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c index 37866768d64c..06b1c913588a 100644 --- a/drivers/gpu/drm/xe/xe_sync.c +++ b/drivers/gpu/drm/xe/xe_sync.c @@ -345,15 +345,9 @@ xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, return ERR_PTR(-EOPNOTSUPP); if (q->flags & EXEC_QUEUE_FLAG_VM) { - struct xe_exec_queue *__q; - struct xe_tile *tile; - u8 id; - - for_each_tile(tile, vm->xe, id) { + num_fence++; + for_each_tlb_inval(q, i) num_fence++; - for_each_tlb_inval(i) - num_fence++; - } fences = kmalloc_objs(*fences, num_fence); if (!fences) @@ -361,17 +355,9 @@ xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, fences[current_fence++] = xe_exec_queue_last_fence_get(q, vm); - for_each_tlb_inval(i) + for_each_tlb_inval(q, i) fences[current_fence++] = xe_exec_queue_tlb_inval_last_fence_get(q, vm, i); - list_for_each_entry(__q, &q->multi_gt_list, - multi_gt_link) { - fences[current_fence++] = - xe_exec_queue_last_fence_get(__q, vm); - for_each_tlb_inval(i) - fences[current_fence++] = - xe_exec_queue_tlb_inval_last_fence_get(__q, vm, i); - } xe_assert(vm->xe, current_fence == num_fence); cf = dma_fence_array_create(num_fence, fences, diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c index 72baf1aa4b3a..e21848ca7594 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c @@ -334,6 +334,9 @@ void xe_sysctrl_create_command(struct xe_sysctrl_mailbox_command *command, u8 gr * If a response is expected, @cmd->data_out must point to a buffer of * size @cmd->data_out_len supplied by caller. * + * @cmd->timeout_ms specifies the response timeout in milliseconds. + * A value of 0 uses %XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS. + * * On success, @rdata_len is updated with number of valid response bytes * returned by firmware, bounded by @cmd->data_out_len. * @@ -347,6 +350,7 @@ int xe_sysctrl_send_command(struct xe_sysctrl *sc, u8 group_id, command_code; u8 *mbox_cmd = NULL; size_t cmd_size = 0; + unsigned int timeout_ms; int ret; guard(xe_pm_runtime_noresume)(xe); @@ -360,6 +364,7 @@ int xe_sysctrl_send_command(struct xe_sysctrl *sc, group_id = XE_SYSCTRL_APP_HDR_GROUP_ID(&cmd->header); command_code = XE_SYSCTRL_APP_HDR_COMMAND(&cmd->header); + timeout_ms = cmd->timeout_ms ?: XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS; might_sleep(); @@ -375,7 +380,7 @@ int xe_sysctrl_send_command(struct xe_sysctrl *sc, ret = sysctrl_send_command(sc, mbox_cmd, cmd_size, cmd->data_out, cmd->data_out_len, rdata_len, - XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS); + timeout_ms); if (ret) xe_err(xe, "sysctrl: Mailbox command failed: %pe\n", ERR_PTR(ret)); diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h index c236e5377f30..e98fa6877fec 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h @@ -105,6 +105,9 @@ struct xe_sysctrl_mailbox_command { /** @data_out_len: Size of output buffer in bytes (0 if no response expected) */ size_t data_out_len; + + /** @timeout_ms: Response timeout in ms, or 0 for %XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS */ + unsigned int timeout_ms; }; /* Modify as needed */ diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c index dd1030de9ab6..c9cc5842d65b 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval.c +++ b/drivers/gpu/drm/xe/xe_tlb_inval.c @@ -5,6 +5,7 @@ #include <drm/drm_managed.h> +#include "xe_devcoredump.h" #include "xe_device_types.h" #include "xe_force_wake.h" #include "xe_gt_stats.h" @@ -13,6 +14,7 @@ #include "xe_guc_tlb_inval.h" #include "xe_mmio.h" #include "xe_pm.h" +#include "xe_printk.h" #include "xe_tlb_inval.h" #include "xe_trace.h" @@ -29,6 +31,12 @@ #define FENCE_STACK_BIT DMA_FENCE_FLAG_USER_BITS +/* The frontend is only ever embedded in a GT */ +static struct xe_gt *tlb_inval_to_gt(struct xe_tlb_inval *tlb_inval) +{ + return container_of(tlb_inval, struct xe_gt, tlb_inval); +} + static void xe_tlb_inval_fence_fini(struct xe_tlb_inval_fence *fence) { if (WARN_ON_ONCE(!fence->tlb_inval)) @@ -73,6 +81,7 @@ static void xe_tlb_inval_fence_timeout(struct work_struct *work) struct xe_device *xe = tlb_inval->xe; struct xe_tlb_inval_fence *fence, *next; long timeout_delay = tlb_inval->ops->timeout_delay(tlb_inval); + int timedout_seqno = 0, seqno_recv = 0; tlb_inval->ops->flush(tlb_inval); @@ -90,13 +99,45 @@ static void xe_tlb_inval_fence_timeout(struct work_struct *work) "TLB invalidation fence timeout, seqno=%d recv=%d", fence->seqno, tlb_inval->seqno_recv); + if (!timedout_seqno) { + /* + * Hold a PM reference across the capture below. Every + * pending fence holds one, so the device is awake + * here, but signalling them may drop the last + * reference and let it autosuspend before the + * snapshot touches the hardware. + */ + xe_pm_runtime_get_noresume(xe); + } + + timedout_seqno = fence->seqno; + if (!tlb_inval->timedout_seqno) { + tlb_inval->timedout_seqno = fence->seqno; + tlb_inval->timedout_inval_time = fence->inval_time; + tlb_inval->timedout_time = ktime_get(); + } + fence->base.error = -ETIME; xe_tlb_inval_fence_signal(fence); } if (!list_empty(&tlb_inval->pending_fences)) queue_delayed_work(tlb_inval->timeout_wq, &tlb_inval->fence_tdr, timeout_delay); + seqno_recv = tlb_inval->seqno_recv; spin_unlock_irq(&tlb_inval->pending_lock); + + /* + * Capture the GuC log and CT state so the firmware side of the hang + * can be inspected; there is no queue or job to blame here. Must be + * outside pending_lock as the capture takes sleeping locks, hence + * @seqno_recv is sampled above while the lock is still held. + */ + if (timedout_seqno) { + xe_devcoredump_gt(tlb_inval_to_gt(tlb_inval), + "TLB invalidation fence timeout, seqno=%d recv=%d", + timedout_seqno, seqno_recv); + xe_pm_runtime_put(xe); + } } /** @@ -207,6 +248,7 @@ void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval) else pending_seqno = tlb_inval->seqno - 1; WRITE_ONCE(tlb_inval->seqno_recv, pending_seqno); + tlb_inval->timedout_seqno = 0; list_for_each_entry_safe(fence, next, &tlb_inval->pending_fences, link) @@ -435,6 +477,18 @@ void xe_tlb_inval_done_handler(struct xe_tlb_inval *tlb_inval, int seqno) WRITE_ONCE(tlb_inval->seqno_recv, seqno); + if (tlb_inval->timedout_seqno && + xe_tlb_inval_seqno_past(tlb_inval, tlb_inval->timedout_seqno)) { + ktime_t now = ktime_get(); + + xe_warn(xe, + "TLB invalidation late ack: seqno=%d recv=%d, request-to-ack=%lldms, timeout-to-ack=%lldms", + tlb_inval->timedout_seqno, seqno, + ktime_ms_delta(now, tlb_inval->timedout_inval_time), + ktime_ms_delta(now, tlb_inval->timedout_time)); + tlb_inval->timedout_seqno = 0; + } + list_for_each_entry_safe(fence, next, &tlb_inval->pending_fences, link) { trace_xe_tlb_inval_fence_recv(xe, fence); diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_job.c b/drivers/gpu/drm/xe/xe_tlb_inval_job.c index 04d21015cd5d..7378cfe6e855 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval_job.c +++ b/drivers/gpu/drm/xe/xe_tlb_inval_job.c @@ -4,6 +4,7 @@ */ #include "xe_assert.h" +#include "xe_cpu_bind.h" #include "xe_dep_job_types.h" #include "xe_dep_scheduler.h" #include "xe_exec_queue.h" @@ -12,7 +13,6 @@ #include "xe_page_reclaim.h" #include "xe_tlb_inval.h" #include "xe_tlb_inval_job.h" -#include "xe_migrate.h" #include "xe_pm.h" #include "xe_vm.h" @@ -39,8 +39,8 @@ struct xe_tlb_inval_job { u64 start; /** @end: End address to invalidate */ u64 end; - /** @type: GT type */ - int type; + /** @idx: Index of tlb invalidation */ + int idx; /** @fence_armed: Fence has been armed */ bool fence_armed; }; @@ -87,7 +87,7 @@ static const struct xe_dep_job_ops dep_job_ops = { * @vm: VM which TLB invalidation is being issued for * @start: Start address to invalidate * @end: End address to invalidate - * @type: GT type + * @idx: Index of tlb invalidation * * Create a TLB invalidation job and initialize internal fields. The caller is * responsible for releasing the creation reference. @@ -97,7 +97,7 @@ static const struct xe_dep_job_ops dep_job_ops = { struct xe_tlb_inval_job * xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval, struct xe_dep_scheduler *dep_scheduler, - struct xe_vm *vm, u64 start, u64 end, int type) + struct xe_vm *vm, u64 start, u64 end, int idx) { struct xe_tlb_inval_job *job; struct drm_sched_entity *entity = @@ -105,8 +105,7 @@ xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval, struct xe_tlb_inval_fence *ifence; int err; - xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || - type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT); + xe_assert(vm->xe, idx < XE_EXEC_QUEUE_TLB_INVAL_COUNT); job = kmalloc_obj(*job); if (!job) @@ -120,7 +119,7 @@ xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval, job->fence_armed = false; xe_page_reclaim_list_init(&job->prl); job->dep.ops = &dep_job_ops; - job->type = type; + job->idx = idx; kref_init(&job->refcount); xe_exec_queue_get(q); /* Pairs with put in xe_tlb_inval_job_destroy */ xe_vm_get(vm); /* Pairs with put in xe_tlb_inval_job_destroy */ @@ -219,7 +218,6 @@ int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job) /** * xe_tlb_inval_job_push() - TLB invalidation job push * @job: TLB invalidation job to push - * @m: The migration object being used * @fence: Dependency for TLB invalidation job * * Pushes a TLB invalidation job for execution, using @fence as a dependency. @@ -231,11 +229,11 @@ int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job) * Return: Job's finished fence on success, cannot fail */ struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job, - struct xe_migrate *m, struct dma_fence *fence) { struct xe_tlb_inval_fence *ifence = container_of(job->fence, typeof(*ifence), base); + struct xe_cpu_bind *cpu_bind = gt_to_xe(job->q->gt)->cpu_bind; if (!dma_fence_is_signaled(fence)) { void *ptr; @@ -259,11 +257,11 @@ struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job, job->fence_armed = true; /* - * We need the migration lock to protect the job's seqno and the spsc - * queue, only taken on migration queue, user queues protected dma-resv + * We need the cpu_bind lock to protect the job's seqno and the spsc + * queue, only taken on cpu_bind queue, user queues protected dma-resv * VM lock. */ - xe_migrate_job_lock(m, job->q); + xe_cpu_bind_job_lock(cpu_bind, job->q); /* Creation ref pairs with put in xe_tlb_inval_job_destroy */ xe_tlb_inval_fence_init(job->tlb_inval, ifence, false); @@ -280,9 +278,9 @@ struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job, /* Let the upper layers fish this out */ xe_exec_queue_tlb_inval_last_fence_set(job->q, job->vm, &job->dep.drm.s_fence->finished, - job->type); + job->idx); - xe_migrate_job_unlock(m, job->q); + xe_cpu_bind_job_unlock(cpu_bind, job->q); /* * Not using job->fence, as it has its own dma-fence context, which does diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_job.h b/drivers/gpu/drm/xe/xe_tlb_inval_job.h index 03d6e21cd611..97e032ea21c3 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval_job.h +++ b/drivers/gpu/drm/xe/xe_tlb_inval_job.h @@ -11,7 +11,6 @@ struct dma_fence; struct xe_dep_scheduler; struct xe_exec_queue; -struct xe_migrate; struct xe_page_reclaim_list; struct xe_tlb_inval; struct xe_tlb_inval_job; @@ -20,7 +19,7 @@ struct xe_vm; struct xe_tlb_inval_job * xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval, struct xe_dep_scheduler *dep_scheduler, - struct xe_vm *vm, u64 start, u64 end, int type); + struct xe_vm *vm, u64 start, u64 end, int idx); void xe_tlb_inval_job_add_page_reclaim(struct xe_tlb_inval_job *job, struct xe_page_reclaim_list *prl); @@ -28,7 +27,6 @@ void xe_tlb_inval_job_add_page_reclaim(struct xe_tlb_inval_job *job, int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job); struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job, - struct xe_migrate *m, struct dma_fence *fence); void xe_tlb_inval_job_get(struct xe_tlb_inval_job *job); diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_types.h b/drivers/gpu/drm/xe/xe_tlb_inval_types.h index d77be1aedc91..80d2019fa20b 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval_types.h +++ b/drivers/gpu/drm/xe/xe_tlb_inval_types.h @@ -113,6 +113,23 @@ struct xe_tlb_inval { */ spinlock_t pending_lock; /** + * @timedout_seqno: seqno of the most recent timed out TLB + * invalidation, 0 if none. Used to measure how late the ack for a + * timed out invalidation actually arrives. Protected by + * @pending_lock. + */ + int timedout_seqno; + /** + * @timedout_inval_time: request time of @timedout_seqno. Protected by + * @pending_lock. + */ + ktime_t timedout_inval_time; + /** + * @timedout_time: time @timedout_seqno was signaled with -ETIME. + * Protected by @pending_lock. + */ + ktime_t timedout_time; + /** * @fence_tdr: schedules a delayed call to xe_tlb_fence_timeout after * the timeout interval is over. */ diff --git a/drivers/gpu/drm/xe/xe_trace.h b/drivers/gpu/drm/xe/xe_trace.h index 2fe8f89a1e34..d4e9d91f6f7f 100644 --- a/drivers/gpu/drm/xe/xe_trace.h +++ b/drivers/gpu/drm/xe/xe_trace.h @@ -261,7 +261,7 @@ DECLARE_EVENT_CLASS(xe_sched_job, __entry->flags = job->q->flags; __entry->error = job->fence ? job->fence->error : 0; __entry->fence = job->fence; - __entry->batch_addr = (u64)job->ptrs[0].batch_addr; + __entry->batch_addr = job->is_pt_job ? 0 : (u64)job->ptrs[0].batch_addr; ), TP_printk("dev=%s, fence=%p, seqno=%u, lrc_seqno=%u, gt=%u, guc_id=%d, batch_addr=0x%012llx, guc_state=0x%x, flags=0x%x, error=%d", diff --git a/drivers/gpu/drm/xe/xe_trace_bo.h b/drivers/gpu/drm/xe/xe_trace_bo.h index 86323cf3be2c..36fc7eb65882 100644 --- a/drivers/gpu/drm/xe/xe_trace_bo.h +++ b/drivers/gpu/drm/xe/xe_trace_bo.h @@ -16,10 +16,15 @@ #include "xe_bo_types.h" #include "xe_vm.h" +#define __dev_name_xe(xe) dev_name((xe)->drm.dev) #define __dev_name_bo(bo) dev_name(xe_bo_device(bo)->drm.dev) #define __dev_name_vm(vm) dev_name((vm)->xe->drm.dev) #define __dev_name_vma(vma) __dev_name_vm(xe_vma_vm(vma)) +#define XE_VM_TRACE_MODE_DMA_FENCE 0 +#define XE_VM_TRACE_MODE_PREEMPT_FENCE 1 +#define XE_VM_TRACE_MODE_FAULT 2 + DECLARE_EVENT_CLASS(xe_bo, TP_PROTO(struct xe_bo *bo), TP_ARGS(bo), @@ -194,6 +199,7 @@ DECLARE_EVENT_CLASS(xe_vm, __field(struct xe_vm *, vm) __field(u32, asid) __field(u32, flags) + __field(u8, mode) ), TP_fast_assign( @@ -201,11 +207,20 @@ DECLARE_EVENT_CLASS(xe_vm, __entry->vm = vm; __entry->asid = vm->usm.asid; __entry->flags = vm->flags; + __entry->mode = xe_vm_in_fault_mode(vm) ? + XE_VM_TRACE_MODE_FAULT : + xe_vm_in_preempt_fence_mode(vm) ? + XE_VM_TRACE_MODE_PREEMPT_FENCE : + XE_VM_TRACE_MODE_DMA_FENCE; ), - TP_printk("dev=%s, vm=%p, asid=0x%05x, vm flags=0x%05x", + TP_printk("dev=%s, vm=%p, asid=0x%05x, vm flags=0x%05x, mode=%s", __get_str(dev), __entry->vm, __entry->asid, - __entry->flags) + __entry->flags, + __print_symbolic(__entry->mode, + { XE_VM_TRACE_MODE_DMA_FENCE, "dma-fence" }, + { XE_VM_TRACE_MODE_PREEMPT_FENCE, "preempt-fence" }, + { XE_VM_TRACE_MODE_FAULT, "fault" })) ); DEFINE_EVENT(xe_vm, xe_vm_kill, @@ -223,6 +238,16 @@ DEFINE_EVENT(xe_vm, xe_vm_free, TP_ARGS(vm) ); +DEFINE_EVENT(xe_vm, xe_vm_close_and_put, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +DEFINE_EVENT(xe_vm, xe_vm_asid_release, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + DEFINE_EVENT(xe_vm, xe_vm_cpu_bind, TP_PROTO(struct xe_vm *vm), TP_ARGS(vm) @@ -253,6 +278,35 @@ DEFINE_EVENT(xe_vm, xe_vm_ops_fail, TP_ARGS(vm) ); +TRACE_EVENT(xe_pagefault_fail, + TP_PROTO(struct xe_device *xe, struct xe_vm *vm, struct xe_vma *vma, + u32 asid, u64 page_addr, u8 reason, int err), + TP_ARGS(xe, vm, vma, asid, page_addr, reason, err), + + TP_STRUCT__entry(__string(dev, __dev_name_xe(xe)) + __field(struct xe_vm *, vm) + __field(struct xe_vma *, vma) + __field(u32, asid) + __field(u64, page_addr) + __field(u8, reason) + __field(int, err) + ), + + TP_fast_assign(__assign_str(dev); + __entry->vm = vm; + __entry->vma = vma; + __entry->asid = asid; + __entry->page_addr = page_addr; + __entry->reason = reason; + __entry->err = err; + ), + + TP_printk("dev=%s, vm=%p, vma=%p, asid=0x%05x, page_addr=0x%012llx, reason=%u, err=%d", + __get_str(dev), __entry->vm, __entry->vma, + __entry->asid, __entry->page_addr, __entry->reason, + __entry->err) +); + #endif /* This part must be outside protection */ diff --git a/drivers/gpu/drm/xe/xe_tuning.c b/drivers/gpu/drm/xe/xe_tuning.c index bcec40ca2d35..a9bd3f8969a5 100644 --- a/drivers/gpu/drm/xe/xe_tuning.c +++ b/drivers/gpu/drm/xe/xe_tuning.c @@ -100,6 +100,10 @@ VISIBLE_IF_KUNIT const struct xe_rtp_table_sr gt_tunings = XE_RTP_TABLE_SR( XE_RTP_ACTIONS(FIELD_SET(GAMSTLB_CTRL, BANK_HASH_MODE, BANK_HASH_4KB_MODE)) }, + { XE_RTP_NAME("Tuning: Force All Non-Compressible Memory Reads to be 256B Overfetches"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3510, XE_RTP_END_VERSION_UNDEFINED)), + XE_RTP_ACTIONS(SET(L3SQCREG2, NONCOMPMEMRD256BOVRFETCHEN)) + }, ); EXPORT_SYMBOL_IF_KUNIT(gt_tunings); diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c index a8e6f18cc9b4..f61910431cf2 100644 --- a/drivers/gpu/drm/xe/xe_uc_fw.c +++ b/drivers/gpu/drm/xe/xe_uc_fw.c @@ -141,6 +141,7 @@ struct fw_blobs_by_type { /* for the GSC FW we match the compatibility version and not the release one */ #define XE_GSC_FIRMWARE_DEFS(fw_def, major_ver) \ + fw_def(NOVALAKE_S, GT_TYPE_ANY, major_ver(xe, gsc, nvl, 108, 0, 0)) \ fw_def(PANTHERLAKE, GT_TYPE_ANY, major_ver(xe, gsc, ptl, 105, 1, 0)) \ fw_def(LUNARLAKE, GT_TYPE_ANY, major_ver(xe, gsc, lnl, 104, 1, 0)) \ fw_def(METEORLAKE, GT_TYPE_ANY, major_ver(i915, gsc, mtl, 102, 1, 0)) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 17dc4debe7c1..2422bfde9177 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -24,6 +24,7 @@ #include "regs/xe_gtt_defs.h" #include "xe_assert.h" #include "xe_bo.h" +#include "xe_cpu_bind.h" #include "xe_device.h" #include "xe_drm_client.h" #include "xe_exec_queue.h" @@ -148,7 +149,7 @@ static bool xe_vm_is_idle(struct xe_vm *vm) xe_vm_assert_held(vm); list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { - if (!xe_exec_queue_is_idle(q)) + if (!xe_exec_queue_is_idle(q, 0)) return false; } @@ -682,11 +683,9 @@ static int xe_vma_ops_alloc(struct xe_vma_ops *vops, bool array_of_binds) if (!vops->pt_update_ops[i].num_ops) continue; - vops->pt_update_ops[i].ops = - kmalloc_objs(*vops->pt_update_ops[i].ops, - vops->pt_update_ops[i].num_ops, - GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); - if (!vops->pt_update_ops[i].ops) + vops->pt_update_ops[i].pt_job_ops = + xe_pt_job_ops_alloc(vops->pt_update_ops[i].num_ops); + if (!vops->pt_update_ops[i].pt_job_ops) return array_of_binds ? -ENOBUFS : -ENOMEM; } @@ -733,7 +732,7 @@ static void xe_vma_ops_fini(struct xe_vma_ops *vops) xe_vma_svm_prefetch_ops_fini(vops); for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) - kfree(vops->pt_update_ops[i].ops); + xe_pt_job_ops_put(vops->pt_update_ops[i].pt_job_ops); } static void xe_vma_ops_incr_pt_update_ops(struct xe_vma_ops *vops, u8 tile_mask, int inc_val) @@ -798,16 +797,16 @@ int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker) struct xe_vma *vma, *next; struct xe_vma_ops vops; struct xe_vma_op *op, *next_op; - int err, i; + int err; lockdep_assert_held(&vm->lock); if ((xe_vm_in_lr_mode(vm) && !rebind_worker) || list_empty(&vm->rebind_list)) return 0; - xe_vma_ops_init(&vops, vm, NULL, NULL, 0); - for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) - vops.pt_update_ops[i].wait_vm_bookkeep = true; + xe_vma_ops_init(&vops, vm, xe_cpu_bind_queue(vm->xe->cpu_bind), + NULL, 0); + vops.flags |= XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP; xe_vm_assert_held(vm); list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) { @@ -847,28 +846,32 @@ free_ops: return err; } +static u8 adjust_rebind_tile_mask(struct xe_vm *vm, u8 tile_mask) +{ + if (vm->xe->info.has_pt_mirror) + return tile_mask; + + return (0x1 << vm->xe->info.tile_count) - 1; +} + struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma, u8 tile_mask) { struct dma_fence *fence = NULL; struct xe_vma_ops vops; struct xe_vma_op *op, *next_op; - struct xe_tile *tile; - u8 id; int err; lockdep_assert_held(&vm->lock); xe_vm_assert_held(vm); xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); - xe_vma_ops_init(&vops, vm, NULL, NULL, 0); - vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT; - for_each_tile(tile, vm->xe, id) { - vops.pt_update_ops[id].wait_vm_bookkeep = true; - vops.pt_update_ops[tile->id].q = - xe_migrate_exec_queue(tile->migrate); - } + xe_vma_ops_init(&vops, vm, xe_cpu_bind_queue(vm->xe->cpu_bind), + NULL, 0); + vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT | + XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP; - err = xe_vm_ops_add_rebind(&vops, vma, tile_mask); + err = xe_vm_ops_add_rebind(&vops, vma, + adjust_rebind_tile_mask(vm, tile_mask)); if (err) return ERR_PTR(err); @@ -942,8 +945,6 @@ struct dma_fence *xe_vm_range_rebind(struct xe_vm *vm, struct dma_fence *fence = NULL; struct xe_vma_ops vops; struct xe_vma_op *op, *next_op; - struct xe_tile *tile; - u8 id; int err; lockdep_assert_held(&range->lock); @@ -952,15 +953,13 @@ struct dma_fence *xe_vm_range_rebind(struct xe_vm *vm, xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(vma)); - xe_vma_ops_init(&vops, vm, NULL, NULL, 0); - vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT; - for_each_tile(tile, vm->xe, id) { - vops.pt_update_ops[id].wait_vm_bookkeep = true; - vops.pt_update_ops[tile->id].q = - xe_migrate_exec_queue(tile->migrate); - } + xe_vma_ops_init(&vops, vm, xe_cpu_bind_queue(vm->xe->cpu_bind), + NULL, 0); + vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT | + XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP; - err = xe_vm_ops_add_range_rebind(&vops, vma, range, tile_mask); + err = xe_vm_ops_add_range_rebind(&vops, vma, range, + adjust_rebind_tile_mask(vm, tile_mask)); if (err) return ERR_PTR(err); @@ -1025,8 +1024,6 @@ struct dma_fence *xe_vm_range_unbind(struct xe_vm *vm, struct dma_fence *fence = NULL; struct xe_vma_ops vops; struct xe_vma_op *op, *next_op; - struct xe_tile *tile; - u8 id; int err; lockdep_assert_held(&range->lock); @@ -1037,12 +1034,9 @@ struct dma_fence *xe_vm_range_unbind(struct xe_vm *vm, if (!range->tile_present) return dma_fence_get_stub(); - xe_vma_ops_init(&vops, vm, NULL, NULL, 0); - for_each_tile(tile, vm->xe, id) { - vops.pt_update_ops[id].wait_vm_bookkeep = true; - vops.pt_update_ops[tile->id].q = - xe_migrate_exec_queue(tile->migrate); - } + xe_vma_ops_init(&vops, vm, xe_cpu_bind_queue(vm->xe->cpu_bind), + NULL, 0); + vops.flags |= XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP; err = xe_vm_ops_add_range_unbind(&vops, range); if (err) @@ -1713,9 +1707,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef) init_rwsem(&vm->exec_queues.lock); xe_vm_init_prove_locking(xe, vm); - - for_each_tile(tile, xe, id) - xe_range_fence_tree_init(&vm->rftree[id]); + xe_range_fence_tree_init(&vm->rftree); vm->pt_ops = &xelp_pt_ops; @@ -1757,7 +1749,8 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef) for_each_tile(tile, xe, id) { if (flags & XE_VM_FLAG_MIGRATION && - tile->id != XE_VM_FLAG_TILE_ID(flags)) + tile->id != XE_VM_FLAG_TILE_ID(flags) && + (vm->xe->info.has_pt_mirror || id)) continue; vm->pt_root[id] = xe_pt_create(vm, tile, xe->info.vm_max_level, @@ -1813,7 +1806,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef) struct xe_exec_queue *q; u32 create_flags = EXEC_QUEUE_FLAG_VM; - if (!vm->pt_root[id]) + if (!vm->pt_root[id] || vm->q) continue; if (!xef) /* Not from userspace */ @@ -1824,7 +1817,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef) err = PTR_ERR(q); goto err_close; } - vm->q[id] = q; + vm->q = q; } } @@ -1857,8 +1850,7 @@ err_svm_fini: xe_svm_fini(vm); err_no_resv: mutex_destroy(&vm->snap_mutex); - for_each_tile(tile, xe, id) - xe_range_fence_tree_fini(&vm->rftree[id]); + xe_range_fence_tree_fini(&vm->rftree); ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move); if (vm->xef) xe_file_put(vm->xef); @@ -1918,13 +1910,13 @@ void xe_vm_close_and_put(struct xe_vm *vm) { LIST_HEAD(contested); struct xe_device *xe = vm->xe; - struct xe_tile *tile; struct xe_vma *vma, *next_vma; struct drm_gpuva *gpuva, *next; - u8 id; xe_assert(xe, !vm->preempt.num_exec_queues); + trace_xe_vm_close_and_put(vm); + xe_vm_close(vm); if (xe_vm_in_preempt_fence_mode(vm)) { mutex_lock(&xe->rebind_resume_lock); @@ -1935,24 +1927,18 @@ void xe_vm_close_and_put(struct xe_vm *vm) if (xe_vm_in_fault_mode(vm)) xe_svm_close(vm); - down_write(&vm->lock); - for_each_tile(tile, xe, id) { - if (vm->q[id]) { - int i; + if (vm->q) { + int i; - xe_exec_queue_last_fence_put(vm->q[id], vm); - for_each_tlb_inval(i) - xe_exec_queue_tlb_inval_last_fence_put(vm->q[id], vm, i); - } - } - up_write(&vm->lock); + down_write(&vm->lock); + xe_exec_queue_last_fence_put(vm->q, vm); + for_each_tlb_inval(vm->q, i) + xe_exec_queue_tlb_inval_last_fence_put(vm->q, vm, i); + up_write(&vm->lock); - for_each_tile(tile, xe, id) { - if (vm->q[id]) { - xe_exec_queue_kill(vm->q[id]); - xe_exec_queue_put(vm->q[id]); - vm->q[id] = NULL; - } + xe_exec_queue_kill(vm->q); + xe_exec_queue_put(vm->q); + vm->q = NULL; } down_write(&vm->lock); @@ -1979,21 +1965,13 @@ void xe_vm_close_and_put(struct xe_vm *vm) vma->gpuva.flags |= XE_VMA_DESTROYED; } - /* - * All vm operations will add shared fences to resv. - * The only exception is eviction for a shared object, - * but even so, the unbind when evicted would still - * install a fence to resv. Hence it's safe to - * destroy the pagetables immediately. - */ - xe_vm_free_scratch(vm); - xe_vm_pt_destroy(vm); xe_vm_unlock(vm); /* - * VM is now dead, cannot re-add nodes to vm->vmas if it's NULL - * Since we hold a refcount to the bo, we can remove and free - * the members safely without locking. + * Unlink and destroy all contested external-BO VMAs before destroying + * the page tables. Otherwise, concurrent eviction holding only bo->resv + * can walk the BO's VMAs and attempt to invalidate/zap page tables that + * have already been freed. */ list_for_each_entry_safe(vma, next_vma, &contested, combined_links.destroy) { @@ -2001,6 +1979,11 @@ void xe_vm_close_and_put(struct xe_vm *vm) xe_vma_destroy_unlocked(vma); } + xe_vm_lock(vm, false); + xe_vm_free_scratch(vm); + xe_vm_pt_destroy(vm); + xe_vm_unlock(vm); + xe_svm_fini(vm); up_write(&vm->lock); @@ -2012,6 +1995,8 @@ void xe_vm_close_and_put(struct xe_vm *vm) xe_assert(xe, xe->info.has_asid); xe_assert(xe, !(vm->flags & XE_VM_FLAG_MIGRATION)); + trace_xe_vm_asid_release(vm); + lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid); xe_assert(xe, lookup == vm); } @@ -2019,8 +2004,7 @@ void xe_vm_close_and_put(struct xe_vm *vm) xe_vm_clear_fault_entries(vm); - for_each_tile(tile, xe, id) - xe_range_fence_tree_fini(&vm->rftree[id]); + xe_range_fence_tree_fini(&vm->rftree); xe_vm_put(vm); } @@ -2080,13 +2064,13 @@ struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id) u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile) { - return vm->pt_ops->pde_encode_bo(vm->pt_root[tile->id]->bo, 0); + return vm->pt_ops->pde_encode_bo(xe_vm_pt_root(vm, tile->id)->bo, 0); } static struct xe_exec_queue * to_wait_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) { - return q ? q : vm->q[0]; + return q ? q : vm->q; } static struct xe_user_fence * @@ -2548,8 +2532,10 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops, ctx.devmem_possible = IS_DGFX(vm->xe) && IS_ENABLED(CONFIG_DRM_XE_PAGEMAP); - for_each_tile(tile, vm->xe, id) + for_each_tile(tile, vm->xe, id) { + xe_migrate_ulls_enter(tile->migrate); tile_mask |= 0x1 << id; + } if (prefetch_region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC) { dpagemap = xe_vma_resolve_pagemap(vma, @@ -2584,6 +2570,11 @@ alloc_next_range: dpagemap, &valid_pages)) { xe_svm_range_debug(svm_range, "PREFETCH - RANGE IS VALID"); xe_assert(vm->xe, valid_pages); + + if (dpagemap) + xe_svm_range_prefetch_lru_bump(vm, vma, svm_range, + dpagemap); + need_put = true; goto check_next_range; } @@ -3260,6 +3251,7 @@ static int prefetch_ranges(struct xe_vm *vm, struct xe_vma_ops *vops, ctx.devmem_possible = devmem_possible; ctx.check_pages_threshold = devmem_possible ? SZ_64K : 0; ctx.device_private_page_owner = xe_svm_private_page_owner(vm, !dpagemap); + ctx.devmem_fn = xe_svm_devmem_lru_bump; skip_threads = op->prefetch_range.ranges_count == 1 || (!dpagemap && !(vops->flags & @@ -3515,28 +3507,18 @@ static void trace_xe_vm_ops_execute(struct xe_vma_ops *vops) op_trace(op); } -static int vm_ops_setup_tile_args(struct xe_vm *vm, struct xe_vma_ops *vops) +static int vm_ops_setup(struct xe_vm *vm, struct xe_vma_ops *vops) { - struct xe_exec_queue *q = vops->q; struct xe_tile *tile; int number_tiles = 0; u8 id; - for_each_tile(tile, vm->xe, id) { + for_each_tile(tile, vm->xe, id) if (vops->pt_update_ops[id].num_ops) ++number_tiles; - if (vops->pt_update_ops[id].q) - continue; - - if (q) { - vops->pt_update_ops[id].q = q; - if (vm->pt_root[id] && !list_empty(&q->multi_gt_list)) - q = list_next_entry(q, multi_gt_list); - } else { - vops->pt_update_ops[id].q = vm->q[id]; - } - } + if (!vops->q) + vops->q = vm->q; return number_tiles; } @@ -3544,26 +3526,21 @@ static int vm_ops_setup_tile_args(struct xe_vm *vm, struct xe_vma_ops *vops) static struct dma_fence *ops_execute(struct xe_vm *vm, struct xe_vma_ops *vops) { - struct xe_tile *tile; + struct xe_device *xe = vm->xe; struct dma_fence *fence = NULL; struct dma_fence **fences = NULL; struct dma_fence_array *cf = NULL; - int number_tiles = 0, current_fence = 0, n_fence = 0, err, i; - u8 id; + int current_fence = 0, n_fence = 1, err, i; - number_tiles = vm_ops_setup_tile_args(vm, vops); - if (number_tiles == 0) + if (!vm_ops_setup(vm, vops)) return ERR_PTR(-ENODATA); - for_each_tile(tile, vm->xe, id) { - ++n_fence; - - if (!(vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT)) - for_each_tlb_inval(i) - ++n_fence; + if (!(vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT)) { + for_each_tlb_inval(vops->q, i) + ++n_fence; } - fences = kmalloc_objs(*fences, n_fence); + fences = kcalloc(n_fence, sizeof(*fences), GFP_KERNEL); if (!fences) { fence = ERR_PTR(-ENOMEM); goto err_trace; @@ -3575,65 +3552,39 @@ static struct dma_fence *ops_execute(struct xe_vm *vm, goto err_out; } - for_each_tile(tile, vm->xe, id) { - if (!vops->pt_update_ops[id].num_ops) - continue; - - err = xe_pt_update_ops_prepare(tile, vops); - if (err) { - fence = ERR_PTR(err); - goto err_out; - } + err = xe_pt_update_ops_prepare(xe, vops); + if (err) { + fence = ERR_PTR(err); + goto err_out; } trace_xe_vm_ops_execute(vops); - for_each_tile(tile, vm->xe, id) { - struct xe_exec_queue *q = vops->pt_update_ops[tile->id].q; - - fence = NULL; - if (!vops->pt_update_ops[id].num_ops) - goto collect_fences; - - fence = xe_pt_update_ops_run(tile, vops); - if (IS_ERR(fence)) - goto err_out; - -collect_fences: - fences[current_fence++] = fence ?: dma_fence_get_stub(); - if (vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT) - continue; + fence = xe_pt_update_ops_run(xe, vops); + if (IS_ERR(fence)) + goto err_out; + fences[current_fence++] = fence; - xe_migrate_job_lock(tile->migrate, q); - for_each_tlb_inval(i) + if (!(vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT)) { + xe_cpu_bind_job_lock(xe->cpu_bind, vops->q); + for_each_tlb_inval(vops->q, i) fences[current_fence++] = - xe_exec_queue_tlb_inval_last_fence_get(q, vm, i); - xe_migrate_job_unlock(tile->migrate, q); + xe_exec_queue_tlb_inval_last_fence_get(vops->q, + vm, i); + xe_cpu_bind_job_unlock(xe->cpu_bind, vops->q); } - xe_assert(vm->xe, current_fence == n_fence); + xe_assert(xe, current_fence == n_fence); dma_fence_array_init(cf, n_fence, fences, dma_fence_context_alloc(1), 1); fence = &cf->base; - for_each_tile(tile, vm->xe, id) { - if (!vops->pt_update_ops[id].num_ops) - continue; - - xe_pt_update_ops_fini(tile, vops); - } + xe_pt_update_ops_fini(xe, vops); return fence; err_out: - for_each_tile(tile, vm->xe, id) { - if (!vops->pt_update_ops[id].num_ops) - continue; - - xe_pt_update_ops_abort(tile, vops); - } - while (current_fence) - dma_fence_put(fences[--current_fence]); + xe_pt_update_ops_abort(xe, vops); kfree(fences); kfree(cf); @@ -3943,6 +3894,8 @@ static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm, vops->syncs = syncs; vops->num_syncs = num_syncs; vops->flags = 0; + vops->start = ~0x0ull; + vops->last = 0x0ull; } static int xe_vm_bind_ioctl_validate_bo(struct xe_device *xe, struct xe_bo *bo, @@ -4137,7 +4090,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) syncs_user = u64_to_user_ptr(args->syncs); for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) { - struct xe_exec_queue *__q = q ?: vm->q[0]; + struct xe_exec_queue *__q = q ?: vm->q; err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs], &syncs_user[num_syncs], @@ -5142,3 +5095,21 @@ void xe_vm_remove_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) } up_write(&vm->exec_queues.lock); } + +/** + * xe_vm_pt_root() - Retrieve VM page-table root + * @vm: The VM. + * @tile_id: Tile ID + * + * Retrieve VM page-table root for a tile ID, used to abstract if PT mirroring is + * enabled across tiles. + * + * Return: VM page-table root for a tile ID + */ +struct xe_pt *xe_vm_pt_root(struct xe_vm *vm, u8 tile_id) +{ + if (vm->xe->info.has_pt_mirror) + return vm->pt_root[tile_id]; + + return vm->pt_root[0]; +} diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index c5b900f38ded..89c4f95984a2 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -436,4 +436,7 @@ static inline struct drm_exec *xe_vm_validation_exec(struct xe_vm *vm) ((READ_ONCE(tile_present) & ~READ_ONCE(tile_invalidated)) & BIT((tile)->id)) void xe_vma_mem_attr_copy(struct xe_vma_mem_attr *to, struct xe_vma_mem_attr *from); + +struct xe_pt *xe_vm_pt_root(struct xe_vm *vm, u8 tile_id); + #endif diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c index 0474768a38aa..5c7aefc641ee 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.c +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -713,12 +713,6 @@ int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *fil attr_type = array_index_nospec(args->type, ARRAY_SIZE(madvise_funcs)); - /* Ensure the madvise function exists for this type */ - if (!madvise_funcs[attr_type]) { - err = -EINVAL; - goto err_fini; - } - madvise_funcs[attr_type](xe, vm, madvise_range.vmas, madvise_range.num_vmas, args, &details); diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index 648031e64145..fb9305124679 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -262,7 +262,7 @@ struct xe_vm { struct xe_device *xe; /* exec queue used for (un)binding vma's */ - struct xe_exec_queue *q[XE_MAX_TILES_PER_DEVICE]; + struct xe_exec_queue *q; /** @lru_bulk_move: Bulk LRU move list for this VM's BOs */ struct ttm_lru_bulk_move lru_bulk_move; @@ -317,7 +317,7 @@ struct xe_vm { * @rftree: range fence tree to track updates to page table structure. * Used to implement conflict tracking between independent bind engines. */ - struct xe_range_fence_tree rftree[XE_MAX_TILES_PER_DEVICE]; + struct xe_range_fence_tree rftree; const struct xe_pt_ops *pt_ops; @@ -557,6 +557,10 @@ struct xe_vma_ops { u32 num_syncs; /** @pt_update_ops: page table update operations */ struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE]; + /** @start: start address of ops */ + u64 start; + /** @last: last address of ops */ + u64 last; /** @flag: signify the properties within xe_vma_ops*/ #define XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH BIT(0) #define XE_VMA_OPS_FLAG_MADVISE BIT(1) @@ -566,6 +570,10 @@ struct xe_vma_ops { #define XE_VMA_OPS_FLAG_MODIFIES_GPUVA BIT(5) #define XE_VMA_OPS_FLAG_DOWNGRADE_LOCK BIT(6) #define XE_VMA_OPS_FLAG_HAS_SVM_VALID_RANGE BIT(7) +#define XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP BIT(8) +#define XE_VMA_OPS_FLAG_WAIT_VM_KERNEL BIT(9) +#define XE_VMA_OPS_FLAG_NEEDS_INVALIDATION BIT(10) +#define XE_VMA_OPS_FLAG_NEEDS_SVM_LOCK BIT(11) u32 flags; #ifdef TEST_VM_OPS_ERROR /** @inject_error: inject error to test error handling */ diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules index dd69ad07f7a9..f35181fc4e49 100644 --- a/drivers/gpu/drm/xe/xe_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_wa_oob.rules @@ -14,6 +14,7 @@ 16017236439 PLATFORM(PVC) 14019821291 MEDIA_VERSION_RANGE(1300, 2000) 14015076503 MEDIA_VERSION(1300) +22016122933 MEDIA_VERSION(1300) 14018913170 GRAPHICS_VERSION_RANGE(1270, 1274) MEDIA_VERSION(1300) PLATFORM(DG2) @@ -72,3 +73,6 @@ 16029897822 MEDIA_VERSION(3500) GRAPHICS_VERSION(3510) 14027054324 GRAPHICS_VERSION(3511) +14025941587 GRAPHICS_VERSION_RANGE(2001, 3511), FUNC(xe_rtp_match_not_sriov_vf) + MEDIA_VERSION_RANGE(1301, 3503), FUNC(xe_rtp_match_not_sriov_vf) +16030224309 GRAPHICS_VERSION(3510) diff --git a/include/drm/drm_gpusvm.h b/include/drm/drm_gpusvm.h index 9e35584812fd..4d31c6bba745 100644 --- a/include/drm/drm_gpusvm.h +++ b/include/drm/drm_gpusvm.h @@ -274,6 +274,11 @@ struct drm_gpusvm { * pages as valid; the caller revalidates the snapshot itself, see * drm_gpusvm_get_pages(). @devmem_only is rejected and no page * type check is performed, so @allow_mixed has no effect. + * @devmem_fn: Optional callback invoked under the notifier lock, once per + * contiguous run of pages backed by the same &drm_pagemap_devmem. + * Must not sleep and must be idempotent, as a non-contiguous + * allocation is reported more than once. Fires even with + * @no_dma_map. May be NULL. * * Context that is DRM GPUSVM is operating in (i.e. user arguments). */ @@ -281,6 +286,7 @@ struct drm_gpusvm_ctx { void *device_private_page_owner; unsigned long check_pages_threshold; unsigned long timeslice_ms; + void (*devmem_fn)(struct drm_pagemap_devmem *devmem); unsigned int in_notifier :1; unsigned int read_only :1; unsigned int devmem_possible :1; diff --git a/include/drm/drm_pagemap.h b/include/drm/drm_pagemap.h index ebbd3b0ddf36..d12a509d6baf 100644 --- a/include/drm/drm_pagemap.h +++ b/include/drm/drm_pagemap.h @@ -258,6 +258,8 @@ struct drm_pagemap *drm_pagemap_create(struct drm_device *drm, struct drm_pagemap *drm_pagemap_page_to_dpagemap(struct page *page); +struct drm_pagemap_devmem *drm_pagemap_page_to_devmem(struct page *page); + void drm_pagemap_put(struct drm_pagemap *dpagemap); #else @@ -267,6 +269,11 @@ static inline struct drm_pagemap *drm_pagemap_page_to_dpagemap(struct page *page return NULL; } +static inline struct drm_pagemap_devmem *drm_pagemap_page_to_devmem(struct page *page) +{ + return NULL; +} + static inline void drm_pagemap_put(struct drm_pagemap *dpagemap) { } diff --git a/include/drm/intel/pciids.h b/include/drm/intel/pciids.h index dff389b56eb3..32c88a690ad8 100644 --- a/include/drm/intel/pciids.h +++ b/include/drm/intel/pciids.h @@ -880,6 +880,7 @@ MACRO__(0xB08F, ## __VA_ARGS__), \ MACRO__(0xB090, ## __VA_ARGS__), \ MACRO__(0xB0A0, ## __VA_ARGS__), \ + MACRO__(0xB0A1, ## __VA_ARGS__), \ MACRO__(0xB0B0, ## __VA_ARGS__) /* WCL */ |