diff options
| author | Dave Airlie <airlied@redhat.com> | 2026-10-28 16:22:37 +1000 |
|---|---|---|
| committer | Dave Airlie <airlied@redhat.com> | 2026-10-28 16:26:16 +1000 |
| commit | af16ecbdf81301e88bb03e114706e0fc7c2a8e42 (patch) | |
| tree | 267703e25921b4fa37ec0b2f7ce48f77febe0c44 /drivers/gpu | |
| parent | fad06cfaf385787b9ffa2f73f049d42dbe1de57f (diff) | |
| parent | 9131cc85d0dedd8ba4cae60dbe5985d081551f30 (diff) | |
Merge tag 'amd-drm-next-fixes-7.4-2026-10-21' of https://gitlab.freedesktop.org/drm/amdgpu/kernel into drm-next
amd-drm-next-fixes-7.4-2026-10-21:
amdgpu:
- UALink fixes
- Runtime PM fixes
- GPUVM fixes
- Mac fixes
- HDMI fixes
- DP MST fix
- Userptr fix
- SMU fixes
- DSC fix
amdkfd:
- SVM fixes
- Fix doorbell allocation race
- GC 12.1 fix
- GPU reset fix
- Buffer accounting fixes
Signed-off-by: Dave Airlie <airlied@redhat.com>
From: Alex Deucher <alexander.deucher@amd.com>
Link: https://patch.msgid.link/20261021173450.2959689-1-alexander.deucher@amd.com
Diffstat (limited to 'drivers/gpu')
47 files changed, 950 insertions, 234 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index ce671731b700..c7577b608f3e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -844,8 +844,17 @@ struct amdgpu_device { struct mutex mn_lock; DECLARE_HASHTABLE(mn_hash, 7); - /* tracking pinned memory */ + /* tracking pinned memory + * + * vram_pin_size = sum(vram_pin_size_xcp[]) + vram_pin_size_common + * + * vram_pin_size total pin size in device + * vram_pin_size_xcp[i] pins owned by partition i (bo->xcp_id = i, i >= 0) + * vram_pin_size_common pins not owned by any partition, bo->xcp_id = -1 + */ atomic64_t vram_pin_size; + atomic64_t vram_pin_size_xcp[MAX_XCP]; + atomic64_t vram_pin_size_common; atomic64_t visible_pin_size; atomic64_t gart_pin_size; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index e3da382ceb3c..77c0504cc158 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -42,6 +42,12 @@ extern uint64_t amdgpu_amdkfd_total_mem_size; +/* + * Align VRAM availability to 2MB to avoid fragmentation caused by 4K + * allocations in the tail 2MB BO chunk. + */ +#define VRAM_AVAILABLITY_ALIGN (1 << 21) + enum TLB_FLUSH_TYPE { TLB_FLUSH_LEGACY = 0, TLB_FLUSH_LIGHTWEIGHT, @@ -106,6 +112,7 @@ struct amdgpu_kfd_dev { struct kfd_dev *dev; int64_t vram_used[MAX_XCP]; uint64_t vram_used_aligned[MAX_XCP]; + atomic64_t vram_pinned[MAX_XCP]; bool init_complete; struct work_struct reset_work; @@ -389,6 +396,8 @@ void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, * Allows KFD to release its resources associated with the GEM object. */ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo); +void amdgpu_amdkfd_account_vram_pin(struct amdgpu_bo *bo, int xcp_id); +void amdgpu_amdkfd_account_vram_unpin(struct amdgpu_bo *bo, int xcp_id); void amdgpu_amdkfd_reserve_system_mem(uint64_t size); #else static inline @@ -406,6 +415,16 @@ static inline void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo) { } + +static inline +void amdgpu_amdkfd_account_vram_pin(struct amdgpu_bo *bo, int xcp_id) +{ +} + +static inline +void amdgpu_amdkfd_account_vram_unpin(struct amdgpu_bo *bo, int xcp_id) +{ +} #endif #if IS_ENABLED(CONFIG_HSA_AMD_SVM) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c index 0694bff871d2..5b526add8645 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c @@ -77,25 +77,21 @@ static uint32_t get_sdma_rlc_reg_offset(struct amdgpu_device *adev, unsigned int engine_id, unsigned int queue_id) { - uint32_t sdma_engine_reg_base = 0; - uint32_t sdma_rlc_reg_offset; uint32_t dev_inst = GET_INST(SDMA0, engine_id); + int xcc_id = adev->sdma.instance[engine_id].xcc_id; + int xcc_inst = dev_inst % adev->sdma.num_inst_per_xcc; + uint32_t sdma_engine_reg_base; + uint32_t sdma_rlc_reg_offset; - switch (dev_inst % adev->sdma.num_inst_per_xcc) { - case 0: - sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA0, - dev_inst / adev->sdma.num_inst_per_xcc, - regSDMA0_SDMA_QUEUE0_RB_CNTL) - regSDMA0_SDMA_QUEUE0_RB_CNTL; - break; - case 1: - sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA1, - dev_inst / adev->sdma.num_inst_per_xcc, - regSDMA1_SDMA_QUEUE0_RB_CNTL) - regSDMA0_SDMA_QUEUE0_RB_CNTL; - break; - default: - WARN(1, "Invalid SDMA engine id %d\n", engine_id); - break; - } + /* + * On SDMA v7 (gc_12_x) all SDMA queue registers live in the GC_HWIP + * aperture. The second SDMA per XCC is reached by adding a fixed + * offset (regSDMA1_* - regSDMA0_* == SDMA1_REG_OFFSET + */ + sdma_engine_reg_base = adev->reg_offset[GC_HWIP][xcc_id][0]; + if (xcc_inst != 0) + sdma_engine_reg_base += + regSDMA1_SDMA_QUEUE0_RB_CNTL - regSDMA0_SDMA_QUEUE0_RB_CNTL; sdma_rlc_reg_offset = sdma_engine_reg_base + queue_id * (regSDMA0_SDMA_QUEUE1_RB_CNTL - regSDMA0_SDMA_QUEUE0_RB_CNTL); @@ -141,8 +137,7 @@ static int hqd_sdma_dump_v12_1(struct amdgpu_device *adev, uint32_t engine_id, uint32_t queue_id, uint32_t (**dump)[2], uint32_t *n_regs) { - uint32_t sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, - engine_id, queue_id); + uint32_t sdma_rlc_reg_offset; uint32_t i = 0, reg; const uint32_t first_reg = regSDMA0_SDMA_QUEUE0_RB_CNTL; @@ -150,6 +145,17 @@ static int hqd_sdma_dump_v12_1(struct amdgpu_device *adev, #undef HQD_N_REGS #define HQD_N_REGS (last_reg - first_reg + 1) + /* + * dqm_debugfs_hqds() iterates engine_id over + * node_id * get_num_all_sdma_engines()..+get_num_all_sdma_engines(). + * On partitioned devices that range can address a logical SDMA engine + * beyond the instances that actually exist on this node; reject it + */ + if (engine_id >= adev->sdma.num_instances) + return -EINVAL; + + sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, engine_id, queue_id); + *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL); if (*dump == NULL) return -ENOMEM; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 9a7e14a3c719..ff77e8ae9fe6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -46,12 +46,6 @@ #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1 #define AMDGPU_RESERVE_MEM_LIMIT (3UL << 29) -/* - * Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB - * BO chunk - */ -#define VRAM_AVAILABLITY_ALIGN (1 << 21) - /* Impose limit on how much memory KFD can use */ static struct { uint64_t max_system_mem_limit; @@ -73,6 +67,8 @@ static const char * const domain_bit_to_string[] = { #define domain_string(domain) domain_bit_to_string[ffs(domain)-1] static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work); +static size_t __get_available_memory(struct amdgpu_device *adev, + uint8_t xcp_id, int64_t vram_used); static bool kfd_mem_is_attached(struct amdgpu_vm *avm, struct kgd_mem *mem) @@ -169,13 +165,8 @@ void amdgpu_amdkfd_reserve_system_mem(uint64_t size) int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 alloc_flag, int8_t xcp_id) { - uint64_t reserved_for_pt = - ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size); - struct amdgpu_ras *con = amdgpu_ras_get_context(adev); - uint64_t reserved_for_ras = (con ? con->reserved_pages_in_bytes : 0); size_t system_mem_needed, ttm_mem_needed, vram_needed; int ret = 0; - uint64_t vram_size = 0; system_mem_needed = 0; ttm_mem_needed = 0; @@ -196,7 +187,6 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id)) return -EINVAL; - vram_size = KFD_XCP_MEMORY_SIZE(adev, xcp_id); if (adev->apu_prefer_gtt) { system_mem_needed = size; ttm_mem_needed = size; @@ -232,14 +222,11 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, * VRAM check since ttm_mem_limit check already cover this allocation */ - if (adev && xcp_id >= 0 && (!adev->apu_prefer_gtt || adev->gmc.is_app_apu)) { - uint64_t vram_available = - vram_size - reserved_for_pt - reserved_for_ras - - atomic64_read(&adev->vram_pin_size); - if (adev->kfd.vram_used[xcp_id] + vram_needed > vram_available) { - ret = -ENOMEM; - goto release; - } + if (adev && xcp_id >= 0 && (!adev->apu_prefer_gtt || adev->gmc.is_app_apu) && + vram_needed > __get_available_memory(adev, xcp_id, + adev->kfd.vram_used[xcp_id])) { + ret = -ENOMEM; + goto release; } /* Update memory accounting by decreasing available system @@ -318,6 +305,30 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo) kfree(bo->kfd_bo); } +void amdgpu_amdkfd_account_vram_pin(struct amdgpu_bo *bo, int xcp_id) +{ + if (!bo->kfd_bo) + return; + + if (WARN_ONCE(xcp_id < 0, "KFD BO pinned in VRAM with no partition")) + return; + + atomic64_add(amdgpu_bo_size(bo), + &amdgpu_ttm_adev(bo->tbo.bdev)->kfd.vram_pinned[xcp_id]); +} + +void amdgpu_amdkfd_account_vram_unpin(struct amdgpu_bo *bo, int xcp_id) +{ + if (!bo->kfd_bo) + return; + + if (WARN_ONCE(xcp_id < 0, "KFD BO unpinned in VRAM with no partition")) + return; + + atomic64_sub(amdgpu_bo_size(bo), + &amdgpu_ttm_adev(bo->tbo.bdev)->kfd.vram_pinned[xcp_id]); +} + /** * create_dmamap_sg_bo() - Creates a amdgpu_bo object to reflect information * about USERPTR or DOOREBELL or MMIO BO. @@ -1536,6 +1547,7 @@ static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain) pr_err("Error in Pinning BO to domain: %d\n", domain); amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); + out: amdgpu_bo_unreserve(bo); return ret; @@ -1558,6 +1570,7 @@ static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo) return; amdgpu_bo_unpin(bo); + amdgpu_bo_unreserve(bo); } @@ -1661,8 +1674,9 @@ out_unlock: return ret; } -size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev, - uint8_t xcp_id) +/* Caller must hold kfd_mem_limit.mem_limit_lock. */ +static size_t __get_available_memory(struct amdgpu_device *adev, + uint8_t xcp_id, int64_t vram_used) { uint64_t reserved_for_pt = ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size); @@ -1671,14 +1685,17 @@ size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev, ssize_t available; uint64_t vram_available, system_mem_available, ttm_mem_available; - spin_lock(&kfd_mem_limit.mem_limit_lock); + lockdep_assert_held(&kfd_mem_limit.mem_limit_lock); + if (adev->apu_prefer_gtt && !adev->gmc.is_app_apu) vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id) - - adev->kfd.vram_used_aligned[xcp_id]; + - vram_used; else vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id) - - adev->kfd.vram_used_aligned[xcp_id] - - atomic64_read(&adev->vram_pin_size) + - vram_used + - atomic64_read(&adev->vram_pin_size_xcp[xcp_id]) + - atomic64_read(&adev->vram_pin_size_common) + + atomic64_read(&adev->kfd.vram_pinned[xcp_id]) - reserved_for_pt - reserved_for_ras; @@ -1693,19 +1710,29 @@ size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev, available = min3(system_mem_available, ttm_mem_available, vram_available); - available = ALIGN_DOWN(available, PAGE_SIZE); } else { - available = ALIGN_DOWN(vram_available, VRAM_AVAILABLITY_ALIGN); + available = vram_available; } - spin_unlock(&kfd_mem_limit.mem_limit_lock); - if (available < 0) available = 0; return available; } +size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev, + uint8_t xcp_id) +{ + size_t available; + + spin_lock(&kfd_mem_limit.mem_limit_lock); + available = __get_available_memory(adev, xcp_id, + adev->kfd.vram_used_aligned[xcp_id]); + spin_unlock(&kfd_mem_limit.mem_limit_lock); + + return available; +} + int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( struct amdgpu_device *adev, uint64_t va, uint64_t size, void *drm_priv, struct kgd_mem **mem, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 48279392d385..f147c0c459da 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -593,10 +593,8 @@ bool amdgpu_device_supports_boco(struct amdgpu_device *adev) if (!IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)) return false; - if (adev->has_pr3 || - ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid())) - return true; - return false; + /* BOCO needs a platform power resource (_PR3) to cut the power */ + return adev->has_pr3; } /** @@ -2783,6 +2781,12 @@ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE); amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE); + r = amdgpu_dpm_reapply_umd_profile(adev); + if (r) { + dev_err(adev->dev, "Failed to restore UMD profile state: %d\n", r); + return r; + } + amdgpu_device_fill_reset_magic(adev); r = amdgpu_device_enable_mgpu_fan_boost(); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index ab6f9131ac11..3e5634de077f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -2974,19 +2974,6 @@ static int amdgpu_pmops_runtime_suspend(struct device *dev) return 0; } -static void amdgpu_restore_umd_profile_pstate_after_runpm(struct amdgpu_device *adev) -{ - enum amd_dpm_forced_level level; - uint32_t profile_mode_mask = AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD | - AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK | - AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK | - AMD_DPM_FORCED_LEVEL_PROFILE_PEAK; - - level = amdgpu_dpm_get_performance_level(adev); - if (level & profile_mode_mask) - amdgpu_asic_update_umd_stable_pstate(adev, true); -} - static int amdgpu_pmops_runtime_resume(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); @@ -3032,7 +3019,6 @@ static int amdgpu_pmops_runtime_resume(struct device *dev) if (adev->pm.rpm_mode == AMDGPU_RUNPM_PX) drm_dev->switch_power_state = DRM_SWITCH_POWER_ON; - amdgpu_restore_umd_profile_pstate_after_runpm(adev); adev->in_runpm = false; return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 8e15b204f117..59ea7af5b0e4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -780,9 +780,16 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) return copy_to_user(out, &count, min(size, 4u)) ? -EFAULT : 0; } - case AMDGPU_INFO_TIMESTAMP: + case AMDGPU_INFO_TIMESTAMP: { + PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm); + + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + ui64 = amdgpu_gfx_get_gpu_clock_counter(adev); return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0; + } case AMDGPU_INFO_FW_VERSION: { struct drm_amdgpu_info_firmware fw_info; @@ -906,6 +913,12 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) if (info->read_mmr_reg.count > 128) return -EINVAL; + PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm); + + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + regs = kmalloc_array(info->read_mmr_reg.count, sizeof(*regs), GFP_KERNEL); if (!regs) @@ -940,6 +953,13 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) uint64_t vm_size; uint32_t pcie_gen_mask, pcie_width_mask; + /* The clock limits are only available while the SMU is up. */ + PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm); + + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + dev_info = kzalloc_obj(*dev_info); if (!dev_info) return -ENOMEM; @@ -1167,6 +1187,12 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) } } case AMDGPU_INFO_SENSOR: { + PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm); + + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + if (!adev->pm.dpm_enabled) return -ENOENT; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index 2a6a5bc01eb4..6aa895f63aea 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -923,6 +923,19 @@ void amdgpu_bo_unref(struct amdgpu_bo **bo) *bo = NULL; } +/* Map a BO onto vram_pin_size_xcp[]. bo->xcp_id < 0 means "any partition": + * such pins are not owned by a single partition and go to + * vram_pin_size_common, which every partition subtracts. + */ +static int amdgpu_bo_get_xcp(struct amdgpu_bo *bo) +{ + s8 xcp = bo->xcp_id; + + if (xcp >= 0 && xcp < MAX_XCP) + return xcp; + return -1; +} + /** * amdgpu_bo_pin - pin an &amdgpu_bo buffer object * @bo: &amdgpu_bo buffer object to be pinned @@ -1003,7 +1016,17 @@ int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain) ttm_bo_pin(&bo->tbo); if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { + int xcp = amdgpu_bo_get_xcp(bo); + atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size); + if (xcp >= 0) { + atomic64_add(amdgpu_bo_size(bo), + &adev->vram_pin_size_xcp[xcp]); + } else { + atomic64_add(amdgpu_bo_size(bo), + &adev->vram_pin_size_common); + } + amdgpu_amdkfd_account_vram_pin(bo, xcp); atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo), &adev->visible_pin_size); } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { @@ -1036,7 +1059,17 @@ void amdgpu_bo_unpin(struct amdgpu_bo *bo) dma_buf_unpin(bo->tbo.base.import_attach); if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { + int xcp = amdgpu_bo_get_xcp(bo); + atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size); + if (xcp >= 0) { + atomic64_sub(amdgpu_bo_size(bo), + &adev->vram_pin_size_xcp[xcp]); + } else { + atomic64_sub(amdgpu_bo_size(bo), + &adev->vram_pin_size_common); + } + amdgpu_amdkfd_account_vram_unpin(bo, xcp); atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo), &adev->visible_pin_size); } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 6b3fbf79abc9..756ba0dad85d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -784,7 +784,8 @@ int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, goto out_unlock; } if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) && - vma->vm_file)) { + (vma->vm_file || + vma->vm_end < start + (u64)ttm->num_pages * PAGE_SIZE))) { r = -EPERM; goto out_unlock; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c index 17888cf54447..6cd3927a7e82 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c @@ -137,14 +137,21 @@ static bool __check_vpod_info(struct amdgpu_device *adev, vpod->size, ppod->size); return false; } - if (vpod->addr_mode >= AMDGPU_UALINK_ADDR_MODE_MAX) { + + switch (vpod->addr_mode) { + case AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS: + break; + case AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT: dev_warn(adev->dev, - "UALINK: invalid addr mode %u\n", + "UALINK: source-identification addr mode is not supported\n"); + return false; + default: + dev_warn(adev->dev, "UALINK: invalid addr mode %u\n", vpod->addr_mode); return false; } - weight = - bitmap_weight(vpod->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX); + + weight = bitmap_weight(vpod->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX); if (weight != vpod->size) { dev_warn(adev->dev, "UALINK: vPod size doesn't match vpod_active_accels list: %u != %u\n", @@ -1351,8 +1358,7 @@ static ssize_t ualink_vpod_config_commit_store(struct kobject *kobj, if (info->vpod.id != AMDGPU_UALINK_VPOD_ID_INVALID && !__check_vpod_info(adev, info)) { dev_err(adev->dev, - "UALINK: vpod-commit __check_vpod_info() failed: %d\n", - r); + "UALINK: vpod-commit __check_vpod_info() failed\n"); return -EINVAL; } @@ -6075,9 +6081,8 @@ static int amdgpu_ualink_remote_shootdown(struct amdgpu_device *adev, struct amdgpu_ualink_ring *ring; int r; - dev_dbg(adev->dev, "skip remote shootdown to remote_accel_id %u\n", + dev_dbg(adev->dev, "send remote shootdown to remote_accel_id %u\n", remote_accel_id); - return 0; peer = &remote->peer[remote_accel_id]; ring = &peer->shootdown; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 2bd07007def4..c41e9c0916b7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -2737,10 +2737,7 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm) if (vm->use_cpu_for_update) { /* Sync with last SDMA update/clear before switching to CPU */ - r = amdgpu_bo_sync_wait(vm->root.bo, - AMDGPU_FENCE_OWNER_UNDEFINED, true); - if (r) - goto unreserve_bo; + dma_fence_wait(vm->last_update, false); vm->update_funcs = &amdgpu_vm_cpu_funcs; r = amdgpu_vm_pt_map_tables(adev, vm); @@ -2797,6 +2794,7 @@ int amdgpu_vm_make_npa(struct amdgpu_device *adev, struct amdgpu_vm *vm) return r; vm->is_npa = true; + vm->need_tlb_fence = false; return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index c03327f1242d..c73eb416c0e2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c @@ -427,7 +427,7 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm, if (r) goto exit; - r = vm->update_funcs->commit(¶ms, NULL); + r = vm->update_funcs->commit(¶ms, &vm->last_update); exit: drm_dev_exit(idx); return r; @@ -531,13 +531,16 @@ static int amdgpu_vm_pt_alloc(struct amdgpu_vm_update_params *p, r2 = amdgpu_vm_begin_critical(p); if (r) return r; - if (r2) - goto error_free_pt; + + pt_bo = &pt->bo; + if (r2) { + r = r2; + goto error_unpin; + } /* Keep a reference to the root directory to avoid * freeing them up in the wrong order. */ - pt_bo = &pt->bo; pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo); amdgpu_vm_bo_base_init(entry, p->vm, pt_bo); r = amdgpu_vm_pt_clear(p->adev, p->vm, pt, p->immediate); @@ -549,7 +552,6 @@ static int amdgpu_vm_pt_alloc(struct amdgpu_vm_update_params *p, error_unpin: if (p->vm->is_npa) amdgpu_bo_unpin(pt_bo); -error_free_pt: amdgpu_bo_unref(&pt_bo); return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c index d29991a4af4c..e1029239b2c6 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik.c +++ b/drivers/gpu/drm/amd/amdgpu/cik.c @@ -1927,6 +1927,16 @@ static bool cik_need_reset_on_init(struct amdgpu_device *adev) { u32 clock_cntl, pc; + /* + * MacBookPros with switchable graphics put VRAM at 0 when the iGPU + * is enabled. Any later post (S3 resume, GPU reset) moves it back to + * the vbios default, so reset and post on load to start with the + * vbios layout. + */ + if (adev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE && + !(RREG32(mmMC_VM_FB_LOCATION) & MC_VM_FB_LOCATION__FB_BASE_MASK)) + return true; + if (adev->flags & AMD_IS_APU) return false; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index 85ba228f002d..822681635a65 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -2179,10 +2179,22 @@ static int gmc_v9_0_hw_init(struct amdgpu_ip_block *ip_block) * (type 2), which flushes both. Due to a race condition with * concurrent memory accesses using the same TLB cache line, we still * need a second TLB flush after this. + * + * GC 9.4.3, 9.4.4 and 9.5.0 hit a similar race. */ - adev->gmc.flush_tlb_needs_extra_type_2 = - amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 0) && - adev->gmc.xgmi.num_physical_nodes; + switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { + case IP_VERSION(9, 4, 0): + adev->gmc.flush_tlb_needs_extra_type_2 = + adev->gmc.xgmi.num_physical_nodes; + break; + case IP_VERSION(9, 4, 3): + case IP_VERSION(9, 4, 4): + case IP_VERSION(9, 5, 0): + adev->gmc.flush_tlb_needs_extra_type_2 = true; + break; + default: + break; + } /* The sequence of these two function calls matters.*/ gmc_v9_0_init_golden_registers(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c index 0adfd4b101aa..e5e590425f42 100644 --- a/drivers/gpu/drm/amd/amdgpu/si.c +++ b/drivers/gpu/drm/amd/amdgpu/si.c @@ -1512,6 +1512,16 @@ static void si_invalidate_hdp(struct amdgpu_device *adev, static bool si_need_reset_on_init(struct amdgpu_device *adev) { + /* + * MacBookPros with switchable graphics put VRAM at 0 when the iGPU + * is enabled. Any later post (S3 resume, GPU reset) moves it back to + * the vbios default, so reset and post on load to start with the + * vbios layout. + */ + if (adev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE && + !(RREG32(mmMC_VM_FB_LOCATION) & MC_VM_FB_LOCATION__FB_BASE_MASK)) + return true; + return false; } diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c index a0435fd5525b..cad6bbfcedf3 100644 --- a/drivers/gpu/drm/amd/amdgpu/vi.c +++ b/drivers/gpu/drm/amd/amdgpu/vi.c @@ -1405,6 +1405,16 @@ static bool vi_need_reset_on_init(struct amdgpu_device *adev) unsigned int i; u32 clock_cntl, pc; + /* + * MacBookPros with switchable graphics put VRAM at 0 when the iGPU + * is enabled. Any later post (S3 resume, GPU reset) moves it back to + * the vbios default, so reset and post on load to start with the + * vbios layout. + */ + if (adev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE && + !(RREG32(mmMC_VM_FB_LOCATION) & MC_VM_FB_LOCATION__FB_BASE_MASK)) + return true; + for (i = 0; i < ARRAY_SIZE(vi_reset_quirks); i++) if (adev->pdev->device == vi_reset_quirks[i].device && adev->pdev->subsystem_vendor == vi_reset_quirks[i].subsystem_vendor && diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 344da6c0e96a..cad60ccdb216 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -427,12 +427,10 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, } } - if (!pdd->qpd.proc_doorbells) { - err = kfd_alloc_process_doorbells(dev->kfd, pdd); - if (err) { - pr_debug("failed to allocate process doorbells\n"); - goto err_bind_process; - } + err = kfd_alloc_process_doorbells(dev->kfd, pdd); + if (err) { + pr_debug("failed to allocate process doorbells\n"); + goto err_bind_process; } err = kfd_queue_acquire_buffers(pdd, &q_properties); @@ -1112,11 +1110,19 @@ static int kfd_ioctl_get_available_memory(struct file *filep, { struct kfd_ioctl_get_available_memory_args *args = data; struct kfd_process_device *pdd = kfd_lock_pdd_by_id(p, args->gpu_id); + struct amdgpu_device *adev; + size_t available; if (!pdd) return -EINVAL; - args->available = amdgpu_amdkfd_get_available_memory(pdd->dev->adev, - pdd->dev->node_id); + + adev = pdd->dev->adev; + available = amdgpu_amdkfd_get_available_memory(adev, pdd->dev->node_id); + if (adev->apu_prefer_gtt) + args->available = ALIGN_DOWN(available, PAGE_SIZE); + else + args->available = ALIGN_DOWN(available, VRAM_AVAILABLITY_ALIGN); + kfd_unlock_pdd(pdd); return 0; } @@ -1305,6 +1311,9 @@ static int kfd_ioctl_free_memory_of_gpu(struct file *filep, goto err_unlock; } + /* Queues a reset could not stop may still access the buffer */ + kfd_process_wait_gpu_reset_complete(p); + ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, (struct kgd_mem *)mem, pdd->drm_priv, &size); @@ -2455,11 +2464,9 @@ static int criu_restore_devices(struct kfd_process *p, goto exit; } - if (!pdd->qpd.proc_doorbells) { - ret = kfd_alloc_process_doorbells(dev->kfd, pdd); - if (ret) - goto exit; - } + ret = kfd_alloc_process_doorbells(dev->kfd, pdd); + if (ret) + goto exit; } /* diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 5e57576fba21..ce67e56efa8c 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -4215,6 +4215,15 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data) queue++) { r = dqm->dev->kfd2kgd->hqd_sdma_dump( dqm->dev->adev, pipe, queue, &dump, &n_regs); + if (r == -EINVAL) { + /* + * SDMA engine not present/mapped on this + * device; skip it instead of aborting the + * whole dump. + */ + r = 0; + break; + } if (r) break; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c index fdcf7f2d1b5b..6af268a031f2 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c @@ -239,11 +239,9 @@ phys_addr_t kfd_get_process_doorbells(struct kfd_process_device *pdd) struct amdgpu_device *adev = pdd->dev->adev; uint32_t first_db_index; - if (!pdd->qpd.proc_doorbells) { - if (kfd_alloc_process_doorbells(pdd->dev->kfd, pdd)) - /* phys_addr_t 0 is error */ - return 0; - } + if (kfd_alloc_process_doorbells(pdd->dev->kfd, pdd)) + /* phys_addr_t 0 is error */ + return 0; first_db_index = amdgpu_doorbell_index_on_bar(adev, pdd->qpd.proc_doorbells, @@ -257,12 +255,21 @@ int kfd_alloc_process_doorbells(struct kfd_dev *kfd, struct kfd_process_device * int r; struct qcm_process_device *qpd = &pdd->qpd; + mutex_lock(&pdd->doorbell_mutex); + + /* Another thread may have won the check-and-allocate race */ + if (qpd->proc_doorbells) { + mutex_unlock(&pdd->doorbell_mutex); + return 0; + } + /* Allocate bitmap for dynamic doorbell allocation */ qpd->doorbell_bitmap = bitmap_zalloc(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, GFP_KERNEL); if (!qpd->doorbell_bitmap) { DRM_ERROR("Failed to allocate process doorbell bitmap\n"); - return -ENOMEM; + r = -ENOMEM; + goto unlock; } r = init_doorbell_bitmap(&pdd->qpd, kfd); @@ -284,11 +291,15 @@ int kfd_alloc_process_doorbells(struct kfd_dev *kfd, struct kfd_process_device * DRM_ERROR("Failed to allocate process doorbells\n"); goto err; } + + mutex_unlock(&pdd->doorbell_mutex); return 0; err: bitmap_free(qpd->doorbell_bitmap); qpd->doorbell_bitmap = NULL; +unlock: + mutex_unlock(&pdd->doorbell_mutex); return r; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index b9a31489b1c8..288731acc928 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -784,6 +784,9 @@ struct kfd_process_device { /* per-process-per device QCM data structure */ struct qcm_process_device qpd; + /* Serializes process doorbell allocation for this device */ + struct mutex doorbell_mutex; + /*Apertures*/ uint64_t lds_base; uint64_t lds_limit; @@ -1130,6 +1133,7 @@ void kfd_unref_process(struct kfd_process *p); int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger); int kfd_process_restore_queues(struct kfd_process *p); void kfd_suspend_all_processes(void); +void kfd_process_wait_gpu_reset_complete(struct kfd_process *p); int kfd_resume_all_processes(void); struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *process, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index 226f52626e84..6663718fb83c 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -1233,6 +1233,7 @@ static void kfd_process_destroy_pdds(struct kfd_process *p) } atomic_dec(&pdd->dev->kfd->kfd_processes_count); + mutex_destroy(&pdd->doorbell_mutex); kfree(pdd); p->pdds[i] = NULL; @@ -1293,7 +1294,7 @@ static void kfd_process_remove_sysfs(struct kfd_process *p) /* * If any GPU is ongoing reset, wait for reset complete. */ -static void kfd_process_wait_gpu_reset_complete(struct kfd_process *p) +void kfd_process_wait_gpu_reset_complete(struct kfd_process *p) { int i; @@ -1802,6 +1803,7 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev, pdd->dev = dev; INIT_LIST_HEAD(&pdd->qpd.queues_list); INIT_LIST_HEAD(&pdd->qpd.priv_queue_list); + mutex_init(&pdd->doorbell_mutex); pdd->qpd.dqm = dev->dqm; pdd->qpd.pqm = &p->pqm; pdd->qpd.evicted = 0; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c index ae3e7c54014a..6010a5d7a7eb 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c @@ -230,6 +230,15 @@ void kfd_queue_buffer_put(struct amdgpu_bo **bo) amdgpu_bo_unref(bo); } +static void kfd_queue_put_bos(struct queue_properties *properties) +{ + kfd_queue_buffer_put(&properties->wptr_bo); + kfd_queue_buffer_put(&properties->rptr_bo); + kfd_queue_buffer_put(&properties->ring_bo); + kfd_queue_buffer_put(&properties->eop_buf_bo); + kfd_queue_buffer_put(&properties->cwsr_bo); +} + int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_properties *properties) { struct kfd_topology_device *topo_dev; @@ -343,7 +352,13 @@ out_err_release: * dropping the VM reservation. */ kfd_queue_unref_bo_vas(pdd, properties); - kfd_queue_release_buffers(pdd, properties); + /* + * The SVM CWSR range is the last thing acquired and a failed + * kfd_queue_buffer_svm_get() takes nothing, so only the BOs are held + * here. Dropping the SVM reference would drop the one of another queue + * using the same CWSR address. + */ + kfd_queue_put_bos(properties); return err; } @@ -352,11 +367,7 @@ int kfd_queue_release_buffers(struct kfd_process_device *pdd, struct queue_prope struct kfd_topology_device *topo_dev; u64 total_cwsr_size; - kfd_queue_buffer_put(&properties->wptr_bo); - kfd_queue_buffer_put(&properties->rptr_bo); - kfd_queue_buffer_put(&properties->ring_bo); - kfd_queue_buffer_put(&properties->eop_buf_bo); - kfd_queue_buffer_put(&properties->cwsr_bo); + kfd_queue_put_bos(properties); topo_dev = kfd_topology_device_by_id(pdd->dev->id); if (!topo_dev) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index cfb428c72686..1f5d73f0ebce 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -2158,14 +2158,25 @@ static struct svm_range *svm_range_clone(struct svm_range *old) svm_range_free(new, false); return NULL; } - if (old->svm_bo) { + + /* old->lock keeps old->svm_bo alive while taking the reference */ + mutex_lock(&old->lock); + if (svm_bo_ref_unless_zero(old->svm_bo)) { new->ttm_res = old->ttm_res; new->offset = old->offset; - new->svm_bo = svm_range_bo_ref(old->svm_bo); + new->svm_bo = old->svm_bo; spin_lock(&new->svm_bo->list_lock); list_add(&new->svm_bo_list, &new->svm_bo->range_list); spin_unlock(&new->svm_bo->list_lock); } + mutex_unlock(&old->lock); + + /* prange only holds an svm_bo reference while ttm_res is set, unref + * outside old->lock, svm_range_bo_release takes it and would deadlock. + */ + if (new->svm_bo && !new->ttm_res) + svm_range_bo_unref(new->svm_bo); + new->flags = old->flags; new->preferred_loc = old->preferred_loc; new->prefetch_loc = old->prefetch_loc; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c index 3217aa82bea2..6febc15ebc2b 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c @@ -309,7 +309,7 @@ fill_dc_mst_payload_table_from_drm(struct dc_link *link, break; } } - ASSERT(i != copy_of_link_table.stream_count); + /* Payload absent from HW table when never allocated: nothing to remove. */ } /* Fill payload info*/ diff --git a/drivers/gpu/drm/amd/display/dc/dc_hdmi_types.h b/drivers/gpu/drm/amd/display/dc/dc_hdmi_types.h index eb6e7f4043fd..9e0fe41f28a5 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_hdmi_types.h +++ b/drivers/gpu/drm/amd/display/dc/dc_hdmi_types.h @@ -235,7 +235,8 @@ enum link_result { LINK_RESULT_SUCCESS, LINK_RESULT_LOWER_LINKRATE, LINK_RESULT_TIMEOUT, - LINK_RESULT_FALLBACK + LINK_RESULT_FALLBACK, + LINK_RESULT_NO_TIMEOUT }; enum hdmi_frl_link_rate { @@ -274,6 +275,7 @@ struct dc_hdmi_frl_flags { bool force_frl_dsc; bool force_frl_max; bool apply_vsdb_rcc_wa; + bool no_timeout; }; struct dc_hdmi_frl_link_training_overrides { diff --git a/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c index 71eedb3301e4..62673dc8b04a 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c @@ -45,7 +45,7 @@ static int median3(int a, int b, int c) if (b > c) swap(b, c); if (a > b) - swap(b, c); + swap(a, b); return b; } diff --git a/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_frl.c b/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_frl.c index a2c0fa21ee94..62288133cd55 100644 --- a/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_frl.c +++ b/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_frl.c @@ -59,6 +59,9 @@ static void disable_hpo_frl_link_output(struct dc_link *link, enum signal_type signal) { (void)link_res; + if (link->frl_flags.no_timeout) + return; + if (dc_is_hdmi_frl_signal(signal)) link->hpo_frl_link_enc->funcs->disable_link_encoder(link->hpo_frl_link_enc); link->link_enc->funcs->disable_output(link->link_enc, signal); diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c index 0d3b65d40807..59a3f4b88a37 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c @@ -2038,6 +2038,8 @@ static enum dc_status enable_link_hdmi_frl(struct pipe_ctx *pipe_ctx) pipe_ctx->stream_res.hpo_frl_stream_enc->funcs->hdmi_frl_enable( pipe_ctx->stream_res.hpo_frl_stream_enc, pipe_ctx->stream_res.tg->inst); + } else if (link_stat == LINK_RESULT_NO_TIMEOUT) { + return status; } else { status = DC_FAIL_HDMI_FRL_LINK_TRAINING; stream->link->frl_link_settings.frl_link_rate = 0; diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_hdmi_frl.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_hdmi_frl.c index 5e0979ac3dcf..960f18fe275b 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_hdmi_frl.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_hdmi_frl.c @@ -442,7 +442,7 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s union hdmi_scdc_status_flags_data status_data = {0}; union hdmi_scdc_source_test_req test_req = {0}; union hdmi_scdc_LTP_req_data ltp_req = {0}; - unsigned long long poll_timeout_ns = 200 * 1000 * 1000; + unsigned long long poll_timeout_ns = 220 * 1000 * 1000; unsigned long long wait_time_ns = 2 * 1000 * 1000; unsigned long long num_polls = 0; struct hpo_frl_link_encoder *hpo_frl_link_enc = ddc_service->link->hpo_frl_link_enc; @@ -451,11 +451,37 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s uint8_t FFE_Levels = hdmi_frl_get_max_ffe_level(ddc_service->link, link_settings); uint8_t current_FFE = 0; bool override_FFE = false; - bool flt_no_timeout = false; DC_LOGGER_INIT(ddc_service->link->ctx->logger); FRL_INFO("FRL LINK TRAINING: Starting FRL Link Training.\n"); + /*Test Condition - FLT_no_timeout avoid link training*/ + offset = HDMI_SCDC_SOURCE_TEST_REQ; + link_query_ddc_data(ddc_service, slave_address, &offset, + sizeof(offset), &test_req.byte, sizeof(test_req.byte)); + FRL_INFO("FRL TEST REQ: FLT_no_timeout = %d \n", test_req.fields.FLT_NO_TIMEOUT); + + if (test_req.fields.FLT_NO_TIMEOUT && ddc_service->link->frl_flags.no_timeout) { + bool de_emphasis_only = false; + bool pre_shoot_only = false; + bool no_ffe = false; + + read_idcc_data(ddc_service, HDMI_IDCC_SCOPE_RW_CA, + ddc_service->link->hdmi_cable_id.raw, 0, + sizeof(ddc_service->link->hdmi_cable_id.raw)); + + hdmi_return_preeshoot_and_deemphasis(ddc_service->link, &test_req, + &de_emphasis_only, &pre_shoot_only, &no_ffe); + + dio_link_enc->funcs->prog_eq_setting(dio_link_enc, 0, + de_emphasis_only, + pre_shoot_only, + no_ffe, + link_settings); + + return LINK_RESULT_NO_TIMEOUT; + } + hdmi_frl_LTS_clear_Link_Setting(ddc_service); offset = HDMI_SCDC_SINK_VERSION; @@ -511,13 +537,68 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s } } while (elapsed < poll_timeout_ns); - /*Test Condition - FLT_no_timeout avoid link training*/ - offset = HDMI_SCDC_SOURCE_TEST_REQ; - link_query_ddc_data(ddc_service, slave_address, &offset, - sizeof(offset), &test_req.byte, sizeof(test_req.byte)); - FRL_INFO("FRL TEST REQ: FLT_no_timeout = %d \n", test_req.fields.FLT_NO_TIMEOUT); - if (test_req.fields.FLT_NO_TIMEOUT) - flt_no_timeout = true; + if (test_req.fields.FLT_NO_TIMEOUT) { + ddc_service->link->frl_flags.no_timeout = true; + + /*Specify FRL rate*/ + write_buffer[0] = HDMI_SCDC_CONFIG_1; + /*FRL_RATE*/ + write_buffer[1] = (uint8_t)(link_settings->frl_link_rate | (FFE_Levels << 4)); + link_query_ddc_data(ddc_service, slave_address, + write_buffer, sizeof(write_buffer), NULL, 0); + FRL_INFO("FRL LINK TRAINING: Write link rate = %d. Max FFE_Levels = %d\n", + link_settings->frl_link_rate, FFE_Levels); + + offset = HDMI_SCDC_UPDATE_0; + /*Check FLT_update flag*/ + link_query_ddc_data(ddc_service, slave_address, + &offset, sizeof(offset), &scdc_update.byte[0], + sizeof(scdc_update.byte[0])); + + offset = HDMI_SCDC_LTP_REQ; + link_query_ddc_data(ddc_service, slave_address, + &offset, sizeof(offset), ltp_req.byte, + sizeof(ltp_req.byte)); + + ln0_pattern = ltp_req.fields.LN0_LTP_REQ; + ln1_pattern = ltp_req.fields.LN1_LTP_REQ; + ln2_pattern = ltp_req.fields.LN2_LTP_REQ; + ln3_pattern = ltp_req.fields.LN3_LTP_REQ; + + hpo_frl_link_enc->funcs->set_hdmi_training_pattern( + hpo_frl_link_enc, + ln0_pattern - 1, + ln1_pattern - 1, + ln2_pattern - 1, + ln3_pattern - 1); + + read_idcc_data(ddc_service, HDMI_IDCC_SCOPE_RW_CA, + ddc_service->link->hdmi_cable_id.raw, 0, + sizeof(ddc_service->link->hdmi_cable_id.raw)); + + bool de_emphasis_only = false; + bool pre_shoot_only = false; + bool no_ffe = false; + + hdmi_return_preeshoot_and_deemphasis(ddc_service->link, &test_req, + &de_emphasis_only, &pre_shoot_only, &no_ffe); + + dio_link_enc->funcs->prog_eq_setting(dio_link_enc, 0, + de_emphasis_only, + pre_shoot_only, + no_ffe, + link_settings); + + write_buffer[0] = HDMI_SCDC_UPDATE_0; + /*FLT_update - bit 5*/ + write_buffer[1] = (scdc_update.fields.FLT_UPDATE << 5); + link_query_ddc_data(ddc_service, slave_address, + write_buffer, sizeof(write_buffer), NULL, 0); + + return LINK_RESULT_NO_TIMEOUT; + } + + ddc_service->link->frl_flags.no_timeout = false; if (status_data.fields.FLT_READY) { /*Specify FRL rate*/ @@ -534,9 +615,7 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s /*Start FLT Timer = 200 ms, or 300ms if link rate >= 16Gbps*/ num_polls = 0; if (link_settings->frl_link_rate >= HDMI_FRL_LINK_RATE_16GBPS) - poll_timeout_ns = 300 * 1000 * 1000; - if (flt_no_timeout) - poll_timeout_ns = 1000 * 1000 * 1000; + poll_timeout_ns = 320 * 1000 * 1000; flt_poll_start = dm_get_timestamp(ddc_service->ctx); uint8_t prev_flt_update = 0; @@ -561,21 +640,9 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s prev_flt_update = scdc_update.fields.FLT_UPDATE; } - if (flt_no_timeout) { - offset = HDMI_SCDC_SOURCE_TEST_REQ; - link_query_ddc_data(ddc_service, slave_address, &offset, - sizeof(offset), &test_req.byte, sizeof(test_req.byte)); - if (test_req.fields.FLT_NO_TIMEOUT == 0) { - FRL_INFO("FRL TEST REQ: FLT_no_timeout = %d \n", test_req.fields.FLT_NO_TIMEOUT); - return LINK_RESULT_SUCCESS; - } - } - /*Set TxFFE = TxFFE0*/ /*Program FFE_Levels - scdc_config has this field at 0 */ if (override_FFE) { - if (flt_no_timeout) - current_FFE = 0; if (current_FFE == 0) dio_link_enc->funcs->prog_eq_setting(dio_link_enc, current_FFE, test_req.fields.TXFFE_DEEMPHASIS, @@ -626,8 +693,7 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s if (ln0_pattern == 0x03 || ln1_pattern == 0x03 || ln2_pattern == 0x03 || ln3_pattern == 0x03) - if (!flt_no_timeout) - continue; + continue; if (link_settings->frl_num_lanes == 3) { ln3_pattern = 1; @@ -651,8 +717,6 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s override_FFE = true; if (current_FFE > FFE_Levels) current_FFE = 0; - if (flt_no_timeout) - current_FFE = 0; } } else { if (!ln0_pattern && !ln1_pattern && !ln2_pattern && !ln3_pattern) { @@ -677,8 +741,6 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s override_FFE = true; if (current_FFE > FFE_Levels) current_FFE = 0; - if (flt_no_timeout) - current_FFE = 0; } } @@ -733,7 +795,7 @@ enum link_result hdmi_frl_perform_link_training_with_retries( /* FRL Link Training */ while (status != LINK_RESULT_FALLBACK) { - if (status == LINK_RESULT_SUCCESS) { + if (status == LINK_RESULT_SUCCESS || status == LINK_RESULT_NO_TIMEOUT) { break; } else if (status == LINK_RESULT_LOWER_LINKRATE) { FRL_INFO("FRL LINK TRAINING: Sink requested lower link rate during link enable. \n"); @@ -780,6 +842,9 @@ enum link_result hdmi_frl_perform_link_training_with_fallback( status = hdmi_frl_perform_link_training(link->ddc, &link->frl_link_settings); + if (status == LINK_RESULT_NO_TIMEOUT) + break; + link->dc->hwss.disable_link_output(link, link_res, SIGNAL_TYPE_HDMI_FRL); if (status == LINK_RESULT_SUCCESS) @@ -892,7 +957,7 @@ void hdmi_frl_verify_link_cap(struct dc_link *link, status = hdmi_frl_perform_link_training_with_fallback(link, &link_res, frl_phy_clock_source_id); - if (status == LINK_RESULT_SUCCESS) { + if (status == LINK_RESULT_SUCCESS || status == LINK_RESULT_NO_TIMEOUT) { cur->frl_link_rate = link->frl_link_settings.frl_link_rate; cur->frl_num_lanes = link->frl_link_settings.frl_num_lanes; success = true; diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h index 4b863a2e38f1..7db44fa24042 100644 --- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h @@ -449,6 +449,8 @@ struct amd_pm_funcs { int (*print_clock_levels)(void *handle, enum pp_clock_type type, char *buf); int (*emit_clock_levels)(void *handle, enum pp_clock_type type, char *buf, int *offset); int (*force_performance_level)(void *handle, enum amd_dpm_forced_level level); + int (*reapply_performance_level)(void *handle, + enum amd_dpm_forced_level level); int (*get_sclk_od)(void *handle); int (*set_sclk_od)(void *handle, uint32_t value); int (*get_mclk_od)(void *handle); @@ -530,6 +532,7 @@ struct amd_pm_funcs { ssize_t (*get_temp_metrics)(void *handle, enum smu_temp_metric_type type, void *table); bool (*temp_metrics_is_supported)(void *handle, enum smu_temp_metric_type type); u64 (*get_npm_cap)(void *handle); + int (*set_npm_limit)(void *handle, uint32_t limit); ssize_t (*get_xcp_metrics)(void *handle, int xcp_id, void *table); ssize_t (*get_pm_metrics)(void *handle, void *pmmetrics, size_t size); int (*set_watermarks_for_clock_ranges)(void *handle, diff --git a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c index df5bbdab1ea3..75d8bb4868e8 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c @@ -1082,6 +1082,91 @@ static void amdgpu_dpm_exit_umd_state(struct amdgpu_device *adev) AMD_PG_STATE_GATE); } +/** + * amdgpu_dpm_reapply_umd_profile - Restore the active UMD profile + * @adev: amdgpu device + * + * Resume and reset retain the selected DPM level in software, but the + * underlying GFX and PMFW state is lost. Reapply a retained UMD profile + * after the common resume path has restored its default CG and PG state. + */ +int amdgpu_dpm_reapply_umd_profile(struct amdgpu_device *adev) +{ + const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs; + enum amd_dpm_forced_level level; + int ret, fallback_ret; + + if (!pp_funcs || !pp_funcs->reapply_performance_level || + !pp_funcs->force_performance_level) + return 0; + + /* Serialize the retained level with the GFX and PMFW reprogramming. */ + mutex_lock(&adev->pm.mutex); + if (adev->pm.dpm.thermal_active) { + dev_warn(adev->dev, + "Cannot restore UMD profile while thermal control is active\n"); + ret = 0; + goto unlock; + } + + if (pp_funcs->get_performance_level) + level = pp_funcs->get_performance_level(adev->powerplay.pp_handle); + else + level = adev->pm.dpm.forced_level; + + switch (level) { + case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD: + case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK: + case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK: + case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK: + break; + default: + ret = 0; + goto unlock; + } + + /* The active profile already owns the GFX PG reference. */ + ret = amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_GFX, + AMD_CG_STATE_UNGATE); + if (ret) + goto fallback_auto; + + ret = amdgpu_asic_update_umd_stable_pstate(adev, true); + if (ret) + goto fallback_auto; + + ret = pp_funcs->reapply_performance_level(adev->powerplay.pp_handle, + level); + if (ret) + goto fallback_auto; + +unlock: + mutex_unlock(&adev->pm.mutex); + + return ret; + +fallback_auto: + dev_warn(adev->dev, "Failed to restore UMD profile state: %d\n", ret); + + /* Restore normal CG before releasing the profile's PG reference. */ + amdgpu_asic_update_umd_stable_pstate(adev, false); + amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_GFX, + AMD_CG_STATE_GATE); + mutex_unlock(&adev->pm.mutex); + + mutex_lock(&adev->pm.stable_pstate_ctx_lock); + fallback_ret = amdgpu_dpm_force_performance_level(adev, + AMD_DPM_FORCED_LEVEL_AUTO); + if (fallback_ret) + dev_warn(adev->dev, "Failed to fall back to AUTO performance level: %d\n", + fallback_ret); + else + adev->pm.stable_pstate_ctx = NULL; + mutex_unlock(&adev->pm.stable_pstate_ctx_lock); + + return 0; +} + int amdgpu_dpm_force_performance_level(struct amdgpu_device *adev, enum amd_dpm_forced_level level) { @@ -2124,6 +2209,24 @@ u64 amdgpu_dpm_get_npm_cap(struct amdgpu_device *adev) return cap; } +int amdgpu_dpm_set_npm_limit(struct amdgpu_device *adev, uint32_t limit) +{ + const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs; + int ret; + + if (!pp_funcs || !pp_funcs->set_npm_limit) + return -EOPNOTSUPP; + + if (amdgpu_sriov_vf(adev)) + return -EPERM; + + mutex_lock(&adev->pm.mutex); + ret = pp_funcs->set_npm_limit(adev->powerplay.pp_handle, limit); + mutex_unlock(&adev->pm.mutex); + + return ret; +} + /** * amdgpu_dpm_get_xcp_metrics - Retrieve metrics for a specific compute * partition diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c index c6e3748bb11b..48c52ac0fc77 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c @@ -2211,8 +2211,10 @@ out: /** * DOC: cur_node_power_limit * - * The amdgpu driver provides a sysfs API for retrieving current node power limit. - * The file cur_node_power_limit is used for this. + * The amdgpu driver provides a sysfs API for reading and writing the current node + * power limit in watts. The file cur_node_power_limit is used for this. Writing a + * new limit sends SetNpmLimit to PMFW. The file is writable only when the ASIC + * reports the write capability bit for this attribute. */ static ssize_t amdgpu_show_cur_node_power_limit(struct device *dev, struct device_attribute *attr, char *buf) @@ -2231,6 +2233,27 @@ static ssize_t amdgpu_show_cur_node_power_limit(struct device *dev, return sysfs_emit(buf, "%u\n", nplimit); } +static ssize_t amdgpu_store_cur_node_power_limit(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + u32 limit; + int r; + + r = amdgpu_pm_get_access_if_active(adev); + if (r) + return r; + + r = kstrtou32(buf, 0, &limit); + if (!r) + r = amdgpu_dpm_set_npm_limit(adev, limit); + + amdgpu_pm_put_access(adev); + return r ? r : count; +} + /** * DOC: node_power * @@ -2259,7 +2282,8 @@ static ssize_t amdgpu_show_node_power(struct device *dev, * * The amdgpu driver provides a sysfs API for retrieving current node power management status. * The file npm_status is used for this. It shows the status as enabled or disabled based on - * current node power value. If node power is zero, status is disabled else enabled. + * current node power or npm status value. If node power or npm status is zero, status is + * disabled else enabled. */ static ssize_t amdgpu_show_npm_status(struct device *dev, struct device_attribute *attr, char *buf) @@ -2371,7 +2395,8 @@ static ssize_t amdgpu_show_baseboard_power_limit(struct device *dev, static DEVICE_ATTR(baseboard_temp, 0444, amdgpu_get_baseboard_temp_metrics, NULL); static DEVICE_ATTR(gpuboard_temp, 0444, amdgpu_get_gpuboard_temp_metrics, NULL); -static DEVICE_ATTR(cur_node_power_limit, 0444, amdgpu_show_cur_node_power_limit, NULL); +static DEVICE_ATTR(cur_node_power_limit, 0644, amdgpu_show_cur_node_power_limit, + amdgpu_store_cur_node_power_limit); static DEVICE_ATTR(node_power, 0444, amdgpu_show_node_power, NULL); static DEVICE_ATTR(global_ppt_resid, 0444, amdgpu_show_global_ppt_resid, NULL); static DEVICE_ATTR(max_node_power_limit, 0444, amdgpu_show_max_node_power_limit, NULL); diff --git a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h index 64d68a905f86..ba8624d28f28 100644 --- a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h +++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h @@ -484,6 +484,7 @@ void amdgpu_dpm_set_power_state(struct amdgpu_device *adev, enum amd_dpm_forced_level amdgpu_dpm_get_performance_level(struct amdgpu_device *adev); int amdgpu_dpm_force_performance_level(struct amdgpu_device *adev, enum amd_dpm_forced_level level); +int amdgpu_dpm_reapply_umd_profile(struct amdgpu_device *adev); int amdgpu_dpm_get_pp_num_states(struct amdgpu_device *adev, struct pp_states_info *states); int amdgpu_dpm_dispatch_task(struct amdgpu_device *adev, @@ -617,5 +618,6 @@ bool amdgpu_dpm_reset_vcn_is_supported(struct amdgpu_device *adev); bool amdgpu_dpm_is_temp_metrics_supported(struct amdgpu_device *adev, enum smu_temp_metric_type type); u64 amdgpu_dpm_get_npm_cap(struct amdgpu_device *adev); +int amdgpu_dpm_set_npm_limit(struct amdgpu_device *adev, uint32_t limit); #endif diff --git a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c index 239c64b8edca..7b0ff5a0993e 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +++ b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c @@ -358,24 +358,46 @@ static void pp_dpm_en_umd_pstate(struct pp_hwmgr *hwmgr, } } -static int pp_dpm_force_performance_level(void *handle, - enum amd_dpm_forced_level level) +static int pp_dpm_apply_performance_level(struct pp_hwmgr *hwmgr, + enum amd_dpm_forced_level level, + bool reapply) { - struct pp_hwmgr *hwmgr = handle; + int ret; if (!hwmgr || !hwmgr->pm_en) return -EINVAL; - if (level == hwmgr->dpm_level) + if (!reapply && level == hwmgr->dpm_level) return 0; pp_dpm_en_umd_pstate(hwmgr, &level); hwmgr->request_dpm_level = level; - hwmgr_handle_task(hwmgr, AMD_PP_TASK_READJUST_POWER_STATE, NULL); + ret = hwmgr_handle_task(hwmgr, AMD_PP_TASK_READJUST_POWER_STATE, NULL); + if (ret) + return ret; + + ret = phm_force_dpm_levels(hwmgr, level); + if (ret) + return ret; + + hwmgr->dpm_level = level; return 0; } +static int pp_dpm_force_performance_level(void *handle, + enum amd_dpm_forced_level level) +{ + return pp_dpm_apply_performance_level(handle, level, false); +} + +static int +pp_dpm_reapply_performance_level(void *handle, + enum amd_dpm_forced_level level) +{ + return pp_dpm_apply_performance_level(handle, level, true); +} + static enum amd_dpm_forced_level pp_dpm_get_performance_level( void *handle) { @@ -1536,6 +1558,7 @@ static const struct amd_pm_funcs pp_dpm_funcs = { .load_firmware = pp_dpm_load_fw, .wait_for_fw_loading_complete = pp_dpm_fw_loading_complete, .force_performance_level = pp_dpm_force_performance_level, + .reapply_performance_level = pp_dpm_reapply_performance_level, .get_performance_level = pp_dpm_get_performance_level, .get_current_power_state = pp_dpm_get_current_power_state, .dispatch_tasks = pp_dpm_dispatch_tasks, diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index a65aa2129845..c9fe323fa624 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -2366,6 +2366,13 @@ static int smu_set_powergating_state(struct amdgpu_ip_block *ip_block, return 0; } +static void smu_enter_umd_pstate(struct smu_context *smu) +{ + smu_gpo_control(smu, false); + smu_gfx_ulv_control(smu, false); + smu_deep_sleep_control(smu, false); +} + static int smu_enable_umd_pstate(void *handle, enum amd_dpm_forced_level *level) { @@ -2384,9 +2391,7 @@ static int smu_enable_umd_pstate(void *handle, /* enter umd pstate, save current level, disable gfx cg*/ if (*level & profile_mode_mask) { smu_dpm_ctx->saved_dpm_level = smu_dpm_ctx->dpm_level; - smu_gpo_control(smu, false); - smu_gfx_ulv_control(smu, false); - smu_deep_sleep_control(smu, false); + smu_enter_umd_pstate(smu); amdgpu_asic_update_umd_stable_pstate(smu->adev, true); } } else { @@ -2445,7 +2450,8 @@ static void smu_power_profile_mode_put(struct smu_context *smu, static int smu_adjust_power_state_dynamic(struct smu_context *smu, enum amd_dpm_forced_level level, - bool skip_display_settings) + bool skip_display_settings, + bool force_performance_level) { int ret = 0; struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm); @@ -2472,7 +2478,7 @@ static int smu_adjust_power_state_dynamic(struct smu_context *smu, } } - if (smu_dpm_ctx->dpm_level != level) { + if (force_performance_level || smu_dpm_ctx->dpm_level != level) { ret = smu_asic_set_performance_level(smu, level); if (ret) { if (ret == -EOPNOTSUPP) @@ -2495,6 +2501,39 @@ static int smu_adjust_power_state_dynamic(struct smu_context *smu, return ret; } +static int +smu_reapply_power_state_dynamic(struct smu_context *smu, + enum amd_dpm_forced_level level) +{ + struct smu_dpm_context *smu_dpm_ctx = &smu->smu_dpm; + int ret; + + ret = smu_apply_clocks_adjust_rules(smu); + if (ret) { + dev_err(smu->adev->dev, "Failed to apply clocks adjust rules!"); + return ret; + } + + ret = smu_asic_set_performance_level(smu, level); + if (ret) { + if (ret == -EOPNOTSUPP) + dev_info(smu->adev->dev, "set performance level %d not supported", + level); + else + dev_err(smu->adev->dev, "Failed to set performance level %d", + level); + return ret; + } + + smu_dpm_ctx->dpm_level = level; + + if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL || + smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM) + return 0; + + return smu_bump_power_profile_mode(smu, NULL, 0); +} + static int smu_handle_task(struct smu_context *smu, enum amd_dpm_forced_level level, enum amd_pp_task task_id) @@ -2509,13 +2548,13 @@ static int smu_handle_task(struct smu_context *smu, ret = smu_pre_display_config_changed(smu); if (ret) return ret; - ret = smu_adjust_power_state_dynamic(smu, level, false); + ret = smu_adjust_power_state_dynamic(smu, level, false, false); break; case AMD_PP_TASK_COMPLETE_INIT: - ret = smu_adjust_power_state_dynamic(smu, level, true); + ret = smu_adjust_power_state_dynamic(smu, level, true, false); break; case AMD_PP_TASK_READJUST_POWER_STATE: - ret = smu_adjust_power_state_dynamic(smu, level, true); + ret = smu_adjust_power_state_dynamic(smu, level, true, false); break; default: break; @@ -2644,6 +2683,37 @@ static int smu_force_performance_level(void *handle, return ret; } +static int +smu_reapply_performance_level(void *handle, + enum amd_dpm_forced_level level) +{ + struct smu_context *smu = handle; + struct smu_dpm_context *smu_dpm_ctx = &smu->smu_dpm; + + if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) + return -EOPNOTSUPP; + + if (!smu->is_apu && !smu_dpm_ctx->dpm_context) + return -EINVAL; + + switch (level) { + case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD: + case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK: + case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK: + case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK: + break; + default: + return -EINVAL; + } + + if (smu_dpm_ctx->dpm_level != level) + return -EINVAL; + + smu_enter_umd_pstate(smu); + + return smu_reapply_power_state_dynamic(smu, level); +} + static int smu_set_display_count(void *handle, uint32_t count) { struct smu_context *smu = handle; @@ -3995,6 +4065,19 @@ static u64 smu_get_npm_cap(void *handle) return smu->ppt_funcs->get_npm_cap(smu); } +static int smu_set_npm_limit(void *handle, uint32_t limit) +{ + struct smu_context *smu = handle; + + if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) + return -EOPNOTSUPP; + + if (!smu->ppt_funcs || !smu->ppt_funcs->set_npm_limit) + return -EOPNOTSUPP; + + return smu->ppt_funcs->set_npm_limit(smu, limit); +} + static ssize_t smu_sys_get_xcp_metrics(void *handle, int xcp_id, void *table) { struct smu_context *smu = handle; @@ -4017,6 +4100,7 @@ static const struct amd_pm_funcs swsmu_pm_funcs = { .force_clock_level = smu_force_ppclk_levels, .emit_clock_levels = smu_emit_ppclk_levels, .force_performance_level = smu_force_performance_level, + .reapply_performance_level = smu_reapply_performance_level, .read_sensor = smu_read_sensor, .get_apu_thermal_limit = smu_get_apu_thermal_limit, .set_apu_thermal_limit = smu_set_apu_thermal_limit, @@ -4069,6 +4153,7 @@ static const struct amd_pm_funcs swsmu_pm_funcs = { .get_xcp_metrics = smu_sys_get_xcp_metrics, .get_temp_metrics = smu_sys_get_temp_metrics, .temp_metrics_is_supported = smu_temp_metrics_is_supported, + .set_npm_limit = smu_set_npm_limit, .get_npm_cap = smu_get_npm_cap, }; diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h index 1e5fc6ff76c8..9d47d2ab1b49 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h @@ -433,6 +433,8 @@ struct smu_table_context { void *boot_overdrive_table; void *user_overdrive_table; + uint32_t user_od_feature_mask; + struct smu_driver_table driver_tables[SMU_DRIVER_TABLE_COUNT]; }; @@ -1626,6 +1628,12 @@ struct pptable_funcs { * unsupported. */ u64 (*get_npm_cap)(struct smu_context *smu); + + /** + * @set_npm_limit: Set node power limit in watts. + */ + int (*set_npm_limit)(struct smu_context *smu, uint32_t limit); + /** * @ras_send_msg: Send a message with a parameter from Ras * &msg: Type of message. diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v15_0_8_pmfw.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v15_0_8_pmfw.h index a3401c4cc20b..61d76ebfdbd4 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v15_0_8_pmfw.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v15_0_8_pmfw.h @@ -328,7 +328,7 @@ typedef struct __attribute__((packed, aligned(4))) { uint64_t GfxclkLowUtilizationAcc[8]; } MetricsTable_t; -#define SMU_SYSTEM_METRICS_TABLE_VERSION 0x1 +#define SMU_SYSTEM_METRICS_TABLE_VERSION 0x2 #pragma pack(push, 4) typedef struct { @@ -343,9 +343,13 @@ typedef struct { //NPM: NODE POWER MANAGEMENT uint32_t NodePowerLimit; uint32_t NodePower; - uint32_t GlobalPPTResidencyAcc; + uint32_t NPMResidencyAcc; uint16_t SystemPower[SYSTEM_POWER_MAX_ENTRIES]; // UBB Current Power and Power Threshold + + // NPM flags + uint32_t NpmSupported; + uint32_t NpmEnabled; } SystemMetricsTable_t; #pragma pack(pop) @@ -368,7 +372,7 @@ typedef struct __attribute__((aligned(4))) { uint8_t FruId[PRODUCT_FRU_ID_LEN]; } FRUProductInfo_t; -#define SMU_STATIC_METRICS_TABLE_VERSION 0x1 +#define SMU_STATIC_METRICS_TABLE_VERSION 0x2 #pragma pack(push, 4) typedef struct { @@ -421,6 +425,8 @@ typedef struct { uint32_t PPT1Max; uint32_t PPT1Min; uint32_t PPT1Default; + + uint32_t MaxNodePowerLimit; } StaticMetricsTable_t; #pragma pack(pop) diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v15_0_8_ppsmc.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v15_0_8_ppsmc.h index 7ffb445f4c0c..888261c26409 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v15_0_8_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v15_0_8_ppsmc.h @@ -83,7 +83,10 @@ #define PPSMC_MSG_SetSoftMaxGl2clk 0x32 #define PPSMC_MSG_SetSoftMinFclk 0x33 #define PPSMC_MSG_SetSoftMaxFclk 0x34 -#define PPSMC_Message_Count 0x35 +/* NPM limit setter; NPM reads are served from the SystemMetricsTable. */ +#define PPSMC_MSG_SetNpmLimit 0x40 +#define PPSMC_MSG_PrepareNodeForUnload 0x41 +#define PPSMC_Message_Count 0x42 /* PSMC Reset Types for driver msg argument */ #define PPSMC_RESET_TYPE_DRIVER_MODE_1_RESET 0x1 diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h index 2631e12b5e59..31b4ef025351 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h @@ -305,7 +305,9 @@ __SMU_DUMMY_MAP(SetSoftMaxFclk), \ __SMU_DUMMY_MAP(SetSoftMaxGl2clk), \ __SMU_DUMMY_MAP(SetSoftMinGl2clk), \ - __SMU_DUMMY_MAP(GetSystemMetricsVersion), + __SMU_DUMMY_MAP(GetSystemMetricsVersion), \ + __SMU_DUMMY_MAP(SetNpmLimit), \ + __SMU_DUMMY_MAP(PrepareNodeForUnload), #undef __SMU_DUMMY_MAP #define __SMU_DUMMY_MAP(type) SMU_MSG_##type diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h index 0c97bee2c4bb..f443000ba657 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h @@ -89,6 +89,7 @@ struct smu_15_0_dpm_context { uint32_t workload_policy_mask; uint32_t dcef_min_ds_clk; uint64_t caps; + bool node_unload; uint32_t board_volt; }; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c index 6beafabf1306..468aaeb79ffb 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c @@ -1569,7 +1569,6 @@ static int smu_v13_0_0_od_edit_dpm_table(struct smu_context *smu, struct amdgpu_device *adev = smu->adev; uint32_t offset_of_voltageoffset; int32_t minimum, maximum; - uint32_t feature_ctrlmask; int i, ret = 0; switch (type) { @@ -1891,21 +1890,16 @@ static int smu_v13_0_0_od_edit_dpm_table(struct smu_context *smu, if (ret) return ret; } else { - feature_ctrlmask = od_table->OverDriveTable.FeatureCtrlMask; memcpy(od_table, table_context->boot_overdrive_table, sizeof(OverDriveTableExternal_t)); - od_table->OverDriveTable.FeatureCtrlMask = feature_ctrlmask; + od_table->OverDriveTable.FeatureCtrlMask = 0; + table_context->user_od_feature_mask = 0; } fallthrough; case PP_OD_COMMIT_DPM_TABLE: - /* - * The member below instructs PMFW the settings focused in - * this single operation. - * `uint32_t FeatureCtrlMask;` - * It does not contain actual informations about user's custom - * settings. Thus we do not cache it. - */ + table_context->user_od_feature_mask = + od_table->OverDriveTable.FeatureCtrlMask; offset_of_voltageoffset = offsetof(OverDriveTable_t, VoltageOffsetPerZoneBoundary); if (memcmp((u8 *)od_table + offset_of_voltageoffset, table_context->user_overdrive_table + offset_of_voltageoffset, @@ -1918,7 +1912,6 @@ static int smu_v13_0_0_od_edit_dpm_table(struct smu_context *smu, return ret; } - od_table->OverDriveTable.FeatureCtrlMask = 0; memcpy(table_context->user_overdrive_table + offset_of_voltageoffset, (u8 *)od_table + offset_of_voltageoffset, sizeof(OverDriveTableExternal_t) - offset_of_voltageoffset); @@ -2201,6 +2194,7 @@ static int smu_v13_0_0_set_default_od_settings(struct smu_context *smu) boot_od_table, sizeof(OverDriveTableExternal_t)); smu->user_dpm_profile.user_od = false; + smu->smu_table.user_od_feature_mask = 0; } else if (smu->user_dpm_profile.user_od) { memcpy(&user_od_table_bak, user_od_table, @@ -2250,12 +2244,8 @@ static int smu_v13_0_0_restore_user_od_settings(struct smu_context *smu) OverDriveTableExternal_t *od_table = table_context->overdrive_table; OverDriveTableExternal_t *user_od_table = table_context->user_overdrive_table; int res; - - user_od_table->OverDriveTable.FeatureCtrlMask = BIT(PP_OD_FEATURE_GFXCLK_BIT) | - BIT(PP_OD_FEATURE_UCLK_BIT) | - BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT) | - BIT(PP_OD_FEATURE_FAN_CURVE_BIT) | - BIT(PP_OD_FEATURE_ZERO_FAN_BIT); + user_od_table->OverDriveTable.FeatureCtrlMask = + table_context->user_od_feature_mask; res = smu_v13_0_0_upload_overdrive_table(smu, user_od_table); user_od_table->OverDriveTable.FeatureCtrlMask = 0; if (res == 0) @@ -3069,8 +3059,10 @@ static int smu_v13_0_0_set_ppt_limit(struct smu_context *smu, if (current_od_table.OverDriveTable.Ppt) { od_table->OverDriveTable.Ppt = 0; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT; - + table_context->user_od_feature_mask &= + ~(1U << PP_OD_FEATURE_PPT_BIT); + od_table->OverDriveTable.FeatureCtrlMask = + table_context->user_od_feature_mask; ret = smu_v13_0_0_upload_overdrive_table(smu, od_table); if (ret) { dev_err(smu->adev->dev, "Failed to upload overdrive table!\n"); @@ -3084,8 +3076,9 @@ static int smu_v13_0_0_set_ppt_limit(struct smu_context *smu, return ret; od_table->OverDriveTable.Ppt = (limit * 100) / msg_limit - 100; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT; - + table_context->user_od_feature_mask |= 1U << PP_OD_FEATURE_PPT_BIT; + od_table->OverDriveTable.FeatureCtrlMask = + table_context->user_od_feature_mask; ret = smu_v13_0_0_upload_overdrive_table(smu, od_table); if (ret) { dev_err(smu->adev->dev, "Failed to upload overdrive table!\n"); diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c index 5fe409a23772..51e419e6380c 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c @@ -1578,7 +1578,6 @@ static int smu_v13_0_7_od_edit_dpm_table(struct smu_context *smu, struct amdgpu_device *adev = smu->adev; uint32_t offset_of_voltageoffset; int32_t minimum, maximum; - uint32_t feature_ctrlmask; int i, ret = 0; switch (type) { @@ -1900,22 +1899,17 @@ static int smu_v13_0_7_od_edit_dpm_table(struct smu_context *smu, if (ret) return ret; } else { - feature_ctrlmask = od_table->OverDriveTable.FeatureCtrlMask; memcpy(od_table, table_context->boot_overdrive_table, sizeof(OverDriveTableExternal_t)); - od_table->OverDriveTable.FeatureCtrlMask = feature_ctrlmask; + od_table->OverDriveTable.FeatureCtrlMask = 0; + table_context->user_od_feature_mask = 0; } fallthrough; case PP_OD_COMMIT_DPM_TABLE: - /* - * The member below instructs PMFW the settings focused in - * this single operation. - * `uint32_t FeatureCtrlMask;` - * It does not contain actual informations about user's custom - * settings. Thus we do not cache it. - */ + table_context->user_od_feature_mask = + od_table->OverDriveTable.FeatureCtrlMask; offset_of_voltageoffset = offsetof(OverDriveTable_t, VoltageOffsetPerZoneBoundary); if (memcmp((u8 *)od_table + offset_of_voltageoffset, table_context->user_overdrive_table + offset_of_voltageoffset, @@ -1928,7 +1922,6 @@ static int smu_v13_0_7_od_edit_dpm_table(struct smu_context *smu, return ret; } - od_table->OverDriveTable.FeatureCtrlMask = 0; memcpy(table_context->user_overdrive_table + offset_of_voltageoffset, (u8 *)od_table + offset_of_voltageoffset, sizeof(OverDriveTableExternal_t) - offset_of_voltageoffset); @@ -2203,6 +2196,7 @@ static int smu_v13_0_7_set_default_od_settings(struct smu_context *smu) boot_od_table, sizeof(OverDriveTableExternal_t)); smu->user_dpm_profile.user_od = false; + smu->smu_table.user_od_feature_mask = 0; } else if (smu->user_dpm_profile.user_od) { memcpy(&user_od_table_bak, user_od_table, @@ -2253,11 +2247,8 @@ static int smu_v13_0_7_restore_user_od_settings(struct smu_context *smu) OverDriveTableExternal_t *user_od_table = table_context->user_overdrive_table; int res; - user_od_table->OverDriveTable.FeatureCtrlMask = BIT(PP_OD_FEATURE_GFXCLK_BIT) | - BIT(PP_OD_FEATURE_UCLK_BIT) | - BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT) | - BIT(PP_OD_FEATURE_FAN_CURVE_BIT) | - BIT(PP_OD_FEATURE_ZERO_FAN_BIT); + user_od_table->OverDriveTable.FeatureCtrlMask = + table_context->user_od_feature_mask; res = smu_v13_0_7_upload_overdrive_table(smu, user_od_table); user_od_table->OverDriveTable.FeatureCtrlMask = 0; if (res == 0) @@ -2689,7 +2680,10 @@ static int smu_v13_0_7_set_ppt_limit(struct smu_context *smu, if (current_od_table.OverDriveTable.Ppt) { od_table->OverDriveTable.Ppt = 0; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT; + table_context->user_od_feature_mask &= + ~(1U << PP_OD_FEATURE_PPT_BIT); + od_table->OverDriveTable.FeatureCtrlMask = + table_context->user_od_feature_mask; ret = smu_v13_0_7_upload_overdrive_table(smu, od_table); if (ret) { @@ -2704,7 +2698,9 @@ static int smu_v13_0_7_set_ppt_limit(struct smu_context *smu, return ret; od_table->OverDriveTable.Ppt = (limit * 100) / msg_limit - 100; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT; + table_context->user_od_feature_mask |= 1U << PP_OD_FEATURE_PPT_BIT; + od_table->OverDriveTable.FeatureCtrlMask = + table_context->user_od_feature_mask; ret = smu_v13_0_7_upload_overdrive_table(smu, od_table); if (ret) { diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c index ed99e61f18e1..549716349117 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c @@ -2360,6 +2360,7 @@ static int smu_v14_0_2_set_default_od_settings(struct smu_context *smu) boot_od_table, sizeof(OverDriveTableExternal_t)); smu->user_dpm_profile.user_od = false; + smu->smu_table.user_od_feature_mask = 0; } else if (smu->user_dpm_profile.user_od) { memcpy(&user_od_table_bak, user_od_table, @@ -2406,11 +2407,8 @@ static int smu_v14_0_2_restore_user_od_settings(struct smu_context *smu) OverDriveTableExternal_t *user_od_table = table_context->user_overdrive_table; int res; - user_od_table->OverDriveTable.FeatureCtrlMask = BIT(PP_OD_FEATURE_GFXCLK_BIT) | - BIT(PP_OD_FEATURE_UCLK_BIT) | - BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT) | - BIT(PP_OD_FEATURE_FAN_CURVE_BIT) | - BIT(PP_OD_FEATURE_ZERO_FAN_BIT); + user_od_table->OverDriveTable.FeatureCtrlMask = + table_context->user_od_feature_mask; res = smu_v14_0_2_upload_overdrive_table(smu, user_od_table); user_od_table->OverDriveTable.FeatureCtrlMask = 0; if (res == 0) @@ -2490,7 +2488,6 @@ static int smu_v14_0_2_od_edit_dpm_table(struct smu_context *smu, struct amdgpu_device *adev = smu->adev; uint32_t offset_of_voltageoffset; int32_t minimum, maximum; - uint32_t feature_ctrlmask; int i, ret = 0; switch (type) { @@ -2758,21 +2755,26 @@ static int smu_v14_0_2_od_edit_dpm_table(struct smu_context *smu, if (ret) return ret; } else { - feature_ctrlmask = od_table->OverDriveTable.FeatureCtrlMask; memcpy(od_table, table_context->boot_overdrive_table, sizeof(OverDriveTableExternal_t)); - od_table->OverDriveTable.FeatureCtrlMask = feature_ctrlmask; + od_table->OverDriveTable.FeatureCtrlMask = 0; + table_context->user_od_feature_mask = 0; } fallthrough; case PP_OD_COMMIT_DPM_TABLE: /* - * The member below instructs PMFW the settings focused in - * this single operation. - * `uint32_t FeatureCtrlMask;` - * It does not contain actual informations about user's custom - * settings. Thus we do not cache it. + * PMFW interprets FeatureCtrlMask as the complete set of OD + * features that are currently overridden, not as a delta + * describing this single operation. Any feature whose bit is + * clear gets reset to its default on import. od_table's mask + * has accumulated every feature the user touched since the + * last full restore, so record it and keep restating it on + * subsequent uploads. */ + table_context->user_od_feature_mask = + od_table->OverDriveTable.FeatureCtrlMask; + offset_of_voltageoffset = offsetof(OverDriveTable_t, VoltageOffsetPerZoneBoundary); if (memcmp((u8 *)od_table + offset_of_voltageoffset, table_context->user_overdrive_table + offset_of_voltageoffset, @@ -2785,7 +2787,6 @@ static int smu_v14_0_2_od_edit_dpm_table(struct smu_context *smu, return ret; } - od_table->OverDriveTable.FeatureCtrlMask = 0; memcpy(table_context->user_overdrive_table + offset_of_voltageoffset, (u8 *)od_table + offset_of_voltageoffset, sizeof(OverDriveTableExternal_t) - offset_of_voltageoffset); @@ -2829,7 +2830,10 @@ static int smu_v14_0_2_set_ppt_limit(struct smu_context *smu, if (current_od_table.OverDriveTable.Ppt) { od_table->OverDriveTable.Ppt = 0; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT; + table_context->user_od_feature_mask &= + ~(1U << PP_OD_FEATURE_PPT_BIT); + od_table->OverDriveTable.FeatureCtrlMask = + table_context->user_od_feature_mask; ret = smu_v14_0_2_upload_overdrive_table(smu, od_table); if (ret) { @@ -2844,7 +2848,9 @@ static int smu_v14_0_2_set_ppt_limit(struct smu_context *smu, return ret; od_table->OverDriveTable.Ppt = (limit * 100) / msg_limit - 100; - od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT; + table_context->user_od_feature_mask |= 1U << PP_OD_FEATURE_PPT_BIT; + od_table->OverDriveTable.FeatureCtrlMask = + table_context->user_od_feature_mask; ret = smu_v14_0_2_upload_overdrive_table(smu, od_table); if (ret) { diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c index 9e42c27f60ee..ce56d540e85e 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c @@ -25,6 +25,7 @@ #include <linux/firmware.h> #include "amdgpu.h" +#include "amdgpu_xgmi.h" #include "amdgpu_smu.h" #include "smu_v15_0_8_pmfw.h" #include "smu15_driver_if_v15_0_8.h" @@ -137,6 +138,8 @@ static const struct cmn2asic_msg_mapping smu_v15_0_8_message_map[SMU_MSG_MAX_COU MSG_MAP(SetSoftMaxGl2clk, PPSMC_MSG_SetSoftMaxGl2clk, 0), MSG_MAP(SetSoftMinFclk, PPSMC_MSG_SetSoftMinFclk, 0), MSG_MAP(SetSoftMaxFclk, PPSMC_MSG_SetSoftMaxFclk, 0), + MSG_MAP(SetNpmLimit, PPSMC_MSG_SetNpmLimit, 0), + MSG_MAP(PrepareNodeForUnload, PPSMC_MSG_PrepareNodeForUnload, 0), }; /* TODO: Update the clk map once enum PPCLK is updated in smu15_driver_if_v15_0_8.h */ @@ -588,19 +591,99 @@ static int smu_v15_0_8_get_system_metrics_table(struct smu_context *smu) return 0; } +static inline void smu_v15_0_8_cap_set(struct smu_context *smu, u32 cap) +{ + struct smu_15_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context; + + dpm_context->caps |= BIT_ULL(cap); +} + +static bool smu_v15_0_8_cap_supported(struct smu_context *smu, u32 cap) +{ + struct smu_15_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context; + + return !!(dpm_context->caps & BIT_ULL(cap)); +} + +static void smu_v15_0_8_init_npm_cap(struct smu_context *smu) +{ + struct smu_table_context *smu_table = &smu->smu_table; + struct smu_table *tables = smu_table->tables; + SystemMetricsTable_t *metrics; + struct smu_table *sys_table; + uint32_t table_version; + int ret; + + if (smu->adev->gmc.xgmi.physical_node_id != 0) + return; + + ret = smu_cmn_send_smc_msg(smu, SMU_MSG_GetSystemMetricsVersion, + &table_version); + if (ret) + return; + + if (table_version < SMU_SYSTEM_METRICS_TABLE_VERSION) + return; + + ret = smu_v15_0_8_get_system_metrics_table(smu); + if (ret) + return; + + sys_table = &tables[SMU_TABLE_PMFW_SYSTEM_METRICS]; + metrics = (SystemMetricsTable_t *)sys_table->cache.buffer; + if (!metrics->NpmSupported) + return; + + smu_v15_0_8_cap_set(smu, SMU_15_0_8_CAP_NPM); +} + +static void smu_v15_0_8_init_node_unload_cap(struct smu_context *smu) +{ + uint32_t fw_ver = smu->smc_fw_version; + uint32_t pgm = (fw_ver >> 24) & 0xFF; + + if (smu_v15_0_8_cap_supported(smu, SMU_15_0_8_CAP_NODE_UNLOAD)) + return; + + if (pgm == 2 && fw_ver >= SMU_15_0_8_NODE_UNLOAD_MIN_FW_VERSION) + smu_v15_0_8_cap_set(smu, SMU_15_0_8_CAP_NODE_UNLOAD); +} + +static void smu_v15_0_8_init_caps(struct smu_context *smu) +{ + smu_v15_0_8_init_npm_cap(smu); + smu_v15_0_8_init_node_unload_cap(smu); +} + +static u64 smu_v15_0_8_get_npm_cap(struct smu_context *smu) +{ + if (!smu_v15_0_8_cap_supported(smu, SMU_15_0_8_CAP_NPM)) + return 0; + + return AMDGPU_NPM_CAP_R(AMDGPU_NPM_CAP_CUR_NODE_POWER_LIMIT) | + AMDGPU_NPM_CAP_W(AMDGPU_NPM_CAP_CUR_NODE_POWER_LIMIT) | + AMDGPU_NPM_CAP_R(AMDGPU_NPM_CAP_NODE_POWER) | + AMDGPU_NPM_CAP_R(AMDGPU_NPM_CAP_GLOBAL_PPT_RESID) | + AMDGPU_NPM_CAP_R(AMDGPU_NPM_CAP_MAX_NODE_POWER_LIMIT) | + AMDGPU_NPM_CAP_R(AMDGPU_NPM_CAP_NPM_STATUS); +} + static int smu_v15_0_8_get_npm_data(struct smu_context *smu, enum amd_pp_sensors sensor, uint32_t *value) { struct smu_table_context *smu_table = &smu->smu_table; + PPTable_t *pptable = (PPTable_t *)smu_table->driver_pptable; struct smu_table *tables = smu_table->tables; SystemMetricsTable_t *metrics; struct smu_table *sys_table; int ret; + if (!smu_v15_0_8_cap_supported(smu, SMU_15_0_8_CAP_NPM)) + return -EOPNOTSUPP; + if (sensor == AMDGPU_PP_SENSOR_MAXNODEPOWERLIMIT) { - /*TBD as of now put 0 */ - *value = 0; + *value = pptable->MaxNodePowerLimit; return 0; } @@ -619,10 +702,10 @@ static int smu_v15_0_8_get_npm_data(struct smu_context *smu, *value = SMUQ10_ROUND(metrics->NodePower); break; case AMDGPU_PP_SENSOR_NPMSTATUS: - *value = !!SMUQ10_ROUND(metrics->NodePower); + *value = !!metrics->NpmEnabled; break; case AMDGPU_PP_SENSOR_GPPTRESIDENCY: - *value = SMUQ10_ROUND(metrics->GlobalPPTResidencyAcc); + *value = SMUQ10_ROUND(metrics->NPMResidencyAcc); break; default: return -EINVAL; @@ -631,6 +714,15 @@ static int smu_v15_0_8_get_npm_data(struct smu_context *smu, return 0; } +static int smu_v15_0_8_set_npm_limit(struct smu_context *smu, uint32_t limit) +{ + if (!smu_v15_0_8_cap_supported(smu, SMU_15_0_8_CAP_NPM)) + return -EOPNOTSUPP; + + return smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetNpmLimit, + limit, NULL); +} + static int smu_v15_0_8_read_sensor(struct smu_context *smu, enum amd_pp_sensors sensor, void *data, uint32_t *size) @@ -1149,6 +1241,10 @@ static int smu_v15_0_8_set_driver_pptable(struct smu_context *smu) static_metrics->pldmVersion[0]; dpm_context->board_volt = static_metrics->InputTelemetryVoltageInmV; smu_v15_0_8_init_xgmi_data(smu, static_metrics); + smu_v15_0_8_init_caps(smu); + if (smu_v15_0_8_cap_supported(smu, SMU_15_0_8_CAP_NPM)) + pptable->MaxNodePowerLimit = + static_metrics->MaxNodePowerLimit; pptable->init = true; } @@ -1285,8 +1381,46 @@ static int smu_v15_0_8_register_irq_handler(struct smu_context *smu) return ret; } +static void smu_v15_0_8_notify_node_unload(struct smu_context *smu) +{ + if (!smu_v15_0_8_cap_supported(smu, SMU_15_0_8_CAP_NODE_UNLOAD)) + return; + + smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PrepareNodeForUnload, 0, NULL); +} + +static void smu_v15_0_8_mark_node_unload(struct smu_context *smu) +{ + struct amdgpu_device *adev = smu->adev; + struct amdgpu_hive_info *hive; + struct amdgpu_device *peer; + struct smu_context *peer_smu; + struct smu_15_0_dpm_context *dpm; + + hive = amdgpu_get_xgmi_hive(adev); + if (!hive) { + dpm = smu->smu_dpm.dpm_context; + if (dpm) + dpm->node_unload = true; + return; + } + + list_for_each_entry(peer, &hive->device_list, gmc.xgmi.head) { + peer_smu = peer->powerplay.pp_handle; + if (!peer_smu) + continue; + dpm = peer_smu->smu_dpm.dpm_context; + if (dpm) + dpm->node_unload = true; + } + + amdgpu_put_xgmi_hive(hive); +} + static int smu_v15_0_8_notify_unload(struct smu_context *smu) { + struct smu_context *primary_smu; + struct smu_15_0_dpm_context *dpm = smu->smu_dpm.dpm_context; struct smu_msg_ctl *ctl = &smu->msg_ctl; struct smu_msg_args args = { .msg = SMU_MSG_PrepareMp1ForUnload, @@ -1296,6 +1430,13 @@ static int smu_v15_0_8_notify_unload(struct smu_context *smu) if (amdgpu_in_reset(smu->adev)) return 0; + if (dpm && !dpm->node_unload) { + primary_smu = smu_get_primary_dev(smu); + if (primary_smu) + smu_v15_0_8_notify_node_unload(primary_smu); + smu_v15_0_8_mark_node_unload(smu); + } + dev_dbg(smu->adev->dev, "Notify PMFW about driver unload"); /* Ignore return, just intimate FW that driver is not going to be there */ ctl->ops->send_msg(ctl, &args); @@ -2382,6 +2523,8 @@ static const struct pptable_funcs smu_v15_0_8_ppt_funcs = { .get_unique_id = smu_v15_0_8_get_unique_id, .get_ppt_limit = smu_v15_0_8_get_ppt_limit, .set_ppt_limit = smu_v15_0_8_set_ppt_limit, + .get_npm_cap = smu_v15_0_8_get_npm_cap, + .set_npm_limit = smu_v15_0_8_set_npm_limit, .emit_clk_levels = smu_v15_0_8_emit_clk_levels, .read_sensor = smu_v15_0_8_read_sensor, .populate_umd_state_clk = smu_v15_0_8_populate_umd_state_clk, diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h index 46fce0a79c21..dff601bc0f9a 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h +++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h @@ -32,6 +32,9 @@ #define SMU_15_0_8_MAX_AID 2 #define SMU_15_0_8_MAX_MID 2 #define SMU_15_0_8_MAX_HBM_STACKS 12 +#define SMU_15_0_8_CAP_NPM 1 +#define SMU_15_0_8_CAP_NODE_UNLOAD 2 +#define SMU_15_0_8_NODE_UNLOAD_MIN_FW_VERSION 0x027d0c00u /* 2.125.12.0 */ extern void smu_v15_0_8_set_ppt_funcs(struct smu_context *smu); typedef struct { @@ -61,6 +64,7 @@ typedef struct { uint32_t PPT1Max; uint32_t PPT1Min; uint32_t PPT1Default; + uint32_t MaxNodePowerLimit; bool init; } PPTable_t; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c index 2bd3ea17e789..ff59bf19c3e7 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c @@ -27,6 +27,32 @@ #include "smu_cmn.h" #include "soc15_common.h" +#include "amdgpu_xgmi.h" + +struct smu_context *smu_get_primary_dev(struct smu_context *smu) +{ + struct amdgpu_device *adev = smu->adev; + struct amdgpu_hive_info *hive; + struct amdgpu_device *peer; + + if (adev->gmc.xgmi.physical_node_id == 0) + return smu; + + hive = amdgpu_get_xgmi_hive(adev); + if (!hive) + return NULL; + + list_for_each_entry(peer, &hive->device_list, gmc.xgmi.head) { + if (peer->gmc.xgmi.physical_node_id == 0) { + amdgpu_put_xgmi_hive(hive); + return peer->powerplay.pp_handle; + } + } + + amdgpu_put_xgmi_hive(hive); + return NULL; +} + /* * DO NOT use these for err/warn/info/debug messages. * Use dev_err, dev_warn, dev_info and dev_dbg instead. diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h index ae6742f5298f..6d7f47c934f2 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h @@ -249,6 +249,8 @@ int smu_cmn_dpm_pcie_gen_idx(int gen); int smu_cmn_dpm_pcie_width_idx(int width); int smu_cmn_check_fw_version(struct smu_context *smu); +struct smu_context *smu_get_primary_dev(struct smu_context *smu); + int smu_cmn_get_pptable_from_firmware(struct smu_context *smu, void **table, uint32_t *size, uint32_t pptable_id); |