summaryrefslogtreecommitdiff
path: root/drivers/accel
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/accel')
-rw-r--r--drivers/accel/amdxdna/aie2_ctx.c87
-rw-r--r--drivers/accel/amdxdna/aie2_message.c4
-rw-r--r--drivers/accel/amdxdna/amdxdna_ctx.c74
-rw-r--r--drivers/accel/amdxdna/amdxdna_ctx.h1
-rw-r--r--drivers/accel/amdxdna/amdxdna_gem.c50
-rw-r--r--drivers/accel/amdxdna/amdxdna_gem.h12
-rw-r--r--drivers/accel/amdxdna/amdxdna_iommu.c43
-rw-r--r--drivers/accel/amdxdna/amdxdna_pci_drv.c59
-rw-r--r--drivers/accel/amdxdna/amdxdna_pci_drv.h1
-rw-r--r--drivers/accel/ethosu/ethosu_drv.c1
-rw-r--r--drivers/accel/ivpu/ivpu_fw_log.c4
-rw-r--r--drivers/accel/ivpu/ivpu_hw_btrs.c2
-rw-r--r--drivers/accel/qaic/qaic_timesync.c1
-rw-r--r--drivers/accel/qaic/sahara.c1
14 files changed, 235 insertions, 105 deletions
diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c
index e9fbd8c14364..101f324ee178 100644
--- a/drivers/accel/amdxdna/aie2_ctx.c
+++ b/drivers/accel/amdxdna/aie2_ctx.c
@@ -59,6 +59,18 @@ static bool aie2_tdr_detect(struct amdxdna_dev *xdna)
return false;
}
+static void aie2_cmd_release(struct kref *ref)
+{
+ struct amdxdna_drv_cmd *drv_cmd = container_of(ref, struct amdxdna_drv_cmd, refcnt);
+
+ kfree(drv_cmd);
+}
+
+static void aie2_cmd_put(struct amdxdna_drv_cmd *drv_cmd)
+{
+ kref_put(&drv_cmd->refcnt, aie2_cmd_release);
+}
+
static void aie2_job_release(struct kref *ref)
{
struct amdxdna_sched_job *job;
@@ -70,6 +82,8 @@ static void aie2_job_release(struct kref *ref)
wake_up(&job->hwctx->priv->job_free_wq);
if (job->out_fence)
dma_fence_put(job->out_fence);
+ if (job->drv_cmd)
+ aie2_cmd_put(job->drv_cmd);
kfree(job->aie2_job_health);
kfree(job);
}
@@ -861,7 +875,7 @@ static int aie2_hwctx_cu_config(struct amdxdna_hwctx *hwctx, void *buf, u32 size
if (!hwctx->cus)
return -ENOMEM;
- ret = amdxdna_pm_resume_get_locked(xdna);
+ ret = amdxdna_pm_resume_get(xdna);
if (ret)
goto free_cus;
@@ -886,13 +900,16 @@ free_cus:
static void aie2_cmd_wait(struct amdxdna_hwctx *hwctx, u64 seq)
{
struct dma_fence *out_fence = aie2_cmd_get_out_fence(hwctx, seq);
+ struct amdxdna_dev *xdna = hwctx->client->xdna;
if (!out_fence) {
- XDNA_ERR(hwctx->client->xdna, "Failed to get fence");
+ XDNA_ERR(xdna, "Failed to get fence");
return;
}
+ mutex_unlock(&xdna->dev_lock);
dma_fence_wait_timeout(out_fence, false, MAX_SCHEDULE_TIMEOUT);
+ mutex_lock(&xdna->dev_lock);
dma_fence_put(out_fence);
}
@@ -901,7 +918,7 @@ static int aie2_hwctx_cfg_debug_bo(struct amdxdna_hwctx *hwctx, u32 bo_hdl,
{
struct amdxdna_client *client = hwctx->client;
struct amdxdna_dev *xdna = client->xdna;
- struct amdxdna_drv_cmd cmd = { 0 };
+ struct amdxdna_drv_cmd *cmd;
struct amdxdna_gem_obj *abo;
u64 seq;
int ret;
@@ -912,32 +929,39 @@ static int aie2_hwctx_cfg_debug_bo(struct amdxdna_hwctx *hwctx, u32 bo_hdl,
return -EINVAL;
}
+ cmd = kzalloc_obj(*cmd);
+ if (!cmd) {
+ ret = -ENOMEM;
+ goto put_obj;
+ }
+ kref_init(&cmd->refcnt);
+
if (attach) {
if (abo->assigned_hwctx != AMDXDNA_INVALID_CTX_HANDLE) {
ret = -EBUSY;
- goto put_obj;
+ goto put_cmd;
}
- cmd.opcode = ATTACH_DEBUG_BO;
+ cmd->opcode = ATTACH_DEBUG_BO;
} else {
if (abo->assigned_hwctx != hwctx->id) {
ret = -EINVAL;
- goto put_obj;
+ goto put_cmd;
}
- cmd.opcode = DETACH_DEBUG_BO;
+ cmd->opcode = DETACH_DEBUG_BO;
}
- ret = amdxdna_cmd_submit(client, &cmd, AMDXDNA_INVALID_BO_HANDLE,
+ ret = amdxdna_cmd_submit(client, cmd, AMDXDNA_INVALID_BO_HANDLE,
&bo_hdl, 1, hwctx->id, &seq);
if (ret) {
XDNA_ERR(xdna, "Submit command failed");
- goto put_obj;
+ goto put_cmd;
}
aie2_cmd_wait(hwctx, seq);
- if (cmd.result) {
- XDNA_ERR(xdna, "Response failure 0x%x", cmd.result);
+ if (cmd->result) {
+ XDNA_ERR(xdna, "Response failure 0x%x", cmd->result);
ret = -EINVAL;
- goto put_obj;
+ goto put_cmd;
}
if (attach)
@@ -947,6 +971,8 @@ static int aie2_hwctx_cfg_debug_bo(struct amdxdna_hwctx *hwctx, u32 bo_hdl,
XDNA_DBG(xdna, "Config debug BO %d to %s", bo_hdl, hwctx->name);
+put_cmd:
+ aie2_cmd_put(cmd);
put_obj:
amdxdna_gem_put_obj(abo);
return ret;
@@ -974,25 +1000,32 @@ int aie2_hwctx_sync_debug_bo(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl)
{
struct amdxdna_client *client = hwctx->client;
struct amdxdna_dev *xdna = client->xdna;
- struct amdxdna_drv_cmd cmd = { 0 };
+ struct amdxdna_drv_cmd *cmd;
u64 seq;
int ret;
- cmd.opcode = SYNC_DEBUG_BO;
- ret = amdxdna_cmd_submit(client, &cmd, AMDXDNA_INVALID_BO_HANDLE,
+ cmd = kzalloc_obj(*cmd);
+ if (!cmd)
+ return -ENOMEM;
+ kref_init(&cmd->refcnt);
+
+ cmd->opcode = SYNC_DEBUG_BO;
+ ret = amdxdna_cmd_submit(client, cmd, AMDXDNA_INVALID_BO_HANDLE,
&debug_bo_hdl, 1, hwctx->id, &seq);
if (ret) {
XDNA_ERR(xdna, "Submit command failed");
- return ret;
+ goto put_cmd;
}
aie2_cmd_wait(hwctx, seq);
- if (cmd.result) {
- XDNA_ERR(xdna, "Response failure 0x%x", cmd.result);
- return -EINVAL;
+ if (cmd->result) {
+ XDNA_ERR(xdna, "Response failure 0x%x", cmd->result);
+ ret = -EINVAL;
}
- return 0;
+put_cmd:
+ aie2_cmd_put(cmd);
+ return ret;
}
static int aie2_populate_range(struct amdxdna_gem_obj *abo)
@@ -1009,7 +1042,7 @@ again:
found = false;
down_write(&xdna->notifier_lock);
list_for_each_entry(mapp, &abo->mem.umap_list, node) {
- if (mapp->invalid) {
+ if (mapp->invalid && kref_get_unless_zero(&mapp->refcnt)) {
found = true;
break;
}
@@ -1020,11 +1053,9 @@ again:
up_write(&xdna->notifier_lock);
return 0;
}
- kref_get(&mapp->refcnt);
+
up_write(&xdna->notifier_lock);
- XDNA_DBG(xdna, "populate memory range %lx %lx",
- mapp->vma->vm_start, mapp->vma->vm_end);
mm = mapp->notifier.mm;
if (!mmget_not_zero(mm)) {
amdxdna_umap_put(mapp);
@@ -1142,6 +1173,8 @@ retry:
dma_resv_add_fence(job->bos[i]->resv, job->out_fence, DMA_RESV_USAGE_WRITE);
job->seq = hwctx->priv->seq++;
kref_get(&job->refcnt);
+ if (job->drv_cmd)
+ kref_get(&job->drv_cmd->refcnt);
drm_sched_entity_push_job(&job->base);
*seq = job->seq;
@@ -1189,10 +1222,6 @@ int aie2_hwctx_heap_expand(struct amdxdna_hwctx *hwctx,
u64 addr;
int ret;
- ret = amdxdna_pm_resume_get_locked(xdna);
- if (ret)
- return ret;
-
addr = amdxdna_obj_dma_addr(heap);
ret = aie2_add_host_buf(xdna->dev_handle, hwctx->fw_ctx_id,
addr, heap->mem.size);
@@ -1201,7 +1230,5 @@ int aie2_hwctx_heap_expand(struct amdxdna_hwctx *hwctx,
hwctx->name, heap->mem.size, ret);
}
- amdxdna_pm_suspend_put(xdna);
-
return ret;
}
diff --git a/drivers/accel/amdxdna/aie2_message.c b/drivers/accel/amdxdna/aie2_message.c
index c4b364801cc0..dfe0fbdf066d 100644
--- a/drivers/accel/amdxdna/aie2_message.c
+++ b/drivers/accel/amdxdna/aie2_message.c
@@ -840,7 +840,7 @@ static struct aie2_exec_msg_ops npu_exec_message_ops = {
static int aie2_init_exec_req(void *req, struct amdxdna_gem_obj *cmd_abo,
size_t *size, u32 *msg_op)
{
- struct amdxdna_dev *xdna = cmd_abo->client->xdna;
+ struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(cmd_abo)->dev);
int ret;
u32 op;
@@ -874,7 +874,7 @@ static int
aie2_cmdlist_fill_slot(void *slot, struct amdxdna_gem_obj *cmd_abo,
size_t *size, u32 *cmd_op)
{
- struct amdxdna_dev *xdna = cmd_abo->client->xdna;
+ struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(cmd_abo)->dev);
int ret;
u32 op;
diff --git a/drivers/accel/amdxdna/amdxdna_ctx.c b/drivers/accel/amdxdna/amdxdna_ctx.c
index 855da8c79a1c..31a414c3f0d9 100644
--- a/drivers/accel/amdxdna/amdxdna_ctx.c
+++ b/drivers/accel/amdxdna/amdxdna_ctx.c
@@ -310,6 +310,7 @@ int amdxdna_drm_destroy_hwctx_ioctl(struct drm_device *dev, void *data, struct d
if (!drm_dev_enter(dev, &idx))
return -ENODEV;
+ mutex_lock(&xdna->client_lock);
mutex_lock(&xdna->dev_lock);
hwctx = xa_erase(&client->hwctx_xa, args->handle);
if (!hwctx) {
@@ -328,6 +329,7 @@ int amdxdna_drm_destroy_hwctx_ioctl(struct drm_device *dev, void *data, struct d
XDNA_DBG(xdna, "PID %d destroyed HW context %d", client->pid, args->handle);
out:
mutex_unlock(&xdna->dev_lock);
+ mutex_unlock(&xdna->client_lock);
drm_dev_exit(idx);
return ret;
}
@@ -382,16 +384,27 @@ int amdxdna_drm_config_hwctx_ioctl(struct drm_device *dev, void *data, struct dr
return -EINVAL;
}
- guard(mutex)(&xdna->dev_lock);
+ ret = amdxdna_pm_resume_get(xdna);
+ if (ret) {
+ XDNA_ERR(xdna, "Resume failed, ret %d", ret);
+ goto free_buf;
+ }
+
+ mutex_lock(&xdna->client_lock);
+ mutex_lock(&xdna->dev_lock);
hwctx = xa_load(&client->hwctx_xa, args->handle);
if (!hwctx) {
XDNA_DBG(xdna, "PID %d failed to get hwctx %d", client->pid, args->handle);
ret = -EINVAL;
- goto free_buf;
+ goto unlock;
}
ret = xdna->dev_info->ops->hwctx_config(hwctx, args->param_type, val, buf, buf_size);
+unlock:
+ mutex_unlock(&xdna->dev_lock);
+ mutex_unlock(&xdna->client_lock);
+ amdxdna_pm_suspend_put(xdna);
free_buf:
kfree(buf);
return ret;
@@ -412,16 +425,27 @@ int amdxdna_hwctx_sync_debug_bo(struct amdxdna_client *client, u32 debug_bo_hdl)
if (!gobj)
return -EINVAL;
+ ret = amdxdna_pm_resume_get(xdna);
+ if (ret) {
+ XDNA_ERR(xdna, "Resume failed, ret %d", ret);
+ goto put_obj;
+ }
+
abo = to_xdna_obj(gobj);
- guard(mutex)(&xdna->dev_lock);
+ mutex_lock(&xdna->client_lock);
+ mutex_lock(&xdna->dev_lock);
hwctx = xa_load(&client->hwctx_xa, abo->assigned_hwctx);
if (!hwctx) {
ret = -EINVAL;
- goto put_obj;
+ goto unlock;
}
ret = xdna->dev_info->ops->hwctx_sync_debug_bo(hwctx, debug_bo_hdl);
+unlock:
+ mutex_unlock(&xdna->dev_lock);
+ mutex_unlock(&xdna->client_lock);
+ amdxdna_pm_suspend_put(xdna);
put_obj:
drm_gem_object_put(gobj);
return ret;
@@ -448,9 +472,7 @@ static int amdxdna_hwctx_expand_heap(struct amdxdna_hwctx *hwctx)
break;
}
- mutex_unlock(&client->mm_lock);
ret = xdna->dev_info->ops->hwctx_heap_expand(hwctx, heap);
- mutex_lock(&client->mm_lock);
if (ret) {
amdxdna_gem_unpin(heap);
drm_gem_object_put(to_gobj(heap));
@@ -469,18 +491,26 @@ int amdxdna_update_heap(struct amdxdna_client *client, struct amdxdna_hwctx *hwc
unsigned long hwctx_id;
int ret;
- guard(mutex)(&client->mm_lock);
+ ret = amdxdna_pm_resume_get_locked(client->xdna);
+ if (ret)
+ return ret;
- if (hwctx)
- return amdxdna_hwctx_expand_heap(hwctx);
+ mutex_lock(&client->mm_lock);
- amdxdna_for_each_hwctx(client, hwctx_id, hwctx) {
+ if (hwctx) {
ret = amdxdna_hwctx_expand_heap(hwctx);
- if (ret)
- return ret;
+ } else {
+ amdxdna_for_each_hwctx(client, hwctx_id, hwctx) {
+ ret = amdxdna_hwctx_expand_heap(hwctx);
+ if (ret)
+ break;
+ }
}
+ mutex_unlock(&client->mm_lock);
- return 0;
+ amdxdna_pm_suspend_put(client->xdna);
+
+ return ret;
}
static void
@@ -547,6 +577,7 @@ void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job)
amdxdna_arg_bos_put(job);
amdxdna_gem_put_obj(job->cmd_bo);
dma_fence_put(job->fence);
+ mmdrop(job->mm);
}
int amdxdna_cmd_submit(struct amdxdna_client *client,
@@ -560,6 +591,10 @@ int amdxdna_cmd_submit(struct amdxdna_client *client,
int ret, idx;
XDNA_DBG(xdna, "Command BO hdl %d, Arg BO count %d", cmd_bo_hdl, arg_bo_cnt);
+
+ if (!xdna->dev_info->ops->cmd_submit)
+ return -EOPNOTSUPP;
+
job = kzalloc_flex(*job, bos, arg_bo_cnt);
if (!job)
return -ENOMEM;
@@ -573,6 +608,16 @@ int amdxdna_cmd_submit(struct amdxdna_client *client,
ret = -EINVAL;
goto free_job;
}
+ } else if (!drv_cmd) {
+ /*
+ * Only internal driver commands (drv_cmd != NULL) may omit a
+ * command BO. A user command submission with the invalid handle
+ * would leave job->cmd_bo NULL and later fault when the scheduler
+ * dereferences it in amdxdna_cmd_set_state().
+ */
+ XDNA_DBG(xdna, "Command BO handle required for user submission");
+ ret = -EINVAL;
+ goto free_job;
}
ret = amdxdna_arg_bos_lookup(client, job, arg_bo_hdls, arg_bo_cnt);
@@ -598,6 +643,7 @@ int amdxdna_cmd_submit(struct amdxdna_client *client,
job->hwctx = hwctx;
job->mm = current->mm;
+ mmgrab(job->mm);
job->fence = amdxdna_fence_create(hwctx);
if (!job->fence) {
@@ -632,6 +678,8 @@ put_bos:
cmd_put:
amdxdna_gem_put_obj(job->cmd_bo);
free_job:
+ if (job->mm)
+ mmdrop(job->mm);
kfree(job);
return ret;
}
diff --git a/drivers/accel/amdxdna/amdxdna_ctx.h b/drivers/accel/amdxdna/amdxdna_ctx.h
index aaae16430466..b6bef3af7dab 100644
--- a/drivers/accel/amdxdna/amdxdna_ctx.h
+++ b/drivers/accel/amdxdna/amdxdna_ctx.h
@@ -132,6 +132,7 @@ enum amdxdna_job_opcode {
struct amdxdna_drv_cmd {
enum amdxdna_job_opcode opcode;
u32 result;
+ struct kref refcnt;
};
struct app_health_report;
diff --git a/drivers/accel/amdxdna/amdxdna_gem.c b/drivers/accel/amdxdna/amdxdna_gem.c
index 891112c2cddf..4628a2787265 100644
--- a/drivers/accel/amdxdna/amdxdna_gem.c
+++ b/drivers/accel/amdxdna/amdxdna_gem.c
@@ -198,6 +198,7 @@ amdxdna_gem_destroy_obj(struct amdxdna_gem_obj *abo)
*/
void *amdxdna_gem_vmap(struct amdxdna_gem_obj *abo)
{
+ struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
struct iosys_map map = IOSYS_MAP_INIT_VADDR(NULL);
int ret;
@@ -210,7 +211,7 @@ void *amdxdna_gem_vmap(struct amdxdna_gem_obj *abo)
if (!abo->mem.kva) {
ret = drm_gem_vmap(to_gobj(abo), &map);
if (ret)
- XDNA_ERR(abo->client->xdna, "Vmap bo failed, ret %d", ret);
+ XDNA_ERR(xdna, "Vmap bo failed, ret %d", ret);
else
abo->mem.kva = map.vaddr;
}
@@ -254,7 +255,7 @@ static bool amdxdna_hmm_invalidate(struct mmu_interval_notifier *mni,
xdna = to_xdna_dev(to_gobj(abo)->dev);
XDNA_DBG(xdna, "Invalidating range 0x%lx, 0x%lx, type %d",
- mapp->vma->vm_start, mapp->vma->vm_end, abo->type);
+ mapp->range.start, mapp->range.end, abo->type);
if (!mmu_notifier_range_blockable(range))
return false;
@@ -284,15 +285,23 @@ static const struct mmu_interval_notifier_ops amdxdna_hmm_ops = {
.invalidate = amdxdna_hmm_invalidate,
};
+static inline bool compare_range(struct amdxdna_umap *mapp,
+ struct mm_struct *mm,
+ unsigned long start, unsigned long end)
+{
+ return (!mapp->unmapped && mapp->notifier.mm == mm &&
+ mapp->range.start == start && mapp->range.end == end);
+}
+
static void amdxdna_hmm_unregister(struct amdxdna_gem_obj *abo,
struct vm_area_struct *vma)
{
struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
struct amdxdna_umap *mapp;
- down_read(&xdna->notifier_lock);
+ down_write(&xdna->notifier_lock);
list_for_each_entry(mapp, &abo->mem.umap_list, node) {
- if (!vma || mapp->vma == vma) {
+ if (!vma || compare_range(mapp, vma->vm_mm, vma->vm_start, vma->vm_end)) {
if (!mapp->unmapped) {
queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work);
mapp->unmapped = true;
@@ -301,19 +310,16 @@ static void amdxdna_hmm_unregister(struct amdxdna_gem_obj *abo,
break;
}
}
- up_read(&xdna->notifier_lock);
+ up_write(&xdna->notifier_lock);
}
static void amdxdna_umap_release(struct kref *ref)
{
struct amdxdna_umap *mapp = container_of(ref, struct amdxdna_umap, refcnt);
struct amdxdna_gem_obj *abo = mapp->abo;
- struct vm_area_struct *vma = mapp->vma;
struct amdxdna_dev *xdna;
mmu_interval_notifier_remove(&mapp->notifier);
- if (is_import_bo(abo) && vma->vm_file && vma->vm_file->f_mapping)
- mapping_clear_unevictable(vma->vm_file->f_mapping);
xdna = to_xdna_dev(to_gobj(mapp->abo)->dev);
down_write(&xdna->notifier_lock);
@@ -346,15 +352,30 @@ static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo,
unsigned long len = vma->vm_end - vma->vm_start;
unsigned long addr = vma->vm_start;
struct amdxdna_umap *mapp;
- u32 nr_pages;
+ unsigned long nr_pages;
int ret;
- if (!amdxdna_pasid_on(abo->client)) {
+ /*
+ * When PASID is off, amdxdna_gem_obj_open() called amdxdna_dma_map_bo()
+ * and mem.dma_addr is valid; use the DMA address directly and skip HMM.
+ * Avoid dereferencing abo->client which may be NULL (cleared in close())
+ * while internal kernel references are still held.
+ */
+ if (abo->mem.dma_addr != AMDXDNA_INVALID_ADDR) {
/* Need to set uva for heap uva validation */
abo->mem.uva = addr;
return 0;
}
+ down_read(&xdna->notifier_lock);
+ list_for_each_entry(mapp, &abo->mem.umap_list, node) {
+ if (compare_range(mapp, current->mm, addr, addr + len)) {
+ up_read(&xdna->notifier_lock);
+ return 0;
+ }
+ }
+ up_read(&xdna->notifier_lock);
+
mapp = kzalloc_obj(*mapp);
if (!mapp)
return -ENOMEM;
@@ -380,13 +401,10 @@ static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo,
mapp->range.start = vma->vm_start;
mapp->range.end = vma->vm_end;
mapp->range.default_flags = HMM_PFN_REQ_FAULT;
- mapp->vma = vma;
mapp->abo = abo;
kref_init(&mapp->refcnt);
INIT_WORK(&mapp->hmm_unreg_work, amdxdna_hmm_unreg_work);
- if (is_import_bo(abo) && vma->vm_file && vma->vm_file->f_mapping)
- mapping_set_unevictable(vma->vm_file->f_mapping);
down_write(&xdna->notifier_lock);
if (list_empty(&abo->mem.umap_list))
@@ -527,6 +545,7 @@ static int amdxdna_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struc
close_vma:
vma->vm_ops->close(vma);
+ return ret;
put_obj:
drm_gem_object_put(gobj);
return ret;
@@ -652,8 +671,11 @@ static int amdxdna_gem_obj_open(struct drm_gem_object *gobj, struct drm_file *fi
/* No need to set up dma addr mapping in PASID mode. */
if (!amdxdna_pasid_on(abo->client)) {
ret = amdxdna_dma_map_bo(xdna, abo);
- if (ret)
+ if (ret) {
+ abo->open_ref--;
+ abo->client = NULL;
return ret;
+ }
}
amdxdna_gem_add_bo_usage(abo);
diff --git a/drivers/accel/amdxdna/amdxdna_gem.h b/drivers/accel/amdxdna/amdxdna_gem.h
index a3e44c7a2395..1e90e32bf3cd 100644
--- a/drivers/accel/amdxdna/amdxdna_gem.h
+++ b/drivers/accel/amdxdna/amdxdna_gem.h
@@ -12,7 +12,6 @@
#include "amdxdna_pci_drv.h"
struct amdxdna_umap {
- struct vm_area_struct *vma;
struct mmu_interval_notifier notifier;
struct hmm_range range;
struct work_struct hmm_unreg_work;
@@ -89,12 +88,19 @@ u64 amdxdna_gem_dev_addr(struct amdxdna_gem_obj *abo);
static inline u64 amdxdna_dev_bo_offset(struct amdxdna_gem_obj *abo)
{
- return amdxdna_gem_dev_addr(abo) - abo->client->xdna->dev_info->dev_mem_base;
+ return amdxdna_gem_dev_addr(abo) - to_xdna_dev(to_gobj(abo)->dev)->dev_info->dev_mem_base;
}
static inline u64 amdxdna_obj_dma_addr(struct amdxdna_gem_obj *abo)
{
- return amdxdna_pasid_on(abo->client) ? amdxdna_gem_uva(abo) : abo->mem.dma_addr;
+ /*
+ * amdxdna_gem_obj_open() calls amdxdna_dma_map_bo() only when PASID is
+ * off, leaving mem.dma_addr at AMDXDNA_INVALID_ADDR when PASID is on.
+ * Avoid dereferencing abo->client, which is cleared to NULL by
+ * amdxdna_gem_obj_close() while internal kernel references remain.
+ */
+ return (abo->mem.dma_addr != AMDXDNA_INVALID_ADDR) ?
+ abo->mem.dma_addr : amdxdna_gem_uva(abo);
}
void amdxdna_umap_put(struct amdxdna_umap *mapp);
diff --git a/drivers/accel/amdxdna/amdxdna_iommu.c b/drivers/accel/amdxdna/amdxdna_iommu.c
index eff00131d0f8..4f245b969eef 100644
--- a/drivers/accel/amdxdna/amdxdna_iommu.c
+++ b/drivers/accel/amdxdna/amdxdna_iommu.c
@@ -4,6 +4,7 @@
*/
#include <drm/amdxdna_accel.h>
+#include <drm/drm_managed.h>
#include <linux/iommu.h>
#include <linux/iova.h>
@@ -153,10 +154,30 @@ void amdxdna_iommu_free(struct amdxdna_dev *xdna, size_t size,
free_pages((unsigned long)cpu_addr, get_order(size));
}
+static void amdxdna_cleanup_force_iova(struct drm_device *dev, void *res)
+{
+ struct amdxdna_dev *xdna = to_xdna_dev(dev);
+
+ if (xdna->domain) {
+ iommu_detach_group(xdna->domain, xdna->group);
+ put_iova_domain(&xdna->iovad);
+ iova_cache_put();
+ iommu_domain_free(xdna->domain);
+ }
+
+ iommu_group_put(xdna->group);
+}
+
+void amdxdna_iommu_fini(struct amdxdna_dev *xdna)
+{
+ if (xdna->group && !xdna->domain)
+ iommu_group_put(xdna->group);
+}
+
int amdxdna_iommu_init(struct amdxdna_dev *xdna)
{
unsigned long order;
- int ret;
+ int ret = 0;
xdna->group = iommu_group_get(xdna->ddev.dev);
if (!xdna->group || !force_iova)
@@ -182,8 +203,14 @@ int amdxdna_iommu_init(struct amdxdna_dev *xdna)
if (ret)
goto put_iova;
+ ret = drmm_add_action(&xdna->ddev, amdxdna_cleanup_force_iova, NULL);
+ if (ret)
+ goto detach_group;
+
return 0;
+detach_group:
+ iommu_detach_group(xdna->domain, xdna->group);
put_iova:
put_iova_domain(&xdna->iovad);
iova_cache_put();
@@ -191,20 +218,8 @@ free_domain:
iommu_domain_free(xdna->domain);
put_group:
iommu_group_put(xdna->group);
+ xdna->group = NULL;
xdna->domain = NULL;
return ret;
}
-
-void amdxdna_iommu_fini(struct amdxdna_dev *xdna)
-{
- if (xdna->domain) {
- iommu_detach_group(xdna->domain, xdna->group);
- put_iova_domain(&xdna->iovad);
- iova_cache_put();
- iommu_domain_free(xdna->domain);
- }
-
- if (xdna->group)
- iommu_group_put(xdna->group);
-}
diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.c b/drivers/accel/amdxdna/amdxdna_pci_drv.c
index 65489bb3f2b0..bb339e641416 100644
--- a/drivers/accel/amdxdna/amdxdna_pci_drv.c
+++ b/drivers/accel/amdxdna/amdxdna_pci_drv.c
@@ -109,11 +109,16 @@ static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp)
{
struct amdxdna_dev *xdna = to_xdna_dev(ddev);
struct amdxdna_client *client;
+ int ret;
client = kzalloc_obj(*client);
if (!client)
return -ENOMEM;
+ ret = init_srcu_struct(&client->hwctx_srcu);
+ if (ret)
+ goto free_client;
+
client->pid = pid_nr(rcu_access_pointer(filp->pid));
client->xdna = xdna;
client->pasid = IOMMU_PASID_INVALID;
@@ -125,28 +130,35 @@ static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp)
XDNA_WARN(xdna, "PASID not available for pid %d", client->pid);
if (!amdxdna_use_carveout(xdna)) {
XDNA_ERR(xdna, "PASID unavailable and carveout not configured");
- kfree(client);
- return -EINVAL;
+ ret = -EINVAL;
+ goto cleanup_srcu;
}
}
}
mmgrab(client->mm);
- init_srcu_struct(&client->hwctx_srcu);
xa_init_flags(&client->hwctx_xa, XA_FLAGS_ALLOC);
xa_init_flags(&client->dev_heap_xa, XA_FLAGS_ALLOC);
drm_mm_init(&client->dev_heap_mm, xdna->dev_info->dev_mem_base,
xdna->dev_info->dev_heap_max_size);
mutex_init(&client->mm_lock);
+ mutex_lock(&xdna->client_lock);
mutex_lock(&xdna->dev_lock);
list_add_tail(&client->node, &xdna->client_list);
mutex_unlock(&xdna->dev_lock);
+ mutex_unlock(&xdna->client_lock);
filp->driver_priv = client;
client->filp = filp;
XDNA_DBG(xdna, "pid %d opened", client->pid);
return 0;
+
+cleanup_srcu:
+ cleanup_srcu_struct(&client->hwctx_srcu);
+free_client:
+ kfree(client);
+ return ret;
}
static void amdxdna_client_cleanup(struct amdxdna_client *client)
@@ -174,18 +186,14 @@ static void amdxdna_drm_close(struct drm_device *ddev, struct drm_file *filp)
{
struct amdxdna_client *client = filp->driver_priv;
struct amdxdna_dev *xdna = to_xdna_dev(ddev);
- int idx;
XDNA_DBG(xdna, "closing pid %d", client->pid);
- if (!drm_dev_enter(&xdna->ddev, &idx))
- return;
-
+ mutex_lock(&xdna->client_lock);
mutex_lock(&xdna->dev_lock);
amdxdna_client_cleanup(client);
mutex_unlock(&xdna->dev_lock);
-
- drm_dev_exit(idx);
+ mutex_unlock(&xdna->client_lock);
}
static int amdxdna_drm_get_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
@@ -371,7 +379,14 @@ static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (!xdna->dev_info)
return -ENODEV;
- drmm_mutex_init(ddev, &xdna->dev_lock);
+ ret = drmm_mutex_init(ddev, &xdna->client_lock);
+ if (ret)
+ return ret;
+
+ ret = drmm_mutex_init(ddev, &xdna->dev_lock);
+ if (ret)
+ return ret;
+
init_rwsem(&xdna->notifier_lock);
INIT_LIST_HEAD(&xdna->client_list);
pci_set_drvdata(pdev, xdna);
@@ -390,9 +405,9 @@ static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (ret)
return ret;
- xdna->notifier_wq = alloc_ordered_workqueue("notifier_wq", WQ_MEM_RECLAIM);
- if (!xdna->notifier_wq) {
- ret = -ENOMEM;
+ xdna->notifier_wq = drmm_alloc_ordered_workqueue(ddev, "notifier_wq", WQ_MEM_RECLAIM);
+ if (IS_ERR(xdna->notifier_wq)) {
+ ret = PTR_ERR(xdna->notifier_wq);
goto iommu_fini;
}
@@ -401,7 +416,7 @@ static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id)
mutex_unlock(&xdna->dev_lock);
if (ret) {
XDNA_ERR(xdna, "Hardware init failed, ret %d", ret);
- goto destroy_notifier_wq;
+ goto iommu_fini;
}
ret = amdxdna_sysfs_init(xdna);
@@ -425,8 +440,6 @@ failed_dev_fini:
mutex_lock(&xdna->dev_lock);
xdna->dev_info->ops->fini(xdna);
mutex_unlock(&xdna->dev_lock);
-destroy_notifier_wq:
- destroy_workqueue(xdna->notifier_wq);
iommu_fini:
amdxdna_iommu_fini(xdna);
return ret;
@@ -437,23 +450,19 @@ static void amdxdna_remove(struct pci_dev *pdev)
struct amdxdna_dev *xdna = pci_get_drvdata(pdev);
struct amdxdna_client *client;
- destroy_workqueue(xdna->notifier_wq);
-
drm_dev_unplug(&xdna->ddev);
amdxdna_sysfs_fini(xdna);
+ mutex_lock(&xdna->client_lock);
mutex_lock(&xdna->dev_lock);
- client = list_first_entry_or_null(&xdna->client_list,
- struct amdxdna_client, node);
- while (client) {
- amdxdna_client_cleanup(client);
-
- client = list_first_entry_or_null(&xdna->client_list,
- struct amdxdna_client, node);
+ list_for_each_entry(client, &xdna->client_list, node) {
+ amdxdna_hwctx_remove_all(client);
+ amdxdna_sva_fini(client);
}
xdna->dev_info->ops->fini(xdna);
mutex_unlock(&xdna->dev_lock);
+ mutex_unlock(&xdna->client_lock);
amdxdna_iommu_fini(xdna);
}
diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.h b/drivers/accel/amdxdna/amdxdna_pci_drv.h
index 34271c14d359..a997d27a504d 100644
--- a/drivers/accel/amdxdna/amdxdna_pci_drv.h
+++ b/drivers/accel/amdxdna/amdxdna_pci_drv.h
@@ -120,6 +120,7 @@ struct amdxdna_dev {
struct mutex dev_lock; /* per device lock */
struct list_head client_list;
+ struct mutex client_lock; /* client_list */
struct amdxdna_fw_ver fw_ver;
struct rw_semaphore notifier_lock; /* for mmu notifier*/
struct workqueue_struct *notifier_wq;
diff --git a/drivers/accel/ethosu/ethosu_drv.c b/drivers/accel/ethosu/ethosu_drv.c
index 9992193d7338..ed9c748a54ad 100644
--- a/drivers/accel/ethosu/ethosu_drv.c
+++ b/drivers/accel/ethosu/ethosu_drv.c
@@ -7,7 +7,6 @@
#include <linux/io.h>
#include <linux/iopoll.h>
#include <linux/module.h>
-#include <linux/mod_devicetable.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
diff --git a/drivers/accel/ivpu/ivpu_fw_log.c b/drivers/accel/ivpu/ivpu_fw_log.c
index 275baf844b56..716467aa3156 100644
--- a/drivers/accel/ivpu/ivpu_fw_log.c
+++ b/drivers/accel/ivpu/ivpu_fw_log.c
@@ -43,6 +43,10 @@ static int fw_log_from_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, u32 *off
ivpu_dbg(vdev, FW_BOOT, "Invalid header size 0x%x\n", log->header_size);
return -EINVAL;
}
+ if (log->size < log->header_size) {
+ ivpu_dbg(vdev, FW_BOOT, "Invalid log size 0x%x\n", log->size);
+ return -EINVAL;
+ }
if ((char *)log + log->size > (char *)ivpu_bo_vaddr(bo) + ivpu_bo_size(bo)) {
ivpu_dbg(vdev, FW_BOOT, "Invalid log size 0x%x\n", log->size);
return -EINVAL;
diff --git a/drivers/accel/ivpu/ivpu_hw_btrs.c b/drivers/accel/ivpu/ivpu_hw_btrs.c
index dac935164e11..a17c829adb89 100644
--- a/drivers/accel/ivpu/ivpu_hw_btrs.c
+++ b/drivers/accel/ivpu/ivpu_hw_btrs.c
@@ -927,7 +927,7 @@ static void diagnose_failure_mtl(struct ivpu_device *vdev)
static void diagnose_failure_lnl(struct ivpu_device *vdev)
{
- u32 reg = REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) & BTRS_LNL_IRQ_MASK;
+ u32 reg = REGB_RD32(VPU_HW_BTRS_LNL_INTERRUPT_STAT) & BTRS_LNL_IRQ_MASK;
if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR, reg)) {
ivpu_err(vdev, "ATS_ERR_LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n",
diff --git a/drivers/accel/qaic/qaic_timesync.c b/drivers/accel/qaic/qaic_timesync.c
index 9faf71f47bdc..45e5f0728ebe 100644
--- a/drivers/accel/qaic/qaic_timesync.c
+++ b/drivers/accel/qaic/qaic_timesync.c
@@ -6,7 +6,6 @@
#include <linux/kernel.h>
#include <linux/math64.h>
#include <linux/mhi.h>
-#include <linux/mod_devicetable.h>
#include <linux/module.h>
#include <linux/time64.h>
#include <linux/timer.h>
diff --git a/drivers/accel/qaic/sahara.c b/drivers/accel/qaic/sahara.c
index 9fea294e1d7b..c7c0b3eb4b65 100644
--- a/drivers/accel/qaic/sahara.c
+++ b/drivers/accel/qaic/sahara.c
@@ -7,7 +7,6 @@
#include <linux/limits.h>
#include <linux/mhi.h>
#include <linux/minmax.h>
-#include <linux/mod_devicetable.h>
#include <linux/overflow.h>
#include <linux/types.h>
#include <linux/vmalloc.h>
generated by cgit v1.2.3 (git 2.0.4) at 2026-07-24 15:15:31 +0000