diff options
Diffstat (limited to 'drivers/gpu/drm/amd/amdkfd')
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 31 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 33 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_device.c | 5 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c | 40 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_module.c | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 44 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_process.c | 97 |
9 files changed, 185 insertions, 71 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index b7b16adb0615..222f1df1a6b6 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -97,6 +97,7 @@ void kfd_chardev_exit(void) device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0)); class_destroy(kfd_class); unregister_chrdev(kfd_char_dev_major, kfd_dev_name); + kfd_device = NULL; } struct device *kfd_chardev(void) @@ -1290,18 +1291,6 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep, return -EINVAL; } - if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) { - if (args->size != kfd_doorbell_process_slice(dev)) - return -EINVAL; - offset = kfd_get_process_doorbells(dev, p); - } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) { - if (args->size != PAGE_SIZE) - return -EINVAL; - offset = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd); - if (!offset) - return -ENOMEM; - } - mutex_lock(&p->mutex); pdd = kfd_bind_process_to_device(dev, p); @@ -1310,6 +1299,24 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep, goto err_unlock; } + if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) { + if (args->size != kfd_doorbell_process_slice(dev)) { + err = -EINVAL; + goto err_unlock; + } + offset = kfd_get_process_doorbells(pdd); + } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) { + if (args->size != PAGE_SIZE) { + err = -EINVAL; + goto err_unlock; + } + offset = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd); + if (!offset) { + err = -ENOMEM; + goto err_unlock; + } + } + err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( dev->kgd, args->va_addr, args->size, pdd->vm, (struct kgd_mem **) &mem, &offset, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index 3fac06b281ce..d2981524dba0 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -797,7 +797,8 @@ int kfd_create_crat_image_acpi(void **crat_image, size_t *size) return -ENODATA; } - pcrat_image = kmemdup(crat_table, crat_table->length, GFP_KERNEL); + pcrat_image = kvmalloc(crat_table->length, GFP_KERNEL); + memcpy(pcrat_image, crat_table, crat_table->length); if (!pcrat_image) return -ENOMEM; @@ -809,11 +810,10 @@ int kfd_create_crat_image_acpi(void **crat_image, size_t *size) /* Memory required to create Virtual CRAT. * Since there is no easy way to predict the amount of memory required, the - * following amount are allocated for CPU and GPU Virtual CRAT. This is + * following amount is allocated for GPU Virtual CRAT. This is * expected to cover all known conditions. But to be safe additional check * is put in the code to ensure we don't overwrite. */ -#define VCRAT_SIZE_FOR_CPU (2 * PAGE_SIZE) #define VCRAT_SIZE_FOR_GPU (4 * PAGE_SIZE) /* kfd_fill_cu_for_cpu - Fill in Compute info for the given CPU NUMA node @@ -964,7 +964,7 @@ static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size) #endif int ret = 0; - if (!pcrat_image || avail_size < VCRAT_SIZE_FOR_CPU) + if (!pcrat_image) return -EINVAL; /* Fill in CRAT Header. @@ -1364,30 +1364,37 @@ int kfd_create_crat_image_virtual(void **crat_image, size_t *size, uint32_t proximity_domain) { void *pcrat_image = NULL; - int ret = 0; + int ret = 0, num_nodes; + size_t dyn_size; if (!crat_image) return -EINVAL; *crat_image = NULL; - /* Allocate one VCRAT_SIZE_FOR_CPU for CPU virtual CRAT image and - * VCRAT_SIZE_FOR_GPU for GPU virtual CRAT image. This should cover - * all the current conditions. A check is put not to overwrite beyond - * allocated size + /* Allocate the CPU Virtual CRAT size based on the number of online + * nodes. Allocate VCRAT_SIZE_FOR_GPU for GPU virtual CRAT image. + * This should cover all the current conditions. A check is put not + * to overwrite beyond allocated size for GPUs */ switch (flags) { case COMPUTE_UNIT_CPU: - pcrat_image = kmalloc(VCRAT_SIZE_FOR_CPU, GFP_KERNEL); + num_nodes = num_online_nodes(); + dyn_size = sizeof(struct crat_header) + + num_nodes * (sizeof(struct crat_subtype_computeunit) + + sizeof(struct crat_subtype_memory) + + (num_nodes - 1) * sizeof(struct crat_subtype_iolink)); + pcrat_image = kvmalloc(dyn_size, GFP_KERNEL); if (!pcrat_image) return -ENOMEM; - *size = VCRAT_SIZE_FOR_CPU; + *size = dyn_size; + pr_debug("CRAT size is %ld", dyn_size); ret = kfd_create_vcrat_image_cpu(pcrat_image, size); break; case COMPUTE_UNIT_GPU: if (!kdev) return -EINVAL; - pcrat_image = kmalloc(VCRAT_SIZE_FOR_GPU, GFP_KERNEL); + pcrat_image = kvmalloc(VCRAT_SIZE_FOR_GPU, GFP_KERNEL); if (!pcrat_image) return -ENOMEM; *size = VCRAT_SIZE_FOR_GPU; @@ -1406,7 +1413,7 @@ int kfd_create_crat_image_virtual(void **crat_image, size_t *size, if (!ret) *crat_image = pcrat_image; else - kfree(pcrat_image); + kvfree(pcrat_image); return ret; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index e3fc6ed7b79c..903170e59342 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -583,6 +583,8 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, atomic_set(&kfd->sram_ecc_flag, 0); + ida_init(&kfd->doorbell_ida); + return kfd; } @@ -716,6 +718,8 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, kfd->unique_id = amdgpu_amdkfd_get_unique_id(kfd->kgd); + kfd->noretry = amdgpu_amdkfd_get_noretry(kfd->kgd); + if (kfd_interrupt_init(kfd)) { dev_err(kfd_device, "Error initializing interrupts\n"); goto kfd_interrupt_error; @@ -798,6 +802,7 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd) kfd_interrupt_exit(kfd); kfd_topology_remove_device(kfd); kfd_doorbell_fini(kfd); + ida_destroy(&kfd->doorbell_ida); kfd_gtt_sa_fini(kfd); amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem); if (kfd->gws) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index ed362ab8ec21..62504d5fa42b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -191,9 +191,8 @@ static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q) } q->properties.doorbell_off = - kfd_get_doorbell_dw_offset_in_bar(dev, q->process, + kfd_get_doorbell_dw_offset_in_bar(dev, qpd_to_pdd(qpd), q->doorbell_id); - return 0; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c index 309f63a0b34a..eca6331efa94 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c @@ -61,7 +61,7 @@ static int update_qpd_v9(struct device_queue_manager *dqm, qpd->sh_mem_config = SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; - if (amdgpu_noretry && + if (dqm->dev->noretry && !dqm->dev->use_iommu_v2) qpd->sh_mem_config |= 1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c index 8e0c00b9555e..768d153acff4 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c @@ -31,9 +31,6 @@ * kernel queues using the first doorbell page reserved for the kernel. */ -static DEFINE_IDA(doorbell_ida); -static unsigned int max_doorbell_slices; - /* * Each device exposes a doorbell aperture, a PCI MMIO aperture that * receives 32-bit writes that are passed to queues as wptr values. @@ -84,9 +81,9 @@ int kfd_doorbell_init(struct kfd_dev *kfd) else return -ENOSPC; - if (!max_doorbell_slices || - doorbell_process_limit < max_doorbell_slices) - max_doorbell_slices = doorbell_process_limit; + if (!kfd->max_doorbell_slices || + doorbell_process_limit < kfd->max_doorbell_slices) + kfd->max_doorbell_slices = doorbell_process_limit; kfd->doorbell_base = kfd->shared_resources.doorbell_physical_address + doorbell_start_offset; @@ -130,6 +127,7 @@ int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process, struct vm_area_struct *vma) { phys_addr_t address; + struct kfd_process_device *pdd; /* * For simplicitly we only allow mapping of the entire doorbell @@ -138,9 +136,12 @@ int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process, if (vma->vm_end - vma->vm_start != kfd_doorbell_process_slice(dev)) return -EINVAL; - /* Calculate physical address of doorbell */ - address = kfd_get_process_doorbells(dev, process); + pdd = kfd_get_process_device_data(dev, process); + if (!pdd) + return -EINVAL; + /* Calculate physical address of doorbell */ + address = kfd_get_process_doorbells(pdd); vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP; @@ -226,7 +227,7 @@ void write_kernel_doorbell64(void __iomem *db, u64 value) } unsigned int kfd_get_doorbell_dw_offset_in_bar(struct kfd_dev *kfd, - struct kfd_process *process, + struct kfd_process_device *pdd, unsigned int doorbell_id) { /* @@ -236,7 +237,7 @@ unsigned int kfd_get_doorbell_dw_offset_in_bar(struct kfd_dev *kfd, * units regardless of the ASIC-dependent doorbell size. */ return kfd->doorbell_base_dw_offset + - process->doorbell_index + pdd->doorbell_index * kfd_doorbell_process_slice(kfd) / sizeof(u32) + doorbell_id * kfd->device_info->doorbell_size / sizeof(u32); } @@ -251,25 +252,24 @@ uint64_t kfd_get_number_elems(struct kfd_dev *kfd) } -phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev, - struct kfd_process *process) +phys_addr_t kfd_get_process_doorbells(struct kfd_process_device *pdd) { - return dev->doorbell_base + - process->doorbell_index * kfd_doorbell_process_slice(dev); + return pdd->dev->doorbell_base + + pdd->doorbell_index * kfd_doorbell_process_slice(pdd->dev); } -int kfd_alloc_process_doorbells(struct kfd_process *process) +int kfd_alloc_process_doorbells(struct kfd_dev *kfd, unsigned int *doorbell_index) { - int r = ida_simple_get(&doorbell_ida, 1, max_doorbell_slices, + int r = ida_simple_get(&kfd->doorbell_ida, 1, kfd->max_doorbell_slices, GFP_KERNEL); if (r > 0) - process->doorbell_index = r; + *doorbell_index = r; return r; } -void kfd_free_process_doorbells(struct kfd_process *process) +void kfd_free_process_doorbells(struct kfd_dev *kfd, unsigned int doorbell_index) { - if (process->doorbell_index) - ida_simple_remove(&doorbell_ida, process->doorbell_index); + if (doorbell_index) + ida_simple_remove(&kfd->doorbell_ida, doorbell_index); } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_module.c b/drivers/gpu/drm/amd/amdkfd/kfd_module.c index f4b7f7e6c40e..5e90fe642192 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_module.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_module.c @@ -70,6 +70,7 @@ err_create_wq: err_topology: kfd_chardev_exit(); err_ioctl: + pr_err("KFD is disabled due to module initialization failure\n"); return err; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 8c2b8ccd27fb..b7be5c5751b7 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -314,6 +314,11 @@ struct kfd_dev { spinlock_t smi_lock; uint32_t reset_seq_num; + + struct ida doorbell_ida; + unsigned int max_doorbell_slices; + + int noretry; }; enum kfd_mempool { @@ -699,6 +704,32 @@ struct kfd_process_device { struct attribute attr_evict; struct kobject *kobj_stats; + unsigned int doorbell_index; + + /* + * @cu_occupancy: Reports occupancy of Compute Units (CU) of a process + * that is associated with device encoded by "this" struct instance. The + * value reflects CU usage by all of the waves launched by this process + * on this device. A very important property of occupancy parameter is + * that its value is a snapshot of current use. + * + * Following is to be noted regarding how this parameter is reported: + * + * The number of waves that a CU can launch is limited by couple of + * parameters. These are encoded by struct amdgpu_cu_info instance + * that is part of every device definition. For GFX9 devices this + * translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves + * do not use scratch memory and 32 waves (max_scratch_slots_per_cu) + * when they do use scratch memory. This could change for future + * devices and therefore this example should be considered as a guide. + * + * All CU's of a device are available for the process. This may not be true + * under certain conditions - e.g. CU masking. + * + * Finally number of CU's that are occupied by a process is affected by both + * number of CU's a device has along with number of other competing processes + */ + struct attribute attr_cu_occupancy; }; #define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd) @@ -736,7 +767,6 @@ struct kfd_process { struct mmu_notifier mmu_notifier; uint16_t pasid; - unsigned int doorbell_index; /* * List of kfd_process_device structures, @@ -869,13 +899,13 @@ u32 read_kernel_doorbell(u32 __iomem *db); void write_kernel_doorbell(void __iomem *db, u32 value); void write_kernel_doorbell64(void __iomem *db, u64 value); unsigned int kfd_get_doorbell_dw_offset_in_bar(struct kfd_dev *kfd, - struct kfd_process *process, + struct kfd_process_device *pdd, unsigned int doorbell_id); -phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev, - struct kfd_process *process); -int kfd_alloc_process_doorbells(struct kfd_process *process); -void kfd_free_process_doorbells(struct kfd_process *process); - +phys_addr_t kfd_get_process_doorbells(struct kfd_process_device *pdd); +int kfd_alloc_process_doorbells(struct kfd_dev *kfd, + unsigned int *doorbell_index); +void kfd_free_process_doorbells(struct kfd_dev *kfd, + unsigned int doorbell_index); /* GTT Sub-Allocator */ int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index ad53b2668221..2807e1c4d59b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -249,6 +249,52 @@ cleanup: } } +/** + * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device + * by current process. Translates acquired wave count into number of compute units + * that are occupied. + * + * @atr: Handle of attribute that allows reporting of wave count. The attribute + * handle encapsulates GPU device it is associated with, thereby allowing collection + * of waves in flight, etc + * + * @buffer: Handle of user provided buffer updated with wave count + * + * Return: Number of bytes written to user buffer or an error value + */ +static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer) +{ + int cu_cnt; + int wave_cnt; + int max_waves_per_cu; + struct kfd_dev *dev = NULL; + struct kfd_process *proc = NULL; + struct kfd_process_device *pdd = NULL; + + pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy); + dev = pdd->dev; + if (dev->kfd2kgd->get_cu_occupancy == NULL) + return -EINVAL; + + cu_cnt = 0; + proc = pdd->process; + if (pdd->qpd.queue_count == 0) { + pr_debug("Gpu-Id: %d has no active queues for process %d\n", + dev->id, proc->pasid); + return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt); + } + + /* Collect wave count from device if it supports */ + wave_cnt = 0; + max_waves_per_cu = 0; + dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt, + &max_waves_per_cu); + + /* Translate wave count to number of compute units */ + cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu; + return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt); +} + static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr, char *buffer) { @@ -344,6 +390,7 @@ static ssize_t kfd_procfs_queue_show(struct kobject *kobj, return 0; } + static ssize_t kfd_procfs_stats_show(struct kobject *kobj, struct attribute *attr, char *buffer) { @@ -359,8 +406,13 @@ static ssize_t kfd_procfs_stats_show(struct kobject *kobj, PAGE_SIZE, "%llu\n", jiffies64_to_msecs(evict_jiffies)); - } else + + /* Sysfs handle that gets CU occupancy is per device */ + } else if (strcmp(attr->name, "cu_occupancy") == 0) { + return kfd_get_cu_occupancy(attr, buffer); + } else { pr_err("Invalid attribute"); + } return 0; } @@ -466,6 +518,7 @@ static int kfd_procfs_add_sysfs_stats(struct kfd_process *p) * Create sysfs files for each GPU: * - proc/<pid>/stats_<gpuid>/ * - proc/<pid>/stats_<gpuid>/evicted_ms + * - proc/<pid>/stats_<gpuid>/cu_occupancy */ list_for_each_entry(pdd, &p->per_device_data, per_device_list) { struct kobject *kobj_stats; @@ -496,6 +549,19 @@ static int kfd_procfs_add_sysfs_stats(struct kfd_process *p) if (ret) pr_warn("Creating eviction stats for gpuid %d failed", (int)pdd->dev->id); + + /* Add sysfs file to report compute unit occupancy */ + if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL) { + pdd->attr_cu_occupancy.name = "cu_occupancy"; + pdd->attr_cu_occupancy.mode = KFD_SYSFS_FILE_MODE; + sysfs_attr_init(&pdd->attr_cu_occupancy); + ret = sysfs_create_file(kobj_stats, + &pdd->attr_cu_occupancy); + if (ret) + pr_warn("Creating %s failed for gpuid: %d", + pdd->attr_cu_occupancy.name, + (int)pdd->dev->id); + } } err: return ret; @@ -537,7 +603,6 @@ static int kfd_procfs_add_sysfs_files(struct kfd_process *p) return ret; } - void kfd_procfs_del_queue(struct queue *q) { if (!q) @@ -750,11 +815,6 @@ struct kfd_process *kfd_create_process(struct file *filep) pr_warn("Creating sysfs stats dir for pid %d failed", (int)process->lead_thread->pid); - ret = kfd_procfs_add_sysfs_stats(process); - if (ret) - pr_warn("Creating sysfs stats dir for pid %d failed", - (int)process->lead_thread->pid); - ret = kfd_procfs_add_sysfs_files(process); if (ret) pr_warn("Creating sysfs usage file for pid %d failed", @@ -876,6 +936,8 @@ static void kfd_process_destroy_pdds(struct kfd_process *p) kfree(pdd->qpd.doorbell_bitmap); idr_destroy(&pdd->alloc_idr); + kfd_free_process_doorbells(pdd->dev, pdd->doorbell_index); + /* * before destroying pdd, make sure to report availability * for auto suspend @@ -912,6 +974,8 @@ static void kfd_process_wq_release(struct work_struct *work) sysfs_remove_file(p->kobj, &pdd->attr_vram); sysfs_remove_file(p->kobj, &pdd->attr_sdma); sysfs_remove_file(p->kobj, &pdd->attr_evict); + if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL) + sysfs_remove_file(p->kobj, &pdd->attr_cu_occupancy); kobject_del(pdd->kobj_stats); kobject_put(pdd->kobj_stats); pdd->kobj_stats = NULL; @@ -932,8 +996,6 @@ static void kfd_process_wq_release(struct work_struct *work) kfd_event_free_process(p); kfd_pasid_free(p->pasid); - kfd_free_process_doorbells(p); - mutex_destroy(&p->mutex); put_task_struct(p->lead_thread); @@ -1111,9 +1173,6 @@ static struct kfd_process *create_process(const struct task_struct *thread) if (process->pasid == 0) goto err_alloc_pasid; - if (kfd_alloc_process_doorbells(process) < 0) - goto err_alloc_doorbells; - err = pqm_init(&process->pqm, process); if (err != 0) goto err_process_pqm_init; @@ -1141,8 +1200,6 @@ err_register_notifier: err_init_apertures: pqm_uninit(&process->pqm); err_process_pqm_init: - kfd_free_process_doorbells(process); -err_alloc_doorbells: kfd_pasid_free(process->pasid); err_alloc_pasid: mutex_destroy(&process->mutex); @@ -1205,10 +1262,14 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev, if (!pdd) return NULL; + if (kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) { + pr_err("Failed to alloc doorbell for pdd\n"); + goto err_free_pdd; + } + if (init_doorbell_bitmap(&pdd->qpd, dev)) { pr_err("Failed to init doorbell for process\n"); - kfree(pdd); - return NULL; + goto err_free_pdd; } pdd->dev = dev; @@ -1231,6 +1292,10 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev, idr_init(&pdd->alloc_idr); return pdd; + +err_free_pdd: + kfree(pdd); + return NULL; } /** |