diff options
Diffstat (limited to 'drivers/gpu/drm/vc4')
| -rw-r--r-- | drivers/gpu/drm/vc4/vc4_bo.c | 43 | ||||
| -rw-r--r-- | drivers/gpu/drm/vc4/vc4_crtc.c | 126 | ||||
| -rw-r--r-- | drivers/gpu/drm/vc4/vc4_drv.h | 21 | ||||
| -rw-r--r-- | drivers/gpu/drm/vc4/vc4_dsi.c | 7 | ||||
| -rw-r--r-- | drivers/gpu/drm/vc4/vc4_gem.c | 20 | ||||
| -rw-r--r-- | drivers/gpu/drm/vc4/vc4_validate.c | 15 | ||||
| -rw-r--r-- | drivers/gpu/drm/vc4/vc4_validate_shaders.c | 7 |
7 files changed, 145 insertions, 94 deletions
diff --git a/drivers/gpu/drm/vc4/vc4_bo.c b/drivers/gpu/drm/vc4/vc4_bo.c index da4b11b7ce9f..2ab7c158b7ff 100644 --- a/drivers/gpu/drm/vc4/vc4_bo.c +++ b/drivers/gpu/drm/vc4/vc4_bo.c @@ -17,6 +17,7 @@ */ #include <linux/dma-buf.h> +#include <linux/overflow.h> #include <drm/drm_fourcc.h> #include <drm/drm_print.h> @@ -836,6 +837,19 @@ int vc4_mmap_bo_ioctl(struct drm_device *dev, void *data, return 0; } +/* + * QPUs sharing a slice's instruction cache can livelock evicting each + * other's fills. Considering that shader BOs are page aligned, any code + * at offset 0 maps every shader to the same cache sets. This makes a + * livelock more likely to happen due to GFXH-16. + * + * Start each shader at one of four offsets, cycled per client, to spread + * them across sets. The use of four different offsets is motivated by the + * number of QPUs per slice (4 QPUs). + */ +#define VC4_SHADER_OFFSET_STEP 256 +#define VC4_SHADER_OFFSET_NUM 4 + int vc4_create_shader_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) @@ -844,6 +858,7 @@ vc4_create_shader_bo_ioctl(struct drm_device *dev, void *data, struct vc4_file *vc4file = file_priv->driver_priv; struct vc4_dev *vc4 = to_vc4_dev(dev); struct vc4_bo *bo = NULL; + u32 offset, size; int ret; if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4)) @@ -865,29 +880,37 @@ vc4_create_shader_bo_ioctl(struct drm_device *dev, void *data, return -EINVAL; } + offset = ((u32)atomic_inc_return(&vc4file->shader_offset_idx) % + VC4_SHADER_OFFSET_NUM) * VC4_SHADER_OFFSET_STEP; + + if (check_add_overflow(offset, args->size, &size)) + return -EINVAL; + ret = vc4_grab_bin_bo(vc4, vc4file); if (ret) return ret; - bo = vc4_bo_create(dev, args->size, true, VC4_BO_TYPE_V3D_SHADER); + bo = vc4_bo_create(dev, size, true, VC4_BO_TYPE_V3D_SHADER); if (IS_ERR(bo)) return PTR_ERR(bo); bo->madv = VC4_MADV_WILLNEED; + bo->shader_offset = offset; - if (copy_from_user(bo->base.vaddr, - (void __user *)(uintptr_t)args->data, - args->size)) { + /* Clear the memory around the code, which may come from the BO + * cache. + */ + memset(bo->base.vaddr, 0, offset); + if (copy_from_user(bo->base.vaddr + offset, + (void __user *)(uintptr_t)args->data, + args->size)) { ret = -EFAULT; goto fail; } - /* Clear the rest of the memory from allocating from the BO - * cache. - */ - memset(bo->base.vaddr + args->size, 0, - bo->base.base.size - args->size); + memset(bo->base.vaddr + offset + args->size, 0, + bo->base.base.size - offset - args->size); - bo->validated_shader = vc4_validate_shader(&bo->base); + bo->validated_shader = vc4_validate_shader(&bo->base, offset); if (!bo->validated_shader) { ret = -EINVAL; goto fail; diff --git a/drivers/gpu/drm/vc4/vc4_crtc.c b/drivers/gpu/drm/vc4/vc4_crtc.c index f42cd5353230..4b6d4a5f7bc9 100644 --- a/drivers/gpu/drm/vc4/vc4_crtc.c +++ b/drivers/gpu/drm/vc4/vc4_crtc.c @@ -884,17 +884,21 @@ struct vc4_async_flip_state { struct drm_framebuffer *fb; struct drm_framebuffer *old_fb; struct drm_pending_vblank_event *event; + struct dma_fence *fence; struct dma_fence_cb cb; + struct work_struct work; }; -/* Called when the V3D execution for the BO being flipped to is done, so that - * we can actually update the plane's address to point to it. +/* Executed when the V3D execution for the BO being flipped to is done, so + * that we can actually update the plane's address to point to it. */ -static void -vc4_async_page_flip_complete(struct vc4_async_flip_state *flip_state) +static void vc4_async_page_flip_work(struct work_struct *work) { + struct vc4_async_flip_state *flip_state = + container_of(work, struct vc4_async_flip_state, work); struct drm_crtc *crtc = flip_state->crtc; struct drm_device *dev = crtc->dev; + struct vc4_dev *vc4 = to_vc4_dev(dev); struct drm_plane *plane = crtc->primary; vc4_plane_async_set_fb(plane, flip_state->fb); @@ -909,82 +913,57 @@ vc4_async_page_flip_complete(struct vc4_async_flip_state *flip_state) drm_crtc_vblank_put(crtc); drm_framebuffer_put(flip_state->fb); - if (flip_state->old_fb) - drm_framebuffer_put(flip_state->old_fb); - - kfree(flip_state); -} + if (flip_state->old_fb) { + /* + * Decrement the BO usecnt in order to keep the inc/dec + * calls balanced when the planes are updated through + * the async update path. + * + * FIXME: we should move to generic async-page-flip when + * it's available, so that we can get rid of this + * hand-made cleanup_fb() logic. + */ + if (vc4->gen == VC4_GEN_4) { + struct drm_gem_dma_object *dma_bo = + drm_fb_dma_get_gem_obj(flip_state->old_fb, 0); -static void vc4_async_page_flip_complete_with_cleanup(struct dma_fence *fence, - struct dma_fence_cb *cb) -{ - struct vc4_async_flip_state *flip_state = - container_of(cb, struct vc4_async_flip_state, cb); - struct vc4_bo *bo = NULL; + vc4_bo_dec_usecnt(to_vc4_bo(&dma_bo->base)); + } - if (flip_state->old_fb) { - struct drm_gem_dma_object *dma_bo = - drm_fb_dma_get_gem_obj(flip_state->old_fb, 0); - bo = to_vc4_bo(&dma_bo->base); + drm_framebuffer_put(flip_state->old_fb); } - vc4_async_page_flip_complete(flip_state); - dma_fence_put(fence); - - /* - * Decrement the BO usecnt in order to keep the inc/dec - * calls balanced when the planes are updated through - * the async update path. - * - * FIXME: we should move to generic async-page-flip when - * it's available, so that we can get rid of this - * hand-made cleanup_fb() logic. - */ - if (bo) - vc4_bo_dec_usecnt(bo); + dma_fence_put(flip_state->fence); + kfree(flip_state); } -static void vc4_async_page_flip_fence_complete(struct dma_fence *fence, - struct dma_fence_cb *cb) +static void vc4_async_page_flip_fence_cb(struct dma_fence *fence, + struct dma_fence_cb *cb) { struct vc4_async_flip_state *flip_state = container_of(cb, struct vc4_async_flip_state, cb); + struct vc4_crtc *vc4_crtc = to_vc4_crtc(flip_state->crtc); - vc4_async_page_flip_complete(flip_state); - dma_fence_put(fence); + /* Completing the flip may sleep, so defer it to a worker. */ + queue_work(vc4_crtc->async_flip_wq, &flip_state->work); } -static int vc4_async_set_fence_cb(struct drm_device *dev, - struct vc4_async_flip_state *flip_state) +static void vc4_async_set_fence_cb(struct vc4_async_flip_state *flip_state) { - struct drm_framebuffer *fb = flip_state->fb; - struct drm_gem_dma_object *dma_bo = drm_fb_dma_get_gem_obj(fb, 0); - dma_fence_func_t async_page_flip_complete_function; - struct vc4_dev *vc4 = to_vc4_dev(dev); - struct dma_fence *fence; - int ret; + struct vc4_crtc *vc4_crtc = to_vc4_crtc(flip_state->crtc); - if (vc4->gen == VC4_GEN_4) - async_page_flip_complete_function = vc4_async_page_flip_complete_with_cleanup; - else - async_page_flip_complete_function = vc4_async_page_flip_fence_complete; - - ret = dma_resv_get_singleton(dma_bo->base.resv, DMA_RESV_USAGE_READ, &fence); - if (ret) - return ret; - - /* If there's no fence, complete the page flip immediately */ - if (!fence) { - async_page_flip_complete_function(fence, &flip_state->cb); - return 0; - } + INIT_WORK(&flip_state->work, vc4_async_page_flip_work); - /* If the fence has already been completed, complete the page flip */ - if (dma_fence_add_callback(fence, &flip_state->cb, - async_page_flip_complete_function)) - async_page_flip_complete_function(fence, &flip_state->cb); + if (flip_state->fence && + !dma_fence_add_callback(flip_state->fence, &flip_state->cb, + vc4_async_page_flip_fence_cb)) + return; - return 0; + /* No fence or the fence has already signaled. Complete the flip + * immediately, after earlier queued flips. + */ + flush_workqueue(vc4_crtc->async_flip_wq); + vc4_async_page_flip_work(&flip_state->work); } static int @@ -993,14 +972,22 @@ vc4_async_page_flip_common(struct drm_crtc *crtc, struct drm_pending_vblank_event *event, uint32_t flags) { - struct drm_device *dev = crtc->dev; + struct drm_gem_dma_object *dma_bo = drm_fb_dma_get_gem_obj(fb, 0); struct drm_plane *plane = crtc->primary; struct vc4_async_flip_state *flip_state; + int ret; flip_state = kzalloc_obj(*flip_state); if (!flip_state) return -ENOMEM; + ret = dma_resv_get_singleton(dma_bo->base.resv, DMA_RESV_USAGE_READ, + &flip_state->fence); + if (ret) { + kfree(flip_state); + return ret; + } + drm_framebuffer_get(fb); flip_state->fb = fb; flip_state->crtc = crtc; @@ -1008,7 +995,7 @@ vc4_async_page_flip_common(struct drm_crtc *crtc, /* Save the current FB before it's replaced by the new one in * drm_atomic_set_fb_for_plane(). We'll need the old FB in - * vc4_async_page_flip_complete() to decrement the BO usecnt and keep + * vc4_async_page_flip_work() to decrement the BO usecnt and keep * it consistent. * FIXME: we should move to generic async-page-flip when it's * available, so that we can get rid of this hand-made cleanup_fb() @@ -1026,7 +1013,7 @@ vc4_async_page_flip_common(struct drm_crtc *crtc, */ drm_atomic_set_fb_for_plane(plane->state, fb); - vc4_async_set_fence_cb(dev, flip_state); + vc4_async_set_fence_cb(flip_state); /* Driver takes ownership of state on successful async commit. */ return 0; @@ -1413,6 +1400,11 @@ int __vc4_crtc_init(struct drm_device *drm, drm_crtc_helper_add(crtc, crtc_helper_funcs); + vc4_crtc->async_flip_wq = drmm_alloc_ordered_workqueue(drm, "%s-async-flip", + 0, crtc->name); + if (IS_ERR(vc4_crtc->async_flip_wq)) + return PTR_ERR(vc4_crtc->async_flip_wq); + if (vc4->gen == VC4_GEN_4) { drm_mode_crtc_set_gamma_size(crtc, ARRAY_SIZE(vc4_crtc->lut_r)); drm_crtc_enable_color_mgmt(crtc, 0, false, crtc->gamma_size); diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h index 09134809bf15..dae325a3854d 100644 --- a/drivers/gpu/drm/vc4/vc4_drv.h +++ b/drivers/gpu/drm/vc4/vc4_drv.h @@ -218,6 +218,11 @@ struct vc4_dev { struct { struct timer_list timer; struct work_struct reset_work; + + /* The instruction and uniforms caches were cleared at the + * last check that found no progress. + */ + bool caches_cleared; } hangcheck; struct drm_modeset_lock ctm_state_lock; @@ -255,6 +260,9 @@ struct vc4_bo { */ struct vc4_validated_shader_info *validated_shader; + /* Byte offset of the shader code in the BO. */ + u32 shader_offset; + /* One of enum vc4_kernel_bo_type, or VC4_BO_TYPE_COUNT + i * for user-allocated labels. */ @@ -596,6 +604,12 @@ struct vc4_crtc { * access to that value. */ unsigned int current_hvs_channel; + + /** + * @async_flip_wq: Ordered workqueue completing the CRTC's async + * page flips. + */ + struct workqueue_struct *async_flip_wq; }; #define to_vc4_crtc(_crtc) \ @@ -796,6 +810,11 @@ struct vc4_file { struct xarray perfmons; bool bin_bo_used; + + /* Counter picking the start offset of the next shader BO's code, + * see vc4_create_shader_bo_ioctl(). + */ + atomic_t shader_offset_idx; }; static inline struct vc4_exec_info * @@ -1083,7 +1102,7 @@ bool vc4_check_tex_size(struct vc4_exec_info *exec, /* vc4_validate_shader.c */ struct vc4_validated_shader_info * -vc4_validate_shader(struct drm_gem_dma_object *shader_obj); +vc4_validate_shader(struct drm_gem_dma_object *shader_obj, u32 offset); /* vc4_perfmon.c */ void vc4_perfmon_get(struct vc4_perfmon *perfmon); diff --git a/drivers/gpu/drm/vc4/vc4_dsi.c b/drivers/gpu/drm/vc4/vc4_dsi.c index 0148f2befed1..762929ec471f 100644 --- a/drivers/gpu/drm/vc4/vc4_dsi.c +++ b/drivers/gpu/drm/vc4/vc4_dsi.c @@ -1527,8 +1527,8 @@ static irqreturn_t vc4_dsi_irq_handler(int irq, void *data) complete(&dsi->xfer_completion); ret = IRQ_HANDLED; } else if (stat & DSI_PORT_BIT(INT_HSTX_TO)) { - complete(&dsi->xfer_completion); dsi->xfer_result = -ETIMEDOUT; + complete(&dsi->xfer_completion); ret = IRQ_HANDLED; } @@ -1599,9 +1599,8 @@ vc4_dsi_init_phy_clocks(struct vc4_dsi *dsi) dsi->clk_onecell->hws[i] = &fix->hw; } - return of_clk_add_hw_provider(dev->of_node, - of_clk_hw_onecell_get, - dsi->clk_onecell); + return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, + dsi->clk_onecell); } static void vc4_dsi_dma_mem_release(void *ptr) diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c index 956be198f8a7..9898eb1ef077 100644 --- a/drivers/gpu/drm/vc4/vc4_gem.c +++ b/drivers/gpu/drm/vc4/vc4_gem.c @@ -43,8 +43,7 @@ vc4_queue_hangcheck(struct drm_device *dev) { struct vc4_dev *vc4 = to_vc4_dev(dev); - mod_timer(&vc4->hangcheck.timer, - round_jiffies_up(jiffies + msecs_to_jiffies(100))); + mod_timer(&vc4->hangcheck.timer, jiffies + msecs_to_jiffies(100)); } struct vc4_hang_state { @@ -355,6 +354,7 @@ vc4_hangcheck_elapsed(struct timer_list *t) /* If idle, we can stop watching for hangs. */ if (!bin_exec && !render_exec) { + vc4->hangcheck.caches_cleared = false; spin_unlock_irqrestore(&vc4->job_lock, irqflags); return; } @@ -371,11 +371,27 @@ vc4_hangcheck_elapsed(struct timer_list *t) bin_exec->last_ct0ca = ct0ca; if (render_exec) render_exec->last_ct1ca = ct1ca; + + vc4->hangcheck.caches_cleared = false; + spin_unlock_irqrestore(&vc4->job_lock, irqflags); vc4_queue_hangcheck(dev); return; } + /* Clear the slice caches to workaround GFXH-16 */ + if (!vc4->hangcheck.caches_cleared) { + V3D_WRITE(V3D_SLCACTL, VC4_SET_FIELD(0xf, V3D_SLCACTL_ICC) | + VC4_SET_FIELD(0xf, V3D_SLCACTL_UCC)); + + vc4->hangcheck.caches_cleared = true; + spin_unlock_irqrestore(&vc4->job_lock, irqflags); + vc4_queue_hangcheck(dev); + return; + } + + vc4->hangcheck.caches_cleared = false; + spin_unlock_irqrestore(&vc4->job_lock, irqflags); /* We've gone too long with no progress, reset. This has to diff --git a/drivers/gpu/drm/vc4/vc4_validate.c b/drivers/gpu/drm/vc4/vc4_validate.c index 565227657b43..b67ac1b21265 100644 --- a/drivers/gpu/drm/vc4/vc4_validate.c +++ b/drivers/gpu/drm/vc4/vc4_validate.c @@ -815,7 +815,7 @@ validate_gl_shader_rec(struct drm_device *dev, exec->shader_rec_size -= packet_size; for (i = 0; i < shader_reloc_count; i++) { - if (src_handles[i] > exec->bo_count) { + if (src_handles[i] >= exec->bo_count) { DRM_DEBUG("Shader handle %d too big\n", src_handles[i]); return -EINVAL; } @@ -823,6 +823,9 @@ validate_gl_shader_rec(struct drm_device *dev, bo[i] = to_drm_gem_dma_obj(exec->bo[src_handles[i]]); if (!bo[i]) return -EINVAL; + + if (!to_vc4_bo(&bo[i]->base)->validated_shader) + return -EINVAL; } for (i = shader_reloc_count; i < nr_relocs; i++) { bo[i] = vc4_use_bo(exec, src_handles[i]); @@ -843,14 +846,16 @@ validate_gl_shader_rec(struct drm_device *dev, } for (i = 0; i < shader_reloc_count; i++) { - struct vc4_validated_shader_info *validated_shader; + struct vc4_validated_shader_info *validated_shader = + to_vc4_bo(&bo[i]->base)->validated_shader; uint32_t o = shader_reloc_offsets[i]; uint32_t src_offset = *(uint32_t *)(pkt_u + o); uint32_t *texture_handles_u; void *uniform_data_u; uint32_t tex, uni; - *(uint32_t *)(pkt_v + o) = bo[i]->dma_addr + src_offset; + *(uint32_t *)(pkt_v + o) = bo[i]->dma_addr + src_offset + + to_vc4_bo(&bo[i]->base)->shader_offset; if (src_offset != 0) { DRM_DEBUG("Shaders must be at offset 0 of " @@ -858,10 +863,6 @@ validate_gl_shader_rec(struct drm_device *dev, return -EINVAL; } - validated_shader = to_vc4_bo(&bo[i]->base)->validated_shader; - if (!validated_shader) - return -EINVAL; - if (validated_shader->uniforms_src_size > exec->uniforms_size) { DRM_DEBUG("Uniforms src buffer overflow\n"); diff --git a/drivers/gpu/drm/vc4/vc4_validate_shaders.c b/drivers/gpu/drm/vc4/vc4_validate_shaders.c index db7c0dbdf60e..09ebd8f92bf5 100644 --- a/drivers/gpu/drm/vc4/vc4_validate_shaders.c +++ b/drivers/gpu/drm/vc4/vc4_validate_shaders.c @@ -779,7 +779,7 @@ vc4_handle_branch_target(struct vc4_shader_validation_state *validation_state) } struct vc4_validated_shader_info * -vc4_validate_shader(struct drm_gem_dma_object *shader_obj) +vc4_validate_shader(struct drm_gem_dma_object *shader_obj, u32 offset) { struct vc4_dev *vc4 = to_vc4_dev(shader_obj->base.dev); bool found_shader_end = false; @@ -793,8 +793,9 @@ vc4_validate_shader(struct drm_gem_dma_object *shader_obj) return NULL; memset(&validation_state, 0, sizeof(validation_state)); - validation_state.shader = shader_obj->vaddr; - validation_state.max_ip = shader_obj->base.size / sizeof(uint64_t); + validation_state.shader = shader_obj->vaddr + offset; + validation_state.max_ip = (shader_obj->base.size - offset) / + sizeof(uint64_t); reset_validation_state(&validation_state); |
