drm/i915: More accurately track last fence usage by the GPU

Based on a patch by Daniel Vetter.

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>

3 files changed, 78 insertions, 52 deletions

diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index addb9392e1c4..5faae476954c 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -110,7 +110,7 @@ static const char *get_tiling_flag(struct drm_i915_gem_object *obj)
 static void
 describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
 {
-	seq_printf(m, "%p: %s%s %8zd %08x %08x %d%s%s",
+	seq_printf(m, "%p: %s%s %8zd %04x %04x %d %d%s%s",
 		   &obj->base,
 		   get_pin_flag(obj),
 		   get_tiling_flag(obj),
@@ -118,6 +118,7 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
 		   obj->base.read_domains,
 		   obj->base.write_domain,
 		   obj->last_rendering_seqno,
+		   obj->last_fenced_seqno,
 		   obj->dirty ? " dirty" : "",
 		   obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
 	if (obj->base.name)
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 42d3e901619d..ee7df1d2b8c8 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -124,9 +124,8 @@ struct drm_i915_master_private {
 #define I915_FENCE_REG_NONE -1
 
 struct drm_i915_fence_reg {
-	struct drm_i915_gem_object *obj;
 	struct list_head lru_list;
-	bool gpu;
+	struct drm_i915_gem_object *obj;
 };
 
 struct sdvo_device_mapping {
@@ -787,6 +786,12 @@ struct drm_i915_gem_object {
 	unsigned int fault_mappable : 1;
 	unsigned int pin_mappable : 1;
 
+	/*
+	 * Is the GPU currently using a fence to access this buffer,
+	 */
+	unsigned int pending_fenced_gpu_access:1;
+	unsigned int fenced_gpu_access:1;
+
 	struct page **pages;
 
 	/**
@@ -802,11 +807,13 @@ struct drm_i915_gem_object {
 	 */
 	uint32_t gtt_offset;
 
-	/* Which ring is refering to is this object */
-	struct intel_ring_buffer *ring;
-
 	/** Breadcrumb of last rendering to the buffer. */
 	uint32_t last_rendering_seqno;
+	struct intel_ring_buffer *ring;
+
+	/** Breadcrumb of last fenced GPU access to the buffer. */
+	uint32_t last_fenced_seqno;
+	struct intel_ring_buffer *last_fenced_ring;
 
 	/** Current tiling stride for the object, if it's tiled. */
 	uint32_t stride;
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 061426e1bbf4..2cfdee8811c4 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -1688,7 +1688,27 @@ i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
 	/* Move from whatever list we were on to the tail of execution. */
 	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
 	list_move_tail(&obj->ring_list, &ring->active_list);
+
 	obj->last_rendering_seqno = seqno;
+	if (obj->fenced_gpu_access) {
+		struct drm_i915_fence_reg *reg;
+
+		BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
+
+		obj->last_fenced_seqno = seqno;
+		obj->last_fenced_ring = ring;
+
+		reg = &dev_priv->fence_regs[obj->fence_reg];
+		list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
+	}
+}
+
+static void
+i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
+{
+	list_del_init(&obj->ring_list);
+	obj->last_rendering_seqno = 0;
+	obj->last_fenced_seqno = 0;
 }
 
 static void
@@ -1699,8 +1719,33 @@ i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
 
 	BUG_ON(!obj->active);
 	list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
-	list_del_init(&obj->ring_list);
-	obj->last_rendering_seqno = 0;
+
+	i915_gem_object_move_off_active(obj);
+}
+
+static void
+i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
+{
+	struct drm_device *dev = obj->base.dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	if (obj->pin_count != 0)
+		list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
+	else
+		list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
+
+	BUG_ON(!list_empty(&obj->gpu_write_list));
+	BUG_ON(!obj->active);
+	obj->ring = NULL;
+
+	i915_gem_object_move_off_active(obj);
+	obj->fenced_gpu_access = false;
+	obj->last_fenced_ring = NULL;
+
+	obj->active = 0;
+	drm_gem_object_unreference(&obj->base);
+
+	WARN_ON(i915_verify_lists(dev));
 }
 
 /* Immediately discard the backing storage */
@@ -1730,34 +1775,10 @@ i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
 }
 
 static void
-i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
-{
-	struct drm_device *dev = obj->base.dev;
-	drm_i915_private_t *dev_priv = dev->dev_private;
-
-	if (obj->pin_count != 0)
-		list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
-	else
-		list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
-	list_del_init(&obj->ring_list);
-
-	BUG_ON(!list_empty(&obj->gpu_write_list));
-
-	obj->last_rendering_seqno = 0;
-	obj->ring = NULL;
-	if (obj->active) {
-		obj->active = 0;
-		drm_gem_object_unreference(&obj->base);
-	}
-	WARN_ON(i915_verify_lists(dev));
-}
-
-static void
 i915_gem_process_flushing_list(struct drm_device *dev,
 			       uint32_t flush_domains,
 			       struct intel_ring_buffer *ring)
 {
-	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct drm_i915_gem_object *obj, *next;
 
 	list_for_each_entry_safe(obj, next,
@@ -1770,14 +1791,6 @@ i915_gem_process_flushing_list(struct drm_device *dev,
 			list_del_init(&obj->gpu_write_list);
 			i915_gem_object_move_to_active(obj, ring);
 
-			/* update the fence lru list */
-			if (obj->fence_reg != I915_FENCE_REG_NONE) {
-				struct drm_i915_fence_reg *reg =
-					&dev_priv->fence_regs[obj->fence_reg];
-				list_move_tail(&reg->lru_list,
-						&dev_priv->mm.fence_list);
-			}
-
 			trace_i915_gem_object_change_domain(obj,
 							    obj->base.read_domains,
 							    old_write_domain);
@@ -2615,8 +2628,7 @@ i915_gem_object_put_fence_reg(struct drm_i915_gem_object *obj,
 			      bool interruptible)
 {
 	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct drm_i915_fence_reg *reg;
+	int ret;
 
 	if (obj->fence_reg == I915_FENCE_REG_NONE)
 		return 0;
@@ -2631,19 +2643,23 @@ i915_gem_object_put_fence_reg(struct drm_i915_gem_object *obj,
 	 * therefore we must wait for any outstanding access to complete
 	 * before clearing the fence.
 	 */
-	reg = &dev_priv->fence_regs[obj->fence_reg];
-	if (reg->gpu) {
-		int ret;
-
+	if (obj->fenced_gpu_access) {
 		ret = i915_gem_object_flush_gpu_write_domain(obj, NULL);
 		if (ret)
 			return ret;
 
-		ret = i915_gem_object_wait_rendering(obj, interruptible);
+		obj->fenced_gpu_access = false;
+	}
+
+	if (obj->last_fenced_seqno) {
+		ret = i915_do_wait_request(dev,
+					   obj->last_fenced_seqno,
+					   interruptible,
+					   obj->last_fenced_ring);
 		if (ret)
 			return ret;
 
-		reg->gpu = false;
+		obj->last_fenced_seqno = false;
 	}
 
 	i915_gem_object_flush_gtt_write_domain(obj);
@@ -3166,8 +3182,9 @@ i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj,
 	 * write domain
 	 */
 	if (obj->base.write_domain &&
-	    (obj->base.write_domain != obj->base.pending_read_domains ||
-	     obj->ring != ring)) {
+	    (((obj->base.write_domain != obj->base.pending_read_domains ||
+	       obj->ring != ring)) ||
+	     (obj->fenced_gpu_access && !obj->pending_fenced_gpu_access))) {
 		flush_domains |= obj->base.write_domain;
 		invalidate_domains |=
 			obj->base.pending_read_domains & ~obj->base.write_domain;
@@ -3528,7 +3545,6 @@ i915_gem_execbuffer_reserve(struct drm_device *dev,
 			    struct drm_i915_gem_exec_object2 *exec_list,
 			    int count)
 {
-	struct drm_i915_private *dev_priv = dev->dev_private;
 	int ret, i, retry;
 
 	/* Attempt to pin all of the buffers into the GTT.
@@ -3601,7 +3617,7 @@ i915_gem_execbuffer_reserve(struct drm_device *dev,
 				if (ret)
 					break;
 
-				dev_priv->fence_regs[obj->fence_reg].gpu = true;
+				obj->pending_fenced_gpu_access = true;
 			}
 
 			entry->offset = obj->gtt_offset;
@@ -3981,6 +3997,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 			goto err;
 		}
 		obj->in_execbuffer = true;
+		obj->pending_fenced_gpu_access = false;
 	}
 
 	/* Move the objects en-masse into the GTT, evicting if necessary. */
@@ -4085,6 +4102,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 
 		obj->base.read_domains = obj->base.pending_read_domains;
 		obj->base.write_domain = obj->base.pending_write_domain;
+		obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
 
 		i915_gem_object_move_to_active(obj, ring);
 		if (obj->base.write_domain) {