[PATCH 39/50] cache1

[Chris Wilson]

---
 .../gpu/drm/i915/gem/i915_gem_execbuffer.c    | 62 ++++++++++++++-----
 .../i915/gem/selftests/i915_gem_execbuffer.c  | 18 ++++--
 .../drm/i915/gt/intel_gt_buffer_pool_types.h  |  2 +-
 3 files changed, 59 insertions(+), 23 deletions(-)

diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
index c44103a8a02f..2728b4b322ab 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
@@ -1958,6 +1958,26 @@ static unsigned int reloc_bb_flags(const struct reloc_cache *cache)
 	return cache->gen > 5 ? 0 : I915_DISPATCH_SECURE;
 }
 
+static struct intel_gt_buffer_pool_node *reloc_node(struct i915_vma *vma)
+{
+	return vma->private;
+}
+
+static void reloc_close(struct reloc_cache *c)
+{
+	struct i915_vma *vma = c->head.vma;
+
+	while (vma) {
+		struct i915_vma *next =
+			((struct eb_relocs_link *)(c->map + c->max))->vma;
+
+		i915_gem_object_unpin_map(vma->obj);
+		intel_gt_buffer_pool_put(reloc_node(vma));
+
+		vma = next;
+	}
+}
+
 static int
 reloc_gpu_flush(struct i915_execbuffer *eb, struct i915_request *rq, int err)
 {
@@ -1968,7 +1988,6 @@ reloc_gpu_flush(struct i915_execbuffer *eb, struct i915_request *rq, int err)
 	*cs++ = MI_BATCH_BUFFER_END;
 	__i915_gem_object_flush_map(cache->head.vma->obj,
 				    0, (void *)cs - (void *)cache->map);
-	i915_gem_object_unpin_map(cache->head.vma->obj);
 
 	if (rq->engine->emit_init_breadcrumb)
 		err = rq->engine->emit_init_breadcrumb(rq);
@@ -2178,32 +2197,33 @@ eb_relocs_grow(struct i915_execbuffer *eb, unsigned long *count)
 	GEM_BUG_ON(c->pos > c->max);
 	remain = c->max - c->pos;
 	if (remain == 0) {
-		struct drm_i915_gem_object *obj;
+		struct intel_gt_buffer_pool_node *pool;
 		struct i915_vma *vma;
 		struct eb_vma *ev;
 
-		obj = i915_gem_object_create_internal(eb->i915, c->bufsz);
-		if (IS_ERR(obj))
-			return ERR_CAST(obj);
+		pool = intel_gt_get_buffer_pool(eb->engine->gt, c->bufsz);
+		if (IS_ERR(pool))
+			return ERR_CAST(pool);
 
+		i915_gem_object_set_readonly(pool->obj);
 		if (c->gen >= 6)
-			i915_gem_object_set_cache_coherency(obj,
+			i915_gem_object_set_cache_coherency(pool->obj,
 							    I915_CACHE_LLC);
 
-		vma = i915_vma_instance(obj, eb->context->vm, NULL);
+		vma = i915_vma_instance(pool->obj, eb->context->vm, NULL);
 		if (IS_ERR(vma)) {
-			i915_gem_object_put(obj);
+			intel_gt_buffer_pool_put(pool);
 			return ERR_CAST(vma);
 		}
 
 		ev = kzalloc(sizeof(*ev), GFP_KERNEL);
 		if (!ev) {
-			i915_gem_object_put(obj);
+			intel_gt_buffer_pool_put(pool);
 			return ERR_PTR(-ENOMEM);
 		}
 
-		vma->private = ev;
-		ev->vma = vma;
+		vma->private = pool;
+		ev->vma = i915_vma_get(vma);
 		ev->exec = &no_entry;
 		ev->flags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
 		list_add_tail(&ev->bind_link, &eb->bind_list);
@@ -2219,10 +2239,11 @@ eb_relocs_grow(struct i915_execbuffer *eb, unsigned long *count)
 		}
 
 		c->pos = 0;
-		c->map = i915_gem_object_pin_map(obj, I915_MAP_WB);
+		c->map = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
 		if (IS_ERR(c->map))
 			return ERR_CAST(c->map);
 
+		memset(c->map + c->max, 0, sizeof(struct eb_relocs_link));
 		remain = c->max;
 	}
 	*count = min(remain, *count);
@@ -2325,6 +2346,8 @@ get_gpu_relocs(struct i915_execbuffer *eb,
 		i915_gem_object_flush_map(c->head.vma->obj);
 		i915_gem_object_unpin_map(c->head.vma->obj);
 
+		intel_gt_buffer_pool_put(reloc_node(c->head.vma));
+
 		c->head = link;
 		c->map = NULL;
 	}
@@ -2334,6 +2357,11 @@ get_gpu_relocs(struct i915_execbuffer *eb,
 		int err;
 
 		GEM_BUG_ON(!vma);
+
+		err = i915_active_add_request(&reloc_node(vma)->active, rq);
+		if (err < 0)
+			return ERR_PTR(err);
+
 		i915_vma_lock(vma);
 		err = i915_request_await_object(rq, vma->obj, false);
 		if (err == 0)
@@ -2483,19 +2511,19 @@ static int eb_relocate(struct i915_execbuffer *eb)
 	/* Copy the user's relocations into plain system memory */
 	err = eb_relocs_copy_user(eb);
 	if (err)
-		return err;
+		goto err_close;
 
 	/* Now reacquire everything, including the extra reloc bo */
 	err = eb_reserve_vm(eb);
 	if (err)
-		return err;
+		goto err_close;
 
 	/* The objects are now final, convert the relocations into commands. */
 	err = eb_relocs_gpu(eb);
-	if (err)
-		return err;
 
-	return 0;
+err_close:
+	reloc_close(c);
+	return err;
 }
 
 static int eb_reserve(struct i915_execbuffer *eb)
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c
index 8776f2750fa7..74134a2290d0 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c
@@ -92,6 +92,18 @@ static int check_relocs(struct i915_execbuffer *eb, struct eb_vma *ev)
 	return err;
 }
 
+static int mock_reloc_gpu(struct i915_execbuffer *eb)
+{
+	int err;
+
+	err = eb_reserve_vm(eb);
+	if (err == 0)
+		err = eb_relocs_gpu(eb);
+
+	reloc_close(&eb->reloc_cache);
+	return err;
+}
+
 static int __igt_gpu_reloc(struct i915_execbuffer *eb, struct eb_vma *ev)
 {
 	int err;
@@ -101,11 +113,7 @@ static int __igt_gpu_reloc(struct i915_execbuffer *eb, struct eb_vma *ev)
 		return err;
 	ev->exec->relocation_count = err;
 
-	err = eb_reserve_vm(eb);
-	if (err)
-		return err;
-
-	err = eb_relocs_gpu(eb);
+	err = mock_reloc_gpu(eb);
 	if (err)
 		return err;
 
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool_types.h b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool_types.h
index bcf1658c9633..bb8b0312cb74 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool_types.h
@@ -17,7 +17,7 @@ struct drm_i915_gem_object;
 
 struct intel_gt_buffer_pool {
 	spinlock_t lock;
-	struct list_head cache_list[4];
+	struct list_head cache_list[5];
 	struct delayed_work work;
 };
 
-- 
2.20.1