[PATCH 55/59] timeline-hwsp

[Chris Wilson]

---
 drivers/gpu/drm/i915/i915_drv.h      |  4 ++
 drivers/gpu/drm/i915/i915_gem.c      |  1 +
 drivers/gpu/drm/i915/i915_timeline.c | 98 +++++++++++++++++++++++++++-
 drivers/gpu/drm/i915/i915_timeline.h | 11 ++++
 4 files changed, 112 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 2aeea38140b1..06451efdc8f1 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -1961,6 +1961,10 @@ struct drm_i915_private {
 		struct mutex timeline_lock;
 		struct list_head timelines;
 
+		/* Pack multiple timelines' seqnos into the same page */
+		struct i915_vma *timeline_hwsp;
+		u64 timeline_free;
+
 		struct list_head active_rings;
 		struct list_head closed_vma;
 		u32 active_requests;
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 19f71d7c4806..29e23f0bb02d 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -5324,6 +5324,7 @@ void i915_gem_fini(struct drm_i915_private *dev_priv)
 
 	intel_uc_fini_misc(dev_priv);
 	i915_gem_cleanup_userptr(dev_priv);
+	i915_timelines_fini(dev_priv);
 
 	i915_gem_drain_freed_objects(dev_priv);
 
diff --git a/drivers/gpu/drm/i915/i915_timeline.c b/drivers/gpu/drm/i915/i915_timeline.c
index c4e7ad179d86..4cacc82bacbd 100644
--- a/drivers/gpu/drm/i915/i915_timeline.c
+++ b/drivers/gpu/drm/i915/i915_timeline.c
@@ -9,6 +9,52 @@
 #include "i915_timeline.h"
 #include "i915_syncmap.h"
 
+static void get_seqno_locked(struct drm_i915_private *i915,
+			     struct i915_timeline *timeline)
+{
+	struct i915_vma *vma;
+	void *vaddr;
+	int offset;
+
+	offset = __builtin_ffsll(i915->gt.timeline_free);
+	if (offset-- == 0 && i915->gt.timeline_hwsp) {
+		i915_vma_put(i915->gt.timeline_hwsp);
+		i915->gt.timeline_hwsp = NULL;
+	}
+
+	vma = i915->gt.timeline_hwsp;
+	if (!vma) {
+		struct drm_i915_gem_object *bo;
+
+		BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE);
+		bo = i915_gem_object_create_internal(i915, PAGE_SIZE);
+		if (!bo)
+			return;
+
+		vma = i915_vma_instance(bo, &i915->ggtt.vm, NULL);
+		if (IS_ERR(vma))
+			return;
+
+		i915->gt.timeline_hwsp = vma;
+		i915->gt.timeline_free = ~0ull;
+		offset = 0;
+	}
+
+	vaddr = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
+	if (IS_ERR(vaddr))
+		return;
+
+	timeline->hwsp_ggtt = i915_vma_get(vma);
+	timeline->hwsp_offset = offset * CACHELINE_BYTES;
+
+	timeline->hwsp_seqno =
+		memset(vaddr + timeline->hwsp_offset,
+		       0,
+		       sizeof(*timeline->hwsp_seqno));
+
+	i915->gt.timeline_free &= ~BIT_ULL(offset);
+}
+
 void i915_timeline_init(struct drm_i915_private *i915,
 			struct i915_timeline *timeline,
 			const char *name)
@@ -23,8 +69,10 @@ void i915_timeline_init(struct drm_i915_private *i915,
 
 	timeline->i915 = i915;
 	timeline->name = name;
+	timeline->pin_count = 0;
 
 	mutex_lock(&i915->gt.timeline_lock);
+	get_seqno_locked(i915, timeline);
 	list_add(&timeline->link, &i915->gt.timelines);
 	mutex_unlock(&i915->gt.timeline_lock);
 
@@ -69,13 +117,22 @@ void i915_timelines_park(struct drm_i915_private *i915)
 
 void i915_timeline_fini(struct i915_timeline *timeline)
 {
+	struct drm_i915_private *i915 = timeline->i915;
+
+	GEM_BUG_ON(timeline->pin_count);
 	GEM_BUG_ON(!list_empty(&timeline->requests));
 
 	i915_syncmap_free(&timeline->sync);
 
-	mutex_lock(&timeline->i915->gt.timeline_lock);
+	mutex_lock(&i915->gt.timeline_lock);
 	list_del(&timeline->link);
-	mutex_unlock(&timeline->i915->gt.timeline_lock);
+	if (timeline->hwsp_ggtt == i915->gt.timeline_hwsp)
+		i915->gt.timeline_free |=
+			BIT_ULL(timeline->hwsp_offset / CACHELINE_BYTES);
+	mutex_unlock(&i915->gt.timeline_lock);
+
+	i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
+	i915_vma_put(timeline->hwsp_ggtt);
 }
 
 struct i915_timeline *
@@ -93,6 +150,34 @@ i915_timeline_create(struct drm_i915_private *i915, const char *name)
 	return timeline;
 }
 
+int i915_timeline_pin(struct i915_timeline *tl)
+{
+	int err;
+
+	if (tl->pin_count++)
+		return 0;
+	GEM_BUG_ON(!tl->pin_count);
+
+	err = i915_vma_pin(tl->hwsp_ggtt, 0, 0, PIN_GLOBAL | PIN_HIGH);
+	if (err)
+		goto unpin;
+
+	return 0;
+
+unpin:
+	tl->pin_count = 0;
+	return err;
+}
+
+void i915_timeline_unpin(struct i915_timeline *tl)
+{
+	GEM_BUG_ON(!tl->pin_count);
+	if (--tl->pin_count)
+		return;
+
+	__i915_vma_unpin(tl->hwsp_ggtt);
+}
+
 void __i915_timeline_free(struct kref *kref)
 {
 	struct i915_timeline *timeline =
@@ -102,6 +187,15 @@ void __i915_timeline_free(struct kref *kref)
 	kfree(timeline);
 }
 
+void i915_timelines_fini(struct drm_i915_private *i915)
+{
+	if (!i915->gt.timeline_hwsp)
+		return;
+
+	i915_vma_put(i915->gt.timeline_hwsp);
+	i915->gt.timeline_hwsp = NULL;
+}
+
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 #include "selftests/mock_timeline.c"
 #include "selftests/i915_timeline.c"
diff --git a/drivers/gpu/drm/i915/i915_timeline.h b/drivers/gpu/drm/i915/i915_timeline.h
index df0ddcd6459d..edd4353e7ac6 100644
--- a/drivers/gpu/drm/i915/i915_timeline.h
+++ b/drivers/gpu/drm/i915/i915_timeline.h
@@ -32,6 +32,8 @@
 #include "i915_syncmap.h"
 #include "i915_utils.h"
 
+struct i915_vma;
+
 struct i915_timeline {
 	u64 fence_context;
 	u32 seqno;
@@ -41,6 +43,11 @@ struct i915_timeline {
 #define TIMELINE_ENGINE 1
 #define TIMELINE_VIRTUAL 2
 
+	unsigned int pin_count;
+	const u32 *hwsp_seqno;
+	struct i915_vma *hwsp_ggtt;
+	u32 hwsp_offset;
+
 	/**
 	 * List of breadcrumbs associated with GPU requests currently
 	 * outstanding.
@@ -136,6 +143,10 @@ static inline bool i915_timeline_sync_is_later(struct i915_timeline *tl,
 	return __i915_timeline_sync_is_later(tl, fence->context, fence->seqno);
 }
 
+int i915_timeline_pin(struct i915_timeline *tl);
+void i915_timeline_unpin(struct i915_timeline *tl);
+
 void i915_timelines_park(struct drm_i915_private *i915);
+void i915_timelines_fini(struct drm_i915_private *i915);
 
 #endif
-- 
2.20.1