[PATCH 21/25] drm/i915: Keep all partially allocated HWSP on a freelist

[Chris Wilson]

Keep track of partially allocated pages for use in allocating future
timeline HWSP. This is still without migration, so it is possible for
the system to end up with each timeline in its own page, but we ensure
that no new allocation would needless allocate a fresh page!

Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
Cc: John Harrison <John.C.Harrison at Intel.com>
---
 drivers/gpu/drm/i915/i915_drv.h      |  3 +-
 drivers/gpu/drm/i915/i915_timeline.c | 90 ++++++++++++++++------------
 2 files changed, 54 insertions(+), 39 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 0b003cd10a4c..de9d203c09de 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -1979,8 +1979,7 @@ struct drm_i915_private {
 		struct list_head timelines;
 
 		/* Pack multiple timelines' seqnos into the same page */
-		struct i915_vma *timeline_hwsp;
-		u64 timeline_free;
+		struct list_head timeline_hwsp;
 
 		struct list_head active_rings;
 		struct list_head closed_vma;
diff --git a/drivers/gpu/drm/i915/i915_timeline.c b/drivers/gpu/drm/i915/i915_timeline.c
index bcac88434896..11eb0e9d5480 100644
--- a/drivers/gpu/drm/i915/i915_timeline.c
+++ b/drivers/gpu/drm/i915/i915_timeline.c
@@ -9,80 +9,100 @@
 #include "i915_timeline.h"
 #include "i915_syncmap.h"
 
-#define NBITS BITS_PER_TYPE(typeof(i915->gt.timeline_free))
-
-static int find_first_cacheline(struct drm_i915_private *i915)
-{
-	return find_first_bit((unsigned long *)&i915->gt.timeline_free, NBITS);
-}
+struct i915_timeline_hwsp {
+	struct list_head link;
+	struct i915_vma *vma;
+	u64 bitmap;
+};
 
 static int hwsp_alloc(struct i915_timeline *timeline)
 {
+#define NBITS BITS_PER_TYPE(typeof(hwsp->bitmap))
 	struct drm_i915_private *i915 = timeline->i915;
-	struct i915_vma *vma;
-	int offset;
+	struct i915_timeline_hwsp *hwsp;
+	int offset = NBITS;
 
 	mutex_lock(&i915->gt.timeline_lock);
 
-restart:
-	offset = find_first_cacheline(i915);
-	if (offset == NBITS && i915->gt.timeline_hwsp) {
-		i915_vma_put(i915->gt.timeline_hwsp);
-		i915->gt.timeline_hwsp = NULL;
+	list_for_each_entry(hwsp, &i915->gt.timeline_hwsp, link) {
+		offset = find_first_bit((unsigned long *)&hwsp->bitmap, NBITS);
+		if (offset != NBITS)
+			break;
 	}
 
-	vma = i915->gt.timeline_hwsp;
-	if (!vma) {
+	if (offset == NBITS) {
 		struct drm_i915_gem_object *bo;
+		struct i915_vma *vma;
 
 		/* Drop the lock before allocations */
 		mutex_unlock(&i915->gt.timeline_lock);
 
+		hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL);
+		if (!hwsp)
+			return -ENOMEM;
+
 		BUILD_BUG_ON(NBITS * CACHELINE_BYTES > PAGE_SIZE);
 		bo = i915_gem_object_create_internal(i915, PAGE_SIZE);
-		if (IS_ERR(bo))
+		if (IS_ERR(bo)) {
+			kfree(hwsp);
 			return PTR_ERR(bo);
+		}
 
 		i915_gem_object_set_cache_level(bo, I915_CACHE_LLC);
 
 		vma = i915_vma_instance(bo, &i915->ggtt.vm, NULL);
 		if (IS_ERR(vma)) {
 			i915_gem_object_put(bo);
+			kfree(hwsp);
 			return PTR_ERR(vma);
 		}
 
-		mutex_lock(&i915->gt.timeline_lock);
-		if (i915->gt.timeline_hwsp) {
-			i915_gem_object_put(bo);
-			goto restart;
-		}
-
-		i915->gt.timeline_hwsp = vma;
-		i915->gt.timeline_free = ~0ull;
+		vma->private = hwsp;
+		hwsp->vma = vma;
+		hwsp->bitmap = ~0ull;
 		offset = 0;
+
+		mutex_lock(&i915->gt.timeline_lock);
+		list_add(&hwsp->link, &i915->gt.timeline_hwsp);
 	}
 
-	i915->gt.timeline_free &= ~BIT_ULL(offset);
+	hwsp->bitmap &= ~BIT_ULL(offset);
+	if (!hwsp->bitmap)
+		list_del(&hwsp->link);
+
+	mutex_unlock(&i915->gt.timeline_lock);
 
-	timeline->hwsp_ggtt = i915_vma_get(vma);
+	timeline->hwsp_ggtt = i915_vma_get(hwsp->vma);
 	timeline->hwsp_offset = offset * CACHELINE_BYTES;
 
-	mutex_unlock(&i915->gt.timeline_lock);
+	GEM_BUG_ON(timeline->hwsp_ggtt->private != hwsp);
 
 	return 0;
+#undef NBITS
 }
 
 static void hwsp_free(struct i915_timeline *timeline)
 {
 	struct drm_i915_private *i915 = timeline->i915;
+	struct i915_timeline_hwsp *hwsp;
 
-	if (timeline->hwsp_ggtt != i915->gt.timeline_hwsp)
+	hwsp = timeline->hwsp_ggtt->private;
+	if (!hwsp)
 		return;
 
 	mutex_lock(&i915->gt.timeline_lock);
-	if (timeline->hwsp_ggtt == i915->gt.timeline_hwsp)
-		i915->gt.timeline_free |=
-			BIT_ULL(timeline->hwsp_offset / CACHELINE_BYTES);
+
+	if (!hwsp->bitmap)
+		list_add_tail(&hwsp->link, &i915->gt.timeline_hwsp);
+
+	hwsp->bitmap |= BIT_ULL(timeline->hwsp_offset / CACHELINE_BYTES);
+
+	if (hwsp->bitmap == ~0ull) {
+		i915_vma_put(hwsp->vma);
+		list_del(&hwsp->link);
+		kfree(hwsp);
+	}
+
 	mutex_unlock(&i915->gt.timeline_lock);
 }
 
@@ -149,6 +169,7 @@ void i915_timelines_init(struct drm_i915_private *i915)
 {
 	mutex_init(&i915->gt.timeline_lock);
 	INIT_LIST_HEAD(&i915->gt.timelines);
+	INIT_LIST_HEAD(&i915->gt.timeline_hwsp);
 
 	/* via i915_gem_wait_for_idle() */
 	i915_gem_shrinker_taints_mutex(i915, &i915->gt.timeline_lock);
@@ -261,13 +282,8 @@ void __i915_timeline_free(struct kref *kref)
 
 void i915_timelines_fini(struct drm_i915_private *i915)
 {
-	struct i915_vma *vma;
-
 	GEM_BUG_ON(!list_empty(&i915->gt.timelines));
-
-	vma = fetch_and_zero(&i915->gt.timeline_hwsp);
-	if (vma)
-		i915_vma_put(vma);
+	GEM_BUG_ON(!list_empty(&i915->gt.timeline_hwsp));
 
 	mutex_destroy(&i915->gt.timeline_lock);
 }
-- 
2.20.1