[Intel-gfx] [PATCH 17/38] drm/i915: Keep all partially allocated HWSP on a freelist

[Chris Wilson]

Keep track of partially allocated pages for use in allocating future
timeline HWSP. This is still without migration, so it is possible for
the system to end up with each timeline in its own page, but we ensure
that no new allocation would needless allocate a fresh page!

Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
Cc: John Harrison <John.C.Harrison at Intel.com>
---
 drivers/gpu/drm/i915/i915_drv.h      |  3 +-
 drivers/gpu/drm/i915/i915_timeline.c | 85 +++++++++++++++++-----------
 2 files changed, 54 insertions(+), 34 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index d59228dabb6e..cd2ea6e13fa6 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -1981,8 +1981,7 @@ struct drm_i915_private {
 
 			/* Pack multiple timelines' seqnos into the same page */
 			spinlock_t hwsp_lock;
-			struct i915_vma *hwsp;
-			u64 bitmap;
+			struct list_head hwsp_free_list;
 		} timelines;
 
 		struct list_head active_rings;
diff --git a/drivers/gpu/drm/i915/i915_timeline.c b/drivers/gpu/drm/i915/i915_timeline.c
index 1579ace82cfd..92532f713dd7 100644
--- a/drivers/gpu/drm/i915/i915_timeline.c
+++ b/drivers/gpu/drm/i915/i915_timeline.c
@@ -9,74 +9,98 @@
 #include "i915_timeline.h"
 #include "i915_syncmap.h"
 
+struct i915_timeline_hwsp {
+	struct list_head link;
+	struct i915_vma *vma;
+	u64 bitmap;
+};
+
 static int hwsp_alloc(struct i915_timeline *timeline)
 {
-#define NBITS BITS_PER_TYPE(typeof(gt->bitmap))
 	struct drm_i915_private *i915 = timeline->i915;
 	struct i915_gt_timelines *gt = &i915->gt.timelines;
-	struct i915_vma *vma;
+	struct i915_timeline_hwsp *hwsp;
 	int offset;
 
 	spin_lock(&gt->hwsp_lock);
 
-restart:
-	offset = find_first_bit((unsigned long *)&gt->bitmap, NBITS);
-	if (offset == NBITS && gt->hwsp) {
-		i915_vma_put(gt->hwsp);
-		gt->hwsp = NULL;
-	}
-
-	vma = gt->hwsp;
-	if (!vma) {
+	/* hwsp_free_list only contains HWSP that have available cachelines */
+	hwsp = list_first_entry_or_null(&gt->hwsp_free_list,
+					typeof(*hwsp), link);
+	if (!hwsp) {
 		struct drm_i915_gem_object *obj;
+		struct i915_vma *vma;
 
 		spin_unlock(&gt->hwsp_lock);
 
-		BUILD_BUG_ON(NBITS * CACHELINE_BYTES > PAGE_SIZE);
+		hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL);
+		if (!hwsp)
+			return -ENOMEM;
+
+		BUILD_BUG_ON(BITS_PER_TYPE(hwsp->bitmap) * CACHELINE_BYTES > PAGE_SIZE);
 		obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
-		if (IS_ERR(obj))
+		if (IS_ERR(obj)) {
+			kfree(hwsp);
 			return PTR_ERR(obj);
+		}
 
 		i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 
 		vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
 		if (IS_ERR(vma)) {
 			i915_gem_object_put(obj);
+			kfree(hwsp);
 			return PTR_ERR(vma);
 		}
 
-		spin_lock(&gt->hwsp_lock);
-		if (gt->hwsp) {
-			i915_gem_object_put(obj);
-			goto restart;
-		}
+		vma->private = hwsp;
+		hwsp->vma = vma;
+		hwsp->bitmap = ~0ull;
 
-		gt->hwsp = vma;
-		gt->bitmap = ~0ull;
-		offset = 0;
+		spin_lock(&gt->hwsp_lock);
+		list_add(&hwsp->link, &gt->hwsp_free_list);
 	}
 
-	gt->bitmap &= ~BIT_ULL(offset);
+	GEM_BUG_ON(!hwsp->bitmap);
+	offset = __ffs64(hwsp->bitmap);
+	hwsp->bitmap &= ~BIT_ULL(offset);
+	if (!hwsp->bitmap)
+		list_del(&hwsp->link);
 
 	spin_unlock(&gt->hwsp_lock);
 
-	timeline->hwsp_ggtt = i915_vma_get(vma);
+	timeline->hwsp_ggtt = i915_vma_get(hwsp->vma);
 	timeline->hwsp_offset = offset * CACHELINE_BYTES;
 
+	GEM_BUG_ON(timeline->hwsp_ggtt->private != hwsp);
+
 	return 0;
-#undef NBITS
 }
 
 static void hwsp_free(struct i915_timeline *timeline)
 {
 	struct i915_gt_timelines *gt = &timeline->i915->gt.timelines;
+	struct i915_timeline_hwsp *hwsp;
 
-	if (timeline->hwsp_ggtt != gt->hwsp)
+	hwsp = timeline->hwsp_ggtt->private;
+	if (!hwsp)
 		return;
 
 	spin_lock(&gt->hwsp_lock);
-	if (timeline->hwsp_ggtt == gt->hwsp)
-		gt->bitmap |= BIT_ULL(timeline->hwsp_offset / CACHELINE_BYTES);
+
+	/* As a cacheline becomes available, publish the HWSP on the freelist */
+	if (!hwsp->bitmap)
+		list_add_tail(&hwsp->link, &gt->hwsp_free_list);
+
+	hwsp->bitmap |= BIT_ULL(timeline->hwsp_offset / CACHELINE_BYTES);
+
+	/* And if no one is left using it, give the page back to the system */
+	if (hwsp->bitmap == ~0ull) {
+		i915_vma_put(hwsp->vma);
+		list_del(&hwsp->link);
+		kfree(hwsp);
+	}
+
 	spin_unlock(&gt->hwsp_lock);
 }
 
@@ -148,6 +172,7 @@ void i915_timelines_init(struct drm_i915_private *i915)
 	INIT_LIST_HEAD(&gt->list);
 
 	spin_lock_init(&gt->hwsp_lock);
+	INIT_LIST_HEAD(&gt->hwsp_free_list);
 
 	/* via i915_gem_wait_for_idle() */
 	i915_gem_shrinker_taints_mutex(i915, &gt->mutex);
@@ -264,13 +289,9 @@ void __i915_timeline_free(struct kref *kref)
 void i915_timelines_fini(struct drm_i915_private *i915)
 {
 	struct i915_gt_timelines *gt = &i915->gt.timelines;
-	struct i915_vma *vma;
 
 	GEM_BUG_ON(!list_empty(&gt->list));
-
-	vma = fetch_and_zero(&i915->gt.timelines.hwsp);
-	if (vma)
-		i915_vma_put(vma);
+	GEM_BUG_ON(!list_empty(&gt->hwsp_free_list));
 
 	mutex_destroy(&gt->mutex);
 }
-- 
2.20.1