[Mesa-dev] [PATCH] i965: Put freed but busy buffer to the head of the free list

[Chris Wilson]

We treat out free list as a rough LRU, at the tail we have active
buffers to be allocated for rendering, and at the head we have the
older, inactive buffers that we can allocate for use by the CPU.

At the time of freeing, we can inspect the busyness of the buffer to
decide if we should place it at the tail (for reuse of active buffers)
or head (for reuse of inactive buffers). Furthermore, we can reduce the
frequency of calling madvise by only using it for buffers that are
inactive (busy buffers will be pinned by they use by the GPU, so should
not significantly add to mempressure, but will still be reaped by our
timed cache.)
---
 src/mesa/drivers/dri/i965/brw_bufmgr.c | 48 ++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 19 deletions(-)

diff --git a/src/mesa/drivers/dri/i965/brw_bufmgr.c b/src/mesa/drivers/dri/i965/brw_bufmgr.c
index dc693f7d63..a869e7665e 100644
--- a/src/mesa/drivers/dri/i965/brw_bufmgr.c
+++ b/src/mesa/drivers/dri/i965/brw_bufmgr.c
@@ -112,7 +112,7 @@ struct brw_bufmgr {
    /** Array of lists of cached gem objects of power-of-two sizes */
    struct bo_cache_bucket cache_bucket[14 * 4];
    int num_buckets;
-   time_t time;
+   unsigned time;
 
    struct hash_table *name_table;
    struct hash_table *handle_table;
@@ -336,7 +336,7 @@ retry:
       }
 
       if (alloc_from_cache) {
-         if (!brw_bo_madvise(bo, I915_MADV_WILLNEED)) {
+         if (bo->free_time && !brw_bo_madvise(bo, I915_MADV_WILLNEED)) {
             bo_free(bo);
             brw_bo_cache_purge_bucket(bufmgr, bucket);
             goto retry;
@@ -408,7 +408,7 @@ retry:
 
    bo->name = name;
    p_atomic_set(&bo->refcount, 1);
-   bo->reusable = true;
+   bo->reusable = bufmgr->bo_reuse;
    bo->cache_coherent = bufmgr->has_llc;
    bo->index = -1;
 
@@ -678,10 +678,17 @@ bo_free(struct brw_bo *bo)
 
 /** Frees all cached buffers significantly older than @time. */
 static void
-cleanup_bo_cache(struct brw_bufmgr *bufmgr, time_t time)
+cleanup_bo_cache(struct brw_bufmgr *bufmgr)
 {
+   struct timespec tv;
+   unsigned time;
    int i;
 
+   clock_gettime(CLOCK_MONOTONIC, &tv);
+   time = tv.tv_sec;
+   if (unlikely(!time)) /* 0 is reserved for unset */
+      time = 1;
+
    if (bufmgr->time == time)
       return;
 
@@ -689,12 +696,16 @@ cleanup_bo_cache(struct brw_bufmgr *bufmgr, time_t time)
       struct bo_cache_bucket *bucket = &bufmgr->cache_bucket[i];
 
       list_for_each_entry_safe(struct brw_bo, bo, &bucket->head, head) {
-         if (time - bo->free_time <= 1)
-            break;
-
-         list_del(&bo->head);
+         if (!bo->free_time) {
+            if (brw_bo_busy(bo))
+               break;
 
-         bo_free(bo);
+            brw_bo_madvise(bo, I915_MADV_DONTNEED);
+            bo->free_time = time;
+         } else if (time - bo->free_time > 1) {
+            list_del(&bo->head);
+            bo_free(bo);
+         }
       }
    }
 
@@ -702,7 +713,7 @@ cleanup_bo_cache(struct brw_bufmgr *bufmgr, time_t time)
 }
 
 static void
-bo_unreference_final(struct brw_bo *bo, time_t time)
+bo_unreference_final(struct brw_bo *bo)
 {
    struct brw_bufmgr *bufmgr = bo->bufmgr;
    struct bo_cache_bucket *bucket;
@@ -711,14 +722,16 @@ bo_unreference_final(struct brw_bo *bo, time_t time)
 
    bucket = bucket_for_size(bufmgr, bo->size);
    /* Put the buffer into our internal cache for reuse if we can. */
-   if (bufmgr->bo_reuse && bo->reusable && bucket != NULL &&
-       brw_bo_madvise(bo, I915_MADV_DONTNEED)) {
-      bo->free_time = time;
+   if (bo->reusable && bucket != NULL) {
 
       bo->name = NULL;
       bo->kflags = 0;
 
-      list_addtail(&bo->head, &bucket->head);
+      bo->free_time = 0;
+      if (brw_bo_busy(bo))
+         list_addtail(&bo->head, &bucket->head);
+      else
+         list_add(&bo->head, &bucket->head);
    } else {
       bo_free(bo);
    }
@@ -734,15 +747,12 @@ brw_bo_unreference(struct brw_bo *bo)
 
    if (atomic_add_unless(&bo->refcount, -1, 1)) {
       struct brw_bufmgr *bufmgr = bo->bufmgr;
-      struct timespec time;
-
-      clock_gettime(CLOCK_MONOTONIC, &time);
 
       pthread_mutex_lock(&bufmgr->lock);
 
       if (p_atomic_dec_zero(&bo->refcount)) {
-         bo_unreference_final(bo, time.tv_sec);
-         cleanup_bo_cache(bufmgr, time.tv_sec);
+         cleanup_bo_cache(bufmgr);
+         bo_unreference_final(bo);
       }
 
       pthread_mutex_unlock(&bufmgr->lock);
-- 
2.13.3