[PATCH] dma-buf: Replace reservation shared fence array with a compressed radixtree
Chris Wilson
chris at chris-wilson.co.uk
Fri Nov 11 18:42:58 UTC 2016
Replace the array used for tracking shared fences with a compressed
radix tree. The primary operation on the shared fence arrays is
insertion and retrieval. Retrieval is reasonably fast, as we just copy
the array, but insertion into the shared fence array is slow as we must
iterate over all current fences to discard an older fence. To speed up
insertion, we want a direct lookup of the per-context slot. Enter the
radix tree.
---
drivers/dma-buf/dma-buf.c | 34 +-
drivers/dma-buf/reservation.c | 562 ++++++++++++++++----------------
drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 14 +-
drivers/gpu/drm/i915/i915_gem.c | 17 +-
drivers/gpu/drm/msm/msm_gem.c | 14 +-
drivers/gpu/drm/nouveau/nouveau_fence.c | 15 +-
drivers/gpu/drm/nouveau/nouveau_gem.c | 21 +-
drivers/gpu/drm/radeon/radeon_sync.c | 12 +-
drivers/gpu/drm/ttm/ttm_bo.c | 10 +-
include/linux/reservation.h | 147 ++++++---
10 files changed, 425 insertions(+), 421 deletions(-)
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index e72e64484131..3596bdd752e6 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -139,10 +139,9 @@ static unsigned int dma_buf_poll(struct file *file, poll_table *poll)
{
struct dma_buf *dmabuf;
struct reservation_object *resv;
- struct reservation_object_list *fobj;
- struct dma_fence *fence_excl;
+ struct dma_fence *excl;
unsigned long events;
- unsigned shared_count, seq;
+ unsigned seq;
dmabuf = file->private_data;
if (!dmabuf || !dmabuf->resv)
@@ -160,22 +159,17 @@ static unsigned int dma_buf_poll(struct file *file, poll_table *poll)
seq = read_seqcount_begin(&resv->seq);
rcu_read_lock();
- fobj = rcu_dereference(resv->fence);
- if (fobj)
- shared_count = fobj->shared_count;
- else
- shared_count = 0;
- fence_excl = rcu_dereference(resv->fence_excl);
+ excl = rcu_dereference(resv->excl);
if (read_seqcount_retry(&resv->seq, seq)) {
rcu_read_unlock();
goto retry;
}
- if (fence_excl && (!(events & POLLOUT) || shared_count == 0)) {
+ if (excl && (!(events & POLLOUT) || !reservation_object_has_shared(resv))) {
struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_excl;
unsigned long pevents = POLLIN;
- if (shared_count == 0)
+ if (!reservation_object_has_shared(resv))
pevents |= POLLOUT;
spin_lock_irq(&dmabuf->poll.lock);
@@ -187,28 +181,28 @@ static unsigned int dma_buf_poll(struct file *file, poll_table *poll)
spin_unlock_irq(&dmabuf->poll.lock);
if (events & pevents) {
- if (!dma_fence_get_rcu(fence_excl)) {
+ if (!dma_fence_get_rcu(excl)) {
/* force a recheck */
events &= ~pevents;
dma_buf_poll_cb(NULL, &dcb->cb);
- } else if (!dma_fence_add_callback(fence_excl, &dcb->cb,
+ } else if (!dma_fence_add_callback(excl, &dcb->cb,
dma_buf_poll_cb)) {
events &= ~pevents;
- dma_fence_put(fence_excl);
+ dma_fence_put(excl);
} else {
/*
* No callback queued, wake up any additional
* waiters.
*/
- dma_fence_put(fence_excl);
+ dma_fence_put(excl);
dma_buf_poll_cb(NULL, &dcb->cb);
}
}
}
- if ((events & POLLOUT) && shared_count > 0) {
+ if ((events & POLLOUT) && reservation_object_has_shared(resv)) {
struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_shared;
- int i;
+ struct reservation_shared_iter iter;
/* Only queue a new callback if no event has fired yet */
spin_lock_irq(&dmabuf->poll.lock);
@@ -221,8 +215,8 @@ static unsigned int dma_buf_poll(struct file *file, poll_table *poll)
if (!(events & POLLOUT))
goto out;
- for (i = 0; i < shared_count; ++i) {
- struct dma_fence *fence = rcu_dereference(fobj->shared[i]);
+ reservation_object_for_each_shared(resv, iter) {
+ struct dma_fence *fence = iter.fence;
if (!dma_fence_get_rcu(fence)) {
/*
@@ -245,7 +239,7 @@ static unsigned int dma_buf_poll(struct file *file, poll_table *poll)
}
/* No callback queued, wake up any additional waiters. */
- if (i == shared_count)
+ if (events & POLLOUT)
dma_buf_poll_cb(NULL, &dcb->cb);
}
diff --git a/drivers/dma-buf/reservation.c b/drivers/dma-buf/reservation.c
index 64748a7ddc0c..c834c4618aae 100644
--- a/drivers/dma-buf/reservation.c
+++ b/drivers/dma-buf/reservation.c
@@ -55,151 +55,205 @@ EXPORT_SYMBOL(reservation_seqcount_class);
const char reservation_seqcount_string[] = "reservation_seqcount";
EXPORT_SYMBOL(reservation_seqcount_string);
-/**
- * reservation_object_reserve_shared - Reserve space to add a shared
- * fence to a reservation_object.
- * @obj: reservation object
- *
- * Should be called before reservation_object_add_shared_fence(). Must
- * be called with obj->lock held.
- *
- * RETURNS
- * Zero for success, or -errno
- */
-int reservation_object_reserve_shared(struct reservation_object *obj)
+#define SHIFT ilog2(NSHARED)
+#define MASK (NSHARED-1)
+
+void __reservation_shared_iter_next(struct reservation_shared_iter *iter)
{
- struct reservation_object_list *fobj, *old;
- u32 max;
-
- old = reservation_object_get_list(obj);
-
- if (old && old->shared_max) {
- if (old->shared_count < old->shared_max) {
- /* perform an in-place update */
- kfree(obj->staged);
- obj->staged = NULL;
- return 0;
- } else
- max = old->shared_max * 2;
- } else
- max = 4;
-
- /*
- * resize obj->staged or allocate if it doesn't exist,
- * noop if already correct size
- */
- fobj = krealloc(obj->staged, offsetof(typeof(*fobj), shared[max]),
- GFP_KERNEL);
- if (!fobj)
- return -ENOMEM;
-
- obj->staged = fobj;
- fobj->shared_max = max;
- return 0;
+ struct reservation_shared_layer *p = iter->p;
+ int pos, h;
+
+ do {
+ p = p->parent;
+ if (!p) {
+ iter->fence = NULL;
+ return;
+ }
+
+ h = p->height / SHIFT;
+ pos = fns(p->bitmap, iter->pos[h] + 1) - 1;
+ } while (pos == -1);
+
+ h = p->height / SHIFT;
+ iter->pos[h] = pos;
+ p = p->slot[pos];
+
+ do {
+ pos = ffs(p->bitmap) - 1;
+
+ h = p->height / SHIFT;
+ iter->pos[h] = pos;
+
+ iter->p = p;
+ p = p->slot[pos];
+ } while (h);
+
+ iter->fence = (void *)p;
}
-EXPORT_SYMBOL(reservation_object_reserve_shared);
+EXPORT_SYMBOL(__reservation_shared_iter_next);
-static void
-reservation_object_add_shared_inplace(struct reservation_object *obj,
- struct reservation_object_list *fobj,
- struct dma_fence *fence)
+static void __shared_layer_free(struct reservation_shared *shared,
+ struct reservation_shared_layer *p)
{
- u32 i;
+ p->bitmap = shared->freed ? shared->freed->bitmap + 1 : 0;
+ p->parent = shared->freed;
+ shared->freed = p;
+}
- dma_fence_get(fence);
+static struct reservation_shared_layer *
+__shared_layer_alloc(struct reservation_shared *shared)
+{
+ struct reservation_shared_layer *p;
- preempt_disable();
- write_seqcount_begin(&obj->seq);
+ p = shared->freed;
+ shared->freed = p->parent;
- for (i = 0; i < fobj->shared_count; ++i) {
- struct dma_fence *old_fence;
+ return memset(p, 0, sizeof(*p));
+}
- old_fence = rcu_dereference_protected(fobj->shared[i],
- reservation_object_held(obj));
+static void shared_reset_layer(struct reservation_shared *shared,
+ struct reservation_shared_layer *p)
+{
+ if (p->height) { /* internal nodes, fill the freelist */
+ unsigned int i;
- if (old_fence->context == fence->context) {
- /* memory barrier is added by write_seqcount_begin */
- RCU_INIT_POINTER(fobj->shared[i], fence);
- write_seqcount_end(&obj->seq);
- preempt_enable();
+ for (i = 0; i < NSHARED; i++) {
+ if (!p->slot[i])
+ continue;
- dma_fence_put(old_fence);
- return;
+ shared_reset_layer(shared, p->slot[i]);
}
- }
- /*
- * memory barrier is added by write_seqcount_begin,
- * fobj->shared_count is protected by this lock too
- */
- RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);
- fobj->shared_count++;
-
- write_seqcount_end(&obj->seq);
- preempt_enable();
+ __shared_layer_free(shared, p);
+ } else /* external nodes -> RCU safe */
+ kfree_rcu(p, rcu);
}
-static void
-reservation_object_add_shared_replace(struct reservation_object *obj,
- struct reservation_object_list *old,
- struct reservation_object_list *fobj,
- struct dma_fence *fence)
+static void shared_reset(struct reservation_shared *shared)
{
- unsigned i;
- struct dma_fence *old_fence = NULL;
+ if (!shared->top)
+ return;
- dma_fence_get(fence);
+ shared_reset_layer(shared, shared->top);
+}
- if (!old) {
- RCU_INIT_POINTER(fobj->shared[0], fence);
- fobj->shared_count = 1;
- goto done;
- }
+void reservation_shared_destroy(struct reservation_shared *shared)
+{
+ shared_reset(shared);
- /*
- * no need to bump fence refcounts, rcu_read access
- * requires the use of kref_get_unless_zero, and the
- * references from the old struct are carried over to
- * the new.
- */
- fobj->shared_count = old->shared_count;
-
- for (i = 0; i < old->shared_count; ++i) {
- struct dma_fence *check;
-
- check = rcu_dereference_protected(old->shared[i],
- reservation_object_held(obj));
-
- if (!old_fence && check->context == fence->context) {
- old_fence = check;
- RCU_INIT_POINTER(fobj->shared[i], fence);
- } else
- RCU_INIT_POINTER(fobj->shared[i], check);
+ while (shared->freed) {
+ struct reservation_shared_layer *p = shared->freed;
+ shared->freed = p->parent;
+ kfree(p);
}
- if (!old_fence) {
- RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);
- fobj->shared_count++;
+}
+EXPORT_SYMBOL(reservation_shared_destroy);
+
+static void *
+shared_fence_replace(struct reservation_shared *shared, u64 id, void *item)
+{
+ struct reservation_shared_layer *p;
+ struct reservation_shared_layer *cur;
+ unsigned idx;
+
+ p = shared->hint;
+ if (p && (id & ~p->mask) == 0)
+ goto found_layer;
+
+ p = shared->top;
+ if (!p) {
+ shared->top = cur = __shared_layer_alloc(shared);
+ goto new_layer;
}
-done:
- preempt_disable();
- write_seqcount_begin(&obj->seq);
- /*
- * RCU_INIT_POINTER can be used here,
- * seqcount provides the necessary barriers
- */
- RCU_INIT_POINTER(obj->fence, fobj);
- write_seqcount_end(&obj->seq);
- preempt_enable();
+ do {
+ if (id & ~p->mask) {
+ /* insert a join above the current layer */
+ cur = __shared_layer_alloc(shared);
+ cur->height = ALIGN(fls(id & ~p->mask), SHIFT);
+ cur->mask = ~0ull >> (64 - cur->height);
+ cur->mask |= id;
- if (old)
- kfree_rcu(old, rcu);
+ if (p->parent)
+ p->parent->slot[0] = cur;
+ else
+ shared->top = cur;
+ cur->parent = p->parent;
+
+ idx = (p->mask >> (cur->height-SHIFT)) & MASK;
+ cur->slot[idx] = p;
+ cur->bitmap |= BIT(idx);
+ p->parent = cur;
+ } else if (!p->height) {
+ /* matching base layer */
+ shared->hint = p;
+ goto found_layer;
+ } else {
+ /* descend into the next layer */
+ idx = (id >> (p->height-SHIFT)) & MASK;
+ cur = p->slot[idx];
+ if (unlikely(cur == NULL)) {
+ cur = __shared_layer_alloc(shared);
+ p->slot[idx] = cur;
+ p->bitmap |= BIT(idx);
+ cur->parent = p;
+ goto new_layer;
+ }
+ }
- if (old_fence)
- dma_fence_put(old_fence);
+ p = cur;
+ } while (1);
+
+found_layer:
+ idx = id & MASK;
+ cur = p->slot[idx];
+ p->slot[idx] = item;
+ p->bitmap |= BIT(idx);
+ return (void *)cur;
+
+new_layer:
+ cur->mask = id | MASK;
+ cur->slot[id & MASK] = item;
+ cur->bitmap |= BIT(id & MASK);
+ shared->hint = cur;
+ return NULL;
}
/**
+ * reservation_object_reserve_shared - Reserve space to add a shared
+ * fence to a reservation_object.
+ * @obj: reservation object
+ *
+ * Should be called before reservation_object_add_shared_fence(). Must
+ * be called with obj->lock held.
+ *
+ * RETURNS
+ * Zero for success, or -errno
+ */
+int reservation_object_reserve_shared(struct reservation_object *obj)
+{
+ struct reservation_shared *shared = &obj->shared;
+ int count;
+
+ reservation_object_assert_held(obj);
+ might_sleep();
+
+ count = shared->freed ? shared->freed->bitmap : 0;
+ while (count++ < 2) {
+ struct reservation_shared_layer *p;
+
+ p = kmalloc(sizeof(*p), GFP_KERNEL);
+ if (unlikely(!p))
+ return -ENOMEM;
+
+ __shared_layer_free(shared, p);
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(reservation_object_reserve_shared);
+
+/**
* reservation_object_add_shared_fence - Add a fence to a shared slot
* @obj: the reservation object
* @fence: the shared fence to add
@@ -210,16 +264,21 @@ reservation_object_add_shared_replace(struct reservation_object *obj,
void reservation_object_add_shared_fence(struct reservation_object *obj,
struct dma_fence *fence)
{
- struct reservation_object_list *old, *fobj = obj->staged;
+ struct dma_fence *old_fence;
+
+ reservation_object_assert_held(obj);
- old = reservation_object_get_list(obj);
- obj->staged = NULL;
+ dma_fence_get(fence);
- if (!fobj) {
- BUG_ON(old->shared_count >= old->shared_max);
- reservation_object_add_shared_inplace(obj, old, fence);
- } else
- reservation_object_add_shared_replace(obj, old, fobj, fence);
+ preempt_disable();
+ write_seqcount_begin(&obj->seq);
+
+ old_fence = shared_fence_replace(&obj->shared, fence->context, fence);
+
+ write_seqcount_end(&obj->seq);
+ preempt_enable();
+
+ dma_fence_put(old_fence);
}
EXPORT_SYMBOL(reservation_object_add_shared_fence);
@@ -233,33 +292,27 @@ EXPORT_SYMBOL(reservation_object_add_shared_fence);
void reservation_object_add_excl_fence(struct reservation_object *obj,
struct dma_fence *fence)
{
- struct dma_fence *old_fence = reservation_object_get_excl(obj);
- struct reservation_object_list *old;
- u32 i = 0;
+ struct dma_fence *old_fence = obj->excl;
+ struct reservation_shared old_shared = obj->shared;
- old = reservation_object_get_list(obj);
- if (old)
- i = old->shared_count;
+ reservation_object_assert_held(obj);
- if (fence)
- dma_fence_get(fence);
+ dma_fence_get(fence);
preempt_disable();
write_seqcount_begin(&obj->seq);
+
/* write_seqcount_begin provides the necessary memory barrier */
- RCU_INIT_POINTER(obj->fence_excl, fence);
- if (old)
- old->shared_count = 0;
+ RCU_INIT_POINTER(obj->excl, fence);
+ reservation_shared_init(&obj->shared);
+
write_seqcount_end(&obj->seq);
preempt_enable();
- /* inplace update, no shared fences */
- while (i--)
- dma_fence_put(rcu_dereference_protected(old->shared[i],
- reservation_object_held(obj)));
+ shared_reset(&old_shared);
+ obj->shared.freed = old_shared.freed;
- if (old_fence)
- dma_fence_put(old_fence);
+ dma_fence_put(old_fence);
}
EXPORT_SYMBOL(reservation_object_add_excl_fence);
@@ -281,71 +334,76 @@ int reservation_object_get_fences_rcu(struct reservation_object *obj,
struct dma_fence ***pshared)
{
struct dma_fence **shared = NULL;
- struct dma_fence *fence_excl;
- unsigned int shared_count;
+ struct dma_fence *excl;
+ unsigned int count = 0;
+ unsigned int sz = 0;
rcu_read_lock();
for (;;) {
- struct reservation_object_list *fobj;
- unsigned int seq, i;
-
- shared_count = i = 0;
+ struct reservation_shared_iter iter;
+ unsigned int seq;
+restart:
seq = read_seqcount_begin(&obj->seq);
- fence_excl = rcu_dereference(obj->fence_excl);
- if (fence_excl && !dma_fence_get_rcu(fence_excl))
+ excl = rcu_dereference(obj->excl);
+ if (excl && !dma_fence_get_rcu(excl))
continue;
- fobj = rcu_dereference(obj->fence);
- if (fobj && fobj->shared_count) {
- struct dma_fence **nshared;
- size_t sz = sizeof(*shared) * fobj->shared_max;
+ reservation_object_for_each_shared(obj, iter) {
+ if (dma_fence_is_signaled(iter.fence))
+ continue;
+
+ if (!dma_fence_get_rcu(iter.fence))
+ break;
- nshared = krealloc(shared, sz,
- GFP_NOWAIT | __GFP_NOWARN);
- if (!nshared) {
- rcu_read_unlock();
- dma_fence_put(fence_excl);
+ if (count == sz) {
+ struct dma_fence **nshared;
- nshared = krealloc(shared, sz, GFP_KERNEL);
+ sz = sz ? 2*sz : 4;
+ nshared = krealloc(shared,
+ sz * sizeof(*shared),
+ GFP_NOWAIT | __GFP_NOWARN);
if (!nshared) {
+ rcu_read_unlock();
+
+ dma_fence_put(excl);
+ dma_fence_put(iter.fence);
+ while (count--)
+ dma_fence_put(shared[count]);
kfree(shared);
- return -ENOMEM;
+
+ shared = kmalloc(sz, GFP_TEMPORARY);
+ if (!nshared)
+ return -ENOMEM;
+
+ rcu_read_lock();
+ goto restart;
}
shared = nshared;
- rcu_read_lock();
- continue;
}
- shared = nshared;
- shared_count = fobj->shared_count;
-
- for (i = 0; i < shared_count; ++i) {
- shared[i] = rcu_dereference(fobj->shared[i]);
- if (!dma_fence_get_rcu(shared[i]))
- break;
- }
+ shared[count++] = iter.fence;
}
- if (i == shared_count && !read_seqcount_retry(&obj->seq, seq))
+ if (!read_seqcount_retry(&obj->seq, seq))
break;
- while (i--)
- dma_fence_put(shared[i]);
- dma_fence_put(fence_excl);
+ while (count--)
+ dma_fence_put(shared[count]);
+ dma_fence_put(excl);
}
rcu_read_unlock();
- if (!shared_count) {
+ if (!count) {
kfree(shared);
shared = NULL;
}
- *pshared_count = shared_count;
+ *pshared_count = count;
*pshared = shared;
- *pfence_excl = fence_excl;
+ *pfence_excl = excl;
return 0;
}
EXPORT_SYMBOL_GPL(reservation_object_get_fences_rcu);
@@ -364,99 +422,50 @@ EXPORT_SYMBOL_GPL(reservation_object_get_fences_rcu);
*/
long reservation_object_wait_timeout_rcu(struct reservation_object *obj,
bool wait_all, bool intr,
- unsigned long timeout)
+ long timeout)
{
struct dma_fence *fence;
- unsigned seq, shared_count, i = 0;
- long ret = timeout ? timeout : 1;
+ unsigned seq;
+ rcu_read_lock();
retry:
- fence = NULL;
- shared_count = 0;
seq = read_seqcount_begin(&obj->seq);
- rcu_read_lock();
if (wait_all) {
- struct reservation_object_list *fobj =
- rcu_dereference(obj->fence);
-
- if (fobj)
- shared_count = fobj->shared_count;
-
- for (i = 0; i < shared_count; ++i) {
- struct dma_fence *lfence = rcu_dereference(fobj->shared[i]);
-
- if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
- &lfence->flags))
- continue;
+ struct reservation_shared_iter iter;
- if (!dma_fence_get_rcu(lfence))
- goto unlock_retry;
-
- if (dma_fence_is_signaled(lfence)) {
- dma_fence_put(lfence);
- continue;
- }
-
- fence = lfence;
- break;
+ reservation_object_for_each_shared(obj, iter) {
+ fence = iter.fence;
+ if (!dma_fence_is_signaled(fence))
+ goto wait;
}
}
- if (!shared_count) {
- struct dma_fence *fence_excl = rcu_dereference(obj->fence_excl);
+ fence = rcu_dereference(obj->excl);
+ if (fence && !dma_fence_is_signaled(fence))
+ goto wait;
- if (fence_excl &&
- !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
- &fence_excl->flags)) {
- if (!dma_fence_get_rcu(fence_excl))
- goto unlock_retry;
+ if (read_seqcount_retry(&obj->seq, seq))
+ goto retry;
- if (dma_fence_is_signaled(fence_excl))
- dma_fence_put(fence_excl);
- else
- fence = fence_excl;
- }
- }
+ rcu_read_unlock();
+ return timeout;
+wait:
+ if (!dma_fence_get_rcu(fence))
+ goto retry;
rcu_read_unlock();
- if (fence) {
- if (read_seqcount_retry(&obj->seq, seq)) {
- dma_fence_put(fence);
- goto retry;
- }
- ret = dma_fence_wait_timeout(fence, intr, ret);
- dma_fence_put(fence);
- if (ret > 0 && wait_all && (i + 1 < shared_count))
- goto retry;
- }
- return ret;
+ timeout = dma_fence_wait_timeout(fence, intr, timeout);
+ dma_fence_put(fence);
+ if (timeout < 0)
+ return timeout;
-unlock_retry:
- rcu_read_unlock();
+ rcu_read_lock();
goto retry;
}
EXPORT_SYMBOL_GPL(reservation_object_wait_timeout_rcu);
-
-static inline int
-reservation_object_test_signaled_single(struct dma_fence *passed_fence)
-{
- struct dma_fence *fence, *lfence = passed_fence;
- int ret = 1;
-
- if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &lfence->flags)) {
- fence = dma_fence_get_rcu(lfence);
- if (!fence)
- return -1;
-
- ret = !!dma_fence_is_signaled(fence);
- dma_fence_put(fence);
- }
- return ret;
-}
-
/**
* reservation_object_test_signaled_rcu - Test if a reservation object's
* fences have been signaled.
@@ -470,52 +479,33 @@ reservation_object_test_signaled_single(struct dma_fence *passed_fence)
bool reservation_object_test_signaled_rcu(struct reservation_object *obj,
bool test_all)
{
- unsigned seq, shared_count;
- int ret;
+ struct dma_fence *fence;
+ bool ret = false;
+ unsigned seq;
rcu_read_lock();
retry:
- ret = true;
- shared_count = 0;
seq = read_seqcount_begin(&obj->seq);
if (test_all) {
- unsigned i;
-
- struct reservation_object_list *fobj =
- rcu_dereference(obj->fence);
+ struct reservation_shared_iter iter;
- if (fobj)
- shared_count = fobj->shared_count;
-
- for (i = 0; i < shared_count; ++i) {
- struct dma_fence *fence = rcu_dereference(fobj->shared[i]);
-
- ret = reservation_object_test_signaled_single(fence);
- if (ret < 0)
- goto retry;
- else if (!ret)
- break;
+ reservation_object_for_each_shared(obj, iter) {
+ fence = iter.fence;
+ if (!dma_fence_is_signaled(fence))
+ goto busy;
}
-
- if (read_seqcount_retry(&obj->seq, seq))
- goto retry;
}
- if (!shared_count) {
- struct dma_fence *fence_excl = rcu_dereference(obj->fence_excl);
+ fence = rcu_dereference(obj->excl);
+ if (fence && !dma_fence_is_signaled(fence))
+ goto busy;
- if (fence_excl) {
- ret = reservation_object_test_signaled_single(
- fence_excl);
- if (ret < 0)
- goto retry;
-
- if (read_seqcount_retry(&obj->seq, seq))
- goto retry;
- }
- }
+ if (read_seqcount_retry(&obj->seq, seq))
+ goto retry;
+ ret = true;
+busy:
rcu_read_unlock();
return ret;
}
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index d2211825e5c8..7fcdf9dc86f8 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -1020,9 +1020,9 @@ int etnaviv_gpu_fence_sync_obj(struct etnaviv_gem_object *etnaviv_obj,
unsigned int context, bool exclusive)
{
struct reservation_object *robj = etnaviv_obj->resv;
- struct reservation_object_list *fobj;
+ struct reservation_shared_iter iter;
struct dma_fence *fence;
- int i, ret;
+ int ret;
if (!exclusive) {
ret = reservation_object_reserve_shared(robj);
@@ -1034,8 +1034,7 @@ int etnaviv_gpu_fence_sync_obj(struct etnaviv_gem_object *etnaviv_obj,
* If we have any shared fences, then the exclusive fence
* should be ignored as it will already have been signalled.
*/
- fobj = reservation_object_get_list(robj);
- if (!fobj || fobj->shared_count == 0) {
+ if (!reservation_object_has_shared(robj)) {
/* Wait on any existing exclusive fence which isn't our own */
fence = reservation_object_get_excl(robj);
if (fence && fence->context != context) {
@@ -1045,12 +1044,11 @@ int etnaviv_gpu_fence_sync_obj(struct etnaviv_gem_object *etnaviv_obj,
}
}
- if (!exclusive || !fobj)
+ if (!exclusive)
return 0;
- for (i = 0; i < fobj->shared_count; i++) {
- fence = rcu_dereference_protected(fobj->shared[i],
- reservation_object_held(robj));
+ reservation_object_for_each_shared(resv, iter) {
+ fence = iter.fence;
if (fence->context != context) {
ret = dma_fence_wait(fence, true);
if (ret)
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 9fcee91be774..cc178b3a2fd2 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -3776,7 +3776,7 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
{
struct drm_i915_gem_busy *args = data;
struct drm_i915_gem_object *obj;
- struct reservation_object_list *list;
+ struct reservation_shared_iter iter;
unsigned int seq;
int err;
@@ -3806,20 +3806,11 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
seq = raw_read_seqcount(&obj->resv->seq);
/* Translate the exclusive fence to the READ *and* WRITE engine */
- args->busy = busy_check_writer(rcu_dereference(obj->resv->fence_excl));
+ args->busy = busy_check_writer(rcu_dereference(obj->resv->excl));
/* Translate shared fences to READ set of engines */
- list = rcu_dereference(obj->resv->fence);
- if (list) {
- unsigned int shared_count = list->shared_count, i;
-
- for (i = 0; i < shared_count; ++i) {
- struct dma_fence *fence =
- rcu_dereference(list->shared[i]);
-
- args->busy |= busy_check_reader(fence);
- }
- }
+ reservation_object_for_each_shared(obj->resv, iter)
+ args->busy |= busy_check_reader(iter.fence);
if (args->busy && read_seqcount_retry(&obj->resv->seq, seq))
goto retry;
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index 57db7dbbb618..5cdb94f0573e 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -520,9 +520,9 @@ int msm_gem_sync_object(struct drm_gem_object *obj,
struct msm_fence_context *fctx, bool exclusive)
{
struct msm_gem_object *msm_obj = to_msm_bo(obj);
- struct reservation_object_list *fobj;
+ struct reservation_shared_iter iter;
struct dma_fence *fence;
- int i, ret;
+ int ret;
if (!exclusive) {
/* NOTE: _reserve_shared() must happen before _add_shared_fence(),
@@ -535,8 +535,7 @@ int msm_gem_sync_object(struct drm_gem_object *obj,
return ret;
}
- fobj = reservation_object_get_list(msm_obj->resv);
- if (!fobj || (fobj->shared_count == 0)) {
+ if (!reservation_object_has_shared(msm_obj->resv)) {
fence = reservation_object_get_excl(msm_obj->resv);
/* don't need to wait on our own fences, since ring is fifo */
if (fence && (fence->context != fctx->context)) {
@@ -546,12 +545,11 @@ int msm_gem_sync_object(struct drm_gem_object *obj,
}
}
- if (!exclusive || !fobj)
+ if (!exclusive)
return 0;
- for (i = 0; i < fobj->shared_count; i++) {
- fence = rcu_dereference_protected(fobj->shared[i],
- reservation_object_held(msm_obj->resv));
+ reservation_object_for_each_shared(msm_obj->resv, iter) {
+ fence = iter.fence;
if (fence->context != fctx->context) {
ret = dma_fence_wait(fence, true);
if (ret)
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c
index f2f348f0160c..c5b0967f5f6e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.c
@@ -393,21 +393,19 @@ nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool e
struct nouveau_fence_chan *fctx = chan->fence;
struct dma_fence *fence;
struct reservation_object *resv = nvbo->bo.resv;
- struct reservation_object_list *fobj;
+ struct reservation_shared_iter iter;
struct nouveau_fence *f;
- int ret = 0, i;
+ int ret = 0;
if (!exclusive) {
ret = reservation_object_reserve_shared(resv);
-
if (ret)
return ret;
}
- fobj = reservation_object_get_list(resv);
fence = reservation_object_get_excl(resv);
- if (fence && (!exclusive || !fobj || !fobj->shared_count)) {
+ if (fence && (!exclusive || !reservation_object_has_shared(resv))) {
struct nouveau_channel *prev = NULL;
bool must_wait = true;
@@ -426,15 +424,14 @@ nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool e
return ret;
}
- if (!exclusive || !fobj)
+ if (!exclusive)
return ret;
- for (i = 0; i < fobj->shared_count && !ret; ++i) {
+ reservation_object_for_each_shared(resv, iter) {
struct nouveau_channel *prev = NULL;
bool must_wait = true;
- fence = rcu_dereference_protected(fobj->shared[i],
- reservation_object_held(resv));
+ fence = iter.fence;
f = nouveau_local_fence(fence, chan->drm);
if (f) {
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index 201b52b750dd..dead9630e472 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -118,19 +118,22 @@ nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nvkm_vma *vma)
{
const bool mapped = nvbo->bo.mem.mem_type != TTM_PL_SYSTEM;
struct reservation_object *resv = nvbo->bo.resv;
- struct reservation_object_list *fobj;
+ struct reservation_shared_iter iter;
struct dma_fence *fence = NULL;
- fobj = reservation_object_get_list(resv);
-
list_del(&vma->head);
- if (fobj && fobj->shared_count > 1)
- ttm_bo_wait(&nvbo->bo, false, false);
- else if (fobj && fobj->shared_count == 1)
- fence = rcu_dereference_protected(fobj->shared[0],
- reservation_object_held(resv));
- else
+ reservation_object_for_each_shared(resv, iter) {
+ if (fence) {
+ ttm_bo_wait(&nvbo->bo, false, false);
+ fence = NULL;
+ break;
+ }
+
+ fence = iter.fence;
+ }
+
+ if (!fence)
fence = reservation_object_get_excl(nvbo->bo.resv);
if (fence && mapped) {
diff --git a/drivers/gpu/drm/radeon/radeon_sync.c b/drivers/gpu/drm/radeon/radeon_sync.c
index be5d7a38d3aa..72b56ed09df4 100644
--- a/drivers/gpu/drm/radeon/radeon_sync.c
+++ b/drivers/gpu/drm/radeon/radeon_sync.c
@@ -91,10 +91,9 @@ int radeon_sync_resv(struct radeon_device *rdev,
struct reservation_object *resv,
bool shared)
{
- struct reservation_object_list *flist;
+ struct reservation_shared_iter iter;
struct dma_fence *f;
struct radeon_fence *fence;
- unsigned i;
int r = 0;
/* always sync to the exclusive fence */
@@ -105,14 +104,11 @@ int radeon_sync_resv(struct radeon_device *rdev,
else if (f)
r = dma_fence_wait(f, true);
- flist = reservation_object_get_list(resv);
- if (shared || !flist || r)
+ if (shared || !reservation_object_has_shared(resv) || r)
return r;
- for (i = 0; i < flist->shared_count; ++i) {
- f = rcu_dereference_protected(flist->shared[i],
- reservation_object_held(resv));
- fence = to_radeon_fence(f);
+ resevation_object_for_each_shared(resv, iter) {
+ fence = to_radeon_fence(iter.fence);
if (fence && fence->rdev == rdev)
radeon_sync_fence(sync, fence);
else
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index d5063618efa7..5844c4aa6e1c 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -425,19 +425,15 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
{
- struct reservation_object_list *fobj;
+ struct reservation_shared_iter iter;
struct dma_fence *fence;
- int i;
- fobj = reservation_object_get_list(bo->resv);
fence = reservation_object_get_excl(bo->resv);
if (fence && !fence->ops->signaled)
dma_fence_enable_sw_signaling(fence);
- for (i = 0; fobj && i < fobj->shared_count; ++i) {
- fence = rcu_dereference_protected(fobj->shared[i],
- reservation_object_held(bo->resv));
-
+ reservation_object_for_each_shared(bo->resv, iter) {
+ fence = iter.fence;
if (!fence->ops->signaled)
dma_fence_enable_sw_signaling(fence);
}
diff --git a/include/linux/reservation.h b/include/linux/reservation.h
index 2e313cca08f0..a6e2f60157d1 100644
--- a/include/linux/reservation.h
+++ b/include/linux/reservation.h
@@ -43,40 +43,54 @@
#include <linux/dma-fence.h>
#include <linux/slab.h>
#include <linux/seqlock.h>
+#include <linux/radix-tree.h>
#include <linux/rcupdate.h>
extern struct ww_class reservation_ww_class;
extern struct lock_class_key reservation_seqcount_class;
extern const char reservation_seqcount_string[];
-/**
- * struct reservation_object_list - a list of shared fences
- * @rcu: for internal use
- * @shared_count: table of shared fences
- * @shared_max: for growing shared fence table
- * @shared: shared fence table
- */
-struct reservation_object_list {
- struct rcu_head rcu;
- u32 shared_count, shared_max;
- struct dma_fence __rcu *shared[];
+struct reservation_shared_layer;
+
+#define NSHARED 16
+
+struct reservation_shared_layer {
+ u64 mask;
+ unsigned int height;
+ unsigned int bitmap;
+ void *slot[NSHARED];
+ union {
+ struct reservation_shared_layer *parent;
+ struct rcu_head rcu;
+ };
+};
+
+struct reservation_shared {
+ struct reservation_shared_layer *hint;
+ struct reservation_shared_layer *top;
+ struct reservation_shared_layer *freed;
};
+static inline void reservation_shared_init(struct reservation_shared *shared)
+{
+ memset(shared, 0, sizeof(*shared));
+}
+
+void reservation_shared_destroy(struct reservation_shared *shared);
+
/**
* struct reservation_object - a reservation object manages fences for a buffer
* @lock: update side lock
* @seq: sequence count for managing RCU read-side synchronization
- * @fence_excl: the exclusive fence, if there is one currently
- * @fence: list of current shared fences
- * @staged: staged copy of shared fences for RCU updates
+ * @excl: the exclusive fence, if there is one currently
+ * @shared: list of current shared fences
*/
struct reservation_object {
struct ww_mutex lock;
seqcount_t seq;
- struct dma_fence __rcu *fence_excl;
- struct reservation_object_list __rcu *fence;
- struct reservation_object_list *staged;
+ struct dma_fence __rcu *excl;
+ struct reservation_shared shared;
};
#define reservation_object_held(obj) lockdep_is_held(&(obj)->lock.base)
@@ -93,9 +107,8 @@ reservation_object_init(struct reservation_object *obj)
ww_mutex_init(&obj->lock, &reservation_ww_class);
__seqcount_init(&obj->seq, reservation_seqcount_string, &reservation_seqcount_class);
- RCU_INIT_POINTER(obj->fence, NULL);
- RCU_INIT_POINTER(obj->fence_excl, NULL);
- obj->staged = NULL;
+ RCU_INIT_POINTER(obj->excl, NULL);
+ reservation_shared_init(&obj->shared);
}
/**
@@ -105,46 +118,18 @@ reservation_object_init(struct reservation_object *obj)
static inline void
reservation_object_fini(struct reservation_object *obj)
{
- int i;
- struct reservation_object_list *fobj;
- struct dma_fence *excl;
-
/*
* This object should be dead and all references must have
* been released to it, so no need to be protected with rcu.
*/
- excl = rcu_dereference_protected(obj->fence_excl, 1);
- if (excl)
- dma_fence_put(excl);
+ dma_fence_put(rcu_dereference_protected(obj->excl, 1));
- fobj = rcu_dereference_protected(obj->fence, 1);
- if (fobj) {
- for (i = 0; i < fobj->shared_count; ++i)
- dma_fence_put(rcu_dereference_protected(fobj->shared[i], 1));
-
- kfree(fobj);
- }
- kfree(obj->staged);
+ reservation_shared_destroy(&obj->shared);
ww_mutex_destroy(&obj->lock);
}
/**
- * reservation_object_get_list - get the reservation object's
- * shared fence list, with update-side lock held
- * @obj: the reservation object
- *
- * Returns the shared fence list. Does NOT take references to
- * the fence. The obj->lock must be held.
- */
-static inline struct reservation_object_list *
-reservation_object_get_list(struct reservation_object *obj)
-{
- return rcu_dereference_protected(obj->fence,
- reservation_object_held(obj));
-}
-
-/**
* reservation_object_get_excl - get the reservation object's
* exclusive fence, with update-side lock held
* @obj: the reservation object
@@ -158,7 +143,7 @@ reservation_object_get_list(struct reservation_object *obj)
static inline struct dma_fence *
reservation_object_get_excl(struct reservation_object *obj)
{
- return rcu_dereference_protected(obj->fence_excl,
+ return rcu_dereference_protected(obj->excl,
reservation_object_held(obj));
}
@@ -181,7 +166,7 @@ reservation_object_get_excl_rcu(struct reservation_object *obj)
retry:
seq = read_seqcount_begin(&obj->seq);
rcu_read_lock();
- fence = rcu_dereference(obj->fence_excl);
+ fence = rcu_dereference(obj->excl);
if (read_seqcount_retry(&obj->seq, seq)) {
rcu_read_unlock();
goto retry;
@@ -191,6 +176,12 @@ reservation_object_get_excl_rcu(struct reservation_object *obj)
return fence;
}
+static inline bool
+reservation_object_has_shared(struct reservation_object *obj)
+{
+ return READ_ONCE(obj->shared.top);
+}
+
int reservation_object_reserve_shared(struct reservation_object *obj);
void reservation_object_add_shared_fence(struct reservation_object *obj,
struct dma_fence *fence);
@@ -205,9 +196,59 @@ int reservation_object_get_fences_rcu(struct reservation_object *obj,
long reservation_object_wait_timeout_rcu(struct reservation_object *obj,
bool wait_all, bool intr,
- unsigned long timeout);
+ long timeout);
bool reservation_object_test_signaled_rcu(struct reservation_object *obj,
bool test_all);
+struct reservation_shared_iter {
+ struct dma_fence *fence;
+ struct reservation_shared_layer *p;
+ u8 pos[16];
+};
+
+static inline void reservation_shared_iter_init(struct reservation_object *obj,
+ struct reservation_shared_iter *iter)
+{
+ struct reservation_shared_layer *p;
+ int h;
+
+ p = obj->shared.top;
+ if (!p) {
+ iter->fence = NULL;
+ return;
+ }
+
+ do {
+ int pos = ffs(p->bitmap) - 1;
+
+ h = p->height / ilog2(NSHARED);
+ iter->pos[h] = pos;
+
+ iter->p = p;
+ p = p->slot[pos];
+ } while (h);
+
+ iter->fence = (void *)p;
+}
+
+#define fns(x, bit) ffs((x) & (~(typeof(x))0 << (bit)))
+
+void __reservation_shared_iter_next(struct reservation_shared_iter *iter);
+static inline void reservation_shared_iter_next(struct reservation_shared_iter *iter)
+{
+ int pos;
+
+ pos = fns(iter->p->bitmap, iter->pos[0] + 1) - 1;
+ if (unlikely(pos < 0)) {
+ __reservation_shared_iter_next(iter);
+ } else {
+ iter->pos[0] = pos;
+ iter->fence = iter->p->slot[pos];
+ }
+}
+
+#define reservation_object_for_each_shared(obj, __i) \
+ for (reservation_shared_iter_init(obj, &(__i)); __i.fence; reservation_shared_iter_next(&(__i)))
+
#endif /* _LINUX_RESERVATION_H */
--
2.10.2
More information about the Intel-gfx-trybot
mailing list