[PATCH 5/5] drm/i915: Squash repeated awaits on the same fence
Chris Wilson
chris at chris-wilson.co.uk
Sun Apr 9 13:52:12 UTC 2017
Track the latest fence waited upon on each context, and only add a new
asynchronous wait if the new fence is more recent than the recorded
fence for that context. This requires us to filter out unordered
timelines, which are noted by DMA_FENCE_NO_CONTEXT. However, in the
absence of a universal identifier, we have to use our own
i915->mm.unordered_timeline token.
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
Cc: Tvrtko Ursulin <tvrtko.ursulin at intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen at linux.intel.com>
---
drivers/gpu/drm/i915/i915_gem_request.c | 72 +++++----
drivers/gpu/drm/i915/i915_gem_timeline.c | 251 +++++++++++++++++++++++++++++++
drivers/gpu/drm/i915/i915_gem_timeline.h | 23 +++
lib/radix-tree.c | 1 +
4 files changed, 320 insertions(+), 27 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c
index 31874a38752e..6e68387aa097 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.c
+++ b/drivers/gpu/drm/i915/i915_gem_request.c
@@ -714,9 +714,7 @@ int
i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req,
struct dma_fence *fence)
{
- struct dma_fence_array *array;
int ret;
- int i;
if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
return 0;
@@ -728,39 +726,59 @@ i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req,
if (fence->context == req->fence.context)
return 0;
- if (dma_fence_is_i915(fence))
- return i915_gem_request_await_request(req, to_request(fence));
+ /* Squash repeated waits to the same timelines, picking the latest */
+ if (fence->context != req->i915->mm.unordered_timeline) {
+ if (intel_timeline_sync_get(req->timeline,
+ fence->context,
+ fence->seqno))
+ return 0;
- if (!dma_fence_is_array(fence)) {
+ ret = intel_timeline_sync_reserve(req->timeline);
+ if (unlikely(ret))
+ return ret;
+ }
+
+ if (dma_fence_is_i915(fence)) {
+ ret = i915_gem_request_await_request(req, to_request(fence));
+ if (ret < 0)
+ return ret;
+ } else if (!dma_fence_is_array(fence)) {
ret = i915_sw_fence_await_dma_fence(&req->submit,
fence, I915_FENCE_TIMEOUT,
GFP_KERNEL);
- return ret < 0 ? ret : 0;
- }
-
- /* Note that if the fence-array was created in signal-on-any mode,
- * we should *not* decompose it into its individual fences. However,
- * we don't currently store which mode the fence-array is operating
- * in. Fortunately, the only user of signal-on-any is private to
- * amdgpu and we should not see any incoming fence-array from
- * sync-file being in signal-on-any mode.
- */
-
- array = to_dma_fence_array(fence);
- for (i = 0; i < array->num_fences; i++) {
- struct dma_fence *child = array->fences[i];
-
- if (dma_fence_is_i915(child))
- ret = i915_gem_request_await_request(req,
- to_request(child));
- else
- ret = i915_sw_fence_await_dma_fence(&req->submit,
- child, I915_FENCE_TIMEOUT,
- GFP_KERNEL);
if (ret < 0)
return ret;
+ } else {
+ struct dma_fence_array *array = to_dma_fence_array(fence);
+ int i;
+
+ /* Note that if the fence-array was created in signal-on-any mode,
+ * we should *not* decompose it into its individual fences. However,
+ * we don't currently store which mode the fence-array is operating
+ * in. Fortunately, the only user of signal-on-any is private to
+ * amdgpu and we should not see any incoming fence-array from
+ * sync-file being in signal-on-any mode.
+ */
+
+ for (i = 0; i < array->num_fences; i++) {
+ struct dma_fence *child = array->fences[i];
+
+ if (dma_fence_is_i915(child))
+ ret = i915_gem_request_await_request(req,
+ to_request(child));
+ else
+ ret = i915_sw_fence_await_dma_fence(&req->submit,
+ child, I915_FENCE_TIMEOUT,
+ GFP_KERNEL);
+ if (ret < 0)
+ return ret;
+ }
}
+ if (fence->context != req->i915->mm.unordered_timeline)
+ intel_timeline_sync_set(req->timeline,
+ fence->context, fence->seqno);
+
return 0;
}
diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.c b/drivers/gpu/drm/i915/i915_gem_timeline.c
index b596ca7ee058..6f6103b3f240 100644
--- a/drivers/gpu/drm/i915/i915_gem_timeline.c
+++ b/drivers/gpu/drm/i915/i915_gem_timeline.c
@@ -24,6 +24,255 @@
#include "i915_drv.h"
+#if 0
+#define assert(x) BUG_ON(!(x))
+#define dbg(x) pr_err x
+#else
+#define assert(x) do { } while (0)
+#define dbg(x) do { } while (0)
+#endif
+
+#define SHIFT ilog2(NSEQMAP)
+#define MASK (NSEQMAP - 1)
+
+static void seqmap_free_layers(struct seqmap_layer *p)
+{
+ unsigned int i;
+
+ if (p->height) {
+ for (; (i = ffs(p->bitmap)); p->bitmap &= ~0 << i)
+ seqmap_free_layers(p->slot[i - 1]);
+ }
+
+ kfree(p);
+}
+
+static void seqmap_free(struct seqmap *seqmap)
+{
+ if (seqmap->top)
+ seqmap_free_layers(seqmap->top);
+
+ while (seqmap->freed) {
+ struct seqmap_layer *p;
+
+ p = ptr_mask_bits(seqmap->freed, 2);
+ seqmap->freed = p->parent;
+ kfree(p);
+ }
+}
+
+__malloc static struct seqmap_layer *
+seqmap_alloc_layer(struct seqmap *shared)
+{
+ struct seqmap_layer *p;
+
+ p = ptr_mask_bits(shared->freed, 2);
+ shared->freed = p->parent;
+
+ return p;
+}
+
+static int layer_idx(const struct seqmap_layer *p, u64 id)
+{
+ return (id >> p->height) & MASK;
+}
+
+bool intel_timeline_sync_get(struct intel_timeline *tl, u64 id, u32 seqno)
+{
+ struct seqmap *shared = &tl->sync;
+ struct seqmap_layer *p, *cur;
+ unsigned int idx;
+
+ dbg(("%s(id=%llx)\n", __func__, id));
+
+ p = shared->top;
+ if (!p)
+ return false;
+
+ /* First see if this fence is in the same layer as the previous fence */
+ p = shared->hint;
+ if ((id >> SHIFT) == p->prefix)
+ goto found;
+
+ do {
+ dbg(("id=%llx, p->(prefix=%llx, height=%d), delta=%llx, idx=%x\n",
+ id, p->prefix, p->height,
+ id >> p->height >> SHIFT,
+ layer_idx(p, id)));
+
+ if ((id >> p->height >> SHIFT) != p->prefix) {
+ return false;
+ } else if (!p->height) {
+ /* matching base layer */
+ shared->hint = p;
+ break;
+ } else {
+ /* descend into the next layer */
+ cur = p->slot[layer_idx(p, id)];
+ if (unlikely(!cur))
+ return false;
+ }
+
+ p = cur;
+ } while (1);
+
+found:
+ idx = id & MASK;
+ if (!(p->bitmap & BIT(idx)))
+ return false;
+
+ idx = (u32)(uintptr_t)p->slot[idx];
+ return i915_seqno_passed(idx, seqno);
+}
+
+void intel_timeline_sync_set(struct intel_timeline *tl, u64 id, u32 seqno)
+{
+ struct seqmap *shared = &tl->sync;
+ struct seqmap_layer *p, *cur;
+ unsigned int idx;
+
+ dbg(("%s(id=%llx)\n", __func__, id));
+
+ /* First see if this fence is in the same layer as the previous fence */
+ p = shared->hint;
+ if (p && (id >> SHIFT) == p->prefix) {
+ assert(p->height == 0);
+ goto found_layer;
+ }
+
+ p = shared->top;
+ if (!p) {
+ cur = seqmap_alloc_layer(shared);
+ cur->parent = NULL;
+ shared->top = cur;
+ goto new_layer;
+ }
+
+ /* No shortcut, we have to descend the tree to find the right layer
+ * containing this fence.
+ *
+ * Each layer in the tree holds 16 (NSHARED) pointers, either fences
+ * or lower layers. Leaf nodes (height = 0) contain the fences, all
+ * other nodes (height > 0) are internal layers that point to a lower
+ * node. Each internal layer has at least 2 descendents.
+ *
+ * Starting at the top, we check whether the current prefix matches. If
+ * it doesn't, we have gone passed our layer and need to insert a join
+ * into the tree, and a new leaf node as a descendent as well as the
+ * original layer.
+ *
+ * The matching prefix means we are still following the right branch
+ * of the tree. If it has height 0, we have found our leaf and just
+ * need to replace the fence slot with ourselves. If the height is
+ * not zero, our slot contains the next layer in the tree (unless
+ * it is empty, in which case we can add ourselves as a new leaf).
+ * As descend the tree the prefix grows (and height decreases).
+ */
+ do {
+ dbg(("id=%llx, p->(prefix=%llx, height=%d), delta=%llx, idx=%x\n",
+ id, p->prefix, p->height,
+ id >> p->height >> SHIFT,
+ layer_idx(p, id)));
+
+ if ((id >> p->height >> SHIFT) != p->prefix) {
+ /* insert a join above the current layer */
+ cur = seqmap_alloc_layer(shared);
+ cur->height = ALIGN(fls64((id >> p->height >> SHIFT) ^ p->prefix),
+ SHIFT) + p->height;
+ cur->prefix = id >> cur->height >> SHIFT;
+
+ dbg(("id=%llx, join prefix=%llu, height=%d\n",
+ id, cur->prefix, cur->height));
+
+ assert((id >> cur->height >> SHIFT) == cur->prefix);
+ assert(cur->height > p->height);
+
+ if (p->parent) {
+ assert(p->parent->slot[layer_idx(p->parent, id)] == p);
+ assert(cur->height < p->parent->height);
+ p->parent->slot[layer_idx(p->parent, id)] = cur;
+ } else {
+ shared->top = cur;
+ }
+ cur->parent = p->parent;
+
+ idx = p->prefix >> (cur->height - p->height - SHIFT) & MASK;
+ cur->slot[idx] = p;
+ cur->bitmap |= BIT(idx);
+ p->parent = cur;
+ } else if (!p->height) {
+ /* matching base layer */
+ shared->hint = p;
+ goto found_layer;
+ } else {
+ /* descend into the next layer */
+ idx = layer_idx(p, id);
+ cur = p->slot[idx];
+ if (unlikely(!cur)) {
+ cur = seqmap_alloc_layer(shared);
+ p->slot[idx] = cur;
+ p->bitmap |= BIT(idx);
+ cur->parent = p;
+ goto new_layer;
+ }
+ }
+
+ p = cur;
+ } while (1);
+
+found_layer:
+ assert(p->height == 0);
+ idx = id & MASK;
+ cur = p->slot[idx];
+ p->slot[idx] = (void *)(uintptr_t)seqno;
+ p->bitmap |= BIT(idx);
+ dbg(("id=%llx, found existing layer prefix=%llx, idx=%x [bitmap=%x]\n",
+ id, p->prefix, idx, p->bitmap));
+ return;
+
+new_layer:
+ assert(cur->height == 0);
+ assert(cur->bitmap == 0);
+ cur->prefix = id >> SHIFT;
+ cur->slot[id & MASK] = (void *)(uintptr_t)seqno;
+ cur->bitmap = BIT(id & MASK);
+ shared->hint = cur;
+ dbg(("id=%llx, new layer prefix=%llx, idx=%x [bitmap=%x]\n",
+ id, cur->prefix, (int)(id & MASK), cur->bitmap));
+ return;
+}
+
+int intel_timeline_sync_reserve(struct intel_timeline *tl)
+{
+ struct seqmap *shared = &tl->sync;
+ int count;
+
+ might_sleep();
+
+ /* To guarantee being able to replace a fence in the radixtree,
+ * we need at most 2 layers: one to create a join in the tree,
+ * and one to contain the fence. Typically we expect to reuse
+ * a layer and so avoid any insertions.
+ *
+ * We use the low bits of the freed list to track its length
+ * since we only need a couple of bits.
+ */
+ count = ptr_unmask_bits(shared->freed, 2);
+ while (count++ < 2) {
+ struct seqmap_layer *p;
+
+ p = kzalloc(sizeof(*p), GFP_KERNEL);
+ if (unlikely(!p))
+ return -ENOMEM;
+
+ p->parent = shared->freed;
+ shared->freed = ptr_pack_bits(p, count, 2);
+ }
+
+ return 0;
+}
+
+
static int __i915_gem_timeline_init(struct drm_i915_private *i915,
struct i915_gem_timeline *timeline,
const char *name,
@@ -90,6 +339,8 @@ void i915_gem_timeline_fini(struct i915_gem_timeline *timeline)
for (i = 0; i < ARRAY_SIZE(timeline->engine); i++) {
struct intel_timeline *tl = &timeline->engine[i];
+ seqmap_free(&tl->sync);
+
GEM_BUG_ON(!list_empty(&tl->requests));
}
diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.h b/drivers/gpu/drm/i915/i915_gem_timeline.h
index 6c53e14cab2a..5d0e37f212ac 100644
--- a/drivers/gpu/drm/i915/i915_gem_timeline.h
+++ b/drivers/gpu/drm/i915/i915_gem_timeline.h
@@ -26,11 +26,28 @@
#define I915_GEM_TIMELINE_H
#include <linux/list.h>
+#include <linux/radix-tree.h>
#include "i915_gem_request.h"
struct i915_gem_timeline;
+#define NSEQMAP 16
+
+struct seqmap_layer {
+ u64 prefix;
+ unsigned int height;
+ unsigned int bitmap;
+ struct seqmap_layer *parent;
+ void *slot[NSEQMAP];
+};
+
+struct seqmap {
+ struct seqmap_layer *hint;
+ struct seqmap_layer *top;
+ struct seqmap_layer *freed;
+};
+
struct intel_timeline {
u64 fence_context;
u32 seqno;
@@ -55,6 +72,8 @@ struct intel_timeline {
* struct_mutex.
*/
struct i915_gem_active last_request;
+
+ struct seqmap sync;
u32 sync_seqno[I915_NUM_ENGINES];
struct i915_gem_timeline *common;
@@ -75,4 +94,8 @@ int i915_gem_timeline_init(struct drm_i915_private *i915,
int i915_gem_timeline_init__global(struct drm_i915_private *i915);
void i915_gem_timeline_fini(struct i915_gem_timeline *tl);
+bool intel_timeline_sync_get(struct intel_timeline *tl, u64 id, u32 seqno);
+int intel_timeline_sync_reserve(struct intel_timeline *tl);
+void intel_timeline_sync_set(struct intel_timeline *tl, u64 id, u32 seqno);
+
#endif
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index 691a9ad48497..84cccf7138c4 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -2022,6 +2022,7 @@ void radix_tree_iter_delete(struct radix_tree_root *root,
if (__radix_tree_delete(root, iter->node, slot))
iter->index = iter->next_index;
}
+EXPORT_SYMBOL(radix_tree_iter_delete);
/**
* radix_tree_delete_item - delete an item from a radix tree
--
2.11.0
More information about the Intel-gfx-trybot
mailing list