[PATCH 8/8] eb-unbind
Chris Wilson
chris at chris-wilson.co.uk
Fri Mar 13 00:47:47 UTC 2020
---
drivers/gpu/drm/i915/Kconfig.debug | 14 +
.../gpu/drm/i915/gem/i915_gem_execbuffer.c | 643 ++++++++++++++++--
drivers/gpu/drm/i915/gt/intel_ppgtt.c | 3 +-
drivers/gpu/drm/i915/i915_vma.c | 28 +-
drivers/gpu/drm/i915/i915_vma.h | 17 +-
5 files changed, 617 insertions(+), 88 deletions(-)
diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug
index 206882e154bc..38a6d0908a61 100644
--- a/drivers/gpu/drm/i915/Kconfig.debug
+++ b/drivers/gpu/drm/i915/Kconfig.debug
@@ -104,6 +104,20 @@ config DRM_I915_TRACE_GEM
If in doubt, say "N".
+config DRM_I915_TRACE_EXECBUF
+ bool "Insert extra ftrace output from the execbuf internals"
+ depends on DRM_I915_DEBUG_GEM
+ select TRACING
+ default n
+ help
+ Enable additional and verbose debugging output that will spam
+ ordinary tests, but may be vital for post-mortem debugging when
+ used with /proc/sys/kernel/ftrace_dump_on_oops
+
+ Recommended for driver developers only.
+
+ If in doubt, say "N".
+
config DRM_I915_TRACE_GTT
bool "Insert extra ftrace output from the GTT internals"
depends on DRM_I915_DEBUG_GEM
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
index 19fba22f7ea0..2abdf09c3316 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
@@ -27,10 +27,19 @@
#include "i915_sw_fence_work.h"
#include "i915_trace.h"
+#if IS_ENABLED(CONFIG_DRM_I915_TRACE_EXECBUF)
+#define EB_DBG(...) pr_err(__VA_ARGS__)
+#else
+#define EB_DBG(...)
+#endif
+
struct eb_vma {
struct i915_vma *vma;
unsigned int flags;
+ struct drm_mm_node hole;
+ unsigned int bind_flags;
+
/** This vma's place in the execbuf reservation list */
struct drm_i915_gem_exec_object2 *exec;
struct list_head bind_link;
@@ -43,6 +52,7 @@ struct eb_vma {
struct eb_vma_array {
struct kref kref;
+ unsigned int count;
struct eb_vma vma[];
};
@@ -55,11 +65,18 @@ static struct eb_vma_array *eb_vma_array_create(unsigned int count)
return NULL;
kref_init(&arr->kref);
+ arr->count = count;
arr->vma[0].vma = NULL;
return arr;
}
+static struct eb_vma_array *eb_vma_array_get(struct eb_vma_array *arr)
+{
+ kref_get(&arr->kref);
+ return arr;
+}
+
static void eb_vma_array_destroy(struct kref *kref)
{
struct eb_vma_array *arr = container_of(kref, typeof(*arr), kref);
@@ -83,7 +100,8 @@ enum {
#define __EXEC_OBJECT_HAS_FENCE BIT(30)
#define __EXEC_OBJECT_NEEDS_MAP BIT(29)
#define __EXEC_OBJECT_NEEDS_BIAS BIT(28)
-#define __EXEC_OBJECT_INTERNAL_FLAGS (~0u << 28) /* all of the above */
+#define __EXEC_OBJECT_HAS_PAGES BIT(27)
+#define __EXEC_OBJECT_INTERNAL_FLAGS (~0u << 27) /* all of the above */
#define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE)
#define __EXEC_HAS_RELOC BIT(31)
@@ -379,6 +397,10 @@ static int eb_create(struct i915_execbuffer *eb)
eb->lut_size = -eb->buffer_count;
}
+ INIT_LIST_HEAD(&eb->relocs);
+ INIT_LIST_HEAD(&eb->unbound);
+ INIT_LIST_HEAD(&eb->lock);
+
return 0;
}
@@ -459,10 +481,15 @@ static inline void __eb_unreserve_vma(struct i915_vma *vma, unsigned int flags)
static inline void
eb_unreserve_vma(struct eb_vma *ev)
{
- if (!(ev->flags & __EXEC_OBJECT_HAS_PIN))
- return;
+ if (ev->flags & __EXEC_OBJECT_HAS_PIN)
+ __eb_unreserve_vma(ev->vma, ev->flags);
- __eb_unreserve_vma(ev->vma, ev->flags);
+ if (ev->flags & __EXEC_OBJECT_HAS_PAGES) {
+ i915_vma_put_pages(ev->vma);
+ ev->flags &= ~__EXEC_OBJECT_HAS_PAGES;
+ }
+
+ GEM_BUG_ON(drm_mm_node_allocated(&ev->hole));
ev->flags &= ~__EXEC_OBJECT_RESERVED;
}
@@ -525,6 +552,7 @@ eb_add_vma(struct i915_execbuffer *eb,
GEM_BUG_ON(i915_vma_is_closed(vma));
+ memset(&ev->hole, 0, sizeof(ev->hole));
ev->vma = i915_vma_get(vma);
ev->exec = entry;
ev->flags = entry->flags;
@@ -566,6 +594,7 @@ eb_add_vma(struct i915_execbuffer *eb,
} else {
eb_unreserve_vma(ev);
list_add_tail(&ev->bind_link, &eb->unbound);
+ i915_vma_get(vma);
}
list_add_tail(&ev->lock_link, &eb->lock);
@@ -588,55 +617,334 @@ static inline int use_cpu_reloc(const struct reloc_cache *cache,
obj->cache_level != I915_CACHE_NONE);
}
-static int eb_reserve_vma(const struct i915_execbuffer *eb,
- struct eb_vma *ev,
- u64 pin_flags)
+struct eb_vma_work {
+ struct dma_fence_work base;
+ struct list_head unbound;
+ struct eb_vma_array *array;
+ struct i915_address_space *vm;
+ struct list_head eviction_list;
+ u64 *p_flags;
+};
+
+static inline u64 node_end(const struct drm_mm_node *node)
+{
+ return node->start + node->size;
+}
+
+static void __evict(struct i915_vma *vma)
+{
+ lockdep_assert_held(&vma->vm->mutex);
+
+ if (i915_vma_is_map_and_fenceable(vma)) {
+ /*
+ * Check that we have flushed all writes through the GGTT
+ * before the unbind, other due to non-strict nature of those
+ * indirect writes they may end up referencing the GGTT PTE
+ * after the unbind.
+ *
+ * Note that we may be concurrently poking at the GGTT_WRITE
+ * bit from set-domain, as we mark all GGTT vma associated
+ * with an object. We know this is for another vma, as we
+ * are currently unbinding this one -- so if this vma will be
+ * reused, it will be refaulted and have its dirty bit set
+ * before the next write.
+ */
+ i915_vma_flush_writes(vma);
+
+ /* release the fence reg _after_ flushing */
+ i915_vma_revoke_fence(vma);
+
+ /* Force a pagefault for domain tracking on next user access */
+ i915_vma_revoke_mmap(vma);
+
+ __i915_vma_iounmap(vma);
+ clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
+ }
+
+ if (likely(atomic_read(&vma->vm->open)))
+ vma->ops->unbind_vma(vma);
+
+ i915_vma_unbind_pages(vma);
+}
+
+static bool evict_for_node(struct eb_vma_work *ew,
+ struct eb_vma *const target,
+ unsigned int flags)
+{
+ struct i915_address_space *vm = target->vma->vm;
+ const unsigned long color = target->vma->node.color;
+ const u64 start = target->vma->node.start;
+ const u64 end = start + target->vma->node.size;
+ u64 hole_start = start, hole_end = end;
+ struct drm_mm_node *node;
+ LIST_HEAD(eviction_list);
+ struct i915_vma *vma;
+
+ EB_DBG("check [%llx, %llx]\n", start, end);
+
+ lockdep_assert_held(&vm->mutex);
+ GEM_BUG_ON(drm_mm_node_allocated(&target->vma->node));
+ GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
+ GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
+
+ if (i915_vm_has_cache_coloring(vm)) {
+ /* Expand search to cover neighbouring guard pages (or lack!) */
+ if (hole_start)
+ hole_start -= I915_GTT_PAGE_SIZE;
+
+ /* Always look at the page afterwards to avoid the end-of-GTT */
+ hole_end += I915_GTT_PAGE_SIZE;
+ }
+ GEM_BUG_ON(hole_start >= hole_end);
+
+ drm_mm_for_each_node_in_range(node, &vm->mm, hole_start, hole_end) {
+ EB_DBG("evict? %llx + %llx, colour=%lu\n",
+ node->start, node->size, node->color);
+ GEM_BUG_ON(node == &target->vma->node);
+
+ /* If we find any non-objects (!vma), we cannot evict them */
+ if (node->color == I915_COLOR_UNEVICTABLE)
+ goto err;
+
+ GEM_BUG_ON(!drm_mm_node_allocated(node));
+ vma = container_of(node, typeof(*vma), node);
+
+ /*
+ * If we are using coloring to insert guard pages between
+ * different cache domains within the address space, we have
+ * to check whether the objects on either side of our range
+ * abutt and conflict. If they are in conflict, then we evict
+ * those as well to make room for our guard pages.
+ */
+ if (i915_vm_has_cache_coloring(vm)) {
+ if (node_end(node) == start && node->color == color)
+ continue;
+
+ if (node->start == end && node->color == color)
+ continue;
+ }
+
+ if (i915_vma_is_pinned(vma))
+ goto err;
+
+ if (flags & PIN_NONBLOCK && i915_vma_is_active(vma))
+ goto err;
+
+ if (i915_sw_fence_await_active(&ew->base.chain,
+ &vma->active,
+ I915_ACTIVE_AWAIT_ALL))
+ goto err;
+
+ if (i915_active_acquire(&vma->active))
+ goto err;
+
+ i915_active_set_exclusive(&vma->active, &ew->base.dma);
+ list_move(&vma->vm_link, &eviction_list);
+ i915_active_release(&vma->active);
+ }
+
+ /* No overlapping nodes to evict, claim the slot for ourselves! */
+ if (list_empty(&eviction_list)) {
+ EB_DBG("bind: %llx + %llx\n",
+ target->vma->node.start, target->vma->node.size);
+ return drm_mm_reserve_node(&vm->mm, &target->vma->node) == 0;
+ }
+
+ /*
+ * Mark this range as reserved.
+ *
+ * We have not yet removed the PTEs for the old evicted nodes, so
+ * must prevent this range from being reused for anything else. The
+ * PTE will be cleared when the range is idle (during the rebind
+ * phase in the worker).
+ */
+ target->hole.color = I915_COLOR_UNEVICTABLE;
+ target->hole.start = start;
+ target->hole.size = end;
+
+ list_for_each_entry(vma, &eviction_list, vm_link) {
+ EB_DBG("evicting %llx + %llx\n",
+ vma->node.start, vma->node.size);
+ target->hole.start =
+ min(target->hole.start, vma->node.start);
+ target->hole.size =
+ max(target->hole.size, node_end(&vma->node));
+
+ GEM_BUG_ON(vma->node.mm != &vm->mm);
+ drm_mm_remove_node(&vma->node);
+ atomic_and(~I915_VMA_BIND_MASK, &vma->flags);
+ GEM_BUG_ON(i915_vma_is_pinned(vma));
+ }
+ list_splice(&eviction_list, &ew->eviction_list);
+
+ target->hole.size -= target->hole.start;
+
+ EB_DBG("hole: %llx + %llx\n", target->hole.start, target->hole.size);
+ return drm_mm_reserve_node(&vm->mm, &target->hole) == 0;
+
+err:
+ list_splice(&eviction_list, &vm->bound_list);
+ return false;
+}
+
+static u64 random_offset(u64 start, u64 end, u64 len, u64 align)
+{
+ u64 range, addr;
+
+ GEM_BUG_ON(range_overflows(start, len, end));
+ GEM_BUG_ON(round_up(start, align) > round_down(end - len, align));
+
+ range = round_down(end - len, align) - round_up(start, align);
+ if (range) {
+ if (sizeof(unsigned long) == sizeof(u64)) {
+ addr = get_random_long();
+ } else {
+ addr = get_random_int();
+ if (range > U32_MAX) {
+ addr <<= 32;
+ addr |= get_random_int();
+ }
+ }
+ div64_u64_rem(addr, range, &addr);
+ start += addr;
+ }
+
+ return round_up(start, align);
+}
+
+static int eb_reserve_vma(struct eb_vma_work *ew, struct eb_vma *ev)
{
struct drm_i915_gem_exec_object2 *entry = ev->exec;
unsigned int exec_flags = ev->flags;
struct i915_vma *vma = ev->vma;
+ struct i915_address_space *vm = vma->vm;
+ u64 start = 0, end = vm->total;
+ u64 align = entry->alignment ?: I915_GTT_MIN_ALIGNMENT;
+ unsigned int bind_flags;
int err;
+ bind_flags = PIN_USER;
if (exec_flags & EXEC_OBJECT_NEEDS_GTT)
- pin_flags |= PIN_GLOBAL;
+ bind_flags |= PIN_GLOBAL;
+
+ if (drm_mm_node_allocated(&vma->node)) {
+ EB_DBG("reuse bound node: %llx + %llx\n",
+ vma->node.start, vma->node.size);
+ goto pin;
+ }
+
+ vma->node.start = entry->offset & PIN_OFFSET_MASK;
+ vma->node.size = max(entry->pad_to_size, vma->size);
+ vma->node.color = 0;
+ if (i915_vm_has_cache_coloring(vm))
+ vma->node.color = vma->obj->cache_level;
/*
* Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
* limit address to the first 4GBs for unflagged objects.
*/
if (!(exec_flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
- pin_flags |= PIN_ZONE_4G;
+ end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE);
+
+ align = max(align, vma->display_alignment);
+ if (exec_flags & __EXEC_OBJECT_NEEDS_MAP) {
+ vma->node.size = max_t(u64, vma->node.size, vma->fence_size);
+ end = min_t(u64, end, i915_vm_to_ggtt(vm)->mappable_end);
+ align = max_t(u64, align, vma->fence_alignment);
+ }
+
+ if (exec_flags & __EXEC_OBJECT_NEEDS_BIAS)
+ start = BATCH_OFFSET_BIAS;
+
+ if (vma->node.size > end - start)
+ return -E2BIG;
- if (exec_flags & __EXEC_OBJECT_NEEDS_MAP)
- pin_flags |= PIN_MAPPABLE;
+ if (vma->node.start >= start &&
+ IS_ALIGNED(vma->node.start, align) &&
+ !range_overflows(vma->node.start, vma->node.size, end)) {
+ unsigned int pin_flags;
+
+ if (drm_mm_reserve_node(&vm->mm, &vma->node) == 0) {
+ EB_DBG("reuse node location: %llx + %llx\n",
+ vma->node.start, vma->node.size);
+ goto pin;
+ }
+
+ pin_flags = 0;
+ if (!(exec_flags & EXEC_OBJECT_PINNED))
+ pin_flags = PIN_NONBLOCK;
+
+ if (evict_for_node(ew, ev, pin_flags)) {
+ EB_DBG("evicted old node location: %llx + %llx\n",
+ vma->node.start, vma->node.size);
+ goto pin;
+ }
+ }
if (exec_flags & EXEC_OBJECT_PINNED)
- pin_flags |= entry->offset | PIN_OFFSET_FIXED;
- else if (exec_flags & __EXEC_OBJECT_NEEDS_BIAS)
- pin_flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
+ return -ENOSPC;
+
+ vma->node.start = random_offset(start, end, vma->node.size, align);
+ if (evict_for_node(ew, ev, PIN_NONBLOCK)) {
+ EB_DBG("evicted random location: %llx + %llx\n",
+ vma->node.start, vma->node.size);
+ goto pin;
+ }
+
+ if (drm_mm_insert_node_in_range(&vm->mm, &vma->node,
+ vma->node.size,
+ align <= I915_GTT_MIN_ALIGNMENT ? 0 : align,
+ vma->node.color,
+ start, end, 0) == 0) {
+ EB_DBG("found free space: %llx + %llx\n",
+ vma->node.start, vma->node.size);
+ goto pin;
+ }
+
+#if 0
+ if (i915_gem_evict_something(vm,
+ vma->node.size,
+ align,
+ vma->node.color,
+ start, end, 0) == 0)
+ goto pin;
+#endif
- if (drm_mm_node_allocated(&vma->node) &&
- eb_vma_misplaced(entry, vma, ev->flags)) {
- err = i915_vma_unbind(vma);
- if (err)
- return err;
+ vma->node.start = random_offset(start, end, vma->node.size, align);
+ if (evict_for_node(ew, ev, 0)) {
+ EB_DBG("evicted random location: %llx + %llx\n",
+ vma->node.start, vma->node.size);
+ goto pin;
}
- err = i915_vma_pin(vma,
- entry->pad_to_size, entry->alignment,
- pin_flags);
- if (err)
- return err;
+ return -ENOSPC;
+
+pin:
+ __i915_vma_pin(vma);
+ list_move_tail(&vma->vm_link, &vm->bound_list);
+
+ ev->bind_flags = bind_flags & ~atomic_read(&vma->flags);
+ if (ev->bind_flags) {
+ i915_active_set_exclusive(&vma->active, &ew->base.dma);
+ atomic_add(I915_VMA_PAGES_ACTIVE, &vma->pages_count);
+ atomic_or(bind_flags, &vma->flags);
+ }
+
+ GEM_BUG_ON(!ev->bind_flags && !drm_mm_node_allocated(&vma->node));
+ GEM_BUG_ON(drm_mm_node_allocated(&vma->node) &&
+ drm_mm_node_allocated(&ev->hole));
+ GEM_BUG_ON(!drm_mm_node_allocated(&vma->node) &&
+ !drm_mm_node_allocated(&ev->hole));
if (entry->offset != vma->node.start) {
entry->offset = vma->node.start | UPDATE;
- eb->args->flags |= __EXEC_HAS_RELOC;
+ *ew->p_flags |= __EXEC_HAS_RELOC;
}
if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) {
- err = i915_vma_pin_fence(vma);
+ err = __i915_vma_pin_fence(vma); /* XXX no waiting */
if (unlikely(err)) {
- i915_vma_unpin(vma);
+ __i915_vma_unpin(vma);
return err;
}
@@ -650,44 +958,251 @@ static int eb_reserve_vma(const struct i915_execbuffer *eb,
return 0;
}
+static int eb_bind_vma(struct dma_fence_work *work)
+{
+ struct eb_vma_work *ew = container_of(work, typeof(*ew), base);
+ struct i915_address_space *vm = ew->vm;
+ struct eb_vma *ev;
+ int err = 0;
+
+ if (!list_empty(&ew->eviction_list)) {
+ struct i915_vma *vma, *vn;
+
+ mutex_lock(&vm->mutex);
+ list_for_each_entry_safe(vma, vn, &ew->eviction_list, vm_link) {
+ EB_DBG("evict: [%llx + %llx]\n",
+ vma->node.start, vma->node.size);
+ GEM_BUG_ON(vma->vm != vm);
+ GEM_BUG_ON(!i915_vma_is_active(vma));
+
+ __evict(vma);
+ INIT_LIST_HEAD(&vma->vm_link);
+ }
+ mutex_unlock(&vm->mutex);
+ }
+
+ list_for_each_entry(ev, &ew->unbound, bind_link) {
+ struct i915_vma *vma = ev->vma;
+
+ EB_DBG("hole: [%llx + %llx], node: [%llx + %llx], flags:%x\n",
+ ev->hole.start, ev->hole.size,
+ vma->node.start, vma->node.size,
+ ev->bind_flags);
+ GEM_BUG_ON(vma->vm != vm);
+
+ if (!ev->bind_flags)
+ goto put;
+
+ GEM_BUG_ON(!i915_vma_is_active(vma));
+
+ if (err == 0)
+ err = vma->ops->bind_vma(vma,
+ vma->obj->cache_level,
+ ev->bind_flags |
+ I915_VMA_ALLOC);
+
+ GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_LOCAL_BIND));
+ if (drm_mm_node_allocated(&ev->hole)) {
+ mutex_lock(&vm->mutex);
+ GEM_BUG_ON(ev->hole.mm != &vm->mm);
+ GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
+ drm_mm_remove_node(&ev->hole);
+ drm_mm_reserve_node(&vm->mm, &vma->node);
+ GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
+ mutex_unlock(&vm->mutex);
+ }
+ if (err) {
+ atomic_and(~ev->bind_flags, &vma->flags);
+ set_bit(I915_VMA_ERROR_BIT, __i915_vma_flags(vma));
+ }
+
+put:
+ GEM_BUG_ON(drm_mm_node_allocated(&ev->hole));
+ i915_vma_put(vma);
+ }
+
+ EB_DBG("%s: %d\n", __func__, err);
+ return err;
+}
+
+static void eb_vma_work_release(struct dma_fence_work *work)
+{
+ struct eb_vma_work *ew = container_of(work, typeof(*ew), base);
+
+ eb_vma_array_put(ew->array);
+}
+
+static const struct dma_fence_work_ops eb_bind_ops = {
+ .name = "eb_bind",
+ .work = eb_bind_vma,
+ .release = eb_vma_work_release,
+};
+
+static struct eb_vma_work *eb_vma_work(struct i915_execbuffer *eb)
+{
+ struct eb_vma_work *ew;
+
+ ew = kzalloc(sizeof(*ew), GFP_KERNEL);
+ if (!ew)
+ return NULL;
+
+ dma_fence_work_init(&ew->base, &eb_bind_ops);
+ list_replace_init(&eb->unbound, &ew->unbound);
+ ew->array = eb_vma_array_get(eb->array);
+ ew->p_flags = &eb->args->flags;
+ ew->vm = eb->context->vm;
+
+ INIT_LIST_HEAD(&ew->eviction_list);
+
+ GEM_BUG_ON(list_empty(&ew->unbound));
+ GEM_BUG_ON(!list_empty(&eb->unbound));
+
+ return ew;
+}
+
+static int eb_vm_throttle(struct i915_execbuffer *eb, struct eb_vma_work *ew)
+{
+ struct dma_fence *prev;
+ int err = 0;
+
+ /* Keep async work queued per context */
+ prev = __i915_active_ref(&ew->vm->active,
+ i915_gem_context_async_id(eb->gem_context),
+ &ew->base.dma);
+ if (prev) {
+ err = i915_sw_fence_await_dma_fence(&ew->base.chain,
+ prev, 0, I915_FENCE_GFP);
+ dma_fence_put(prev);
+ }
+
+ return err;
+}
+
+static int i915_vma_wait_unbind(struct i915_vma *vma)
+{
+ struct dma_fence *fence;
+ int err = 0;
+
+ fence = i915_active_fence_get(&vma->active.excl);
+ if (!fence)
+ return 0;
+
+ if (!drm_mm_node_allocated(&vma->node))
+ err = dma_fence_wait(fence, true);
+
+ dma_fence_put(fence);
+ return err;
+}
+
+static int eb_prepare_vma(struct eb_vma *ev)
+{
+ struct i915_vma *vma = ev->vma;
+ int err;
+
+ GEM_BUG_ON(drm_mm_node_allocated(&ev->hole));
+ ev->bind_flags = 0;
+
+ /* Wait for a previous unbind to avoid reusing an active vma->node */
+ err = i915_vma_wait_unbind(vma);
+ if (err)
+ return err;
+
+ err = i915_vma_get_pages(vma);
+ if (err)
+ return err;
+
+ ev->flags |= __EXEC_OBJECT_HAS_PAGES;
+
+ return 0;
+}
+
static int eb_reserve(struct i915_execbuffer *eb)
{
const unsigned int count = eb->buffer_count;
- unsigned int pin_flags = PIN_USER | PIN_NONBLOCK;
+ struct i915_address_space *vm = eb->context->vm;
struct list_head last;
- struct eb_vma *ev;
unsigned int i, pass;
+ struct eb_vma *ev;
int err = 0;
- /*
- * Attempt to pin all of the buffers into the GTT.
- * This is done in 3 phases:
- *
- * 1a. Unbind all objects that do not match the GTT constraints for
- * the execbuffer (fenceable, mappable, alignment etc).
- * 1b. Increment pin count for already bound objects.
- * 2. Bind new objects.
- * 3. Decrement pin count.
- *
- * This avoid unnecessary unbinding of later objects in order to make
- * room for the earlier objects *unless* we need to defragment.
- */
-
- if (mutex_lock_interruptible(&eb->i915->drm.struct_mutex))
- return -EINTR;
-
pass = 0;
do {
+ struct eb_vma_work *work;
+
list_for_each_entry(ev, &eb->unbound, bind_link) {
- err = eb_reserve_vma(eb, ev, pin_flags);
+ err = eb_prepare_vma(ev);
+ switch (err) {
+ case 0:
+ break;
+ case -EAGAIN:
+ goto retry;
+ default:
+ return err;
+ }
+ }
+
+ work = eb_vma_work(eb);
+ if (!work)
+ return -ENOMEM;
+
+ if (mutex_lock_interruptible(&vm->mutex)) {
+ dma_fence_work_commit(&work->base);
+ return -EINTR;
+ }
+
+ err = eb_vm_throttle(eb, work);
+ if (err) {
+ mutex_unlock(&vm->mutex);
+ dma_fence_work_commit(&work->base);
+ return err;
+ }
+
+ list_for_each_entry(ev, &work->unbound, bind_link) {
+ struct i915_vma *vma = ev->vma;
+
+ if (!drm_mm_node_allocated(&vma->node) &&
+ i915_vma_is_active(vma)) {
+ err = -ENOSPC;
+ break;
+ }
+
+ if (drm_mm_node_allocated(&vma->node) &&
+ eb_vma_misplaced(ev->exec, vma, ev->flags)) {
+ EB_DBG("unbinding stale node\n");
+ err = __i915_vma_unbind(vma);
+ if (err)
+ break;
+ }
+
+ err = i915_active_acquire(&vma->active);
+ if (err)
+ break;
+
+ err = eb_reserve_vma(work, ev);
+ i915_active_release(&vma->active);
if (err)
break;
}
- if (!(err == -ENOSPC || err == -EAGAIN))
- break;
+
+ mutex_unlock(&vm->mutex);
+
+ dma_fence_get(&work->base.dma);
+ dma_fence_work_commit(&work->base);
+ if (err == -ENOSPC && dma_fence_wait(&work->base.dma, true))
+ err = -EINTR;
+ dma_fence_put(&work->base.dma);
+ if (err != -ENOSPC)
+ return err;
+
+retry:
+ if (!list_empty(&eb->unbound)) {
+ list_for_each_entry(ev, &eb->unbound, bind_link)
+ i915_vma_put(ev->vma);
+ INIT_LIST_HEAD(&eb->unbound);
+ }
/* Resort *all* the objects into priority order */
- INIT_LIST_HEAD(&eb->unbound);
+ EB_DBG("resort\n");
INIT_LIST_HEAD(&last);
for (i = 0; i < count; i++) {
unsigned int flags;
@@ -711,13 +1226,13 @@ static int eb_reserve(struct i915_execbuffer *eb)
list_add(&ev->bind_link, &last);
else
list_add_tail(&ev->bind_link, &last);
+
+ i915_vma_get(ev->vma);
}
list_splice_tail(&last, &eb->unbound);
if (err == -EAGAIN) {
- mutex_unlock(&eb->i915->drm.struct_mutex);
flush_workqueue(eb->i915->mm.userptr_wq);
- mutex_lock(&eb->i915->drm.struct_mutex);
continue;
}
@@ -727,22 +1242,14 @@ static int eb_reserve(struct i915_execbuffer *eb)
case 1:
/* Too fragmented, unbind everything and retry */
- err = i915_active_wait(&eb->context->vm->active);
- if (err)
- goto unlock;
+ if (i915_active_wait(&vm->active))
+ return -EINTR;
break;
default:
- err = -ENOSPC;
- goto unlock;
+ return err;
}
-
- pin_flags = PIN_USER;
} while (1);
-
-unlock:
- mutex_unlock(&eb->i915->drm.struct_mutex);
- return err;
}
static unsigned int eb_batch_index(const struct i915_execbuffer *eb)
@@ -782,10 +1289,6 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
if (unlikely(i915_gem_context_is_closed(eb->gem_context)))
return -ENOENT;
- INIT_LIST_HEAD(&eb->relocs);
- INIT_LIST_HEAD(&eb->unbound);
- INIT_LIST_HEAD(&eb->lock);
-
batch = eb_batch_index(eb);
for (i = 0; i < eb->buffer_count; i++) {
@@ -890,8 +1393,13 @@ static void eb_release_vmas(const struct i915_execbuffer *eb)
static void eb_destroy(const struct i915_execbuffer *eb)
{
+ struct eb_vma *ev;
+
GEM_BUG_ON(eb->reloc_cache.rq);
+ list_for_each_entry(ev, &eb->unbound, bind_link)
+ i915_vma_put(ev->vma);
+
eb_vma_array_put(eb->array);
if (eb->lut_size > 0)
@@ -1904,6 +2412,7 @@ static int eb_parse(struct i915_execbuffer *eb)
goto err_trampoline;
eb->batch = &eb->vma[eb->buffer_count++];
+ memset(&eb->batch->hole, 0, sizeof(eb->batch->hole));
eb->batch->vma = i915_vma_get(shadow);
eb->batch->flags = __EXEC_OBJECT_HAS_PIN;
list_add_tail(&eb->batch->lock_link, &eb->lock);
diff --git a/drivers/gpu/drm/i915/gt/intel_ppgtt.c b/drivers/gpu/drm/i915/gt/intel_ppgtt.c
index f86f7e68ce5e..ecdd58f4b993 100644
--- a/drivers/gpu/drm/i915/gt/intel_ppgtt.c
+++ b/drivers/gpu/drm/i915/gt/intel_ppgtt.c
@@ -162,7 +162,8 @@ static int ppgtt_bind_vma(struct i915_vma *vma,
u32 pte_flags;
int err;
- if (flags & I915_VMA_ALLOC) {
+ if (flags & I915_VMA_ALLOC &&
+ !test_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma))) {
err = vma->vm->allocate_va_range(vma->vm,
vma->node.start, vma->size);
if (err)
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index ed65d92a7609..b886891a6d6d 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -132,6 +132,7 @@ vma_create(struct drm_i915_gem_object *obj,
fs_reclaim_release(GFP_KERNEL);
}
+ INIT_LIST_HEAD(&vma->vm_link);
INIT_LIST_HEAD(&vma->closed_link);
if (view && view->type != I915_GGTT_VIEW_NORMAL) {
@@ -726,7 +727,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, color));
- list_add_tail(&vma->vm_link, &vma->vm->bound_list);
+ list_move_tail(&vma->vm_link, &vma->vm->bound_list);
return 0;
}
@@ -742,7 +743,7 @@ i915_vma_detach(struct i915_vma *vma)
* vma, we can drop its hold on the backing storage and allow
* it to be reaped by the shrinker.
*/
- list_del(&vma->vm_link);
+ list_del_init(&vma->vm_link);
}
static bool try_qad_pin(struct i915_vma *vma, unsigned int flags)
@@ -788,7 +789,7 @@ static bool try_qad_pin(struct i915_vma *vma, unsigned int flags)
return pinned;
}
-static int vma_get_pages(struct i915_vma *vma)
+int i915_vma_get_pages(struct i915_vma *vma)
{
int err = 0;
@@ -835,7 +836,7 @@ static void __vma_put_pages(struct i915_vma *vma, unsigned int count)
mutex_unlock(&vma->pages_mutex);
}
-static void vma_put_pages(struct i915_vma *vma)
+void i915_vma_put_pages(struct i915_vma *vma)
{
if (atomic_add_unless(&vma->pages_count, -1, 1))
return;
@@ -843,7 +844,7 @@ static void vma_put_pages(struct i915_vma *vma)
__vma_put_pages(vma, 1);
}
-static void vma_unbind_pages(struct i915_vma *vma)
+void i915_vma_unbind_pages(struct i915_vma *vma)
{
unsigned int count;
@@ -874,7 +875,7 @@ int i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
if (try_qad_pin(vma, flags & I915_VMA_BIND_MASK))
return 0;
- err = vma_get_pages(vma);
+ err = i915_vma_get_pages(vma);
if (err)
return err;
@@ -960,7 +961,7 @@ int i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
if (wakeref)
intel_runtime_pm_put(&vma->vm->i915->runtime_pm, wakeref);
err_pages:
- vma_put_pages(vma);
+ i915_vma_put_pages(vma);
return err;
}
@@ -1108,17 +1109,6 @@ void i915_vma_parked(struct intel_gt *gt)
spin_unlock_irq(>->closed_lock);
}
-static void __i915_vma_iounmap(struct i915_vma *vma)
-{
- GEM_BUG_ON(i915_vma_is_pinned(vma));
-
- if (vma->iomap == NULL)
- return;
-
- io_mapping_unmap(vma->iomap);
- vma->iomap = NULL;
-}
-
void i915_vma_revoke_mmap(struct i915_vma *vma)
{
struct drm_vma_offset_node *node;
@@ -1273,7 +1263,7 @@ int __i915_vma_unbind(struct i915_vma *vma)
&vma->flags);
i915_vma_detach(vma);
- vma_unbind_pages(vma);
+ i915_vma_unbind_pages(vma);
drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */
return 0;
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index e1ced1df13e1..00f74b99bebd 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -239,6 +239,10 @@ int __must_check
i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags);
int i915_ggtt_pin(struct i915_vma *vma, u32 align, unsigned int flags);
+int i915_vma_get_pages(struct i915_vma *vma);
+void i915_vma_put_pages(struct i915_vma *vma);
+void i915_vma_unbind_pages(struct i915_vma *vma);
+
static inline int i915_vma_pin_count(const struct i915_vma *vma)
{
return atomic_read(&vma->flags) & I915_VMA_PIN_MASK;
@@ -293,6 +297,17 @@ static inline bool i915_node_color_differs(const struct drm_mm_node *node,
void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
#define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x))
+static inline void __i915_vma_iounmap(struct i915_vma *vma)
+{
+ GEM_BUG_ON(i915_vma_is_pinned(vma));
+
+ if (!vma->iomap)
+ return;
+
+ io_mapping_unmap(vma->iomap);
+ vma->iomap = NULL;
+}
+
/**
* i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap
* @vma: VMA to unpin
@@ -326,7 +341,7 @@ static inline struct page *i915_vma_first_page(struct i915_vma *vma)
* True if the vma has a fence, false otherwise.
*/
int __must_check i915_vma_pin_fence(struct i915_vma *vma);
-int __must_check i915_vma_revoke_fence(struct i915_vma *vma);
+int i915_vma_revoke_fence(struct i915_vma *vma);
int __i915_vma_pin_fence(struct i915_vma *vma);
--
2.20.1
More information about the Intel-gfx-trybot
mailing list